1/*-
2 * Copyright (c) 2010-2011 Juniper Networks, Inc.
3 * All rights reserved.
4 *
5 * This software was developed by Robert N. M. Watson under contract
6 * to Juniper Networks, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30
31#include "opt_inet6.h"
32
33#include <sys/param.h>
34#include <sys/mbuf.h>
35#include <sys/socket.h>
36#include <sys/priv.h>
37#include <sys/kernel.h>
38#include <sys/smp.h>
39#include <sys/sysctl.h>
40#include <sys/sbuf.h>
41
42#include <net/if.h>
43#include <net/if_var.h>
44#include <net/netisr.h>
45#include <net/rss_config.h>
46#include <net/toeplitz.h>
47
48/*-
49 * Operating system parts of receiver-side scaling (RSS), which allows
50 * network cards to direct flows to particular receive queues based on hashes
51 * of header tuples.  This implementation aligns RSS buckets with connection
52 * groups at the TCP/IP layer, so each bucket is associated with exactly one
53 * group.  As a result, the group lookup structures (and lock) should have an
54 * effective affinity with exactly one CPU.
55 *
56 * Network device drivers needing to configure RSS will query this framework
57 * for parameters, such as the current RSS key, hashing policies, number of
58 * bits, and indirection table mapping hashes to buckets and CPUs.  They may
59 * provide their own supplementary information, such as queue<->CPU bindings.
60 * It is the responsibility of the network device driver to inject packets
61 * into the stack on as close to the right CPU as possible, if playing by RSS
62 * rules.
63 *
64 * TODO:
65 *
66 * - Synchronization for rss_key and other future-configurable parameters.
67 * - Event handler drivers can register to pick up RSS configuration changes.
68 * - Should we allow rss_basecpu to be configured?
69 * - Randomize key on boot.
70 * - IPv6 support.
71 * - Statistics on how often there's a misalignment between hardware
72 *   placement and pcbgroup expectations.
73 */
74
75SYSCTL_DECL(_net_inet);
76SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
77    "Receive-side steering");
78
79/*
80 * Toeplitz is the only required hash function in the RSS spec, so use it by
81 * default.
82 */
83static u_int	rss_hashalgo = RSS_HASH_TOEPLITZ;
84SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
85    "RSS hash algorithm");
86
87/*
88 * Size of the indirection table; at most 128 entries per the RSS spec.  We
89 * size it to at least 2 times the number of CPUs by default to allow useful
90 * rebalancing.  If not set explicitly with a loader tunable, we tune based
91 * on the number of CPUs present.
92 *
93 * XXXRW: buckets might be better to use for the tunable than bits.
94 */
95static u_int	rss_bits;
96SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
97    "RSS bits");
98
99static u_int	rss_mask;
100SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
101    "RSS mask");
102
103static const u_int	rss_maxbits = RSS_MAXBITS;
104SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
105    __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
106
107/*
108 * RSS's own count of the number of CPUs it could be using for processing.
109 * Bounded to 64 by RSS constants.
110 */
111static u_int	rss_ncpus;
112SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
113    "Number of CPUs available to RSS");
114
115#define	RSS_MAXCPUS	(1 << (RSS_MAXBITS - 1))
116static const u_int	rss_maxcpus = RSS_MAXCPUS;
117SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
118    __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
119
120/*
121 * Variable exists just for reporting rss_bits in a user-friendly way.
122 */
123static u_int	rss_buckets;
124SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
125    "RSS buckets");
126
127/*
128 * Base CPU number; devices will add this to all CPU numbers returned by the
129 * RSS indirection table.  Currently unmodifable in FreeBSD.
130 */
131static const u_int	rss_basecpu;
132SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
133    __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
134
135/*
136 * Print verbose debugging messages.
137 * 0 - disable
138 * non-zero - enable
139 */
140int	rss_debug = 0;
141SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0,
142    "RSS debug level");
143
144/*
145 * RSS secret key, intended to prevent attacks on load-balancing.  Its
146 * effectiveness may be limited by algorithm choice and available entropy
147 * during the boot.
148 *
149 * XXXRW: And that we don't randomize it yet!
150 *
151 * This is the default Microsoft RSS specification key which is also
152 * the Chelsio T5 firmware default key.
153 */
154static uint8_t rss_key[RSS_KEYSIZE] = {
155	0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
156	0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
157	0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
158	0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
159	0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
160};
161
162/*
163 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
164 * Drivers may supplement this table with a separate CPU<->queue table when
165 * programming devices.
166 */
167struct rss_table_entry {
168	uint8_t		rte_cpu;	/* CPU affinity of bucket. */
169};
170static struct rss_table_entry	rss_table[RSS_TABLE_MAXLEN];
171
172static void
173rss_init(__unused void *arg)
174{
175	u_int i;
176	u_int cpuid;
177
178	/*
179	 * Validate tunables, coerce to sensible values.
180	 */
181	switch (rss_hashalgo) {
182	case RSS_HASH_TOEPLITZ:
183	case RSS_HASH_NAIVE:
184		break;
185
186	default:
187		RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n",
188		    rss_hashalgo, RSS_HASH_TOEPLITZ);
189		rss_hashalgo = RSS_HASH_TOEPLITZ;
190	}
191
192	/*
193	 * Count available CPUs.
194	 *
195	 * XXXRW: Note incorrect assumptions regarding contiguity of this set
196	 * elsewhere.
197	 */
198	rss_ncpus = 0;
199	for (i = 0; i <= mp_maxid; i++) {
200		if (CPU_ABSENT(i))
201			continue;
202		rss_ncpus++;
203	}
204	if (rss_ncpus > RSS_MAXCPUS)
205		rss_ncpus = RSS_MAXCPUS;
206
207	/*
208	 * Tune RSS table entries to be no less than 2x the number of CPUs
209	 * -- unless we're running uniprocessor, in which case there's not
210	 * much point in having buckets to rearrange for load-balancing!
211	 */
212	if (rss_ncpus > 1) {
213		if (rss_bits == 0)
214			rss_bits = fls(rss_ncpus - 1) + 1;
215
216		/*
217		 * Microsoft limits RSS table entries to 128, so apply that
218		 * limit to both auto-detected CPU counts and user-configured
219		 * ones.
220		 */
221		if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
222			RSS_DEBUG("RSS bits %u not valid, coercing to %u\n",
223			    rss_bits, RSS_MAXBITS);
224			rss_bits = RSS_MAXBITS;
225		}
226
227		/*
228		 * Figure out how many buckets to use; warn if less than the
229		 * number of configured CPUs, although this is not a fatal
230		 * problem.
231		 */
232		rss_buckets = (1 << rss_bits);
233		if (rss_buckets < rss_ncpus)
234			RSS_DEBUG("WARNING: rss_buckets (%u) less than "
235			    "rss_ncpus (%u)\n", rss_buckets, rss_ncpus);
236		rss_mask = rss_buckets - 1;
237	} else {
238		rss_bits = 0;
239		rss_buckets = 1;
240		rss_mask = 0;
241	}
242
243	/*
244	 * Set up initial CPU assignments: round-robin by default.
245	 */
246	cpuid = CPU_FIRST();
247	for (i = 0; i < rss_buckets; i++) {
248		rss_table[i].rte_cpu = cpuid;
249		cpuid = CPU_NEXT(cpuid);
250	}
251
252	/*
253	 * Randomize rrs_key.
254	 *
255	 * XXXRW: Not yet.  If nothing else, will require an rss_isbadkey()
256	 * loop to check for "bad" RSS keys.
257	 */
258}
259SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
260
261static uint32_t
262rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
263    const uint8_t *data)
264{
265	uint32_t v;
266	u_int i;
267
268	v = 0;
269	for (i = 0; i < keylen; i++)
270		v += key[i];
271	for (i = 0; i < datalen; i++)
272		v += data[i];
273	return (v);
274}
275
276uint32_t
277rss_hash(u_int datalen, const uint8_t *data)
278{
279
280	switch (rss_hashalgo) {
281	case RSS_HASH_TOEPLITZ:
282		return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
283		    data));
284
285	case RSS_HASH_NAIVE:
286		return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
287		    data));
288
289	default:
290		panic("%s: unsupported/unknown hashalgo %d", __func__,
291		    rss_hashalgo);
292	}
293}
294
295/*
296 * Query the number of RSS bits in use.
297 */
298u_int
299rss_getbits(void)
300{
301
302	return (rss_bits);
303}
304
305/*
306 * Query the RSS bucket associated with an RSS hash.
307 */
308u_int
309rss_getbucket(u_int hash)
310{
311
312	return (hash & rss_mask);
313}
314
315/*
316 * Query the RSS layer bucket associated with the given
317 * entry in the RSS hash space.
318 *
319 * The RSS indirection table is 0 .. rss_buckets-1,
320 * covering the low 'rss_bits' of the total 128 slot
321 * RSS indirection table.  So just mask off rss_bits and
322 * return that.
323 *
324 * NIC drivers can then iterate over the 128 slot RSS
325 * indirection table and fetch which RSS bucket to
326 * map it to.  This will typically be a CPU queue
327 */
328u_int
329rss_get_indirection_to_bucket(u_int index)
330{
331
332	return (index & rss_mask);
333}
334
335/*
336 * Query the RSS CPU associated with an RSS bucket.
337 */
338u_int
339rss_getcpu(u_int bucket)
340{
341
342	return (rss_table[bucket].rte_cpu);
343}
344
345/*
346 * netisr CPU affinity lookup given just the hash and hashtype.
347 */
348u_int
349rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
350{
351
352	switch (hash_type) {
353	case M_HASHTYPE_RSS_IPV4:
354	case M_HASHTYPE_RSS_TCP_IPV4:
355	case M_HASHTYPE_RSS_UDP_IPV4:
356	case M_HASHTYPE_RSS_IPV6:
357	case M_HASHTYPE_RSS_TCP_IPV6:
358	case M_HASHTYPE_RSS_UDP_IPV6:
359		return (rss_getcpu(rss_getbucket(hash_val)));
360	default:
361		return (NETISR_CPUID_NONE);
362	}
363}
364
365/*
366 * Query the RSS bucket associated with the given hash value and
367 * type.
368 */
369int
370rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
371{
372
373	switch (hash_type) {
374	case M_HASHTYPE_RSS_IPV4:
375	case M_HASHTYPE_RSS_TCP_IPV4:
376	case M_HASHTYPE_RSS_UDP_IPV4:
377	case M_HASHTYPE_RSS_IPV6:
378	case M_HASHTYPE_RSS_TCP_IPV6:
379	case M_HASHTYPE_RSS_UDP_IPV6:
380		*bucket_id = rss_getbucket(hash_val);
381		return (0);
382	default:
383		return (-1);
384	}
385}
386
387/*
388 * netisr CPU affinity lookup routine for use by protocols.
389 */
390struct mbuf *
391rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
392{
393
394	M_ASSERTPKTHDR(m);
395	*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
396	return (m);
397}
398
399int
400rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
401{
402
403	M_ASSERTPKTHDR(m);
404
405	return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
406	    bucket_id));
407}
408
409/*
410 * Query the RSS hash algorithm.
411 */
412u_int
413rss_gethashalgo(void)
414{
415
416	return (rss_hashalgo);
417}
418
419/*
420 * Query the current RSS key; likely to be used by device drivers when
421 * configuring hardware RSS.  Caller must pass an array of size RSS_KEYSIZE.
422 *
423 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
424 */
425void
426rss_getkey(uint8_t *key)
427{
428
429	bcopy(rss_key, key, sizeof(rss_key));
430}
431
432/*
433 * Query the number of buckets; this may be used by both network device
434 * drivers, which will need to populate hardware shadows of the software
435 * indirection table, and the network stack itself (such as when deciding how
436 * many connection groups to allocate).
437 */
438u_int
439rss_getnumbuckets(void)
440{
441
442	return (rss_buckets);
443}
444
445/*
446 * Query the number of CPUs in use by RSS; may be useful to device drivers
447 * trying to figure out how to map a larger number of CPUs into a smaller
448 * number of receive queues.
449 */
450u_int
451rss_getnumcpus(void)
452{
453
454	return (rss_ncpus);
455}
456
457/*
458 * Return the supported RSS hash configuration.
459 *
460 * NICs should query this to determine what to configure in their redirection
461 * matching table.
462 */
463inline u_int
464rss_gethashconfig(void)
465{
466
467	/* Return 4-tuple for TCP; 2-tuple for others */
468	/*
469	 * UDP may fragment more often than TCP and thus we'll end up with
470	 * NICs returning 2-tuple fragments.
471	 * udp_init() and udplite_init() both currently initialise things
472	 * as 2-tuple.
473	 * So for now disable UDP 4-tuple hashing until all of the other
474	 * pieces are in place.
475	 */
476	return (
477	    RSS_HASHTYPE_RSS_IPV4
478	|    RSS_HASHTYPE_RSS_TCP_IPV4
479	|    RSS_HASHTYPE_RSS_IPV6
480	|    RSS_HASHTYPE_RSS_TCP_IPV6
481	|    RSS_HASHTYPE_RSS_IPV6_EX
482	|    RSS_HASHTYPE_RSS_TCP_IPV6_EX
483#if 0
484	|    RSS_HASHTYPE_RSS_UDP_IPV4
485	|    RSS_HASHTYPE_RSS_UDP_IPV6
486	|    RSS_HASHTYPE_RSS_UDP_IPV6_EX
487#endif
488	);
489}
490
491/*
492 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
493 * it appearing in debugging output unnecessarily.
494 */
495static int
496sysctl_rss_key(SYSCTL_HANDLER_ARGS)
497{
498	uint8_t temp_rss_key[RSS_KEYSIZE];
499	int error;
500
501	error = priv_check(req->td, PRIV_NETINET_HASHKEY);
502	if (error)
503		return (error);
504
505	bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
506	error = sysctl_handle_opaque(oidp, temp_rss_key,
507	    sizeof(temp_rss_key), req);
508	if (error)
509		return (error);
510	if (req->newptr != NULL) {
511		/* XXXRW: Not yet. */
512		return (EINVAL);
513	}
514	return (0);
515}
516SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
517    CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
518    "", "RSS keying material");
519
520static int
521sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
522{
523	struct sbuf *sb;
524	int error;
525	int i;
526
527	error = 0;
528	error = sysctl_wire_old_buffer(req, 0);
529	if (error != 0)
530		return (error);
531	sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
532	if (sb == NULL)
533		return (ENOMEM);
534	for (i = 0; i < rss_buckets; i++) {
535		sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
536		    i,
537		    rss_getcpu(i));
538	}
539	error = sbuf_finish(sb);
540	sbuf_delete(sb);
541
542	return (error);
543}
544SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
545    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
546    sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
547