1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Codel/FQ_Codel and PIE/FQ-PIE Code: 5 * Copyright (C) 2016 Centre for Advanced Internet Architectures, 6 * Swinburne University of Technology, Melbourne, Australia. 7 * Portions of this code were made possible in part by a gift from 8 * The Comcast Innovation Fund. 9 * Implemented by Rasool Al-Saadi <ralsaadi@swin.edu.au> 10 * 11 * Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa 12 * Portions Copyright (c) 2000 Akamba Corp. 13 * All rights reserved 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 */ 36 37#include <sys/cdefs.h> 38/* 39 * Configuration and internal object management for dummynet. 40 */ 41 42#include "opt_inet6.h" 43 44#include <sys/param.h> 45#include <sys/ck.h> 46#include <sys/systm.h> 47#include <sys/malloc.h> 48#include <sys/mbuf.h> 49#include <sys/kernel.h> 50#include <sys/lock.h> 51#include <sys/module.h> 52#include <sys/mutex.h> 53#include <sys/priv.h> 54#include <sys/proc.h> 55#include <sys/rwlock.h> 56#include <sys/socket.h> 57#include <sys/socketvar.h> 58#include <sys/time.h> 59#include <sys/taskqueue.h> 60#include <net/if.h> /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ 61#include <netinet/in.h> 62#include <netinet/ip_var.h> /* ip_output(), IP_FORWARDING */ 63#include <netinet/ip_fw.h> 64#include <netinet/ip_dummynet.h> 65#include <net/vnet.h> 66 67#include <netpfil/ipfw/ip_fw_private.h> 68#include <netpfil/ipfw/dn_heap.h> 69#include <netpfil/ipfw/ip_dn_private.h> 70#ifdef NEW_AQM 71#include <netpfil/ipfw/dn_aqm.h> 72#endif 73#include <netpfil/ipfw/dn_sched.h> 74 75/* which objects to copy */ 76#define DN_C_LINK 0x01 77#define DN_C_SCH 0x02 78#define DN_C_FLOW 0x04 79#define DN_C_FS 0x08 80#define DN_C_QUEUE 0x10 81 82/* we use this argument in case of a schk_new */ 83struct schk_new_arg { 84 struct dn_alg *fp; 85 struct dn_sch *sch; 86}; 87 88/*---- callout hooks. ----*/ 89static struct callout dn_timeout; 90static int dn_tasks_started = 0; 91static int dn_gone; 92static struct task dn_task; 93static struct taskqueue *dn_tq = NULL; 94 95/* global scheduler list */ 96struct mtx sched_mtx; 97CK_LIST_HEAD(, dn_alg) schedlist; 98#ifdef NEW_AQM 99CK_LIST_HEAD(, dn_aqm) aqmlist; /* list of AQMs */ 100#endif 101 102static void 103dummynet(void *arg) 104{ 105 106 (void)arg; /* UNUSED */ 107 taskqueue_enqueue(dn_tq, &dn_task); 108} 109 110void 111dummynet_sched_lock(void) 112{ 113 mtx_lock(&sched_mtx); 114} 115 116void 117dummynet_sched_unlock(void) 118{ 119 mtx_unlock(&sched_mtx); 120} 121 122void 123dn_reschedule(void) 124{ 125 126 if (dn_gone != 0) 127 return; 128 callout_reset_sbt(&dn_timeout, tick_sbt, 0, dummynet, NULL, 129 C_HARDCLOCK | C_DIRECT_EXEC); 130} 131/*----- end of callout hooks -----*/ 132 133#ifdef NEW_AQM 134/* Return AQM descriptor for given type or name. */ 135static struct dn_aqm * 136find_aqm_type(int type, char *name) 137{ 138 struct dn_aqm *d; 139 140 NET_EPOCH_ASSERT(); 141 142 CK_LIST_FOREACH(d, &aqmlist, next) { 143 if (d->type == type || (name && !strcasecmp(d->name, name))) 144 return d; 145 } 146 return NULL; /* not found */ 147} 148#endif 149 150/* Return a scheduler descriptor given the type or name. */ 151static struct dn_alg * 152find_sched_type(int type, char *name) 153{ 154 struct dn_alg *d; 155 156 NET_EPOCH_ASSERT(); 157 158 CK_LIST_FOREACH(d, &schedlist, next) { 159 if (d->type == type || (name && !strcasecmp(d->name, name))) 160 return d; 161 } 162 return NULL; /* not found */ 163} 164 165int 166ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg) 167{ 168 int oldv = *v; 169 const char *op = NULL; 170 if (dflt < lo) 171 dflt = lo; 172 if (dflt > hi) 173 dflt = hi; 174 if (oldv < lo) { 175 *v = dflt; 176 op = "Bump"; 177 } else if (oldv > hi) { 178 *v = hi; 179 op = "Clamp"; 180 } else 181 return *v; 182 if (op && msg && bootverbose) 183 printf("%s %s to %d (was %d)\n", op, msg, *v, oldv); 184 return *v; 185} 186 187/*---- flow_id mask, hash and compare functions ---*/ 188/* 189 * The flow_id includes the 5-tuple, the queue/pipe number 190 * which we store in the extra area in host order, 191 * and for ipv6 also the flow_id6. 192 * XXX see if we want the tos byte (can store in 'flags') 193 */ 194static struct ipfw_flow_id * 195flow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id) 196{ 197 int is_v6 = IS_IP6_FLOW_ID(id); 198 199 id->dst_port &= mask->dst_port; 200 id->src_port &= mask->src_port; 201 id->proto &= mask->proto; 202 id->extra &= mask->extra; 203 if (is_v6) { 204 APPLY_MASK(&id->dst_ip6, &mask->dst_ip6); 205 APPLY_MASK(&id->src_ip6, &mask->src_ip6); 206 id->flow_id6 &= mask->flow_id6; 207 } else { 208 id->dst_ip &= mask->dst_ip; 209 id->src_ip &= mask->src_ip; 210 } 211 return id; 212} 213 214/* computes an OR of two masks, result in dst and also returned */ 215static struct ipfw_flow_id * 216flow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst) 217{ 218 int is_v6 = IS_IP6_FLOW_ID(dst); 219 220 dst->dst_port |= src->dst_port; 221 dst->src_port |= src->src_port; 222 dst->proto |= src->proto; 223 dst->extra |= src->extra; 224 if (is_v6) { 225#define OR_MASK(_d, _s) \ 226 (_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \ 227 (_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \ 228 (_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \ 229 (_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3]; 230 OR_MASK(&dst->dst_ip6, &src->dst_ip6); 231 OR_MASK(&dst->src_ip6, &src->src_ip6); 232#undef OR_MASK 233 dst->flow_id6 |= src->flow_id6; 234 } else { 235 dst->dst_ip |= src->dst_ip; 236 dst->src_ip |= src->src_ip; 237 } 238 return dst; 239} 240 241static int 242nonzero_mask(struct ipfw_flow_id *m) 243{ 244 if (m->dst_port || m->src_port || m->proto || m->extra) 245 return 1; 246 if (IS_IP6_FLOW_ID(m)) { 247 return 248 m->dst_ip6.__u6_addr.__u6_addr32[0] || 249 m->dst_ip6.__u6_addr.__u6_addr32[1] || 250 m->dst_ip6.__u6_addr.__u6_addr32[2] || 251 m->dst_ip6.__u6_addr.__u6_addr32[3] || 252 m->src_ip6.__u6_addr.__u6_addr32[0] || 253 m->src_ip6.__u6_addr.__u6_addr32[1] || 254 m->src_ip6.__u6_addr.__u6_addr32[2] || 255 m->src_ip6.__u6_addr.__u6_addr32[3] || 256 m->flow_id6; 257 } else { 258 return m->dst_ip || m->src_ip; 259 } 260} 261 262/* XXX we may want a better hash function */ 263static uint32_t 264flow_id_hash(struct ipfw_flow_id *id) 265{ 266 uint32_t i; 267 268 if (IS_IP6_FLOW_ID(id)) { 269 uint32_t *d = (uint32_t *)&id->dst_ip6; 270 uint32_t *s = (uint32_t *)&id->src_ip6; 271 i = (d[0] ) ^ (d[1]) ^ 272 (d[2] ) ^ (d[3]) ^ 273 (d[0] >> 15) ^ (d[1] >> 15) ^ 274 (d[2] >> 15) ^ (d[3] >> 15) ^ 275 (s[0] << 1) ^ (s[1] << 1) ^ 276 (s[2] << 1) ^ (s[3] << 1) ^ 277 (s[0] << 16) ^ (s[1] << 16) ^ 278 (s[2] << 16) ^ (s[3] << 16) ^ 279 (id->dst_port << 1) ^ (id->src_port) ^ 280 (id->extra) ^ 281 (id->proto ) ^ (id->flow_id6); 282 } else { 283 i = (id->dst_ip) ^ (id->dst_ip >> 15) ^ 284 (id->src_ip << 1) ^ (id->src_ip >> 16) ^ 285 (id->extra) ^ 286 (id->dst_port << 1) ^ (id->src_port) ^ (id->proto); 287 } 288 return i; 289} 290 291/* Like bcmp, returns 0 if ids match, 1 otherwise. */ 292static int 293flow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2) 294{ 295 int is_v6 = IS_IP6_FLOW_ID(id1); 296 297 if (!is_v6) { 298 if (IS_IP6_FLOW_ID(id2)) 299 return 1; /* different address families */ 300 301 return (id1->dst_ip == id2->dst_ip && 302 id1->src_ip == id2->src_ip && 303 id1->dst_port == id2->dst_port && 304 id1->src_port == id2->src_port && 305 id1->proto == id2->proto && 306 id1->extra == id2->extra) ? 0 : 1; 307 } 308 /* the ipv6 case */ 309 return ( 310 !bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) && 311 !bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) && 312 id1->dst_port == id2->dst_port && 313 id1->src_port == id2->src_port && 314 id1->proto == id2->proto && 315 id1->extra == id2->extra && 316 id1->flow_id6 == id2->flow_id6) ? 0 : 1; 317} 318/*--------- end of flow-id mask, hash and compare ---------*/ 319 320/*--- support functions for the qht hashtable ---- 321 * Entries are hashed by flow-id 322 */ 323static uint32_t 324q_hash(uintptr_t key, int flags, void *arg) 325{ 326 /* compute the hash slot from the flow id */ 327 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 328 &((struct dn_queue *)key)->ni.fid : 329 (struct ipfw_flow_id *)key; 330 331 return flow_id_hash(id); 332} 333 334static int 335q_match(void *obj, uintptr_t key, int flags, void *arg) 336{ 337 struct dn_queue *o = (struct dn_queue *)obj; 338 struct ipfw_flow_id *id2; 339 340 if (flags & DNHT_KEY_IS_OBJ) { 341 /* compare pointers */ 342 id2 = &((struct dn_queue *)key)->ni.fid; 343 } else { 344 id2 = (struct ipfw_flow_id *)key; 345 } 346 return (0 == flow_id_cmp(&o->ni.fid, id2)); 347} 348 349/* 350 * create a new queue instance for the given 'key'. 351 */ 352static void * 353q_new(uintptr_t key, int flags, void *arg) 354{ 355 struct dn_queue *q, *template = arg; 356 struct dn_fsk *fs = template->fs; 357 int size = sizeof(*q) + fs->sched->fp->q_datalen; 358 359 q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO); 360 if (q == NULL) { 361 D("no memory for new queue"); 362 return NULL; 363 } 364 365 set_oid(&q->ni.oid, DN_QUEUE, size); 366 if (fs->fs.flags & DN_QHT_HASH) 367 q->ni.fid = *(struct ipfw_flow_id *)key; 368 q->fs = fs; 369 q->_si = template->_si; 370 q->_si->q_count++; 371 372 if (fs->sched->fp->new_queue) 373 fs->sched->fp->new_queue(q); 374 375#ifdef NEW_AQM 376 /* call AQM init function after creating a queue*/ 377 if (fs->aqmfp && fs->aqmfp->init) 378 if(fs->aqmfp->init(q)) 379 D("unable to init AQM for fs %d", fs->fs.fs_nr); 380#endif 381 V_dn_cfg.queue_count++; 382 383 return q; 384} 385 386/* 387 * Notify schedulers that a queue is going away. 388 * If (flags & DN_DESTROY), also free the packets. 389 * The version for callbacks is called q_delete_cb(). 390 */ 391static void 392dn_delete_queue(struct dn_queue *q, int flags) 393{ 394 struct dn_fsk *fs = q->fs; 395 396#ifdef NEW_AQM 397 /* clean up AQM status for queue 'q' 398 * cleanup here is called just with MULTIQUEUE 399 */ 400 if (fs && fs->aqmfp && fs->aqmfp->cleanup) 401 fs->aqmfp->cleanup(q); 402#endif 403 // D("fs %p si %p\n", fs, q->_si); 404 /* notify the parent scheduler that the queue is going away */ 405 if (fs && fs->sched->fp->free_queue) 406 fs->sched->fp->free_queue(q); 407 q->_si->q_count--; 408 q->_si = NULL; 409 if (flags & DN_DESTROY) { 410 if (q->mq.head) 411 dn_free_pkts(q->mq.head); 412 bzero(q, sizeof(*q)); // safety 413 free(q, M_DUMMYNET); 414 V_dn_cfg.queue_count--; 415 } 416} 417 418static int 419q_delete_cb(void *q, void *arg) 420{ 421 int flags = (int)(uintptr_t)arg; 422 dn_delete_queue(q, flags); 423 return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0; 424} 425 426/* 427 * calls dn_delete_queue/q_delete_cb on all queues, 428 * which notifies the parent scheduler and possibly drains packets. 429 * flags & DN_DESTROY: drains queues and destroy qht; 430 */ 431static void 432qht_delete(struct dn_fsk *fs, int flags) 433{ 434 ND("fs %d start flags %d qht %p", 435 fs->fs.fs_nr, flags, fs->qht); 436 if (!fs->qht) 437 return; 438 if (fs->fs.flags & DN_QHT_HASH) { 439 dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags); 440 if (flags & DN_DESTROY) { 441 dn_ht_free(fs->qht, 0); 442 fs->qht = NULL; 443 } 444 } else { 445 dn_delete_queue((struct dn_queue *)(fs->qht), flags); 446 if (flags & DN_DESTROY) 447 fs->qht = NULL; 448 } 449} 450 451/* 452 * Find and possibly create the queue for a MULTIQUEUE scheduler. 453 * We never call it for !MULTIQUEUE (the queue is in the sch_inst). 454 */ 455struct dn_queue * 456ipdn_q_find(struct dn_fsk *fs, struct dn_sch_inst *si, 457 struct ipfw_flow_id *id) 458{ 459 struct dn_queue template; 460 461 template._si = si; 462 template.fs = fs; 463 464 if (fs->fs.flags & DN_QHT_HASH) { 465 struct ipfw_flow_id masked_id; 466 if (fs->qht == NULL) { 467 fs->qht = dn_ht_init(NULL, fs->fs.buckets, 468 offsetof(struct dn_queue, q_next), 469 q_hash, q_match, q_new); 470 if (fs->qht == NULL) 471 return NULL; 472 } 473 masked_id = *id; 474 flow_id_mask(&fs->fsk_mask, &masked_id); 475 return dn_ht_find(fs->qht, (uintptr_t)&masked_id, 476 DNHT_INSERT, &template); 477 } else { 478 if (fs->qht == NULL) 479 fs->qht = q_new(0, 0, &template); 480 return (struct dn_queue *)fs->qht; 481 } 482} 483/*--- end of queue hash table ---*/ 484 485/*--- support functions for the sch_inst hashtable ---- 486 * 487 * These are hashed by flow-id 488 */ 489static uint32_t 490si_hash(uintptr_t key, int flags, void *arg) 491{ 492 /* compute the hash slot from the flow id */ 493 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 494 &((struct dn_sch_inst *)key)->ni.fid : 495 (struct ipfw_flow_id *)key; 496 497 return flow_id_hash(id); 498} 499 500static int 501si_match(void *obj, uintptr_t key, int flags, void *arg) 502{ 503 struct dn_sch_inst *o = obj; 504 struct ipfw_flow_id *id2; 505 506 id2 = (flags & DNHT_KEY_IS_OBJ) ? 507 &((struct dn_sch_inst *)key)->ni.fid : 508 (struct ipfw_flow_id *)key; 509 return flow_id_cmp(&o->ni.fid, id2) == 0; 510} 511 512/* 513 * create a new instance for the given 'key' 514 * Allocate memory for instance, delay line and scheduler private data. 515 */ 516static void * 517si_new(uintptr_t key, int flags, void *arg) 518{ 519 struct dn_schk *s = arg; 520 struct dn_sch_inst *si; 521 int l = sizeof(*si) + s->fp->si_datalen; 522 523 si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 524 if (si == NULL) 525 goto error; 526 527 /* Set length only for the part passed up to userland. */ 528 set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow)); 529 set_oid(&(si->dline.oid), DN_DELAY_LINE, 530 sizeof(struct delay_line)); 531 /* mark si and dline as outside the event queue */ 532 si->ni.oid.id = si->dline.oid.id = -1; 533 534 si->sched = s; 535 si->dline.si = si; 536 537 if (s->fp->new_sched && s->fp->new_sched(si)) { 538 D("new_sched error"); 539 goto error; 540 } 541 if (s->sch.flags & DN_HAVE_MASK) 542 si->ni.fid = *(struct ipfw_flow_id *)key; 543 544#ifdef NEW_AQM 545 /* init AQM status for !DN_MULTIQUEUE sched*/ 546 if (!(s->fp->flags & DN_MULTIQUEUE)) 547 if (s->fs->aqmfp && s->fs->aqmfp->init) 548 if(s->fs->aqmfp->init((struct dn_queue *)(si + 1))) { 549 D("unable to init AQM for fs %d", s->fs->fs.fs_nr); 550 goto error; 551 } 552#endif 553 554 V_dn_cfg.si_count++; 555 return si; 556 557error: 558 if (si) { 559 bzero(si, sizeof(*si)); // safety 560 free(si, M_DUMMYNET); 561 } 562 return NULL; 563} 564 565/* 566 * Callback from siht to delete all scheduler instances. Remove 567 * si and delay line from the system heap, destroy all queues. 568 * We assume that all flowset have been notified and do not 569 * point to us anymore. 570 */ 571static int 572si_destroy(void *_si, void *arg) 573{ 574 struct dn_sch_inst *si = _si; 575 struct dn_schk *s = si->sched; 576 struct delay_line *dl = &si->dline; 577 578 if (dl->oid.subtype) /* remove delay line from event heap */ 579 heap_extract(&V_dn_cfg.evheap, dl); 580 dn_free_pkts(dl->mq.head); /* drain delay line */ 581 if (si->kflags & DN_ACTIVE) /* remove si from event heap */ 582 heap_extract(&V_dn_cfg.evheap, si); 583 584#ifdef NEW_AQM 585 /* clean up AQM status for !DN_MULTIQUEUE sched 586 * Note that all queues belong to fs were cleaned up in fsk_detach. 587 * When drain_scheduler is called s->fs and q->fs are pointing 588 * to a correct fs, so we can use fs in this case. 589 */ 590 if (!(s->fp->flags & DN_MULTIQUEUE)) { 591 struct dn_queue *q = (struct dn_queue *)(si + 1); 592 if (q->aqm_status && q->fs->aqmfp) 593 if (q->fs->aqmfp->cleanup) 594 q->fs->aqmfp->cleanup(q); 595 } 596#endif 597 if (s->fp->free_sched) 598 s->fp->free_sched(si); 599 bzero(si, sizeof(*si)); /* safety */ 600 free(si, M_DUMMYNET); 601 V_dn_cfg.si_count--; 602 return DNHT_SCAN_DEL; 603} 604 605/* 606 * Find the scheduler instance for this packet. If we need to apply 607 * a mask, do on a local copy of the flow_id to preserve the original. 608 * Assume siht is always initialized if we have a mask. 609 */ 610struct dn_sch_inst * 611ipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id) 612{ 613 614 if (s->sch.flags & DN_HAVE_MASK) { 615 struct ipfw_flow_id id_t = *id; 616 flow_id_mask(&s->sch.sched_mask, &id_t); 617 return dn_ht_find(s->siht, (uintptr_t)&id_t, 618 DNHT_INSERT, s); 619 } 620 if (!s->siht) 621 s->siht = si_new(0, 0, s); 622 return (struct dn_sch_inst *)s->siht; 623} 624 625/* callback to flush credit for the scheduler instance */ 626static int 627si_reset_credit(void *_si, void *arg) 628{ 629 struct dn_sch_inst *si = _si; 630 struct dn_link *p = &si->sched->link; 631 632 si->credit = p->burst + (V_dn_cfg.io_fast ? p->bandwidth : 0); 633 return 0; 634} 635 636static void 637schk_reset_credit(struct dn_schk *s) 638{ 639 if (s->sch.flags & DN_HAVE_MASK) 640 dn_ht_scan(s->siht, si_reset_credit, NULL); 641 else if (s->siht) 642 si_reset_credit(s->siht, NULL); 643} 644/*---- end of sch_inst hashtable ---------------------*/ 645 646/*------------------------------------------------------- 647 * flowset hash (fshash) support. Entries are hashed by fs_nr. 648 * New allocations are put in the fsunlinked list, from which 649 * they are removed when they point to a specific scheduler. 650 */ 651static uint32_t 652fsk_hash(uintptr_t key, int flags, void *arg) 653{ 654 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 655 ((struct dn_fsk *)key)->fs.fs_nr; 656 657 return ( (i>>8)^(i>>4)^i ); 658} 659 660static int 661fsk_match(void *obj, uintptr_t key, int flags, void *arg) 662{ 663 struct dn_fsk *fs = obj; 664 int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 665 ((struct dn_fsk *)key)->fs.fs_nr; 666 667 return (fs->fs.fs_nr == i); 668} 669 670static void * 671fsk_new(uintptr_t key, int flags, void *arg) 672{ 673 struct dn_fsk *fs; 674 675 fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO); 676 if (fs) { 677 set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs)); 678 V_dn_cfg.fsk_count++; 679 fs->drain_bucket = 0; 680 SLIST_INSERT_HEAD(&V_dn_cfg.fsu, fs, sch_chain); 681 } 682 return fs; 683} 684 685#ifdef NEW_AQM 686/* callback function for cleaning up AQM queue status belongs to a flowset 687 * connected to scheduler instance '_si' (for !DN_MULTIQUEUE only). 688 */ 689static int 690si_cleanup_q(void *_si, void *arg) 691{ 692 struct dn_sch_inst *si = _si; 693 694 if (!(si->sched->fp->flags & DN_MULTIQUEUE)) { 695 if (si->sched->fs->aqmfp && si->sched->fs->aqmfp->cleanup) 696 si->sched->fs->aqmfp->cleanup((struct dn_queue *) (si+1)); 697 } 698 return 0; 699} 700 701/* callback to clean up queue AQM status.*/ 702static int 703q_cleanup_q(void *_q, void *arg) 704{ 705 struct dn_queue *q = _q; 706 q->fs->aqmfp->cleanup(q); 707 return 0; 708} 709 710/* Clean up all AQM queues status belongs to flowset 'fs' and then 711 * deconfig AQM for flowset 'fs' 712 */ 713static void 714aqm_cleanup_deconfig_fs(struct dn_fsk *fs) 715{ 716 struct dn_sch_inst *si; 717 718 /* clean up AQM status for all queues for !DN_MULTIQUEUE sched*/ 719 if (fs->fs.fs_nr > DN_MAX_ID) { 720 if (fs->sched && !(fs->sched->fp->flags & DN_MULTIQUEUE)) { 721 if (fs->sched->sch.flags & DN_HAVE_MASK) 722 dn_ht_scan(fs->sched->siht, si_cleanup_q, NULL); 723 else { 724 /* single si i.e. no sched mask */ 725 si = (struct dn_sch_inst *) fs->sched->siht; 726 if (si && fs->aqmfp && fs->aqmfp->cleanup) 727 fs->aqmfp->cleanup((struct dn_queue *) (si+1)); 728 } 729 } 730 } 731 732 /* clean up AQM status for all queues for DN_MULTIQUEUE sched*/ 733 if (fs->sched && fs->sched->fp->flags & DN_MULTIQUEUE && fs->qht) { 734 if (fs->fs.flags & DN_QHT_HASH) 735 dn_ht_scan(fs->qht, q_cleanup_q, NULL); 736 else 737 fs->aqmfp->cleanup((struct dn_queue *)(fs->qht)); 738 } 739 740 /* deconfig AQM */ 741 if(fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) 742 fs->aqmfp->deconfig(fs); 743} 744#endif 745 746/* 747 * detach flowset from its current scheduler. Flags as follows: 748 * DN_DETACH removes from the fsk_list 749 * DN_DESTROY deletes individual queues 750 * DN_DELETE_FS destroys the flowset (otherwise goes in unlinked). 751 */ 752static void 753fsk_detach(struct dn_fsk *fs, int flags) 754{ 755 if (flags & DN_DELETE_FS) 756 flags |= DN_DESTROY; 757 ND("fs %d from sched %d flags %s %s %s", 758 fs->fs.fs_nr, fs->fs.sched_nr, 759 (flags & DN_DELETE_FS) ? "DEL_FS":"", 760 (flags & DN_DESTROY) ? "DEL":"", 761 (flags & DN_DETACH) ? "DET":""); 762 if (flags & DN_DETACH) { /* detach from the list */ 763 struct dn_fsk_head *h; 764 h = fs->sched ? &fs->sched->fsk_list : &V_dn_cfg.fsu; 765 SLIST_REMOVE(h, fs, dn_fsk, sch_chain); 766 } 767 /* Free the RED parameters, they will be recomputed on 768 * subsequent attach if needed. 769 */ 770 free(fs->w_q_lookup, M_DUMMYNET); 771 fs->w_q_lookup = NULL; 772 qht_delete(fs, flags); 773#ifdef NEW_AQM 774 aqm_cleanup_deconfig_fs(fs); 775#endif 776 777 if (fs->sched && fs->sched->fp->free_fsk) 778 fs->sched->fp->free_fsk(fs); 779 fs->sched = NULL; 780 if (flags & DN_DELETE_FS) { 781 bzero(fs, sizeof(*fs)); /* safety */ 782 free(fs, M_DUMMYNET); 783 V_dn_cfg.fsk_count--; 784 } else { 785 SLIST_INSERT_HEAD(&V_dn_cfg.fsu, fs, sch_chain); 786 } 787} 788 789/* 790 * Detach or destroy all flowsets in a list. 791 * flags specifies what to do: 792 * DN_DESTROY: flush all queues 793 * DN_DELETE_FS: DN_DESTROY + destroy flowset 794 * DN_DELETE_FS implies DN_DESTROY 795 */ 796static void 797fsk_detach_list(struct dn_fsk_head *h, int flags) 798{ 799 struct dn_fsk *fs; 800 int n __unused = 0; /* only for stats */ 801 802 ND("head %p flags %x", h, flags); 803 while ((fs = SLIST_FIRST(h))) { 804 SLIST_REMOVE_HEAD(h, sch_chain); 805 n++; 806 fsk_detach(fs, flags); 807 } 808 ND("done %d flowsets", n); 809} 810 811/* 812 * called on 'queue X delete' -- removes the flowset from fshash, 813 * deletes all queues for the flowset, and removes the flowset. 814 */ 815static int 816delete_fs(int i, int locked) 817{ 818 struct dn_fsk *fs; 819 int err = 0; 820 821 if (!locked) 822 DN_BH_WLOCK(); 823 fs = dn_ht_find(V_dn_cfg.fshash, i, DNHT_REMOVE, NULL); 824 ND("fs %d found %p", i, fs); 825 if (fs) { 826 fsk_detach(fs, DN_DETACH | DN_DELETE_FS); 827 err = 0; 828 } else 829 err = EINVAL; 830 if (!locked) 831 DN_BH_WUNLOCK(); 832 return err; 833} 834 835/*----- end of flowset hashtable support -------------*/ 836 837/*------------------------------------------------------------ 838 * Scheduler hash. When searching by index we pass sched_nr, 839 * otherwise we pass struct dn_sch * which is the first field in 840 * struct dn_schk so we can cast between the two. We use this trick 841 * because in the create phase (but it should be fixed). 842 */ 843static uint32_t 844schk_hash(uintptr_t key, int flags, void *_arg) 845{ 846 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 847 ((struct dn_schk *)key)->sch.sched_nr; 848 return ( (i>>8)^(i>>4)^i ); 849} 850 851static int 852schk_match(void *obj, uintptr_t key, int flags, void *_arg) 853{ 854 struct dn_schk *s = (struct dn_schk *)obj; 855 int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 856 ((struct dn_schk *)key)->sch.sched_nr; 857 return (s->sch.sched_nr == i); 858} 859 860/* 861 * Create the entry and intialize with the sched hash if needed. 862 * Leave s->fp unset so we can tell whether a dn_ht_find() returns 863 * a new object or a previously existing one. 864 */ 865static void * 866schk_new(uintptr_t key, int flags, void *arg) 867{ 868 struct schk_new_arg *a = arg; 869 struct dn_schk *s; 870 int l = sizeof(*s) +a->fp->schk_datalen; 871 872 s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 873 if (s == NULL) 874 return NULL; 875 set_oid(&s->link.oid, DN_LINK, sizeof(s->link)); 876 s->sch = *a->sch; // copy initial values 877 s->link.link_nr = s->sch.sched_nr; 878 SLIST_INIT(&s->fsk_list); 879 /* initialize the hash table or create the single instance */ 880 s->fp = a->fp; /* si_new needs this */ 881 s->drain_bucket = 0; 882 if (s->sch.flags & DN_HAVE_MASK) { 883 s->siht = dn_ht_init(NULL, s->sch.buckets, 884 offsetof(struct dn_sch_inst, si_next), 885 si_hash, si_match, si_new); 886 if (s->siht == NULL) { 887 free(s, M_DUMMYNET); 888 return NULL; 889 } 890 } 891 s->fp = NULL; /* mark as a new scheduler */ 892 V_dn_cfg.schk_count++; 893 return s; 894} 895 896/* 897 * Callback for sched delete. Notify all attached flowsets to 898 * detach from the scheduler, destroy the internal flowset, and 899 * all instances. The scheduler goes away too. 900 * arg is 0 (only detach flowsets and destroy instances) 901 * DN_DESTROY (detach & delete queues, delete schk) 902 * or DN_DELETE_FS (delete queues and flowsets, delete schk) 903 */ 904static int 905schk_delete_cb(void *obj, void *arg) 906{ 907 struct dn_schk *s = obj; 908#if 0 909 int a = (int)arg; 910 ND("sched %d arg %s%s", 911 s->sch.sched_nr, 912 a&DN_DESTROY ? "DEL ":"", 913 a&DN_DELETE_FS ? "DEL_FS":""); 914#endif 915 fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0); 916 /* no more flowset pointing to us now */ 917 if (s->sch.flags & DN_HAVE_MASK) { 918 dn_ht_scan(s->siht, si_destroy, NULL); 919 dn_ht_free(s->siht, 0); 920 } else if (s->siht) 921 si_destroy(s->siht, NULL); 922 923 free(s->profile, M_DUMMYNET); 924 s->profile = NULL; 925 s->siht = NULL; 926 if (s->fp->destroy) 927 s->fp->destroy(s); 928 bzero(s, sizeof(*s)); // safety 929 free(obj, M_DUMMYNET); 930 V_dn_cfg.schk_count--; 931 return DNHT_SCAN_DEL; 932} 933 934/* 935 * called on a 'sched X delete' command. Deletes a single scheduler. 936 * This is done by removing from the schedhash, unlinking all 937 * flowsets and deleting their traffic. 938 */ 939static int 940delete_schk(int i) 941{ 942 struct dn_schk *s; 943 944 s = dn_ht_find(V_dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 945 ND("%d %p", i, s); 946 if (!s) 947 return EINVAL; 948 delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */ 949 /* then detach flowsets, delete traffic */ 950 schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY); 951 return 0; 952} 953/*--- end of schk hashtable support ---*/ 954 955static int 956copy_obj(char **start, char *end, void *_o, const char *msg, int i) 957{ 958 struct dn_id o; 959 union { 960 struct dn_link l; 961 struct dn_schk s; 962 } dn; 963 int have = end - *start; 964 965 memcpy(&o, _o, sizeof(o)); 966 if (have < o.len || o.len == 0 || o.type == 0) { 967 D("(WARN) type %d %s %d have %d need %d", 968 o.type, msg, i, have, o.len); 969 return 1; 970 } 971 ND("type %d %s %d len %d", o.type, msg, i, o.len); 972 if (o.type == DN_LINK) { 973 memcpy(&dn.l, _o, sizeof(dn.l)); 974 /* Adjust burst parameter for link */ 975 dn.l.burst = div64(dn.l.burst, 8 * hz); 976 dn.l.delay = dn.l.delay * 1000 / hz; 977 memcpy(*start, &dn.l, sizeof(dn.l)); 978 } else if (o.type == DN_SCH) { 979 /* Set dn.s.sch.oid.id to the number of instances */ 980 memcpy(&dn.s, _o, sizeof(dn.s)); 981 dn.s.sch.oid.id = (dn.s.sch.flags & DN_HAVE_MASK) ? 982 dn_ht_entries(dn.s.siht) : (dn.s.siht ? 1 : 0); 983 memcpy(*start, &dn.s, sizeof(dn.s)); 984 } else 985 memcpy(*start, _o, o.len); 986 *start += o.len; 987 return 0; 988} 989 990/* Specific function to copy a queue. 991 * Copies only the user-visible part of a queue (which is in 992 * a struct dn_flow), and sets len accordingly. 993 */ 994static int 995copy_obj_q(char **start, char *end, void *_o, const char *msg, int i) 996{ 997 struct dn_id *o = _o; 998 int have = end - *start; 999 int len = sizeof(struct dn_flow); /* see above comment */ 1000 1001 if (have < len || o->len == 0 || o->type != DN_QUEUE) { 1002 D("ERROR type %d %s %d have %d need %d", 1003 o->type, msg, i, have, len); 1004 return 1; 1005 } 1006 ND("type %d %s %d len %d", o->type, msg, i, len); 1007 memcpy(*start, _o, len); 1008 ((struct dn_id*)(*start))->len = len; 1009 *start += len; 1010 return 0; 1011} 1012 1013static int 1014copy_q_cb(void *obj, void *arg) 1015{ 1016 struct dn_queue *q = obj; 1017 struct copy_args *a = arg; 1018 struct dn_flow *ni = (struct dn_flow *)(*a->start); 1019 if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1)) 1020 return DNHT_SCAN_END; 1021 ni->oid.type = DN_FLOW; /* override the DN_QUEUE */ 1022 ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL); 1023 return 0; 1024} 1025 1026static int 1027copy_q(struct copy_args *a, struct dn_fsk *fs, int flags) 1028{ 1029 if (!fs->qht) 1030 return 0; 1031 if (fs->fs.flags & DN_QHT_HASH) 1032 dn_ht_scan(fs->qht, copy_q_cb, a); 1033 else 1034 copy_q_cb(fs->qht, a); 1035 return 0; 1036} 1037 1038/* 1039 * This routine only copies the initial part of a profile ? XXX 1040 */ 1041static int 1042copy_profile(struct copy_args *a, struct dn_profile *p) 1043{ 1044 int have = a->end - *a->start; 1045 /* XXX here we check for max length */ 1046 int profile_len = sizeof(struct dn_profile) - 1047 ED_MAX_SAMPLES_NO*sizeof(int); 1048 1049 if (p == NULL) 1050 return 0; 1051 if (have < profile_len) { 1052 D("error have %d need %d", have, profile_len); 1053 return 1; 1054 } 1055 memcpy(*a->start, p, profile_len); 1056 ((struct dn_id *)(*a->start))->len = profile_len; 1057 *a->start += profile_len; 1058 return 0; 1059} 1060 1061static int 1062copy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags) 1063{ 1064 struct dn_fs *ufs = (struct dn_fs *)(*a->start); 1065 if (!fs) 1066 return 0; 1067 ND("flowset %d", fs->fs.fs_nr); 1068 if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr)) 1069 return DNHT_SCAN_END; 1070 ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ? 1071 dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0); 1072 if (flags) { /* copy queues */ 1073 copy_q(a, fs, 0); 1074 } 1075 return 0; 1076} 1077 1078static int 1079copy_si_cb(void *obj, void *arg) 1080{ 1081 struct dn_sch_inst *si = obj; 1082 struct copy_args *a = arg; 1083 struct dn_flow *ni = (struct dn_flow *)(*a->start); 1084 if (copy_obj(a->start, a->end, &si->ni, "inst", 1085 si->sched->sch.sched_nr)) 1086 return DNHT_SCAN_END; 1087 ni->oid.type = DN_FLOW; /* override the DN_SCH_I */ 1088 ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL); 1089 return 0; 1090} 1091 1092static int 1093copy_si(struct copy_args *a, struct dn_schk *s, int flags) 1094{ 1095 if (s->sch.flags & DN_HAVE_MASK) 1096 dn_ht_scan(s->siht, copy_si_cb, a); 1097 else if (s->siht) 1098 copy_si_cb(s->siht, a); 1099 return 0; 1100} 1101 1102/* 1103 * compute a list of children of a scheduler and copy up 1104 */ 1105static int 1106copy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags) 1107{ 1108 struct dn_fsk *fs; 1109 struct dn_id *o; 1110 uint32_t *p; 1111 1112 int n = 0, space = sizeof(*o); 1113 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 1114 if (fs->fs.fs_nr < DN_MAX_ID) 1115 n++; 1116 } 1117 space += n * sizeof(uint32_t); 1118 DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n); 1119 if (a->end - *(a->start) < space) 1120 return DNHT_SCAN_END; 1121 o = (struct dn_id *)(*(a->start)); 1122 o->len = space; 1123 *a->start += o->len; 1124 o->type = DN_TEXT; 1125 p = (uint32_t *)(o+1); 1126 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) 1127 if (fs->fs.fs_nr < DN_MAX_ID) 1128 *p++ = fs->fs.fs_nr; 1129 return 0; 1130} 1131 1132static int 1133copy_data_helper(void *_o, void *_arg) 1134{ 1135 struct copy_args *a = _arg; 1136 uint32_t *r = a->extra->r; /* start of first range */ 1137 uint32_t *lim; /* first invalid pointer */ 1138 int n; 1139 1140 lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len); 1141 1142 if (a->type == DN_LINK || a->type == DN_SCH) { 1143 /* pipe|sched show, we receive a dn_schk */ 1144 struct dn_schk *s = _o; 1145 1146 n = s->sch.sched_nr; 1147 if (a->type == DN_SCH && n >= DN_MAX_ID) 1148 return 0; /* not a scheduler */ 1149 if (a->type == DN_LINK && n <= DN_MAX_ID) 1150 return 0; /* not a pipe */ 1151 1152 /* see if the object is within one of our ranges */ 1153 for (;r < lim; r += 2) { 1154 if (n < r[0] || n > r[1]) 1155 continue; 1156 /* Found a valid entry, copy and we are done */ 1157 if (a->flags & DN_C_LINK) { 1158 if (copy_obj(a->start, a->end, 1159 &s->link, "link", n)) 1160 return DNHT_SCAN_END; 1161 if (copy_profile(a, s->profile)) 1162 return DNHT_SCAN_END; 1163 if (copy_flowset(a, s->fs, 0)) 1164 return DNHT_SCAN_END; 1165 } 1166 if (a->flags & DN_C_SCH) { 1167 if (copy_obj(a->start, a->end, 1168 &s->sch, "sched", n)) 1169 return DNHT_SCAN_END; 1170 /* list all attached flowsets */ 1171 if (copy_fsk_list(a, s, 0)) 1172 return DNHT_SCAN_END; 1173 } 1174 if (a->flags & DN_C_FLOW) 1175 copy_si(a, s, 0); 1176 break; 1177 } 1178 } else if (a->type == DN_FS) { 1179 /* queue show, skip internal flowsets */ 1180 struct dn_fsk *fs = _o; 1181 1182 n = fs->fs.fs_nr; 1183 if (n >= DN_MAX_ID) 1184 return 0; 1185 /* see if the object is within one of our ranges */ 1186 for (;r < lim; r += 2) { 1187 if (n < r[0] || n > r[1]) 1188 continue; 1189 if (copy_flowset(a, fs, 0)) 1190 return DNHT_SCAN_END; 1191 copy_q(a, fs, 0); 1192 break; /* we are done */ 1193 } 1194 } 1195 return 0; 1196} 1197 1198static inline struct dn_schk * 1199locate_scheduler(int i) 1200{ 1201 return dn_ht_find(V_dn_cfg.schedhash, i, 0, NULL); 1202} 1203 1204/* 1205 * red parameters are in fixed point arithmetic. 1206 */ 1207static int 1208config_red(struct dn_fsk *fs) 1209{ 1210 int64_t s, idle, weight, w0; 1211 int t, i; 1212 1213 fs->w_q = fs->fs.w_q; 1214 fs->max_p = fs->fs.max_p; 1215 ND("called"); 1216 /* Doing stuff that was in userland */ 1217 i = fs->sched->link.bandwidth; 1218 s = (i <= 0) ? 0 : 1219 hz * V_dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i; 1220 1221 idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */ 1222 fs->lookup_step = div64(idle , V_dn_cfg.red_lookup_depth); 1223 /* fs->lookup_step not scaled, */ 1224 if (!fs->lookup_step) 1225 fs->lookup_step = 1; 1226 w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled 1227 1228 for (t = fs->lookup_step; t > 1; --t) 1229 weight = SCALE_MUL(weight, w0); 1230 fs->lookup_weight = (int)(weight); // scaled 1231 1232 /* Now doing stuff that was in kerneland */ 1233 fs->min_th = SCALE(fs->fs.min_th); 1234 fs->max_th = SCALE(fs->fs.max_th); 1235 1236 if (fs->fs.max_th == fs->fs.min_th) 1237 fs->c_1 = fs->max_p; 1238 else 1239 fs->c_1 = SCALE((int64_t)(fs->max_p)) / (fs->fs.max_th - fs->fs.min_th); 1240 fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th)); 1241 1242 if (fs->fs.flags & DN_IS_GENTLE_RED) { 1243 fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th; 1244 fs->c_4 = SCALE(1) - 2 * fs->max_p; 1245 } 1246 1247 /* If the lookup table already exist, free and create it again. */ 1248 free(fs->w_q_lookup, M_DUMMYNET); 1249 fs->w_q_lookup = NULL; 1250 if (V_dn_cfg.red_lookup_depth == 0) { 1251 printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth" 1252 "must be > 0\n"); 1253 fs->fs.flags &= ~DN_IS_RED; 1254 fs->fs.flags &= ~DN_IS_GENTLE_RED; 1255 return (EINVAL); 1256 } 1257 fs->lookup_depth = V_dn_cfg.red_lookup_depth; 1258 fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int), 1259 M_DUMMYNET, M_NOWAIT); 1260 if (fs->w_q_lookup == NULL) { 1261 printf("dummynet: sorry, cannot allocate red lookup table\n"); 1262 fs->fs.flags &= ~DN_IS_RED; 1263 fs->fs.flags &= ~DN_IS_GENTLE_RED; 1264 return(ENOSPC); 1265 } 1266 1267 /* Fill the lookup table with (1 - w_q)^x */ 1268 fs->w_q_lookup[0] = SCALE(1) - fs->w_q; 1269 1270 for (i = 1; i < fs->lookup_depth; i++) 1271 fs->w_q_lookup[i] = 1272 SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight); 1273 1274 if (V_dn_cfg.red_avg_pkt_size < 1) 1275 V_dn_cfg.red_avg_pkt_size = 512; 1276 fs->avg_pkt_size = V_dn_cfg.red_avg_pkt_size; 1277 if (V_dn_cfg.red_max_pkt_size < 1) 1278 V_dn_cfg.red_max_pkt_size = 1500; 1279 fs->max_pkt_size = V_dn_cfg.red_max_pkt_size; 1280 ND("exit"); 1281 return 0; 1282} 1283 1284/* Scan all flowset attached to this scheduler and update red */ 1285static void 1286update_red(struct dn_schk *s) 1287{ 1288 struct dn_fsk *fs; 1289 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 1290 if (fs && (fs->fs.flags & DN_IS_RED)) 1291 config_red(fs); 1292 } 1293} 1294 1295/* attach flowset to scheduler s, possibly requeue */ 1296static void 1297fsk_attach(struct dn_fsk *fs, struct dn_schk *s) 1298{ 1299 ND("remove fs %d from fsunlinked, link to sched %d", 1300 fs->fs.fs_nr, s->sch.sched_nr); 1301 SLIST_REMOVE(&V_dn_cfg.fsu, fs, dn_fsk, sch_chain); 1302 fs->sched = s; 1303 SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain); 1304 if (s->fp->new_fsk) 1305 s->fp->new_fsk(fs); 1306 /* XXX compute fsk_mask */ 1307 fs->fsk_mask = fs->fs.flow_mask; 1308 if (fs->sched->sch.flags & DN_HAVE_MASK) 1309 flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask); 1310 if (fs->qht) { 1311 /* 1312 * we must drain qht according to the old 1313 * type, and reinsert according to the new one. 1314 * The requeue is complex -- in general we need to 1315 * reclassify every single packet. 1316 * For the time being, let's hope qht is never set 1317 * when we reach this point. 1318 */ 1319 D("XXX TODO requeue from fs %d to sch %d", 1320 fs->fs.fs_nr, s->sch.sched_nr); 1321 fs->qht = NULL; 1322 } 1323 /* set the new type for qht */ 1324 if (nonzero_mask(&fs->fsk_mask)) 1325 fs->fs.flags |= DN_QHT_HASH; 1326 else 1327 fs->fs.flags &= ~DN_QHT_HASH; 1328 1329 /* XXX config_red() can fail... */ 1330 if (fs->fs.flags & DN_IS_RED) 1331 config_red(fs); 1332} 1333 1334/* update all flowsets which may refer to this scheduler */ 1335static void 1336update_fs(struct dn_schk *s) 1337{ 1338 struct dn_fsk *fs, *tmp; 1339 1340 SLIST_FOREACH_SAFE(fs, &V_dn_cfg.fsu, sch_chain, tmp) { 1341 if (s->sch.sched_nr != fs->fs.sched_nr) { 1342 D("fs %d for sch %d not %d still unlinked", 1343 fs->fs.fs_nr, fs->fs.sched_nr, 1344 s->sch.sched_nr); 1345 continue; 1346 } 1347 fsk_attach(fs, s); 1348 } 1349} 1350 1351#ifdef NEW_AQM 1352/* Retrieve AQM configurations to ipfw userland 1353 */ 1354static int 1355get_aqm_parms(struct sockopt *sopt) 1356{ 1357 struct dn_extra_parms *ep; 1358 struct dn_fsk *fs; 1359 size_t sopt_valsize; 1360 int l, err = 0; 1361 1362 sopt_valsize = sopt->sopt_valsize; 1363 l = sizeof(*ep); 1364 if (sopt->sopt_valsize < l) { 1365 D("bad len sopt->sopt_valsize %d len %d", 1366 (int) sopt->sopt_valsize , l); 1367 err = EINVAL; 1368 return err; 1369 } 1370 ep = malloc(l, M_DUMMYNET, M_NOWAIT); 1371 if(!ep) { 1372 err = ENOMEM ; 1373 return err; 1374 } 1375 do { 1376 err = sooptcopyin(sopt, ep, l, l); 1377 if(err) 1378 break; 1379 sopt->sopt_valsize = sopt_valsize; 1380 if (ep->oid.len < l) { 1381 err = EINVAL; 1382 break; 1383 } 1384 1385 fs = dn_ht_find(V_dn_cfg.fshash, ep->nr, 0, NULL); 1386 if (!fs) { 1387 D("fs %d not found", ep->nr); 1388 err = EINVAL; 1389 break; 1390 } 1391 1392 if (fs->aqmfp && fs->aqmfp->getconfig) { 1393 if(fs->aqmfp->getconfig(fs, ep)) { 1394 D("Error while trying to get AQM params"); 1395 err = EINVAL; 1396 break; 1397 } 1398 ep->oid.len = l; 1399 err = sooptcopyout(sopt, ep, l); 1400 } 1401 }while(0); 1402 1403 free(ep, M_DUMMYNET); 1404 return err; 1405} 1406 1407/* Retrieve AQM configurations to ipfw userland 1408 */ 1409static int 1410get_sched_parms(struct sockopt *sopt) 1411{ 1412 struct dn_extra_parms *ep; 1413 struct dn_schk *schk; 1414 size_t sopt_valsize; 1415 int l, err = 0; 1416 1417 sopt_valsize = sopt->sopt_valsize; 1418 l = sizeof(*ep); 1419 if (sopt->sopt_valsize < l) { 1420 D("bad len sopt->sopt_valsize %d len %d", 1421 (int) sopt->sopt_valsize , l); 1422 err = EINVAL; 1423 return err; 1424 } 1425 ep = malloc(l, M_DUMMYNET, M_NOWAIT); 1426 if(!ep) { 1427 err = ENOMEM ; 1428 return err; 1429 } 1430 do { 1431 err = sooptcopyin(sopt, ep, l, l); 1432 if(err) 1433 break; 1434 sopt->sopt_valsize = sopt_valsize; 1435 if (ep->oid.len < l) { 1436 err = EINVAL; 1437 break; 1438 } 1439 1440 schk = locate_scheduler(ep->nr); 1441 if (!schk) { 1442 D("sched %d not found", ep->nr); 1443 err = EINVAL; 1444 break; 1445 } 1446 1447 if (schk->fp && schk->fp->getconfig) { 1448 if(schk->fp->getconfig(schk, ep)) { 1449 D("Error while trying to get sched params"); 1450 err = EINVAL; 1451 break; 1452 } 1453 ep->oid.len = l; 1454 err = sooptcopyout(sopt, ep, l); 1455 } 1456 }while(0); 1457 free(ep, M_DUMMYNET); 1458 1459 return err; 1460} 1461 1462/* Configure AQM for flowset 'fs'. 1463 * extra parameters are passed from userland. 1464 */ 1465static int 1466config_aqm(struct dn_fsk *fs, struct dn_extra_parms *ep, int busy) 1467{ 1468 int err = 0; 1469 1470 NET_EPOCH_ASSERT(); 1471 1472 do { 1473 /* no configurations */ 1474 if (!ep) { 1475 err = 0; 1476 break; 1477 } 1478 1479 /* no AQM for this flowset*/ 1480 if (!strcmp(ep->name,"")) { 1481 err = 0; 1482 break; 1483 } 1484 if (ep->oid.len < sizeof(*ep)) { 1485 D("short aqm len %d", ep->oid.len); 1486 err = EINVAL; 1487 break; 1488 } 1489 1490 if (busy) { 1491 D("Unable to configure flowset, flowset busy!"); 1492 err = EINVAL; 1493 break; 1494 } 1495 1496 /* deconfigure old aqm if exist */ 1497 if (fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) { 1498 aqm_cleanup_deconfig_fs(fs); 1499 } 1500 1501 if (!(fs->aqmfp = find_aqm_type(0, ep->name))) { 1502 D("AQM functions not found for type %s!", ep->name); 1503 fs->fs.flags &= ~DN_IS_AQM; 1504 err = EINVAL; 1505 break; 1506 } else 1507 fs->fs.flags |= DN_IS_AQM; 1508 1509 if (ep->oid.subtype != DN_AQM_PARAMS) { 1510 D("Wrong subtype"); 1511 err = EINVAL; 1512 break; 1513 } 1514 1515 if (fs->aqmfp->config) { 1516 err = fs->aqmfp->config(fs, ep, ep->oid.len); 1517 if (err) { 1518 D("Unable to configure AQM for FS %d", fs->fs.fs_nr ); 1519 fs->fs.flags &= ~DN_IS_AQM; 1520 fs->aqmfp = NULL; 1521 break; 1522 } 1523 } 1524 } while(0); 1525 1526 return err; 1527} 1528#endif 1529 1530/* 1531 * Configuration -- to preserve backward compatibility we use 1532 * the following scheme (N is 65536) 1533 * NUMBER SCHED LINK FLOWSET 1534 * 1 .. N-1 (1)WFQ (2)WFQ (3)queue 1535 * N+1 .. 2N-1 (4)FIFO (5)FIFO (6)FIFO for sched 1..N-1 1536 * 2N+1 .. 3N-1 -- -- (7)FIFO for sched N+1..2N-1 1537 * 1538 * "pipe i config" configures #1, #2 and #3 1539 * "sched i config" configures #1 and possibly #6 1540 * "queue i config" configures #3 1541 * #1 is configured with 'pipe i config' or 'sched i config' 1542 * #2 is configured with 'pipe i config', and created if not 1543 * existing with 'sched i config' 1544 * #3 is configured with 'queue i config' 1545 * #4 is automatically configured after #1, can only be FIFO 1546 * #5 is automatically configured after #2 1547 * #6 is automatically created when #1 is !MULTIQUEUE, 1548 * and can be updated. 1549 * #7 is automatically configured after #2 1550 */ 1551 1552/* 1553 * configure a link (and its FIFO instance) 1554 */ 1555static int 1556config_link(struct dn_link *p, struct dn_id *arg) 1557{ 1558 int i; 1559 1560 if (p->oid.len != sizeof(*p)) { 1561 D("invalid pipe len %d", p->oid.len); 1562 return EINVAL; 1563 } 1564 i = p->link_nr; 1565 if (i <= 0 || i >= DN_MAX_ID) 1566 return EINVAL; 1567 /* 1568 * The config program passes parameters as follows: 1569 * bw = bits/second (0 means no limits), 1570 * delay = ms, must be translated into ticks. 1571 * qsize = slots/bytes 1572 * burst ??? 1573 */ 1574 p->delay = (p->delay * hz) / 1000; 1575 /* Scale burst size: bytes -> bits * hz */ 1576 p->burst *= 8 * hz; 1577 1578 DN_BH_WLOCK(); 1579 /* do it twice, base link and FIFO link */ 1580 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1581 struct dn_schk *s = locate_scheduler(i); 1582 if (s == NULL) { 1583 DN_BH_WUNLOCK(); 1584 D("sched %d not found", i); 1585 return EINVAL; 1586 } 1587 /* remove profile if exists */ 1588 free(s->profile, M_DUMMYNET); 1589 s->profile = NULL; 1590 1591 /* copy all parameters */ 1592 s->link.oid = p->oid; 1593 s->link.link_nr = i; 1594 s->link.delay = p->delay; 1595 if (s->link.bandwidth != p->bandwidth) { 1596 /* XXX bandwidth changes, need to update red params */ 1597 s->link.bandwidth = p->bandwidth; 1598 update_red(s); 1599 } 1600 s->link.burst = p->burst; 1601 schk_reset_credit(s); 1602 } 1603 V_dn_cfg.id++; 1604 DN_BH_WUNLOCK(); 1605 return 0; 1606} 1607 1608/* 1609 * configure a flowset. Can be called from inside with locked=1, 1610 */ 1611static struct dn_fsk * 1612config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) 1613{ 1614 int i; 1615 struct dn_fsk *fs; 1616#ifdef NEW_AQM 1617 struct dn_extra_parms *ep; 1618#endif 1619 1620 if (nfs->oid.len != sizeof(*nfs)) { 1621 D("invalid flowset len %d", nfs->oid.len); 1622 return NULL; 1623 } 1624 i = nfs->fs_nr; 1625 if (i <= 0 || i >= 3*DN_MAX_ID) 1626 return NULL; 1627#ifdef NEW_AQM 1628 ep = NULL; 1629 if (arg != NULL) { 1630 ep = malloc(sizeof(*ep), M_TEMP, M_NOWAIT); 1631 if (ep == NULL) 1632 return (NULL); 1633 memcpy(ep, arg, sizeof(*ep)); 1634 } 1635#endif 1636 ND("flowset %d", i); 1637 /* XXX other sanity checks */ 1638 if (nfs->flags & DN_QSIZE_BYTES) { 1639 ipdn_bound_var(&nfs->qsize, 16384, 1640 1500, V_dn_cfg.byte_limit, NULL); // "queue byte size"); 1641 } else { 1642 ipdn_bound_var(&nfs->qsize, 50, 1643 1, V_dn_cfg.slot_limit, NULL); // "queue slot size"); 1644 } 1645 if (nfs->flags & DN_HAVE_MASK) { 1646 /* make sure we have some buckets */ 1647 ipdn_bound_var((int *)&nfs->buckets, V_dn_cfg.hash_size, 1648 1, V_dn_cfg.max_hash_size, "flowset buckets"); 1649 } else { 1650 nfs->buckets = 1; /* we only need 1 */ 1651 } 1652 if (!locked) 1653 DN_BH_WLOCK(); 1654 do { /* exit with break when done */ 1655 struct dn_schk *s; 1656 int flags = nfs->sched_nr ? DNHT_INSERT : 0; 1657 int j; 1658 int oldc = V_dn_cfg.fsk_count; 1659 fs = dn_ht_find(V_dn_cfg.fshash, i, flags, NULL); 1660 if (fs == NULL) { 1661 D("missing sched for flowset %d", i); 1662 break; 1663 } 1664 /* grab some defaults from the existing one */ 1665 if (nfs->sched_nr == 0) /* reuse */ 1666 nfs->sched_nr = fs->fs.sched_nr; 1667 for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) { 1668 if (nfs->par[j] == -1) /* reuse */ 1669 nfs->par[j] = fs->fs.par[j]; 1670 } 1671 if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) { 1672 ND("flowset %d unchanged", i); 1673#ifdef NEW_AQM 1674 if (ep != NULL) { 1675 /* 1676 * Reconfigure AQM as the parameters can be changed. 1677 * We consider the flowset as busy if it has scheduler 1678 * instance(s). 1679 */ 1680 s = locate_scheduler(nfs->sched_nr); 1681 config_aqm(fs, ep, s != NULL && s->siht != NULL); 1682 } 1683#endif 1684 break; /* no change, nothing to do */ 1685 } 1686 if (oldc != V_dn_cfg.fsk_count) /* new item */ 1687 V_dn_cfg.id++; 1688 s = locate_scheduler(nfs->sched_nr); 1689 /* detach from old scheduler if needed, preserving 1690 * queues if we need to reattach. Then update the 1691 * configuration, and possibly attach to the new sched. 1692 */ 1693 DX(2, "fs %d changed sched %d@%p to %d@%p", 1694 fs->fs.fs_nr, 1695 fs->fs.sched_nr, fs->sched, nfs->sched_nr, s); 1696 if (fs->sched) { 1697 int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY); 1698 flags |= DN_DESTROY; /* XXX temporary */ 1699 fsk_detach(fs, flags); 1700 } 1701 fs->fs = *nfs; /* copy configuration */ 1702#ifdef NEW_AQM 1703 fs->aqmfp = NULL; 1704 if (ep != NULL) 1705 config_aqm(fs, ep, s != NULL && 1706 s->siht != NULL); 1707#endif 1708 if (s != NULL) 1709 fsk_attach(fs, s); 1710 } while (0); 1711 if (!locked) 1712 DN_BH_WUNLOCK(); 1713#ifdef NEW_AQM 1714 free(ep, M_TEMP); 1715#endif 1716 return fs; 1717} 1718 1719/* 1720 * config/reconfig a scheduler and its FIFO variant. 1721 * For !MULTIQUEUE schedulers, also set up the flowset. 1722 * 1723 * On reconfigurations (detected because s->fp is set), 1724 * detach existing flowsets preserving traffic, preserve link, 1725 * and delete the old scheduler creating a new one. 1726 */ 1727static int 1728config_sched(struct dn_sch *_nsch, struct dn_id *arg) 1729{ 1730 struct dn_schk *s; 1731 struct schk_new_arg a; /* argument for schk_new */ 1732 int i; 1733 struct dn_link p; /* copy of oldlink */ 1734 struct dn_profile *pf = NULL; /* copy of old link profile */ 1735 /* Used to preserve mask parameter */ 1736 struct ipfw_flow_id new_mask; 1737 int new_buckets = 0; 1738 int new_flags = 0; 1739 int pipe_cmd; 1740 int err = ENOMEM; 1741 1742 NET_EPOCH_ASSERT(); 1743 1744 a.sch = _nsch; 1745 if (a.sch->oid.len != sizeof(*a.sch)) { 1746 D("bad sched len %d", a.sch->oid.len); 1747 return EINVAL; 1748 } 1749 i = a.sch->sched_nr; 1750 if (i <= 0 || i >= DN_MAX_ID) 1751 return EINVAL; 1752 /* make sure we have some buckets */ 1753 if (a.sch->flags & DN_HAVE_MASK) 1754 ipdn_bound_var((int *)&a.sch->buckets, V_dn_cfg.hash_size, 1755 1, V_dn_cfg.max_hash_size, "sched buckets"); 1756 /* XXX other sanity checks */ 1757 bzero(&p, sizeof(p)); 1758 1759 pipe_cmd = a.sch->flags & DN_PIPE_CMD; 1760 a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set? 1761 if (pipe_cmd) { 1762 /* Copy mask parameter */ 1763 new_mask = a.sch->sched_mask; 1764 new_buckets = a.sch->buckets; 1765 new_flags = a.sch->flags; 1766 } 1767 DN_BH_WLOCK(); 1768again: /* run twice, for wfq and fifo */ 1769 /* 1770 * lookup the type. If not supplied, use the previous one 1771 * or default to WF2Q+. Otherwise, return an error. 1772 */ 1773 V_dn_cfg.id++; 1774 a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name); 1775 if (a.fp != NULL) { 1776 /* found. Lookup or create entry */ 1777 s = dn_ht_find(V_dn_cfg.schedhash, i, DNHT_INSERT, &a); 1778 } else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) { 1779 /* No type. search existing s* or retry with WF2Q+ */ 1780 s = dn_ht_find(V_dn_cfg.schedhash, i, 0, &a); 1781 if (s != NULL) { 1782 a.fp = s->fp; 1783 /* Scheduler exists, skip to FIFO scheduler 1784 * if command was pipe config... 1785 */ 1786 if (pipe_cmd) 1787 goto next; 1788 } else { 1789 /* New scheduler, create a wf2q+ with no mask 1790 * if command was pipe config... 1791 */ 1792 if (pipe_cmd) { 1793 /* clear mask parameter */ 1794 bzero(&a.sch->sched_mask, sizeof(new_mask)); 1795 a.sch->buckets = 0; 1796 a.sch->flags &= ~DN_HAVE_MASK; 1797 } 1798 a.sch->oid.subtype = DN_SCHED_WF2QP; 1799 goto again; 1800 } 1801 } else { 1802 D("invalid scheduler type %d %s", 1803 a.sch->oid.subtype, a.sch->name); 1804 err = EINVAL; 1805 goto error; 1806 } 1807 /* normalize name and subtype */ 1808 a.sch->oid.subtype = a.fp->type; 1809 bzero(a.sch->name, sizeof(a.sch->name)); 1810 strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name)); 1811 if (s == NULL) { 1812 D("cannot allocate scheduler %d", i); 1813 goto error; 1814 } 1815 /* restore existing link if any */ 1816 if (p.link_nr) { 1817 s->link = p; 1818 if (!pf || pf->link_nr != p.link_nr) { /* no saved value */ 1819 s->profile = NULL; /* XXX maybe not needed */ 1820 } else { 1821 s->profile = malloc(sizeof(struct dn_profile), 1822 M_DUMMYNET, M_NOWAIT | M_ZERO); 1823 if (s->profile == NULL) { 1824 D("cannot allocate profile"); 1825 goto error; //XXX 1826 } 1827 memcpy(s->profile, pf, sizeof(*pf)); 1828 } 1829 } 1830 p.link_nr = 0; 1831 if (s->fp == NULL) { 1832 DX(2, "sched %d new type %s", i, a.fp->name); 1833 } else if (s->fp != a.fp || 1834 bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) { 1835 /* already existing. */ 1836 DX(2, "sched %d type changed from %s to %s", 1837 i, s->fp->name, a.fp->name); 1838 DX(4, " type/sub %d/%d -> %d/%d", 1839 s->sch.oid.type, s->sch.oid.subtype, 1840 a.sch->oid.type, a.sch->oid.subtype); 1841 if (s->link.link_nr == 0) 1842 D("XXX WARNING link 0 for sched %d", i); 1843 p = s->link; /* preserve link */ 1844 if (s->profile) {/* preserve profile */ 1845 if (!pf) 1846 pf = malloc(sizeof(*pf), 1847 M_DUMMYNET, M_NOWAIT | M_ZERO); 1848 if (pf) /* XXX should issue a warning otherwise */ 1849 memcpy(pf, s->profile, sizeof(*pf)); 1850 } 1851 /* remove from the hash */ 1852 dn_ht_find(V_dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 1853 /* Detach flowsets, preserve queues. */ 1854 // schk_delete_cb(s, NULL); 1855 // XXX temporarily, kill queues 1856 schk_delete_cb(s, (void *)DN_DESTROY); 1857 goto again; 1858 } else { 1859 DX(4, "sched %d unchanged type %s", i, a.fp->name); 1860 } 1861 /* complete initialization */ 1862 s->sch = *a.sch; 1863 s->fp = a.fp; 1864 s->cfg = arg; 1865 // XXX schk_reset_credit(s); 1866 /* create the internal flowset if needed, 1867 * trying to reuse existing ones if available 1868 */ 1869 if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) { 1870 s->fs = dn_ht_find(V_dn_cfg.fshash, i, 0, NULL); 1871 if (!s->fs) { 1872 struct dn_fs fs; 1873 bzero(&fs, sizeof(fs)); 1874 set_oid(&fs.oid, DN_FS, sizeof(fs)); 1875 fs.fs_nr = i + DN_MAX_ID; 1876 fs.sched_nr = i; 1877 s->fs = config_fs(&fs, NULL, 1 /* locked */); 1878 } 1879 if (!s->fs) { 1880 schk_delete_cb(s, (void *)DN_DESTROY); 1881 D("error creating internal fs for %d", i); 1882 goto error; 1883 } 1884 } 1885 /* call init function after the flowset is created */ 1886 if (s->fp->config) 1887 s->fp->config(s); 1888 update_fs(s); 1889next: 1890 if (i < DN_MAX_ID) { /* now configure the FIFO instance */ 1891 i += DN_MAX_ID; 1892 if (pipe_cmd) { 1893 /* Restore mask parameter for FIFO */ 1894 a.sch->sched_mask = new_mask; 1895 a.sch->buckets = new_buckets; 1896 a.sch->flags = new_flags; 1897 } else { 1898 /* sched config shouldn't modify the FIFO scheduler */ 1899 if (dn_ht_find(V_dn_cfg.schedhash, i, 0, &a) != NULL) { 1900 /* FIFO already exist, don't touch it */ 1901 err = 0; /* and this is not an error */ 1902 goto error; 1903 } 1904 } 1905 a.sch->sched_nr = i; 1906 a.sch->oid.subtype = DN_SCHED_FIFO; 1907 bzero(a.sch->name, sizeof(a.sch->name)); 1908 goto again; 1909 } 1910 err = 0; 1911error: 1912 DN_BH_WUNLOCK(); 1913 free(pf, M_DUMMYNET); 1914 return err; 1915} 1916 1917/* 1918 * attach a profile to a link 1919 */ 1920static int 1921config_profile(struct dn_profile *pf, struct dn_id *arg) 1922{ 1923 struct dn_schk *s; 1924 int i, olen, err = 0; 1925 1926 if (pf->oid.len < sizeof(*pf)) { 1927 D("short profile len %d", pf->oid.len); 1928 return EINVAL; 1929 } 1930 i = pf->link_nr; 1931 if (i <= 0 || i >= DN_MAX_ID) 1932 return EINVAL; 1933 /* XXX other sanity checks */ 1934 DN_BH_WLOCK(); 1935 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1936 s = locate_scheduler(i); 1937 1938 if (s == NULL) { 1939 err = EINVAL; 1940 break; 1941 } 1942 V_dn_cfg.id++; 1943 /* 1944 * If we had a profile and the new one does not fit, 1945 * or it is deleted, then we need to free memory. 1946 */ 1947 if (s->profile && (pf->samples_no == 0 || 1948 s->profile->oid.len < pf->oid.len)) { 1949 free(s->profile, M_DUMMYNET); 1950 s->profile = NULL; 1951 } 1952 if (pf->samples_no == 0) 1953 continue; 1954 /* 1955 * new profile, possibly allocate memory 1956 * and copy data. 1957 */ 1958 if (s->profile == NULL) 1959 s->profile = malloc(pf->oid.len, 1960 M_DUMMYNET, M_NOWAIT | M_ZERO); 1961 if (s->profile == NULL) { 1962 D("no memory for profile %d", i); 1963 err = ENOMEM; 1964 break; 1965 } 1966 /* preserve larger length XXX double check */ 1967 olen = s->profile->oid.len; 1968 if (olen < pf->oid.len) 1969 olen = pf->oid.len; 1970 memcpy(s->profile, pf, pf->oid.len); 1971 s->profile->oid.len = olen; 1972 } 1973 DN_BH_WUNLOCK(); 1974 return err; 1975} 1976 1977/* 1978 * Delete all objects: 1979 */ 1980static void 1981dummynet_flush(void) 1982{ 1983 1984 /* delete all schedulers and related links/queues/flowsets */ 1985 dn_ht_scan(V_dn_cfg.schedhash, schk_delete_cb, 1986 (void *)(uintptr_t)DN_DELETE_FS); 1987 /* delete all remaining (unlinked) flowsets */ 1988 DX(4, "still %d unlinked fs", V_dn_cfg.fsk_count); 1989 dn_ht_free(V_dn_cfg.fshash, DNHT_REMOVE); 1990 fsk_detach_list(&V_dn_cfg.fsu, DN_DELETE_FS); 1991 /* Reinitialize system heap... */ 1992 heap_init(&V_dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 1993} 1994 1995/* 1996 * Main handler for configuration. We are guaranteed to be called 1997 * with an oid which is at least a dn_id. 1998 * - the first object is the command (config, delete, flush, ...) 1999 * - config_link must be issued after the corresponding config_sched 2000 * - parameters (DN_TXT) for an object must precede the object 2001 * processed on a config_sched. 2002 */ 2003int 2004do_config(void *p, size_t l) 2005{ 2006 struct dn_id o; 2007 union { 2008 struct dn_profile profile; 2009 struct dn_fs fs; 2010 struct dn_link link; 2011 struct dn_sch sched; 2012 } *dn; 2013 struct dn_id *arg; 2014 uintptr_t a; 2015 int err, err2, off; 2016 2017 memcpy(&o, p, sizeof(o)); 2018 if (o.id != DN_API_VERSION) { 2019 D("invalid api version got %d need %d", o.id, DN_API_VERSION); 2020 return EINVAL; 2021 } 2022 arg = NULL; 2023 dn = NULL; 2024 off = 0; 2025 while (l >= sizeof(o)) { 2026 memcpy(&o, (char *)p + off, sizeof(o)); 2027 if (o.len < sizeof(o) || l < o.len) { 2028 D("bad len o.len %d len %zu", o.len, l); 2029 err = EINVAL; 2030 break; 2031 } 2032 l -= o.len; 2033 err = 0; 2034 switch (o.type) { 2035 default: 2036 D("cmd %d not implemented", o.type); 2037 break; 2038 2039#ifdef EMULATE_SYSCTL 2040 /* sysctl emulation. 2041 * if we recognize the command, jump to the correct 2042 * handler and return 2043 */ 2044 case DN_SYSCTL_SET: 2045 err = kesysctl_emu_set(p, l); 2046 return err; 2047#endif 2048 2049 case DN_CMD_CONFIG: /* simply a header */ 2050 break; 2051 2052 case DN_CMD_DELETE: 2053 /* the argument is in the first uintptr_t after o */ 2054 if (o.len < sizeof(o) + sizeof(a)) { 2055 err = EINVAL; 2056 break; 2057 } 2058 memcpy(&a, (char *)p + off + sizeof(o), sizeof(a)); 2059 switch (o.subtype) { 2060 case DN_LINK: 2061 /* delete base and derived schedulers */ 2062 DN_BH_WLOCK(); 2063 err = delete_schk(a); 2064 err2 = delete_schk(a + DN_MAX_ID); 2065 DN_BH_WUNLOCK(); 2066 if (!err) 2067 err = err2; 2068 break; 2069 2070 default: 2071 D("invalid delete type %d", o.subtype); 2072 err = EINVAL; 2073 break; 2074 2075 case DN_FS: 2076 err = (a < 1 || a >= DN_MAX_ID) ? 2077 EINVAL : delete_fs(a, 0) ; 2078 break; 2079 } 2080 break; 2081 2082 case DN_CMD_FLUSH: 2083 DN_BH_WLOCK(); 2084 dummynet_flush(); 2085 DN_BH_WUNLOCK(); 2086 break; 2087 case DN_TEXT: /* store argument of next block */ 2088 free(arg, M_TEMP); 2089 arg = malloc(o.len, M_TEMP, M_NOWAIT); 2090 if (arg == NULL) { 2091 err = ENOMEM; 2092 break; 2093 } 2094 memcpy(arg, (char *)p + off, o.len); 2095 break; 2096 case DN_LINK: 2097 if (dn == NULL) 2098 dn = malloc(sizeof(*dn), M_TEMP, M_NOWAIT); 2099 if (dn == NULL) { 2100 err = ENOMEM; 2101 break; 2102 } 2103 memcpy(&dn->link, (char *)p + off, sizeof(dn->link)); 2104 err = config_link(&dn->link, arg); 2105 break; 2106 case DN_PROFILE: 2107 if (dn == NULL) 2108 dn = malloc(sizeof(*dn), M_TEMP, M_NOWAIT); 2109 if (dn == NULL) { 2110 err = ENOMEM; 2111 break; 2112 } 2113 memcpy(&dn->profile, (char *)p + off, 2114 sizeof(dn->profile)); 2115 err = config_profile(&dn->profile, arg); 2116 break; 2117 case DN_SCH: 2118 if (dn == NULL) 2119 dn = malloc(sizeof(*dn), M_TEMP, M_NOWAIT); 2120 if (dn == NULL) { 2121 err = ENOMEM; 2122 break; 2123 } 2124 memcpy(&dn->sched, (char *)p + off, 2125 sizeof(dn->sched)); 2126 err = config_sched(&dn->sched, arg); 2127 break; 2128 case DN_FS: 2129 if (dn == NULL) 2130 dn = malloc(sizeof(*dn), M_TEMP, M_NOWAIT); 2131 if (dn == NULL) { 2132 err = ENOMEM; 2133 break; 2134 } 2135 memcpy(&dn->fs, (char *)p + off, sizeof(dn->fs)); 2136 err = (NULL == config_fs(&dn->fs, arg, 0)); 2137 break; 2138 } 2139 if (err != 0) 2140 break; 2141 off += o.len; 2142 } 2143 free(arg, M_TEMP); 2144 free(dn, M_TEMP); 2145 return err; 2146} 2147 2148static int 2149compute_space(struct dn_id *cmd, struct copy_args *a) 2150{ 2151 int x = 0, need = 0; 2152 int profile_size = sizeof(struct dn_profile) - 2153 ED_MAX_SAMPLES_NO*sizeof(int); 2154 2155 /* NOTE about compute space: 2156 * NP = V_dn_cfg.schk_count 2157 * NSI = V_dn_cfg.si_count 2158 * NF = V_dn_cfg.fsk_count 2159 * NQ = V_dn_cfg.queue_count 2160 * - ipfw pipe show 2161 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 2162 * link, scheduler template, flowset 2163 * integrated in scheduler and header 2164 * for flowset list 2165 * (NSI)*(dn_flow) all scheduler instance (includes 2166 * the queue instance) 2167 * - ipfw sched show 2168 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 2169 * link, scheduler template, flowset 2170 * integrated in scheduler and header 2171 * for flowset list 2172 * (NSI * dn_flow) all scheduler instances 2173 * (NF * sizeof(uint_32)) space for flowset list linked to scheduler 2174 * (NQ * dn_queue) all queue [XXXfor now not listed] 2175 * - ipfw queue show 2176 * (NF * dn_fs) all flowset 2177 * (NQ * dn_queue) all queues 2178 */ 2179 switch (cmd->subtype) { 2180 default: 2181 return -1; 2182 /* XXX where do LINK and SCH differ ? */ 2183 /* 'ipfw sched show' could list all queues associated to 2184 * a scheduler. This feature for now is disabled 2185 */ 2186 case DN_LINK: /* pipe show */ 2187 x = DN_C_LINK | DN_C_SCH | DN_C_FLOW; 2188 need += V_dn_cfg.schk_count * 2189 (sizeof(struct dn_fs) + profile_size) / 2; 2190 need += V_dn_cfg.fsk_count * sizeof(uint32_t); 2191 break; 2192 case DN_SCH: /* sched show */ 2193 need += V_dn_cfg.schk_count * 2194 (sizeof(struct dn_fs) + profile_size) / 2; 2195 need += V_dn_cfg.fsk_count * sizeof(uint32_t); 2196 x = DN_C_SCH | DN_C_LINK | DN_C_FLOW; 2197 break; 2198 case DN_FS: /* queue show */ 2199 x = DN_C_FS | DN_C_QUEUE; 2200 break; 2201 case DN_GET_COMPAT: /* compatibility mode */ 2202 need = dn_compat_calc_size(); 2203 break; 2204 } 2205 a->flags = x; 2206 if (x & DN_C_SCH) { 2207 need += V_dn_cfg.schk_count * sizeof(struct dn_sch) / 2; 2208 /* NOT also, each fs might be attached to a sched */ 2209 need += V_dn_cfg.schk_count * sizeof(struct dn_id) / 2; 2210 } 2211 if (x & DN_C_FS) 2212 need += V_dn_cfg.fsk_count * sizeof(struct dn_fs); 2213 if (x & DN_C_LINK) { 2214 need += V_dn_cfg.schk_count * sizeof(struct dn_link) / 2; 2215 } 2216 /* 2217 * When exporting a queue to userland, only pass up the 2218 * struct dn_flow, which is the only visible part. 2219 */ 2220 2221 if (x & DN_C_QUEUE) 2222 need += V_dn_cfg.queue_count * sizeof(struct dn_flow); 2223 if (x & DN_C_FLOW) 2224 need += V_dn_cfg.si_count * (sizeof(struct dn_flow)); 2225 return need; 2226} 2227 2228/* 2229 * If compat != NULL dummynet_get is called in compatibility mode. 2230 * *compat will be the pointer to the buffer to pass to ipfw 2231 */ 2232int 2233dummynet_get(struct sockopt *sopt, void **compat) 2234{ 2235 int have, i, need, error; 2236 char *start = NULL, *buf; 2237 size_t sopt_valsize; 2238 struct dn_id *cmd; 2239 struct copy_args a; 2240 struct copy_range r; 2241 int l = sizeof(struct dn_id); 2242 2243 bzero(&a, sizeof(a)); 2244 bzero(&r, sizeof(r)); 2245 2246 /* save and restore original sopt_valsize around copyin */ 2247 sopt_valsize = sopt->sopt_valsize; 2248 2249 cmd = &r.o; 2250 2251 if (!compat) { 2252 /* copy at least an oid, and possibly a full object */ 2253 error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd)); 2254 sopt->sopt_valsize = sopt_valsize; 2255 if (error) 2256 goto done; 2257 l = cmd->len; 2258#ifdef EMULATE_SYSCTL 2259 /* sysctl emulation. */ 2260 if (cmd->type == DN_SYSCTL_GET) 2261 return kesysctl_emu_get(sopt); 2262#endif 2263 if (l > sizeof(r)) { 2264 /* request larger than default, allocate buffer */ 2265 cmd = malloc(l, M_DUMMYNET, M_NOWAIT); 2266 if (cmd == NULL) { 2267 error = ENOMEM; 2268 goto done; 2269 } 2270 error = sooptcopyin(sopt, cmd, l, l); 2271 sopt->sopt_valsize = sopt_valsize; 2272 if (error) 2273 goto done; 2274 } 2275 } else { /* compatibility */ 2276 error = 0; 2277 cmd->type = DN_CMD_GET; 2278 cmd->len = sizeof(struct dn_id); 2279 cmd->subtype = DN_GET_COMPAT; 2280 // cmd->id = sopt_valsize; 2281 D("compatibility mode"); 2282 } 2283 2284#ifdef NEW_AQM 2285 /* get AQM params */ 2286 if(cmd->subtype == DN_AQM_PARAMS) { 2287 error = get_aqm_parms(sopt); 2288 goto done; 2289 /* get Scheduler params */ 2290 } else if (cmd->subtype == DN_SCH_PARAMS) { 2291 error = get_sched_parms(sopt); 2292 goto done; 2293 } 2294#endif 2295 2296 a.extra = (struct copy_range *)cmd; 2297 if (cmd->len == sizeof(*cmd)) { /* no range, create a default */ 2298 uint32_t *rp = (uint32_t *)(cmd + 1); 2299 cmd->len += 2* sizeof(uint32_t); 2300 rp[0] = 1; 2301 rp[1] = DN_MAX_ID - 1; 2302 if (cmd->subtype == DN_LINK) { 2303 rp[0] += DN_MAX_ID; 2304 rp[1] += DN_MAX_ID; 2305 } 2306 } 2307 /* Count space (under lock) and allocate (outside lock). 2308 * Exit with lock held if we manage to get enough buffer. 2309 * Try a few times then give up. 2310 */ 2311 for (have = 0, i = 0; i < 10; i++) { 2312 DN_BH_WLOCK(); 2313 need = compute_space(cmd, &a); 2314 2315 /* if there is a range, ignore value from compute_space() */ 2316 if (l > sizeof(*cmd)) 2317 need = sopt_valsize - sizeof(*cmd); 2318 2319 if (need < 0) { 2320 DN_BH_WUNLOCK(); 2321 error = EINVAL; 2322 goto done; 2323 } 2324 need += sizeof(*cmd); 2325 cmd->id = need; 2326 if (have >= need) 2327 break; 2328 2329 DN_BH_WUNLOCK(); 2330 free(start, M_DUMMYNET); 2331 start = NULL; 2332 if (need > sopt_valsize) 2333 break; 2334 2335 have = need; 2336 start = malloc(have, M_DUMMYNET, M_NOWAIT | M_ZERO); 2337 } 2338 2339 if (start == NULL) { 2340 if (compat) { 2341 *compat = NULL; 2342 error = 1; // XXX 2343 } else { 2344 error = sooptcopyout(sopt, cmd, sizeof(*cmd)); 2345 } 2346 goto done; 2347 } 2348 ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, " 2349 "%d:%d si %d, %d:%d queues %d", 2350 V_dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH, 2351 V_dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK, 2352 V_dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS, 2353 V_dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I, 2354 V_dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE); 2355 sopt->sopt_valsize = sopt_valsize; 2356 a.type = cmd->subtype; 2357 2358 if (compat == NULL) { 2359 memcpy(start, cmd, sizeof(*cmd)); 2360 ((struct dn_id*)(start))->len = sizeof(struct dn_id); 2361 buf = start + sizeof(*cmd); 2362 } else 2363 buf = start; 2364 a.start = &buf; 2365 a.end = start + have; 2366 /* start copying other objects */ 2367 if (compat) { 2368 a.type = DN_COMPAT_PIPE; 2369 dn_ht_scan(V_dn_cfg.schedhash, copy_data_helper_compat, &a); 2370 a.type = DN_COMPAT_QUEUE; 2371 dn_ht_scan(V_dn_cfg.fshash, copy_data_helper_compat, &a); 2372 } else if (a.type == DN_FS) { 2373 dn_ht_scan(V_dn_cfg.fshash, copy_data_helper, &a); 2374 } else { 2375 dn_ht_scan(V_dn_cfg.schedhash, copy_data_helper, &a); 2376 } 2377 DN_BH_WUNLOCK(); 2378 2379 if (compat) { 2380 *compat = start; 2381 sopt->sopt_valsize = buf - start; 2382 /* free() is done by ip_dummynet_compat() */ 2383 start = NULL; //XXX hack 2384 } else { 2385 error = sooptcopyout(sopt, start, buf - start); 2386 } 2387done: 2388 if (cmd != &r.o) 2389 free(cmd, M_DUMMYNET); 2390 free(start, M_DUMMYNET); 2391 return error; 2392} 2393 2394/* Callback called on scheduler instance to delete it if idle */ 2395static int 2396drain_scheduler_cb(void *_si, void *arg) 2397{ 2398 struct dn_sch_inst *si = _si; 2399 2400 if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL) 2401 return 0; 2402 2403 if (si->sched->fp->flags & DN_MULTIQUEUE) { 2404 if (si->q_count == 0) 2405 return si_destroy(si, NULL); 2406 else 2407 return 0; 2408 } else { /* !DN_MULTIQUEUE */ 2409 if ((si+1)->ni.length == 0) 2410 return si_destroy(si, NULL); 2411 else 2412 return 0; 2413 } 2414 return 0; /* unreachable */ 2415} 2416 2417/* Callback called on scheduler to check if it has instances */ 2418static int 2419drain_scheduler_sch_cb(void *_s, void *arg) 2420{ 2421 struct dn_schk *s = _s; 2422 2423 if (s->sch.flags & DN_HAVE_MASK) { 2424 dn_ht_scan_bucket(s->siht, &s->drain_bucket, 2425 drain_scheduler_cb, NULL); 2426 s->drain_bucket++; 2427 } else { 2428 if (s->siht) { 2429 if (drain_scheduler_cb(s->siht, NULL) == DNHT_SCAN_DEL) 2430 s->siht = NULL; 2431 } 2432 } 2433 return 0; 2434} 2435 2436/* Called every tick, try to delete a 'bucket' of scheduler */ 2437void 2438dn_drain_scheduler(void) 2439{ 2440 dn_ht_scan_bucket(V_dn_cfg.schedhash, &V_dn_cfg.drain_sch, 2441 drain_scheduler_sch_cb, NULL); 2442 V_dn_cfg.drain_sch++; 2443} 2444 2445/* Callback called on queue to delete if it is idle */ 2446static int 2447drain_queue_cb(void *_q, void *arg) 2448{ 2449 struct dn_queue *q = _q; 2450 2451 if (q->ni.length == 0) { 2452 dn_delete_queue(q, DN_DESTROY); 2453 return DNHT_SCAN_DEL; /* queue is deleted */ 2454 } 2455 2456 return 0; /* queue isn't deleted */ 2457} 2458 2459/* Callback called on flowset used to check if it has queues */ 2460static int 2461drain_queue_fs_cb(void *_fs, void *arg) 2462{ 2463 struct dn_fsk *fs = _fs; 2464 2465 if (fs->fs.flags & DN_QHT_HASH) { 2466 /* Flowset has a hash table for queues */ 2467 dn_ht_scan_bucket(fs->qht, &fs->drain_bucket, 2468 drain_queue_cb, NULL); 2469 fs->drain_bucket++; 2470 } else { 2471 /* No hash table for this flowset, null the pointer 2472 * if the queue is deleted 2473 */ 2474 if (fs->qht) { 2475 if (drain_queue_cb(fs->qht, NULL) == DNHT_SCAN_DEL) 2476 fs->qht = NULL; 2477 } 2478 } 2479 return 0; 2480} 2481 2482/* Called every tick, try to delete a 'bucket' of queue */ 2483void 2484dn_drain_queue(void) 2485{ 2486 /* scan a bucket of flowset */ 2487 dn_ht_scan_bucket(V_dn_cfg.fshash, &V_dn_cfg.drain_fs, 2488 drain_queue_fs_cb, NULL); 2489 V_dn_cfg.drain_fs++; 2490} 2491 2492/* 2493 * Handler for the various dummynet socket options 2494 */ 2495static int 2496ip_dn_ctl(struct sockopt *sopt) 2497{ 2498 struct epoch_tracker et; 2499 void *p = NULL; 2500 size_t l; 2501 int error; 2502 2503 error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET); 2504 if (error) 2505 return (error); 2506 2507 /* Disallow sets in really-really secure mode. */ 2508 if (sopt->sopt_dir == SOPT_SET) { 2509 error = securelevel_ge(sopt->sopt_td->td_ucred, 3); 2510 if (error) 2511 return (error); 2512 } 2513 2514 NET_EPOCH_ENTER(et); 2515 2516 switch (sopt->sopt_name) { 2517 default : 2518 D("dummynet: unknown option %d", sopt->sopt_name); 2519 error = EINVAL; 2520 break; 2521 2522 case IP_DUMMYNET_FLUSH: 2523 case IP_DUMMYNET_CONFIGURE: 2524 case IP_DUMMYNET_DEL: /* remove a pipe or queue */ 2525 case IP_DUMMYNET_GET: 2526 D("dummynet: compat option %d", sopt->sopt_name); 2527 error = ip_dummynet_compat(sopt); 2528 break; 2529 2530 case IP_DUMMYNET3: 2531 if (sopt->sopt_dir == SOPT_GET) { 2532 error = dummynet_get(sopt, NULL); 2533 break; 2534 } 2535 l = sopt->sopt_valsize; 2536 if (l < sizeof(struct dn_id) || l > 12000) { 2537 D("argument len %zu invalid", l); 2538 break; 2539 } 2540 p = malloc(l, M_TEMP, M_NOWAIT); 2541 if (p == NULL) { 2542 error = ENOMEM; 2543 break; 2544 } 2545 error = sooptcopyin(sopt, p, l, l); 2546 if (error == 0) 2547 error = do_config(p, l); 2548 break; 2549 } 2550 2551 free(p, M_TEMP); 2552 2553 NET_EPOCH_EXIT(et); 2554 2555 return error ; 2556} 2557 2558static void 2559ip_dn_vnet_init(void) 2560{ 2561 if (V_dn_cfg.init_done) 2562 return; 2563 2564 /* Set defaults here. MSVC does not accept initializers, 2565 * and this is also useful for vimages 2566 */ 2567 /* queue limits */ 2568 V_dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */ 2569 V_dn_cfg.byte_limit = 1024 * 1024; 2570 V_dn_cfg.expire = 1; 2571 2572 /* RED parameters */ 2573 V_dn_cfg.red_lookup_depth = 256; /* default lookup table depth */ 2574 V_dn_cfg.red_avg_pkt_size = 512; /* default medium packet size */ 2575 V_dn_cfg.red_max_pkt_size = 1500; /* default max packet size */ 2576 2577 /* hash tables */ 2578 V_dn_cfg.max_hash_size = 65536; /* max in the hash tables */ 2579 V_dn_cfg.hash_size = 64; /* default hash size */ 2580 2581 /* create hash tables for schedulers and flowsets. 2582 * In both we search by key and by pointer. 2583 */ 2584 V_dn_cfg.schedhash = dn_ht_init(NULL, V_dn_cfg.hash_size, 2585 offsetof(struct dn_schk, schk_next), 2586 schk_hash, schk_match, schk_new); 2587 V_dn_cfg.fshash = dn_ht_init(NULL, V_dn_cfg.hash_size, 2588 offsetof(struct dn_fsk, fsk_next), 2589 fsk_hash, fsk_match, fsk_new); 2590 2591 /* bucket index to drain object */ 2592 V_dn_cfg.drain_fs = 0; 2593 V_dn_cfg.drain_sch = 0; 2594 2595 heap_init(&V_dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 2596 SLIST_INIT(&V_dn_cfg.fsu); 2597 2598 DN_LOCK_INIT(); 2599 2600 /* Initialize curr_time adjustment mechanics. */ 2601 getmicrouptime(&V_dn_cfg.prev_t); 2602 2603 V_dn_cfg.init_done = 1; 2604} 2605 2606static void 2607ip_dn_vnet_destroy(void) 2608{ 2609 DN_BH_WLOCK(); 2610 dummynet_flush(); 2611 DN_BH_WUNLOCK(); 2612 2613 dn_ht_free(V_dn_cfg.schedhash, 0); 2614 dn_ht_free(V_dn_cfg.fshash, 0); 2615 heap_free(&V_dn_cfg.evheap); 2616 2617 DN_LOCK_DESTROY(); 2618} 2619 2620static void 2621ip_dn_init(void) 2622{ 2623 if (dn_tasks_started) 2624 return; 2625 2626 mtx_init(&sched_mtx, "dn_sched", NULL, MTX_DEF); 2627 2628 dn_tasks_started = 1; 2629 TASK_INIT(&dn_task, 0, dummynet_task, NULL); 2630 dn_tq = taskqueue_create_fast("dummynet", M_WAITOK, 2631 taskqueue_thread_enqueue, &dn_tq); 2632 taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet"); 2633 2634 CK_LIST_INIT(&schedlist); 2635 callout_init(&dn_timeout, 1); 2636 dn_reschedule(); 2637} 2638 2639static void 2640ip_dn_destroy(int last) 2641{ 2642 /* ensure no more callouts are started */ 2643 dn_gone = 1; 2644 2645 /* check for last */ 2646 if (last) { 2647 ND("removing last instance\n"); 2648 ip_dn_ctl_ptr = NULL; 2649 ip_dn_io_ptr = NULL; 2650 } 2651 2652 callout_drain(&dn_timeout); 2653 taskqueue_drain(dn_tq, &dn_task); 2654 taskqueue_free(dn_tq); 2655} 2656 2657static int 2658dummynet_modevent(module_t mod, int type, void *data) 2659{ 2660 2661 if (type == MOD_LOAD) { 2662 if (ip_dn_io_ptr) { 2663 printf("DUMMYNET already loaded\n"); 2664 return EEXIST ; 2665 } 2666 ip_dn_init(); 2667 ip_dn_ctl_ptr = ip_dn_ctl; 2668 ip_dn_io_ptr = dummynet_io; 2669 return 0; 2670 } else if (type == MOD_UNLOAD) { 2671 ip_dn_destroy(1 /* last */); 2672 return 0; 2673 } else 2674 return EOPNOTSUPP; 2675} 2676 2677/* modevent helpers for the modules */ 2678static int 2679load_dn_sched(struct dn_alg *d) 2680{ 2681 struct dn_alg *s; 2682 2683 if (d == NULL) 2684 return 1; /* error */ 2685 ip_dn_init(); /* just in case, we need the lock */ 2686 2687 /* Check that mandatory funcs exists */ 2688 if (d->enqueue == NULL || d->dequeue == NULL) { 2689 D("missing enqueue or dequeue for %s", d->name); 2690 return 1; 2691 } 2692 2693 /* Search if scheduler already exists */ 2694 mtx_lock(&sched_mtx); 2695 CK_LIST_FOREACH(s, &schedlist, next) { 2696 if (strcmp(s->name, d->name) == 0) { 2697 D("%s already loaded", d->name); 2698 break; /* scheduler already exists */ 2699 } 2700 } 2701 if (s == NULL) 2702 CK_LIST_INSERT_HEAD(&schedlist, d, next); 2703 mtx_unlock(&sched_mtx); 2704 D("dn_sched %s %sloaded", d->name, s ? "not ":""); 2705 return s ? 1 : 0; 2706} 2707 2708static int 2709unload_dn_sched(struct dn_alg *s) 2710{ 2711 struct dn_alg *tmp, *r; 2712 int err = EINVAL; 2713 2714 ND("called for %s", s->name); 2715 2716 mtx_lock(&sched_mtx); 2717 CK_LIST_FOREACH_SAFE(r, &schedlist, next, tmp) { 2718 if (strcmp(s->name, r->name) != 0) 2719 continue; 2720 ND("ref_count = %d", r->ref_count); 2721 err = (r->ref_count != 0) ? EBUSY : 0; 2722 if (err == 0) 2723 CK_LIST_REMOVE(r, next); 2724 break; 2725 } 2726 mtx_unlock(&sched_mtx); 2727 NET_EPOCH_WAIT(); 2728 D("dn_sched %s %sunloaded", s->name, err ? "not ":""); 2729 return err; 2730} 2731 2732int 2733dn_sched_modevent(module_t mod, int cmd, void *arg) 2734{ 2735 struct dn_alg *sch = arg; 2736 2737 if (cmd == MOD_LOAD) 2738 return load_dn_sched(sch); 2739 else if (cmd == MOD_UNLOAD) 2740 return unload_dn_sched(sch); 2741 else 2742 return EINVAL; 2743} 2744 2745static moduledata_t dummynet_mod = { 2746 "dummynet", dummynet_modevent, NULL 2747}; 2748 2749#define DN_SI_SUB SI_SUB_PROTO_FIREWALL 2750#define DN_MODEV_ORD (SI_ORDER_ANY - 128) /* after ipfw */ 2751DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD); 2752MODULE_VERSION(dummynet, 3); 2753 2754/* 2755 * Starting up. Done in order after dummynet_modevent() has been called. 2756 * VNET_SYSINIT is also called for each existing vnet and each new vnet. 2757 */ 2758VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_vnet_init, NULL); 2759 2760/* 2761 * Shutdown handlers up shop. These are done in REVERSE ORDER, but still 2762 * after dummynet_modevent() has been called. Not called on reboot. 2763 * VNET_SYSUNINIT is also called for each exiting vnet as it exits. 2764 * or when the module is unloaded. 2765 */ 2766VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_vnet_destroy, NULL); 2767 2768#ifdef NEW_AQM 2769 2770/* modevent helpers for the AQM modules */ 2771static int 2772load_dn_aqm(struct dn_aqm *d) 2773{ 2774 struct dn_aqm *aqm=NULL; 2775 2776 if (d == NULL) 2777 return 1; /* error */ 2778 ip_dn_init(); /* just in case, we need the lock */ 2779 2780 /* Check that mandatory funcs exists */ 2781 if (d->enqueue == NULL || d->dequeue == NULL) { 2782 D("missing enqueue or dequeue for %s", d->name); 2783 return 1; 2784 } 2785 2786 mtx_lock(&sched_mtx); 2787 2788 /* Search if AQM already exists */ 2789 CK_LIST_FOREACH(aqm, &aqmlist, next) { 2790 if (strcmp(aqm->name, d->name) == 0) { 2791 D("%s already loaded", d->name); 2792 break; /* AQM already exists */ 2793 } 2794 } 2795 if (aqm == NULL) 2796 CK_LIST_INSERT_HEAD(&aqmlist, d, next); 2797 2798 mtx_unlock(&sched_mtx); 2799 2800 D("dn_aqm %s %sloaded", d->name, aqm ? "not ":""); 2801 return aqm ? 1 : 0; 2802} 2803 2804/* Callback to clean up AQM status for queues connected to a flowset 2805 * and then deconfigure the flowset. 2806 * This function is called before an AQM module is unloaded 2807 */ 2808static int 2809fs_cleanup(void *_fs, void *arg) 2810{ 2811 struct dn_fsk *fs = _fs; 2812 uint32_t type = *(uint32_t *)arg; 2813 2814 if (fs->aqmfp && fs->aqmfp->type == type) 2815 aqm_cleanup_deconfig_fs(fs); 2816 2817 return 0; 2818} 2819 2820static int 2821unload_dn_aqm(struct dn_aqm *aqm) 2822{ 2823 struct dn_aqm *tmp, *r; 2824 int err = EINVAL; 2825 err = 0; 2826 ND("called for %s", aqm->name); 2827 2828 /* clean up AQM status and deconfig flowset */ 2829 dn_ht_scan(V_dn_cfg.fshash, fs_cleanup, &aqm->type); 2830 2831 mtx_lock(&sched_mtx); 2832 2833 CK_LIST_FOREACH_SAFE(r, &aqmlist, next, tmp) { 2834 if (strcmp(aqm->name, r->name) != 0) 2835 continue; 2836 ND("ref_count = %d", r->ref_count); 2837 err = (r->ref_count != 0 || r->cfg_ref_count != 0) ? EBUSY : 0; 2838 if (err == 0) 2839 CK_LIST_REMOVE(r, next); 2840 break; 2841 } 2842 2843 mtx_unlock(&sched_mtx); 2844 NET_EPOCH_WAIT(); 2845 2846 D("%s %sunloaded", aqm->name, err ? "not ":""); 2847 if (err) 2848 D("ref_count=%d, cfg_ref_count=%d", r->ref_count, r->cfg_ref_count); 2849 return err; 2850} 2851 2852int 2853dn_aqm_modevent(module_t mod, int cmd, void *arg) 2854{ 2855 struct dn_aqm *aqm = arg; 2856 2857 if (cmd == MOD_LOAD) 2858 return load_dn_aqm(aqm); 2859 else if (cmd == MOD_UNLOAD) 2860 return unload_dn_aqm(aqm); 2861 else 2862 return EINVAL; 2863} 2864#endif 2865 2866/* end of file */ 2867