1/*- 2 * Codel/FQ_Codel and PIE/FQ_PIE Code: 3 * Copyright (C) 2016 Centre for Advanced Internet Architectures, 4 * Swinburne University of Technology, Melbourne, Australia. 5 * Portions of this code were made possible in part by a gift from 6 * The Comcast Innovation Fund. 7 * Implemented by Rasool Al-Saadi <ralsaadi@swin.edu.au> 8 * 9 * Copyright (c) 2002-2003,2010 Luigi Rizzo 10 * 11 * Redistribution and use in source forms, with and without modification, 12 * are permitted provided that this entire comment appears intact. 13 * 14 * Redistribution in binary form may occur without any restrictions. 15 * Obviously, it would be nice if you gave credit where credit is due 16 * but requiring it would be too onerous. 17 * 18 * This software is provided ``AS IS'' without any warranties of any kind. 19 * 20 * dummynet support 21 */ 22 23#define NEW_AQM 24#include <sys/limits.h> 25#include <sys/param.h> 26#include <sys/socket.h> 27/* XXX there are several sysctl leftover here */ 28#include <sys/sysctl.h> 29 30#include "ipfw2.h" 31 32#ifdef NEW_AQM 33#include <stdint.h> 34#endif 35 36#include <ctype.h> 37#include <err.h> 38#include <errno.h> 39#include <libutil.h> 40#include <netdb.h> 41#include <stdio.h> 42#include <stdlib.h> 43#include <string.h> 44#include <sysexits.h> 45 46#include <net/if.h> 47#include <netinet/in.h> 48#include <netinet/ip_fw.h> 49#include <netinet/ip_dummynet.h> 50#include <arpa/inet.h> /* inet_ntoa */ 51 52 53static struct _s_x dummynet_params[] = { 54 { "plr", TOK_PLR }, 55 { "noerror", TOK_NOERROR }, 56 { "buckets", TOK_BUCKETS }, 57 { "dst-ip", TOK_DSTIP }, 58 { "src-ip", TOK_SRCIP }, 59 { "dst-port", TOK_DSTPORT }, 60 { "src-port", TOK_SRCPORT }, 61 { "proto", TOK_PROTO }, 62 { "weight", TOK_WEIGHT }, 63 { "lmax", TOK_LMAX }, 64 { "maxlen", TOK_LMAX }, 65 { "all", TOK_ALL }, 66 { "mask", TOK_MASK }, /* alias for both */ 67 { "sched_mask", TOK_SCHED_MASK }, 68 { "flow_mask", TOK_FLOW_MASK }, 69 { "droptail", TOK_DROPTAIL }, 70 { "ecn", TOK_ECN }, 71 { "red", TOK_RED }, 72 { "gred", TOK_GRED }, 73#ifdef NEW_AQM 74 { "codel", TOK_CODEL}, /* Codel AQM */ 75 { "fq_codel", TOK_FQ_CODEL}, /* FQ-Codel */ 76 { "pie", TOK_PIE}, /* PIE AQM */ 77 { "fq_pie", TOK_FQ_PIE}, /* FQ-PIE */ 78#endif 79 { "bw", TOK_BW }, 80 { "bandwidth", TOK_BW }, 81 { "delay", TOK_DELAY }, 82 { "link", TOK_LINK }, 83 { "pipe", TOK_PIPE }, 84 { "queue", TOK_QUEUE }, 85 { "flowset", TOK_FLOWSET }, 86 { "sched", TOK_SCHED }, 87 { "pri", TOK_PRI }, 88 { "priority", TOK_PRI }, 89 { "type", TOK_TYPE }, 90 { "flow-id", TOK_FLOWID}, 91 { "dst-ipv6", TOK_DSTIP6}, 92 { "dst-ip6", TOK_DSTIP6}, 93 { "src-ipv6", TOK_SRCIP6}, 94 { "src-ip6", TOK_SRCIP6}, 95 { "profile", TOK_PROFILE}, 96 { "burst", TOK_BURST}, 97 { "dummynet-params", TOK_NULL }, 98 { NULL, 0 } /* terminator */ 99}; 100 101#ifdef NEW_AQM 102/* AQM/extra sched parameters tokens*/ 103static struct _s_x aqm_params[] = { 104 { "target", TOK_TARGET}, 105 { "interval", TOK_INTERVAL}, 106 { "limit", TOK_LIMIT}, 107 { "flows", TOK_FLOWS}, 108 { "quantum", TOK_QUANTUM}, 109 { "ecn", TOK_ECN}, 110 { "noecn", TOK_NO_ECN}, 111 { "tupdate", TOK_TUPDATE}, 112 { "max_burst", TOK_MAX_BURST}, 113 { "max_ecnth", TOK_MAX_ECNTH}, 114 { "alpha", TOK_ALPHA}, 115 { "beta", TOK_BETA}, 116 { "capdrop", TOK_CAPDROP}, 117 { "nocapdrop", TOK_NO_CAPDROP}, 118 { "onoff", TOK_ONOFF}, 119 { "dre", TOK_DRE}, 120 { "ts", TOK_TS}, 121 { "derand", TOK_DERAND}, 122 { "noderand", TOK_NO_DERAND}, 123 { NULL, 0 } /* terminator */ 124}; 125#endif 126 127#define O_NEXT(p, len) ((void *)((char *)p + len)) 128 129static void 130oid_fill(struct dn_id *oid, int len, int type, uintptr_t id) 131{ 132 oid->len = len; 133 oid->type = type; 134 oid->subtype = 0; 135 oid->id = id; 136} 137 138/* make room in the buffer and move the pointer forward */ 139static void * 140o_next(struct dn_id **o, int len, int type) 141{ 142 struct dn_id *ret = *o; 143 oid_fill(ret, len, type, 0); 144 *o = O_NEXT(*o, len); 145 return ret; 146} 147 148#ifdef NEW_AQM 149 150/* Codel flags */ 151enum { 152 CODEL_ECN_ENABLED = 1 153}; 154 155/* PIE flags, from PIE kernel module */ 156enum { 157 PIE_ECN_ENABLED = 1, 158 PIE_CAPDROP_ENABLED = 2, 159 PIE_ON_OFF_MODE_ENABLED = 4, 160 PIE_DEPRATEEST_ENABLED = 8, 161 PIE_DERAND_ENABLED = 16 162}; 163 164#define PIE_FIX_POINT_BITS 13 165#define PIE_SCALE (1L<<PIE_FIX_POINT_BITS) 166 167/* integer to time */ 168static void 169us_to_time(int t, char *strt) 170{ 171 if (t < 0) 172 strt[0]='\0'; 173 else if ( t==0 ) 174 sprintf(strt,"%d", t); 175 else if (t< 1000) 176 sprintf(strt,"%dus", t); 177 else if (t < 1000000) 178 sprintf(strt,"%gms", (float) t / 1000); 179 else 180 sprintf(strt,"%gfs", (float) t / 1000000); 181} 182 183/* 184 * returns -1 if s is not a valid time, otherwise, return time in us 185 */ 186static long 187time_to_us(const char *s) 188{ 189 int i, dots = 0; 190 int len = strlen(s); 191 char strt[16]="", stru[16]=""; 192 193 if (len>15) 194 return -1; 195 for (i = 0; i<len && (isdigit(s[i]) || s[i]=='.') ; i++) 196 if (s[i]=='.') { 197 if (dots) 198 return -1; 199 else 200 dots++; 201 } 202 203 if (!i) 204 return -1; 205 strncpy(strt, s, i); 206 if (i<len) 207 strcpy(stru, s+i); 208 else 209 strcpy(stru, "ms"); 210 211 if (!strcasecmp(stru, "us")) 212 return atol(strt); 213 if (!strcasecmp(stru, "ms")) 214 return (strtod(strt, NULL) * 1000); 215 if (!strcasecmp(stru, "s")) 216 return (strtod(strt, NULL)*1000000); 217 218 return -1; 219} 220 221 222/* Get AQM or scheduler extra parameters */ 223static void 224get_extra_parms(uint32_t nr, char *out, int subtype) 225{ 226 struct dn_extra_parms *ep; 227 int ret; 228 char strt1[15], strt2[15], strt3[15]; 229 u_int l; 230 231 /* prepare the request */ 232 l = sizeof(struct dn_extra_parms); 233 ep = safe_calloc(1, l); 234 memset(ep, 0, sizeof(*ep)); 235 *out = '\0'; 236 237 oid_fill(&ep->oid, l, DN_CMD_GET, DN_API_VERSION); 238 ep->oid.len = l; 239 ep->oid.subtype = subtype; 240 ep->nr = nr; 241 242 ret = do_cmd(-IP_DUMMYNET3, ep, (uintptr_t)&l); 243 if (ret) { 244 free(ep); 245 errx(EX_DATAERR, "Error getting extra parameters\n"); 246 } 247 248 switch (subtype) { 249 case DN_AQM_PARAMS: 250 if( !strcasecmp(ep->name, "codel")) { 251 us_to_time(ep->par[0], strt1); 252 us_to_time(ep->par[1], strt2); 253 l = sprintf(out, " AQM CoDel target %s interval %s", 254 strt1, strt2); 255 if (ep->par[2] & CODEL_ECN_ENABLED) 256 l = sprintf(out + l, " ECN"); 257 else 258 l += sprintf(out + l, " NoECN"); 259 } else if( !strcasecmp(ep->name, "pie")) { 260 us_to_time(ep->par[0], strt1); 261 us_to_time(ep->par[1], strt2); 262 us_to_time(ep->par[2], strt3); 263 l = sprintf(out, " AQM type PIE target %s tupdate %s alpha " 264 "%g beta %g max_burst %s max_ecnth %.3g", 265 strt1, 266 strt2, 267 ep->par[4] / (float) PIE_SCALE, 268 ep->par[5] / (float) PIE_SCALE, 269 strt3, 270 ep->par[3] / (float) PIE_SCALE 271 ); 272 273 if (ep->par[6] & PIE_ECN_ENABLED) 274 l += sprintf(out + l, " ECN"); 275 else 276 l += sprintf(out + l, " NoECN"); 277 if (ep->par[6] & PIE_CAPDROP_ENABLED) 278 l += sprintf(out + l, " CapDrop"); 279 else 280 l += sprintf(out + l, " NoCapDrop"); 281 if (ep->par[6] & PIE_ON_OFF_MODE_ENABLED) 282 l += sprintf(out + l, " OnOff"); 283 if (ep->par[6] & PIE_DEPRATEEST_ENABLED) 284 l += sprintf(out + l, " DRE"); 285 else 286 l += sprintf(out + l, " TS"); 287 if (ep->par[6] & PIE_DERAND_ENABLED) 288 l += sprintf(out + l, " Derand"); 289 else 290 l += sprintf(out + l, " NoDerand"); 291 } 292 break; 293 294 case DN_SCH_PARAMS: 295 if (!strcasecmp(ep->name,"FQ_CODEL")) { 296 us_to_time(ep->par[0], strt1); 297 us_to_time(ep->par[1], strt2); 298 l = sprintf(out," FQ_CODEL target %s interval %s" 299 " quantum %jd limit %jd flows %jd", 300 strt1, strt2, 301 (intmax_t) ep->par[3], 302 (intmax_t) ep->par[4], 303 (intmax_t) ep->par[5] 304 ); 305 if (ep->par[2] & CODEL_ECN_ENABLED) 306 l += sprintf(out + l, " ECN"); 307 else 308 l += sprintf(out + l, " NoECN"); 309 l += sprintf(out + l, "\n"); 310 } else if (!strcasecmp(ep->name,"FQ_PIE")) { 311 us_to_time(ep->par[0], strt1); 312 us_to_time(ep->par[1], strt2); 313 us_to_time(ep->par[2], strt3); 314 l = sprintf(out, " FQ_PIE target %s tupdate %s alpha " 315 "%g beta %g max_burst %s max_ecnth %.3g" 316 " quantum %jd limit %jd flows %jd", 317 strt1, 318 strt2, 319 ep->par[4] / (float) PIE_SCALE, 320 ep->par[5] / (float) PIE_SCALE, 321 strt3, 322 ep->par[3] / (float) PIE_SCALE, 323 (intmax_t) ep->par[7], 324 (intmax_t) ep->par[8], 325 (intmax_t) ep->par[9] 326 ); 327 328 if (ep->par[6] & PIE_ECN_ENABLED) 329 l += sprintf(out + l, " ECN"); 330 else 331 l += sprintf(out + l, " NoECN"); 332 if (ep->par[6] & PIE_CAPDROP_ENABLED) 333 l += sprintf(out + l, " CapDrop"); 334 else 335 l += sprintf(out + l, " NoCapDrop"); 336 if (ep->par[6] & PIE_ON_OFF_MODE_ENABLED) 337 l += sprintf(out + l, " OnOff"); 338 if (ep->par[6] & PIE_DEPRATEEST_ENABLED) 339 l += sprintf(out + l, " DRE"); 340 else 341 l += sprintf(out + l, " TS"); 342 if (ep->par[6] & PIE_DERAND_ENABLED) 343 l += sprintf(out + l, " Derand"); 344 else 345 l += sprintf(out + l, " NoDerand"); 346 l += sprintf(out + l, "\n"); 347 } 348 break; 349 } 350 351 free(ep); 352} 353#endif 354 355 356#if 0 357static int 358sort_q(void *arg, const void *pa, const void *pb) 359{ 360 int rev = (co.do_sort < 0); 361 int field = rev ? -co.do_sort : co.do_sort; 362 long long res = 0; 363 const struct dn_flow_queue *a = pa; 364 const struct dn_flow_queue *b = pb; 365 366 switch (field) { 367 case 1: /* pkts */ 368 res = a->len - b->len; 369 break; 370 case 2: /* bytes */ 371 res = a->len_bytes - b->len_bytes; 372 break; 373 374 case 3: /* tot pkts */ 375 res = a->tot_pkts - b->tot_pkts; 376 break; 377 378 case 4: /* tot bytes */ 379 res = a->tot_bytes - b->tot_bytes; 380 break; 381 } 382 if (res < 0) 383 res = -1; 384 if (res > 0) 385 res = 1; 386 return (int)(rev ? res : -res); 387} 388#endif 389 390/* print a mask and header for the subsequent list of flows */ 391static void 392print_mask(struct ipfw_flow_id *id) 393{ 394 if (!IS_IP6_FLOW_ID(id)) { 395 printf(" " 396 "mask: %s 0x%02x 0x%08x/0x%04x -> 0x%08x/0x%04x\n", 397 id->extra ? "queue," : "", 398 id->proto, 399 id->src_ip, id->src_port, 400 id->dst_ip, id->dst_port); 401 } else { 402 char buf[255]; 403 printf("\n mask: %sproto: 0x%02x, flow_id: 0x%08x, ", 404 id->extra ? "queue," : "", 405 id->proto, id->flow_id6); 406 inet_ntop(AF_INET6, &(id->src_ip6), buf, sizeof(buf)); 407 printf("%s/0x%04x -> ", buf, id->src_port); 408 inet_ntop(AF_INET6, &(id->dst_ip6), buf, sizeof(buf)); 409 printf("%s/0x%04x\n", buf, id->dst_port); 410 } 411} 412 413static void 414print_header(struct ipfw_flow_id *id) 415{ 416 if (!IS_IP6_FLOW_ID(id)) 417 printf("BKT Prot ___Source IP/port____ " 418 "____Dest. IP/port____ " 419 "Tot_pkt/bytes Pkt/Byte Drp\n"); 420 else 421 printf("BKT ___Prot___ _flow-id_ " 422 "______________Source IPv6/port_______________ " 423 "_______________Dest. IPv6/port_______________ " 424 "Tot_pkt/bytes Pkt/Byte Drp\n"); 425} 426 427static void 428list_flow(struct buf_pr *bp, struct dn_flow *ni) 429{ 430 char buff[255]; 431 struct protoent *pe = NULL; 432 struct in_addr ina; 433 struct ipfw_flow_id *id = &ni->fid; 434 435 pe = getprotobynumber(id->proto); 436 /* XXX: Should check for IPv4 flows */ 437 bprintf(bp, "%3u%c", (ni->oid.id) & 0xff, 438 id->extra ? '*' : ' '); 439 if (!IS_IP6_FLOW_ID(id)) { 440 if (pe) 441 bprintf(bp, "%-4s ", pe->p_name); 442 else 443 bprintf(bp, "%4u ", id->proto); 444 ina.s_addr = htonl(id->src_ip); 445 bprintf(bp, "%15s/%-5d ", 446 inet_ntoa(ina), id->src_port); 447 ina.s_addr = htonl(id->dst_ip); 448 bprintf(bp, "%15s/%-5d ", 449 inet_ntoa(ina), id->dst_port); 450 } else { 451 /* Print IPv6 flows */ 452 if (pe != NULL) 453 bprintf(bp, "%9s ", pe->p_name); 454 else 455 bprintf(bp, "%9u ", id->proto); 456 bprintf(bp, "%7d %39s/%-5d ", id->flow_id6, 457 inet_ntop(AF_INET6, &(id->src_ip6), buff, sizeof(buff)), 458 id->src_port); 459 bprintf(bp, " %39s/%-5d ", 460 inet_ntop(AF_INET6, &(id->dst_ip6), buff, sizeof(buff)), 461 id->dst_port); 462 } 463 pr_u64(bp, &ni->tot_pkts, 4); 464 pr_u64(bp, &ni->tot_bytes, 8); 465 bprintf(bp, "%2u %4u %3u", 466 ni->length, ni->len_bytes, ni->drops); 467} 468 469static void 470print_flowset_parms(struct dn_fs *fs, char *prefix) 471{ 472 int l; 473 char qs[30]; 474 char plr[40]; 475 char red[200]; /* Display RED parameters */ 476 477 l = fs->qsize; 478 if (fs->flags & DN_QSIZE_BYTES) { 479 if (l >= 8192) 480 sprintf(qs, "%d KB", l / 1024); 481 else 482 sprintf(qs, "%d B", l); 483 } else 484 sprintf(qs, "%3d sl.", l); 485 if (fs->plr[0] || fs->plr[1]) { 486 if (fs->plr[1] == 0) 487 sprintf(plr, "plr %f", 488 1.0 * fs->plr[0] / (double)(0x7fffffff)); 489 else 490 sprintf(plr, "plr %f,%f,%f,%f", 491 1.0 * fs->plr[0] / (double)(0x7fffffff), 492 1.0 * fs->plr[1] / (double)(0x7fffffff), 493 1.0 * fs->plr[2] / (double)(0x7fffffff), 494 1.0 * fs->plr[3] / (double)(0x7fffffff)); 495 } else 496 plr[0] = '\0'; 497 498 if (fs->flags & DN_IS_RED) { /* RED parameters */ 499 sprintf(red, 500 "\n\t %cRED w_q %f min_th %d max_th %d max_p %f", 501 (fs->flags & DN_IS_GENTLE_RED) ? 'G' : ' ', 502 1.0 * fs->w_q / (double)(1 << SCALE_RED), 503 fs->min_th, 504 fs->max_th, 505 1.0 * fs->max_p / (double)(1 << SCALE_RED)); 506 if (fs->flags & DN_IS_ECN) 507 strlcat(red, " (ecn)", sizeof(red)); 508#ifdef NEW_AQM 509 /* get AQM parameters */ 510 } else if (fs->flags & DN_IS_AQM) { 511 get_extra_parms(fs->fs_nr, red, DN_AQM_PARAMS); 512#endif 513 } else 514 sprintf(red, "droptail"); 515 516 if (prefix[0]) { 517 printf("%s %s%s %d queues (%d buckets) %s\n", 518 prefix, qs, plr, fs->oid.id, fs->buckets, red); 519 prefix[0] = '\0'; 520 } else { 521 printf("q%05d %s%s %d flows (%d buckets) sched %d " 522 "weight %d lmax %d pri %d %s\n", 523 fs->fs_nr, qs, plr, fs->oid.id, fs->buckets, 524 fs->sched_nr, fs->par[0], fs->par[1], fs->par[2], red); 525 if (fs->flags & DN_HAVE_MASK) 526 print_mask(&fs->flow_mask); 527 } 528} 529 530static void 531print_extra_delay_parms(struct dn_profile *p) 532{ 533 double loss; 534 if (p->samples_no <= 0) 535 return; 536 537 loss = p->loss_level; 538 loss /= p->samples_no; 539 printf("\t profile: name \"%s\" loss %f samples %d\n", 540 p->name, loss, p->samples_no); 541} 542 543static void 544flush_buf(char *buf) 545{ 546 if (buf[0]) 547 printf("%s\n", buf); 548 buf[0] = '\0'; 549} 550 551/* 552 * generic list routine. We expect objects in a specific order, i.e. 553 * PIPES AND SCHEDULERS: 554 * link; scheduler; internal flowset if any; instances 555 * we can tell a pipe from the number. 556 * 557 * FLOWSETS: 558 * flowset; queues; 559 * link i (int queue); scheduler i; si(i) { flowsets() : queues } 560 */ 561static void 562list_pipes(struct dn_id *oid, struct dn_id *end) 563{ 564 char buf[160]; /* pending buffer */ 565 int toPrint = 1; /* print header */ 566 struct buf_pr bp; 567 568 buf[0] = '\0'; 569 bp_alloc(&bp, 4096); 570 for (; oid != end; oid = O_NEXT(oid, oid->len)) { 571 if (oid->len < sizeof(*oid)) 572 errx(1, "invalid oid len %d\n", oid->len); 573 574 switch (oid->type) { 575 default: 576 flush_buf(buf); 577 printf("unrecognized object %d size %d\n", oid->type, oid->len); 578 break; 579 case DN_TEXT: /* list of attached flowsets */ 580 { 581 int i, l; 582 struct { 583 struct dn_id id; 584 uint32_t p[0]; 585 } *d = (void *)oid; 586 l = (oid->len - sizeof(*oid))/sizeof(d->p[0]); 587 if (l == 0) 588 break; 589 printf(" Children flowsets: "); 590 for (i = 0; i < l; i++) 591 printf("%u ", d->p[i]); 592 printf("\n"); 593 break; 594 } 595 case DN_CMD_GET: 596 if (g_co.verbose) 597 printf("answer for cmd %d, len %d\n", oid->type, oid->id); 598 break; 599 case DN_SCH: { 600 struct dn_sch *s = (struct dn_sch *)oid; 601 flush_buf(buf); 602 printf(" sched %d type %s flags 0x%x %d buckets %d active\n", 603 s->sched_nr, 604 s->name, s->flags, s->buckets, s->oid.id); 605#ifdef NEW_AQM 606 char parms[200]; 607 get_extra_parms(s->sched_nr, parms, DN_SCH_PARAMS); 608 printf("%s",parms); 609#endif 610 if (s->flags & DN_HAVE_MASK) 611 print_mask(&s->sched_mask); 612 } 613 break; 614 615 case DN_FLOW: 616 if (toPrint != 0) { 617 print_header(&((struct dn_flow *)oid)->fid); 618 toPrint = 0; 619 } 620 list_flow(&bp, (struct dn_flow *)oid); 621 printf("%s\n", bp.buf); 622 bp_flush(&bp); 623 break; 624 625 case DN_LINK: { 626 struct dn_link *p = (struct dn_link *)oid; 627 double b = p->bandwidth; 628 char bwbuf[30]; 629 char burst[5 + 7]; 630 631 /* This starts a new object so flush buffer */ 632 flush_buf(buf); 633 /* data rate */ 634 if (b == 0) 635 sprintf(bwbuf, "unlimited "); 636 else if (b >= 1000000000) 637 sprintf(bwbuf, "%7.3f Gbit/s", b/1000000000); 638 else if (b >= 1000000) 639 sprintf(bwbuf, "%7.3f Mbit/s", b/1000000); 640 else if (b >= 1000) 641 sprintf(bwbuf, "%7.3f Kbit/s", b/1000); 642 else 643 sprintf(bwbuf, "%7.3f bit/s ", b); 644 645 if (humanize_number(burst, sizeof(burst), p->burst, 646 "", HN_AUTOSCALE, 0) < 0 || g_co.verbose) 647 sprintf(burst, "%d", (int)p->burst); 648 sprintf(buf, "%05d: %s %4d ms burst %s", 649 p->link_nr % DN_MAX_ID, bwbuf, p->delay, burst); 650 } 651 break; 652 653 case DN_FS: 654 print_flowset_parms((struct dn_fs *)oid, buf); 655 break; 656 case DN_PROFILE: 657 flush_buf(buf); 658 print_extra_delay_parms((struct dn_profile *)oid); 659 } 660 flush_buf(buf); // XXX does it really go here ? 661 } 662 663 bp_free(&bp); 664} 665 666/* 667 * Delete pipe, queue or scheduler i 668 */ 669int 670ipfw_delete_pipe(int do_pipe, int i) 671{ 672 struct { 673 struct dn_id oid; 674 uintptr_t a[1]; /* add more if we want a list */ 675 } cmd; 676 oid_fill((void *)&cmd, sizeof(cmd), DN_CMD_DELETE, DN_API_VERSION); 677 cmd.oid.subtype = (do_pipe == 1) ? DN_LINK : 678 ( (do_pipe == 2) ? DN_FS : DN_SCH); 679 cmd.a[0] = i; 680 i = do_cmd(IP_DUMMYNET3, &cmd, cmd.oid.len); 681 if (i) { 682 i = 1; 683 warn("rule %u: setsockopt(IP_DUMMYNET_DEL)", i); 684 } 685 return i; 686} 687 688/* 689 * Code to parse delay profiles. 690 * 691 * Some link types introduce extra delays in the transmission 692 * of a packet, e.g. because of MAC level framing, contention on 693 * the use of the channel, MAC level retransmissions and so on. 694 * From our point of view, the channel is effectively unavailable 695 * for this extra time, which is constant or variable depending 696 * on the link type. Additionally, packets may be dropped after this 697 * time (e.g. on a wireless link after too many retransmissions). 698 * We can model the additional delay with an empirical curve 699 * that represents its distribution. 700 * 701 * cumulative probability 702 * 1.0 ^ 703 * | 704 * L +-- loss-level x 705 * | ****** 706 * | * 707 * | ***** 708 * | * 709 * | ** 710 * | * 711 * +-------*-------------------> 712 * delay 713 * 714 * The empirical curve may have both vertical and horizontal lines. 715 * Vertical lines represent constant delay for a range of 716 * probabilities; horizontal lines correspond to a discontinuty 717 * in the delay distribution: the link will use the largest delay 718 * for a given probability. 719 * 720 * To pass the curve to dummynet, we must store the parameters 721 * in a file as described below, and issue the command 722 * 723 * ipfw pipe <n> config ... bw XXX profile <filename> ... 724 * 725 * The file format is the following, with whitespace acting as 726 * a separator and '#' indicating the beginning a comment: 727 * 728 * samples N 729 * the number of samples used in the internal 730 * representation (2..1024; default 100); 731 * 732 * loss-level L 733 * The probability above which packets are lost. 734 * (0.0 <= L <= 1.0, default 1.0 i.e. no loss); 735 * 736 * name identifier 737 * Optional a name (listed by "ipfw pipe show") 738 * to identify the distribution; 739 * 740 * "delay prob" | "prob delay" 741 * One of these two lines is mandatory and defines 742 * the format of the following lines with data points. 743 * 744 * XXX YYY 745 * 2 or more lines representing points in the curve, 746 * with either delay or probability first, according 747 * to the chosen format. 748 * The unit for delay is milliseconds. 749 * 750 * Data points does not need to be ordered or equal to the number 751 * specified in the "samples" line. ipfw will sort and interpolate 752 * the curve as needed. 753 * 754 * Example of a profile file: 755 756 name bla_bla_bla 757 samples 100 758 loss-level 0.86 759 prob delay 760 0 200 # minimum overhead is 200ms 761 0.5 200 762 0.5 300 763 0.8 1000 764 0.9 1300 765 1 1300 766 767 * Internally, we will convert the curve to a fixed number of 768 * samples, and when it is time to transmit a packet we will 769 * model the extra delay as extra bits in the packet. 770 * 771 */ 772 773#define ED_MAX_LINE_LEN 256+ED_MAX_NAME_LEN 774#define ED_TOK_SAMPLES "samples" 775#define ED_TOK_LOSS "loss-level" 776#define ED_TOK_NAME "name" 777#define ED_TOK_DELAY "delay" 778#define ED_TOK_PROB "prob" 779#define ED_TOK_BW "bw" 780#define ED_SEPARATORS " \t\n" 781#define ED_MIN_SAMPLES_NO 2 782 783/* 784 * returns 1 if s is a non-negative number, with at least one '.' 785 */ 786static int 787is_valid_number(const char *s) 788{ 789 int i, dots_found = 0; 790 int len = strlen(s); 791 792 for (i = 0; i<len; ++i) 793 if (!isdigit(s[i]) && (s[i] !='.' || ++dots_found > 1)) 794 return 0; 795 return 1; 796} 797 798/* 799 * Take as input a string describing a bandwidth value 800 * and return the numeric bandwidth value. 801 * set clocking interface or bandwidth value 802 */ 803static void 804read_bandwidth(char *arg, uint32_t *bandwidth, char *if_name, int namelen) 805{ 806 if (*bandwidth != (uint32_t)-1) 807 warnx("duplicate token, override bandwidth value!"); 808 809 if (arg[0] >= 'a' && arg[0] <= 'z') { 810 if (!if_name) { 811 errx(1, "no if support"); 812 } 813 if (namelen >= IFNAMSIZ) 814 warn("interface name truncated"); 815 namelen--; 816 /* interface name */ 817 strlcpy(if_name, arg, namelen); 818 *bandwidth = 0; 819 } else { /* read bandwidth value */ 820 uint64_t bw; 821 char *end = NULL; 822 823 bw = strtoul(arg, &end, 0); 824 if (*end == 'K' || *end == 'k') { 825 end++; 826 bw *= 1000; 827 } else if (*end == 'M' || *end == 'm') { 828 end++; 829 bw *= 1000000; 830 } else if (*end == 'G' || *end == 'g') { 831 end++; 832 bw *= 1000000000; 833 } 834 if ((*end == 'B' && 835 _substrcmp2(end, "Bi", "Bit/s") != 0) || 836 _substrcmp2(end, "by", "bytes") == 0) 837 bw *= 8; 838 839 if (bw > UINT_MAX) 840 errx(EX_DATAERR, "bandwidth too large"); 841 842 *bandwidth = (uint32_t)bw; 843 if (if_name) 844 if_name[0] = '\0'; 845 } 846} 847 848struct point { 849 double prob; 850 double delay; 851}; 852 853static int 854compare_points(const void *vp1, const void *vp2) 855{ 856 const struct point *p1 = vp1; 857 const struct point *p2 = vp2; 858 double res = 0; 859 860 res = p1->prob - p2->prob; 861 if (res == 0) 862 res = p1->delay - p2->delay; 863 if (res < 0) 864 return -1; 865 else if (res > 0) 866 return 1; 867 else 868 return 0; 869} 870 871#define ED_EFMT(s) EX_DATAERR,"error in %s at line %d: "#s,filename,lineno 872 873static void 874load_extra_delays(const char *filename, struct dn_profile *p, 875 struct dn_link *link) 876{ 877 char line[ED_MAX_LINE_LEN]; 878 FILE *f; 879 int lineno = 0; 880 int i; 881 882 int samples = -1; 883 double loss = -1.0; 884 char profile_name[ED_MAX_NAME_LEN]; 885 int delay_first = -1; 886 int do_points = 0; 887 struct point points[ED_MAX_SAMPLES_NO]; 888 int points_no = 0; 889 890 /* XXX link never NULL? */ 891 p->link_nr = link->link_nr; 892 893 profile_name[0] = '\0'; 894 f = fopen(filename, "r"); 895 if (f == NULL) 896 err(EX_UNAVAILABLE, "fopen: %s", filename); 897 898 while (fgets(line, ED_MAX_LINE_LEN, f)) { /* read commands */ 899 char *s, *cur = line, *name = NULL, *arg = NULL; 900 901 ++lineno; 902 903 /* parse the line */ 904 while (cur) { 905 s = strsep(&cur, ED_SEPARATORS); 906 if (s == NULL || *s == '#') 907 break; 908 if (*s == '\0') 909 continue; 910 if (arg) 911 errx(ED_EFMT("too many arguments")); 912 if (name == NULL) 913 name = s; 914 else 915 arg = s; 916 } 917 if (name == NULL) /* empty line */ 918 continue; 919 if (arg == NULL) 920 errx(ED_EFMT("missing arg for %s"), name); 921 922 if (!strcasecmp(name, ED_TOK_SAMPLES)) { 923 if (samples > 0) 924 errx(ED_EFMT("duplicate ``samples'' line")); 925 if (atoi(arg) <=0) 926 errx(ED_EFMT("invalid number of samples")); 927 samples = atoi(arg); 928 if (samples>ED_MAX_SAMPLES_NO) 929 errx(ED_EFMT("too many samples, maximum is %d"), 930 ED_MAX_SAMPLES_NO); 931 do_points = 0; 932 } else if (!strcasecmp(name, ED_TOK_BW)) { 933 char buf[IFNAMSIZ]; 934 read_bandwidth(arg, &link->bandwidth, buf, sizeof(buf)); 935 } else if (!strcasecmp(name, ED_TOK_LOSS)) { 936 if (loss != -1.0) 937 errx(ED_EFMT("duplicated token: %s"), name); 938 if (!is_valid_number(arg)) 939 errx(ED_EFMT("invalid %s"), arg); 940 loss = atof(arg); 941 if (loss > 1) 942 errx(ED_EFMT("%s greater than 1.0"), name); 943 do_points = 0; 944 } else if (!strcasecmp(name, ED_TOK_NAME)) { 945 if (profile_name[0] != '\0') 946 errx(ED_EFMT("duplicated token: %s"), name); 947 strlcpy(profile_name, arg, sizeof(profile_name)); 948 do_points = 0; 949 } else if (!strcasecmp(name, ED_TOK_DELAY)) { 950 if (do_points) 951 errx(ED_EFMT("duplicated token: %s"), name); 952 delay_first = 1; 953 do_points = 1; 954 } else if (!strcasecmp(name, ED_TOK_PROB)) { 955 if (do_points) 956 errx(ED_EFMT("duplicated token: %s"), name); 957 delay_first = 0; 958 do_points = 1; 959 } else if (do_points) { 960 if (!is_valid_number(name) || !is_valid_number(arg)) 961 errx(ED_EFMT("invalid point found")); 962 if (delay_first) { 963 points[points_no].delay = atof(name); 964 points[points_no].prob = atof(arg); 965 } else { 966 points[points_no].delay = atof(arg); 967 points[points_no].prob = atof(name); 968 } 969 if (points[points_no].prob > 1.0) 970 errx(ED_EFMT("probability greater than 1.0")); 971 ++points_no; 972 } else { 973 errx(ED_EFMT("unrecognised command '%s'"), name); 974 } 975 } 976 977 fclose (f); 978 979 if (samples == -1) { 980 warnx("'%s' not found, assuming 100", ED_TOK_SAMPLES); 981 samples = 100; 982 } 983 984 if (loss == -1.0) { 985 warnx("'%s' not found, assuming no loss", ED_TOK_LOSS); 986 loss = 1; 987 } 988 989 /* make sure that there are enough points. */ 990 if (points_no < ED_MIN_SAMPLES_NO) 991 errx(ED_EFMT("too few samples, need at least %d"), 992 ED_MIN_SAMPLES_NO); 993 994 qsort(points, points_no, sizeof(struct point), compare_points); 995 996 /* interpolation */ 997 for (i = 0; i<points_no-1; ++i) { 998 double y1 = points[i].prob * samples; 999 double x1 = points[i].delay; 1000 double y2 = points[i+1].prob * samples; 1001 double x2 = points[i+1].delay; 1002 1003 int ix = y1; 1004 int stop = y2; 1005 1006 if (x1 == x2) { 1007 for (; ix<stop; ++ix) 1008 p->samples[ix] = x1; 1009 } else { 1010 double m = (y2-y1)/(x2-x1); 1011 double c = y1 - m*x1; 1012 for (; ix<stop ; ++ix) 1013 p->samples[ix] = (ix - c)/m; 1014 } 1015 } 1016 p->samples_no = samples; 1017 p->loss_level = loss * samples; 1018 strlcpy(p->name, profile_name, sizeof(p->name)); 1019} 1020 1021#ifdef NEW_AQM 1022 1023/* Parse AQM/extra scheduler parameters */ 1024static int 1025process_extra_parms(int *ac, char **av, struct dn_extra_parms *ep, 1026 uint16_t type) 1027{ 1028 int i; 1029 1030 /* use kernel defaults */ 1031 for (i=0; i<DN_MAX_EXTRA_PARM; i++) 1032 ep->par[i] = -1; 1033 1034 switch(type) { 1035 case TOK_CODEL: 1036 case TOK_FQ_CODEL: 1037 /* Codel 1038 * 0- target, 1- interval, 2- flags, 1039 * FQ_CODEL 1040 * 3- quantum, 4- limit, 5- flows 1041 */ 1042 if (type==TOK_CODEL) 1043 ep->par[2] = 0; 1044 else 1045 ep->par[2] = CODEL_ECN_ENABLED; 1046 1047 while (*ac > 0) { 1048 int tok = match_token(aqm_params, *av); 1049 (*ac)--; av++; 1050 switch(tok) { 1051 case TOK_TARGET: 1052 if (*ac <= 0 || time_to_us(av[0]) < 0) 1053 errx(EX_DATAERR, "target needs time\n"); 1054 1055 ep->par[0] = time_to_us(av[0]); 1056 (*ac)--; av++; 1057 break; 1058 1059 case TOK_INTERVAL: 1060 if (*ac <= 0 || time_to_us(av[0]) < 0) 1061 errx(EX_DATAERR, "interval needs time\n"); 1062 1063 ep->par[1] = time_to_us(av[0]); 1064 (*ac)--; av++; 1065 break; 1066 1067 case TOK_ECN: 1068 ep->par[2] = CODEL_ECN_ENABLED; 1069 break; 1070 case TOK_NO_ECN: 1071 ep->par[2] &= ~CODEL_ECN_ENABLED; 1072 break; 1073 /* Config fq_codel parameters */ 1074 case TOK_QUANTUM: 1075 if (type != TOK_FQ_CODEL) 1076 errx(EX_DATAERR, "quantum is not for codel\n"); 1077 if (*ac <= 0 || !is_valid_number(av[0])) 1078 errx(EX_DATAERR, "quantum needs number\n"); 1079 1080 ep->par[3]= atoi(av[0]); 1081 (*ac)--; av++; 1082 break; 1083 1084 case TOK_LIMIT: 1085 if (type != TOK_FQ_CODEL) 1086 errx(EX_DATAERR, "limit is not for codel, use queue instead\n"); 1087 if (*ac <= 0 || !is_valid_number(av[0])) 1088 errx(EX_DATAERR, "limit needs number\n"); 1089 1090 ep->par[4] = atoi(av[0]); 1091 (*ac)--; av++; 1092 break; 1093 1094 case TOK_FLOWS: 1095 if (type != TOK_FQ_CODEL) 1096 errx(EX_DATAERR, "flows is not for codel\n"); 1097 if (*ac <= 0 || !is_valid_number(av[0])) 1098 errx(EX_DATAERR, "flows needs number\n"); 1099 1100 ep->par[5] = atoi(av[0]); 1101 (*ac)--; av++; 1102 break; 1103 1104 default: 1105 printf("%s is Invalid parameter\n", av[-1]); 1106 } 1107 } 1108 break; 1109 case TOK_PIE: 1110 case TOK_FQ_PIE: 1111 /* PIE 1112 * 0- target , 1- tupdate, 2- max_burst, 1113 * 3- max_ecnth, 4- alpha, 1114 * 5- beta, 6- flags 1115 * FQ_CODEL 1116 * 7- quantum, 8- limit, 9- flows 1117 */ 1118 1119 if ( type == TOK_PIE) 1120 ep->par[6] = PIE_CAPDROP_ENABLED | PIE_DEPRATEEST_ENABLED 1121 | PIE_DERAND_ENABLED; 1122 else 1123 /* for FQ-PIE, use TS mode */ 1124 ep->par[6] = PIE_CAPDROP_ENABLED | PIE_DERAND_ENABLED 1125 | PIE_ECN_ENABLED; 1126 1127 while (*ac > 0) { 1128 int tok = match_token(aqm_params, *av); 1129 (*ac)--; av++; 1130 switch(tok) { 1131 case TOK_TARGET: 1132 if (*ac <= 0 || time_to_us(av[0]) < 0) 1133 errx(EX_DATAERR, "target needs time\n"); 1134 1135 ep->par[0] = time_to_us(av[0]); 1136 (*ac)--; av++; 1137 break; 1138 1139 case TOK_TUPDATE: 1140 if (*ac <= 0 || time_to_us(av[0]) < 0) 1141 errx(EX_DATAERR, "tupdate needs time\n"); 1142 1143 ep->par[1] = time_to_us(av[0]); 1144 (*ac)--; av++; 1145 break; 1146 1147 case TOK_MAX_BURST: 1148 if (*ac <= 0 || time_to_us(av[0]) < 0) 1149 errx(EX_DATAERR, "max_burst needs time\n"); 1150 1151 ep->par[2] = time_to_us(av[0]); 1152 (*ac)--; av++; 1153 break; 1154 1155 case TOK_MAX_ECNTH: 1156 if (*ac <= 0 || !is_valid_number(av[0])) 1157 errx(EX_DATAERR, "max_ecnth needs number\n"); 1158 1159 ep->par[3] = atof(av[0]) * PIE_SCALE; 1160 (*ac)--; av++; 1161 break; 1162 1163 case TOK_ALPHA: 1164 if (*ac <= 0 || !is_valid_number(av[0])) 1165 errx(EX_DATAERR, "alpha needs number\n"); 1166 1167 ep->par[4] = atof(av[0]) * PIE_SCALE; 1168 (*ac)--; av++; 1169 break; 1170 1171 case TOK_BETA: 1172 if (*ac <= 0 || !is_valid_number(av[0])) 1173 errx(EX_DATAERR, "beta needs number\n"); 1174 1175 ep->par[5] = atof(av[0]) * PIE_SCALE; 1176 (*ac)--; av++; 1177 break; 1178 1179 case TOK_ECN: 1180 ep->par[6] |= PIE_ECN_ENABLED; 1181 break; 1182 case TOK_NO_ECN: 1183 ep->par[6] &= ~PIE_ECN_ENABLED; 1184 break; 1185 1186 case TOK_CAPDROP: 1187 ep->par[6] |= PIE_CAPDROP_ENABLED; 1188 break; 1189 case TOK_NO_CAPDROP: 1190 ep->par[6] &= ~PIE_CAPDROP_ENABLED; 1191 break; 1192 1193 case TOK_ONOFF: 1194 ep->par[6] |= PIE_ON_OFF_MODE_ENABLED; 1195 break; 1196 1197 case TOK_DRE: 1198 ep->par[6] |= PIE_DEPRATEEST_ENABLED; 1199 break; 1200 1201 case TOK_TS: 1202 ep->par[6] &= ~PIE_DEPRATEEST_ENABLED; 1203 break; 1204 1205 case TOK_DERAND: 1206 ep->par[6] |= PIE_DERAND_ENABLED; 1207 break; 1208 case TOK_NO_DERAND: 1209 ep->par[6] &= ~PIE_DERAND_ENABLED; 1210 break; 1211 1212 /* Config fq_pie parameters */ 1213 case TOK_QUANTUM: 1214 if (type != TOK_FQ_PIE) 1215 errx(EX_DATAERR, "quantum is not for pie\n"); 1216 if (*ac <= 0 || !is_valid_number(av[0])) 1217 errx(EX_DATAERR, "quantum needs number\n"); 1218 1219 ep->par[7]= atoi(av[0]); 1220 (*ac)--; av++; 1221 break; 1222 1223 case TOK_LIMIT: 1224 if (type != TOK_FQ_PIE) 1225 errx(EX_DATAERR, "limit is not for pie, use queue instead\n"); 1226 if (*ac <= 0 || !is_valid_number(av[0])) 1227 errx(EX_DATAERR, "limit needs number\n"); 1228 1229 ep->par[8] = atoi(av[0]); 1230 (*ac)--; av++; 1231 break; 1232 1233 case TOK_FLOWS: 1234 if (type != TOK_FQ_PIE) 1235 errx(EX_DATAERR, "flows is not for pie\n"); 1236 if (*ac <= 0 || !is_valid_number(av[0])) 1237 errx(EX_DATAERR, "flows needs number\n"); 1238 1239 ep->par[9] = atoi(av[0]); 1240 (*ac)--; av++; 1241 break; 1242 1243 1244 default: 1245 printf("%s is invalid parameter\n", av[-1]); 1246 } 1247 } 1248 break; 1249 } 1250 1251 return 0; 1252} 1253 1254#endif 1255 1256 1257/* 1258 * configuration of pipes, schedulers, flowsets. 1259 * When we configure a new scheduler, an empty pipe is created, so: 1260 * 1261 * do_pipe = 1 -> "pipe N config ..." only for backward compatibility 1262 * sched N+Delta type fifo sched_mask ... 1263 * pipe N+Delta <parameters> 1264 * flowset N+Delta pipe N+Delta (no parameters) 1265 * sched N type wf2q+ sched_mask ... 1266 * pipe N <parameters> 1267 * 1268 * do_pipe = 2 -> flowset N config 1269 * flowset N parameters 1270 * 1271 * do_pipe = 3 -> sched N config 1272 * sched N parameters (default no pipe) 1273 * optional Pipe N config ... 1274 * pipe ==> 1275 */ 1276void 1277ipfw_config_pipe(int ac, char **av) 1278{ 1279 int i; 1280 u_int j; 1281 char *end; 1282 struct dn_id *buf, *base; 1283 struct dn_sch *sch = NULL; 1284 struct dn_link *p = NULL; 1285 struct dn_fs *fs = NULL; 1286 struct dn_profile *pf = NULL; 1287 struct ipfw_flow_id *mask = NULL; 1288#ifdef NEW_AQM 1289 struct dn_extra_parms *aqm_extra = NULL; 1290 struct dn_extra_parms *sch_extra = NULL; 1291 int lmax_extra; 1292#endif 1293 1294 int lmax; 1295 uint32_t _foo = 0, *flags = &_foo , *buckets = &_foo; 1296 1297 /* 1298 * allocate space for 1 header, 1299 * 1 scheduler, 1 link, 1 flowset, 1 profile 1300 */ 1301 lmax = sizeof(struct dn_id); /* command header */ 1302 lmax += sizeof(struct dn_sch) + sizeof(struct dn_link) + 1303 sizeof(struct dn_fs) + sizeof(struct dn_profile); 1304 1305#ifdef NEW_AQM 1306 /* Extra Params */ 1307 lmax_extra = sizeof(struct dn_extra_parms); 1308 /* two lmax_extra because one for AQM params and another 1309 * sch params 1310 */ 1311 lmax += lmax_extra*2; 1312#endif 1313 1314 av++; ac--; 1315 /* Pipe number */ 1316 if (ac && isdigit(**av)) { 1317 i = atoi(*av); av++; ac--; 1318 } else 1319 i = -1; 1320 if (i <= 0) 1321 errx(EX_USAGE, "need a pipe/flowset/sched number"); 1322 base = buf = safe_calloc(1, lmax); 1323 /* all commands start with a 'CONFIGURE' and a version */ 1324 o_next(&buf, sizeof(struct dn_id), DN_CMD_CONFIG); 1325 base->id = DN_API_VERSION; 1326 1327 switch (g_co.do_pipe) { 1328 case 1: /* "pipe N config ..." */ 1329 /* Allocate space for the WF2Q+ scheduler, its link 1330 * and the FIFO flowset. Set the number, but leave 1331 * the scheduler subtype and other parameters to 0 1332 * so the kernel will use appropriate defaults. 1333 * XXX todo: add a flag to record if a parameter 1334 * is actually configured. 1335 * If we do a 'pipe config' mask -> sched_mask. 1336 * The FIFO scheduler and link are derived from the 1337 * WF2Q+ one in the kernel. 1338 */ 1339#ifdef NEW_AQM 1340 sch_extra = o_next(&buf, lmax_extra, DN_TEXT); 1341 sch_extra ->oid.subtype = 0; /* don't configure scheduler */ 1342#endif 1343 sch = o_next(&buf, sizeof(*sch), DN_SCH); 1344 p = o_next(&buf, sizeof(*p), DN_LINK); 1345#ifdef NEW_AQM 1346 aqm_extra = o_next(&buf, lmax_extra, DN_TEXT); 1347 aqm_extra ->oid.subtype = 0; /* don't configure AQM */ 1348#endif 1349 fs = o_next(&buf, sizeof(*fs), DN_FS); 1350 1351 sch->sched_nr = i; 1352 sch->oid.subtype = 0; /* defaults to WF2Q+ */ 1353 mask = &sch->sched_mask; 1354 flags = &sch->flags; 1355 buckets = &sch->buckets; 1356 *flags |= DN_PIPE_CMD; 1357 1358 p->link_nr = i; 1359 1360 /* This flowset is only for the FIFO scheduler */ 1361 fs->fs_nr = i + 2*DN_MAX_ID; 1362 fs->sched_nr = i + DN_MAX_ID; 1363 break; 1364 1365 case 2: /* "queue N config ... " */ 1366#ifdef NEW_AQM 1367 aqm_extra = o_next(&buf, lmax_extra, DN_TEXT); 1368 aqm_extra ->oid.subtype = 0; 1369#endif 1370 fs = o_next(&buf, sizeof(*fs), DN_FS); 1371 fs->fs_nr = i; 1372 mask = &fs->flow_mask; 1373 flags = &fs->flags; 1374 buckets = &fs->buckets; 1375 break; 1376 1377 case 3: /* "sched N config ..." */ 1378#ifdef NEW_AQM 1379 sch_extra = o_next(&buf, lmax_extra, DN_TEXT); 1380 sch_extra ->oid.subtype = 0; 1381#endif 1382 sch = o_next(&buf, sizeof(*sch), DN_SCH); 1383#ifdef NEW_AQM 1384 aqm_extra = o_next(&buf, lmax_extra, DN_TEXT); 1385 aqm_extra ->oid.subtype = 0; 1386#endif 1387 fs = o_next(&buf, sizeof(*fs), DN_FS); 1388 sch->sched_nr = i; 1389 mask = &sch->sched_mask; 1390 flags = &sch->flags; 1391 buckets = &sch->buckets; 1392 /* fs is used only with !MULTIQUEUE schedulers */ 1393 fs->fs_nr = i + DN_MAX_ID; 1394 fs->sched_nr = i; 1395 break; 1396 } 1397 /* set to -1 those fields for which we want to reuse existing 1398 * values from the kernel. 1399 * Also, *_nr and subtype = 0 mean reuse the value from the kernel. 1400 * XXX todo: support reuse of the mask. 1401 */ 1402 if (p) 1403 p->bandwidth = -1; 1404 for (j = 0; j < nitems(fs->par); j++) 1405 fs->par[j] = -1; 1406 while (ac > 0) { 1407 double d; 1408 int tok = match_token(dummynet_params, *av); 1409 ac--; av++; 1410 1411 switch(tok) { 1412 case TOK_NOERROR: 1413 NEED(fs, "noerror is only for pipes"); 1414 fs->flags |= DN_NOERROR; 1415 break; 1416 1417 case TOK_PLR: 1418 NEED(fs, "plr is only for pipes"); 1419 NEED1("plr needs one or four arguments 0..1\n"); 1420 if ((end = strsep(&av[0], ","))) { 1421 d = strtod(end, NULL); 1422 d = (d < 0) ? 0 : (d <= 1) ? d : 1; 1423 fs->plr[0] = (int)(d*0x7fffffff); 1424 } 1425 if ((end = strsep(&av[0], ","))) { 1426 d = strtod(end, NULL); 1427 d = (d < 0) ? 0 : (d <= 1) ? d : 1; 1428 fs->plr[1] = (int)(d*0x7fffffff); 1429 } 1430 if ((end = strsep(&av[0], ","))) { 1431 d = strtod(end, NULL); 1432 d = (d < 0) ? 0 : (d <= 1) ? d : 1; 1433 fs->plr[2] = (int)(d*0x7fffffff); 1434 } 1435 if ((end = strsep(&av[0], ","))) { 1436 d = strtod(end, NULL); 1437 d = (d < 0) ? 0 : (d <= 1) ? d : 1; 1438 fs->plr[3] = (int)(d*0x7fffffff); 1439 } 1440 ac--; av++; 1441 break; 1442 1443 case TOK_QUEUE: 1444 NEED(fs, "queue is only for pipes or flowsets"); 1445 NEED1("queue needs queue size\n"); 1446 end = NULL; 1447 fs->qsize = strtoul(av[0], &end, 0); 1448 if (*end == 'K' || *end == 'k') { 1449 fs->flags |= DN_QSIZE_BYTES; 1450 fs->qsize *= 1024; 1451 } else if (*end == 'B' || 1452 _substrcmp2(end, "by", "bytes") == 0) { 1453 fs->flags |= DN_QSIZE_BYTES; 1454 } 1455 ac--; av++; 1456 break; 1457 1458 case TOK_BUCKETS: 1459 NEED(fs, "buckets is only for pipes or flowsets"); 1460 NEED1("buckets needs argument\n"); 1461 *buckets = strtoul(av[0], NULL, 0); 1462 ac--; av++; 1463 break; 1464 1465 case TOK_FLOW_MASK: 1466 case TOK_SCHED_MASK: 1467 case TOK_MASK: 1468 NEED(mask, "tok_mask"); 1469 NEED1("mask needs mask specifier\n"); 1470 /* 1471 * per-flow queue, mask is dst_ip, dst_port, 1472 * src_ip, src_port, proto measured in bits 1473 */ 1474 1475 bzero(mask, sizeof(*mask)); 1476 end = NULL; 1477 1478 while (ac >= 1) { 1479 uint32_t *p32 = NULL; 1480 uint16_t *p16 = NULL; 1481 uint32_t *p20 = NULL; 1482 struct in6_addr *pa6 = NULL; 1483 uint32_t a; 1484 1485 tok = match_token(dummynet_params, *av); 1486 ac--; av++; 1487 switch(tok) { 1488 case TOK_ALL: 1489 /* 1490 * special case, all bits significant 1491 * except 'extra' (the queue number) 1492 */ 1493 mask->dst_ip = ~0; 1494 mask->src_ip = ~0; 1495 mask->dst_port = ~0; 1496 mask->src_port = ~0; 1497 mask->proto = ~0; 1498 n2mask(&mask->dst_ip6, 128); 1499 n2mask(&mask->src_ip6, 128); 1500 mask->flow_id6 = ~0; 1501 *flags |= DN_HAVE_MASK; 1502 goto end_mask; 1503 1504 case TOK_QUEUE: 1505 mask->extra = ~0; 1506 *flags |= DN_HAVE_MASK; 1507 goto end_mask; 1508 1509 case TOK_DSTIP: 1510 mask->addr_type = 4; 1511 p32 = &mask->dst_ip; 1512 break; 1513 1514 case TOK_SRCIP: 1515 mask->addr_type = 4; 1516 p32 = &mask->src_ip; 1517 break; 1518 1519 case TOK_DSTIP6: 1520 mask->addr_type = 6; 1521 pa6 = &mask->dst_ip6; 1522 break; 1523 1524 case TOK_SRCIP6: 1525 mask->addr_type = 6; 1526 pa6 = &mask->src_ip6; 1527 break; 1528 1529 case TOK_FLOWID: 1530 mask->addr_type = 6; 1531 p20 = &mask->flow_id6; 1532 break; 1533 1534 case TOK_DSTPORT: 1535 p16 = &mask->dst_port; 1536 break; 1537 1538 case TOK_SRCPORT: 1539 p16 = &mask->src_port; 1540 break; 1541 1542 case TOK_PROTO: 1543 break; 1544 1545 default: 1546 ac++; av--; /* backtrack */ 1547 goto end_mask; 1548 } 1549 if (ac < 1) 1550 errx(EX_USAGE, "mask: value missing"); 1551 if (*av[0] == '/') { 1552 a = strtoul(av[0]+1, &end, 0); 1553 if (pa6 == NULL) 1554 a = (a == 32) ? ~0 : (1 << a) - 1; 1555 } else 1556 a = strtoul(av[0], &end, 0); 1557 if (p32 != NULL) 1558 *p32 = a; 1559 else if (p16 != NULL) { 1560 if (a > 0xFFFF) 1561 errx(EX_DATAERR, 1562 "port mask must be 16 bit"); 1563 *p16 = (uint16_t)a; 1564 } else if (p20 != NULL) { 1565 if (a > 0xfffff) 1566 errx(EX_DATAERR, 1567 "flow_id mask must be 20 bit"); 1568 *p20 = (uint32_t)a; 1569 } else if (pa6 != NULL) { 1570 if (a > 128) 1571 errx(EX_DATAERR, 1572 "in6addr invalid mask len"); 1573 else 1574 n2mask(pa6, a); 1575 } else { 1576 if (a > 0xFF) 1577 errx(EX_DATAERR, 1578 "proto mask must be 8 bit"); 1579 mask->proto = (uint8_t)a; 1580 } 1581 if (a != 0) 1582 *flags |= DN_HAVE_MASK; 1583 ac--; av++; 1584 } /* end while, config masks */ 1585end_mask: 1586 break; 1587#ifdef NEW_AQM 1588 case TOK_CODEL: 1589 case TOK_PIE: 1590 NEED(fs, "codel/pie is only for flowsets"); 1591 1592 fs->flags &= ~(DN_IS_RED|DN_IS_GENTLE_RED); 1593 fs->flags |= DN_IS_AQM; 1594 1595 strlcpy(aqm_extra->name, av[-1], 1596 sizeof(aqm_extra->name)); 1597 aqm_extra->oid.subtype = DN_AQM_PARAMS; 1598 1599 process_extra_parms(&ac, av, aqm_extra, tok); 1600 break; 1601 1602 case TOK_FQ_CODEL: 1603 case TOK_FQ_PIE: 1604 if (!strcmp(av[-1],"type")) 1605 errx(EX_DATAERR, "use type before fq_codel/fq_pie"); 1606 1607 NEED(sch, "fq_codel/fq_pie is only for schd"); 1608 strlcpy(sch_extra->name, av[-1], 1609 sizeof(sch_extra->name)); 1610 sch_extra->oid.subtype = DN_SCH_PARAMS; 1611 process_extra_parms(&ac, av, sch_extra, tok); 1612 break; 1613#endif 1614 case TOK_RED: 1615 case TOK_GRED: 1616 NEED1("red/gred needs w_q/min_th/max_th/max_p\n"); 1617 fs->flags |= DN_IS_RED; 1618 if (tok == TOK_GRED) 1619 fs->flags |= DN_IS_GENTLE_RED; 1620 /* 1621 * the format for parameters is w_q/min_th/max_th/max_p 1622 */ 1623 if ((end = strsep(&av[0], "/"))) { 1624 double w_q = strtod(end, NULL); 1625 if (w_q > 1 || w_q <= 0) 1626 errx(EX_DATAERR, "0 < w_q <= 1"); 1627 fs->w_q = (int) (w_q * (1 << SCALE_RED)); 1628 } 1629 if ((end = strsep(&av[0], "/"))) { 1630 fs->min_th = strtoul(end, &end, 0); 1631 if (*end == 'K' || *end == 'k') 1632 fs->min_th *= 1024; 1633 } 1634 if ((end = strsep(&av[0], "/"))) { 1635 fs->max_th = strtoul(end, &end, 0); 1636 if (*end == 'K' || *end == 'k') 1637 fs->max_th *= 1024; 1638 } 1639 if ((end = strsep(&av[0], "/"))) { 1640 double max_p = strtod(end, NULL); 1641 if (max_p > 1 || max_p < 0) 1642 errx(EX_DATAERR, "0 <= max_p <= 1"); 1643 fs->max_p = (int)(max_p * (1 << SCALE_RED)); 1644 } 1645 ac--; av++; 1646 break; 1647 1648 case TOK_ECN: 1649 fs->flags |= DN_IS_ECN; 1650 break; 1651 1652 case TOK_DROPTAIL: 1653 NEED(fs, "droptail is only for flowsets"); 1654 fs->flags &= ~(DN_IS_RED|DN_IS_GENTLE_RED); 1655 break; 1656 1657 case TOK_BW: 1658 NEED(p, "bw is only for links"); 1659 NEED1("bw needs bandwidth or interface\n"); 1660 read_bandwidth(av[0], &p->bandwidth, NULL, 0); 1661 ac--; av++; 1662 break; 1663 1664 case TOK_DELAY: 1665 NEED(p, "delay is only for links"); 1666 NEED1("delay needs argument 0..10000ms\n"); 1667 p->delay = strtoul(av[0], NULL, 0); 1668 ac--; av++; 1669 break; 1670 1671 case TOK_TYPE: { 1672 int l; 1673 NEED(sch, "type is only for schedulers"); 1674 NEED1("type needs a string"); 1675 l = strlen(av[0]); 1676 if (l == 0 || l > 15) 1677 errx(1, "type %s too long\n", av[0]); 1678 strlcpy(sch->name, av[0], sizeof(sch->name)); 1679 sch->oid.subtype = 0; /* use string */ 1680#ifdef NEW_AQM 1681 /* if fq_codel is selected, consider all tokens after it 1682 * as parameters 1683 */ 1684 if (!strcasecmp(av[0],"fq_codel") || !strcasecmp(av[0],"fq_pie")){ 1685 strlcpy(sch_extra->name, av[0], 1686 sizeof(sch_extra->name)); 1687 sch_extra->oid.subtype = DN_SCH_PARAMS; 1688 process_extra_parms(&ac, av, sch_extra, tok); 1689 } else { 1690 ac--;av++; 1691 } 1692#else 1693 ac--;av++; 1694#endif 1695 break; 1696 } 1697 1698 case TOK_WEIGHT: 1699 NEED(fs, "weight is only for flowsets"); 1700 NEED1("weight needs argument\n"); 1701 fs->par[0] = strtol(av[0], &end, 0); 1702 ac--; av++; 1703 break; 1704 1705 case TOK_LMAX: 1706 NEED(fs, "lmax is only for flowsets"); 1707 NEED1("lmax needs argument\n"); 1708 fs->par[1] = strtol(av[0], &end, 0); 1709 ac--; av++; 1710 break; 1711 1712 case TOK_PRI: 1713 NEED(fs, "priority is only for flowsets"); 1714 NEED1("priority needs argument\n"); 1715 fs->par[2] = strtol(av[0], &end, 0); 1716 ac--; av++; 1717 break; 1718 1719 case TOK_SCHED: 1720 case TOK_PIPE: 1721 NEED(fs, "pipe/sched"); 1722 NEED1("pipe/link/sched needs number\n"); 1723 fs->sched_nr = strtoul(av[0], &end, 0); 1724 ac--; av++; 1725 break; 1726 1727 case TOK_PROFILE: 1728 NEED((!pf), "profile already set"); 1729 NEED(p, "profile"); 1730 { 1731 NEED1("extra delay needs the file name\n"); 1732 pf = o_next(&buf, sizeof(*pf), DN_PROFILE); 1733 load_extra_delays(av[0], pf, p); //XXX can't fail? 1734 --ac; ++av; 1735 } 1736 break; 1737 1738 case TOK_BURST: 1739 NEED(p, "burst"); 1740 NEED1("burst needs argument\n"); 1741 errno = 0; 1742 if (expand_number(av[0], &p->burst) < 0) 1743 if (errno != ERANGE) 1744 errx(EX_DATAERR, 1745 "burst: invalid argument"); 1746 if (errno || p->burst > (1ULL << 48) - 1) 1747 errx(EX_DATAERR, 1748 "burst: out of range (0..2^48-1)"); 1749 ac--; av++; 1750 break; 1751 1752 default: 1753 errx(EX_DATAERR, "unrecognised option ``%s''", av[-1]); 1754 } 1755 } 1756 1757 /* check validity of parameters */ 1758 if (p) { 1759 if (p->delay > 10000) 1760 errx(EX_DATAERR, "delay must be < 10000"); 1761 if (p->bandwidth == (uint32_t)-1) 1762 p->bandwidth = 0; 1763 } 1764 if (fs) { 1765 /* XXX accept a 0 scheduler to keep the default */ 1766 if (fs->flags & DN_QSIZE_BYTES) { 1767 size_t len; 1768 long limit; 1769 1770 len = sizeof(limit); 1771 if (sysctlbyname("net.inet.ip.dummynet.pipe_byte_limit", 1772 &limit, &len, NULL, 0) == -1) 1773 limit = 1024*1024; 1774 if (fs->qsize > limit) 1775 errx(EX_DATAERR, "queue size must be < %ldB", limit); 1776 } else { 1777 size_t len; 1778 long limit; 1779 1780 len = sizeof(limit); 1781 if (sysctlbyname("net.inet.ip.dummynet.pipe_slot_limit", 1782 &limit, &len, NULL, 0) == -1) 1783 limit = 100; 1784 if (fs->qsize > limit) 1785 errx(EX_DATAERR, "2 <= queue size <= %ld", limit); 1786 } 1787 1788#ifdef NEW_AQM 1789 if ((fs->flags & DN_IS_ECN) && !((fs->flags & DN_IS_RED)|| 1790 (fs->flags & DN_IS_AQM))) 1791 errx(EX_USAGE, "ECN can be used with red/gred/" 1792 "codel/fq_codel only!"); 1793#else 1794 if ((fs->flags & DN_IS_ECN) && !(fs->flags & DN_IS_RED)) 1795 errx(EX_USAGE, "enable red/gred for ECN"); 1796 1797#endif 1798 1799 if (fs->flags & DN_IS_RED) { 1800 size_t len; 1801 int lookup_depth, avg_pkt_size; 1802 1803 if (!(fs->flags & DN_IS_ECN) && (fs->min_th >= fs->max_th)) 1804 errx(EX_DATAERR, "min_th %d must be < than max_th %d", 1805 fs->min_th, fs->max_th); 1806 else if ((fs->flags & DN_IS_ECN) && (fs->min_th > fs->max_th)) 1807 errx(EX_DATAERR, "min_th %d must be =< than max_th %d", 1808 fs->min_th, fs->max_th); 1809 1810 if (fs->max_th == 0) 1811 errx(EX_DATAERR, "max_th must be > 0"); 1812 1813 len = sizeof(int); 1814 if (sysctlbyname("net.inet.ip.dummynet.red_lookup_depth", 1815 &lookup_depth, &len, NULL, 0) == -1) 1816 lookup_depth = 256; 1817 if (lookup_depth == 0) 1818 errx(EX_DATAERR, "net.inet.ip.dummynet.red_lookup_depth" 1819 " must be greater than zero"); 1820 1821 len = sizeof(int); 1822 if (sysctlbyname("net.inet.ip.dummynet.red_avg_pkt_size", 1823 &avg_pkt_size, &len, NULL, 0) == -1) 1824 avg_pkt_size = 512; 1825 1826 if (avg_pkt_size == 0) 1827 errx(EX_DATAERR, 1828 "net.inet.ip.dummynet.red_avg_pkt_size must" 1829 " be greater than zero"); 1830 1831#if 0 /* the following computation is now done in the kernel */ 1832 /* 1833 * Ticks needed for sending a medium-sized packet. 1834 * Unfortunately, when we are configuring a WF2Q+ queue, we 1835 * do not have bandwidth information, because that is stored 1836 * in the parent pipe, and also we have multiple queues 1837 * competing for it. So we set s=0, which is not very 1838 * correct. But on the other hand, why do we want RED with 1839 * WF2Q+ ? 1840 */ 1841 if (p.bandwidth==0) /* this is a WF2Q+ queue */ 1842 s = 0; 1843 else 1844 s = (double)ck.hz * avg_pkt_size * 8 / p.bandwidth; 1845 /* 1846 * max idle time (in ticks) before avg queue size becomes 0. 1847 * NOTA: (3/w_q) is approx the value x so that 1848 * (1-w_q)^x < 10^-3. 1849 */ 1850 w_q = ((double)fs->w_q) / (1 << SCALE_RED); 1851 idle = s * 3. / w_q; 1852 fs->lookup_step = (int)idle / lookup_depth; 1853 if (!fs->lookup_step) 1854 fs->lookup_step = 1; 1855 weight = 1 - w_q; 1856 for (t = fs->lookup_step; t > 1; --t) 1857 weight *= 1 - w_q; 1858 fs->lookup_weight = (int)(weight * (1 << SCALE_RED)); 1859#endif /* code moved in the kernel */ 1860 } 1861 } 1862 1863 i = do_cmd(IP_DUMMYNET3, base, (char *)buf - (char *)base); 1864 1865 if (i) 1866 err(1, "setsockopt(%s)", "IP_DUMMYNET_CONFIGURE"); 1867} 1868 1869void 1870dummynet_flush(void) 1871{ 1872 struct dn_id oid; 1873 oid_fill(&oid, sizeof(oid), DN_CMD_FLUSH, DN_API_VERSION); 1874 do_cmd(IP_DUMMYNET3, &oid, oid.len); 1875} 1876 1877/* Parse input for 'ipfw [pipe|sched|queue] show [range list]' 1878 * Returns the number of ranges, and possibly stores them 1879 * in the array v of size len. 1880 */ 1881static int 1882parse_range(int ac, char *av[], uint32_t *v, int len) 1883{ 1884 int n = 0; 1885 char *endptr, *s; 1886 uint32_t base[2]; 1887 1888 if (v == NULL || len < 2) { 1889 v = base; 1890 len = 2; 1891 } 1892 1893 for (s = *av; s != NULL; av++, ac--) { 1894 v[0] = strtoul(s, &endptr, 10); 1895 v[1] = (*endptr != '-') ? v[0] : 1896 strtoul(endptr+1, &endptr, 10); 1897 if (*endptr == '\0') { /* prepare for next round */ 1898 s = (ac > 0) ? *(av+1) : NULL; 1899 } else { 1900 if (*endptr != ',') { 1901 warn("invalid number: %s", s); 1902 s = ++endptr; 1903 continue; 1904 } 1905 /* continue processing from here */ 1906 s = ++endptr; 1907 ac++; 1908 av--; 1909 } 1910 if (v[1] < v[0] || 1911 v[0] >= DN_MAX_ID-1 || 1912 v[1] >= DN_MAX_ID-1) { 1913 continue; /* invalid entry */ 1914 } 1915 n++; 1916 /* translate if 'pipe list' */ 1917 if (g_co.do_pipe == 1) { 1918 v[0] += DN_MAX_ID; 1919 v[1] += DN_MAX_ID; 1920 } 1921 v = (n*2 < len) ? v + 2 : base; 1922 } 1923 return n; 1924} 1925 1926/* main entry point for dummynet list functions. co.do_pipe indicates 1927 * which function we want to support. 1928 * av may contain filtering arguments, either individual entries 1929 * or ranges, or lists (space or commas are valid separators). 1930 * Format for a range can be n1-n2 or n3 n4 n5 ... 1931 * In a range n1 must be <= n2, otherwise the range is ignored. 1932 * A number 'n4' is translate in a range 'n4-n4' 1933 * All number must be > 0 and < DN_MAX_ID-1 1934 */ 1935void 1936dummynet_list(int ac, char *av[], int show_counters) 1937{ 1938 struct dn_id *oid, *x = NULL; 1939 int ret, i; 1940 int n; /* # of ranges */ 1941 u_int buflen, l; 1942 u_int max_size; /* largest obj passed up */ 1943 1944 (void)show_counters; // XXX unused, but we should use it. 1945 ac--; 1946 av++; /* skip 'list' | 'show' word */ 1947 1948 n = parse_range(ac, av, NULL, 0); /* Count # of ranges. */ 1949 1950 /* Allocate space to store ranges */ 1951 l = sizeof(*oid) + sizeof(uint32_t) * n * 2; 1952 oid = safe_calloc(1, l); 1953 oid_fill(oid, l, DN_CMD_GET, DN_API_VERSION); 1954 1955 if (n > 0) /* store ranges in idx */ 1956 parse_range(ac, av, (uint32_t *)(oid + 1), n*2); 1957 /* 1958 * Compute the size of the largest object returned. If the 1959 * response leaves at least this much spare space in the 1960 * buffer, then surely the response is complete; otherwise 1961 * there might be a risk of truncation and we will need to 1962 * retry with a larger buffer. 1963 * XXX don't bother with smaller structs. 1964 */ 1965 max_size = sizeof(struct dn_fs); 1966 if (max_size < sizeof(struct dn_sch)) 1967 max_size = sizeof(struct dn_sch); 1968 if (max_size < sizeof(struct dn_flow)) 1969 max_size = sizeof(struct dn_flow); 1970 1971 switch (g_co.do_pipe) { 1972 case 1: 1973 oid->subtype = DN_LINK; /* list pipe */ 1974 break; 1975 case 2: 1976 oid->subtype = DN_FS; /* list queue */ 1977 break; 1978 case 3: 1979 oid->subtype = DN_SCH; /* list sched */ 1980 break; 1981 } 1982 1983 /* 1984 * Ask the kernel an estimate of the required space (result 1985 * in oid.id), unless we are requesting a subset of objects, 1986 * in which case the kernel does not give an exact answer. 1987 * In any case, space might grow in the meantime due to the 1988 * creation of new queues, so we must be prepared to retry. 1989 */ 1990 if (n > 0) { 1991 buflen = 4*1024; 1992 } else { 1993 ret = do_cmd(-IP_DUMMYNET3, oid, (uintptr_t)&l); 1994 if (ret != 0 || oid->id <= sizeof(*oid)) 1995 goto done; 1996 buflen = oid->id + max_size; 1997 oid->len = sizeof(*oid); /* restore */ 1998 } 1999 /* Try a few times, until the buffer fits */ 2000 for (i = 0; i < 20; i++) { 2001 l = buflen; 2002 x = safe_realloc(x, l); 2003 bcopy(oid, x, oid->len); 2004 ret = do_cmd(-IP_DUMMYNET3, x, (uintptr_t)&l); 2005 if (ret != 0 || x->id <= sizeof(*oid)) 2006 goto done; /* no response */ 2007 if (l + max_size <= buflen) 2008 break; /* ok */ 2009 buflen *= 2; /* double for next attempt */ 2010 } 2011 list_pipes(x, O_NEXT(x, l)); 2012done: 2013 if (x) 2014 free(x); 2015 free(oid); 2016} 2017