1/*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 5 * Copyright (C) 2005-2017 Jung-uk Kim <jkim@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. Neither the name of the Politecnico di Torino nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34#include <sys/cdefs.h> 35#ifdef _KERNEL 36#include "opt_bpf.h" 37#include <sys/param.h> 38#include <sys/systm.h> 39#include <sys/kernel.h> 40#include <sys/malloc.h> 41#include <sys/mbuf.h> 42#include <sys/socket.h> 43 44#include <net/if.h> 45#else 46#include <stdlib.h> 47#include <string.h> 48#include <sys/mman.h> 49#include <sys/param.h> 50#endif 51 52#include <sys/types.h> 53 54#include <net/bpf.h> 55#include <net/bpf_jitter.h> 56 57#include <i386/i386/bpf_jit_machdep.h> 58 59/* 60 * Emit routine to update the jump table. 61 */ 62static void 63emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len) 64{ 65 66 if (stream->refs != NULL) 67 (stream->refs)[stream->bpf_pc] += len; 68 stream->cur_ip += len; 69} 70 71/* 72 * Emit routine to output the actual binary code. 73 */ 74static void 75emit_code(bpf_bin_stream *stream, u_int value, u_int len) 76{ 77 78 switch (len) { 79 case 1: 80 stream->ibuf[stream->cur_ip] = (u_char)value; 81 stream->cur_ip++; 82 break; 83 84 case 2: 85 *((u_short *)(void *)(stream->ibuf + stream->cur_ip)) = 86 (u_short)value; 87 stream->cur_ip += 2; 88 break; 89 90 case 4: 91 *((u_int *)(void *)(stream->ibuf + stream->cur_ip)) = value; 92 stream->cur_ip += 4; 93 break; 94 } 95 96 return; 97} 98 99/* 100 * Scan the filter program and find possible optimization. 101 */ 102static int 103bpf_jit_optimize(struct bpf_insn *prog, u_int nins) 104{ 105 int flags; 106 u_int i; 107 108 /* Do we return immediately? */ 109 if (BPF_CLASS(prog[0].code) == BPF_RET) 110 return (BPF_JIT_FRET); 111 112 for (flags = 0, i = 0; i < nins; i++) { 113 switch (prog[i].code) { 114 case BPF_LD|BPF_W|BPF_ABS: 115 case BPF_LD|BPF_H|BPF_ABS: 116 case BPF_LD|BPF_B|BPF_ABS: 117 case BPF_LD|BPF_W|BPF_IND: 118 case BPF_LD|BPF_H|BPF_IND: 119 case BPF_LD|BPF_B|BPF_IND: 120 case BPF_LDX|BPF_MSH|BPF_B: 121 flags |= BPF_JIT_FPKT; 122 break; 123 case BPF_LD|BPF_MEM: 124 case BPF_LDX|BPF_MEM: 125 case BPF_ST: 126 case BPF_STX: 127 flags |= BPF_JIT_FMEM; 128 break; 129 case BPF_JMP|BPF_JA: 130 case BPF_JMP|BPF_JGT|BPF_K: 131 case BPF_JMP|BPF_JGE|BPF_K: 132 case BPF_JMP|BPF_JEQ|BPF_K: 133 case BPF_JMP|BPF_JSET|BPF_K: 134 case BPF_JMP|BPF_JGT|BPF_X: 135 case BPF_JMP|BPF_JGE|BPF_X: 136 case BPF_JMP|BPF_JEQ|BPF_X: 137 case BPF_JMP|BPF_JSET|BPF_X: 138 flags |= BPF_JIT_FJMP; 139 break; 140 case BPF_ALU|BPF_DIV|BPF_K: 141 case BPF_ALU|BPF_MOD|BPF_K: 142 flags |= BPF_JIT_FADK; 143 break; 144 } 145 if (flags == BPF_JIT_FLAG_ALL) 146 break; 147 } 148 149 return (flags); 150} 151 152/* 153 * Function that does the real stuff. 154 */ 155bpf_filter_func 156bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) 157{ 158 bpf_bin_stream stream; 159 struct bpf_insn *ins; 160 int flags, fret, fpkt, fmem, fjmp, fadk; 161 int save_esp; 162 u_int i, pass; 163 164 /* 165 * NOTE: Do not modify the name of this variable, as it's used by 166 * the macros to emit code. 167 */ 168 emit_func emitm; 169 170 flags = bpf_jit_optimize(prog, nins); 171 fret = (flags & BPF_JIT_FRET) != 0; 172 fpkt = (flags & BPF_JIT_FPKT) != 0; 173 fmem = (flags & BPF_JIT_FMEM) != 0; 174 fjmp = (flags & BPF_JIT_FJMP) != 0; 175 fadk = (flags & BPF_JIT_FADK) != 0; 176 save_esp = (fpkt || fmem || fadk); /* Stack is used. */ 177 178 if (fret) 179 nins = 1; 180 181 memset(&stream, 0, sizeof(stream)); 182 183 /* Allocate the reference table for the jumps. */ 184 if (fjmp) { 185#ifdef _KERNEL 186 stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, 187 M_NOWAIT | M_ZERO); 188#else 189 stream.refs = calloc(nins + 1, sizeof(u_int)); 190#endif 191 if (stream.refs == NULL) 192 return (NULL); 193 } 194 195 /* 196 * The first pass will emit the lengths of the instructions 197 * to create the reference table. 198 */ 199 emitm = emit_length; 200 201 for (pass = 0; pass < 2; pass++) { 202 ins = prog; 203 204 /* Create the procedure header. */ 205 if (save_esp) { 206 PUSH(EBP); 207 MOVrd(ESP, EBP); 208 } 209 if (fmem) 210 SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP); 211 if (save_esp) 212 PUSH(ESI); 213 if (fpkt) { 214 PUSH(EDI); 215 PUSH(EBX); 216 MOVodd(8, EBP, EBX); 217 MOVodd(16, EBP, EDI); 218 } 219 220 for (i = 0; i < nins; i++) { 221 stream.bpf_pc++; 222 223 switch (ins->code) { 224 default: 225#ifdef _KERNEL 226 return (NULL); 227#else 228 abort(); 229#endif 230 231 case BPF_RET|BPF_K: 232 MOVid(ins->k, EAX); 233 if (save_esp) { 234 if (fpkt) { 235 POP(EBX); 236 POP(EDI); 237 } 238 POP(ESI); 239 LEAVE(); 240 } 241 RET(); 242 break; 243 244 case BPF_RET|BPF_A: 245 if (save_esp) { 246 if (fpkt) { 247 POP(EBX); 248 POP(EDI); 249 } 250 POP(ESI); 251 LEAVE(); 252 } 253 RET(); 254 break; 255 256 case BPF_LD|BPF_W|BPF_ABS: 257 MOVid(ins->k, ESI); 258 CMPrd(EDI, ESI); 259 JAb(12); 260 MOVrd(EDI, ECX); 261 SUBrd(ESI, ECX); 262 CMPid(sizeof(int32_t), ECX); 263 JAEb(7); 264 ZEROrd(EAX); 265 POP(EBX); 266 POP(EDI); 267 POP(ESI); 268 LEAVE(); 269 RET(); 270 MOVobd(EBX, ESI, EAX); 271 BSWAP(EAX); 272 break; 273 274 case BPF_LD|BPF_H|BPF_ABS: 275 ZEROrd(EAX); 276 MOVid(ins->k, ESI); 277 CMPrd(EDI, ESI); 278 JAb(12); 279 MOVrd(EDI, ECX); 280 SUBrd(ESI, ECX); 281 CMPid(sizeof(int16_t), ECX); 282 JAEb(5); 283 POP(EBX); 284 POP(EDI); 285 POP(ESI); 286 LEAVE(); 287 RET(); 288 MOVobw(EBX, ESI, AX); 289 SWAP_AX(); 290 break; 291 292 case BPF_LD|BPF_B|BPF_ABS: 293 ZEROrd(EAX); 294 MOVid(ins->k, ESI); 295 CMPrd(EDI, ESI); 296 JBb(5); 297 POP(EBX); 298 POP(EDI); 299 POP(ESI); 300 LEAVE(); 301 RET(); 302 MOVobb(EBX, ESI, AL); 303 break; 304 305 case BPF_LD|BPF_W|BPF_LEN: 306 if (save_esp) 307 MOVodd(12, EBP, EAX); 308 else { 309 MOVrd(ESP, ECX); 310 MOVodd(12, ECX, EAX); 311 } 312 break; 313 314 case BPF_LDX|BPF_W|BPF_LEN: 315 if (save_esp) 316 MOVodd(12, EBP, EDX); 317 else { 318 MOVrd(ESP, ECX); 319 MOVodd(12, ECX, EDX); 320 } 321 break; 322 323 case BPF_LD|BPF_W|BPF_IND: 324 CMPrd(EDI, EDX); 325 JAb(27); 326 MOVid(ins->k, ESI); 327 MOVrd(EDI, ECX); 328 SUBrd(EDX, ECX); 329 CMPrd(ESI, ECX); 330 JBb(14); 331 ADDrd(EDX, ESI); 332 MOVrd(EDI, ECX); 333 SUBrd(ESI, ECX); 334 CMPid(sizeof(int32_t), ECX); 335 JAEb(7); 336 ZEROrd(EAX); 337 POP(EBX); 338 POP(EDI); 339 POP(ESI); 340 LEAVE(); 341 RET(); 342 MOVobd(EBX, ESI, EAX); 343 BSWAP(EAX); 344 break; 345 346 case BPF_LD|BPF_H|BPF_IND: 347 ZEROrd(EAX); 348 CMPrd(EDI, EDX); 349 JAb(27); 350 MOVid(ins->k, ESI); 351 MOVrd(EDI, ECX); 352 SUBrd(EDX, ECX); 353 CMPrd(ESI, ECX); 354 JBb(14); 355 ADDrd(EDX, ESI); 356 MOVrd(EDI, ECX); 357 SUBrd(ESI, ECX); 358 CMPid(sizeof(int16_t), ECX); 359 JAEb(5); 360 POP(EBX); 361 POP(EDI); 362 POP(ESI); 363 LEAVE(); 364 RET(); 365 MOVobw(EBX, ESI, AX); 366 SWAP_AX(); 367 break; 368 369 case BPF_LD|BPF_B|BPF_IND: 370 ZEROrd(EAX); 371 CMPrd(EDI, EDX); 372 JAEb(13); 373 MOVid(ins->k, ESI); 374 MOVrd(EDI, ECX); 375 SUBrd(EDX, ECX); 376 CMPrd(ESI, ECX); 377 JAb(5); 378 POP(EBX); 379 POP(EDI); 380 POP(ESI); 381 LEAVE(); 382 RET(); 383 ADDrd(EDX, ESI); 384 MOVobb(EBX, ESI, AL); 385 break; 386 387 case BPF_LDX|BPF_MSH|BPF_B: 388 MOVid(ins->k, ESI); 389 CMPrd(EDI, ESI); 390 JBb(7); 391 ZEROrd(EAX); 392 POP(EBX); 393 POP(EDI); 394 POP(ESI); 395 LEAVE(); 396 RET(); 397 ZEROrd(EDX); 398 MOVobb(EBX, ESI, DL); 399 ANDib(0x0f, DL); 400 SHLib(2, EDX); 401 break; 402 403 case BPF_LD|BPF_IMM: 404 MOVid(ins->k, EAX); 405 break; 406 407 case BPF_LDX|BPF_IMM: 408 MOVid(ins->k, EDX); 409 break; 410 411 case BPF_LD|BPF_MEM: 412 MOVrd(EBP, ECX); 413 MOVid(((int)ins->k - BPF_MEMWORDS) * 414 sizeof(uint32_t), ESI); 415 MOVobd(ECX, ESI, EAX); 416 break; 417 418 case BPF_LDX|BPF_MEM: 419 MOVrd(EBP, ECX); 420 MOVid(((int)ins->k - BPF_MEMWORDS) * 421 sizeof(uint32_t), ESI); 422 MOVobd(ECX, ESI, EDX); 423 break; 424 425 case BPF_ST: 426 /* 427 * XXX this command and the following could 428 * be optimized if the previous instruction 429 * was already of this type 430 */ 431 MOVrd(EBP, ECX); 432 MOVid(((int)ins->k - BPF_MEMWORDS) * 433 sizeof(uint32_t), ESI); 434 MOVomd(EAX, ECX, ESI); 435 break; 436 437 case BPF_STX: 438 MOVrd(EBP, ECX); 439 MOVid(((int)ins->k - BPF_MEMWORDS) * 440 sizeof(uint32_t), ESI); 441 MOVomd(EDX, ECX, ESI); 442 break; 443 444 case BPF_JMP|BPF_JA: 445 JUMP(ins->k); 446 break; 447 448 case BPF_JMP|BPF_JGT|BPF_K: 449 case BPF_JMP|BPF_JGE|BPF_K: 450 case BPF_JMP|BPF_JEQ|BPF_K: 451 case BPF_JMP|BPF_JSET|BPF_K: 452 case BPF_JMP|BPF_JGT|BPF_X: 453 case BPF_JMP|BPF_JGE|BPF_X: 454 case BPF_JMP|BPF_JEQ|BPF_X: 455 case BPF_JMP|BPF_JSET|BPF_X: 456 if (ins->jt == ins->jf) { 457 JUMP(ins->jt); 458 break; 459 } 460 switch (ins->code) { 461 case BPF_JMP|BPF_JGT|BPF_K: 462 CMPid(ins->k, EAX); 463 JCC(JA, JBE); 464 break; 465 466 case BPF_JMP|BPF_JGE|BPF_K: 467 CMPid(ins->k, EAX); 468 JCC(JAE, JB); 469 break; 470 471 case BPF_JMP|BPF_JEQ|BPF_K: 472 CMPid(ins->k, EAX); 473 JCC(JE, JNE); 474 break; 475 476 case BPF_JMP|BPF_JSET|BPF_K: 477 TESTid(ins->k, EAX); 478 JCC(JNE, JE); 479 break; 480 481 case BPF_JMP|BPF_JGT|BPF_X: 482 CMPrd(EDX, EAX); 483 JCC(JA, JBE); 484 break; 485 486 case BPF_JMP|BPF_JGE|BPF_X: 487 CMPrd(EDX, EAX); 488 JCC(JAE, JB); 489 break; 490 491 case BPF_JMP|BPF_JEQ|BPF_X: 492 CMPrd(EDX, EAX); 493 JCC(JE, JNE); 494 break; 495 496 case BPF_JMP|BPF_JSET|BPF_X: 497 TESTrd(EDX, EAX); 498 JCC(JNE, JE); 499 break; 500 } 501 break; 502 503 case BPF_ALU|BPF_ADD|BPF_X: 504 ADDrd(EDX, EAX); 505 break; 506 507 case BPF_ALU|BPF_SUB|BPF_X: 508 SUBrd(EDX, EAX); 509 break; 510 511 case BPF_ALU|BPF_MUL|BPF_X: 512 MOVrd(EDX, ECX); 513 MULrd(EDX); 514 MOVrd(ECX, EDX); 515 break; 516 517 case BPF_ALU|BPF_DIV|BPF_X: 518 case BPF_ALU|BPF_MOD|BPF_X: 519 TESTrd(EDX, EDX); 520 if (save_esp) { 521 if (fpkt) { 522 JNEb(7); 523 ZEROrd(EAX); 524 POP(EBX); 525 POP(EDI); 526 } else { 527 JNEb(5); 528 ZEROrd(EAX); 529 } 530 POP(ESI); 531 LEAVE(); 532 } else { 533 JNEb(3); 534 ZEROrd(EAX); 535 } 536 RET(); 537 MOVrd(EDX, ECX); 538 ZEROrd(EDX); 539 DIVrd(ECX); 540 if (BPF_OP(ins->code) == BPF_MOD) 541 MOVrd(EDX, EAX); 542 MOVrd(ECX, EDX); 543 break; 544 545 case BPF_ALU|BPF_AND|BPF_X: 546 ANDrd(EDX, EAX); 547 break; 548 549 case BPF_ALU|BPF_OR|BPF_X: 550 ORrd(EDX, EAX); 551 break; 552 553 case BPF_ALU|BPF_XOR|BPF_X: 554 XORrd(EDX, EAX); 555 break; 556 557 case BPF_ALU|BPF_LSH|BPF_X: 558 MOVrd(EDX, ECX); 559 SHL_CLrb(EAX); 560 break; 561 562 case BPF_ALU|BPF_RSH|BPF_X: 563 MOVrd(EDX, ECX); 564 SHR_CLrb(EAX); 565 break; 566 567 case BPF_ALU|BPF_ADD|BPF_K: 568 ADD_EAXi(ins->k); 569 break; 570 571 case BPF_ALU|BPF_SUB|BPF_K: 572 SUB_EAXi(ins->k); 573 break; 574 575 case BPF_ALU|BPF_MUL|BPF_K: 576 MOVrd(EDX, ECX); 577 MOVid(ins->k, EDX); 578 MULrd(EDX); 579 MOVrd(ECX, EDX); 580 break; 581 582 case BPF_ALU|BPF_DIV|BPF_K: 583 case BPF_ALU|BPF_MOD|BPF_K: 584 MOVrd(EDX, ECX); 585 ZEROrd(EDX); 586 MOVid(ins->k, ESI); 587 DIVrd(ESI); 588 if (BPF_OP(ins->code) == BPF_MOD) 589 MOVrd(EDX, EAX); 590 MOVrd(ECX, EDX); 591 break; 592 593 case BPF_ALU|BPF_AND|BPF_K: 594 ANDid(ins->k, EAX); 595 break; 596 597 case BPF_ALU|BPF_OR|BPF_K: 598 ORid(ins->k, EAX); 599 break; 600 601 case BPF_ALU|BPF_XOR|BPF_K: 602 XORid(ins->k, EAX); 603 break; 604 605 case BPF_ALU|BPF_LSH|BPF_K: 606 SHLib((ins->k) & 0xff, EAX); 607 break; 608 609 case BPF_ALU|BPF_RSH|BPF_K: 610 SHRib((ins->k) & 0xff, EAX); 611 break; 612 613 case BPF_ALU|BPF_NEG: 614 NEGd(EAX); 615 break; 616 617 case BPF_MISC|BPF_TAX: 618 MOVrd(EAX, EDX); 619 break; 620 621 case BPF_MISC|BPF_TXA: 622 MOVrd(EDX, EAX); 623 break; 624 } 625 ins++; 626 } 627 628 if (pass > 0) 629 continue; 630 631 *size = stream.cur_ip; 632#ifdef _KERNEL 633 stream.ibuf = malloc_exec(*size, M_BPFJIT, M_NOWAIT); 634 if (stream.ibuf == NULL) 635 break; 636#else 637 stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE, 638 MAP_ANON, -1, 0); 639 if (stream.ibuf == MAP_FAILED) { 640 stream.ibuf = NULL; 641 break; 642 } 643#endif 644 645 /* 646 * Modify the reference table to contain the offsets and 647 * not the lengths of the instructions. 648 */ 649 if (fjmp) 650 for (i = 1; i < nins + 1; i++) 651 stream.refs[i] += stream.refs[i - 1]; 652 653 /* Reset the counters. */ 654 stream.cur_ip = 0; 655 stream.bpf_pc = 0; 656 657 /* The second pass creates the actual code. */ 658 emitm = emit_code; 659 } 660 661 /* 662 * The reference table is needed only during compilation, 663 * now we can free it. 664 */ 665 if (fjmp) 666#ifdef _KERNEL 667 free(stream.refs, M_BPFJIT); 668#else 669 free(stream.refs); 670#endif 671 672#ifndef _KERNEL 673 if (stream.ibuf != NULL && 674 mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) { 675 munmap(stream.ibuf, *size); 676 stream.ibuf = NULL; 677 } 678#endif 679 680 return ((bpf_filter_func)(void *)stream.ibuf); 681} 682