1153151Sjkim/*- 2181648Sjkim * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy) 3199492Sjkim * Copyright (C) 2005-2009 Jung-uk Kim <jkim@FreeBSD.org> 4153151Sjkim * All rights reserved. 5153151Sjkim * 6153151Sjkim * Redistribution and use in source and binary forms, with or without 7153151Sjkim * modification, are permitted provided that the following conditions 8153151Sjkim * are met: 9153151Sjkim * 10153151Sjkim * 1. Redistributions of source code must retain the above copyright 11153151Sjkim * notice, this list of conditions and the following disclaimer. 12153151Sjkim * 2. Redistributions in binary form must reproduce the above copyright 13153151Sjkim * notice, this list of conditions and the following disclaimer in the 14153151Sjkim * documentation and/or other materials provided with the distribution. 15153151Sjkim * 3. Neither the name of the Politecnico di Torino nor the names of its 16153151Sjkim * contributors may be used to endorse or promote products derived from 17153151Sjkim * this software without specific prior written permission. 18153151Sjkim * 19153151Sjkim * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20153151Sjkim * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21153151Sjkim * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22153151Sjkim * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23153151Sjkim * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24153151Sjkim * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25153151Sjkim * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26182173Sjkim * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27153151Sjkim * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28153151Sjkim * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29153151Sjkim * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30153151Sjkim */ 31153151Sjkim 32153151Sjkim#include <sys/cdefs.h> 33153151Sjkim__FBSDID("$FreeBSD$"); 34153151Sjkim 35181846Sjkim#ifdef _KERNEL 36153151Sjkim#include "opt_bpf.h" 37153151Sjkim#include <sys/param.h> 38153151Sjkim#include <sys/systm.h> 39153151Sjkim#include <sys/kernel.h> 40153151Sjkim#include <sys/socket.h> 41153151Sjkim#include <sys/malloc.h> 42181846Sjkim#include <net/if.h> 43181846Sjkim#else 44181846Sjkim#include <stdlib.h> 45199615Sjkim#include <string.h> 46199492Sjkim#include <sys/mman.h> 47199492Sjkim#include <sys/param.h> 48181846Sjkim#endif 49153151Sjkim 50181846Sjkim#include <sys/types.h> 51181846Sjkim 52153151Sjkim#include <net/bpf.h> 53153151Sjkim#include <net/bpf_jitter.h> 54153151Sjkim 55153151Sjkim#include <amd64/amd64/bpf_jit_machdep.h> 56153151Sjkim 57199603Sjkimbpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); 58153151Sjkim 59153151Sjkim/* 60199619Sjkim * Emit routine to update the jump table. 61153151Sjkim */ 62153151Sjkimstatic void 63181846Sjkimemit_length(bpf_bin_stream *stream, __unused u_int value, u_int len) 64153151Sjkim{ 65153151Sjkim 66199619Sjkim if (stream->refs != NULL) 67199619Sjkim (stream->refs)[stream->bpf_pc] += len; 68153151Sjkim stream->cur_ip += len; 69153151Sjkim} 70153151Sjkim 71153151Sjkim/* 72199619Sjkim * Emit routine to output the actual binary code. 73153151Sjkim */ 74153151Sjkimstatic void 75153151Sjkimemit_code(bpf_bin_stream *stream, u_int value, u_int len) 76153151Sjkim{ 77153151Sjkim 78153151Sjkim switch (len) { 79153151Sjkim case 1: 80153151Sjkim stream->ibuf[stream->cur_ip] = (u_char)value; 81153151Sjkim stream->cur_ip++; 82153151Sjkim break; 83153151Sjkim 84153151Sjkim case 2: 85153151Sjkim *((u_short *)(stream->ibuf + stream->cur_ip)) = (u_short)value; 86153151Sjkim stream->cur_ip += 2; 87153151Sjkim break; 88153151Sjkim 89153151Sjkim case 4: 90153151Sjkim *((u_int *)(stream->ibuf + stream->cur_ip)) = value; 91153151Sjkim stream->cur_ip += 4; 92153151Sjkim break; 93153151Sjkim } 94153151Sjkim 95153151Sjkim return; 96153151Sjkim} 97153151Sjkim 98153151Sjkim/* 99199619Sjkim * Scan the filter program and find possible optimization. 100153151Sjkim */ 101199619Sjkimstatic int 102199619Sjkimbpf_jit_optimize(struct bpf_insn *prog, u_int nins) 103199619Sjkim{ 104199619Sjkim int flags; 105199619Sjkim u_int i; 106199619Sjkim 107199619Sjkim /* Do we return immediately? */ 108199619Sjkim if (BPF_CLASS(prog[0].code) == BPF_RET) 109199721Sjkim return (BPF_JIT_FRET); 110199619Sjkim 111199619Sjkim for (flags = 0, i = 0; i < nins; i++) { 112199721Sjkim switch (prog[i].code) { 113199721Sjkim case BPF_LD|BPF_W|BPF_ABS: 114199721Sjkim case BPF_LD|BPF_H|BPF_ABS: 115199721Sjkim case BPF_LD|BPF_B|BPF_ABS: 116199721Sjkim case BPF_LD|BPF_W|BPF_IND: 117199721Sjkim case BPF_LD|BPF_H|BPF_IND: 118199721Sjkim case BPF_LD|BPF_B|BPF_IND: 119199721Sjkim case BPF_LDX|BPF_MSH|BPF_B: 120199721Sjkim flags |= BPF_JIT_FPKT; 121199721Sjkim break; 122199721Sjkim case BPF_LD|BPF_MEM: 123199721Sjkim case BPF_LDX|BPF_MEM: 124199721Sjkim case BPF_ST: 125199721Sjkim case BPF_STX: 126199721Sjkim flags |= BPF_JIT_FMEM; 127199721Sjkim break; 128199721Sjkim case BPF_LD|BPF_W|BPF_LEN: 129199721Sjkim case BPF_LDX|BPF_W|BPF_LEN: 130199721Sjkim flags |= BPF_JIT_FLEN; 131199721Sjkim break; 132199721Sjkim case BPF_JMP|BPF_JA: 133199721Sjkim case BPF_JMP|BPF_JGT|BPF_K: 134199721Sjkim case BPF_JMP|BPF_JGE|BPF_K: 135199721Sjkim case BPF_JMP|BPF_JEQ|BPF_K: 136199721Sjkim case BPF_JMP|BPF_JSET|BPF_K: 137199721Sjkim case BPF_JMP|BPF_JGT|BPF_X: 138199721Sjkim case BPF_JMP|BPF_JGE|BPF_X: 139199721Sjkim case BPF_JMP|BPF_JEQ|BPF_X: 140199721Sjkim case BPF_JMP|BPF_JSET|BPF_X: 141199721Sjkim flags |= BPF_JIT_FJMP; 142199721Sjkim break; 143199721Sjkim } 144199619Sjkim if (flags == BPF_JIT_FLAG_ALL) 145199619Sjkim break; 146199619Sjkim } 147199619Sjkim 148199619Sjkim return (flags); 149199619Sjkim} 150199619Sjkim 151199619Sjkim/* 152199619Sjkim * Function that does the real stuff. 153199619Sjkim */ 154153151Sjkimbpf_filter_func 155199603Sjkimbpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) 156153151Sjkim{ 157199492Sjkim bpf_bin_stream stream; 158153151Sjkim struct bpf_insn *ins; 159199721Sjkim int flags, fret, fpkt, fmem, fjmp, flen; 160153151Sjkim u_int i, pass; 161153151Sjkim 162153151Sjkim /* 163199615Sjkim * NOTE: Do not modify the name of this variable, as it's used by 164153151Sjkim * the macros to emit code. 165153151Sjkim */ 166153151Sjkim emit_func emitm; 167153151Sjkim 168199721Sjkim flags = bpf_jit_optimize(prog, nins); 169199721Sjkim fret = (flags & BPF_JIT_FRET) != 0; 170199721Sjkim fpkt = (flags & BPF_JIT_FPKT) != 0; 171199721Sjkim fmem = (flags & BPF_JIT_FMEM) != 0; 172199721Sjkim fjmp = (flags & BPF_JIT_FJMP) != 0; 173199721Sjkim flen = (flags & BPF_JIT_FLEN) != 0; 174199721Sjkim 175199721Sjkim if (fret) 176199721Sjkim nins = 1; 177199721Sjkim 178199619Sjkim memset(&stream, 0, sizeof(stream)); 179199619Sjkim 180199615Sjkim /* Allocate the reference table for the jumps. */ 181199721Sjkim if (fjmp) { 182181846Sjkim#ifdef _KERNEL 183199619Sjkim stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, 184199619Sjkim M_NOWAIT | M_ZERO); 185181846Sjkim#else 186199721Sjkim stream.refs = calloc(nins + 1, sizeof(u_int)); 187181846Sjkim#endif 188199619Sjkim if (stream.refs == NULL) 189199619Sjkim return (NULL); 190199619Sjkim } 191153151Sjkim 192153151Sjkim /* 193199615Sjkim * The first pass will emit the lengths of the instructions 194199615Sjkim * to create the reference table. 195153151Sjkim */ 196153151Sjkim emitm = emit_length; 197153151Sjkim 198199615Sjkim for (pass = 0; pass < 2; pass++) { 199153151Sjkim ins = prog; 200153151Sjkim 201199615Sjkim /* Create the procedure header. */ 202199721Sjkim if (fmem) { 203199619Sjkim PUSH(RBP); 204199619Sjkim MOVrq(RSP, RBP); 205199619Sjkim SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP); 206199619Sjkim } 207199721Sjkim if (flen) 208199721Sjkim MOVrd2(ESI, R9D); 209199721Sjkim if (fpkt) { 210199619Sjkim MOVrq2(RDI, R8); 211199619Sjkim MOVrd(EDX, EDI); 212199619Sjkim } 213153151Sjkim 214153151Sjkim for (i = 0; i < nins; i++) { 215153151Sjkim stream.bpf_pc++; 216153151Sjkim 217153151Sjkim switch (ins->code) { 218153151Sjkim default: 219181846Sjkim#ifdef _KERNEL 220181648Sjkim return (NULL); 221181846Sjkim#else 222181846Sjkim abort(); 223181846Sjkim#endif 224153151Sjkim 225153151Sjkim case BPF_RET|BPF_K: 226179967Sjkim MOVid(ins->k, EAX); 227199721Sjkim if (fmem) 228199619Sjkim LEAVE(); 229199619Sjkim RET(); 230153151Sjkim break; 231153151Sjkim 232153151Sjkim case BPF_RET|BPF_A: 233199721Sjkim if (fmem) 234199619Sjkim LEAVE(); 235199619Sjkim RET(); 236153151Sjkim break; 237153151Sjkim 238153151Sjkim case BPF_LD|BPF_W|BPF_ABS: 239181848Sjkim MOVid(ins->k, ESI); 240181848Sjkim CMPrd(EDI, ESI); 241181848Sjkim JAb(12); 242181848Sjkim MOVrd(EDI, ECX); 243181848Sjkim SUBrd(ESI, ECX); 244181848Sjkim CMPid(sizeof(int32_t), ECX); 245199721Sjkim if (fmem) { 246199619Sjkim JAEb(4); 247199619Sjkim ZEROrd(EAX); 248199619Sjkim LEAVE(); 249199619Sjkim } else { 250199619Sjkim JAEb(3); 251199619Sjkim ZEROrd(EAX); 252199619Sjkim } 253199619Sjkim RET(); 254199603Sjkim MOVrq3(R8, RCX); 255199603Sjkim MOVobd(RCX, RSI, EAX); 256153151Sjkim BSWAP(EAX); 257153151Sjkim break; 258153151Sjkim 259153151Sjkim case BPF_LD|BPF_H|BPF_ABS: 260179977Sjkim ZEROrd(EAX); 261181848Sjkim MOVid(ins->k, ESI); 262181848Sjkim CMPrd(EDI, ESI); 263181848Sjkim JAb(12); 264181848Sjkim MOVrd(EDI, ECX); 265181848Sjkim SUBrd(ESI, ECX); 266181848Sjkim CMPid(sizeof(int16_t), ECX); 267199721Sjkim if (fmem) { 268199619Sjkim JAEb(2); 269199619Sjkim LEAVE(); 270199619Sjkim } else 271199619Sjkim JAEb(1); 272199619Sjkim RET(); 273199603Sjkim MOVrq3(R8, RCX); 274199603Sjkim MOVobw(RCX, RSI, AX); 275153151Sjkim SWAP_AX(); 276153151Sjkim break; 277153151Sjkim 278153151Sjkim case BPF_LD|BPF_B|BPF_ABS: 279179977Sjkim ZEROrd(EAX); 280181848Sjkim MOVid(ins->k, ESI); 281181848Sjkim CMPrd(EDI, ESI); 282199721Sjkim if (fmem) { 283199619Sjkim JBb(2); 284199619Sjkim LEAVE(); 285199619Sjkim } else 286199619Sjkim JBb(1); 287199619Sjkim RET(); 288199603Sjkim MOVrq3(R8, RCX); 289199603Sjkim MOVobb(RCX, RSI, AL); 290153151Sjkim break; 291153151Sjkim 292153151Sjkim case BPF_LD|BPF_W|BPF_LEN: 293181644Sjkim MOVrd3(R9D, EAX); 294153151Sjkim break; 295153151Sjkim 296153151Sjkim case BPF_LDX|BPF_W|BPF_LEN: 297181644Sjkim MOVrd3(R9D, EDX); 298153151Sjkim break; 299153151Sjkim 300153151Sjkim case BPF_LD|BPF_W|BPF_IND: 301181848Sjkim CMPrd(EDI, EDX); 302181848Sjkim JAb(27); 303181848Sjkim MOVid(ins->k, ESI); 304181848Sjkim MOVrd(EDI, ECX); 305181848Sjkim SUBrd(EDX, ECX); 306181848Sjkim CMPrd(ESI, ECX); 307181848Sjkim JBb(14); 308181848Sjkim ADDrd(EDX, ESI); 309181848Sjkim MOVrd(EDI, ECX); 310181848Sjkim SUBrd(ESI, ECX); 311181848Sjkim CMPid(sizeof(int32_t), ECX); 312199721Sjkim if (fmem) { 313199619Sjkim JAEb(4); 314199619Sjkim ZEROrd(EAX); 315199619Sjkim LEAVE(); 316199619Sjkim } else { 317199619Sjkim JAEb(3); 318199619Sjkim ZEROrd(EAX); 319199619Sjkim } 320199619Sjkim RET(); 321199603Sjkim MOVrq3(R8, RCX); 322199603Sjkim MOVobd(RCX, RSI, EAX); 323153151Sjkim BSWAP(EAX); 324153151Sjkim break; 325153151Sjkim 326153151Sjkim case BPF_LD|BPF_H|BPF_IND: 327179977Sjkim ZEROrd(EAX); 328181848Sjkim CMPrd(EDI, EDX); 329181848Sjkim JAb(27); 330181848Sjkim MOVid(ins->k, ESI); 331181848Sjkim MOVrd(EDI, ECX); 332181848Sjkim SUBrd(EDX, ECX); 333181848Sjkim CMPrd(ESI, ECX); 334181848Sjkim JBb(14); 335181848Sjkim ADDrd(EDX, ESI); 336181848Sjkim MOVrd(EDI, ECX); 337181848Sjkim SUBrd(ESI, ECX); 338181848Sjkim CMPid(sizeof(int16_t), ECX); 339199721Sjkim if (fmem) { 340199619Sjkim JAEb(2); 341199619Sjkim LEAVE(); 342199619Sjkim } else 343199619Sjkim JAEb(1); 344199619Sjkim RET(); 345199603Sjkim MOVrq3(R8, RCX); 346199603Sjkim MOVobw(RCX, RSI, AX); 347153151Sjkim SWAP_AX(); 348153151Sjkim break; 349153151Sjkim 350153151Sjkim case BPF_LD|BPF_B|BPF_IND: 351179977Sjkim ZEROrd(EAX); 352181848Sjkim CMPrd(EDI, EDX); 353181848Sjkim JAEb(13); 354181848Sjkim MOVid(ins->k, ESI); 355181848Sjkim MOVrd(EDI, ECX); 356181848Sjkim SUBrd(EDX, ECX); 357181848Sjkim CMPrd(ESI, ECX); 358199721Sjkim if (fmem) { 359199619Sjkim JAb(2); 360199619Sjkim LEAVE(); 361199619Sjkim } else 362199619Sjkim JAb(1); 363199619Sjkim RET(); 364199603Sjkim MOVrq3(R8, RCX); 365181848Sjkim ADDrd(EDX, ESI); 366199603Sjkim MOVobb(RCX, RSI, AL); 367153151Sjkim break; 368153151Sjkim 369153151Sjkim case BPF_LDX|BPF_MSH|BPF_B: 370181848Sjkim MOVid(ins->k, ESI); 371181848Sjkim CMPrd(EDI, ESI); 372199721Sjkim if (fmem) { 373199619Sjkim JBb(4); 374199619Sjkim ZEROrd(EAX); 375199619Sjkim LEAVE(); 376199619Sjkim } else { 377199619Sjkim JBb(3); 378199619Sjkim ZEROrd(EAX); 379199619Sjkim } 380199619Sjkim RET(); 381179977Sjkim ZEROrd(EDX); 382199603Sjkim MOVrq3(R8, RCX); 383199603Sjkim MOVobb(RCX, RSI, DL); 384181648Sjkim ANDib(0x0f, DL); 385179967Sjkim SHLib(2, EDX); 386153151Sjkim break; 387153151Sjkim 388153151Sjkim case BPF_LD|BPF_IMM: 389179967Sjkim MOVid(ins->k, EAX); 390153151Sjkim break; 391153151Sjkim 392153151Sjkim case BPF_LDX|BPF_IMM: 393179967Sjkim MOVid(ins->k, EDX); 394153151Sjkim break; 395153151Sjkim 396153151Sjkim case BPF_LD|BPF_MEM: 397199603Sjkim MOVid(ins->k * sizeof(uint32_t), ESI); 398199603Sjkim MOVobd(RSP, RSI, EAX); 399153151Sjkim break; 400153151Sjkim 401153151Sjkim case BPF_LDX|BPF_MEM: 402199603Sjkim MOVid(ins->k * sizeof(uint32_t), ESI); 403199603Sjkim MOVobd(RSP, RSI, EDX); 404153151Sjkim break; 405153151Sjkim 406153151Sjkim case BPF_ST: 407153151Sjkim /* 408153151Sjkim * XXX this command and the following could 409153151Sjkim * be optimized if the previous instruction 410153151Sjkim * was already of this type 411153151Sjkim */ 412199603Sjkim MOVid(ins->k * sizeof(uint32_t), ESI); 413199603Sjkim MOVomd(EAX, RSP, RSI); 414153151Sjkim break; 415153151Sjkim 416153151Sjkim case BPF_STX: 417199603Sjkim MOVid(ins->k * sizeof(uint32_t), ESI); 418199603Sjkim MOVomd(EDX, RSP, RSI); 419153151Sjkim break; 420153151Sjkim 421153151Sjkim case BPF_JMP|BPF_JA: 422207081Sjkim JUMP(ins->k); 423153151Sjkim break; 424153151Sjkim 425153151Sjkim case BPF_JMP|BPF_JGT|BPF_K: 426207081Sjkim if (ins->jt == ins->jf) { 427207081Sjkim JUMP(ins->jt); 428181697Sjkim break; 429207081Sjkim } 430179967Sjkim CMPid(ins->k, EAX); 431181697Sjkim JCC(JA, JBE); 432153151Sjkim break; 433153151Sjkim 434153151Sjkim case BPF_JMP|BPF_JGE|BPF_K: 435207081Sjkim if (ins->jt == ins->jf) { 436207081Sjkim JUMP(ins->jt); 437181697Sjkim break; 438207081Sjkim } 439179967Sjkim CMPid(ins->k, EAX); 440181697Sjkim JCC(JAE, JB); 441153151Sjkim break; 442153151Sjkim 443153151Sjkim case BPF_JMP|BPF_JEQ|BPF_K: 444207081Sjkim if (ins->jt == ins->jf) { 445207081Sjkim JUMP(ins->jt); 446181697Sjkim break; 447207081Sjkim } 448179967Sjkim CMPid(ins->k, EAX); 449181697Sjkim JCC(JE, JNE); 450153151Sjkim break; 451153151Sjkim 452153151Sjkim case BPF_JMP|BPF_JSET|BPF_K: 453207081Sjkim if (ins->jt == ins->jf) { 454207081Sjkim JUMP(ins->jt); 455181697Sjkim break; 456207081Sjkim } 457181697Sjkim TESTid(ins->k, EAX); 458181697Sjkim JCC(JNE, JE); 459153151Sjkim break; 460153151Sjkim 461153151Sjkim case BPF_JMP|BPF_JGT|BPF_X: 462207081Sjkim if (ins->jt == ins->jf) { 463207081Sjkim JUMP(ins->jt); 464181697Sjkim break; 465207081Sjkim } 466179967Sjkim CMPrd(EDX, EAX); 467181697Sjkim JCC(JA, JBE); 468153151Sjkim break; 469153151Sjkim 470153151Sjkim case BPF_JMP|BPF_JGE|BPF_X: 471207081Sjkim if (ins->jt == ins->jf) { 472207081Sjkim JUMP(ins->jt); 473181697Sjkim break; 474207081Sjkim } 475179967Sjkim CMPrd(EDX, EAX); 476181697Sjkim JCC(JAE, JB); 477153151Sjkim break; 478153151Sjkim 479153151Sjkim case BPF_JMP|BPF_JEQ|BPF_X: 480207081Sjkim if (ins->jt == ins->jf) { 481207081Sjkim JUMP(ins->jt); 482181697Sjkim break; 483207081Sjkim } 484179967Sjkim CMPrd(EDX, EAX); 485181697Sjkim JCC(JE, JNE); 486153151Sjkim break; 487153151Sjkim 488153151Sjkim case BPF_JMP|BPF_JSET|BPF_X: 489207081Sjkim if (ins->jt == ins->jf) { 490207081Sjkim JUMP(ins->jt); 491181697Sjkim break; 492207081Sjkim } 493181697Sjkim TESTrd(EDX, EAX); 494181697Sjkim JCC(JNE, JE); 495153151Sjkim break; 496153151Sjkim 497153151Sjkim case BPF_ALU|BPF_ADD|BPF_X: 498179967Sjkim ADDrd(EDX, EAX); 499153151Sjkim break; 500153151Sjkim 501153151Sjkim case BPF_ALU|BPF_SUB|BPF_X: 502179967Sjkim SUBrd(EDX, EAX); 503153151Sjkim break; 504153151Sjkim 505153151Sjkim case BPF_ALU|BPF_MUL|BPF_X: 506179967Sjkim MOVrd(EDX, ECX); 507179967Sjkim MULrd(EDX); 508153151Sjkim MOVrd(ECX, EDX); 509153151Sjkim break; 510153151Sjkim 511153151Sjkim case BPF_ALU|BPF_DIV|BPF_X: 512181697Sjkim TESTrd(EDX, EDX); 513199721Sjkim if (fmem) { 514199619Sjkim JNEb(4); 515199619Sjkim ZEROrd(EAX); 516199619Sjkim LEAVE(); 517199619Sjkim } else { 518199619Sjkim JNEb(3); 519199619Sjkim ZEROrd(EAX); 520199619Sjkim } 521199619Sjkim RET(); 522179967Sjkim MOVrd(EDX, ECX); 523179977Sjkim ZEROrd(EDX); 524153151Sjkim DIVrd(ECX); 525179967Sjkim MOVrd(ECX, EDX); 526153151Sjkim break; 527153151Sjkim 528153151Sjkim case BPF_ALU|BPF_AND|BPF_X: 529179967Sjkim ANDrd(EDX, EAX); 530153151Sjkim break; 531153151Sjkim 532153151Sjkim case BPF_ALU|BPF_OR|BPF_X: 533179967Sjkim ORrd(EDX, EAX); 534153151Sjkim break; 535153151Sjkim 536153151Sjkim case BPF_ALU|BPF_LSH|BPF_X: 537179967Sjkim MOVrd(EDX, ECX); 538153151Sjkim SHL_CLrb(EAX); 539153151Sjkim break; 540153151Sjkim 541153151Sjkim case BPF_ALU|BPF_RSH|BPF_X: 542179967Sjkim MOVrd(EDX, ECX); 543153151Sjkim SHR_CLrb(EAX); 544153151Sjkim break; 545153151Sjkim 546153151Sjkim case BPF_ALU|BPF_ADD|BPF_K: 547153151Sjkim ADD_EAXi(ins->k); 548153151Sjkim break; 549153151Sjkim 550153151Sjkim case BPF_ALU|BPF_SUB|BPF_K: 551153151Sjkim SUB_EAXi(ins->k); 552153151Sjkim break; 553153151Sjkim 554153151Sjkim case BPF_ALU|BPF_MUL|BPF_K: 555179967Sjkim MOVrd(EDX, ECX); 556179967Sjkim MOVid(ins->k, EDX); 557179967Sjkim MULrd(EDX); 558153151Sjkim MOVrd(ECX, EDX); 559153151Sjkim break; 560153151Sjkim 561153151Sjkim case BPF_ALU|BPF_DIV|BPF_K: 562179967Sjkim MOVrd(EDX, ECX); 563179977Sjkim ZEROrd(EDX); 564179967Sjkim MOVid(ins->k, ESI); 565153151Sjkim DIVrd(ESI); 566179967Sjkim MOVrd(ECX, EDX); 567153151Sjkim break; 568153151Sjkim 569153151Sjkim case BPF_ALU|BPF_AND|BPF_K: 570179967Sjkim ANDid(ins->k, EAX); 571153151Sjkim break; 572153151Sjkim 573153151Sjkim case BPF_ALU|BPF_OR|BPF_K: 574179967Sjkim ORid(ins->k, EAX); 575153151Sjkim break; 576153151Sjkim 577153151Sjkim case BPF_ALU|BPF_LSH|BPF_K: 578179967Sjkim SHLib((ins->k) & 0xff, EAX); 579153151Sjkim break; 580153151Sjkim 581153151Sjkim case BPF_ALU|BPF_RSH|BPF_K: 582179967Sjkim SHRib((ins->k) & 0xff, EAX); 583153151Sjkim break; 584153151Sjkim 585153151Sjkim case BPF_ALU|BPF_NEG: 586153151Sjkim NEGd(EAX); 587153151Sjkim break; 588153151Sjkim 589153151Sjkim case BPF_MISC|BPF_TAX: 590179967Sjkim MOVrd(EAX, EDX); 591153151Sjkim break; 592153151Sjkim 593153151Sjkim case BPF_MISC|BPF_TXA: 594179967Sjkim MOVrd(EDX, EAX); 595153151Sjkim break; 596153151Sjkim } 597153151Sjkim ins++; 598153151Sjkim } 599153151Sjkim 600199615Sjkim if (pass > 0) 601199615Sjkim continue; 602153151Sjkim 603199615Sjkim *size = stream.cur_ip; 604181846Sjkim#ifdef _KERNEL 605199615Sjkim stream.ibuf = malloc(*size, M_BPFJIT, M_NOWAIT); 606199492Sjkim if (stream.ibuf == NULL) 607199492Sjkim break; 608181846Sjkim#else 609199615Sjkim stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE, 610199603Sjkim MAP_ANON, -1, 0); 611199492Sjkim if (stream.ibuf == MAP_FAILED) { 612199492Sjkim stream.ibuf = NULL; 613199492Sjkim break; 614199492Sjkim } 615181846Sjkim#endif 616153151Sjkim 617153151Sjkim /* 618199615Sjkim * Modify the reference table to contain the offsets and 619199615Sjkim * not the lengths of the instructions. 620153151Sjkim */ 621199721Sjkim if (fjmp) 622199619Sjkim for (i = 1; i < nins + 1; i++) 623199619Sjkim stream.refs[i] += stream.refs[i - 1]; 624153151Sjkim 625199615Sjkim /* Reset the counters. */ 626153151Sjkim stream.cur_ip = 0; 627153151Sjkim stream.bpf_pc = 0; 628153151Sjkim 629199615Sjkim /* The second pass creates the actual code. */ 630153151Sjkim emitm = emit_code; 631153151Sjkim } 632153151Sjkim 633153151Sjkim /* 634199615Sjkim * The reference table is needed only during compilation, 635199615Sjkim * now we can free it. 636153151Sjkim */ 637199721Sjkim if (fjmp) 638181846Sjkim#ifdef _KERNEL 639199619Sjkim free(stream.refs, M_BPFJIT); 640181846Sjkim#else 641199619Sjkim free(stream.refs); 642199619Sjkim#endif 643199619Sjkim 644199619Sjkim#ifndef _KERNEL 645199615Sjkim if (stream.ibuf != NULL && 646199615Sjkim mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) { 647199615Sjkim munmap(stream.ibuf, *size); 648199615Sjkim stream.ibuf = NULL; 649199615Sjkim } 650181846Sjkim#endif 651153151Sjkim 652181648Sjkim return ((bpf_filter_func)stream.ibuf); 653153151Sjkim} 654