1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ 3 4#include <errno.h> 5#include <string.h> 6#include <linux/bpf.h> 7#include <bpf/bpf_helpers.h> 8#include "bpf_misc.h" 9#include <../../../tools/include/linux/filter.h> 10 11#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) 12 13int vals[] SEC(".data.vals") = {1, 2, 3, 4}; 14 15__naked __noinline __used 16static unsigned long identity_subprog() 17{ 18 /* the simplest *static* 64-bit identity function */ 19 asm volatile ( 20 "r0 = r1;" 21 "exit;" 22 ); 23} 24 25__noinline __used 26unsigned long global_identity_subprog(__u64 x) 27{ 28 /* the simplest *global* 64-bit identity function */ 29 return x; 30} 31 32__naked __noinline __used 33static unsigned long callback_subprog() 34{ 35 /* the simplest callback function */ 36 asm volatile ( 37 "r0 = 0;" 38 "exit;" 39 ); 40} 41 42SEC("?raw_tp") 43__success __log_level(2) 44__msg("7: (0f) r1 += r0") 45__msg("mark_precise: frame0: regs=r0 stack= before 6: (bf) r1 = r7") 46__msg("mark_precise: frame0: regs=r0 stack= before 5: (27) r0 *= 4") 47__msg("mark_precise: frame0: regs=r0 stack= before 11: (95) exit") 48__msg("mark_precise: frame1: regs=r0 stack= before 10: (bf) r0 = r1") 49__msg("mark_precise: frame1: regs=r1 stack= before 4: (85) call pc+5") 50__msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") 51__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 52__naked int subprog_result_precise(void) 53{ 54 asm volatile ( 55 "r6 = 3;" 56 /* pass r6 through r1 into subprog to get it back as r0; 57 * this whole chain will have to be marked as precise later 58 */ 59 "r1 = r6;" 60 "call identity_subprog;" 61 /* now use subprog's returned value (which is a 62 * r6 -> r1 -> r0 chain), as index into vals array, forcing 63 * all of that to be known precisely 64 */ 65 "r0 *= 4;" 66 "r1 = %[vals];" 67 /* here r0->r1->r6 chain is forced to be precise and has to be 68 * propagated back to the beginning, including through the 69 * subprog call 70 */ 71 "r1 += r0;" 72 "r0 = *(u32 *)(r1 + 0);" 73 "exit;" 74 : 75 : __imm_ptr(vals) 76 : __clobber_common, "r6" 77 ); 78} 79 80__naked __noinline __used 81static unsigned long fp_leaking_subprog() 82{ 83 asm volatile ( 84 ".8byte %[r0_eq_r10_cast_s8];" 85 "exit;" 86 :: __imm_insn(r0_eq_r10_cast_s8, BPF_MOVSX64_REG(BPF_REG_0, BPF_REG_10, 8)) 87 ); 88} 89 90__naked __noinline __used 91static unsigned long sneaky_fp_leaking_subprog() 92{ 93 asm volatile ( 94 "r1 = r10;" 95 ".8byte %[r0_eq_r1_cast_s8];" 96 "exit;" 97 :: __imm_insn(r0_eq_r1_cast_s8, BPF_MOVSX64_REG(BPF_REG_0, BPF_REG_1, 8)) 98 ); 99} 100 101SEC("?raw_tp") 102__success __log_level(2) 103__msg("6: (0f) r1 += r0") 104__msg("mark_precise: frame0: last_idx 6 first_idx 0 subseq_idx -1") 105__msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r1 = r6") 106__msg("mark_precise: frame0: regs=r0 stack= before 4: (27) r0 *= 4") 107__msg("mark_precise: frame0: regs=r0 stack= before 3: (57) r0 &= 3") 108__msg("mark_precise: frame0: regs=r0 stack= before 10: (95) exit") 109__msg("mark_precise: frame1: regs=r0 stack= before 9: (bf) r0 = (s8)r10") 110__msg("7: R0_w=scalar") 111__naked int fp_precise_subprog_result(void) 112{ 113 asm volatile ( 114 "call fp_leaking_subprog;" 115 /* use subprog's returned value (which is derived from r10=fp 116 * register), as index into vals array, forcing all of that to 117 * be known precisely 118 */ 119 "r0 &= 3;" 120 "r0 *= 4;" 121 "r1 = %[vals];" 122 /* force precision marking */ 123 "r1 += r0;" 124 "r0 = *(u32 *)(r1 + 0);" 125 "exit;" 126 : 127 : __imm_ptr(vals) 128 : __clobber_common 129 ); 130} 131 132SEC("?raw_tp") 133__success __log_level(2) 134__msg("6: (0f) r1 += r0") 135__msg("mark_precise: frame0: last_idx 6 first_idx 0 subseq_idx -1") 136__msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r1 = r6") 137__msg("mark_precise: frame0: regs=r0 stack= before 4: (27) r0 *= 4") 138__msg("mark_precise: frame0: regs=r0 stack= before 3: (57) r0 &= 3") 139__msg("mark_precise: frame0: regs=r0 stack= before 11: (95) exit") 140__msg("mark_precise: frame1: regs=r0 stack= before 10: (bf) r0 = (s8)r1") 141/* here r1 is marked precise, even though it's fp register, but that's fine 142 * because by the time we get out of subprogram it has to be derived from r10 143 * anyways, at which point we'll break precision chain 144 */ 145__msg("mark_precise: frame1: regs=r1 stack= before 9: (bf) r1 = r10") 146__msg("7: R0_w=scalar") 147__naked int sneaky_fp_precise_subprog_result(void) 148{ 149 asm volatile ( 150 "call sneaky_fp_leaking_subprog;" 151 /* use subprog's returned value (which is derived from r10=fp 152 * register), as index into vals array, forcing all of that to 153 * be known precisely 154 */ 155 "r0 &= 3;" 156 "r0 *= 4;" 157 "r1 = %[vals];" 158 /* force precision marking */ 159 "r1 += r0;" 160 "r0 = *(u32 *)(r1 + 0);" 161 "exit;" 162 : 163 : __imm_ptr(vals) 164 : __clobber_common 165 ); 166} 167 168SEC("?raw_tp") 169__success __log_level(2) 170__msg("9: (0f) r1 += r0") 171__msg("mark_precise: frame0: last_idx 9 first_idx 0") 172__msg("mark_precise: frame0: regs=r0 stack= before 8: (bf) r1 = r7") 173__msg("mark_precise: frame0: regs=r0 stack= before 7: (27) r0 *= 4") 174__msg("mark_precise: frame0: regs=r0 stack= before 5: (a5) if r0 < 0x4 goto pc+1") 175__msg("mark_precise: frame0: regs=r0 stack= before 4: (85) call pc+7") 176__naked int global_subprog_result_precise(void) 177{ 178 asm volatile ( 179 "r6 = 3;" 180 /* pass r6 through r1 into subprog to get it back as r0; 181 * given global_identity_subprog is global, precision won't 182 * propagate all the way back to r6 183 */ 184 "r1 = r6;" 185 "call global_identity_subprog;" 186 /* now use subprog's returned value (which is unknown now, so 187 * we need to clamp it), as index into vals array, forcing r0 188 * to be marked precise (with no effect on r6, though) 189 */ 190 "if r0 < %[vals_arr_sz] goto 1f;" 191 "r0 = %[vals_arr_sz] - 1;" 192 "1:" 193 "r0 *= 4;" 194 "r1 = %[vals];" 195 /* here r0 is forced to be precise and has to be 196 * propagated back to the global subprog call, but it 197 * shouldn't go all the way to mark r6 as precise 198 */ 199 "r1 += r0;" 200 "r0 = *(u32 *)(r1 + 0);" 201 "exit;" 202 : 203 : __imm_ptr(vals), 204 __imm_const(vals_arr_sz, ARRAY_SIZE(vals)) 205 : __clobber_common, "r6" 206 ); 207} 208 209__naked __noinline __used 210static unsigned long loop_callback_bad() 211{ 212 /* bpf_loop() callback that can return values outside of [0, 1] range */ 213 asm volatile ( 214 "call %[bpf_get_prandom_u32];" 215 "if r0 s> 1000 goto 1f;" 216 "r0 = 0;" 217 "1:" 218 "goto +0;" /* checkpoint */ 219 /* bpf_loop() expects [0, 1] values, so branch above skipping 220 * r0 = 0; should lead to a failure, but if exit instruction 221 * doesn't enforce r0's precision, this callback will be 222 * successfully verified 223 */ 224 "exit;" 225 : 226 : __imm(bpf_get_prandom_u32) 227 : __clobber_common 228 ); 229} 230 231SEC("?raw_tp") 232__failure __log_level(2) 233__flag(BPF_F_TEST_STATE_FREQ) 234/* check that fallthrough code path marks r0 as precise */ 235__msg("mark_precise: frame1: regs=r0 stack= before 11: (b7) r0 = 0") 236/* check that we have branch code path doing its own validation */ 237__msg("from 10 to 12: frame1: R0=scalar(smin=umin=1001") 238/* check that branch code path marks r0 as precise, before failing */ 239__msg("mark_precise: frame1: regs=r0 stack= before 9: (85) call bpf_get_prandom_u32#7") 240__msg("At callback return the register R0 has smin=1001 should have been in [0, 1]") 241__naked int callback_precise_return_fail(void) 242{ 243 asm volatile ( 244 "r1 = 1;" /* nr_loops */ 245 "r2 = %[loop_callback_bad];" /* callback_fn */ 246 "r3 = 0;" /* callback_ctx */ 247 "r4 = 0;" /* flags */ 248 "call %[bpf_loop];" 249 250 "r0 = 0;" 251 "exit;" 252 : 253 : __imm_ptr(loop_callback_bad), 254 __imm(bpf_loop) 255 : __clobber_common 256 ); 257} 258 259SEC("?raw_tp") 260__success __log_level(2) 261/* First simulated path does not include callback body, 262 * r1 and r4 are always precise for bpf_loop() calls. 263 */ 264__msg("9: (85) call bpf_loop#181") 265__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") 266__msg("mark_precise: frame0: parent state regs=r4 stack=:") 267__msg("mark_precise: frame0: last_idx 8 first_idx 0 subseq_idx 9") 268__msg("mark_precise: frame0: regs=r4 stack= before 8: (b7) r4 = 0") 269__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") 270__msg("mark_precise: frame0: parent state regs=r1 stack=:") 271__msg("mark_precise: frame0: last_idx 8 first_idx 0 subseq_idx 9") 272__msg("mark_precise: frame0: regs=r1 stack= before 8: (b7) r4 = 0") 273__msg("mark_precise: frame0: regs=r1 stack= before 7: (b7) r3 = 0") 274__msg("mark_precise: frame0: regs=r1 stack= before 6: (bf) r2 = r8") 275__msg("mark_precise: frame0: regs=r1 stack= before 5: (bf) r1 = r6") 276__msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") 277/* r6 precision propagation */ 278__msg("14: (0f) r1 += r6") 279__msg("mark_precise: frame0: last_idx 14 first_idx 9") 280__msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7") 281__msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4") 282__msg("mark_precise: frame0: regs=r6 stack= before 11: (25) if r6 > 0x3 goto pc+4") 283__msg("mark_precise: frame0: regs=r6 stack= before 10: (bf) r6 = r0") 284__msg("mark_precise: frame0: regs=r0 stack= before 9: (85) call bpf_loop") 285/* State entering callback body popped from states stack */ 286__msg("from 9 to 17: frame1:") 287__msg("17: frame1: R1=scalar() R2=0 R10=fp0 cb") 288__msg("17: (b7) r0 = 0") 289__msg("18: (95) exit") 290__msg("returning from callee:") 291__msg("to caller at 9:") 292__msg("frame 0: propagating r1,r4") 293__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") 294__msg("mark_precise: frame0: regs=r1,r4 stack= before 18: (95) exit") 295__msg("from 18 to 9: safe") 296__naked int callback_result_precise(void) 297{ 298 asm volatile ( 299 "r6 = 3;" 300 301 /* call subprog and use result; r0 shouldn't propagate back to 302 * callback_subprog 303 */ 304 "r1 = r6;" /* nr_loops */ 305 "r2 = %[callback_subprog];" /* callback_fn */ 306 "r3 = 0;" /* callback_ctx */ 307 "r4 = 0;" /* flags */ 308 "call %[bpf_loop];" 309 310 "r6 = r0;" 311 "if r6 > 3 goto 1f;" 312 "r6 *= 4;" 313 "r1 = %[vals];" 314 /* here r6 is forced to be precise and has to be propagated 315 * back to the bpf_loop() call, but not beyond 316 */ 317 "r1 += r6;" 318 "r0 = *(u32 *)(r1 + 0);" 319 "1:" 320 "exit;" 321 : 322 : __imm_ptr(vals), 323 __imm_ptr(callback_subprog), 324 __imm(bpf_loop) 325 : __clobber_common, "r6" 326 ); 327} 328 329SEC("?raw_tp") 330__success __log_level(2) 331__msg("7: (0f) r1 += r6") 332__msg("mark_precise: frame0: last_idx 7 first_idx 0") 333__msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r1 = r7") 334__msg("mark_precise: frame0: regs=r6 stack= before 5: (27) r6 *= 4") 335__msg("mark_precise: frame0: regs=r6 stack= before 11: (95) exit") 336__msg("mark_precise: frame1: regs= stack= before 10: (bf) r0 = r1") 337__msg("mark_precise: frame1: regs= stack= before 4: (85) call pc+5") 338__msg("mark_precise: frame0: regs=r6 stack= before 3: (b7) r1 = 0") 339__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 340__naked int parent_callee_saved_reg_precise(void) 341{ 342 asm volatile ( 343 "r6 = 3;" 344 345 /* call subprog and ignore result; we need this call only to 346 * complicate jump history 347 */ 348 "r1 = 0;" 349 "call identity_subprog;" 350 351 "r6 *= 4;" 352 "r1 = %[vals];" 353 /* here r6 is forced to be precise and has to be propagated 354 * back to the beginning, handling (and ignoring) subprog call 355 */ 356 "r1 += r6;" 357 "r0 = *(u32 *)(r1 + 0);" 358 "exit;" 359 : 360 : __imm_ptr(vals) 361 : __clobber_common, "r6" 362 ); 363} 364 365SEC("?raw_tp") 366__success __log_level(2) 367__msg("7: (0f) r1 += r6") 368__msg("mark_precise: frame0: last_idx 7 first_idx 0") 369__msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r1 = r7") 370__msg("mark_precise: frame0: regs=r6 stack= before 5: (27) r6 *= 4") 371__msg("mark_precise: frame0: regs=r6 stack= before 4: (85) call pc+5") 372__msg("mark_precise: frame0: regs=r6 stack= before 3: (b7) r1 = 0") 373__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 374__naked int parent_callee_saved_reg_precise_global(void) 375{ 376 asm volatile ( 377 "r6 = 3;" 378 379 /* call subprog and ignore result; we need this call only to 380 * complicate jump history 381 */ 382 "r1 = 0;" 383 "call global_identity_subprog;" 384 385 "r6 *= 4;" 386 "r1 = %[vals];" 387 /* here r6 is forced to be precise and has to be propagated 388 * back to the beginning, handling (and ignoring) subprog call 389 */ 390 "r1 += r6;" 391 "r0 = *(u32 *)(r1 + 0);" 392 "exit;" 393 : 394 : __imm_ptr(vals) 395 : __clobber_common, "r6" 396 ); 397} 398 399SEC("?raw_tp") 400__success __log_level(2) 401/* First simulated path does not include callback body */ 402__msg("12: (0f) r1 += r6") 403__msg("mark_precise: frame0: last_idx 12 first_idx 9") 404__msg("mark_precise: frame0: regs=r6 stack= before 11: (bf) r1 = r7") 405__msg("mark_precise: frame0: regs=r6 stack= before 10: (27) r6 *= 4") 406__msg("mark_precise: frame0: regs=r6 stack= before 9: (85) call bpf_loop") 407__msg("mark_precise: frame0: parent state regs=r6 stack=:") 408__msg("mark_precise: frame0: last_idx 8 first_idx 0 subseq_idx 9") 409__msg("mark_precise: frame0: regs=r6 stack= before 8: (b7) r4 = 0") 410__msg("mark_precise: frame0: regs=r6 stack= before 7: (b7) r3 = 0") 411__msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r2 = r8") 412__msg("mark_precise: frame0: regs=r6 stack= before 5: (b7) r1 = 1") 413__msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") 414/* State entering callback body popped from states stack */ 415__msg("from 9 to 15: frame1:") 416__msg("15: frame1: R1=scalar() R2=0 R10=fp0 cb") 417__msg("15: (b7) r0 = 0") 418__msg("16: (95) exit") 419__msg("returning from callee:") 420__msg("to caller at 9:") 421/* r1, r4 are always precise for bpf_loop(), 422 * r6 was marked before backtracking to callback body. 423 */ 424__msg("frame 0: propagating r1,r4,r6") 425__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") 426__msg("mark_precise: frame0: regs=r1,r4,r6 stack= before 16: (95) exit") 427__msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0") 428__msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop") 429__msg("mark_precise: frame0: parent state regs= stack=:") 430__msg("from 16 to 9: safe") 431__naked int parent_callee_saved_reg_precise_with_callback(void) 432{ 433 asm volatile ( 434 "r6 = 3;" 435 436 /* call subprog and ignore result; we need this call only to 437 * complicate jump history 438 */ 439 "r1 = 1;" /* nr_loops */ 440 "r2 = %[callback_subprog];" /* callback_fn */ 441 "r3 = 0;" /* callback_ctx */ 442 "r4 = 0;" /* flags */ 443 "call %[bpf_loop];" 444 445 "r6 *= 4;" 446 "r1 = %[vals];" 447 /* here r6 is forced to be precise and has to be propagated 448 * back to the beginning, handling (and ignoring) callback call 449 */ 450 "r1 += r6;" 451 "r0 = *(u32 *)(r1 + 0);" 452 "exit;" 453 : 454 : __imm_ptr(vals), 455 __imm_ptr(callback_subprog), 456 __imm(bpf_loop) 457 : __clobber_common, "r6" 458 ); 459} 460 461SEC("?raw_tp") 462__success __log_level(2) 463__msg("9: (0f) r1 += r6") 464__msg("mark_precise: frame0: last_idx 9 first_idx 6") 465__msg("mark_precise: frame0: regs=r6 stack= before 8: (bf) r1 = r7") 466__msg("mark_precise: frame0: regs=r6 stack= before 7: (27) r6 *= 4") 467__msg("mark_precise: frame0: regs=r6 stack= before 6: (79) r6 = *(u64 *)(r10 -8)") 468__msg("mark_precise: frame0: parent state regs= stack=-8:") 469__msg("mark_precise: frame0: last_idx 13 first_idx 0") 470__msg("mark_precise: frame0: regs= stack=-8 before 13: (95) exit") 471__msg("mark_precise: frame1: regs= stack= before 12: (bf) r0 = r1") 472__msg("mark_precise: frame1: regs= stack= before 5: (85) call pc+6") 473__msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r1 = 0") 474__msg("mark_precise: frame0: regs= stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r6") 475__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 476__naked int parent_stack_slot_precise(void) 477{ 478 asm volatile ( 479 /* spill reg */ 480 "r6 = 3;" 481 "*(u64 *)(r10 - 8) = r6;" 482 483 /* call subprog and ignore result; we need this call only to 484 * complicate jump history 485 */ 486 "r1 = 0;" 487 "call identity_subprog;" 488 489 /* restore reg from stack; in this case we'll be carrying 490 * stack mask when going back into subprog through jump 491 * history 492 */ 493 "r6 = *(u64 *)(r10 - 8);" 494 495 "r6 *= 4;" 496 "r1 = %[vals];" 497 /* here r6 is forced to be precise and has to be propagated 498 * back to the beginning, handling (and ignoring) subprog call 499 */ 500 "r1 += r6;" 501 "r0 = *(u32 *)(r1 + 0);" 502 "exit;" 503 : 504 : __imm_ptr(vals) 505 : __clobber_common, "r6" 506 ); 507} 508 509SEC("?raw_tp") 510__success __log_level(2) 511__msg("9: (0f) r1 += r6") 512__msg("mark_precise: frame0: last_idx 9 first_idx 0") 513__msg("mark_precise: frame0: regs=r6 stack= before 8: (bf) r1 = r7") 514__msg("mark_precise: frame0: regs=r6 stack= before 7: (27) r6 *= 4") 515__msg("mark_precise: frame0: regs=r6 stack= before 6: (79) r6 = *(u64 *)(r10 -8)") 516__msg("mark_precise: frame0: regs= stack=-8 before 5: (85) call pc+6") 517__msg("mark_precise: frame0: regs= stack=-8 before 4: (b7) r1 = 0") 518__msg("mark_precise: frame0: regs= stack=-8 before 3: (7b) *(u64 *)(r10 -8) = r6") 519__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 3") 520__naked int parent_stack_slot_precise_global(void) 521{ 522 asm volatile ( 523 /* spill reg */ 524 "r6 = 3;" 525 "*(u64 *)(r10 - 8) = r6;" 526 527 /* call subprog and ignore result; we need this call only to 528 * complicate jump history 529 */ 530 "r1 = 0;" 531 "call global_identity_subprog;" 532 533 /* restore reg from stack; in this case we'll be carrying 534 * stack mask when going back into subprog through jump 535 * history 536 */ 537 "r6 = *(u64 *)(r10 - 8);" 538 539 "r6 *= 4;" 540 "r1 = %[vals];" 541 /* here r6 is forced to be precise and has to be propagated 542 * back to the beginning, handling (and ignoring) subprog call 543 */ 544 "r1 += r6;" 545 "r0 = *(u32 *)(r1 + 0);" 546 "exit;" 547 : 548 : __imm_ptr(vals) 549 : __clobber_common, "r6" 550 ); 551} 552 553SEC("?raw_tp") 554__success __log_level(2) 555/* First simulated path does not include callback body */ 556__msg("14: (0f) r1 += r6") 557__msg("mark_precise: frame0: last_idx 14 first_idx 10") 558__msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7") 559__msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4") 560__msg("mark_precise: frame0: regs=r6 stack= before 11: (79) r6 = *(u64 *)(r10 -8)") 561__msg("mark_precise: frame0: regs= stack=-8 before 10: (85) call bpf_loop") 562__msg("mark_precise: frame0: parent state regs= stack=-8:") 563__msg("mark_precise: frame0: last_idx 9 first_idx 0 subseq_idx 10") 564__msg("mark_precise: frame0: regs= stack=-8 before 9: (b7) r4 = 0") 565__msg("mark_precise: frame0: regs= stack=-8 before 8: (b7) r3 = 0") 566__msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r2 = r8") 567__msg("mark_precise: frame0: regs= stack=-8 before 6: (bf) r1 = r6") 568__msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -8) = r6") 569__msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3") 570/* State entering callback body popped from states stack */ 571__msg("from 10 to 17: frame1:") 572__msg("17: frame1: R1=scalar() R2=0 R10=fp0 cb") 573__msg("17: (b7) r0 = 0") 574__msg("18: (95) exit") 575__msg("returning from callee:") 576__msg("to caller at 10:") 577/* r1, r4 are always precise for bpf_loop(), 578 * fp-8 was marked before backtracking to callback body. 579 */ 580__msg("frame 0: propagating r1,r4,fp-8") 581__msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx -1") 582__msg("mark_precise: frame0: regs=r1,r4 stack=-8 before 18: (95) exit") 583__msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0") 584__msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181") 585__msg("mark_precise: frame0: parent state regs= stack=:") 586__msg("from 18 to 10: safe") 587__naked int parent_stack_slot_precise_with_callback(void) 588{ 589 asm volatile ( 590 /* spill reg */ 591 "r6 = 3;" 592 "*(u64 *)(r10 - 8) = r6;" 593 594 /* ensure we have callback frame in jump history */ 595 "r1 = r6;" /* nr_loops */ 596 "r2 = %[callback_subprog];" /* callback_fn */ 597 "r3 = 0;" /* callback_ctx */ 598 "r4 = 0;" /* flags */ 599 "call %[bpf_loop];" 600 601 /* restore reg from stack; in this case we'll be carrying 602 * stack mask when going back into subprog through jump 603 * history 604 */ 605 "r6 = *(u64 *)(r10 - 8);" 606 607 "r6 *= 4;" 608 "r1 = %[vals];" 609 /* here r6 is forced to be precise and has to be propagated 610 * back to the beginning, handling (and ignoring) subprog call 611 */ 612 "r1 += r6;" 613 "r0 = *(u32 *)(r1 + 0);" 614 "exit;" 615 : 616 : __imm_ptr(vals), 617 __imm_ptr(callback_subprog), 618 __imm(bpf_loop) 619 : __clobber_common, "r6" 620 ); 621} 622 623__noinline __used 624static __u64 subprog_with_precise_arg(__u64 x) 625{ 626 return vals[x]; /* x is forced to be precise */ 627} 628 629SEC("?raw_tp") 630__success __log_level(2) 631__msg("8: (0f) r2 += r1") 632__msg("mark_precise: frame1: last_idx 8 first_idx 0") 633__msg("mark_precise: frame1: regs=r1 stack= before 6: (18) r2 = ") 634__msg("mark_precise: frame1: regs=r1 stack= before 5: (67) r1 <<= 2") 635__msg("mark_precise: frame1: regs=r1 stack= before 2: (85) call pc+2") 636__msg("mark_precise: frame0: regs=r1 stack= before 1: (bf) r1 = r6") 637__msg("mark_precise: frame0: regs=r6 stack= before 0: (b7) r6 = 3") 638__naked int subprog_arg_precise(void) 639{ 640 asm volatile ( 641 "r6 = 3;" 642 "r1 = r6;" 643 /* subprog_with_precise_arg expects its argument to be 644 * precise, so r1->r6 will be marked precise from inside the 645 * subprog 646 */ 647 "call subprog_with_precise_arg;" 648 "r0 += r6;" 649 "exit;" 650 : 651 : 652 : __clobber_common, "r6" 653 ); 654} 655 656/* r1 is pointer to stack slot; 657 * r2 is a register to spill into that slot 658 * subprog also spills r2 into its own stack slot 659 */ 660__naked __noinline __used 661static __u64 subprog_spill_reg_precise(void) 662{ 663 asm volatile ( 664 /* spill to parent stack */ 665 "*(u64 *)(r1 + 0) = r2;" 666 /* spill to subprog stack (we use -16 offset to avoid 667 * accidental confusion with parent's -8 stack slot in 668 * verifier log output) 669 */ 670 "*(u64 *)(r10 - 16) = r2;" 671 /* use both spills as return result to propagete precision everywhere */ 672 "r0 = *(u64 *)(r10 - 16);" 673 "r2 = *(u64 *)(r1 + 0);" 674 "r0 += r2;" 675 "exit;" 676 ); 677} 678 679SEC("?raw_tp") 680__success __log_level(2) 681__msg("10: (0f) r1 += r7") 682__msg("mark_precise: frame0: last_idx 10 first_idx 7 subseq_idx -1") 683__msg("mark_precise: frame0: regs=r7 stack= before 9: (bf) r1 = r8") 684__msg("mark_precise: frame0: regs=r7 stack= before 8: (27) r7 *= 4") 685__msg("mark_precise: frame0: regs=r7 stack= before 7: (79) r7 = *(u64 *)(r10 -8)") 686__msg("mark_precise: frame0: parent state regs= stack=-8: R0_w=2 R6_w=1 R8_rw=map_value(map=.data.vals,ks=4,vs=16) R10=fp0 fp-8_rw=P1") 687__msg("mark_precise: frame0: last_idx 18 first_idx 0 subseq_idx 7") 688__msg("mark_precise: frame0: regs= stack=-8 before 18: (95) exit") 689__msg("mark_precise: frame1: regs= stack= before 17: (0f) r0 += r2") 690__msg("mark_precise: frame1: regs= stack= before 16: (79) r2 = *(u64 *)(r1 +0)") 691__msg("mark_precise: frame1: regs= stack= before 15: (79) r0 = *(u64 *)(r10 -16)") 692__msg("mark_precise: frame1: regs= stack= before 14: (7b) *(u64 *)(r10 -16) = r2") 693__msg("mark_precise: frame1: regs= stack= before 13: (7b) *(u64 *)(r1 +0) = r2") 694__msg("mark_precise: frame1: regs=r2 stack= before 6: (85) call pc+6") 695__msg("mark_precise: frame0: regs=r2 stack= before 5: (bf) r2 = r6") 696__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") 697__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") 698__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 1") 699__naked int subprog_spill_into_parent_stack_slot_precise(void) 700{ 701 asm volatile ( 702 "r6 = 1;" 703 704 /* pass pointer to stack slot and r6 to subprog; 705 * r6 will be marked precise and spilled into fp-8 slot, which 706 * also should be marked precise 707 */ 708 "r1 = r10;" 709 "r1 += -8;" 710 "r2 = r6;" 711 "call subprog_spill_reg_precise;" 712 713 /* restore reg from stack; in this case we'll be carrying 714 * stack mask when going back into subprog through jump 715 * history 716 */ 717 "r7 = *(u64 *)(r10 - 8);" 718 719 "r7 *= 4;" 720 "r1 = %[vals];" 721 /* here r7 is forced to be precise and has to be propagated 722 * back to the beginning, handling subprog call and logic 723 */ 724 "r1 += r7;" 725 "r0 = *(u32 *)(r1 + 0);" 726 "exit;" 727 : 728 : __imm_ptr(vals) 729 : __clobber_common, "r6", "r7" 730 ); 731} 732 733SEC("?raw_tp") 734__success __log_level(2) 735__msg("17: (0f) r1 += r0") 736__msg("mark_precise: frame0: last_idx 17 first_idx 0 subseq_idx -1") 737__msg("mark_precise: frame0: regs=r0 stack= before 16: (bf) r1 = r7") 738__msg("mark_precise: frame0: regs=r0 stack= before 15: (27) r0 *= 4") 739__msg("mark_precise: frame0: regs=r0 stack= before 14: (79) r0 = *(u64 *)(r10 -16)") 740__msg("mark_precise: frame0: regs= stack=-16 before 13: (7b) *(u64 *)(r7 -8) = r0") 741__msg("mark_precise: frame0: regs=r0 stack= before 12: (79) r0 = *(u64 *)(r8 +16)") 742__msg("mark_precise: frame0: regs= stack=-16 before 11: (7b) *(u64 *)(r8 +16) = r0") 743__msg("mark_precise: frame0: regs=r0 stack= before 10: (79) r0 = *(u64 *)(r7 -8)") 744__msg("mark_precise: frame0: regs= stack=-16 before 9: (7b) *(u64 *)(r10 -16) = r0") 745__msg("mark_precise: frame0: regs=r0 stack= before 8: (07) r8 += -32") 746__msg("mark_precise: frame0: regs=r0 stack= before 7: (bf) r8 = r10") 747__msg("mark_precise: frame0: regs=r0 stack= before 6: (07) r7 += -8") 748__msg("mark_precise: frame0: regs=r0 stack= before 5: (bf) r7 = r10") 749__msg("mark_precise: frame0: regs=r0 stack= before 21: (95) exit") 750__msg("mark_precise: frame1: regs=r0 stack= before 20: (bf) r0 = r1") 751__msg("mark_precise: frame1: regs=r1 stack= before 4: (85) call pc+15") 752__msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r1 = r6") 753__msg("mark_precise: frame0: regs=r6 stack= before 2: (b7) r6 = 1") 754__naked int stack_slot_aliases_precision(void) 755{ 756 asm volatile ( 757 "r6 = 1;" 758 /* pass r6 through r1 into subprog to get it back as r0; 759 * this whole chain will have to be marked as precise later 760 */ 761 "r1 = r6;" 762 "call identity_subprog;" 763 /* let's setup two registers that are aliased to r10 */ 764 "r7 = r10;" 765 "r7 += -8;" /* r7 = r10 - 8 */ 766 "r8 = r10;" 767 "r8 += -32;" /* r8 = r10 - 32 */ 768 /* now spill subprog's return value (a r6 -> r1 -> r0 chain) 769 * a few times through different stack pointer regs, making 770 * sure to use r10, r7, and r8 both in LDX and STX insns, and 771 * *importantly* also using a combination of const var_off and 772 * insn->off to validate that we record final stack slot 773 * correctly, instead of relying on just insn->off derivation, 774 * which is only valid for r10-based stack offset 775 */ 776 "*(u64 *)(r10 - 16) = r0;" 777 "r0 = *(u64 *)(r7 - 8);" /* r7 - 8 == r10 - 16 */ 778 "*(u64 *)(r8 + 16) = r0;" /* r8 + 16 = r10 - 16 */ 779 "r0 = *(u64 *)(r8 + 16);" 780 "*(u64 *)(r7 - 8) = r0;" 781 "r0 = *(u64 *)(r10 - 16);" 782 /* get ready to use r0 as an index into array to force precision */ 783 "r0 *= 4;" 784 "r1 = %[vals];" 785 /* here r0->r1->r6 chain is forced to be precise and has to be 786 * propagated back to the beginning, including through the 787 * subprog call and all the stack spills and loads 788 */ 789 "r1 += r0;" 790 "r0 = *(u32 *)(r1 + 0);" 791 "exit;" 792 : 793 : __imm_ptr(vals) 794 : __clobber_common, "r6" 795 ); 796} 797 798char _license[] SEC("license") = "GPL"; 799