1/* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */ 2 3/* 4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * Copyright (c) 2021 The FreeBSD Foundation 7 * 8 * Portions of this software were developed by Mark Johnston under sponsorship 9 * from the FreeBSD Foundation. 10 * 11 * This code is part of the KMSAN subsystem of the NetBSD kernel. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35#define SAN_RUNTIME 36 37#include <sys/cdefs.h> 38#if 0 39__KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $"); 40#endif 41 42#include <sys/param.h> 43#include <sys/systm.h> 44#include <sys/bio.h> 45#include <sys/buf.h> 46#include <sys/conf.h> 47#include <sys/kdb.h> 48#include <sys/kernel.h> 49#include <sys/linker.h> 50#include <sys/malloc.h> 51#include <sys/mbuf.h> 52#include <sys/memdesc.h> 53#include <sys/msan.h> 54#include <sys/proc.h> 55#include <sys/stack.h> 56#include <sys/sysctl.h> 57#include <sys/uio.h> 58 59#include <vm/vm.h> 60#include <vm/pmap.h> 61 62#include <machine/msan.h> 63#include <machine/stdarg.h> 64 65void kmsan_init_arg(size_t); 66void kmsan_init_ret(size_t); 67 68/* -------------------------------------------------------------------------- */ 69 70/* 71 * Part of the compiler ABI. 72 */ 73 74typedef struct { 75 uint8_t *shad; 76 msan_orig_t *orig; 77} msan_meta_t; 78 79#define MSAN_PARAM_SIZE 800 80#define MSAN_RETVAL_SIZE 800 81typedef struct { 82 uint8_t param_shadow[MSAN_PARAM_SIZE]; 83 uint8_t retval_shadow[MSAN_RETVAL_SIZE]; 84 uint8_t va_arg_shadow[MSAN_PARAM_SIZE]; 85 uint8_t va_arg_origin[MSAN_PARAM_SIZE]; 86 uint64_t va_arg_overflow_size; 87 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)]; 88 msan_orig_t retval_origin; 89} msan_tls_t; 90 91/* -------------------------------------------------------------------------- */ 92 93#define MSAN_NCONTEXT 4 94#define MSAN_ORIG_MASK (~0x3) 95 96typedef struct kmsan_td { 97 size_t ctx; 98 msan_tls_t tls[MSAN_NCONTEXT]; 99} msan_td_t; 100 101static msan_tls_t dummy_tls; 102 103/* 104 * Use separate dummy regions for loads and stores: stores may mark the region 105 * as uninitialized, and that can trigger false positives. 106 */ 107static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 108static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE); 109static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE); 110static msan_td_t msan_thread0; 111static bool kmsan_reporting = false; 112 113/* 114 * Avoid clobbering any thread-local state before we panic. 115 */ 116#define kmsan_panic(f, ...) do { \ 117 kmsan_disabled = true; \ 118 panic(f, __VA_ARGS__); \ 119} while (0) 120 121#define REPORT(f, ...) do { \ 122 if (panic_on_violation) { \ 123 kmsan_panic(f, __VA_ARGS__); \ 124 } else { \ 125 struct stack st; \ 126 \ 127 stack_save(&st); \ 128 printf(f "\n", __VA_ARGS__); \ 129 stack_print_ddb(&st); \ 130 } \ 131} while (0) 132 133FEATURE(kmsan, "Kernel memory sanitizer"); 134 135static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 136 "KMSAN options"); 137 138static bool panic_on_violation = 1; 139SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN, 140 &panic_on_violation, 0, 141 "Panic if an invalid access is detected"); 142 143static bool kmsan_disabled __read_mostly = true; 144#define kmsan_enabled (!kmsan_disabled) 145SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 146 &kmsan_disabled, 0, "KMSAN is disabled"); 147 148static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer"); 149 150/* -------------------------------------------------------------------------- */ 151 152static inline const char * 153kmsan_orig_name(int type) 154{ 155 switch (type) { 156 case KMSAN_TYPE_STACK: 157 return ("stack"); 158 case KMSAN_TYPE_KMEM: 159 return ("kmem"); 160 case KMSAN_TYPE_MALLOC: 161 return ("malloc"); 162 case KMSAN_TYPE_UMA: 163 return ("UMA"); 164 default: 165 return ("unknown"); 166 } 167} 168 169static void 170kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off, 171 const char *hook) 172{ 173 const char *typename; 174 char *var, *fn; 175 uintptr_t ptr; 176 long foff; 177 char buf[128]; 178 int type; 179 180 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 181 return; 182 183 kmsan_reporting = true; 184 __compiler_membar(); 185 186 if (*orig == 0) { 187 REPORT("MSan: Uninitialized memory in %s, offset %zu", 188 hook, off); 189 goto out; 190 } 191 192 kmsan_md_orig_decode(*orig, &type, &ptr); 193 typename = kmsan_orig_name(type); 194 195 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 196 sizeof(buf), &foff) == 0) { 197 REPORT("MSan: Uninitialized %s memory in %s, " 198 "offset %zu/%zu, addr %p, from %s+%#lx", 199 typename, hook, off, size, addr, buf, foff); 200 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 201 /* 202 * The format of the string is: "----var@function". Parse it to 203 * display a nice warning. 204 */ 205 var = (char *)ptr + 4; 206 strlcpy(buf, var, sizeof(buf)); 207 var = buf; 208 fn = strchr(buf, '@'); 209 *fn++ = '\0'; 210 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, " 211 "variable '%s' from %s", typename, hook, off, var, fn); 212 } else { 213 REPORT("MSan: Uninitialized %s memory in %s, " 214 "offset %zu/%zu, addr %p, PC %p", 215 typename, hook, off, size, addr, (void *)ptr); 216 } 217 218out: 219 __compiler_membar(); 220 kmsan_reporting = false; 221} 222 223static void 224kmsan_report_inline(msan_orig_t orig, unsigned long pc) 225{ 226 const char *typename; 227 char *var, *fn; 228 uintptr_t ptr; 229 char buf[128]; 230 long foff; 231 int type; 232 233 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting)) 234 return; 235 236 kmsan_reporting = true; 237 __compiler_membar(); 238 239 if (orig == 0) { 240 REPORT("MSan: uninitialized variable in %p", (void *)pc); 241 goto out; 242 } 243 244 kmsan_md_orig_decode(orig, &type, &ptr); 245 typename = kmsan_orig_name(type); 246 247 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf, 248 sizeof(buf), &foff) == 0) { 249 REPORT("MSan: Uninitialized %s memory from %s+%#lx", 250 typename, buf, foff); 251 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) { 252 /* 253 * The format of the string is: "----var@function". Parse it to 254 * display a nice warning. 255 */ 256 var = (char *)ptr + 4; 257 strlcpy(buf, var, sizeof(buf)); 258 var = buf; 259 fn = strchr(buf, '@'); 260 *fn++ = '\0'; 261 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn); 262 } else { 263 REPORT("MSan: Uninitialized %s memory, origin %x", 264 typename, orig); 265 } 266 267out: 268 __compiler_membar(); 269 kmsan_reporting = false; 270} 271 272/* -------------------------------------------------------------------------- */ 273 274static inline msan_meta_t 275kmsan_meta_get(const void *addr, size_t size, const bool write) 276{ 277 msan_meta_t ret; 278 279 if (__predict_false(!kmsan_enabled)) { 280 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 281 ret.orig = (msan_orig_t *)msan_dummy_orig; 282 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) { 283 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad; 284 ret.orig = (msan_orig_t *)msan_dummy_orig; 285 } else { 286 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr); 287 ret.orig = 288 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 289 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig & 290 MSAN_ORIG_MASK); 291 } 292 293 return (ret); 294} 295 296static inline void 297kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size) 298{ 299 msan_orig_t *orig; 300 size_t i; 301 302 if (__predict_false(!kmsan_enabled)) 303 return; 304 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) 305 return; 306 307 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr); 308 size += ((uintptr_t)orig & (sizeof(*orig) - 1)); 309 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 310 311 for (i = 0; i < size; i += 4) { 312 orig[i / 4] = o; 313 } 314} 315 316static inline void 317kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size) 318{ 319 uint8_t *shad; 320 321 if (__predict_false(!kmsan_enabled)) 322 return; 323 if (__predict_false(kmsan_md_unsupported(addr))) 324 return; 325 326 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 327 __builtin_memset(shad, c, size); 328} 329 330static inline void 331kmsan_meta_copy(void *dst, const void *src, size_t size) 332{ 333 uint8_t *orig_src, *orig_dst; 334 uint8_t *shad_src, *shad_dst; 335 msan_orig_t *_src, *_dst; 336 size_t i; 337 338 if (__predict_false(!kmsan_enabled)) 339 return; 340 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst))) 341 return; 342 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) { 343 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size); 344 return; 345 } 346 347 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src); 348 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst); 349 __builtin_memmove(shad_dst, shad_src, size); 350 351 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src); 352 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst); 353 for (i = 0; i < size; i++) { 354 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK); 355 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK); 356 *_dst = *_src; 357 orig_src++; 358 orig_dst++; 359 } 360} 361 362static inline void 363kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook) 364{ 365 msan_orig_t *orig; 366 uint8_t *shad; 367 size_t i; 368 369 if (__predict_false(!kmsan_enabled)) 370 return; 371 if (__predict_false(kmsan_md_unsupported(addr))) 372 return; 373 374 shad = (uint8_t *)kmsan_md_addr_to_shad(addr); 375 for (i = 0; i < size; i++) { 376 if (__predict_true(shad[i] == 0)) 377 continue; 378 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i); 379 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK); 380 kmsan_report_hook((const char *)addr + i, orig, size, i, hook); 381 break; 382 } 383} 384 385void 386kmsan_init_arg(size_t n) 387{ 388 msan_td_t *mtd; 389 uint8_t *arg; 390 391 if (__predict_false(!kmsan_enabled)) 392 return; 393 if (__predict_false(curthread == NULL)) 394 return; 395 mtd = curthread->td_kmsan; 396 arg = mtd->tls[mtd->ctx].param_shadow; 397 __builtin_memset(arg, 0, n); 398} 399 400void 401kmsan_init_ret(size_t n) 402{ 403 msan_td_t *mtd; 404 uint8_t *arg; 405 406 if (__predict_false(!kmsan_enabled)) 407 return; 408 if (__predict_false(curthread == NULL)) 409 return; 410 mtd = curthread->td_kmsan; 411 arg = mtd->tls[mtd->ctx].retval_shadow; 412 __builtin_memset(arg, 0, n); 413} 414 415static void 416kmsan_check_arg(size_t size, const char *hook) 417{ 418 msan_orig_t *orig; 419 msan_td_t *mtd; 420 uint8_t *arg; 421 size_t ctx, i; 422 423 if (__predict_false(!kmsan_enabled)) 424 return; 425 if (__predict_false(curthread == NULL)) 426 return; 427 mtd = curthread->td_kmsan; 428 ctx = mtd->ctx; 429 arg = mtd->tls[ctx].param_shadow; 430 431 for (i = 0; i < size; i++) { 432 if (__predict_true(arg[i] == 0)) 433 continue; 434 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)]; 435 kmsan_report_hook((const char *)arg + i, orig, size, i, hook); 436 break; 437 } 438} 439 440void 441kmsan_thread_alloc(struct thread *td) 442{ 443 msan_td_t *mtd; 444 445 if (!kmsan_enabled) 446 return; 447 448 mtd = td->td_kmsan; 449 if (mtd == NULL) { 450 /* We might be recycling a thread. */ 451 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) + 452 sizeof(int)); 453 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK); 454 } 455 __builtin_memset(mtd, 0, sizeof(*mtd)); 456 mtd->ctx = 0; 457 458 if (td->td_kstack != 0) 459 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages), 460 KMSAN_STATE_UNINIT); 461 462 td->td_kmsan = mtd; 463} 464 465void 466kmsan_thread_free(struct thread *td) 467{ 468 msan_td_t *mtd; 469 470 if (!kmsan_enabled) 471 return; 472 if (__predict_false(td == curthread)) 473 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__); 474 475 mtd = td->td_kmsan; 476 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *)); 477 free(mtd, M_KMSAN); 478 td->td_kmsan = NULL; 479} 480 481void kmsan_intr_enter(void); 482void kmsan_intr_leave(void); 483 484void 485kmsan_intr_enter(void) 486{ 487 msan_td_t *mtd; 488 489 if (__predict_false(!kmsan_enabled)) 490 return; 491 492 mtd = curthread->td_kmsan; 493 mtd->ctx++; 494 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT)) 495 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 496} 497 498void 499kmsan_intr_leave(void) 500{ 501 msan_td_t *mtd; 502 503 if (__predict_false(!kmsan_enabled)) 504 return; 505 506 mtd = curthread->td_kmsan; 507 if (__predict_false(mtd->ctx == 0)) 508 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx); 509 mtd->ctx--; 510} 511 512/* -------------------------------------------------------------------------- */ 513 514void 515kmsan_shadow_map(vm_offset_t addr, size_t size) 516{ 517 size_t npages, i; 518 vm_offset_t va; 519 520 MPASS(addr % PAGE_SIZE == 0); 521 MPASS(size % PAGE_SIZE == 0); 522 523 if (!kmsan_enabled) 524 return; 525 526 npages = atop(size); 527 528 va = kmsan_md_addr_to_shad(addr); 529 for (i = 0; i < npages; i++) { 530 pmap_san_enter(va + ptoa(i)); 531 } 532 533 va = kmsan_md_addr_to_orig(addr); 534 for (i = 0; i < npages; i++) { 535 pmap_san_enter(va + ptoa(i)); 536 } 537} 538 539void 540kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc) 541{ 542 msan_orig_t orig; 543 544 orig = kmsan_md_orig_encode(type, pc); 545 kmsan_origin_fill(addr, orig, size); 546} 547 548void 549kmsan_mark(const void *addr, size_t size, uint8_t c) 550{ 551 kmsan_shadow_fill((uintptr_t)addr, c, size); 552} 553 554void 555kmsan_mark_bio(const struct bio *bp, uint8_t c) 556{ 557 kmsan_mark(bp->bio_data, bp->bio_length, c); 558} 559 560void 561kmsan_mark_mbuf(const struct mbuf *m, uint8_t c) 562{ 563 do { 564 if ((m->m_flags & M_EXTPG) == 0) 565 kmsan_mark(m->m_data, m->m_len, c); 566 m = m->m_next; 567 } while (m != NULL); 568} 569 570void 571kmsan_check(const void *p, size_t sz, const char *descr) 572{ 573 kmsan_shadow_check((uintptr_t)p, sz, descr); 574} 575 576void 577kmsan_check_bio(const struct bio *bp, const char *descr) 578{ 579 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr); 580} 581 582void 583kmsan_check_mbuf(const struct mbuf *m, const char *descr) 584{ 585 do { 586 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr); 587 } while ((m = m->m_next) != NULL); 588} 589 590void 591kmsan_check_uio(const struct uio *uio, const char *descr) 592{ 593 for (int i = 0; i < uio->uio_iovcnt; i++) { 594 kmsan_check(uio->uio_iov[i].iov_base, uio->uio_iov[i].iov_len, 595 descr); 596 } 597} 598 599void 600kmsan_init(void) 601{ 602 int disabled; 603 604 disabled = 0; 605 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled); 606 if (disabled) 607 return; 608 609 /* Initialize the TLS for curthread. */ 610 msan_thread0.ctx = 0; 611 thread0.td_kmsan = &msan_thread0; 612 613 /* Now officially enabled. */ 614 kmsan_disabled = false; 615} 616 617/* -------------------------------------------------------------------------- */ 618 619msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t); 620msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t); 621 622msan_meta_t 623__msan_metadata_ptr_for_load_n(void *addr, size_t size) 624{ 625 return (kmsan_meta_get(addr, size, false)); 626} 627 628msan_meta_t 629__msan_metadata_ptr_for_store_n(void *addr, size_t size) 630{ 631 return (kmsan_meta_get(addr, size, true)); 632} 633 634#define MSAN_META_FUNC(size) \ 635 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \ 636 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \ 637 { \ 638 return (kmsan_meta_get(addr, size, false)); \ 639 } \ 640 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \ 641 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \ 642 { \ 643 return (kmsan_meta_get(addr, size, true)); \ 644 } 645 646MSAN_META_FUNC(1) 647MSAN_META_FUNC(2) 648MSAN_META_FUNC(4) 649MSAN_META_FUNC(8) 650 651void __msan_instrument_asm_store(const void *, size_t); 652msan_orig_t __msan_chain_origin(msan_orig_t); 653void __msan_poison(const void *, size_t); 654void __msan_unpoison(const void *, size_t); 655void __msan_poison_alloca(const void *, uint64_t, const char *); 656void __msan_unpoison_alloca(const void *, uint64_t); 657void __msan_warning(msan_orig_t); 658msan_tls_t *__msan_get_context_state(void); 659 660void 661__msan_instrument_asm_store(const void *addr, size_t size) 662{ 663 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 664} 665 666msan_orig_t 667__msan_chain_origin(msan_orig_t origin) 668{ 669 return (origin); 670} 671 672void 673__msan_poison(const void *addr, size_t size) 674{ 675 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 676} 677 678void 679__msan_unpoison(const void *addr, size_t size) 680{ 681 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 682} 683 684void 685__msan_poison_alloca(const void *addr, uint64_t size, const char *descr) 686{ 687 msan_orig_t orig; 688 689 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr); 690 kmsan_origin_fill(addr, orig, size); 691 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size); 692} 693 694void 695__msan_unpoison_alloca(const void *addr, uint64_t size) 696{ 697 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size); 698} 699 700void 701__msan_warning(msan_orig_t origin) 702{ 703 if (__predict_false(!kmsan_enabled)) 704 return; 705 kmsan_report_inline(origin, KMSAN_RET_ADDR); 706} 707 708msan_tls_t * 709__msan_get_context_state(void) 710{ 711 msan_td_t *mtd; 712 713 /* 714 * When APs are started, they execute some C code before curthread is 715 * set. We have to handle that here. 716 */ 717 if (__predict_false(!kmsan_enabled || curthread == NULL)) 718 return (&dummy_tls); 719 mtd = curthread->td_kmsan; 720 return (&mtd->tls[mtd->ctx]); 721} 722 723/* -------------------------------------------------------------------------- */ 724 725/* 726 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle 727 * initialized areas properly. 728 */ 729 730void * 731kmsan_memcpy(void *dst, const void *src, size_t len) 732{ 733 /* No kmsan_check_arg, because inlined. */ 734 kmsan_init_ret(sizeof(void *)); 735 if (__predict_true(len != 0)) { 736 kmsan_meta_copy(dst, src, len); 737 } 738 return (__builtin_memcpy(dst, src, len)); 739} 740 741int 742kmsan_memcmp(const void *b1, const void *b2, size_t len) 743{ 744 const uint8_t *_b1 = b1, *_b2 = b2; 745 size_t i; 746 747 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len), 748 "memcmp():args"); 749 kmsan_init_ret(sizeof(int)); 750 751 for (i = 0; i < len; i++) { 752 if (*_b1 != *_b2) { 753 kmsan_shadow_check((uintptr_t)b1, i + 1, 754 "memcmp():arg1"); 755 kmsan_shadow_check((uintptr_t)b2, i + 1, 756 "memcmp():arg2"); 757 return (*_b1 - *_b2); 758 } 759 _b1++, _b2++; 760 } 761 762 return (0); 763} 764 765void * 766kmsan_memset(void *dst, int c, size_t len) 767{ 768 /* No kmsan_check_arg, because inlined. */ 769 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len); 770 kmsan_init_ret(sizeof(void *)); 771 return (__builtin_memset(dst, c, len)); 772} 773 774void * 775kmsan_memmove(void *dst, const void *src, size_t len) 776{ 777 /* No kmsan_check_arg, because inlined. */ 778 if (__predict_true(len != 0)) { 779 kmsan_meta_copy(dst, src, len); 780 } 781 kmsan_init_ret(sizeof(void *)); 782 return (__builtin_memmove(dst, src, len)); 783} 784 785__strong_reference(kmsan_memcpy, __msan_memcpy); 786__strong_reference(kmsan_memset, __msan_memset); 787__strong_reference(kmsan_memmove, __msan_memmove); 788 789char * 790kmsan_strcpy(char *dst, const char *src) 791{ 792 const char *_src = src; 793 char *_dst = dst; 794 size_t len = 0; 795 796 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args"); 797 798 while (1) { 799 len++; 800 *dst = *src; 801 if (*src == '\0') 802 break; 803 src++, dst++; 804 } 805 806 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2"); 807 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len); 808 kmsan_init_ret(sizeof(char *)); 809 return (_dst); 810} 811 812int 813kmsan_strcmp(const char *s1, const char *s2) 814{ 815 const char *_s1 = s1, *_s2 = s2; 816 size_t len = 0; 817 818 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args"); 819 kmsan_init_ret(sizeof(int)); 820 821 while (1) { 822 len++; 823 if (*s1 != *s2) 824 break; 825 if (*s1 == '\0') { 826 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 827 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 828 return (0); 829 } 830 s1++, s2++; 831 } 832 833 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1"); 834 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2"); 835 836 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 837} 838 839size_t 840kmsan_strlen(const char *str) 841{ 842 const char *s; 843 844 kmsan_check_arg(sizeof(str), "strlen():args"); 845 846 s = str; 847 while (1) { 848 if (*s == '\0') 849 break; 850 s++; 851 } 852 853 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1"); 854 kmsan_init_ret(sizeof(size_t)); 855 return (s - str); 856} 857 858int kmsan_copyin(const void *, void *, size_t); 859int kmsan_copyout(const void *, void *, size_t); 860int kmsan_copyinstr(const void *, void *, size_t, size_t *); 861 862int 863kmsan_copyin(const void *uaddr, void *kaddr, size_t len) 864{ 865 int ret; 866 867 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len), 868 "copyin():args"); 869 ret = copyin(uaddr, kaddr, len); 870 if (ret == 0) 871 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len); 872 kmsan_init_ret(sizeof(int)); 873 return (ret); 874} 875 876int 877kmsan_copyout(const void *kaddr, void *uaddr, size_t len) 878{ 879 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len), 880 "copyout():args"); 881 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1"); 882 kmsan_init_ret(sizeof(int)); 883 return (copyout(kaddr, uaddr, len)); 884} 885 886int 887kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 888{ 889 size_t _done; 890 int ret; 891 892 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + 893 sizeof(len) + sizeof(done), "copyinstr():args"); 894 ret = copyinstr(uaddr, kaddr, len, &_done); 895 if (ret == 0) 896 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done); 897 if (done != NULL) { 898 *done = _done; 899 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t)); 900 } 901 kmsan_init_ret(sizeof(int)); 902 return (ret); 903} 904 905/* -------------------------------------------------------------------------- */ 906 907int 908kmsan_fubyte(volatile const void *base) 909{ 910 int ret; 911 912 kmsan_check_arg(sizeof(base), "fubyte(): args"); 913 ret = fubyte(base); 914 kmsan_init_ret(sizeof(int)); 915 return (ret); 916} 917 918int 919kmsan_fuword16(volatile const void *base) 920{ 921 int ret; 922 923 kmsan_check_arg(sizeof(base), "fuword16(): args"); 924 ret = fuword16(base); 925 kmsan_init_ret(sizeof(int)); 926 return (ret); 927} 928 929int 930kmsan_fueword(volatile const void *base, long *val) 931{ 932 int ret; 933 934 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args"); 935 ret = fueword(base, val); 936 if (ret == 0) 937 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 938 sizeof(*val)); 939 kmsan_init_ret(sizeof(int)); 940 return (ret); 941} 942 943int 944kmsan_fueword32(volatile const void *base, int32_t *val) 945{ 946 int ret; 947 948 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args"); 949 ret = fueword32(base, val); 950 if (ret == 0) 951 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 952 sizeof(*val)); 953 kmsan_init_ret(sizeof(int)); 954 return (ret); 955} 956 957int 958kmsan_fueword64(volatile const void *base, int64_t *val) 959{ 960 int ret; 961 962 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args"); 963 ret = fueword64(base, val); 964 if (ret == 0) 965 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED, 966 sizeof(*val)); 967 kmsan_init_ret(sizeof(int)); 968 return (ret); 969} 970 971int 972kmsan_subyte(volatile void *base, int byte) 973{ 974 int ret; 975 976 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args"); 977 ret = subyte(base, byte); 978 kmsan_init_ret(sizeof(int)); 979 return (ret); 980} 981 982int 983kmsan_suword(volatile void *base, long word) 984{ 985 int ret; 986 987 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args"); 988 ret = suword(base, word); 989 kmsan_init_ret(sizeof(int)); 990 return (ret); 991} 992 993int 994kmsan_suword16(volatile void *base, int word) 995{ 996 int ret; 997 998 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args"); 999 ret = suword16(base, word); 1000 kmsan_init_ret(sizeof(int)); 1001 return (ret); 1002} 1003 1004int 1005kmsan_suword32(volatile void *base, int32_t word) 1006{ 1007 int ret; 1008 1009 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args"); 1010 ret = suword32(base, word); 1011 kmsan_init_ret(sizeof(int)); 1012 return (ret); 1013} 1014 1015int 1016kmsan_suword64(volatile void *base, int64_t word) 1017{ 1018 int ret; 1019 1020 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args"); 1021 ret = suword64(base, word); 1022 kmsan_init_ret(sizeof(int)); 1023 return (ret); 1024} 1025 1026int 1027kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, 1028 uint32_t newval) 1029{ 1030 int ret; 1031 1032 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1033 sizeof(newval), "casueword32(): args"); 1034 ret = casueword32(base, oldval, oldvalp, newval); 1035 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1036 sizeof(*oldvalp)); 1037 kmsan_init_ret(sizeof(int)); 1038 return (ret); 1039} 1040 1041int 1042kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, 1043 u_long newval) 1044{ 1045 int ret; 1046 1047 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) + 1048 sizeof(newval), "casueword32(): args"); 1049 ret = casueword(base, oldval, oldvalp, newval); 1050 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED, 1051 sizeof(*oldvalp)); 1052 kmsan_init_ret(sizeof(int)); 1053 return (ret); 1054} 1055 1056/* -------------------------------------------------------------------------- */ 1057 1058#include <machine/atomic.h> 1059#include <sys/atomic_san.h> 1060 1061#define _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1062 void kmsan_atomic_add_##name(volatile type *ptr, type val) \ 1063 { \ 1064 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1065 "atomic_add_" #name "():args"); \ 1066 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1067 "atomic_add_" #name "():ptr"); \ 1068 atomic_add_##name(ptr, val); \ 1069 } 1070 1071#define MSAN_ATOMIC_FUNC_ADD(name, type) \ 1072 _MSAN_ATOMIC_FUNC_ADD(name, type) \ 1073 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 1074 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type) 1075 1076#define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1077 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \ 1078 { \ 1079 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1080 "atomic_subtract_" #name "():args"); \ 1081 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1082 "atomic_subtract_" #name "():ptr"); \ 1083 atomic_subtract_##name(ptr, val); \ 1084 } 1085 1086#define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1087 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 1088 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 1089 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 1090 1091#define _MSAN_ATOMIC_FUNC_SET(name, type) \ 1092 void kmsan_atomic_set_##name(volatile type *ptr, type val) \ 1093 { \ 1094 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1095 "atomic_set_" #name "():args"); \ 1096 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1097 "atomic_set_" #name "():ptr"); \ 1098 atomic_set_##name(ptr, val); \ 1099 } 1100 1101#define MSAN_ATOMIC_FUNC_SET(name, type) \ 1102 _MSAN_ATOMIC_FUNC_SET(name, type) \ 1103 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 1104 _MSAN_ATOMIC_FUNC_SET(rel_##name, type) 1105 1106#define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1107 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \ 1108 { \ 1109 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1110 "atomic_clear_" #name "():args"); \ 1111 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1112 "atomic_clear_" #name "():ptr"); \ 1113 atomic_clear_##name(ptr, val); \ 1114 } 1115 1116#define MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1117 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \ 1118 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 1119 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 1120 1121#define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 1122 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 1123 { \ 1124 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1125 "atomic_fetchadd_" #name "():args"); \ 1126 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1127 "atomic_fetchadd_" #name "():ptr"); \ 1128 kmsan_init_ret(sizeof(type)); \ 1129 return (atomic_fetchadd_##name(ptr, val)); \ 1130 } 1131 1132#define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 1133 type kmsan_atomic_readandclear_##name(volatile type *ptr) \ 1134 { \ 1135 kmsan_check_arg(sizeof(ptr), \ 1136 "atomic_readandclear_" #name "():args"); \ 1137 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1138 "atomic_readandclear_" #name "():ptr"); \ 1139 kmsan_init_ret(sizeof(type)); \ 1140 return (atomic_readandclear_##name(ptr)); \ 1141 } 1142 1143#define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 1144 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \ 1145 { \ 1146 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1147 "atomic_testandclear_" #name "():args"); \ 1148 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1149 "atomic_testandclear_" #name "():ptr"); \ 1150 kmsan_init_ret(sizeof(int)); \ 1151 return (atomic_testandclear_##name(ptr, v)); \ 1152 } 1153 1154#define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 1155 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \ 1156 { \ 1157 kmsan_check_arg(sizeof(ptr) + sizeof(v), \ 1158 "atomic_testandset_" #name "():args"); \ 1159 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1160 "atomic_testandset_" #name "():ptr"); \ 1161 kmsan_init_ret(sizeof(int)); \ 1162 return (atomic_testandset_##name(ptr, v)); \ 1163 } 1164 1165#define MSAN_ATOMIC_FUNC_SWAP(name, type) \ 1166 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \ 1167 { \ 1168 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1169 "atomic_swap_" #name "():args"); \ 1170 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1171 "atomic_swap_" #name "():ptr"); \ 1172 kmsan_init_ret(sizeof(type)); \ 1173 return (atomic_swap_##name(ptr, val)); \ 1174 } 1175 1176#define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1177 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \ 1178 type nval) \ 1179 { \ 1180 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1181 sizeof(nval), "atomic_cmpset_" #name "():args"); \ 1182 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1183 "atomic_cmpset_" #name "():ptr"); \ 1184 kmsan_init_ret(sizeof(int)); \ 1185 return (atomic_cmpset_##name(ptr, oval, nval)); \ 1186 } 1187 1188#define MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1189 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \ 1190 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 1191 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 1192 1193#define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1194 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \ 1195 type nval) \ 1196 { \ 1197 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \ 1198 sizeof(nval), "atomic_fcmpset_" #name "():args"); \ 1199 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1200 "atomic_fcmpset_" #name "():ptr"); \ 1201 kmsan_init_ret(sizeof(int)); \ 1202 return (atomic_fcmpset_##name(ptr, oval, nval)); \ 1203 } 1204 1205#define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1206 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 1207 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 1208 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 1209 1210#define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 1211 void kmsan_atomic_thread_fence_##name(void) \ 1212 { \ 1213 atomic_thread_fence_##name(); \ 1214 } 1215 1216#define _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1217 type kmsan_atomic_load_##name(volatile type *ptr) \ 1218 { \ 1219 kmsan_check_arg(sizeof(ptr), \ 1220 "atomic_load_" #name "():args"); \ 1221 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \ 1222 "atomic_load_" #name "():ptr"); \ 1223 kmsan_init_ret(sizeof(type)); \ 1224 return (atomic_load_##name(ptr)); \ 1225 } 1226 1227#define MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1228 _MSAN_ATOMIC_FUNC_LOAD(name, type) \ 1229 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type) 1230 1231#define _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1232 void kmsan_atomic_store_##name(volatile type *ptr, type val) \ 1233 { \ 1234 kmsan_check_arg(sizeof(ptr) + sizeof(val), \ 1235 "atomic_store_" #name "():args"); \ 1236 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \ 1237 sizeof(type)); \ 1238 atomic_store_##name(ptr, val); \ 1239 } 1240 1241#define MSAN_ATOMIC_FUNC_STORE(name, type) \ 1242 _MSAN_ATOMIC_FUNC_STORE(name, type) \ 1243 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type) 1244 1245MSAN_ATOMIC_FUNC_ADD(8, uint8_t); 1246MSAN_ATOMIC_FUNC_ADD(16, uint16_t); 1247MSAN_ATOMIC_FUNC_ADD(32, uint32_t); 1248MSAN_ATOMIC_FUNC_ADD(64, uint64_t); 1249MSAN_ATOMIC_FUNC_ADD(int, u_int); 1250MSAN_ATOMIC_FUNC_ADD(long, u_long); 1251MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t); 1252 1253MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t); 1254MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t); 1255MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t); 1256MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t); 1257MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int); 1258MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long); 1259MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t); 1260 1261MSAN_ATOMIC_FUNC_SET(8, uint8_t); 1262MSAN_ATOMIC_FUNC_SET(16, uint16_t); 1263MSAN_ATOMIC_FUNC_SET(32, uint32_t); 1264MSAN_ATOMIC_FUNC_SET(64, uint64_t); 1265MSAN_ATOMIC_FUNC_SET(int, u_int); 1266MSAN_ATOMIC_FUNC_SET(long, u_long); 1267MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t); 1268 1269MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t); 1270MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t); 1271MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t); 1272MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t); 1273MSAN_ATOMIC_FUNC_CLEAR(int, u_int); 1274MSAN_ATOMIC_FUNC_CLEAR(long, u_long); 1275MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t); 1276 1277MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t); 1278MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t); 1279MSAN_ATOMIC_FUNC_FETCHADD(int, u_int); 1280MSAN_ATOMIC_FUNC_FETCHADD(long, u_long); 1281 1282MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t); 1283MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t); 1284MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int); 1285MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long); 1286MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t); 1287 1288MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t); 1289MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t); 1290MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int); 1291MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long); 1292 1293MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t); 1294MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t); 1295MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int); 1296MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long); 1297 1298MSAN_ATOMIC_FUNC_SWAP(32, uint32_t); 1299MSAN_ATOMIC_FUNC_SWAP(64, uint64_t); 1300MSAN_ATOMIC_FUNC_SWAP(int, u_int); 1301MSAN_ATOMIC_FUNC_SWAP(long, u_long); 1302MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t); 1303 1304MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t); 1305MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t); 1306MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t); 1307MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t); 1308MSAN_ATOMIC_FUNC_CMPSET(int, u_int); 1309MSAN_ATOMIC_FUNC_CMPSET(long, u_long); 1310MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t); 1311 1312MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t); 1313MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t); 1314MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t); 1315MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t); 1316MSAN_ATOMIC_FUNC_FCMPSET(int, u_int); 1317MSAN_ATOMIC_FUNC_FCMPSET(long, u_long); 1318MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t); 1319 1320_MSAN_ATOMIC_FUNC_LOAD(bool, bool); 1321MSAN_ATOMIC_FUNC_LOAD(8, uint8_t); 1322MSAN_ATOMIC_FUNC_LOAD(16, uint16_t); 1323MSAN_ATOMIC_FUNC_LOAD(32, uint32_t); 1324MSAN_ATOMIC_FUNC_LOAD(64, uint64_t); 1325MSAN_ATOMIC_FUNC_LOAD(char, u_char); 1326MSAN_ATOMIC_FUNC_LOAD(short, u_short); 1327MSAN_ATOMIC_FUNC_LOAD(int, u_int); 1328MSAN_ATOMIC_FUNC_LOAD(long, u_long); 1329MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t); 1330 1331_MSAN_ATOMIC_FUNC_STORE(bool, bool); 1332MSAN_ATOMIC_FUNC_STORE(8, uint8_t); 1333MSAN_ATOMIC_FUNC_STORE(16, uint16_t); 1334MSAN_ATOMIC_FUNC_STORE(32, uint32_t); 1335MSAN_ATOMIC_FUNC_STORE(64, uint64_t); 1336MSAN_ATOMIC_FUNC_STORE(char, u_char); 1337MSAN_ATOMIC_FUNC_STORE(short, u_short); 1338MSAN_ATOMIC_FUNC_STORE(int, u_int); 1339MSAN_ATOMIC_FUNC_STORE(long, u_long); 1340MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t); 1341 1342MSAN_ATOMIC_FUNC_THREAD_FENCE(acq); 1343MSAN_ATOMIC_FUNC_THREAD_FENCE(rel); 1344MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel); 1345MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst); 1346 1347void 1348kmsan_atomic_interrupt_fence(void) 1349{ 1350 atomic_interrupt_fence(); 1351} 1352 1353/* -------------------------------------------------------------------------- */ 1354 1355#include <sys/bus.h> 1356#include <machine/bus.h> 1357#include <sys/bus_san.h> 1358 1359int 1360kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 1361 int flags, bus_space_handle_t *handlep) 1362{ 1363 return (bus_space_map(tag, hnd, size, flags, handlep)); 1364} 1365 1366void 1367kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 1368 bus_size_t size) 1369{ 1370 bus_space_unmap(tag, hnd, size); 1371} 1372 1373int 1374kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 1375 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 1376{ 1377 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 1378} 1379 1380void 1381kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 1382 bus_size_t size) 1383{ 1384 bus_space_free(tag, hnd, size); 1385} 1386 1387void 1388kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 1389 bus_size_t offset, bus_size_t size, int flags) 1390{ 1391 bus_space_barrier(tag, hnd, offset, size, flags); 1392} 1393 1394#if defined(__amd64__) 1395#define BUS_SPACE_IO(tag) ((tag) == X86_BUS_SPACE_IO) 1396#else 1397#define BUS_SPACE_IO(tag) (false) 1398#endif 1399 1400#define MSAN_BUS_READ_FUNC(func, width, type) \ 1401 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 1402 bus_space_handle_t hnd, bus_size_t offset) \ 1403 { \ 1404 type ret; \ 1405 if (!BUS_SPACE_IO(tag)) \ 1406 kmsan_shadow_fill((uintptr_t)(hnd + offset), \ 1407 KMSAN_STATE_INITED, (width)); \ 1408 ret = bus_space_read##func##_##width(tag, hnd, offset); \ 1409 kmsan_init_ret(sizeof(type)); \ 1410 return (ret); \ 1411 } \ 1412 1413#define MSAN_BUS_READ_PTR_FUNC(func, width, type) \ 1414 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 1415 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 1416 bus_size_t count) \ 1417 { \ 1418 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \ 1419 (width) * count); \ 1420 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 1421 count); \ 1422 } 1423 1424MSAN_BUS_READ_FUNC(, 1, uint8_t) 1425MSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 1426MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 1427MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 1428MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 1429MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 1430 1431MSAN_BUS_READ_FUNC(, 2, uint16_t) 1432MSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 1433MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 1434MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 1435MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 1436MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 1437 1438MSAN_BUS_READ_FUNC(, 4, uint32_t) 1439MSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 1440MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 1441MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 1442MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 1443MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 1444 1445MSAN_BUS_READ_FUNC(, 8, uint64_t) 1446#ifndef __amd64__ 1447MSAN_BUS_READ_FUNC(_stream, 8, uint64_t) 1448MSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) 1449MSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) 1450MSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) 1451MSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) 1452#endif 1453 1454#define MSAN_BUS_WRITE_FUNC(func, width, type) \ 1455 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 1456 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1457 { \ 1458 bus_space_write##func##_##width(tag, hnd, offset, value);\ 1459 } \ 1460 1461#define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 1462 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\ 1463 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 1464 bus_size_t count) \ 1465 { \ 1466 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\ 1467 "bus_space_write()"); \ 1468 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 1469 count); \ 1470 } 1471 1472MSAN_BUS_WRITE_FUNC(, 1, uint8_t) 1473MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 1474MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 1475MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 1476MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 1477MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 1478 1479MSAN_BUS_WRITE_FUNC(, 2, uint16_t) 1480MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 1481MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 1482MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 1483MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 1484MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 1485 1486MSAN_BUS_WRITE_FUNC(, 4, uint32_t) 1487MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 1488MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 1489MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 1490MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 1491MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 1492 1493MSAN_BUS_WRITE_FUNC(, 8, uint64_t) 1494 1495#define MSAN_BUS_SET_FUNC(func, width, type) \ 1496 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 1497 bus_space_handle_t hnd, bus_size_t offset, type value, \ 1498 bus_size_t count) \ 1499 { \ 1500 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 1501 count); \ 1502 } 1503 1504MSAN_BUS_SET_FUNC(multi, 1, uint8_t) 1505MSAN_BUS_SET_FUNC(region, 1, uint8_t) 1506MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 1507MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 1508 1509MSAN_BUS_SET_FUNC(multi, 2, uint16_t) 1510MSAN_BUS_SET_FUNC(region, 2, uint16_t) 1511MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 1512MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 1513 1514MSAN_BUS_SET_FUNC(multi, 4, uint32_t) 1515MSAN_BUS_SET_FUNC(region, 4, uint32_t) 1516MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 1517MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 1518 1519#define MSAN_BUS_PEEK_FUNC(width, type) \ 1520 int kmsan_bus_space_peek_##width(bus_space_tag_t tag, \ 1521 bus_space_handle_t hnd, bus_size_t offset, type *value) \ 1522 { \ 1523 return (bus_space_peek_##width(tag, hnd, offset, value)); \ 1524 } 1525 1526MSAN_BUS_PEEK_FUNC(1, uint8_t) 1527MSAN_BUS_PEEK_FUNC(2, uint16_t) 1528MSAN_BUS_PEEK_FUNC(4, uint32_t) 1529 1530#define MSAN_BUS_POKE_FUNC(width, type) \ 1531 int kmsan_bus_space_poke_##width(bus_space_tag_t tag, \ 1532 bus_space_handle_t hnd, bus_size_t offset, type value) \ 1533 { \ 1534 return (bus_space_poke_##width(tag, hnd, offset, value)); \ 1535 } 1536 1537MSAN_BUS_POKE_FUNC(1, uint8_t) 1538MSAN_BUS_POKE_FUNC(2, uint16_t) 1539MSAN_BUS_POKE_FUNC(4, uint32_t) 1540 1541/* -------------------------------------------------------------------------- */ 1542 1543void 1544kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op) 1545{ 1546 /* 1547 * Some drivers, e.g., nvme, use the same code path for loading device 1548 * read and write requests, and will thus specify both flags. In this 1549 * case we should not do any checking since it will generally lead to 1550 * false positives. 1551 */ 1552 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == 1553 BUS_DMASYNC_PREWRITE) { 1554 switch (desc->md_type) { 1555 case MEMDESC_VADDR: 1556 kmsan_check(desc->u.md_vaddr, desc->md_len, 1557 "dmasync"); 1558 break; 1559 case MEMDESC_MBUF: 1560 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync"); 1561 break; 1562 case 0: 1563 break; 1564 default: 1565 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1566 desc->md_type); 1567 } 1568 } 1569 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1570 switch (desc->md_type) { 1571 case MEMDESC_VADDR: 1572 kmsan_mark(desc->u.md_vaddr, desc->md_len, 1573 KMSAN_STATE_INITED); 1574 break; 1575 case MEMDESC_MBUF: 1576 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED); 1577 break; 1578 case 0: 1579 break; 1580 default: 1581 kmsan_panic("%s: unhandled memdesc type %d", __func__, 1582 desc->md_type); 1583 } 1584 } 1585} 1586