1/* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */ 2 3/* 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * Copyright (c) 2019 Andrew Turner 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Maxime Villard. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33#define SAN_RUNTIME 34 35#include "opt_ddb.h" 36 37#include <sys/param.h> 38#include <sys/kernel.h> 39#include <sys/bus.h> 40#include <sys/conf.h> 41#include <sys/cpu.h> 42#include <sys/csan.h> 43#include <sys/proc.h> 44#include <sys/smp.h> 45#include <sys/systm.h> 46 47#include <ddb/ddb.h> 48#include <ddb/db_sym.h> 49 50#ifdef KCSAN_PANIC 51#define REPORT panic 52#else 53#define REPORT printf 54#endif 55 56typedef struct { 57 uintptr_t addr; 58 uint32_t size; 59 bool write:1; 60 bool atomic:1; 61 uintptr_t pc; 62} csan_cell_t; 63 64typedef struct { 65 bool inited; 66 uint32_t cnt; 67 csan_cell_t cell; 68} csan_cpu_t; 69 70static csan_cpu_t kcsan_cpus[MAXCPU]; 71static bool kcsan_enabled __read_mostly; 72 73#define __RET_ADDR (uintptr_t)__builtin_return_address(0) 74 75#define KCSAN_NACCESSES 1024 76#define KCSAN_DELAY 10 /* 10 microseconds */ 77 78/* -------------------------------------------------------------------------- */ 79 80/* The MD code. */ 81#include <machine/csan.h> 82 83/* -------------------------------------------------------------------------- */ 84 85static void 86kcsan_enable(void *dummy __unused) 87{ 88 89 printf("Enabling KCSCAN, expect reduced performance.\n"); 90 kcsan_enabled = true; 91} 92SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); 93 94void 95kcsan_cpu_init(u_int cpu) 96{ 97 kcsan_cpus[cpu].inited = true; 98} 99 100/* -------------------------------------------------------------------------- */ 101 102static inline void 103kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) 104{ 105 const char *newsym, *oldsym; 106#ifdef DDB 107 c_db_sym_t sym; 108 db_expr_t offset; 109 110 sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); 111 db_symbol_values(sym, &newsym, NULL); 112 113 sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); 114 db_symbol_values(sym, &oldsym, NULL); 115#else 116 newsym = ""; 117 oldsym = ""; 118#endif 119 REPORT("CSan: Racy Access " 120 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " 121 "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", 122 newcpu, 123 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), 124 (void *)new->addr, new->size, (void *)new->pc, newsym, 125 oldcpu, 126 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), 127 (void *)old->addr, old->size, (void *)old->pc, oldsym); 128 kcsan_md_unwind(); 129} 130 131static inline bool 132kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) 133{ 134 if (new->write && !new->atomic) 135 return false; 136 if (old->write && !old->atomic) 137 return false; 138 return true; 139} 140 141static inline void 142kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) 143{ 144 csan_cell_t old, new; 145 csan_cpu_t *cpu; 146 uint64_t intr; 147 size_t i; 148 149 if (__predict_false(!kcsan_enabled)) 150 return; 151 if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr))) 152 return; 153 if (KERNEL_PANICKED()) 154 return; 155 156 new.addr = addr; 157 new.size = size; 158 new.write = write; 159 new.atomic = atomic; 160 new.pc = pc; 161 162 CPU_FOREACH(i) { 163 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); 164 165 if (old.addr + old.size <= new.addr) 166 continue; 167 if (new.addr + new.size <= old.addr) 168 continue; 169 if (__predict_true(!old.write && !new.write)) 170 continue; 171 if (__predict_true(kcsan_access_is_atomic(&new, &old))) 172 continue; 173 174 kcsan_report(&new, PCPU_GET(cpuid), &old, i); 175 break; 176 } 177 178 if (__predict_false(!kcsan_md_is_avail())) 179 return; 180 181 kcsan_md_disable_intrs(&intr); 182 183 cpu = &kcsan_cpus[PCPU_GET(cpuid)]; 184 if (__predict_false(!cpu->inited)) 185 goto out; 186 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; 187 if (__predict_true(cpu->cnt != 0)) 188 goto out; 189 190 __builtin_memcpy(&cpu->cell, &new, sizeof(new)); 191 kcsan_md_delay(KCSAN_DELAY); 192 __builtin_memset(&cpu->cell, 0, sizeof(new)); 193 194out: 195 kcsan_md_enable_intrs(&intr); 196} 197 198#define CSAN_READ(size) \ 199 void __tsan_read##size(uintptr_t); \ 200 void __tsan_read##size(uintptr_t addr) \ 201 { \ 202 kcsan_access(addr, size, false, false, __RET_ADDR); \ 203 } \ 204 void __tsan_unaligned_read##size(uintptr_t); \ 205 void __tsan_unaligned_read##size(uintptr_t addr) \ 206 { \ 207 kcsan_access(addr, size, false, false, __RET_ADDR); \ 208 } 209 210CSAN_READ(1) 211CSAN_READ(2) 212CSAN_READ(4) 213CSAN_READ(8) 214CSAN_READ(16) 215 216#define CSAN_WRITE(size) \ 217 void __tsan_write##size(uintptr_t); \ 218 void __tsan_write##size(uintptr_t addr) \ 219 { \ 220 kcsan_access(addr, size, true, false, __RET_ADDR); \ 221 } \ 222 void __tsan_unaligned_write##size(uintptr_t); \ 223 void __tsan_unaligned_write##size(uintptr_t addr) \ 224 { \ 225 kcsan_access(addr, size, true, false, __RET_ADDR); \ 226 } 227 228CSAN_WRITE(1) 229CSAN_WRITE(2) 230CSAN_WRITE(4) 231CSAN_WRITE(8) 232CSAN_WRITE(16) 233 234void __tsan_read_range(uintptr_t, size_t); 235void __tsan_write_range(uintptr_t, size_t); 236 237void 238__tsan_read_range(uintptr_t addr, size_t size) 239{ 240 kcsan_access(addr, size, false, false, __RET_ADDR); 241} 242 243void 244__tsan_write_range(uintptr_t addr, size_t size) 245{ 246 kcsan_access(addr, size, true, false, __RET_ADDR); 247} 248 249void __tsan_init(void); 250void __tsan_func_entry(void *); 251void __tsan_func_exit(void); 252 253void 254__tsan_init(void) 255{ 256} 257 258void 259__tsan_func_entry(void *call_pc) 260{ 261} 262 263void 264__tsan_func_exit(void) 265{ 266} 267 268/* -------------------------------------------------------------------------- */ 269 270void * 271kcsan_memcpy(void *dst, const void *src, size_t len) 272{ 273 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 274 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 275 return __builtin_memcpy(dst, src, len); 276} 277 278int 279kcsan_memcmp(const void *b1, const void *b2, size_t len) 280{ 281 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); 282 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); 283 return __builtin_memcmp(b1, b2, len); 284} 285 286void * 287kcsan_memset(void *b, int c, size_t len) 288{ 289 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); 290 return __builtin_memset(b, c, len); 291} 292 293void * 294kcsan_memmove(void *dst, const void *src, size_t len) 295{ 296 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 297 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 298 return __builtin_memmove(dst, src, len); 299} 300 301__strong_reference(kcsan_memcpy, __tsan_memcpy); 302__strong_reference(kcsan_memset, __tsan_memset); 303__strong_reference(kcsan_memmove, __tsan_memmove); 304 305char * 306kcsan_strcpy(char *dst, const char *src) 307{ 308 char *save = dst; 309 310 while (1) { 311 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); 312 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); 313 *dst = *src; 314 if (*src == '\0') 315 break; 316 src++, dst++; 317 } 318 319 return save; 320} 321 322int 323kcsan_strcmp(const char *s1, const char *s2) 324{ 325 while (1) { 326 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); 327 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); 328 if (*s1 != *s2) 329 break; 330 if (*s1 == '\0') 331 return 0; 332 s1++, s2++; 333 } 334 335 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 336} 337 338size_t 339kcsan_strlen(const char *str) 340{ 341 const char *s; 342 343 s = str; 344 while (1) { 345 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); 346 if (*s == '\0') 347 break; 348 s++; 349 } 350 351 return (s - str); 352} 353 354int 355kcsan_copyin(const void *uaddr, void *kaddr, size_t len) 356{ 357 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 358 return copyin(uaddr, kaddr, len); 359} 360 361int 362kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 363{ 364 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 365 return copyinstr(uaddr, kaddr, len, done); 366} 367 368int 369kcsan_copyout(const void *kaddr, void *uaddr, size_t len) 370{ 371 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 372 return copyout(kaddr, uaddr, len); 373} 374 375/* -------------------------------------------------------------------------- */ 376 377#include <machine/atomic.h> 378#include <sys/atomic_san.h> 379 380#define _CSAN_ATOMIC_FUNC_ADD(name, type) \ 381 void kcsan_atomic_add_##name(volatile type *ptr, type val) \ 382 { \ 383 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 384 __RET_ADDR); \ 385 atomic_add_##name(ptr, val); \ 386 } 387 388#define CSAN_ATOMIC_FUNC_ADD(name, type) \ 389 _CSAN_ATOMIC_FUNC_ADD(name, type) \ 390 _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ 391 _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) 392 393#define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ 394 void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ 395 { \ 396 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 397 __RET_ADDR); \ 398 atomic_clear_##name(ptr, val); \ 399 } 400 401#define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ 402 _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ 403 _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ 404 _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) 405 406#define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ 407 int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ 408 type val2) \ 409 { \ 410 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 411 __RET_ADDR); \ 412 return (atomic_cmpset_##name(ptr, val1, val2)); \ 413 } 414 415#define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ 416 _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ 417 _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ 418 _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) 419 420#define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 421 int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ 422 type val2) \ 423 { \ 424 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 425 __RET_ADDR); \ 426 return (atomic_fcmpset_##name(ptr, val1, val2)); \ 427 } 428 429#define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 430 _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ 431 _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ 432 _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) 433 434#define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ 435 type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ 436 { \ 437 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 438 __RET_ADDR); \ 439 return (atomic_fetchadd_##name(ptr, val)); \ 440 } 441 442#define _CSAN_ATOMIC_FUNC_LOAD(name, type) \ 443 type kcsan_atomic_load_##name(volatile type *ptr) \ 444 { \ 445 kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \ 446 __RET_ADDR); \ 447 return (atomic_load_##name(ptr)); \ 448 } 449 450#define CSAN_ATOMIC_FUNC_LOAD(name, type) \ 451 _CSAN_ATOMIC_FUNC_LOAD(name, type) \ 452 _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \ 453 454#define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ 455 type kcsan_atomic_readandclear_##name(volatile type *ptr) \ 456 { \ 457 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 458 __RET_ADDR); \ 459 return (atomic_readandclear_##name(ptr)); \ 460 } 461 462#define _CSAN_ATOMIC_FUNC_SET(name, type) \ 463 void kcsan_atomic_set_##name(volatile type *ptr, type val) \ 464 { \ 465 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 466 __RET_ADDR); \ 467 atomic_set_##name(ptr, val); \ 468 } 469 470#define CSAN_ATOMIC_FUNC_SET(name, type) \ 471 _CSAN_ATOMIC_FUNC_SET(name, type) \ 472 _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \ 473 _CSAN_ATOMIC_FUNC_SET(rel_##name, type) 474 475#define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 476 void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \ 477 { \ 478 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 479 __RET_ADDR); \ 480 atomic_subtract_##name(ptr, val); \ 481 } 482 483#define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 484 _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ 485 _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ 486 _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) 487 488#define _CSAN_ATOMIC_FUNC_STORE(name, type) \ 489 void kcsan_atomic_store_##name(volatile type *ptr, type val) \ 490 { \ 491 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 492 __RET_ADDR); \ 493 atomic_store_##name(ptr, val); \ 494 } 495 496#define CSAN_ATOMIC_FUNC_STORE(name, type) \ 497 _CSAN_ATOMIC_FUNC_STORE(name, type) \ 498 _CSAN_ATOMIC_FUNC_STORE(rel_##name, type) 499 500#define CSAN_ATOMIC_FUNC_SWAP(name, type) \ 501 type kcsan_atomic_swap_##name(volatile type *ptr, type val) \ 502 { \ 503 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 504 __RET_ADDR); \ 505 return(atomic_swap_##name(ptr, val)); \ 506 } 507 508#define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ 509 int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \ 510 { \ 511 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 512 __RET_ADDR); \ 513 return(atomic_testandclear_##name(ptr, val)); \ 514 } 515 516#define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ 517 int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \ 518 { \ 519 kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ 520 __RET_ADDR); \ 521 return (atomic_testandset_##name(ptr, val)); \ 522 } 523 524_CSAN_ATOMIC_FUNC_LOAD(bool, bool) 525_CSAN_ATOMIC_FUNC_STORE(bool, bool) 526 527CSAN_ATOMIC_FUNC_ADD(8, uint8_t) 528CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) 529CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) 530CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) 531CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) 532CSAN_ATOMIC_FUNC_SET(8, uint8_t) 533CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) 534_CSAN_ATOMIC_FUNC_STORE(8, uint8_t) 535#if 0 536CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t) 537CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t) 538CSAN_ATOMIC_FUNC_SWAP(8, uint8_t) 539CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t) 540CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t) 541#endif 542 543CSAN_ATOMIC_FUNC_ADD(16, uint16_t) 544CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) 545CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) 546CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) 547CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) 548CSAN_ATOMIC_FUNC_SET(16, uint16_t) 549CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) 550_CSAN_ATOMIC_FUNC_STORE(16, uint16_t) 551#if 0 552CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t) 553CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t) 554CSAN_ATOMIC_FUNC_SWAP(16, uint16_t) 555CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t) 556CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t) 557#endif 558 559CSAN_ATOMIC_FUNC_ADD(32, uint32_t) 560CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t) 561CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t) 562CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t) 563CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t) 564CSAN_ATOMIC_FUNC_LOAD(32, uint32_t) 565CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t) 566CSAN_ATOMIC_FUNC_SET(32, uint32_t) 567CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t) 568CSAN_ATOMIC_FUNC_STORE(32, uint32_t) 569CSAN_ATOMIC_FUNC_SWAP(32, uint32_t) 570CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t) 571CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t) 572 573CSAN_ATOMIC_FUNC_ADD(64, uint64_t) 574CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t) 575CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t) 576CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t) 577CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t) 578CSAN_ATOMIC_FUNC_LOAD(64, uint64_t) 579CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t) 580CSAN_ATOMIC_FUNC_SET(64, uint64_t) 581CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t) 582CSAN_ATOMIC_FUNC_STORE(64, uint64_t) 583CSAN_ATOMIC_FUNC_SWAP(64, uint64_t) 584CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t) 585CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t) 586 587CSAN_ATOMIC_FUNC_ADD(char, uint8_t) 588CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t) 589CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t) 590CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t) 591CSAN_ATOMIC_FUNC_LOAD(char, uint8_t) 592CSAN_ATOMIC_FUNC_SET(char, uint8_t) 593CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t) 594_CSAN_ATOMIC_FUNC_STORE(char, uint8_t) 595#if 0 596CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t) 597CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t) 598CSAN_ATOMIC_FUNC_SWAP(char, uint8_t) 599CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t) 600CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t) 601#endif 602 603CSAN_ATOMIC_FUNC_ADD(short, uint16_t) 604CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t) 605CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t) 606CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t) 607CSAN_ATOMIC_FUNC_LOAD(short, uint16_t) 608CSAN_ATOMIC_FUNC_SET(short, uint16_t) 609CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t) 610_CSAN_ATOMIC_FUNC_STORE(short, uint16_t) 611#if 0 612CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t) 613CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t) 614CSAN_ATOMIC_FUNC_SWAP(short, uint16_t) 615CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t) 616CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t) 617#endif 618 619CSAN_ATOMIC_FUNC_ADD(int, u_int) 620CSAN_ATOMIC_FUNC_CLEAR(int, u_int) 621CSAN_ATOMIC_FUNC_CMPSET(int, u_int) 622CSAN_ATOMIC_FUNC_FCMPSET(int, u_int) 623CSAN_ATOMIC_FUNC_FETCHADD(int, u_int) 624CSAN_ATOMIC_FUNC_LOAD(int, u_int) 625CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int) 626CSAN_ATOMIC_FUNC_SET(int, u_int) 627CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int) 628CSAN_ATOMIC_FUNC_STORE(int, u_int) 629CSAN_ATOMIC_FUNC_SWAP(int, u_int) 630CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int) 631CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int) 632 633CSAN_ATOMIC_FUNC_ADD(long, u_long) 634CSAN_ATOMIC_FUNC_CLEAR(long, u_long) 635CSAN_ATOMIC_FUNC_CMPSET(long, u_long) 636CSAN_ATOMIC_FUNC_FCMPSET(long, u_long) 637CSAN_ATOMIC_FUNC_FETCHADD(long, u_long) 638CSAN_ATOMIC_FUNC_LOAD(long, u_long) 639CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long) 640CSAN_ATOMIC_FUNC_SET(long, u_long) 641CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long) 642CSAN_ATOMIC_FUNC_STORE(long, u_long) 643CSAN_ATOMIC_FUNC_SWAP(long, u_long) 644CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) 645CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) 646CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long) 647 648CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) 649CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t) 650CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t) 651CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t) 652#if !defined(__amd64__) 653CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t) 654#endif 655CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t) 656CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t) 657CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t) 658CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t) 659CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t) 660CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t) 661#if 0 662CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t) 663CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t) 664#endif 665 666#define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ 667 void kcsan_atomic_thread_fence_##name(void) \ 668 { \ 669 atomic_thread_fence_##name(); \ 670 } 671 672CSAN_ATOMIC_FUNC_THREAD_FENCE(acq) 673CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel) 674CSAN_ATOMIC_FUNC_THREAD_FENCE(rel) 675CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst) 676 677void 678kcsan_atomic_interrupt_fence(void) 679{ 680 atomic_interrupt_fence(); 681} 682 683/* -------------------------------------------------------------------------- */ 684 685#include <sys/bus.h> 686#include <machine/bus.h> 687#include <sys/bus_san.h> 688 689int 690kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, 691 int flags, bus_space_handle_t *handlep) 692{ 693 694 return (bus_space_map(tag, hnd, size, flags, handlep)); 695} 696 697void 698kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, 699 bus_size_t size) 700{ 701 702 bus_space_unmap(tag, hnd, size); 703} 704 705int 706kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, 707 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) 708{ 709 710 return (bus_space_subregion(tag, hnd, offset, size, handlep)); 711} 712 713#if !defined(__amd64__) 714int 715kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, 716 bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, 717 bus_size_t boundary, int flags, bus_addr_t *addrp, 718 bus_space_handle_t *handlep) 719{ 720 721 return (bus_space_alloc(tag, reg_start, reg_end, size, alignment, 722 boundary, flags, addrp, handlep)); 723} 724#endif 725 726void 727kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, 728 bus_size_t size) 729{ 730 731 bus_space_free(tag, hnd, size); 732} 733 734void 735kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, 736 bus_size_t offset, bus_size_t size, int flags) 737{ 738 739 bus_space_barrier(tag, hnd, offset, size, flags); 740} 741 742#define CSAN_BUS_READ_FUNC(func, width, type) \ 743 type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ 744 bus_space_handle_t hnd, bus_size_t offset) \ 745 { \ 746 return (bus_space_read##func##_##width(tag, hnd, \ 747 offset)); \ 748 } \ 749 750#define CSAN_BUS_READ_PTR_FUNC(func, width, type) \ 751 void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ 752 bus_space_handle_t hnd, bus_size_t size, type *buf, \ 753 bus_size_t count) \ 754 { \ 755 kcsan_access((uintptr_t)buf, sizeof(type) * count, \ 756 false, false, __RET_ADDR); \ 757 bus_space_read_##func##_##width(tag, hnd, size, buf, \ 758 count); \ 759 } 760 761CSAN_BUS_READ_FUNC(, 1, uint8_t) 762CSAN_BUS_READ_FUNC(_stream, 1, uint8_t) 763CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) 764CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) 765CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) 766CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) 767 768CSAN_BUS_READ_FUNC(, 2, uint16_t) 769CSAN_BUS_READ_FUNC(_stream, 2, uint16_t) 770CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) 771CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) 772CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) 773CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) 774 775CSAN_BUS_READ_FUNC(, 4, uint32_t) 776CSAN_BUS_READ_FUNC(_stream, 4, uint32_t) 777CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) 778CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) 779CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) 780CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) 781 782CSAN_BUS_READ_FUNC(, 8, uint64_t) 783#if defined(__aarch64__) 784CSAN_BUS_READ_FUNC(_stream, 8, uint64_t) 785CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) 786CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) 787CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) 788CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) 789#endif 790 791#define CSAN_BUS_WRITE_FUNC(func, width, type) \ 792 void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ 793 bus_space_handle_t hnd, bus_size_t offset, type value) \ 794 { \ 795 bus_space_write##func##_##width(tag, hnd, offset, value); \ 796 } \ 797 798#define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ 799 void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \ 800 bus_space_handle_t hnd, bus_size_t size, const type *buf, \ 801 bus_size_t count) \ 802 { \ 803 kcsan_access((uintptr_t)buf, sizeof(type) * count, \ 804 true, false, __RET_ADDR); \ 805 bus_space_write_##func##_##width(tag, hnd, size, buf, \ 806 count); \ 807 } 808 809CSAN_BUS_WRITE_FUNC(, 1, uint8_t) 810CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) 811CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) 812CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) 813CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) 814CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) 815 816CSAN_BUS_WRITE_FUNC(, 2, uint16_t) 817CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) 818CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) 819CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) 820CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) 821CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) 822 823CSAN_BUS_WRITE_FUNC(, 4, uint32_t) 824CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) 825CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) 826CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) 827CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) 828CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) 829 830CSAN_BUS_WRITE_FUNC(, 8, uint64_t) 831#if defined(__aarch64__) 832CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t) 833CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t) 834CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t) 835CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t) 836CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t) 837#endif 838 839#define CSAN_BUS_SET_FUNC(func, width, type) \ 840 void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ 841 bus_space_handle_t hnd, bus_size_t offset, type value, \ 842 bus_size_t count) \ 843 { \ 844 bus_space_set_##func##_##width(tag, hnd, offset, value, \ 845 count); \ 846 } 847 848CSAN_BUS_SET_FUNC(multi, 1, uint8_t) 849CSAN_BUS_SET_FUNC(region, 1, uint8_t) 850#if !defined(__aarch64__) 851CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) 852CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) 853#endif 854 855CSAN_BUS_SET_FUNC(multi, 2, uint16_t) 856CSAN_BUS_SET_FUNC(region, 2, uint16_t) 857#if !defined(__aarch64__) 858CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) 859CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) 860#endif 861 862CSAN_BUS_SET_FUNC(multi, 4, uint32_t) 863CSAN_BUS_SET_FUNC(region, 4, uint32_t) 864#if !defined(__aarch64__) 865CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) 866CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) 867#endif 868 869#if !defined(__amd64__) 870CSAN_BUS_SET_FUNC(multi, 8, uint64_t) 871CSAN_BUS_SET_FUNC(region, 8, uint64_t) 872#if !defined(__aarch64__) 873CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t) 874CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t) 875#endif 876#endif 877 878#define CSAN_BUS_PEEK_FUNC(width, type) \ 879 int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \ 880 bus_space_handle_t hnd, bus_size_t offset, type *value) \ 881 { \ 882 kcsan_access((uintptr_t)value, sizeof(type), true, false, \ 883 __RET_ADDR); \ 884 return (bus_space_peek_##width(tag, hnd, offset, value)); \ 885 } 886 887CSAN_BUS_PEEK_FUNC(1, uint8_t) 888CSAN_BUS_PEEK_FUNC(2, uint16_t) 889CSAN_BUS_PEEK_FUNC(4, uint32_t) 890#if !defined(__i386__) 891CSAN_BUS_PEEK_FUNC(8, uint64_t) 892#endif 893 894#define CSAN_BUS_POKE_FUNC(width, type) \ 895 int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \ 896 bus_space_handle_t hnd, bus_size_t offset, type value) \ 897 { \ 898 return (bus_space_poke_##width(tag, hnd, offset, value)); \ 899 } 900 901CSAN_BUS_POKE_FUNC(1, uint8_t) 902CSAN_BUS_POKE_FUNC(2, uint16_t) 903CSAN_BUS_POKE_FUNC(4, uint32_t) 904#if !defined(__i386__) 905CSAN_BUS_POKE_FUNC(8, uint64_t) 906#endif 907