1198934Sdelphij/*- 2198934Sdelphij * SPDX-License-Identifier: BSD-3-Clause 3198934Sdelphij * 4198934Sdelphij * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. 5198934Sdelphij * All rights reserved. 6198934Sdelphij * 7198934Sdelphij * Redistribution and use in source and binary forms, with or without 8198934Sdelphij * modification, are permitted provided that the following conditions 9198934Sdelphij * are met: 10198934Sdelphij * 11198934Sdelphij * * Redistributions of source code must retain the above copyright 12198934Sdelphij * notice, this list of conditions and the following disclaimer. 13198934Sdelphij * * Redistributions in binary form must reproduce the above copyright 14198934Sdelphij * notice, this list of conditions and the following disclaimer in 15198934Sdelphij * the documentation and/or other materials provided with the 16198934Sdelphij * distribution. 17198934Sdelphij * * Neither the name of copyright holder nor the names of its 18198934Sdelphij * contributors may be used to endorse or promote products derived 19198934Sdelphij * from this software without specific prior written permission. 20198934Sdelphij * 21198934Sdelphij * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22198934Sdelphij * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23198934Sdelphij * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24198934Sdelphij * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25198934Sdelphij * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26198934Sdelphij * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27198934Sdelphij * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28198934Sdelphij * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29198934Sdelphij * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30198934Sdelphij * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31198934Sdelphij * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32198934Sdelphij */ 33198934Sdelphij 34198934Sdelphij#ifndef ENA_PLAT_H_ 35292121Sbapt#define ENA_PLAT_H_ 36292121Sbapt 37292121Sbapt#include <sys/cdefs.h> 38__FBSDID("$FreeBSD$"); 39 40#include <sys/param.h> 41#include <sys/systm.h> 42 43#include <sys/bus.h> 44#include <sys/condvar.h> 45#include <sys/domainset.h> 46#include <sys/endian.h> 47#include <sys/kernel.h> 48#include <sys/kthread.h> 49#include <sys/malloc.h> 50#include <sys/mbuf.h> 51#include <sys/module.h> 52#include <sys/rman.h> 53#include <sys/proc.h> 54#include <sys/smp.h> 55#include <sys/socket.h> 56#include <sys/sockio.h> 57#include <sys/sysctl.h> 58#include <sys/taskqueue.h> 59#include <sys/eventhandler.h> 60#include <sys/types.h> 61#include <sys/timetc.h> 62#include <sys/cdefs.h> 63 64#include <machine/atomic.h> 65#include <machine/bus.h> 66#include <machine/in_cksum.h> 67#include <machine/pcpu.h> 68#include <machine/resource.h> 69#include <machine/_inttypes.h> 70 71#include <net/bpf.h> 72#include <net/ethernet.h> 73#include <net/if.h> 74#include <net/if_var.h> 75#include <net/if_arp.h> 76#include <net/if_dl.h> 77#include <net/if_media.h> 78 79#include <net/if_types.h> 80#include <net/if_vlan_var.h> 81 82#include <netinet/in_systm.h> 83#include <netinet/in.h> 84#include <netinet/if_ether.h> 85#include <netinet/ip.h> 86#include <netinet/ip6.h> 87#include <netinet/tcp.h> 88#include <netinet/tcp_lro.h> 89#include <netinet/udp.h> 90 91#include <dev/led/led.h> 92#include <dev/pci/pcivar.h> 93#include <dev/pci/pcireg.h> 94 95enum ena_log_t { 96 ENA_ERR = 0, 97 ENA_WARN, 98 ENA_INFO, 99 ENA_DBG, 100}; 101 102extern int ena_log_level; 103 104#define ena_log(dev, level, fmt, args...) \ 105 do { \ 106 if (ENA_ ## level <= ena_log_level) \ 107 device_printf((dev), fmt, ##args); \ 108 } while (0) 109 110#define ena_log_raw(level, fmt, args...) \ 111 do { \ 112 if (ENA_ ## level <= ena_log_level) \ 113 printf(fmt, ##args); \ 114 } while (0) 115 116#define ena_log_unused(dev, level, fmt, args...) \ 117 do { \ 118 (void)(dev); \ 119 } while (0) 120 121#ifdef ENA_LOG_IO_ENABLE 122#define ena_log_io(dev, level, fmt, args...) \ 123 ena_log((dev), level, fmt, ##args) 124#else 125#define ena_log_io(dev, level, fmt, args...) \ 126 ena_log_unused((dev), level, fmt, ##args) 127#endif 128 129#define ena_log_nm(dev, level, fmt, args...) \ 130 ena_log((dev), level, "[nm] " fmt, ##args) 131 132extern struct ena_bus_space ebs; 133 134#define DEFAULT_ALLOC_ALIGNMENT 8 135#define ENA_CDESC_RING_SIZE_ALIGNMENT (1 << 12) /* 4K */ 136 137#define container_of(ptr, type, member) \ 138 ({ \ 139 const __typeof(((type *)0)->member) *__p = (ptr); \ 140 (type *)((uintptr_t)__p - offsetof(type, member)); \ 141 }) 142 143#define ena_trace(ctx, level, fmt, args...) \ 144 ena_log((ctx)->dmadev, level, "%s() [TID:%d]: " \ 145 fmt, __func__, curthread->td_tid, ##args) 146 147#define ena_trc_dbg(ctx, format, arg...) \ 148 ena_trace(ctx, DBG, format, ##arg) 149#define ena_trc_info(ctx, format, arg...) \ 150 ena_trace(ctx, INFO, format, ##arg) 151#define ena_trc_warn(ctx, format, arg...) \ 152 ena_trace(ctx, WARN, format, ##arg) 153#define ena_trc_err(ctx, format, arg...) \ 154 ena_trace(ctx, ERR, format, ##arg) 155 156#define unlikely(x) __predict_false(!!(x)) 157#define likely(x) __predict_true(!!(x)) 158 159#define __iomem 160#define ____cacheline_aligned __aligned(CACHE_LINE_SIZE) 161 162#define MAX_ERRNO 4095 163#define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO) 164 165#define ENA_WARN(cond, ctx, format, arg...) \ 166 do { \ 167 if (unlikely((cond))) { \ 168 ena_trc_warn(ctx, format, ##arg); \ 169 } \ 170 } while (0) 171 172static inline long IS_ERR(const void *ptr) 173{ 174 return IS_ERR_VALUE((unsigned long)ptr); 175} 176 177static inline void *ERR_PTR(long error) 178{ 179 return (void *)error; 180} 181 182static inline long PTR_ERR(const void *ptr) 183{ 184 return (long) ptr; 185} 186 187#define GENMASK(h, l) (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h)))) 188#define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h)))) 189#define BIT(x) (1UL << (x)) 190#define BIT64(x) BIT(x) 191#define ENA_ABORT() BUG() 192#define BUG() panic("ENA BUG") 193 194#define SZ_256 (256) 195#define SZ_4K (4096) 196 197#define ENA_COM_OK 0 198#define ENA_COM_FAULT EFAULT 199#define ENA_COM_INVAL EINVAL 200#define ENA_COM_NO_MEM ENOMEM 201#define ENA_COM_NO_SPACE ENOSPC 202#define ENA_COM_TRY_AGAIN -1 203#define ENA_COM_UNSUPPORTED EOPNOTSUPP 204#define ENA_COM_NO_DEVICE ENODEV 205#define ENA_COM_PERMISSION EPERM 206#define ENA_COM_TIMER_EXPIRED ETIMEDOUT 207#define ENA_COM_EIO EIO 208#define ENA_COM_DEVICE_BUSY EBUSY 209 210#define ENA_NODE_ANY (-1) 211 212#define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0) 213#define ENA_USLEEP(x) pause_sbt("ena", SBT_1US * (x), SBT_1US, 0) 214#define ENA_UDELAY(x) DELAY(x) 215#define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ 216 ((long)cputick2usec(cpu_ticks()) + (timeout_us)) 217#define ENA_TIME_EXPIRE(timeout) ((timeout) < cputick2usec(cpu_ticks())) 218#define ENA_TIME_EXPIRE_HIGH_RES ENA_TIME_EXPIRE 219#define ENA_TIME_INIT_HIGH_RES() (0) 220#define ENA_TIME_COMPARE_HIGH_RES(time1, time2) \ 221 ((time1 < time2) ? -1 : ((time1 > time2) ? 1 : 0)) 222#define ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(current_time, timeout_us) \ 223 ((long)cputick2usec(cpu_ticks()) + (timeout_us)) 224#define ENA_GET_SYSTEM_TIME_HIGH_RES() ENA_GET_SYSTEM_TIMEOUT(0) 225#define ENA_MIGHT_SLEEP() 226 227#define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y)) 228#define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y)) 229 230#define ENA_MIN32(x,y) MIN(x, y) 231#define ENA_MIN16(x,y) MIN(x, y) 232#define ENA_MIN8(x,y) MIN(x, y) 233 234#define ENA_MAX32(x,y) MAX(x, y) 235#define ENA_MAX16(x,y) MAX(x, y) 236#define ENA_MAX8(x,y) MAX(x, y) 237 238/* Spinlock related methods */ 239#define ena_spinlock_t struct mtx 240#define ENA_SPINLOCK_INIT(spinlock) \ 241 mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN) 242#define ENA_SPINLOCK_DESTROY(spinlock) \ 243 do { \ 244 if (mtx_initialized(&(spinlock))) \ 245 mtx_destroy(&(spinlock)); \ 246 } while (0) 247#define ENA_SPINLOCK_LOCK(spinlock, flags) \ 248 do { \ 249 (void)(flags); \ 250 mtx_lock_spin(&(spinlock)); \ 251 } while (0) 252#define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ 253 do { \ 254 (void)(flags); \ 255 mtx_unlock_spin(&(spinlock)); \ 256 } while (0) 257 258 259/* Wait queue related methods */ 260#define ena_wait_event_t struct { struct cv wq; struct mtx mtx; } 261#define ENA_WAIT_EVENT_INIT(waitqueue) \ 262 do { \ 263 cv_init(&((waitqueue).wq), "cv"); \ 264 mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \ 265 } while (0) 266#define ENA_WAIT_EVENTS_DESTROY(admin_queue) \ 267 do { \ 268 struct ena_comp_ctx *comp_ctx; \ 269 int i; \ 270 for (i = 0; i < admin_queue->q_depth; i++) { \ 271 comp_ctx = get_comp_ctxt(admin_queue, i, false); \ 272 if (comp_ctx != NULL) { \ 273 cv_destroy(&((comp_ctx->wait_event).wq)); \ 274 mtx_destroy(&((comp_ctx->wait_event).mtx)); \ 275 } \ 276 } \ 277 } while (0) 278#define ENA_WAIT_EVENT_CLEAR(waitqueue) \ 279 cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description) 280#define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \ 281 do { \ 282 mtx_lock(&((waitqueue).mtx)); \ 283 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \ 284 timeout_us * hz / 1000 / 1000 ); \ 285 mtx_unlock(&((waitqueue).mtx)); \ 286 } while (0) 287#define ENA_WAIT_EVENT_SIGNAL(waitqueue) \ 288 do { \ 289 mtx_lock(&((waitqueue).mtx)); \ 290 cv_broadcast(&((waitqueue).wq)); \ 291 mtx_unlock(&((waitqueue).mtx)); \ 292 } while (0) 293 294#define dma_addr_t bus_addr_t 295#define u8 uint8_t 296#define u16 uint16_t 297#define u32 uint32_t 298#define u64 uint64_t 299 300typedef struct { 301 bus_addr_t paddr; 302 caddr_t vaddr; 303 bus_dma_tag_t tag; 304 bus_dmamap_t map; 305 bus_dma_segment_t seg; 306 int nseg; 307} ena_mem_handle_t; 308 309struct ena_bus { 310 bus_space_handle_t reg_bar_h; 311 bus_space_tag_t reg_bar_t; 312 bus_space_handle_t mem_bar_h; 313 bus_space_tag_t mem_bar_t; 314}; 315 316typedef uint32_t ena_atomic32_t; 317 318#define ENA_PRIu64 PRIu64 319 320typedef uint64_t ena_time_t; 321typedef uint64_t ena_time_high_res_t; 322typedef struct ifnet ena_netdev; 323 324void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, 325 int error); 326int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma, 327 int mapflags, bus_size_t alignment, int domain); 328 329static inline uint32_t 330ena_reg_read32(struct ena_bus *bus, bus_size_t offset) 331{ 332 uint32_t v = bus_space_read_4(bus->reg_bar_t, bus->reg_bar_h, offset); 333 rmb(); 334 return v; 335} 336 337#define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \ 338 do { \ 339 int count, i; \ 340 volatile uint64_t *to = (volatile uint64_t *)(dst); \ 341 const uint64_t *from = (const uint64_t *)(src); \ 342 count = (size) / 8; \ 343 \ 344 for (i = 0; i < count; i++, from++, to++) \ 345 *to = *from; \ 346 } while (0) 347 348#define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO) 349 350#define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \ 351 do { \ 352 (virt) = malloc_domainset((size), M_DEVBUF, \ 353 (node) < 0 ? DOMAINSET_RR() : DOMAINSET_PREF(node), \ 354 M_NOWAIT | M_ZERO); \ 355 (void)(dev_node); \ 356 } while (0) 357 358#define ENA_MEM_FREE(dmadev, ptr, size) \ 359 do { \ 360 (void)(size); \ 361 free(ptr, M_DEVBUF); \ 362 } while (0) 363#define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys, \ 364 dma, node, dev_node, alignment) \ 365 do { \ 366 ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment), \ 367 (node)); \ 368 (virt) = (void *)(dma).vaddr; \ 369 (phys) = (dma).paddr; \ 370 (void)(dev_node); \ 371 } while (0) 372 373#define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, \ 374 node, dev_node) \ 375 ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, \ 376 phys, handle, node, dev_node, DEFAULT_ALLOC_ALIGNMENT) 377 378#define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma, \ 379 alignment) \ 380 do { \ 381 ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment), \ 382 ENA_NODE_ANY); \ 383 (virt) = (void *)(dma).vaddr; \ 384 (phys) = (dma).paddr; \ 385 } while (0) 386 387#define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \ 388 ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, \ 389 phys, dma, DEFAULT_ALLOC_ALIGNMENT) 390 391#define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \ 392 do { \ 393 (void)size; \ 394 bus_dmamap_unload((dma).tag, (dma).map); \ 395 bus_dmamem_free((dma).tag, (virt), (dma).map); \ 396 bus_dma_tag_destroy((dma).tag); \ 397 (dma).tag = NULL; \ 398 (virt) = NULL; \ 399 } while (0) 400 401/* Register R/W methods */ 402#define ENA_REG_WRITE32(bus, value, offset) \ 403 do { \ 404 wmb(); \ 405 ENA_REG_WRITE32_RELAXED(bus, value, offset); \ 406 } while (0) 407 408#define ENA_REG_WRITE32_RELAXED(bus, value, offset) \ 409 bus_space_write_4( \ 410 ((struct ena_bus*)bus)->reg_bar_t, \ 411 ((struct ena_bus*)bus)->reg_bar_h, \ 412 (bus_size_t)(offset), (value)) 413 414#define ENA_REG_READ32(bus, offset) \ 415 ena_reg_read32((struct ena_bus*)(bus), (bus_size_t)(offset)) 416 417#define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync( \ 418 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE) 419#define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync( \ 420 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD) 421#define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync( \ 422 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD) 423#define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle) 424 425#define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0) 426 427#define VLAN_HLEN sizeof(struct ether_vlan_header) 428 429#define prefetch(x) (void)(x) 430#define prefetchw(x) (void)(x) 431 432/* DMA buffers access */ 433#define dma_unmap_addr(p, name) ((p)->dma->name) 434#define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v)) 435#define dma_unmap_len(p, name) ((p)->name) 436#define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) 437 438#define memcpy_toio memcpy 439 440#define ATOMIC32_INC(I32_PTR) atomic_add_int(I32_PTR, 1) 441#define ATOMIC32_DEC(I32_PTR) atomic_add_int(I32_PTR, -1) 442#define ATOMIC32_READ(I32_PTR) atomic_load_acq_int(I32_PTR) 443#define ATOMIC32_SET(I32_PTR, VAL) atomic_store_rel_int(I32_PTR, VAL) 444 445#define barrier() __asm__ __volatile__("": : :"memory") 446#define dma_rmb() barrier() 447#define mmiowb() barrier() 448 449#define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) 450#define READ_ONCE(x) ({ \ 451 __typeof(x) __var; \ 452 barrier(); \ 453 __var = ACCESS_ONCE(x); \ 454 barrier(); \ 455 __var; \ 456 }) 457#define READ_ONCE8(x) READ_ONCE(x) 458#define READ_ONCE16(x) READ_ONCE(x) 459#define READ_ONCE32(x) READ_ONCE(x) 460 461#define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16)) 462#define lower_32_bits(n) ((uint32_t)(n)) 463 464#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 465 466#define ENA_FFS(x) ffs(x) 467 468void ena_rss_key_fill(void *key, size_t size); 469 470#define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size) 471 472#include "ena_defs/ena_includes.h" 473 474#define ENA_BITS_PER_U64(bitmap) (bitcount64(bitmap)) 475 476#endif /* ENA_PLAT_H_ */ 477