1/* SPDX-License-Identifier: GPL-2.0 */ 2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3#ifndef _IDXD_H_ 4#define _IDXD_H_ 5 6#include <linux/sbitmap.h> 7#include <linux/dmaengine.h> 8#include <linux/percpu-rwsem.h> 9#include <linux/wait.h> 10#include <linux/cdev.h> 11#include <linux/idr.h> 12#include <linux/pci.h> 13#include <linux/bitmap.h> 14#include <linux/perf_event.h> 15#include <linux/iommu.h> 16#include <linux/crypto.h> 17#include <uapi/linux/idxd.h> 18#include "registers.h" 19 20#define IDXD_DRIVER_VERSION "1.00" 21 22extern struct kmem_cache *idxd_desc_pool; 23extern bool tc_override; 24 25struct idxd_wq; 26struct idxd_dev; 27 28enum idxd_dev_type { 29 IDXD_DEV_NONE = -1, 30 IDXD_DEV_DSA = 0, 31 IDXD_DEV_IAX, 32 IDXD_DEV_WQ, 33 IDXD_DEV_GROUP, 34 IDXD_DEV_ENGINE, 35 IDXD_DEV_CDEV, 36 IDXD_DEV_CDEV_FILE, 37 IDXD_DEV_MAX_TYPE, 38}; 39 40struct idxd_dev { 41 struct device conf_dev; 42 enum idxd_dev_type type; 43}; 44 45#define IDXD_REG_TIMEOUT 50 46#define IDXD_DRAIN_TIMEOUT 5000 47 48enum idxd_type { 49 IDXD_TYPE_UNKNOWN = -1, 50 IDXD_TYPE_DSA = 0, 51 IDXD_TYPE_IAX, 52 IDXD_TYPE_MAX, 53}; 54 55#define IDXD_NAME_SIZE 128 56#define IDXD_PMU_EVENT_MAX 64 57 58#define IDXD_ENQCMDS_RETRIES 32 59#define IDXD_ENQCMDS_MAX_RETRIES 64 60 61enum idxd_complete_type { 62 IDXD_COMPLETE_NORMAL = 0, 63 IDXD_COMPLETE_ABORT, 64 IDXD_COMPLETE_DEV_FAIL, 65}; 66 67struct idxd_desc; 68 69struct idxd_device_driver { 70 const char *name; 71 enum idxd_dev_type *type; 72 int (*probe)(struct idxd_dev *idxd_dev); 73 void (*remove)(struct idxd_dev *idxd_dev); 74 void (*desc_complete)(struct idxd_desc *desc, 75 enum idxd_complete_type comp_type, 76 bool free_desc, 77 void *ctx, u32 *status); 78 struct device_driver drv; 79}; 80 81extern struct idxd_device_driver dsa_drv; 82extern struct idxd_device_driver idxd_drv; 83extern struct idxd_device_driver idxd_dmaengine_drv; 84extern struct idxd_device_driver idxd_user_drv; 85 86#define INVALID_INT_HANDLE -1 87struct idxd_irq_entry { 88 int id; 89 int vector; 90 struct llist_head pending_llist; 91 struct list_head work_list; 92 /* 93 * Lock to protect access between irq thread process descriptor 94 * and irq thread processing error descriptor. 95 */ 96 spinlock_t list_lock; 97 int int_handle; 98 ioasid_t pasid; 99}; 100 101struct idxd_group { 102 struct idxd_dev idxd_dev; 103 struct idxd_device *idxd; 104 struct grpcfg grpcfg; 105 int id; 106 int num_engines; 107 int num_wqs; 108 bool use_rdbuf_limit; 109 u8 rdbufs_allowed; 110 u8 rdbufs_reserved; 111 int tc_a; 112 int tc_b; 113 int desc_progress_limit; 114 int batch_progress_limit; 115}; 116 117struct idxd_pmu { 118 struct idxd_device *idxd; 119 120 struct perf_event *event_list[IDXD_PMU_EVENT_MAX]; 121 int n_events; 122 123 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX); 124 125 struct pmu pmu; 126 char name[IDXD_NAME_SIZE]; 127 int cpu; 128 129 int n_counters; 130 int counter_width; 131 int n_event_categories; 132 133 bool per_counter_caps_supported; 134 unsigned long supported_event_categories; 135 136 unsigned long supported_filters; 137 int n_filters; 138 139 struct hlist_node cpuhp_node; 140}; 141 142#define IDXD_MAX_PRIORITY 0xf 143 144enum { 145 COUNTER_FAULTS = 0, 146 COUNTER_FAULT_FAILS, 147 COUNTER_MAX 148}; 149 150enum idxd_wq_state { 151 IDXD_WQ_DISABLED = 0, 152 IDXD_WQ_ENABLED, 153}; 154 155enum idxd_wq_flag { 156 WQ_FLAG_DEDICATED = 0, 157 WQ_FLAG_BLOCK_ON_FAULT, 158 WQ_FLAG_ATS_DISABLE, 159 WQ_FLAG_PRS_DISABLE, 160}; 161 162enum idxd_wq_type { 163 IDXD_WQT_NONE = 0, 164 IDXD_WQT_KERNEL, 165 IDXD_WQT_USER, 166}; 167 168struct idxd_cdev { 169 struct idxd_wq *wq; 170 struct cdev cdev; 171 struct idxd_dev idxd_dev; 172 int minor; 173}; 174 175#define DRIVER_NAME_SIZE 128 176 177#define IDXD_ALLOCATED_BATCH_SIZE 128U 178#define WQ_NAME_SIZE 1024 179#define WQ_TYPE_SIZE 10 180 181#define WQ_DEFAULT_QUEUE_DEPTH 16 182#define WQ_DEFAULT_MAX_XFER SZ_2M 183#define WQ_DEFAULT_MAX_BATCH 32 184 185enum idxd_op_type { 186 IDXD_OP_BLOCK = 0, 187 IDXD_OP_NONBLOCK = 1, 188}; 189 190struct idxd_dma_chan { 191 struct dma_chan chan; 192 struct idxd_wq *wq; 193}; 194 195struct idxd_wq { 196 void __iomem *portal; 197 u32 portal_offset; 198 unsigned int enqcmds_retries; 199 struct percpu_ref wq_active; 200 struct completion wq_dead; 201 struct completion wq_resurrect; 202 struct idxd_dev idxd_dev; 203 struct idxd_cdev *idxd_cdev; 204 struct wait_queue_head err_queue; 205 struct workqueue_struct *wq; 206 struct idxd_device *idxd; 207 int id; 208 struct idxd_irq_entry ie; 209 enum idxd_wq_type type; 210 struct idxd_group *group; 211 int client_count; 212 struct mutex wq_lock; /* mutex for workqueue */ 213 u32 size; 214 u32 threshold; 215 u32 priority; 216 enum idxd_wq_state state; 217 unsigned long flags; 218 union wqcfg *wqcfg; 219 unsigned long *opcap_bmap; 220 221 struct dsa_hw_desc **hw_descs; 222 int num_descs; 223 union { 224 struct dsa_completion_record *compls; 225 struct iax_completion_record *iax_compls; 226 }; 227 dma_addr_t compls_addr; 228 int compls_size; 229 struct idxd_desc **descs; 230 struct sbitmap_queue sbq; 231 struct idxd_dma_chan *idxd_chan; 232 char name[WQ_NAME_SIZE + 1]; 233 u64 max_xfer_bytes; 234 u32 max_batch_size; 235 236 /* Lock to protect upasid_xa access. */ 237 struct mutex uc_lock; 238 struct xarray upasid_xa; 239 240 char driver_name[DRIVER_NAME_SIZE + 1]; 241}; 242 243struct idxd_engine { 244 struct idxd_dev idxd_dev; 245 int id; 246 struct idxd_group *group; 247 struct idxd_device *idxd; 248}; 249 250/* shadow registers */ 251struct idxd_hw { 252 u32 version; 253 union gen_cap_reg gen_cap; 254 union wq_cap_reg wq_cap; 255 union group_cap_reg group_cap; 256 union engine_cap_reg engine_cap; 257 struct opcap opcap; 258 u32 cmd_cap; 259 union iaa_cap_reg iaa_cap; 260}; 261 262enum idxd_device_state { 263 IDXD_DEV_HALTED = -1, 264 IDXD_DEV_DISABLED = 0, 265 IDXD_DEV_ENABLED, 266}; 267 268enum idxd_device_flag { 269 IDXD_FLAG_CONFIGURABLE = 0, 270 IDXD_FLAG_CMD_RUNNING, 271 IDXD_FLAG_PASID_ENABLED, 272 IDXD_FLAG_USER_PASID_ENABLED, 273}; 274 275struct idxd_dma_dev { 276 struct idxd_device *idxd; 277 struct dma_device dma; 278}; 279 280typedef int (*load_device_defaults_fn_t) (struct idxd_device *idxd); 281 282struct idxd_driver_data { 283 const char *name_prefix; 284 enum idxd_type type; 285 const struct device_type *dev_type; 286 int compl_size; 287 int align; 288 int evl_cr_off; 289 int cr_status_off; 290 int cr_result_off; 291 bool user_submission_safe; 292 load_device_defaults_fn_t load_device_defaults; 293}; 294 295struct idxd_evl { 296 /* Lock to protect event log access. */ 297 struct mutex lock; 298 void *log; 299 dma_addr_t dma; 300 /* Total size of event log = number of entries * entry size. */ 301 unsigned int log_size; 302 /* The number of entries in the event log. */ 303 u16 size; 304 unsigned long *bmap; 305 bool batch_fail[IDXD_MAX_BATCH_IDENT]; 306}; 307 308struct idxd_evl_fault { 309 struct work_struct work; 310 struct idxd_wq *wq; 311 u8 status; 312 313 /* make this last member always */ 314 struct __evl_entry entry[]; 315}; 316 317struct idxd_device { 318 struct idxd_dev idxd_dev; 319 struct idxd_driver_data *data; 320 struct list_head list; 321 struct idxd_hw hw; 322 enum idxd_device_state state; 323 unsigned long flags; 324 int id; 325 int major; 326 u32 cmd_status; 327 struct idxd_irq_entry ie; /* misc irq, msix 0 */ 328 329 struct pci_dev *pdev; 330 void __iomem *reg_base; 331 332 spinlock_t dev_lock; /* spinlock for device */ 333 spinlock_t cmd_lock; /* spinlock for device commands */ 334 struct completion *cmd_done; 335 struct idxd_group **groups; 336 struct idxd_wq **wqs; 337 struct idxd_engine **engines; 338 339 struct iommu_sva *sva; 340 unsigned int pasid; 341 342 int num_groups; 343 int irq_cnt; 344 bool request_int_handles; 345 346 u32 msix_perm_offset; 347 u32 wqcfg_offset; 348 u32 grpcfg_offset; 349 u32 perfmon_offset; 350 351 u64 max_xfer_bytes; 352 u32 max_batch_size; 353 int max_groups; 354 int max_engines; 355 int max_rdbufs; 356 int max_wqs; 357 int max_wq_size; 358 int rdbuf_limit; 359 int nr_rdbufs; /* non-reserved read buffers */ 360 unsigned int wqcfg_size; 361 unsigned long *wq_enable_map; 362 363 union sw_err_reg sw_err; 364 wait_queue_head_t cmd_waitq; 365 366 struct idxd_dma_dev *idxd_dma; 367 struct workqueue_struct *wq; 368 struct work_struct work; 369 370 struct idxd_pmu *idxd_pmu; 371 372 unsigned long *opcap_bmap; 373 struct idxd_evl *evl; 374 struct kmem_cache *evl_cache; 375 376 struct dentry *dbgfs_dir; 377 struct dentry *dbgfs_evl_file; 378 379 bool user_submission_safe; 380}; 381 382static inline unsigned int evl_ent_size(struct idxd_device *idxd) 383{ 384 return idxd->hw.gen_cap.evl_support ? 385 (32 * (1 << idxd->hw.gen_cap.evl_support)) : 0; 386} 387 388static inline unsigned int evl_size(struct idxd_device *idxd) 389{ 390 return idxd->evl->size * evl_ent_size(idxd); 391} 392 393struct crypto_ctx { 394 struct acomp_req *req; 395 struct crypto_tfm *tfm; 396 dma_addr_t src_addr; 397 dma_addr_t dst_addr; 398 bool compress; 399}; 400 401/* IDXD software descriptor */ 402struct idxd_desc { 403 union { 404 struct dsa_hw_desc *hw; 405 struct iax_hw_desc *iax_hw; 406 }; 407 dma_addr_t desc_dma; 408 union { 409 struct dsa_completion_record *completion; 410 struct iax_completion_record *iax_completion; 411 }; 412 dma_addr_t compl_dma; 413 union { 414 struct dma_async_tx_descriptor txd; 415 struct crypto_ctx crypto; 416 }; 417 struct llist_node llnode; 418 struct list_head list; 419 int id; 420 int cpu; 421 struct idxd_wq *wq; 422}; 423 424/* 425 * This is software defined error for the completion status. We overload the error code 426 * that will never appear in completion status and only SWERR register. 427 */ 428enum idxd_completion_status { 429 IDXD_COMP_DESC_ABORT = 0xff, 430}; 431 432#define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev 433#define wq_confdev(wq) &wq->idxd_dev.conf_dev 434#define engine_confdev(engine) &engine->idxd_dev.conf_dev 435#define group_confdev(group) &group->idxd_dev.conf_dev 436#define cdev_dev(cdev) &cdev->idxd_dev.conf_dev 437#define user_ctx_dev(ctx) (&(ctx)->idxd_dev.conf_dev) 438 439#define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) 440#define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) 441#define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) 442 443static inline struct idxd_device_driver *wq_to_idxd_drv(struct idxd_wq *wq) 444{ 445 struct device *dev = wq_confdev(wq); 446 struct idxd_device_driver *idxd_drv = 447 container_of(dev->driver, struct idxd_device_driver, drv); 448 449 return idxd_drv; 450} 451 452static inline struct idxd_device *confdev_to_idxd(struct device *dev) 453{ 454 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 455 456 return idxd_dev_to_idxd(idxd_dev); 457} 458 459static inline struct idxd_wq *confdev_to_wq(struct device *dev) 460{ 461 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 462 463 return idxd_dev_to_wq(idxd_dev); 464} 465 466static inline struct idxd_engine *confdev_to_engine(struct device *dev) 467{ 468 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 469 470 return container_of(idxd_dev, struct idxd_engine, idxd_dev); 471} 472 473static inline struct idxd_group *confdev_to_group(struct device *dev) 474{ 475 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 476 477 return container_of(idxd_dev, struct idxd_group, idxd_dev); 478} 479 480static inline struct idxd_cdev *dev_to_cdev(struct device *dev) 481{ 482 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 483 484 return container_of(idxd_dev, struct idxd_cdev, idxd_dev); 485} 486 487static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) 488{ 489 if (type >= IDXD_DEV_MAX_TYPE) { 490 idev->type = IDXD_DEV_NONE; 491 return; 492 } 493 494 idev->type = type; 495} 496 497static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx) 498{ 499 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie; 500} 501 502static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie) 503{ 504 return container_of(ie, struct idxd_wq, ie); 505} 506 507static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie) 508{ 509 return container_of(ie, struct idxd_device, ie); 510} 511 512static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable) 513{ 514 union gencfg_reg reg; 515 516 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); 517 reg.user_int_en = enable; 518 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); 519} 520 521extern const struct bus_type dsa_bus_type; 522 523extern bool support_enqcmd; 524extern struct ida idxd_ida; 525extern const struct device_type dsa_device_type; 526extern const struct device_type iax_device_type; 527extern const struct device_type idxd_wq_device_type; 528extern const struct device_type idxd_engine_device_type; 529extern const struct device_type idxd_group_device_type; 530 531static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) 532{ 533 return idxd_dev->type == IDXD_DEV_DSA; 534} 535 536static inline bool is_iax_dev(struct idxd_dev *idxd_dev) 537{ 538 return idxd_dev->type == IDXD_DEV_IAX; 539} 540 541static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) 542{ 543 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); 544} 545 546static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) 547{ 548 return idxd_dev->type == IDXD_DEV_WQ; 549} 550 551static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) 552{ 553 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) 554 return true; 555 return false; 556} 557 558static inline bool is_idxd_wq_user(struct idxd_wq *wq) 559{ 560 return wq->type == IDXD_WQT_USER; 561} 562 563static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) 564{ 565 return wq->type == IDXD_WQT_KERNEL; 566} 567 568static inline bool wq_dedicated(struct idxd_wq *wq) 569{ 570 return test_bit(WQ_FLAG_DEDICATED, &wq->flags); 571} 572 573static inline bool wq_shared(struct idxd_wq *wq) 574{ 575 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags); 576} 577 578static inline bool device_pasid_enabled(struct idxd_device *idxd) 579{ 580 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 581} 582 583static inline bool device_user_pasid_enabled(struct idxd_device *idxd) 584{ 585 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 586} 587 588static inline bool wq_pasid_enabled(struct idxd_wq *wq) 589{ 590 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) || 591 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd)); 592} 593 594static inline bool wq_shared_supported(struct idxd_wq *wq) 595{ 596 return (support_enqcmd && wq_pasid_enabled(wq)); 597} 598 599enum idxd_portal_prot { 600 IDXD_PORTAL_UNLIMITED = 0, 601 IDXD_PORTAL_LIMITED, 602}; 603 604enum idxd_interrupt_type { 605 IDXD_IRQ_MSIX = 0, 606 IDXD_IRQ_IMS, 607}; 608 609static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) 610{ 611 return prot * 0x1000; 612} 613 614static inline int idxd_get_wq_portal_full_offset(int wq_id, 615 enum idxd_portal_prot prot) 616{ 617 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); 618} 619 620#define IDXD_PORTAL_MASK (PAGE_SIZE - 1) 621 622/* 623 * Even though this function can be accessed by multiple threads, it is safe to use. 624 * At worst the address gets used more than once before it gets incremented. We don't 625 * hit a threshold until iops becomes many million times a second. So the occasional 626 * reuse of the same address is tolerable compare to using an atomic variable. This is 627 * safe on a system that has atomic load/store for 32bit integers. Given that this is an 628 * Intel iEP device, that should not be a problem. 629 */ 630static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq) 631{ 632 int ofs = wq->portal_offset; 633 634 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK; 635 return wq->portal + ofs; 636} 637 638static inline void idxd_wq_get(struct idxd_wq *wq) 639{ 640 wq->client_count++; 641} 642 643static inline void idxd_wq_put(struct idxd_wq *wq) 644{ 645 wq->client_count--; 646} 647 648static inline int idxd_wq_refcount(struct idxd_wq *wq) 649{ 650 return wq->client_count; 651}; 652 653static inline void idxd_wq_set_private(struct idxd_wq *wq, void *private) 654{ 655 dev_set_drvdata(wq_confdev(wq), private); 656} 657 658static inline void *idxd_wq_get_private(struct idxd_wq *wq) 659{ 660 return dev_get_drvdata(wq_confdev(wq)); 661} 662 663/* 664 * Intel IAA does not support batch processing. 665 * The max batch size of device, max batch size of wq and 666 * max batch shift of wqcfg should be always 0 on IAA. 667 */ 668static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd, 669 u32 max_batch_size) 670{ 671 if (idxd_type == IDXD_TYPE_IAX) 672 idxd->max_batch_size = 0; 673 else 674 idxd->max_batch_size = max_batch_size; 675} 676 677static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq, 678 u32 max_batch_size) 679{ 680 if (idxd_type == IDXD_TYPE_IAX) 681 wq->max_batch_size = 0; 682 else 683 wq->max_batch_size = max_batch_size; 684} 685 686static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg, 687 u32 max_batch_shift) 688{ 689 if (idxd_type == IDXD_TYPE_IAX) 690 wqcfg->max_batch_shift = 0; 691 else 692 wqcfg->max_batch_shift = max_batch_shift; 693} 694 695static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev) 696{ 697 return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0); 698} 699 700#define MODULE_ALIAS_IDXD_DEVICE(type) MODULE_ALIAS("idxd:t" __stringify(type) "*") 701#define IDXD_DEVICES_MODALIAS_FMT "idxd:t%d" 702 703int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, 704 struct module *module, const char *mod_name); 705#define idxd_driver_register(driver) \ 706 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) 707 708void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); 709 710#define module_idxd_driver(__idxd_driver) \ 711 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) 712 713void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); 714void idxd_dma_complete_txd(struct idxd_desc *desc, 715 enum idxd_complete_type comp_type, 716 bool free_desc, void *ctx, u32 *status); 717 718static inline void idxd_desc_complete(struct idxd_desc *desc, 719 enum idxd_complete_type comp_type, 720 bool free_desc) 721{ 722 struct idxd_device_driver *drv; 723 u32 status; 724 725 drv = wq_to_idxd_drv(desc->wq); 726 if (drv->desc_complete) 727 drv->desc_complete(desc, comp_type, free_desc, 728 &desc->txd, &status); 729} 730 731int idxd_register_bus_type(void); 732void idxd_unregister_bus_type(void); 733int idxd_register_devices(struct idxd_device *idxd); 734void idxd_unregister_devices(struct idxd_device *idxd); 735void idxd_wqs_quiesce(struct idxd_device *idxd); 736bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); 737void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count); 738int idxd_load_iaa_device_defaults(struct idxd_device *idxd); 739 740/* device interrupt control */ 741irqreturn_t idxd_misc_thread(int vec, void *data); 742irqreturn_t idxd_wq_thread(int irq, void *data); 743void idxd_mask_error_interrupts(struct idxd_device *idxd); 744void idxd_unmask_error_interrupts(struct idxd_device *idxd); 745 746/* device control */ 747int idxd_device_drv_probe(struct idxd_dev *idxd_dev); 748void idxd_device_drv_remove(struct idxd_dev *idxd_dev); 749int idxd_drv_enable_wq(struct idxd_wq *wq); 750void idxd_drv_disable_wq(struct idxd_wq *wq); 751int idxd_device_init_reset(struct idxd_device *idxd); 752int idxd_device_enable(struct idxd_device *idxd); 753int idxd_device_disable(struct idxd_device *idxd); 754void idxd_device_reset(struct idxd_device *idxd); 755void idxd_device_clear_state(struct idxd_device *idxd); 756int idxd_device_config(struct idxd_device *idxd); 757void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); 758int idxd_device_load_config(struct idxd_device *idxd); 759int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, 760 enum idxd_interrupt_type irq_type); 761int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, 762 enum idxd_interrupt_type irq_type); 763 764/* work queue control */ 765void idxd_wqs_unmap_portal(struct idxd_device *idxd); 766int idxd_wq_alloc_resources(struct idxd_wq *wq); 767void idxd_wq_free_resources(struct idxd_wq *wq); 768int idxd_wq_enable(struct idxd_wq *wq); 769int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); 770void idxd_wq_drain(struct idxd_wq *wq); 771void idxd_wq_reset(struct idxd_wq *wq); 772int idxd_wq_map_portal(struct idxd_wq *wq); 773void idxd_wq_unmap_portal(struct idxd_wq *wq); 774int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); 775int idxd_wq_disable_pasid(struct idxd_wq *wq); 776void __idxd_wq_quiesce(struct idxd_wq *wq); 777void idxd_wq_quiesce(struct idxd_wq *wq); 778int idxd_wq_init_percpu_ref(struct idxd_wq *wq); 779void idxd_wq_free_irq(struct idxd_wq *wq); 780int idxd_wq_request_irq(struct idxd_wq *wq); 781 782/* submission */ 783int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); 784struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); 785int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); 786 787/* dmaengine */ 788int idxd_register_dma_device(struct idxd_device *idxd); 789void idxd_unregister_dma_device(struct idxd_device *idxd); 790 791/* cdev */ 792int idxd_cdev_register(void); 793void idxd_cdev_remove(void); 794int idxd_cdev_get_major(struct idxd_device *idxd); 795int idxd_wq_add_cdev(struct idxd_wq *wq); 796void idxd_wq_del_cdev(struct idxd_wq *wq); 797int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, 798 void *buf, int len); 799void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index); 800 801/* perfmon */ 802#if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) 803int perfmon_pmu_init(struct idxd_device *idxd); 804void perfmon_pmu_remove(struct idxd_device *idxd); 805void perfmon_counter_overflow(struct idxd_device *idxd); 806void perfmon_init(void); 807void perfmon_exit(void); 808#else 809static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } 810static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} 811static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} 812static inline void perfmon_init(void) {} 813static inline void perfmon_exit(void) {} 814#endif 815 816/* debugfs */ 817int idxd_device_init_debugfs(struct idxd_device *idxd); 818void idxd_device_remove_debugfs(struct idxd_device *idxd); 819int idxd_init_debugfs(void); 820void idxd_remove_debugfs(void); 821 822#endif 823