1178784Skmacy/* 2178784Skmacy * Copyright (c) 2005 Ammasso, Inc. All rights reserved. 3256829Snp * Copyright (c) 2006-2009 Open Grid Computing, Inc. All rights reserved. 4178784Skmacy * 5178784Skmacy * This software is available to you under a choice of one of two 6178784Skmacy * licenses. You may choose to be licensed under the terms of the GNU 7178784Skmacy * General Public License (GPL) Version 2, available from the file 8178784Skmacy * COPYING in the main directory of this source tree, or the 9178784Skmacy * OpenIB.org BSD license below: 10178784Skmacy * 11178784Skmacy * Redistribution and use in source and binary forms, with or 12178784Skmacy * without modification, are permitted provided that the following 13178784Skmacy * conditions are met: 14178784Skmacy * 15178784Skmacy * - Redistributions of source code must retain the above 16178784Skmacy * copyright notice, this list of conditions and the following 17178784Skmacy * disclaimer. 18178784Skmacy * 19178784Skmacy * - Redistributions in binary form must reproduce the above 20178784Skmacy * copyright notice, this list of conditions and the following 21178784Skmacy * disclaimer in the documentation and/or other materials 22178784Skmacy * provided with the distribution. 23178784Skmacy * 24178784Skmacy * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25178784Skmacy * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26178784Skmacy * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27178784Skmacy * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28178784Skmacy * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29178784Skmacy * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30178784Skmacy * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31178784Skmacy * SOFTWARE. 32178784Skmacy */ 33178784Skmacy 34178784Skmacy#include <sys/cdefs.h> 35178784Skmacy__FBSDID("$FreeBSD: stable/10/sys/contrib/rdma/krping/krping.c 325602 2017-11-09 17:51:55Z hselasky $"); 36178784Skmacy 37256829Snp#include <linux/module.h> 38256829Snp#include <linux/moduleparam.h> 39256829Snp#include <linux/slab.h> 40256829Snp#include <linux/err.h> 41256829Snp#include <linux/string.h> 42256829Snp#include <linux/list.h> 43256829Snp#include <linux/in.h> 44256829Snp#include <linux/device.h> 45256829Snp#include <linux/pci.h> 46256829Snp#include <linux/sched.h> 47178784Skmacy 48256829Snp#include <asm/atomic.h> 49178784Skmacy 50256829Snp#include <rdma/ib_verbs.h> 51237263Snp#include <rdma/rdma_cm.h> 52178784Skmacy 53256829Snp#include "krping.h" 54178784Skmacy#include "getopt.h" 55178784Skmacy 56256829Snpextern int krping_debug; 57256829Snp#define DEBUG_LOG(cb, x...) if (krping_debug) krping_printf((cb)->cookie, x) 58256829Snp#define PRINTF(cb, x...) krping_printf((cb)->cookie, x) 59178784Skmacy 60256829SnpMODULE_AUTHOR("Steve Wise"); 61256829SnpMODULE_DESCRIPTION("RDMA ping client/server"); 62256829SnpMODULE_LICENSE("Dual BSD/GPL"); 63178784Skmacy 64256829Snpstatic __inline uint64_t 65256829Snpget_cycles(void) 66256829Snp{ 67256829Snp uint32_t low, high; 68256829Snp __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); 69256829Snp return (low | ((u_int64_t)high << 32)); 70256829Snp} 71256829Snp 72256829Snptypedef uint64_t cycles_t; 73256829Snp 74256829Snpenum mem_type { 75256829Snp DMA = 1, 76256829Snp FASTREG = 2, 77256829Snp MW = 3, 78256829Snp MR = 4 79256829Snp}; 80256829Snp 81178784Skmacystatic const struct krping_option krping_opts[] = { 82178784Skmacy {"count", OPT_INT, 'C'}, 83178784Skmacy {"size", OPT_INT, 'S'}, 84178784Skmacy {"addr", OPT_STRING, 'a'}, 85325602Shselasky {"addr6", OPT_STRING, 'A'}, 86178784Skmacy {"port", OPT_INT, 'p'}, 87178784Skmacy {"verbose", OPT_NOPARAM, 'v'}, 88178784Skmacy {"validate", OPT_NOPARAM, 'V'}, 89178784Skmacy {"server", OPT_NOPARAM, 's'}, 90178784Skmacy {"client", OPT_NOPARAM, 'c'}, 91256829Snp {"mem_mode", OPT_STRING, 'm'}, 92256829Snp {"server_inv", OPT_NOPARAM, 'I'}, 93256829Snp {"wlat", OPT_NOPARAM, 'l'}, 94256829Snp {"rlat", OPT_NOPARAM, 'L'}, 95256829Snp {"bw", OPT_NOPARAM, 'B'}, 96256829Snp {"duplex", OPT_NOPARAM, 'd'}, 97256829Snp {"txdepth", OPT_INT, 'T'}, 98256829Snp {"poll", OPT_NOPARAM, 'P'}, 99256829Snp {"local_dma_lkey", OPT_NOPARAM, 'Z'}, 100256829Snp {"read_inv", OPT_NOPARAM, 'R'}, 101256829Snp {"fr", OPT_NOPARAM, 'f'}, 102178784Skmacy {NULL, 0, 0} 103178784Skmacy}; 104178784Skmacy 105256829Snp#define htonll(x) cpu_to_be64((x)) 106256829Snp#define ntohll(x) cpu_to_be64((x)) 107178784Skmacy 108256829Snpstatic struct mutex krping_mutex; 109256829Snp 110178784Skmacy/* 111178784Skmacy * List of running krping threads. 112178784Skmacy */ 113256829Snpstatic LIST_HEAD(krping_cbs); 114178784Skmacy 115178784Skmacy/* 116178784Skmacy * krping "ping/pong" loop: 117178784Skmacy * client sends source rkey/addr/len 118178784Skmacy * server receives source rkey/add/len 119178784Skmacy * server rdma reads "ping" data from source 120178784Skmacy * server sends "go ahead" on rdma read completion 121178784Skmacy * client sends sink rkey/addr/len 122178784Skmacy * server receives sink rkey/addr/len 123178784Skmacy * server rdma writes "pong" data to sink 124178784Skmacy * server sends "go ahead" on rdma write completion 125178784Skmacy * <repeat loop> 126178784Skmacy */ 127178784Skmacy 128178784Skmacy/* 129256829Snp * These states are used to signal events between the completion handler 130256829Snp * and the main client or server thread. 131256829Snp * 132256829Snp * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, 133256829Snp * and RDMA_WRITE_COMPLETE for each ping. 134256829Snp */ 135256829Snpenum test_state { 136256829Snp IDLE = 1, 137256829Snp CONNECT_REQUEST, 138256829Snp ADDR_RESOLVED, 139256829Snp ROUTE_RESOLVED, 140256829Snp CONNECTED, 141256829Snp RDMA_READ_ADV, 142256829Snp RDMA_READ_COMPLETE, 143256829Snp RDMA_WRITE_ADV, 144256829Snp RDMA_WRITE_COMPLETE, 145256829Snp ERROR 146256829Snp}; 147256829Snp 148256829Snpstruct krping_rdma_info { 149256829Snp uint64_t buf; 150256829Snp uint32_t rkey; 151256829Snp uint32_t size; 152256829Snp}; 153256829Snp 154256829Snp/* 155178784Skmacy * Default max buffer size for IO... 156178784Skmacy */ 157178784Skmacy#define RPING_BUFSIZE 128*1024 158256829Snp#define RPING_SQ_DEPTH 64 159178784Skmacy 160256829Snp/* 161256829Snp * Control block struct. 162256829Snp */ 163256829Snpstruct krping_cb { 164256829Snp void *cookie; 165256829Snp int server; /* 0 iff client */ 166256829Snp struct ib_cq *cq; 167256829Snp struct ib_pd *pd; 168256829Snp struct ib_qp *qp; 169178784Skmacy 170256829Snp enum mem_type mem; 171256829Snp struct ib_mr *dma_mr; 172256829Snp 173256829Snp struct ib_fast_reg_page_list *page_list; 174256829Snp int page_list_len; 175256829Snp struct ib_send_wr fastreg_wr; 176256829Snp struct ib_send_wr invalidate_wr; 177256829Snp struct ib_mr *fastreg_mr; 178256829Snp int server_invalidate; 179256829Snp int read_inv; 180256829Snp u8 key; 181256829Snp 182256829Snp struct ib_mw *mw; 183256829Snp struct ib_mw_bind bind_attr; 184256829Snp 185256829Snp struct ib_recv_wr rq_wr; /* recv work request record */ 186256829Snp struct ib_sge recv_sgl; /* recv single SGE */ 187256829Snp struct krping_rdma_info recv_buf;/* malloc'd buffer */ 188256829Snp u64 recv_dma_addr; 189256829Snp DECLARE_PCI_UNMAP_ADDR(recv_mapping) 190256829Snp struct ib_mr *recv_mr; 191256829Snp 192256829Snp struct ib_send_wr sq_wr; /* send work requrest record */ 193256829Snp struct ib_sge send_sgl; 194256829Snp struct krping_rdma_info send_buf;/* single send buf */ 195256829Snp u64 send_dma_addr; 196256829Snp DECLARE_PCI_UNMAP_ADDR(send_mapping) 197256829Snp struct ib_mr *send_mr; 198256829Snp 199256829Snp struct ib_send_wr rdma_sq_wr; /* rdma work request record */ 200256829Snp struct ib_sge rdma_sgl; /* rdma single SGE */ 201256829Snp char *rdma_buf; /* used as rdma sink */ 202256829Snp u64 rdma_dma_addr; 203256829Snp DECLARE_PCI_UNMAP_ADDR(rdma_mapping) 204256829Snp struct ib_mr *rdma_mr; 205256829Snp 206256829Snp uint32_t remote_rkey; /* remote guys RKEY */ 207256829Snp uint64_t remote_addr; /* remote guys TO */ 208256829Snp uint32_t remote_len; /* remote guys LEN */ 209256829Snp 210256829Snp char *start_buf; /* rdma read src */ 211256829Snp u64 start_dma_addr; 212256829Snp DECLARE_PCI_UNMAP_ADDR(start_mapping) 213256829Snp struct ib_mr *start_mr; 214256829Snp 215256829Snp enum test_state state; /* used for cond/signalling */ 216256829Snp wait_queue_head_t sem; 217256829Snp struct krping_stats stats; 218256829Snp 219256829Snp uint16_t port; /* dst port in NBO */ 220325602Shselasky union { 221325602Shselasky struct in_addr v4; 222325602Shselasky struct in6_addr v6; 223325602Shselasky } addr; /* dst addr in NBO */ 224325602Shselasky int addr_type; /* AF_INET or AF_INET6 */ 225256829Snp char *addr_str; /* dst addr string */ 226256829Snp int verbose; /* verbose logging */ 227256829Snp int count; /* ping count */ 228256829Snp int size; /* ping data size */ 229256829Snp int validate; /* validate ping data */ 230256829Snp int wlat; /* run wlat test */ 231256829Snp int rlat; /* run rlat test */ 232256829Snp int bw; /* run bw test */ 233256829Snp int duplex; /* run bw full duplex test */ 234256829Snp int poll; /* poll or block for rlat test */ 235256829Snp int txdepth; /* SQ depth */ 236256829Snp int local_dma_lkey; /* use 0 for lkey */ 237256829Snp int frtest; /* fastreg test */ 238256829Snp 239256829Snp /* CM stuff */ 240256829Snp struct rdma_cm_id *cm_id; /* connection on client side,*/ 241256829Snp /* listener on server side. */ 242256829Snp struct rdma_cm_id *child_cm_id; /* connection on server side */ 243256829Snp struct list_head list; 244256829Snp}; 245256829Snp 246178784Skmacystatic int krping_cma_event_handler(struct rdma_cm_id *cma_id, 247178784Skmacy struct rdma_cm_event *event) 248178784Skmacy{ 249178784Skmacy int ret; 250178784Skmacy struct krping_cb *cb = cma_id->context; 251178784Skmacy 252256829Snp DEBUG_LOG(cb, "cma_event type %d cma_id %p (%s)\n", event->event, 253256829Snp cma_id, (cma_id == cb->cm_id) ? "parent" : "child"); 254178784Skmacy 255178784Skmacy switch (event->event) { 256178784Skmacy case RDMA_CM_EVENT_ADDR_RESOLVED: 257178784Skmacy cb->state = ADDR_RESOLVED; 258178784Skmacy ret = rdma_resolve_route(cma_id, 2000); 259178784Skmacy if (ret) { 260256829Snp PRINTF(cb, "rdma_resolve_route error %d\n", ret); 261256829Snp wake_up_interruptible(&cb->sem); 262178784Skmacy } 263178784Skmacy break; 264178784Skmacy 265178784Skmacy case RDMA_CM_EVENT_ROUTE_RESOLVED: 266178784Skmacy cb->state = ROUTE_RESOLVED; 267297655Shselasky cb->child_cm_id = cma_id; 268256829Snp wake_up_interruptible(&cb->sem); 269178784Skmacy break; 270178784Skmacy 271178784Skmacy case RDMA_CM_EVENT_CONNECT_REQUEST: 272178784Skmacy cb->state = CONNECT_REQUEST; 273178784Skmacy cb->child_cm_id = cma_id; 274256829Snp DEBUG_LOG(cb, "child cma %p\n", cb->child_cm_id); 275256829Snp wake_up_interruptible(&cb->sem); 276178784Skmacy break; 277178784Skmacy 278178784Skmacy case RDMA_CM_EVENT_ESTABLISHED: 279256829Snp DEBUG_LOG(cb, "ESTABLISHED\n"); 280178784Skmacy if (!cb->server) { 281178784Skmacy cb->state = CONNECTED; 282178784Skmacy } 283256829Snp wake_up_interruptible(&cb->sem); 284178784Skmacy break; 285178784Skmacy 286178784Skmacy case RDMA_CM_EVENT_ADDR_ERROR: 287178784Skmacy case RDMA_CM_EVENT_ROUTE_ERROR: 288178784Skmacy case RDMA_CM_EVENT_CONNECT_ERROR: 289178784Skmacy case RDMA_CM_EVENT_UNREACHABLE: 290178784Skmacy case RDMA_CM_EVENT_REJECTED: 291256829Snp PRINTF(cb, "cma event %d, error %d\n", event->event, 292178784Skmacy event->status); 293178784Skmacy cb->state = ERROR; 294256829Snp wake_up_interruptible(&cb->sem); 295178784Skmacy break; 296178784Skmacy 297178784Skmacy case RDMA_CM_EVENT_DISCONNECTED: 298256829Snp PRINTF(cb, "DISCONNECT EVENT...\n"); 299178784Skmacy cb->state = ERROR; 300256829Snp wake_up_interruptible(&cb->sem); 301178784Skmacy break; 302178784Skmacy 303178784Skmacy case RDMA_CM_EVENT_DEVICE_REMOVAL: 304256829Snp PRINTF(cb, "cma detected device removal!!!!\n"); 305256829Snp break; 306178784Skmacy 307178784Skmacy default: 308256829Snp PRINTF(cb, "oof bad type!\n"); 309256829Snp wake_up_interruptible(&cb->sem); 310178784Skmacy break; 311178784Skmacy } 312178784Skmacy return 0; 313178784Skmacy} 314178784Skmacy 315178784Skmacystatic int server_recv(struct krping_cb *cb, struct ib_wc *wc) 316178784Skmacy{ 317178784Skmacy if (wc->byte_len != sizeof(cb->recv_buf)) { 318256829Snp PRINTF(cb, "Received bogus data, size %d\n", 319178784Skmacy wc->byte_len); 320178784Skmacy return -1; 321178784Skmacy } 322178784Skmacy 323178784Skmacy cb->remote_rkey = ntohl(cb->recv_buf.rkey); 324178784Skmacy cb->remote_addr = ntohll(cb->recv_buf.buf); 325178784Skmacy cb->remote_len = ntohl(cb->recv_buf.size); 326256829Snp DEBUG_LOG(cb, "Received rkey %x addr %llx len %d from peer\n", 327178784Skmacy cb->remote_rkey, (unsigned long long)cb->remote_addr, 328178784Skmacy cb->remote_len); 329178784Skmacy 330178784Skmacy if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE) 331178784Skmacy cb->state = RDMA_READ_ADV; 332178784Skmacy else 333178784Skmacy cb->state = RDMA_WRITE_ADV; 334178784Skmacy 335178784Skmacy return 0; 336178784Skmacy} 337178784Skmacy 338178784Skmacystatic int client_recv(struct krping_cb *cb, struct ib_wc *wc) 339178784Skmacy{ 340178784Skmacy if (wc->byte_len != sizeof(cb->recv_buf)) { 341256829Snp PRINTF(cb, "Received bogus data, size %d\n", 342178784Skmacy wc->byte_len); 343178784Skmacy return -1; 344178784Skmacy } 345178784Skmacy 346178784Skmacy if (cb->state == RDMA_READ_ADV) 347178784Skmacy cb->state = RDMA_WRITE_ADV; 348178784Skmacy else 349178784Skmacy cb->state = RDMA_WRITE_COMPLETE; 350178784Skmacy 351178784Skmacy return 0; 352178784Skmacy} 353178784Skmacy 354178784Skmacystatic void krping_cq_event_handler(struct ib_cq *cq, void *ctx) 355178784Skmacy{ 356178784Skmacy struct krping_cb *cb = ctx; 357178784Skmacy struct ib_wc wc; 358178784Skmacy struct ib_recv_wr *bad_wr; 359178784Skmacy int ret; 360178784Skmacy 361256829Snp BUG_ON(cb->cq != cq); 362178784Skmacy if (cb->state == ERROR) { 363256829Snp PRINTF(cb, "cq completion in ERROR state\n"); 364178784Skmacy return; 365178784Skmacy } 366256829Snp if (cb->frtest) { 367256829Snp PRINTF(cb, "cq completion event in frtest!\n"); 368256829Snp return; 369256829Snp } 370178784Skmacy if (!cb->wlat && !cb->rlat && !cb->bw) 371178784Skmacy ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); 372178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { 373178784Skmacy if (wc.status) { 374237263Snp if (wc.status == IB_WC_WR_FLUSH_ERR) { 375256829Snp DEBUG_LOG(cb, "cq flushed\n"); 376237263Snp continue; 377237263Snp } else { 378256829Snp PRINTF(cb, "cq completion failed with " 379256829Snp "wr_id %Lx status %d opcode %d vender_err %x\n", 380256829Snp wc.wr_id, wc.status, wc.opcode, wc.vendor_err); 381237263Snp goto error; 382237263Snp } 383178784Skmacy } 384178784Skmacy 385178784Skmacy switch (wc.opcode) { 386178784Skmacy case IB_WC_SEND: 387256829Snp DEBUG_LOG(cb, "send completion\n"); 388178784Skmacy cb->stats.send_bytes += cb->send_sgl.length; 389178784Skmacy cb->stats.send_msgs++; 390178784Skmacy break; 391178784Skmacy 392178784Skmacy case IB_WC_RDMA_WRITE: 393256829Snp DEBUG_LOG(cb, "rdma write completion\n"); 394178784Skmacy cb->stats.write_bytes += cb->rdma_sq_wr.sg_list->length; 395178784Skmacy cb->stats.write_msgs++; 396178784Skmacy cb->state = RDMA_WRITE_COMPLETE; 397256829Snp wake_up_interruptible(&cb->sem); 398178784Skmacy break; 399178784Skmacy 400178784Skmacy case IB_WC_RDMA_READ: 401256829Snp DEBUG_LOG(cb, "rdma read completion\n"); 402178784Skmacy cb->stats.read_bytes += cb->rdma_sq_wr.sg_list->length; 403178784Skmacy cb->stats.read_msgs++; 404178784Skmacy cb->state = RDMA_READ_COMPLETE; 405256829Snp wake_up_interruptible(&cb->sem); 406178784Skmacy break; 407178784Skmacy 408178784Skmacy case IB_WC_RECV: 409256829Snp DEBUG_LOG(cb, "recv completion\n"); 410178784Skmacy cb->stats.recv_bytes += sizeof(cb->recv_buf); 411178784Skmacy cb->stats.recv_msgs++; 412178784Skmacy if (cb->wlat || cb->rlat || cb->bw) 413178784Skmacy ret = server_recv(cb, &wc); 414178784Skmacy else 415178784Skmacy ret = cb->server ? server_recv(cb, &wc) : 416256829Snp client_recv(cb, &wc); 417178784Skmacy if (ret) { 418256829Snp PRINTF(cb, "recv wc error: %d\n", ret); 419178784Skmacy goto error; 420178784Skmacy } 421178784Skmacy 422178784Skmacy ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); 423178784Skmacy if (ret) { 424256829Snp PRINTF(cb, "post recv error: %d\n", 425178784Skmacy ret); 426178784Skmacy goto error; 427178784Skmacy } 428256829Snp wake_up_interruptible(&cb->sem); 429178784Skmacy break; 430178784Skmacy 431178784Skmacy default: 432256829Snp PRINTF(cb, 433256829Snp "%s:%d Unexpected opcode %d, Shutting down\n", 434256829Snp __func__, __LINE__, wc.opcode); 435178784Skmacy goto error; 436178784Skmacy } 437178784Skmacy } 438178784Skmacy if (ret) { 439256829Snp PRINTF(cb, "poll error %d\n", ret); 440178784Skmacy goto error; 441178784Skmacy } 442178784Skmacy return; 443178784Skmacyerror: 444178784Skmacy cb->state = ERROR; 445256829Snp wake_up_interruptible(&cb->sem); 446178784Skmacy} 447178784Skmacy 448178784Skmacystatic int krping_accept(struct krping_cb *cb) 449178784Skmacy{ 450178784Skmacy struct rdma_conn_param conn_param; 451178784Skmacy int ret; 452178784Skmacy 453256829Snp DEBUG_LOG(cb, "accepting client connection request\n"); 454178784Skmacy 455178784Skmacy memset(&conn_param, 0, sizeof conn_param); 456178784Skmacy conn_param.responder_resources = 1; 457178784Skmacy conn_param.initiator_depth = 1; 458178784Skmacy 459178784Skmacy ret = rdma_accept(cb->child_cm_id, &conn_param); 460178784Skmacy if (ret) { 461256829Snp PRINTF(cb, "rdma_accept error: %d\n", ret); 462178784Skmacy return ret; 463178784Skmacy } 464178784Skmacy 465178784Skmacy if (!cb->wlat && !cb->rlat && !cb->bw) { 466256829Snp wait_event_interruptible(cb->sem, cb->state >= CONNECTED); 467178784Skmacy if (cb->state == ERROR) { 468256829Snp PRINTF(cb, "wait for CONNECTED state %d\n", 469256829Snp cb->state); 470178784Skmacy return -1; 471178784Skmacy } 472178784Skmacy } 473178784Skmacy return 0; 474178784Skmacy} 475178784Skmacy 476178784Skmacystatic void krping_setup_wr(struct krping_cb *cb) 477178784Skmacy{ 478256829Snp cb->recv_sgl.addr = cb->recv_dma_addr; 479178784Skmacy cb->recv_sgl.length = sizeof cb->recv_buf; 480256829Snp if (cb->local_dma_lkey) 481256829Snp cb->recv_sgl.lkey = cb->qp->device->local_dma_lkey; 482256829Snp else if (cb->mem == DMA) 483178784Skmacy cb->recv_sgl.lkey = cb->dma_mr->lkey; 484178784Skmacy else 485178784Skmacy cb->recv_sgl.lkey = cb->recv_mr->lkey; 486178784Skmacy cb->rq_wr.sg_list = &cb->recv_sgl; 487178784Skmacy cb->rq_wr.num_sge = 1; 488178784Skmacy 489256829Snp cb->send_sgl.addr = cb->send_dma_addr; 490178784Skmacy cb->send_sgl.length = sizeof cb->send_buf; 491256829Snp if (cb->local_dma_lkey) 492256829Snp cb->send_sgl.lkey = cb->qp->device->local_dma_lkey; 493256829Snp else if (cb->mem == DMA) 494178784Skmacy cb->send_sgl.lkey = cb->dma_mr->lkey; 495178784Skmacy else 496178784Skmacy cb->send_sgl.lkey = cb->send_mr->lkey; 497178784Skmacy 498178784Skmacy cb->sq_wr.opcode = IB_WR_SEND; 499178784Skmacy cb->sq_wr.send_flags = IB_SEND_SIGNALED; 500178784Skmacy cb->sq_wr.sg_list = &cb->send_sgl; 501178784Skmacy cb->sq_wr.num_sge = 1; 502178784Skmacy 503256829Snp if (cb->server || cb->wlat || cb->rlat || cb->bw) { 504256829Snp cb->rdma_sgl.addr = cb->rdma_dma_addr; 505256829Snp if (cb->mem == MR) 506256829Snp cb->rdma_sgl.lkey = cb->rdma_mr->lkey; 507256829Snp cb->rdma_sq_wr.send_flags = IB_SEND_SIGNALED; 508256829Snp cb->rdma_sq_wr.sg_list = &cb->rdma_sgl; 509256829Snp cb->rdma_sq_wr.num_sge = 1; 510256829Snp } 511178784Skmacy 512256829Snp switch(cb->mem) { 513256829Snp case FASTREG: 514256829Snp 515256829Snp /* 516256829Snp * A chain of 2 WRs, INVALDATE_MR + FAST_REG_MR. 517256829Snp * both unsignaled. The client uses them to reregister 518256829Snp * the rdma buffers with a new key each iteration. 519256829Snp */ 520256829Snp cb->fastreg_wr.opcode = IB_WR_FAST_REG_MR; 521256829Snp cb->fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 522256829Snp cb->fastreg_wr.wr.fast_reg.length = cb->size; 523256829Snp cb->fastreg_wr.wr.fast_reg.page_list = cb->page_list; 524256829Snp cb->fastreg_wr.wr.fast_reg.page_list_len = cb->page_list_len; 525256829Snp 526256829Snp cb->invalidate_wr.next = &cb->fastreg_wr; 527256829Snp cb->invalidate_wr.opcode = IB_WR_LOCAL_INV; 528256829Snp break; 529256829Snp case MW: 530256829Snp cb->bind_attr.wr_id = 0xabbaabba; 531256829Snp cb->bind_attr.send_flags = 0; /* unsignaled */ 532256829Snp cb->bind_attr.length = cb->size; 533256829Snp break; 534256829Snp default: 535256829Snp break; 536178784Skmacy } 537178784Skmacy} 538178784Skmacy 539178784Skmacystatic int krping_setup_buffers(struct krping_cb *cb) 540178784Skmacy{ 541178784Skmacy int ret; 542178784Skmacy struct ib_phys_buf buf; 543178784Skmacy u64 iovbase; 544178784Skmacy 545256829Snp DEBUG_LOG(cb, "krping_setup_buffers called on cb %p\n", cb); 546178784Skmacy 547256829Snp cb->recv_dma_addr = dma_map_single(cb->pd->device->dma_device, 548256829Snp &cb->recv_buf, 549256829Snp sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); 550256829Snp pci_unmap_addr_set(cb, recv_mapping, cb->recv_dma_addr); 551256829Snp cb->send_dma_addr = dma_map_single(cb->pd->device->dma_device, 552256829Snp &cb->send_buf, sizeof(cb->send_buf), 553256829Snp DMA_BIDIRECTIONAL); 554256829Snp pci_unmap_addr_set(cb, send_mapping, cb->send_dma_addr); 555256829Snp 556256829Snp if (cb->mem == DMA) { 557178784Skmacy cb->dma_mr = ib_get_dma_mr(cb->pd, IB_ACCESS_LOCAL_WRITE| 558178784Skmacy IB_ACCESS_REMOTE_READ| 559178784Skmacy IB_ACCESS_REMOTE_WRITE); 560178784Skmacy if (IS_ERR(cb->dma_mr)) { 561256829Snp DEBUG_LOG(cb, "reg_dmamr failed\n"); 562256829Snp ret = PTR_ERR(cb->dma_mr); 563256829Snp goto bail; 564178784Skmacy } 565178784Skmacy } else { 566256829Snp if (!cb->local_dma_lkey) { 567256829Snp buf.addr = cb->recv_dma_addr; 568256829Snp buf.size = sizeof cb->recv_buf; 569256829Snp DEBUG_LOG(cb, "recv buf dma_addr %llx size %d\n", buf.addr, 570256829Snp (int)buf.size); 571256829Snp iovbase = cb->recv_dma_addr; 572256829Snp cb->recv_mr = ib_reg_phys_mr(cb->pd, &buf, 1, 573256829Snp IB_ACCESS_LOCAL_WRITE, 574256829Snp &iovbase); 575178784Skmacy 576256829Snp if (IS_ERR(cb->recv_mr)) { 577256829Snp DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); 578256829Snp ret = PTR_ERR(cb->recv_mr); 579256829Snp goto bail; 580256829Snp } 581178784Skmacy 582256829Snp buf.addr = cb->send_dma_addr; 583256829Snp buf.size = sizeof cb->send_buf; 584256829Snp DEBUG_LOG(cb, "send buf dma_addr %llx size %d\n", buf.addr, 585256829Snp (int)buf.size); 586256829Snp iovbase = cb->send_dma_addr; 587256829Snp cb->send_mr = ib_reg_phys_mr(cb->pd, &buf, 1, 588256829Snp 0, &iovbase); 589178784Skmacy 590256829Snp if (IS_ERR(cb->send_mr)) { 591256829Snp DEBUG_LOG(cb, "send_buf reg_mr failed\n"); 592256829Snp ret = PTR_ERR(cb->send_mr); 593256829Snp goto bail; 594256829Snp } 595178784Skmacy } 596178784Skmacy } 597178784Skmacy 598256829Snp cb->rdma_buf = kmalloc(cb->size, GFP_KERNEL); 599178784Skmacy if (!cb->rdma_buf) { 600256829Snp DEBUG_LOG(cb, "rdma_buf malloc failed\n"); 601256829Snp ret = -ENOMEM; 602256829Snp goto bail; 603178784Skmacy } 604178784Skmacy 605256829Snp cb->rdma_dma_addr = dma_map_single(cb->pd->device->dma_device, 606256829Snp cb->rdma_buf, cb->size, 607256829Snp DMA_BIDIRECTIONAL); 608256829Snp pci_unmap_addr_set(cb, rdma_mapping, cb->rdma_dma_addr); 609256829Snp if (cb->mem != DMA) { 610256829Snp switch (cb->mem) { 611256829Snp case FASTREG: 612256829Snp cb->page_list_len = (((cb->size - 1) & PAGE_MASK) + 613256829Snp PAGE_SIZE) >> PAGE_SHIFT; 614256829Snp cb->page_list = ib_alloc_fast_reg_page_list( 615256829Snp cb->pd->device, 616256829Snp cb->page_list_len); 617256829Snp if (IS_ERR(cb->page_list)) { 618256829Snp DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); 619256829Snp ret = PTR_ERR(cb->page_list); 620256829Snp goto bail; 621256829Snp } 622256829Snp cb->fastreg_mr = ib_alloc_fast_reg_mr(cb->pd, 623256829Snp cb->page_list->max_page_list_len); 624256829Snp if (IS_ERR(cb->fastreg_mr)) { 625256829Snp DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); 626256829Snp ret = PTR_ERR(cb->fastreg_mr); 627256829Snp goto bail; 628256829Snp } 629256829Snp DEBUG_LOG(cb, "fastreg rkey 0x%x page_list %p" 630256829Snp " page_list_len %u\n", cb->fastreg_mr->rkey, 631256829Snp cb->page_list, cb->page_list_len); 632256829Snp break; 633256829Snp case MW: 634256829Snp cb->mw = ib_alloc_mw(cb->pd); 635256829Snp if (IS_ERR(cb->mw)) { 636256829Snp DEBUG_LOG(cb, "recv_buf alloc_mw failed\n"); 637256829Snp ret = PTR_ERR(cb->mw); 638256829Snp goto bail; 639256829Snp } 640256829Snp DEBUG_LOG(cb, "mw rkey 0x%x\n", cb->mw->rkey); 641256829Snp /*FALLTHROUGH*/ 642256829Snp case MR: 643256829Snp buf.addr = cb->rdma_dma_addr; 644256829Snp buf.size = cb->size; 645256829Snp iovbase = cb->rdma_dma_addr; 646256829Snp cb->rdma_mr = ib_reg_phys_mr(cb->pd, &buf, 1, 647178784Skmacy IB_ACCESS_REMOTE_READ| 648178784Skmacy IB_ACCESS_REMOTE_WRITE, 649178784Skmacy &iovbase); 650256829Snp if (IS_ERR(cb->rdma_mr)) { 651256829Snp DEBUG_LOG(cb, "rdma_buf reg_mr failed\n"); 652256829Snp ret = PTR_ERR(cb->rdma_mr); 653256829Snp goto bail; 654256829Snp } 655256829Snp DEBUG_LOG(cb, "rdma buf dma_addr %llx size %d mr rkey 0x%x\n", 656256829Snp buf.addr, (int)buf.size, cb->rdma_mr->rkey); 657256829Snp break; 658256829Snp default: 659256829Snp ret = -EINVAL; 660256829Snp goto bail; 661256829Snp break; 662178784Skmacy } 663178784Skmacy } 664178784Skmacy 665178784Skmacy if (!cb->server || cb->wlat || cb->rlat || cb->bw) { 666256829Snp 667256829Snp cb->start_buf = kmalloc(cb->size, GFP_KERNEL); 668178784Skmacy if (!cb->start_buf) { 669256829Snp DEBUG_LOG(cb, "start_buf malloc failed\n"); 670256829Snp ret = -ENOMEM; 671256829Snp goto bail; 672178784Skmacy } 673256829Snp 674256829Snp cb->start_dma_addr = dma_map_single(cb->pd->device->dma_device, 675256829Snp cb->start_buf, cb->size, 676256829Snp DMA_BIDIRECTIONAL); 677256829Snp pci_unmap_addr_set(cb, start_mapping, cb->start_dma_addr); 678256829Snp 679256829Snp if (cb->mem == MR || cb->mem == MW) { 680178784Skmacy unsigned flags = IB_ACCESS_REMOTE_READ; 681178784Skmacy 682256829Snp if (cb->wlat || cb->rlat || cb->bw) 683178784Skmacy flags |= IB_ACCESS_REMOTE_WRITE; 684256829Snp 685256829Snp buf.addr = cb->start_dma_addr; 686178784Skmacy buf.size = cb->size; 687256829Snp DEBUG_LOG(cb, "start buf dma_addr %llx size %d\n", 688256829Snp buf.addr, (int)buf.size); 689256829Snp iovbase = cb->start_dma_addr; 690178784Skmacy cb->start_mr = ib_reg_phys_mr(cb->pd, &buf, 1, 691178784Skmacy flags, 692178784Skmacy &iovbase); 693178784Skmacy 694178784Skmacy if (IS_ERR(cb->start_mr)) { 695256829Snp DEBUG_LOG(cb, "start_buf reg_mr failed\n"); 696178784Skmacy ret = PTR_ERR(cb->start_mr); 697256829Snp goto bail; 698178784Skmacy } 699178784Skmacy } 700178784Skmacy } 701178784Skmacy 702178784Skmacy krping_setup_wr(cb); 703256829Snp DEBUG_LOG(cb, "allocated & registered buffers...\n"); 704178784Skmacy return 0; 705256829Snpbail: 706256829Snp if (cb->fastreg_mr && !IS_ERR(cb->fastreg_mr)) 707256829Snp ib_dereg_mr(cb->fastreg_mr); 708256829Snp if (cb->mw && !IS_ERR(cb->mw)) 709256829Snp ib_dealloc_mw(cb->mw); 710256829Snp if (cb->rdma_mr && !IS_ERR(cb->rdma_mr)) 711178784Skmacy ib_dereg_mr(cb->rdma_mr); 712256829Snp if (cb->page_list && !IS_ERR(cb->page_list)) 713256829Snp ib_free_fast_reg_page_list(cb->page_list); 714256829Snp if (cb->dma_mr && !IS_ERR(cb->dma_mr)) 715178784Skmacy ib_dereg_mr(cb->dma_mr); 716256829Snp if (cb->recv_mr && !IS_ERR(cb->recv_mr)) 717178784Skmacy ib_dereg_mr(cb->recv_mr); 718256829Snp if (cb->send_mr && !IS_ERR(cb->send_mr)) 719178784Skmacy ib_dereg_mr(cb->send_mr); 720256829Snp if (cb->rdma_buf) 721256829Snp kfree(cb->rdma_buf); 722256829Snp if (cb->start_buf) 723256829Snp kfree(cb->start_buf); 724178784Skmacy return ret; 725178784Skmacy} 726178784Skmacy 727178784Skmacystatic void krping_free_buffers(struct krping_cb *cb) 728178784Skmacy{ 729256829Snp DEBUG_LOG(cb, "krping_free_buffers called on cb %p\n", cb); 730178784Skmacy 731256829Snp if (cb->dma_mr) 732256829Snp ib_dereg_mr(cb->dma_mr); 733256829Snp if (cb->send_mr) 734256829Snp ib_dereg_mr(cb->send_mr); 735256829Snp if (cb->recv_mr) 736256829Snp ib_dereg_mr(cb->recv_mr); 737256829Snp if (cb->rdma_mr) 738256829Snp ib_dereg_mr(cb->rdma_mr); 739256829Snp if (cb->start_mr) 740256829Snp ib_dereg_mr(cb->start_mr); 741256829Snp if (cb->fastreg_mr) 742256829Snp ib_dereg_mr(cb->fastreg_mr); 743256829Snp if (cb->mw) 744256829Snp ib_dealloc_mw(cb->mw); 745256829Snp 746178784Skmacy dma_unmap_single(cb->pd->device->dma_device, 747178784Skmacy pci_unmap_addr(cb, recv_mapping), 748178784Skmacy sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); 749178784Skmacy dma_unmap_single(cb->pd->device->dma_device, 750178784Skmacy pci_unmap_addr(cb, send_mapping), 751178784Skmacy sizeof(cb->send_buf), DMA_BIDIRECTIONAL); 752178784Skmacy dma_unmap_single(cb->pd->device->dma_device, 753178784Skmacy pci_unmap_addr(cb, rdma_mapping), 754178784Skmacy cb->size, DMA_BIDIRECTIONAL); 755256829Snp kfree(cb->rdma_buf); 756256829Snp if (cb->start_buf) { 757178784Skmacy dma_unmap_single(cb->pd->device->dma_device, 758178784Skmacy pci_unmap_addr(cb, start_mapping), 759178784Skmacy cb->size, DMA_BIDIRECTIONAL); 760256829Snp kfree(cb->start_buf); 761178784Skmacy } 762178784Skmacy} 763178784Skmacy 764178784Skmacystatic int krping_create_qp(struct krping_cb *cb) 765178784Skmacy{ 766178784Skmacy struct ib_qp_init_attr init_attr; 767178784Skmacy int ret; 768178784Skmacy 769178784Skmacy memset(&init_attr, 0, sizeof(init_attr)); 770178784Skmacy init_attr.cap.max_send_wr = cb->txdepth; 771178784Skmacy init_attr.cap.max_recv_wr = 2; 772178784Skmacy init_attr.cap.max_recv_sge = 1; 773178784Skmacy init_attr.cap.max_send_sge = 1; 774178784Skmacy init_attr.qp_type = IB_QPT_RC; 775178784Skmacy init_attr.send_cq = cb->cq; 776178784Skmacy init_attr.recv_cq = cb->cq; 777256829Snp init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 778178784Skmacy 779178784Skmacy if (cb->server) { 780178784Skmacy ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr); 781178784Skmacy if (!ret) 782178784Skmacy cb->qp = cb->child_cm_id->qp; 783178784Skmacy } else { 784178784Skmacy ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr); 785178784Skmacy if (!ret) 786178784Skmacy cb->qp = cb->cm_id->qp; 787178784Skmacy } 788178784Skmacy 789178784Skmacy return ret; 790178784Skmacy} 791178784Skmacy 792178784Skmacystatic void krping_free_qp(struct krping_cb *cb) 793178784Skmacy{ 794178784Skmacy ib_destroy_qp(cb->qp); 795178784Skmacy ib_destroy_cq(cb->cq); 796178784Skmacy ib_dealloc_pd(cb->pd); 797178784Skmacy} 798178784Skmacy 799178784Skmacystatic int krping_setup_qp(struct krping_cb *cb, struct rdma_cm_id *cm_id) 800178784Skmacy{ 801178784Skmacy int ret; 802178784Skmacy cb->pd = ib_alloc_pd(cm_id->device); 803178784Skmacy if (IS_ERR(cb->pd)) { 804256829Snp PRINTF(cb, "ib_alloc_pd failed\n"); 805178784Skmacy return PTR_ERR(cb->pd); 806178784Skmacy } 807256829Snp DEBUG_LOG(cb, "created pd %p\n", cb->pd); 808178784Skmacy 809256829Snp strlcpy(cb->stats.name, cb->pd->device->name, sizeof(cb->stats.name)); 810254735Snp 811178784Skmacy cb->cq = ib_create_cq(cm_id->device, krping_cq_event_handler, NULL, 812178784Skmacy cb, cb->txdepth * 2, 0); 813178784Skmacy if (IS_ERR(cb->cq)) { 814256829Snp PRINTF(cb, "ib_create_cq failed\n"); 815178784Skmacy ret = PTR_ERR(cb->cq); 816178784Skmacy goto err1; 817178784Skmacy } 818256829Snp DEBUG_LOG(cb, "created cq %p\n", cb->cq); 819178784Skmacy 820256829Snp if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { 821178784Skmacy ret = ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); 822178784Skmacy if (ret) { 823256829Snp PRINTF(cb, "ib_create_cq failed\n"); 824178784Skmacy goto err2; 825178784Skmacy } 826178784Skmacy } 827178784Skmacy 828178784Skmacy ret = krping_create_qp(cb); 829178784Skmacy if (ret) { 830256829Snp PRINTF(cb, "krping_create_qp failed: %d\n", ret); 831178784Skmacy goto err2; 832178784Skmacy } 833256829Snp DEBUG_LOG(cb, "created qp %p\n", cb->qp); 834178784Skmacy return 0; 835178784Skmacyerr2: 836178784Skmacy ib_destroy_cq(cb->cq); 837178784Skmacyerr1: 838178784Skmacy ib_dealloc_pd(cb->pd); 839178784Skmacy return ret; 840178784Skmacy} 841178784Skmacy 842256829Snp/* 843256829Snp * return the (possibly rebound) rkey for the rdma buffer. 844256829Snp * FASTREG mode: invalidate and rebind via fastreg wr. 845256829Snp * MW mode: rebind the MW. 846256829Snp * other modes: just return the mr rkey. 847256829Snp */ 848256829Snpstatic u32 krping_rdma_rkey(struct krping_cb *cb, u64 buf, int post_inv) 849178784Skmacy{ 850256829Snp u32 rkey = 0xffffffff; 851256829Snp u64 p; 852256829Snp struct ib_send_wr *bad_wr; 853256829Snp int i; 854256829Snp int ret; 855178784Skmacy 856256829Snp switch (cb->mem) { 857256829Snp case FASTREG: 858256829Snp cb->invalidate_wr.ex.invalidate_rkey = cb->fastreg_mr->rkey; 859178784Skmacy 860256829Snp /* 861256829Snp * Update the fastreg key. 862256829Snp */ 863256829Snp ib_update_fast_reg_key(cb->fastreg_mr, ++cb->key); 864256829Snp cb->fastreg_wr.wr.fast_reg.rkey = cb->fastreg_mr->rkey; 865256829Snp 866256829Snp /* 867256829Snp * Update the fastreg WR with new buf info. 868256829Snp */ 869256829Snp if (buf == (u64)cb->start_dma_addr) 870256829Snp cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_READ; 871256829Snp else 872256829Snp cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; 873256829Snp cb->fastreg_wr.wr.fast_reg.iova_start = buf; 874256829Snp p = (u64)(buf & PAGE_MASK); 875256829Snp for (i=0; i < cb->fastreg_wr.wr.fast_reg.page_list_len; 876256829Snp i++, p += PAGE_SIZE) { 877256829Snp cb->page_list->page_list[i] = p; 878256829Snp DEBUG_LOG(cb, "page_list[%d] 0x%llx\n", i, p); 879256829Snp } 880256829Snp 881256829Snp DEBUG_LOG(cb, "post_inv = %d, fastreg new rkey 0x%x shift %u len %u" 882256829Snp " iova_start %llx page_list_len %u\n", 883256829Snp post_inv, 884256829Snp cb->fastreg_wr.wr.fast_reg.rkey, 885256829Snp cb->fastreg_wr.wr.fast_reg.page_shift, 886256829Snp cb->fastreg_wr.wr.fast_reg.length, 887256829Snp cb->fastreg_wr.wr.fast_reg.iova_start, 888256829Snp cb->fastreg_wr.wr.fast_reg.page_list_len); 889256829Snp 890256829Snp if (post_inv) 891256829Snp ret = ib_post_send(cb->qp, &cb->invalidate_wr, &bad_wr); 892256829Snp else 893256829Snp ret = ib_post_send(cb->qp, &cb->fastreg_wr, &bad_wr); 894256829Snp if (ret) { 895256829Snp PRINTF(cb, "post send error %d\n", ret); 896256829Snp cb->state = ERROR; 897256829Snp } 898256829Snp rkey = cb->fastreg_mr->rkey; 899256829Snp break; 900256829Snp case MW: 901256829Snp /* 902256829Snp * Update the MW with new buf info. 903256829Snp */ 904256829Snp if (buf == (u64)cb->start_dma_addr) { 905256829Snp cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_READ; 906256829Snp cb->bind_attr.mr = cb->start_mr; 907256829Snp } else { 908256829Snp cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_WRITE; 909256829Snp cb->bind_attr.mr = cb->rdma_mr; 910256829Snp } 911256829Snp cb->bind_attr.addr = buf; 912256829Snp DEBUG_LOG(cb, "binding mw rkey 0x%x to buf %llx mr rkey 0x%x\n", 913256829Snp cb->mw->rkey, buf, cb->bind_attr.mr->rkey); 914256829Snp ret = ib_bind_mw(cb->qp, cb->mw, &cb->bind_attr); 915256829Snp if (ret) { 916256829Snp PRINTF(cb, "bind mw error %d\n", ret); 917256829Snp cb->state = ERROR; 918256829Snp } else 919256829Snp rkey = cb->mw->rkey; 920256829Snp break; 921256829Snp case MR: 922256829Snp if (buf == (u64)cb->start_dma_addr) 923256829Snp rkey = cb->start_mr->rkey; 924256829Snp else 925256829Snp rkey = cb->rdma_mr->rkey; 926256829Snp break; 927256829Snp case DMA: 928256829Snp rkey = cb->dma_mr->rkey; 929256829Snp break; 930256829Snp default: 931256829Snp PRINTF(cb, "%s:%d case ERROR\n", __func__, __LINE__); 932256829Snp cb->state = ERROR; 933256829Snp break; 934256829Snp } 935256829Snp return rkey; 936178784Skmacy} 937178784Skmacy 938256829Snpstatic void krping_format_send(struct krping_cb *cb, u64 buf) 939256829Snp{ 940256829Snp struct krping_rdma_info *info = &cb->send_buf; 941256829Snp u32 rkey; 942256829Snp 943256829Snp /* 944256829Snp * Client side will do fastreg or mw bind before 945256829Snp * advertising the rdma buffer. Server side 946256829Snp * sends have no data. 947256829Snp */ 948256829Snp if (!cb->server || cb->wlat || cb->rlat || cb->bw) { 949256829Snp rkey = krping_rdma_rkey(cb, buf, !cb->server_invalidate); 950256829Snp info->buf = htonll(buf); 951256829Snp info->rkey = htonl(rkey); 952256829Snp info->size = htonl(cb->size); 953256829Snp DEBUG_LOG(cb, "RDMA addr %llx rkey %x len %d\n", 954256829Snp (unsigned long long)buf, rkey, cb->size); 955256829Snp } 956256829Snp} 957256829Snp 958178784Skmacystatic void krping_test_server(struct krping_cb *cb) 959178784Skmacy{ 960256829Snp struct ib_send_wr *bad_wr, inv; 961178784Skmacy int ret; 962178784Skmacy 963178784Skmacy while (1) { 964178784Skmacy /* Wait for client's Start STAG/TO/Len */ 965256829Snp wait_event_interruptible(cb->sem, cb->state >= RDMA_READ_ADV); 966178784Skmacy if (cb->state != RDMA_READ_ADV) { 967256829Snp PRINTF(cb, "wait for RDMA_READ_ADV state %d\n", 968178784Skmacy cb->state); 969178784Skmacy break; 970178784Skmacy } 971178784Skmacy 972256829Snp DEBUG_LOG(cb, "server received sink adv\n"); 973178784Skmacy 974178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 975178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 976178784Skmacy cb->rdma_sq_wr.sg_list->length = cb->remote_len; 977256829Snp cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 1); 978256829Snp cb->rdma_sq_wr.next = NULL; 979178784Skmacy 980256829Snp /* Issue RDMA Read. */ 981256829Snp if (cb->read_inv) 982256829Snp cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ_WITH_INV; 983256829Snp else { 984256829Snp 985256829Snp cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; 986256829Snp if (cb->mem == FASTREG) { 987256829Snp /* 988256829Snp * Immediately follow the read with a 989256829Snp * fenced LOCAL_INV. 990256829Snp */ 991256829Snp cb->rdma_sq_wr.next = &inv; 992256829Snp memset(&inv, 0, sizeof inv); 993256829Snp inv.opcode = IB_WR_LOCAL_INV; 994256829Snp inv.ex.invalidate_rkey = cb->fastreg_mr->rkey; 995256829Snp inv.send_flags = IB_SEND_FENCE; 996256829Snp } 997256829Snp } 998256829Snp 999178784Skmacy ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); 1000178784Skmacy if (ret) { 1001256829Snp PRINTF(cb, "post send error %d\n", ret); 1002178784Skmacy break; 1003178784Skmacy } 1004256829Snp cb->rdma_sq_wr.next = NULL; 1005178784Skmacy 1006256829Snp DEBUG_LOG(cb, "server posted rdma read req \n"); 1007256829Snp 1008178784Skmacy /* Wait for read completion */ 1009256829Snp wait_event_interruptible(cb->sem, 1010256829Snp cb->state >= RDMA_READ_COMPLETE); 1011178784Skmacy if (cb->state != RDMA_READ_COMPLETE) { 1012256829Snp PRINTF(cb, 1013178784Skmacy "wait for RDMA_READ_COMPLETE state %d\n", 1014178784Skmacy cb->state); 1015178784Skmacy break; 1016178784Skmacy } 1017256829Snp DEBUG_LOG(cb, "server received read complete\n"); 1018178784Skmacy 1019178784Skmacy /* Display data in recv buf */ 1020178784Skmacy if (cb->verbose) 1021256829Snp PRINTF(cb, "server ping data: %s\n", 1022256829Snp cb->rdma_buf); 1023178784Skmacy 1024178784Skmacy /* Tell client to continue */ 1025256829Snp if (cb->server && cb->server_invalidate) { 1026256829Snp cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; 1027256829Snp cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; 1028256829Snp DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); 1029256829Snp } 1030178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1031178784Skmacy if (ret) { 1032256829Snp PRINTF(cb, "post send error %d\n", ret); 1033178784Skmacy break; 1034178784Skmacy } 1035256829Snp DEBUG_LOG(cb, "server posted go ahead\n"); 1036178784Skmacy 1037178784Skmacy /* Wait for client's RDMA STAG/TO/Len */ 1038256829Snp wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); 1039178784Skmacy if (cb->state != RDMA_WRITE_ADV) { 1040256829Snp PRINTF(cb, 1041178784Skmacy "wait for RDMA_WRITE_ADV state %d\n", 1042178784Skmacy cb->state); 1043178784Skmacy break; 1044178784Skmacy } 1045256829Snp DEBUG_LOG(cb, "server received sink adv\n"); 1046178784Skmacy 1047178784Skmacy /* RDMA Write echo data */ 1048178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; 1049178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1050178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1051178784Skmacy cb->rdma_sq_wr.sg_list->length = strlen(cb->rdma_buf) + 1; 1052256829Snp if (cb->local_dma_lkey) 1053256829Snp cb->rdma_sgl.lkey = cb->qp->device->local_dma_lkey; 1054256829Snp else 1055256829Snp cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 0); 1056256829Snp 1057256829Snp DEBUG_LOG(cb, "rdma write from lkey %x laddr %llx len %d\n", 1058178784Skmacy cb->rdma_sq_wr.sg_list->lkey, 1059178784Skmacy (unsigned long long)cb->rdma_sq_wr.sg_list->addr, 1060178784Skmacy cb->rdma_sq_wr.sg_list->length); 1061178784Skmacy 1062178784Skmacy ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); 1063178784Skmacy if (ret) { 1064256829Snp PRINTF(cb, "post send error %d\n", ret); 1065178784Skmacy break; 1066178784Skmacy } 1067178784Skmacy 1068178784Skmacy /* Wait for completion */ 1069256829Snp ret = wait_event_interruptible(cb->sem, cb->state >= 1070256829Snp RDMA_WRITE_COMPLETE); 1071178784Skmacy if (cb->state != RDMA_WRITE_COMPLETE) { 1072256829Snp PRINTF(cb, 1073178784Skmacy "wait for RDMA_WRITE_COMPLETE state %d\n", 1074178784Skmacy cb->state); 1075178784Skmacy break; 1076178784Skmacy } 1077256829Snp DEBUG_LOG(cb, "server rdma write complete \n"); 1078178784Skmacy 1079178784Skmacy cb->state = CONNECTED; 1080178784Skmacy 1081178784Skmacy /* Tell client to begin again */ 1082256829Snp if (cb->server && cb->server_invalidate) { 1083256829Snp cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; 1084256829Snp cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; 1085256829Snp DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); 1086256829Snp } 1087178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1088178784Skmacy if (ret) { 1089256829Snp PRINTF(cb, "post send error %d\n", ret); 1090178784Skmacy break; 1091178784Skmacy } 1092256829Snp DEBUG_LOG(cb, "server posted go ahead\n"); 1093178784Skmacy } 1094178784Skmacy} 1095178784Skmacy 1096178784Skmacystatic void rlat_test(struct krping_cb *cb) 1097178784Skmacy{ 1098178784Skmacy int scnt; 1099178784Skmacy int iters = cb->count; 1100178784Skmacy struct timeval start_tv, stop_tv; 1101178784Skmacy int ret; 1102178784Skmacy struct ib_wc wc; 1103178784Skmacy struct ib_send_wr *bad_wr; 1104178784Skmacy int ne; 1105178784Skmacy 1106178784Skmacy scnt = 0; 1107178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; 1108178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1109178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1110178784Skmacy cb->rdma_sq_wr.sg_list->length = cb->size; 1111178784Skmacy 1112178784Skmacy microtime(&start_tv); 1113256829Snp if (!cb->poll) { 1114256829Snp cb->state = RDMA_READ_ADV; 1115256829Snp ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); 1116256829Snp } 1117178784Skmacy while (scnt < iters) { 1118178784Skmacy 1119256829Snp cb->state = RDMA_READ_ADV; 1120178784Skmacy ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); 1121178784Skmacy if (ret) { 1122256829Snp PRINTF(cb, 1123178784Skmacy "Couldn't post send: ret=%d scnt %d\n", 1124178784Skmacy ret, scnt); 1125178784Skmacy return; 1126178784Skmacy } 1127178784Skmacy 1128178784Skmacy do { 1129178784Skmacy if (!cb->poll) { 1130256829Snp wait_event_interruptible(cb->sem, 1131256829Snp cb->state != RDMA_READ_ADV); 1132178784Skmacy if (cb->state == RDMA_READ_COMPLETE) { 1133178784Skmacy ne = 1; 1134256829Snp ib_req_notify_cq(cb->cq, 1135256829Snp IB_CQ_NEXT_COMP); 1136178784Skmacy } else { 1137178784Skmacy ne = -1; 1138178784Skmacy } 1139178784Skmacy } else 1140178784Skmacy ne = ib_poll_cq(cb->cq, 1, &wc); 1141178784Skmacy if (cb->state == ERROR) { 1142256829Snp PRINTF(cb, 1143256829Snp "state == ERROR...bailing scnt %d\n", 1144256829Snp scnt); 1145178784Skmacy return; 1146178784Skmacy } 1147178784Skmacy } while (ne == 0); 1148178784Skmacy 1149178784Skmacy if (ne < 0) { 1150256829Snp PRINTF(cb, "poll CQ failed %d\n", ne); 1151178784Skmacy return; 1152178784Skmacy } 1153256829Snp if (cb->poll && wc.status != IB_WC_SUCCESS) { 1154256829Snp PRINTF(cb, "Completion wth error at %s:\n", 1155178784Skmacy cb->server ? "server" : "client"); 1156256829Snp PRINTF(cb, "Failed status %d: wr_id %d\n", 1157178784Skmacy wc.status, (int) wc.wr_id); 1158178784Skmacy return; 1159178784Skmacy } 1160178784Skmacy ++scnt; 1161178784Skmacy } 1162178784Skmacy microtime(&stop_tv); 1163178784Skmacy 1164178784Skmacy if (stop_tv.tv_usec < start_tv.tv_usec) { 1165178784Skmacy stop_tv.tv_usec += 1000000; 1166178784Skmacy stop_tv.tv_sec -= 1; 1167178784Skmacy } 1168178784Skmacy 1169256829Snp PRINTF(cb, "delta sec %lu delta usec %lu iter %d size %d\n", 1170178784Skmacy stop_tv.tv_sec - start_tv.tv_sec, 1171178784Skmacy stop_tv.tv_usec - start_tv.tv_usec, 1172178784Skmacy scnt, cb->size); 1173178784Skmacy} 1174178784Skmacy 1175178784Skmacystatic void wlat_test(struct krping_cb *cb) 1176178784Skmacy{ 1177178784Skmacy int ccnt, scnt, rcnt; 1178178784Skmacy int iters=cb->count; 1179178784Skmacy volatile char *poll_buf = (char *) cb->start_buf; 1180178784Skmacy char *buf = (char *)cb->rdma_buf; 1181178784Skmacy struct timeval start_tv, stop_tv; 1182178784Skmacy cycles_t *post_cycles_start, *post_cycles_stop; 1183178784Skmacy cycles_t *poll_cycles_start, *poll_cycles_stop; 1184178784Skmacy cycles_t *last_poll_cycles_start; 1185178784Skmacy cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; 1186178784Skmacy int i; 1187178784Skmacy int cycle_iters = 1000; 1188178784Skmacy 1189256829Snp ccnt = 0; 1190256829Snp scnt = 0; 1191256829Snp rcnt = 0; 1192256829Snp 1193256829Snp post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1194256829Snp if (!post_cycles_start) { 1195256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1196178784Skmacy return; 1197178784Skmacy } 1198256829Snp post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1199256829Snp if (!post_cycles_stop) { 1200256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1201256829Snp return; 1202256829Snp } 1203256829Snp poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1204256829Snp if (!poll_cycles_start) { 1205256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1206256829Snp return; 1207256829Snp } 1208256829Snp poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1209256829Snp if (!poll_cycles_stop) { 1210256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1211256829Snp return; 1212256829Snp } 1213256829Snp last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), 1214256829Snp GFP_KERNEL); 1215256829Snp if (!last_poll_cycles_start) { 1216256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1217256829Snp return; 1218256829Snp } 1219178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; 1220178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1221178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1222178784Skmacy cb->rdma_sq_wr.sg_list->length = cb->size; 1223178784Skmacy 1224178784Skmacy if (cycle_iters > iters) 1225178784Skmacy cycle_iters = iters; 1226178784Skmacy microtime(&start_tv); 1227178784Skmacy while (scnt < iters || ccnt < iters || rcnt < iters) { 1228178784Skmacy 1229178784Skmacy /* Wait till buffer changes. */ 1230178784Skmacy if (rcnt < iters && !(scnt < 1 && !cb->server)) { 1231178784Skmacy ++rcnt; 1232178784Skmacy while (*poll_buf != (char)rcnt) { 1233178784Skmacy if (cb->state == ERROR) { 1234256829Snp PRINTF(cb, 1235256829Snp "state = ERROR, bailing\n"); 1236178784Skmacy return; 1237178784Skmacy } 1238178784Skmacy } 1239178784Skmacy } 1240178784Skmacy 1241178784Skmacy if (scnt < iters) { 1242178784Skmacy struct ib_send_wr *bad_wr; 1243178784Skmacy 1244178784Skmacy *buf = (char)scnt+1; 1245178784Skmacy if (scnt < cycle_iters) 1246178784Skmacy post_cycles_start[scnt] = get_cycles(); 1247178784Skmacy if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { 1248256829Snp PRINTF(cb, 1249256829Snp "Couldn't post send: scnt=%d\n", 1250178784Skmacy scnt); 1251178784Skmacy return; 1252178784Skmacy } 1253178784Skmacy if (scnt < cycle_iters) 1254178784Skmacy post_cycles_stop[scnt] = get_cycles(); 1255178784Skmacy scnt++; 1256178784Skmacy } 1257178784Skmacy 1258178784Skmacy if (ccnt < iters) { 1259178784Skmacy struct ib_wc wc; 1260178784Skmacy int ne; 1261178784Skmacy 1262178784Skmacy if (ccnt < cycle_iters) 1263178784Skmacy poll_cycles_start[ccnt] = get_cycles(); 1264178784Skmacy do { 1265178784Skmacy if (ccnt < cycle_iters) 1266256829Snp last_poll_cycles_start[ccnt] = 1267256829Snp get_cycles(); 1268178784Skmacy ne = ib_poll_cq(cb->cq, 1, &wc); 1269178784Skmacy } while (ne == 0); 1270178784Skmacy if (ccnt < cycle_iters) 1271178784Skmacy poll_cycles_stop[ccnt] = get_cycles(); 1272178784Skmacy ++ccnt; 1273178784Skmacy 1274178784Skmacy if (ne < 0) { 1275256829Snp PRINTF(cb, "poll CQ failed %d\n", ne); 1276178784Skmacy return; 1277178784Skmacy } 1278178784Skmacy if (wc.status != IB_WC_SUCCESS) { 1279256829Snp PRINTF(cb, 1280256829Snp "Completion wth error at %s:\n", 1281178784Skmacy cb->server ? "server" : "client"); 1282256829Snp PRINTF(cb, 1283256829Snp "Failed status %d: wr_id %d\n", 1284178784Skmacy wc.status, (int) wc.wr_id); 1285256829Snp PRINTF(cb, 1286256829Snp "scnt=%d, rcnt=%d, ccnt=%d\n", 1287178784Skmacy scnt, rcnt, ccnt); 1288178784Skmacy return; 1289178784Skmacy } 1290178784Skmacy } 1291178784Skmacy } 1292178784Skmacy microtime(&stop_tv); 1293178784Skmacy 1294178784Skmacy if (stop_tv.tv_usec < start_tv.tv_usec) { 1295178784Skmacy stop_tv.tv_usec += 1000000; 1296178784Skmacy stop_tv.tv_sec -= 1; 1297178784Skmacy } 1298178784Skmacy 1299178784Skmacy for (i=0; i < cycle_iters; i++) { 1300178784Skmacy sum_post += post_cycles_stop[i] - post_cycles_start[i]; 1301178784Skmacy sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; 1302256829Snp sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; 1303178784Skmacy } 1304256829Snp PRINTF(cb, 1305256829Snp "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" 1306256829Snp " sum_post %llu sum_poll %llu sum_last_poll %llu\n", 1307178784Skmacy stop_tv.tv_sec - start_tv.tv_sec, 1308178784Skmacy stop_tv.tv_usec - start_tv.tv_usec, 1309178784Skmacy scnt, cb->size, cycle_iters, 1310178784Skmacy (unsigned long long)sum_post, (unsigned long long)sum_poll, 1311178784Skmacy (unsigned long long)sum_last_poll); 1312256829Snp kfree(post_cycles_start); 1313256829Snp kfree(post_cycles_stop); 1314256829Snp kfree(poll_cycles_start); 1315256829Snp kfree(poll_cycles_stop); 1316256829Snp kfree(last_poll_cycles_start); 1317178784Skmacy} 1318178784Skmacy 1319178784Skmacystatic void bw_test(struct krping_cb *cb) 1320178784Skmacy{ 1321178784Skmacy int ccnt, scnt, rcnt; 1322178784Skmacy int iters=cb->count; 1323178784Skmacy struct timeval start_tv, stop_tv; 1324178784Skmacy cycles_t *post_cycles_start, *post_cycles_stop; 1325178784Skmacy cycles_t *poll_cycles_start, *poll_cycles_stop; 1326178784Skmacy cycles_t *last_poll_cycles_start; 1327178784Skmacy cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; 1328178784Skmacy int i; 1329178784Skmacy int cycle_iters = 1000; 1330178784Skmacy 1331256829Snp ccnt = 0; 1332256829Snp scnt = 0; 1333256829Snp rcnt = 0; 1334256829Snp 1335256829Snp post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1336256829Snp if (!post_cycles_start) { 1337256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1338178784Skmacy return; 1339178784Skmacy } 1340256829Snp post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1341256829Snp if (!post_cycles_stop) { 1342256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1343256829Snp return; 1344256829Snp } 1345256829Snp poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1346256829Snp if (!poll_cycles_start) { 1347256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1348256829Snp return; 1349256829Snp } 1350256829Snp poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1351256829Snp if (!poll_cycles_stop) { 1352256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1353256829Snp return; 1354256829Snp } 1355256829Snp last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), 1356256829Snp GFP_KERNEL); 1357256829Snp if (!last_poll_cycles_start) { 1358256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1359256829Snp return; 1360256829Snp } 1361178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; 1362178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1363178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1364178784Skmacy cb->rdma_sq_wr.sg_list->length = cb->size; 1365178784Skmacy 1366178784Skmacy if (cycle_iters > iters) 1367178784Skmacy cycle_iters = iters; 1368178784Skmacy microtime(&start_tv); 1369178784Skmacy while (scnt < iters || ccnt < iters) { 1370178784Skmacy 1371178784Skmacy while (scnt < iters && scnt - ccnt < cb->txdepth) { 1372178784Skmacy struct ib_send_wr *bad_wr; 1373178784Skmacy 1374178784Skmacy if (scnt < cycle_iters) 1375178784Skmacy post_cycles_start[scnt] = get_cycles(); 1376178784Skmacy if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { 1377256829Snp PRINTF(cb, 1378256829Snp "Couldn't post send: scnt=%d\n", 1379178784Skmacy scnt); 1380178784Skmacy return; 1381178784Skmacy } 1382178784Skmacy if (scnt < cycle_iters) 1383178784Skmacy post_cycles_stop[scnt] = get_cycles(); 1384178784Skmacy ++scnt; 1385178784Skmacy } 1386178784Skmacy 1387178784Skmacy if (ccnt < iters) { 1388178784Skmacy int ne; 1389178784Skmacy struct ib_wc wc; 1390178784Skmacy 1391178784Skmacy if (ccnt < cycle_iters) 1392178784Skmacy poll_cycles_start[ccnt] = get_cycles(); 1393178784Skmacy do { 1394178784Skmacy if (ccnt < cycle_iters) 1395256829Snp last_poll_cycles_start[ccnt] = 1396256829Snp get_cycles(); 1397178784Skmacy ne = ib_poll_cq(cb->cq, 1, &wc); 1398178784Skmacy } while (ne == 0); 1399178784Skmacy if (ccnt < cycle_iters) 1400178784Skmacy poll_cycles_stop[ccnt] = get_cycles(); 1401178784Skmacy ccnt += 1; 1402178784Skmacy 1403178784Skmacy if (ne < 0) { 1404256829Snp PRINTF(cb, "poll CQ failed %d\n", ne); 1405178784Skmacy return; 1406178784Skmacy } 1407178784Skmacy if (wc.status != IB_WC_SUCCESS) { 1408256829Snp PRINTF(cb, 1409256829Snp "Completion wth error at %s:\n", 1410178784Skmacy cb->server ? "server" : "client"); 1411256829Snp PRINTF(cb, 1412256829Snp "Failed status %d: wr_id %d\n", 1413178784Skmacy wc.status, (int) wc.wr_id); 1414178784Skmacy return; 1415178784Skmacy } 1416178784Skmacy } 1417178784Skmacy } 1418178784Skmacy microtime(&stop_tv); 1419178784Skmacy 1420178784Skmacy if (stop_tv.tv_usec < start_tv.tv_usec) { 1421178784Skmacy stop_tv.tv_usec += 1000000; 1422178784Skmacy stop_tv.tv_sec -= 1; 1423178784Skmacy } 1424178784Skmacy 1425178784Skmacy for (i=0; i < cycle_iters; i++) { 1426178784Skmacy sum_post += post_cycles_stop[i] - post_cycles_start[i]; 1427178784Skmacy sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; 1428256829Snp sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; 1429178784Skmacy } 1430256829Snp PRINTF(cb, 1431256829Snp "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" 1432256829Snp " sum_post %llu sum_poll %llu sum_last_poll %llu\n", 1433178784Skmacy stop_tv.tv_sec - start_tv.tv_sec, 1434178784Skmacy stop_tv.tv_usec - start_tv.tv_usec, 1435178784Skmacy scnt, cb->size, cycle_iters, 1436178784Skmacy (unsigned long long)sum_post, (unsigned long long)sum_poll, 1437178784Skmacy (unsigned long long)sum_last_poll); 1438256829Snp kfree(post_cycles_start); 1439256829Snp kfree(post_cycles_stop); 1440256829Snp kfree(poll_cycles_start); 1441256829Snp kfree(poll_cycles_stop); 1442256829Snp kfree(last_poll_cycles_start); 1443178784Skmacy} 1444178784Skmacy 1445178784Skmacystatic void krping_rlat_test_server(struct krping_cb *cb) 1446178784Skmacy{ 1447178784Skmacy struct ib_send_wr *bad_wr; 1448178784Skmacy struct ib_wc wc; 1449178784Skmacy int ret; 1450178784Skmacy 1451178784Skmacy /* Spin waiting for client's Start STAG/TO/Len */ 1452178784Skmacy while (cb->state < RDMA_READ_ADV) { 1453178784Skmacy krping_cq_event_handler(cb->cq, cb); 1454178784Skmacy } 1455178784Skmacy 1456178784Skmacy /* Send STAG/TO/Len to client */ 1457256829Snp krping_format_send(cb, cb->start_dma_addr); 1458178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1459178784Skmacy if (ret) { 1460256829Snp PRINTF(cb, "post send error %d\n", ret); 1461178784Skmacy return; 1462178784Skmacy } 1463178784Skmacy 1464178784Skmacy /* Spin waiting for send completion */ 1465178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1466178784Skmacy if (ret < 0) { 1467256829Snp PRINTF(cb, "poll error %d\n", ret); 1468178784Skmacy return; 1469178784Skmacy } 1470178784Skmacy if (wc.status) { 1471256829Snp PRINTF(cb, "send completiong error %d\n", wc.status); 1472178784Skmacy return; 1473178784Skmacy } 1474178784Skmacy 1475256829Snp wait_event_interruptible(cb->sem, cb->state == ERROR); 1476178784Skmacy} 1477178784Skmacy 1478178784Skmacystatic void krping_wlat_test_server(struct krping_cb *cb) 1479178784Skmacy{ 1480178784Skmacy struct ib_send_wr *bad_wr; 1481178784Skmacy struct ib_wc wc; 1482178784Skmacy int ret; 1483178784Skmacy 1484178784Skmacy /* Spin waiting for client's Start STAG/TO/Len */ 1485178784Skmacy while (cb->state < RDMA_READ_ADV) { 1486178784Skmacy krping_cq_event_handler(cb->cq, cb); 1487178784Skmacy } 1488178784Skmacy 1489178784Skmacy /* Send STAG/TO/Len to client */ 1490256829Snp krping_format_send(cb, cb->start_dma_addr); 1491178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1492178784Skmacy if (ret) { 1493256829Snp PRINTF(cb, "post send error %d\n", ret); 1494178784Skmacy return; 1495178784Skmacy } 1496178784Skmacy 1497178784Skmacy /* Spin waiting for send completion */ 1498178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1499178784Skmacy if (ret < 0) { 1500256829Snp PRINTF(cb, "poll error %d\n", ret); 1501178784Skmacy return; 1502178784Skmacy } 1503178784Skmacy if (wc.status) { 1504256829Snp PRINTF(cb, "send completiong error %d\n", wc.status); 1505178784Skmacy return; 1506178784Skmacy } 1507178784Skmacy 1508178784Skmacy wlat_test(cb); 1509256829Snp wait_event_interruptible(cb->sem, cb->state == ERROR); 1510178784Skmacy} 1511178784Skmacy 1512178784Skmacystatic void krping_bw_test_server(struct krping_cb *cb) 1513178784Skmacy{ 1514178784Skmacy struct ib_send_wr *bad_wr; 1515178784Skmacy struct ib_wc wc; 1516178784Skmacy int ret; 1517178784Skmacy 1518178784Skmacy /* Spin waiting for client's Start STAG/TO/Len */ 1519178784Skmacy while (cb->state < RDMA_READ_ADV) { 1520178784Skmacy krping_cq_event_handler(cb->cq, cb); 1521178784Skmacy } 1522178784Skmacy 1523178784Skmacy /* Send STAG/TO/Len to client */ 1524256829Snp krping_format_send(cb, cb->start_dma_addr); 1525178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1526178784Skmacy if (ret) { 1527256829Snp PRINTF(cb, "post send error %d\n", ret); 1528178784Skmacy return; 1529178784Skmacy } 1530178784Skmacy 1531178784Skmacy /* Spin waiting for send completion */ 1532178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1533178784Skmacy if (ret < 0) { 1534256829Snp PRINTF(cb, "poll error %d\n", ret); 1535178784Skmacy return; 1536178784Skmacy } 1537178784Skmacy if (wc.status) { 1538256829Snp PRINTF(cb, "send completiong error %d\n", wc.status); 1539178784Skmacy return; 1540178784Skmacy } 1541178784Skmacy 1542178784Skmacy if (cb->duplex) 1543178784Skmacy bw_test(cb); 1544256829Snp wait_event_interruptible(cb->sem, cb->state == ERROR); 1545178784Skmacy} 1546178784Skmacy 1547256829Snpstatic int fastreg_supported(struct krping_cb *cb) 1548256829Snp{ 1549256829Snp struct ib_device *dev = cb->child_cm_id->device; 1550256829Snp struct ib_device_attr attr; 1551256829Snp int ret; 1552256829Snp 1553256829Snp ret = ib_query_device(dev, &attr); 1554256829Snp if (ret) { 1555256829Snp PRINTF(cb, "ib_query_device failed ret %d\n", ret); 1556256829Snp return 0; 1557256829Snp } 1558256829Snp if (!(attr.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) { 1559256829Snp PRINTF(cb, "Fastreg not supported - device_cap_flags 0x%x\n", 1560256829Snp attr.device_cap_flags); 1561256829Snp return 0; 1562256829Snp } 1563256829Snp DEBUG_LOG(cb, "Fastreg supported - device_cap_flags 0x%x\n", 1564256829Snp attr.device_cap_flags); 1565256829Snp return 1; 1566256829Snp} 1567256829Snp 1568178784Skmacystatic int krping_bind_server(struct krping_cb *cb) 1569178784Skmacy{ 1570325602Shselasky union { 1571325602Shselasky struct sockaddr_in v4; 1572325602Shselasky struct sockaddr_in6 v6; 1573325602Shselasky } sin; 1574178784Skmacy int ret; 1575178784Skmacy 1576178784Skmacy memset(&sin, 0, sizeof(sin)); 1577178784Skmacy 1578325602Shselasky switch (cb->addr_type) { 1579325602Shselasky case AF_INET: 1580325602Shselasky sin.v4.sin_len = sizeof sin.v4; 1581325602Shselasky sin.v4.sin_family = AF_INET; 1582325602Shselasky sin.v4.sin_addr = cb->addr.v4; 1583325602Shselasky sin.v4.sin_port = cb->port; 1584325602Shselasky break; 1585325602Shselasky case AF_INET6: 1586325602Shselasky sin.v6.sin6_len = sizeof sin.v6; 1587325602Shselasky sin.v6.sin6_family = AF_INET6; 1588325602Shselasky sin.v6.sin6_addr = cb->addr.v6; 1589325602Shselasky sin.v6.sin6_port = cb->port; 1590325602Shselasky break; 1591325602Shselasky default: 1592325602Shselasky return (-EINVAL); 1593325602Shselasky } 1594325602Shselasky 1595178784Skmacy ret = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &sin); 1596178784Skmacy if (ret) { 1597256829Snp PRINTF(cb, "rdma_bind_addr error %d\n", ret); 1598178784Skmacy return ret; 1599178784Skmacy } 1600256829Snp DEBUG_LOG(cb, "rdma_bind_addr successful\n"); 1601178784Skmacy 1602256829Snp DEBUG_LOG(cb, "rdma_listen\n"); 1603178784Skmacy ret = rdma_listen(cb->cm_id, 3); 1604178784Skmacy if (ret) { 1605256829Snp PRINTF(cb, "rdma_listen failed: %d\n", ret); 1606178784Skmacy return ret; 1607178784Skmacy } 1608178784Skmacy 1609256829Snp wait_event_interruptible(cb->sem, cb->state >= CONNECT_REQUEST); 1610178784Skmacy if (cb->state != CONNECT_REQUEST) { 1611256829Snp PRINTF(cb, "wait for CONNECT_REQUEST state %d\n", 1612178784Skmacy cb->state); 1613178784Skmacy return -1; 1614178784Skmacy } 1615178784Skmacy 1616256829Snp if (cb->mem == FASTREG && !fastreg_supported(cb)) 1617256829Snp return -EINVAL; 1618256829Snp 1619178784Skmacy return 0; 1620178784Skmacy} 1621178784Skmacy 1622178784Skmacystatic void krping_run_server(struct krping_cb *cb) 1623178784Skmacy{ 1624178784Skmacy struct ib_recv_wr *bad_wr; 1625178784Skmacy int ret; 1626178784Skmacy 1627178784Skmacy ret = krping_bind_server(cb); 1628178784Skmacy if (ret) 1629178784Skmacy return; 1630178784Skmacy 1631178784Skmacy ret = krping_setup_qp(cb, cb->child_cm_id); 1632178784Skmacy if (ret) { 1633256829Snp PRINTF(cb, "setup_qp failed: %d\n", ret); 1634256829Snp goto err0; 1635178784Skmacy } 1636178784Skmacy 1637178784Skmacy ret = krping_setup_buffers(cb); 1638178784Skmacy if (ret) { 1639256829Snp PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); 1640178784Skmacy goto err1; 1641178784Skmacy } 1642178784Skmacy 1643178784Skmacy ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); 1644178784Skmacy if (ret) { 1645256829Snp PRINTF(cb, "ib_post_recv failed: %d\n", ret); 1646178784Skmacy goto err2; 1647178784Skmacy } 1648178784Skmacy 1649178784Skmacy ret = krping_accept(cb); 1650178784Skmacy if (ret) { 1651256829Snp PRINTF(cb, "connect error %d\n", ret); 1652178784Skmacy goto err2; 1653178784Skmacy } 1654178784Skmacy 1655178784Skmacy if (cb->wlat) 1656178784Skmacy krping_wlat_test_server(cb); 1657178784Skmacy else if (cb->rlat) 1658178784Skmacy krping_rlat_test_server(cb); 1659178784Skmacy else if (cb->bw) 1660178784Skmacy krping_bw_test_server(cb); 1661178784Skmacy else 1662178784Skmacy krping_test_server(cb); 1663178784Skmacy rdma_disconnect(cb->child_cm_id); 1664178784Skmacyerr2: 1665178784Skmacy krping_free_buffers(cb); 1666178784Skmacyerr1: 1667178784Skmacy krping_free_qp(cb); 1668256829Snperr0: 1669256829Snp rdma_destroy_id(cb->child_cm_id); 1670178784Skmacy} 1671178784Skmacy 1672178784Skmacystatic void krping_test_client(struct krping_cb *cb) 1673178784Skmacy{ 1674178784Skmacy int ping, start, cc, i, ret; 1675178784Skmacy struct ib_send_wr *bad_wr; 1676178784Skmacy unsigned char c; 1677178784Skmacy 1678178784Skmacy start = 65; 1679178784Skmacy for (ping = 0; !cb->count || ping < cb->count; ping++) { 1680178784Skmacy cb->state = RDMA_READ_ADV; 1681178784Skmacy 1682178784Skmacy /* Put some ascii text in the buffer. */ 1683178784Skmacy cc = sprintf(cb->start_buf, "rdma-ping-%d: ", ping); 1684178784Skmacy for (i = cc, c = start; i < cb->size; i++) { 1685178784Skmacy cb->start_buf[i] = c; 1686178784Skmacy c++; 1687178784Skmacy if (c > 122) 1688178784Skmacy c = 65; 1689178784Skmacy } 1690178784Skmacy start++; 1691178784Skmacy if (start > 122) 1692178784Skmacy start = 65; 1693178784Skmacy cb->start_buf[cb->size - 1] = 0; 1694178784Skmacy 1695256829Snp krping_format_send(cb, cb->start_dma_addr); 1696256829Snp if (cb->state == ERROR) { 1697256829Snp PRINTF(cb, "krping_format_send failed\n"); 1698256829Snp break; 1699256829Snp } 1700178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1701178784Skmacy if (ret) { 1702256829Snp PRINTF(cb, "post send error %d\n", ret); 1703178784Skmacy break; 1704178784Skmacy } 1705178784Skmacy 1706178784Skmacy /* Wait for server to ACK */ 1707256829Snp wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); 1708178784Skmacy if (cb->state != RDMA_WRITE_ADV) { 1709256829Snp PRINTF(cb, 1710178784Skmacy "wait for RDMA_WRITE_ADV state %d\n", 1711178784Skmacy cb->state); 1712178784Skmacy break; 1713178784Skmacy } 1714178784Skmacy 1715256829Snp krping_format_send(cb, cb->rdma_dma_addr); 1716178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1717178784Skmacy if (ret) { 1718256829Snp PRINTF(cb, "post send error %d\n", ret); 1719178784Skmacy break; 1720178784Skmacy } 1721178784Skmacy 1722178784Skmacy /* Wait for the server to say the RDMA Write is complete. */ 1723256829Snp wait_event_interruptible(cb->sem, 1724256829Snp cb->state >= RDMA_WRITE_COMPLETE); 1725178784Skmacy if (cb->state != RDMA_WRITE_COMPLETE) { 1726256829Snp PRINTF(cb, 1727178784Skmacy "wait for RDMA_WRITE_COMPLETE state %d\n", 1728178784Skmacy cb->state); 1729178784Skmacy break; 1730178784Skmacy } 1731178784Skmacy 1732178784Skmacy if (cb->validate) 1733178784Skmacy if (memcmp(cb->start_buf, cb->rdma_buf, cb->size)) { 1734256829Snp PRINTF(cb, "data mismatch!\n"); 1735178784Skmacy break; 1736178784Skmacy } 1737178784Skmacy 1738178784Skmacy if (cb->verbose) 1739256829Snp PRINTF(cb, "ping data: %s\n", cb->rdma_buf); 1740256829Snp#ifdef SLOW_KRPING 1741256829Snp wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); 1742256829Snp#endif 1743178784Skmacy } 1744178784Skmacy} 1745178784Skmacy 1746178784Skmacystatic void krping_rlat_test_client(struct krping_cb *cb) 1747178784Skmacy{ 1748178784Skmacy struct ib_send_wr *bad_wr; 1749178784Skmacy struct ib_wc wc; 1750178784Skmacy int ret; 1751178784Skmacy 1752178784Skmacy cb->state = RDMA_READ_ADV; 1753178784Skmacy 1754178784Skmacy /* Send STAG/TO/Len to client */ 1755256829Snp krping_format_send(cb, cb->start_dma_addr); 1756256829Snp if (cb->state == ERROR) { 1757256829Snp PRINTF(cb, "krping_format_send failed\n"); 1758256829Snp return; 1759256829Snp } 1760178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1761178784Skmacy if (ret) { 1762256829Snp PRINTF(cb, "post send error %d\n", ret); 1763178784Skmacy return; 1764178784Skmacy } 1765178784Skmacy 1766178784Skmacy /* Spin waiting for send completion */ 1767178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1768178784Skmacy if (ret < 0) { 1769256829Snp PRINTF(cb, "poll error %d\n", ret); 1770178784Skmacy return; 1771178784Skmacy } 1772178784Skmacy if (wc.status) { 1773256829Snp PRINTF(cb, "send completion error %d\n", wc.status); 1774178784Skmacy return; 1775178784Skmacy } 1776178784Skmacy 1777178784Skmacy /* Spin waiting for server's Start STAG/TO/Len */ 1778178784Skmacy while (cb->state < RDMA_WRITE_ADV) { 1779178784Skmacy krping_cq_event_handler(cb->cq, cb); 1780178784Skmacy } 1781178784Skmacy 1782178784Skmacy#if 0 1783178784Skmacy{ 1784178784Skmacy int i; 1785178784Skmacy struct timeval start, stop; 1786178784Skmacy time_t sec; 1787178784Skmacy suseconds_t usec; 1788178784Skmacy unsigned long long elapsed; 1789178784Skmacy struct ib_wc wc; 1790178784Skmacy struct ib_send_wr *bad_wr; 1791178784Skmacy int ne; 1792178784Skmacy 1793178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; 1794178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1795178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1796178784Skmacy cb->rdma_sq_wr.sg_list->length = 0; 1797178784Skmacy cb->rdma_sq_wr.num_sge = 0; 1798178784Skmacy 1799178784Skmacy microtime(&start); 1800178784Skmacy for (i=0; i < 100000; i++) { 1801178784Skmacy if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { 1802256829Snp PRINTF(cb, "Couldn't post send\n"); 1803178784Skmacy return; 1804178784Skmacy } 1805178784Skmacy do { 1806178784Skmacy ne = ib_poll_cq(cb->cq, 1, &wc); 1807178784Skmacy } while (ne == 0); 1808178784Skmacy if (ne < 0) { 1809256829Snp PRINTF(cb, "poll CQ failed %d\n", ne); 1810178784Skmacy return; 1811178784Skmacy } 1812178784Skmacy if (wc.status != IB_WC_SUCCESS) { 1813256829Snp PRINTF(cb, "Completion wth error at %s:\n", 1814178784Skmacy cb->server ? "server" : "client"); 1815256829Snp PRINTF(cb, "Failed status %d: wr_id %d\n", 1816178784Skmacy wc.status, (int) wc.wr_id); 1817178784Skmacy return; 1818178784Skmacy } 1819178784Skmacy } 1820178784Skmacy microtime(&stop); 1821178784Skmacy 1822178784Skmacy if (stop.tv_usec < start.tv_usec) { 1823178784Skmacy stop.tv_usec += 1000000; 1824178784Skmacy stop.tv_sec -= 1; 1825178784Skmacy } 1826178784Skmacy sec = stop.tv_sec - start.tv_sec; 1827178784Skmacy usec = stop.tv_usec - start.tv_usec; 1828178784Skmacy elapsed = sec * 1000000 + usec; 1829256829Snp PRINTF(cb, "0B-write-lat iters 100000 usec %llu\n", elapsed); 1830178784Skmacy} 1831178784Skmacy#endif 1832178784Skmacy 1833178784Skmacy rlat_test(cb); 1834178784Skmacy} 1835178784Skmacy 1836178784Skmacystatic void krping_wlat_test_client(struct krping_cb *cb) 1837178784Skmacy{ 1838178784Skmacy struct ib_send_wr *bad_wr; 1839178784Skmacy struct ib_wc wc; 1840178784Skmacy int ret; 1841178784Skmacy 1842178784Skmacy cb->state = RDMA_READ_ADV; 1843178784Skmacy 1844178784Skmacy /* Send STAG/TO/Len to client */ 1845256829Snp krping_format_send(cb, cb->start_dma_addr); 1846256829Snp if (cb->state == ERROR) { 1847256829Snp PRINTF(cb, "krping_format_send failed\n"); 1848256829Snp return; 1849256829Snp } 1850178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1851178784Skmacy if (ret) { 1852256829Snp PRINTF(cb, "post send error %d\n", ret); 1853178784Skmacy return; 1854178784Skmacy } 1855178784Skmacy 1856178784Skmacy /* Spin waiting for send completion */ 1857178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1858178784Skmacy if (ret < 0) { 1859256829Snp PRINTF(cb, "poll error %d\n", ret); 1860178784Skmacy return; 1861178784Skmacy } 1862178784Skmacy if (wc.status) { 1863256829Snp PRINTF(cb, "send completion error %d\n", wc.status); 1864178784Skmacy return; 1865178784Skmacy } 1866178784Skmacy 1867178784Skmacy /* Spin waiting for server's Start STAG/TO/Len */ 1868178784Skmacy while (cb->state < RDMA_WRITE_ADV) { 1869178784Skmacy krping_cq_event_handler(cb->cq, cb); 1870178784Skmacy } 1871178784Skmacy 1872178784Skmacy wlat_test(cb); 1873178784Skmacy} 1874178784Skmacy 1875178784Skmacystatic void krping_bw_test_client(struct krping_cb *cb) 1876178784Skmacy{ 1877178784Skmacy struct ib_send_wr *bad_wr; 1878178784Skmacy struct ib_wc wc; 1879178784Skmacy int ret; 1880178784Skmacy 1881178784Skmacy cb->state = RDMA_READ_ADV; 1882178784Skmacy 1883178784Skmacy /* Send STAG/TO/Len to client */ 1884256829Snp krping_format_send(cb, cb->start_dma_addr); 1885256829Snp if (cb->state == ERROR) { 1886256829Snp PRINTF(cb, "krping_format_send failed\n"); 1887256829Snp return; 1888256829Snp } 1889178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1890178784Skmacy if (ret) { 1891256829Snp PRINTF(cb, "post send error %d\n", ret); 1892178784Skmacy return; 1893178784Skmacy } 1894178784Skmacy 1895178784Skmacy /* Spin waiting for send completion */ 1896178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1897178784Skmacy if (ret < 0) { 1898256829Snp PRINTF(cb, "poll error %d\n", ret); 1899178784Skmacy return; 1900178784Skmacy } 1901178784Skmacy if (wc.status) { 1902256829Snp PRINTF(cb, "send completion error %d\n", wc.status); 1903178784Skmacy return; 1904178784Skmacy } 1905178784Skmacy 1906178784Skmacy /* Spin waiting for server's Start STAG/TO/Len */ 1907178784Skmacy while (cb->state < RDMA_WRITE_ADV) { 1908178784Skmacy krping_cq_event_handler(cb->cq, cb); 1909178784Skmacy } 1910178784Skmacy 1911178784Skmacy bw_test(cb); 1912178784Skmacy} 1913178784Skmacy 1914256829Snpstatic void krping_fr_test(struct krping_cb *cb) 1915256829Snp{ 1916256829Snp struct ib_fast_reg_page_list *pl; 1917256829Snp struct ib_send_wr fr, inv, *bad; 1918256829Snp struct ib_wc wc; 1919256829Snp u8 key = 0; 1920256829Snp struct ib_mr *mr; 1921256829Snp int i; 1922256829Snp int ret; 1923256829Snp int size = cb->size; 1924256829Snp int plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; 1925256829Snp time_t start; 1926256829Snp int count = 0; 1927256829Snp int scnt = 0; 1928256829Snp 1929256829Snp pl = ib_alloc_fast_reg_page_list(cb->qp->device, plen); 1930256829Snp if (IS_ERR(pl)) { 1931256829Snp PRINTF(cb, "ib_alloc_fast_reg_page_list failed %ld\n", PTR_ERR(pl)); 1932256829Snp return; 1933256829Snp } 1934256829Snp 1935256829Snp mr = ib_alloc_fast_reg_mr(cb->pd, plen); 1936256829Snp if (IS_ERR(mr)) { 1937256829Snp PRINTF(cb, "ib_alloc_fast_reg_mr failed %ld\n", PTR_ERR(pl)); 1938256829Snp goto err1; 1939256829Snp } 1940256829Snp 1941256829Snp for (i=0; i<plen; i++) 1942256829Snp pl->page_list[i] = 0xcafebabe | i; 1943256829Snp 1944256829Snp memset(&fr, 0, sizeof fr); 1945256829Snp fr.opcode = IB_WR_FAST_REG_MR; 1946256829Snp fr.wr.fast_reg.page_shift = PAGE_SHIFT; 1947256829Snp fr.wr.fast_reg.length = size; 1948256829Snp fr.wr.fast_reg.page_list = pl; 1949256829Snp fr.wr.fast_reg.page_list_len = plen; 1950256829Snp fr.wr.fast_reg.iova_start = 0; 1951256829Snp fr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; 1952256829Snp fr.next = &inv; 1953256829Snp memset(&inv, 0, sizeof inv); 1954256829Snp inv.opcode = IB_WR_LOCAL_INV; 1955256829Snp inv.send_flags = IB_SEND_SIGNALED; 1956256829Snp 1957256829Snp DEBUG_LOG(cb, "fr_test: stag index 0x%x plen %u size %u depth %u\n", mr->rkey >> 8, plen, cb->size, cb->txdepth); 1958256829Snp start = time_uptime; 1959256829Snp while (1) { 1960256829Snp if ((time_uptime - start) >= 9) { 1961256829Snp DEBUG_LOG(cb, "fr_test: pausing 1 second! count %u latest size %u plen %u\n", count, size, plen); 1962256829Snp wait_event_interruptible(cb->sem, cb->state == ERROR); 1963256829Snp if (cb->state == ERROR) 1964256829Snp break; 1965256829Snp start = time_uptime; 1966256829Snp } 1967256829Snp while (scnt < (cb->txdepth>>1)) { 1968256829Snp ib_update_fast_reg_key(mr, ++key); 1969256829Snp fr.wr.fast_reg.rkey = mr->rkey; 1970256829Snp inv.ex.invalidate_rkey = mr->rkey; 1971256829Snp size = arc4random() % cb->size; 1972256829Snp if (size == 0) 1973256829Snp size = cb->size; 1974256829Snp plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; 1975256829Snp fr.wr.fast_reg.length = size; 1976256829Snp fr.wr.fast_reg.page_list_len = plen; 1977256829Snp ret = ib_post_send(cb->qp, &fr, &bad); 1978256829Snp if (ret) { 1979256829Snp PRINTF(cb, "ib_post_send failed %d\n", ret); 1980256829Snp goto err2; 1981256829Snp } 1982256829Snp scnt++; 1983256829Snp } 1984256829Snp 1985256829Snp do { 1986256829Snp ret = ib_poll_cq(cb->cq, 1, &wc); 1987256829Snp if (ret < 0) { 1988256829Snp PRINTF(cb, "ib_poll_cq failed %d\n", ret); 1989256829Snp goto err2; 1990256829Snp } 1991256829Snp if (ret == 1) { 1992256829Snp if (wc.status) { 1993256829Snp PRINTF(cb, "completion error %u\n", wc.status); 1994256829Snp goto err2; 1995256829Snp } 1996256829Snp count++; 1997256829Snp scnt--; 1998256829Snp } 1999256829Snp else if (krping_sigpending()) { 2000256829Snp PRINTF(cb, "signal!\n"); 2001256829Snp goto err2; 2002256829Snp } 2003256829Snp } while (ret == 1); 2004256829Snp } 2005256829Snperr2: 2006256829Snp#if 0 2007256829Snp DEBUG_LOG(cb, "sleeping 1 second\n"); 2008256829Snp wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); 2009256829Snp#endif 2010256829Snp DEBUG_LOG(cb, "draining the cq...\n"); 2011256829Snp do { 2012256829Snp ret = ib_poll_cq(cb->cq, 1, &wc); 2013256829Snp if (ret < 0) { 2014256829Snp PRINTF(cb, "ib_poll_cq failed %d\n", ret); 2015256829Snp break; 2016256829Snp } 2017256829Snp if (ret == 1) { 2018256829Snp if (wc.status) { 2019256829Snp PRINTF(cb, "completion error %u opcode %u\n", wc.status, wc.opcode); 2020256829Snp } 2021256829Snp } 2022256829Snp } while (ret == 1); 2023256829Snp DEBUG_LOG(cb, "fr_test: done!\n"); 2024256829Snp ib_dereg_mr(mr); 2025256829Snperr1: 2026256829Snp ib_free_fast_reg_page_list(pl); 2027256829Snp} 2028256829Snp 2029178784Skmacystatic int krping_connect_client(struct krping_cb *cb) 2030178784Skmacy{ 2031178784Skmacy struct rdma_conn_param conn_param; 2032178784Skmacy int ret; 2033178784Skmacy 2034178784Skmacy memset(&conn_param, 0, sizeof conn_param); 2035178784Skmacy conn_param.responder_resources = 1; 2036178784Skmacy conn_param.initiator_depth = 1; 2037178784Skmacy conn_param.retry_count = 10; 2038178784Skmacy 2039178784Skmacy ret = rdma_connect(cb->cm_id, &conn_param); 2040178784Skmacy if (ret) { 2041256829Snp PRINTF(cb, "rdma_connect error %d\n", ret); 2042178784Skmacy return ret; 2043178784Skmacy } 2044178784Skmacy 2045256829Snp wait_event_interruptible(cb->sem, cb->state >= CONNECTED); 2046178784Skmacy if (cb->state == ERROR) { 2047256829Snp PRINTF(cb, "wait for CONNECTED state %d\n", cb->state); 2048178784Skmacy return -1; 2049178784Skmacy } 2050178784Skmacy 2051256829Snp DEBUG_LOG(cb, "rdma_connect successful\n"); 2052178784Skmacy return 0; 2053178784Skmacy} 2054178784Skmacy 2055178784Skmacystatic int krping_bind_client(struct krping_cb *cb) 2056178784Skmacy{ 2057325602Shselasky union { 2058325602Shselasky struct sockaddr_in v4; 2059325602Shselasky struct sockaddr_in6 v6; 2060325602Shselasky } sin; 2061178784Skmacy int ret; 2062178784Skmacy 2063178784Skmacy memset(&sin, 0, sizeof(sin)); 2064178784Skmacy 2065325602Shselasky switch (cb->addr_type) { 2066325602Shselasky case AF_INET: 2067325602Shselasky sin.v4.sin_len = sizeof sin.v4; 2068325602Shselasky sin.v4.sin_family = AF_INET; 2069325602Shselasky sin.v4.sin_addr = cb->addr.v4; 2070325602Shselasky sin.v4.sin_port = cb->port; 2071325602Shselasky break; 2072325602Shselasky case AF_INET6: 2073325602Shselasky sin.v6.sin6_len = sizeof sin.v6; 2074325602Shselasky sin.v6.sin6_family = AF_INET6; 2075325602Shselasky sin.v6.sin6_addr = cb->addr.v6; 2076325602Shselasky sin.v6.sin6_port = cb->port; 2077325602Shselasky break; 2078325602Shselasky default: 2079325602Shselasky return (-EINVAL); 2080325602Shselasky } 2081325602Shselasky 2082178784Skmacy ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *) &sin, 2083178784Skmacy 2000); 2084178784Skmacy if (ret) { 2085256829Snp PRINTF(cb, "rdma_resolve_addr error %d\n", ret); 2086178784Skmacy return ret; 2087178784Skmacy } 2088178784Skmacy 2089256829Snp wait_event_interruptible(cb->sem, cb->state >= ROUTE_RESOLVED); 2090178784Skmacy if (cb->state != ROUTE_RESOLVED) { 2091256829Snp PRINTF(cb, 2092178784Skmacy "addr/route resolution did not resolve: state %d\n", 2093178784Skmacy cb->state); 2094256829Snp return -EINTR; 2095178784Skmacy } 2096178784Skmacy 2097256829Snp if (cb->mem == FASTREG && !fastreg_supported(cb)) 2098256829Snp return -EINVAL; 2099256829Snp 2100256829Snp DEBUG_LOG(cb, "rdma_resolve_addr - rdma_resolve_route successful\n"); 2101178784Skmacy return 0; 2102178784Skmacy} 2103178784Skmacy 2104178784Skmacystatic void krping_run_client(struct krping_cb *cb) 2105178784Skmacy{ 2106178784Skmacy struct ib_recv_wr *bad_wr; 2107178784Skmacy int ret; 2108178784Skmacy 2109178784Skmacy ret = krping_bind_client(cb); 2110178784Skmacy if (ret) 2111178784Skmacy return; 2112178784Skmacy 2113178784Skmacy ret = krping_setup_qp(cb, cb->cm_id); 2114178784Skmacy if (ret) { 2115256829Snp PRINTF(cb, "setup_qp failed: %d\n", ret); 2116178784Skmacy return; 2117178784Skmacy } 2118178784Skmacy 2119178784Skmacy ret = krping_setup_buffers(cb); 2120178784Skmacy if (ret) { 2121256829Snp PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); 2122178784Skmacy goto err1; 2123178784Skmacy } 2124178784Skmacy 2125178784Skmacy ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); 2126178784Skmacy if (ret) { 2127256829Snp PRINTF(cb, "ib_post_recv failed: %d\n", ret); 2128178784Skmacy goto err2; 2129178784Skmacy } 2130178784Skmacy 2131178784Skmacy ret = krping_connect_client(cb); 2132178784Skmacy if (ret) { 2133256829Snp PRINTF(cb, "connect error %d\n", ret); 2134178784Skmacy goto err2; 2135178784Skmacy } 2136178784Skmacy 2137178784Skmacy if (cb->wlat) 2138178784Skmacy krping_wlat_test_client(cb); 2139178784Skmacy else if (cb->rlat) 2140178784Skmacy krping_rlat_test_client(cb); 2141178784Skmacy else if (cb->bw) 2142178784Skmacy krping_bw_test_client(cb); 2143256829Snp else if (cb->frtest) 2144256829Snp krping_fr_test(cb); 2145178784Skmacy else 2146178784Skmacy krping_test_client(cb); 2147178784Skmacy rdma_disconnect(cb->cm_id); 2148178784Skmacyerr2: 2149178784Skmacy krping_free_buffers(cb); 2150178784Skmacyerr1: 2151178784Skmacy krping_free_qp(cb); 2152178784Skmacy} 2153178784Skmacy 2154325602Shselaskystatic uint16_t 2155325602Shselaskykrping_get_ipv6_scope_id(char *name) 2156325602Shselasky{ 2157325602Shselasky struct ifnet *ifp; 2158325602Shselasky uint16_t retval; 2159325602Shselasky 2160325602Shselasky if (name == NULL) 2161325602Shselasky return (0); 2162325602Shselasky ifp = ifunit_ref(name); 2163325602Shselasky if (ifp == NULL) 2164325602Shselasky return (0); 2165325602Shselasky retval = ifp->if_index; 2166325602Shselasky if_rele(ifp); 2167325602Shselasky return (retval); 2168325602Shselasky} 2169325602Shselasky 2170256829Snpint krping_doit(char *cmd, void *cookie) 2171178784Skmacy{ 2172178784Skmacy struct krping_cb *cb; 2173178784Skmacy int op; 2174178784Skmacy int ret = 0; 2175178784Skmacy char *optarg; 2176325602Shselasky char *scope; 2177178784Skmacy unsigned long optint; 2178178784Skmacy 2179256829Snp cb = kzalloc(sizeof(*cb), GFP_KERNEL); 2180178784Skmacy if (!cb) 2181256829Snp return -ENOMEM; 2182178784Skmacy 2183256829Snp mutex_lock(&krping_mutex); 2184256829Snp list_add_tail(&cb->list, &krping_cbs); 2185256829Snp mutex_unlock(&krping_mutex); 2186178784Skmacy 2187256829Snp cb->cookie = cookie; 2188178784Skmacy cb->server = -1; 2189178784Skmacy cb->state = IDLE; 2190178784Skmacy cb->size = 64; 2191178784Skmacy cb->txdepth = RPING_SQ_DEPTH; 2192256829Snp cb->mem = DMA; 2193325602Shselasky cb->addr_type = AF_INET; 2194256829Snp init_waitqueue_head(&cb->sem); 2195178784Skmacy 2196178784Skmacy while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg, 2197178784Skmacy &optint)) != 0) { 2198178784Skmacy switch (op) { 2199178784Skmacy case 'a': 2200178784Skmacy cb->addr_str = optarg; 2201325602Shselasky cb->addr_type = AF_INET; 2202325602Shselasky DEBUG_LOG(cb, "ipv4addr (%s)\n", optarg); 2203325602Shselasky if (inet_pton(AF_INET, optarg, &cb->addr) != 1) { 2204256829Snp PRINTF(cb, "bad addr string %s\n", 2205256829Snp optarg); 2206178784Skmacy ret = EINVAL; 2207178784Skmacy } 2208178784Skmacy break; 2209325602Shselasky case 'A': 2210325602Shselasky cb->addr_str = optarg; 2211325602Shselasky cb->addr_type = AF_INET6; 2212325602Shselasky DEBUG_LOG(cb, "ipv6addr (%s)\n", optarg); 2213325602Shselasky scope = strstr(optarg, "%"); 2214325602Shselasky /* extract scope ID, if any */ 2215325602Shselasky if (scope != NULL) 2216325602Shselasky *scope++ = 0; 2217325602Shselasky /* extract IPv6 network address */ 2218325602Shselasky if (inet_pton(AF_INET6, optarg, &cb->addr) != 1) { 2219325602Shselasky PRINTF(cb, "bad addr string %s\n", 2220325602Shselasky optarg); 2221325602Shselasky ret = EINVAL; 2222325602Shselasky } else if (IN6_IS_SCOPE_LINKLOCAL(&cb->addr.v6) || 2223325602Shselasky IN6_IS_ADDR_MC_INTFACELOCAL(&cb->addr.v6)) { 2224325602Shselasky uint16_t scope_id = krping_get_ipv6_scope_id(scope); 2225325602Shselasky DEBUG_LOG(cb, "ipv6 scope ID = %d\n", scope_id); 2226325602Shselasky cb->addr.v6.s6_addr[2] = scope_id >> 8; 2227325602Shselasky cb->addr.v6.s6_addr[3] = scope_id & 0xFF; 2228325602Shselasky } 2229325602Shselasky break; 2230178784Skmacy case 'p': 2231178784Skmacy cb->port = htons(optint); 2232256829Snp DEBUG_LOG(cb, "port %d\n", (int)optint); 2233178784Skmacy break; 2234178784Skmacy case 'P': 2235178784Skmacy cb->poll = 1; 2236256829Snp DEBUG_LOG(cb, "server\n"); 2237178784Skmacy break; 2238178784Skmacy case 's': 2239178784Skmacy cb->server = 1; 2240256829Snp DEBUG_LOG(cb, "server\n"); 2241178784Skmacy break; 2242178784Skmacy case 'c': 2243178784Skmacy cb->server = 0; 2244256829Snp DEBUG_LOG(cb, "client\n"); 2245178784Skmacy break; 2246178784Skmacy case 'S': 2247178784Skmacy cb->size = optint; 2248178784Skmacy if ((cb->size < 1) || 2249178784Skmacy (cb->size > RPING_BUFSIZE)) { 2250256829Snp PRINTF(cb, "Invalid size %d " 2251178784Skmacy "(valid range is 1 to %d)\n", 2252178784Skmacy cb->size, RPING_BUFSIZE); 2253178784Skmacy ret = EINVAL; 2254178784Skmacy } else 2255256829Snp DEBUG_LOG(cb, "size %d\n", (int)optint); 2256178784Skmacy break; 2257178784Skmacy case 'C': 2258178784Skmacy cb->count = optint; 2259178784Skmacy if (cb->count < 0) { 2260256829Snp PRINTF(cb, "Invalid count %d\n", 2261178784Skmacy cb->count); 2262178784Skmacy ret = EINVAL; 2263178784Skmacy } else 2264256829Snp DEBUG_LOG(cb, "count %d\n", (int) cb->count); 2265178784Skmacy break; 2266178784Skmacy case 'v': 2267178784Skmacy cb->verbose++; 2268256829Snp DEBUG_LOG(cb, "verbose\n"); 2269178784Skmacy break; 2270178784Skmacy case 'V': 2271178784Skmacy cb->validate++; 2272256829Snp DEBUG_LOG(cb, "validate data\n"); 2273178784Skmacy break; 2274256829Snp case 'l': 2275256829Snp cb->wlat++; 2276256829Snp break; 2277178784Skmacy case 'L': 2278178784Skmacy cb->rlat++; 2279178784Skmacy break; 2280178784Skmacy case 'B': 2281178784Skmacy cb->bw++; 2282178784Skmacy break; 2283256829Snp case 'd': 2284256829Snp cb->duplex++; 2285256829Snp break; 2286256829Snp case 'm': 2287256829Snp if (!strncmp(optarg, "dma", 3)) 2288256829Snp cb->mem = DMA; 2289256829Snp else if (!strncmp(optarg, "fastreg", 7)) 2290256829Snp cb->mem = FASTREG; 2291256829Snp else if (!strncmp(optarg, "mw", 2)) 2292256829Snp cb->mem = MW; 2293256829Snp else if (!strncmp(optarg, "mr", 2)) 2294256829Snp cb->mem = MR; 2295256829Snp else { 2296256829Snp PRINTF(cb, "unknown mem mode %s. " 2297256829Snp "Must be dma, fastreg, mw, or mr\n", 2298256829Snp optarg); 2299256829Snp ret = -EINVAL; 2300256829Snp break; 2301256829Snp } 2302256829Snp break; 2303256829Snp case 'I': 2304256829Snp cb->server_invalidate = 1; 2305256829Snp break; 2306256829Snp case 'T': 2307178784Skmacy cb->txdepth = optint; 2308256829Snp DEBUG_LOG(cb, "txdepth %d\n", (int) cb->txdepth); 2309178784Skmacy break; 2310256829Snp case 'Z': 2311256829Snp cb->local_dma_lkey = 1; 2312256829Snp DEBUG_LOG(cb, "using local dma lkey\n"); 2313178784Skmacy break; 2314256829Snp case 'R': 2315256829Snp cb->read_inv = 1; 2316256829Snp DEBUG_LOG(cb, "using read-with-inv\n"); 2317256829Snp break; 2318256829Snp case 'f': 2319256829Snp cb->frtest = 1; 2320256829Snp DEBUG_LOG(cb, "fast-reg test!\n"); 2321256829Snp break; 2322178784Skmacy default: 2323256829Snp PRINTF(cb, "unknown opt %s\n", optarg); 2324256829Snp ret = -EINVAL; 2325178784Skmacy break; 2326178784Skmacy } 2327178784Skmacy } 2328178784Skmacy if (ret) 2329178784Skmacy goto out; 2330178784Skmacy 2331178784Skmacy if (cb->server == -1) { 2332256829Snp PRINTF(cb, "must be either client or server\n"); 2333256829Snp ret = -EINVAL; 2334178784Skmacy goto out; 2335178784Skmacy } 2336256829Snp 2337256829Snp if (cb->server && cb->frtest) { 2338256829Snp PRINTF(cb, "must be client to run frtest\n"); 2339256829Snp ret = -EINVAL; 2340178784Skmacy goto out; 2341178784Skmacy } 2342178784Skmacy 2343256829Snp if ((cb->frtest + cb->bw + cb->rlat + cb->wlat) > 1) { 2344256829Snp PRINTF(cb, "Pick only one test: fr, bw, rlat, wlat\n"); 2345256829Snp ret = -EINVAL; 2346256829Snp goto out; 2347256829Snp } 2348178784Skmacy 2349256829Snp if (cb->server_invalidate && cb->mem != FASTREG) { 2350256829Snp PRINTF(cb, "server_invalidate only valid with fastreg mem_mode\n"); 2351256829Snp ret = -EINVAL; 2352256829Snp goto out; 2353256829Snp } 2354256829Snp 2355256829Snp if (cb->read_inv && cb->mem != FASTREG) { 2356256829Snp PRINTF(cb, "read_inv only valid with fastreg mem_mode\n"); 2357256829Snp ret = -EINVAL; 2358256829Snp goto out; 2359256829Snp } 2360256829Snp 2361256829Snp if (cb->mem != MR && (cb->wlat || cb->rlat || cb->bw)) { 2362256829Snp PRINTF(cb, "wlat, rlat, and bw tests only support mem_mode MR\n"); 2363256829Snp ret = -EINVAL; 2364256829Snp goto out; 2365256829Snp } 2366256829Snp 2367178784Skmacy cb->cm_id = rdma_create_id(krping_cma_event_handler, cb, RDMA_PS_TCP); 2368178784Skmacy if (IS_ERR(cb->cm_id)) { 2369178784Skmacy ret = PTR_ERR(cb->cm_id); 2370256829Snp PRINTF(cb, "rdma_create_id error %d\n", ret); 2371178784Skmacy goto out; 2372178784Skmacy } 2373256829Snp DEBUG_LOG(cb, "created cm_id %p\n", cb->cm_id); 2374256829Snp 2375178784Skmacy if (cb->server) 2376178784Skmacy krping_run_server(cb); 2377178784Skmacy else 2378178784Skmacy krping_run_client(cb); 2379254735Snp 2380256829Snp DEBUG_LOG(cb, "destroy cm_id %p\n", cb->cm_id); 2381178784Skmacy rdma_destroy_id(cb->cm_id); 2382178784Skmacyout: 2383256829Snp mutex_lock(&krping_mutex); 2384256829Snp list_del(&cb->list); 2385256829Snp mutex_unlock(&krping_mutex); 2386256829Snp kfree(cb); 2387178784Skmacy return ret; 2388178784Skmacy} 2389178784Skmacy 2390256829Snpvoid 2391256829Snpkrping_walk_cb_list(void (*f)(struct krping_stats *, void *), void *arg) 2392256829Snp{ 2393256829Snp struct krping_cb *cb; 2394256829Snp 2395256829Snp mutex_lock(&krping_mutex); 2396256829Snp list_for_each_entry(cb, &krping_cbs, list) 2397256829Snp (*f)(cb->pd ? &cb->stats : NULL, arg); 2398256829Snp mutex_unlock(&krping_mutex); 2399256829Snp} 2400256829Snp 2401178784Skmacyvoid krping_init(void) 2402178784Skmacy{ 2403256829Snp 2404256829Snp mutex_init(&krping_mutex); 2405178784Skmacy} 2406