1178784Skmacy/* 2178784Skmacy * Copyright (c) 2005 Ammasso, Inc. All rights reserved. 3256829Snp * Copyright (c) 2006-2009 Open Grid Computing, Inc. All rights reserved. 4178784Skmacy * 5178784Skmacy * This software is available to you under a choice of one of two 6178784Skmacy * licenses. You may choose to be licensed under the terms of the GNU 7178784Skmacy * General Public License (GPL) Version 2, available from the file 8178784Skmacy * COPYING in the main directory of this source tree, or the 9178784Skmacy * OpenIB.org BSD license below: 10178784Skmacy * 11178784Skmacy * Redistribution and use in source and binary forms, with or 12178784Skmacy * without modification, are permitted provided that the following 13178784Skmacy * conditions are met: 14178784Skmacy * 15178784Skmacy * - Redistributions of source code must retain the above 16178784Skmacy * copyright notice, this list of conditions and the following 17178784Skmacy * disclaimer. 18178784Skmacy * 19178784Skmacy * - Redistributions in binary form must reproduce the above 20178784Skmacy * copyright notice, this list of conditions and the following 21178784Skmacy * disclaimer in the documentation and/or other materials 22178784Skmacy * provided with the distribution. 23178784Skmacy * 24178784Skmacy * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25178784Skmacy * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26178784Skmacy * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27178784Skmacy * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28178784Skmacy * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29178784Skmacy * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30178784Skmacy * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31178784Skmacy * SOFTWARE. 32178784Skmacy */ 33178784Skmacy 34178784Skmacy#include <sys/cdefs.h> 35178784Skmacy__FBSDID("$FreeBSD$"); 36178784Skmacy 37256829Snp#include <linux/module.h> 38256829Snp#include <linux/moduleparam.h> 39256829Snp#include <linux/init.h> 40256829Snp#include <linux/slab.h> 41256829Snp#include <linux/err.h> 42256829Snp#include <linux/string.h> 43256829Snp#include <linux/inet.h> 44256829Snp#include <linux/list.h> 45256829Snp#include <linux/in.h> 46256829Snp#include <linux/device.h> 47256829Snp#include <linux/pci.h> 48256829Snp#include <linux/sched.h> 49256829Snp#include <asm/system.h> 50178784Skmacy 51256829Snp#include <asm/atomic.h> 52178784Skmacy 53256829Snp#include <rdma/ib_verbs.h> 54237263Snp#include <rdma/rdma_cm.h> 55178784Skmacy 56256829Snp#include "krping.h" 57178784Skmacy#include "getopt.h" 58178784Skmacy 59256829Snpextern int krping_debug; 60256829Snp#define DEBUG_LOG(cb, x...) if (krping_debug) krping_printf((cb)->cookie, x) 61256829Snp#define PRINTF(cb, x...) krping_printf((cb)->cookie, x) 62178784Skmacy 63256829SnpMODULE_AUTHOR("Steve Wise"); 64256829SnpMODULE_DESCRIPTION("RDMA ping client/server"); 65256829SnpMODULE_LICENSE("Dual BSD/GPL"); 66178784Skmacy 67256829Snpstatic __inline uint64_t 68256829Snpget_cycles(void) 69256829Snp{ 70256829Snp uint32_t low, high; 71256829Snp __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); 72256829Snp return (low | ((u_int64_t)high << 32)); 73256829Snp} 74256829Snp 75256829Snptypedef uint64_t cycles_t; 76256829Snp 77256829Snpenum mem_type { 78256829Snp DMA = 1, 79256829Snp FASTREG = 2, 80256829Snp MW = 3, 81256829Snp MR = 4 82256829Snp}; 83256829Snp 84178784Skmacystatic const struct krping_option krping_opts[] = { 85178784Skmacy {"count", OPT_INT, 'C'}, 86178784Skmacy {"size", OPT_INT, 'S'}, 87178784Skmacy {"addr", OPT_STRING, 'a'}, 88178784Skmacy {"port", OPT_INT, 'p'}, 89178784Skmacy {"verbose", OPT_NOPARAM, 'v'}, 90178784Skmacy {"validate", OPT_NOPARAM, 'V'}, 91178784Skmacy {"server", OPT_NOPARAM, 's'}, 92178784Skmacy {"client", OPT_NOPARAM, 'c'}, 93256829Snp {"mem_mode", OPT_STRING, 'm'}, 94256829Snp {"server_inv", OPT_NOPARAM, 'I'}, 95256829Snp {"wlat", OPT_NOPARAM, 'l'}, 96256829Snp {"rlat", OPT_NOPARAM, 'L'}, 97256829Snp {"bw", OPT_NOPARAM, 'B'}, 98256829Snp {"duplex", OPT_NOPARAM, 'd'}, 99256829Snp {"txdepth", OPT_INT, 'T'}, 100256829Snp {"poll", OPT_NOPARAM, 'P'}, 101256829Snp {"local_dma_lkey", OPT_NOPARAM, 'Z'}, 102256829Snp {"read_inv", OPT_NOPARAM, 'R'}, 103256829Snp {"fr", OPT_NOPARAM, 'f'}, 104178784Skmacy {NULL, 0, 0} 105178784Skmacy}; 106178784Skmacy 107256829Snp#define htonll(x) cpu_to_be64((x)) 108256829Snp#define ntohll(x) cpu_to_be64((x)) 109178784Skmacy 110256829Snpstatic struct mutex krping_mutex; 111256829Snp 112178784Skmacy/* 113178784Skmacy * List of running krping threads. 114178784Skmacy */ 115256829Snpstatic LIST_HEAD(krping_cbs); 116178784Skmacy 117178784Skmacy/* 118178784Skmacy * krping "ping/pong" loop: 119178784Skmacy * client sends source rkey/addr/len 120178784Skmacy * server receives source rkey/add/len 121178784Skmacy * server rdma reads "ping" data from source 122178784Skmacy * server sends "go ahead" on rdma read completion 123178784Skmacy * client sends sink rkey/addr/len 124178784Skmacy * server receives sink rkey/addr/len 125178784Skmacy * server rdma writes "pong" data to sink 126178784Skmacy * server sends "go ahead" on rdma write completion 127178784Skmacy * <repeat loop> 128178784Skmacy */ 129178784Skmacy 130178784Skmacy/* 131256829Snp * These states are used to signal events between the completion handler 132256829Snp * and the main client or server thread. 133256829Snp * 134256829Snp * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, 135256829Snp * and RDMA_WRITE_COMPLETE for each ping. 136256829Snp */ 137256829Snpenum test_state { 138256829Snp IDLE = 1, 139256829Snp CONNECT_REQUEST, 140256829Snp ADDR_RESOLVED, 141256829Snp ROUTE_RESOLVED, 142256829Snp CONNECTED, 143256829Snp RDMA_READ_ADV, 144256829Snp RDMA_READ_COMPLETE, 145256829Snp RDMA_WRITE_ADV, 146256829Snp RDMA_WRITE_COMPLETE, 147256829Snp ERROR 148256829Snp}; 149256829Snp 150256829Snpstruct krping_rdma_info { 151256829Snp uint64_t buf; 152256829Snp uint32_t rkey; 153256829Snp uint32_t size; 154256829Snp}; 155256829Snp 156256829Snp/* 157178784Skmacy * Default max buffer size for IO... 158178784Skmacy */ 159178784Skmacy#define RPING_BUFSIZE 128*1024 160256829Snp#define RPING_SQ_DEPTH 64 161178784Skmacy 162256829Snp/* 163256829Snp * Control block struct. 164256829Snp */ 165256829Snpstruct krping_cb { 166256829Snp void *cookie; 167256829Snp int server; /* 0 iff client */ 168256829Snp struct ib_cq *cq; 169256829Snp struct ib_pd *pd; 170256829Snp struct ib_qp *qp; 171178784Skmacy 172256829Snp enum mem_type mem; 173256829Snp struct ib_mr *dma_mr; 174256829Snp 175256829Snp struct ib_fast_reg_page_list *page_list; 176256829Snp int page_list_len; 177256829Snp struct ib_send_wr fastreg_wr; 178256829Snp struct ib_send_wr invalidate_wr; 179256829Snp struct ib_mr *fastreg_mr; 180256829Snp int server_invalidate; 181256829Snp int read_inv; 182256829Snp u8 key; 183256829Snp 184256829Snp struct ib_mw *mw; 185256829Snp struct ib_mw_bind bind_attr; 186256829Snp 187256829Snp struct ib_recv_wr rq_wr; /* recv work request record */ 188256829Snp struct ib_sge recv_sgl; /* recv single SGE */ 189256829Snp struct krping_rdma_info recv_buf;/* malloc'd buffer */ 190256829Snp u64 recv_dma_addr; 191256829Snp DECLARE_PCI_UNMAP_ADDR(recv_mapping) 192256829Snp struct ib_mr *recv_mr; 193256829Snp 194256829Snp struct ib_send_wr sq_wr; /* send work requrest record */ 195256829Snp struct ib_sge send_sgl; 196256829Snp struct krping_rdma_info send_buf;/* single send buf */ 197256829Snp u64 send_dma_addr; 198256829Snp DECLARE_PCI_UNMAP_ADDR(send_mapping) 199256829Snp struct ib_mr *send_mr; 200256829Snp 201256829Snp struct ib_send_wr rdma_sq_wr; /* rdma work request record */ 202256829Snp struct ib_sge rdma_sgl; /* rdma single SGE */ 203256829Snp char *rdma_buf; /* used as rdma sink */ 204256829Snp u64 rdma_dma_addr; 205256829Snp DECLARE_PCI_UNMAP_ADDR(rdma_mapping) 206256829Snp struct ib_mr *rdma_mr; 207256829Snp 208256829Snp uint32_t remote_rkey; /* remote guys RKEY */ 209256829Snp uint64_t remote_addr; /* remote guys TO */ 210256829Snp uint32_t remote_len; /* remote guys LEN */ 211256829Snp 212256829Snp char *start_buf; /* rdma read src */ 213256829Snp u64 start_dma_addr; 214256829Snp DECLARE_PCI_UNMAP_ADDR(start_mapping) 215256829Snp struct ib_mr *start_mr; 216256829Snp 217256829Snp enum test_state state; /* used for cond/signalling */ 218256829Snp wait_queue_head_t sem; 219256829Snp struct krping_stats stats; 220256829Snp 221256829Snp uint16_t port; /* dst port in NBO */ 222256829Snp struct in_addr addr; /* dst addr in NBO */ 223256829Snp char *addr_str; /* dst addr string */ 224256829Snp int verbose; /* verbose logging */ 225256829Snp int count; /* ping count */ 226256829Snp int size; /* ping data size */ 227256829Snp int validate; /* validate ping data */ 228256829Snp int wlat; /* run wlat test */ 229256829Snp int rlat; /* run rlat test */ 230256829Snp int bw; /* run bw test */ 231256829Snp int duplex; /* run bw full duplex test */ 232256829Snp int poll; /* poll or block for rlat test */ 233256829Snp int txdepth; /* SQ depth */ 234256829Snp int local_dma_lkey; /* use 0 for lkey */ 235256829Snp int frtest; /* fastreg test */ 236256829Snp 237256829Snp /* CM stuff */ 238256829Snp struct rdma_cm_id *cm_id; /* connection on client side,*/ 239256829Snp /* listener on server side. */ 240256829Snp struct rdma_cm_id *child_cm_id; /* connection on server side */ 241256829Snp struct list_head list; 242256829Snp}; 243256829Snp 244178784Skmacystatic int krping_cma_event_handler(struct rdma_cm_id *cma_id, 245178784Skmacy struct rdma_cm_event *event) 246178784Skmacy{ 247178784Skmacy int ret; 248178784Skmacy struct krping_cb *cb = cma_id->context; 249178784Skmacy 250256829Snp DEBUG_LOG(cb, "cma_event type %d cma_id %p (%s)\n", event->event, 251256829Snp cma_id, (cma_id == cb->cm_id) ? "parent" : "child"); 252178784Skmacy 253178784Skmacy switch (event->event) { 254178784Skmacy case RDMA_CM_EVENT_ADDR_RESOLVED: 255178784Skmacy cb->state = ADDR_RESOLVED; 256178784Skmacy ret = rdma_resolve_route(cma_id, 2000); 257178784Skmacy if (ret) { 258256829Snp PRINTF(cb, "rdma_resolve_route error %d\n", ret); 259256829Snp wake_up_interruptible(&cb->sem); 260178784Skmacy } 261178784Skmacy break; 262178784Skmacy 263178784Skmacy case RDMA_CM_EVENT_ROUTE_RESOLVED: 264178784Skmacy cb->state = ROUTE_RESOLVED; 265256829Snp wake_up_interruptible(&cb->sem); 266178784Skmacy break; 267178784Skmacy 268178784Skmacy case RDMA_CM_EVENT_CONNECT_REQUEST: 269178784Skmacy cb->state = CONNECT_REQUEST; 270178784Skmacy cb->child_cm_id = cma_id; 271256829Snp DEBUG_LOG(cb, "child cma %p\n", cb->child_cm_id); 272256829Snp wake_up_interruptible(&cb->sem); 273178784Skmacy break; 274178784Skmacy 275178784Skmacy case RDMA_CM_EVENT_ESTABLISHED: 276256829Snp DEBUG_LOG(cb, "ESTABLISHED\n"); 277178784Skmacy if (!cb->server) { 278178784Skmacy cb->state = CONNECTED; 279178784Skmacy } 280256829Snp wake_up_interruptible(&cb->sem); 281178784Skmacy break; 282178784Skmacy 283178784Skmacy case RDMA_CM_EVENT_ADDR_ERROR: 284178784Skmacy case RDMA_CM_EVENT_ROUTE_ERROR: 285178784Skmacy case RDMA_CM_EVENT_CONNECT_ERROR: 286178784Skmacy case RDMA_CM_EVENT_UNREACHABLE: 287178784Skmacy case RDMA_CM_EVENT_REJECTED: 288256829Snp PRINTF(cb, "cma event %d, error %d\n", event->event, 289178784Skmacy event->status); 290178784Skmacy cb->state = ERROR; 291256829Snp wake_up_interruptible(&cb->sem); 292178784Skmacy break; 293178784Skmacy 294178784Skmacy case RDMA_CM_EVENT_DISCONNECTED: 295256829Snp PRINTF(cb, "DISCONNECT EVENT...\n"); 296178784Skmacy cb->state = ERROR; 297256829Snp wake_up_interruptible(&cb->sem); 298178784Skmacy break; 299178784Skmacy 300178784Skmacy case RDMA_CM_EVENT_DEVICE_REMOVAL: 301256829Snp PRINTF(cb, "cma detected device removal!!!!\n"); 302256829Snp break; 303178784Skmacy 304178784Skmacy default: 305256829Snp PRINTF(cb, "oof bad type!\n"); 306256829Snp wake_up_interruptible(&cb->sem); 307178784Skmacy break; 308178784Skmacy } 309178784Skmacy return 0; 310178784Skmacy} 311178784Skmacy 312178784Skmacystatic int server_recv(struct krping_cb *cb, struct ib_wc *wc) 313178784Skmacy{ 314178784Skmacy if (wc->byte_len != sizeof(cb->recv_buf)) { 315256829Snp PRINTF(cb, "Received bogus data, size %d\n", 316178784Skmacy wc->byte_len); 317178784Skmacy return -1; 318178784Skmacy } 319178784Skmacy 320178784Skmacy cb->remote_rkey = ntohl(cb->recv_buf.rkey); 321178784Skmacy cb->remote_addr = ntohll(cb->recv_buf.buf); 322178784Skmacy cb->remote_len = ntohl(cb->recv_buf.size); 323256829Snp DEBUG_LOG(cb, "Received rkey %x addr %llx len %d from peer\n", 324178784Skmacy cb->remote_rkey, (unsigned long long)cb->remote_addr, 325178784Skmacy cb->remote_len); 326178784Skmacy 327178784Skmacy if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE) 328178784Skmacy cb->state = RDMA_READ_ADV; 329178784Skmacy else 330178784Skmacy cb->state = RDMA_WRITE_ADV; 331178784Skmacy 332178784Skmacy return 0; 333178784Skmacy} 334178784Skmacy 335178784Skmacystatic int client_recv(struct krping_cb *cb, struct ib_wc *wc) 336178784Skmacy{ 337178784Skmacy if (wc->byte_len != sizeof(cb->recv_buf)) { 338256829Snp PRINTF(cb, "Received bogus data, size %d\n", 339178784Skmacy wc->byte_len); 340178784Skmacy return -1; 341178784Skmacy } 342178784Skmacy 343178784Skmacy if (cb->state == RDMA_READ_ADV) 344178784Skmacy cb->state = RDMA_WRITE_ADV; 345178784Skmacy else 346178784Skmacy cb->state = RDMA_WRITE_COMPLETE; 347178784Skmacy 348178784Skmacy return 0; 349178784Skmacy} 350178784Skmacy 351178784Skmacystatic void krping_cq_event_handler(struct ib_cq *cq, void *ctx) 352178784Skmacy{ 353178784Skmacy struct krping_cb *cb = ctx; 354178784Skmacy struct ib_wc wc; 355178784Skmacy struct ib_recv_wr *bad_wr; 356178784Skmacy int ret; 357178784Skmacy 358256829Snp BUG_ON(cb->cq != cq); 359178784Skmacy if (cb->state == ERROR) { 360256829Snp PRINTF(cb, "cq completion in ERROR state\n"); 361178784Skmacy return; 362178784Skmacy } 363256829Snp if (cb->frtest) { 364256829Snp PRINTF(cb, "cq completion event in frtest!\n"); 365256829Snp return; 366256829Snp } 367178784Skmacy if (!cb->wlat && !cb->rlat && !cb->bw) 368178784Skmacy ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); 369178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { 370178784Skmacy if (wc.status) { 371237263Snp if (wc.status == IB_WC_WR_FLUSH_ERR) { 372256829Snp DEBUG_LOG(cb, "cq flushed\n"); 373237263Snp continue; 374237263Snp } else { 375256829Snp PRINTF(cb, "cq completion failed with " 376256829Snp "wr_id %Lx status %d opcode %d vender_err %x\n", 377256829Snp wc.wr_id, wc.status, wc.opcode, wc.vendor_err); 378237263Snp goto error; 379237263Snp } 380178784Skmacy } 381178784Skmacy 382178784Skmacy switch (wc.opcode) { 383178784Skmacy case IB_WC_SEND: 384256829Snp DEBUG_LOG(cb, "send completion\n"); 385178784Skmacy cb->stats.send_bytes += cb->send_sgl.length; 386178784Skmacy cb->stats.send_msgs++; 387178784Skmacy break; 388178784Skmacy 389178784Skmacy case IB_WC_RDMA_WRITE: 390256829Snp DEBUG_LOG(cb, "rdma write completion\n"); 391178784Skmacy cb->stats.write_bytes += cb->rdma_sq_wr.sg_list->length; 392178784Skmacy cb->stats.write_msgs++; 393178784Skmacy cb->state = RDMA_WRITE_COMPLETE; 394256829Snp wake_up_interruptible(&cb->sem); 395178784Skmacy break; 396178784Skmacy 397178784Skmacy case IB_WC_RDMA_READ: 398256829Snp DEBUG_LOG(cb, "rdma read completion\n"); 399178784Skmacy cb->stats.read_bytes += cb->rdma_sq_wr.sg_list->length; 400178784Skmacy cb->stats.read_msgs++; 401178784Skmacy cb->state = RDMA_READ_COMPLETE; 402256829Snp wake_up_interruptible(&cb->sem); 403178784Skmacy break; 404178784Skmacy 405178784Skmacy case IB_WC_RECV: 406256829Snp DEBUG_LOG(cb, "recv completion\n"); 407178784Skmacy cb->stats.recv_bytes += sizeof(cb->recv_buf); 408178784Skmacy cb->stats.recv_msgs++; 409178784Skmacy if (cb->wlat || cb->rlat || cb->bw) 410178784Skmacy ret = server_recv(cb, &wc); 411178784Skmacy else 412178784Skmacy ret = cb->server ? server_recv(cb, &wc) : 413256829Snp client_recv(cb, &wc); 414178784Skmacy if (ret) { 415256829Snp PRINTF(cb, "recv wc error: %d\n", ret); 416178784Skmacy goto error; 417178784Skmacy } 418178784Skmacy 419178784Skmacy ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); 420178784Skmacy if (ret) { 421256829Snp PRINTF(cb, "post recv error: %d\n", 422178784Skmacy ret); 423178784Skmacy goto error; 424178784Skmacy } 425256829Snp wake_up_interruptible(&cb->sem); 426178784Skmacy break; 427178784Skmacy 428178784Skmacy default: 429256829Snp PRINTF(cb, 430256829Snp "%s:%d Unexpected opcode %d, Shutting down\n", 431256829Snp __func__, __LINE__, wc.opcode); 432178784Skmacy goto error; 433178784Skmacy } 434178784Skmacy } 435178784Skmacy if (ret) { 436256829Snp PRINTF(cb, "poll error %d\n", ret); 437178784Skmacy goto error; 438178784Skmacy } 439178784Skmacy return; 440178784Skmacyerror: 441178784Skmacy cb->state = ERROR; 442256829Snp wake_up_interruptible(&cb->sem); 443178784Skmacy} 444178784Skmacy 445178784Skmacystatic int krping_accept(struct krping_cb *cb) 446178784Skmacy{ 447178784Skmacy struct rdma_conn_param conn_param; 448178784Skmacy int ret; 449178784Skmacy 450256829Snp DEBUG_LOG(cb, "accepting client connection request\n"); 451178784Skmacy 452178784Skmacy memset(&conn_param, 0, sizeof conn_param); 453178784Skmacy conn_param.responder_resources = 1; 454178784Skmacy conn_param.initiator_depth = 1; 455178784Skmacy 456178784Skmacy ret = rdma_accept(cb->child_cm_id, &conn_param); 457178784Skmacy if (ret) { 458256829Snp PRINTF(cb, "rdma_accept error: %d\n", ret); 459178784Skmacy return ret; 460178784Skmacy } 461178784Skmacy 462178784Skmacy if (!cb->wlat && !cb->rlat && !cb->bw) { 463256829Snp wait_event_interruptible(cb->sem, cb->state >= CONNECTED); 464178784Skmacy if (cb->state == ERROR) { 465256829Snp PRINTF(cb, "wait for CONNECTED state %d\n", 466256829Snp cb->state); 467178784Skmacy return -1; 468178784Skmacy } 469178784Skmacy } 470178784Skmacy return 0; 471178784Skmacy} 472178784Skmacy 473178784Skmacystatic void krping_setup_wr(struct krping_cb *cb) 474178784Skmacy{ 475256829Snp cb->recv_sgl.addr = cb->recv_dma_addr; 476178784Skmacy cb->recv_sgl.length = sizeof cb->recv_buf; 477256829Snp if (cb->local_dma_lkey) 478256829Snp cb->recv_sgl.lkey = cb->qp->device->local_dma_lkey; 479256829Snp else if (cb->mem == DMA) 480178784Skmacy cb->recv_sgl.lkey = cb->dma_mr->lkey; 481178784Skmacy else 482178784Skmacy cb->recv_sgl.lkey = cb->recv_mr->lkey; 483178784Skmacy cb->rq_wr.sg_list = &cb->recv_sgl; 484178784Skmacy cb->rq_wr.num_sge = 1; 485178784Skmacy 486256829Snp cb->send_sgl.addr = cb->send_dma_addr; 487178784Skmacy cb->send_sgl.length = sizeof cb->send_buf; 488256829Snp if (cb->local_dma_lkey) 489256829Snp cb->send_sgl.lkey = cb->qp->device->local_dma_lkey; 490256829Snp else if (cb->mem == DMA) 491178784Skmacy cb->send_sgl.lkey = cb->dma_mr->lkey; 492178784Skmacy else 493178784Skmacy cb->send_sgl.lkey = cb->send_mr->lkey; 494178784Skmacy 495178784Skmacy cb->sq_wr.opcode = IB_WR_SEND; 496178784Skmacy cb->sq_wr.send_flags = IB_SEND_SIGNALED; 497178784Skmacy cb->sq_wr.sg_list = &cb->send_sgl; 498178784Skmacy cb->sq_wr.num_sge = 1; 499178784Skmacy 500256829Snp if (cb->server || cb->wlat || cb->rlat || cb->bw) { 501256829Snp cb->rdma_sgl.addr = cb->rdma_dma_addr; 502256829Snp if (cb->mem == MR) 503256829Snp cb->rdma_sgl.lkey = cb->rdma_mr->lkey; 504256829Snp cb->rdma_sq_wr.send_flags = IB_SEND_SIGNALED; 505256829Snp cb->rdma_sq_wr.sg_list = &cb->rdma_sgl; 506256829Snp cb->rdma_sq_wr.num_sge = 1; 507256829Snp } 508178784Skmacy 509256829Snp switch(cb->mem) { 510256829Snp case FASTREG: 511256829Snp 512256829Snp /* 513256829Snp * A chain of 2 WRs, INVALDATE_MR + FAST_REG_MR. 514256829Snp * both unsignaled. The client uses them to reregister 515256829Snp * the rdma buffers with a new key each iteration. 516256829Snp */ 517256829Snp cb->fastreg_wr.opcode = IB_WR_FAST_REG_MR; 518256829Snp cb->fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 519256829Snp cb->fastreg_wr.wr.fast_reg.length = cb->size; 520256829Snp cb->fastreg_wr.wr.fast_reg.page_list = cb->page_list; 521256829Snp cb->fastreg_wr.wr.fast_reg.page_list_len = cb->page_list_len; 522256829Snp 523256829Snp cb->invalidate_wr.next = &cb->fastreg_wr; 524256829Snp cb->invalidate_wr.opcode = IB_WR_LOCAL_INV; 525256829Snp break; 526256829Snp case MW: 527256829Snp cb->bind_attr.wr_id = 0xabbaabba; 528256829Snp cb->bind_attr.send_flags = 0; /* unsignaled */ 529256829Snp cb->bind_attr.length = cb->size; 530256829Snp break; 531256829Snp default: 532256829Snp break; 533178784Skmacy } 534178784Skmacy} 535178784Skmacy 536178784Skmacystatic int krping_setup_buffers(struct krping_cb *cb) 537178784Skmacy{ 538178784Skmacy int ret; 539178784Skmacy struct ib_phys_buf buf; 540178784Skmacy u64 iovbase; 541178784Skmacy 542256829Snp DEBUG_LOG(cb, "krping_setup_buffers called on cb %p\n", cb); 543178784Skmacy 544256829Snp cb->recv_dma_addr = dma_map_single(cb->pd->device->dma_device, 545256829Snp &cb->recv_buf, 546256829Snp sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); 547256829Snp pci_unmap_addr_set(cb, recv_mapping, cb->recv_dma_addr); 548256829Snp cb->send_dma_addr = dma_map_single(cb->pd->device->dma_device, 549256829Snp &cb->send_buf, sizeof(cb->send_buf), 550256829Snp DMA_BIDIRECTIONAL); 551256829Snp pci_unmap_addr_set(cb, send_mapping, cb->send_dma_addr); 552256829Snp 553256829Snp if (cb->mem == DMA) { 554178784Skmacy cb->dma_mr = ib_get_dma_mr(cb->pd, IB_ACCESS_LOCAL_WRITE| 555178784Skmacy IB_ACCESS_REMOTE_READ| 556178784Skmacy IB_ACCESS_REMOTE_WRITE); 557178784Skmacy if (IS_ERR(cb->dma_mr)) { 558256829Snp DEBUG_LOG(cb, "reg_dmamr failed\n"); 559256829Snp ret = PTR_ERR(cb->dma_mr); 560256829Snp goto bail; 561178784Skmacy } 562178784Skmacy } else { 563256829Snp if (!cb->local_dma_lkey) { 564256829Snp buf.addr = cb->recv_dma_addr; 565256829Snp buf.size = sizeof cb->recv_buf; 566256829Snp DEBUG_LOG(cb, "recv buf dma_addr %llx size %d\n", buf.addr, 567256829Snp (int)buf.size); 568256829Snp iovbase = cb->recv_dma_addr; 569256829Snp cb->recv_mr = ib_reg_phys_mr(cb->pd, &buf, 1, 570256829Snp IB_ACCESS_LOCAL_WRITE, 571256829Snp &iovbase); 572178784Skmacy 573256829Snp if (IS_ERR(cb->recv_mr)) { 574256829Snp DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); 575256829Snp ret = PTR_ERR(cb->recv_mr); 576256829Snp goto bail; 577256829Snp } 578178784Skmacy 579256829Snp buf.addr = cb->send_dma_addr; 580256829Snp buf.size = sizeof cb->send_buf; 581256829Snp DEBUG_LOG(cb, "send buf dma_addr %llx size %d\n", buf.addr, 582256829Snp (int)buf.size); 583256829Snp iovbase = cb->send_dma_addr; 584256829Snp cb->send_mr = ib_reg_phys_mr(cb->pd, &buf, 1, 585256829Snp 0, &iovbase); 586178784Skmacy 587256829Snp if (IS_ERR(cb->send_mr)) { 588256829Snp DEBUG_LOG(cb, "send_buf reg_mr failed\n"); 589256829Snp ret = PTR_ERR(cb->send_mr); 590256829Snp goto bail; 591256829Snp } 592178784Skmacy } 593178784Skmacy } 594178784Skmacy 595256829Snp cb->rdma_buf = kmalloc(cb->size, GFP_KERNEL); 596178784Skmacy if (!cb->rdma_buf) { 597256829Snp DEBUG_LOG(cb, "rdma_buf malloc failed\n"); 598256829Snp ret = -ENOMEM; 599256829Snp goto bail; 600178784Skmacy } 601178784Skmacy 602256829Snp cb->rdma_dma_addr = dma_map_single(cb->pd->device->dma_device, 603256829Snp cb->rdma_buf, cb->size, 604256829Snp DMA_BIDIRECTIONAL); 605256829Snp pci_unmap_addr_set(cb, rdma_mapping, cb->rdma_dma_addr); 606256829Snp if (cb->mem != DMA) { 607256829Snp switch (cb->mem) { 608256829Snp case FASTREG: 609256829Snp cb->page_list_len = (((cb->size - 1) & PAGE_MASK) + 610256829Snp PAGE_SIZE) >> PAGE_SHIFT; 611256829Snp cb->page_list = ib_alloc_fast_reg_page_list( 612256829Snp cb->pd->device, 613256829Snp cb->page_list_len); 614256829Snp if (IS_ERR(cb->page_list)) { 615256829Snp DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); 616256829Snp ret = PTR_ERR(cb->page_list); 617256829Snp goto bail; 618256829Snp } 619256829Snp cb->fastreg_mr = ib_alloc_fast_reg_mr(cb->pd, 620256829Snp cb->page_list->max_page_list_len); 621256829Snp if (IS_ERR(cb->fastreg_mr)) { 622256829Snp DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); 623256829Snp ret = PTR_ERR(cb->fastreg_mr); 624256829Snp goto bail; 625256829Snp } 626256829Snp DEBUG_LOG(cb, "fastreg rkey 0x%x page_list %p" 627256829Snp " page_list_len %u\n", cb->fastreg_mr->rkey, 628256829Snp cb->page_list, cb->page_list_len); 629256829Snp break; 630256829Snp case MW: 631256829Snp cb->mw = ib_alloc_mw(cb->pd); 632256829Snp if (IS_ERR(cb->mw)) { 633256829Snp DEBUG_LOG(cb, "recv_buf alloc_mw failed\n"); 634256829Snp ret = PTR_ERR(cb->mw); 635256829Snp goto bail; 636256829Snp } 637256829Snp DEBUG_LOG(cb, "mw rkey 0x%x\n", cb->mw->rkey); 638256829Snp /*FALLTHROUGH*/ 639256829Snp case MR: 640256829Snp buf.addr = cb->rdma_dma_addr; 641256829Snp buf.size = cb->size; 642256829Snp iovbase = cb->rdma_dma_addr; 643256829Snp cb->rdma_mr = ib_reg_phys_mr(cb->pd, &buf, 1, 644178784Skmacy IB_ACCESS_REMOTE_READ| 645178784Skmacy IB_ACCESS_REMOTE_WRITE, 646178784Skmacy &iovbase); 647256829Snp if (IS_ERR(cb->rdma_mr)) { 648256829Snp DEBUG_LOG(cb, "rdma_buf reg_mr failed\n"); 649256829Snp ret = PTR_ERR(cb->rdma_mr); 650256829Snp goto bail; 651256829Snp } 652256829Snp DEBUG_LOG(cb, "rdma buf dma_addr %llx size %d mr rkey 0x%x\n", 653256829Snp buf.addr, (int)buf.size, cb->rdma_mr->rkey); 654256829Snp break; 655256829Snp default: 656256829Snp ret = -EINVAL; 657256829Snp goto bail; 658256829Snp break; 659178784Skmacy } 660178784Skmacy } 661178784Skmacy 662178784Skmacy if (!cb->server || cb->wlat || cb->rlat || cb->bw) { 663256829Snp 664256829Snp cb->start_buf = kmalloc(cb->size, GFP_KERNEL); 665178784Skmacy if (!cb->start_buf) { 666256829Snp DEBUG_LOG(cb, "start_buf malloc failed\n"); 667256829Snp ret = -ENOMEM; 668256829Snp goto bail; 669178784Skmacy } 670256829Snp 671256829Snp cb->start_dma_addr = dma_map_single(cb->pd->device->dma_device, 672256829Snp cb->start_buf, cb->size, 673256829Snp DMA_BIDIRECTIONAL); 674256829Snp pci_unmap_addr_set(cb, start_mapping, cb->start_dma_addr); 675256829Snp 676256829Snp if (cb->mem == MR || cb->mem == MW) { 677178784Skmacy unsigned flags = IB_ACCESS_REMOTE_READ; 678178784Skmacy 679256829Snp if (cb->wlat || cb->rlat || cb->bw) 680178784Skmacy flags |= IB_ACCESS_REMOTE_WRITE; 681256829Snp 682256829Snp buf.addr = cb->start_dma_addr; 683178784Skmacy buf.size = cb->size; 684256829Snp DEBUG_LOG(cb, "start buf dma_addr %llx size %d\n", 685256829Snp buf.addr, (int)buf.size); 686256829Snp iovbase = cb->start_dma_addr; 687178784Skmacy cb->start_mr = ib_reg_phys_mr(cb->pd, &buf, 1, 688178784Skmacy flags, 689178784Skmacy &iovbase); 690178784Skmacy 691178784Skmacy if (IS_ERR(cb->start_mr)) { 692256829Snp DEBUG_LOG(cb, "start_buf reg_mr failed\n"); 693178784Skmacy ret = PTR_ERR(cb->start_mr); 694256829Snp goto bail; 695178784Skmacy } 696178784Skmacy } 697178784Skmacy } 698178784Skmacy 699178784Skmacy krping_setup_wr(cb); 700256829Snp DEBUG_LOG(cb, "allocated & registered buffers...\n"); 701178784Skmacy return 0; 702256829Snpbail: 703256829Snp if (cb->fastreg_mr && !IS_ERR(cb->fastreg_mr)) 704256829Snp ib_dereg_mr(cb->fastreg_mr); 705256829Snp if (cb->mw && !IS_ERR(cb->mw)) 706256829Snp ib_dealloc_mw(cb->mw); 707256829Snp if (cb->rdma_mr && !IS_ERR(cb->rdma_mr)) 708178784Skmacy ib_dereg_mr(cb->rdma_mr); 709256829Snp if (cb->page_list && !IS_ERR(cb->page_list)) 710256829Snp ib_free_fast_reg_page_list(cb->page_list); 711256829Snp if (cb->dma_mr && !IS_ERR(cb->dma_mr)) 712178784Skmacy ib_dereg_mr(cb->dma_mr); 713256829Snp if (cb->recv_mr && !IS_ERR(cb->recv_mr)) 714178784Skmacy ib_dereg_mr(cb->recv_mr); 715256829Snp if (cb->send_mr && !IS_ERR(cb->send_mr)) 716178784Skmacy ib_dereg_mr(cb->send_mr); 717256829Snp if (cb->rdma_buf) 718256829Snp kfree(cb->rdma_buf); 719256829Snp if (cb->start_buf) 720256829Snp kfree(cb->start_buf); 721178784Skmacy return ret; 722178784Skmacy} 723178784Skmacy 724178784Skmacystatic void krping_free_buffers(struct krping_cb *cb) 725178784Skmacy{ 726256829Snp DEBUG_LOG(cb, "krping_free_buffers called on cb %p\n", cb); 727178784Skmacy 728256829Snp if (cb->dma_mr) 729256829Snp ib_dereg_mr(cb->dma_mr); 730256829Snp if (cb->send_mr) 731256829Snp ib_dereg_mr(cb->send_mr); 732256829Snp if (cb->recv_mr) 733256829Snp ib_dereg_mr(cb->recv_mr); 734256829Snp if (cb->rdma_mr) 735256829Snp ib_dereg_mr(cb->rdma_mr); 736256829Snp if (cb->start_mr) 737256829Snp ib_dereg_mr(cb->start_mr); 738256829Snp if (cb->fastreg_mr) 739256829Snp ib_dereg_mr(cb->fastreg_mr); 740256829Snp if (cb->mw) 741256829Snp ib_dealloc_mw(cb->mw); 742256829Snp 743178784Skmacy dma_unmap_single(cb->pd->device->dma_device, 744178784Skmacy pci_unmap_addr(cb, recv_mapping), 745178784Skmacy sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); 746178784Skmacy dma_unmap_single(cb->pd->device->dma_device, 747178784Skmacy pci_unmap_addr(cb, send_mapping), 748178784Skmacy sizeof(cb->send_buf), DMA_BIDIRECTIONAL); 749178784Skmacy dma_unmap_single(cb->pd->device->dma_device, 750178784Skmacy pci_unmap_addr(cb, rdma_mapping), 751178784Skmacy cb->size, DMA_BIDIRECTIONAL); 752256829Snp kfree(cb->rdma_buf); 753256829Snp if (cb->start_buf) { 754178784Skmacy dma_unmap_single(cb->pd->device->dma_device, 755178784Skmacy pci_unmap_addr(cb, start_mapping), 756178784Skmacy cb->size, DMA_BIDIRECTIONAL); 757256829Snp kfree(cb->start_buf); 758178784Skmacy } 759178784Skmacy} 760178784Skmacy 761178784Skmacystatic int krping_create_qp(struct krping_cb *cb) 762178784Skmacy{ 763178784Skmacy struct ib_qp_init_attr init_attr; 764178784Skmacy int ret; 765178784Skmacy 766178784Skmacy memset(&init_attr, 0, sizeof(init_attr)); 767178784Skmacy init_attr.cap.max_send_wr = cb->txdepth; 768178784Skmacy init_attr.cap.max_recv_wr = 2; 769178784Skmacy init_attr.cap.max_recv_sge = 1; 770178784Skmacy init_attr.cap.max_send_sge = 1; 771178784Skmacy init_attr.qp_type = IB_QPT_RC; 772178784Skmacy init_attr.send_cq = cb->cq; 773178784Skmacy init_attr.recv_cq = cb->cq; 774256829Snp init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 775178784Skmacy 776178784Skmacy if (cb->server) { 777178784Skmacy ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr); 778178784Skmacy if (!ret) 779178784Skmacy cb->qp = cb->child_cm_id->qp; 780178784Skmacy } else { 781178784Skmacy ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr); 782178784Skmacy if (!ret) 783178784Skmacy cb->qp = cb->cm_id->qp; 784178784Skmacy } 785178784Skmacy 786178784Skmacy return ret; 787178784Skmacy} 788178784Skmacy 789178784Skmacystatic void krping_free_qp(struct krping_cb *cb) 790178784Skmacy{ 791178784Skmacy ib_destroy_qp(cb->qp); 792178784Skmacy ib_destroy_cq(cb->cq); 793178784Skmacy ib_dealloc_pd(cb->pd); 794178784Skmacy} 795178784Skmacy 796178784Skmacystatic int krping_setup_qp(struct krping_cb *cb, struct rdma_cm_id *cm_id) 797178784Skmacy{ 798178784Skmacy int ret; 799178784Skmacy cb->pd = ib_alloc_pd(cm_id->device); 800178784Skmacy if (IS_ERR(cb->pd)) { 801256829Snp PRINTF(cb, "ib_alloc_pd failed\n"); 802178784Skmacy return PTR_ERR(cb->pd); 803178784Skmacy } 804256829Snp DEBUG_LOG(cb, "created pd %p\n", cb->pd); 805178784Skmacy 806256829Snp strlcpy(cb->stats.name, cb->pd->device->name, sizeof(cb->stats.name)); 807254735Snp 808178784Skmacy cb->cq = ib_create_cq(cm_id->device, krping_cq_event_handler, NULL, 809178784Skmacy cb, cb->txdepth * 2, 0); 810178784Skmacy if (IS_ERR(cb->cq)) { 811256829Snp PRINTF(cb, "ib_create_cq failed\n"); 812178784Skmacy ret = PTR_ERR(cb->cq); 813178784Skmacy goto err1; 814178784Skmacy } 815256829Snp DEBUG_LOG(cb, "created cq %p\n", cb->cq); 816178784Skmacy 817256829Snp if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { 818178784Skmacy ret = ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); 819178784Skmacy if (ret) { 820256829Snp PRINTF(cb, "ib_create_cq failed\n"); 821178784Skmacy goto err2; 822178784Skmacy } 823178784Skmacy } 824178784Skmacy 825178784Skmacy ret = krping_create_qp(cb); 826178784Skmacy if (ret) { 827256829Snp PRINTF(cb, "krping_create_qp failed: %d\n", ret); 828178784Skmacy goto err2; 829178784Skmacy } 830256829Snp DEBUG_LOG(cb, "created qp %p\n", cb->qp); 831178784Skmacy return 0; 832178784Skmacyerr2: 833178784Skmacy ib_destroy_cq(cb->cq); 834178784Skmacyerr1: 835178784Skmacy ib_dealloc_pd(cb->pd); 836178784Skmacy return ret; 837178784Skmacy} 838178784Skmacy 839256829Snp/* 840256829Snp * return the (possibly rebound) rkey for the rdma buffer. 841256829Snp * FASTREG mode: invalidate and rebind via fastreg wr. 842256829Snp * MW mode: rebind the MW. 843256829Snp * other modes: just return the mr rkey. 844256829Snp */ 845256829Snpstatic u32 krping_rdma_rkey(struct krping_cb *cb, u64 buf, int post_inv) 846178784Skmacy{ 847256829Snp u32 rkey = 0xffffffff; 848256829Snp u64 p; 849256829Snp struct ib_send_wr *bad_wr; 850256829Snp int i; 851256829Snp int ret; 852178784Skmacy 853256829Snp switch (cb->mem) { 854256829Snp case FASTREG: 855256829Snp cb->invalidate_wr.ex.invalidate_rkey = cb->fastreg_mr->rkey; 856178784Skmacy 857256829Snp /* 858256829Snp * Update the fastreg key. 859256829Snp */ 860256829Snp ib_update_fast_reg_key(cb->fastreg_mr, ++cb->key); 861256829Snp cb->fastreg_wr.wr.fast_reg.rkey = cb->fastreg_mr->rkey; 862256829Snp 863256829Snp /* 864256829Snp * Update the fastreg WR with new buf info. 865256829Snp */ 866256829Snp if (buf == (u64)cb->start_dma_addr) 867256829Snp cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_READ; 868256829Snp else 869256829Snp cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; 870256829Snp cb->fastreg_wr.wr.fast_reg.iova_start = buf; 871256829Snp p = (u64)(buf & PAGE_MASK); 872256829Snp for (i=0; i < cb->fastreg_wr.wr.fast_reg.page_list_len; 873256829Snp i++, p += PAGE_SIZE) { 874256829Snp cb->page_list->page_list[i] = p; 875256829Snp DEBUG_LOG(cb, "page_list[%d] 0x%llx\n", i, p); 876256829Snp } 877256829Snp 878256829Snp DEBUG_LOG(cb, "post_inv = %d, fastreg new rkey 0x%x shift %u len %u" 879256829Snp " iova_start %llx page_list_len %u\n", 880256829Snp post_inv, 881256829Snp cb->fastreg_wr.wr.fast_reg.rkey, 882256829Snp cb->fastreg_wr.wr.fast_reg.page_shift, 883256829Snp cb->fastreg_wr.wr.fast_reg.length, 884256829Snp cb->fastreg_wr.wr.fast_reg.iova_start, 885256829Snp cb->fastreg_wr.wr.fast_reg.page_list_len); 886256829Snp 887256829Snp if (post_inv) 888256829Snp ret = ib_post_send(cb->qp, &cb->invalidate_wr, &bad_wr); 889256829Snp else 890256829Snp ret = ib_post_send(cb->qp, &cb->fastreg_wr, &bad_wr); 891256829Snp if (ret) { 892256829Snp PRINTF(cb, "post send error %d\n", ret); 893256829Snp cb->state = ERROR; 894256829Snp } 895256829Snp rkey = cb->fastreg_mr->rkey; 896256829Snp break; 897256829Snp case MW: 898256829Snp /* 899256829Snp * Update the MW with new buf info. 900256829Snp */ 901256829Snp if (buf == (u64)cb->start_dma_addr) { 902256829Snp cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_READ; 903256829Snp cb->bind_attr.mr = cb->start_mr; 904256829Snp } else { 905256829Snp cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_WRITE; 906256829Snp cb->bind_attr.mr = cb->rdma_mr; 907256829Snp } 908256829Snp cb->bind_attr.addr = buf; 909256829Snp DEBUG_LOG(cb, "binding mw rkey 0x%x to buf %llx mr rkey 0x%x\n", 910256829Snp cb->mw->rkey, buf, cb->bind_attr.mr->rkey); 911256829Snp ret = ib_bind_mw(cb->qp, cb->mw, &cb->bind_attr); 912256829Snp if (ret) { 913256829Snp PRINTF(cb, "bind mw error %d\n", ret); 914256829Snp cb->state = ERROR; 915256829Snp } else 916256829Snp rkey = cb->mw->rkey; 917256829Snp break; 918256829Snp case MR: 919256829Snp if (buf == (u64)cb->start_dma_addr) 920256829Snp rkey = cb->start_mr->rkey; 921256829Snp else 922256829Snp rkey = cb->rdma_mr->rkey; 923256829Snp break; 924256829Snp case DMA: 925256829Snp rkey = cb->dma_mr->rkey; 926256829Snp break; 927256829Snp default: 928256829Snp PRINTF(cb, "%s:%d case ERROR\n", __func__, __LINE__); 929256829Snp cb->state = ERROR; 930256829Snp break; 931256829Snp } 932256829Snp return rkey; 933178784Skmacy} 934178784Skmacy 935256829Snpstatic void krping_format_send(struct krping_cb *cb, u64 buf) 936256829Snp{ 937256829Snp struct krping_rdma_info *info = &cb->send_buf; 938256829Snp u32 rkey; 939256829Snp 940256829Snp /* 941256829Snp * Client side will do fastreg or mw bind before 942256829Snp * advertising the rdma buffer. Server side 943256829Snp * sends have no data. 944256829Snp */ 945256829Snp if (!cb->server || cb->wlat || cb->rlat || cb->bw) { 946256829Snp rkey = krping_rdma_rkey(cb, buf, !cb->server_invalidate); 947256829Snp info->buf = htonll(buf); 948256829Snp info->rkey = htonl(rkey); 949256829Snp info->size = htonl(cb->size); 950256829Snp DEBUG_LOG(cb, "RDMA addr %llx rkey %x len %d\n", 951256829Snp (unsigned long long)buf, rkey, cb->size); 952256829Snp } 953256829Snp} 954256829Snp 955178784Skmacystatic void krping_test_server(struct krping_cb *cb) 956178784Skmacy{ 957256829Snp struct ib_send_wr *bad_wr, inv; 958178784Skmacy int ret; 959178784Skmacy 960178784Skmacy while (1) { 961178784Skmacy /* Wait for client's Start STAG/TO/Len */ 962256829Snp wait_event_interruptible(cb->sem, cb->state >= RDMA_READ_ADV); 963178784Skmacy if (cb->state != RDMA_READ_ADV) { 964256829Snp PRINTF(cb, "wait for RDMA_READ_ADV state %d\n", 965178784Skmacy cb->state); 966178784Skmacy break; 967178784Skmacy } 968178784Skmacy 969256829Snp DEBUG_LOG(cb, "server received sink adv\n"); 970178784Skmacy 971178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 972178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 973178784Skmacy cb->rdma_sq_wr.sg_list->length = cb->remote_len; 974256829Snp cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 1); 975256829Snp cb->rdma_sq_wr.next = NULL; 976178784Skmacy 977256829Snp /* Issue RDMA Read. */ 978256829Snp if (cb->read_inv) 979256829Snp cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ_WITH_INV; 980256829Snp else { 981256829Snp 982256829Snp cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; 983256829Snp if (cb->mem == FASTREG) { 984256829Snp /* 985256829Snp * Immediately follow the read with a 986256829Snp * fenced LOCAL_INV. 987256829Snp */ 988256829Snp cb->rdma_sq_wr.next = &inv; 989256829Snp memset(&inv, 0, sizeof inv); 990256829Snp inv.opcode = IB_WR_LOCAL_INV; 991256829Snp inv.ex.invalidate_rkey = cb->fastreg_mr->rkey; 992256829Snp inv.send_flags = IB_SEND_FENCE; 993256829Snp } 994256829Snp } 995256829Snp 996178784Skmacy ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); 997178784Skmacy if (ret) { 998256829Snp PRINTF(cb, "post send error %d\n", ret); 999178784Skmacy break; 1000178784Skmacy } 1001256829Snp cb->rdma_sq_wr.next = NULL; 1002178784Skmacy 1003256829Snp DEBUG_LOG(cb, "server posted rdma read req \n"); 1004256829Snp 1005178784Skmacy /* Wait for read completion */ 1006256829Snp wait_event_interruptible(cb->sem, 1007256829Snp cb->state >= RDMA_READ_COMPLETE); 1008178784Skmacy if (cb->state != RDMA_READ_COMPLETE) { 1009256829Snp PRINTF(cb, 1010178784Skmacy "wait for RDMA_READ_COMPLETE state %d\n", 1011178784Skmacy cb->state); 1012178784Skmacy break; 1013178784Skmacy } 1014256829Snp DEBUG_LOG(cb, "server received read complete\n"); 1015178784Skmacy 1016178784Skmacy /* Display data in recv buf */ 1017178784Skmacy if (cb->verbose) 1018256829Snp PRINTF(cb, "server ping data: %s\n", 1019256829Snp cb->rdma_buf); 1020178784Skmacy 1021178784Skmacy /* Tell client to continue */ 1022256829Snp if (cb->server && cb->server_invalidate) { 1023256829Snp cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; 1024256829Snp cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; 1025256829Snp DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); 1026256829Snp } 1027178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1028178784Skmacy if (ret) { 1029256829Snp PRINTF(cb, "post send error %d\n", ret); 1030178784Skmacy break; 1031178784Skmacy } 1032256829Snp DEBUG_LOG(cb, "server posted go ahead\n"); 1033178784Skmacy 1034178784Skmacy /* Wait for client's RDMA STAG/TO/Len */ 1035256829Snp wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); 1036178784Skmacy if (cb->state != RDMA_WRITE_ADV) { 1037256829Snp PRINTF(cb, 1038178784Skmacy "wait for RDMA_WRITE_ADV state %d\n", 1039178784Skmacy cb->state); 1040178784Skmacy break; 1041178784Skmacy } 1042256829Snp DEBUG_LOG(cb, "server received sink adv\n"); 1043178784Skmacy 1044178784Skmacy /* RDMA Write echo data */ 1045178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; 1046178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1047178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1048178784Skmacy cb->rdma_sq_wr.sg_list->length = strlen(cb->rdma_buf) + 1; 1049256829Snp if (cb->local_dma_lkey) 1050256829Snp cb->rdma_sgl.lkey = cb->qp->device->local_dma_lkey; 1051256829Snp else 1052256829Snp cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 0); 1053256829Snp 1054256829Snp DEBUG_LOG(cb, "rdma write from lkey %x laddr %llx len %d\n", 1055178784Skmacy cb->rdma_sq_wr.sg_list->lkey, 1056178784Skmacy (unsigned long long)cb->rdma_sq_wr.sg_list->addr, 1057178784Skmacy cb->rdma_sq_wr.sg_list->length); 1058178784Skmacy 1059178784Skmacy ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); 1060178784Skmacy if (ret) { 1061256829Snp PRINTF(cb, "post send error %d\n", ret); 1062178784Skmacy break; 1063178784Skmacy } 1064178784Skmacy 1065178784Skmacy /* Wait for completion */ 1066256829Snp ret = wait_event_interruptible(cb->sem, cb->state >= 1067256829Snp RDMA_WRITE_COMPLETE); 1068178784Skmacy if (cb->state != RDMA_WRITE_COMPLETE) { 1069256829Snp PRINTF(cb, 1070178784Skmacy "wait for RDMA_WRITE_COMPLETE state %d\n", 1071178784Skmacy cb->state); 1072178784Skmacy break; 1073178784Skmacy } 1074256829Snp DEBUG_LOG(cb, "server rdma write complete \n"); 1075178784Skmacy 1076178784Skmacy cb->state = CONNECTED; 1077178784Skmacy 1078178784Skmacy /* Tell client to begin again */ 1079256829Snp if (cb->server && cb->server_invalidate) { 1080256829Snp cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; 1081256829Snp cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; 1082256829Snp DEBUG_LOG(cb, "send-w-inv rkey 0x%x\n", cb->remote_rkey); 1083256829Snp } 1084178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1085178784Skmacy if (ret) { 1086256829Snp PRINTF(cb, "post send error %d\n", ret); 1087178784Skmacy break; 1088178784Skmacy } 1089256829Snp DEBUG_LOG(cb, "server posted go ahead\n"); 1090178784Skmacy } 1091178784Skmacy} 1092178784Skmacy 1093178784Skmacystatic void rlat_test(struct krping_cb *cb) 1094178784Skmacy{ 1095178784Skmacy int scnt; 1096178784Skmacy int iters = cb->count; 1097178784Skmacy struct timeval start_tv, stop_tv; 1098178784Skmacy int ret; 1099178784Skmacy struct ib_wc wc; 1100178784Skmacy struct ib_send_wr *bad_wr; 1101178784Skmacy int ne; 1102178784Skmacy 1103178784Skmacy scnt = 0; 1104178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; 1105178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1106178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1107178784Skmacy cb->rdma_sq_wr.sg_list->length = cb->size; 1108178784Skmacy 1109178784Skmacy microtime(&start_tv); 1110256829Snp if (!cb->poll) { 1111256829Snp cb->state = RDMA_READ_ADV; 1112256829Snp ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); 1113256829Snp } 1114178784Skmacy while (scnt < iters) { 1115178784Skmacy 1116256829Snp cb->state = RDMA_READ_ADV; 1117178784Skmacy ret = ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr); 1118178784Skmacy if (ret) { 1119256829Snp PRINTF(cb, 1120178784Skmacy "Couldn't post send: ret=%d scnt %d\n", 1121178784Skmacy ret, scnt); 1122178784Skmacy return; 1123178784Skmacy } 1124178784Skmacy 1125178784Skmacy do { 1126178784Skmacy if (!cb->poll) { 1127256829Snp wait_event_interruptible(cb->sem, 1128256829Snp cb->state != RDMA_READ_ADV); 1129178784Skmacy if (cb->state == RDMA_READ_COMPLETE) { 1130178784Skmacy ne = 1; 1131256829Snp ib_req_notify_cq(cb->cq, 1132256829Snp IB_CQ_NEXT_COMP); 1133178784Skmacy } else { 1134178784Skmacy ne = -1; 1135178784Skmacy } 1136178784Skmacy } else 1137178784Skmacy ne = ib_poll_cq(cb->cq, 1, &wc); 1138178784Skmacy if (cb->state == ERROR) { 1139256829Snp PRINTF(cb, 1140256829Snp "state == ERROR...bailing scnt %d\n", 1141256829Snp scnt); 1142178784Skmacy return; 1143178784Skmacy } 1144178784Skmacy } while (ne == 0); 1145178784Skmacy 1146178784Skmacy if (ne < 0) { 1147256829Snp PRINTF(cb, "poll CQ failed %d\n", ne); 1148178784Skmacy return; 1149178784Skmacy } 1150256829Snp if (cb->poll && wc.status != IB_WC_SUCCESS) { 1151256829Snp PRINTF(cb, "Completion wth error at %s:\n", 1152178784Skmacy cb->server ? "server" : "client"); 1153256829Snp PRINTF(cb, "Failed status %d: wr_id %d\n", 1154178784Skmacy wc.status, (int) wc.wr_id); 1155178784Skmacy return; 1156178784Skmacy } 1157178784Skmacy ++scnt; 1158178784Skmacy } 1159178784Skmacy microtime(&stop_tv); 1160178784Skmacy 1161178784Skmacy if (stop_tv.tv_usec < start_tv.tv_usec) { 1162178784Skmacy stop_tv.tv_usec += 1000000; 1163178784Skmacy stop_tv.tv_sec -= 1; 1164178784Skmacy } 1165178784Skmacy 1166256829Snp PRINTF(cb, "delta sec %lu delta usec %lu iter %d size %d\n", 1167178784Skmacy stop_tv.tv_sec - start_tv.tv_sec, 1168178784Skmacy stop_tv.tv_usec - start_tv.tv_usec, 1169178784Skmacy scnt, cb->size); 1170178784Skmacy} 1171178784Skmacy 1172178784Skmacystatic void wlat_test(struct krping_cb *cb) 1173178784Skmacy{ 1174178784Skmacy int ccnt, scnt, rcnt; 1175178784Skmacy int iters=cb->count; 1176178784Skmacy volatile char *poll_buf = (char *) cb->start_buf; 1177178784Skmacy char *buf = (char *)cb->rdma_buf; 1178178784Skmacy struct timeval start_tv, stop_tv; 1179178784Skmacy cycles_t *post_cycles_start, *post_cycles_stop; 1180178784Skmacy cycles_t *poll_cycles_start, *poll_cycles_stop; 1181178784Skmacy cycles_t *last_poll_cycles_start; 1182178784Skmacy cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; 1183178784Skmacy int i; 1184178784Skmacy int cycle_iters = 1000; 1185178784Skmacy 1186256829Snp ccnt = 0; 1187256829Snp scnt = 0; 1188256829Snp rcnt = 0; 1189256829Snp 1190256829Snp post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1191256829Snp if (!post_cycles_start) { 1192256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1193178784Skmacy return; 1194178784Skmacy } 1195256829Snp post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1196256829Snp if (!post_cycles_stop) { 1197256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1198256829Snp return; 1199256829Snp } 1200256829Snp poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1201256829Snp if (!poll_cycles_start) { 1202256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1203256829Snp return; 1204256829Snp } 1205256829Snp poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1206256829Snp if (!poll_cycles_stop) { 1207256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1208256829Snp return; 1209256829Snp } 1210256829Snp last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), 1211256829Snp GFP_KERNEL); 1212256829Snp if (!last_poll_cycles_start) { 1213256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1214256829Snp return; 1215256829Snp } 1216178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; 1217178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1218178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1219178784Skmacy cb->rdma_sq_wr.sg_list->length = cb->size; 1220178784Skmacy 1221178784Skmacy if (cycle_iters > iters) 1222178784Skmacy cycle_iters = iters; 1223178784Skmacy microtime(&start_tv); 1224178784Skmacy while (scnt < iters || ccnt < iters || rcnt < iters) { 1225178784Skmacy 1226178784Skmacy /* Wait till buffer changes. */ 1227178784Skmacy if (rcnt < iters && !(scnt < 1 && !cb->server)) { 1228178784Skmacy ++rcnt; 1229178784Skmacy while (*poll_buf != (char)rcnt) { 1230178784Skmacy if (cb->state == ERROR) { 1231256829Snp PRINTF(cb, 1232256829Snp "state = ERROR, bailing\n"); 1233178784Skmacy return; 1234178784Skmacy } 1235178784Skmacy } 1236178784Skmacy } 1237178784Skmacy 1238178784Skmacy if (scnt < iters) { 1239178784Skmacy struct ib_send_wr *bad_wr; 1240178784Skmacy 1241178784Skmacy *buf = (char)scnt+1; 1242178784Skmacy if (scnt < cycle_iters) 1243178784Skmacy post_cycles_start[scnt] = get_cycles(); 1244178784Skmacy if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { 1245256829Snp PRINTF(cb, 1246256829Snp "Couldn't post send: scnt=%d\n", 1247178784Skmacy scnt); 1248178784Skmacy return; 1249178784Skmacy } 1250178784Skmacy if (scnt < cycle_iters) 1251178784Skmacy post_cycles_stop[scnt] = get_cycles(); 1252178784Skmacy scnt++; 1253178784Skmacy } 1254178784Skmacy 1255178784Skmacy if (ccnt < iters) { 1256178784Skmacy struct ib_wc wc; 1257178784Skmacy int ne; 1258178784Skmacy 1259178784Skmacy if (ccnt < cycle_iters) 1260178784Skmacy poll_cycles_start[ccnt] = get_cycles(); 1261178784Skmacy do { 1262178784Skmacy if (ccnt < cycle_iters) 1263256829Snp last_poll_cycles_start[ccnt] = 1264256829Snp get_cycles(); 1265178784Skmacy ne = ib_poll_cq(cb->cq, 1, &wc); 1266178784Skmacy } while (ne == 0); 1267178784Skmacy if (ccnt < cycle_iters) 1268178784Skmacy poll_cycles_stop[ccnt] = get_cycles(); 1269178784Skmacy ++ccnt; 1270178784Skmacy 1271178784Skmacy if (ne < 0) { 1272256829Snp PRINTF(cb, "poll CQ failed %d\n", ne); 1273178784Skmacy return; 1274178784Skmacy } 1275178784Skmacy if (wc.status != IB_WC_SUCCESS) { 1276256829Snp PRINTF(cb, 1277256829Snp "Completion wth error at %s:\n", 1278178784Skmacy cb->server ? "server" : "client"); 1279256829Snp PRINTF(cb, 1280256829Snp "Failed status %d: wr_id %d\n", 1281178784Skmacy wc.status, (int) wc.wr_id); 1282256829Snp PRINTF(cb, 1283256829Snp "scnt=%d, rcnt=%d, ccnt=%d\n", 1284178784Skmacy scnt, rcnt, ccnt); 1285178784Skmacy return; 1286178784Skmacy } 1287178784Skmacy } 1288178784Skmacy } 1289178784Skmacy microtime(&stop_tv); 1290178784Skmacy 1291178784Skmacy if (stop_tv.tv_usec < start_tv.tv_usec) { 1292178784Skmacy stop_tv.tv_usec += 1000000; 1293178784Skmacy stop_tv.tv_sec -= 1; 1294178784Skmacy } 1295178784Skmacy 1296178784Skmacy for (i=0; i < cycle_iters; i++) { 1297178784Skmacy sum_post += post_cycles_stop[i] - post_cycles_start[i]; 1298178784Skmacy sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; 1299256829Snp sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; 1300178784Skmacy } 1301256829Snp PRINTF(cb, 1302256829Snp "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" 1303256829Snp " sum_post %llu sum_poll %llu sum_last_poll %llu\n", 1304178784Skmacy stop_tv.tv_sec - start_tv.tv_sec, 1305178784Skmacy stop_tv.tv_usec - start_tv.tv_usec, 1306178784Skmacy scnt, cb->size, cycle_iters, 1307178784Skmacy (unsigned long long)sum_post, (unsigned long long)sum_poll, 1308178784Skmacy (unsigned long long)sum_last_poll); 1309256829Snp kfree(post_cycles_start); 1310256829Snp kfree(post_cycles_stop); 1311256829Snp kfree(poll_cycles_start); 1312256829Snp kfree(poll_cycles_stop); 1313256829Snp kfree(last_poll_cycles_start); 1314178784Skmacy} 1315178784Skmacy 1316178784Skmacystatic void bw_test(struct krping_cb *cb) 1317178784Skmacy{ 1318178784Skmacy int ccnt, scnt, rcnt; 1319178784Skmacy int iters=cb->count; 1320178784Skmacy struct timeval start_tv, stop_tv; 1321178784Skmacy cycles_t *post_cycles_start, *post_cycles_stop; 1322178784Skmacy cycles_t *poll_cycles_start, *poll_cycles_stop; 1323178784Skmacy cycles_t *last_poll_cycles_start; 1324178784Skmacy cycles_t sum_poll = 0, sum_post = 0, sum_last_poll = 0; 1325178784Skmacy int i; 1326178784Skmacy int cycle_iters = 1000; 1327178784Skmacy 1328256829Snp ccnt = 0; 1329256829Snp scnt = 0; 1330256829Snp rcnt = 0; 1331256829Snp 1332256829Snp post_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1333256829Snp if (!post_cycles_start) { 1334256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1335178784Skmacy return; 1336178784Skmacy } 1337256829Snp post_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1338256829Snp if (!post_cycles_stop) { 1339256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1340256829Snp return; 1341256829Snp } 1342256829Snp poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1343256829Snp if (!poll_cycles_start) { 1344256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1345256829Snp return; 1346256829Snp } 1347256829Snp poll_cycles_stop = kmalloc(cycle_iters * sizeof(cycles_t), GFP_KERNEL); 1348256829Snp if (!poll_cycles_stop) { 1349256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1350256829Snp return; 1351256829Snp } 1352256829Snp last_poll_cycles_start = kmalloc(cycle_iters * sizeof(cycles_t), 1353256829Snp GFP_KERNEL); 1354256829Snp if (!last_poll_cycles_start) { 1355256829Snp PRINTF(cb, "%s kmalloc failed\n", __FUNCTION__); 1356256829Snp return; 1357256829Snp } 1358178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; 1359178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1360178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1361178784Skmacy cb->rdma_sq_wr.sg_list->length = cb->size; 1362178784Skmacy 1363178784Skmacy if (cycle_iters > iters) 1364178784Skmacy cycle_iters = iters; 1365178784Skmacy microtime(&start_tv); 1366178784Skmacy while (scnt < iters || ccnt < iters) { 1367178784Skmacy 1368178784Skmacy while (scnt < iters && scnt - ccnt < cb->txdepth) { 1369178784Skmacy struct ib_send_wr *bad_wr; 1370178784Skmacy 1371178784Skmacy if (scnt < cycle_iters) 1372178784Skmacy post_cycles_start[scnt] = get_cycles(); 1373178784Skmacy if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { 1374256829Snp PRINTF(cb, 1375256829Snp "Couldn't post send: scnt=%d\n", 1376178784Skmacy scnt); 1377178784Skmacy return; 1378178784Skmacy } 1379178784Skmacy if (scnt < cycle_iters) 1380178784Skmacy post_cycles_stop[scnt] = get_cycles(); 1381178784Skmacy ++scnt; 1382178784Skmacy } 1383178784Skmacy 1384178784Skmacy if (ccnt < iters) { 1385178784Skmacy int ne; 1386178784Skmacy struct ib_wc wc; 1387178784Skmacy 1388178784Skmacy if (ccnt < cycle_iters) 1389178784Skmacy poll_cycles_start[ccnt] = get_cycles(); 1390178784Skmacy do { 1391178784Skmacy if (ccnt < cycle_iters) 1392256829Snp last_poll_cycles_start[ccnt] = 1393256829Snp get_cycles(); 1394178784Skmacy ne = ib_poll_cq(cb->cq, 1, &wc); 1395178784Skmacy } while (ne == 0); 1396178784Skmacy if (ccnt < cycle_iters) 1397178784Skmacy poll_cycles_stop[ccnt] = get_cycles(); 1398178784Skmacy ccnt += 1; 1399178784Skmacy 1400178784Skmacy if (ne < 0) { 1401256829Snp PRINTF(cb, "poll CQ failed %d\n", ne); 1402178784Skmacy return; 1403178784Skmacy } 1404178784Skmacy if (wc.status != IB_WC_SUCCESS) { 1405256829Snp PRINTF(cb, 1406256829Snp "Completion wth error at %s:\n", 1407178784Skmacy cb->server ? "server" : "client"); 1408256829Snp PRINTF(cb, 1409256829Snp "Failed status %d: wr_id %d\n", 1410178784Skmacy wc.status, (int) wc.wr_id); 1411178784Skmacy return; 1412178784Skmacy } 1413178784Skmacy } 1414178784Skmacy } 1415178784Skmacy microtime(&stop_tv); 1416178784Skmacy 1417178784Skmacy if (stop_tv.tv_usec < start_tv.tv_usec) { 1418178784Skmacy stop_tv.tv_usec += 1000000; 1419178784Skmacy stop_tv.tv_sec -= 1; 1420178784Skmacy } 1421178784Skmacy 1422178784Skmacy for (i=0; i < cycle_iters; i++) { 1423178784Skmacy sum_post += post_cycles_stop[i] - post_cycles_start[i]; 1424178784Skmacy sum_poll += poll_cycles_stop[i] - poll_cycles_start[i]; 1425256829Snp sum_last_poll += poll_cycles_stop[i]-last_poll_cycles_start[i]; 1426178784Skmacy } 1427256829Snp PRINTF(cb, 1428256829Snp "delta sec %lu delta usec %lu iter %d size %d cycle_iters %d" 1429256829Snp " sum_post %llu sum_poll %llu sum_last_poll %llu\n", 1430178784Skmacy stop_tv.tv_sec - start_tv.tv_sec, 1431178784Skmacy stop_tv.tv_usec - start_tv.tv_usec, 1432178784Skmacy scnt, cb->size, cycle_iters, 1433178784Skmacy (unsigned long long)sum_post, (unsigned long long)sum_poll, 1434178784Skmacy (unsigned long long)sum_last_poll); 1435256829Snp kfree(post_cycles_start); 1436256829Snp kfree(post_cycles_stop); 1437256829Snp kfree(poll_cycles_start); 1438256829Snp kfree(poll_cycles_stop); 1439256829Snp kfree(last_poll_cycles_start); 1440178784Skmacy} 1441178784Skmacy 1442178784Skmacystatic void krping_rlat_test_server(struct krping_cb *cb) 1443178784Skmacy{ 1444178784Skmacy struct ib_send_wr *bad_wr; 1445178784Skmacy struct ib_wc wc; 1446178784Skmacy int ret; 1447178784Skmacy 1448178784Skmacy /* Spin waiting for client's Start STAG/TO/Len */ 1449178784Skmacy while (cb->state < RDMA_READ_ADV) { 1450178784Skmacy krping_cq_event_handler(cb->cq, cb); 1451178784Skmacy } 1452178784Skmacy 1453178784Skmacy /* Send STAG/TO/Len to client */ 1454256829Snp krping_format_send(cb, cb->start_dma_addr); 1455178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1456178784Skmacy if (ret) { 1457256829Snp PRINTF(cb, "post send error %d\n", ret); 1458178784Skmacy return; 1459178784Skmacy } 1460178784Skmacy 1461178784Skmacy /* Spin waiting for send completion */ 1462178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1463178784Skmacy if (ret < 0) { 1464256829Snp PRINTF(cb, "poll error %d\n", ret); 1465178784Skmacy return; 1466178784Skmacy } 1467178784Skmacy if (wc.status) { 1468256829Snp PRINTF(cb, "send completiong error %d\n", wc.status); 1469178784Skmacy return; 1470178784Skmacy } 1471178784Skmacy 1472256829Snp wait_event_interruptible(cb->sem, cb->state == ERROR); 1473178784Skmacy} 1474178784Skmacy 1475178784Skmacystatic void krping_wlat_test_server(struct krping_cb *cb) 1476178784Skmacy{ 1477178784Skmacy struct ib_send_wr *bad_wr; 1478178784Skmacy struct ib_wc wc; 1479178784Skmacy int ret; 1480178784Skmacy 1481178784Skmacy /* Spin waiting for client's Start STAG/TO/Len */ 1482178784Skmacy while (cb->state < RDMA_READ_ADV) { 1483178784Skmacy krping_cq_event_handler(cb->cq, cb); 1484178784Skmacy } 1485178784Skmacy 1486178784Skmacy /* Send STAG/TO/Len to client */ 1487256829Snp krping_format_send(cb, cb->start_dma_addr); 1488178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1489178784Skmacy if (ret) { 1490256829Snp PRINTF(cb, "post send error %d\n", ret); 1491178784Skmacy return; 1492178784Skmacy } 1493178784Skmacy 1494178784Skmacy /* Spin waiting for send completion */ 1495178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1496178784Skmacy if (ret < 0) { 1497256829Snp PRINTF(cb, "poll error %d\n", ret); 1498178784Skmacy return; 1499178784Skmacy } 1500178784Skmacy if (wc.status) { 1501256829Snp PRINTF(cb, "send completiong error %d\n", wc.status); 1502178784Skmacy return; 1503178784Skmacy } 1504178784Skmacy 1505178784Skmacy wlat_test(cb); 1506256829Snp wait_event_interruptible(cb->sem, cb->state == ERROR); 1507178784Skmacy} 1508178784Skmacy 1509178784Skmacystatic void krping_bw_test_server(struct krping_cb *cb) 1510178784Skmacy{ 1511178784Skmacy struct ib_send_wr *bad_wr; 1512178784Skmacy struct ib_wc wc; 1513178784Skmacy int ret; 1514178784Skmacy 1515178784Skmacy /* Spin waiting for client's Start STAG/TO/Len */ 1516178784Skmacy while (cb->state < RDMA_READ_ADV) { 1517178784Skmacy krping_cq_event_handler(cb->cq, cb); 1518178784Skmacy } 1519178784Skmacy 1520178784Skmacy /* Send STAG/TO/Len to client */ 1521256829Snp krping_format_send(cb, cb->start_dma_addr); 1522178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1523178784Skmacy if (ret) { 1524256829Snp PRINTF(cb, "post send error %d\n", ret); 1525178784Skmacy return; 1526178784Skmacy } 1527178784Skmacy 1528178784Skmacy /* Spin waiting for send completion */ 1529178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1530178784Skmacy if (ret < 0) { 1531256829Snp PRINTF(cb, "poll error %d\n", ret); 1532178784Skmacy return; 1533178784Skmacy } 1534178784Skmacy if (wc.status) { 1535256829Snp PRINTF(cb, "send completiong error %d\n", wc.status); 1536178784Skmacy return; 1537178784Skmacy } 1538178784Skmacy 1539178784Skmacy if (cb->duplex) 1540178784Skmacy bw_test(cb); 1541256829Snp wait_event_interruptible(cb->sem, cb->state == ERROR); 1542178784Skmacy} 1543178784Skmacy 1544256829Snpstatic int fastreg_supported(struct krping_cb *cb) 1545256829Snp{ 1546256829Snp struct ib_device *dev = cb->child_cm_id->device; 1547256829Snp struct ib_device_attr attr; 1548256829Snp int ret; 1549256829Snp 1550256829Snp ret = ib_query_device(dev, &attr); 1551256829Snp if (ret) { 1552256829Snp PRINTF(cb, "ib_query_device failed ret %d\n", ret); 1553256829Snp return 0; 1554256829Snp } 1555256829Snp if (!(attr.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) { 1556256829Snp PRINTF(cb, "Fastreg not supported - device_cap_flags 0x%x\n", 1557256829Snp attr.device_cap_flags); 1558256829Snp return 0; 1559256829Snp } 1560256829Snp DEBUG_LOG(cb, "Fastreg supported - device_cap_flags 0x%x\n", 1561256829Snp attr.device_cap_flags); 1562256829Snp return 1; 1563256829Snp} 1564256829Snp 1565178784Skmacystatic int krping_bind_server(struct krping_cb *cb) 1566178784Skmacy{ 1567178784Skmacy struct sockaddr_in sin; 1568178784Skmacy int ret; 1569178784Skmacy 1570178784Skmacy memset(&sin, 0, sizeof(sin)); 1571178784Skmacy sin.sin_len = sizeof sin; 1572178784Skmacy sin.sin_family = AF_INET; 1573178784Skmacy sin.sin_addr.s_addr = cb->addr.s_addr; 1574178784Skmacy sin.sin_port = cb->port; 1575178784Skmacy 1576178784Skmacy ret = rdma_bind_addr(cb->cm_id, (struct sockaddr *) &sin); 1577178784Skmacy if (ret) { 1578256829Snp PRINTF(cb, "rdma_bind_addr error %d\n", ret); 1579178784Skmacy return ret; 1580178784Skmacy } 1581256829Snp DEBUG_LOG(cb, "rdma_bind_addr successful\n"); 1582178784Skmacy 1583256829Snp DEBUG_LOG(cb, "rdma_listen\n"); 1584178784Skmacy ret = rdma_listen(cb->cm_id, 3); 1585178784Skmacy if (ret) { 1586256829Snp PRINTF(cb, "rdma_listen failed: %d\n", ret); 1587178784Skmacy return ret; 1588178784Skmacy } 1589178784Skmacy 1590256829Snp wait_event_interruptible(cb->sem, cb->state >= CONNECT_REQUEST); 1591178784Skmacy if (cb->state != CONNECT_REQUEST) { 1592256829Snp PRINTF(cb, "wait for CONNECT_REQUEST state %d\n", 1593178784Skmacy cb->state); 1594178784Skmacy return -1; 1595178784Skmacy } 1596178784Skmacy 1597256829Snp if (cb->mem == FASTREG && !fastreg_supported(cb)) 1598256829Snp return -EINVAL; 1599256829Snp 1600178784Skmacy return 0; 1601178784Skmacy} 1602178784Skmacy 1603178784Skmacystatic void krping_run_server(struct krping_cb *cb) 1604178784Skmacy{ 1605178784Skmacy struct ib_recv_wr *bad_wr; 1606178784Skmacy int ret; 1607178784Skmacy 1608178784Skmacy ret = krping_bind_server(cb); 1609178784Skmacy if (ret) 1610178784Skmacy return; 1611178784Skmacy 1612178784Skmacy ret = krping_setup_qp(cb, cb->child_cm_id); 1613178784Skmacy if (ret) { 1614256829Snp PRINTF(cb, "setup_qp failed: %d\n", ret); 1615256829Snp goto err0; 1616178784Skmacy } 1617178784Skmacy 1618178784Skmacy ret = krping_setup_buffers(cb); 1619178784Skmacy if (ret) { 1620256829Snp PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); 1621178784Skmacy goto err1; 1622178784Skmacy } 1623178784Skmacy 1624178784Skmacy ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); 1625178784Skmacy if (ret) { 1626256829Snp PRINTF(cb, "ib_post_recv failed: %d\n", ret); 1627178784Skmacy goto err2; 1628178784Skmacy } 1629178784Skmacy 1630178784Skmacy ret = krping_accept(cb); 1631178784Skmacy if (ret) { 1632256829Snp PRINTF(cb, "connect error %d\n", ret); 1633178784Skmacy goto err2; 1634178784Skmacy } 1635178784Skmacy 1636178784Skmacy if (cb->wlat) 1637178784Skmacy krping_wlat_test_server(cb); 1638178784Skmacy else if (cb->rlat) 1639178784Skmacy krping_rlat_test_server(cb); 1640178784Skmacy else if (cb->bw) 1641178784Skmacy krping_bw_test_server(cb); 1642178784Skmacy else 1643178784Skmacy krping_test_server(cb); 1644178784Skmacy rdma_disconnect(cb->child_cm_id); 1645178784Skmacyerr2: 1646178784Skmacy krping_free_buffers(cb); 1647178784Skmacyerr1: 1648178784Skmacy krping_free_qp(cb); 1649256829Snperr0: 1650256829Snp rdma_destroy_id(cb->child_cm_id); 1651178784Skmacy} 1652178784Skmacy 1653178784Skmacystatic void krping_test_client(struct krping_cb *cb) 1654178784Skmacy{ 1655178784Skmacy int ping, start, cc, i, ret; 1656178784Skmacy struct ib_send_wr *bad_wr; 1657178784Skmacy unsigned char c; 1658178784Skmacy 1659178784Skmacy start = 65; 1660178784Skmacy for (ping = 0; !cb->count || ping < cb->count; ping++) { 1661178784Skmacy cb->state = RDMA_READ_ADV; 1662178784Skmacy 1663178784Skmacy /* Put some ascii text in the buffer. */ 1664178784Skmacy cc = sprintf(cb->start_buf, "rdma-ping-%d: ", ping); 1665178784Skmacy for (i = cc, c = start; i < cb->size; i++) { 1666178784Skmacy cb->start_buf[i] = c; 1667178784Skmacy c++; 1668178784Skmacy if (c > 122) 1669178784Skmacy c = 65; 1670178784Skmacy } 1671178784Skmacy start++; 1672178784Skmacy if (start > 122) 1673178784Skmacy start = 65; 1674178784Skmacy cb->start_buf[cb->size - 1] = 0; 1675178784Skmacy 1676256829Snp krping_format_send(cb, cb->start_dma_addr); 1677256829Snp if (cb->state == ERROR) { 1678256829Snp PRINTF(cb, "krping_format_send failed\n"); 1679256829Snp break; 1680256829Snp } 1681178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1682178784Skmacy if (ret) { 1683256829Snp PRINTF(cb, "post send error %d\n", ret); 1684178784Skmacy break; 1685178784Skmacy } 1686178784Skmacy 1687178784Skmacy /* Wait for server to ACK */ 1688256829Snp wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); 1689178784Skmacy if (cb->state != RDMA_WRITE_ADV) { 1690256829Snp PRINTF(cb, 1691178784Skmacy "wait for RDMA_WRITE_ADV state %d\n", 1692178784Skmacy cb->state); 1693178784Skmacy break; 1694178784Skmacy } 1695178784Skmacy 1696256829Snp krping_format_send(cb, cb->rdma_dma_addr); 1697178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1698178784Skmacy if (ret) { 1699256829Snp PRINTF(cb, "post send error %d\n", ret); 1700178784Skmacy break; 1701178784Skmacy } 1702178784Skmacy 1703178784Skmacy /* Wait for the server to say the RDMA Write is complete. */ 1704256829Snp wait_event_interruptible(cb->sem, 1705256829Snp cb->state >= RDMA_WRITE_COMPLETE); 1706178784Skmacy if (cb->state != RDMA_WRITE_COMPLETE) { 1707256829Snp PRINTF(cb, 1708178784Skmacy "wait for RDMA_WRITE_COMPLETE state %d\n", 1709178784Skmacy cb->state); 1710178784Skmacy break; 1711178784Skmacy } 1712178784Skmacy 1713178784Skmacy if (cb->validate) 1714178784Skmacy if (memcmp(cb->start_buf, cb->rdma_buf, cb->size)) { 1715256829Snp PRINTF(cb, "data mismatch!\n"); 1716178784Skmacy break; 1717178784Skmacy } 1718178784Skmacy 1719178784Skmacy if (cb->verbose) 1720256829Snp PRINTF(cb, "ping data: %s\n", cb->rdma_buf); 1721256829Snp#ifdef SLOW_KRPING 1722256829Snp wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); 1723256829Snp#endif 1724178784Skmacy } 1725178784Skmacy} 1726178784Skmacy 1727178784Skmacystatic void krping_rlat_test_client(struct krping_cb *cb) 1728178784Skmacy{ 1729178784Skmacy struct ib_send_wr *bad_wr; 1730178784Skmacy struct ib_wc wc; 1731178784Skmacy int ret; 1732178784Skmacy 1733178784Skmacy cb->state = RDMA_READ_ADV; 1734178784Skmacy 1735178784Skmacy /* Send STAG/TO/Len to client */ 1736256829Snp krping_format_send(cb, cb->start_dma_addr); 1737256829Snp if (cb->state == ERROR) { 1738256829Snp PRINTF(cb, "krping_format_send failed\n"); 1739256829Snp return; 1740256829Snp } 1741178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1742178784Skmacy if (ret) { 1743256829Snp PRINTF(cb, "post send error %d\n", ret); 1744178784Skmacy return; 1745178784Skmacy } 1746178784Skmacy 1747178784Skmacy /* Spin waiting for send completion */ 1748178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1749178784Skmacy if (ret < 0) { 1750256829Snp PRINTF(cb, "poll error %d\n", ret); 1751178784Skmacy return; 1752178784Skmacy } 1753178784Skmacy if (wc.status) { 1754256829Snp PRINTF(cb, "send completion error %d\n", wc.status); 1755178784Skmacy return; 1756178784Skmacy } 1757178784Skmacy 1758178784Skmacy /* Spin waiting for server's Start STAG/TO/Len */ 1759178784Skmacy while (cb->state < RDMA_WRITE_ADV) { 1760178784Skmacy krping_cq_event_handler(cb->cq, cb); 1761178784Skmacy } 1762178784Skmacy 1763178784Skmacy#if 0 1764178784Skmacy{ 1765178784Skmacy int i; 1766178784Skmacy struct timeval start, stop; 1767178784Skmacy time_t sec; 1768178784Skmacy suseconds_t usec; 1769178784Skmacy unsigned long long elapsed; 1770178784Skmacy struct ib_wc wc; 1771178784Skmacy struct ib_send_wr *bad_wr; 1772178784Skmacy int ne; 1773178784Skmacy 1774178784Skmacy cb->rdma_sq_wr.opcode = IB_WR_RDMA_WRITE; 1775178784Skmacy cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; 1776178784Skmacy cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; 1777178784Skmacy cb->rdma_sq_wr.sg_list->length = 0; 1778178784Skmacy cb->rdma_sq_wr.num_sge = 0; 1779178784Skmacy 1780178784Skmacy microtime(&start); 1781178784Skmacy for (i=0; i < 100000; i++) { 1782178784Skmacy if (ib_post_send(cb->qp, &cb->rdma_sq_wr, &bad_wr)) { 1783256829Snp PRINTF(cb, "Couldn't post send\n"); 1784178784Skmacy return; 1785178784Skmacy } 1786178784Skmacy do { 1787178784Skmacy ne = ib_poll_cq(cb->cq, 1, &wc); 1788178784Skmacy } while (ne == 0); 1789178784Skmacy if (ne < 0) { 1790256829Snp PRINTF(cb, "poll CQ failed %d\n", ne); 1791178784Skmacy return; 1792178784Skmacy } 1793178784Skmacy if (wc.status != IB_WC_SUCCESS) { 1794256829Snp PRINTF(cb, "Completion wth error at %s:\n", 1795178784Skmacy cb->server ? "server" : "client"); 1796256829Snp PRINTF(cb, "Failed status %d: wr_id %d\n", 1797178784Skmacy wc.status, (int) wc.wr_id); 1798178784Skmacy return; 1799178784Skmacy } 1800178784Skmacy } 1801178784Skmacy microtime(&stop); 1802178784Skmacy 1803178784Skmacy if (stop.tv_usec < start.tv_usec) { 1804178784Skmacy stop.tv_usec += 1000000; 1805178784Skmacy stop.tv_sec -= 1; 1806178784Skmacy } 1807178784Skmacy sec = stop.tv_sec - start.tv_sec; 1808178784Skmacy usec = stop.tv_usec - start.tv_usec; 1809178784Skmacy elapsed = sec * 1000000 + usec; 1810256829Snp PRINTF(cb, "0B-write-lat iters 100000 usec %llu\n", elapsed); 1811178784Skmacy} 1812178784Skmacy#endif 1813178784Skmacy 1814178784Skmacy rlat_test(cb); 1815178784Skmacy} 1816178784Skmacy 1817178784Skmacystatic void krping_wlat_test_client(struct krping_cb *cb) 1818178784Skmacy{ 1819178784Skmacy struct ib_send_wr *bad_wr; 1820178784Skmacy struct ib_wc wc; 1821178784Skmacy int ret; 1822178784Skmacy 1823178784Skmacy cb->state = RDMA_READ_ADV; 1824178784Skmacy 1825178784Skmacy /* Send STAG/TO/Len to client */ 1826256829Snp krping_format_send(cb, cb->start_dma_addr); 1827256829Snp if (cb->state == ERROR) { 1828256829Snp PRINTF(cb, "krping_format_send failed\n"); 1829256829Snp return; 1830256829Snp } 1831178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1832178784Skmacy if (ret) { 1833256829Snp PRINTF(cb, "post send error %d\n", ret); 1834178784Skmacy return; 1835178784Skmacy } 1836178784Skmacy 1837178784Skmacy /* Spin waiting for send completion */ 1838178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1839178784Skmacy if (ret < 0) { 1840256829Snp PRINTF(cb, "poll error %d\n", ret); 1841178784Skmacy return; 1842178784Skmacy } 1843178784Skmacy if (wc.status) { 1844256829Snp PRINTF(cb, "send completion error %d\n", wc.status); 1845178784Skmacy return; 1846178784Skmacy } 1847178784Skmacy 1848178784Skmacy /* Spin waiting for server's Start STAG/TO/Len */ 1849178784Skmacy while (cb->state < RDMA_WRITE_ADV) { 1850178784Skmacy krping_cq_event_handler(cb->cq, cb); 1851178784Skmacy } 1852178784Skmacy 1853178784Skmacy wlat_test(cb); 1854178784Skmacy} 1855178784Skmacy 1856178784Skmacystatic void krping_bw_test_client(struct krping_cb *cb) 1857178784Skmacy{ 1858178784Skmacy struct ib_send_wr *bad_wr; 1859178784Skmacy struct ib_wc wc; 1860178784Skmacy int ret; 1861178784Skmacy 1862178784Skmacy cb->state = RDMA_READ_ADV; 1863178784Skmacy 1864178784Skmacy /* Send STAG/TO/Len to client */ 1865256829Snp krping_format_send(cb, cb->start_dma_addr); 1866256829Snp if (cb->state == ERROR) { 1867256829Snp PRINTF(cb, "krping_format_send failed\n"); 1868256829Snp return; 1869256829Snp } 1870178784Skmacy ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); 1871178784Skmacy if (ret) { 1872256829Snp PRINTF(cb, "post send error %d\n", ret); 1873178784Skmacy return; 1874178784Skmacy } 1875178784Skmacy 1876178784Skmacy /* Spin waiting for send completion */ 1877178784Skmacy while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); 1878178784Skmacy if (ret < 0) { 1879256829Snp PRINTF(cb, "poll error %d\n", ret); 1880178784Skmacy return; 1881178784Skmacy } 1882178784Skmacy if (wc.status) { 1883256829Snp PRINTF(cb, "send completion error %d\n", wc.status); 1884178784Skmacy return; 1885178784Skmacy } 1886178784Skmacy 1887178784Skmacy /* Spin waiting for server's Start STAG/TO/Len */ 1888178784Skmacy while (cb->state < RDMA_WRITE_ADV) { 1889178784Skmacy krping_cq_event_handler(cb->cq, cb); 1890178784Skmacy } 1891178784Skmacy 1892178784Skmacy bw_test(cb); 1893178784Skmacy} 1894178784Skmacy 1895256829Snpstatic void krping_fr_test(struct krping_cb *cb) 1896256829Snp{ 1897256829Snp struct ib_fast_reg_page_list *pl; 1898256829Snp struct ib_send_wr fr, inv, *bad; 1899256829Snp struct ib_wc wc; 1900256829Snp u8 key = 0; 1901256829Snp struct ib_mr *mr; 1902256829Snp int i; 1903256829Snp int ret; 1904256829Snp int size = cb->size; 1905256829Snp int plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; 1906256829Snp time_t start; 1907256829Snp int count = 0; 1908256829Snp int scnt = 0; 1909256829Snp 1910256829Snp pl = ib_alloc_fast_reg_page_list(cb->qp->device, plen); 1911256829Snp if (IS_ERR(pl)) { 1912256829Snp PRINTF(cb, "ib_alloc_fast_reg_page_list failed %ld\n", PTR_ERR(pl)); 1913256829Snp return; 1914256829Snp } 1915256829Snp 1916256829Snp mr = ib_alloc_fast_reg_mr(cb->pd, plen); 1917256829Snp if (IS_ERR(mr)) { 1918256829Snp PRINTF(cb, "ib_alloc_fast_reg_mr failed %ld\n", PTR_ERR(pl)); 1919256829Snp goto err1; 1920256829Snp } 1921256829Snp 1922256829Snp for (i=0; i<plen; i++) 1923256829Snp pl->page_list[i] = 0xcafebabe | i; 1924256829Snp 1925256829Snp memset(&fr, 0, sizeof fr); 1926256829Snp fr.opcode = IB_WR_FAST_REG_MR; 1927256829Snp fr.wr.fast_reg.page_shift = PAGE_SHIFT; 1928256829Snp fr.wr.fast_reg.length = size; 1929256829Snp fr.wr.fast_reg.page_list = pl; 1930256829Snp fr.wr.fast_reg.page_list_len = plen; 1931256829Snp fr.wr.fast_reg.iova_start = 0; 1932256829Snp fr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; 1933256829Snp fr.next = &inv; 1934256829Snp memset(&inv, 0, sizeof inv); 1935256829Snp inv.opcode = IB_WR_LOCAL_INV; 1936256829Snp inv.send_flags = IB_SEND_SIGNALED; 1937256829Snp 1938256829Snp DEBUG_LOG(cb, "fr_test: stag index 0x%x plen %u size %u depth %u\n", mr->rkey >> 8, plen, cb->size, cb->txdepth); 1939256829Snp start = time_uptime; 1940256829Snp while (1) { 1941256829Snp if ((time_uptime - start) >= 9) { 1942256829Snp DEBUG_LOG(cb, "fr_test: pausing 1 second! count %u latest size %u plen %u\n", count, size, plen); 1943256829Snp wait_event_interruptible(cb->sem, cb->state == ERROR); 1944256829Snp if (cb->state == ERROR) 1945256829Snp break; 1946256829Snp start = time_uptime; 1947256829Snp } 1948256829Snp while (scnt < (cb->txdepth>>1)) { 1949256829Snp ib_update_fast_reg_key(mr, ++key); 1950256829Snp fr.wr.fast_reg.rkey = mr->rkey; 1951256829Snp inv.ex.invalidate_rkey = mr->rkey; 1952256829Snp size = arc4random() % cb->size; 1953256829Snp if (size == 0) 1954256829Snp size = cb->size; 1955256829Snp plen = (((size - 1) & PAGE_MASK) + PAGE_SIZE) >> PAGE_SHIFT; 1956256829Snp fr.wr.fast_reg.length = size; 1957256829Snp fr.wr.fast_reg.page_list_len = plen; 1958256829Snp ret = ib_post_send(cb->qp, &fr, &bad); 1959256829Snp if (ret) { 1960256829Snp PRINTF(cb, "ib_post_send failed %d\n", ret); 1961256829Snp goto err2; 1962256829Snp } 1963256829Snp scnt++; 1964256829Snp } 1965256829Snp 1966256829Snp do { 1967256829Snp ret = ib_poll_cq(cb->cq, 1, &wc); 1968256829Snp if (ret < 0) { 1969256829Snp PRINTF(cb, "ib_poll_cq failed %d\n", ret); 1970256829Snp goto err2; 1971256829Snp } 1972256829Snp if (ret == 1) { 1973256829Snp if (wc.status) { 1974256829Snp PRINTF(cb, "completion error %u\n", wc.status); 1975256829Snp goto err2; 1976256829Snp } 1977256829Snp count++; 1978256829Snp scnt--; 1979256829Snp } 1980256829Snp else if (krping_sigpending()) { 1981256829Snp PRINTF(cb, "signal!\n"); 1982256829Snp goto err2; 1983256829Snp } 1984256829Snp } while (ret == 1); 1985256829Snp } 1986256829Snperr2: 1987256829Snp#if 0 1988256829Snp DEBUG_LOG(cb, "sleeping 1 second\n"); 1989256829Snp wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); 1990256829Snp#endif 1991256829Snp DEBUG_LOG(cb, "draining the cq...\n"); 1992256829Snp do { 1993256829Snp ret = ib_poll_cq(cb->cq, 1, &wc); 1994256829Snp if (ret < 0) { 1995256829Snp PRINTF(cb, "ib_poll_cq failed %d\n", ret); 1996256829Snp break; 1997256829Snp } 1998256829Snp if (ret == 1) { 1999256829Snp if (wc.status) { 2000256829Snp PRINTF(cb, "completion error %u opcode %u\n", wc.status, wc.opcode); 2001256829Snp } 2002256829Snp } 2003256829Snp } while (ret == 1); 2004256829Snp DEBUG_LOG(cb, "fr_test: done!\n"); 2005256829Snp ib_dereg_mr(mr); 2006256829Snperr1: 2007256829Snp ib_free_fast_reg_page_list(pl); 2008256829Snp} 2009256829Snp 2010178784Skmacystatic int krping_connect_client(struct krping_cb *cb) 2011178784Skmacy{ 2012178784Skmacy struct rdma_conn_param conn_param; 2013178784Skmacy int ret; 2014178784Skmacy 2015178784Skmacy memset(&conn_param, 0, sizeof conn_param); 2016178784Skmacy conn_param.responder_resources = 1; 2017178784Skmacy conn_param.initiator_depth = 1; 2018178784Skmacy conn_param.retry_count = 10; 2019178784Skmacy 2020178784Skmacy ret = rdma_connect(cb->cm_id, &conn_param); 2021178784Skmacy if (ret) { 2022256829Snp PRINTF(cb, "rdma_connect error %d\n", ret); 2023178784Skmacy return ret; 2024178784Skmacy } 2025178784Skmacy 2026256829Snp wait_event_interruptible(cb->sem, cb->state >= CONNECTED); 2027178784Skmacy if (cb->state == ERROR) { 2028256829Snp PRINTF(cb, "wait for CONNECTED state %d\n", cb->state); 2029178784Skmacy return -1; 2030178784Skmacy } 2031178784Skmacy 2032256829Snp DEBUG_LOG(cb, "rdma_connect successful\n"); 2033178784Skmacy return 0; 2034178784Skmacy} 2035178784Skmacy 2036178784Skmacystatic int krping_bind_client(struct krping_cb *cb) 2037178784Skmacy{ 2038178784Skmacy struct sockaddr_in sin; 2039178784Skmacy int ret; 2040178784Skmacy 2041178784Skmacy memset(&sin, 0, sizeof(sin)); 2042178784Skmacy sin.sin_len = sizeof sin; 2043178784Skmacy sin.sin_family = AF_INET; 2044178784Skmacy sin.sin_addr.s_addr = cb->addr.s_addr; 2045178784Skmacy sin.sin_port = cb->port; 2046178784Skmacy 2047178784Skmacy ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *) &sin, 2048178784Skmacy 2000); 2049178784Skmacy if (ret) { 2050256829Snp PRINTF(cb, "rdma_resolve_addr error %d\n", ret); 2051178784Skmacy return ret; 2052178784Skmacy } 2053178784Skmacy 2054256829Snp wait_event_interruptible(cb->sem, cb->state >= ROUTE_RESOLVED); 2055178784Skmacy if (cb->state != ROUTE_RESOLVED) { 2056256829Snp PRINTF(cb, 2057178784Skmacy "addr/route resolution did not resolve: state %d\n", 2058178784Skmacy cb->state); 2059256829Snp return -EINTR; 2060178784Skmacy } 2061178784Skmacy 2062256829Snp if (cb->mem == FASTREG && !fastreg_supported(cb)) 2063256829Snp return -EINVAL; 2064256829Snp 2065256829Snp DEBUG_LOG(cb, "rdma_resolve_addr - rdma_resolve_route successful\n"); 2066178784Skmacy return 0; 2067178784Skmacy} 2068178784Skmacy 2069178784Skmacystatic void krping_run_client(struct krping_cb *cb) 2070178784Skmacy{ 2071178784Skmacy struct ib_recv_wr *bad_wr; 2072178784Skmacy int ret; 2073178784Skmacy 2074178784Skmacy ret = krping_bind_client(cb); 2075178784Skmacy if (ret) 2076178784Skmacy return; 2077178784Skmacy 2078178784Skmacy ret = krping_setup_qp(cb, cb->cm_id); 2079178784Skmacy if (ret) { 2080256829Snp PRINTF(cb, "setup_qp failed: %d\n", ret); 2081178784Skmacy return; 2082178784Skmacy } 2083178784Skmacy 2084178784Skmacy ret = krping_setup_buffers(cb); 2085178784Skmacy if (ret) { 2086256829Snp PRINTF(cb, "krping_setup_buffers failed: %d\n", ret); 2087178784Skmacy goto err1; 2088178784Skmacy } 2089178784Skmacy 2090178784Skmacy ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); 2091178784Skmacy if (ret) { 2092256829Snp PRINTF(cb, "ib_post_recv failed: %d\n", ret); 2093178784Skmacy goto err2; 2094178784Skmacy } 2095178784Skmacy 2096178784Skmacy ret = krping_connect_client(cb); 2097178784Skmacy if (ret) { 2098256829Snp PRINTF(cb, "connect error %d\n", ret); 2099178784Skmacy goto err2; 2100178784Skmacy } 2101178784Skmacy 2102178784Skmacy if (cb->wlat) 2103178784Skmacy krping_wlat_test_client(cb); 2104178784Skmacy else if (cb->rlat) 2105178784Skmacy krping_rlat_test_client(cb); 2106178784Skmacy else if (cb->bw) 2107178784Skmacy krping_bw_test_client(cb); 2108256829Snp else if (cb->frtest) 2109256829Snp krping_fr_test(cb); 2110178784Skmacy else 2111178784Skmacy krping_test_client(cb); 2112178784Skmacy rdma_disconnect(cb->cm_id); 2113178784Skmacyerr2: 2114178784Skmacy krping_free_buffers(cb); 2115178784Skmacyerr1: 2116178784Skmacy krping_free_qp(cb); 2117178784Skmacy} 2118178784Skmacy 2119256829Snpint krping_doit(char *cmd, void *cookie) 2120178784Skmacy{ 2121178784Skmacy struct krping_cb *cb; 2122178784Skmacy int op; 2123178784Skmacy int ret = 0; 2124178784Skmacy char *optarg; 2125178784Skmacy unsigned long optint; 2126178784Skmacy 2127256829Snp cb = kzalloc(sizeof(*cb), GFP_KERNEL); 2128178784Skmacy if (!cb) 2129256829Snp return -ENOMEM; 2130178784Skmacy 2131256829Snp mutex_lock(&krping_mutex); 2132256829Snp list_add_tail(&cb->list, &krping_cbs); 2133256829Snp mutex_unlock(&krping_mutex); 2134178784Skmacy 2135256829Snp cb->cookie = cookie; 2136178784Skmacy cb->server = -1; 2137178784Skmacy cb->state = IDLE; 2138178784Skmacy cb->size = 64; 2139178784Skmacy cb->txdepth = RPING_SQ_DEPTH; 2140256829Snp cb->mem = DMA; 2141256829Snp init_waitqueue_head(&cb->sem); 2142178784Skmacy 2143178784Skmacy while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg, 2144178784Skmacy &optint)) != 0) { 2145178784Skmacy switch (op) { 2146178784Skmacy case 'a': 2147178784Skmacy cb->addr_str = optarg; 2148256829Snp DEBUG_LOG(cb, "ipaddr (%s)\n", optarg); 2149178784Skmacy if (!inet_aton(optarg, &cb->addr)) { 2150256829Snp PRINTF(cb, "bad addr string %s\n", 2151256829Snp optarg); 2152178784Skmacy ret = EINVAL; 2153178784Skmacy } 2154178784Skmacy break; 2155178784Skmacy case 'p': 2156178784Skmacy cb->port = htons(optint); 2157256829Snp DEBUG_LOG(cb, "port %d\n", (int)optint); 2158178784Skmacy break; 2159178784Skmacy case 'P': 2160178784Skmacy cb->poll = 1; 2161256829Snp DEBUG_LOG(cb, "server\n"); 2162178784Skmacy break; 2163178784Skmacy case 's': 2164178784Skmacy cb->server = 1; 2165256829Snp DEBUG_LOG(cb, "server\n"); 2166178784Skmacy break; 2167178784Skmacy case 'c': 2168178784Skmacy cb->server = 0; 2169256829Snp DEBUG_LOG(cb, "client\n"); 2170178784Skmacy break; 2171178784Skmacy case 'S': 2172178784Skmacy cb->size = optint; 2173178784Skmacy if ((cb->size < 1) || 2174178784Skmacy (cb->size > RPING_BUFSIZE)) { 2175256829Snp PRINTF(cb, "Invalid size %d " 2176178784Skmacy "(valid range is 1 to %d)\n", 2177178784Skmacy cb->size, RPING_BUFSIZE); 2178178784Skmacy ret = EINVAL; 2179178784Skmacy } else 2180256829Snp DEBUG_LOG(cb, "size %d\n", (int)optint); 2181178784Skmacy break; 2182178784Skmacy case 'C': 2183178784Skmacy cb->count = optint; 2184178784Skmacy if (cb->count < 0) { 2185256829Snp PRINTF(cb, "Invalid count %d\n", 2186178784Skmacy cb->count); 2187178784Skmacy ret = EINVAL; 2188178784Skmacy } else 2189256829Snp DEBUG_LOG(cb, "count %d\n", (int) cb->count); 2190178784Skmacy break; 2191178784Skmacy case 'v': 2192178784Skmacy cb->verbose++; 2193256829Snp DEBUG_LOG(cb, "verbose\n"); 2194178784Skmacy break; 2195178784Skmacy case 'V': 2196178784Skmacy cb->validate++; 2197256829Snp DEBUG_LOG(cb, "validate data\n"); 2198178784Skmacy break; 2199256829Snp case 'l': 2200256829Snp cb->wlat++; 2201256829Snp break; 2202178784Skmacy case 'L': 2203178784Skmacy cb->rlat++; 2204178784Skmacy break; 2205178784Skmacy case 'B': 2206178784Skmacy cb->bw++; 2207178784Skmacy break; 2208256829Snp case 'd': 2209256829Snp cb->duplex++; 2210256829Snp break; 2211256829Snp case 'm': 2212256829Snp if (!strncmp(optarg, "dma", 3)) 2213256829Snp cb->mem = DMA; 2214256829Snp else if (!strncmp(optarg, "fastreg", 7)) 2215256829Snp cb->mem = FASTREG; 2216256829Snp else if (!strncmp(optarg, "mw", 2)) 2217256829Snp cb->mem = MW; 2218256829Snp else if (!strncmp(optarg, "mr", 2)) 2219256829Snp cb->mem = MR; 2220256829Snp else { 2221256829Snp PRINTF(cb, "unknown mem mode %s. " 2222256829Snp "Must be dma, fastreg, mw, or mr\n", 2223256829Snp optarg); 2224256829Snp ret = -EINVAL; 2225256829Snp break; 2226256829Snp } 2227256829Snp break; 2228256829Snp case 'I': 2229256829Snp cb->server_invalidate = 1; 2230256829Snp break; 2231256829Snp case 'T': 2232178784Skmacy cb->txdepth = optint; 2233256829Snp DEBUG_LOG(cb, "txdepth %d\n", (int) cb->txdepth); 2234178784Skmacy break; 2235256829Snp case 'Z': 2236256829Snp cb->local_dma_lkey = 1; 2237256829Snp DEBUG_LOG(cb, "using local dma lkey\n"); 2238178784Skmacy break; 2239256829Snp case 'R': 2240256829Snp cb->read_inv = 1; 2241256829Snp DEBUG_LOG(cb, "using read-with-inv\n"); 2242256829Snp break; 2243256829Snp case 'f': 2244256829Snp cb->frtest = 1; 2245256829Snp DEBUG_LOG(cb, "fast-reg test!\n"); 2246256829Snp break; 2247178784Skmacy default: 2248256829Snp PRINTF(cb, "unknown opt %s\n", optarg); 2249256829Snp ret = -EINVAL; 2250178784Skmacy break; 2251178784Skmacy } 2252178784Skmacy } 2253178784Skmacy if (ret) 2254178784Skmacy goto out; 2255178784Skmacy 2256178784Skmacy if (cb->server == -1) { 2257256829Snp PRINTF(cb, "must be either client or server\n"); 2258256829Snp ret = -EINVAL; 2259178784Skmacy goto out; 2260178784Skmacy } 2261256829Snp 2262256829Snp if (cb->server && cb->frtest) { 2263256829Snp PRINTF(cb, "must be client to run frtest\n"); 2264256829Snp ret = -EINVAL; 2265178784Skmacy goto out; 2266178784Skmacy } 2267178784Skmacy 2268256829Snp if ((cb->frtest + cb->bw + cb->rlat + cb->wlat) > 1) { 2269256829Snp PRINTF(cb, "Pick only one test: fr, bw, rlat, wlat\n"); 2270256829Snp ret = -EINVAL; 2271256829Snp goto out; 2272256829Snp } 2273178784Skmacy 2274256829Snp if (cb->server_invalidate && cb->mem != FASTREG) { 2275256829Snp PRINTF(cb, "server_invalidate only valid with fastreg mem_mode\n"); 2276256829Snp ret = -EINVAL; 2277256829Snp goto out; 2278256829Snp } 2279256829Snp 2280256829Snp if (cb->read_inv && cb->mem != FASTREG) { 2281256829Snp PRINTF(cb, "read_inv only valid with fastreg mem_mode\n"); 2282256829Snp ret = -EINVAL; 2283256829Snp goto out; 2284256829Snp } 2285256829Snp 2286256829Snp if (cb->mem != MR && (cb->wlat || cb->rlat || cb->bw)) { 2287256829Snp PRINTF(cb, "wlat, rlat, and bw tests only support mem_mode MR\n"); 2288256829Snp ret = -EINVAL; 2289256829Snp goto out; 2290256829Snp } 2291256829Snp 2292178784Skmacy cb->cm_id = rdma_create_id(krping_cma_event_handler, cb, RDMA_PS_TCP); 2293178784Skmacy if (IS_ERR(cb->cm_id)) { 2294178784Skmacy ret = PTR_ERR(cb->cm_id); 2295256829Snp PRINTF(cb, "rdma_create_id error %d\n", ret); 2296178784Skmacy goto out; 2297178784Skmacy } 2298256829Snp DEBUG_LOG(cb, "created cm_id %p\n", cb->cm_id); 2299256829Snp 2300178784Skmacy if (cb->server) 2301178784Skmacy krping_run_server(cb); 2302178784Skmacy else 2303178784Skmacy krping_run_client(cb); 2304254735Snp 2305256829Snp DEBUG_LOG(cb, "destroy cm_id %p\n", cb->cm_id); 2306178784Skmacy rdma_destroy_id(cb->cm_id); 2307178784Skmacyout: 2308256829Snp mutex_lock(&krping_mutex); 2309256829Snp list_del(&cb->list); 2310256829Snp mutex_unlock(&krping_mutex); 2311256829Snp kfree(cb); 2312178784Skmacy return ret; 2313178784Skmacy} 2314178784Skmacy 2315256829Snpvoid 2316256829Snpkrping_walk_cb_list(void (*f)(struct krping_stats *, void *), void *arg) 2317256829Snp{ 2318256829Snp struct krping_cb *cb; 2319256829Snp 2320256829Snp mutex_lock(&krping_mutex); 2321256829Snp list_for_each_entry(cb, &krping_cbs, list) 2322256829Snp (*f)(cb->pd ? &cb->stats : NULL, arg); 2323256829Snp mutex_unlock(&krping_mutex); 2324256829Snp} 2325256829Snp 2326178784Skmacyvoid krping_init(void) 2327178784Skmacy{ 2328256829Snp 2329256829Snp mutex_init(&krping_mutex); 2330178784Skmacy} 2331