1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005 Topspin Communications. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff */ 32331769Shselasky#define _GNU_SOURCE 33331769Shselasky#include <config.h> 34219820Sjeff 35219820Sjeff#include <stdio.h> 36219820Sjeff#include <stdlib.h> 37219820Sjeff#include <unistd.h> 38219820Sjeff#include <string.h> 39219820Sjeff#include <sys/types.h> 40219820Sjeff#include <sys/socket.h> 41219820Sjeff#include <sys/time.h> 42219820Sjeff#include <netdb.h> 43331769Shselasky#include <stdlib.h> 44219820Sjeff#include <getopt.h> 45219820Sjeff#include <arpa/inet.h> 46219820Sjeff#include <time.h> 47219820Sjeff 48219820Sjeff#include "pingpong.h" 49219820Sjeff 50219820Sjeffenum { 51219820Sjeff PINGPONG_RECV_WRID = 1, 52219820Sjeff PINGPONG_SEND_WRID = 2, 53219820Sjeff}; 54219820Sjeff 55219820Sjeffstatic int page_size; 56219820Sjeff 57219820Sjeffstruct pingpong_context { 58219820Sjeff struct ibv_context *context; 59219820Sjeff struct ibv_comp_channel *channel; 60219820Sjeff struct ibv_pd *pd; 61219820Sjeff struct ibv_mr *mr; 62219820Sjeff struct ibv_cq *cq; 63219820Sjeff struct ibv_qp *qp; 64219820Sjeff void *buf; 65219820Sjeff int size; 66331769Shselasky int send_flags; 67219820Sjeff int rx_depth; 68219820Sjeff int pending; 69331769Shselasky struct ibv_port_attr portinfo; 70219820Sjeff}; 71219820Sjeff 72219820Sjeffstruct pingpong_dest { 73219820Sjeff int lid; 74219820Sjeff int qpn; 75219820Sjeff int psn; 76219820Sjeff union ibv_gid gid; 77219820Sjeff}; 78219820Sjeff 79219820Sjeffstatic int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 80219820Sjeff enum ibv_mtu mtu, int sl, 81219820Sjeff struct pingpong_dest *dest, int sgid_idx) 82219820Sjeff{ 83219820Sjeff struct ibv_qp_attr attr = { 84219820Sjeff .qp_state = IBV_QPS_RTR, 85219820Sjeff .path_mtu = mtu, 86219820Sjeff .dest_qp_num = dest->qpn, 87219820Sjeff .rq_psn = dest->psn, 88219820Sjeff .ah_attr = { 89219820Sjeff .is_global = 0, 90219820Sjeff .dlid = dest->lid, 91219820Sjeff .sl = sl, 92219820Sjeff .src_path_bits = 0, 93219820Sjeff .port_num = port 94219820Sjeff } 95219820Sjeff }; 96219820Sjeff 97219820Sjeff if (dest->gid.global.interface_id) { 98219820Sjeff attr.ah_attr.is_global = 1; 99219820Sjeff attr.ah_attr.grh.hop_limit = 1; 100219820Sjeff attr.ah_attr.grh.dgid = dest->gid; 101219820Sjeff attr.ah_attr.grh.sgid_index = sgid_idx; 102219820Sjeff } 103219820Sjeff if (ibv_modify_qp(ctx->qp, &attr, 104219820Sjeff IBV_QP_STATE | 105219820Sjeff IBV_QP_AV | 106219820Sjeff IBV_QP_PATH_MTU | 107219820Sjeff IBV_QP_DEST_QPN | 108219820Sjeff IBV_QP_RQ_PSN)) { 109219820Sjeff fprintf(stderr, "Failed to modify QP to RTR\n"); 110219820Sjeff return 1; 111219820Sjeff } 112219820Sjeff 113219820Sjeff attr.qp_state = IBV_QPS_RTS; 114219820Sjeff attr.sq_psn = my_psn; 115219820Sjeff if (ibv_modify_qp(ctx->qp, &attr, 116219820Sjeff IBV_QP_STATE | 117219820Sjeff IBV_QP_SQ_PSN)) { 118219820Sjeff fprintf(stderr, "Failed to modify QP to RTS\n"); 119219820Sjeff return 1; 120219820Sjeff } 121219820Sjeff 122219820Sjeff return 0; 123219820Sjeff} 124219820Sjeff 125219820Sjeffstatic struct pingpong_dest *pp_client_exch_dest(const char *servername, int port, 126219820Sjeff const struct pingpong_dest *my_dest) 127219820Sjeff{ 128219820Sjeff struct addrinfo *res, *t; 129219820Sjeff struct addrinfo hints = { 130337119Shselasky .ai_family = AF_UNSPEC, 131219820Sjeff .ai_socktype = SOCK_STREAM 132219820Sjeff }; 133219820Sjeff char *service; 134219820Sjeff char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 135219820Sjeff int n; 136219820Sjeff int sockfd = -1; 137219820Sjeff struct pingpong_dest *rem_dest = NULL; 138219820Sjeff char gid[33]; 139219820Sjeff 140219820Sjeff if (asprintf(&service, "%d", port) < 0) 141219820Sjeff return NULL; 142219820Sjeff 143219820Sjeff n = getaddrinfo(servername, service, &hints, &res); 144219820Sjeff 145219820Sjeff if (n < 0) { 146219820Sjeff fprintf(stderr, "%s for %s:%d\n", gai_strerror(n), servername, port); 147219820Sjeff free(service); 148219820Sjeff return NULL; 149219820Sjeff } 150219820Sjeff 151219820Sjeff for (t = res; t; t = t->ai_next) { 152219820Sjeff sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 153219820Sjeff if (sockfd >= 0) { 154219820Sjeff if (!connect(sockfd, t->ai_addr, t->ai_addrlen)) 155219820Sjeff break; 156219820Sjeff close(sockfd); 157219820Sjeff sockfd = -1; 158219820Sjeff } 159219820Sjeff } 160219820Sjeff 161331769Shselasky freeaddrinfo_null(res); 162219820Sjeff free(service); 163219820Sjeff 164219820Sjeff if (sockfd < 0) { 165219820Sjeff fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port); 166219820Sjeff return NULL; 167219820Sjeff } 168219820Sjeff 169219820Sjeff gid_to_wire_gid(&my_dest->gid, gid); 170331769Shselasky sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, 171331769Shselasky my_dest->psn, gid); 172219820Sjeff if (write(sockfd, msg, sizeof msg) != sizeof msg) { 173219820Sjeff fprintf(stderr, "Couldn't send local address\n"); 174219820Sjeff goto out; 175219820Sjeff } 176219820Sjeff 177331769Shselasky if (read(sockfd, msg, sizeof msg) != sizeof msg || 178331769Shselasky write(sockfd, "done", sizeof "done") != sizeof "done") { 179331769Shselasky perror("client read/write"); 180331769Shselasky fprintf(stderr, "Couldn't read/write remote address\n"); 181219820Sjeff goto out; 182219820Sjeff } 183219820Sjeff 184219820Sjeff 185219820Sjeff rem_dest = malloc(sizeof *rem_dest); 186219820Sjeff if (!rem_dest) 187219820Sjeff goto out; 188219820Sjeff 189331769Shselasky sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, 190331769Shselasky &rem_dest->psn, gid); 191219820Sjeff wire_gid_to_gid(gid, &rem_dest->gid); 192219820Sjeff 193219820Sjeffout: 194219820Sjeff close(sockfd); 195219820Sjeff return rem_dest; 196219820Sjeff} 197219820Sjeff 198219820Sjeffstatic struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 199219820Sjeff int ib_port, enum ibv_mtu mtu, 200219820Sjeff int port, int sl, 201219820Sjeff const struct pingpong_dest *my_dest, 202219820Sjeff int sgid_idx) 203219820Sjeff{ 204219820Sjeff struct addrinfo *res, *t; 205219820Sjeff struct addrinfo hints = { 206219820Sjeff .ai_flags = AI_PASSIVE, 207219820Sjeff .ai_family = AF_INET, 208219820Sjeff .ai_socktype = SOCK_STREAM 209219820Sjeff }; 210219820Sjeff char *service; 211219820Sjeff char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 212219820Sjeff int n; 213219820Sjeff int sockfd = -1, connfd; 214219820Sjeff struct pingpong_dest *rem_dest = NULL; 215219820Sjeff char gid[33]; 216219820Sjeff 217219820Sjeff if (asprintf(&service, "%d", port) < 0) 218219820Sjeff return NULL; 219219820Sjeff 220219820Sjeff n = getaddrinfo(NULL, service, &hints, &res); 221219820Sjeff 222219820Sjeff if (n < 0) { 223219820Sjeff fprintf(stderr, "%s for port %d\n", gai_strerror(n), port); 224219820Sjeff free(service); 225219820Sjeff return NULL; 226219820Sjeff } 227219820Sjeff 228219820Sjeff for (t = res; t; t = t->ai_next) { 229219820Sjeff sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol); 230219820Sjeff if (sockfd >= 0) { 231219820Sjeff n = 1; 232219820Sjeff 233219820Sjeff setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n); 234219820Sjeff 235219820Sjeff if (!bind(sockfd, t->ai_addr, t->ai_addrlen)) 236219820Sjeff break; 237219820Sjeff close(sockfd); 238219820Sjeff sockfd = -1; 239219820Sjeff } 240219820Sjeff } 241219820Sjeff 242331769Shselasky freeaddrinfo_null(res); 243219820Sjeff free(service); 244219820Sjeff 245219820Sjeff if (sockfd < 0) { 246219820Sjeff fprintf(stderr, "Couldn't listen to port %d\n", port); 247219820Sjeff return NULL; 248219820Sjeff } 249219820Sjeff 250219820Sjeff listen(sockfd, 1); 251331769Shselasky connfd = accept(sockfd, NULL, NULL); 252219820Sjeff close(sockfd); 253219820Sjeff if (connfd < 0) { 254219820Sjeff fprintf(stderr, "accept() failed\n"); 255219820Sjeff return NULL; 256219820Sjeff } 257219820Sjeff 258219820Sjeff n = read(connfd, msg, sizeof msg); 259219820Sjeff if (n != sizeof msg) { 260219820Sjeff perror("server read"); 261219820Sjeff fprintf(stderr, "%d/%d: Couldn't read remote address\n", n, (int) sizeof msg); 262219820Sjeff goto out; 263219820Sjeff } 264219820Sjeff 265219820Sjeff rem_dest = malloc(sizeof *rem_dest); 266219820Sjeff if (!rem_dest) 267219820Sjeff goto out; 268219820Sjeff 269331769Shselasky sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, 270331769Shselasky &rem_dest->psn, gid); 271219820Sjeff wire_gid_to_gid(gid, &rem_dest->gid); 272219820Sjeff 273331769Shselasky if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, 274331769Shselasky sgid_idx)) { 275219820Sjeff fprintf(stderr, "Couldn't connect to remote QP\n"); 276219820Sjeff free(rem_dest); 277219820Sjeff rem_dest = NULL; 278219820Sjeff goto out; 279219820Sjeff } 280219820Sjeff 281331769Shselasky 282219820Sjeff gid_to_wire_gid(&my_dest->gid, gid); 283331769Shselasky sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, 284331769Shselasky my_dest->psn, gid); 285331769Shselasky if (write(connfd, msg, sizeof msg) != sizeof msg || 286331769Shselasky read(connfd, msg, sizeof msg) != sizeof "done") { 287331769Shselasky fprintf(stderr, "Couldn't send/recv local address\n"); 288219820Sjeff free(rem_dest); 289219820Sjeff rem_dest = NULL; 290219820Sjeff goto out; 291219820Sjeff } 292219820Sjeff 293219820Sjeffout: 294219820Sjeff close(connfd); 295219820Sjeff return rem_dest; 296219820Sjeff} 297219820Sjeff 298219820Sjeffstatic struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, 299219820Sjeff int rx_depth, int port, 300219820Sjeff int use_event) 301219820Sjeff{ 302219820Sjeff struct pingpong_context *ctx; 303219820Sjeff 304219820Sjeff ctx = calloc(1, sizeof *ctx); 305219820Sjeff if (!ctx) 306219820Sjeff return NULL; 307219820Sjeff 308331769Shselasky ctx->size = size; 309331769Shselasky ctx->send_flags = IBV_SEND_SIGNALED; 310331769Shselasky ctx->rx_depth = rx_depth; 311219820Sjeff 312331769Shselasky ctx->buf = memalign(page_size, size); 313219820Sjeff if (!ctx->buf) { 314219820Sjeff fprintf(stderr, "Couldn't allocate work buf.\n"); 315331769Shselasky goto clean_ctx; 316219820Sjeff } 317219820Sjeff 318331769Shselasky /* FIXME memset(ctx->buf, 0, size); */ 319331769Shselasky memset(ctx->buf, 0x7b, size); 320219820Sjeff 321219820Sjeff ctx->context = ibv_open_device(ib_dev); 322219820Sjeff if (!ctx->context) { 323219820Sjeff fprintf(stderr, "Couldn't get context for %s\n", 324219820Sjeff ibv_get_device_name(ib_dev)); 325331769Shselasky goto clean_buffer; 326219820Sjeff } 327219820Sjeff 328219820Sjeff if (use_event) { 329219820Sjeff ctx->channel = ibv_create_comp_channel(ctx->context); 330219820Sjeff if (!ctx->channel) { 331219820Sjeff fprintf(stderr, "Couldn't create completion channel\n"); 332331769Shselasky goto clean_device; 333219820Sjeff } 334219820Sjeff } else 335219820Sjeff ctx->channel = NULL; 336219820Sjeff 337219820Sjeff ctx->pd = ibv_alloc_pd(ctx->context); 338219820Sjeff if (!ctx->pd) { 339219820Sjeff fprintf(stderr, "Couldn't allocate PD\n"); 340331769Shselasky goto clean_comp_channel; 341219820Sjeff } 342219820Sjeff 343219820Sjeff ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE); 344219820Sjeff if (!ctx->mr) { 345219820Sjeff fprintf(stderr, "Couldn't register MR\n"); 346331769Shselasky goto clean_pd; 347219820Sjeff } 348219820Sjeff 349219820Sjeff ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, 350219820Sjeff ctx->channel, 0); 351219820Sjeff if (!ctx->cq) { 352219820Sjeff fprintf(stderr, "Couldn't create CQ\n"); 353331769Shselasky goto clean_mr; 354219820Sjeff } 355219820Sjeff 356219820Sjeff { 357331769Shselasky struct ibv_qp_attr attr; 358331769Shselasky struct ibv_qp_init_attr init_attr = { 359219820Sjeff .send_cq = ctx->cq, 360219820Sjeff .recv_cq = ctx->cq, 361219820Sjeff .cap = { 362219820Sjeff .max_send_wr = 1, 363219820Sjeff .max_recv_wr = rx_depth, 364219820Sjeff .max_send_sge = 1, 365219820Sjeff .max_recv_sge = 1 366219820Sjeff }, 367219820Sjeff .qp_type = IBV_QPT_UC 368219820Sjeff }; 369219820Sjeff 370331769Shselasky ctx->qp = ibv_create_qp(ctx->pd, &init_attr); 371219820Sjeff if (!ctx->qp) { 372219820Sjeff fprintf(stderr, "Couldn't create QP\n"); 373331769Shselasky goto clean_cq; 374219820Sjeff } 375331769Shselasky ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr); 376331769Shselasky if (init_attr.cap.max_inline_data >= size) { 377331769Shselasky ctx->send_flags |= IBV_SEND_INLINE; 378331769Shselasky } 379219820Sjeff } 380219820Sjeff 381219820Sjeff { 382219820Sjeff struct ibv_qp_attr attr = { 383219820Sjeff .qp_state = IBV_QPS_INIT, 384219820Sjeff .pkey_index = 0, 385219820Sjeff .port_num = port, 386219820Sjeff .qp_access_flags = 0 387219820Sjeff }; 388219820Sjeff 389219820Sjeff if (ibv_modify_qp(ctx->qp, &attr, 390219820Sjeff IBV_QP_STATE | 391219820Sjeff IBV_QP_PKEY_INDEX | 392219820Sjeff IBV_QP_PORT | 393219820Sjeff IBV_QP_ACCESS_FLAGS)) { 394219820Sjeff fprintf(stderr, "Failed to modify QP to INIT\n"); 395331769Shselasky goto clean_qp; 396219820Sjeff } 397219820Sjeff } 398219820Sjeff 399219820Sjeff return ctx; 400331769Shselasky 401331769Shselaskyclean_qp: 402331769Shselasky ibv_destroy_qp(ctx->qp); 403331769Shselasky 404331769Shselaskyclean_cq: 405331769Shselasky ibv_destroy_cq(ctx->cq); 406331769Shselasky 407331769Shselaskyclean_mr: 408331769Shselasky ibv_dereg_mr(ctx->mr); 409331769Shselasky 410331769Shselaskyclean_pd: 411331769Shselasky ibv_dealloc_pd(ctx->pd); 412331769Shselasky 413331769Shselaskyclean_comp_channel: 414331769Shselasky if (ctx->channel) 415331769Shselasky ibv_destroy_comp_channel(ctx->channel); 416331769Shselasky 417331769Shselaskyclean_device: 418331769Shselasky ibv_close_device(ctx->context); 419331769Shselasky 420331769Shselaskyclean_buffer: 421331769Shselasky free(ctx->buf); 422331769Shselasky 423331769Shselaskyclean_ctx: 424331769Shselasky free(ctx); 425331769Shselasky 426331769Shselasky return NULL; 427219820Sjeff} 428219820Sjeff 429331769Shselaskystatic int pp_close_ctx(struct pingpong_context *ctx) 430219820Sjeff{ 431219820Sjeff if (ibv_destroy_qp(ctx->qp)) { 432219820Sjeff fprintf(stderr, "Couldn't destroy QP\n"); 433219820Sjeff return 1; 434219820Sjeff } 435219820Sjeff 436219820Sjeff if (ibv_destroy_cq(ctx->cq)) { 437219820Sjeff fprintf(stderr, "Couldn't destroy CQ\n"); 438219820Sjeff return 1; 439219820Sjeff } 440219820Sjeff 441219820Sjeff if (ibv_dereg_mr(ctx->mr)) { 442219820Sjeff fprintf(stderr, "Couldn't deregister MR\n"); 443219820Sjeff return 1; 444219820Sjeff } 445219820Sjeff 446219820Sjeff if (ibv_dealloc_pd(ctx->pd)) { 447219820Sjeff fprintf(stderr, "Couldn't deallocate PD\n"); 448219820Sjeff return 1; 449219820Sjeff } 450219820Sjeff 451219820Sjeff if (ctx->channel) { 452219820Sjeff if (ibv_destroy_comp_channel(ctx->channel)) { 453219820Sjeff fprintf(stderr, "Couldn't destroy completion channel\n"); 454219820Sjeff return 1; 455219820Sjeff } 456219820Sjeff } 457219820Sjeff 458219820Sjeff if (ibv_close_device(ctx->context)) { 459219820Sjeff fprintf(stderr, "Couldn't release context\n"); 460219820Sjeff return 1; 461219820Sjeff } 462219820Sjeff 463219820Sjeff free(ctx->buf); 464219820Sjeff free(ctx); 465219820Sjeff 466219820Sjeff return 0; 467219820Sjeff} 468219820Sjeff 469219820Sjeffstatic int pp_post_recv(struct pingpong_context *ctx, int n) 470219820Sjeff{ 471219820Sjeff struct ibv_sge list = { 472219820Sjeff .addr = (uintptr_t) ctx->buf, 473219820Sjeff .length = ctx->size, 474219820Sjeff .lkey = ctx->mr->lkey 475219820Sjeff }; 476219820Sjeff struct ibv_recv_wr wr = { 477219820Sjeff .wr_id = PINGPONG_RECV_WRID, 478219820Sjeff .sg_list = &list, 479219820Sjeff .num_sge = 1, 480219820Sjeff }; 481219820Sjeff struct ibv_recv_wr *bad_wr; 482219820Sjeff int i; 483219820Sjeff 484219820Sjeff for (i = 0; i < n; ++i) 485219820Sjeff if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) 486219820Sjeff break; 487219820Sjeff 488219820Sjeff return i; 489219820Sjeff} 490219820Sjeff 491219820Sjeffstatic int pp_post_send(struct pingpong_context *ctx) 492219820Sjeff{ 493219820Sjeff struct ibv_sge list = { 494219820Sjeff .addr = (uintptr_t) ctx->buf, 495219820Sjeff .length = ctx->size, 496219820Sjeff .lkey = ctx->mr->lkey 497219820Sjeff }; 498219820Sjeff struct ibv_send_wr wr = { 499219820Sjeff .wr_id = PINGPONG_SEND_WRID, 500219820Sjeff .sg_list = &list, 501219820Sjeff .num_sge = 1, 502219820Sjeff .opcode = IBV_WR_SEND, 503331769Shselasky .send_flags = ctx->send_flags, 504219820Sjeff }; 505219820Sjeff struct ibv_send_wr *bad_wr; 506219820Sjeff 507219820Sjeff return ibv_post_send(ctx->qp, &wr, &bad_wr); 508219820Sjeff} 509219820Sjeff 510219820Sjeffstatic void usage(const char *argv0) 511219820Sjeff{ 512219820Sjeff printf("Usage:\n"); 513219820Sjeff printf(" %s start a server and wait for connection\n", argv0); 514219820Sjeff printf(" %s <host> connect to server at <host>\n", argv0); 515219820Sjeff printf("\n"); 516219820Sjeff printf("Options:\n"); 517219820Sjeff printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); 518219820Sjeff printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); 519219820Sjeff printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); 520219820Sjeff printf(" -s, --size=<size> size of message to exchange (default 4096)\n"); 521219820Sjeff printf(" -m, --mtu=<size> path MTU (default 1024)\n"); 522219820Sjeff printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); 523219820Sjeff printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 524219820Sjeff printf(" -l, --sl=<sl> service level value\n"); 525219820Sjeff printf(" -e, --events sleep on CQ events (default poll)\n"); 526219820Sjeff printf(" -g, --gid-idx=<gid index> local port gid index\n"); 527219820Sjeff} 528219820Sjeff 529219820Sjeffint main(int argc, char *argv[]) 530219820Sjeff{ 531219820Sjeff struct ibv_device **dev_list; 532219820Sjeff struct ibv_device *ib_dev; 533219820Sjeff struct pingpong_context *ctx; 534219820Sjeff struct pingpong_dest my_dest; 535219820Sjeff struct pingpong_dest *rem_dest; 536219820Sjeff struct timeval start, end; 537219820Sjeff char *ib_devname = NULL; 538219820Sjeff char *servername = NULL; 539331769Shselasky unsigned int port = 18515; 540219820Sjeff int ib_port = 1; 541331769Shselasky unsigned int size = 4096; 542219820Sjeff enum ibv_mtu mtu = IBV_MTU_1024; 543331769Shselasky unsigned int rx_depth = 500; 544331769Shselasky unsigned int iters = 1000; 545219820Sjeff int use_event = 0; 546219820Sjeff int routs; 547219820Sjeff int rcnt, scnt; 548219820Sjeff int num_cq_events = 0; 549219820Sjeff int sl = 0; 550219820Sjeff int gidx = -1; 551219820Sjeff char gid[33]; 552219820Sjeff 553219820Sjeff srand48(getpid() * time(NULL)); 554219820Sjeff 555219820Sjeff while (1) { 556219820Sjeff int c; 557219820Sjeff 558219820Sjeff static struct option long_options[] = { 559219820Sjeff { .name = "port", .has_arg = 1, .val = 'p' }, 560219820Sjeff { .name = "ib-dev", .has_arg = 1, .val = 'd' }, 561219820Sjeff { .name = "ib-port", .has_arg = 1, .val = 'i' }, 562219820Sjeff { .name = "size", .has_arg = 1, .val = 's' }, 563219820Sjeff { .name = "mtu", .has_arg = 1, .val = 'm' }, 564219820Sjeff { .name = "rx-depth", .has_arg = 1, .val = 'r' }, 565219820Sjeff { .name = "iters", .has_arg = 1, .val = 'n' }, 566219820Sjeff { .name = "sl", .has_arg = 1, .val = 'l' }, 567219820Sjeff { .name = "events", .has_arg = 0, .val = 'e' }, 568219820Sjeff { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 569331769Shselasky {} 570219820Sjeff }; 571219820Sjeff 572331769Shselasky c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", 573331769Shselasky long_options, NULL); 574219820Sjeff if (c == -1) 575219820Sjeff break; 576219820Sjeff 577219820Sjeff switch (c) { 578219820Sjeff case 'p': 579331769Shselasky port = strtoul(optarg, NULL, 0); 580331769Shselasky if (port > 65535) { 581219820Sjeff usage(argv[0]); 582219820Sjeff return 1; 583219820Sjeff } 584219820Sjeff break; 585219820Sjeff 586219820Sjeff case 'd': 587331769Shselasky ib_devname = strdupa(optarg); 588219820Sjeff break; 589219820Sjeff 590219820Sjeff case 'i': 591219820Sjeff ib_port = strtol(optarg, NULL, 0); 592331769Shselasky if (ib_port < 1) { 593219820Sjeff usage(argv[0]); 594219820Sjeff return 1; 595219820Sjeff } 596219820Sjeff break; 597219820Sjeff 598219820Sjeff case 's': 599331769Shselasky size = strtoul(optarg, NULL, 0); 600219820Sjeff break; 601219820Sjeff 602219820Sjeff case 'm': 603219820Sjeff mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0)); 604331769Shselasky if (mtu == 0) { 605219820Sjeff usage(argv[0]); 606219820Sjeff return 1; 607219820Sjeff } 608219820Sjeff break; 609219820Sjeff 610219820Sjeff case 'r': 611331769Shselasky rx_depth = strtoul(optarg, NULL, 0); 612219820Sjeff break; 613219820Sjeff 614219820Sjeff case 'n': 615331769Shselasky iters = strtoul(optarg, NULL, 0); 616219820Sjeff break; 617219820Sjeff 618219820Sjeff case 'l': 619219820Sjeff sl = strtol(optarg, NULL, 0); 620219820Sjeff break; 621219820Sjeff 622219820Sjeff case 'e': 623219820Sjeff ++use_event; 624219820Sjeff break; 625219820Sjeff 626219820Sjeff case 'g': 627219820Sjeff gidx = strtol(optarg, NULL, 0); 628219820Sjeff break; 629219820Sjeff 630219820Sjeff default: 631219820Sjeff usage(argv[0]); 632219820Sjeff return 1; 633219820Sjeff } 634219820Sjeff } 635219820Sjeff 636219820Sjeff if (optind == argc - 1) 637331769Shselasky servername = strdupa(argv[optind]); 638219820Sjeff else if (optind < argc) { 639219820Sjeff usage(argv[0]); 640219820Sjeff return 1; 641219820Sjeff } 642219820Sjeff 643219820Sjeff page_size = sysconf(_SC_PAGESIZE); 644219820Sjeff 645219820Sjeff dev_list = ibv_get_device_list(NULL); 646219820Sjeff if (!dev_list) { 647219820Sjeff perror("Failed to get IB devices list"); 648219820Sjeff return 1; 649219820Sjeff } 650219820Sjeff 651219820Sjeff if (!ib_devname) { 652219820Sjeff ib_dev = *dev_list; 653219820Sjeff if (!ib_dev) { 654219820Sjeff fprintf(stderr, "No IB devices found\n"); 655219820Sjeff return 1; 656219820Sjeff } 657219820Sjeff } else { 658219820Sjeff int i; 659219820Sjeff for (i = 0; dev_list[i]; ++i) 660219820Sjeff if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname)) 661219820Sjeff break; 662219820Sjeff ib_dev = dev_list[i]; 663219820Sjeff if (!ib_dev) { 664219820Sjeff fprintf(stderr, "IB device %s not found\n", ib_devname); 665219820Sjeff return 1; 666219820Sjeff } 667219820Sjeff } 668219820Sjeff 669219820Sjeff ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); 670219820Sjeff if (!ctx) 671219820Sjeff return 1; 672219820Sjeff 673219820Sjeff routs = pp_post_recv(ctx, ctx->rx_depth); 674219820Sjeff if (routs < ctx->rx_depth) { 675219820Sjeff fprintf(stderr, "Couldn't post receive (%d)\n", routs); 676219820Sjeff return 1; 677219820Sjeff } 678219820Sjeff 679219820Sjeff if (use_event) 680219820Sjeff if (ibv_req_notify_cq(ctx->cq, 0)) { 681219820Sjeff fprintf(stderr, "Couldn't request CQ notification\n"); 682219820Sjeff return 1; 683219820Sjeff } 684219820Sjeff 685331769Shselasky 686219820Sjeff if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 687219820Sjeff fprintf(stderr, "Couldn't get port info\n"); 688219820Sjeff return 1; 689219820Sjeff } 690219820Sjeff 691219820Sjeff my_dest.lid = ctx->portinfo.lid; 692331769Shselasky if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET && 693331769Shselasky !my_dest.lid) { 694219820Sjeff fprintf(stderr, "Couldn't get local LID\n"); 695219820Sjeff return 1; 696219820Sjeff } 697219820Sjeff 698219820Sjeff if (gidx >= 0) { 699219820Sjeff if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 700331769Shselasky fprintf(stderr, "can't read sgid of index %d\n", gidx); 701219820Sjeff return 1; 702219820Sjeff } 703219820Sjeff } else 704219820Sjeff memset(&my_dest.gid, 0, sizeof my_dest.gid); 705219820Sjeff 706219820Sjeff my_dest.qpn = ctx->qp->qp_num; 707219820Sjeff my_dest.psn = lrand48() & 0xffffff; 708219820Sjeff inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 709219820Sjeff printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 710219820Sjeff my_dest.lid, my_dest.qpn, my_dest.psn, gid); 711219820Sjeff 712331769Shselasky 713219820Sjeff if (servername) 714219820Sjeff rem_dest = pp_client_exch_dest(servername, port, &my_dest); 715219820Sjeff else 716331769Shselasky rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, 717331769Shselasky &my_dest, gidx); 718219820Sjeff 719219820Sjeff if (!rem_dest) 720219820Sjeff return 1; 721219820Sjeff 722219820Sjeff inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 723219820Sjeff printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 724219820Sjeff rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 725219820Sjeff 726219820Sjeff if (servername) 727331769Shselasky if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, 728331769Shselasky gidx)) 729219820Sjeff return 1; 730219820Sjeff 731219820Sjeff ctx->pending = PINGPONG_RECV_WRID; 732219820Sjeff 733219820Sjeff if (servername) { 734219820Sjeff if (pp_post_send(ctx)) { 735219820Sjeff fprintf(stderr, "Couldn't post send\n"); 736219820Sjeff return 1; 737219820Sjeff } 738219820Sjeff ctx->pending |= PINGPONG_SEND_WRID; 739219820Sjeff } 740219820Sjeff 741219820Sjeff if (gettimeofday(&start, NULL)) { 742219820Sjeff perror("gettimeofday"); 743219820Sjeff return 1; 744219820Sjeff } 745219820Sjeff 746219820Sjeff rcnt = scnt = 0; 747219820Sjeff while (rcnt < iters || scnt < iters) { 748219820Sjeff if (use_event) { 749219820Sjeff struct ibv_cq *ev_cq; 750219820Sjeff void *ev_ctx; 751219820Sjeff 752219820Sjeff if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { 753219820Sjeff fprintf(stderr, "Failed to get cq_event\n"); 754219820Sjeff return 1; 755219820Sjeff } 756219820Sjeff 757219820Sjeff ++num_cq_events; 758219820Sjeff 759219820Sjeff if (ev_cq != ctx->cq) { 760219820Sjeff fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq); 761219820Sjeff return 1; 762219820Sjeff } 763219820Sjeff 764219820Sjeff if (ibv_req_notify_cq(ctx->cq, 0)) { 765219820Sjeff fprintf(stderr, "Couldn't request CQ notification\n"); 766219820Sjeff return 1; 767219820Sjeff } 768219820Sjeff } 769219820Sjeff 770219820Sjeff { 771219820Sjeff struct ibv_wc wc[2]; 772219820Sjeff int ne, i; 773219820Sjeff 774219820Sjeff do { 775219820Sjeff ne = ibv_poll_cq(ctx->cq, 2, wc); 776219820Sjeff if (ne < 0) { 777219820Sjeff fprintf(stderr, "poll CQ failed %d\n", ne); 778219820Sjeff return 1; 779219820Sjeff } 780331769Shselasky 781219820Sjeff } while (!use_event && ne < 1); 782219820Sjeff 783219820Sjeff for (i = 0; i < ne; ++i) { 784219820Sjeff if (wc[i].status != IBV_WC_SUCCESS) { 785219820Sjeff fprintf(stderr, "Failed status %s (%d) for wr_id %d\n", 786219820Sjeff ibv_wc_status_str(wc[i].status), 787219820Sjeff wc[i].status, (int) wc[i].wr_id); 788219820Sjeff return 1; 789219820Sjeff } 790219820Sjeff 791219820Sjeff switch ((int) wc[i].wr_id) { 792219820Sjeff case PINGPONG_SEND_WRID: 793219820Sjeff ++scnt; 794219820Sjeff break; 795219820Sjeff 796219820Sjeff case PINGPONG_RECV_WRID: 797219820Sjeff if (--routs <= 1) { 798219820Sjeff routs += pp_post_recv(ctx, ctx->rx_depth - routs); 799219820Sjeff if (routs < ctx->rx_depth) { 800219820Sjeff fprintf(stderr, 801219820Sjeff "Couldn't post receive (%d)\n", 802219820Sjeff routs); 803219820Sjeff return 1; 804219820Sjeff } 805219820Sjeff } 806219820Sjeff 807219820Sjeff ++rcnt; 808219820Sjeff break; 809219820Sjeff 810219820Sjeff default: 811219820Sjeff fprintf(stderr, "Completion for unknown wr_id %d\n", 812219820Sjeff (int) wc[i].wr_id); 813219820Sjeff return 1; 814219820Sjeff } 815219820Sjeff 816219820Sjeff ctx->pending &= ~(int) wc[i].wr_id; 817219820Sjeff if (scnt < iters && !ctx->pending) { 818219820Sjeff if (pp_post_send(ctx)) { 819219820Sjeff fprintf(stderr, "Couldn't post send\n"); 820219820Sjeff return 1; 821219820Sjeff } 822219820Sjeff ctx->pending = PINGPONG_RECV_WRID | 823219820Sjeff PINGPONG_SEND_WRID; 824219820Sjeff } 825219820Sjeff } 826219820Sjeff } 827219820Sjeff } 828219820Sjeff 829219820Sjeff if (gettimeofday(&end, NULL)) { 830219820Sjeff perror("gettimeofday"); 831219820Sjeff return 1; 832219820Sjeff } 833219820Sjeff 834219820Sjeff { 835219820Sjeff float usec = (end.tv_sec - start.tv_sec) * 1000000 + 836219820Sjeff (end.tv_usec - start.tv_usec); 837219820Sjeff long long bytes = (long long) size * iters * 2; 838219820Sjeff 839219820Sjeff printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", 840219820Sjeff bytes, usec / 1000000., bytes * 8. / usec); 841219820Sjeff printf("%d iters in %.2f seconds = %.2f usec/iter\n", 842219820Sjeff iters, usec / 1000000., usec / iters); 843219820Sjeff } 844219820Sjeff 845219820Sjeff ibv_ack_cq_events(ctx->cq, num_cq_events); 846219820Sjeff 847219820Sjeff if (pp_close_ctx(ctx)) 848219820Sjeff return 1; 849219820Sjeff 850219820Sjeff ibv_free_device_list(dev_list); 851219820Sjeff free(rem_dest); 852219820Sjeff 853219820Sjeff return 0; 854219820Sjeff} 855