1/*
2 * Copyright (c) 2017 Pure Storage, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. The name of the author may not be used to endorse or promote
15 * products derived from this software without specific prior written
16 * permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#ifdef HAVE_CONFIG_H
32#include "config.h"
33#endif
34
35#include "pcap-int.h"
36#include "pcap-rdmasniff.h"
37
38#include <infiniband/verbs.h>
39#include <stdlib.h>
40#include <string.h>
41#include <limits.h> /* for INT_MAX */
42#include <sys/time.h>
43
44#if !defined(IBV_FLOW_ATTR_SNIFFER)
45#define IBV_FLOW_ATTR_SNIFFER	3
46#endif
47
48static const int RDMASNIFF_NUM_RECEIVES = 128;
49static const int RDMASNIFF_RECEIVE_SIZE = 10000;
50
51struct pcap_rdmasniff {
52	struct ibv_device *		rdma_device;
53	struct ibv_context *		context;
54	struct ibv_comp_channel *	channel;
55	struct ibv_pd *			pd;
56	struct ibv_cq *			cq;
57	struct ibv_qp *			qp;
58	struct ibv_flow *               flow;
59	struct ibv_mr *			mr;
60	u_char *			oneshot_buffer;
61	unsigned long			port_num;
62	int                             cq_event;
63	u_int                           packets_recv;
64};
65
66static int
67rdmasniff_stats(pcap_t *handle, struct pcap_stat *stat)
68{
69	struct pcap_rdmasniff *priv = handle->priv;
70
71	stat->ps_recv = priv->packets_recv;
72	stat->ps_drop = 0;
73	stat->ps_ifdrop = 0;
74
75	return 0;
76}
77
78static void
79rdmasniff_cleanup(pcap_t *handle)
80{
81	struct pcap_rdmasniff *priv = handle->priv;
82
83	ibv_dereg_mr(priv->mr);
84	ibv_destroy_flow(priv->flow);
85	ibv_destroy_qp(priv->qp);
86	ibv_destroy_cq(priv->cq);
87	ibv_dealloc_pd(priv->pd);
88	ibv_destroy_comp_channel(priv->channel);
89	ibv_close_device(priv->context);
90	free(priv->oneshot_buffer);
91
92	pcap_cleanup_live_common(handle);
93}
94
95static void
96rdmasniff_post_recv(pcap_t *handle, uint64_t wr_id)
97{
98	struct pcap_rdmasniff *priv = handle->priv;
99	struct ibv_sge sg_entry;
100	struct ibv_recv_wr wr, *bad_wr;
101
102	sg_entry.length = RDMASNIFF_RECEIVE_SIZE;
103	sg_entry.addr = (uintptr_t) handle->buffer + RDMASNIFF_RECEIVE_SIZE * wr_id;
104	sg_entry.lkey = priv->mr->lkey;
105
106	wr.wr_id = wr_id;
107	wr.num_sge = 1;
108	wr.sg_list = &sg_entry;
109	wr.next = NULL;
110
111	ibv_post_recv(priv->qp, &wr, &bad_wr);
112}
113
114static int
115rdmasniff_read(pcap_t *handle, int max_packets, pcap_handler callback, u_char *user)
116{
117	struct pcap_rdmasniff *priv = handle->priv;
118	struct ibv_cq *ev_cq;
119	void *ev_ctx;
120	struct ibv_wc wc;
121	struct pcap_pkthdr pkth;
122	u_char *pktd;
123	int count = 0;
124
125	if (!priv->cq_event) {
126		while (ibv_get_cq_event(priv->channel, &ev_cq, &ev_ctx) < 0) {
127			if (errno != EINTR) {
128				return PCAP_ERROR;
129			}
130			if (handle->break_loop) {
131				handle->break_loop = 0;
132				return PCAP_ERROR_BREAK;
133			}
134		}
135		ibv_ack_cq_events(priv->cq, 1);
136		ibv_req_notify_cq(priv->cq, 0);
137		priv->cq_event = 1;
138	}
139
140	/*
141	 * This can conceivably process more than INT_MAX packets,
142	 * which would overflow the packet count, causing it either
143	 * to look like a negative number, and thus cause us to
144	 * return a value that looks like an error, or overflow
145	 * back into positive territory, and thus cause us to
146	 * return a too-low count.
147	 *
148	 * Therefore, if the packet count is unlimited, we clip
149	 * it at INT_MAX; this routine is not expected to
150	 * process packets indefinitely, so that's not an issue.
151	 */
152	if (PACKET_COUNT_IS_UNLIMITED(max_packets))
153		max_packets = INT_MAX;
154
155	while (count < max_packets) {
156		if (ibv_poll_cq(priv->cq, 1, &wc) != 1) {
157			priv->cq_event = 0;
158			break;
159		}
160
161		if (wc.status != IBV_WC_SUCCESS) {
162			fprintf(stderr, "failed WC wr_id %" PRIu64 " status %d/%s\n",
163				wc.wr_id,
164				wc.status, ibv_wc_status_str(wc.status));
165			continue;
166		}
167
168		pkth.len = wc.byte_len;
169		pkth.caplen = min(pkth.len, (u_int)handle->snapshot);
170		gettimeofday(&pkth.ts, NULL);
171
172		pktd = (u_char *) handle->buffer + wc.wr_id * RDMASNIFF_RECEIVE_SIZE;
173
174		if (handle->fcode.bf_insns == NULL ||
175		    pcap_filter(handle->fcode.bf_insns, pktd, pkth.len, pkth.caplen)) {
176			callback(user, &pkth, pktd);
177			++priv->packets_recv;
178			++count;
179		}
180
181		rdmasniff_post_recv(handle, wc.wr_id);
182
183		if (handle->break_loop) {
184			handle->break_loop = 0;
185			return PCAP_ERROR_BREAK;
186		}
187	}
188
189	return count;
190}
191
192static void
193rdmasniff_oneshot(u_char *user, const struct pcap_pkthdr *h, const u_char *bytes)
194{
195	struct oneshot_userdata *sp = (struct oneshot_userdata *) user;
196	pcap_t *handle = sp->pd;
197	struct pcap_rdmasniff *priv = handle->priv;
198
199	*sp->hdr = *h;
200	memcpy(priv->oneshot_buffer, bytes, h->caplen);
201	*sp->pkt = priv->oneshot_buffer;
202}
203
204static int
205rdmasniff_activate(pcap_t *handle)
206{
207	struct pcap_rdmasniff *priv = handle->priv;
208	struct ibv_qp_init_attr qp_init_attr;
209	struct ibv_qp_attr qp_attr;
210	struct ibv_flow_attr flow_attr;
211	struct ibv_port_attr port_attr;
212	int i;
213
214	priv->context = ibv_open_device(priv->rdma_device);
215	if (!priv->context) {
216		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
217			      "Failed to open device %s", handle->opt.device);
218		goto error;
219	}
220
221	priv->pd = ibv_alloc_pd(priv->context);
222	if (!priv->pd) {
223		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
224			      "Failed to alloc PD for device %s", handle->opt.device);
225		goto error;
226	}
227
228	priv->channel = ibv_create_comp_channel(priv->context);
229	if (!priv->channel) {
230		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
231			      "Failed to create comp channel for device %s", handle->opt.device);
232		goto error;
233	}
234
235	priv->cq = ibv_create_cq(priv->context, RDMASNIFF_NUM_RECEIVES,
236				 NULL, priv->channel, 0);
237	if (!priv->cq) {
238		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
239			      "Failed to create CQ for device %s", handle->opt.device);
240		goto error;
241	}
242
243	ibv_req_notify_cq(priv->cq, 0);
244
245	memset(&qp_init_attr, 0, sizeof qp_init_attr);
246	qp_init_attr.send_cq = qp_init_attr.recv_cq = priv->cq;
247	qp_init_attr.cap.max_recv_wr = RDMASNIFF_NUM_RECEIVES;
248	qp_init_attr.cap.max_recv_sge = 1;
249	qp_init_attr.qp_type = IBV_QPT_RAW_PACKET;
250	priv->qp = ibv_create_qp(priv->pd, &qp_init_attr);
251	if (!priv->qp) {
252		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
253			      "Failed to create QP for device %s", handle->opt.device);
254		goto error;
255	}
256
257	memset(&qp_attr, 0, sizeof qp_attr);
258	qp_attr.qp_state = IBV_QPS_INIT;
259	qp_attr.port_num = priv->port_num;
260	if (ibv_modify_qp(priv->qp, &qp_attr, IBV_QP_STATE | IBV_QP_PORT)) {
261		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
262			      "Failed to modify QP to INIT for device %s", handle->opt.device);
263		goto error;
264	}
265
266	memset(&qp_attr, 0, sizeof qp_attr);
267	qp_attr.qp_state = IBV_QPS_RTR;
268	if (ibv_modify_qp(priv->qp, &qp_attr, IBV_QP_STATE)) {
269		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
270			      "Failed to modify QP to RTR for device %s", handle->opt.device);
271		goto error;
272	}
273
274	memset(&flow_attr, 0, sizeof flow_attr);
275	flow_attr.type = IBV_FLOW_ATTR_SNIFFER;
276	flow_attr.size = sizeof flow_attr;
277	flow_attr.port = priv->port_num;
278	priv->flow = ibv_create_flow(priv->qp, &flow_attr);
279	if (!priv->flow) {
280		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
281			      "Failed to create flow for device %s", handle->opt.device);
282		goto error;
283	}
284
285	handle->bufsize = RDMASNIFF_NUM_RECEIVES * RDMASNIFF_RECEIVE_SIZE;
286	handle->buffer = malloc(handle->bufsize);
287	if (!handle->buffer) {
288		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
289			      "Failed to allocate receive buffer for device %s", handle->opt.device);
290		goto error;
291	}
292
293	priv->oneshot_buffer = malloc(RDMASNIFF_RECEIVE_SIZE);
294	if (!priv->oneshot_buffer) {
295		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
296			      "Failed to allocate oneshot buffer for device %s", handle->opt.device);
297		goto error;
298	}
299
300	priv->mr = ibv_reg_mr(priv->pd, handle->buffer, handle->bufsize, IBV_ACCESS_LOCAL_WRITE);
301	if (!priv->mr) {
302		snprintf(handle->errbuf, PCAP_ERRBUF_SIZE,
303			      "Failed to register MR for device %s", handle->opt.device);
304		goto error;
305	}
306
307
308	for (i = 0; i < RDMASNIFF_NUM_RECEIVES; ++i) {
309		rdmasniff_post_recv(handle, i);
310	}
311
312	if (!ibv_query_port(priv->context, priv->port_num, &port_attr) &&
313	    port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) {
314		handle->linktype = DLT_INFINIBAND;
315	} else {
316		handle->linktype = DLT_EN10MB;
317	}
318
319	if (handle->snapshot <= 0 || handle->snapshot > RDMASNIFF_RECEIVE_SIZE)
320		handle->snapshot = RDMASNIFF_RECEIVE_SIZE;
321
322	handle->offset = 0;
323	handle->read_op = rdmasniff_read;
324	handle->stats_op = rdmasniff_stats;
325	handle->cleanup_op = rdmasniff_cleanup;
326	handle->setfilter_op = install_bpf_program;
327	handle->setdirection_op = NULL;
328	handle->set_datalink_op = NULL;
329	handle->getnonblock_op = pcap_getnonblock_fd;
330	handle->setnonblock_op = pcap_setnonblock_fd;
331	handle->oneshot_callback = rdmasniff_oneshot;
332	handle->selectable_fd = priv->channel->fd;
333
334	return 0;
335
336error:
337	if (priv->mr) {
338		ibv_dereg_mr(priv->mr);
339	}
340
341	if (priv->flow) {
342		ibv_destroy_flow(priv->flow);
343	}
344
345	if (priv->qp) {
346		ibv_destroy_qp(priv->qp);
347	}
348
349	if (priv->cq) {
350		ibv_destroy_cq(priv->cq);
351	}
352
353	if (priv->channel) {
354		ibv_destroy_comp_channel(priv->channel);
355	}
356
357	if (priv->pd) {
358		ibv_dealloc_pd(priv->pd);
359	}
360
361	if (priv->context) {
362		ibv_close_device(priv->context);
363	}
364
365	if (priv->oneshot_buffer) {
366		free(priv->oneshot_buffer);
367	}
368
369	return PCAP_ERROR;
370}
371
372pcap_t *
373rdmasniff_create(const char *device, char *ebuf, int *is_ours)
374{
375	struct pcap_rdmasniff *priv;
376	struct ibv_device **dev_list;
377	int numdev;
378	size_t namelen;
379	const char *port;
380	unsigned long port_num;
381	int i;
382	pcap_t *p = NULL;
383
384	*is_ours = 0;
385
386	dev_list = ibv_get_device_list(&numdev);
387	if (!dev_list) {
388		return NULL;
389	}
390	if (!numdev) {
391		ibv_free_device_list(dev_list);
392		return NULL;
393	}
394
395	namelen = strlen(device);
396
397	port = strchr(device, ':');
398	if (port) {
399		port_num = strtoul(port + 1, NULL, 10);
400		if (port_num > 0) {
401			namelen = port - device;
402		} else {
403			port_num = 1;
404		}
405	} else {
406		port_num = 1;
407	}
408
409	for (i = 0; i < numdev; ++i) {
410		if (strlen(dev_list[i]->name) == namelen &&
411		    !strncmp(device, dev_list[i]->name, namelen)) {
412			*is_ours = 1;
413
414			p = PCAP_CREATE_COMMON(ebuf, struct pcap_rdmasniff);
415			if (p) {
416				p->activate_op = rdmasniff_activate;
417				priv = p->priv;
418				priv->rdma_device = dev_list[i];
419				priv->port_num = port_num;
420			}
421			break;
422		}
423	}
424
425	ibv_free_device_list(dev_list);
426	return p;
427}
428
429int
430rdmasniff_findalldevs(pcap_if_list_t *devlistp, char *err_str)
431{
432	struct ibv_device **dev_list;
433	int numdev;
434	int i;
435	int ret = 0;
436
437	dev_list = ibv_get_device_list(&numdev);
438	if (!dev_list) {
439		return 0;
440	}
441
442	for (i = 0; i < numdev; ++i) {
443		/*
444		 * XXX - do the notions of "up", "running", or
445		 * "connected" apply here?
446		 */
447		if (!add_dev(devlistp, dev_list[i]->name, 0, "RDMA sniffer", err_str)) {
448			ret = -1;
449			break;
450		}
451	}
452
453	ibv_free_device_list(dev_list);
454	return ret;
455}
456