1/*-
2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
3 *
4 * Copyright (c) 2017 Mellanox Technologies Ltd.  All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses.  You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 *     Redistribution and use in source and binary forms, with or
13 *     without modification, are permitted provided that the following
14 *     conditions are met:
15 *
16 *      - Redistributions of source code must retain the above
17 *        copyright notice, this list of conditions and the following
18 *        disclaimer.
19 *
20 *      - Redistributions in binary form must reproduce the above
21 *        copyright notice, this list of conditions and the following
22 *        disclaimer in the documentation and/or other materials
23 *        provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#include <sys/cdefs.h>
36#include <linux/module.h>
37#include <linux/err.h>
38#include <linux/slab.h>
39
40#include <rdma/ib_verbs.h>
41
42#define	IB_CQ_POLL_MAX	16
43/* maximum number of completions per poll loop */
44#define	IB_CQ_POLL_BUDGET 65536
45#define	IB_CQ_POLL_FLAGS (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)
46
47static void
48ib_cq_poll_work(struct work_struct *work)
49{
50	struct ib_wc ib_wc[IB_CQ_POLL_MAX];
51	struct ib_cq *cq = container_of(work, struct ib_cq, work);
52	int total = 0;
53	int i;
54	int n;
55
56	while (1) {
57		n = ib_poll_cq(cq, IB_CQ_POLL_MAX, ib_wc);
58		for (i = 0; i < n; i++) {
59			struct ib_wc *wc = ib_wc + i;
60
61			if (wc->wr_cqe != NULL)
62				wc->wr_cqe->done(cq, wc);
63		}
64
65		if (n != IB_CQ_POLL_MAX) {
66			if (ib_req_notify_cq(cq, IB_CQ_POLL_FLAGS) > 0)
67				break;
68			else
69				return;
70		}
71		total += n;
72		if (total >= IB_CQ_POLL_BUDGET)
73			break;
74	}
75
76	/* give other work structs a chance */
77	queue_work(ib_comp_wq, &cq->work);
78}
79
80static void
81ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
82{
83	queue_work(ib_comp_wq, &cq->work);
84}
85
86struct ib_cq *
87__ib_alloc_cq_user(struct ib_device *dev, void *private,
88		   int nr_cqe, int comp_vector,
89		   enum ib_poll_context poll_ctx,
90		   const char *caller, struct ib_udata *udata)
91{
92	struct ib_cq_init_attr cq_attr = {
93		.cqe = nr_cqe,
94		.comp_vector = comp_vector,
95	};
96	struct ib_cq *cq;
97	int ret;
98
99	/*
100	 * Check for invalid parameters early on to avoid
101	 * extra error handling code:
102	 */
103	switch (poll_ctx) {
104	case IB_POLL_DIRECT:
105	case IB_POLL_SOFTIRQ:
106	case IB_POLL_WORKQUEUE:
107		break;
108	default:
109		return (ERR_PTR(-EINVAL));
110	}
111
112	cq = rdma_zalloc_drv_obj(dev, ib_cq);
113	if (!cq)
114		return ERR_PTR(-ENOMEM);
115
116	cq->device = dev;
117	cq->cq_context = private;
118	cq->poll_ctx = poll_ctx;
119	atomic_set(&cq->usecnt, 0);
120
121	ret = dev->create_cq(cq, &cq_attr, NULL);
122	if (ret)
123		goto out_free_cq;
124
125	switch (poll_ctx) {
126	case IB_POLL_DIRECT:
127		cq->comp_handler = NULL;	/* no hardware completions */
128		break;
129	case IB_POLL_SOFTIRQ:
130	case IB_POLL_WORKQUEUE:
131		cq->comp_handler = ib_cq_completion_workqueue;
132		INIT_WORK(&cq->work, ib_cq_poll_work);
133		ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
134		break;
135	default:
136		break;
137	}
138	return (cq);
139
140out_free_cq:
141	kfree(cq);
142	return (ERR_PTR(ret));
143}
144EXPORT_SYMBOL(__ib_alloc_cq_user);
145
146void
147ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
148{
149
150	if (WARN_ON_ONCE(atomic_read(&cq->usecnt) != 0))
151		return;
152
153	switch (cq->poll_ctx) {
154	case IB_POLL_DIRECT:
155		break;
156	case IB_POLL_SOFTIRQ:
157	case IB_POLL_WORKQUEUE:
158		flush_work(&cq->work);
159		break;
160	default:
161		break;
162	}
163
164	cq->device->destroy_cq(cq, udata);
165	kfree(cq);
166}
167EXPORT_SYMBOL(ib_free_cq_user);
168