iw_cxgbe.h revision 309450 
1/*
2 * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses.  You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 *     Redistribution and use in source and binary forms, with or
11 *     without modification, are permitted provided that the following
12 *     conditions are met:
13 *
14 *      - Redistributions of source code must retain the above
15 *	  copyright notice, this list of conditions and the following
16 *	  disclaimer.
17 *      - Redistributions in binary form must reproduce the above
18 *	  copyright notice, this list of conditions and the following
19 *	  disclaimer in the documentation and/or other materials
20 *	  provided with the distribution.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29 * SOFTWARE.
30 *
31 * $FreeBSD: stable/10/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h 309450 2016-12-03 00:18:38Z jhb $
32 */
33#ifndef __IW_CXGB4_H__
34#define __IW_CXGB4_H__
35
36#include <linux/list.h>
37#include <linux/spinlock.h>
38#include <linux/idr.h>
39#include <linux/completion.h>
40#include <linux/netdevice.h>
41#include <linux/sched.h>
42#include <linux/pci.h>
43#include <linux/dma-mapping.h>
44#include <linux/wait.h>
45#include <linux/kref.h>
46#include <linux/timer.h>
47#include <linux/io.h>
48#include <sys/vmem.h>
49
50#include <asm/byteorder.h>
51
52#include <netinet/in.h>
53#include <netinet/toecore.h>
54
55#include <rdma/ib_verbs.h>
56#include <rdma/iw_cm.h>
57
58#undef prefetch
59
60#include "common/common.h"
61#include "common/t4_msg.h"
62#include "common/t4_regs.h"
63#include "common/t4_tcb.h"
64#include "t4_l2t.h"
65
66#define DRV_NAME "iw_cxgbe"
67#define MOD DRV_NAME ":"
68#define KTR_IW_CXGBE	KTR_SPARE3
69
70extern int c4iw_debug;
71#define PDBG(fmt, args...) \
72do { \
73	if (c4iw_debug) \
74		printf(MOD fmt, ## args); \
75} while (0)
76
77#include "t4.h"
78
79static inline void *cplhdr(struct mbuf *m)
80{
81	return mtod(m, void*);
82}
83
84#define PBL_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.pbl.start)
85#define RQT_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.rq.start)
86
87#define C4IW_ID_TABLE_F_RANDOM 1       /* Pseudo-randomize the id's returned */
88#define C4IW_ID_TABLE_F_EMPTY  2       /* Table is initially empty */
89
90struct c4iw_id_table {
91	u32 flags;
92	u32 start;              /* logical minimal id */
93	u32 last;               /* hint for find */
94	u32 max;
95	spinlock_t lock;
96	unsigned long *table;
97};
98
99struct c4iw_resource {
100	struct c4iw_id_table tpt_table;
101	struct c4iw_id_table qid_table;
102	struct c4iw_id_table pdid_table;
103};
104
105struct c4iw_qid_list {
106	struct list_head entry;
107	u32 qid;
108};
109
110struct c4iw_dev_ucontext {
111	struct list_head qpids;
112	struct list_head cqids;
113	struct mutex lock;
114};
115
116enum c4iw_rdev_flags {
117	T4_FATAL_ERROR = (1<<0),
118};
119
120struct c4iw_stat {
121	u64 total;
122	u64 cur;
123	u64 max;
124	u64 fail;
125};
126
127struct c4iw_stats {
128	struct mutex lock;
129	struct c4iw_stat qid;
130	struct c4iw_stat pd;
131	struct c4iw_stat stag;
132	struct c4iw_stat pbl;
133	struct c4iw_stat rqt;
134};
135
136struct c4iw_rdev {
137	struct adapter *adap;
138	struct c4iw_resource resource;
139	unsigned long qpshift;
140	u32 qpmask;
141	unsigned long cqshift;
142	u32 cqmask;
143	struct c4iw_dev_ucontext uctx;
144	vmem_t          *rqt_arena;
145	vmem_t          *pbl_arena;
146	u32 flags;
147	struct c4iw_stats stats;
148};
149
150static inline int c4iw_fatal_error(struct c4iw_rdev *rdev)
151{
152	return rdev->flags & T4_FATAL_ERROR;
153}
154
155static inline int c4iw_num_stags(struct c4iw_rdev *rdev)
156{
157	return (int)(rdev->adap->vres.stag.size >> 5);
158}
159
160#define C4IW_WR_TO (10*HZ)
161
162struct c4iw_wr_wait {
163	int ret;
164	atomic_t completion;
165};
166
167static inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
168{
169	wr_waitp->ret = 0;
170	atomic_set(&wr_waitp->completion, 0);
171}
172
173static inline void c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret)
174{
175	wr_waitp->ret = ret;
176	atomic_set(&wr_waitp->completion, 1);
177	wakeup(wr_waitp);
178}
179
180static inline int
181c4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp,
182    u32 hwtid, u32 qpid, const char *func)
183{
184	struct adapter *sc = rdev->adap;
185	unsigned to = C4IW_WR_TO;
186
187	while (!atomic_read(&wr_waitp->completion)) {
188                tsleep(wr_waitp, 0, "c4iw_wait", to);
189                if (SIGPENDING(curthread)) {
190			printf("%s - Device %s not responding - "
191			    "tid %u qpid %u\n", func,
192			    device_get_nameunit(sc->dev), hwtid, qpid);
193                        if (c4iw_fatal_error(rdev)) {
194                                wr_waitp->ret = -EIO;
195                                break;
196                        }
197                        to = to << 2;
198                }
199        }
200	if (wr_waitp->ret)
201		CTR4(KTR_IW_CXGBE, "%s: FW reply %d tid %u qpid %u",
202		    device_get_nameunit(sc->dev), wr_waitp->ret, hwtid, qpid);
203	return (wr_waitp->ret);
204}
205
206struct c4iw_dev {
207	struct ib_device ibdev;
208	struct c4iw_rdev rdev;
209	u32 device_cap_flags;
210	struct idr cqidr;
211	struct idr qpidr;
212	struct idr mmidr;
213	spinlock_t lock;
214	struct dentry *debugfs_root;
215};
216
217static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)
218{
219	return container_of(ibdev, struct c4iw_dev, ibdev);
220}
221
222static inline struct c4iw_dev *rdev_to_c4iw_dev(struct c4iw_rdev *rdev)
223{
224	return container_of(rdev, struct c4iw_dev, rdev);
225}
226
227static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid)
228{
229	return idr_find(&rhp->cqidr, cqid);
230}
231
232static inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid)
233{
234	return idr_find(&rhp->qpidr, qpid);
235}
236
237static inline struct c4iw_mr *get_mhp(struct c4iw_dev *rhp, u32 mmid)
238{
239	return idr_find(&rhp->mmidr, mmid);
240}
241
242static inline int _insert_handle(struct c4iw_dev *rhp, struct idr *idr,
243				 void *handle, u32 id, int lock)
244{
245	int ret;
246	int newid;
247
248	do {
249		if (!idr_pre_get(idr, lock ? GFP_KERNEL : GFP_ATOMIC))
250			return -ENOMEM;
251		if (lock)
252			spin_lock_irq(&rhp->lock);
253		ret = idr_get_new_above(idr, handle, id, &newid);
254		BUG_ON(!ret && newid != id);
255		if (lock)
256			spin_unlock_irq(&rhp->lock);
257	} while (ret == -EAGAIN);
258
259	return ret;
260}
261
262static inline int insert_handle(struct c4iw_dev *rhp, struct idr *idr,
263				void *handle, u32 id)
264{
265	return _insert_handle(rhp, idr, handle, id, 1);
266}
267
268static inline int insert_handle_nolock(struct c4iw_dev *rhp, struct idr *idr,
269				       void *handle, u32 id)
270{
271	return _insert_handle(rhp, idr, handle, id, 0);
272}
273
274static inline void _remove_handle(struct c4iw_dev *rhp, struct idr *idr,
275				   u32 id, int lock)
276{
277	if (lock)
278		spin_lock_irq(&rhp->lock);
279	idr_remove(idr, id);
280	if (lock)
281		spin_unlock_irq(&rhp->lock);
282}
283
284static inline void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id)
285{
286	_remove_handle(rhp, idr, id, 1);
287}
288
289static inline void remove_handle_nolock(struct c4iw_dev *rhp,
290					 struct idr *idr, u32 id)
291{
292	_remove_handle(rhp, idr, id, 0);
293}
294
295struct c4iw_pd {
296	struct ib_pd ibpd;
297	u32 pdid;
298	struct c4iw_dev *rhp;
299};
300
301static inline struct c4iw_pd *to_c4iw_pd(struct ib_pd *ibpd)
302{
303	return container_of(ibpd, struct c4iw_pd, ibpd);
304}
305
306struct tpt_attributes {
307	u64 len;
308	u64 va_fbo;
309	enum fw_ri_mem_perms perms;
310	u32 stag;
311	u32 pdid;
312	u32 qpid;
313	u32 pbl_addr;
314	u32 pbl_size;
315	u32 state:1;
316	u32 type:2;
317	u32 rsvd:1;
318	u32 remote_invaliate_disable:1;
319	u32 zbva:1;
320	u32 mw_bind_enable:1;
321	u32 page_size:5;
322};
323
324struct c4iw_mr {
325	struct ib_mr ibmr;
326	struct ib_umem *umem;
327	struct c4iw_dev *rhp;
328	u64 kva;
329	struct tpt_attributes attr;
330};
331
332static inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr)
333{
334	return container_of(ibmr, struct c4iw_mr, ibmr);
335}
336
337struct c4iw_mw {
338	struct ib_mw ibmw;
339	struct c4iw_dev *rhp;
340	u64 kva;
341	struct tpt_attributes attr;
342};
343
344static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw)
345{
346	return container_of(ibmw, struct c4iw_mw, ibmw);
347}
348
349struct c4iw_fr_page_list {
350	struct ib_fast_reg_page_list ibpl;
351	DECLARE_PCI_UNMAP_ADDR(mapping);
352	dma_addr_t dma_addr;
353	struct c4iw_dev *dev;
354	int size;
355};
356
357static inline struct c4iw_fr_page_list *to_c4iw_fr_page_list(
358					struct ib_fast_reg_page_list *ibpl)
359{
360	return container_of(ibpl, struct c4iw_fr_page_list, ibpl);
361}
362
363struct c4iw_cq {
364	struct ib_cq ibcq;
365	struct c4iw_dev *rhp;
366	struct t4_cq cq;
367	spinlock_t lock;
368	spinlock_t comp_handler_lock;
369	atomic_t refcnt;
370	wait_queue_head_t wait;
371};
372
373static inline struct c4iw_cq *to_c4iw_cq(struct ib_cq *ibcq)
374{
375	return container_of(ibcq, struct c4iw_cq, ibcq);
376}
377
378struct c4iw_mpa_attributes {
379	u8 initiator;
380	u8 recv_marker_enabled;
381	u8 xmit_marker_enabled;
382	u8 crc_enabled;
383	u8 enhanced_rdma_conn;
384	u8 version;
385	u8 p2p_type;
386};
387
388struct c4iw_qp_attributes {
389	u32 scq;
390	u32 rcq;
391	u32 sq_num_entries;
392	u32 rq_num_entries;
393	u32 sq_max_sges;
394	u32 sq_max_sges_rdma_write;
395	u32 rq_max_sges;
396	u32 state;
397	u8 enable_rdma_read;
398	u8 enable_rdma_write;
399	u8 enable_bind;
400	u8 enable_mmid0_fastreg;
401	u32 max_ord;
402	u32 max_ird;
403	u32 pd;
404	u32 next_state;
405	char terminate_buffer[52];
406	u32 terminate_msg_len;
407	u8 is_terminate_local;
408	struct c4iw_mpa_attributes mpa_attr;
409	struct c4iw_ep *llp_stream_handle;
410	u8 layer_etype;
411	u8 ecode;
412	u16 sq_db_inc;
413	u16 rq_db_inc;
414};
415
416struct c4iw_qp {
417	struct ib_qp ibqp;
418	struct c4iw_dev *rhp;
419	struct c4iw_ep *ep;
420	struct c4iw_qp_attributes attr;
421	struct t4_wq wq;
422	spinlock_t lock;
423	struct mutex mutex;
424	atomic_t refcnt;
425	wait_queue_head_t wait;
426	struct timer_list timer;
427	int sq_sig_all;
428};
429
430static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)
431{
432	return container_of(ibqp, struct c4iw_qp, ibqp);
433}
434
435struct c4iw_ucontext {
436	struct ib_ucontext ibucontext;
437	struct c4iw_dev_ucontext uctx;
438	u32 key;
439	spinlock_t mmap_lock;
440	struct list_head mmaps;
441};
442
443static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
444{
445	return container_of(c, struct c4iw_ucontext, ibucontext);
446}
447
448struct c4iw_mm_entry {
449	struct list_head entry;
450	u64 addr;
451	u32 key;
452	unsigned len;
453};
454
455static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext,
456						u32 key, unsigned len)
457{
458	struct list_head *pos, *nxt;
459	struct c4iw_mm_entry *mm;
460
461	spin_lock(&ucontext->mmap_lock);
462	list_for_each_safe(pos, nxt, &ucontext->mmaps) {
463
464		mm = list_entry(pos, struct c4iw_mm_entry, entry);
465		if (mm->key == key && mm->len == len) {
466			list_del_init(&mm->entry);
467			spin_unlock(&ucontext->mmap_lock);
468			CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d",
469			     __func__, key, (unsigned long long) mm->addr,
470			     mm->len);
471			return mm;
472		}
473	}
474	spin_unlock(&ucontext->mmap_lock);
475	return NULL;
476}
477
478static inline void insert_mmap(struct c4iw_ucontext *ucontext,
479			       struct c4iw_mm_entry *mm)
480{
481	spin_lock(&ucontext->mmap_lock);
482	CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d", __func__, mm->key,
483	    (unsigned long long) mm->addr, mm->len);
484	list_add_tail(&mm->entry, &ucontext->mmaps);
485	spin_unlock(&ucontext->mmap_lock);
486}
487
488enum c4iw_qp_attr_mask {
489	C4IW_QP_ATTR_NEXT_STATE = 1 << 0,
490	C4IW_QP_ATTR_SQ_DB = 1<<1,
491	C4IW_QP_ATTR_RQ_DB = 1<<2,
492	C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7,
493	C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8,
494	C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9,
495	C4IW_QP_ATTR_MAX_ORD = 1 << 11,
496	C4IW_QP_ATTR_MAX_IRD = 1 << 12,
497	C4IW_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22,
498	C4IW_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23,
499	C4IW_QP_ATTR_MPA_ATTR = 1 << 24,
500	C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25,
501	C4IW_QP_ATTR_VALID_MODIFY = (C4IW_QP_ATTR_ENABLE_RDMA_READ |
502				     C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
503				     C4IW_QP_ATTR_MAX_ORD |
504				     C4IW_QP_ATTR_MAX_IRD |
505				     C4IW_QP_ATTR_LLP_STREAM_HANDLE |
506				     C4IW_QP_ATTR_STREAM_MSG_BUFFER |
507				     C4IW_QP_ATTR_MPA_ATTR |
508				     C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE)
509};
510
511int c4iw_modify_qp(struct c4iw_dev *rhp,
512				struct c4iw_qp *qhp,
513				enum c4iw_qp_attr_mask mask,
514				struct c4iw_qp_attributes *attrs,
515				int internal);
516
517enum c4iw_qp_state {
518	C4IW_QP_STATE_IDLE,
519	C4IW_QP_STATE_RTS,
520	C4IW_QP_STATE_ERROR,
521	C4IW_QP_STATE_TERMINATE,
522	C4IW_QP_STATE_CLOSING,
523	C4IW_QP_STATE_TOT
524};
525
526static inline int c4iw_convert_state(enum ib_qp_state ib_state)
527{
528	switch (ib_state) {
529	case IB_QPS_RESET:
530	case IB_QPS_INIT:
531		return C4IW_QP_STATE_IDLE;
532	case IB_QPS_RTS:
533		return C4IW_QP_STATE_RTS;
534	case IB_QPS_SQD:
535		return C4IW_QP_STATE_CLOSING;
536	case IB_QPS_SQE:
537		return C4IW_QP_STATE_TERMINATE;
538	case IB_QPS_ERR:
539		return C4IW_QP_STATE_ERROR;
540	default:
541		return -1;
542	}
543}
544
545static inline int to_ib_qp_state(int c4iw_qp_state)
546{
547	switch (c4iw_qp_state) {
548	case C4IW_QP_STATE_IDLE:
549		return IB_QPS_INIT;
550	case C4IW_QP_STATE_RTS:
551		return IB_QPS_RTS;
552	case C4IW_QP_STATE_CLOSING:
553		return IB_QPS_SQD;
554	case C4IW_QP_STATE_TERMINATE:
555		return IB_QPS_SQE;
556	case C4IW_QP_STATE_ERROR:
557		return IB_QPS_ERR;
558	}
559	return IB_QPS_ERR;
560}
561
562static inline u32 c4iw_ib_to_tpt_access(int a)
563{
564	return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
565	       (a & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0) |
566	       (a & IB_ACCESS_LOCAL_WRITE ? FW_RI_MEM_ACCESS_LOCAL_WRITE : 0) |
567	       FW_RI_MEM_ACCESS_LOCAL_READ;
568}
569
570static inline u32 c4iw_ib_to_tpt_bind_access(int acc)
571{
572	return (acc & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
573	       (acc & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0);
574}
575
576enum c4iw_mmid_state {
577	C4IW_STAG_STATE_VALID,
578	C4IW_STAG_STATE_INVALID
579};
580
581#define C4IW_NODE_DESC "iw_cxgbe Chelsio Communications"
582
583#define MPA_KEY_REQ "MPA ID Req Frame"
584#define MPA_KEY_REP "MPA ID Rep Frame"
585
586#define MPA_MAX_PRIVATE_DATA	256
587#define MPA_ENHANCED_RDMA_CONN	0x10
588#define MPA_REJECT		0x20
589#define MPA_CRC			0x40
590#define MPA_MARKERS		0x80
591#define MPA_FLAGS_MASK		0xE0
592
593#define MPA_V2_PEER2PEER_MODEL          0x8000
594#define MPA_V2_ZERO_LEN_FPDU_RTR        0x4000
595#define MPA_V2_RDMA_WRITE_RTR           0x8000
596#define MPA_V2_RDMA_READ_RTR            0x4000
597#define MPA_V2_IRD_ORD_MASK             0x3FFF
598
599#define c4iw_put_ep(ep) { \
600	CTR4(KTR_IW_CXGBE, "put_ep (%s:%u) ep %p, refcnt %d", \
601	     __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \
602	WARN_ON(atomic_read(&(ep)->kref.refcount) < 1); \
603        kref_put(&((ep)->kref), _c4iw_free_ep); \
604}
605
606#define c4iw_get_ep(ep) { \
607	CTR4(KTR_IW_CXGBE, "get_ep (%s:%u) ep %p, refcnt %d", \
608	      __func__, __LINE__, ep, atomic_read(&(ep)->kref.refcount)); \
609        kref_get(&((ep)->kref));  \
610}
611
612void _c4iw_free_ep(struct kref *kref);
613
614struct mpa_message {
615	u8 key[16];
616	u8 flags;
617	u8 revision;
618	__be16 private_data_size;
619	u8 private_data[0];
620};
621
622struct mpa_v2_conn_params {
623	__be16 ird;
624	__be16 ord;
625};
626
627struct terminate_message {
628	u8 layer_etype;
629	u8 ecode;
630	__be16 hdrct_rsvd;
631	u8 len_hdrs[0];
632};
633
634#define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)
635
636enum c4iw_layers_types {
637	LAYER_RDMAP		= 0x00,
638	LAYER_DDP		= 0x10,
639	LAYER_MPA		= 0x20,
640	RDMAP_LOCAL_CATA	= 0x00,
641	RDMAP_REMOTE_PROT	= 0x01,
642	RDMAP_REMOTE_OP		= 0x02,
643	DDP_LOCAL_CATA		= 0x00,
644	DDP_TAGGED_ERR		= 0x01,
645	DDP_UNTAGGED_ERR	= 0x02,
646	DDP_LLP			= 0x03
647};
648
649enum c4iw_rdma_ecodes {
650	RDMAP_INV_STAG		= 0x00,
651	RDMAP_BASE_BOUNDS	= 0x01,
652	RDMAP_ACC_VIOL		= 0x02,
653	RDMAP_STAG_NOT_ASSOC	= 0x03,
654	RDMAP_TO_WRAP		= 0x04,
655	RDMAP_INV_VERS		= 0x05,
656	RDMAP_INV_OPCODE	= 0x06,
657	RDMAP_STREAM_CATA	= 0x07,
658	RDMAP_GLOBAL_CATA	= 0x08,
659	RDMAP_CANT_INV_STAG	= 0x09,
660	RDMAP_UNSPECIFIED	= 0xff
661};
662
663enum c4iw_ddp_ecodes {
664	DDPT_INV_STAG		= 0x00,
665	DDPT_BASE_BOUNDS	= 0x01,
666	DDPT_STAG_NOT_ASSOC	= 0x02,
667	DDPT_TO_WRAP		= 0x03,
668	DDPT_INV_VERS		= 0x04,
669	DDPU_INV_QN		= 0x01,
670	DDPU_INV_MSN_NOBUF	= 0x02,
671	DDPU_INV_MSN_RANGE	= 0x03,
672	DDPU_INV_MO		= 0x04,
673	DDPU_MSG_TOOBIG		= 0x05,
674	DDPU_INV_VERS		= 0x06
675};
676
677enum c4iw_mpa_ecodes {
678	MPA_CRC_ERR		= 0x02,
679	MPA_MARKER_ERR		= 0x03,
680	MPA_LOCAL_CATA          = 0x05,
681	MPA_INSUFF_IRD          = 0x06,
682	MPA_NOMATCH_RTR         = 0x07,
683};
684
685enum c4iw_ep_state {
686	IDLE = 0,
687	LISTEN,
688	CONNECTING,
689	MPA_REQ_WAIT,
690	MPA_REQ_SENT,
691	MPA_REQ_RCVD,
692	MPA_REP_SENT,
693	FPDU_MODE,
694	ABORTING,
695	CLOSING,
696	MORIBUND,
697	DEAD,
698};
699
700enum c4iw_ep_flags {
701	PEER_ABORT_IN_PROGRESS	= 0,
702	ABORT_REQ_IN_PROGRESS	= 1,
703	RELEASE_RESOURCES	= 2,
704	CLOSE_SENT		= 3,
705	TIMEOUT                 = 4,
706	QP_REFERENCED		= 5
707};
708
709enum c4iw_ep_history {
710        ACT_OPEN_REQ            = 0,
711        ACT_OFLD_CONN           = 1,
712        ACT_OPEN_RPL            = 2,
713        ACT_ESTAB               = 3,
714        PASS_ACCEPT_REQ         = 4,
715        PASS_ESTAB              = 5,
716        ABORT_UPCALL            = 6,
717        ESTAB_UPCALL            = 7,
718        CLOSE_UPCALL            = 8,
719        ULP_ACCEPT              = 9,
720        ULP_REJECT              = 10,
721        TIMEDOUT                = 11,
722        PEER_ABORT              = 12,
723        PEER_CLOSE              = 13,
724        CONNREQ_UPCALL          = 14,
725        ABORT_CONN              = 15,
726        DISCONN_UPCALL          = 16,
727        EP_DISC_CLOSE           = 17,
728        EP_DISC_ABORT           = 18,
729        CONN_RPL_UPCALL         = 19,
730        ACT_RETRY_NOMEM         = 20,
731        ACT_RETRY_INUSE         = 21,
732        CLOSE_CON_RPL           = 22,
733        EP_DISC_FAIL            = 24,
734        QP_REFED                = 25,
735        QP_DEREFED              = 26,
736        CM_ID_REFED             = 27,
737        CM_ID_DEREFED           = 28
738};
739
740struct c4iw_ep_common {
741	TAILQ_ENTRY(c4iw_ep_common) entry;	/* Work queue attachment */
742	struct iw_cm_id *cm_id;
743	struct c4iw_qp *qp;
744	struct c4iw_dev *dev;
745	enum c4iw_ep_state state;
746	struct kref kref;
747	struct mutex mutex;
748	struct sockaddr_in local_addr;
749	struct sockaddr_in remote_addr;
750	struct c4iw_wr_wait wr_wait;
751	unsigned long flags;
752	unsigned long history;
753        int rpl_err;
754        int rpl_done;
755        struct thread *thread;
756        struct socket *so;
757	struct mutex so_mutex;
758};
759
760struct c4iw_listen_ep {
761	struct c4iw_ep_common com;
762	unsigned int stid;
763	int backlog;
764};
765
766struct c4iw_ep {
767	struct c4iw_ep_common com;
768	struct c4iw_ep *parent_ep;
769	struct timer_list timer;
770	struct list_head entry;
771	unsigned int atid;
772	u32 hwtid;
773	u32 snd_seq;
774	u32 rcv_seq;
775	struct l2t_entry *l2t;
776	struct dst_entry *dst;
777	struct c4iw_mpa_attributes mpa_attr;
778	u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA];
779	unsigned int mpa_pkt_len;
780	u32 ird;
781	u32 ord;
782	u32 smac_idx;
783	u32 tx_chan;
784	u32 mtu;
785	u16 mss;
786	u16 emss;
787	u16 plen;
788	u16 rss_qid;
789	u16 txq_idx;
790	u16 ctrlq_idx;
791	u8 tos;
792	u8 retry_with_mpa_v1;
793	u8 tried_with_mpa_v1;
794};
795
796static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id)
797{
798	return cm_id->provider_data;
799}
800
801static inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id)
802{
803	return cm_id->provider_data;
804}
805
806static inline int compute_wscale(int win)
807{
808	int wscale = 0;
809
810	while (wscale < 14 && (65535<<wscale) < win)
811		wscale++;
812	return wscale;
813}
814
815u32 c4iw_id_alloc(struct c4iw_id_table *alloc);
816void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj);
817int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,
818			u32 reserved, u32 flags);
819void c4iw_id_table_free(struct c4iw_id_table *alloc);
820
821typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct mbuf *m);
822
823int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
824		     struct l2t_entry *l2t);
825u32 c4iw_get_resource(struct c4iw_id_table *id_table);
826void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry);
827int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt, u32 nr_pdid);
828int c4iw_init_ctrl_qp(struct c4iw_rdev *rdev);
829int c4iw_pblpool_create(struct c4iw_rdev *rdev);
830int c4iw_rqtpool_create(struct c4iw_rdev *rdev);
831void c4iw_pblpool_destroy(struct c4iw_rdev *rdev);
832void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev);
833void c4iw_destroy_resource(struct c4iw_resource *rscp);
834int c4iw_destroy_ctrl_qp(struct c4iw_rdev *rdev);
835int c4iw_register_device(struct c4iw_dev *dev);
836void c4iw_unregister_device(struct c4iw_dev *dev);
837int __init c4iw_cm_init(void);
838void __exit c4iw_cm_term(void);
839void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,
840			       struct c4iw_dev_ucontext *uctx);
841void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev,
842			    struct c4iw_dev_ucontext *uctx);
843int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
844int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
845		      struct ib_send_wr **bad_wr);
846int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
847		      struct ib_recv_wr **bad_wr);
848int c4iw_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
849		 struct ib_mw_bind *mw_bind);
850int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
851int c4iw_create_listen_ep(struct iw_cm_id *cm_id, int backlog);
852void c4iw_destroy_listen_ep(struct iw_cm_id *cm_id);
853int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
854int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
855void c4iw_qp_add_ref(struct ib_qp *qp);
856void c4iw_qp_rem_ref(struct ib_qp *qp);
857void c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list *page_list);
858struct ib_fast_reg_page_list *c4iw_alloc_fastreg_pbl(
859					struct ib_device *device,
860					int page_list_len);
861struct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth);
862int c4iw_dealloc_mw(struct ib_mw *mw);
863struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd);
864struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64
865    virt, int acc, struct ib_udata *udata, int mr_id);
866struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
867struct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd,
868					struct ib_phys_buf *buffer_list,
869					int num_phys_buf,
870					int acc,
871					u64 *iova_start);
872int c4iw_reregister_phys_mem(struct ib_mr *mr,
873				     int mr_rereg_mask,
874				     struct ib_pd *pd,
875				     struct ib_phys_buf *buffer_list,
876				     int num_phys_buf,
877				     int acc, u64 *iova_start);
878int c4iw_dereg_mr(struct ib_mr *ib_mr);
879int c4iw_destroy_cq(struct ib_cq *ib_cq);
880struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, int entries,
881					int vector,
882					struct ib_ucontext *ib_context,
883					struct ib_udata *udata);
884int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
885int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
886int c4iw_destroy_qp(struct ib_qp *ib_qp);
887struct ib_qp *c4iw_create_qp(struct ib_pd *pd,
888			     struct ib_qp_init_attr *attrs,
889			     struct ib_udata *udata);
890int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
891				 int attr_mask, struct ib_udata *udata);
892int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
893		     int attr_mask, struct ib_qp_init_attr *init_attr);
894struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn);
895u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size);
896void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
897u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size);
898void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
899int c4iw_ofld_send(struct c4iw_rdev *rdev, struct mbuf *m);
900void c4iw_flush_hw_cq(struct t4_cq *cq);
901void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
902void c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
903int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
904int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);
905int c4iw_flush_sq(struct t4_wq *wq, struct t4_cq *cq, int count);
906int c4iw_ev_handler(struct sge_iq *, const struct rsp_ctrl *);
907u16 c4iw_rqes_posted(struct c4iw_qp *qhp);
908int c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe);
909u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
910void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,
911		struct c4iw_dev_ucontext *uctx);
912u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
913void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
914		struct c4iw_dev_ucontext *uctx);
915void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe);
916void process_newconn(struct iw_cm_id *parent_cm_id,
917		struct socket *child_so);
918
919extern struct cxgb4_client t4c_client;
920extern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];
921extern int c4iw_max_read_depth;
922
923#if defined(__i386__) || defined(__amd64__)
924#define L1_CACHE_BYTES 128
925#else
926#define L1_CACHE_BYTES 32
927#endif
928
929static inline
930int idr_for_each(struct idr *idp,
931                 int (*fn)(int id, void *p, void *data), void *data)
932{
933        int n, id, max, error = 0;
934        struct idr_layer *p;
935        struct idr_layer *pa[MAX_LEVEL];
936        struct idr_layer **paa = &pa[0];
937
938        n = idp->layers * IDR_BITS;
939        p = idp->top;
940        max = 1 << n;
941
942        id = 0;
943        while (id < max) {
944                while (n > 0 && p) {
945                        n -= IDR_BITS;
946                        *paa++ = p;
947                        p = p->ary[(id >> n) & IDR_MASK];
948                }
949
950                if (p) {
951                        error = fn(id, (void *)p, data);
952                        if (error)
953                                break;
954                }
955
956                id += 1 << n;
957                while (n < fls(id)) {
958                        n += IDR_BITS;
959                        p = *--paa;
960                }
961        }
962
963        return error;
964}
965
966void your_reg_device(struct c4iw_dev *dev);
967
968#define SGE_CTRLQ_NUM	0
969
970#endif
971