1/*-
2 * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD$");
28
29#include "efsys.h"
30#include "efx.h"
31#include "efx_types.h"
32#include "efx_regs.h"
33#include "efx_impl.h"
34
35#if EFSYS_OPT_QSTATS
36#define	EFX_TX_QSTAT_INCR(_etp, _stat)					\
37	do {								\
38		(_etp)->et_stat[_stat]++;				\
39	_NOTE(CONSTANTCONDITION)					\
40	} while (B_FALSE)
41#else
42#define	EFX_TX_QSTAT_INCR(_etp, _stat)
43#endif
44
45	__checkReturn	int
46efx_tx_init(
47	__in		efx_nic_t *enp)
48{
49	efx_oword_t oword;
50	int rc;
51
52	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
53	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
54
55	if (!(enp->en_mod_flags & EFX_MOD_EV)) {
56		rc = EINVAL;
57		goto fail1;
58	}
59
60	if (enp->en_mod_flags & EFX_MOD_TX) {
61		rc = EINVAL;
62		goto fail2;
63	}
64
65	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
66
67	/*
68	 * Disable the timer-based TX DMA backoff and allow TX DMA to be
69	 * controlled by the RX FIFO fill level (although always allow a
70	 * minimal trickle).
71	 */
72	EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
73	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
74	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
75	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
76	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
77	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
78	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
79	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
80
81	/*
82	 * Filter all packets less than 14 bytes to avoid parsing
83	 * errors.
84	 */
85	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
86	EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
87
88	/*
89	 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
90	 * descriptors (which is bad).
91	 */
92	EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
93	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
94	EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
95
96	enp->en_mod_flags |= EFX_MOD_TX;
97	return (0);
98
99fail2:
100	EFSYS_PROBE(fail2);
101fail1:
102	EFSYS_PROBE1(fail1, int, rc);
103
104	return (rc);
105}
106
107#if EFSYS_OPT_FILTER
108extern	__checkReturn	int
109efx_tx_filter_insert(
110	__in		efx_txq_t *etp,
111	__inout		efx_filter_spec_t *spec)
112{
113	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
114	EFSYS_ASSERT3P(spec, !=, NULL);
115
116	spec->efs_dmaq_id = (uint16_t)etp->et_index;
117	return efx_filter_insert_filter(etp->et_enp, spec, B_FALSE);
118}
119#endif
120
121#if EFSYS_OPT_FILTER
122extern	__checkReturn	int
123efx_tx_filter_remove(
124	__in		efx_txq_t *etp,
125	__inout		efx_filter_spec_t *spec)
126{
127	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
128	EFSYS_ASSERT3P(spec, !=, NULL);
129
130	spec->efs_dmaq_id = (uint16_t)etp->et_index;
131	return efx_filter_remove_filter(etp->et_enp, spec);
132}
133#endif
134
135#define	EFX_TX_DESC(_etp, _addr, _size, _eop, _added)			\
136	do {								\
137		unsigned int id;					\
138		size_t offset;						\
139		efx_qword_t qword;					\
140									\
141		id = (_added)++ & (_etp)->et_mask;			\
142		offset = id * sizeof (efx_qword_t);			\
143									\
144		EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index,	\
145		    unsigned int, id, efsys_dma_addr_t, (_addr),	\
146		    size_t, (_size), boolean_t, (_eop));		\
147									\
148		EFX_POPULATE_QWORD_4(qword,				\
149		    FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1,			\
150		    FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size),	\
151		    FSF_AZ_TX_KER_BUF_ADDR_DW0,				\
152		    (uint32_t)((_addr) & 0xffffffff),			\
153		    FSF_AZ_TX_KER_BUF_ADDR_DW1,				\
154		    (uint32_t)((_addr) >> 32));				\
155		EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword);	\
156									\
157		_NOTE(CONSTANTCONDITION)				\
158	} while (B_FALSE)
159
160	__checkReturn	int
161efx_tx_qpost(
162	__in		efx_txq_t *etp,
163	__in_ecount(n)	efx_buffer_t *eb,
164	__in		unsigned int n,
165	__in		unsigned int completed,
166	__inout		unsigned int *addedp)
167{
168	unsigned int added = *addedp;
169	unsigned int i;
170	int rc = ENOSPC;
171
172	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
173
174	if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
175		goto fail1;
176
177	for (i = 0; i < n; i++) {
178		efx_buffer_t *ebp = &eb[i];
179		efsys_dma_addr_t start = ebp->eb_addr;
180		size_t size = ebp->eb_size;
181		efsys_dma_addr_t end = start + size;
182
183		/* Fragments must not span 4k boundaries. */
184		EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
185
186		EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
187	}
188
189	EFX_TX_QSTAT_INCR(etp, TX_POST);
190
191	*addedp = added;
192	return (0);
193
194fail1:
195	EFSYS_PROBE1(fail1, int, rc);
196
197	return (rc);
198}
199
200		void
201efx_tx_qpush(
202	__in	efx_txq_t *etp,
203	__in	unsigned int added)
204{
205	efx_nic_t *enp = etp->et_enp;
206	uint32_t wptr;
207	efx_dword_t dword;
208	efx_oword_t oword;
209
210	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
211
212	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
213	EFSYS_PIO_WRITE_BARRIER();
214
215	/* Push the populated descriptors out */
216	wptr = added & etp->et_mask;
217
218	EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
219
220	/* Only write the third DWORD */
221	EFX_POPULATE_DWORD_1(dword,
222	    EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
223	EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
224			    etp->et_index, &dword, B_FALSE);
225}
226
227		void
228efx_tx_qflush(
229	__in	efx_txq_t *etp)
230{
231	efx_nic_t *enp = etp->et_enp;
232	efx_oword_t oword;
233	uint32_t label;
234
235	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
236
237	label = etp->et_index;
238
239	/* Flush the queue */
240	EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
241	    FRF_AZ_TX_FLUSH_DESCQ, label);
242	EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
243}
244
245		void
246efx_tx_qenable(
247	__in	efx_txq_t *etp)
248{
249	efx_nic_t *enp = etp->et_enp;
250	efx_oword_t oword;
251
252	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
253
254	EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
255			    etp->et_index, &oword);
256
257	EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
258	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
259	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
260	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
261	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
262
263	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
264	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
265	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
266
267	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
268			    etp->et_index, &oword);
269}
270
271	__checkReturn	int
272efx_tx_qcreate(
273	__in		efx_nic_t *enp,
274	__in		unsigned int index,
275	__in		unsigned int label,
276	__in		efsys_mem_t *esmp,
277	__in		size_t n,
278	__in		uint32_t id,
279	__in		uint16_t flags,
280	__in		efx_evq_t *eep,
281	__deref_out	efx_txq_t **etpp)
282{
283	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
284	efx_txq_t *etp;
285	efx_oword_t oword;
286	uint32_t size;
287	int rc;
288
289	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
291
292	EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
293	/*	EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);*/
294	EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
295
296	if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) {
297		rc = EINVAL;
298		goto fail1;
299	}
300	if (index >= encp->enc_txq_limit) {
301		rc = EINVAL;
302		goto fail2;
303	}
304	for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS);
305	    size++)
306		if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
307			break;
308	if (id + (1 << size) >= encp->enc_buftbl_limit) {
309		rc = EINVAL;
310		goto fail3;
311	}
312
313	/* Allocate an TXQ object */
314	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
315
316	if (etp == NULL) {
317		rc = ENOMEM;
318		goto fail4;
319	}
320
321	etp->et_magic = EFX_TXQ_MAGIC;
322	etp->et_enp = enp;
323	etp->et_index = index;
324	etp->et_mask = n - 1;
325	etp->et_esmp = esmp;
326
327	/* Set up the new descriptor queue */
328	EFX_POPULATE_OWORD_6(oword,
329	    FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
330	    FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
331	    FRF_AZ_TX_DESCQ_OWNER_ID, 0,
332	    FRF_AZ_TX_DESCQ_LABEL, label,
333	    FRF_AZ_TX_DESCQ_SIZE, size,
334	    FRF_AZ_TX_DESCQ_TYPE, 0);
335
336	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
337	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
338	    (flags & EFX_CKSUM_IPV4) ? 0 : 1);
339	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
340	    (flags & EFX_CKSUM_TCPUDP) ? 0 : 1);
341
342	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
343	    etp->et_index, &oword);
344
345	enp->en_tx_qcount++;
346	*etpp = etp;
347	return (0);
348
349fail4:
350	EFSYS_PROBE(fail4);
351fail3:
352	EFSYS_PROBE(fail3);
353fail2:
354	EFSYS_PROBE(fail2);
355fail1:
356	EFSYS_PROBE1(fail1, int, rc);
357
358	return (rc);
359}
360
361#if EFSYS_OPT_NAMES
362/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */
363static const char 	__cs * __cs __efx_tx_qstat_name[] = {
364	"post",
365	"unaligned_split",
366};
367/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
368
369		const char __cs *
370efx_tx_qstat_name(
371	__in	efx_nic_t *enp,
372	__in	unsigned int id)
373{
374	_NOTE(ARGUNUSED(enp))
375	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
376	EFSYS_ASSERT3U(id, <, TX_NQSTATS);
377
378	return (__efx_tx_qstat_name[id]);
379}
380#endif	/* EFSYS_OPT_NAMES */
381
382#if EFSYS_OPT_QSTATS
383					void
384efx_tx_qstats_update(
385	__in				efx_txq_t *etp,
386	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat)
387{
388	unsigned int id;
389
390	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
391
392	for (id = 0; id < TX_NQSTATS; id++) {
393		efsys_stat_t *essp = &stat[id];
394
395		EFSYS_STAT_INCR(essp, etp->et_stat[id]);
396		etp->et_stat[id] = 0;
397	}
398}
399#endif	/* EFSYS_OPT_QSTATS */
400
401		void
402efx_tx_qdestroy(
403	__in	efx_txq_t *etp)
404{
405	efx_nic_t *enp = etp->et_enp;
406	efx_oword_t oword;
407
408	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
409
410	EFSYS_ASSERT(enp->en_tx_qcount != 0);
411	--enp->en_tx_qcount;
412
413	/* Purge descriptor queue */
414	EFX_ZERO_OWORD(oword);
415
416	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
417			    etp->et_index, &oword);
418
419	/* Free the TXQ object */
420	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
421}
422
423		void
424efx_tx_fini(
425	__in	efx_nic_t *enp)
426{
427	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
428	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
429	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
430	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
431
432	enp->en_mod_flags &= ~EFX_MOD_TX;
433}
434