1/*
2 * Copyright (c) 2014, ETH Zurich. All rights reserved.
3 *
4 * This file is distributed under the terms in the attached LICENSE file.
5 * If you do not find this file, copies can be found by writing to:
6 * ETH Zurich D-INFK, Universitaetsstrasse 6, CH-8092 Zurich. Attn: Systems Group.
7 */
8
9#include <barrelfish/barrelfish.h>
10
11#include <dma_internal.h>
12#include <dma_mem_utils.h>
13
14#include <dma_internal.h>
15#include <dma_descriptor_internal.h>
16#include <dma_ring_internal.h>
17
18#include <debug.h>
19#include "include/dma_channel_internal.h"
20#include "include/dma_device_internal.h"
21
22/**
23 * represents the IOAT DMA ring with its internal state
24 */
25struct dma_ring
26{
27    uint16_t size;                 ///< size of the descriptor ring
28    uint16_t write_next;           ///< next descriptor to be allocated
29    uint16_t issued;               ///< hardware notification point
30    uint16_t tail;                 ///< cleanup index
31    uint16_t head;                 ///< head of the descriptor chain
32    uint8_t use_modulo;           ///< return values module ringsize
33    struct dma_descriptor **desc;  ///< descriptor pointer array
34    struct dma_channel *chan;      ///< channel associated with this ring
35};
36
37/*
38 * ============================================================================
39 * Library Internal Interface
40 * ============================================================================
41 */
42
43/*
44 * ----------------------------------------------------------------------------
45 * Ring Manipulation
46 * ----------------------------------------------------------------------------
47 */
48
49/**
50 * \brief gets the next descriptor based on the head pointer and increments the
51 *        head pointer
52 *
53 * \param ring the DMA ring
54 *
55 * \returns pointer to a DMA descriptor
56 */
57inline struct dma_descriptor *dma_ring_get_next_desc(struct dma_ring *ring)
58{
59    struct dma_descriptor *desc = dma_ring_get_desc(ring, ring->write_next++);
60
61    DMADESC_DEBUG("ring getting next head desc:%p @ [%016lx], new head:%u\n", desc,
62                  dma_desc_get_paddr(desc), (ring->write_next & (ring->size - 1)));
63
64    return desc;
65}
66
67/**
68 * \brief gets the next descriptor based on the tail pointer and increases the
69 *        tail pointer index
70 *
71 * \param ring the DMA ring
72 *
73 * \returns pointer to a DMA descriptor
74 *
75 */
76inline struct dma_descriptor *dma_ring_get_tail_desc(struct dma_ring *ring)
77{
78    struct dma_descriptor *desc = dma_ring_get_desc(ring, ring->tail++);
79
80    DMADESC_DEBUG("ring getting tail desc:%p @ [%016lx], new tail: %u\n", desc,
81                  dma_desc_get_paddr(desc), (ring->tail & (ring->size - 1)));
82
83    return desc;
84}
85
86/**
87 * \brief submits the pending descriptors and updates the DMA count value
88 *
89 * \param ring DMA ring to submit the pending descriptors
90 *
91 * \returns the current head of the descriptors (dmacount)
92 */
93uint16_t dma_ring_submit_pending(struct dma_ring *ring)
94{
95    uint16_t num_pending = dma_ring_get_pendig(ring);
96
97    if (num_pending != 0) {
98        ring->head += num_pending;
99        ring->issued = ring->write_next;
100
101        DMADESC_DEBUG("ring submit pending head: %u, head = %u\n",
102                      (ring->head & (ring->size - 1)),
103                      (ring->write_next & (ring->size - 1)));
104    }
105    if (ring->use_modulo) {
106        return (ring->head & (ring->size - 1));
107    }
108    return ring->head;
109}
110
111/**
112 * \brief obtains the physical address of the descriptor chain
113 *        (pending descriptors)
114 *
115 * \param ring the DMA ring
116 *
117 * \returns physical address of the pending descriptor chain
118 */
119inline lpaddr_t dma_ring_get_chain_addr(struct dma_ring *ring)
120{
121    return dma_desc_get_paddr(dma_ring_get_desc(ring, ring->issued));
122}
123
124/**
125 * \brief obtains the physical address of the descriptor ring
126 *
127 * \param ring the DMA ring
128 *
129 * \returns physical address of the pending descriptor chain
130 */
131lpaddr_t dma_ring_get_base_addr(struct dma_ring *ring)
132{
133    return dma_desc_get_paddr(dma_ring_get_desc(ring, 0));
134}
135
136/*
137 * ============================================================================
138 * Public Interface
139 * ============================================================================
140 */
141
142/*
143 * ----------------------------------------------------------------------------
144 * Descriptor Ring Allocation / free
145 * ----------------------------------------------------------------------------
146 */
147
148/**
149 * \brief allocates a descriptor ring of a given element count and descriptor
150 *        sizes
151 *
152 * \param ndesc_bits number of descriptors for this ring in bits
153 * \param desc_align alignment constraints of the descriptors
154 * \param desc_size  size of the descriptors in bytes
155 * \param use_modulo return the head pointer modulo ring size
156 * \param chan       DMA channel of this ring
157 * \param ret_ring   where the ring pointer is returned
158 *
159 * \returns SYS_ERR_OK on succes
160 *          errval on error
161 */
162errval_t dma_ring_alloc(uint8_t ndesc_bits,
163                        uint32_t desc_align,
164                        uint32_t desc_size,
165                        uint8_t use_modulo,
166                        struct dma_channel *chan,
167                        struct dma_ring **ret_ring)
168{
169    errval_t err;
170
171    DMADESC_DEBUG("allocating descriptor ring of size %u\n", (1 << ndesc_bits));
172
173    struct dma_ring *ring;
174
175    if (ndesc_bits > DMA_RING_SIZE_MAX) {
176        ndesc_bits = DMA_RING_SIZE_MAX;
177    }
178
179    uint16_t ndesc = (1UL << ndesc_bits);
180
181    ring = calloc(1, sizeof(struct dma_ring) + ndesc * sizeof(void *));
182    if (ring == NULL) {
183        return LIB_ERR_MALLOC_FAIL;
184    }
185
186    ring->chan = chan;
187    ring->size = ndesc;
188    ring->desc = (void *) (ring + 1);
189    ring->use_modulo = use_modulo;
190
191    err = dma_desc_alloc(desc_size, desc_align, ndesc_bits,
192                         chan->device, ring->desc);
193    if (err_is_fail(err)) {
194        free(ring);
195        return err;
196    }
197
198    *ret_ring = ring;
199
200    return SYS_ERR_OK;
201}
202
203/**
204 * \brief frees a previously allocated descriptor ring
205 *
206 * \param ring IAT DMA descriptor ring to be freed
207 *
208 * \returns SYS_ERR_OK on success
209 *          errval on failure
210 */
211errval_t dma_ring_free(struct dma_ring *ring)
212{
213    errval_t err;
214
215    DMADESC_DEBUG("freeing descriptor ring %p\n", ring);
216
217    err = dma_desc_free(ring->desc[0]);
218    if (err_is_fail(err)) {
219        return err;
220    }
221
222    free(ring);
223
224    return SYS_ERR_OK;
225}
226
227/*
228 * ----------------------------------------------------------------------------
229 * Ring Status Queries
230 * ----------------------------------------------------------------------------
231 */
232
233/**
234 * \brief returns the number of active descriptors i.e. those who have not yet
235 *        been finished by the DMA hardware. this includes submitted and pending
236 *        DMA descriptors
237 *
238 * \param ring  IOAT DMA descriptor ring
239 *
240 * \returns number of active descriptors
241 */
242inline uint16_t dma_ring_get_active(struct dma_ring *ring)
243{
244    return (ring->write_next - ring->tail) & (ring->size - 1);
245}
246
247/**
248 * \brief returns the number of prepared but not submitted descriptors
249 *
250 * \param ring  IOAT DMA descriptor ring
251 *
252 * \returns number of pending descriptors
253 */
254inline uint16_t dma_ring_get_pendig(struct dma_ring *ring)
255{
256    return (ring->write_next - ring->issued) & (ring->size - 1);
257}
258
259/*
260 * ----------------------------------------------------------------------------
261 * Getter Functions
262 * ----------------------------------------------------------------------------
263 */
264
265/**
266 * \brief returns the size of the ring
267 *
268 * \param ring  IOAT DMA descriptor ring
269 *
270 * \returns number of descriptors in the ring
271 */
272inline uint16_t dma_ring_get_size(struct dma_ring *ring)
273{
274    return ring->size;
275}
276
277/**
278 * \brief returns the head pointer index of the ring
279 *
280 * \param ring  DMA descriptor ring
281 *
282 * \returns head element index
283 */
284inline uint16_t dma_ring_get_write_next(struct dma_ring *ring)
285{
286    return (ring->write_next & (ring->size - 1));
287}
288
289/**
290 * \brief returns the tail pointer index of the ring
291 *
292 * \param ring  DMA descriptor ring
293 *
294 * \returns tail element index
295 */
296inline uint16_t dma_ring_get_tail(struct dma_ring *ring)
297{
298    return (ring->tail & (ring->size - 1));
299}
300
301/**
302 * \brief returns the issued pointer index of the ring
303 *
304 * \param ring  DMA descriptor ring
305 *
306 * \returns issued element index
307 */
308inline uint16_t dma_ring_get_issued(struct dma_ring *ring)
309{
310    return (ring->issued & (ring->size - 1));
311}
312
313/**
314 * \brief returns the DMA count of the ring for setting the DMA count register
315 *
316 * \param ring  DMA descriptor ring
317 *
318 * \returns dmacount value
319 */
320inline uint16_t dma_ring_get_head(struct dma_ring *ring)
321{
322    if (ring->use_modulo) {
323        return (ring->head & (ring->size - 1));
324    }
325    return ring->head;
326}
327
328/**
329 * \brief gets the next descriptor based on the index
330 *
331 * \param ring  the DMA ring
332 * \param index the index of the ring
333 *
334 * \returns pointer to a DMA descriptor
335 */
336inline struct dma_descriptor *dma_ring_get_desc(struct dma_ring *ring,
337                                                uint16_t idx)
338{
339    return ring->desc[idx & (ring->size - 1)];
340}
341