1/**
2 * \file
3 * \brief Kernel management of dispatchers (implementation).
4 */
5
6/*
7 * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2013, ETH Zurich.
8 * All rights reserved.
9 *
10 * This file is distributed under the terms in the attached LICENSE file.
11 * If you do not find this file, copies can be found by writing to:
12 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
13 */
14
15#include <kernel.h>
16#include <barrelfish_kpi/cpu.h>
17#include <exec.h> /* XXX wait_for_interrupt, resume, execute */
18#include <paging_kernel_arch.h>
19#include <dispatch.h>
20#include <kcb.h>
21#include <wakeup.h>
22#include <systime.h>
23#include <barrelfish_kpi/syscalls.h>
24#include <barrelfish_kpi/lmp.h>
25#include <trace/trace.h>
26#include <trace_definitions/trace_defs.h>
27#include <barrelfish_kpi/dispatcher_shared_target.h>
28#include <barrelfish_kpi/cpu_arch.h>
29#include <barrelfish_kpi/registers_arch.h>
30
31#include <bitmacros.h>
32
33#if defined(__x86_64__) || defined(__i386__)
34#  include <arch/x86/apic.h>
35#endif
36
37#if defined(__x86_64__) && !defined(__k1om__)
38#  include <vmkit.h>
39#endif
40
41
42/**
43 * \brief The kernel timeslice given in system ticks
44 */
45systime_t kernel_timeslice;
46
47unsigned int config_timeslice = CONFIG_TIMESLICE;
48
49/// Counter for number of context switches
50uint64_t context_switch_counter = 0;
51
52/// Current execution dispatcher (when in system call or exception)
53struct dcb *dcb_current = NULL;
54
55#if CONFIG_TRACE && NETWORK_STACK_BENCHMARK
56#define TRACE_N_BM 1
57#endif // CONFIG_TRACE && NETWORK_STACK_BENCHMARK
58
59
60void __attribute__ ((noreturn)) dispatch(struct dcb *dcb)
61{
62    // XXX FIXME: Why is this null pointer check on the fast path ?
63    // If we have nothing to do we should call something other than dispatch
64    if (dcb == NULL) {
65        dcb_current = NULL;
66        wait_for_interrupt();
67    }
68
69    // Don't context switch if we are current already
70    if (dcb_current != dcb) {
71
72#ifdef TRACE_CSWITCH
73        trace_event(TRACE_SUBSYS_KERNEL,
74                    TRACE_EVENT_KERNEL_CSWITCH,
75                    (uint32_t)(lvaddr_t)dcb & 0xFFFFFFFF);
76#endif
77
78        context_switch(dcb);
79        dcb_current = dcb;
80    }
81
82    assert(dcb != NULL);
83
84    dispatcher_handle_t handle = dcb->disp;
85    struct dispatcher_shared_generic *disp =
86        get_dispatcher_shared_generic(handle);
87    arch_registers_state_t *disabled_area =
88        dispatcher_get_disabled_save_area(handle);
89
90    if (disp != NULL) {
91        disp->systime = systime_now() + kcb_current->kernel_off;
92    }
93    TRACE(KERNEL, SC_YIELD, 1);
94
95    if (dcb->disabled) {
96        if (disp != NULL) {
97            debug(SUBSYS_DISPATCH, "resume %.*s at 0x%" PRIx64 ", %s\n",
98		  DISP_NAME_LEN, disp->name,
99		  (uint64_t)registers_get_ip(disabled_area),
100		  disp->disabled ? "disp->disabled" : "disp->enabled"
101		);
102        }
103
104#if defined(__x86_64__) && !defined(__k1om__)
105        if(!dcb->is_vm_guest) {
106            resume(disabled_area);
107        } else {
108            vmkit_vmenter(dcb);
109        }
110#else
111        resume(disabled_area);
112#endif
113    } else {
114        if (disp != NULL) {
115            debug(SUBSYS_DISPATCH, "dispatch %.*s\n", DISP_NAME_LEN, disp->name);
116            assert(disp->dispatcher_run != 0);
117            disp->disabled = true;
118        }
119#if defined(__x86_64__) && !defined(__k1om__)
120        if(!dcb->is_vm_guest) {
121            execute(disp->dispatcher_run);
122        } else {
123            vmkit_vmexec(dcb, (disp) ? disp->dispatcher_run : 0);
124        }
125#else
126        execute(disp->dispatcher_run);
127#endif
128    }
129} // end function: dispatch
130
131/**
132 * \brief Transfer cap from 'send' to 'ep', according to 'msg'.
133 *
134 * Reads the cap transfer spec in the LMP message 'msg' and transfers
135 * the cap from CSpace in DCB 'send' accordingly.
136 *
137 * \param ep    Endpoint capability of destination
138 * \param send  Pointer to sending DCB.
139 * \param send_cptr Address of capability in sender's cspace
140 * \param send_level Depth/level of capability in sender's cspace
141 *
142 * \return      Error code
143 */
144static errval_t lmp_transfer_cap(struct capability *ep, struct dcb *send,
145                                 capaddr_t send_cptr, uint8_t send_level,
146                                 bool give_away)
147{
148    errval_t err;
149    /* Parameter checking */
150    assert(send_cptr != CPTR_NULL);
151    assert(send != NULL);
152    assert(ep != NULL);
153    assert(ep->type == ObjType_EndPoint);
154    struct dcb *recv = ep->u.endpoint.listener;
155    assert(recv != NULL);
156    assert(ep->u.endpoint.epoffset != 0);
157
158    // printk(LOG_NOTE, "%s: ep->u.endpoint.epoffset = %"PRIuLVADDR"\n", __FUNCTION__, ep->u.endpoint.epoffset);
159    /* Look up the slot receiver can receive caps in */
160    struct lmp_endpoint_kern *recv_ep
161        = (void *)((uint8_t *)recv->disp + ep->u.endpoint.epoffset);
162
163    // Lookup cspace root for receiving
164    struct capability *recv_cspace_cap;
165    // XXX: do we want a level into receiver's cspace here?
166    // printk(LOG_NOTE, "recv_cspace_ptr = %"PRIxCADDR"\n", recv_ep->recv_cspc);
167    err = caps_lookup_cap(&recv->cspace.cap, recv_ep->recv_cspc, 2,
168                          &recv_cspace_cap, CAPRIGHTS_READ_WRITE);
169    if (err_is_fail(err) || recv_cspace_cap->type != ObjType_L1CNode) {
170        return SYS_ERR_LMP_CAPTRANSFER_DST_CNODE_INVALID;
171    }
172    // Check index into L1 cnode
173    capaddr_t l1index = recv_ep->recv_cptr >> L2_CNODE_BITS;
174    if (l1index >= cnode_get_slots(recv_cspace_cap)) {
175        return SYS_ERR_LMP_CAPTRANSFER_DST_CNODE_INVALID;
176    }
177    // Get the cnode
178    struct cte *recv_cnode_cte = caps_locate_slot(get_address(recv_cspace_cap),
179                                                  l1index);
180    struct capability *recv_cnode_cap = &recv_cnode_cte->cap;
181    // Check for cnode type
182    if (recv_cnode_cap->type != ObjType_L2CNode) {
183        return SYS_ERR_LMP_CAPTRANSFER_DST_CNODE_INVALID;
184    }
185    // The slot within the cnode
186    struct cte *recv_cte;
187    recv_cte = caps_locate_slot(get_address(recv_cnode_cap),
188                                recv_ep->recv_cptr & MASK(L2_CNODE_BITS));
189
190    /* Look up source slot in sender */
191    struct cte *send_cte;
192    err = caps_lookup_slot(&send->cspace.cap, send_cptr, send_level,
193                           &send_cte, CAPRIGHTS_READ);
194    if (err_is_fail(err)) {
195        return err_push(err, SYS_ERR_LMP_CAPTRANSFER_SRC_LOOKUP);
196    }
197
198    /* Is destination empty */
199    if (recv_cte->cap.type != ObjType_Null) {
200        debug(SUBSYS_DISPATCH, "%s: dest slot occupied\n", __FUNCTION__);
201        return SYS_ERR_LMP_CAPTRANSFER_DST_SLOT_OCCUPIED;
202    }
203
204    //caps_trace(__func__, __LINE__, send_cte, "transferring");
205    //TRACE_CAP_MSG("transferring", send_cte);
206
207    /* Insert send cap into recv cap */
208    err = caps_copy_to_cte(recv_cte, send_cte, false, 0, 0);
209    assert(err_is_ok(err)); // Cannot fail after checking that slot is empty
210
211    if (give_away) {
212        err = caps_delete(send_cte);
213        if (err_is_fail(err)) {
214            printk(LOG_NOTE, "deleting source of lmp captransfer failed: %"PRIuERRV"\n", err);
215        }
216        assert(err_is_ok(err)); // A copy now exists in the recv slot, so this
217                                // should not fail
218    }
219
220    return SYS_ERR_OK;
221}
222
223/**
224 * \brief Check if it would be possible to deliver LMP payload, but do not deliver it
225 *
226 * \param ep     Endpoint capability to send to
227 * \param payload_len Length (in number of words) of payload
228 */
229errval_t lmp_can_deliver_payload(struct capability *ep,
230                                 size_t payload_len)
231{
232    assert(ep != NULL);
233    assert(ep->type == ObjType_EndPoint);
234    struct dcb *recv = ep->u.endpoint.listener;
235    assert(recv != NULL);
236
237    /* check that receiver exists and has specified an endpoint buffer */
238    if (recv->disp == 0 || ep->u.endpoint.epoffset == 0) {
239        return SYS_ERR_LMP_NO_TARGET;
240    }
241
242    /* locate receiver's endpoint buffer */
243    struct lmp_endpoint_kern *recv_ep
244        = (void *)((uint8_t *)recv->disp + ep->u.endpoint.epoffset);
245
246    /* check delivered/consumed state */
247    uint32_t epbuflen = ep->u.endpoint.epbuflen;
248    uint32_t pos = recv_ep->delivered;
249    uint32_t consumed = recv_ep->consumed;
250    if (pos >= epbuflen || consumed >= epbuflen) {
251        return SYS_ERR_LMP_EP_STATE_INVALID;
252    }
253
254    /* compute space available in endpoint */
255    uint32_t epspace;
256    if (pos >= consumed) {
257        epspace = epbuflen - (pos - consumed);
258    } else {
259        epspace = consumed - pos;
260    }
261
262    /* Check if there's enough space for another msg.
263     * We always keep one word free, to avoid having the special case where
264     * delivered == consumed may mean the buffer is both completely full and
265     * completely empty */
266    if (epspace <= payload_len + LMP_RECV_HEADER_LENGTH) {
267        return SYS_ERR_LMP_BUF_OVERFLOW;
268    }
269
270    return SYS_ERR_OK;
271}
272
273/**
274 * \brief Deliver the payload of an LMP message to a dispatcher.
275 *
276 * \param ep     Endpoint capability to send to
277 * \param send   DCB of the sender. Can be NULL for kernel-originated messages
278 * \param payload     Message payload
279 * \param payload_len Length (in number of words) of payload
280 * \param captransfer True iff a cap has also been delivered
281 *
282 * \return Error code
283 */
284errval_t lmp_deliver_payload(struct capability *ep, struct dcb *send,
285                             uintptr_t *payload, size_t payload_len,
286                             bool captransfer, bool now)
287{
288    assert(ep != NULL);
289    assert(ep->type == ObjType_EndPoint);
290    struct dcb *recv = ep->u.endpoint.listener;
291    assert(recv != NULL);
292    assert(payload != NULL || payload_len == 0);
293
294    errval_t err;
295
296    err = lmp_can_deliver_payload(ep, payload_len);
297    if (err_is_fail(err)) {
298        return err;
299    }
300
301    /* locate receiver's endpoint buffer */
302    struct lmp_endpoint_kern *recv_ep
303        = (void *)((uint8_t *)recv->disp + ep->u.endpoint.epoffset);
304
305    /* read current pos and buflen */
306    uint32_t epbuflen = ep->u.endpoint.epbuflen;
307    uint32_t pos = recv_ep->delivered;
308
309    struct dispatcher_shared_generic *send_disp =
310        send ? get_dispatcher_shared_generic(send->disp) : NULL;
311    struct dispatcher_shared_generic *recv_disp =
312        get_dispatcher_shared_generic(recv->disp);
313    debug(SUBSYS_DISPATCH, "LMP %.*s -> %.*s\n",
314          DISP_NAME_LEN, send ? send_disp->name : "kernel",
315          DISP_NAME_LEN, recv_disp->name);
316
317    // Setup receiver's message flags
318    union lmp_recv_header recvheader = { .raw = 0 };
319    recvheader.x.flags.captransfer = captransfer;
320    recvheader.x.length = payload_len;
321
322    /* Deliver header */
323    recv_ep->buf[pos] = recvheader.raw;
324    if (++pos == epbuflen) {
325        pos = 0;
326    }
327
328    /* Transfer the msg */
329    for(int i = 0; i < payload_len; i++) {
330        recv_ep->buf[pos] = payload[i];
331        if (++pos == epbuflen) {
332            pos = 0;
333        }
334    }
335
336    // update the delivered pos
337    recv_ep->delivered = pos;
338
339    // tell the dispatcher that it has an outstanding message in one of its EPs
340    recv_disp->lmp_delivered += payload_len + LMP_RECV_HEADER_LENGTH;
341
342    // ... and give it a hint which one to look at
343    recv_disp->lmp_hint = ep->u.endpoint.epoffset;
344
345    // Make target runnable
346    make_runnable(recv);
347    if (now)
348        schedule_now(recv);
349
350    return SYS_ERR_OK;
351}
352
353/**
354 * \brief Deliver an LMP message to a dispatcher.
355 *
356 * \param ep     Endpoint capability to send to
357 * \param send   DCB of the sender. Can be NULL for kernel-originated messages
358 * \param payload Buffer containing message payload
359 * \param len    Length of message payload, as number of words
360 * \param send_cptr Capability to be transferred with LMP
361 * \param send_level CSpace level of cptr
362 */
363errval_t lmp_deliver(struct capability *ep, struct dcb *send,
364                     uintptr_t *payload, size_t len,
365                     capaddr_t send_cptr, uint8_t send_level, bool give_away)
366{
367    bool captransfer;
368    assert(ep != NULL);
369    assert(ep->type == ObjType_EndPoint);
370    struct dcb *recv = ep->u.endpoint.listener;
371    assert(recv != NULL);
372    assert(payload != NULL);
373
374    errval_t err;
375
376    /* Is the sender trying to send a cap? */
377    if (send_cptr != CPTR_NULL) {
378        /* Don't attempt to transfer the cap if we can't send the payload */
379        err = lmp_can_deliver_payload(ep, len);
380        if (err_is_fail(err)) {
381            return err;
382        }
383
384        err = lmp_transfer_cap(ep, send, send_cptr, send_level, give_away);
385        if (err_is_fail(err)) {
386            return err;
387        }
388
389        captransfer = true;
390    } else {
391        captransfer = false;
392    }
393
394    /* Send msg */
395    err = lmp_deliver_payload(ep, send, payload, len, captransfer, false);
396    // shouldn't fail, if we delivered the cap successfully
397    assert(!(captransfer && err_is_fail(err)));
398    return err;
399}
400