1/** 2 * \file 3 * \brief Kernel management of dispatchers (implementation). 4 */ 5 6/* 7 * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2013, ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15#include <kernel.h> 16#include <barrelfish_kpi/cpu.h> 17#include <exec.h> /* XXX wait_for_interrupt, resume, execute */ 18#include <paging_kernel_arch.h> 19#include <dispatch.h> 20#include <kcb.h> 21#include <wakeup.h> 22#include <systime.h> 23#include <barrelfish_kpi/syscalls.h> 24#include <barrelfish_kpi/lmp.h> 25#include <trace/trace.h> 26#include <trace_definitions/trace_defs.h> 27#include <barrelfish_kpi/dispatcher_shared_target.h> 28#include <barrelfish_kpi/cpu_arch.h> 29#include <barrelfish_kpi/registers_arch.h> 30 31#include <bitmacros.h> 32 33#if defined(__x86_64__) || defined(__i386__) 34# include <arch/x86/apic.h> 35#endif 36 37#if defined(__x86_64__) && !defined(__k1om__) 38# include <vmkit.h> 39#endif 40 41 42/** 43 * \brief The kernel timeslice given in system ticks 44 */ 45systime_t kernel_timeslice; 46 47unsigned int config_timeslice = CONFIG_TIMESLICE; 48 49/// Counter for number of context switches 50uint64_t context_switch_counter = 0; 51 52/// Current execution dispatcher (when in system call or exception) 53struct dcb *dcb_current = NULL; 54 55#if CONFIG_TRACE && NETWORK_STACK_BENCHMARK 56#define TRACE_N_BM 1 57#endif // CONFIG_TRACE && NETWORK_STACK_BENCHMARK 58 59 60void __attribute__ ((noreturn)) dispatch(struct dcb *dcb) 61{ 62 // XXX FIXME: Why is this null pointer check on the fast path ? 63 // If we have nothing to do we should call something other than dispatch 64 if (dcb == NULL) { 65 dcb_current = NULL; 66 wait_for_interrupt(); 67 } 68 69 // Don't context switch if we are current already 70 if (dcb_current != dcb) { 71 72#ifdef TRACE_CSWITCH 73 trace_event(TRACE_SUBSYS_KERNEL, 74 TRACE_EVENT_KERNEL_CSWITCH, 75 (uint32_t)(lvaddr_t)dcb & 0xFFFFFFFF); 76#endif 77 78 context_switch(dcb); 79 dcb_current = dcb; 80 } 81 82 assert(dcb != NULL); 83 84 dispatcher_handle_t handle = dcb->disp; 85 struct dispatcher_shared_generic *disp = 86 get_dispatcher_shared_generic(handle); 87 arch_registers_state_t *disabled_area = 88 dispatcher_get_disabled_save_area(handle); 89 90 if (disp != NULL) { 91 disp->systime = systime_now() + kcb_current->kernel_off; 92 } 93 TRACE(KERNEL, SC_YIELD, 1); 94 95 if (dcb->disabled) { 96 if (disp != NULL) { 97 debug(SUBSYS_DISPATCH, "resume %.*s at 0x%" PRIx64 ", %s\n", 98 DISP_NAME_LEN, disp->name, 99 (uint64_t)registers_get_ip(disabled_area), 100 disp->disabled ? "disp->disabled" : "disp->enabled" 101 ); 102 } 103 104#if defined(__x86_64__) && !defined(__k1om__) 105 if(!dcb->is_vm_guest) { 106 resume(disabled_area); 107 } else { 108 vmkit_vmenter(dcb); 109 } 110#else 111 resume(disabled_area); 112#endif 113 } else { 114 if (disp != NULL) { 115 debug(SUBSYS_DISPATCH, "dispatch %.*s\n", DISP_NAME_LEN, disp->name); 116 assert(disp->dispatcher_run != 0); 117 disp->disabled = true; 118 } 119#if defined(__x86_64__) && !defined(__k1om__) 120 if(!dcb->is_vm_guest) { 121 execute(disp->dispatcher_run); 122 } else { 123 vmkit_vmexec(dcb, (disp) ? disp->dispatcher_run : 0); 124 } 125#else 126 execute(disp->dispatcher_run); 127#endif 128 } 129} // end function: dispatch 130 131/** 132 * \brief Transfer cap from 'send' to 'ep', according to 'msg'. 133 * 134 * Reads the cap transfer spec in the LMP message 'msg' and transfers 135 * the cap from CSpace in DCB 'send' accordingly. 136 * 137 * \param ep Endpoint capability of destination 138 * \param send Pointer to sending DCB. 139 * \param send_cptr Address of capability in sender's cspace 140 * \param send_level Depth/level of capability in sender's cspace 141 * 142 * \return Error code 143 */ 144static errval_t lmp_transfer_cap(struct capability *ep, struct dcb *send, 145 capaddr_t send_cptr, uint8_t send_level, 146 bool give_away) 147{ 148 errval_t err; 149 /* Parameter checking */ 150 assert(send_cptr != CPTR_NULL); 151 assert(send != NULL); 152 assert(ep != NULL); 153 assert(ep->type == ObjType_EndPoint); 154 struct dcb *recv = ep->u.endpoint.listener; 155 assert(recv != NULL); 156 assert(ep->u.endpoint.epoffset != 0); 157 158 // printk(LOG_NOTE, "%s: ep->u.endpoint.epoffset = %"PRIuLVADDR"\n", __FUNCTION__, ep->u.endpoint.epoffset); 159 /* Look up the slot receiver can receive caps in */ 160 struct lmp_endpoint_kern *recv_ep 161 = (void *)((uint8_t *)recv->disp + ep->u.endpoint.epoffset); 162 163 // Lookup cspace root for receiving 164 struct capability *recv_cspace_cap; 165 // XXX: do we want a level into receiver's cspace here? 166 // printk(LOG_NOTE, "recv_cspace_ptr = %"PRIxCADDR"\n", recv_ep->recv_cspc); 167 err = caps_lookup_cap(&recv->cspace.cap, recv_ep->recv_cspc, 2, 168 &recv_cspace_cap, CAPRIGHTS_READ_WRITE); 169 if (err_is_fail(err) || recv_cspace_cap->type != ObjType_L1CNode) { 170 return SYS_ERR_LMP_CAPTRANSFER_DST_CNODE_INVALID; 171 } 172 // Check index into L1 cnode 173 capaddr_t l1index = recv_ep->recv_cptr >> L2_CNODE_BITS; 174 if (l1index >= cnode_get_slots(recv_cspace_cap)) { 175 return SYS_ERR_LMP_CAPTRANSFER_DST_CNODE_INVALID; 176 } 177 // Get the cnode 178 struct cte *recv_cnode_cte = caps_locate_slot(get_address(recv_cspace_cap), 179 l1index); 180 struct capability *recv_cnode_cap = &recv_cnode_cte->cap; 181 // Check for cnode type 182 if (recv_cnode_cap->type != ObjType_L2CNode) { 183 return SYS_ERR_LMP_CAPTRANSFER_DST_CNODE_INVALID; 184 } 185 // The slot within the cnode 186 struct cte *recv_cte; 187 recv_cte = caps_locate_slot(get_address(recv_cnode_cap), 188 recv_ep->recv_cptr & MASK(L2_CNODE_BITS)); 189 190 /* Look up source slot in sender */ 191 struct cte *send_cte; 192 err = caps_lookup_slot(&send->cspace.cap, send_cptr, send_level, 193 &send_cte, CAPRIGHTS_READ); 194 if (err_is_fail(err)) { 195 return err_push(err, SYS_ERR_LMP_CAPTRANSFER_SRC_LOOKUP); 196 } 197 198 /* Is destination empty */ 199 if (recv_cte->cap.type != ObjType_Null) { 200 debug(SUBSYS_DISPATCH, "%s: dest slot occupied\n", __FUNCTION__); 201 return SYS_ERR_LMP_CAPTRANSFER_DST_SLOT_OCCUPIED; 202 } 203 204 //caps_trace(__func__, __LINE__, send_cte, "transferring"); 205 //TRACE_CAP_MSG("transferring", send_cte); 206 207 /* Insert send cap into recv cap */ 208 err = caps_copy_to_cte(recv_cte, send_cte, false, 0, 0); 209 assert(err_is_ok(err)); // Cannot fail after checking that slot is empty 210 211 if (give_away) { 212 err = caps_delete(send_cte); 213 if (err_is_fail(err)) { 214 printk(LOG_NOTE, "deleting source of lmp captransfer failed: %"PRIuERRV"\n", err); 215 } 216 assert(err_is_ok(err)); // A copy now exists in the recv slot, so this 217 // should not fail 218 } 219 220 return SYS_ERR_OK; 221} 222 223/** 224 * \brief Check if it would be possible to deliver LMP payload, but do not deliver it 225 * 226 * \param ep Endpoint capability to send to 227 * \param payload_len Length (in number of words) of payload 228 */ 229errval_t lmp_can_deliver_payload(struct capability *ep, 230 size_t payload_len) 231{ 232 assert(ep != NULL); 233 assert(ep->type == ObjType_EndPoint); 234 struct dcb *recv = ep->u.endpoint.listener; 235 assert(recv != NULL); 236 237 /* check that receiver exists and has specified an endpoint buffer */ 238 if (recv->disp == 0 || ep->u.endpoint.epoffset == 0) { 239 return SYS_ERR_LMP_NO_TARGET; 240 } 241 242 /* locate receiver's endpoint buffer */ 243 struct lmp_endpoint_kern *recv_ep 244 = (void *)((uint8_t *)recv->disp + ep->u.endpoint.epoffset); 245 246 /* check delivered/consumed state */ 247 uint32_t epbuflen = ep->u.endpoint.epbuflen; 248 uint32_t pos = recv_ep->delivered; 249 uint32_t consumed = recv_ep->consumed; 250 if (pos >= epbuflen || consumed >= epbuflen) { 251 return SYS_ERR_LMP_EP_STATE_INVALID; 252 } 253 254 /* compute space available in endpoint */ 255 uint32_t epspace; 256 if (pos >= consumed) { 257 epspace = epbuflen - (pos - consumed); 258 } else { 259 epspace = consumed - pos; 260 } 261 262 /* Check if there's enough space for another msg. 263 * We always keep one word free, to avoid having the special case where 264 * delivered == consumed may mean the buffer is both completely full and 265 * completely empty */ 266 if (epspace <= payload_len + LMP_RECV_HEADER_LENGTH) { 267 return SYS_ERR_LMP_BUF_OVERFLOW; 268 } 269 270 return SYS_ERR_OK; 271} 272 273/** 274 * \brief Deliver the payload of an LMP message to a dispatcher. 275 * 276 * \param ep Endpoint capability to send to 277 * \param send DCB of the sender. Can be NULL for kernel-originated messages 278 * \param payload Message payload 279 * \param payload_len Length (in number of words) of payload 280 * \param captransfer True iff a cap has also been delivered 281 * 282 * \return Error code 283 */ 284errval_t lmp_deliver_payload(struct capability *ep, struct dcb *send, 285 uintptr_t *payload, size_t payload_len, 286 bool captransfer, bool now) 287{ 288 assert(ep != NULL); 289 assert(ep->type == ObjType_EndPoint); 290 struct dcb *recv = ep->u.endpoint.listener; 291 assert(recv != NULL); 292 assert(payload != NULL || payload_len == 0); 293 294 errval_t err; 295 296 err = lmp_can_deliver_payload(ep, payload_len); 297 if (err_is_fail(err)) { 298 return err; 299 } 300 301 /* locate receiver's endpoint buffer */ 302 struct lmp_endpoint_kern *recv_ep 303 = (void *)((uint8_t *)recv->disp + ep->u.endpoint.epoffset); 304 305 /* read current pos and buflen */ 306 uint32_t epbuflen = ep->u.endpoint.epbuflen; 307 uint32_t pos = recv_ep->delivered; 308 309 struct dispatcher_shared_generic *send_disp = 310 send ? get_dispatcher_shared_generic(send->disp) : NULL; 311 struct dispatcher_shared_generic *recv_disp = 312 get_dispatcher_shared_generic(recv->disp); 313 debug(SUBSYS_DISPATCH, "LMP %.*s -> %.*s\n", 314 DISP_NAME_LEN, send ? send_disp->name : "kernel", 315 DISP_NAME_LEN, recv_disp->name); 316 317 // Setup receiver's message flags 318 union lmp_recv_header recvheader = { .raw = 0 }; 319 recvheader.x.flags.captransfer = captransfer; 320 recvheader.x.length = payload_len; 321 322 /* Deliver header */ 323 recv_ep->buf[pos] = recvheader.raw; 324 if (++pos == epbuflen) { 325 pos = 0; 326 } 327 328 /* Transfer the msg */ 329 for(int i = 0; i < payload_len; i++) { 330 recv_ep->buf[pos] = payload[i]; 331 if (++pos == epbuflen) { 332 pos = 0; 333 } 334 } 335 336 // update the delivered pos 337 recv_ep->delivered = pos; 338 339 // tell the dispatcher that it has an outstanding message in one of its EPs 340 recv_disp->lmp_delivered += payload_len + LMP_RECV_HEADER_LENGTH; 341 342 // ... and give it a hint which one to look at 343 recv_disp->lmp_hint = ep->u.endpoint.epoffset; 344 345 // Make target runnable 346 make_runnable(recv); 347 if (now) 348 schedule_now(recv); 349 350 return SYS_ERR_OK; 351} 352 353/** 354 * \brief Deliver an LMP message to a dispatcher. 355 * 356 * \param ep Endpoint capability to send to 357 * \param send DCB of the sender. Can be NULL for kernel-originated messages 358 * \param payload Buffer containing message payload 359 * \param len Length of message payload, as number of words 360 * \param send_cptr Capability to be transferred with LMP 361 * \param send_level CSpace level of cptr 362 */ 363errval_t lmp_deliver(struct capability *ep, struct dcb *send, 364 uintptr_t *payload, size_t len, 365 capaddr_t send_cptr, uint8_t send_level, bool give_away) 366{ 367 bool captransfer; 368 assert(ep != NULL); 369 assert(ep->type == ObjType_EndPoint); 370 struct dcb *recv = ep->u.endpoint.listener; 371 assert(recv != NULL); 372 assert(payload != NULL); 373 374 errval_t err; 375 376 /* Is the sender trying to send a cap? */ 377 if (send_cptr != CPTR_NULL) { 378 /* Don't attempt to transfer the cap if we can't send the payload */ 379 err = lmp_can_deliver_payload(ep, len); 380 if (err_is_fail(err)) { 381 return err; 382 } 383 384 err = lmp_transfer_cap(ep, send, send_cptr, send_level, give_away); 385 if (err_is_fail(err)) { 386 return err; 387 } 388 389 captransfer = true; 390 } else { 391 captransfer = false; 392 } 393 394 /* Send msg */ 395 err = lmp_deliver_payload(ep, send, payload, len, captransfer, false); 396 // shouldn't fail, if we delivered the cap successfully 397 assert(!(captransfer && err_is_fail(err))); 398 return err; 399} 400