1181624Skmacy/****************************************************************************** 2181624Skmacy * gnttab.c 3181624Skmacy * 4181624Skmacy * Two sets of functionality: 5181624Skmacy * 1. Granting foreign access to our memory reservation. 6181624Skmacy * 2. Accessing others' memory reservations via grant references. 7181624Skmacy * (i.e., mechanisms for both sender and recipient of grant references) 8181624Skmacy * 9181624Skmacy * Copyright (c) 2005, Christopher Clark 10181624Skmacy * Copyright (c) 2004, K A Fraser 11181624Skmacy */ 12181624Skmacy 13181624Skmacy#include <sys/cdefs.h> 14181624Skmacy__FBSDID("$FreeBSD$"); 15181624Skmacy 16181624Skmacy#include "opt_global.h" 17181624Skmacy#include "opt_pmap.h" 18183340Skmacy 19181624Skmacy#include <sys/param.h> 20181624Skmacy#include <sys/systm.h> 21181624Skmacy#include <sys/bus.h> 22181624Skmacy#include <sys/conf.h> 23181624Skmacy#include <sys/module.h> 24181624Skmacy#include <sys/kernel.h> 25181624Skmacy#include <sys/lock.h> 26181624Skmacy#include <sys/malloc.h> 27181624Skmacy#include <sys/mman.h> 28181624Skmacy 29255040Sgibbs#include <xen/xen-os.h> 30189699Sdfr#include <xen/hypervisor.h> 31189699Sdfr#include <machine/xen/synch_bitops.h> 32181624Skmacy 33186557Skmacy#include <xen/hypervisor.h> 34181624Skmacy#include <xen/gnttab.h> 35181624Skmacy 36189699Sdfr#include <vm/vm.h> 37189699Sdfr#include <vm/vm_kern.h> 38189699Sdfr#include <vm/vm_extern.h> 39189699Sdfr#include <vm/pmap.h> 40189699Sdfr 41181624Skmacy#define cmpxchg(a, b, c) atomic_cmpset_int((volatile u_int *)(a),(b),(c)) 42181624Skmacy 43181624Skmacy/* External tools reserve first few grant table entries. */ 44181624Skmacy#define NR_RESERVED_ENTRIES 8 45181624Skmacy#define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(grant_entry_t)) 46181624Skmacy 47181624Skmacystatic grant_ref_t **gnttab_list; 48181624Skmacystatic unsigned int nr_grant_frames; 49181624Skmacystatic unsigned int boot_max_nr_grant_frames; 50181624Skmacystatic int gnttab_free_count; 51181624Skmacystatic grant_ref_t gnttab_free_head; 52181624Skmacystatic struct mtx gnttab_list_lock; 53181624Skmacy 54181624Skmacystatic grant_entry_t *shared; 55181624Skmacy 56181624Skmacystatic struct gnttab_free_callback *gnttab_free_callback_list = NULL; 57181624Skmacy 58181624Skmacystatic int gnttab_expand(unsigned int req_entries); 59181624Skmacy 60181624Skmacy#define RPP (PAGE_SIZE / sizeof(grant_ref_t)) 61181624Skmacy#define gnttab_entry(entry) (gnttab_list[(entry) / RPP][(entry) % RPP]) 62181624Skmacy 63181624Skmacystatic int 64186557Skmacyget_free_entries(int count, int *entries) 65181624Skmacy{ 66189699Sdfr int ref, error; 67181624Skmacy grant_ref_t head; 68214077Sgibbs 69181624Skmacy mtx_lock(&gnttab_list_lock); 70181624Skmacy if ((gnttab_free_count < count) && 71189699Sdfr ((error = gnttab_expand(count - gnttab_free_count)) != 0)) { 72181624Skmacy mtx_unlock(&gnttab_list_lock); 73189699Sdfr return (error); 74181624Skmacy } 75181624Skmacy ref = head = gnttab_free_head; 76181624Skmacy gnttab_free_count -= count; 77181624Skmacy while (count-- > 1) 78181624Skmacy head = gnttab_entry(head); 79181624Skmacy gnttab_free_head = gnttab_entry(head); 80181624Skmacy gnttab_entry(head) = GNTTAB_LIST_END; 81214077Sgibbs mtx_unlock(&gnttab_list_lock); 82186557Skmacy 83186557Skmacy *entries = ref; 84186557Skmacy return (0); 85181624Skmacy} 86181624Skmacy 87181624Skmacystatic void 88181624Skmacydo_free_callbacks(void) 89181624Skmacy{ 90181624Skmacy struct gnttab_free_callback *callback, *next; 91181624Skmacy 92181624Skmacy callback = gnttab_free_callback_list; 93181624Skmacy gnttab_free_callback_list = NULL; 94181624Skmacy 95181624Skmacy while (callback != NULL) { 96181624Skmacy next = callback->next; 97181624Skmacy if (gnttab_free_count >= callback->count) { 98181624Skmacy callback->next = NULL; 99181624Skmacy callback->fn(callback->arg); 100181624Skmacy } else { 101181624Skmacy callback->next = gnttab_free_callback_list; 102181624Skmacy gnttab_free_callback_list = callback; 103181624Skmacy } 104181624Skmacy callback = next; 105181624Skmacy } 106181624Skmacy} 107181624Skmacy 108181624Skmacystatic inline void 109181624Skmacycheck_free_callbacks(void) 110181624Skmacy{ 111255040Sgibbs if (__predict_false(gnttab_free_callback_list != NULL)) 112181624Skmacy do_free_callbacks(); 113181624Skmacy} 114181624Skmacy 115181624Skmacystatic void 116181624Skmacyput_free_entry(grant_ref_t ref) 117181624Skmacy{ 118181624Skmacy 119181624Skmacy mtx_lock(&gnttab_list_lock); 120181624Skmacy gnttab_entry(ref) = gnttab_free_head; 121181624Skmacy gnttab_free_head = ref; 122181624Skmacy gnttab_free_count++; 123181624Skmacy check_free_callbacks(); 124214077Sgibbs mtx_unlock(&gnttab_list_lock); 125181624Skmacy} 126181624Skmacy 127181624Skmacy/* 128181624Skmacy * Public grant-issuing interface functions 129181624Skmacy */ 130181624Skmacy 131181624Skmacyint 132186557Skmacygnttab_grant_foreign_access(domid_t domid, unsigned long frame, int readonly, 133186557Skmacy grant_ref_t *result) 134181624Skmacy{ 135186557Skmacy int error, ref; 136181624Skmacy 137186557Skmacy error = get_free_entries(1, &ref); 138214077Sgibbs 139255040Sgibbs if (__predict_false(error)) 140186557Skmacy return (error); 141181624Skmacy 142181624Skmacy shared[ref].frame = frame; 143181624Skmacy shared[ref].domid = domid; 144181624Skmacy wmb(); 145181624Skmacy shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 146181624Skmacy 147186557Skmacy if (result) 148186557Skmacy *result = ref; 149186557Skmacy 150186557Skmacy return (0); 151181624Skmacy} 152181624Skmacy 153181624Skmacyvoid 154181624Skmacygnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 155181624Skmacy unsigned long frame, int readonly) 156181624Skmacy{ 157189699Sdfr 158181624Skmacy shared[ref].frame = frame; 159181624Skmacy shared[ref].domid = domid; 160181624Skmacy wmb(); 161181624Skmacy shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 162181624Skmacy} 163181624Skmacy 164181624Skmacyint 165181624Skmacygnttab_query_foreign_access(grant_ref_t ref) 166181624Skmacy{ 167181624Skmacy uint16_t nflags; 168214077Sgibbs 169181624Skmacy nflags = shared[ref].flags; 170214077Sgibbs 171181624Skmacy return (nflags & (GTF_reading|GTF_writing)); 172181624Skmacy} 173181624Skmacy 174181624Skmacyint 175183375Skmacygnttab_end_foreign_access_ref(grant_ref_t ref) 176181624Skmacy{ 177181624Skmacy uint16_t flags, nflags; 178181624Skmacy 179181624Skmacy nflags = shared[ref].flags; 180181624Skmacy do { 181181624Skmacy if ( (flags = nflags) & (GTF_reading|GTF_writing) ) { 182214077Sgibbs printf("%s: WARNING: g.e. still in use!\n", __func__); 183181624Skmacy return (0); 184181624Skmacy } 185181624Skmacy } while ((nflags = synch_cmpxchg(&shared[ref].flags, flags, 0)) != 186181624Skmacy flags); 187181624Skmacy 188181624Skmacy return (1); 189181624Skmacy} 190181624Skmacy 191181624Skmacyvoid 192183375Skmacygnttab_end_foreign_access(grant_ref_t ref, void *page) 193181624Skmacy{ 194183375Skmacy if (gnttab_end_foreign_access_ref(ref)) { 195181624Skmacy put_free_entry(ref); 196181624Skmacy if (page != NULL) { 197181624Skmacy free(page, M_DEVBUF); 198181624Skmacy } 199181624Skmacy } 200181624Skmacy else { 201181624Skmacy /* XXX This needs to be fixed so that the ref and page are 202181624Skmacy placed on a list to be freed up later. */ 203214077Sgibbs printf("%s: WARNING: leaking g.e. and page still in use!\n", 204214077Sgibbs __func__); 205181624Skmacy } 206181624Skmacy} 207181624Skmacy 208214077Sgibbsvoid 209214077Sgibbsgnttab_end_foreign_access_references(u_int count, grant_ref_t *refs) 210214077Sgibbs{ 211214077Sgibbs grant_ref_t *last_ref; 212214077Sgibbs grant_ref_t head; 213214077Sgibbs grant_ref_t tail; 214214077Sgibbs 215214077Sgibbs head = GNTTAB_LIST_END; 216214077Sgibbs tail = *refs; 217214077Sgibbs last_ref = refs + count; 218214077Sgibbs while (refs != last_ref) { 219214077Sgibbs 220214077Sgibbs if (gnttab_end_foreign_access_ref(*refs)) { 221214077Sgibbs gnttab_entry(*refs) = head; 222214077Sgibbs head = *refs; 223214077Sgibbs } else { 224214077Sgibbs /* 225214077Sgibbs * XXX This needs to be fixed so that the ref 226214077Sgibbs * is placed on a list to be freed up later. 227214077Sgibbs */ 228214077Sgibbs printf("%s: WARNING: leaking g.e. still in use!\n", 229214077Sgibbs __func__); 230214077Sgibbs count--; 231214077Sgibbs } 232214077Sgibbs refs++; 233214077Sgibbs } 234214077Sgibbs 235214077Sgibbs if (count != 0) { 236214077Sgibbs mtx_lock(&gnttab_list_lock); 237214077Sgibbs gnttab_free_count += count; 238214077Sgibbs gnttab_entry(tail) = gnttab_free_head; 239214077Sgibbs gnttab_free_head = head; 240214077Sgibbs mtx_unlock(&gnttab_list_lock); 241214077Sgibbs } 242214077Sgibbs} 243214077Sgibbs 244181624Skmacyint 245189699Sdfrgnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn, 246189699Sdfr grant_ref_t *result) 247181624Skmacy{ 248186557Skmacy int error, ref; 249181624Skmacy 250186557Skmacy error = get_free_entries(1, &ref); 251255040Sgibbs if (__predict_false(error)) 252186557Skmacy return (error); 253186557Skmacy 254181624Skmacy gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 255214077Sgibbs 256189699Sdfr *result = ref; 257189699Sdfr return (0); 258181624Skmacy} 259181624Skmacy 260181624Skmacyvoid 261181624Skmacygnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, 262181624Skmacy unsigned long pfn) 263181624Skmacy{ 264181624Skmacy shared[ref].frame = pfn; 265181624Skmacy shared[ref].domid = domid; 266181624Skmacy wmb(); 267181624Skmacy shared[ref].flags = GTF_accept_transfer; 268181624Skmacy} 269181624Skmacy 270181624Skmacyunsigned long 271181624Skmacygnttab_end_foreign_transfer_ref(grant_ref_t ref) 272181624Skmacy{ 273181624Skmacy unsigned long frame; 274181624Skmacy uint16_t flags; 275181624Skmacy 276181624Skmacy /* 277181624Skmacy * If a transfer is not even yet started, try to reclaim the grant 278181624Skmacy * reference and return failure (== 0). 279181624Skmacy */ 280181624Skmacy while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 281181624Skmacy if ( synch_cmpxchg(&shared[ref].flags, flags, 0) == flags ) 282181624Skmacy return (0); 283181624Skmacy cpu_relax(); 284181624Skmacy } 285181624Skmacy 286181624Skmacy /* If a transfer is in progress then wait until it is completed. */ 287181624Skmacy while (!(flags & GTF_transfer_completed)) { 288181624Skmacy flags = shared[ref].flags; 289181624Skmacy cpu_relax(); 290181624Skmacy } 291181624Skmacy 292181624Skmacy /* Read the frame number /after/ reading completion status. */ 293181624Skmacy rmb(); 294181624Skmacy frame = shared[ref].frame; 295189699Sdfr KASSERT(frame != 0, ("grant table inconsistent")); 296181624Skmacy 297181624Skmacy return (frame); 298181624Skmacy} 299181624Skmacy 300181624Skmacyunsigned long 301181624Skmacygnttab_end_foreign_transfer(grant_ref_t ref) 302181624Skmacy{ 303181624Skmacy unsigned long frame = gnttab_end_foreign_transfer_ref(ref); 304181624Skmacy 305181624Skmacy put_free_entry(ref); 306181624Skmacy return (frame); 307181624Skmacy} 308181624Skmacy 309181624Skmacyvoid 310181624Skmacygnttab_free_grant_reference(grant_ref_t ref) 311181624Skmacy{ 312181624Skmacy 313181624Skmacy put_free_entry(ref); 314181624Skmacy} 315181624Skmacy 316181624Skmacyvoid 317181624Skmacygnttab_free_grant_references(grant_ref_t head) 318181624Skmacy{ 319181624Skmacy grant_ref_t ref; 320181624Skmacy int count = 1; 321214077Sgibbs 322181624Skmacy if (head == GNTTAB_LIST_END) 323181624Skmacy return; 324214077Sgibbs 325181624Skmacy ref = head; 326181624Skmacy while (gnttab_entry(ref) != GNTTAB_LIST_END) { 327181624Skmacy ref = gnttab_entry(ref); 328181624Skmacy count++; 329181624Skmacy } 330214077Sgibbs mtx_lock(&gnttab_list_lock); 331181624Skmacy gnttab_entry(ref) = gnttab_free_head; 332181624Skmacy gnttab_free_head = head; 333181624Skmacy gnttab_free_count += count; 334181624Skmacy check_free_callbacks(); 335181624Skmacy mtx_unlock(&gnttab_list_lock); 336181624Skmacy} 337181624Skmacy 338181624Skmacyint 339181624Skmacygnttab_alloc_grant_references(uint16_t count, grant_ref_t *head) 340181624Skmacy{ 341186557Skmacy int ref, error; 342181624Skmacy 343186557Skmacy error = get_free_entries(count, &ref); 344255040Sgibbs if (__predict_false(error)) 345186557Skmacy return (error); 346181624Skmacy 347186557Skmacy *head = ref; 348186557Skmacy return (0); 349181624Skmacy} 350181624Skmacy 351181624Skmacyint 352181624Skmacygnttab_empty_grant_references(const grant_ref_t *private_head) 353181624Skmacy{ 354189699Sdfr 355181624Skmacy return (*private_head == GNTTAB_LIST_END); 356181624Skmacy} 357181624Skmacy 358181624Skmacyint 359181624Skmacygnttab_claim_grant_reference(grant_ref_t *private_head) 360181624Skmacy{ 361181624Skmacy grant_ref_t g = *private_head; 362181624Skmacy 363255040Sgibbs if (__predict_false(g == GNTTAB_LIST_END)) 364201234Sgibbs return (g); 365181624Skmacy *private_head = gnttab_entry(g); 366181624Skmacy return (g); 367181624Skmacy} 368181624Skmacy 369181624Skmacyvoid 370181624Skmacygnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release) 371181624Skmacy{ 372189699Sdfr 373181624Skmacy gnttab_entry(release) = *private_head; 374181624Skmacy *private_head = release; 375181624Skmacy} 376181624Skmacy 377181624Skmacyvoid 378181624Skmacygnttab_request_free_callback(struct gnttab_free_callback *callback, 379189699Sdfr void (*fn)(void *), void *arg, uint16_t count) 380181624Skmacy{ 381181624Skmacy 382181624Skmacy mtx_lock(&gnttab_list_lock); 383181624Skmacy if (callback->next) 384181624Skmacy goto out; 385181624Skmacy callback->fn = fn; 386181624Skmacy callback->arg = arg; 387181624Skmacy callback->count = count; 388181624Skmacy callback->next = gnttab_free_callback_list; 389181624Skmacy gnttab_free_callback_list = callback; 390181624Skmacy check_free_callbacks(); 391181624Skmacy out: 392181624Skmacy mtx_unlock(&gnttab_list_lock); 393181624Skmacy 394181624Skmacy} 395181624Skmacy 396181624Skmacyvoid 397181624Skmacygnttab_cancel_free_callback(struct gnttab_free_callback *callback) 398181624Skmacy{ 399181624Skmacy struct gnttab_free_callback **pcb; 400181624Skmacy 401181624Skmacy mtx_lock(&gnttab_list_lock); 402181624Skmacy for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 403181624Skmacy if (*pcb == callback) { 404181624Skmacy *pcb = callback->next; 405181624Skmacy break; 406181624Skmacy } 407181624Skmacy } 408181624Skmacy mtx_unlock(&gnttab_list_lock); 409181624Skmacy} 410181624Skmacy 411181624Skmacy 412181624Skmacystatic int 413181624Skmacygrow_gnttab_list(unsigned int more_frames) 414181624Skmacy{ 415181624Skmacy unsigned int new_nr_grant_frames, extra_entries, i; 416181624Skmacy 417181624Skmacy new_nr_grant_frames = nr_grant_frames + more_frames; 418181624Skmacy extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 419181624Skmacy 420181624Skmacy for (i = nr_grant_frames; i < new_nr_grant_frames; i++) 421181624Skmacy { 422189699Sdfr gnttab_list[i] = (grant_ref_t *) 423189699Sdfr malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT); 424181624Skmacy 425181624Skmacy if (!gnttab_list[i]) 426181624Skmacy goto grow_nomem; 427181624Skmacy } 428181624Skmacy 429181624Skmacy for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 430181624Skmacy i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 431181624Skmacy gnttab_entry(i) = i + 1; 432181624Skmacy 433181624Skmacy gnttab_entry(i) = gnttab_free_head; 434181624Skmacy gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 435181624Skmacy gnttab_free_count += extra_entries; 436181624Skmacy 437181624Skmacy nr_grant_frames = new_nr_grant_frames; 438181624Skmacy 439181624Skmacy check_free_callbacks(); 440181624Skmacy 441189699Sdfr return (0); 442214077Sgibbs 443181624Skmacygrow_nomem: 444181624Skmacy for ( ; i >= nr_grant_frames; i--) 445181624Skmacy free(gnttab_list[i], M_DEVBUF); 446189699Sdfr return (ENOMEM); 447181624Skmacy} 448181624Skmacy 449181624Skmacystatic unsigned int 450181624Skmacy__max_nr_grant_frames(void) 451181624Skmacy{ 452181624Skmacy struct gnttab_query_size query; 453181624Skmacy int rc; 454181624Skmacy 455181624Skmacy query.dom = DOMID_SELF; 456181624Skmacy 457181624Skmacy rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 458181624Skmacy if ((rc < 0) || (query.status != GNTST_okay)) 459181624Skmacy return (4); /* Legacy max supported number of frames */ 460181624Skmacy 461181624Skmacy return (query.max_nr_frames); 462181624Skmacy} 463181624Skmacy 464181624Skmacystatic inline 465181624Skmacyunsigned int max_nr_grant_frames(void) 466181624Skmacy{ 467181624Skmacy unsigned int xen_max = __max_nr_grant_frames(); 468181624Skmacy 469181624Skmacy if (xen_max > boot_max_nr_grant_frames) 470181624Skmacy return (boot_max_nr_grant_frames); 471181624Skmacy return (xen_max); 472181624Skmacy} 473181624Skmacy 474181624Skmacy#ifdef notyet 475181624Skmacy/* 476181624Skmacy * XXX needed for backend support 477181624Skmacy * 478181624Skmacy */ 479181624Skmacystatic int 480181624Skmacymap_pte_fn(pte_t *pte, struct page *pmd_page, 481181624Skmacy unsigned long addr, void *data) 482181624Skmacy{ 483181624Skmacy unsigned long **frames = (unsigned long **)data; 484181624Skmacy 485181624Skmacy set_pte_at(&init_mm, addr, pte, pfn_pte_ma((*frames)[0], PAGE_KERNEL)); 486181624Skmacy (*frames)++; 487181624Skmacy return 0; 488181624Skmacy} 489181624Skmacy 490181624Skmacystatic int 491181624Skmacyunmap_pte_fn(pte_t *pte, struct page *pmd_page, 492181624Skmacy unsigned long addr, void *data) 493181624Skmacy{ 494181624Skmacy 495181624Skmacy set_pte_at(&init_mm, addr, pte, __pte(0)); 496181624Skmacy return 0; 497181624Skmacy} 498181624Skmacy#endif 499181624Skmacy 500189699Sdfr#ifndef XENHVM 501189699Sdfr 502181624Skmacystatic int 503181624Skmacygnttab_map(unsigned int start_idx, unsigned int end_idx) 504181624Skmacy{ 505181624Skmacy struct gnttab_setup_table setup; 506183375Skmacy u_long *frames; 507183375Skmacy 508181624Skmacy unsigned int nr_gframes = end_idx + 1; 509181624Skmacy int i, rc; 510181624Skmacy 511181624Skmacy frames = malloc(nr_gframes * sizeof(unsigned long), M_DEVBUF, M_NOWAIT); 512181624Skmacy if (!frames) 513186557Skmacy return (ENOMEM); 514181624Skmacy 515181624Skmacy setup.dom = DOMID_SELF; 516181624Skmacy setup.nr_frames = nr_gframes; 517181624Skmacy set_xen_guest_handle(setup.frame_list, frames); 518181624Skmacy 519181624Skmacy rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1); 520181624Skmacy if (rc == -ENOSYS) { 521181624Skmacy free(frames, M_DEVBUF); 522186557Skmacy return (ENOSYS); 523181624Skmacy } 524189699Sdfr KASSERT(!(rc || setup.status), 525189699Sdfr ("unexpected result from grant_table_op")); 526181624Skmacy 527181624Skmacy if (shared == NULL) { 528181624Skmacy vm_offset_t area; 529214077Sgibbs 530254025Sjeff area = kva_alloc(PAGE_SIZE * max_nr_grant_frames()); 531189699Sdfr KASSERT(area, ("can't allocate VM space for grant table")); 532181624Skmacy shared = (grant_entry_t *)area; 533181624Skmacy } 534189699Sdfr 535181624Skmacy for (i = 0; i < nr_gframes; i++) 536181624Skmacy PT_SET_MA(((caddr_t)shared) + i*PAGE_SIZE, 537181804Skmacy ((vm_paddr_t)frames[i]) << PAGE_SHIFT | PG_RW | PG_V); 538181624Skmacy 539181624Skmacy free(frames, M_DEVBUF); 540214077Sgibbs 541189699Sdfr return (0); 542181624Skmacy} 543181624Skmacy 544181624Skmacyint 545181624Skmacygnttab_resume(void) 546181624Skmacy{ 547189699Sdfr 548181624Skmacy if (max_nr_grant_frames() < nr_grant_frames) 549189699Sdfr return (ENOSYS); 550189699Sdfr return (gnttab_map(0, nr_grant_frames - 1)); 551181624Skmacy} 552181624Skmacy 553181624Skmacyint 554181624Skmacygnttab_suspend(void) 555214077Sgibbs{ 556189699Sdfr int i; 557181624Skmacy 558189699Sdfr for (i = 0; i < nr_grant_frames; i++) 559189699Sdfr pmap_kremove((vm_offset_t) shared + i * PAGE_SIZE); 560181624Skmacy 561189699Sdfr return (0); 562189699Sdfr} 563181624Skmacy 564189699Sdfr#else /* XENHVM */ 565189699Sdfr 566189699Sdfr#include <dev/xen/xenpci/xenpcivar.h> 567189699Sdfr 568189699Sdfrstatic vm_paddr_t resume_frames; 569189699Sdfr 570214077Sgibbsstatic int 571214077Sgibbsgnttab_map(unsigned int start_idx, unsigned int end_idx) 572189699Sdfr{ 573189699Sdfr struct xen_add_to_physmap xatp; 574189699Sdfr unsigned int i = end_idx; 575189699Sdfr 576189699Sdfr /* 577189699Sdfr * Loop backwards, so that the first hypercall has the largest index, 578189699Sdfr * ensuring that the table will grow only once. 579189699Sdfr */ 580189699Sdfr do { 581189699Sdfr xatp.domid = DOMID_SELF; 582189699Sdfr xatp.idx = i; 583189699Sdfr xatp.space = XENMAPSPACE_grant_table; 584189699Sdfr xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i; 585189699Sdfr if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp)) 586189699Sdfr panic("HYPERVISOR_memory_op failed to map gnttab"); 587189699Sdfr } while (i-- > start_idx); 588189699Sdfr 589189699Sdfr if (shared == NULL) { 590189699Sdfr vm_offset_t area; 591214077Sgibbs 592254025Sjeff area = kva_alloc(PAGE_SIZE * max_nr_grant_frames()); 593189699Sdfr KASSERT(area, ("can't allocate VM space for grant table")); 594189699Sdfr shared = (grant_entry_t *)area; 595189699Sdfr } 596189699Sdfr 597189699Sdfr for (i = start_idx; i <= end_idx; i++) { 598189699Sdfr pmap_kenter((vm_offset_t) shared + i * PAGE_SIZE, 599189699Sdfr resume_frames + i * PAGE_SIZE); 600189699Sdfr } 601189699Sdfr 602181624Skmacy return (0); 603181624Skmacy} 604181624Skmacy 605189699Sdfrint 606189699Sdfrgnttab_resume(void) 607189699Sdfr{ 608189699Sdfr int error; 609189699Sdfr unsigned int max_nr_gframes, nr_gframes; 610189699Sdfr 611189699Sdfr nr_gframes = nr_grant_frames; 612189699Sdfr max_nr_gframes = max_nr_grant_frames(); 613189699Sdfr if (max_nr_gframes < nr_gframes) 614189699Sdfr return (ENOSYS); 615189699Sdfr 616189699Sdfr if (!resume_frames) { 617189699Sdfr error = xenpci_alloc_space(PAGE_SIZE * max_nr_gframes, 618189699Sdfr &resume_frames); 619189699Sdfr if (error) { 620189699Sdfr printf("error mapping gnttab share frames\n"); 621189699Sdfr return (error); 622189699Sdfr } 623189699Sdfr } 624189699Sdfr 625189699Sdfr return (gnttab_map(0, nr_gframes - 1)); 626189699Sdfr} 627189699Sdfr 628189699Sdfr#endif 629189699Sdfr 630181624Skmacystatic int 631181624Skmacygnttab_expand(unsigned int req_entries) 632181624Skmacy{ 633189699Sdfr int error; 634181624Skmacy unsigned int cur, extra; 635181624Skmacy 636181624Skmacy cur = nr_grant_frames; 637181624Skmacy extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) / 638181624Skmacy GREFS_PER_GRANT_FRAME); 639181624Skmacy if (cur + extra > max_nr_grant_frames()) 640186557Skmacy return (ENOSPC); 641181624Skmacy 642189699Sdfr error = gnttab_map(cur, cur + extra - 1); 643189699Sdfr if (!error) 644189699Sdfr error = grow_gnttab_list(extra); 645181624Skmacy 646189699Sdfr return (error); 647181624Skmacy} 648181624Skmacy 649185605Skmacyint 650185605Skmacygnttab_init() 651181624Skmacy{ 652181624Skmacy int i; 653181624Skmacy unsigned int max_nr_glist_frames; 654181624Skmacy unsigned int nr_init_grefs; 655181624Skmacy 656181624Skmacy if (!is_running_on_xen()) 657189699Sdfr return (ENODEV); 658181624Skmacy 659181624Skmacy nr_grant_frames = 1; 660181624Skmacy boot_max_nr_grant_frames = __max_nr_grant_frames(); 661181624Skmacy 662181624Skmacy /* Determine the maximum number of frames required for the 663181624Skmacy * grant reference free list on the current hypervisor. 664181624Skmacy */ 665181624Skmacy max_nr_glist_frames = (boot_max_nr_grant_frames * 666181624Skmacy GREFS_PER_GRANT_FRAME / 667181624Skmacy (PAGE_SIZE / sizeof(grant_ref_t))); 668181624Skmacy 669181624Skmacy gnttab_list = malloc(max_nr_glist_frames * sizeof(grant_ref_t *), 670181624Skmacy M_DEVBUF, M_NOWAIT); 671181624Skmacy 672181624Skmacy if (gnttab_list == NULL) 673186557Skmacy return (ENOMEM); 674181624Skmacy 675181624Skmacy for (i = 0; i < nr_grant_frames; i++) { 676189699Sdfr gnttab_list[i] = (grant_ref_t *) 677189699Sdfr malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT); 678181624Skmacy if (gnttab_list[i] == NULL) 679181624Skmacy goto ini_nomem; 680181624Skmacy } 681214077Sgibbs 682186557Skmacy if (gnttab_resume()) 683186557Skmacy return (ENODEV); 684214077Sgibbs 685181624Skmacy nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 686181624Skmacy 687181624Skmacy for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 688181624Skmacy gnttab_entry(i) = i + 1; 689181624Skmacy 690181624Skmacy gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END; 691181624Skmacy gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 692181624Skmacy gnttab_free_head = NR_RESERVED_ENTRIES; 693181624Skmacy 694189699Sdfr if (bootverbose) 695189699Sdfr printf("Grant table initialized\n"); 696181624Skmacy 697189699Sdfr return (0); 698189699Sdfr 699181624Skmacyini_nomem: 700181624Skmacy for (i--; i >= 0; i--) 701181624Skmacy free(gnttab_list[i], M_DEVBUF); 702181624Skmacy free(gnttab_list, M_DEVBUF); 703186557Skmacy return (ENOMEM); 704181624Skmacy 705181624Skmacy} 706181624Skmacy 707181624SkmacyMTX_SYSINIT(gnttab, &gnttab_list_lock, "GNTTAB LOCK", MTX_DEF); 708