1/* 2 * Copyright (c) 2014 Roger Pau Monn�� <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD$"); 29 30#include <sys/param.h> 31#include <sys/bus.h> 32#include <sys/kernel.h> 33#include <sys/malloc.h> 34#include <sys/proc.h> 35#include <sys/smp.h> 36#include <sys/systm.h> 37 38#include <vm/vm.h> 39#include <vm/pmap.h> 40 41#include <machine/cpufunc.h> 42#include <machine/cpu.h> 43#include <machine/intr_machdep.h> 44#include <machine/md_var.h> 45#include <machine/smp.h> 46 47#include <x86/apicreg.h> 48#include <x86/apicvar.h> 49 50#include <xen/xen-os.h> 51#include <xen/features.h> 52#include <xen/gnttab.h> 53#include <xen/hypervisor.h> 54#include <xen/hvm.h> 55#include <xen/xen_intr.h> 56 57#include <xen/interface/vcpu.h> 58 59/*--------------------------------- Macros -----------------------------------*/ 60 61#define XEN_APIC_UNSUPPORTED \ 62 panic("%s: not available in Xen PV port.", __func__) 63 64 65/*--------------------------- Forward Declarations ---------------------------*/ 66#ifdef SMP 67static driver_filter_t xen_smp_rendezvous_action; 68static driver_filter_t xen_invltlb; 69static driver_filter_t xen_invlpg; 70static driver_filter_t xen_invlrng; 71static driver_filter_t xen_invlcache; 72static driver_filter_t xen_ipi_bitmap_handler; 73static driver_filter_t xen_cpustop_handler; 74static driver_filter_t xen_cpususpend_handler; 75static driver_filter_t xen_ipi_swi_handler; 76#endif 77 78/*---------------------------------- Macros ----------------------------------*/ 79#define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 80 81/*--------------------------------- Xen IPIs ---------------------------------*/ 82#ifdef SMP 83struct xen_ipi_handler 84{ 85 driver_filter_t *filter; 86 const char *description; 87}; 88 89static struct xen_ipi_handler xen_ipis[] = 90{ 91 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 92 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 93 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 94 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 95 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 96 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 97 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 98 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 99 [IPI_TO_IDX(IPI_SWI)] = { xen_ipi_swi_handler, "sw" }, 100}; 101#endif 102 103/*------------------------------- Per-CPU Data -------------------------------*/ 104#ifdef SMP 105DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 106#endif 107 108/*------------------------------- Xen PV APIC --------------------------------*/ 109 110static void 111xen_pv_lapic_create(u_int apic_id, int boot_cpu) 112{ 113#ifdef SMP 114 cpu_add(apic_id, boot_cpu); 115#endif 116} 117 118static void 119xen_pv_lapic_init(vm_paddr_t addr) 120{ 121 122} 123 124static void 125xen_pv_lapic_setup(int boot) 126{ 127 128} 129 130static void 131xen_pv_lapic_dump(const char *str) 132{ 133 134 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 135} 136 137static void 138xen_pv_lapic_disable(void) 139{ 140 141} 142 143static bool 144xen_pv_lapic_is_x2apic(void) 145{ 146 147 return (false); 148} 149 150static void 151xen_pv_lapic_eoi(void) 152{ 153 154 XEN_APIC_UNSUPPORTED; 155} 156 157static int 158xen_pv_lapic_id(void) 159{ 160 161 return (PCPU_GET(apic_id)); 162} 163 164static int 165xen_pv_lapic_intr_pending(u_int vector) 166{ 167 168 XEN_APIC_UNSUPPORTED; 169 return (0); 170} 171 172static u_int 173xen_pv_apic_cpuid(u_int apic_id) 174{ 175#ifdef SMP 176 return (apic_cpuids[apic_id]); 177#else 178 return (0); 179#endif 180} 181 182static u_int 183xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 184{ 185 186 XEN_APIC_UNSUPPORTED; 187 return (0); 188} 189 190static u_int 191xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 192{ 193 194 XEN_APIC_UNSUPPORTED; 195 return (0); 196} 197 198static void 199xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 200{ 201 202 XEN_APIC_UNSUPPORTED; 203} 204 205static void 206xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 207{ 208 209 XEN_APIC_UNSUPPORTED; 210} 211 212static void 213xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 214{ 215 216 XEN_APIC_UNSUPPORTED; 217} 218 219static void 220xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 221{ 222 223 XEN_APIC_UNSUPPORTED; 224} 225 226static int 227xen_pv_lapic_enable_pmc(void) 228{ 229 230 XEN_APIC_UNSUPPORTED; 231 return (0); 232} 233 234static void 235xen_pv_lapic_disable_pmc(void) 236{ 237 238 XEN_APIC_UNSUPPORTED; 239} 240 241static void 242xen_pv_lapic_reenable_pmc(void) 243{ 244 245 XEN_APIC_UNSUPPORTED; 246} 247 248static void 249xen_pv_lapic_enable_cmc(void) 250{ 251 252} 253 254#ifdef SMP 255static void 256xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 257{ 258 259 XEN_APIC_UNSUPPORTED; 260} 261 262#define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field) 263static void 264send_nmi(int dest) 265{ 266 unsigned int cpu; 267 268 /* 269 * NMIs are not routed over event channels, and instead delivered as on 270 * native using the exception vector (#2). Triggering them can be done 271 * using the local APIC, or an hypercall as a shortcut like it's done 272 * below. 273 */ 274 switch(dest) { 275 case APIC_IPI_DEST_SELF: 276 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL); 277 break; 278 case APIC_IPI_DEST_ALL: 279 CPU_FOREACH(cpu) 280 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 281 PCPU_ID_GET(cpu, vcpu_id), NULL); 282 break; 283 case APIC_IPI_DEST_OTHERS: 284 CPU_FOREACH(cpu) 285 if (cpu != PCPU_GET(cpuid)) 286 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 287 PCPU_ID_GET(cpu, vcpu_id), NULL); 288 break; 289 default: 290 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 291 PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL); 292 break; 293 } 294} 295#undef PCPU_ID_GET 296 297static void 298xen_pv_lapic_ipi_vectored(u_int vector, int dest) 299{ 300 xen_intr_handle_t *ipi_handle; 301 int ipi_idx, to_cpu, self; 302 303 if (vector >= IPI_NMI_FIRST) { 304 send_nmi(dest); 305 return; 306 } 307 308 ipi_idx = IPI_TO_IDX(vector); 309 if (ipi_idx >= nitems(xen_ipis)) 310 panic("IPI out of range"); 311 312 switch(dest) { 313 case APIC_IPI_DEST_SELF: 314 ipi_handle = DPCPU_GET(ipi_handle); 315 xen_intr_signal(ipi_handle[ipi_idx]); 316 break; 317 case APIC_IPI_DEST_ALL: 318 CPU_FOREACH(to_cpu) { 319 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 320 xen_intr_signal(ipi_handle[ipi_idx]); 321 } 322 break; 323 case APIC_IPI_DEST_OTHERS: 324 self = PCPU_GET(cpuid); 325 CPU_FOREACH(to_cpu) { 326 if (to_cpu != self) { 327 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 328 xen_intr_signal(ipi_handle[ipi_idx]); 329 } 330 } 331 break; 332 default: 333 to_cpu = apic_cpuid(dest); 334 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 335 xen_intr_signal(ipi_handle[ipi_idx]); 336 break; 337 } 338} 339 340static int 341xen_pv_lapic_ipi_wait(int delay) 342{ 343 344 XEN_APIC_UNSUPPORTED; 345 return (0); 346} 347#endif /* SMP */ 348 349static int 350xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 351{ 352 353 XEN_APIC_UNSUPPORTED; 354 return (-1); 355} 356 357static void 358xen_pv_lapic_ipi_free(int vector) 359{ 360 361 XEN_APIC_UNSUPPORTED; 362} 363 364static int 365xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 366{ 367 368 XEN_APIC_UNSUPPORTED; 369 return (0); 370} 371 372static int 373xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 374{ 375 376 XEN_APIC_UNSUPPORTED; 377 return (0); 378} 379 380static int 381xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 382{ 383 384 XEN_APIC_UNSUPPORTED; 385 return (0); 386} 387 388static int 389xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 390 enum intr_trigger trigger) 391{ 392 393 XEN_APIC_UNSUPPORTED; 394 return (0); 395} 396 397/* Xen apic_ops implementation */ 398struct apic_ops xen_apic_ops = { 399 .create = xen_pv_lapic_create, 400 .init = xen_pv_lapic_init, 401 .xapic_mode = xen_pv_lapic_disable, 402 .is_x2apic = xen_pv_lapic_is_x2apic, 403 .setup = xen_pv_lapic_setup, 404 .dump = xen_pv_lapic_dump, 405 .disable = xen_pv_lapic_disable, 406 .eoi = xen_pv_lapic_eoi, 407 .id = xen_pv_lapic_id, 408 .intr_pending = xen_pv_lapic_intr_pending, 409 .set_logical_id = xen_pv_lapic_set_logical_id, 410 .cpuid = xen_pv_apic_cpuid, 411 .alloc_vector = xen_pv_apic_alloc_vector, 412 .alloc_vectors = xen_pv_apic_alloc_vectors, 413 .enable_vector = xen_pv_apic_enable_vector, 414 .disable_vector = xen_pv_apic_disable_vector, 415 .free_vector = xen_pv_apic_free_vector, 416 .enable_pmc = xen_pv_lapic_enable_pmc, 417 .disable_pmc = xen_pv_lapic_disable_pmc, 418 .reenable_pmc = xen_pv_lapic_reenable_pmc, 419 .enable_cmc = xen_pv_lapic_enable_cmc, 420#ifdef SMP 421 .ipi_raw = xen_pv_lapic_ipi_raw, 422 .ipi_vectored = xen_pv_lapic_ipi_vectored, 423 .ipi_wait = xen_pv_lapic_ipi_wait, 424#endif 425 .ipi_alloc = xen_pv_lapic_ipi_alloc, 426 .ipi_free = xen_pv_lapic_ipi_free, 427 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 428 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 429 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 430 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 431}; 432 433#ifdef SMP 434/*---------------------------- XEN PV IPI Handlers ---------------------------*/ 435/* 436 * These are C clones of the ASM functions found in apic_vector. 437 */ 438static int 439xen_ipi_bitmap_handler(void *arg) 440{ 441 struct trapframe *frame; 442 443 frame = arg; 444 ipi_bitmap_handler(*frame); 445 return (FILTER_HANDLED); 446} 447 448static int 449xen_smp_rendezvous_action(void *arg) 450{ 451#ifdef COUNT_IPIS 452 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 453#endif /* COUNT_IPIS */ 454 455 smp_rendezvous_action(); 456 return (FILTER_HANDLED); 457} 458 459static int 460xen_invltlb(void *arg) 461{ 462 463 invltlb_handler(); 464 return (FILTER_HANDLED); 465} 466 467#ifdef __amd64__ 468static int 469xen_invltlb_invpcid(void *arg) 470{ 471 472 invltlb_invpcid_handler(); 473 return (FILTER_HANDLED); 474} 475 476static int 477xen_invltlb_pcid(void *arg) 478{ 479 480 invltlb_pcid_handler(); 481 return (FILTER_HANDLED); 482} 483 484static int 485xen_invltlb_invpcid_pti(void *arg) 486{ 487 488 invltlb_invpcid_pti_handler(); 489 return (FILTER_HANDLED); 490} 491 492static int 493xen_invlpg_invpcid_handler(void *arg) 494{ 495 496 invlpg_invpcid_handler(); 497 return (FILTER_HANDLED); 498} 499 500static int 501xen_invlpg_pcid_handler(void *arg) 502{ 503 504 invlpg_pcid_handler(); 505 return (FILTER_HANDLED); 506} 507 508static int 509xen_invlrng_invpcid_handler(void *arg) 510{ 511 512 invlrng_invpcid_handler(); 513 return (FILTER_HANDLED); 514} 515 516static int 517xen_invlrng_pcid_handler(void *arg) 518{ 519 520 invlrng_pcid_handler(); 521 return (FILTER_HANDLED); 522} 523#endif 524 525static int 526xen_invlpg(void *arg) 527{ 528 529 invlpg_handler(); 530 return (FILTER_HANDLED); 531} 532 533static int 534xen_invlrng(void *arg) 535{ 536 537 invlrng_handler(); 538 return (FILTER_HANDLED); 539} 540 541static int 542xen_invlcache(void *arg) 543{ 544 545 invlcache_handler(); 546 return (FILTER_HANDLED); 547} 548 549static int 550xen_cpustop_handler(void *arg) 551{ 552 553 cpustop_handler(); 554 return (FILTER_HANDLED); 555} 556 557static int 558xen_cpususpend_handler(void *arg) 559{ 560 561 cpususpend_handler(); 562 return (FILTER_HANDLED); 563} 564 565static int 566xen_ipi_swi_handler(void *arg) 567{ 568 struct trapframe *frame = arg; 569 570 ipi_swi_handler(*frame); 571 return (FILTER_HANDLED); 572} 573 574/*----------------------------- XEN PV IPI setup -----------------------------*/ 575/* 576 * Those functions are provided outside of the Xen PV APIC implementation 577 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 578 * because on PVHVM there's an emulated LAPIC provided by Xen. 579 */ 580static void 581xen_cpu_ipi_init(int cpu) 582{ 583 xen_intr_handle_t *ipi_handle; 584 const struct xen_ipi_handler *ipi; 585 int idx, rc; 586 587 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 588 589 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 590 591 if (ipi->filter == NULL) { 592 ipi_handle[idx] = NULL; 593 continue; 594 } 595 596 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 597 INTR_TYPE_TTY, &ipi_handle[idx]); 598 if (rc != 0) 599 panic("Unable to allocate a XEN IPI port"); 600 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 601 } 602} 603 604static void 605xen_setup_cpus(void) 606{ 607 int i; 608 609 if (!xen_vector_callback_enabled) 610 return; 611 612#ifdef __amd64__ 613 if (pmap_pcid_enabled) { 614 if (pti) 615 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = 616 invpcid_works ? xen_invltlb_invpcid_pti : 617 xen_invltlb_pcid; 618 else 619 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = 620 invpcid_works ? xen_invltlb_invpcid : 621 xen_invltlb_pcid; 622 xen_ipis[IPI_TO_IDX(IPI_INVLPG)].filter = invpcid_works ? 623 xen_invlpg_invpcid_handler : xen_invlpg_pcid_handler; 624 xen_ipis[IPI_TO_IDX(IPI_INVLRNG)].filter = invpcid_works ? 625 xen_invlrng_invpcid_handler : xen_invlrng_pcid_handler; 626 } 627#endif 628 CPU_FOREACH(i) 629 xen_cpu_ipi_init(i); 630 631 /* Set the xen pv ipi ops to replace the native ones */ 632 if (xen_hvm_domain()) 633 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 634} 635 636/* Switch to using PV IPIs as soon as the vcpu_id is set. */ 637SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL); 638#endif /* SMP */ 639