1/* $OpenBSD: virtio_mmio.c,v 1.13 2024/05/17 16:37:10 sf Exp $ */ 2/* $NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $ */ 3 4/* 5 * Copyright (c) 2014 Patrick Wildt <patrick@blueri.se> 6 * Copyright (c) 2012 Stefan Fritsch. 7 * Copyright (c) 2010 Minoura Makoto. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31#include <sys/param.h> 32#include <sys/systm.h> 33#include <sys/kernel.h> 34#include <sys/device.h> 35#include <sys/mutex.h> 36 37#include <dev/pv/virtioreg.h> 38#include <dev/pv/virtiovar.h> 39 40#include <machine/fdt.h> 41 42#include <dev/ofw/fdt.h> 43#include <dev/ofw/openfirm.h> 44 45#define VIRTIO_MMIO_MAGIC ('v' | 'i' << 8 | 'r' << 16 | 't' << 24) 46 47#define VIRTIO_MMIO_MAGIC_VALUE 0x000 48#define VIRTIO_MMIO_VERSION 0x004 49#define VIRTIO_MMIO_DEVICE_ID 0x008 50#define VIRTIO_MMIO_VENDOR_ID 0x00c 51#define VIRTIO_MMIO_HOST_FEATURES 0x010 52#define VIRTIO_MMIO_HOST_FEATURES_SEL 0x014 53#define VIRTIO_MMIO_GUEST_FEATURES 0x020 54#define VIRTIO_MMIO_GUEST_FEATURES_SEL 0x024 55#define VIRTIO_MMIO_GUEST_PAGE_SIZE 0x028 56#define VIRTIO_MMIO_QUEUE_SEL 0x030 57#define VIRTIO_MMIO_QUEUE_NUM_MAX 0x034 58#define VIRTIO_MMIO_QUEUE_NUM 0x038 59#define VIRTIO_MMIO_QUEUE_ALIGN 0x03c 60#define VIRTIO_MMIO_QUEUE_PFN 0x040 61#define VIRTIO_MMIO_QUEUE_READY 0x044 62#define VIRTIO_MMIO_QUEUE_NOTIFY 0x050 63#define VIRTIO_MMIO_INTERRUPT_STATUS 0x060 64#define VIRTIO_MMIO_INTERRUPT_ACK 0x064 65#define VIRTIO_MMIO_STATUS 0x070 66#define VIRTIO_MMIO_QUEUE_DESC_LOW 0x080 67#define VIRTIO_MMIO_QUEUE_DESC_HIGH 0x084 68#define VIRTIO_MMIO_QUEUE_AVAIL_LOW 0x090 69#define VIRTIO_MMIO_QUEUE_AVAIL_HIGH 0x094 70#define VIRTIO_MMIO_QUEUE_USED_LOW 0x0a0 71#define VIRTIO_MMIO_QUEUE_USED_HIGH 0x0a4 72#define VIRTIO_MMIO_CONFIG 0x100 73 74#define VIRTIO_MMIO_INT_VRING (1 << 0) 75#define VIRTIO_MMIO_INT_CONFIG (1 << 1) 76 77#define DEVNAME(sc) (sc)->sc_dev.dv_xname 78 79/* 80 * XXX: Before being used on big endian arches, the access to config registers 81 * XXX: needs to be reviewed/fixed. The non-device specific registers are 82 * XXX: PCI-endian while the device specific registers are native endian. 83 */ 84 85int virtio_mmio_match(struct device *, void *, void *); 86void virtio_mmio_attach(struct device *, struct device *, void *); 87int virtio_mmio_detach(struct device *, int); 88 89void virtio_mmio_kick(struct virtio_softc *, uint16_t); 90uint8_t virtio_mmio_read_device_config_1(struct virtio_softc *, int); 91uint16_t virtio_mmio_read_device_config_2(struct virtio_softc *, int); 92uint32_t virtio_mmio_read_device_config_4(struct virtio_softc *, int); 93uint64_t virtio_mmio_read_device_config_8(struct virtio_softc *, int); 94void virtio_mmio_write_device_config_1(struct virtio_softc *, int, uint8_t); 95void virtio_mmio_write_device_config_2(struct virtio_softc *, int, uint16_t); 96void virtio_mmio_write_device_config_4(struct virtio_softc *, int, uint32_t); 97void virtio_mmio_write_device_config_8(struct virtio_softc *, int, uint64_t); 98uint16_t virtio_mmio_read_queue_size(struct virtio_softc *, uint16_t); 99void virtio_mmio_setup_queue(struct virtio_softc *, struct virtqueue *, uint64_t); 100int virtio_mmio_get_status(struct virtio_softc *); 101void virtio_mmio_set_status(struct virtio_softc *, int); 102int virtio_mmio_negotiate_features(struct virtio_softc *, 103 const struct virtio_feature_name *); 104int virtio_mmio_intr(void *); 105 106struct virtio_mmio_softc { 107 struct virtio_softc sc_sc; 108 109 bus_space_tag_t sc_iot; 110 bus_space_handle_t sc_ioh; 111 bus_size_t sc_iosize; 112 bus_dma_tag_t sc_dmat; 113 114 void *sc_ih; 115 116 int sc_config_offset; 117 uint32_t sc_version; 118}; 119 120const struct cfattach virtio_mmio_ca = { 121 sizeof(struct virtio_mmio_softc), 122 virtio_mmio_match, 123 virtio_mmio_attach, 124 virtio_mmio_detach, 125 NULL 126}; 127 128const struct cfattach virtio_mmio_fdt_ca = { 129 sizeof(struct virtio_mmio_softc), 130 NULL, 131 virtio_mmio_attach, 132 virtio_mmio_detach, 133 NULL 134}; 135 136struct virtio_ops virtio_mmio_ops = { 137 virtio_mmio_kick, 138 virtio_mmio_read_device_config_1, 139 virtio_mmio_read_device_config_2, 140 virtio_mmio_read_device_config_4, 141 virtio_mmio_read_device_config_8, 142 virtio_mmio_write_device_config_1, 143 virtio_mmio_write_device_config_2, 144 virtio_mmio_write_device_config_4, 145 virtio_mmio_write_device_config_8, 146 virtio_mmio_read_queue_size, 147 virtio_mmio_setup_queue, 148 virtio_mmio_get_status, 149 virtio_mmio_set_status, 150 virtio_mmio_negotiate_features, 151 virtio_mmio_intr, 152}; 153 154uint16_t 155virtio_mmio_read_queue_size(struct virtio_softc *vsc, uint16_t idx) 156{ 157 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 158 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_SEL, idx); 159 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 160 VIRTIO_MMIO_QUEUE_NUM_MAX); 161} 162 163void 164virtio_mmio_setup_queue(struct virtio_softc *vsc, struct virtqueue *vq, 165 uint64_t addr) 166{ 167 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 168 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_SEL, 169 vq->vq_index); 170 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_NUM, 171 bus_space_read_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_NUM_MAX)); 172 if (sc->sc_version == 1) { 173 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 174 VIRTIO_MMIO_QUEUE_ALIGN, PAGE_SIZE); 175 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 176 VIRTIO_MMIO_QUEUE_PFN, addr / VIRTIO_PAGE_SIZE); 177 } else { 178 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 179 VIRTIO_MMIO_QUEUE_DESC_LOW, addr); 180 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 181 VIRTIO_MMIO_QUEUE_DESC_HIGH, addr >> 32); 182 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 183 VIRTIO_MMIO_QUEUE_AVAIL_LOW, 184 addr + vq->vq_availoffset); 185 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 186 VIRTIO_MMIO_QUEUE_AVAIL_HIGH, 187 (addr + vq->vq_availoffset) >> 32); 188 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 189 VIRTIO_MMIO_QUEUE_USED_LOW, 190 addr + vq->vq_usedoffset); 191 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 192 VIRTIO_MMIO_QUEUE_USED_HIGH, 193 (addr + vq->vq_usedoffset) >> 32); 194 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 195 VIRTIO_MMIO_QUEUE_READY, 1); 196 } 197} 198 199int 200virtio_mmio_get_status(struct virtio_softc *vsc) 201{ 202 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 203 204 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 205 VIRTIO_MMIO_STATUS); 206} 207 208void 209virtio_mmio_set_status(struct virtio_softc *vsc, int status) 210{ 211 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 212 int old = 0; 213 214 if (status == 0) { 215 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_STATUS, 216 0); 217 while (bus_space_read_4(sc->sc_iot, sc->sc_ioh, 218 VIRTIO_MMIO_STATUS) != 0) { 219 CPU_BUSY_CYCLE(); 220 } 221 } else { 222 old = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 223 VIRTIO_MMIO_STATUS); 224 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_STATUS, 225 status|old); 226 } 227} 228 229int 230virtio_mmio_match(struct device *parent, void *cfdata, void *aux) 231{ 232 struct fdt_attach_args *faa = aux; 233 234 return OF_is_compatible(faa->fa_node, "virtio,mmio"); 235} 236 237void 238virtio_mmio_attach(struct device *parent, struct device *self, void *aux) 239{ 240 struct fdt_attach_args *faa = aux; 241 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)self; 242 struct virtio_softc *vsc = &sc->sc_sc; 243 uint32_t id, magic; 244 245 if (faa->fa_nreg < 1) { 246 printf(": no register data\n"); 247 return; 248 } 249 250 sc->sc_iosize = faa->fa_reg[0].size; 251 sc->sc_iot = faa->fa_iot; 252 sc->sc_dmat = faa->fa_dmat; 253 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, faa->fa_reg[0].size, 254 0, &sc->sc_ioh)) 255 panic("%s: bus_space_map failed!", __func__); 256 257 magic = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 258 VIRTIO_MMIO_MAGIC_VALUE); 259 if (magic != VIRTIO_MMIO_MAGIC) { 260 printf(": wrong magic value 0x%08x; giving up\n", magic); 261 return; 262 } 263 264 sc->sc_version = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 265 VIRTIO_MMIO_VERSION); 266 if (sc->sc_version < 1 || sc->sc_version > 2) { 267 printf(": unknown version 0x%02x; giving up\n", sc->sc_version); 268 return; 269 } 270 271 id = bus_space_read_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_DEVICE_ID); 272 printf(": Virtio %s Device", virtio_device_string(id)); 273 274 if (sc->sc_version == 1) 275 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 276 VIRTIO_MMIO_GUEST_PAGE_SIZE, PAGE_SIZE); 277 278 printf("\n"); 279 280 /* No device connected. */ 281 if (id == 0) 282 return; 283 284 vsc->sc_ops = &virtio_mmio_ops; 285 vsc->sc_dmat = sc->sc_dmat; 286 sc->sc_config_offset = VIRTIO_MMIO_CONFIG; 287 288 virtio_device_reset(vsc); 289 virtio_mmio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 290 virtio_mmio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 291 292 /* XXX: use softc as aux... */ 293 vsc->sc_childdevid = id; 294 vsc->sc_child = NULL; 295 config_found(self, sc, NULL); 296 if (vsc->sc_child == NULL) { 297 printf("%s: no matching child driver; not configured\n", 298 vsc->sc_dev.dv_xname); 299 goto fail_1; 300 } 301 if (vsc->sc_child == VIRTIO_CHILD_ERROR) { 302 printf("%s: virtio configuration failed\n", 303 vsc->sc_dev.dv_xname); 304 goto fail_1; 305 } 306 307 sc->sc_ih = fdt_intr_establish(faa->fa_node, vsc->sc_ipl, 308 virtio_mmio_intr, sc, vsc->sc_dev.dv_xname); 309 if (sc->sc_ih == NULL) { 310 printf("%s: couldn't establish interrupt\n", 311 vsc->sc_dev.dv_xname); 312 goto fail_2; 313 } 314 315 return; 316 317fail_2: 318 config_detach(vsc->sc_child, 0); 319fail_1: 320 /* no mmio_mapreg_unmap() or mmio_intr_unmap() */ 321 virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 322} 323 324int 325virtio_mmio_detach(struct device *self, int flags) 326{ 327 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)self; 328 struct virtio_softc *vsc = &sc->sc_sc; 329 int r; 330 331 if (vsc->sc_child != 0 && vsc->sc_child != VIRTIO_CHILD_ERROR) { 332 r = config_detach(vsc->sc_child, flags); 333 if (r) 334 return r; 335 } 336 KASSERT(vsc->sc_child == 0 || vsc->sc_child == VIRTIO_CHILD_ERROR); 337 KASSERT(vsc->sc_vqs == 0); 338 fdt_intr_disestablish(sc->sc_ih); 339 sc->sc_ih = 0; 340 if (sc->sc_iosize) 341 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 342 sc->sc_iosize = 0; 343 344 return 0; 345} 346 347/* 348 * Feature negotiation. 349 * Prints available / negotiated features if guest_feature_names != NULL and 350 * VIRTIO_DEBUG is 1 351 */ 352int 353virtio_mmio_negotiate_features(struct virtio_softc *vsc, 354 const struct virtio_feature_name *guest_feature_names) 355{ 356 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 357 uint64_t host, neg; 358 359 vsc->sc_active_features = 0; 360 361 /* 362 * We enable indirect descriptors by default. They can be switched 363 * off by setting bit 1 in the driver flags, see config(8). 364 */ 365 if (!(vsc->sc_dev.dv_cfdata->cf_flags & VIRTIO_CF_NO_INDIRECT) && 366 !(vsc->sc_child->dv_cfdata->cf_flags & VIRTIO_CF_NO_INDIRECT)) { 367 vsc->sc_driver_features |= VIRTIO_F_RING_INDIRECT_DESC; 368 } else if (guest_feature_names != NULL) { 369 printf("RingIndirectDesc disabled by UKC\n"); 370 } 371 /* 372 * The driver must add VIRTIO_F_RING_EVENT_IDX if it supports it. 373 * If it did, check if it is disabled by bit 2 in the driver flags. 374 */ 375 if ((vsc->sc_driver_features & VIRTIO_F_RING_EVENT_IDX) && 376 ((vsc->sc_dev.dv_cfdata->cf_flags & VIRTIO_CF_NO_EVENT_IDX) || 377 (vsc->sc_child->dv_cfdata->cf_flags & VIRTIO_CF_NO_EVENT_IDX))) { 378 if (guest_feature_names != NULL) 379 printf(" RingEventIdx disabled by UKC"); 380 vsc->sc_driver_features &= ~(VIRTIO_F_RING_EVENT_IDX); 381 } 382 383 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 384 VIRTIO_MMIO_HOST_FEATURES_SEL, 0); 385 host = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 386 VIRTIO_MMIO_HOST_FEATURES); 387 neg = host & vsc->sc_driver_features; 388#if VIRTIO_DEBUG 389 if (guest_feature_names) 390 virtio_log_features(host, neg, guest_feature_names); 391#endif 392 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 393 VIRTIO_MMIO_GUEST_FEATURES_SEL, 0); 394 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 395 VIRTIO_MMIO_GUEST_FEATURES, neg); 396 vsc->sc_active_features = neg; 397 if (neg & VIRTIO_F_RING_INDIRECT_DESC) 398 vsc->sc_indirect = 1; 399 else 400 vsc->sc_indirect = 0; 401 402 return 0; 403} 404 405/* 406 * Device configuration registers. 407 */ 408uint8_t 409virtio_mmio_read_device_config_1(struct virtio_softc *vsc, int index) 410{ 411 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 412 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 413 sc->sc_config_offset + index); 414} 415 416uint16_t 417virtio_mmio_read_device_config_2(struct virtio_softc *vsc, int index) 418{ 419 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 420 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 421 sc->sc_config_offset + index); 422} 423 424uint32_t 425virtio_mmio_read_device_config_4(struct virtio_softc *vsc, int index) 426{ 427 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 428 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 429 sc->sc_config_offset + index); 430} 431 432uint64_t 433virtio_mmio_read_device_config_8(struct virtio_softc *vsc, int index) 434{ 435 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 436 uint64_t r; 437 438 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 439 sc->sc_config_offset + index + sizeof(uint32_t)); 440 r <<= 32; 441 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 442 sc->sc_config_offset + index); 443 return r; 444} 445 446void 447virtio_mmio_write_device_config_1(struct virtio_softc *vsc, 448 int index, uint8_t value) 449{ 450 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 451 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 452 sc->sc_config_offset + index, value); 453} 454 455void 456virtio_mmio_write_device_config_2(struct virtio_softc *vsc, 457 int index, uint16_t value) 458{ 459 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 460 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 461 sc->sc_config_offset + index, value); 462} 463 464void 465virtio_mmio_write_device_config_4(struct virtio_softc *vsc, 466 int index, uint32_t value) 467{ 468 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 469 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 470 sc->sc_config_offset + index, value); 471} 472 473void 474virtio_mmio_write_device_config_8(struct virtio_softc *vsc, 475 int index, uint64_t value) 476{ 477 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 478 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 479 sc->sc_config_offset + index, 480 value & 0xffffffff); 481 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 482 sc->sc_config_offset + index + sizeof(uint32_t), 483 value >> 32); 484} 485 486/* 487 * Interrupt handler. 488 */ 489int 490virtio_mmio_intr(void *arg) 491{ 492 struct virtio_mmio_softc *sc = arg; 493 struct virtio_softc *vsc = &sc->sc_sc; 494 int isr, r = 0; 495 496 /* check and ack the interrupt */ 497 isr = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 498 VIRTIO_MMIO_INTERRUPT_STATUS); 499 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 500 VIRTIO_MMIO_INTERRUPT_ACK, isr); 501 if ((isr & VIRTIO_MMIO_INT_CONFIG) && 502 (vsc->sc_config_change != NULL)) 503 r = (vsc->sc_config_change)(vsc); 504 if ((isr & VIRTIO_MMIO_INT_VRING)) 505 r |= virtio_check_vqs(vsc); 506 507 return r; 508} 509 510void 511virtio_mmio_kick(struct virtio_softc *vsc, uint16_t idx) 512{ 513 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 514 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_NOTIFY, 515 idx); 516} 517