1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (c) 2009, Microsoft Corporation. 4 * 5 * Authors: 6 * Haiyang Zhang <haiyangz@microsoft.com> 7 * Hank Janssen <hjanssen@microsoft.com> 8 */ 9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11#include <linux/kernel.h> 12#include <linux/interrupt.h> 13#include <linux/sched.h> 14#include <linux/wait.h> 15#include <linux/mm.h> 16#include <linux/slab.h> 17#include <linux/list.h> 18#include <linux/module.h> 19#include <linux/completion.h> 20#include <linux/delay.h> 21#include <linux/cpu.h> 22#include <linux/hyperv.h> 23#include <asm/mshyperv.h> 24#include <linux/sched/isolation.h> 25 26#include "hyperv_vmbus.h" 27 28static void init_vp_index(struct vmbus_channel *channel); 29 30const struct vmbus_device vmbus_devs[] = { 31 /* IDE */ 32 { .dev_type = HV_IDE, 33 HV_IDE_GUID, 34 .perf_device = true, 35 .allowed_in_isolated = false, 36 }, 37 38 /* SCSI */ 39 { .dev_type = HV_SCSI, 40 HV_SCSI_GUID, 41 .perf_device = true, 42 .allowed_in_isolated = true, 43 }, 44 45 /* Fibre Channel */ 46 { .dev_type = HV_FC, 47 HV_SYNTHFC_GUID, 48 .perf_device = true, 49 .allowed_in_isolated = false, 50 }, 51 52 /* Synthetic NIC */ 53 { .dev_type = HV_NIC, 54 HV_NIC_GUID, 55 .perf_device = true, 56 .allowed_in_isolated = true, 57 }, 58 59 /* Network Direct */ 60 { .dev_type = HV_ND, 61 HV_ND_GUID, 62 .perf_device = true, 63 .allowed_in_isolated = false, 64 }, 65 66 /* PCIE */ 67 { .dev_type = HV_PCIE, 68 HV_PCIE_GUID, 69 .perf_device = false, 70 .allowed_in_isolated = true, 71 }, 72 73 /* Synthetic Frame Buffer */ 74 { .dev_type = HV_FB, 75 HV_SYNTHVID_GUID, 76 .perf_device = false, 77 .allowed_in_isolated = false, 78 }, 79 80 /* Synthetic Keyboard */ 81 { .dev_type = HV_KBD, 82 HV_KBD_GUID, 83 .perf_device = false, 84 .allowed_in_isolated = false, 85 }, 86 87 /* Synthetic MOUSE */ 88 { .dev_type = HV_MOUSE, 89 HV_MOUSE_GUID, 90 .perf_device = false, 91 .allowed_in_isolated = false, 92 }, 93 94 /* KVP */ 95 { .dev_type = HV_KVP, 96 HV_KVP_GUID, 97 .perf_device = false, 98 .allowed_in_isolated = false, 99 }, 100 101 /* Time Synch */ 102 { .dev_type = HV_TS, 103 HV_TS_GUID, 104 .perf_device = false, 105 .allowed_in_isolated = true, 106 }, 107 108 /* Heartbeat */ 109 { .dev_type = HV_HB, 110 HV_HEART_BEAT_GUID, 111 .perf_device = false, 112 .allowed_in_isolated = true, 113 }, 114 115 /* Shutdown */ 116 { .dev_type = HV_SHUTDOWN, 117 HV_SHUTDOWN_GUID, 118 .perf_device = false, 119 .allowed_in_isolated = true, 120 }, 121 122 /* File copy */ 123 /* fcopy always uses 16KB ring buffer size and is working well for last many years */ 124 { .pref_ring_size = 0x4000, 125 .dev_type = HV_FCOPY, 126 HV_FCOPY_GUID, 127 .perf_device = false, 128 .allowed_in_isolated = false, 129 }, 130 131 /* Backup */ 132 { .dev_type = HV_BACKUP, 133 HV_VSS_GUID, 134 .perf_device = false, 135 .allowed_in_isolated = false, 136 }, 137 138 /* Dynamic Memory */ 139 { .dev_type = HV_DM, 140 HV_DM_GUID, 141 .perf_device = false, 142 .allowed_in_isolated = false, 143 }, 144 145 /* 146 * Unknown GUID 147 * 64 KB ring buffer + 4 KB header should be sufficient size for any Hyper-V device apart 148 * from HV_NIC and HV_SCSI. This case avoid the fallback for unknown devices to allocate 149 * much bigger (2 MB) of ring size. 150 */ 151 { .pref_ring_size = 0x11000, 152 .dev_type = HV_UNKNOWN, 153 .perf_device = false, 154 .allowed_in_isolated = false, 155 }, 156}; 157EXPORT_SYMBOL_GPL(vmbus_devs); 158 159static const struct { 160 guid_t guid; 161} vmbus_unsupported_devs[] = { 162 { HV_AVMA1_GUID }, 163 { HV_AVMA2_GUID }, 164 { HV_RDV_GUID }, 165 { HV_IMC_GUID }, 166}; 167 168/* 169 * The rescinded channel may be blocked waiting for a response from the host; 170 * take care of that. 171 */ 172static void vmbus_rescind_cleanup(struct vmbus_channel *channel) 173{ 174 struct vmbus_channel_msginfo *msginfo; 175 unsigned long flags; 176 177 178 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 179 channel->rescind = true; 180 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 181 msglistentry) { 182 183 if (msginfo->waiting_channel == channel) { 184 complete(&msginfo->waitevent); 185 break; 186 } 187 } 188 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 189} 190 191static bool is_unsupported_vmbus_devs(const guid_t *guid) 192{ 193 int i; 194 195 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++) 196 if (guid_equal(guid, &vmbus_unsupported_devs[i].guid)) 197 return true; 198 return false; 199} 200 201static u16 hv_get_dev_type(const struct vmbus_channel *channel) 202{ 203 const guid_t *guid = &channel->offermsg.offer.if_type; 204 u16 i; 205 206 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid)) 207 return HV_UNKNOWN; 208 209 for (i = HV_IDE; i < HV_UNKNOWN; i++) { 210 if (guid_equal(guid, &vmbus_devs[i].guid)) 211 return i; 212 } 213 pr_info("Unknown GUID: %pUl\n", guid); 214 return i; 215} 216 217/** 218 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message 219 * @icmsghdrp: Pointer to msg header structure 220 * @buf: Raw buffer channel data 221 * @buflen: Length of the raw buffer channel data. 222 * @fw_version: The framework versions we can support. 223 * @fw_vercnt: The size of @fw_version. 224 * @srv_version: The service versions we can support. 225 * @srv_vercnt: The size of @srv_version. 226 * @nego_fw_version: The selected framework version. 227 * @nego_srv_version: The selected service version. 228 * 229 * Note: Versions are given in decreasing order. 230 * 231 * Set up and fill in default negotiate response message. 232 * Mainly used by Hyper-V drivers. 233 */ 234bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf, 235 u32 buflen, const int *fw_version, int fw_vercnt, 236 const int *srv_version, int srv_vercnt, 237 int *nego_fw_version, int *nego_srv_version) 238{ 239 int icframe_major, icframe_minor; 240 int icmsg_major, icmsg_minor; 241 int fw_major, fw_minor; 242 int srv_major, srv_minor; 243 int i, j; 244 bool found_match = false; 245 struct icmsg_negotiate *negop; 246 247 /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */ 248 if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) { 249 pr_err_ratelimited("Invalid icmsg negotiate\n"); 250 return false; 251 } 252 253 icmsghdrp->icmsgsize = 0x10; 254 negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR]; 255 256 icframe_major = negop->icframe_vercnt; 257 icframe_minor = 0; 258 259 icmsg_major = negop->icmsg_vercnt; 260 icmsg_minor = 0; 261 262 /* Validate negop packet */ 263 if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT || 264 icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT || 265 ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) { 266 pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n", 267 icframe_major, icmsg_major); 268 goto fw_error; 269 } 270 271 /* 272 * Select the framework version number we will 273 * support. 274 */ 275 276 for (i = 0; i < fw_vercnt; i++) { 277 fw_major = (fw_version[i] >> 16); 278 fw_minor = (fw_version[i] & 0xFFFF); 279 280 for (j = 0; j < negop->icframe_vercnt; j++) { 281 if ((negop->icversion_data[j].major == fw_major) && 282 (negop->icversion_data[j].minor == fw_minor)) { 283 icframe_major = negop->icversion_data[j].major; 284 icframe_minor = negop->icversion_data[j].minor; 285 found_match = true; 286 break; 287 } 288 } 289 290 if (found_match) 291 break; 292 } 293 294 if (!found_match) 295 goto fw_error; 296 297 found_match = false; 298 299 for (i = 0; i < srv_vercnt; i++) { 300 srv_major = (srv_version[i] >> 16); 301 srv_minor = (srv_version[i] & 0xFFFF); 302 303 for (j = negop->icframe_vercnt; 304 (j < negop->icframe_vercnt + negop->icmsg_vercnt); 305 j++) { 306 307 if ((negop->icversion_data[j].major == srv_major) && 308 (negop->icversion_data[j].minor == srv_minor)) { 309 310 icmsg_major = negop->icversion_data[j].major; 311 icmsg_minor = negop->icversion_data[j].minor; 312 found_match = true; 313 break; 314 } 315 } 316 317 if (found_match) 318 break; 319 } 320 321 /* 322 * Respond with the framework and service 323 * version numbers we can support. 324 */ 325 326fw_error: 327 if (!found_match) { 328 negop->icframe_vercnt = 0; 329 negop->icmsg_vercnt = 0; 330 } else { 331 negop->icframe_vercnt = 1; 332 negop->icmsg_vercnt = 1; 333 } 334 335 if (nego_fw_version) 336 *nego_fw_version = (icframe_major << 16) | icframe_minor; 337 338 if (nego_srv_version) 339 *nego_srv_version = (icmsg_major << 16) | icmsg_minor; 340 341 negop->icversion_data[0].major = icframe_major; 342 negop->icversion_data[0].minor = icframe_minor; 343 negop->icversion_data[1].major = icmsg_major; 344 negop->icversion_data[1].minor = icmsg_minor; 345 return found_match; 346} 347EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp); 348 349/* 350 * alloc_channel - Allocate and initialize a vmbus channel object 351 */ 352static struct vmbus_channel *alloc_channel(void) 353{ 354 struct vmbus_channel *channel; 355 356 channel = kzalloc(sizeof(*channel), GFP_ATOMIC); 357 if (!channel) 358 return NULL; 359 360 spin_lock_init(&channel->sched_lock); 361 init_completion(&channel->rescind_event); 362 363 INIT_LIST_HEAD(&channel->sc_list); 364 365 tasklet_init(&channel->callback_event, 366 vmbus_on_event, (unsigned long)channel); 367 368 hv_ringbuffer_pre_init(channel); 369 370 return channel; 371} 372 373/* 374 * free_channel - Release the resources used by the vmbus channel object 375 */ 376static void free_channel(struct vmbus_channel *channel) 377{ 378 tasklet_kill(&channel->callback_event); 379 vmbus_remove_channel_attr_group(channel); 380 381 kobject_put(&channel->kobj); 382} 383 384void vmbus_channel_map_relid(struct vmbus_channel *channel) 385{ 386 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS)) 387 return; 388 /* 389 * The mapping of the channel's relid is visible from the CPUs that 390 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will 391 * execute: 392 * 393 * (a) In the "normal (i.e., not resuming from hibernation)" path, 394 * the full barrier in virt_store_mb() guarantees that the store 395 * is propagated to all CPUs before the add_channel_work work 396 * is queued. In turn, add_channel_work is queued before the 397 * channel's ring buffer is allocated/initialized and the 398 * OPENCHANNEL message for the channel is sent in vmbus_open(). 399 * Hyper-V won't start sending the interrupts for the channel 400 * before the OPENCHANNEL message is acked. The memory barrier 401 * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures 402 * that vmbus_chan_sched() must find the channel's relid in 403 * recv_int_page before retrieving the channel pointer from the 404 * array of channels. 405 * 406 * (b) In the "resuming from hibernation" path, the virt_store_mb() 407 * guarantees that the store is propagated to all CPUs before 408 * the VMBus connection is marked as ready for the resume event 409 * (cf. check_ready_for_resume_event()). The interrupt handler 410 * of the VMBus driver and vmbus_chan_sched() can not run before 411 * vmbus_bus_resume() has completed execution (cf. resume_noirq). 412 */ 413 virt_store_mb( 414 vmbus_connection.channels[channel->offermsg.child_relid], 415 channel); 416} 417 418void vmbus_channel_unmap_relid(struct vmbus_channel *channel) 419{ 420 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS)) 421 return; 422 WRITE_ONCE( 423 vmbus_connection.channels[channel->offermsg.child_relid], 424 NULL); 425} 426 427static void vmbus_release_relid(u32 relid) 428{ 429 struct vmbus_channel_relid_released msg; 430 int ret; 431 432 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released)); 433 msg.child_relid = relid; 434 msg.header.msgtype = CHANNELMSG_RELID_RELEASED; 435 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released), 436 true); 437 438 trace_vmbus_release_relid(&msg, ret); 439} 440 441void hv_process_channel_removal(struct vmbus_channel *channel) 442{ 443 lockdep_assert_held(&vmbus_connection.channel_mutex); 444 BUG_ON(!channel->rescind); 445 446 /* 447 * hv_process_channel_removal() could find INVALID_RELID only for 448 * hv_sock channels. See the inline comments in vmbus_onoffer(). 449 */ 450 WARN_ON(channel->offermsg.child_relid == INVALID_RELID && 451 !is_hvsock_channel(channel)); 452 453 /* 454 * Upon suspend, an in-use hv_sock channel is removed from the array of 455 * channels and the relid is invalidated. After hibernation, when the 456 * user-space application destroys the channel, it's unnecessary and 457 * unsafe to remove the channel from the array of channels. See also 458 * the inline comments before the call of vmbus_release_relid() below. 459 */ 460 if (channel->offermsg.child_relid != INVALID_RELID) 461 vmbus_channel_unmap_relid(channel); 462 463 if (channel->primary_channel == NULL) 464 list_del(&channel->listentry); 465 else 466 list_del(&channel->sc_list); 467 468 /* 469 * If this is a "perf" channel, updates the hv_numa_map[] masks so that 470 * init_vp_index() can (re-)use the CPU. 471 */ 472 if (hv_is_perf_channel(channel)) 473 hv_clear_allocated_cpu(channel->target_cpu); 474 475 /* 476 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and 477 * the relid is invalidated; after hibernation, when the user-space app 478 * destroys the channel, the relid is INVALID_RELID, and in this case 479 * it's unnecessary and unsafe to release the old relid, since the same 480 * relid can refer to a completely different channel now. 481 */ 482 if (channel->offermsg.child_relid != INVALID_RELID) 483 vmbus_release_relid(channel->offermsg.child_relid); 484 485 free_channel(channel); 486} 487 488void vmbus_free_channels(void) 489{ 490 struct vmbus_channel *channel, *tmp; 491 492 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list, 493 listentry) { 494 /* hv_process_channel_removal() needs this */ 495 channel->rescind = true; 496 497 vmbus_device_unregister(channel->device_obj); 498 } 499} 500 501/* Note: the function can run concurrently for primary/sub channels. */ 502static void vmbus_add_channel_work(struct work_struct *work) 503{ 504 struct vmbus_channel *newchannel = 505 container_of(work, struct vmbus_channel, add_channel_work); 506 struct vmbus_channel *primary_channel = newchannel->primary_channel; 507 int ret; 508 509 /* 510 * This state is used to indicate a successful open 511 * so that when we do close the channel normally, we 512 * can cleanup properly. 513 */ 514 newchannel->state = CHANNEL_OPEN_STATE; 515 516 if (primary_channel != NULL) { 517 /* newchannel is a sub-channel. */ 518 struct hv_device *dev = primary_channel->device_obj; 519 520 if (vmbus_add_channel_kobj(dev, newchannel)) 521 goto err_deq_chan; 522 523 if (primary_channel->sc_creation_callback != NULL) 524 primary_channel->sc_creation_callback(newchannel); 525 526 newchannel->probe_done = true; 527 return; 528 } 529 530 /* 531 * Start the process of binding the primary channel to the driver 532 */ 533 newchannel->device_obj = vmbus_device_create( 534 &newchannel->offermsg.offer.if_type, 535 &newchannel->offermsg.offer.if_instance, 536 newchannel); 537 if (!newchannel->device_obj) 538 goto err_deq_chan; 539 540 newchannel->device_obj->device_id = newchannel->device_id; 541 /* 542 * Add the new device to the bus. This will kick off device-driver 543 * binding which eventually invokes the device driver's AddDevice() 544 * method. 545 * 546 * If vmbus_device_register() fails, the 'device_obj' is freed in 547 * vmbus_device_release() as called by device_unregister() in the 548 * error path of vmbus_device_register(). In the outside error 549 * path, there's no need to free it. 550 */ 551 ret = vmbus_device_register(newchannel->device_obj); 552 553 if (ret != 0) { 554 pr_err("unable to add child device object (relid %d)\n", 555 newchannel->offermsg.child_relid); 556 goto err_deq_chan; 557 } 558 559 newchannel->probe_done = true; 560 return; 561 562err_deq_chan: 563 mutex_lock(&vmbus_connection.channel_mutex); 564 565 /* 566 * We need to set the flag, otherwise 567 * vmbus_onoffer_rescind() can be blocked. 568 */ 569 newchannel->probe_done = true; 570 571 if (primary_channel == NULL) 572 list_del(&newchannel->listentry); 573 else 574 list_del(&newchannel->sc_list); 575 576 /* vmbus_process_offer() has mapped the channel. */ 577 vmbus_channel_unmap_relid(newchannel); 578 579 mutex_unlock(&vmbus_connection.channel_mutex); 580 581 vmbus_release_relid(newchannel->offermsg.child_relid); 582 583 free_channel(newchannel); 584} 585 586/* 587 * vmbus_process_offer - Process the offer by creating a channel/device 588 * associated with this offer 589 */ 590static void vmbus_process_offer(struct vmbus_channel *newchannel) 591{ 592 struct vmbus_channel *channel; 593 struct workqueue_struct *wq; 594 bool fnew = true; 595 596 /* 597 * Synchronize vmbus_process_offer() and CPU hotplugging: 598 * 599 * CPU1 CPU2 600 * 601 * [vmbus_process_offer()] [Hot removal of the CPU] 602 * 603 * CPU_READ_LOCK CPUS_WRITE_LOCK 604 * LOAD cpu_online_mask SEARCH chn_list 605 * STORE target_cpu LOAD target_cpu 606 * INSERT chn_list STORE cpu_online_mask 607 * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK 608 * 609 * Forbids: CPU1's LOAD from *not* seing CPU2's STORE && 610 * CPU2's SEARCH from *not* seeing CPU1's INSERT 611 * 612 * Forbids: CPU2's SEARCH from seeing CPU1's INSERT && 613 * CPU2's LOAD from *not* seing CPU1's STORE 614 */ 615 cpus_read_lock(); 616 617 /* 618 * Serializes the modifications of the chn_list list as well as 619 * the accesses to next_numa_node_id in init_vp_index(). 620 */ 621 mutex_lock(&vmbus_connection.channel_mutex); 622 623 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { 624 if (guid_equal(&channel->offermsg.offer.if_type, 625 &newchannel->offermsg.offer.if_type) && 626 guid_equal(&channel->offermsg.offer.if_instance, 627 &newchannel->offermsg.offer.if_instance)) { 628 fnew = false; 629 newchannel->primary_channel = channel; 630 break; 631 } 632 } 633 634 init_vp_index(newchannel); 635 636 /* Remember the channels that should be cleaned up upon suspend. */ 637 if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel)) 638 atomic_inc(&vmbus_connection.nr_chan_close_on_suspend); 639 640 /* 641 * Now that we have acquired the channel_mutex, 642 * we can release the potentially racing rescind thread. 643 */ 644 atomic_dec(&vmbus_connection.offer_in_progress); 645 646 if (fnew) { 647 list_add_tail(&newchannel->listentry, 648 &vmbus_connection.chn_list); 649 } else { 650 /* 651 * Check to see if this is a valid sub-channel. 652 */ 653 if (newchannel->offermsg.offer.sub_channel_index == 0) { 654 mutex_unlock(&vmbus_connection.channel_mutex); 655 cpus_read_unlock(); 656 /* 657 * Don't call free_channel(), because newchannel->kobj 658 * is not initialized yet. 659 */ 660 kfree(newchannel); 661 WARN_ON_ONCE(1); 662 return; 663 } 664 /* 665 * Process the sub-channel. 666 */ 667 list_add_tail(&newchannel->sc_list, &channel->sc_list); 668 } 669 670 vmbus_channel_map_relid(newchannel); 671 672 mutex_unlock(&vmbus_connection.channel_mutex); 673 cpus_read_unlock(); 674 675 /* 676 * vmbus_process_offer() mustn't call channel->sc_creation_callback() 677 * directly for sub-channels, because sc_creation_callback() -> 678 * vmbus_open() may never get the host's response to the 679 * OPEN_CHANNEL message (the host may rescind a channel at any time, 680 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind() 681 * may not wake up the vmbus_open() as it's blocked due to a non-zero 682 * vmbus_connection.offer_in_progress, and finally we have a deadlock. 683 * 684 * The above is also true for primary channels, if the related device 685 * drivers use sync probing mode by default. 686 * 687 * And, usually the handling of primary channels and sub-channels can 688 * depend on each other, so we should offload them to different 689 * workqueues to avoid possible deadlock, e.g. in sync-probing mode, 690 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() -> 691 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock 692 * and waits for all the sub-channels to appear, but the latter 693 * can't get the rtnl_lock and this blocks the handling of 694 * sub-channels. 695 */ 696 INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work); 697 wq = fnew ? vmbus_connection.handle_primary_chan_wq : 698 vmbus_connection.handle_sub_chan_wq; 699 queue_work(wq, &newchannel->add_channel_work); 700} 701 702/* 703 * Check if CPUs used by other channels of the same device. 704 * It should only be called by init_vp_index(). 705 */ 706static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn) 707{ 708 struct vmbus_channel *primary = chn->primary_channel; 709 struct vmbus_channel *sc; 710 711 lockdep_assert_held(&vmbus_connection.channel_mutex); 712 713 if (!primary) 714 return false; 715 716 if (primary->target_cpu == cpu) 717 return true; 718 719 list_for_each_entry(sc, &primary->sc_list, sc_list) 720 if (sc != chn && sc->target_cpu == cpu) 721 return true; 722 723 return false; 724} 725 726/* 727 * We use this state to statically distribute the channel interrupt load. 728 */ 729static int next_numa_node_id; 730 731/* 732 * We can statically distribute the incoming channel interrupt load 733 * by binding a channel to VCPU. 734 * 735 * For non-performance critical channels we assign the VMBUS_CONNECT_CPU. 736 * Performance critical channels will be distributed evenly among all 737 * the available NUMA nodes. Once the node is assigned, we will assign 738 * the CPU based on a simple round robin scheme. 739 */ 740static void init_vp_index(struct vmbus_channel *channel) 741{ 742 bool perf_chn = hv_is_perf_channel(channel); 743 u32 i, ncpu = num_online_cpus(); 744 cpumask_var_t available_mask; 745 struct cpumask *allocated_mask; 746 const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ); 747 u32 target_cpu; 748 int numa_node; 749 750 if (!perf_chn || 751 !alloc_cpumask_var(&available_mask, GFP_KERNEL) || 752 cpumask_empty(hk_mask)) { 753 /* 754 * If the channel is not a performance critical 755 * channel, bind it to VMBUS_CONNECT_CPU. 756 * In case alloc_cpumask_var() fails, bind it to 757 * VMBUS_CONNECT_CPU. 758 * If all the cpus are isolated, bind it to 759 * VMBUS_CONNECT_CPU. 760 */ 761 channel->target_cpu = VMBUS_CONNECT_CPU; 762 if (perf_chn) 763 hv_set_allocated_cpu(VMBUS_CONNECT_CPU); 764 return; 765 } 766 767 for (i = 1; i <= ncpu + 1; i++) { 768 while (true) { 769 numa_node = next_numa_node_id++; 770 if (numa_node == nr_node_ids) { 771 next_numa_node_id = 0; 772 continue; 773 } 774 if (cpumask_empty(cpumask_of_node(numa_node))) 775 continue; 776 break; 777 } 778 allocated_mask = &hv_context.hv_numa_map[numa_node]; 779 780retry: 781 cpumask_xor(available_mask, allocated_mask, cpumask_of_node(numa_node)); 782 cpumask_and(available_mask, available_mask, hk_mask); 783 784 if (cpumask_empty(available_mask)) { 785 /* 786 * We have cycled through all the CPUs in the node; 787 * reset the allocated map. 788 */ 789 cpumask_clear(allocated_mask); 790 goto retry; 791 } 792 793 target_cpu = cpumask_first(available_mask); 794 cpumask_set_cpu(target_cpu, allocated_mask); 795 796 if (channel->offermsg.offer.sub_channel_index >= ncpu || 797 i > ncpu || !hv_cpuself_used(target_cpu, channel)) 798 break; 799 } 800 801 channel->target_cpu = target_cpu; 802 803 free_cpumask_var(available_mask); 804} 805 806#define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */ 807#define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */ 808#define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS) 809#define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */ 810#define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS) 811 812static void vmbus_wait_for_unload(void) 813{ 814 int cpu; 815 void *page_addr; 816 struct hv_message *msg; 817 struct vmbus_channel_message_header *hdr; 818 u32 message_type, i; 819 820 /* 821 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was 822 * used for initial contact or to CPU0 depending on host version. When 823 * we're crashing on a different CPU let's hope that IRQ handler on 824 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still 825 * functional and vmbus_unload_response() will complete 826 * vmbus_connection.unload_event. If not, the last thing we can do is 827 * read message pages for all CPUs directly. 828 * 829 * Wait up to 100 seconds since an Azure host must writeback any dirty 830 * data in its disk cache before the VMbus UNLOAD request will 831 * complete. This flushing has been empirically observed to take up 832 * to 50 seconds in cases with a lot of dirty data, so allow additional 833 * leeway and for inaccuracies in mdelay(). But eventually time out so 834 * that the panic path can't get hung forever in case the response 835 * message isn't seen. 836 */ 837 for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) { 838 if (completion_done(&vmbus_connection.unload_event)) 839 goto completed; 840 841 for_each_present_cpu(cpu) { 842 struct hv_per_cpu_context *hv_cpu 843 = per_cpu_ptr(hv_context.cpu_context, cpu); 844 845 /* 846 * In a CoCo VM the synic_message_page is not allocated 847 * in hv_synic_alloc(). Instead it is set/cleared in 848 * hv_synic_enable_regs() and hv_synic_disable_regs() 849 * such that it is set only when the CPU is online. If 850 * not all present CPUs are online, the message page 851 * might be NULL, so skip such CPUs. 852 */ 853 page_addr = hv_cpu->synic_message_page; 854 if (!page_addr) 855 continue; 856 857 msg = (struct hv_message *)page_addr 858 + VMBUS_MESSAGE_SINT; 859 860 message_type = READ_ONCE(msg->header.message_type); 861 if (message_type == HVMSG_NONE) 862 continue; 863 864 hdr = (struct vmbus_channel_message_header *) 865 msg->u.payload; 866 867 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE) 868 complete(&vmbus_connection.unload_event); 869 870 vmbus_signal_eom(msg, message_type); 871 } 872 873 /* 874 * Give a notice periodically so someone watching the 875 * serial output won't think it is completely hung. 876 */ 877 if (!(i % UNLOAD_MSG_LOOPS)) 878 pr_notice("Waiting for VMBus UNLOAD to complete\n"); 879 880 mdelay(UNLOAD_DELAY_UNIT_MS); 881 } 882 pr_err("Continuing even though VMBus UNLOAD did not complete\n"); 883 884completed: 885 /* 886 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all 887 * maybe-pending messages on all CPUs to be able to receive new 888 * messages after we reconnect. 889 */ 890 for_each_present_cpu(cpu) { 891 struct hv_per_cpu_context *hv_cpu 892 = per_cpu_ptr(hv_context.cpu_context, cpu); 893 894 page_addr = hv_cpu->synic_message_page; 895 if (!page_addr) 896 continue; 897 898 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT; 899 msg->header.message_type = HVMSG_NONE; 900 } 901} 902 903/* 904 * vmbus_unload_response - Handler for the unload response. 905 */ 906static void vmbus_unload_response(struct vmbus_channel_message_header *hdr) 907{ 908 /* 909 * This is a global event; just wakeup the waiting thread. 910 * Once we successfully unload, we can cleanup the monitor state. 911 * 912 * NB. A malicious or compromised Hyper-V could send a spurious 913 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call 914 * of the complete() below. Make sure that unload_event has been 915 * initialized by the time this complete() is executed. 916 */ 917 complete(&vmbus_connection.unload_event); 918} 919 920void vmbus_initiate_unload(bool crash) 921{ 922 struct vmbus_channel_message_header hdr; 923 924 if (xchg(&vmbus_connection.conn_state, DISCONNECTED) == DISCONNECTED) 925 return; 926 927 /* Pre-Win2012R2 hosts don't support reconnect */ 928 if (vmbus_proto_version < VERSION_WIN8_1) 929 return; 930 931 reinit_completion(&vmbus_connection.unload_event); 932 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header)); 933 hdr.msgtype = CHANNELMSG_UNLOAD; 934 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header), 935 !crash); 936 937 /* 938 * vmbus_initiate_unload() is also called on crash and the crash can be 939 * happening in an interrupt context, where scheduling is impossible. 940 */ 941 if (!crash) 942 wait_for_completion(&vmbus_connection.unload_event); 943 else 944 vmbus_wait_for_unload(); 945} 946 947static void check_ready_for_resume_event(void) 948{ 949 /* 950 * If all the old primary channels have been fixed up, then it's safe 951 * to resume. 952 */ 953 if (atomic_dec_and_test(&vmbus_connection.nr_chan_fixup_on_resume)) 954 complete(&vmbus_connection.ready_for_resume_event); 955} 956 957static void vmbus_setup_channel_state(struct vmbus_channel *channel, 958 struct vmbus_channel_offer_channel *offer) 959{ 960 /* 961 * Setup state for signalling the host. 962 */ 963 channel->sig_event = VMBUS_EVENT_CONNECTION_ID; 964 965 channel->is_dedicated_interrupt = 966 (offer->is_dedicated_interrupt != 0); 967 channel->sig_event = offer->connection_id; 968 969 memcpy(&channel->offermsg, offer, 970 sizeof(struct vmbus_channel_offer_channel)); 971 channel->monitor_grp = (u8)offer->monitorid / 32; 972 channel->monitor_bit = (u8)offer->monitorid % 32; 973 channel->device_id = hv_get_dev_type(channel); 974} 975 976/* 977 * find_primary_channel_by_offer - Get the channel object given the new offer. 978 * This is only used in the resume path of hibernation. 979 */ 980static struct vmbus_channel * 981find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer) 982{ 983 struct vmbus_channel *channel = NULL, *iter; 984 const guid_t *inst1, *inst2; 985 986 /* Ignore sub-channel offers. */ 987 if (offer->offer.sub_channel_index != 0) 988 return NULL; 989 990 mutex_lock(&vmbus_connection.channel_mutex); 991 992 list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) { 993 inst1 = &iter->offermsg.offer.if_instance; 994 inst2 = &offer->offer.if_instance; 995 996 if (guid_equal(inst1, inst2)) { 997 channel = iter; 998 break; 999 } 1000 } 1001 1002 mutex_unlock(&vmbus_connection.channel_mutex); 1003 1004 return channel; 1005} 1006 1007static bool vmbus_is_valid_offer(const struct vmbus_channel_offer_channel *offer) 1008{ 1009 const guid_t *guid = &offer->offer.if_type; 1010 u16 i; 1011 1012 if (!hv_is_isolation_supported()) 1013 return true; 1014 1015 if (is_hvsock_offer(offer)) 1016 return true; 1017 1018 for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) { 1019 if (guid_equal(guid, &vmbus_devs[i].guid)) 1020 return vmbus_devs[i].allowed_in_isolated; 1021 } 1022 return false; 1023} 1024 1025/* 1026 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition. 1027 * 1028 */ 1029static void vmbus_onoffer(struct vmbus_channel_message_header *hdr) 1030{ 1031 struct vmbus_channel_offer_channel *offer; 1032 struct vmbus_channel *oldchannel, *newchannel; 1033 size_t offer_sz; 1034 1035 offer = (struct vmbus_channel_offer_channel *)hdr; 1036 1037 trace_vmbus_onoffer(offer); 1038 1039 if (!vmbus_is_valid_offer(offer)) { 1040 pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n", 1041 offer->child_relid); 1042 atomic_dec(&vmbus_connection.offer_in_progress); 1043 return; 1044 } 1045 1046 oldchannel = find_primary_channel_by_offer(offer); 1047 1048 if (oldchannel != NULL) { 1049 /* 1050 * We're resuming from hibernation: all the sub-channel and 1051 * hv_sock channels we had before the hibernation should have 1052 * been cleaned up, and now we must be seeing a re-offered 1053 * primary channel that we had before the hibernation. 1054 */ 1055 1056 /* 1057 * { Initially: channel relid = INVALID_RELID, 1058 * channels[valid_relid] = NULL } 1059 * 1060 * CPU1 CPU2 1061 * 1062 * [vmbus_onoffer()] [vmbus_device_release()] 1063 * 1064 * LOCK channel_mutex LOCK channel_mutex 1065 * STORE channel relid = valid_relid LOAD r1 = channel relid 1066 * MAP_RELID channel if (r1 != INVALID_RELID) 1067 * UNLOCK channel_mutex UNMAP_RELID channel 1068 * UNLOCK channel_mutex 1069 * 1070 * Forbids: r1 == valid_relid && 1071 * channels[valid_relid] == channel 1072 * 1073 * Note. r1 can be INVALID_RELID only for an hv_sock channel. 1074 * None of the hv_sock channels which were present before the 1075 * suspend are re-offered upon the resume. See the WARN_ON() 1076 * in hv_process_channel_removal(). 1077 */ 1078 mutex_lock(&vmbus_connection.channel_mutex); 1079 1080 atomic_dec(&vmbus_connection.offer_in_progress); 1081 1082 WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID); 1083 /* Fix up the relid. */ 1084 oldchannel->offermsg.child_relid = offer->child_relid; 1085 1086 offer_sz = sizeof(*offer); 1087 if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) { 1088 /* 1089 * This is not an error, since the host can also change 1090 * the other field(s) of the offer, e.g. on WS RS5 1091 * (Build 17763), the offer->connection_id of the 1092 * Mellanox VF vmbus device can change when the host 1093 * reoffers the device upon resume. 1094 */ 1095 pr_debug("vmbus offer changed: relid=%d\n", 1096 offer->child_relid); 1097 1098 print_hex_dump_debug("Old vmbus offer: ", 1099 DUMP_PREFIX_OFFSET, 16, 4, 1100 &oldchannel->offermsg, offer_sz, 1101 false); 1102 print_hex_dump_debug("New vmbus offer: ", 1103 DUMP_PREFIX_OFFSET, 16, 4, 1104 offer, offer_sz, false); 1105 1106 /* Fix up the old channel. */ 1107 vmbus_setup_channel_state(oldchannel, offer); 1108 } 1109 1110 /* Add the channel back to the array of channels. */ 1111 vmbus_channel_map_relid(oldchannel); 1112 check_ready_for_resume_event(); 1113 1114 mutex_unlock(&vmbus_connection.channel_mutex); 1115 return; 1116 } 1117 1118 /* Allocate the channel object and save this offer. */ 1119 newchannel = alloc_channel(); 1120 if (!newchannel) { 1121 vmbus_release_relid(offer->child_relid); 1122 atomic_dec(&vmbus_connection.offer_in_progress); 1123 pr_err("Unable to allocate channel object\n"); 1124 return; 1125 } 1126 1127 vmbus_setup_channel_state(newchannel, offer); 1128 1129 vmbus_process_offer(newchannel); 1130} 1131 1132static void check_ready_for_suspend_event(void) 1133{ 1134 /* 1135 * If all the sub-channels or hv_sock channels have been cleaned up, 1136 * then it's safe to suspend. 1137 */ 1138 if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend)) 1139 complete(&vmbus_connection.ready_for_suspend_event); 1140} 1141 1142/* 1143 * vmbus_onoffer_rescind - Rescind offer handler. 1144 * 1145 * We queue a work item to process this offer synchronously 1146 */ 1147static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) 1148{ 1149 struct vmbus_channel_rescind_offer *rescind; 1150 struct vmbus_channel *channel; 1151 struct device *dev; 1152 bool clean_up_chan_for_suspend; 1153 1154 rescind = (struct vmbus_channel_rescind_offer *)hdr; 1155 1156 trace_vmbus_onoffer_rescind(rescind); 1157 1158 /* 1159 * The offer msg and the corresponding rescind msg 1160 * from the host are guranteed to be ordered - 1161 * offer comes in first and then the rescind. 1162 * Since we process these events in work elements, 1163 * and with preemption, we may end up processing 1164 * the events out of order. We rely on the synchronization 1165 * provided by offer_in_progress and by channel_mutex for 1166 * ordering these events: 1167 * 1168 * { Initially: offer_in_progress = 1 } 1169 * 1170 * CPU1 CPU2 1171 * 1172 * [vmbus_onoffer()] [vmbus_onoffer_rescind()] 1173 * 1174 * LOCK channel_mutex WAIT_ON offer_in_progress == 0 1175 * DECREMENT offer_in_progress LOCK channel_mutex 1176 * STORE channels[] LOAD channels[] 1177 * UNLOCK channel_mutex UNLOCK channel_mutex 1178 * 1179 * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE 1180 */ 1181 1182 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) { 1183 /* 1184 * We wait here until any channel offer is currently 1185 * being processed. 1186 */ 1187 msleep(1); 1188 } 1189 1190 mutex_lock(&vmbus_connection.channel_mutex); 1191 channel = relid2channel(rescind->child_relid); 1192 if (channel != NULL) { 1193 /* 1194 * Guarantee that no other instance of vmbus_onoffer_rescind() 1195 * has got a reference to the channel object. Synchronize on 1196 * &vmbus_connection.channel_mutex. 1197 */ 1198 if (channel->rescind_ref) { 1199 mutex_unlock(&vmbus_connection.channel_mutex); 1200 return; 1201 } 1202 channel->rescind_ref = true; 1203 } 1204 mutex_unlock(&vmbus_connection.channel_mutex); 1205 1206 if (channel == NULL) { 1207 /* 1208 * We failed in processing the offer message; 1209 * we would have cleaned up the relid in that 1210 * failure path. 1211 */ 1212 return; 1213 } 1214 1215 clean_up_chan_for_suspend = is_hvsock_channel(channel) || 1216 is_sub_channel(channel); 1217 /* 1218 * Before setting channel->rescind in vmbus_rescind_cleanup(), we 1219 * should make sure the channel callback is not running any more. 1220 */ 1221 vmbus_reset_channel_cb(channel); 1222 1223 /* 1224 * Now wait for offer handling to complete. 1225 */ 1226 vmbus_rescind_cleanup(channel); 1227 while (READ_ONCE(channel->probe_done) == false) { 1228 /* 1229 * We wait here until any channel offer is currently 1230 * being processed. 1231 */ 1232 msleep(1); 1233 } 1234 1235 /* 1236 * At this point, the rescind handling can proceed safely. 1237 */ 1238 1239 if (channel->device_obj) { 1240 if (channel->chn_rescind_callback) { 1241 channel->chn_rescind_callback(channel); 1242 1243 if (clean_up_chan_for_suspend) 1244 check_ready_for_suspend_event(); 1245 1246 return; 1247 } 1248 /* 1249 * We will have to unregister this device from the 1250 * driver core. 1251 */ 1252 dev = get_device(&channel->device_obj->device); 1253 if (dev) { 1254 vmbus_device_unregister(channel->device_obj); 1255 put_device(dev); 1256 } 1257 } else if (channel->primary_channel != NULL) { 1258 /* 1259 * Sub-channel is being rescinded. Following is the channel 1260 * close sequence when initiated from the driveri (refer to 1261 * vmbus_close() for details): 1262 * 1. Close all sub-channels first 1263 * 2. Then close the primary channel. 1264 */ 1265 mutex_lock(&vmbus_connection.channel_mutex); 1266 if (channel->state == CHANNEL_OPEN_STATE) { 1267 /* 1268 * The channel is currently not open; 1269 * it is safe for us to cleanup the channel. 1270 */ 1271 hv_process_channel_removal(channel); 1272 } else { 1273 complete(&channel->rescind_event); 1274 } 1275 mutex_unlock(&vmbus_connection.channel_mutex); 1276 } 1277 1278 /* The "channel" may have been freed. Do not access it any longer. */ 1279 1280 if (clean_up_chan_for_suspend) 1281 check_ready_for_suspend_event(); 1282} 1283 1284void vmbus_hvsock_device_unregister(struct vmbus_channel *channel) 1285{ 1286 BUG_ON(!is_hvsock_channel(channel)); 1287 1288 /* We always get a rescind msg when a connection is closed. */ 1289 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind)) 1290 msleep(1); 1291 1292 vmbus_device_unregister(channel->device_obj); 1293} 1294EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister); 1295 1296 1297/* 1298 * vmbus_onoffers_delivered - 1299 * This is invoked when all offers have been delivered. 1300 * 1301 * Nothing to do here. 1302 */ 1303static void vmbus_onoffers_delivered( 1304 struct vmbus_channel_message_header *hdr) 1305{ 1306} 1307 1308/* 1309 * vmbus_onopen_result - Open result handler. 1310 * 1311 * This is invoked when we received a response to our channel open request. 1312 * Find the matching request, copy the response and signal the requesting 1313 * thread. 1314 */ 1315static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr) 1316{ 1317 struct vmbus_channel_open_result *result; 1318 struct vmbus_channel_msginfo *msginfo; 1319 struct vmbus_channel_message_header *requestheader; 1320 struct vmbus_channel_open_channel *openmsg; 1321 unsigned long flags; 1322 1323 result = (struct vmbus_channel_open_result *)hdr; 1324 1325 trace_vmbus_onopen_result(result); 1326 1327 /* 1328 * Find the open msg, copy the result and signal/unblock the wait event 1329 */ 1330 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1331 1332 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1333 msglistentry) { 1334 requestheader = 1335 (struct vmbus_channel_message_header *)msginfo->msg; 1336 1337 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) { 1338 openmsg = 1339 (struct vmbus_channel_open_channel *)msginfo->msg; 1340 if (openmsg->child_relid == result->child_relid && 1341 openmsg->openid == result->openid) { 1342 memcpy(&msginfo->response.open_result, 1343 result, 1344 sizeof( 1345 struct vmbus_channel_open_result)); 1346 complete(&msginfo->waitevent); 1347 break; 1348 } 1349 } 1350 } 1351 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1352} 1353 1354/* 1355 * vmbus_ongpadl_created - GPADL created handler. 1356 * 1357 * This is invoked when we received a response to our gpadl create request. 1358 * Find the matching request, copy the response and signal the requesting 1359 * thread. 1360 */ 1361static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr) 1362{ 1363 struct vmbus_channel_gpadl_created *gpadlcreated; 1364 struct vmbus_channel_msginfo *msginfo; 1365 struct vmbus_channel_message_header *requestheader; 1366 struct vmbus_channel_gpadl_header *gpadlheader; 1367 unsigned long flags; 1368 1369 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr; 1370 1371 trace_vmbus_ongpadl_created(gpadlcreated); 1372 1373 /* 1374 * Find the establish msg, copy the result and signal/unblock the wait 1375 * event 1376 */ 1377 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1378 1379 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1380 msglistentry) { 1381 requestheader = 1382 (struct vmbus_channel_message_header *)msginfo->msg; 1383 1384 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) { 1385 gpadlheader = 1386 (struct vmbus_channel_gpadl_header *)requestheader; 1387 1388 if ((gpadlcreated->child_relid == 1389 gpadlheader->child_relid) && 1390 (gpadlcreated->gpadl == gpadlheader->gpadl)) { 1391 memcpy(&msginfo->response.gpadl_created, 1392 gpadlcreated, 1393 sizeof( 1394 struct vmbus_channel_gpadl_created)); 1395 complete(&msginfo->waitevent); 1396 break; 1397 } 1398 } 1399 } 1400 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1401} 1402 1403/* 1404 * vmbus_onmodifychannel_response - Modify Channel response handler. 1405 * 1406 * This is invoked when we received a response to our channel modify request. 1407 * Find the matching request, copy the response and signal the requesting thread. 1408 */ 1409static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr) 1410{ 1411 struct vmbus_channel_modifychannel_response *response; 1412 struct vmbus_channel_msginfo *msginfo; 1413 unsigned long flags; 1414 1415 response = (struct vmbus_channel_modifychannel_response *)hdr; 1416 1417 trace_vmbus_onmodifychannel_response(response); 1418 1419 /* 1420 * Find the modify msg, copy the response and signal/unblock the wait event. 1421 */ 1422 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1423 1424 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) { 1425 struct vmbus_channel_message_header *responseheader = 1426 (struct vmbus_channel_message_header *)msginfo->msg; 1427 1428 if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) { 1429 struct vmbus_channel_modifychannel *modifymsg; 1430 1431 modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg; 1432 if (modifymsg->child_relid == response->child_relid) { 1433 memcpy(&msginfo->response.modify_response, response, 1434 sizeof(*response)); 1435 complete(&msginfo->waitevent); 1436 break; 1437 } 1438 } 1439 } 1440 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1441} 1442 1443/* 1444 * vmbus_ongpadl_torndown - GPADL torndown handler. 1445 * 1446 * This is invoked when we received a response to our gpadl teardown request. 1447 * Find the matching request, copy the response and signal the requesting 1448 * thread. 1449 */ 1450static void vmbus_ongpadl_torndown( 1451 struct vmbus_channel_message_header *hdr) 1452{ 1453 struct vmbus_channel_gpadl_torndown *gpadl_torndown; 1454 struct vmbus_channel_msginfo *msginfo; 1455 struct vmbus_channel_message_header *requestheader; 1456 struct vmbus_channel_gpadl_teardown *gpadl_teardown; 1457 unsigned long flags; 1458 1459 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr; 1460 1461 trace_vmbus_ongpadl_torndown(gpadl_torndown); 1462 1463 /* 1464 * Find the open msg, copy the result and signal/unblock the wait event 1465 */ 1466 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1467 1468 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1469 msglistentry) { 1470 requestheader = 1471 (struct vmbus_channel_message_header *)msginfo->msg; 1472 1473 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) { 1474 gpadl_teardown = 1475 (struct vmbus_channel_gpadl_teardown *)requestheader; 1476 1477 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) { 1478 memcpy(&msginfo->response.gpadl_torndown, 1479 gpadl_torndown, 1480 sizeof( 1481 struct vmbus_channel_gpadl_torndown)); 1482 complete(&msginfo->waitevent); 1483 break; 1484 } 1485 } 1486 } 1487 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1488} 1489 1490/* 1491 * vmbus_onversion_response - Version response handler 1492 * 1493 * This is invoked when we received a response to our initiate contact request. 1494 * Find the matching request, copy the response and signal the requesting 1495 * thread. 1496 */ 1497static void vmbus_onversion_response( 1498 struct vmbus_channel_message_header *hdr) 1499{ 1500 struct vmbus_channel_msginfo *msginfo; 1501 struct vmbus_channel_message_header *requestheader; 1502 struct vmbus_channel_version_response *version_response; 1503 unsigned long flags; 1504 1505 version_response = (struct vmbus_channel_version_response *)hdr; 1506 1507 trace_vmbus_onversion_response(version_response); 1508 1509 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); 1510 1511 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, 1512 msglistentry) { 1513 requestheader = 1514 (struct vmbus_channel_message_header *)msginfo->msg; 1515 1516 if (requestheader->msgtype == 1517 CHANNELMSG_INITIATE_CONTACT) { 1518 memcpy(&msginfo->response.version_response, 1519 version_response, 1520 sizeof(struct vmbus_channel_version_response)); 1521 complete(&msginfo->waitevent); 1522 } 1523 } 1524 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); 1525} 1526 1527/* Channel message dispatch table */ 1528const struct vmbus_channel_message_table_entry 1529channel_message_table[CHANNELMSG_COUNT] = { 1530 { CHANNELMSG_INVALID, 0, NULL, 0}, 1531 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer, 1532 sizeof(struct vmbus_channel_offer_channel)}, 1533 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind, 1534 sizeof(struct vmbus_channel_rescind_offer) }, 1535 { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0}, 1536 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0}, 1537 { CHANNELMSG_OPENCHANNEL, 0, NULL, 0}, 1538 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result, 1539 sizeof(struct vmbus_channel_open_result)}, 1540 { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0}, 1541 { CHANNELMSG_GPADL_HEADER, 0, NULL, 0}, 1542 { CHANNELMSG_GPADL_BODY, 0, NULL, 0}, 1543 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created, 1544 sizeof(struct vmbus_channel_gpadl_created)}, 1545 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0}, 1546 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown, 1547 sizeof(struct vmbus_channel_gpadl_torndown) }, 1548 { CHANNELMSG_RELID_RELEASED, 0, NULL, 0}, 1549 { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0}, 1550 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response, 1551 sizeof(struct vmbus_channel_version_response)}, 1552 { CHANNELMSG_UNLOAD, 0, NULL, 0}, 1553 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0}, 1554 { CHANNELMSG_18, 0, NULL, 0}, 1555 { CHANNELMSG_19, 0, NULL, 0}, 1556 { CHANNELMSG_20, 0, NULL, 0}, 1557 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0}, 1558 { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0}, 1559 { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0}, 1560 { CHANNELMSG_MODIFYCHANNEL_RESPONSE, 1, vmbus_onmodifychannel_response, 1561 sizeof(struct vmbus_channel_modifychannel_response)}, 1562}; 1563 1564/* 1565 * vmbus_onmessage - Handler for channel protocol messages. 1566 * 1567 * This is invoked in the vmbus worker thread context. 1568 */ 1569void vmbus_onmessage(struct vmbus_channel_message_header *hdr) 1570{ 1571 trace_vmbus_on_message(hdr); 1572 1573 /* 1574 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go 1575 * out of bound and the message_handler pointer can not be NULL. 1576 */ 1577 channel_message_table[hdr->msgtype].message_handler(hdr); 1578} 1579 1580/* 1581 * vmbus_request_offers - Send a request to get all our pending offers. 1582 */ 1583int vmbus_request_offers(void) 1584{ 1585 struct vmbus_channel_message_header *msg; 1586 struct vmbus_channel_msginfo *msginfo; 1587 int ret; 1588 1589 msginfo = kzalloc(sizeof(*msginfo) + 1590 sizeof(struct vmbus_channel_message_header), 1591 GFP_KERNEL); 1592 if (!msginfo) 1593 return -ENOMEM; 1594 1595 msg = (struct vmbus_channel_message_header *)msginfo->msg; 1596 1597 msg->msgtype = CHANNELMSG_REQUESTOFFERS; 1598 1599 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header), 1600 true); 1601 1602 trace_vmbus_request_offers(ret); 1603 1604 if (ret != 0) { 1605 pr_err("Unable to request offers - %d\n", ret); 1606 1607 goto cleanup; 1608 } 1609 1610cleanup: 1611 kfree(msginfo); 1612 1613 return ret; 1614} 1615 1616void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel, 1617 void (*sc_cr_cb)(struct vmbus_channel *new_sc)) 1618{ 1619 primary_channel->sc_creation_callback = sc_cr_cb; 1620} 1621EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback); 1622 1623void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel, 1624 void (*chn_rescind_cb)(struct vmbus_channel *)) 1625{ 1626 channel->chn_rescind_callback = chn_rescind_cb; 1627} 1628EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback); 1629