1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2003 Sistina Software Limited. 4 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. 5 * 6 * This file is released under the GPL. 7 */ 8 9#include <linux/device-mapper.h> 10 11#include "dm-rq.h" 12#include "dm-bio-record.h" 13#include "dm-path-selector.h" 14#include "dm-uevent.h" 15 16#include <linux/blkdev.h> 17#include <linux/ctype.h> 18#include <linux/init.h> 19#include <linux/mempool.h> 20#include <linux/module.h> 21#include <linux/pagemap.h> 22#include <linux/slab.h> 23#include <linux/time.h> 24#include <linux/timer.h> 25#include <linux/workqueue.h> 26#include <linux/delay.h> 27#include <scsi/scsi_dh.h> 28#include <linux/atomic.h> 29#include <linux/blk-mq.h> 30 31static struct workqueue_struct *dm_mpath_wq; 32 33#define DM_MSG_PREFIX "multipath" 34#define DM_PG_INIT_DELAY_MSECS 2000 35#define DM_PG_INIT_DELAY_DEFAULT ((unsigned int) -1) 36#define QUEUE_IF_NO_PATH_TIMEOUT_DEFAULT 0 37 38static unsigned long queue_if_no_path_timeout_secs = QUEUE_IF_NO_PATH_TIMEOUT_DEFAULT; 39 40/* Path properties */ 41struct pgpath { 42 struct list_head list; 43 44 struct priority_group *pg; /* Owning PG */ 45 unsigned int fail_count; /* Cumulative failure count */ 46 47 struct dm_path path; 48 struct delayed_work activate_path; 49 50 bool is_active:1; /* Path status */ 51}; 52 53#define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path) 54 55/* 56 * Paths are grouped into Priority Groups and numbered from 1 upwards. 57 * Each has a path selector which controls which path gets used. 58 */ 59struct priority_group { 60 struct list_head list; 61 62 struct multipath *m; /* Owning multipath instance */ 63 struct path_selector ps; 64 65 unsigned int pg_num; /* Reference number */ 66 unsigned int nr_pgpaths; /* Number of paths in PG */ 67 struct list_head pgpaths; 68 69 bool bypassed:1; /* Temporarily bypass this PG? */ 70}; 71 72/* Multipath context */ 73struct multipath { 74 unsigned long flags; /* Multipath state flags */ 75 76 spinlock_t lock; 77 enum dm_queue_mode queue_mode; 78 79 struct pgpath *current_pgpath; 80 struct priority_group *current_pg; 81 struct priority_group *next_pg; /* Switch to this PG if set */ 82 83 atomic_t nr_valid_paths; /* Total number of usable paths */ 84 unsigned int nr_priority_groups; 85 struct list_head priority_groups; 86 87 const char *hw_handler_name; 88 char *hw_handler_params; 89 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */ 90 unsigned int pg_init_retries; /* Number of times to retry pg_init */ 91 unsigned int pg_init_delay_msecs; /* Number of msecs before pg_init retry */ 92 atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */ 93 atomic_t pg_init_count; /* Number of times pg_init called */ 94 95 struct mutex work_mutex; 96 struct work_struct trigger_event; 97 struct dm_target *ti; 98 99 struct work_struct process_queued_bios; 100 struct bio_list queued_bios; 101 102 struct timer_list nopath_timer; /* Timeout for queue_if_no_path */ 103}; 104 105/* 106 * Context information attached to each io we process. 107 */ 108struct dm_mpath_io { 109 struct pgpath *pgpath; 110 size_t nr_bytes; 111 u64 start_time_ns; 112}; 113 114typedef int (*action_fn) (struct pgpath *pgpath); 115 116static struct workqueue_struct *kmultipathd, *kmpath_handlerd; 117static void trigger_event(struct work_struct *work); 118static void activate_or_offline_path(struct pgpath *pgpath); 119static void activate_path_work(struct work_struct *work); 120static void process_queued_bios(struct work_struct *work); 121static void queue_if_no_path_timeout_work(struct timer_list *t); 122 123/* 124 *----------------------------------------------- 125 * Multipath state flags. 126 *----------------------------------------------- 127 */ 128#define MPATHF_QUEUE_IO 0 /* Must we queue all I/O? */ 129#define MPATHF_QUEUE_IF_NO_PATH 1 /* Queue I/O if last path fails? */ 130#define MPATHF_SAVED_QUEUE_IF_NO_PATH 2 /* Saved state during suspension */ 131#define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3 /* If there's already a hw_handler present, don't change it. */ 132#define MPATHF_PG_INIT_DISABLED 4 /* pg_init is not currently allowed */ 133#define MPATHF_PG_INIT_REQUIRED 5 /* pg_init needs calling? */ 134#define MPATHF_PG_INIT_DELAY_RETRY 6 /* Delay pg_init retry? */ 135 136static bool mpath_double_check_test_bit(int MPATHF_bit, struct multipath *m) 137{ 138 bool r = test_bit(MPATHF_bit, &m->flags); 139 140 if (r) { 141 unsigned long flags; 142 143 spin_lock_irqsave(&m->lock, flags); 144 r = test_bit(MPATHF_bit, &m->flags); 145 spin_unlock_irqrestore(&m->lock, flags); 146 } 147 148 return r; 149} 150 151/* 152 *----------------------------------------------- 153 * Allocation routines 154 *----------------------------------------------- 155 */ 156static struct pgpath *alloc_pgpath(void) 157{ 158 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL); 159 160 if (!pgpath) 161 return NULL; 162 163 pgpath->is_active = true; 164 165 return pgpath; 166} 167 168static void free_pgpath(struct pgpath *pgpath) 169{ 170 kfree(pgpath); 171} 172 173static struct priority_group *alloc_priority_group(void) 174{ 175 struct priority_group *pg; 176 177 pg = kzalloc(sizeof(*pg), GFP_KERNEL); 178 179 if (pg) 180 INIT_LIST_HEAD(&pg->pgpaths); 181 182 return pg; 183} 184 185static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti) 186{ 187 struct pgpath *pgpath, *tmp; 188 189 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) { 190 list_del(&pgpath->list); 191 dm_put_device(ti, pgpath->path.dev); 192 free_pgpath(pgpath); 193 } 194} 195 196static void free_priority_group(struct priority_group *pg, 197 struct dm_target *ti) 198{ 199 struct path_selector *ps = &pg->ps; 200 201 if (ps->type) { 202 ps->type->destroy(ps); 203 dm_put_path_selector(ps->type); 204 } 205 206 free_pgpaths(&pg->pgpaths, ti); 207 kfree(pg); 208} 209 210static struct multipath *alloc_multipath(struct dm_target *ti) 211{ 212 struct multipath *m; 213 214 m = kzalloc(sizeof(*m), GFP_KERNEL); 215 if (m) { 216 INIT_LIST_HEAD(&m->priority_groups); 217 spin_lock_init(&m->lock); 218 atomic_set(&m->nr_valid_paths, 0); 219 INIT_WORK(&m->trigger_event, trigger_event); 220 mutex_init(&m->work_mutex); 221 222 m->queue_mode = DM_TYPE_NONE; 223 224 m->ti = ti; 225 ti->private = m; 226 227 timer_setup(&m->nopath_timer, queue_if_no_path_timeout_work, 0); 228 } 229 230 return m; 231} 232 233static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m) 234{ 235 if (m->queue_mode == DM_TYPE_NONE) { 236 m->queue_mode = DM_TYPE_REQUEST_BASED; 237 } else if (m->queue_mode == DM_TYPE_BIO_BASED) { 238 INIT_WORK(&m->process_queued_bios, process_queued_bios); 239 /* 240 * bio-based doesn't support any direct scsi_dh management; 241 * it just discovers if a scsi_dh is attached. 242 */ 243 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags); 244 } 245 246 dm_table_set_type(ti->table, m->queue_mode); 247 248 /* 249 * Init fields that are only used when a scsi_dh is attached 250 * - must do this unconditionally (really doesn't hurt non-SCSI uses) 251 */ 252 set_bit(MPATHF_QUEUE_IO, &m->flags); 253 atomic_set(&m->pg_init_in_progress, 0); 254 atomic_set(&m->pg_init_count, 0); 255 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; 256 init_waitqueue_head(&m->pg_init_wait); 257 258 return 0; 259} 260 261static void free_multipath(struct multipath *m) 262{ 263 struct priority_group *pg, *tmp; 264 265 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) { 266 list_del(&pg->list); 267 free_priority_group(pg, m->ti); 268 } 269 270 kfree(m->hw_handler_name); 271 kfree(m->hw_handler_params); 272 mutex_destroy(&m->work_mutex); 273 kfree(m); 274} 275 276static struct dm_mpath_io *get_mpio(union map_info *info) 277{ 278 return info->ptr; 279} 280 281static size_t multipath_per_bio_data_size(void) 282{ 283 return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details); 284} 285 286static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio) 287{ 288 return dm_per_bio_data(bio, multipath_per_bio_data_size()); 289} 290 291static struct dm_bio_details *get_bio_details_from_mpio(struct dm_mpath_io *mpio) 292{ 293 /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */ 294 void *bio_details = mpio + 1; 295 return bio_details; 296} 297 298static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p) 299{ 300 struct dm_mpath_io *mpio = get_mpio_from_bio(bio); 301 struct dm_bio_details *bio_details = get_bio_details_from_mpio(mpio); 302 303 mpio->nr_bytes = bio->bi_iter.bi_size; 304 mpio->pgpath = NULL; 305 mpio->start_time_ns = 0; 306 *mpio_p = mpio; 307 308 dm_bio_record(bio_details, bio); 309} 310 311/* 312 *----------------------------------------------- 313 * Path selection 314 *----------------------------------------------- 315 */ 316static int __pg_init_all_paths(struct multipath *m) 317{ 318 struct pgpath *pgpath; 319 unsigned long pg_init_delay = 0; 320 321 lockdep_assert_held(&m->lock); 322 323 if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) 324 return 0; 325 326 atomic_inc(&m->pg_init_count); 327 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 328 329 /* Check here to reset pg_init_required */ 330 if (!m->current_pg) 331 return 0; 332 333 if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags)) 334 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ? 335 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS); 336 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) { 337 /* Skip failed paths */ 338 if (!pgpath->is_active) 339 continue; 340 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path, 341 pg_init_delay)) 342 atomic_inc(&m->pg_init_in_progress); 343 } 344 return atomic_read(&m->pg_init_in_progress); 345} 346 347static int pg_init_all_paths(struct multipath *m) 348{ 349 int ret; 350 unsigned long flags; 351 352 spin_lock_irqsave(&m->lock, flags); 353 ret = __pg_init_all_paths(m); 354 spin_unlock_irqrestore(&m->lock, flags); 355 356 return ret; 357} 358 359static void __switch_pg(struct multipath *m, struct priority_group *pg) 360{ 361 lockdep_assert_held(&m->lock); 362 363 m->current_pg = pg; 364 365 /* Must we initialise the PG first, and queue I/O till it's ready? */ 366 if (m->hw_handler_name) { 367 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 368 set_bit(MPATHF_QUEUE_IO, &m->flags); 369 } else { 370 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 371 clear_bit(MPATHF_QUEUE_IO, &m->flags); 372 } 373 374 atomic_set(&m->pg_init_count, 0); 375} 376 377static struct pgpath *choose_path_in_pg(struct multipath *m, 378 struct priority_group *pg, 379 size_t nr_bytes) 380{ 381 unsigned long flags; 382 struct dm_path *path; 383 struct pgpath *pgpath; 384 385 path = pg->ps.type->select_path(&pg->ps, nr_bytes); 386 if (!path) 387 return ERR_PTR(-ENXIO); 388 389 pgpath = path_to_pgpath(path); 390 391 if (unlikely(READ_ONCE(m->current_pg) != pg)) { 392 /* Only update current_pgpath if pg changed */ 393 spin_lock_irqsave(&m->lock, flags); 394 m->current_pgpath = pgpath; 395 __switch_pg(m, pg); 396 spin_unlock_irqrestore(&m->lock, flags); 397 } 398 399 return pgpath; 400} 401 402static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes) 403{ 404 unsigned long flags; 405 struct priority_group *pg; 406 struct pgpath *pgpath; 407 unsigned int bypassed = 1; 408 409 if (!atomic_read(&m->nr_valid_paths)) { 410 spin_lock_irqsave(&m->lock, flags); 411 clear_bit(MPATHF_QUEUE_IO, &m->flags); 412 spin_unlock_irqrestore(&m->lock, flags); 413 goto failed; 414 } 415 416 /* Were we instructed to switch PG? */ 417 if (READ_ONCE(m->next_pg)) { 418 spin_lock_irqsave(&m->lock, flags); 419 pg = m->next_pg; 420 if (!pg) { 421 spin_unlock_irqrestore(&m->lock, flags); 422 goto check_current_pg; 423 } 424 m->next_pg = NULL; 425 spin_unlock_irqrestore(&m->lock, flags); 426 pgpath = choose_path_in_pg(m, pg, nr_bytes); 427 if (!IS_ERR_OR_NULL(pgpath)) 428 return pgpath; 429 } 430 431 /* Don't change PG until it has no remaining paths */ 432check_current_pg: 433 pg = READ_ONCE(m->current_pg); 434 if (pg) { 435 pgpath = choose_path_in_pg(m, pg, nr_bytes); 436 if (!IS_ERR_OR_NULL(pgpath)) 437 return pgpath; 438 } 439 440 /* 441 * Loop through priority groups until we find a valid path. 442 * First time we skip PGs marked 'bypassed'. 443 * Second time we only try the ones we skipped, but set 444 * pg_init_delay_retry so we do not hammer controllers. 445 */ 446 do { 447 list_for_each_entry(pg, &m->priority_groups, list) { 448 if (pg->bypassed == !!bypassed) 449 continue; 450 pgpath = choose_path_in_pg(m, pg, nr_bytes); 451 if (!IS_ERR_OR_NULL(pgpath)) { 452 if (!bypassed) { 453 spin_lock_irqsave(&m->lock, flags); 454 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 455 spin_unlock_irqrestore(&m->lock, flags); 456 } 457 return pgpath; 458 } 459 } 460 } while (bypassed--); 461 462failed: 463 spin_lock_irqsave(&m->lock, flags); 464 m->current_pgpath = NULL; 465 m->current_pg = NULL; 466 spin_unlock_irqrestore(&m->lock, flags); 467 468 return NULL; 469} 470 471/* 472 * dm_report_EIO() is a macro instead of a function to make pr_debug_ratelimited() 473 * report the function name and line number of the function from which 474 * it has been invoked. 475 */ 476#define dm_report_EIO(m) \ 477 DMDEBUG_LIMIT("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d", \ 478 dm_table_device_name((m)->ti->table), \ 479 test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags), \ 480 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags), \ 481 dm_noflush_suspending((m)->ti)) 482 483/* 484 * Check whether bios must be queued in the device-mapper core rather 485 * than here in the target. 486 */ 487static bool __must_push_back(struct multipath *m) 488{ 489 return dm_noflush_suspending(m->ti); 490} 491 492static bool must_push_back_rq(struct multipath *m) 493{ 494 unsigned long flags; 495 bool ret; 496 497 spin_lock_irqsave(&m->lock, flags); 498 ret = (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) || __must_push_back(m)); 499 spin_unlock_irqrestore(&m->lock, flags); 500 501 return ret; 502} 503 504/* 505 * Map cloned requests (request-based multipath) 506 */ 507static int multipath_clone_and_map(struct dm_target *ti, struct request *rq, 508 union map_info *map_context, 509 struct request **__clone) 510{ 511 struct multipath *m = ti->private; 512 size_t nr_bytes = blk_rq_bytes(rq); 513 struct pgpath *pgpath; 514 struct block_device *bdev; 515 struct dm_mpath_io *mpio = get_mpio(map_context); 516 struct request_queue *q; 517 struct request *clone; 518 519 /* Do we need to select a new pgpath? */ 520 pgpath = READ_ONCE(m->current_pgpath); 521 if (!pgpath || !mpath_double_check_test_bit(MPATHF_QUEUE_IO, m)) 522 pgpath = choose_pgpath(m, nr_bytes); 523 524 if (!pgpath) { 525 if (must_push_back_rq(m)) 526 return DM_MAPIO_DELAY_REQUEUE; 527 dm_report_EIO(m); /* Failed */ 528 return DM_MAPIO_KILL; 529 } else if (mpath_double_check_test_bit(MPATHF_QUEUE_IO, m) || 530 mpath_double_check_test_bit(MPATHF_PG_INIT_REQUIRED, m)) { 531 pg_init_all_paths(m); 532 return DM_MAPIO_DELAY_REQUEUE; 533 } 534 535 mpio->pgpath = pgpath; 536 mpio->nr_bytes = nr_bytes; 537 538 bdev = pgpath->path.dev->bdev; 539 q = bdev_get_queue(bdev); 540 clone = blk_mq_alloc_request(q, rq->cmd_flags | REQ_NOMERGE, 541 BLK_MQ_REQ_NOWAIT); 542 if (IS_ERR(clone)) { 543 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */ 544 if (blk_queue_dying(q)) { 545 atomic_inc(&m->pg_init_in_progress); 546 activate_or_offline_path(pgpath); 547 return DM_MAPIO_DELAY_REQUEUE; 548 } 549 550 /* 551 * blk-mq's SCHED_RESTART can cover this requeue, so we 552 * needn't deal with it by DELAY_REQUEUE. More importantly, 553 * we have to return DM_MAPIO_REQUEUE so that blk-mq can 554 * get the queue busy feedback (via BLK_STS_RESOURCE), 555 * otherwise I/O merging can suffer. 556 */ 557 return DM_MAPIO_REQUEUE; 558 } 559 clone->bio = clone->biotail = NULL; 560 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT; 561 *__clone = clone; 562 563 if (pgpath->pg->ps.type->start_io) 564 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, 565 &pgpath->path, 566 nr_bytes); 567 return DM_MAPIO_REMAPPED; 568} 569 570static void multipath_release_clone(struct request *clone, 571 union map_info *map_context) 572{ 573 if (unlikely(map_context)) { 574 /* 575 * non-NULL map_context means caller is still map 576 * method; must undo multipath_clone_and_map() 577 */ 578 struct dm_mpath_io *mpio = get_mpio(map_context); 579 struct pgpath *pgpath = mpio->pgpath; 580 581 if (pgpath && pgpath->pg->ps.type->end_io) 582 pgpath->pg->ps.type->end_io(&pgpath->pg->ps, 583 &pgpath->path, 584 mpio->nr_bytes, 585 clone->io_start_time_ns); 586 } 587 588 blk_mq_free_request(clone); 589} 590 591/* 592 * Map cloned bios (bio-based multipath) 593 */ 594 595static void __multipath_queue_bio(struct multipath *m, struct bio *bio) 596{ 597 /* Queue for the daemon to resubmit */ 598 bio_list_add(&m->queued_bios, bio); 599 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) 600 queue_work(kmultipathd, &m->process_queued_bios); 601} 602 603static void multipath_queue_bio(struct multipath *m, struct bio *bio) 604{ 605 unsigned long flags; 606 607 spin_lock_irqsave(&m->lock, flags); 608 __multipath_queue_bio(m, bio); 609 spin_unlock_irqrestore(&m->lock, flags); 610} 611 612static struct pgpath *__map_bio(struct multipath *m, struct bio *bio) 613{ 614 struct pgpath *pgpath; 615 unsigned long flags; 616 617 /* Do we need to select a new pgpath? */ 618 pgpath = READ_ONCE(m->current_pgpath); 619 if (!pgpath || !mpath_double_check_test_bit(MPATHF_QUEUE_IO, m)) 620 pgpath = choose_pgpath(m, bio->bi_iter.bi_size); 621 622 if (!pgpath) { 623 spin_lock_irqsave(&m->lock, flags); 624 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { 625 __multipath_queue_bio(m, bio); 626 pgpath = ERR_PTR(-EAGAIN); 627 } 628 spin_unlock_irqrestore(&m->lock, flags); 629 630 } else if (mpath_double_check_test_bit(MPATHF_QUEUE_IO, m) || 631 mpath_double_check_test_bit(MPATHF_PG_INIT_REQUIRED, m)) { 632 multipath_queue_bio(m, bio); 633 pg_init_all_paths(m); 634 return ERR_PTR(-EAGAIN); 635 } 636 637 return pgpath; 638} 639 640static int __multipath_map_bio(struct multipath *m, struct bio *bio, 641 struct dm_mpath_io *mpio) 642{ 643 struct pgpath *pgpath = __map_bio(m, bio); 644 645 if (IS_ERR(pgpath)) 646 return DM_MAPIO_SUBMITTED; 647 648 if (!pgpath) { 649 if (__must_push_back(m)) 650 return DM_MAPIO_REQUEUE; 651 dm_report_EIO(m); 652 return DM_MAPIO_KILL; 653 } 654 655 mpio->pgpath = pgpath; 656 657 if (dm_ps_use_hr_timer(pgpath->pg->ps.type)) 658 mpio->start_time_ns = ktime_get_ns(); 659 660 bio->bi_status = 0; 661 bio_set_dev(bio, pgpath->path.dev->bdev); 662 bio->bi_opf |= REQ_FAILFAST_TRANSPORT; 663 664 if (pgpath->pg->ps.type->start_io) 665 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, 666 &pgpath->path, 667 mpio->nr_bytes); 668 return DM_MAPIO_REMAPPED; 669} 670 671static int multipath_map_bio(struct dm_target *ti, struct bio *bio) 672{ 673 struct multipath *m = ti->private; 674 struct dm_mpath_io *mpio = NULL; 675 676 multipath_init_per_bio_data(bio, &mpio); 677 return __multipath_map_bio(m, bio, mpio); 678} 679 680static void process_queued_io_list(struct multipath *m) 681{ 682 if (m->queue_mode == DM_TYPE_REQUEST_BASED) 683 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table)); 684 else if (m->queue_mode == DM_TYPE_BIO_BASED) 685 queue_work(kmultipathd, &m->process_queued_bios); 686} 687 688static void process_queued_bios(struct work_struct *work) 689{ 690 int r; 691 unsigned long flags; 692 struct bio *bio; 693 struct bio_list bios; 694 struct blk_plug plug; 695 struct multipath *m = 696 container_of(work, struct multipath, process_queued_bios); 697 698 bio_list_init(&bios); 699 700 spin_lock_irqsave(&m->lock, flags); 701 702 if (bio_list_empty(&m->queued_bios)) { 703 spin_unlock_irqrestore(&m->lock, flags); 704 return; 705 } 706 707 bio_list_merge_init(&bios, &m->queued_bios); 708 709 spin_unlock_irqrestore(&m->lock, flags); 710 711 blk_start_plug(&plug); 712 while ((bio = bio_list_pop(&bios))) { 713 struct dm_mpath_io *mpio = get_mpio_from_bio(bio); 714 715 dm_bio_restore(get_bio_details_from_mpio(mpio), bio); 716 r = __multipath_map_bio(m, bio, mpio); 717 switch (r) { 718 case DM_MAPIO_KILL: 719 bio->bi_status = BLK_STS_IOERR; 720 bio_endio(bio); 721 break; 722 case DM_MAPIO_REQUEUE: 723 bio->bi_status = BLK_STS_DM_REQUEUE; 724 bio_endio(bio); 725 break; 726 case DM_MAPIO_REMAPPED: 727 submit_bio_noacct(bio); 728 break; 729 case DM_MAPIO_SUBMITTED: 730 break; 731 default: 732 WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r); 733 } 734 } 735 blk_finish_plug(&plug); 736} 737 738/* 739 * If we run out of usable paths, should we queue I/O or error it? 740 */ 741static int queue_if_no_path(struct multipath *m, bool f_queue_if_no_path, 742 bool save_old_value, const char *caller) 743{ 744 unsigned long flags; 745 bool queue_if_no_path_bit, saved_queue_if_no_path_bit; 746 const char *dm_dev_name = dm_table_device_name(m->ti->table); 747 748 DMDEBUG("%s: %s caller=%s f_queue_if_no_path=%d save_old_value=%d", 749 dm_dev_name, __func__, caller, f_queue_if_no_path, save_old_value); 750 751 spin_lock_irqsave(&m->lock, flags); 752 753 queue_if_no_path_bit = test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags); 754 saved_queue_if_no_path_bit = test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags); 755 756 if (save_old_value) { 757 if (unlikely(!queue_if_no_path_bit && saved_queue_if_no_path_bit)) { 758 DMERR("%s: QIFNP disabled but saved as enabled, saving again loses state, not saving!", 759 dm_dev_name); 760 } else 761 assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path_bit); 762 } else if (!f_queue_if_no_path && saved_queue_if_no_path_bit) { 763 /* due to "fail_if_no_path" message, need to honor it. */ 764 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags); 765 } 766 assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, f_queue_if_no_path); 767 768 DMDEBUG("%s: after %s changes; QIFNP = %d; SQIFNP = %d; DNFS = %d", 769 dm_dev_name, __func__, 770 test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags), 771 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags), 772 dm_noflush_suspending(m->ti)); 773 774 spin_unlock_irqrestore(&m->lock, flags); 775 776 if (!f_queue_if_no_path) { 777 dm_table_run_md_queue_async(m->ti->table); 778 process_queued_io_list(m); 779 } 780 781 return 0; 782} 783 784/* 785 * If the queue_if_no_path timeout fires, turn off queue_if_no_path and 786 * process any queued I/O. 787 */ 788static void queue_if_no_path_timeout_work(struct timer_list *t) 789{ 790 struct multipath *m = from_timer(m, t, nopath_timer); 791 792 DMWARN("queue_if_no_path timeout on %s, failing queued IO", 793 dm_table_device_name(m->ti->table)); 794 queue_if_no_path(m, false, false, __func__); 795} 796 797/* 798 * Enable the queue_if_no_path timeout if necessary. 799 * Called with m->lock held. 800 */ 801static void enable_nopath_timeout(struct multipath *m) 802{ 803 unsigned long queue_if_no_path_timeout = 804 READ_ONCE(queue_if_no_path_timeout_secs) * HZ; 805 806 lockdep_assert_held(&m->lock); 807 808 if (queue_if_no_path_timeout > 0 && 809 atomic_read(&m->nr_valid_paths) == 0 && 810 test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { 811 mod_timer(&m->nopath_timer, 812 jiffies + queue_if_no_path_timeout); 813 } 814} 815 816static void disable_nopath_timeout(struct multipath *m) 817{ 818 del_timer_sync(&m->nopath_timer); 819} 820 821/* 822 * An event is triggered whenever a path is taken out of use. 823 * Includes path failure and PG bypass. 824 */ 825static void trigger_event(struct work_struct *work) 826{ 827 struct multipath *m = 828 container_of(work, struct multipath, trigger_event); 829 830 dm_table_event(m->ti->table); 831} 832 833/* 834 *--------------------------------------------------------------- 835 * Constructor/argument parsing: 836 * <#multipath feature args> [<arg>]* 837 * <#hw_handler args> [hw_handler [<arg>]*] 838 * <#priority groups> 839 * <initial priority group> 840 * [<selector> <#selector args> [<arg>]* 841 * <#paths> <#per-path selector args> 842 * [<path> [<arg>]* ]+ ]+ 843 *--------------------------------------------------------------- 844 */ 845static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg, 846 struct dm_target *ti) 847{ 848 int r; 849 struct path_selector_type *pst; 850 unsigned int ps_argc; 851 852 static const struct dm_arg _args[] = { 853 {0, 1024, "invalid number of path selector args"}, 854 }; 855 856 pst = dm_get_path_selector(dm_shift_arg(as)); 857 if (!pst) { 858 ti->error = "unknown path selector type"; 859 return -EINVAL; 860 } 861 862 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error); 863 if (r) { 864 dm_put_path_selector(pst); 865 return -EINVAL; 866 } 867 868 r = pst->create(&pg->ps, ps_argc, as->argv); 869 if (r) { 870 dm_put_path_selector(pst); 871 ti->error = "path selector constructor failed"; 872 return r; 873 } 874 875 pg->ps.type = pst; 876 dm_consume_args(as, ps_argc); 877 878 return 0; 879} 880 881static int setup_scsi_dh(struct block_device *bdev, struct multipath *m, 882 const char **attached_handler_name, char **error) 883{ 884 struct request_queue *q = bdev_get_queue(bdev); 885 int r; 886 887 if (mpath_double_check_test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, m)) { 888retain: 889 if (*attached_handler_name) { 890 /* 891 * Clear any hw_handler_params associated with a 892 * handler that isn't already attached. 893 */ 894 if (m->hw_handler_name && strcmp(*attached_handler_name, m->hw_handler_name)) { 895 kfree(m->hw_handler_params); 896 m->hw_handler_params = NULL; 897 } 898 899 /* 900 * Reset hw_handler_name to match the attached handler 901 * 902 * NB. This modifies the table line to show the actual 903 * handler instead of the original table passed in. 904 */ 905 kfree(m->hw_handler_name); 906 m->hw_handler_name = *attached_handler_name; 907 *attached_handler_name = NULL; 908 } 909 } 910 911 if (m->hw_handler_name) { 912 r = scsi_dh_attach(q, m->hw_handler_name); 913 if (r == -EBUSY) { 914 DMINFO("retaining handler on device %pg", bdev); 915 goto retain; 916 } 917 if (r < 0) { 918 *error = "error attaching hardware handler"; 919 return r; 920 } 921 922 if (m->hw_handler_params) { 923 r = scsi_dh_set_params(q, m->hw_handler_params); 924 if (r < 0) { 925 *error = "unable to set hardware handler parameters"; 926 return r; 927 } 928 } 929 } 930 931 return 0; 932} 933 934static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps, 935 struct dm_target *ti) 936{ 937 int r; 938 struct pgpath *p; 939 struct multipath *m = ti->private; 940 struct request_queue *q; 941 const char *attached_handler_name = NULL; 942 943 /* we need at least a path arg */ 944 if (as->argc < 1) { 945 ti->error = "no device given"; 946 return ERR_PTR(-EINVAL); 947 } 948 949 p = alloc_pgpath(); 950 if (!p) 951 return ERR_PTR(-ENOMEM); 952 953 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table), 954 &p->path.dev); 955 if (r) { 956 ti->error = "error getting device"; 957 goto bad; 958 } 959 960 q = bdev_get_queue(p->path.dev->bdev); 961 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL); 962 if (attached_handler_name || m->hw_handler_name) { 963 INIT_DELAYED_WORK(&p->activate_path, activate_path_work); 964 r = setup_scsi_dh(p->path.dev->bdev, m, &attached_handler_name, &ti->error); 965 kfree(attached_handler_name); 966 if (r) { 967 dm_put_device(ti, p->path.dev); 968 goto bad; 969 } 970 } 971 972 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error); 973 if (r) { 974 dm_put_device(ti, p->path.dev); 975 goto bad; 976 } 977 978 return p; 979 bad: 980 free_pgpath(p); 981 return ERR_PTR(r); 982} 983 984static struct priority_group *parse_priority_group(struct dm_arg_set *as, 985 struct multipath *m) 986{ 987 static const struct dm_arg _args[] = { 988 {1, 1024, "invalid number of paths"}, 989 {0, 1024, "invalid number of selector args"} 990 }; 991 992 int r; 993 unsigned int i, nr_selector_args, nr_args; 994 struct priority_group *pg; 995 struct dm_target *ti = m->ti; 996 997 if (as->argc < 2) { 998 as->argc = 0; 999 ti->error = "not enough priority group arguments"; 1000 return ERR_PTR(-EINVAL); 1001 } 1002 1003 pg = alloc_priority_group(); 1004 if (!pg) { 1005 ti->error = "couldn't allocate priority group"; 1006 return ERR_PTR(-ENOMEM); 1007 } 1008 pg->m = m; 1009 1010 r = parse_path_selector(as, pg, ti); 1011 if (r) 1012 goto bad; 1013 1014 /* 1015 * read the paths 1016 */ 1017 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error); 1018 if (r) 1019 goto bad; 1020 1021 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error); 1022 if (r) 1023 goto bad; 1024 1025 nr_args = 1 + nr_selector_args; 1026 for (i = 0; i < pg->nr_pgpaths; i++) { 1027 struct pgpath *pgpath; 1028 struct dm_arg_set path_args; 1029 1030 if (as->argc < nr_args) { 1031 ti->error = "not enough path parameters"; 1032 r = -EINVAL; 1033 goto bad; 1034 } 1035 1036 path_args.argc = nr_args; 1037 path_args.argv = as->argv; 1038 1039 pgpath = parse_path(&path_args, &pg->ps, ti); 1040 if (IS_ERR(pgpath)) { 1041 r = PTR_ERR(pgpath); 1042 goto bad; 1043 } 1044 1045 pgpath->pg = pg; 1046 list_add_tail(&pgpath->list, &pg->pgpaths); 1047 dm_consume_args(as, nr_args); 1048 } 1049 1050 return pg; 1051 1052 bad: 1053 free_priority_group(pg, ti); 1054 return ERR_PTR(r); 1055} 1056 1057static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m) 1058{ 1059 unsigned int hw_argc; 1060 int ret; 1061 struct dm_target *ti = m->ti; 1062 1063 static const struct dm_arg _args[] = { 1064 {0, 1024, "invalid number of hardware handler args"}, 1065 }; 1066 1067 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error)) 1068 return -EINVAL; 1069 1070 if (!hw_argc) 1071 return 0; 1072 1073 if (m->queue_mode == DM_TYPE_BIO_BASED) { 1074 dm_consume_args(as, hw_argc); 1075 DMERR("bio-based multipath doesn't allow hardware handler args"); 1076 return 0; 1077 } 1078 1079 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL); 1080 if (!m->hw_handler_name) 1081 return -EINVAL; 1082 1083 if (hw_argc > 1) { 1084 char *p; 1085 int i, j, len = 4; 1086 1087 for (i = 0; i <= hw_argc - 2; i++) 1088 len += strlen(as->argv[i]) + 1; 1089 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL); 1090 if (!p) { 1091 ti->error = "memory allocation failed"; 1092 ret = -ENOMEM; 1093 goto fail; 1094 } 1095 j = sprintf(p, "%d", hw_argc - 1); 1096 for (i = 0, p += j + 1; i <= hw_argc - 2; i++, p += j + 1) 1097 j = sprintf(p, "%s", as->argv[i]); 1098 } 1099 dm_consume_args(as, hw_argc - 1); 1100 1101 return 0; 1102fail: 1103 kfree(m->hw_handler_name); 1104 m->hw_handler_name = NULL; 1105 return ret; 1106} 1107 1108static int parse_features(struct dm_arg_set *as, struct multipath *m) 1109{ 1110 int r; 1111 unsigned int argc; 1112 struct dm_target *ti = m->ti; 1113 const char *arg_name; 1114 1115 static const struct dm_arg _args[] = { 1116 {0, 8, "invalid number of feature args"}, 1117 {1, 50, "pg_init_retries must be between 1 and 50"}, 1118 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"}, 1119 }; 1120 1121 r = dm_read_arg_group(_args, as, &argc, &ti->error); 1122 if (r) 1123 return -EINVAL; 1124 1125 if (!argc) 1126 return 0; 1127 1128 do { 1129 arg_name = dm_shift_arg(as); 1130 argc--; 1131 1132 if (!strcasecmp(arg_name, "queue_if_no_path")) { 1133 r = queue_if_no_path(m, true, false, __func__); 1134 continue; 1135 } 1136 1137 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) { 1138 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags); 1139 continue; 1140 } 1141 1142 if (!strcasecmp(arg_name, "pg_init_retries") && 1143 (argc >= 1)) { 1144 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error); 1145 argc--; 1146 continue; 1147 } 1148 1149 if (!strcasecmp(arg_name, "pg_init_delay_msecs") && 1150 (argc >= 1)) { 1151 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error); 1152 argc--; 1153 continue; 1154 } 1155 1156 if (!strcasecmp(arg_name, "queue_mode") && 1157 (argc >= 1)) { 1158 const char *queue_mode_name = dm_shift_arg(as); 1159 1160 if (!strcasecmp(queue_mode_name, "bio")) 1161 m->queue_mode = DM_TYPE_BIO_BASED; 1162 else if (!strcasecmp(queue_mode_name, "rq") || 1163 !strcasecmp(queue_mode_name, "mq")) 1164 m->queue_mode = DM_TYPE_REQUEST_BASED; 1165 else { 1166 ti->error = "Unknown 'queue_mode' requested"; 1167 r = -EINVAL; 1168 } 1169 argc--; 1170 continue; 1171 } 1172 1173 ti->error = "Unrecognised multipath feature request"; 1174 r = -EINVAL; 1175 } while (argc && !r); 1176 1177 return r; 1178} 1179 1180static int multipath_ctr(struct dm_target *ti, unsigned int argc, char **argv) 1181{ 1182 /* target arguments */ 1183 static const struct dm_arg _args[] = { 1184 {0, 1024, "invalid number of priority groups"}, 1185 {0, 1024, "invalid initial priority group number"}, 1186 }; 1187 1188 int r; 1189 struct multipath *m; 1190 struct dm_arg_set as; 1191 unsigned int pg_count = 0; 1192 unsigned int next_pg_num; 1193 unsigned long flags; 1194 1195 as.argc = argc; 1196 as.argv = argv; 1197 1198 m = alloc_multipath(ti); 1199 if (!m) { 1200 ti->error = "can't allocate multipath"; 1201 return -EINVAL; 1202 } 1203 1204 r = parse_features(&as, m); 1205 if (r) 1206 goto bad; 1207 1208 r = alloc_multipath_stage2(ti, m); 1209 if (r) 1210 goto bad; 1211 1212 r = parse_hw_handler(&as, m); 1213 if (r) 1214 goto bad; 1215 1216 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error); 1217 if (r) 1218 goto bad; 1219 1220 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error); 1221 if (r) 1222 goto bad; 1223 1224 if ((!m->nr_priority_groups && next_pg_num) || 1225 (m->nr_priority_groups && !next_pg_num)) { 1226 ti->error = "invalid initial priority group"; 1227 r = -EINVAL; 1228 goto bad; 1229 } 1230 1231 /* parse the priority groups */ 1232 while (as.argc) { 1233 struct priority_group *pg; 1234 unsigned int nr_valid_paths = atomic_read(&m->nr_valid_paths); 1235 1236 pg = parse_priority_group(&as, m); 1237 if (IS_ERR(pg)) { 1238 r = PTR_ERR(pg); 1239 goto bad; 1240 } 1241 1242 nr_valid_paths += pg->nr_pgpaths; 1243 atomic_set(&m->nr_valid_paths, nr_valid_paths); 1244 1245 list_add_tail(&pg->list, &m->priority_groups); 1246 pg_count++; 1247 pg->pg_num = pg_count; 1248 if (!--next_pg_num) 1249 m->next_pg = pg; 1250 } 1251 1252 if (pg_count != m->nr_priority_groups) { 1253 ti->error = "priority group count mismatch"; 1254 r = -EINVAL; 1255 goto bad; 1256 } 1257 1258 spin_lock_irqsave(&m->lock, flags); 1259 enable_nopath_timeout(m); 1260 spin_unlock_irqrestore(&m->lock, flags); 1261 1262 ti->num_flush_bios = 1; 1263 ti->num_discard_bios = 1; 1264 ti->num_write_zeroes_bios = 1; 1265 if (m->queue_mode == DM_TYPE_BIO_BASED) 1266 ti->per_io_data_size = multipath_per_bio_data_size(); 1267 else 1268 ti->per_io_data_size = sizeof(struct dm_mpath_io); 1269 1270 return 0; 1271 1272 bad: 1273 free_multipath(m); 1274 return r; 1275} 1276 1277static void multipath_wait_for_pg_init_completion(struct multipath *m) 1278{ 1279 DEFINE_WAIT(wait); 1280 1281 while (1) { 1282 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE); 1283 1284 if (!atomic_read(&m->pg_init_in_progress)) 1285 break; 1286 1287 io_schedule(); 1288 } 1289 finish_wait(&m->pg_init_wait, &wait); 1290} 1291 1292static void flush_multipath_work(struct multipath *m) 1293{ 1294 if (m->hw_handler_name) { 1295 unsigned long flags; 1296 1297 if (!atomic_read(&m->pg_init_in_progress)) 1298 goto skip; 1299 1300 spin_lock_irqsave(&m->lock, flags); 1301 if (atomic_read(&m->pg_init_in_progress) && 1302 !test_and_set_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) { 1303 spin_unlock_irqrestore(&m->lock, flags); 1304 1305 flush_workqueue(kmpath_handlerd); 1306 multipath_wait_for_pg_init_completion(m); 1307 1308 spin_lock_irqsave(&m->lock, flags); 1309 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags); 1310 } 1311 spin_unlock_irqrestore(&m->lock, flags); 1312 } 1313skip: 1314 if (m->queue_mode == DM_TYPE_BIO_BASED) 1315 flush_work(&m->process_queued_bios); 1316 flush_work(&m->trigger_event); 1317} 1318 1319static void multipath_dtr(struct dm_target *ti) 1320{ 1321 struct multipath *m = ti->private; 1322 1323 disable_nopath_timeout(m); 1324 flush_multipath_work(m); 1325 free_multipath(m); 1326} 1327 1328/* 1329 * Take a path out of use. 1330 */ 1331static int fail_path(struct pgpath *pgpath) 1332{ 1333 unsigned long flags; 1334 struct multipath *m = pgpath->pg->m; 1335 1336 spin_lock_irqsave(&m->lock, flags); 1337 1338 if (!pgpath->is_active) 1339 goto out; 1340 1341 DMWARN("%s: Failing path %s.", 1342 dm_table_device_name(m->ti->table), 1343 pgpath->path.dev->name); 1344 1345 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path); 1346 pgpath->is_active = false; 1347 pgpath->fail_count++; 1348 1349 atomic_dec(&m->nr_valid_paths); 1350 1351 if (pgpath == m->current_pgpath) 1352 m->current_pgpath = NULL; 1353 1354 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti, 1355 pgpath->path.dev->name, atomic_read(&m->nr_valid_paths)); 1356 1357 queue_work(dm_mpath_wq, &m->trigger_event); 1358 1359 enable_nopath_timeout(m); 1360 1361out: 1362 spin_unlock_irqrestore(&m->lock, flags); 1363 1364 return 0; 1365} 1366 1367/* 1368 * Reinstate a previously-failed path 1369 */ 1370static int reinstate_path(struct pgpath *pgpath) 1371{ 1372 int r = 0, run_queue = 0; 1373 unsigned long flags; 1374 struct multipath *m = pgpath->pg->m; 1375 unsigned int nr_valid_paths; 1376 1377 spin_lock_irqsave(&m->lock, flags); 1378 1379 if (pgpath->is_active) 1380 goto out; 1381 1382 DMWARN("%s: Reinstating path %s.", 1383 dm_table_device_name(m->ti->table), 1384 pgpath->path.dev->name); 1385 1386 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path); 1387 if (r) 1388 goto out; 1389 1390 pgpath->is_active = true; 1391 1392 nr_valid_paths = atomic_inc_return(&m->nr_valid_paths); 1393 if (nr_valid_paths == 1) { 1394 m->current_pgpath = NULL; 1395 run_queue = 1; 1396 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) { 1397 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work)) 1398 atomic_inc(&m->pg_init_in_progress); 1399 } 1400 1401 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti, 1402 pgpath->path.dev->name, nr_valid_paths); 1403 1404 schedule_work(&m->trigger_event); 1405 1406out: 1407 spin_unlock_irqrestore(&m->lock, flags); 1408 if (run_queue) { 1409 dm_table_run_md_queue_async(m->ti->table); 1410 process_queued_io_list(m); 1411 } 1412 1413 if (pgpath->is_active) 1414 disable_nopath_timeout(m); 1415 1416 return r; 1417} 1418 1419/* 1420 * Fail or reinstate all paths that match the provided struct dm_dev. 1421 */ 1422static int action_dev(struct multipath *m, struct dm_dev *dev, 1423 action_fn action) 1424{ 1425 int r = -EINVAL; 1426 struct pgpath *pgpath; 1427 struct priority_group *pg; 1428 1429 list_for_each_entry(pg, &m->priority_groups, list) { 1430 list_for_each_entry(pgpath, &pg->pgpaths, list) { 1431 if (pgpath->path.dev == dev) 1432 r = action(pgpath); 1433 } 1434 } 1435 1436 return r; 1437} 1438 1439/* 1440 * Temporarily try to avoid having to use the specified PG 1441 */ 1442static void bypass_pg(struct multipath *m, struct priority_group *pg, 1443 bool bypassed) 1444{ 1445 unsigned long flags; 1446 1447 spin_lock_irqsave(&m->lock, flags); 1448 1449 pg->bypassed = bypassed; 1450 m->current_pgpath = NULL; 1451 m->current_pg = NULL; 1452 1453 spin_unlock_irqrestore(&m->lock, flags); 1454 1455 schedule_work(&m->trigger_event); 1456} 1457 1458/* 1459 * Switch to using the specified PG from the next I/O that gets mapped 1460 */ 1461static int switch_pg_num(struct multipath *m, const char *pgstr) 1462{ 1463 struct priority_group *pg; 1464 unsigned int pgnum; 1465 unsigned long flags; 1466 char dummy; 1467 1468 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 1469 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) { 1470 DMWARN("invalid PG number supplied to %s", __func__); 1471 return -EINVAL; 1472 } 1473 1474 spin_lock_irqsave(&m->lock, flags); 1475 list_for_each_entry(pg, &m->priority_groups, list) { 1476 pg->bypassed = false; 1477 if (--pgnum) 1478 continue; 1479 1480 m->current_pgpath = NULL; 1481 m->current_pg = NULL; 1482 m->next_pg = pg; 1483 } 1484 spin_unlock_irqrestore(&m->lock, flags); 1485 1486 schedule_work(&m->trigger_event); 1487 return 0; 1488} 1489 1490/* 1491 * Set/clear bypassed status of a PG. 1492 * PGs are numbered upwards from 1 in the order they were declared. 1493 */ 1494static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed) 1495{ 1496 struct priority_group *pg; 1497 unsigned int pgnum; 1498 char dummy; 1499 1500 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 1501 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) { 1502 DMWARN("invalid PG number supplied to bypass_pg"); 1503 return -EINVAL; 1504 } 1505 1506 list_for_each_entry(pg, &m->priority_groups, list) { 1507 if (!--pgnum) 1508 break; 1509 } 1510 1511 bypass_pg(m, pg, bypassed); 1512 return 0; 1513} 1514 1515/* 1516 * Should we retry pg_init immediately? 1517 */ 1518static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath) 1519{ 1520 unsigned long flags; 1521 bool limit_reached = false; 1522 1523 spin_lock_irqsave(&m->lock, flags); 1524 1525 if (atomic_read(&m->pg_init_count) <= m->pg_init_retries && 1526 !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) 1527 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 1528 else 1529 limit_reached = true; 1530 1531 spin_unlock_irqrestore(&m->lock, flags); 1532 1533 return limit_reached; 1534} 1535 1536static void pg_init_done(void *data, int errors) 1537{ 1538 struct pgpath *pgpath = data; 1539 struct priority_group *pg = pgpath->pg; 1540 struct multipath *m = pg->m; 1541 unsigned long flags; 1542 bool delay_retry = false; 1543 1544 /* device or driver problems */ 1545 switch (errors) { 1546 case SCSI_DH_OK: 1547 break; 1548 case SCSI_DH_NOSYS: 1549 if (!m->hw_handler_name) { 1550 errors = 0; 1551 break; 1552 } 1553 DMERR("Could not failover the device: Handler scsi_dh_%s " 1554 "Error %d.", m->hw_handler_name, errors); 1555 /* 1556 * Fail path for now, so we do not ping pong 1557 */ 1558 fail_path(pgpath); 1559 break; 1560 case SCSI_DH_DEV_TEMP_BUSY: 1561 /* 1562 * Probably doing something like FW upgrade on the 1563 * controller so try the other pg. 1564 */ 1565 bypass_pg(m, pg, true); 1566 break; 1567 case SCSI_DH_RETRY: 1568 /* Wait before retrying. */ 1569 delay_retry = true; 1570 fallthrough; 1571 case SCSI_DH_IMM_RETRY: 1572 case SCSI_DH_RES_TEMP_UNAVAIL: 1573 if (pg_init_limit_reached(m, pgpath)) 1574 fail_path(pgpath); 1575 errors = 0; 1576 break; 1577 case SCSI_DH_DEV_OFFLINED: 1578 default: 1579 /* 1580 * We probably do not want to fail the path for a device 1581 * error, but this is what the old dm did. In future 1582 * patches we can do more advanced handling. 1583 */ 1584 fail_path(pgpath); 1585 } 1586 1587 spin_lock_irqsave(&m->lock, flags); 1588 if (errors) { 1589 if (pgpath == m->current_pgpath) { 1590 DMERR("Could not failover device. Error %d.", errors); 1591 m->current_pgpath = NULL; 1592 m->current_pg = NULL; 1593 } 1594 } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 1595 pg->bypassed = false; 1596 1597 if (atomic_dec_return(&m->pg_init_in_progress) > 0) 1598 /* Activations of other paths are still on going */ 1599 goto out; 1600 1601 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) { 1602 if (delay_retry) 1603 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 1604 else 1605 clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 1606 1607 if (__pg_init_all_paths(m)) 1608 goto out; 1609 } 1610 clear_bit(MPATHF_QUEUE_IO, &m->flags); 1611 1612 process_queued_io_list(m); 1613 1614 /* 1615 * Wake up any thread waiting to suspend. 1616 */ 1617 wake_up(&m->pg_init_wait); 1618 1619out: 1620 spin_unlock_irqrestore(&m->lock, flags); 1621} 1622 1623static void activate_or_offline_path(struct pgpath *pgpath) 1624{ 1625 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev); 1626 1627 if (pgpath->is_active && !blk_queue_dying(q)) 1628 scsi_dh_activate(q, pg_init_done, pgpath); 1629 else 1630 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED); 1631} 1632 1633static void activate_path_work(struct work_struct *work) 1634{ 1635 struct pgpath *pgpath = 1636 container_of(work, struct pgpath, activate_path.work); 1637 1638 activate_or_offline_path(pgpath); 1639} 1640 1641static int multipath_end_io(struct dm_target *ti, struct request *clone, 1642 blk_status_t error, union map_info *map_context) 1643{ 1644 struct dm_mpath_io *mpio = get_mpio(map_context); 1645 struct pgpath *pgpath = mpio->pgpath; 1646 int r = DM_ENDIO_DONE; 1647 1648 /* 1649 * We don't queue any clone request inside the multipath target 1650 * during end I/O handling, since those clone requests don't have 1651 * bio clones. If we queue them inside the multipath target, 1652 * we need to make bio clones, that requires memory allocation. 1653 * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests 1654 * don't have bio clones.) 1655 * Instead of queueing the clone request here, we queue the original 1656 * request into dm core, which will remake a clone request and 1657 * clone bios for it and resubmit it later. 1658 */ 1659 if (error && blk_path_error(error)) { 1660 struct multipath *m = ti->private; 1661 1662 if (error == BLK_STS_RESOURCE) 1663 r = DM_ENDIO_DELAY_REQUEUE; 1664 else 1665 r = DM_ENDIO_REQUEUE; 1666 1667 if (pgpath) 1668 fail_path(pgpath); 1669 1670 if (!atomic_read(&m->nr_valid_paths) && 1671 !must_push_back_rq(m)) { 1672 if (error == BLK_STS_IOERR) 1673 dm_report_EIO(m); 1674 /* complete with the original error */ 1675 r = DM_ENDIO_DONE; 1676 } 1677 } 1678 1679 if (pgpath) { 1680 struct path_selector *ps = &pgpath->pg->ps; 1681 1682 if (ps->type->end_io) 1683 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes, 1684 clone->io_start_time_ns); 1685 } 1686 1687 return r; 1688} 1689 1690static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone, 1691 blk_status_t *error) 1692{ 1693 struct multipath *m = ti->private; 1694 struct dm_mpath_io *mpio = get_mpio_from_bio(clone); 1695 struct pgpath *pgpath = mpio->pgpath; 1696 unsigned long flags; 1697 int r = DM_ENDIO_DONE; 1698 1699 if (!*error || !blk_path_error(*error)) 1700 goto done; 1701 1702 if (pgpath) 1703 fail_path(pgpath); 1704 1705 if (!atomic_read(&m->nr_valid_paths)) { 1706 spin_lock_irqsave(&m->lock, flags); 1707 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { 1708 if (__must_push_back(m)) { 1709 r = DM_ENDIO_REQUEUE; 1710 } else { 1711 dm_report_EIO(m); 1712 *error = BLK_STS_IOERR; 1713 } 1714 spin_unlock_irqrestore(&m->lock, flags); 1715 goto done; 1716 } 1717 spin_unlock_irqrestore(&m->lock, flags); 1718 } 1719 1720 multipath_queue_bio(m, clone); 1721 r = DM_ENDIO_INCOMPLETE; 1722done: 1723 if (pgpath) { 1724 struct path_selector *ps = &pgpath->pg->ps; 1725 1726 if (ps->type->end_io) 1727 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes, 1728 (mpio->start_time_ns ?: 1729 dm_start_time_ns_from_clone(clone))); 1730 } 1731 1732 return r; 1733} 1734 1735/* 1736 * Suspend with flush can't complete until all the I/O is processed 1737 * so if the last path fails we must error any remaining I/O. 1738 * - Note that if the freeze_bdev fails while suspending, the 1739 * queue_if_no_path state is lost - userspace should reset it. 1740 * Otherwise, during noflush suspend, queue_if_no_path will not change. 1741 */ 1742static void multipath_presuspend(struct dm_target *ti) 1743{ 1744 struct multipath *m = ti->private; 1745 1746 /* FIXME: bio-based shouldn't need to always disable queue_if_no_path */ 1747 if (m->queue_mode == DM_TYPE_BIO_BASED || !dm_noflush_suspending(m->ti)) 1748 queue_if_no_path(m, false, true, __func__); 1749} 1750 1751static void multipath_postsuspend(struct dm_target *ti) 1752{ 1753 struct multipath *m = ti->private; 1754 1755 mutex_lock(&m->work_mutex); 1756 flush_multipath_work(m); 1757 mutex_unlock(&m->work_mutex); 1758} 1759 1760/* 1761 * Restore the queue_if_no_path setting. 1762 */ 1763static void multipath_resume(struct dm_target *ti) 1764{ 1765 struct multipath *m = ti->private; 1766 unsigned long flags; 1767 1768 spin_lock_irqsave(&m->lock, flags); 1769 if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) { 1770 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags); 1771 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags); 1772 } 1773 1774 DMDEBUG("%s: %s finished; QIFNP = %d; SQIFNP = %d", 1775 dm_table_device_name(m->ti->table), __func__, 1776 test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags), 1777 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)); 1778 1779 spin_unlock_irqrestore(&m->lock, flags); 1780} 1781 1782/* 1783 * Info output has the following format: 1784 * num_multipath_feature_args [multipath_feature_args]* 1785 * num_handler_status_args [handler_status_args]* 1786 * num_groups init_group_number 1787 * [A|D|E num_ps_status_args [ps_status_args]* 1788 * num_paths num_selector_args 1789 * [path_dev A|F fail_count [selector_args]* ]+ ]+ 1790 * 1791 * Table output has the following format (identical to the constructor string): 1792 * num_feature_args [features_args]* 1793 * num_handler_args hw_handler [hw_handler_args]* 1794 * num_groups init_group_number 1795 * [priority selector-name num_ps_args [ps_args]* 1796 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+ 1797 */ 1798static void multipath_status(struct dm_target *ti, status_type_t type, 1799 unsigned int status_flags, char *result, unsigned int maxlen) 1800{ 1801 int sz = 0, pg_counter, pgpath_counter; 1802 unsigned long flags; 1803 struct multipath *m = ti->private; 1804 struct priority_group *pg; 1805 struct pgpath *p; 1806 unsigned int pg_num; 1807 char state; 1808 1809 spin_lock_irqsave(&m->lock, flags); 1810 1811 /* Features */ 1812 if (type == STATUSTYPE_INFO) 1813 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags), 1814 atomic_read(&m->pg_init_count)); 1815 else { 1816 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) + 1817 (m->pg_init_retries > 0) * 2 + 1818 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 + 1819 test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) + 1820 (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2); 1821 1822 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1823 DMEMIT("queue_if_no_path "); 1824 if (m->pg_init_retries) 1825 DMEMIT("pg_init_retries %u ", m->pg_init_retries); 1826 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) 1827 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs); 1828 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) 1829 DMEMIT("retain_attached_hw_handler "); 1830 if (m->queue_mode != DM_TYPE_REQUEST_BASED) { 1831 switch (m->queue_mode) { 1832 case DM_TYPE_BIO_BASED: 1833 DMEMIT("queue_mode bio "); 1834 break; 1835 default: 1836 WARN_ON_ONCE(true); 1837 break; 1838 } 1839 } 1840 } 1841 1842 if (!m->hw_handler_name || type == STATUSTYPE_INFO) 1843 DMEMIT("0 "); 1844 else 1845 DMEMIT("1 %s ", m->hw_handler_name); 1846 1847 DMEMIT("%u ", m->nr_priority_groups); 1848 1849 if (m->next_pg) 1850 pg_num = m->next_pg->pg_num; 1851 else if (m->current_pg) 1852 pg_num = m->current_pg->pg_num; 1853 else 1854 pg_num = (m->nr_priority_groups ? 1 : 0); 1855 1856 DMEMIT("%u ", pg_num); 1857 1858 switch (type) { 1859 case STATUSTYPE_INFO: 1860 list_for_each_entry(pg, &m->priority_groups, list) { 1861 if (pg->bypassed) 1862 state = 'D'; /* Disabled */ 1863 else if (pg == m->current_pg) 1864 state = 'A'; /* Currently Active */ 1865 else 1866 state = 'E'; /* Enabled */ 1867 1868 DMEMIT("%c ", state); 1869 1870 if (pg->ps.type->status) 1871 sz += pg->ps.type->status(&pg->ps, NULL, type, 1872 result + sz, 1873 maxlen - sz); 1874 else 1875 DMEMIT("0 "); 1876 1877 DMEMIT("%u %u ", pg->nr_pgpaths, 1878 pg->ps.type->info_args); 1879 1880 list_for_each_entry(p, &pg->pgpaths, list) { 1881 DMEMIT("%s %s %u ", p->path.dev->name, 1882 p->is_active ? "A" : "F", 1883 p->fail_count); 1884 if (pg->ps.type->status) 1885 sz += pg->ps.type->status(&pg->ps, 1886 &p->path, type, result + sz, 1887 maxlen - sz); 1888 } 1889 } 1890 break; 1891 1892 case STATUSTYPE_TABLE: 1893 list_for_each_entry(pg, &m->priority_groups, list) { 1894 DMEMIT("%s ", pg->ps.type->name); 1895 1896 if (pg->ps.type->status) 1897 sz += pg->ps.type->status(&pg->ps, NULL, type, 1898 result + sz, 1899 maxlen - sz); 1900 else 1901 DMEMIT("0 "); 1902 1903 DMEMIT("%u %u ", pg->nr_pgpaths, 1904 pg->ps.type->table_args); 1905 1906 list_for_each_entry(p, &pg->pgpaths, list) { 1907 DMEMIT("%s ", p->path.dev->name); 1908 if (pg->ps.type->status) 1909 sz += pg->ps.type->status(&pg->ps, 1910 &p->path, type, result + sz, 1911 maxlen - sz); 1912 } 1913 } 1914 break; 1915 1916 case STATUSTYPE_IMA: 1917 sz = 0; /*reset the result pointer*/ 1918 1919 DMEMIT_TARGET_NAME_VERSION(ti->type); 1920 DMEMIT(",nr_priority_groups=%u", m->nr_priority_groups); 1921 1922 pg_counter = 0; 1923 list_for_each_entry(pg, &m->priority_groups, list) { 1924 if (pg->bypassed) 1925 state = 'D'; /* Disabled */ 1926 else if (pg == m->current_pg) 1927 state = 'A'; /* Currently Active */ 1928 else 1929 state = 'E'; /* Enabled */ 1930 DMEMIT(",pg_state_%d=%c", pg_counter, state); 1931 DMEMIT(",nr_pgpaths_%d=%u", pg_counter, pg->nr_pgpaths); 1932 DMEMIT(",path_selector_name_%d=%s", pg_counter, pg->ps.type->name); 1933 1934 pgpath_counter = 0; 1935 list_for_each_entry(p, &pg->pgpaths, list) { 1936 DMEMIT(",path_name_%d_%d=%s,is_active_%d_%d=%c,fail_count_%d_%d=%u", 1937 pg_counter, pgpath_counter, p->path.dev->name, 1938 pg_counter, pgpath_counter, p->is_active ? 'A' : 'F', 1939 pg_counter, pgpath_counter, p->fail_count); 1940 if (pg->ps.type->status) { 1941 DMEMIT(",path_selector_status_%d_%d=", 1942 pg_counter, pgpath_counter); 1943 sz += pg->ps.type->status(&pg->ps, &p->path, 1944 type, result + sz, 1945 maxlen - sz); 1946 } 1947 pgpath_counter++; 1948 } 1949 pg_counter++; 1950 } 1951 DMEMIT(";"); 1952 break; 1953 } 1954 1955 spin_unlock_irqrestore(&m->lock, flags); 1956} 1957 1958static int multipath_message(struct dm_target *ti, unsigned int argc, char **argv, 1959 char *result, unsigned int maxlen) 1960{ 1961 int r = -EINVAL; 1962 struct dm_dev *dev; 1963 struct multipath *m = ti->private; 1964 action_fn action; 1965 unsigned long flags; 1966 1967 mutex_lock(&m->work_mutex); 1968 1969 if (dm_suspended(ti)) { 1970 r = -EBUSY; 1971 goto out; 1972 } 1973 1974 if (argc == 1) { 1975 if (!strcasecmp(argv[0], "queue_if_no_path")) { 1976 r = queue_if_no_path(m, true, false, __func__); 1977 spin_lock_irqsave(&m->lock, flags); 1978 enable_nopath_timeout(m); 1979 spin_unlock_irqrestore(&m->lock, flags); 1980 goto out; 1981 } else if (!strcasecmp(argv[0], "fail_if_no_path")) { 1982 r = queue_if_no_path(m, false, false, __func__); 1983 disable_nopath_timeout(m); 1984 goto out; 1985 } 1986 } 1987 1988 if (argc != 2) { 1989 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc); 1990 goto out; 1991 } 1992 1993 if (!strcasecmp(argv[0], "disable_group")) { 1994 r = bypass_pg_num(m, argv[1], true); 1995 goto out; 1996 } else if (!strcasecmp(argv[0], "enable_group")) { 1997 r = bypass_pg_num(m, argv[1], false); 1998 goto out; 1999 } else if (!strcasecmp(argv[0], "switch_group")) { 2000 r = switch_pg_num(m, argv[1]); 2001 goto out; 2002 } else if (!strcasecmp(argv[0], "reinstate_path")) 2003 action = reinstate_path; 2004 else if (!strcasecmp(argv[0], "fail_path")) 2005 action = fail_path; 2006 else { 2007 DMWARN("Unrecognised multipath message received: %s", argv[0]); 2008 goto out; 2009 } 2010 2011 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev); 2012 if (r) { 2013 DMWARN("message: error getting device %s", 2014 argv[1]); 2015 goto out; 2016 } 2017 2018 r = action_dev(m, dev, action); 2019 2020 dm_put_device(ti, dev); 2021 2022out: 2023 mutex_unlock(&m->work_mutex); 2024 return r; 2025} 2026 2027static int multipath_prepare_ioctl(struct dm_target *ti, 2028 struct block_device **bdev) 2029{ 2030 struct multipath *m = ti->private; 2031 struct pgpath *pgpath; 2032 unsigned long flags; 2033 int r; 2034 2035 pgpath = READ_ONCE(m->current_pgpath); 2036 if (!pgpath || !mpath_double_check_test_bit(MPATHF_QUEUE_IO, m)) 2037 pgpath = choose_pgpath(m, 0); 2038 2039 if (pgpath) { 2040 if (!mpath_double_check_test_bit(MPATHF_QUEUE_IO, m)) { 2041 *bdev = pgpath->path.dev->bdev; 2042 r = 0; 2043 } else { 2044 /* pg_init has not started or completed */ 2045 r = -ENOTCONN; 2046 } 2047 } else { 2048 /* No path is available */ 2049 r = -EIO; 2050 spin_lock_irqsave(&m->lock, flags); 2051 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 2052 r = -ENOTCONN; 2053 spin_unlock_irqrestore(&m->lock, flags); 2054 } 2055 2056 if (r == -ENOTCONN) { 2057 if (!READ_ONCE(m->current_pg)) { 2058 /* Path status changed, redo selection */ 2059 (void) choose_pgpath(m, 0); 2060 } 2061 spin_lock_irqsave(&m->lock, flags); 2062 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 2063 (void) __pg_init_all_paths(m); 2064 spin_unlock_irqrestore(&m->lock, flags); 2065 dm_table_run_md_queue_async(m->ti->table); 2066 process_queued_io_list(m); 2067 } 2068 2069 /* 2070 * Only pass ioctls through if the device sizes match exactly. 2071 */ 2072 if (!r && ti->len != bdev_nr_sectors((*bdev))) 2073 return 1; 2074 return r; 2075} 2076 2077static int multipath_iterate_devices(struct dm_target *ti, 2078 iterate_devices_callout_fn fn, void *data) 2079{ 2080 struct multipath *m = ti->private; 2081 struct priority_group *pg; 2082 struct pgpath *p; 2083 int ret = 0; 2084 2085 list_for_each_entry(pg, &m->priority_groups, list) { 2086 list_for_each_entry(p, &pg->pgpaths, list) { 2087 ret = fn(ti, p->path.dev, ti->begin, ti->len, data); 2088 if (ret) 2089 goto out; 2090 } 2091 } 2092 2093out: 2094 return ret; 2095} 2096 2097static int pgpath_busy(struct pgpath *pgpath) 2098{ 2099 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev); 2100 2101 return blk_lld_busy(q); 2102} 2103 2104/* 2105 * We return "busy", only when we can map I/Os but underlying devices 2106 * are busy (so even if we map I/Os now, the I/Os will wait on 2107 * the underlying queue). 2108 * In other words, if we want to kill I/Os or queue them inside us 2109 * due to map unavailability, we don't return "busy". Otherwise, 2110 * dm core won't give us the I/Os and we can't do what we want. 2111 */ 2112static int multipath_busy(struct dm_target *ti) 2113{ 2114 bool busy = false, has_active = false; 2115 struct multipath *m = ti->private; 2116 struct priority_group *pg, *next_pg; 2117 struct pgpath *pgpath; 2118 2119 /* pg_init in progress */ 2120 if (atomic_read(&m->pg_init_in_progress)) 2121 return true; 2122 2123 /* no paths available, for blk-mq: rely on IO mapping to delay requeue */ 2124 if (!atomic_read(&m->nr_valid_paths)) { 2125 unsigned long flags; 2126 2127 spin_lock_irqsave(&m->lock, flags); 2128 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { 2129 spin_unlock_irqrestore(&m->lock, flags); 2130 return (m->queue_mode != DM_TYPE_REQUEST_BASED); 2131 } 2132 spin_unlock_irqrestore(&m->lock, flags); 2133 } 2134 2135 /* Guess which priority_group will be used at next mapping time */ 2136 pg = READ_ONCE(m->current_pg); 2137 next_pg = READ_ONCE(m->next_pg); 2138 if (unlikely(!READ_ONCE(m->current_pgpath) && next_pg)) 2139 pg = next_pg; 2140 2141 if (!pg) { 2142 /* 2143 * We don't know which pg will be used at next mapping time. 2144 * We don't call choose_pgpath() here to avoid to trigger 2145 * pg_init just by busy checking. 2146 * So we don't know whether underlying devices we will be using 2147 * at next mapping time are busy or not. Just try mapping. 2148 */ 2149 return busy; 2150 } 2151 2152 /* 2153 * If there is one non-busy active path at least, the path selector 2154 * will be able to select it. So we consider such a pg as not busy. 2155 */ 2156 busy = true; 2157 list_for_each_entry(pgpath, &pg->pgpaths, list) { 2158 if (pgpath->is_active) { 2159 has_active = true; 2160 if (!pgpath_busy(pgpath)) { 2161 busy = false; 2162 break; 2163 } 2164 } 2165 } 2166 2167 if (!has_active) { 2168 /* 2169 * No active path in this pg, so this pg won't be used and 2170 * the current_pg will be changed at next mapping time. 2171 * We need to try mapping to determine it. 2172 */ 2173 busy = false; 2174 } 2175 2176 return busy; 2177} 2178 2179/* 2180 *--------------------------------------------------------------- 2181 * Module setup 2182 *--------------------------------------------------------------- 2183 */ 2184static struct target_type multipath_target = { 2185 .name = "multipath", 2186 .version = {1, 14, 0}, 2187 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE | 2188 DM_TARGET_PASSES_INTEGRITY, 2189 .module = THIS_MODULE, 2190 .ctr = multipath_ctr, 2191 .dtr = multipath_dtr, 2192 .clone_and_map_rq = multipath_clone_and_map, 2193 .release_clone_rq = multipath_release_clone, 2194 .rq_end_io = multipath_end_io, 2195 .map = multipath_map_bio, 2196 .end_io = multipath_end_io_bio, 2197 .presuspend = multipath_presuspend, 2198 .postsuspend = multipath_postsuspend, 2199 .resume = multipath_resume, 2200 .status = multipath_status, 2201 .message = multipath_message, 2202 .prepare_ioctl = multipath_prepare_ioctl, 2203 .iterate_devices = multipath_iterate_devices, 2204 .busy = multipath_busy, 2205}; 2206 2207static int __init dm_multipath_init(void) 2208{ 2209 int r = -ENOMEM; 2210 2211 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0); 2212 if (!kmultipathd) { 2213 DMERR("failed to create workqueue kmpathd"); 2214 goto bad_alloc_kmultipathd; 2215 } 2216 2217 /* 2218 * A separate workqueue is used to handle the device handlers 2219 * to avoid overloading existing workqueue. Overloading the 2220 * old workqueue would also create a bottleneck in the 2221 * path of the storage hardware device activation. 2222 */ 2223 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd", 2224 WQ_MEM_RECLAIM); 2225 if (!kmpath_handlerd) { 2226 DMERR("failed to create workqueue kmpath_handlerd"); 2227 goto bad_alloc_kmpath_handlerd; 2228 } 2229 2230 dm_mpath_wq = alloc_workqueue("dm_mpath_wq", 0, 0); 2231 if (!dm_mpath_wq) { 2232 DMERR("failed to create workqueue dm_mpath_wq"); 2233 goto bad_alloc_dm_mpath_wq; 2234 } 2235 2236 r = dm_register_target(&multipath_target); 2237 if (r < 0) 2238 goto bad_register_target; 2239 2240 return 0; 2241 2242bad_register_target: 2243 destroy_workqueue(dm_mpath_wq); 2244bad_alloc_dm_mpath_wq: 2245 destroy_workqueue(kmpath_handlerd); 2246bad_alloc_kmpath_handlerd: 2247 destroy_workqueue(kmultipathd); 2248bad_alloc_kmultipathd: 2249 return r; 2250} 2251 2252static void __exit dm_multipath_exit(void) 2253{ 2254 destroy_workqueue(dm_mpath_wq); 2255 destroy_workqueue(kmpath_handlerd); 2256 destroy_workqueue(kmultipathd); 2257 2258 dm_unregister_target(&multipath_target); 2259} 2260 2261module_init(dm_multipath_init); 2262module_exit(dm_multipath_exit); 2263 2264module_param_named(queue_if_no_path_timeout_secs, queue_if_no_path_timeout_secs, ulong, 0644); 2265MODULE_PARM_DESC(queue_if_no_path_timeout_secs, "No available paths queue IO timeout in seconds"); 2266 2267MODULE_DESCRIPTION(DM_NAME " multipath target"); 2268MODULE_AUTHOR("Sistina Software <dm-devel@lists.linux.dev>"); 2269MODULE_LICENSE("GPL"); 2270