1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * fs/userfaultfd.c 4 * 5 * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org> 6 * Copyright (C) 2008-2009 Red Hat, Inc. 7 * Copyright (C) 2015 Red Hat, Inc. 8 * 9 * Some part derived from fs/eventfd.c (anon inode setup) and 10 * mm/ksm.c (mm hashing). 11 */ 12 13#include <linux/list.h> 14#include <linux/hashtable.h> 15#include <linux/sched/signal.h> 16#include <linux/sched/mm.h> 17#include <linux/mm.h> 18#include <linux/mm_inline.h> 19#include <linux/mmu_notifier.h> 20#include <linux/poll.h> 21#include <linux/slab.h> 22#include <linux/seq_file.h> 23#include <linux/file.h> 24#include <linux/bug.h> 25#include <linux/anon_inodes.h> 26#include <linux/syscalls.h> 27#include <linux/userfaultfd_k.h> 28#include <linux/mempolicy.h> 29#include <linux/ioctl.h> 30#include <linux/security.h> 31#include <linux/hugetlb.h> 32#include <linux/swapops.h> 33#include <linux/miscdevice.h> 34#include <linux/uio.h> 35 36static int sysctl_unprivileged_userfaultfd __read_mostly; 37 38#ifdef CONFIG_SYSCTL 39static struct ctl_table vm_userfaultfd_table[] = { 40 { 41 .procname = "unprivileged_userfaultfd", 42 .data = &sysctl_unprivileged_userfaultfd, 43 .maxlen = sizeof(sysctl_unprivileged_userfaultfd), 44 .mode = 0644, 45 .proc_handler = proc_dointvec_minmax, 46 .extra1 = SYSCTL_ZERO, 47 .extra2 = SYSCTL_ONE, 48 }, 49}; 50#endif 51 52static struct kmem_cache *userfaultfd_ctx_cachep __ro_after_init; 53 54struct userfaultfd_fork_ctx { 55 struct userfaultfd_ctx *orig; 56 struct userfaultfd_ctx *new; 57 struct list_head list; 58}; 59 60struct userfaultfd_unmap_ctx { 61 struct userfaultfd_ctx *ctx; 62 unsigned long start; 63 unsigned long end; 64 struct list_head list; 65}; 66 67struct userfaultfd_wait_queue { 68 struct uffd_msg msg; 69 wait_queue_entry_t wq; 70 struct userfaultfd_ctx *ctx; 71 bool waken; 72}; 73 74struct userfaultfd_wake_range { 75 unsigned long start; 76 unsigned long len; 77}; 78 79/* internal indication that UFFD_API ioctl was successfully executed */ 80#define UFFD_FEATURE_INITIALIZED (1u << 31) 81 82static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx) 83{ 84 return ctx->features & UFFD_FEATURE_INITIALIZED; 85} 86 87static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx) 88{ 89 return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC); 90} 91 92/* 93 * Whether WP_UNPOPULATED is enabled on the uffd context. It is only 94 * meaningful when userfaultfd_wp()==true on the vma and when it's 95 * anonymous. 96 */ 97bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma) 98{ 99 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; 100 101 if (!ctx) 102 return false; 103 104 return ctx->features & UFFD_FEATURE_WP_UNPOPULATED; 105} 106 107static void userfaultfd_set_vm_flags(struct vm_area_struct *vma, 108 vm_flags_t flags) 109{ 110 const bool uffd_wp_changed = (vma->vm_flags ^ flags) & VM_UFFD_WP; 111 112 vm_flags_reset(vma, flags); 113 /* 114 * For shared mappings, we want to enable writenotify while 115 * userfaultfd-wp is enabled (see vma_wants_writenotify()). We'll simply 116 * recalculate vma->vm_page_prot whenever userfaultfd-wp changes. 117 */ 118 if ((vma->vm_flags & VM_SHARED) && uffd_wp_changed) 119 vma_set_page_prot(vma); 120} 121 122static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode, 123 int wake_flags, void *key) 124{ 125 struct userfaultfd_wake_range *range = key; 126 int ret; 127 struct userfaultfd_wait_queue *uwq; 128 unsigned long start, len; 129 130 uwq = container_of(wq, struct userfaultfd_wait_queue, wq); 131 ret = 0; 132 /* len == 0 means wake all */ 133 start = range->start; 134 len = range->len; 135 if (len && (start > uwq->msg.arg.pagefault.address || 136 start + len <= uwq->msg.arg.pagefault.address)) 137 goto out; 138 WRITE_ONCE(uwq->waken, true); 139 /* 140 * The Program-Order guarantees provided by the scheduler 141 * ensure uwq->waken is visible before the task is woken. 142 */ 143 ret = wake_up_state(wq->private, mode); 144 if (ret) { 145 /* 146 * Wake only once, autoremove behavior. 147 * 148 * After the effect of list_del_init is visible to the other 149 * CPUs, the waitqueue may disappear from under us, see the 150 * !list_empty_careful() in handle_userfault(). 151 * 152 * try_to_wake_up() has an implicit smp_mb(), and the 153 * wq->private is read before calling the extern function 154 * "wake_up_state" (which in turns calls try_to_wake_up). 155 */ 156 list_del_init(&wq->entry); 157 } 158out: 159 return ret; 160} 161 162/** 163 * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd 164 * context. 165 * @ctx: [in] Pointer to the userfaultfd context. 166 */ 167static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx) 168{ 169 refcount_inc(&ctx->refcount); 170} 171 172/** 173 * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd 174 * context. 175 * @ctx: [in] Pointer to userfaultfd context. 176 * 177 * The userfaultfd context reference must have been previously acquired either 178 * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget(). 179 */ 180static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx) 181{ 182 if (refcount_dec_and_test(&ctx->refcount)) { 183 VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock)); 184 VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh)); 185 VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock)); 186 VM_BUG_ON(waitqueue_active(&ctx->fault_wqh)); 187 VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock)); 188 VM_BUG_ON(waitqueue_active(&ctx->event_wqh)); 189 VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock)); 190 VM_BUG_ON(waitqueue_active(&ctx->fd_wqh)); 191 mmdrop(ctx->mm); 192 kmem_cache_free(userfaultfd_ctx_cachep, ctx); 193 } 194} 195 196static inline void msg_init(struct uffd_msg *msg) 197{ 198 BUILD_BUG_ON(sizeof(struct uffd_msg) != 32); 199 /* 200 * Must use memset to zero out the paddings or kernel data is 201 * leaked to userland. 202 */ 203 memset(msg, 0, sizeof(struct uffd_msg)); 204} 205 206static inline struct uffd_msg userfault_msg(unsigned long address, 207 unsigned long real_address, 208 unsigned int flags, 209 unsigned long reason, 210 unsigned int features) 211{ 212 struct uffd_msg msg; 213 214 msg_init(&msg); 215 msg.event = UFFD_EVENT_PAGEFAULT; 216 217 msg.arg.pagefault.address = (features & UFFD_FEATURE_EXACT_ADDRESS) ? 218 real_address : address; 219 220 /* 221 * These flags indicate why the userfault occurred: 222 * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault. 223 * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault. 224 * - Neither of these flags being set indicates a MISSING fault. 225 * 226 * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write 227 * fault. Otherwise, it was a read fault. 228 */ 229 if (flags & FAULT_FLAG_WRITE) 230 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE; 231 if (reason & VM_UFFD_WP) 232 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP; 233 if (reason & VM_UFFD_MINOR) 234 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR; 235 if (features & UFFD_FEATURE_THREAD_ID) 236 msg.arg.pagefault.feat.ptid = task_pid_vnr(current); 237 return msg; 238} 239 240#ifdef CONFIG_HUGETLB_PAGE 241/* 242 * Same functionality as userfaultfd_must_wait below with modifications for 243 * hugepmd ranges. 244 */ 245static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, 246 struct vm_fault *vmf, 247 unsigned long reason) 248{ 249 struct vm_area_struct *vma = vmf->vma; 250 pte_t *ptep, pte; 251 bool ret = true; 252 253 assert_fault_locked(vmf); 254 255 ptep = hugetlb_walk(vma, vmf->address, vma_mmu_pagesize(vma)); 256 if (!ptep) 257 goto out; 258 259 ret = false; 260 pte = huge_ptep_get(ptep); 261 262 /* 263 * Lockless access: we're in a wait_event so it's ok if it 264 * changes under us. PTE markers should be handled the same as none 265 * ptes here. 266 */ 267 if (huge_pte_none_mostly(pte)) 268 ret = true; 269 if (!huge_pte_write(pte) && (reason & VM_UFFD_WP)) 270 ret = true; 271out: 272 return ret; 273} 274#else 275static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, 276 struct vm_fault *vmf, 277 unsigned long reason) 278{ 279 return false; /* should never get here */ 280} 281#endif /* CONFIG_HUGETLB_PAGE */ 282 283/* 284 * Verify the pagetables are still not ok after having reigstered into 285 * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any 286 * userfault that has already been resolved, if userfaultfd_read_iter and 287 * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different 288 * threads. 289 */ 290static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, 291 struct vm_fault *vmf, 292 unsigned long reason) 293{ 294 struct mm_struct *mm = ctx->mm; 295 unsigned long address = vmf->address; 296 pgd_t *pgd; 297 p4d_t *p4d; 298 pud_t *pud; 299 pmd_t *pmd, _pmd; 300 pte_t *pte; 301 pte_t ptent; 302 bool ret = true; 303 304 assert_fault_locked(vmf); 305 306 pgd = pgd_offset(mm, address); 307 if (!pgd_present(*pgd)) 308 goto out; 309 p4d = p4d_offset(pgd, address); 310 if (!p4d_present(*p4d)) 311 goto out; 312 pud = pud_offset(p4d, address); 313 if (!pud_present(*pud)) 314 goto out; 315 pmd = pmd_offset(pud, address); 316again: 317 _pmd = pmdp_get_lockless(pmd); 318 if (pmd_none(_pmd)) 319 goto out; 320 321 ret = false; 322 if (!pmd_present(_pmd) || pmd_devmap(_pmd)) 323 goto out; 324 325 if (pmd_trans_huge(_pmd)) { 326 if (!pmd_write(_pmd) && (reason & VM_UFFD_WP)) 327 ret = true; 328 goto out; 329 } 330 331 pte = pte_offset_map(pmd, address); 332 if (!pte) { 333 ret = true; 334 goto again; 335 } 336 /* 337 * Lockless access: we're in a wait_event so it's ok if it 338 * changes under us. PTE markers should be handled the same as none 339 * ptes here. 340 */ 341 ptent = ptep_get(pte); 342 if (pte_none_mostly(ptent)) 343 ret = true; 344 if (!pte_write(ptent) && (reason & VM_UFFD_WP)) 345 ret = true; 346 pte_unmap(pte); 347 348out: 349 return ret; 350} 351 352static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags) 353{ 354 if (flags & FAULT_FLAG_INTERRUPTIBLE) 355 return TASK_INTERRUPTIBLE; 356 357 if (flags & FAULT_FLAG_KILLABLE) 358 return TASK_KILLABLE; 359 360 return TASK_UNINTERRUPTIBLE; 361} 362 363/* 364 * The locking rules involved in returning VM_FAULT_RETRY depending on 365 * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and 366 * FAULT_FLAG_KILLABLE are not straightforward. The "Caution" 367 * recommendation in __lock_page_or_retry is not an understatement. 368 * 369 * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released 370 * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is 371 * not set. 372 * 373 * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not 374 * set, VM_FAULT_RETRY can still be returned if and only if there are 375 * fatal_signal_pending()s, and the mmap_lock must be released before 376 * returning it. 377 */ 378vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) 379{ 380 struct vm_area_struct *vma = vmf->vma; 381 struct mm_struct *mm = vma->vm_mm; 382 struct userfaultfd_ctx *ctx; 383 struct userfaultfd_wait_queue uwq; 384 vm_fault_t ret = VM_FAULT_SIGBUS; 385 bool must_wait; 386 unsigned int blocking_state; 387 388 /* 389 * We don't do userfault handling for the final child pid update. 390 * 391 * We also don't do userfault handling during 392 * coredumping. hugetlbfs has the special 393 * hugetlb_follow_page_mask() to skip missing pages in the 394 * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with 395 * the no_page_table() helper in follow_page_mask(), but the 396 * shmem_vm_ops->fault method is invoked even during 397 * coredumping and it ends up here. 398 */ 399 if (current->flags & (PF_EXITING|PF_DUMPCORE)) 400 goto out; 401 402 assert_fault_locked(vmf); 403 404 ctx = vma->vm_userfaultfd_ctx.ctx; 405 if (!ctx) 406 goto out; 407 408 BUG_ON(ctx->mm != mm); 409 410 /* Any unrecognized flag is a bug. */ 411 VM_BUG_ON(reason & ~__VM_UFFD_FLAGS); 412 /* 0 or > 1 flags set is a bug; we expect exactly 1. */ 413 VM_BUG_ON(!reason || (reason & (reason - 1))); 414 415 if (ctx->features & UFFD_FEATURE_SIGBUS) 416 goto out; 417 if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY)) 418 goto out; 419 420 /* 421 * If it's already released don't get it. This avoids to loop 422 * in __get_user_pages if userfaultfd_release waits on the 423 * caller of handle_userfault to release the mmap_lock. 424 */ 425 if (unlikely(READ_ONCE(ctx->released))) { 426 /* 427 * Don't return VM_FAULT_SIGBUS in this case, so a non 428 * cooperative manager can close the uffd after the 429 * last UFFDIO_COPY, without risking to trigger an 430 * involuntary SIGBUS if the process was starting the 431 * userfaultfd while the userfaultfd was still armed 432 * (but after the last UFFDIO_COPY). If the uffd 433 * wasn't already closed when the userfault reached 434 * this point, that would normally be solved by 435 * userfaultfd_must_wait returning 'false'. 436 * 437 * If we were to return VM_FAULT_SIGBUS here, the non 438 * cooperative manager would be instead forced to 439 * always call UFFDIO_UNREGISTER before it can safely 440 * close the uffd. 441 */ 442 ret = VM_FAULT_NOPAGE; 443 goto out; 444 } 445 446 /* 447 * Check that we can return VM_FAULT_RETRY. 448 * 449 * NOTE: it should become possible to return VM_FAULT_RETRY 450 * even if FAULT_FLAG_TRIED is set without leading to gup() 451 * -EBUSY failures, if the userfaultfd is to be extended for 452 * VM_UFFD_WP tracking and we intend to arm the userfault 453 * without first stopping userland access to the memory. For 454 * VM_UFFD_MISSING userfaults this is enough for now. 455 */ 456 if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) { 457 /* 458 * Validate the invariant that nowait must allow retry 459 * to be sure not to return SIGBUS erroneously on 460 * nowait invocations. 461 */ 462 BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT); 463#ifdef CONFIG_DEBUG_VM 464 if (printk_ratelimit()) { 465 printk(KERN_WARNING 466 "FAULT_FLAG_ALLOW_RETRY missing %x\n", 467 vmf->flags); 468 dump_stack(); 469 } 470#endif 471 goto out; 472 } 473 474 /* 475 * Handle nowait, not much to do other than tell it to retry 476 * and wait. 477 */ 478 ret = VM_FAULT_RETRY; 479 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 480 goto out; 481 482 /* take the reference before dropping the mmap_lock */ 483 userfaultfd_ctx_get(ctx); 484 485 init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function); 486 uwq.wq.private = current; 487 uwq.msg = userfault_msg(vmf->address, vmf->real_address, vmf->flags, 488 reason, ctx->features); 489 uwq.ctx = ctx; 490 uwq.waken = false; 491 492 blocking_state = userfaultfd_get_blocking_state(vmf->flags); 493 494 /* 495 * Take the vma lock now, in order to safely call 496 * userfaultfd_huge_must_wait() later. Since acquiring the 497 * (sleepable) vma lock can modify the current task state, that 498 * must be before explicitly calling set_current_state(). 499 */ 500 if (is_vm_hugetlb_page(vma)) 501 hugetlb_vma_lock_read(vma); 502 503 spin_lock_irq(&ctx->fault_pending_wqh.lock); 504 /* 505 * After the __add_wait_queue the uwq is visible to userland 506 * through poll/read(). 507 */ 508 __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq); 509 /* 510 * The smp_mb() after __set_current_state prevents the reads 511 * following the spin_unlock to happen before the list_add in 512 * __add_wait_queue. 513 */ 514 set_current_state(blocking_state); 515 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 516 517 if (!is_vm_hugetlb_page(vma)) 518 must_wait = userfaultfd_must_wait(ctx, vmf, reason); 519 else 520 must_wait = userfaultfd_huge_must_wait(ctx, vmf, reason); 521 if (is_vm_hugetlb_page(vma)) 522 hugetlb_vma_unlock_read(vma); 523 release_fault_lock(vmf); 524 525 if (likely(must_wait && !READ_ONCE(ctx->released))) { 526 wake_up_poll(&ctx->fd_wqh, EPOLLIN); 527 schedule(); 528 } 529 530 __set_current_state(TASK_RUNNING); 531 532 /* 533 * Here we race with the list_del; list_add in 534 * userfaultfd_ctx_read(), however because we don't ever run 535 * list_del_init() to refile across the two lists, the prev 536 * and next pointers will never point to self. list_add also 537 * would never let any of the two pointers to point to 538 * self. So list_empty_careful won't risk to see both pointers 539 * pointing to self at any time during the list refile. The 540 * only case where list_del_init() is called is the full 541 * removal in the wake function and there we don't re-list_add 542 * and it's fine not to block on the spinlock. The uwq on this 543 * kernel stack can be released after the list_del_init. 544 */ 545 if (!list_empty_careful(&uwq.wq.entry)) { 546 spin_lock_irq(&ctx->fault_pending_wqh.lock); 547 /* 548 * No need of list_del_init(), the uwq on the stack 549 * will be freed shortly anyway. 550 */ 551 list_del(&uwq.wq.entry); 552 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 553 } 554 555 /* 556 * ctx may go away after this if the userfault pseudo fd is 557 * already released. 558 */ 559 userfaultfd_ctx_put(ctx); 560 561out: 562 return ret; 563} 564 565static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, 566 struct userfaultfd_wait_queue *ewq) 567{ 568 struct userfaultfd_ctx *release_new_ctx; 569 570 if (WARN_ON_ONCE(current->flags & PF_EXITING)) 571 goto out; 572 573 ewq->ctx = ctx; 574 init_waitqueue_entry(&ewq->wq, current); 575 release_new_ctx = NULL; 576 577 spin_lock_irq(&ctx->event_wqh.lock); 578 /* 579 * After the __add_wait_queue the uwq is visible to userland 580 * through poll/read(). 581 */ 582 __add_wait_queue(&ctx->event_wqh, &ewq->wq); 583 for (;;) { 584 set_current_state(TASK_KILLABLE); 585 if (ewq->msg.event == 0) 586 break; 587 if (READ_ONCE(ctx->released) || 588 fatal_signal_pending(current)) { 589 /* 590 * &ewq->wq may be queued in fork_event, but 591 * __remove_wait_queue ignores the head 592 * parameter. It would be a problem if it 593 * didn't. 594 */ 595 __remove_wait_queue(&ctx->event_wqh, &ewq->wq); 596 if (ewq->msg.event == UFFD_EVENT_FORK) { 597 struct userfaultfd_ctx *new; 598 599 new = (struct userfaultfd_ctx *) 600 (unsigned long) 601 ewq->msg.arg.reserved.reserved1; 602 release_new_ctx = new; 603 } 604 break; 605 } 606 607 spin_unlock_irq(&ctx->event_wqh.lock); 608 609 wake_up_poll(&ctx->fd_wqh, EPOLLIN); 610 schedule(); 611 612 spin_lock_irq(&ctx->event_wqh.lock); 613 } 614 __set_current_state(TASK_RUNNING); 615 spin_unlock_irq(&ctx->event_wqh.lock); 616 617 if (release_new_ctx) { 618 struct vm_area_struct *vma; 619 struct mm_struct *mm = release_new_ctx->mm; 620 VMA_ITERATOR(vmi, mm, 0); 621 622 /* the various vma->vm_userfaultfd_ctx still points to it */ 623 mmap_write_lock(mm); 624 for_each_vma(vmi, vma) { 625 if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { 626 vma_start_write(vma); 627 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; 628 userfaultfd_set_vm_flags(vma, 629 vma->vm_flags & ~__VM_UFFD_FLAGS); 630 } 631 } 632 mmap_write_unlock(mm); 633 634 userfaultfd_ctx_put(release_new_ctx); 635 } 636 637 /* 638 * ctx may go away after this if the userfault pseudo fd is 639 * already released. 640 */ 641out: 642 atomic_dec(&ctx->mmap_changing); 643 VM_BUG_ON(atomic_read(&ctx->mmap_changing) < 0); 644 userfaultfd_ctx_put(ctx); 645} 646 647static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx, 648 struct userfaultfd_wait_queue *ewq) 649{ 650 ewq->msg.event = 0; 651 wake_up_locked(&ctx->event_wqh); 652 __remove_wait_queue(&ctx->event_wqh, &ewq->wq); 653} 654 655int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs) 656{ 657 struct userfaultfd_ctx *ctx = NULL, *octx; 658 struct userfaultfd_fork_ctx *fctx; 659 660 octx = vma->vm_userfaultfd_ctx.ctx; 661 if (!octx) 662 return 0; 663 664 if (!(octx->features & UFFD_FEATURE_EVENT_FORK)) { 665 vma_start_write(vma); 666 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; 667 userfaultfd_set_vm_flags(vma, vma->vm_flags & ~__VM_UFFD_FLAGS); 668 return 0; 669 } 670 671 list_for_each_entry(fctx, fcs, list) 672 if (fctx->orig == octx) { 673 ctx = fctx->new; 674 break; 675 } 676 677 if (!ctx) { 678 fctx = kmalloc(sizeof(*fctx), GFP_KERNEL); 679 if (!fctx) 680 return -ENOMEM; 681 682 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL); 683 if (!ctx) { 684 kfree(fctx); 685 return -ENOMEM; 686 } 687 688 refcount_set(&ctx->refcount, 1); 689 ctx->flags = octx->flags; 690 ctx->features = octx->features; 691 ctx->released = false; 692 init_rwsem(&ctx->map_changing_lock); 693 atomic_set(&ctx->mmap_changing, 0); 694 ctx->mm = vma->vm_mm; 695 mmgrab(ctx->mm); 696 697 userfaultfd_ctx_get(octx); 698 down_write(&octx->map_changing_lock); 699 atomic_inc(&octx->mmap_changing); 700 up_write(&octx->map_changing_lock); 701 fctx->orig = octx; 702 fctx->new = ctx; 703 list_add_tail(&fctx->list, fcs); 704 } 705 706 vma->vm_userfaultfd_ctx.ctx = ctx; 707 return 0; 708} 709 710static void dup_fctx(struct userfaultfd_fork_ctx *fctx) 711{ 712 struct userfaultfd_ctx *ctx = fctx->orig; 713 struct userfaultfd_wait_queue ewq; 714 715 msg_init(&ewq.msg); 716 717 ewq.msg.event = UFFD_EVENT_FORK; 718 ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new; 719 720 userfaultfd_event_wait_completion(ctx, &ewq); 721} 722 723void dup_userfaultfd_complete(struct list_head *fcs) 724{ 725 struct userfaultfd_fork_ctx *fctx, *n; 726 727 list_for_each_entry_safe(fctx, n, fcs, list) { 728 dup_fctx(fctx); 729 list_del(&fctx->list); 730 kfree(fctx); 731 } 732} 733 734void mremap_userfaultfd_prep(struct vm_area_struct *vma, 735 struct vm_userfaultfd_ctx *vm_ctx) 736{ 737 struct userfaultfd_ctx *ctx; 738 739 ctx = vma->vm_userfaultfd_ctx.ctx; 740 741 if (!ctx) 742 return; 743 744 if (ctx->features & UFFD_FEATURE_EVENT_REMAP) { 745 vm_ctx->ctx = ctx; 746 userfaultfd_ctx_get(ctx); 747 down_write(&ctx->map_changing_lock); 748 atomic_inc(&ctx->mmap_changing); 749 up_write(&ctx->map_changing_lock); 750 } else { 751 /* Drop uffd context if remap feature not enabled */ 752 vma_start_write(vma); 753 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; 754 userfaultfd_set_vm_flags(vma, vma->vm_flags & ~__VM_UFFD_FLAGS); 755 } 756} 757 758void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx, 759 unsigned long from, unsigned long to, 760 unsigned long len) 761{ 762 struct userfaultfd_ctx *ctx = vm_ctx->ctx; 763 struct userfaultfd_wait_queue ewq; 764 765 if (!ctx) 766 return; 767 768 if (to & ~PAGE_MASK) { 769 userfaultfd_ctx_put(ctx); 770 return; 771 } 772 773 msg_init(&ewq.msg); 774 775 ewq.msg.event = UFFD_EVENT_REMAP; 776 ewq.msg.arg.remap.from = from; 777 ewq.msg.arg.remap.to = to; 778 ewq.msg.arg.remap.len = len; 779 780 userfaultfd_event_wait_completion(ctx, &ewq); 781} 782 783bool userfaultfd_remove(struct vm_area_struct *vma, 784 unsigned long start, unsigned long end) 785{ 786 struct mm_struct *mm = vma->vm_mm; 787 struct userfaultfd_ctx *ctx; 788 struct userfaultfd_wait_queue ewq; 789 790 ctx = vma->vm_userfaultfd_ctx.ctx; 791 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE)) 792 return true; 793 794 userfaultfd_ctx_get(ctx); 795 down_write(&ctx->map_changing_lock); 796 atomic_inc(&ctx->mmap_changing); 797 up_write(&ctx->map_changing_lock); 798 mmap_read_unlock(mm); 799 800 msg_init(&ewq.msg); 801 802 ewq.msg.event = UFFD_EVENT_REMOVE; 803 ewq.msg.arg.remove.start = start; 804 ewq.msg.arg.remove.end = end; 805 806 userfaultfd_event_wait_completion(ctx, &ewq); 807 808 return false; 809} 810 811static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps, 812 unsigned long start, unsigned long end) 813{ 814 struct userfaultfd_unmap_ctx *unmap_ctx; 815 816 list_for_each_entry(unmap_ctx, unmaps, list) 817 if (unmap_ctx->ctx == ctx && unmap_ctx->start == start && 818 unmap_ctx->end == end) 819 return true; 820 821 return false; 822} 823 824int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, 825 unsigned long end, struct list_head *unmaps) 826{ 827 struct userfaultfd_unmap_ctx *unmap_ctx; 828 struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; 829 830 if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) || 831 has_unmap_ctx(ctx, unmaps, start, end)) 832 return 0; 833 834 unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL); 835 if (!unmap_ctx) 836 return -ENOMEM; 837 838 userfaultfd_ctx_get(ctx); 839 down_write(&ctx->map_changing_lock); 840 atomic_inc(&ctx->mmap_changing); 841 up_write(&ctx->map_changing_lock); 842 unmap_ctx->ctx = ctx; 843 unmap_ctx->start = start; 844 unmap_ctx->end = end; 845 list_add_tail(&unmap_ctx->list, unmaps); 846 847 return 0; 848} 849 850void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf) 851{ 852 struct userfaultfd_unmap_ctx *ctx, *n; 853 struct userfaultfd_wait_queue ewq; 854 855 list_for_each_entry_safe(ctx, n, uf, list) { 856 msg_init(&ewq.msg); 857 858 ewq.msg.event = UFFD_EVENT_UNMAP; 859 ewq.msg.arg.remove.start = ctx->start; 860 ewq.msg.arg.remove.end = ctx->end; 861 862 userfaultfd_event_wait_completion(ctx->ctx, &ewq); 863 864 list_del(&ctx->list); 865 kfree(ctx); 866 } 867} 868 869static int userfaultfd_release(struct inode *inode, struct file *file) 870{ 871 struct userfaultfd_ctx *ctx = file->private_data; 872 struct mm_struct *mm = ctx->mm; 873 struct vm_area_struct *vma, *prev; 874 /* len == 0 means wake all */ 875 struct userfaultfd_wake_range range = { .len = 0, }; 876 unsigned long new_flags; 877 VMA_ITERATOR(vmi, mm, 0); 878 879 WRITE_ONCE(ctx->released, true); 880 881 if (!mmget_not_zero(mm)) 882 goto wakeup; 883 884 /* 885 * Flush page faults out of all CPUs. NOTE: all page faults 886 * must be retried without returning VM_FAULT_SIGBUS if 887 * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx 888 * changes while handle_userfault released the mmap_lock. So 889 * it's critical that released is set to true (above), before 890 * taking the mmap_lock for writing. 891 */ 892 mmap_write_lock(mm); 893 prev = NULL; 894 for_each_vma(vmi, vma) { 895 cond_resched(); 896 BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ 897 !!(vma->vm_flags & __VM_UFFD_FLAGS)); 898 if (vma->vm_userfaultfd_ctx.ctx != ctx) { 899 prev = vma; 900 continue; 901 } 902 /* Reset ptes for the whole vma range if wr-protected */ 903 if (userfaultfd_wp(vma)) 904 uffd_wp_range(vma, vma->vm_start, 905 vma->vm_end - vma->vm_start, false); 906 new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS; 907 vma = vma_modify_flags_uffd(&vmi, prev, vma, vma->vm_start, 908 vma->vm_end, new_flags, 909 NULL_VM_UFFD_CTX); 910 911 vma_start_write(vma); 912 userfaultfd_set_vm_flags(vma, new_flags); 913 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; 914 915 prev = vma; 916 } 917 mmap_write_unlock(mm); 918 mmput(mm); 919wakeup: 920 /* 921 * After no new page faults can wait on this fault_*wqh, flush 922 * the last page faults that may have been already waiting on 923 * the fault_*wqh. 924 */ 925 spin_lock_irq(&ctx->fault_pending_wqh.lock); 926 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range); 927 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range); 928 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 929 930 /* Flush pending events that may still wait on event_wqh */ 931 wake_up_all(&ctx->event_wqh); 932 933 wake_up_poll(&ctx->fd_wqh, EPOLLHUP); 934 userfaultfd_ctx_put(ctx); 935 return 0; 936} 937 938/* fault_pending_wqh.lock must be hold by the caller */ 939static inline struct userfaultfd_wait_queue *find_userfault_in( 940 wait_queue_head_t *wqh) 941{ 942 wait_queue_entry_t *wq; 943 struct userfaultfd_wait_queue *uwq; 944 945 lockdep_assert_held(&wqh->lock); 946 947 uwq = NULL; 948 if (!waitqueue_active(wqh)) 949 goto out; 950 /* walk in reverse to provide FIFO behavior to read userfaults */ 951 wq = list_last_entry(&wqh->head, typeof(*wq), entry); 952 uwq = container_of(wq, struct userfaultfd_wait_queue, wq); 953out: 954 return uwq; 955} 956 957static inline struct userfaultfd_wait_queue *find_userfault( 958 struct userfaultfd_ctx *ctx) 959{ 960 return find_userfault_in(&ctx->fault_pending_wqh); 961} 962 963static inline struct userfaultfd_wait_queue *find_userfault_evt( 964 struct userfaultfd_ctx *ctx) 965{ 966 return find_userfault_in(&ctx->event_wqh); 967} 968 969static __poll_t userfaultfd_poll(struct file *file, poll_table *wait) 970{ 971 struct userfaultfd_ctx *ctx = file->private_data; 972 __poll_t ret; 973 974 poll_wait(file, &ctx->fd_wqh, wait); 975 976 if (!userfaultfd_is_initialized(ctx)) 977 return EPOLLERR; 978 979 /* 980 * poll() never guarantees that read won't block. 981 * userfaults can be waken before they're read(). 982 */ 983 if (unlikely(!(file->f_flags & O_NONBLOCK))) 984 return EPOLLERR; 985 /* 986 * lockless access to see if there are pending faults 987 * __pollwait last action is the add_wait_queue but 988 * the spin_unlock would allow the waitqueue_active to 989 * pass above the actual list_add inside 990 * add_wait_queue critical section. So use a full 991 * memory barrier to serialize the list_add write of 992 * add_wait_queue() with the waitqueue_active read 993 * below. 994 */ 995 ret = 0; 996 smp_mb(); 997 if (waitqueue_active(&ctx->fault_pending_wqh)) 998 ret = EPOLLIN; 999 else if (waitqueue_active(&ctx->event_wqh)) 1000 ret = EPOLLIN; 1001 1002 return ret; 1003} 1004 1005static const struct file_operations userfaultfd_fops; 1006 1007static int resolve_userfault_fork(struct userfaultfd_ctx *new, 1008 struct inode *inode, 1009 struct uffd_msg *msg) 1010{ 1011 int fd; 1012 1013 fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, new, 1014 O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode); 1015 if (fd < 0) 1016 return fd; 1017 1018 msg->arg.reserved.reserved1 = 0; 1019 msg->arg.fork.ufd = fd; 1020 return 0; 1021} 1022 1023static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, 1024 struct uffd_msg *msg, struct inode *inode) 1025{ 1026 ssize_t ret; 1027 DECLARE_WAITQUEUE(wait, current); 1028 struct userfaultfd_wait_queue *uwq; 1029 /* 1030 * Handling fork event requires sleeping operations, so 1031 * we drop the event_wqh lock, then do these ops, then 1032 * lock it back and wake up the waiter. While the lock is 1033 * dropped the ewq may go away so we keep track of it 1034 * carefully. 1035 */ 1036 LIST_HEAD(fork_event); 1037 struct userfaultfd_ctx *fork_nctx = NULL; 1038 1039 /* always take the fd_wqh lock before the fault_pending_wqh lock */ 1040 spin_lock_irq(&ctx->fd_wqh.lock); 1041 __add_wait_queue(&ctx->fd_wqh, &wait); 1042 for (;;) { 1043 set_current_state(TASK_INTERRUPTIBLE); 1044 spin_lock(&ctx->fault_pending_wqh.lock); 1045 uwq = find_userfault(ctx); 1046 if (uwq) { 1047 /* 1048 * Use a seqcount to repeat the lockless check 1049 * in wake_userfault() to avoid missing 1050 * wakeups because during the refile both 1051 * waitqueue could become empty if this is the 1052 * only userfault. 1053 */ 1054 write_seqcount_begin(&ctx->refile_seq); 1055 1056 /* 1057 * The fault_pending_wqh.lock prevents the uwq 1058 * to disappear from under us. 1059 * 1060 * Refile this userfault from 1061 * fault_pending_wqh to fault_wqh, it's not 1062 * pending anymore after we read it. 1063 * 1064 * Use list_del() by hand (as 1065 * userfaultfd_wake_function also uses 1066 * list_del_init() by hand) to be sure nobody 1067 * changes __remove_wait_queue() to use 1068 * list_del_init() in turn breaking the 1069 * !list_empty_careful() check in 1070 * handle_userfault(). The uwq->wq.head list 1071 * must never be empty at any time during the 1072 * refile, or the waitqueue could disappear 1073 * from under us. The "wait_queue_head_t" 1074 * parameter of __remove_wait_queue() is unused 1075 * anyway. 1076 */ 1077 list_del(&uwq->wq.entry); 1078 add_wait_queue(&ctx->fault_wqh, &uwq->wq); 1079 1080 write_seqcount_end(&ctx->refile_seq); 1081 1082 /* careful to always initialize msg if ret == 0 */ 1083 *msg = uwq->msg; 1084 spin_unlock(&ctx->fault_pending_wqh.lock); 1085 ret = 0; 1086 break; 1087 } 1088 spin_unlock(&ctx->fault_pending_wqh.lock); 1089 1090 spin_lock(&ctx->event_wqh.lock); 1091 uwq = find_userfault_evt(ctx); 1092 if (uwq) { 1093 *msg = uwq->msg; 1094 1095 if (uwq->msg.event == UFFD_EVENT_FORK) { 1096 fork_nctx = (struct userfaultfd_ctx *) 1097 (unsigned long) 1098 uwq->msg.arg.reserved.reserved1; 1099 list_move(&uwq->wq.entry, &fork_event); 1100 /* 1101 * fork_nctx can be freed as soon as 1102 * we drop the lock, unless we take a 1103 * reference on it. 1104 */ 1105 userfaultfd_ctx_get(fork_nctx); 1106 spin_unlock(&ctx->event_wqh.lock); 1107 ret = 0; 1108 break; 1109 } 1110 1111 userfaultfd_event_complete(ctx, uwq); 1112 spin_unlock(&ctx->event_wqh.lock); 1113 ret = 0; 1114 break; 1115 } 1116 spin_unlock(&ctx->event_wqh.lock); 1117 1118 if (signal_pending(current)) { 1119 ret = -ERESTARTSYS; 1120 break; 1121 } 1122 if (no_wait) { 1123 ret = -EAGAIN; 1124 break; 1125 } 1126 spin_unlock_irq(&ctx->fd_wqh.lock); 1127 schedule(); 1128 spin_lock_irq(&ctx->fd_wqh.lock); 1129 } 1130 __remove_wait_queue(&ctx->fd_wqh, &wait); 1131 __set_current_state(TASK_RUNNING); 1132 spin_unlock_irq(&ctx->fd_wqh.lock); 1133 1134 if (!ret && msg->event == UFFD_EVENT_FORK) { 1135 ret = resolve_userfault_fork(fork_nctx, inode, msg); 1136 spin_lock_irq(&ctx->event_wqh.lock); 1137 if (!list_empty(&fork_event)) { 1138 /* 1139 * The fork thread didn't abort, so we can 1140 * drop the temporary refcount. 1141 */ 1142 userfaultfd_ctx_put(fork_nctx); 1143 1144 uwq = list_first_entry(&fork_event, 1145 typeof(*uwq), 1146 wq.entry); 1147 /* 1148 * If fork_event list wasn't empty and in turn 1149 * the event wasn't already released by fork 1150 * (the event is allocated on fork kernel 1151 * stack), put the event back to its place in 1152 * the event_wq. fork_event head will be freed 1153 * as soon as we return so the event cannot 1154 * stay queued there no matter the current 1155 * "ret" value. 1156 */ 1157 list_del(&uwq->wq.entry); 1158 __add_wait_queue(&ctx->event_wqh, &uwq->wq); 1159 1160 /* 1161 * Leave the event in the waitqueue and report 1162 * error to userland if we failed to resolve 1163 * the userfault fork. 1164 */ 1165 if (likely(!ret)) 1166 userfaultfd_event_complete(ctx, uwq); 1167 } else { 1168 /* 1169 * Here the fork thread aborted and the 1170 * refcount from the fork thread on fork_nctx 1171 * has already been released. We still hold 1172 * the reference we took before releasing the 1173 * lock above. If resolve_userfault_fork 1174 * failed we've to drop it because the 1175 * fork_nctx has to be freed in such case. If 1176 * it succeeded we'll hold it because the new 1177 * uffd references it. 1178 */ 1179 if (ret) 1180 userfaultfd_ctx_put(fork_nctx); 1181 } 1182 spin_unlock_irq(&ctx->event_wqh.lock); 1183 } 1184 1185 return ret; 1186} 1187 1188static ssize_t userfaultfd_read_iter(struct kiocb *iocb, struct iov_iter *to) 1189{ 1190 struct file *file = iocb->ki_filp; 1191 struct userfaultfd_ctx *ctx = file->private_data; 1192 ssize_t _ret, ret = 0; 1193 struct uffd_msg msg; 1194 struct inode *inode = file_inode(file); 1195 bool no_wait; 1196 1197 if (!userfaultfd_is_initialized(ctx)) 1198 return -EINVAL; 1199 1200 no_wait = file->f_flags & O_NONBLOCK || iocb->ki_flags & IOCB_NOWAIT; 1201 for (;;) { 1202 if (iov_iter_count(to) < sizeof(msg)) 1203 return ret ? ret : -EINVAL; 1204 _ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode); 1205 if (_ret < 0) 1206 return ret ? ret : _ret; 1207 _ret = !copy_to_iter_full(&msg, sizeof(msg), to); 1208 if (_ret) 1209 return ret ? ret : -EFAULT; 1210 ret += sizeof(msg); 1211 /* 1212 * Allow to read more than one fault at time but only 1213 * block if waiting for the very first one. 1214 */ 1215 no_wait = true; 1216 } 1217} 1218 1219static void __wake_userfault(struct userfaultfd_ctx *ctx, 1220 struct userfaultfd_wake_range *range) 1221{ 1222 spin_lock_irq(&ctx->fault_pending_wqh.lock); 1223 /* wake all in the range and autoremove */ 1224 if (waitqueue_active(&ctx->fault_pending_wqh)) 1225 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, 1226 range); 1227 if (waitqueue_active(&ctx->fault_wqh)) 1228 __wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range); 1229 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 1230} 1231 1232static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, 1233 struct userfaultfd_wake_range *range) 1234{ 1235 unsigned seq; 1236 bool need_wakeup; 1237 1238 /* 1239 * To be sure waitqueue_active() is not reordered by the CPU 1240 * before the pagetable update, use an explicit SMP memory 1241 * barrier here. PT lock release or mmap_read_unlock(mm) still 1242 * have release semantics that can allow the 1243 * waitqueue_active() to be reordered before the pte update. 1244 */ 1245 smp_mb(); 1246 1247 /* 1248 * Use waitqueue_active because it's very frequent to 1249 * change the address space atomically even if there are no 1250 * userfaults yet. So we take the spinlock only when we're 1251 * sure we've userfaults to wake. 1252 */ 1253 do { 1254 seq = read_seqcount_begin(&ctx->refile_seq); 1255 need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) || 1256 waitqueue_active(&ctx->fault_wqh); 1257 cond_resched(); 1258 } while (read_seqcount_retry(&ctx->refile_seq, seq)); 1259 if (need_wakeup) 1260 __wake_userfault(ctx, range); 1261} 1262 1263static __always_inline int validate_unaligned_range( 1264 struct mm_struct *mm, __u64 start, __u64 len) 1265{ 1266 __u64 task_size = mm->task_size; 1267 1268 if (len & ~PAGE_MASK) 1269 return -EINVAL; 1270 if (!len) 1271 return -EINVAL; 1272 if (start < mmap_min_addr) 1273 return -EINVAL; 1274 if (start >= task_size) 1275 return -EINVAL; 1276 if (len > task_size - start) 1277 return -EINVAL; 1278 if (start + len <= start) 1279 return -EINVAL; 1280 return 0; 1281} 1282 1283static __always_inline int validate_range(struct mm_struct *mm, 1284 __u64 start, __u64 len) 1285{ 1286 if (start & ~PAGE_MASK) 1287 return -EINVAL; 1288 1289 return validate_unaligned_range(mm, start, len); 1290} 1291 1292static int userfaultfd_register(struct userfaultfd_ctx *ctx, 1293 unsigned long arg) 1294{ 1295 struct mm_struct *mm = ctx->mm; 1296 struct vm_area_struct *vma, *prev, *cur; 1297 int ret; 1298 struct uffdio_register uffdio_register; 1299 struct uffdio_register __user *user_uffdio_register; 1300 unsigned long vm_flags, new_flags; 1301 bool found; 1302 bool basic_ioctls; 1303 unsigned long start, end, vma_end; 1304 struct vma_iterator vmi; 1305 bool wp_async = userfaultfd_wp_async_ctx(ctx); 1306 1307 user_uffdio_register = (struct uffdio_register __user *) arg; 1308 1309 ret = -EFAULT; 1310 if (copy_from_user(&uffdio_register, user_uffdio_register, 1311 sizeof(uffdio_register)-sizeof(__u64))) 1312 goto out; 1313 1314 ret = -EINVAL; 1315 if (!uffdio_register.mode) 1316 goto out; 1317 if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES) 1318 goto out; 1319 vm_flags = 0; 1320 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) 1321 vm_flags |= VM_UFFD_MISSING; 1322 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { 1323#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP 1324 goto out; 1325#endif 1326 vm_flags |= VM_UFFD_WP; 1327 } 1328 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) { 1329#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 1330 goto out; 1331#endif 1332 vm_flags |= VM_UFFD_MINOR; 1333 } 1334 1335 ret = validate_range(mm, uffdio_register.range.start, 1336 uffdio_register.range.len); 1337 if (ret) 1338 goto out; 1339 1340 start = uffdio_register.range.start; 1341 end = start + uffdio_register.range.len; 1342 1343 ret = -ENOMEM; 1344 if (!mmget_not_zero(mm)) 1345 goto out; 1346 1347 ret = -EINVAL; 1348 mmap_write_lock(mm); 1349 vma_iter_init(&vmi, mm, start); 1350 vma = vma_find(&vmi, end); 1351 if (!vma) 1352 goto out_unlock; 1353 1354 /* 1355 * If the first vma contains huge pages, make sure start address 1356 * is aligned to huge page size. 1357 */ 1358 if (is_vm_hugetlb_page(vma)) { 1359 unsigned long vma_hpagesize = vma_kernel_pagesize(vma); 1360 1361 if (start & (vma_hpagesize - 1)) 1362 goto out_unlock; 1363 } 1364 1365 /* 1366 * Search for not compatible vmas. 1367 */ 1368 found = false; 1369 basic_ioctls = false; 1370 cur = vma; 1371 do { 1372 cond_resched(); 1373 1374 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ 1375 !!(cur->vm_flags & __VM_UFFD_FLAGS)); 1376 1377 /* check not compatible vmas */ 1378 ret = -EINVAL; 1379 if (!vma_can_userfault(cur, vm_flags, wp_async)) 1380 goto out_unlock; 1381 1382 /* 1383 * UFFDIO_COPY will fill file holes even without 1384 * PROT_WRITE. This check enforces that if this is a 1385 * MAP_SHARED, the process has write permission to the backing 1386 * file. If VM_MAYWRITE is set it also enforces that on a 1387 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further 1388 * F_WRITE_SEAL can be taken until the vma is destroyed. 1389 */ 1390 ret = -EPERM; 1391 if (unlikely(!(cur->vm_flags & VM_MAYWRITE))) 1392 goto out_unlock; 1393 1394 /* 1395 * If this vma contains ending address, and huge pages 1396 * check alignment. 1397 */ 1398 if (is_vm_hugetlb_page(cur) && end <= cur->vm_end && 1399 end > cur->vm_start) { 1400 unsigned long vma_hpagesize = vma_kernel_pagesize(cur); 1401 1402 ret = -EINVAL; 1403 1404 if (end & (vma_hpagesize - 1)) 1405 goto out_unlock; 1406 } 1407 if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE)) 1408 goto out_unlock; 1409 1410 /* 1411 * Check that this vma isn't already owned by a 1412 * different userfaultfd. We can't allow more than one 1413 * userfaultfd to own a single vma simultaneously or we 1414 * wouldn't know which one to deliver the userfaults to. 1415 */ 1416 ret = -EBUSY; 1417 if (cur->vm_userfaultfd_ctx.ctx && 1418 cur->vm_userfaultfd_ctx.ctx != ctx) 1419 goto out_unlock; 1420 1421 /* 1422 * Note vmas containing huge pages 1423 */ 1424 if (is_vm_hugetlb_page(cur)) 1425 basic_ioctls = true; 1426 1427 found = true; 1428 } for_each_vma_range(vmi, cur, end); 1429 BUG_ON(!found); 1430 1431 vma_iter_set(&vmi, start); 1432 prev = vma_prev(&vmi); 1433 if (vma->vm_start < start) 1434 prev = vma; 1435 1436 ret = 0; 1437 for_each_vma_range(vmi, vma, end) { 1438 cond_resched(); 1439 1440 BUG_ON(!vma_can_userfault(vma, vm_flags, wp_async)); 1441 BUG_ON(vma->vm_userfaultfd_ctx.ctx && 1442 vma->vm_userfaultfd_ctx.ctx != ctx); 1443 WARN_ON(!(vma->vm_flags & VM_MAYWRITE)); 1444 1445 /* 1446 * Nothing to do: this vma is already registered into this 1447 * userfaultfd and with the right tracking mode too. 1448 */ 1449 if (vma->vm_userfaultfd_ctx.ctx == ctx && 1450 (vma->vm_flags & vm_flags) == vm_flags) 1451 goto skip; 1452 1453 if (vma->vm_start > start) 1454 start = vma->vm_start; 1455 vma_end = min(end, vma->vm_end); 1456 1457 new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags; 1458 vma = vma_modify_flags_uffd(&vmi, prev, vma, start, vma_end, 1459 new_flags, 1460 (struct vm_userfaultfd_ctx){ctx}); 1461 if (IS_ERR(vma)) { 1462 ret = PTR_ERR(vma); 1463 break; 1464 } 1465 1466 /* 1467 * In the vma_merge() successful mprotect-like case 8: 1468 * the next vma was merged into the current one and 1469 * the current one has not been updated yet. 1470 */ 1471 vma_start_write(vma); 1472 userfaultfd_set_vm_flags(vma, new_flags); 1473 vma->vm_userfaultfd_ctx.ctx = ctx; 1474 1475 if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma)) 1476 hugetlb_unshare_all_pmds(vma); 1477 1478 skip: 1479 prev = vma; 1480 start = vma->vm_end; 1481 } 1482 1483out_unlock: 1484 mmap_write_unlock(mm); 1485 mmput(mm); 1486 if (!ret) { 1487 __u64 ioctls_out; 1488 1489 ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : 1490 UFFD_API_RANGE_IOCTLS; 1491 1492 /* 1493 * Declare the WP ioctl only if the WP mode is 1494 * specified and all checks passed with the range 1495 */ 1496 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP)) 1497 ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT); 1498 1499 /* CONTINUE ioctl is only supported for MINOR ranges. */ 1500 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR)) 1501 ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE); 1502 1503 /* 1504 * Now that we scanned all vmas we can already tell 1505 * userland which ioctls methods are guaranteed to 1506 * succeed on this range. 1507 */ 1508 if (put_user(ioctls_out, &user_uffdio_register->ioctls)) 1509 ret = -EFAULT; 1510 } 1511out: 1512 return ret; 1513} 1514 1515static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, 1516 unsigned long arg) 1517{ 1518 struct mm_struct *mm = ctx->mm; 1519 struct vm_area_struct *vma, *prev, *cur; 1520 int ret; 1521 struct uffdio_range uffdio_unregister; 1522 unsigned long new_flags; 1523 bool found; 1524 unsigned long start, end, vma_end; 1525 const void __user *buf = (void __user *)arg; 1526 struct vma_iterator vmi; 1527 bool wp_async = userfaultfd_wp_async_ctx(ctx); 1528 1529 ret = -EFAULT; 1530 if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) 1531 goto out; 1532 1533 ret = validate_range(mm, uffdio_unregister.start, 1534 uffdio_unregister.len); 1535 if (ret) 1536 goto out; 1537 1538 start = uffdio_unregister.start; 1539 end = start + uffdio_unregister.len; 1540 1541 ret = -ENOMEM; 1542 if (!mmget_not_zero(mm)) 1543 goto out; 1544 1545 mmap_write_lock(mm); 1546 ret = -EINVAL; 1547 vma_iter_init(&vmi, mm, start); 1548 vma = vma_find(&vmi, end); 1549 if (!vma) 1550 goto out_unlock; 1551 1552 /* 1553 * If the first vma contains huge pages, make sure start address 1554 * is aligned to huge page size. 1555 */ 1556 if (is_vm_hugetlb_page(vma)) { 1557 unsigned long vma_hpagesize = vma_kernel_pagesize(vma); 1558 1559 if (start & (vma_hpagesize - 1)) 1560 goto out_unlock; 1561 } 1562 1563 /* 1564 * Search for not compatible vmas. 1565 */ 1566 found = false; 1567 cur = vma; 1568 do { 1569 cond_resched(); 1570 1571 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ 1572 !!(cur->vm_flags & __VM_UFFD_FLAGS)); 1573 1574 /* 1575 * Check not compatible vmas, not strictly required 1576 * here as not compatible vmas cannot have an 1577 * userfaultfd_ctx registered on them, but this 1578 * provides for more strict behavior to notice 1579 * unregistration errors. 1580 */ 1581 if (!vma_can_userfault(cur, cur->vm_flags, wp_async)) 1582 goto out_unlock; 1583 1584 found = true; 1585 } for_each_vma_range(vmi, cur, end); 1586 BUG_ON(!found); 1587 1588 vma_iter_set(&vmi, start); 1589 prev = vma_prev(&vmi); 1590 if (vma->vm_start < start) 1591 prev = vma; 1592 1593 ret = 0; 1594 for_each_vma_range(vmi, vma, end) { 1595 cond_resched(); 1596 1597 BUG_ON(!vma_can_userfault(vma, vma->vm_flags, wp_async)); 1598 1599 /* 1600 * Nothing to do: this vma is already registered into this 1601 * userfaultfd and with the right tracking mode too. 1602 */ 1603 if (!vma->vm_userfaultfd_ctx.ctx) 1604 goto skip; 1605 1606 WARN_ON(!(vma->vm_flags & VM_MAYWRITE)); 1607 1608 if (vma->vm_start > start) 1609 start = vma->vm_start; 1610 vma_end = min(end, vma->vm_end); 1611 1612 if (userfaultfd_missing(vma)) { 1613 /* 1614 * Wake any concurrent pending userfault while 1615 * we unregister, so they will not hang 1616 * permanently and it avoids userland to call 1617 * UFFDIO_WAKE explicitly. 1618 */ 1619 struct userfaultfd_wake_range range; 1620 range.start = start; 1621 range.len = vma_end - start; 1622 wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range); 1623 } 1624 1625 /* Reset ptes for the whole vma range if wr-protected */ 1626 if (userfaultfd_wp(vma)) 1627 uffd_wp_range(vma, start, vma_end - start, false); 1628 1629 new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS; 1630 vma = vma_modify_flags_uffd(&vmi, prev, vma, start, vma_end, 1631 new_flags, NULL_VM_UFFD_CTX); 1632 if (IS_ERR(vma)) { 1633 ret = PTR_ERR(vma); 1634 break; 1635 } 1636 1637 /* 1638 * In the vma_merge() successful mprotect-like case 8: 1639 * the next vma was merged into the current one and 1640 * the current one has not been updated yet. 1641 */ 1642 vma_start_write(vma); 1643 userfaultfd_set_vm_flags(vma, new_flags); 1644 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; 1645 1646 skip: 1647 prev = vma; 1648 start = vma->vm_end; 1649 } 1650 1651out_unlock: 1652 mmap_write_unlock(mm); 1653 mmput(mm); 1654out: 1655 return ret; 1656} 1657 1658/* 1659 * userfaultfd_wake may be used in combination with the 1660 * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches. 1661 */ 1662static int userfaultfd_wake(struct userfaultfd_ctx *ctx, 1663 unsigned long arg) 1664{ 1665 int ret; 1666 struct uffdio_range uffdio_wake; 1667 struct userfaultfd_wake_range range; 1668 const void __user *buf = (void __user *)arg; 1669 1670 ret = -EFAULT; 1671 if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake))) 1672 goto out; 1673 1674 ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len); 1675 if (ret) 1676 goto out; 1677 1678 range.start = uffdio_wake.start; 1679 range.len = uffdio_wake.len; 1680 1681 /* 1682 * len == 0 means wake all and we don't want to wake all here, 1683 * so check it again to be sure. 1684 */ 1685 VM_BUG_ON(!range.len); 1686 1687 wake_userfault(ctx, &range); 1688 ret = 0; 1689 1690out: 1691 return ret; 1692} 1693 1694static int userfaultfd_copy(struct userfaultfd_ctx *ctx, 1695 unsigned long arg) 1696{ 1697 __s64 ret; 1698 struct uffdio_copy uffdio_copy; 1699 struct uffdio_copy __user *user_uffdio_copy; 1700 struct userfaultfd_wake_range range; 1701 uffd_flags_t flags = 0; 1702 1703 user_uffdio_copy = (struct uffdio_copy __user *) arg; 1704 1705 ret = -EAGAIN; 1706 if (atomic_read(&ctx->mmap_changing)) 1707 goto out; 1708 1709 ret = -EFAULT; 1710 if (copy_from_user(&uffdio_copy, user_uffdio_copy, 1711 /* don't copy "copy" last field */ 1712 sizeof(uffdio_copy)-sizeof(__s64))) 1713 goto out; 1714 1715 ret = validate_unaligned_range(ctx->mm, uffdio_copy.src, 1716 uffdio_copy.len); 1717 if (ret) 1718 goto out; 1719 ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); 1720 if (ret) 1721 goto out; 1722 1723 ret = -EINVAL; 1724 if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP)) 1725 goto out; 1726 if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP) 1727 flags |= MFILL_ATOMIC_WP; 1728 if (mmget_not_zero(ctx->mm)) { 1729 ret = mfill_atomic_copy(ctx, uffdio_copy.dst, uffdio_copy.src, 1730 uffdio_copy.len, flags); 1731 mmput(ctx->mm); 1732 } else { 1733 return -ESRCH; 1734 } 1735 if (unlikely(put_user(ret, &user_uffdio_copy->copy))) 1736 return -EFAULT; 1737 if (ret < 0) 1738 goto out; 1739 BUG_ON(!ret); 1740 /* len == 0 would wake all */ 1741 range.len = ret; 1742 if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) { 1743 range.start = uffdio_copy.dst; 1744 wake_userfault(ctx, &range); 1745 } 1746 ret = range.len == uffdio_copy.len ? 0 : -EAGAIN; 1747out: 1748 return ret; 1749} 1750 1751static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, 1752 unsigned long arg) 1753{ 1754 __s64 ret; 1755 struct uffdio_zeropage uffdio_zeropage; 1756 struct uffdio_zeropage __user *user_uffdio_zeropage; 1757 struct userfaultfd_wake_range range; 1758 1759 user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg; 1760 1761 ret = -EAGAIN; 1762 if (atomic_read(&ctx->mmap_changing)) 1763 goto out; 1764 1765 ret = -EFAULT; 1766 if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage, 1767 /* don't copy "zeropage" last field */ 1768 sizeof(uffdio_zeropage)-sizeof(__s64))) 1769 goto out; 1770 1771 ret = validate_range(ctx->mm, uffdio_zeropage.range.start, 1772 uffdio_zeropage.range.len); 1773 if (ret) 1774 goto out; 1775 ret = -EINVAL; 1776 if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE) 1777 goto out; 1778 1779 if (mmget_not_zero(ctx->mm)) { 1780 ret = mfill_atomic_zeropage(ctx, uffdio_zeropage.range.start, 1781 uffdio_zeropage.range.len); 1782 mmput(ctx->mm); 1783 } else { 1784 return -ESRCH; 1785 } 1786 if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage))) 1787 return -EFAULT; 1788 if (ret < 0) 1789 goto out; 1790 /* len == 0 would wake all */ 1791 BUG_ON(!ret); 1792 range.len = ret; 1793 if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) { 1794 range.start = uffdio_zeropage.range.start; 1795 wake_userfault(ctx, &range); 1796 } 1797 ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN; 1798out: 1799 return ret; 1800} 1801 1802static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, 1803 unsigned long arg) 1804{ 1805 int ret; 1806 struct uffdio_writeprotect uffdio_wp; 1807 struct uffdio_writeprotect __user *user_uffdio_wp; 1808 struct userfaultfd_wake_range range; 1809 bool mode_wp, mode_dontwake; 1810 1811 if (atomic_read(&ctx->mmap_changing)) 1812 return -EAGAIN; 1813 1814 user_uffdio_wp = (struct uffdio_writeprotect __user *) arg; 1815 1816 if (copy_from_user(&uffdio_wp, user_uffdio_wp, 1817 sizeof(struct uffdio_writeprotect))) 1818 return -EFAULT; 1819 1820 ret = validate_range(ctx->mm, uffdio_wp.range.start, 1821 uffdio_wp.range.len); 1822 if (ret) 1823 return ret; 1824 1825 if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE | 1826 UFFDIO_WRITEPROTECT_MODE_WP)) 1827 return -EINVAL; 1828 1829 mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP; 1830 mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE; 1831 1832 if (mode_wp && mode_dontwake) 1833 return -EINVAL; 1834 1835 if (mmget_not_zero(ctx->mm)) { 1836 ret = mwriteprotect_range(ctx, uffdio_wp.range.start, 1837 uffdio_wp.range.len, mode_wp); 1838 mmput(ctx->mm); 1839 } else { 1840 return -ESRCH; 1841 } 1842 1843 if (ret) 1844 return ret; 1845 1846 if (!mode_wp && !mode_dontwake) { 1847 range.start = uffdio_wp.range.start; 1848 range.len = uffdio_wp.range.len; 1849 wake_userfault(ctx, &range); 1850 } 1851 return ret; 1852} 1853 1854static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg) 1855{ 1856 __s64 ret; 1857 struct uffdio_continue uffdio_continue; 1858 struct uffdio_continue __user *user_uffdio_continue; 1859 struct userfaultfd_wake_range range; 1860 uffd_flags_t flags = 0; 1861 1862 user_uffdio_continue = (struct uffdio_continue __user *)arg; 1863 1864 ret = -EAGAIN; 1865 if (atomic_read(&ctx->mmap_changing)) 1866 goto out; 1867 1868 ret = -EFAULT; 1869 if (copy_from_user(&uffdio_continue, user_uffdio_continue, 1870 /* don't copy the output fields */ 1871 sizeof(uffdio_continue) - (sizeof(__s64)))) 1872 goto out; 1873 1874 ret = validate_range(ctx->mm, uffdio_continue.range.start, 1875 uffdio_continue.range.len); 1876 if (ret) 1877 goto out; 1878 1879 ret = -EINVAL; 1880 if (uffdio_continue.mode & ~(UFFDIO_CONTINUE_MODE_DONTWAKE | 1881 UFFDIO_CONTINUE_MODE_WP)) 1882 goto out; 1883 if (uffdio_continue.mode & UFFDIO_CONTINUE_MODE_WP) 1884 flags |= MFILL_ATOMIC_WP; 1885 1886 if (mmget_not_zero(ctx->mm)) { 1887 ret = mfill_atomic_continue(ctx, uffdio_continue.range.start, 1888 uffdio_continue.range.len, flags); 1889 mmput(ctx->mm); 1890 } else { 1891 return -ESRCH; 1892 } 1893 1894 if (unlikely(put_user(ret, &user_uffdio_continue->mapped))) 1895 return -EFAULT; 1896 if (ret < 0) 1897 goto out; 1898 1899 /* len == 0 would wake all */ 1900 BUG_ON(!ret); 1901 range.len = ret; 1902 if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) { 1903 range.start = uffdio_continue.range.start; 1904 wake_userfault(ctx, &range); 1905 } 1906 ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN; 1907 1908out: 1909 return ret; 1910} 1911 1912static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long arg) 1913{ 1914 __s64 ret; 1915 struct uffdio_poison uffdio_poison; 1916 struct uffdio_poison __user *user_uffdio_poison; 1917 struct userfaultfd_wake_range range; 1918 1919 user_uffdio_poison = (struct uffdio_poison __user *)arg; 1920 1921 ret = -EAGAIN; 1922 if (atomic_read(&ctx->mmap_changing)) 1923 goto out; 1924 1925 ret = -EFAULT; 1926 if (copy_from_user(&uffdio_poison, user_uffdio_poison, 1927 /* don't copy the output fields */ 1928 sizeof(uffdio_poison) - (sizeof(__s64)))) 1929 goto out; 1930 1931 ret = validate_range(ctx->mm, uffdio_poison.range.start, 1932 uffdio_poison.range.len); 1933 if (ret) 1934 goto out; 1935 1936 ret = -EINVAL; 1937 if (uffdio_poison.mode & ~UFFDIO_POISON_MODE_DONTWAKE) 1938 goto out; 1939 1940 if (mmget_not_zero(ctx->mm)) { 1941 ret = mfill_atomic_poison(ctx, uffdio_poison.range.start, 1942 uffdio_poison.range.len, 0); 1943 mmput(ctx->mm); 1944 } else { 1945 return -ESRCH; 1946 } 1947 1948 if (unlikely(put_user(ret, &user_uffdio_poison->updated))) 1949 return -EFAULT; 1950 if (ret < 0) 1951 goto out; 1952 1953 /* len == 0 would wake all */ 1954 BUG_ON(!ret); 1955 range.len = ret; 1956 if (!(uffdio_poison.mode & UFFDIO_POISON_MODE_DONTWAKE)) { 1957 range.start = uffdio_poison.range.start; 1958 wake_userfault(ctx, &range); 1959 } 1960 ret = range.len == uffdio_poison.range.len ? 0 : -EAGAIN; 1961 1962out: 1963 return ret; 1964} 1965 1966bool userfaultfd_wp_async(struct vm_area_struct *vma) 1967{ 1968 return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx); 1969} 1970 1971static inline unsigned int uffd_ctx_features(__u64 user_features) 1972{ 1973 /* 1974 * For the current set of features the bits just coincide. Set 1975 * UFFD_FEATURE_INITIALIZED to mark the features as enabled. 1976 */ 1977 return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED; 1978} 1979 1980static int userfaultfd_move(struct userfaultfd_ctx *ctx, 1981 unsigned long arg) 1982{ 1983 __s64 ret; 1984 struct uffdio_move uffdio_move; 1985 struct uffdio_move __user *user_uffdio_move; 1986 struct userfaultfd_wake_range range; 1987 struct mm_struct *mm = ctx->mm; 1988 1989 user_uffdio_move = (struct uffdio_move __user *) arg; 1990 1991 if (atomic_read(&ctx->mmap_changing)) 1992 return -EAGAIN; 1993 1994 if (copy_from_user(&uffdio_move, user_uffdio_move, 1995 /* don't copy "move" last field */ 1996 sizeof(uffdio_move)-sizeof(__s64))) 1997 return -EFAULT; 1998 1999 /* Do not allow cross-mm moves. */ 2000 if (mm != current->mm) 2001 return -EINVAL; 2002 2003 ret = validate_range(mm, uffdio_move.dst, uffdio_move.len); 2004 if (ret) 2005 return ret; 2006 2007 ret = validate_range(mm, uffdio_move.src, uffdio_move.len); 2008 if (ret) 2009 return ret; 2010 2011 if (uffdio_move.mode & ~(UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES| 2012 UFFDIO_MOVE_MODE_DONTWAKE)) 2013 return -EINVAL; 2014 2015 if (mmget_not_zero(mm)) { 2016 ret = move_pages(ctx, uffdio_move.dst, uffdio_move.src, 2017 uffdio_move.len, uffdio_move.mode); 2018 mmput(mm); 2019 } else { 2020 return -ESRCH; 2021 } 2022 2023 if (unlikely(put_user(ret, &user_uffdio_move->move))) 2024 return -EFAULT; 2025 if (ret < 0) 2026 goto out; 2027 2028 /* len == 0 would wake all */ 2029 VM_WARN_ON(!ret); 2030 range.len = ret; 2031 if (!(uffdio_move.mode & UFFDIO_MOVE_MODE_DONTWAKE)) { 2032 range.start = uffdio_move.dst; 2033 wake_userfault(ctx, &range); 2034 } 2035 ret = range.len == uffdio_move.len ? 0 : -EAGAIN; 2036 2037out: 2038 return ret; 2039} 2040 2041/* 2042 * userland asks for a certain API version and we return which bits 2043 * and ioctl commands are implemented in this kernel for such API 2044 * version or -EINVAL if unknown. 2045 */ 2046static int userfaultfd_api(struct userfaultfd_ctx *ctx, 2047 unsigned long arg) 2048{ 2049 struct uffdio_api uffdio_api; 2050 void __user *buf = (void __user *)arg; 2051 unsigned int ctx_features; 2052 int ret; 2053 __u64 features; 2054 2055 ret = -EFAULT; 2056 if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api))) 2057 goto out; 2058 features = uffdio_api.features; 2059 ret = -EINVAL; 2060 if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES)) 2061 goto err_out; 2062 ret = -EPERM; 2063 if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE)) 2064 goto err_out; 2065 2066 /* WP_ASYNC relies on WP_UNPOPULATED, choose it unconditionally */ 2067 if (features & UFFD_FEATURE_WP_ASYNC) 2068 features |= UFFD_FEATURE_WP_UNPOPULATED; 2069 2070 /* report all available features and ioctls to userland */ 2071 uffdio_api.features = UFFD_API_FEATURES; 2072#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 2073 uffdio_api.features &= 2074 ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); 2075#endif 2076#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP 2077 uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP; 2078#endif 2079#ifndef CONFIG_PTE_MARKER_UFFD_WP 2080 uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM; 2081 uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; 2082 uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC; 2083#endif 2084 uffdio_api.ioctls = UFFD_API_IOCTLS; 2085 ret = -EFAULT; 2086 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) 2087 goto out; 2088 2089 /* only enable the requested features for this uffd context */ 2090 ctx_features = uffd_ctx_features(features); 2091 ret = -EINVAL; 2092 if (cmpxchg(&ctx->features, 0, ctx_features) != 0) 2093 goto err_out; 2094 2095 ret = 0; 2096out: 2097 return ret; 2098err_out: 2099 memset(&uffdio_api, 0, sizeof(uffdio_api)); 2100 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) 2101 ret = -EFAULT; 2102 goto out; 2103} 2104 2105static long userfaultfd_ioctl(struct file *file, unsigned cmd, 2106 unsigned long arg) 2107{ 2108 int ret = -EINVAL; 2109 struct userfaultfd_ctx *ctx = file->private_data; 2110 2111 if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx)) 2112 return -EINVAL; 2113 2114 switch(cmd) { 2115 case UFFDIO_API: 2116 ret = userfaultfd_api(ctx, arg); 2117 break; 2118 case UFFDIO_REGISTER: 2119 ret = userfaultfd_register(ctx, arg); 2120 break; 2121 case UFFDIO_UNREGISTER: 2122 ret = userfaultfd_unregister(ctx, arg); 2123 break; 2124 case UFFDIO_WAKE: 2125 ret = userfaultfd_wake(ctx, arg); 2126 break; 2127 case UFFDIO_COPY: 2128 ret = userfaultfd_copy(ctx, arg); 2129 break; 2130 case UFFDIO_ZEROPAGE: 2131 ret = userfaultfd_zeropage(ctx, arg); 2132 break; 2133 case UFFDIO_MOVE: 2134 ret = userfaultfd_move(ctx, arg); 2135 break; 2136 case UFFDIO_WRITEPROTECT: 2137 ret = userfaultfd_writeprotect(ctx, arg); 2138 break; 2139 case UFFDIO_CONTINUE: 2140 ret = userfaultfd_continue(ctx, arg); 2141 break; 2142 case UFFDIO_POISON: 2143 ret = userfaultfd_poison(ctx, arg); 2144 break; 2145 } 2146 return ret; 2147} 2148 2149#ifdef CONFIG_PROC_FS 2150static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f) 2151{ 2152 struct userfaultfd_ctx *ctx = f->private_data; 2153 wait_queue_entry_t *wq; 2154 unsigned long pending = 0, total = 0; 2155 2156 spin_lock_irq(&ctx->fault_pending_wqh.lock); 2157 list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) { 2158 pending++; 2159 total++; 2160 } 2161 list_for_each_entry(wq, &ctx->fault_wqh.head, entry) { 2162 total++; 2163 } 2164 spin_unlock_irq(&ctx->fault_pending_wqh.lock); 2165 2166 /* 2167 * If more protocols will be added, there will be all shown 2168 * separated by a space. Like this: 2169 * protocols: aa:... bb:... 2170 */ 2171 seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n", 2172 pending, total, UFFD_API, ctx->features, 2173 UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS); 2174} 2175#endif 2176 2177static const struct file_operations userfaultfd_fops = { 2178#ifdef CONFIG_PROC_FS 2179 .show_fdinfo = userfaultfd_show_fdinfo, 2180#endif 2181 .release = userfaultfd_release, 2182 .poll = userfaultfd_poll, 2183 .read_iter = userfaultfd_read_iter, 2184 .unlocked_ioctl = userfaultfd_ioctl, 2185 .compat_ioctl = compat_ptr_ioctl, 2186 .llseek = noop_llseek, 2187}; 2188 2189static void init_once_userfaultfd_ctx(void *mem) 2190{ 2191 struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem; 2192 2193 init_waitqueue_head(&ctx->fault_pending_wqh); 2194 init_waitqueue_head(&ctx->fault_wqh); 2195 init_waitqueue_head(&ctx->event_wqh); 2196 init_waitqueue_head(&ctx->fd_wqh); 2197 seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock); 2198} 2199 2200static int new_userfaultfd(int flags) 2201{ 2202 struct userfaultfd_ctx *ctx; 2203 struct file *file; 2204 int fd; 2205 2206 BUG_ON(!current->mm); 2207 2208 /* Check the UFFD_* constants for consistency. */ 2209 BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS); 2210 BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC); 2211 BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK); 2212 2213 if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY)) 2214 return -EINVAL; 2215 2216 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL); 2217 if (!ctx) 2218 return -ENOMEM; 2219 2220 refcount_set(&ctx->refcount, 1); 2221 ctx->flags = flags; 2222 ctx->features = 0; 2223 ctx->released = false; 2224 init_rwsem(&ctx->map_changing_lock); 2225 atomic_set(&ctx->mmap_changing, 0); 2226 ctx->mm = current->mm; 2227 2228 fd = get_unused_fd_flags(flags & UFFD_SHARED_FCNTL_FLAGS); 2229 if (fd < 0) 2230 goto err_out; 2231 2232 /* Create a new inode so that the LSM can block the creation. */ 2233 file = anon_inode_create_getfile("[userfaultfd]", &userfaultfd_fops, ctx, 2234 O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL); 2235 if (IS_ERR(file)) { 2236 put_unused_fd(fd); 2237 fd = PTR_ERR(file); 2238 goto err_out; 2239 } 2240 /* prevent the mm struct to be freed */ 2241 mmgrab(ctx->mm); 2242 file->f_mode |= FMODE_NOWAIT; 2243 fd_install(fd, file); 2244 return fd; 2245err_out: 2246 kmem_cache_free(userfaultfd_ctx_cachep, ctx); 2247 return fd; 2248} 2249 2250static inline bool userfaultfd_syscall_allowed(int flags) 2251{ 2252 /* Userspace-only page faults are always allowed */ 2253 if (flags & UFFD_USER_MODE_ONLY) 2254 return true; 2255 2256 /* 2257 * The user is requesting a userfaultfd which can handle kernel faults. 2258 * Privileged users are always allowed to do this. 2259 */ 2260 if (capable(CAP_SYS_PTRACE)) 2261 return true; 2262 2263 /* Otherwise, access to kernel fault handling is sysctl controlled. */ 2264 return sysctl_unprivileged_userfaultfd; 2265} 2266 2267SYSCALL_DEFINE1(userfaultfd, int, flags) 2268{ 2269 if (!userfaultfd_syscall_allowed(flags)) 2270 return -EPERM; 2271 2272 return new_userfaultfd(flags); 2273} 2274 2275static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags) 2276{ 2277 if (cmd != USERFAULTFD_IOC_NEW) 2278 return -EINVAL; 2279 2280 return new_userfaultfd(flags); 2281} 2282 2283static const struct file_operations userfaultfd_dev_fops = { 2284 .unlocked_ioctl = userfaultfd_dev_ioctl, 2285 .compat_ioctl = userfaultfd_dev_ioctl, 2286 .owner = THIS_MODULE, 2287 .llseek = noop_llseek, 2288}; 2289 2290static struct miscdevice userfaultfd_misc = { 2291 .minor = MISC_DYNAMIC_MINOR, 2292 .name = "userfaultfd", 2293 .fops = &userfaultfd_dev_fops 2294}; 2295 2296static int __init userfaultfd_init(void) 2297{ 2298 int ret; 2299 2300 ret = misc_register(&userfaultfd_misc); 2301 if (ret) 2302 return ret; 2303 2304 userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache", 2305 sizeof(struct userfaultfd_ctx), 2306 0, 2307 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 2308 init_once_userfaultfd_ctx); 2309#ifdef CONFIG_SYSCTL 2310 register_sysctl_init("vm", vm_userfaultfd_table); 2311#endif 2312 return 0; 2313} 2314__initcall(userfaultfd_init); 2315