1251881Speter// SPDX-License-Identifier: GPL-2.0-only 2251881Speter/* Kernel thread helper functions. 3251881Speter * Copyright (C) 2004 IBM Corporation, Rusty Russell. 4251881Speter * Copyright (C) 2009 Red Hat, Inc. 5251881Speter * 6251881Speter * Creation is done via kthreadd, so that we get a clean environment 7251881Speter * even if we're invoked from userspace (think modprobe, hotplug cpu, 8251881Speter * etc.). 9251881Speter */ 10251881Speter#include <uapi/linux/sched/types.h> 11251881Speter#include <linux/mm.h> 12251881Speter#include <linux/mmu_context.h> 13251881Speter#include <linux/sched.h> 14251881Speter#include <linux/sched/mm.h> 15251881Speter#include <linux/sched/task.h> 16251881Speter#include <linux/kthread.h> 17251881Speter#include <linux/completion.h> 18251881Speter#include <linux/err.h> 19251881Speter#include <linux/cgroup.h> 20251881Speter#include <linux/cpuset.h> 21251881Speter#include <linux/unistd.h> 22251881Speter#include <linux/file.h> 23251881Speter#include <linux/export.h> 24251881Speter#include <linux/mutex.h> 25251881Speter#include <linux/slab.h> 26251881Speter#include <linux/freezer.h> 27251881Speter#include <linux/ptrace.h> 28251881Speter#include <linux/uaccess.h> 29251881Speter#include <linux/numa.h> 30251881Speter#include <linux/sched/isolation.h> 31251881Speter#include <trace/events/sched.h> 32251881Speter 33251881Speter 34251881Speterstatic DEFINE_SPINLOCK(kthread_create_lock); 35251881Speterstatic LIST_HEAD(kthread_create_list); 36251881Speterstruct task_struct *kthreadd_task; 37289180Speter 38251881Speterstruct kthread_create_info 39251881Speter{ 40251881Speter /* Information passed to kthread() from kthreadd. */ 41251881Speter char *full_name; 42369302Sdim int (*threadfn)(void *data); 43369302Sdim void *data; 44362181Sdim int node; 45369302Sdim 46369302Sdim /* Result passed back to kthread_create() from kthreadd. */ 47369302Sdim struct task_struct *result; 48251881Speter struct completion *done; 49251881Speter 50369302Sdim struct list_head list; 51251881Speter}; 52362181Sdim 53251881Speterstruct kthread { 54251881Speter unsigned long flags; 55251881Speter unsigned int cpu; 56251881Speter int result; 57251881Speter int (*threadfn)(void *); 58251881Speter void *data; 59251881Speter struct completion parked; 60251881Speter struct completion exited; 61251881Speter#ifdef CONFIG_BLK_CGROUP 62251881Speter struct cgroup_subsys_state *blkcg_css; 63251881Speter#endif 64251881Speter /* To store the full name if task comm is truncated. */ 65251881Speter char *full_name; 66251881Speter}; 67251881Speter 68251881Speterenum KTHREAD_BITS { 69251881Speter KTHREAD_IS_PER_CPU = 0, 70251881Speter KTHREAD_SHOULD_STOP, 71251881Speter KTHREAD_SHOULD_PARK, 72251881Speter}; 73251881Speter 74289180Speterstatic inline struct kthread *to_kthread(struct task_struct *k) 75251881Speter{ 76251881Speter WARN_ON(!(k->flags & PF_KTHREAD)); 77251881Speter return k->worker_private; 78251881Speter} 79251881Speter 80251881Speter/* 81289180Speter * Variant of to_kthread() that doesn't assume @p is a kthread. 82289180Speter * 83289180Speter * Per construction; when: 84289180Speter * 85289180Speter * (p->flags & PF_KTHREAD) && p->worker_private 86251881Speter * 87251881Speter * the task is both a kthread and struct kthread is persistent. However 88251881Speter * PF_KTHREAD on it's own is not, kernel_thread() can exec() (See umh.c and 89251881Speter * begin_new_exec()). 90251881Speter */ 91289180Speterstatic inline struct kthread *__to_kthread(struct task_struct *p) 92289180Speter{ 93289180Speter void *kthread = p->worker_private; 94289180Speter if (kthread && !(p->flags & PF_KTHREAD)) 95251881Speter kthread = NULL; 96289180Speter return kthread; 97289180Speter} 98289180Speter 99289180Spetervoid get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk) 100251881Speter{ 101289180Speter struct kthread *kthread = to_kthread(tsk); 102251881Speter 103251881Speter if (!kthread || !kthread->full_name) { 104251881Speter __get_task_comm(buf, buf_size, tsk); 105251881Speter return; 106251881Speter } 107251881Speter 108251881Speter strscpy_pad(buf, kthread->full_name, buf_size); 109251881Speter} 110251881Speter 111251881Speterbool set_kthread_struct(struct task_struct *p) 112251881Speter{ 113251881Speter struct kthread *kthread; 114251881Speter 115251881Speter if (WARN_ON_ONCE(to_kthread(p))) 116251881Speter return false; 117251881Speter 118251881Speter kthread = kzalloc(sizeof(*kthread), GFP_KERNEL); 119251881Speter if (!kthread) 120251881Speter return false; 121251881Speter 122251881Speter init_completion(&kthread->exited); 123251881Speter init_completion(&kthread->parked); 124251881Speter p->vfork_done = &kthread->exited; 125251881Speter 126251881Speter p->worker_private = kthread; 127251881Speter return true; 128251881Speter} 129251881Speter 130251881Spetervoid free_kthread_struct(struct task_struct *k) 131251881Speter{ 132251881Speter struct kthread *kthread; 133251881Speter 134251881Speter /* 135251881Speter * Can be NULL if kmalloc() in set_kthread_struct() failed. 136251881Speter */ 137251881Speter kthread = to_kthread(k); 138251881Speter if (!kthread) 139251881Speter return; 140251881Speter 141289180Speter#ifdef CONFIG_BLK_CGROUP 142251881Speter WARN_ON_ONCE(kthread->blkcg_css); 143251881Speter#endif 144251881Speter k->worker_private = NULL; 145251881Speter kfree(kthread->full_name); 146251881Speter kfree(kthread); 147251881Speter} 148251881Speter 149251881Speter/** 150251881Speter * kthread_should_stop - should this kthread return now? 151251881Speter * 152251881Speter * When someone calls kthread_stop() on your kthread, it will be woken 153251881Speter * and this will return true. You should then return, and your return 154251881Speter * value will be passed through to kthread_stop(). 155289180Speter */ 156251881Speterbool kthread_should_stop(void) 157251881Speter{ 158289180Speter return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags); 159289180Speter} 160289180SpeterEXPORT_SYMBOL(kthread_should_stop); 161289180Speter 162251881Speterstatic bool __kthread_should_park(struct task_struct *k) 163251881Speter{ 164289180Speter return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags); 165289180Speter} 166289180Speter 167289180Speter/** 168289180Speter * kthread_should_park - should this kthread park now? 169289180Speter * 170289180Speter * When someone calls kthread_park() on your kthread, it will be woken 171289180Speter * and this will return true. You should then do the necessary 172251881Speter * cleanup and call kthread_parkme() 173289180Speter * 174289180Speter * Similar to kthread_should_stop(), but this keeps the thread alive 175289180Speter * and in a park position. kthread_unpark() "restarts" the thread and 176289180Speter * calls the thread function again. 177289180Speter */ 178289180Speterbool kthread_should_park(void) 179289180Speter{ 180289180Speter return __kthread_should_park(current); 181289180Speter} 182289180SpeterEXPORT_SYMBOL_GPL(kthread_should_park); 183251881Speter 184251881Speterbool kthread_should_stop_or_park(void) 185251881Speter{ 186251881Speter struct kthread *kthread = __to_kthread(current); 187251881Speter 188251881Speter if (!kthread) 189251881Speter return false; 190251881Speter 191251881Speter return kthread->flags & (BIT(KTHREAD_SHOULD_STOP) | BIT(KTHREAD_SHOULD_PARK)); 192251881Speter} 193251881Speter 194251881Speter/** 195251881Speter * kthread_freezable_should_stop - should this freezable kthread return now? 196251881Speter * @was_frozen: optional out parameter, indicates whether %current was frozen 197251881Speter * 198251881Speter * kthread_should_stop() for freezable kthreads, which will enter 199251881Speter * refrigerator if necessary. This function is safe from kthread_stop() / 200251881Speter * freezer deadlock and freezable kthreads should use this function instead 201251881Speter * of calling try_to_freeze() directly. 202251881Speter */ 203251881Speterbool kthread_freezable_should_stop(bool *was_frozen) 204251881Speter{ 205251881Speter bool frozen = false; 206251881Speter 207251881Speter might_sleep(); 208251881Speter 209251881Speter if (unlikely(freezing(current))) 210251881Speter frozen = __refrigerator(true); 211251881Speter 212251881Speter if (was_frozen) 213251881Speter *was_frozen = frozen; 214251881Speter 215251881Speter return kthread_should_stop(); 216251881Speter} 217251881SpeterEXPORT_SYMBOL_GPL(kthread_freezable_should_stop); 218251881Speter 219251881Speter/** 220251881Speter * kthread_func - return the function specified on kthread creation 221251881Speter * @task: kthread task in question 222251881Speter * 223251881Speter * Returns NULL if the task is not a kthread. 224251881Speter */ 225251881Spetervoid *kthread_func(struct task_struct *task) 226251881Speter{ 227251881Speter struct kthread *kthread = __to_kthread(task); 228251881Speter if (kthread) 229251881Speter return kthread->threadfn; 230251881Speter return NULL; 231251881Speter} 232251881SpeterEXPORT_SYMBOL_GPL(kthread_func); 233251881Speter 234251881Speter/** 235251881Speter * kthread_data - return data value specified on kthread creation 236251881Speter * @task: kthread task in question 237251881Speter * 238251881Speter * Return the data value specified when kthread @task was created. 239251881Speter * The caller is responsible for ensuring the validity of @task when 240251881Speter * calling this function. 241251881Speter */ 242251881Spetervoid *kthread_data(struct task_struct *task) 243251881Speter{ 244251881Speter return to_kthread(task)->data; 245251881Speter} 246251881SpeterEXPORT_SYMBOL_GPL(kthread_data); 247251881Speter 248251881Speter/** 249251881Speter * kthread_probe_data - speculative version of kthread_data() 250251881Speter * @task: possible kthread task in question 251251881Speter * 252251881Speter * @task could be a kthread task. Return the data value specified when it 253251881Speter * was created if accessible. If @task isn't a kthread task or its data is 254251881Speter * inaccessible for any reason, %NULL is returned. This function requires 255251881Speter * that @task itself is safe to dereference. 256251881Speter */ 257251881Spetervoid *kthread_probe_data(struct task_struct *task) 258251881Speter{ 259251881Speter struct kthread *kthread = __to_kthread(task); 260251881Speter void *data = NULL; 261251881Speter 262251881Speter if (kthread) 263251881Speter copy_from_kernel_nofault(&data, &kthread->data, sizeof(data)); 264251881Speter return data; 265251881Speter} 266251881Speter 267251881Speterstatic void __kthread_parkme(struct kthread *self) 268251881Speter{ 269251881Speter for (;;) { 270251881Speter /* 271251881Speter * TASK_PARKED is a special state; we must serialize against 272251881Speter * possible pending wakeups to avoid store-store collisions on 273251881Speter * task->state. 274251881Speter * 275251881Speter * Such a collision might possibly result in the task state 276289180Speter * changin from TASK_PARKED and us failing the 277289180Speter * wait_task_inactive() in kthread_park(). 278289180Speter */ 279289180Speter set_special_state(TASK_PARKED); 280289180Speter if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags)) 281289180Speter break; 282289180Speter 283289180Speter /* 284289180Speter * Thread is going to call schedule(), do not preempt it, 285289180Speter * or the caller of kthread_park() may spend more time in 286289180Speter * wait_task_inactive(). 287289180Speter */ 288289180Speter preempt_disable(); 289289180Speter complete(&self->parked); 290289180Speter schedule_preempt_disabled(); 291289180Speter preempt_enable(); 292289180Speter } 293289180Speter __set_current_state(TASK_RUNNING); 294289180Speter} 295289180Speter 296289180Spetervoid kthread_parkme(void) 297289180Speter{ 298289180Speter __kthread_parkme(to_kthread(current)); 299289180Speter} 300289180SpeterEXPORT_SYMBOL_GPL(kthread_parkme); 301251881Speter 302251881Speter/** 303251881Speter * kthread_exit - Cause the current kthread return @result to kthread_stop(). 304251881Speter * @result: The integer value to return to kthread_stop(). 305251881Speter * 306251881Speter * While kthread_exit can be called directly, it exists so that 307251881Speter * functions which do some additional work in non-modular code such as 308251881Speter * module_put_and_kthread_exit can be implemented. 309251881Speter * 310289180Speter * Does not return. 311289180Speter */ 312251881Spetervoid __noreturn kthread_exit(long result) 313289180Speter{ 314289180Speter struct kthread *kthread = to_kthread(current); 315289180Speter kthread->result = result; 316251881Speter do_exit(0); 317251881Speter} 318251881SpeterEXPORT_SYMBOL(kthread_exit); 319251881Speter 320251881Speter/** 321251881Speter * kthread_complete_and_exit - Exit the current kthread. 322251881Speter * @comp: Completion to complete 323251881Speter * @code: The integer value to return to kthread_stop(). 324251881Speter * 325251881Speter * If present, complete @comp and then return code to kthread_stop(). 326251881Speter * 327251881Speter * A kernel thread whose module may be removed after the completion of 328251881Speter * @comp can use this function to exit safely. 329251881Speter * 330251881Speter * Does not return. 331251881Speter */ 332251881Spetervoid __noreturn kthread_complete_and_exit(struct completion *comp, long code) 333289180Speter{ 334289180Speter if (comp) 335251881Speter complete(comp); 336289180Speter 337289180Speter kthread_exit(code); 338289180Speter} 339251881SpeterEXPORT_SYMBOL(kthread_complete_and_exit); 340251881Speter 341251881Speterstatic int kthread(void *_create) 342251881Speter{ 343251881Speter static const struct sched_param param = { .sched_priority = 0 }; 344251881Speter /* Copy data: it's on kthread's stack */ 345251881Speter struct kthread_create_info *create = _create; 346251881Speter int (*threadfn)(void *data) = create->threadfn; 347251881Speter void *data = create->data; 348251881Speter struct completion *done; 349251881Speter struct kthread *self; 350251881Speter int ret; 351251881Speter 352251881Speter self = to_kthread(current); 353362181Sdim 354362181Sdim /* Release the structure when caller killed by a fatal signal. */ 355362181Sdim done = xchg(&create->done, NULL); 356362181Sdim if (!done) { 357362181Sdim kfree(create->full_name); 358362181Sdim kfree(create); 359362181Sdim kthread_exit(-EINTR); 360362181Sdim } 361362181Sdim 362362181Sdim self->full_name = create->full_name; 363362181Sdim self->threadfn = threadfn; 364362181Sdim self->data = data; 365362181Sdim 366362181Sdim /* 367362181Sdim * The new thread inherited kthreadd's priority and CPU mask. Reset 368362181Sdim * back to default in case they have been changed. 369362181Sdim */ 370251881Speter sched_setscheduler_nocheck(current, SCHED_NORMAL, ¶m); 371251881Speter set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_KTHREAD)); 372251881Speter 373251881Speter /* OK, tell user we're spawned, wait for stop or wakeup */ 374251881Speter __set_current_state(TASK_UNINTERRUPTIBLE); 375251881Speter create->result = current; 376251881Speter /* 377251881Speter * Thread is going to call schedule(), do not preempt it, 378251881Speter * or the creator may spend more time in wait_task_inactive(). 379251881Speter */ 380251881Speter preempt_disable(); 381251881Speter complete(done); 382251881Speter schedule_preempt_disabled(); 383251881Speter preempt_enable(); 384251881Speter 385251881Speter ret = -EINTR; 386251881Speter if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) { 387251881Speter cgroup_kthread_ready(); 388251881Speter __kthread_parkme(self); 389251881Speter ret = threadfn(data); 390251881Speter } 391251881Speter kthread_exit(ret); 392251881Speter} 393251881Speter 394251881Speter/* called from kernel_clone() to get node information for about to be created task */ 395251881Speterint tsk_fork_get_node(struct task_struct *tsk) 396251881Speter{ 397251881Speter#ifdef CONFIG_NUMA 398251881Speter if (tsk == kthreadd_task) 399251881Speter return tsk->pref_node_fork; 400251881Speter#endif 401251881Speter return NUMA_NO_NODE; 402251881Speter} 403251881Speter 404251881Speterstatic void create_kthread(struct kthread_create_info *create) 405251881Speter{ 406251881Speter int pid; 407251881Speter 408251881Speter#ifdef CONFIG_NUMA 409251881Speter current->pref_node_fork = create->node; 410251881Speter#endif 411289180Speter /* We want our own signal handler (we take no signals by default). */ 412289180Speter pid = kernel_thread(kthread, create, create->full_name, 413289180Speter CLONE_FS | CLONE_FILES | SIGCHLD); 414289180Speter if (pid < 0) { 415289180Speter /* Release the structure when caller killed by a fatal signal. */ 416289180Speter struct completion *done = xchg(&create->done, NULL); 417289180Speter 418289180Speter kfree(create->full_name); 419289180Speter if (!done) { 420289180Speter kfree(create); 421289180Speter return; 422289180Speter } 423289180Speter create->result = ERR_PTR(pid); 424289180Speter complete(done); 425289180Speter } 426289180Speter} 427289180Speter 428289180Speterstatic __printf(4, 0) 429289180Speterstruct task_struct *__kthread_create_on_node(int (*threadfn)(void *data), 430289180Speter void *data, int node, 431289180Speter const char namefmt[], 432289180Speter va_list args) 433289180Speter{ 434289180Speter DECLARE_COMPLETION_ONSTACK(done); 435289180Speter struct task_struct *task; 436289180Speter struct kthread_create_info *create = kmalloc(sizeof(*create), 437289180Speter GFP_KERNEL); 438289180Speter 439289180Speter if (!create) 440289180Speter return ERR_PTR(-ENOMEM); 441289180Speter create->threadfn = threadfn; 442289180Speter create->data = data; 443289180Speter create->node = node; 444289180Speter create->done = &done; 445289180Speter create->full_name = kvasprintf(GFP_KERNEL, namefmt, args); 446289180Speter if (!create->full_name) { 447289180Speter task = ERR_PTR(-ENOMEM); 448289180Speter goto free_create; 449289180Speter } 450251881Speter 451251881Speter spin_lock(&kthread_create_lock); 452251881Speter list_add_tail(&create->list, &kthread_create_list); 453251881Speter spin_unlock(&kthread_create_lock); 454251881Speter 455251881Speter wake_up_process(kthreadd_task); 456251881Speter /* 457251881Speter * Wait for completion in killable state, for I might be chosen by 458251881Speter * the OOM killer while kthreadd is trying to allocate memory for 459251881Speter * new kernel thread. 460251881Speter */ 461251881Speter if (unlikely(wait_for_completion_killable(&done))) { 462251881Speter /* 463251881Speter * If I was killed by a fatal signal before kthreadd (or new 464251881Speter * kernel thread) calls complete(), leave the cleanup of this 465251881Speter * structure to that thread. 466251881Speter */ 467251881Speter if (xchg(&create->done, NULL)) 468251881Speter return ERR_PTR(-EINTR); 469251881Speter /* 470257936Speter * kthreadd (or new kernel thread) will call complete() 471251881Speter * shortly. 472251881Speter */ 473251881Speter wait_for_completion(&done); 474251881Speter } 475251881Speter task = create->result; 476251881Speterfree_create: 477251881Speter kfree(create); 478251881Speter return task; 479251881Speter} 480251881Speter 481251881Speter/** 482251881Speter * kthread_create_on_node - create a kthread. 483251881Speter * @threadfn: the function to run until signal_pending(current). 484251881Speter * @data: data ptr for @threadfn. 485251881Speter * @node: task and thread structures for the thread are allocated on this node 486251881Speter * @namefmt: printf-style name for the thread. 487251881Speter * 488251881Speter * Description: This helper function creates and names a kernel 489251881Speter * thread. The thread will be stopped: use wake_up_process() to start 490251881Speter * it. See also kthread_run(). The new thread has SCHED_NORMAL policy and 491251881Speter * is affine to all CPUs. 492251881Speter * 493257936Speter * If thread is going to be bound on a particular cpu, give its node 494251881Speter * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE. 495251881Speter * When woken, the thread will run @threadfn() with @data as its 496251881Speter * argument. @threadfn() can either return directly if it is a 497251881Speter * standalone thread for which no one will call kthread_stop(), or 498251881Speter * return when 'kthread_should_stop()' is true (which means 499251881Speter * kthread_stop() has been called). The return value should be zero 500251881Speter * or a negative error number; it will be passed to kthread_stop(). 501251881Speter * 502251881Speter * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR). 503251881Speter */ 504251881Speterstruct task_struct *kthread_create_on_node(int (*threadfn)(void *data), 505251881Speter void *data, int node, 506251881Speter const char namefmt[], 507289180Speter ...) 508251881Speter{ 509251881Speter struct task_struct *task; 510289180Speter va_list args; 511289180Speter 512289180Speter va_start(args, namefmt); 513251881Speter task = __kthread_create_on_node(threadfn, data, node, namefmt, args); 514251881Speter va_end(args); 515251881Speter 516251881Speter return task; 517251881Speter} 518251881SpeterEXPORT_SYMBOL(kthread_create_on_node); 519251881Speter 520251881Speterstatic void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, unsigned int state) 521251881Speter{ 522251881Speter unsigned long flags; 523251881Speter 524251881Speter if (!wait_task_inactive(p, state)) { 525251881Speter WARN_ON(1); 526251881Speter return; 527251881Speter } 528251881Speter 529251881Speter /* It's safe because the task is inactive. */ 530251881Speter raw_spin_lock_irqsave(&p->pi_lock, flags); 531251881Speter do_set_cpus_allowed(p, mask); 532251881Speter p->flags |= PF_NO_SETAFFINITY; 533251881Speter raw_spin_unlock_irqrestore(&p->pi_lock, flags); 534251881Speter} 535289180Speter 536289180Speterstatic void __kthread_bind(struct task_struct *p, unsigned int cpu, unsigned int state) 537289180Speter{ 538289180Speter __kthread_bind_mask(p, cpumask_of(cpu), state); 539289180Speter} 540289180Speter 541289180Spetervoid kthread_bind_mask(struct task_struct *p, const struct cpumask *mask) 542289180Speter{ 543251881Speter __kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE); 544251881Speter} 545251881Speter 546289180Speter/** 547289180Speter * kthread_bind - bind a just-created kthread to a cpu. 548289180Speter * @p: thread created by kthread_create(). 549251881Speter * @cpu: cpu (might not be online, must be possible) for @k to run on. 550251881Speter * 551251881Speter * Description: This function is equivalent to set_cpus_allowed(), 552289180Speter * except that @cpu doesn't need to be online, and the thread must be 553251881Speter * stopped (i.e., just returned from kthread_create()). 554251881Speter */ 555251881Spetervoid kthread_bind(struct task_struct *p, unsigned int cpu) 556289180Speter{ 557289180Speter __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE); 558289180Speter} 559289180SpeterEXPORT_SYMBOL(kthread_bind); 560289180Speter 561289180Speter/** 562289180Speter * kthread_create_on_cpu - Create a cpu bound kthread 563289180Speter * @threadfn: the function to run until signal_pending(current). 564251881Speter * @data: data ptr for @threadfn. 565289180Speter * @cpu: The cpu on which the thread should be bound, 566289180Speter * @namefmt: printf-style name for the thread. Format is restricted 567251881Speter * to "name.*%u". Code fills in cpu number. 568251881Speter * 569289180Speter * Description: This helper function creates and names a kernel thread 570289180Speter */ 571289180Speterstruct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data), 572289180Speter void *data, unsigned int cpu, 573289180Speter const char *namefmt) 574289180Speter{ 575289180Speter struct task_struct *p; 576289180Speter 577289180Speter p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt, 578289180Speter cpu); 579289180Speter if (IS_ERR(p)) 580251881Speter return p; 581251881Speter kthread_bind(p, cpu); 582251881Speter /* CPU hotplug need to bind once again when unparking the thread. */ 583251881Speter to_kthread(p)->cpu = cpu; 584251881Speter return p; 585251881Speter} 586251881SpeterEXPORT_SYMBOL(kthread_create_on_cpu); 587251881Speter 588251881Spetervoid kthread_set_per_cpu(struct task_struct *k, int cpu) 589251881Speter{ 590289180Speter struct kthread *kthread = to_kthread(k); 591289180Speter if (!kthread) 592289180Speter return; 593289180Speter 594289180Speter WARN_ON_ONCE(!(k->flags & PF_NO_SETAFFINITY)); 595289180Speter 596289180Speter if (cpu < 0) { 597289180Speter clear_bit(KTHREAD_IS_PER_CPU, &kthread->flags); 598289180Speter return; 599289180Speter } 600289180Speter 601289180Speter kthread->cpu = cpu; 602289180Speter set_bit(KTHREAD_IS_PER_CPU, &kthread->flags); 603289180Speter} 604289180Speter 605289180Speterbool kthread_is_per_cpu(struct task_struct *p) 606251881Speter{ 607251881Speter struct kthread *kthread = __to_kthread(p); 608251881Speter if (!kthread) 609251881Speter return false; 610251881Speter 611251881Speter return test_bit(KTHREAD_IS_PER_CPU, &kthread->flags); 612251881Speter} 613251881Speter 614251881Speter/** 615251881Speter * kthread_unpark - unpark a thread created by kthread_create(). 616251881Speter * @k: thread created by kthread_create(). 617251881Speter * 618251881Speter * Sets kthread_should_park() for @k to return false, wakes it, and 619251881Speter * waits for it to return. If the thread is marked percpu then its 620251881Speter * bound to the cpu again. 621251881Speter */ 622251881Spetervoid kthread_unpark(struct task_struct *k) 623251881Speter{ 624251881Speter struct kthread *kthread = to_kthread(k); 625251881Speter 626251881Speter /* 627251881Speter * Newly created kthread was parked when the CPU was offline. 628251881Speter * The binding was lost and we need to set it again. 629251881Speter */ 630251881Speter if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags)) 631251881Speter __kthread_bind(k, kthread->cpu, TASK_PARKED); 632251881Speter 633251881Speter clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 634251881Speter /* 635251881Speter * __kthread_parkme() will either see !SHOULD_PARK or get the wakeup. 636251881Speter */ 637251881Speter wake_up_state(k, TASK_PARKED); 638251881Speter} 639251881SpeterEXPORT_SYMBOL_GPL(kthread_unpark); 640251881Speter 641251881Speter/** 642251881Speter * kthread_park - park a thread created by kthread_create(). 643251881Speter * @k: thread created by kthread_create(). 644251881Speter * 645251881Speter * Sets kthread_should_park() for @k to return true, wakes it, and 646251881Speter * waits for it to return. This can also be called after kthread_create() 647251881Speter * instead of calling wake_up_process(): the thread will park without 648251881Speter * calling threadfn(). 649251881Speter * 650251881Speter * Returns 0 if the thread is parked, -ENOSYS if the thread exited. 651251881Speter * If called by the kthread itself just the park bit is set. 652251881Speter */ 653251881Speterint kthread_park(struct task_struct *k) 654251881Speter{ 655251881Speter struct kthread *kthread = to_kthread(k); 656251881Speter 657251881Speter if (WARN_ON(k->flags & PF_EXITING)) 658251881Speter return -ENOSYS; 659251881Speter 660251881Speter if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags))) 661251881Speter return -EBUSY; 662251881Speter 663251881Speter set_bit(KTHREAD_SHOULD_PARK, &kthread->flags); 664251881Speter if (k != current) { 665251881Speter wake_up_process(k); 666251881Speter /* 667251881Speter * Wait for __kthread_parkme() to complete(), this means we 668251881Speter * _will_ have TASK_PARKED and are about to call schedule(). 669251881Speter */ 670251881Speter wait_for_completion(&kthread->parked); 671251881Speter /* 672251881Speter * Now wait for that schedule() to complete and the task to 673251881Speter * get scheduled out. 674251881Speter */ 675251881Speter WARN_ON_ONCE(!wait_task_inactive(k, TASK_PARKED)); 676251881Speter } 677251881Speter 678251881Speter return 0; 679251881Speter} 680251881SpeterEXPORT_SYMBOL_GPL(kthread_park); 681251881Speter 682251881Speter/** 683251881Speter * kthread_stop - stop a thread created by kthread_create(). 684251881Speter * @k: thread created by kthread_create(). 685251881Speter * 686251881Speter * Sets kthread_should_stop() for @k to return true, wakes it, and 687251881Speter * waits for it to exit. This can also be called after kthread_create() 688251881Speter * instead of calling wake_up_process(): the thread will exit without 689251881Speter * calling threadfn(). 690251881Speter * 691251881Speter * If threadfn() may call kthread_exit() itself, the caller must ensure 692251881Speter * task_struct can't go away. 693251881Speter * 694251881Speter * Returns the result of threadfn(), or %-EINTR if wake_up_process() 695251881Speter * was never called. 696251881Speter */ 697251881Speterint kthread_stop(struct task_struct *k) 698251881Speter{ 699251881Speter struct kthread *kthread; 700251881Speter int ret; 701251881Speter 702251881Speter trace_sched_kthread_stop(k); 703251881Speter 704251881Speter get_task_struct(k); 705251881Speter kthread = to_kthread(k); 706289180Speter set_bit(KTHREAD_SHOULD_STOP, &kthread->flags); 707289180Speter kthread_unpark(k); 708289180Speter set_tsk_thread_flag(k, TIF_NOTIFY_SIGNAL); 709251881Speter wake_up_process(k); 710289180Speter wait_for_completion(&kthread->exited); 711289180Speter ret = kthread->result; 712289180Speter put_task_struct(k); 713289180Speter 714289180Speter trace_sched_kthread_stop_ret(ret); 715289180Speter return ret; 716289180Speter} 717289180SpeterEXPORT_SYMBOL(kthread_stop); 718289180Speter 719251881Speter/** 720251881Speter * kthread_stop_put - stop a thread and put its task struct 721289180Speter * @k: thread created by kthread_create(). 722251881Speter * 723251881Speter * Stops a thread created by kthread_create() and put its task_struct. 724251881Speter * Only use when holding an extra task struct reference obtained by 725251881Speter * calling get_task_struct(). 726251881Speter */ 727251881Speterint kthread_stop_put(struct task_struct *k) 728251881Speter{ 729251881Speter int ret; 730251881Speter 731251881Speter ret = kthread_stop(k); 732251881Speter put_task_struct(k); 733251881Speter return ret; 734251881Speter} 735251881SpeterEXPORT_SYMBOL(kthread_stop_put); 736251881Speter 737251881Speterint kthreadd(void *unused) 738251881Speter{ 739251881Speter struct task_struct *tsk = current; 740251881Speter 741251881Speter /* Setup a clean context for our children to inherit. */ 742251881Speter set_task_comm(tsk, "kthreadd"); 743251881Speter ignore_signals(tsk); 744251881Speter set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_TYPE_KTHREAD)); 745251881Speter set_mems_allowed(node_states[N_MEMORY]); 746251881Speter 747251881Speter current->flags |= PF_NOFREEZE; 748251881Speter cgroup_init_kthreadd(); 749251881Speter 750251881Speter for (;;) { 751251881Speter set_current_state(TASK_INTERRUPTIBLE); 752251881Speter if (list_empty(&kthread_create_list)) 753251881Speter schedule(); 754251881Speter __set_current_state(TASK_RUNNING); 755251881Speter 756251881Speter spin_lock(&kthread_create_lock); 757251881Speter while (!list_empty(&kthread_create_list)) { 758251881Speter struct kthread_create_info *create; 759251881Speter 760251881Speter create = list_entry(kthread_create_list.next, 761251881Speter struct kthread_create_info, list); 762251881Speter list_del_init(&create->list); 763251881Speter spin_unlock(&kthread_create_lock); 764251881Speter 765251881Speter create_kthread(create); 766251881Speter 767251881Speter spin_lock(&kthread_create_lock); 768251881Speter } 769251881Speter spin_unlock(&kthread_create_lock); 770251881Speter } 771251881Speter 772251881Speter return 0; 773251881Speter} 774251881Speter 775251881Spetervoid __kthread_init_worker(struct kthread_worker *worker, 776251881Speter const char *name, 777251881Speter struct lock_class_key *key) 778251881Speter{ 779251881Speter memset(worker, 0, sizeof(struct kthread_worker)); 780251881Speter raw_spin_lock_init(&worker->lock); 781251881Speter lockdep_set_class_and_name(&worker->lock, key, name); 782251881Speter INIT_LIST_HEAD(&worker->work_list); 783251881Speter INIT_LIST_HEAD(&worker->delayed_work_list); 784251881Speter} 785251881SpeterEXPORT_SYMBOL_GPL(__kthread_init_worker); 786251881Speter 787251881Speter/** 788251881Speter * kthread_worker_fn - kthread function to process kthread_worker 789251881Speter * @worker_ptr: pointer to initialized kthread_worker 790251881Speter * 791251881Speter * This function implements the main cycle of kthread worker. It processes 792251881Speter * work_list until it is stopped with kthread_stop(). It sleeps when the queue 793251881Speter * is empty. 794251881Speter * 795251881Speter * The works are not allowed to keep any locks, disable preemption or interrupts 796251881Speter * when they finish. There is defined a safe point for freezing when one work 797251881Speter * finishes and before a new one is started. 798251881Speter * 799251881Speter * Also the works must not be handled by more than one worker at the same time, 800251881Speter * see also kthread_queue_work(). 801251881Speter */ 802251881Speterint kthread_worker_fn(void *worker_ptr) 803251881Speter{ 804251881Speter struct kthread_worker *worker = worker_ptr; 805251881Speter struct kthread_work *work; 806251881Speter 807251881Speter /* 808251881Speter * FIXME: Update the check and remove the assignment when all kthread 809251881Speter * worker users are created using kthread_create_worker*() functions. 810251881Speter */ 811251881Speter WARN_ON(worker->task && worker->task != current); 812251881Speter worker->task = current; 813251881Speter 814251881Speter if (worker->flags & KTW_FREEZABLE) 815251881Speter set_freezable(); 816251881Speter 817251881Speterrepeat: 818251881Speter set_current_state(TASK_INTERRUPTIBLE); /* mb paired w/ kthread_stop */ 819251881Speter 820251881Speter if (kthread_should_stop()) { 821251881Speter __set_current_state(TASK_RUNNING); 822251881Speter raw_spin_lock_irq(&worker->lock); 823289180Speter worker->task = NULL; 824251881Speter raw_spin_unlock_irq(&worker->lock); 825251881Speter return 0; 826251881Speter } 827251881Speter 828289180Speter work = NULL; 829251881Speter raw_spin_lock_irq(&worker->lock); 830251881Speter if (!list_empty(&worker->work_list)) { 831251881Speter work = list_first_entry(&worker->work_list, 832251881Speter struct kthread_work, node); 833251881Speter list_del_init(&work->node); 834251881Speter } 835251881Speter worker->current_work = work; 836251881Speter raw_spin_unlock_irq(&worker->lock); 837251881Speter 838251881Speter if (work) { 839289180Speter kthread_work_func_t func = work->func; 840289180Speter __set_current_state(TASK_RUNNING); 841289180Speter trace_sched_kthread_work_execute_start(work); 842289180Speter work->func(work); 843289180Speter /* 844289180Speter * Avoid dereferencing work after this point. The trace 845289180Speter * event only cares about the address. 846289180Speter */ 847289180Speter trace_sched_kthread_work_execute_end(work, func); 848289180Speter } else if (!freezing(current)) 849362181Sdim schedule(); 850289180Speter 851289180Speter try_to_freeze(); 852289180Speter cond_resched(); 853289180Speter goto repeat; 854289180Speter} 855289180SpeterEXPORT_SYMBOL_GPL(kthread_worker_fn); 856289180Speter 857289180Speterstatic __printf(3, 0) struct kthread_worker * 858289180Speter__kthread_create_worker(int cpu, unsigned int flags, 859289180Speter const char namefmt[], va_list args) 860289180Speter{ 861289180Speter struct kthread_worker *worker; 862289180Speter struct task_struct *task; 863289180Speter int node = NUMA_NO_NODE; 864289180Speter 865289180Speter worker = kzalloc(sizeof(*worker), GFP_KERNEL); 866289180Speter if (!worker) 867289180Speter return ERR_PTR(-ENOMEM); 868289180Speter 869289180Speter kthread_init_worker(worker); 870289180Speter 871289180Speter if (cpu >= 0) 872289180Speter node = cpu_to_node(cpu); 873289180Speter 874362181Sdim task = __kthread_create_on_node(kthread_worker_fn, worker, 875289180Speter node, namefmt, args); 876289180Speter if (IS_ERR(task)) 877289180Speter goto fail_task; 878289180Speter 879289180Speter if (cpu >= 0) 880289180Speter kthread_bind(task, cpu); 881289180Speter 882289180Speter worker->flags = flags; 883289180Speter worker->task = task; 884289180Speter wake_up_process(task); 885289180Speter return worker; 886289180Speter 887289180Speterfail_task: 888289180Speter kfree(worker); 889289180Speter return ERR_CAST(task); 890289180Speter} 891289180Speter 892289180Speter/** 893289180Speter * kthread_create_worker - create a kthread worker 894289180Speter * @flags: flags modifying the default behavior of the worker 895289180Speter * @namefmt: printf-style name for the kthread worker (task). 896289180Speter * 897289180Speter * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM) 898289180Speter * when the needed structures could not get allocated, and ERR_PTR(-EINTR) 899289180Speter * when the caller was killed by a fatal signal. 900289180Speter */ 901289180Speterstruct kthread_worker * 902289180Speterkthread_create_worker(unsigned int flags, const char namefmt[], ...) 903289180Speter{ 904289180Speter struct kthread_worker *worker; 905289180Speter va_list args; 906289180Speter 907289180Speter va_start(args, namefmt); 908362181Sdim worker = __kthread_create_worker(-1, flags, namefmt, args); 909289180Speter va_end(args); 910289180Speter 911289180Speter return worker; 912289180Speter} 913289180SpeterEXPORT_SYMBOL(kthread_create_worker); 914289180Speter 915289180Speter/** 916289180Speter * kthread_create_worker_on_cpu - create a kthread worker and bind it 917289180Speter * to a given CPU and the associated NUMA node. 918289180Speter * @cpu: CPU number 919289180Speter * @flags: flags modifying the default behavior of the worker 920289180Speter * @namefmt: printf-style name for the kthread worker (task). 921289180Speter * 922289180Speter * Use a valid CPU number if you want to bind the kthread worker 923289180Speter * to the given CPU and the associated NUMA node. 924289180Speter * 925289180Speter * A good practice is to add the cpu number also into the worker name. 926289180Speter * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu). 927289180Speter * 928289180Speter * CPU hotplug: 929289180Speter * The kthread worker API is simple and generic. It just provides a way 930289180Speter * to create, use, and destroy workers. 931289180Speter * 932289180Speter * It is up to the API user how to handle CPU hotplug. They have to decide 933289180Speter * how to handle pending work items, prevent queuing new ones, and 934289180Speter * restore the functionality when the CPU goes off and on. There are a 935289180Speter * few catches: 936289180Speter * 937289180Speter * - CPU affinity gets lost when it is scheduled on an offline CPU. 938289180Speter * 939289180Speter * - The worker might not exist when the CPU was off when the user 940289180Speter * created the workers. 941289180Speter * 942289180Speter * Good practice is to implement two CPU hotplug callbacks and to 943289180Speter * destroy/create the worker when the CPU goes down/up. 944289180Speter * 945289180Speter * Return: 946289180Speter * The pointer to the allocated worker on success, ERR_PTR(-ENOMEM) 947289180Speter * when the needed structures could not get allocated, and ERR_PTR(-EINTR) 948289180Speter * when the caller was killed by a fatal signal. 949289180Speter */ 950251881Speterstruct kthread_worker * 951251881Speterkthread_create_worker_on_cpu(int cpu, unsigned int flags, 952251881Speter const char namefmt[], ...) 953289180Speter{ 954251881Speter struct kthread_worker *worker; 955251881Speter va_list args; 956251881Speter 957251881Speter va_start(args, namefmt); 958251881Speter worker = __kthread_create_worker(cpu, flags, namefmt, args); 959251881Speter va_end(args); 960251881Speter 961251881Speter return worker; 962251881Speter} 963251881SpeterEXPORT_SYMBOL(kthread_create_worker_on_cpu); 964251881Speter 965251881Speter/* 966251881Speter * Returns true when the work could not be queued at the moment. 967289180Speter * It happens when it is already pending in a worker list 968289180Speter * or when it is being cancelled. 969251881Speter */ 970251881Speterstatic inline bool queuing_blocked(struct kthread_worker *worker, 971251881Speter struct kthread_work *work) 972251881Speter{ 973251881Speter lockdep_assert_held(&worker->lock); 974251881Speter 975251881Speter return !list_empty(&work->node) || work->canceling; 976251881Speter} 977289180Speter 978289180Speterstatic void kthread_insert_work_sanity_check(struct kthread_worker *worker, 979289180Speter struct kthread_work *work) 980289180Speter{ 981289180Speter lockdep_assert_held(&worker->lock); 982289180Speter WARN_ON_ONCE(!list_empty(&work->node)); 983289180Speter /* Do not use a work with >1 worker, see kthread_queue_work() */ 984251881Speter WARN_ON_ONCE(work->worker && work->worker != worker); 985251881Speter} 986251881Speter 987251881Speter/* insert @work before @pos in @worker */ 988251881Speterstatic void kthread_insert_work(struct kthread_worker *worker, 989251881Speter struct kthread_work *work, 990251881Speter struct list_head *pos) 991251881Speter{ 992251881Speter kthread_insert_work_sanity_check(worker, work); 993251881Speter 994251881Speter trace_sched_kthread_work_queue_work(worker, work); 995251881Speter 996251881Speter list_add_tail(&work->node, pos); 997251881Speter work->worker = worker; 998251881Speter if (!worker->current_work && likely(worker->task)) 999251881Speter wake_up_process(worker->task); 1000251881Speter} 1001251881Speter 1002251881Speter/** 1003251881Speter * kthread_queue_work - queue a kthread_work 1004251881Speter * @worker: target kthread_worker 1005251881Speter * @work: kthread_work to queue 1006251881Speter * 1007251881Speter * Queue @work to work processor @task for async execution. @task 1008251881Speter * must have been created with kthread_worker_create(). Returns %true 1009251881Speter * if @work was successfully queued, %false if it was already pending. 1010251881Speter * 1011251881Speter * Reinitialize the work if it needs to be used by another worker. 1012251881Speter * For example, when the worker was stopped and started again. 1013251881Speter */ 1014251881Speterbool kthread_queue_work(struct kthread_worker *worker, 1015251881Speter struct kthread_work *work) 1016251881Speter{ 1017251881Speter bool ret = false; 1018251881Speter unsigned long flags; 1019251881Speter 1020251881Speter raw_spin_lock_irqsave(&worker->lock, flags); 1021251881Speter if (!queuing_blocked(worker, work)) { 1022251881Speter kthread_insert_work(worker, work, &worker->work_list); 1023251881Speter ret = true; 1024251881Speter } 1025251881Speter raw_spin_unlock_irqrestore(&worker->lock, flags); 1026251881Speter return ret; 1027251881Speter} 1028251881SpeterEXPORT_SYMBOL_GPL(kthread_queue_work); 1029251881Speter 1030251881Speter/** 1031251881Speter * kthread_delayed_work_timer_fn - callback that queues the associated kthread 1032251881Speter * delayed work when the timer expires. 1033251881Speter * @t: pointer to the expired timer 1034251881Speter * 1035251881Speter * The format of the function is defined by struct timer_list. 1036251881Speter * It should have been called from irqsafe timer with irq already off. 1037251881Speter */ 1038251881Spetervoid kthread_delayed_work_timer_fn(struct timer_list *t) 1039251881Speter{ 1040251881Speter struct kthread_delayed_work *dwork = from_timer(dwork, t, timer); 1041251881Speter struct kthread_work *work = &dwork->work; 1042251881Speter struct kthread_worker *worker = work->worker; 1043251881Speter unsigned long flags; 1044251881Speter 1045251881Speter /* 1046251881Speter * This might happen when a pending work is reinitialized. 1047251881Speter * It means that it is used a wrong way. 1048251881Speter */ 1049251881Speter if (WARN_ON_ONCE(!worker)) 1050251881Speter return; 1051251881Speter 1052251881Speter raw_spin_lock_irqsave(&worker->lock, flags); 1053251881Speter /* Work must not be used with >1 worker, see kthread_queue_work(). */ 1054251881Speter WARN_ON_ONCE(work->worker != worker); 1055251881Speter 1056251881Speter /* Move the work from worker->delayed_work_list. */ 1057251881Speter WARN_ON_ONCE(list_empty(&work->node)); 1058251881Speter list_del_init(&work->node); 1059251881Speter if (!work->canceling) 1060251881Speter kthread_insert_work(worker, work, &worker->work_list); 1061251881Speter 1062251881Speter raw_spin_unlock_irqrestore(&worker->lock, flags); 1063251881Speter} 1064251881SpeterEXPORT_SYMBOL(kthread_delayed_work_timer_fn); 1065251881Speter 1066251881Speterstatic void __kthread_queue_delayed_work(struct kthread_worker *worker, 1067251881Speter struct kthread_delayed_work *dwork, 1068251881Speter unsigned long delay) 1069251881Speter{ 1070251881Speter struct timer_list *timer = &dwork->timer; 1071251881Speter struct kthread_work *work = &dwork->work; 1072251881Speter 1073251881Speter WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn); 1074251881Speter 1075251881Speter /* 1076251881Speter * If @delay is 0, queue @dwork->work immediately. This is for 1077251881Speter * both optimization and correctness. The earliest @timer can 1078251881Speter * expire is on the closest next tick and delayed_work users depend 1079251881Speter * on that there's no such delay when @delay is 0. 1080251881Speter */ 1081251881Speter if (!delay) { 1082251881Speter kthread_insert_work(worker, work, &worker->work_list); 1083251881Speter return; 1084251881Speter } 1085251881Speter 1086251881Speter /* Be paranoid and try to detect possible races already now. */ 1087251881Speter kthread_insert_work_sanity_check(worker, work); 1088251881Speter 1089251881Speter list_add(&work->node, &worker->delayed_work_list); 1090251881Speter work->worker = worker; 1091251881Speter timer->expires = jiffies + delay; 1092251881Speter add_timer(timer); 1093251881Speter} 1094251881Speter 1095251881Speter/** 1096251881Speter * kthread_queue_delayed_work - queue the associated kthread work 1097251881Speter * after a delay. 1098251881Speter * @worker: target kthread_worker 1099251881Speter * @dwork: kthread_delayed_work to queue 1100251881Speter * @delay: number of jiffies to wait before queuing 1101251881Speter * 1102251881Speter * If the work has not been pending it starts a timer that will queue 1103251881Speter * the work after the given @delay. If @delay is zero, it queues the 1104251881Speter * work immediately. 1105251881Speter * 1106251881Speter * Return: %false if the @work has already been pending. It means that 1107251881Speter * either the timer was running or the work was queued. It returns %true 1108251881Speter * otherwise. 1109251881Speter */ 1110251881Speterbool kthread_queue_delayed_work(struct kthread_worker *worker, 1111251881Speter struct kthread_delayed_work *dwork, 1112251881Speter unsigned long delay) 1113251881Speter{ 1114251881Speter struct kthread_work *work = &dwork->work; 1115251881Speter unsigned long flags; 1116251881Speter bool ret = false; 1117251881Speter 1118251881Speter raw_spin_lock_irqsave(&worker->lock, flags); 1119251881Speter 1120251881Speter if (!queuing_blocked(worker, work)) { 1121251881Speter __kthread_queue_delayed_work(worker, dwork, delay); 1122251881Speter ret = true; 1123251881Speter } 1124251881Speter 1125251881Speter raw_spin_unlock_irqrestore(&worker->lock, flags); 1126251881Speter return ret; 1127251881Speter} 1128251881SpeterEXPORT_SYMBOL_GPL(kthread_queue_delayed_work); 1129251881Speter 1130251881Speterstruct kthread_flush_work { 1131251881Speter struct kthread_work work; 1132251881Speter struct completion done; 1133251881Speter}; 1134251881Speter 1135251881Speterstatic void kthread_flush_work_fn(struct kthread_work *work) 1136251881Speter{ 1137251881Speter struct kthread_flush_work *fwork = 1138251881Speter container_of(work, struct kthread_flush_work, work); 1139251881Speter complete(&fwork->done); 1140251881Speter} 1141251881Speter 1142251881Speter/** 1143251881Speter * kthread_flush_work - flush a kthread_work 1144251881Speter * @work: work to flush 1145251881Speter * 1146251881Speter * If @work is queued or executing, wait for it to finish execution. 1147251881Speter */ 1148251881Spetervoid kthread_flush_work(struct kthread_work *work) 1149251881Speter{ 1150251881Speter struct kthread_flush_work fwork = { 1151251881Speter KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 1152251881Speter COMPLETION_INITIALIZER_ONSTACK(fwork.done), 1153251881Speter }; 1154251881Speter struct kthread_worker *worker; 1155251881Speter bool noop = false; 1156251881Speter 1157251881Speter worker = work->worker; 1158251881Speter if (!worker) 1159251881Speter return; 1160251881Speter 1161251881Speter raw_spin_lock_irq(&worker->lock); 1162251881Speter /* Work must not be used with >1 worker, see kthread_queue_work(). */ 1163251881Speter WARN_ON_ONCE(work->worker != worker); 1164251881Speter 1165251881Speter if (!list_empty(&work->node)) 1166251881Speter kthread_insert_work(worker, &fwork.work, work->node.next); 1167251881Speter else if (worker->current_work == work) 1168251881Speter kthread_insert_work(worker, &fwork.work, 1169251881Speter worker->work_list.next); 1170251881Speter else 1171251881Speter noop = true; 1172251881Speter 1173251881Speter raw_spin_unlock_irq(&worker->lock); 1174251881Speter 1175289180Speter if (!noop) 1176251881Speter wait_for_completion(&fwork.done); 1177251881Speter} 1178251881SpeterEXPORT_SYMBOL_GPL(kthread_flush_work); 1179251881Speter 1180251881Speter/* 1181251881Speter * Make sure that the timer is neither set nor running and could 1182251881Speter * not manipulate the work list_head any longer. 1183251881Speter * 1184251881Speter * The function is called under worker->lock. The lock is temporary 1185251881Speter * released but the timer can't be set again in the meantime. 1186251881Speter */ 1187251881Speterstatic void kthread_cancel_delayed_work_timer(struct kthread_work *work, 1188251881Speter unsigned long *flags) 1189251881Speter{ 1190251881Speter struct kthread_delayed_work *dwork = 1191251881Speter container_of(work, struct kthread_delayed_work, work); 1192251881Speter struct kthread_worker *worker = work->worker; 1193251881Speter 1194251881Speter /* 1195251881Speter * del_timer_sync() must be called to make sure that the timer 1196362181Sdim * callback is not running. The lock must be temporary released 1197362181Sdim * to avoid a deadlock with the callback. In the meantime, 1198362181Sdim * any queuing is blocked by setting the canceling counter. 1199362181Sdim */ 1200362181Sdim work->canceling++; 1201362181Sdim raw_spin_unlock_irqrestore(&worker->lock, *flags); 1202362181Sdim del_timer_sync(&dwork->timer); 1203362181Sdim raw_spin_lock_irqsave(&worker->lock, *flags); 1204362181Sdim work->canceling--; 1205362181Sdim} 1206362181Sdim 1207362181Sdim/* 1208362181Sdim * This function removes the work from the worker queue. 1209362181Sdim * 1210362181Sdim * It is called under worker->lock. The caller must make sure that 1211362181Sdim * the timer used by delayed work is not running, e.g. by calling 1212362181Sdim * kthread_cancel_delayed_work_timer(). 1213362181Sdim * 1214362181Sdim * The work might still be in use when this function finishes. See the 1215362181Sdim * current_work proceed by the worker. 1216362181Sdim * 1217362181Sdim * Return: %true if @work was pending and successfully canceled, 1218362181Sdim * %false if @work was not pending 1219362181Sdim */ 1220362181Sdimstatic bool __kthread_cancel_work(struct kthread_work *work) 1221362181Sdim{ 1222362181Sdim /* 1223362181Sdim * Try to remove the work from a worker list. It might either 1224362181Sdim * be from worker->work_list or from worker->delayed_work_list. 1225362181Sdim */ 1226251881Speter if (!list_empty(&work->node)) { 1227251881Speter list_del_init(&work->node); 1228251881Speter return true; 1229251881Speter } 1230251881Speter 1231251881Speter return false; 1232251881Speter} 1233251881Speter 1234362181Sdim/** 1235251881Speter * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work 1236251881Speter * @worker: kthread worker to use 1237251881Speter * @dwork: kthread delayed work to queue 1238251881Speter * @delay: number of jiffies to wait before queuing 1239251881Speter * 1240251881Speter * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise, 1241251881Speter * modify @dwork's timer so that it expires after @delay. If @delay is zero, 1242251881Speter * @work is guaranteed to be queued immediately. 1243362181Sdim * 1244369302Sdim * Return: %false if @dwork was idle and queued, %true otherwise. 1245369302Sdim * 1246369302Sdim * A special case is when the work is being canceled in parallel. 1247251881Speter * It might be caused either by the real kthread_cancel_delayed_work_sync() 1248251881Speter * or yet another kthread_mod_delayed_work() call. We let the other command 1249251881Speter * win and return %true here. The return value can be used for reference 1250251881Speter * counting and the number of queued works stays the same. Anyway, the caller 1251251881Speter * is supposed to synchronize these operations a reasonable way. 1252369302Sdim * 1253369302Sdim * This function is safe to call from any context including IRQ handler. 1254251881Speter * See __kthread_cancel_work() and kthread_delayed_work_timer_fn() 1255251881Speter * for details. 1256369302Sdim */ 1257251881Speterbool kthread_mod_delayed_work(struct kthread_worker *worker, 1258369302Sdim struct kthread_delayed_work *dwork, 1259251881Speter unsigned long delay) 1260251881Speter{ 1261369302Sdim struct kthread_work *work = &dwork->work; 1262369302Sdim unsigned long flags; 1263369302Sdim int ret; 1264369302Sdim 1265369302Sdim raw_spin_lock_irqsave(&worker->lock, flags); 1266369302Sdim 1267369302Sdim /* Do not bother with canceling when never queued. */ 1268369302Sdim if (!work->worker) { 1269369302Sdim ret = false; 1270369302Sdim goto fast_queue; 1271369302Sdim } 1272369302Sdim 1273369302Sdim /* Work must not be used with >1 worker, see kthread_queue_work() */ 1274251881Speter WARN_ON_ONCE(work->worker != worker); 1275369302Sdim 1276251881Speter /* 1277251881Speter * Temporary cancel the work but do not fight with another command 1278251881Speter * that is canceling the work as well. 1279369302Sdim * 1280369302Sdim * It is a bit tricky because of possible races with another 1281369302Sdim * mod_delayed_work() and cancel_delayed_work() callers. 1282369302Sdim * 1283369302Sdim * The timer must be canceled first because worker->lock is released 1284369302Sdim * when doing so. But the work can be removed from the queue (list) 1285369302Sdim * only when it can be queued again so that the return value can 1286251881Speter * be used for reference counting. 1287251881Speter */ 1288251881Speter kthread_cancel_delayed_work_timer(work, &flags); 1289251881Speter if (work->canceling) { 1290251881Speter /* The number of works in the queue does not change. */ 1291369302Sdim ret = true; 1292251881Speter goto out; 1293369302Sdim } 1294369302Sdim ret = __kthread_cancel_work(work); 1295369302Sdim 1296369302Sdimfast_queue: 1297369302Sdim __kthread_queue_delayed_work(worker, dwork, delay); 1298369302Sdimout: 1299369302Sdim raw_spin_unlock_irqrestore(&worker->lock, flags); 1300369302Sdim return ret; 1301369302Sdim} 1302369302SdimEXPORT_SYMBOL_GPL(kthread_mod_delayed_work); 1303251881Speter 1304251881Speterstatic bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork) 1305251881Speter{ 1306251881Speter struct kthread_worker *worker = work->worker; 1307369302Sdim unsigned long flags; 1308369302Sdim int ret = false; 1309369302Sdim 1310369302Sdim if (!worker) 1311369302Sdim goto out; 1312369302Sdim 1313369302Sdim raw_spin_lock_irqsave(&worker->lock, flags); 1314251881Speter /* Work must not be used with >1 worker, see kthread_queue_work(). */ 1315369302Sdim WARN_ON_ONCE(work->worker != worker); 1316369302Sdim 1317369302Sdim if (is_dwork) 1318369302Sdim kthread_cancel_delayed_work_timer(work, &flags); 1319369302Sdim 1320369302Sdim ret = __kthread_cancel_work(work); 1321369302Sdim 1322251881Speter if (worker->current_work != work) 1323251881Speter goto out_fast; 1324251881Speter 1325251881Speter /* 1326251881Speter * The work is in progress and we need to wait with the lock released. 1327251881Speter * In the meantime, block any queuing by setting the canceling counter. 1328251881Speter */ 1329251881Speter work->canceling++; 1330251881Speter raw_spin_unlock_irqrestore(&worker->lock, flags); 1331251881Speter kthread_flush_work(work); 1332251881Speter raw_spin_lock_irqsave(&worker->lock, flags); 1333251881Speter work->canceling--; 1334251881Speter 1335251881Speterout_fast: 1336251881Speter raw_spin_unlock_irqrestore(&worker->lock, flags); 1337251881Speterout: 1338251881Speter return ret; 1339251881Speter} 1340251881Speter 1341251881Speter/** 1342251881Speter * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish 1343251881Speter * @work: the kthread work to cancel 1344251881Speter * 1345251881Speter * Cancel @work and wait for its execution to finish. This function 1346251881Speter * can be used even if the work re-queues itself. On return from this 1347251881Speter * function, @work is guaranteed to be not pending or executing on any CPU. 1348251881Speter * 1349251881Speter * kthread_cancel_work_sync(&delayed_work->work) must not be used for 1350251881Speter * delayed_work's. Use kthread_cancel_delayed_work_sync() instead. 1351251881Speter * 1352251881Speter * The caller must ensure that the worker on which @work was last 1353251881Speter * queued can't be destroyed before this function returns. 1354251881Speter * 1355362181Sdim * Return: %true if @work was pending, %false otherwise. 1356362181Sdim */ 1357362181Sdimbool kthread_cancel_work_sync(struct kthread_work *work) 1358362181Sdim{ 1359362181Sdim return __kthread_cancel_work_sync(work, false); 1360362181Sdim} 1361251881SpeterEXPORT_SYMBOL_GPL(kthread_cancel_work_sync); 1362362181Sdim 1363362181Sdim/** 1364362181Sdim * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and 1365362181Sdim * wait for it to finish. 1366362181Sdim * @dwork: the kthread delayed work to cancel 1367362181Sdim * 1368362181Sdim * This is kthread_cancel_work_sync() for delayed works. 1369362181Sdim * 1370362181Sdim * Return: %true if @dwork was pending, %false otherwise. 1371362181Sdim */ 1372362181Sdimbool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork) 1373362181Sdim{ 1374362181Sdim return __kthread_cancel_work_sync(&dwork->work, true); 1375362181Sdim} 1376362181SdimEXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync); 1377362181Sdim 1378362181Sdim/** 1379362181Sdim * kthread_flush_worker - flush all current works on a kthread_worker 1380362181Sdim * @worker: worker to flush 1381362181Sdim * 1382362181Sdim * Wait until all currently executing or pending works on @worker are 1383362181Sdim * finished. 1384362181Sdim */ 1385362181Sdimvoid kthread_flush_worker(struct kthread_worker *worker) 1386362181Sdim{ 1387362181Sdim struct kthread_flush_work fwork = { 1388362181Sdim KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn), 1389362181Sdim COMPLETION_INITIALIZER_ONSTACK(fwork.done), 1390362181Sdim }; 1391362181Sdim 1392362181Sdim kthread_queue_work(worker, &fwork.work); 1393362181Sdim wait_for_completion(&fwork.done); 1394362181Sdim} 1395362181SdimEXPORT_SYMBOL_GPL(kthread_flush_worker); 1396362181Sdim 1397362181Sdim/** 1398362181Sdim * kthread_destroy_worker - destroy a kthread worker 1399362181Sdim * @worker: worker to be destroyed 1400362181Sdim * 1401362181Sdim * Flush and destroy @worker. The simple flush is enough because the kthread 1402362181Sdim * worker API is used only in trivial scenarios. There are no multi-step state 1403362181Sdim * machines needed. 1404362181Sdim * 1405362181Sdim * Note that this function is not responsible for handling delayed work, so 1406362181Sdim * caller should be responsible for queuing or canceling all delayed work items 1407362181Sdim * before invoke this function. 1408362181Sdim */ 1409362181Sdimvoid kthread_destroy_worker(struct kthread_worker *worker) 1410362181Sdim{ 1411362181Sdim struct task_struct *task; 1412362181Sdim 1413362181Sdim task = worker->task; 1414362181Sdim if (WARN_ON(!task)) 1415362181Sdim return; 1416362181Sdim 1417362181Sdim kthread_flush_worker(worker); 1418362181Sdim kthread_stop(task); 1419362181Sdim WARN_ON(!list_empty(&worker->delayed_work_list)); 1420362181Sdim WARN_ON(!list_empty(&worker->work_list)); 1421362181Sdim kfree(worker); 1422362181Sdim} 1423362181SdimEXPORT_SYMBOL(kthread_destroy_worker); 1424362181Sdim 1425362181Sdim/** 1426362181Sdim * kthread_use_mm - make the calling kthread operate on an address space 1427362181Sdim * @mm: address space to operate on 1428362181Sdim */ 1429362181Sdimvoid kthread_use_mm(struct mm_struct *mm) 1430362181Sdim{ 1431362181Sdim struct mm_struct *active_mm; 1432362181Sdim struct task_struct *tsk = current; 1433362181Sdim 1434362181Sdim WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); 1435362181Sdim WARN_ON_ONCE(tsk->mm); 1436362181Sdim 1437362181Sdim /* 1438362181Sdim * It is possible for mm to be the same as tsk->active_mm, but 1439362181Sdim * we must still mmgrab(mm) and mmdrop_lazy_tlb(active_mm), 1440362181Sdim * because these references are not equivalent. 1441362181Sdim */ 1442362181Sdim mmgrab(mm); 1443362181Sdim 1444362181Sdim task_lock(tsk); 1445362181Sdim /* Hold off tlb flush IPIs while switching mm's */ 1446362181Sdim local_irq_disable(); 1447362181Sdim active_mm = tsk->active_mm; 1448251881Speter tsk->active_mm = mm; 1449251881Speter tsk->mm = mm; 1450251881Speter membarrier_update_current_mm(mm); 1451251881Speter switch_mm_irqs_off(active_mm, mm, tsk); 1452251881Speter local_irq_enable(); 1453251881Speter task_unlock(tsk); 1454251881Speter#ifdef finish_arch_post_lock_switch 1455369302Sdim finish_arch_post_lock_switch(); 1456369302Sdim#endif 1457369302Sdim 1458369302Sdim /* 1459251881Speter * When a kthread starts operating on an address space, the loop 1460251881Speter * in membarrier_{private,global}_expedited() may not observe 1461251881Speter * that tsk->mm, and not issue an IPI. Membarrier requires a 1462251881Speter * memory barrier after storing to tsk->mm, before accessing 1463251881Speter * user-space memory. A full memory barrier for membarrier 1464251881Speter * {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by 1465369302Sdim * mmdrop_lazy_tlb(). 1466251881Speter */ 1467251881Speter mmdrop_lazy_tlb(active_mm); 1468251881Speter} 1469251881SpeterEXPORT_SYMBOL_GPL(kthread_use_mm); 1470251881Speter 1471251881Speter/** 1472251881Speter * kthread_unuse_mm - reverse the effect of kthread_use_mm() 1473251881Speter * @mm: address space to operate on 1474251881Speter */ 1475251881Spetervoid kthread_unuse_mm(struct mm_struct *mm) 1476251881Speter{ 1477251881Speter struct task_struct *tsk = current; 1478251881Speter 1479251881Speter WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); 1480251881Speter WARN_ON_ONCE(!tsk->mm); 1481251881Speter 1482369302Sdim task_lock(tsk); 1483369302Sdim /* 1484362181Sdim * When a kthread stops operating on an address space, the loop 1485362181Sdim * in membarrier_{private,global}_expedited() may not observe 1486369302Sdim * that tsk->mm, and not issue an IPI. Membarrier requires a 1487369302Sdim * memory barrier after accessing user-space memory, before 1488251881Speter * clearing tsk->mm. 1489369302Sdim */ 1490369302Sdim smp_mb__after_spinlock(); 1491369302Sdim local_irq_disable(); 1492369302Sdim tsk->mm = NULL; 1493369302Sdim membarrier_update_current_mm(NULL); 1494251881Speter mmgrab_lazy_tlb(mm); 1495251881Speter /* active_mm is still 'mm' */ 1496251881Speter enter_lazy_tlb(mm, tsk); 1497251881Speter local_irq_enable(); 1498251881Speter task_unlock(tsk); 1499251881Speter 1500251881Speter mmdrop(mm); 1501251881Speter} 1502251881SpeterEXPORT_SYMBOL_GPL(kthread_unuse_mm); 1503251881Speter 1504251881Speter#ifdef CONFIG_BLK_CGROUP 1505251881Speter/** 1506251881Speter * kthread_associate_blkcg - associate blkcg to current kthread 1507251881Speter * @css: the cgroup info 1508251881Speter * 1509251881Speter * Current thread must be a kthread. The thread is running jobs on behalf of 1510251881Speter * other threads. In some cases, we expect the jobs attach cgroup info of 1511251881Speter * original threads instead of that of current thread. This function stores 1512251881Speter * original thread's cgroup info in current kthread context for later 1513251881Speter * retrieval. 1514251881Speter */ 1515251881Spetervoid kthread_associate_blkcg(struct cgroup_subsys_state *css) 1516251881Speter{ 1517251881Speter struct kthread *kthread; 1518251881Speter 1519251881Speter if (!(current->flags & PF_KTHREAD)) 1520251881Speter return; 1521251881Speter kthread = to_kthread(current); 1522251881Speter if (!kthread) 1523251881Speter return; 1524251881Speter 1525369302Sdim if (kthread->blkcg_css) { 1526369302Sdim css_put(kthread->blkcg_css); 1527369302Sdim kthread->blkcg_css = NULL; 1528251881Speter } 1529251881Speter if (css) { 1530251881Speter css_get(css); 1531362181Sdim kthread->blkcg_css = css; 1532251881Speter } 1533362181Sdim} 1534251881SpeterEXPORT_SYMBOL(kthread_associate_blkcg); 1535251881Speter 1536362181Sdim/** 1537251881Speter * kthread_blkcg - get associated blkcg css of current kthread 1538251881Speter * 1539251881Speter * Current thread must be a kthread. 1540251881Speter */ 1541369302Sdimstruct cgroup_subsys_state *kthread_blkcg(void) 1542251881Speter{ 1543251881Speter struct kthread *kthread; 1544251881Speter 1545251881Speter if (current->flags & PF_KTHREAD) { 1546251881Speter kthread = to_kthread(current); 1547251881Speter if (kthread) 1548251881Speter return kthread->blkcg_css; 1549251881Speter } 1550251881Speter return NULL; 1551251881Speter} 1552251881Speter#endif 1553251881Speter