summaryrefslogtreecommitdiff
path: root/kernel/cpu.c (plain)
blob: fe604bbc57f2542eddad101d19f4f8ee058470be
1/* CPU control.
2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 *
4 * This code is licenced under the GPL.
5 */
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/sched/smt.h>
12#include <linux/unistd.h>
13#include <linux/cpu.h>
14#include <linux/oom.h>
15#include <linux/rcupdate.h>
16#include <linux/export.h>
17#include <linux/bug.h>
18#include <linux/kthread.h>
19#include <linux/stop_machine.h>
20#include <linux/mutex.h>
21#include <linux/gfp.h>
22#include <linux/suspend.h>
23#include <linux/lockdep.h>
24#include <linux/tick.h>
25#include <linux/irq.h>
26#include <linux/smpboot.h>
27#include <linux/relay.h>
28#include <linux/slab.h>
29
30#include <trace/events/power.h>
31#define CREATE_TRACE_POINTS
32#include <trace/events/cpuhp.h>
33
34#include "smpboot.h"
35
36/**
37 * cpuhp_cpu_state - Per cpu hotplug state storage
38 * @state: The current cpu state
39 * @target: The target state
40 * @thread: Pointer to the hotplug thread
41 * @should_run: Thread should execute
42 * @rollback: Perform a rollback
43 * @single: Single callback invocation
44 * @bringup: Single callback bringup or teardown selector
45 * @cb_state: The state for a single callback (install/uninstall)
46 * @result: Result of the operation
47 * @done: Signal completion to the issuer of the task
48 */
49struct cpuhp_cpu_state {
50 enum cpuhp_state state;
51 enum cpuhp_state target;
52#ifdef CONFIG_SMP
53 struct task_struct *thread;
54 bool should_run;
55 bool rollback;
56 bool single;
57 bool bringup;
58 bool booted_once;
59 struct hlist_node *node;
60 enum cpuhp_state cb_state;
61 int result;
62 struct completion done;
63#endif
64};
65
66static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state);
67
68#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
69static struct lock_class_key cpuhp_state_key;
70static struct lockdep_map cpuhp_state_lock_map =
71 STATIC_LOCKDEP_MAP_INIT("cpuhp_state", &cpuhp_state_key);
72#endif
73
74/**
75 * cpuhp_step - Hotplug state machine step
76 * @name: Name of the step
77 * @startup: Startup function of the step
78 * @teardown: Teardown function of the step
79 * @skip_onerr: Do not invoke the functions on error rollback
80 * Will go away once the notifiers are gone
81 * @cant_stop: Bringup/teardown can't be stopped at this step
82 */
83struct cpuhp_step {
84 const char *name;
85 union {
86 int (*single)(unsigned int cpu);
87 int (*multi)(unsigned int cpu,
88 struct hlist_node *node);
89 } startup;
90 union {
91 int (*single)(unsigned int cpu);
92 int (*multi)(unsigned int cpu,
93 struct hlist_node *node);
94 } teardown;
95 struct hlist_head list;
96 bool skip_onerr;
97 bool cant_stop;
98 bool multi_instance;
99};
100
101static DEFINE_MUTEX(cpuhp_state_mutex);
102static struct cpuhp_step cpuhp_bp_states[];
103static struct cpuhp_step cpuhp_ap_states[];
104
105static bool cpuhp_is_ap_state(enum cpuhp_state state)
106{
107 /*
108 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
109 * purposes as that state is handled explicitly in cpu_down.
110 */
111 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
112}
113
114static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
115{
116 struct cpuhp_step *sp;
117
118 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
119 return sp + state;
120}
121
122/**
123 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
124 * @cpu: The cpu for which the callback should be invoked
125 * @step: The step in the state machine
126 * @bringup: True if the bringup callback should be invoked
127 *
128 * Called from cpu hotplug and from the state register machinery.
129 */
130static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
131 bool bringup, struct hlist_node *node)
132{
133 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
134 struct cpuhp_step *step = cpuhp_get_step(state);
135 int (*cbm)(unsigned int cpu, struct hlist_node *node);
136 int (*cb)(unsigned int cpu);
137 int ret, cnt;
138
139 if (!step->multi_instance) {
140 cb = bringup ? step->startup.single : step->teardown.single;
141 if (!cb)
142 return 0;
143 trace_cpuhp_enter(cpu, st->target, state, cb);
144 ret = cb(cpu);
145 trace_cpuhp_exit(cpu, st->state, state, ret);
146 return ret;
147 }
148 cbm = bringup ? step->startup.multi : step->teardown.multi;
149 if (!cbm)
150 return 0;
151
152 /* Single invocation for instance add/remove */
153 if (node) {
154 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
155 ret = cbm(cpu, node);
156 trace_cpuhp_exit(cpu, st->state, state, ret);
157 return ret;
158 }
159
160 /* State transition. Invoke on all instances */
161 cnt = 0;
162 hlist_for_each(node, &step->list) {
163 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
164 ret = cbm(cpu, node);
165 trace_cpuhp_exit(cpu, st->state, state, ret);
166 if (ret)
167 goto err;
168 cnt++;
169 }
170 return 0;
171err:
172 /* Rollback the instances if one failed */
173 cbm = !bringup ? step->startup.multi : step->teardown.multi;
174 if (!cbm)
175 return ret;
176
177 hlist_for_each(node, &step->list) {
178 if (!cnt--)
179 break;
180 cbm(cpu, node);
181 }
182 return ret;
183}
184
185#ifdef CONFIG_SMP
186/* Serializes the updates to cpu_online_mask, cpu_present_mask */
187static DEFINE_MUTEX(cpu_add_remove_lock);
188bool cpuhp_tasks_frozen;
189EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
190
191/*
192 * The following two APIs (cpu_maps_update_begin/done) must be used when
193 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
194 * The APIs cpu_notifier_register_begin/done() must be used to protect CPU
195 * hotplug callback (un)registration performed using __register_cpu_notifier()
196 * or __unregister_cpu_notifier().
197 */
198void cpu_maps_update_begin(void)
199{
200 mutex_lock(&cpu_add_remove_lock);
201}
202EXPORT_SYMBOL(cpu_notifier_register_begin);
203
204void cpu_maps_update_done(void)
205{
206 mutex_unlock(&cpu_add_remove_lock);
207}
208EXPORT_SYMBOL(cpu_notifier_register_done);
209
210static RAW_NOTIFIER_HEAD(cpu_chain);
211
212/* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
213 * Should always be manipulated under cpu_add_remove_lock
214 */
215static int cpu_hotplug_disabled;
216
217#ifdef CONFIG_HOTPLUG_CPU
218
219static struct {
220 struct task_struct *active_writer;
221 /* wait queue to wake up the active_writer */
222 wait_queue_head_t wq;
223 /* verifies that no writer will get active while readers are active */
224 struct mutex lock;
225 /*
226 * Also blocks the new readers during
227 * an ongoing cpu hotplug operation.
228 */
229 atomic_t refcount;
230
231#ifdef CONFIG_DEBUG_LOCK_ALLOC
232 struct lockdep_map dep_map;
233#endif
234} cpu_hotplug = {
235 .active_writer = NULL,
236 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
237 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
238#ifdef CONFIG_DEBUG_LOCK_ALLOC
239 .dep_map = STATIC_LOCKDEP_MAP_INIT("cpu_hotplug.dep_map", &cpu_hotplug.dep_map),
240#endif
241};
242
243/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */
244#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map)
245#define cpuhp_lock_acquire_tryread() \
246 lock_map_acquire_tryread(&cpu_hotplug.dep_map)
247#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
248#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
249
250
251void get_online_cpus(void)
252{
253 might_sleep();
254 if (cpu_hotplug.active_writer == current)
255 return;
256 cpuhp_lock_acquire_read();
257 mutex_lock(&cpu_hotplug.lock);
258 atomic_inc(&cpu_hotplug.refcount);
259 mutex_unlock(&cpu_hotplug.lock);
260}
261EXPORT_SYMBOL_GPL(get_online_cpus);
262
263void put_online_cpus(void)
264{
265 int refcount;
266
267 if (cpu_hotplug.active_writer == current)
268 return;
269
270 refcount = atomic_dec_return(&cpu_hotplug.refcount);
271 if (WARN_ON(refcount < 0)) /* try to fix things up */
272 atomic_inc(&cpu_hotplug.refcount);
273
274 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq))
275 wake_up(&cpu_hotplug.wq);
276
277 cpuhp_lock_release();
278
279}
280EXPORT_SYMBOL_GPL(put_online_cpus);
281
282/*
283 * This ensures that the hotplug operation can begin only when the
284 * refcount goes to zero.
285 *
286 * Note that during a cpu-hotplug operation, the new readers, if any,
287 * will be blocked by the cpu_hotplug.lock
288 *
289 * Since cpu_hotplug_begin() is always called after invoking
290 * cpu_maps_update_begin(), we can be sure that only one writer is active.
291 *
292 * Note that theoretically, there is a possibility of a livelock:
293 * - Refcount goes to zero, last reader wakes up the sleeping
294 * writer.
295 * - Last reader unlocks the cpu_hotplug.lock.
296 * - A new reader arrives at this moment, bumps up the refcount.
297 * - The writer acquires the cpu_hotplug.lock finds the refcount
298 * non zero and goes to sleep again.
299 *
300 * However, this is very difficult to achieve in practice since
301 * get_online_cpus() not an api which is called all that often.
302 *
303 */
304void cpu_hotplug_begin(void)
305{
306 DEFINE_WAIT(wait);
307
308 cpu_hotplug.active_writer = current;
309 cpuhp_lock_acquire();
310
311 for (;;) {
312 mutex_lock(&cpu_hotplug.lock);
313 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE);
314 if (likely(!atomic_read(&cpu_hotplug.refcount)))
315 break;
316 mutex_unlock(&cpu_hotplug.lock);
317 schedule();
318 }
319 finish_wait(&cpu_hotplug.wq, &wait);
320}
321
322void cpu_hotplug_done(void)
323{
324 cpu_hotplug.active_writer = NULL;
325 mutex_unlock(&cpu_hotplug.lock);
326 cpuhp_lock_release();
327}
328
329/*
330 * Wait for currently running CPU hotplug operations to complete (if any) and
331 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
332 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
333 * hotplug path before performing hotplug operations. So acquiring that lock
334 * guarantees mutual exclusion from any currently running hotplug operations.
335 */
336void cpu_hotplug_disable(void)
337{
338 cpu_maps_update_begin();
339 cpu_hotplug_disabled++;
340 cpu_maps_update_done();
341}
342EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
343
344static void __cpu_hotplug_enable(void)
345{
346 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
347 return;
348 cpu_hotplug_disabled--;
349}
350
351void cpu_hotplug_enable(void)
352{
353 cpu_maps_update_begin();
354 __cpu_hotplug_enable();
355 cpu_maps_update_done();
356}
357EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
358#endif /* CONFIG_HOTPLUG_CPU */
359
360/*
361 * Architectures that need SMT-specific errata handling during SMT hotplug
362 * should override this.
363 */
364void __weak arch_smt_update(void) { }
365
366#ifdef CONFIG_HOTPLUG_SMT
367enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
368EXPORT_SYMBOL_GPL(cpu_smt_control);
369
370static bool cpu_smt_available __read_mostly;
371
372void __init cpu_smt_disable(bool force)
373{
374 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
375 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
376 return;
377
378 if (force) {
379 pr_info("SMT: Force disabled\n");
380 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
381 } else {
382 cpu_smt_control = CPU_SMT_DISABLED;
383 }
384}
385
386/*
387 * The decision whether SMT is supported can only be done after the full
388 * CPU identification. Called from architecture code before non boot CPUs
389 * are brought up.
390 */
391void __init cpu_smt_check_topology_early(void)
392{
393 if (!topology_smt_supported())
394 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
395}
396
397/*
398 * If SMT was disabled by BIOS, detect it here, after the CPUs have been
399 * brought online. This ensures the smt/l1tf sysfs entries are consistent
400 * with reality. cpu_smt_available is set to true during the bringup of non
401 * boot CPUs when a SMT sibling is detected. Note, this may overwrite
402 * cpu_smt_control's previous setting.
403 */
404void __init cpu_smt_check_topology(void)
405{
406 if (!cpu_smt_available)
407 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
408}
409
410static int __init smt_cmdline_disable(char *str)
411{
412 cpu_smt_disable(str && !strcmp(str, "force"));
413 return 0;
414}
415early_param("nosmt", smt_cmdline_disable);
416
417static inline bool cpu_smt_allowed(unsigned int cpu)
418{
419 if (topology_is_primary_thread(cpu))
420 return true;
421
422 /*
423 * If the CPU is not a 'primary' thread and the booted_once bit is
424 * set then the processor has SMT support. Store this information
425 * for the late check of SMT support in cpu_smt_check_topology().
426 */
427 if (per_cpu(cpuhp_state, cpu).booted_once)
428 cpu_smt_available = true;
429
430 if (cpu_smt_control == CPU_SMT_ENABLED)
431 return true;
432
433 /*
434 * On x86 it's required to boot all logical CPUs at least once so
435 * that the init code can get a chance to set CR4.MCE on each
436 * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any
437 * core will shutdown the machine.
438 */
439 return !per_cpu(cpuhp_state, cpu).booted_once;
440}
441#else
442static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
443#endif
444
445/* Need to know about CPUs going up/down? */
446int register_cpu_notifier(struct notifier_block *nb)
447{
448 int ret;
449 cpu_maps_update_begin();
450 ret = raw_notifier_chain_register(&cpu_chain, nb);
451 cpu_maps_update_done();
452 return ret;
453}
454
455int __register_cpu_notifier(struct notifier_block *nb)
456{
457 return raw_notifier_chain_register(&cpu_chain, nb);
458}
459
460static int __cpu_notify(unsigned long val, unsigned int cpu, int nr_to_call,
461 int *nr_calls)
462{
463 unsigned long mod = cpuhp_tasks_frozen ? CPU_TASKS_FROZEN : 0;
464 void *hcpu = (void *)(long)cpu;
465
466 int ret;
467
468 ret = __raw_notifier_call_chain(&cpu_chain, val | mod, hcpu, nr_to_call,
469 nr_calls);
470
471 return notifier_to_errno(ret);
472}
473
474static int cpu_notify(unsigned long val, unsigned int cpu)
475{
476 return __cpu_notify(val, cpu, -1, NULL);
477}
478
479static void cpu_notify_nofail(unsigned long val, unsigned int cpu)
480{
481 BUG_ON(cpu_notify(val, cpu));
482}
483
484/* Notifier wrappers for transitioning to state machine */
485static int notify_prepare(unsigned int cpu)
486{
487 int nr_calls = 0;
488 int ret;
489
490 ret = __cpu_notify(CPU_UP_PREPARE, cpu, -1, &nr_calls);
491 if (ret) {
492 nr_calls--;
493 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
494 __func__, cpu);
495 __cpu_notify(CPU_UP_CANCELED, cpu, nr_calls, NULL);
496 }
497 return ret;
498}
499
500static int notify_online(unsigned int cpu)
501{
502 cpu_notify(CPU_ONLINE, cpu);
503 return 0;
504}
505
506static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st);
507
508static int bringup_wait_for_ap(unsigned int cpu)
509{
510 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
511
512 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
513 wait_for_completion(&st->done);
514 if (WARN_ON_ONCE((!cpu_online(cpu))))
515 return -ECANCELED;
516
517 /* Unpark the stopper thread and the hotplug thread of the target cpu */
518 stop_machine_unpark(cpu);
519 kthread_unpark(st->thread);
520
521 /*
522 * SMT soft disabling on X86 requires to bring the CPU out of the
523 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The
524 * CPU marked itself as booted_once in cpu_notify_starting() so the
525 * cpu_smt_allowed() check will now return false if this is not the
526 * primary sibling.
527 */
528 if (!cpu_smt_allowed(cpu))
529 return -ECANCELED;
530
531 /* Should we go further up ? */
532 if (st->target > CPUHP_AP_ONLINE_IDLE) {
533 __cpuhp_kick_ap_work(st);
534 wait_for_completion(&st->done);
535 }
536 return st->result;
537}
538
539static int bringup_cpu(unsigned int cpu)
540{
541 struct task_struct *idle = idle_thread_get(cpu);
542 int ret;
543
544 /*
545 * Some architectures have to walk the irq descriptors to
546 * setup the vector space for the cpu which comes online.
547 * Prevent irq alloc/free across the bringup.
548 */
549 irq_lock_sparse();
550
551 /* Arch-specific enabling code. */
552 ret = __cpu_up(cpu, idle);
553 irq_unlock_sparse();
554 if (ret) {
555 cpu_notify(CPU_UP_CANCELED, cpu);
556 return ret;
557 }
558 return bringup_wait_for_ap(cpu);
559}
560
561/*
562 * Hotplug state machine related functions
563 */
564static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
565{
566 for (st->state++; st->state < st->target; st->state++) {
567 struct cpuhp_step *step = cpuhp_get_step(st->state);
568
569 if (!step->skip_onerr)
570 cpuhp_invoke_callback(cpu, st->state, true, NULL);
571 }
572}
573
574static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
575 enum cpuhp_state target)
576{
577 enum cpuhp_state prev_state = st->state;
578 int ret = 0;
579
580 for (; st->state > target; st->state--) {
581 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL);
582 if (ret) {
583 st->target = prev_state;
584 undo_cpu_down(cpu, st);
585 break;
586 }
587 }
588 return ret;
589}
590
591static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
592{
593 for (st->state--; st->state > st->target; st->state--) {
594 struct cpuhp_step *step = cpuhp_get_step(st->state);
595
596 if (!step->skip_onerr)
597 cpuhp_invoke_callback(cpu, st->state, false, NULL);
598 }
599}
600
601static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
602{
603 if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
604 return true;
605 /*
606 * When CPU hotplug is disabled, then taking the CPU down is not
607 * possible because takedown_cpu() and the architecture and
608 * subsystem specific mechanisms are not available. So the CPU
609 * which would be completely unplugged again needs to stay around
610 * in the current state.
611 */
612 return st->state <= CPUHP_BRINGUP_CPU;
613}
614
615static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
616 enum cpuhp_state target)
617{
618 enum cpuhp_state prev_state = st->state;
619 int ret = 0;
620
621 while (st->state < target) {
622 st->state++;
623 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL);
624 if (ret) {
625 if (can_rollback_cpu(st)) {
626 st->target = prev_state;
627 undo_cpu_up(cpu, st);
628 }
629 break;
630 }
631 }
632 return ret;
633}
634
635/*
636 * The cpu hotplug threads manage the bringup and teardown of the cpus
637 */
638static void cpuhp_create(unsigned int cpu)
639{
640 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
641
642 init_completion(&st->done);
643}
644
645static int cpuhp_should_run(unsigned int cpu)
646{
647 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
648
649 return st->should_run;
650}
651
652/* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */
653static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st)
654{
655 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU);
656
657 return cpuhp_down_callbacks(cpu, st, target);
658}
659
660/* Execute the online startup callbacks. Used to be CPU_ONLINE */
661static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st)
662{
663 return cpuhp_up_callbacks(cpu, st, st->target);
664}
665
666/*
667 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
668 * callbacks when a state gets [un]installed at runtime.
669 */
670static void cpuhp_thread_fun(unsigned int cpu)
671{
672 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
673 int ret = 0;
674
675 /*
676 * Paired with the mb() in cpuhp_kick_ap_work and
677 * cpuhp_invoke_ap_callback, so the work set is consistent visible.
678 */
679 smp_mb();
680 if (!st->should_run)
681 return;
682
683 st->should_run = false;
684
685 lock_map_acquire(&cpuhp_state_lock_map);
686 /* Single callback invocation for [un]install ? */
687 if (st->single) {
688 if (st->cb_state < CPUHP_AP_ONLINE) {
689 local_irq_disable();
690 ret = cpuhp_invoke_callback(cpu, st->cb_state,
691 st->bringup, st->node);
692 local_irq_enable();
693 } else {
694 ret = cpuhp_invoke_callback(cpu, st->cb_state,
695 st->bringup, st->node);
696 }
697 } else if (st->rollback) {
698 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
699
700 undo_cpu_down(cpu, st);
701 /*
702 * This is a momentary workaround to keep the notifier users
703 * happy. Will go away once we got rid of the notifiers.
704 */
705 cpu_notify_nofail(CPU_DOWN_FAILED, cpu);
706 st->rollback = false;
707 } else {
708 /* Cannot happen .... */
709 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
710
711 /* Regular hotplug work */
712 if (st->state < st->target)
713 ret = cpuhp_ap_online(cpu, st);
714 else if (st->state > st->target)
715 ret = cpuhp_ap_offline(cpu, st);
716 }
717 lock_map_release(&cpuhp_state_lock_map);
718 st->result = ret;
719 complete(&st->done);
720}
721
722/* Invoke a single callback on a remote cpu */
723static int
724cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
725 struct hlist_node *node)
726{
727 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
728
729 if (!cpu_online(cpu))
730 return 0;
731
732 lock_map_acquire(&cpuhp_state_lock_map);
733 lock_map_release(&cpuhp_state_lock_map);
734
735 /*
736 * If we are up and running, use the hotplug thread. For early calls
737 * we invoke the thread function directly.
738 */
739 if (!st->thread)
740 return cpuhp_invoke_callback(cpu, state, bringup, node);
741
742 st->cb_state = state;
743 st->single = true;
744 st->bringup = bringup;
745 st->node = node;
746
747 /*
748 * Make sure the above stores are visible before should_run becomes
749 * true. Paired with the mb() above in cpuhp_thread_fun()
750 */
751 smp_mb();
752 st->should_run = true;
753 wake_up_process(st->thread);
754 wait_for_completion(&st->done);
755 return st->result;
756}
757
758/* Regular hotplug invocation of the AP hotplug thread */
759static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st)
760{
761 st->result = 0;
762 st->single = false;
763 /*
764 * Make sure the above stores are visible before should_run becomes
765 * true. Paired with the mb() above in cpuhp_thread_fun()
766 */
767 smp_mb();
768 st->should_run = true;
769 wake_up_process(st->thread);
770}
771
772static int cpuhp_kick_ap_work(unsigned int cpu)
773{
774 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
775 enum cpuhp_state state = st->state;
776
777 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work);
778 lock_map_acquire(&cpuhp_state_lock_map);
779 lock_map_release(&cpuhp_state_lock_map);
780 __cpuhp_kick_ap_work(st);
781 wait_for_completion(&st->done);
782 trace_cpuhp_exit(cpu, st->state, state, st->result);
783 return st->result;
784}
785
786static struct smp_hotplug_thread cpuhp_threads = {
787 .store = &cpuhp_state.thread,
788 .create = &cpuhp_create,
789 .thread_should_run = cpuhp_should_run,
790 .thread_fn = cpuhp_thread_fun,
791 .thread_comm = "cpuhp/%u",
792 .selfparking = true,
793};
794
795void __init cpuhp_threads_init(void)
796{
797 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
798 kthread_unpark(this_cpu_read(cpuhp_state.thread));
799}
800
801EXPORT_SYMBOL(register_cpu_notifier);
802EXPORT_SYMBOL(__register_cpu_notifier);
803void unregister_cpu_notifier(struct notifier_block *nb)
804{
805 cpu_maps_update_begin();
806 raw_notifier_chain_unregister(&cpu_chain, nb);
807 cpu_maps_update_done();
808}
809EXPORT_SYMBOL(unregister_cpu_notifier);
810
811void __unregister_cpu_notifier(struct notifier_block *nb)
812{
813 raw_notifier_chain_unregister(&cpu_chain, nb);
814}
815EXPORT_SYMBOL(__unregister_cpu_notifier);
816
817#ifdef CONFIG_HOTPLUG_CPU
818/**
819 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
820 * @cpu: a CPU id
821 *
822 * This function walks all processes, finds a valid mm struct for each one and
823 * then clears a corresponding bit in mm's cpumask. While this all sounds
824 * trivial, there are various non-obvious corner cases, which this function
825 * tries to solve in a safe manner.
826 *
827 * Also note that the function uses a somewhat relaxed locking scheme, so it may
828 * be called only for an already offlined CPU.
829 */
830void clear_tasks_mm_cpumask(int cpu)
831{
832 struct task_struct *p;
833
834 /*
835 * This function is called after the cpu is taken down and marked
836 * offline, so its not like new tasks will ever get this cpu set in
837 * their mm mask. -- Peter Zijlstra
838 * Thus, we may use rcu_read_lock() here, instead of grabbing
839 * full-fledged tasklist_lock.
840 */
841 WARN_ON(cpu_online(cpu));
842 rcu_read_lock();
843 for_each_process(p) {
844 struct task_struct *t;
845
846 /*
847 * Main thread might exit, but other threads may still have
848 * a valid mm. Find one.
849 */
850 t = find_lock_task_mm(p);
851 if (!t)
852 continue;
853 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
854 task_unlock(t);
855 }
856 rcu_read_unlock();
857}
858
859static inline void check_for_tasks(int dead_cpu)
860{
861 struct task_struct *g, *p;
862
863 read_lock(&tasklist_lock);
864 for_each_process_thread(g, p) {
865 if (!p->on_rq)
866 continue;
867 /*
868 * We do the check with unlocked task_rq(p)->lock.
869 * Order the reading to do not warn about a task,
870 * which was running on this cpu in the past, and
871 * it's just been woken on another cpu.
872 */
873 rmb();
874 if (task_cpu(p) != dead_cpu)
875 continue;
876
877 pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n",
878 p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags);
879 }
880 read_unlock(&tasklist_lock);
881}
882
883static int notify_down_prepare(unsigned int cpu)
884{
885 int err, nr_calls = 0;
886
887 err = __cpu_notify(CPU_DOWN_PREPARE, cpu, -1, &nr_calls);
888 if (err) {
889 nr_calls--;
890 __cpu_notify(CPU_DOWN_FAILED, cpu, nr_calls, NULL);
891 pr_warn("%s: attempt to take down CPU %u failed\n",
892 __func__, cpu);
893 }
894 return err;
895}
896
897/* Take this CPU down. */
898static int take_cpu_down(void *_param)
899{
900 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
901 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
902 int err, cpu = smp_processor_id();
903
904 /* Ensure this CPU doesn't handle any more interrupts. */
905 err = __cpu_disable();
906 if (err < 0)
907 return err;
908
909 /*
910 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
911 * do this step again.
912 */
913 WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
914 st->state--;
915 /* Invoke the former CPU_DYING callbacks */
916 for (; st->state > target; st->state--)
917 cpuhp_invoke_callback(cpu, st->state, false, NULL);
918
919 /* Give up timekeeping duties */
920 tick_handover_do_timer();
921 /* Park the stopper thread */
922 stop_machine_park(cpu);
923 return 0;
924}
925
926static int takedown_cpu(unsigned int cpu)
927{
928 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
929 int err;
930
931 /* Park the smpboot threads */
932 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
933
934 /*
935 * Prevent irq alloc/free while the dying cpu reorganizes the
936 * interrupt affinities.
937 */
938 irq_lock_sparse();
939
940 /*
941 * So now all preempt/rcu users must observe !cpu_active().
942 */
943 err = stop_machine(take_cpu_down, NULL, cpumask_of(cpu));
944 if (err) {
945 /* CPU refused to die */
946 irq_unlock_sparse();
947 /* Unpark the hotplug thread so we can rollback there */
948 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
949 return err;
950 }
951 BUG_ON(cpu_online(cpu));
952
953 /*
954 * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all
955 * runnable tasks from the cpu, there's only the idle task left now
956 * that the migration thread is done doing the stop_machine thing.
957 *
958 * Wait for the stop thread to go away.
959 */
960 wait_for_completion(&st->done);
961 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
962
963 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
964 irq_unlock_sparse();
965
966 hotplug_cpu__broadcast_tick_pull(cpu);
967 /* This actually kills the CPU. */
968 __cpu_die(cpu);
969
970 tick_cleanup_dead_cpu(cpu);
971 return 0;
972}
973
974static int notify_dead(unsigned int cpu)
975{
976 cpu_notify_nofail(CPU_DEAD, cpu);
977 check_for_tasks(cpu);
978 return 0;
979}
980
981static void cpuhp_complete_idle_dead(void *arg)
982{
983 struct cpuhp_cpu_state *st = arg;
984
985 complete(&st->done);
986}
987
988void cpuhp_report_idle_dead(void)
989{
990 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
991
992 BUG_ON(st->state != CPUHP_AP_OFFLINE);
993 rcu_report_dead(smp_processor_id());
994 st->state = CPUHP_AP_IDLE_DEAD;
995 /*
996 * We cannot call complete after rcu_report_dead() so we delegate it
997 * to an online cpu.
998 */
999 smp_call_function_single(cpumask_first(cpu_online_mask),
1000 cpuhp_complete_idle_dead, st, 0);
1001}
1002
1003#else
1004#define notify_down_prepare NULL
1005#define takedown_cpu NULL
1006#define notify_dead NULL
1007#endif
1008
1009#ifdef CONFIG_HOTPLUG_CPU
1010
1011/* Requires cpu_add_remove_lock to be held */
1012static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1013 enum cpuhp_state target)
1014{
1015 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1016 int prev_state, ret = 0;
1017 bool hasdied = false;
1018
1019 if (num_online_cpus() == 1)
1020 return -EBUSY;
1021
1022 if (!cpu_present(cpu))
1023 return -EINVAL;
1024
1025 cpu_hotplug_begin();
1026
1027 cpuhp_tasks_frozen = tasks_frozen;
1028
1029 prev_state = st->state;
1030 st->target = target;
1031 /*
1032 * If the current CPU state is in the range of the AP hotplug thread,
1033 * then we need to kick the thread.
1034 */
1035 if (st->state > CPUHP_TEARDOWN_CPU) {
1036 ret = cpuhp_kick_ap_work(cpu);
1037 /*
1038 * The AP side has done the error rollback already. Just
1039 * return the error code..
1040 */
1041 if (ret)
1042 goto out;
1043
1044 /*
1045 * We might have stopped still in the range of the AP hotplug
1046 * thread. Nothing to do anymore.
1047 */
1048 if (st->state > CPUHP_TEARDOWN_CPU)
1049 goto out;
1050 }
1051 /*
1052 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1053 * to do the further cleanups.
1054 */
1055 ret = cpuhp_down_callbacks(cpu, st, target);
1056 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) {
1057 st->target = prev_state;
1058 st->rollback = true;
1059 cpuhp_kick_ap_work(cpu);
1060 }
1061
1062 hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE;
1063out:
1064 cpu_hotplug_done();
1065 /* This post dead nonsense must die */
1066 if (!ret && hasdied)
1067 cpu_notify_nofail(CPU_POST_DEAD, cpu);
1068 arch_smt_update();
1069 return ret;
1070}
1071
1072static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1073{
1074 if (cpu_hotplug_disabled)
1075 return -EBUSY;
1076 return _cpu_down(cpu, 0, target);
1077}
1078
1079static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
1080{
1081 int err;
1082
1083 cpu_maps_update_begin();
1084 err = cpu_down_maps_locked(cpu, target);
1085 cpu_maps_update_done();
1086 return err;
1087}
1088int cpu_down(unsigned int cpu)
1089{
1090 return do_cpu_down(cpu, CPUHP_OFFLINE);
1091}
1092EXPORT_SYMBOL(cpu_down);
1093#endif /*CONFIG_HOTPLUG_CPU*/
1094
1095/**
1096 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1097 * @cpu: cpu that just started
1098 *
1099 * It must be called by the arch code on the new cpu, before the new cpu
1100 * enables interrupts and before the "boot" cpu returns from __cpu_up().
1101 */
1102void notify_cpu_starting(unsigned int cpu)
1103{
1104 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1105 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1106
1107 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
1108 st->booted_once = true;
1109 while (st->state < target) {
1110 st->state++;
1111 cpuhp_invoke_callback(cpu, st->state, true, NULL);
1112 }
1113}
1114
1115/*
1116 * Called from the idle task. Wake up the controlling task which brings the
1117 * stopper and the hotplug thread of the upcoming CPU up and then delegates
1118 * the rest of the online bringup to the hotplug thread.
1119 */
1120void cpuhp_online_idle(enum cpuhp_state state)
1121{
1122 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1123
1124 /* Happens for the boot cpu */
1125 if (state != CPUHP_AP_ONLINE_IDLE)
1126 return;
1127
1128 st->state = CPUHP_AP_ONLINE_IDLE;
1129 complete(&st->done);
1130}
1131
1132/* Requires cpu_add_remove_lock to be held */
1133static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1134{
1135 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1136 struct task_struct *idle;
1137 int ret = 0;
1138
1139 cpu_hotplug_begin();
1140
1141 if (!cpu_present(cpu)) {
1142 ret = -EINVAL;
1143 goto out;
1144 }
1145
1146 /*
1147 * The caller of do_cpu_up might have raced with another
1148 * caller. Ignore it for now.
1149 */
1150 if (st->state >= target)
1151 goto out;
1152
1153 if (st->state == CPUHP_OFFLINE) {
1154 /* Let it fail before we try to bring the cpu up */
1155 idle = idle_thread_get(cpu);
1156 if (IS_ERR(idle)) {
1157 ret = PTR_ERR(idle);
1158 goto out;
1159 }
1160 }
1161
1162 cpuhp_tasks_frozen = tasks_frozen;
1163
1164 st->target = target;
1165 /*
1166 * If the current CPU state is in the range of the AP hotplug thread,
1167 * then we need to kick the thread once more.
1168 */
1169 if (st->state > CPUHP_BRINGUP_CPU) {
1170 ret = cpuhp_kick_ap_work(cpu);
1171 /*
1172 * The AP side has done the error rollback already. Just
1173 * return the error code..
1174 */
1175 if (ret)
1176 goto out;
1177 }
1178
1179 /*
1180 * Try to reach the target state. We max out on the BP at
1181 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1182 * responsible for bringing it up to the target state.
1183 */
1184 target = min((int)target, CPUHP_BRINGUP_CPU);
1185 ret = cpuhp_up_callbacks(cpu, st, target);
1186out:
1187 cpu_hotplug_done();
1188 arch_smt_update();
1189 return ret;
1190}
1191
1192static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1193{
1194 int err = 0;
1195
1196 if (!cpu_possible(cpu)) {
1197 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1198 cpu);
1199#if defined(CONFIG_IA64)
1200 pr_err("please check additional_cpus= boot parameter\n");
1201#endif
1202 return -EINVAL;
1203 }
1204
1205 err = try_online_node(cpu_to_node(cpu));
1206 if (err)
1207 return err;
1208
1209 cpu_maps_update_begin();
1210
1211 if (cpu_hotplug_disabled) {
1212 err = -EBUSY;
1213 goto out;
1214 }
1215 if (!cpu_smt_allowed(cpu)) {
1216 err = -EPERM;
1217 goto out;
1218 }
1219
1220 err = _cpu_up(cpu, 0, target);
1221out:
1222 cpu_maps_update_done();
1223 return err;
1224}
1225
1226int cpu_up(unsigned int cpu)
1227{
1228 return do_cpu_up(cpu, CPUHP_ONLINE);
1229}
1230EXPORT_SYMBOL_GPL(cpu_up);
1231
1232#ifdef CONFIG_PM_SLEEP_SMP
1233static cpumask_var_t frozen_cpus;
1234
1235int freeze_secondary_cpus(int primary)
1236{
1237 int cpu, error = 0;
1238
1239 cpu_maps_update_begin();
1240 if (!cpu_online(primary))
1241 primary = cpumask_first(cpu_online_mask);
1242 /*
1243 * We take down all of the non-boot CPUs in one shot to avoid races
1244 * with the userspace trying to use the CPU hotplug at the same time
1245 */
1246 cpumask_clear(frozen_cpus);
1247
1248 pr_info("Disabling non-boot CPUs ...\n");
1249 for_each_online_cpu(cpu) {
1250 if (cpu == primary)
1251 continue;
1252 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1253 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1254 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1255 if (!error)
1256 cpumask_set_cpu(cpu, frozen_cpus);
1257 else {
1258 pr_err("Error taking CPU%d down: %d\n", cpu, error);
1259 break;
1260 }
1261 }
1262
1263 if (!error)
1264 BUG_ON(num_online_cpus() > 1);
1265 else
1266 pr_err("Non-boot CPUs are not disabled\n");
1267
1268 /*
1269 * Make sure the CPUs won't be enabled by someone else. We need to do
1270 * this even in case of failure as all disable_nonboot_cpus() users are
1271 * supposed to do enable_nonboot_cpus() on the failure path.
1272 */
1273 cpu_hotplug_disabled++;
1274
1275 cpu_maps_update_done();
1276 return error;
1277}
1278
1279void __weak arch_enable_nonboot_cpus_begin(void)
1280{
1281}
1282
1283void __weak arch_enable_nonboot_cpus_end(void)
1284{
1285}
1286
1287void enable_nonboot_cpus(void)
1288{
1289 int cpu, error;
1290 struct device *cpu_device;
1291
1292 /* Allow everyone to use the CPU hotplug again */
1293 cpu_maps_update_begin();
1294 __cpu_hotplug_enable();
1295 if (cpumask_empty(frozen_cpus))
1296 goto out;
1297
1298 pr_info("Enabling non-boot CPUs ...\n");
1299
1300 arch_enable_nonboot_cpus_begin();
1301
1302 for_each_cpu(cpu, frozen_cpus) {
1303 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1304 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1305 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1306 if (!error) {
1307 pr_info("CPU%d is up\n", cpu);
1308 cpu_device = get_cpu_device(cpu);
1309 if (!cpu_device)
1310 pr_err("%s: failed to get cpu%d device\n",
1311 __func__, cpu);
1312 else
1313 kobject_uevent(&cpu_device->kobj, KOBJ_ONLINE);
1314 continue;
1315 }
1316 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1317 }
1318
1319 arch_enable_nonboot_cpus_end();
1320
1321 cpumask_clear(frozen_cpus);
1322out:
1323 cpu_maps_update_done();
1324}
1325
1326static int __init alloc_frozen_cpus(void)
1327{
1328 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1329 return -ENOMEM;
1330 return 0;
1331}
1332core_initcall(alloc_frozen_cpus);
1333
1334/*
1335 * When callbacks for CPU hotplug notifications are being executed, we must
1336 * ensure that the state of the system with respect to the tasks being frozen
1337 * or not, as reported by the notification, remains unchanged *throughout the
1338 * duration* of the execution of the callbacks.
1339 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1340 *
1341 * This synchronization is implemented by mutually excluding regular CPU
1342 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1343 * Hibernate notifications.
1344 */
1345static int
1346cpu_hotplug_pm_callback(struct notifier_block *nb,
1347 unsigned long action, void *ptr)
1348{
1349 switch (action) {
1350
1351 case PM_SUSPEND_PREPARE:
1352 case PM_HIBERNATION_PREPARE:
1353 cpu_hotplug_disable();
1354 break;
1355
1356 case PM_POST_SUSPEND:
1357 case PM_POST_HIBERNATION:
1358 cpu_hotplug_enable();
1359 break;
1360
1361 default:
1362 return NOTIFY_DONE;
1363 }
1364
1365 return NOTIFY_OK;
1366}
1367
1368
1369static int __init cpu_hotplug_pm_sync_init(void)
1370{
1371 /*
1372 * cpu_hotplug_pm_callback has higher priority than x86
1373 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1374 * to disable cpu hotplug to avoid cpu hotplug race.
1375 */
1376 pm_notifier(cpu_hotplug_pm_callback, 0);
1377 return 0;
1378}
1379core_initcall(cpu_hotplug_pm_sync_init);
1380
1381#endif /* CONFIG_PM_SLEEP_SMP */
1382
1383#endif /* CONFIG_SMP */
1384
1385/* Boot processor state steps */
1386static struct cpuhp_step cpuhp_bp_states[] = {
1387 [CPUHP_OFFLINE] = {
1388 .name = "offline",
1389 .startup.single = NULL,
1390 .teardown.single = NULL,
1391 },
1392#ifdef CONFIG_SMP
1393 [CPUHP_CREATE_THREADS]= {
1394 .name = "threads:prepare",
1395 .startup.single = smpboot_create_threads,
1396 .teardown.single = NULL,
1397 .cant_stop = true,
1398 },
1399 [CPUHP_PERF_PREPARE] = {
1400 .name = "perf:prepare",
1401 .startup.single = perf_event_init_cpu,
1402 .teardown.single = perf_event_exit_cpu,
1403 },
1404 [CPUHP_WORKQUEUE_PREP] = {
1405 .name = "workqueue:prepare",
1406 .startup.single = workqueue_prepare_cpu,
1407 .teardown.single = NULL,
1408 },
1409 [CPUHP_HRTIMERS_PREPARE] = {
1410 .name = "hrtimers:prepare",
1411 .startup.single = hrtimers_prepare_cpu,
1412 .teardown.single = hrtimers_dead_cpu,
1413 },
1414 [CPUHP_SMPCFD_PREPARE] = {
1415 .name = "smpcfd:prepare",
1416 .startup.single = smpcfd_prepare_cpu,
1417 .teardown.single = smpcfd_dead_cpu,
1418 },
1419 [CPUHP_RELAY_PREPARE] = {
1420 .name = "relay:prepare",
1421 .startup.single = relay_prepare_cpu,
1422 .teardown.single = NULL,
1423 },
1424 [CPUHP_SLAB_PREPARE] = {
1425 .name = "slab:prepare",
1426 .startup.single = slab_prepare_cpu,
1427 .teardown.single = slab_dead_cpu,
1428 },
1429 [CPUHP_RCUTREE_PREP] = {
1430 .name = "RCU/tree:prepare",
1431 .startup.single = rcutree_prepare_cpu,
1432 .teardown.single = rcutree_dead_cpu,
1433 },
1434 /*
1435 * Preparatory and dead notifiers. Will be replaced once the notifiers
1436 * are converted to states.
1437 */
1438 [CPUHP_NOTIFY_PREPARE] = {
1439 .name = "notify:prepare",
1440 .startup.single = notify_prepare,
1441 .teardown.single = notify_dead,
1442 .skip_onerr = true,
1443 .cant_stop = true,
1444 },
1445 /*
1446 * On the tear-down path, timers_dead_cpu() must be invoked
1447 * before blk_mq_queue_reinit_notify() from notify_dead(),
1448 * otherwise a RCU stall occurs.
1449 */
1450 [CPUHP_TIMERS_PREPARE] = {
1451 .name = "timers:dead",
1452 .startup.single = timers_prepare_cpu,
1453 .teardown.single = timers_dead_cpu,
1454 },
1455 /* Kicks the plugged cpu into life */
1456 [CPUHP_BRINGUP_CPU] = {
1457 .name = "cpu:bringup",
1458 .startup.single = bringup_cpu,
1459 .teardown.single = NULL,
1460 .cant_stop = true,
1461 },
1462 /*
1463 * Handled on controll processor until the plugged processor manages
1464 * this itself.
1465 */
1466 [CPUHP_TEARDOWN_CPU] = {
1467 .name = "cpu:teardown",
1468 .startup.single = NULL,
1469 .teardown.single = takedown_cpu,
1470 .cant_stop = true,
1471 },
1472#else
1473 [CPUHP_BRINGUP_CPU] = { },
1474#endif
1475};
1476
1477/* Application processor state steps */
1478static struct cpuhp_step cpuhp_ap_states[] = {
1479#ifdef CONFIG_SMP
1480 /* Final state before CPU kills itself */
1481 [CPUHP_AP_IDLE_DEAD] = {
1482 .name = "idle:dead",
1483 },
1484 /*
1485 * Last state before CPU enters the idle loop to die. Transient state
1486 * for synchronization.
1487 */
1488 [CPUHP_AP_OFFLINE] = {
1489 .name = "ap:offline",
1490 .cant_stop = true,
1491 },
1492 /* First state is scheduler control. Interrupts are disabled */
1493 [CPUHP_AP_SCHED_STARTING] = {
1494 .name = "sched:starting",
1495 .startup.single = sched_cpu_starting,
1496 .teardown.single = sched_cpu_dying,
1497 },
1498 [CPUHP_AP_RCUTREE_DYING] = {
1499 .name = "RCU/tree:dying",
1500 .startup.single = NULL,
1501 .teardown.single = rcutree_dying_cpu,
1502 },
1503 [CPUHP_AP_SMPCFD_DYING] = {
1504 .name = "smpcfd:dying",
1505 .startup.single = NULL,
1506 .teardown.single = smpcfd_dying_cpu,
1507 },
1508 /* Entry state on starting. Interrupts enabled from here on. Transient
1509 * state for synchronsization */
1510 [CPUHP_AP_ONLINE] = {
1511 .name = "ap:online",
1512 },
1513 /* Handle smpboot threads park/unpark */
1514 [CPUHP_AP_SMPBOOT_THREADS] = {
1515 .name = "smpboot/threads:online",
1516 .startup.single = smpboot_unpark_threads,
1517 .teardown.single = smpboot_park_threads,
1518 },
1519 [CPUHP_AP_PERF_ONLINE] = {
1520 .name = "perf:online",
1521 .startup.single = perf_event_init_cpu,
1522 .teardown.single = perf_event_exit_cpu,
1523 },
1524 [CPUHP_AP_WORKQUEUE_ONLINE] = {
1525 .name = "workqueue:online",
1526 .startup.single = workqueue_online_cpu,
1527 .teardown.single = workqueue_offline_cpu,
1528 },
1529 [CPUHP_AP_RCUTREE_ONLINE] = {
1530 .name = "RCU/tree:online",
1531 .startup.single = rcutree_online_cpu,
1532 .teardown.single = rcutree_offline_cpu,
1533 },
1534
1535 /*
1536 * Online/down_prepare notifiers. Will be removed once the notifiers
1537 * are converted to states.
1538 */
1539 [CPUHP_AP_NOTIFY_ONLINE] = {
1540 .name = "notify:online",
1541 .startup.single = notify_online,
1542 .teardown.single = notify_down_prepare,
1543 .skip_onerr = true,
1544 },
1545#endif
1546 /*
1547 * The dynamically registered state space is here
1548 */
1549
1550#ifdef CONFIG_SMP
1551 /* Last state is scheduler control setting the cpu active */
1552 [CPUHP_AP_ACTIVE] = {
1553 .name = "sched:active",
1554 .startup.single = sched_cpu_activate,
1555 .teardown.single = sched_cpu_deactivate,
1556 },
1557#endif
1558
1559 /* CPU is fully up and running. */
1560 [CPUHP_ONLINE] = {
1561 .name = "online",
1562 .startup.single = NULL,
1563 .teardown.single = NULL,
1564 },
1565};
1566
1567/* Sanity check for callbacks */
1568static int cpuhp_cb_check(enum cpuhp_state state)
1569{
1570 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1571 return -EINVAL;
1572 return 0;
1573}
1574
1575static void cpuhp_store_callbacks(enum cpuhp_state state,
1576 const char *name,
1577 int (*startup)(unsigned int cpu),
1578 int (*teardown)(unsigned int cpu),
1579 bool multi_instance)
1580{
1581 /* (Un)Install the callbacks for further cpu hotplug operations */
1582 struct cpuhp_step *sp;
1583
1584 sp = cpuhp_get_step(state);
1585 sp->startup.single = startup;
1586 sp->teardown.single = teardown;
1587 sp->name = name;
1588 sp->multi_instance = multi_instance;
1589 INIT_HLIST_HEAD(&sp->list);
1590}
1591
1592static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1593{
1594 return cpuhp_get_step(state)->teardown.single;
1595}
1596
1597/*
1598 * Call the startup/teardown function for a step either on the AP or
1599 * on the current CPU.
1600 */
1601static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1602 struct hlist_node *node)
1603{
1604 struct cpuhp_step *sp = cpuhp_get_step(state);
1605 int ret;
1606
1607 if ((bringup && !sp->startup.single) ||
1608 (!bringup && !sp->teardown.single))
1609 return 0;
1610 /*
1611 * The non AP bound callbacks can fail on bringup. On teardown
1612 * e.g. module removal we crash for now.
1613 */
1614#ifdef CONFIG_SMP
1615 if (cpuhp_is_ap_state(state))
1616 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1617 else
1618 ret = cpuhp_invoke_callback(cpu, state, bringup, node);
1619#else
1620 ret = cpuhp_invoke_callback(cpu, state, bringup, node);
1621#endif
1622 BUG_ON(ret && !bringup);
1623 return ret;
1624}
1625
1626/*
1627 * Called from __cpuhp_setup_state on a recoverable failure.
1628 *
1629 * Note: The teardown callbacks for rollback are not allowed to fail!
1630 */
1631static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1632 struct hlist_node *node)
1633{
1634 int cpu;
1635
1636 /* Roll back the already executed steps on the other cpus */
1637 for_each_present_cpu(cpu) {
1638 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1639 int cpustate = st->state;
1640
1641 if (cpu >= failedcpu)
1642 break;
1643
1644 /* Did we invoke the startup call on that cpu ? */
1645 if (cpustate >= state)
1646 cpuhp_issue_call(cpu, state, false, node);
1647 }
1648}
1649
1650/*
1651 * Returns a free for dynamic slot assignment of the Online state. The states
1652 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1653 * by having no name assigned.
1654 */
1655static int cpuhp_reserve_state(enum cpuhp_state state)
1656{
1657 enum cpuhp_state i;
1658
1659 for (i = CPUHP_AP_ONLINE_DYN; i <= CPUHP_AP_ONLINE_DYN_END; i++) {
1660 if (cpuhp_ap_states[i].name)
1661 continue;
1662
1663 cpuhp_ap_states[i].name = "Reserved";
1664 return i;
1665 }
1666 WARN(1, "No more dynamic states available for CPU hotplug\n");
1667 return -ENOSPC;
1668}
1669
1670int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1671 bool invoke)
1672{
1673 struct cpuhp_step *sp;
1674 int cpu;
1675 int ret;
1676
1677 sp = cpuhp_get_step(state);
1678 if (sp->multi_instance == false)
1679 return -EINVAL;
1680
1681 get_online_cpus();
1682 mutex_lock(&cpuhp_state_mutex);
1683
1684 if (!invoke || !sp->startup.multi)
1685 goto add_node;
1686
1687 /*
1688 * Try to call the startup callback for each present cpu
1689 * depending on the hotplug state of the cpu.
1690 */
1691 for_each_present_cpu(cpu) {
1692 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1693 int cpustate = st->state;
1694
1695 if (cpustate < state)
1696 continue;
1697
1698 ret = cpuhp_issue_call(cpu, state, true, node);
1699 if (ret) {
1700 if (sp->teardown.multi)
1701 cpuhp_rollback_install(cpu, state, node);
1702 goto err;
1703 }
1704 }
1705add_node:
1706 ret = 0;
1707 hlist_add_head(node, &sp->list);
1708
1709err:
1710 mutex_unlock(&cpuhp_state_mutex);
1711 put_online_cpus();
1712 return ret;
1713}
1714EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1715
1716/**
1717 * __cpuhp_setup_state - Setup the callbacks for an hotplug machine state
1718 * @state: The state to setup
1719 * @invoke: If true, the startup function is invoked for cpus where
1720 * cpu state >= @state
1721 * @startup: startup callback function
1722 * @teardown: teardown callback function
1723 *
1724 * Returns 0 if successful, otherwise a proper error code
1725 */
1726int __cpuhp_setup_state(enum cpuhp_state state,
1727 const char *name, bool invoke,
1728 int (*startup)(unsigned int cpu),
1729 int (*teardown)(unsigned int cpu),
1730 bool multi_instance)
1731{
1732 int cpu, ret = 0;
1733 int dyn_state = 0;
1734
1735 if (cpuhp_cb_check(state) || !name)
1736 return -EINVAL;
1737
1738 get_online_cpus();
1739 mutex_lock(&cpuhp_state_mutex);
1740
1741 /* currently assignments for the ONLINE state are possible */
1742 if (state == CPUHP_AP_ONLINE_DYN) {
1743 dyn_state = 1;
1744 ret = cpuhp_reserve_state(state);
1745 if (ret < 0)
1746 goto out;
1747 state = ret;
1748 }
1749
1750 cpuhp_store_callbacks(state, name, startup, teardown, multi_instance);
1751
1752 if (!invoke || !startup)
1753 goto out;
1754
1755 /*
1756 * Try to call the startup callback for each present cpu
1757 * depending on the hotplug state of the cpu.
1758 */
1759 for_each_present_cpu(cpu) {
1760 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1761 int cpustate = st->state;
1762
1763 if (cpustate < state)
1764 continue;
1765
1766 ret = cpuhp_issue_call(cpu, state, true, NULL);
1767 if (ret) {
1768 if (teardown)
1769 cpuhp_rollback_install(cpu, state, NULL);
1770 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1771 goto out;
1772 }
1773 }
1774out:
1775 mutex_unlock(&cpuhp_state_mutex);
1776
1777 put_online_cpus();
1778 if (!ret && dyn_state)
1779 return state;
1780 return ret;
1781}
1782EXPORT_SYMBOL(__cpuhp_setup_state);
1783
1784int __cpuhp_state_remove_instance(enum cpuhp_state state,
1785 struct hlist_node *node, bool invoke)
1786{
1787 struct cpuhp_step *sp = cpuhp_get_step(state);
1788 int cpu;
1789
1790 BUG_ON(cpuhp_cb_check(state));
1791
1792 if (!sp->multi_instance)
1793 return -EINVAL;
1794
1795 get_online_cpus();
1796 mutex_lock(&cpuhp_state_mutex);
1797
1798 if (!invoke || !cpuhp_get_teardown_cb(state))
1799 goto remove;
1800 /*
1801 * Call the teardown callback for each present cpu depending
1802 * on the hotplug state of the cpu. This function is not
1803 * allowed to fail currently!
1804 */
1805 for_each_present_cpu(cpu) {
1806 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1807 int cpustate = st->state;
1808
1809 if (cpustate >= state)
1810 cpuhp_issue_call(cpu, state, false, node);
1811 }
1812
1813remove:
1814 hlist_del(node);
1815 mutex_unlock(&cpuhp_state_mutex);
1816 put_online_cpus();
1817
1818 return 0;
1819}
1820EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1821/**
1822 * __cpuhp_remove_state - Remove the callbacks for an hotplug machine state
1823 * @state: The state to remove
1824 * @invoke: If true, the teardown function is invoked for cpus where
1825 * cpu state >= @state
1826 *
1827 * The teardown callback is currently not allowed to fail. Think
1828 * about module removal!
1829 */
1830void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1831{
1832 struct cpuhp_step *sp = cpuhp_get_step(state);
1833 int cpu;
1834
1835 BUG_ON(cpuhp_cb_check(state));
1836
1837 get_online_cpus();
1838 mutex_lock(&cpuhp_state_mutex);
1839
1840 if (sp->multi_instance) {
1841 WARN(!hlist_empty(&sp->list),
1842 "Error: Removing state %d which has instances left.\n",
1843 state);
1844 goto remove;
1845 }
1846
1847 if (!invoke || !cpuhp_get_teardown_cb(state))
1848 goto remove;
1849
1850 /*
1851 * Call the teardown callback for each present cpu depending
1852 * on the hotplug state of the cpu. This function is not
1853 * allowed to fail currently!
1854 */
1855 for_each_present_cpu(cpu) {
1856 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1857 int cpustate = st->state;
1858
1859 if (cpustate >= state)
1860 cpuhp_issue_call(cpu, state, false, NULL);
1861 }
1862remove:
1863 cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1864 mutex_unlock(&cpuhp_state_mutex);
1865 put_online_cpus();
1866}
1867EXPORT_SYMBOL(__cpuhp_remove_state);
1868
1869#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1870static ssize_t show_cpuhp_state(struct device *dev,
1871 struct device_attribute *attr, char *buf)
1872{
1873 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1874
1875 return sprintf(buf, "%d\n", st->state);
1876}
1877static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1878
1879static ssize_t write_cpuhp_target(struct device *dev,
1880 struct device_attribute *attr,
1881 const char *buf, size_t count)
1882{
1883 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1884 struct cpuhp_step *sp;
1885 int target, ret;
1886
1887 ret = kstrtoint(buf, 10, &target);
1888 if (ret)
1889 return ret;
1890
1891#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1892 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1893 return -EINVAL;
1894#else
1895 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1896 return -EINVAL;
1897#endif
1898
1899 ret = lock_device_hotplug_sysfs();
1900 if (ret)
1901 return ret;
1902
1903 mutex_lock(&cpuhp_state_mutex);
1904 sp = cpuhp_get_step(target);
1905 ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1906 mutex_unlock(&cpuhp_state_mutex);
1907 if (ret)
1908 goto out;
1909
1910 if (st->state < target)
1911 ret = do_cpu_up(dev->id, target);
1912 else
1913 ret = do_cpu_down(dev->id, target);
1914out:
1915 unlock_device_hotplug();
1916 return ret ? ret : count;
1917}
1918
1919static ssize_t show_cpuhp_target(struct device *dev,
1920 struct device_attribute *attr, char *buf)
1921{
1922 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1923
1924 return sprintf(buf, "%d\n", st->target);
1925}
1926static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1927
1928static struct attribute *cpuhp_cpu_attrs[] = {
1929 &dev_attr_state.attr,
1930 &dev_attr_target.attr,
1931 NULL
1932};
1933
1934static struct attribute_group cpuhp_cpu_attr_group = {
1935 .attrs = cpuhp_cpu_attrs,
1936 .name = "hotplug",
1937 NULL
1938};
1939
1940static ssize_t show_cpuhp_states(struct device *dev,
1941 struct device_attribute *attr, char *buf)
1942{
1943 ssize_t cur, res = 0;
1944 int i;
1945
1946 mutex_lock(&cpuhp_state_mutex);
1947 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
1948 struct cpuhp_step *sp = cpuhp_get_step(i);
1949
1950 if (sp->name) {
1951 cur = sprintf(buf, "%3d: %s\n", i, sp->name);
1952 buf += cur;
1953 res += cur;
1954 }
1955 }
1956 mutex_unlock(&cpuhp_state_mutex);
1957 return res;
1958}
1959static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
1960
1961static struct attribute *cpuhp_cpu_root_attrs[] = {
1962 &dev_attr_states.attr,
1963 NULL
1964};
1965
1966static struct attribute_group cpuhp_cpu_root_attr_group = {
1967 .attrs = cpuhp_cpu_root_attrs,
1968 .name = "hotplug",
1969 NULL
1970};
1971
1972#ifdef CONFIG_HOTPLUG_SMT
1973
1974static const char *smt_states[] = {
1975 [CPU_SMT_ENABLED] = "on",
1976 [CPU_SMT_DISABLED] = "off",
1977 [CPU_SMT_FORCE_DISABLED] = "forceoff",
1978 [CPU_SMT_NOT_SUPPORTED] = "notsupported",
1979};
1980
1981static ssize_t
1982show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
1983{
1984 return snprintf(buf, PAGE_SIZE - 2, "%s\n", smt_states[cpu_smt_control]);
1985}
1986
1987static void cpuhp_offline_cpu_device(unsigned int cpu)
1988{
1989 struct device *dev = get_cpu_device(cpu);
1990
1991 dev->offline = true;
1992 /* Tell user space about the state change */
1993 kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
1994}
1995
1996static void cpuhp_online_cpu_device(unsigned int cpu)
1997{
1998 struct device *dev = get_cpu_device(cpu);
1999
2000 dev->offline = false;
2001 /* Tell user space about the state change */
2002 kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2003}
2004
2005static int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2006{
2007 int cpu, ret = 0;
2008
2009 cpu_maps_update_begin();
2010 for_each_online_cpu(cpu) {
2011 if (topology_is_primary_thread(cpu))
2012 continue;
2013 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2014 if (ret)
2015 break;
2016 /*
2017 * As this needs to hold the cpu maps lock it's impossible
2018 * to call device_offline() because that ends up calling
2019 * cpu_down() which takes cpu maps lock. cpu maps lock
2020 * needs to be held as this might race against in kernel
2021 * abusers of the hotplug machinery (thermal management).
2022 *
2023 * So nothing would update device:offline state. That would
2024 * leave the sysfs entry stale and prevent onlining after
2025 * smt control has been changed to 'off' again. This is
2026 * called under the sysfs hotplug lock, so it is properly
2027 * serialized against the regular offline usage.
2028 */
2029 cpuhp_offline_cpu_device(cpu);
2030 }
2031 if (!ret) {
2032 cpu_smt_control = ctrlval;
2033 arch_smt_update();
2034 }
2035 cpu_maps_update_done();
2036 return ret;
2037}
2038
2039static int cpuhp_smt_enable(void)
2040{
2041 int cpu, ret = 0;
2042
2043 cpu_maps_update_begin();
2044 cpu_smt_control = CPU_SMT_ENABLED;
2045 arch_smt_update();
2046 for_each_present_cpu(cpu) {
2047 /* Skip online CPUs and CPUs on offline nodes */
2048 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2049 continue;
2050 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2051 if (ret)
2052 break;
2053 /* See comment in cpuhp_smt_disable() */
2054 cpuhp_online_cpu_device(cpu);
2055 }
2056 cpu_maps_update_done();
2057 return ret;
2058}
2059
2060static ssize_t
2061store_smt_control(struct device *dev, struct device_attribute *attr,
2062 const char *buf, size_t count)
2063{
2064 int ctrlval, ret;
2065
2066 if (sysfs_streq(buf, "on"))
2067 ctrlval = CPU_SMT_ENABLED;
2068 else if (sysfs_streq(buf, "off"))
2069 ctrlval = CPU_SMT_DISABLED;
2070 else if (sysfs_streq(buf, "forceoff"))
2071 ctrlval = CPU_SMT_FORCE_DISABLED;
2072 else
2073 return -EINVAL;
2074
2075 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2076 return -EPERM;
2077
2078 if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2079 return -ENODEV;
2080
2081 ret = lock_device_hotplug_sysfs();
2082 if (ret)
2083 return ret;
2084
2085 if (ctrlval != cpu_smt_control) {
2086 switch (ctrlval) {
2087 case CPU_SMT_ENABLED:
2088 ret = cpuhp_smt_enable();
2089 break;
2090 case CPU_SMT_DISABLED:
2091 case CPU_SMT_FORCE_DISABLED:
2092 ret = cpuhp_smt_disable(ctrlval);
2093 break;
2094 }
2095 }
2096
2097 unlock_device_hotplug();
2098 return ret ? ret : count;
2099}
2100static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2101
2102static ssize_t
2103show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2104{
2105 bool active = topology_max_smt_threads() > 1;
2106
2107 return snprintf(buf, PAGE_SIZE - 2, "%d\n", active);
2108}
2109static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2110
2111static struct attribute *cpuhp_smt_attrs[] = {
2112 &dev_attr_control.attr,
2113 &dev_attr_active.attr,
2114 NULL
2115};
2116
2117static const struct attribute_group cpuhp_smt_attr_group = {
2118 .attrs = cpuhp_smt_attrs,
2119 .name = "smt",
2120 NULL
2121};
2122
2123static int __init cpu_smt_state_init(void)
2124{
2125 return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2126 &cpuhp_smt_attr_group);
2127}
2128
2129#else
2130static inline int cpu_smt_state_init(void) { return 0; }
2131#endif
2132
2133static int __init cpuhp_sysfs_init(void)
2134{
2135 int cpu, ret;
2136
2137 ret = cpu_smt_state_init();
2138 if (ret)
2139 return ret;
2140
2141 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2142 &cpuhp_cpu_root_attr_group);
2143 if (ret)
2144 return ret;
2145
2146 for_each_possible_cpu(cpu) {
2147 struct device *dev = get_cpu_device(cpu);
2148
2149 if (!dev)
2150 continue;
2151 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2152 if (ret)
2153 return ret;
2154 }
2155 return 0;
2156}
2157device_initcall(cpuhp_sysfs_init);
2158#endif
2159
2160/*
2161 * cpu_bit_bitmap[] is a special, "compressed" data structure that
2162 * represents all NR_CPUS bits binary values of 1<<nr.
2163 *
2164 * It is used by cpumask_of() to get a constant address to a CPU
2165 * mask value that has a single bit set only.
2166 */
2167
2168/* cpu_bit_bitmap[0] is empty - so we can back into it */
2169#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
2170#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2171#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2172#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2173
2174const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2175
2176 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
2177 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
2178#if BITS_PER_LONG > 32
2179 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
2180 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
2181#endif
2182};
2183EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2184
2185const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2186EXPORT_SYMBOL(cpu_all_bits);
2187
2188#ifdef CONFIG_INIT_ALL_POSSIBLE
2189struct cpumask __cpu_possible_mask __read_mostly
2190 = {CPU_BITS_ALL};
2191#else
2192struct cpumask __cpu_possible_mask __read_mostly;
2193#endif
2194EXPORT_SYMBOL(__cpu_possible_mask);
2195
2196struct cpumask __cpu_online_mask __read_mostly;
2197EXPORT_SYMBOL(__cpu_online_mask);
2198
2199struct cpumask __cpu_present_mask __read_mostly;
2200EXPORT_SYMBOL(__cpu_present_mask);
2201
2202struct cpumask __cpu_active_mask __read_mostly;
2203EXPORT_SYMBOL(__cpu_active_mask);
2204
2205void init_cpu_present(const struct cpumask *src)
2206{
2207 cpumask_copy(&__cpu_present_mask, src);
2208}
2209
2210void init_cpu_possible(const struct cpumask *src)
2211{
2212 cpumask_copy(&__cpu_possible_mask, src);
2213}
2214
2215void init_cpu_online(const struct cpumask *src)
2216{
2217 cpumask_copy(&__cpu_online_mask, src);
2218}
2219
2220/*
2221 * Activate the first processor.
2222 */
2223void __init boot_cpu_init(void)
2224{
2225 int cpu = smp_processor_id();
2226
2227 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2228 set_cpu_online(cpu, true);
2229 set_cpu_active(cpu, true);
2230 set_cpu_present(cpu, true);
2231 set_cpu_possible(cpu, true);
2232}
2233
2234/*
2235 * Must be called _AFTER_ setting up the per_cpu areas
2236 */
2237void __init boot_cpu_hotplug_init(void)
2238{
2239#ifdef CONFIG_SMP
2240 this_cpu_write(cpuhp_state.booted_once, true);
2241#endif
2242 this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2243}
2244
2245enum cpu_mitigations cpu_mitigations __ro_after_init = CPU_MITIGATIONS_AUTO;
2246
2247static int __init mitigations_parse_cmdline(char *arg)
2248{
2249 if (!strcmp(arg, "off"))
2250 cpu_mitigations = CPU_MITIGATIONS_OFF;
2251 else if (!strcmp(arg, "auto"))
2252 cpu_mitigations = CPU_MITIGATIONS_AUTO;
2253 else if (!strcmp(arg, "auto,nosmt"))
2254 cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2255
2256 return 0;
2257}
2258early_param("mitigations", mitigations_parse_cmdline);
2259
2260static ATOMIC_NOTIFIER_HEAD(idle_notifier);
2261
2262void idle_notifier_register(struct notifier_block *n)
2263{
2264 atomic_notifier_chain_register(&idle_notifier, n);
2265}
2266EXPORT_SYMBOL_GPL(idle_notifier_register);
2267
2268void idle_notifier_unregister(struct notifier_block *n)
2269{
2270 atomic_notifier_chain_unregister(&idle_notifier, n);
2271}
2272EXPORT_SYMBOL_GPL(idle_notifier_unregister);
2273
2274void idle_notifier_call_chain(unsigned long val)
2275{
2276 atomic_notifier_call_chain(&idle_notifier, val, NULL);
2277}
2278EXPORT_SYMBOL_GPL(idle_notifier_call_chain);
2279
generated by cgit at 2024-05-18 14:16:57 (GMT)