path: root/kernel/watchdog.c (plain)
blob: 9df5ac2fe8fa35b10aec80d3cf315986879c8ee7

1	/*
2	* Detect hard and soft lockups on a system
3	*
4	* started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
5	*
6	* Note: Most of this code is borrowed heavily from the original softlockup
7	* detector, so thanks to Ingo for the initial implementation.
8	* Some chunks also taken from the old x86-specific nmi watchdog code, thanks
9	* to those contributors as well.
10	*/
11
12	#define pr_fmt(fmt) "NMI watchdog: " fmt
13
14	#include <linux/mm.h>
15	#include <linux/cpu.h>
16	#include <linux/nmi.h>
17	#include <linux/init.h>
18	#include <linux/module.h>
19	#include <linux/sysctl.h>
20	#include <linux/smpboot.h>
21	#include <linux/sched/rt.h>
22	#include <linux/tick.h>
23	#include <linux/workqueue.h>
24
25	#include <asm/irq_regs.h>
26	#include <linux/kvm_para.h>
27	#include <linux/kthread.h>
28
29	#include <linux/amlogic/debug_lockup.h>
30
31	static DEFINE_MUTEX(watchdog_proc_mutex);
32
33	#if defined(CONFIG_HAVE_NMI_WATCHDOG) || defined(CONFIG_HARDLOCKUP_DETECTOR)
34	unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED|NMI_WATCHDOG_ENABLED;
35	#else
36	unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED;
37	#endif
38	int __read_mostly nmi_watchdog_enabled;
39	int __read_mostly soft_watchdog_enabled;
40	int __read_mostly watchdog_user_enabled;
41	int __read_mostly watchdog_thresh = 10;
42
43	#ifdef CONFIG_SMP
44	int __read_mostly sysctl_softlockup_all_cpu_backtrace;
45	int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
46	#endif
47	static struct cpumask watchdog_cpumask __read_mostly;
48	unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
49
50	/* Helper for online, unparked cpus. */
51	#define for_each_watchdog_cpu(cpu) \
52	for_each_cpu_and((cpu), cpu_online_mask, &watchdog_cpumask)
53
54	atomic_t watchdog_park_in_progress = ATOMIC_INIT(0);
55
56	/*
57	* The 'watchdog_running' variable is set to 1 when the watchdog threads
58	* are registered/started and is set to 0 when the watchdog threads are
59	* unregistered/stopped, so it is an indicator whether the threads exist.
60	*/
61	static int __read_mostly watchdog_running;
62	/*
63	* If a subsystem has a need to deactivate the watchdog temporarily, it
64	* can use the suspend/resume interface to achieve this. The content of
65	* the 'watchdog_suspended' variable reflects this state. Existing threads
66	* are parked/unparked by the lockup_detector_{suspend|resume} functions
67	* (see comment blocks pertaining to those functions for further details).
68	*
69	* 'watchdog_suspended' also prevents threads from being registered/started
70	* or unregistered/stopped via parameters in /proc/sys/kernel, so the state
71	* of 'watchdog_running' cannot change while the watchdog is deactivated
72	* temporarily (see related code in 'proc' handlers).
73	*/
74	static int __read_mostly watchdog_suspended;
75
76	static u64 __read_mostly sample_period;
77
78	static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
79	static DEFINE_PER_CPU(struct task_struct *, softlockup_watchdog);
80	static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
81	static DEFINE_PER_CPU(bool, softlockup_touch_sync);
82	static DEFINE_PER_CPU(bool, soft_watchdog_warn);
83	DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
84	static DEFINE_PER_CPU(unsigned long, soft_lockup_hrtimer_cnt);
85	static DEFINE_PER_CPU(struct task_struct *, softlockup_task_ptr_saved);
86	DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
87	static unsigned long soft_lockup_nmi_warn;
88
89	unsigned int __read_mostly softlockup_panic =
90	CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE;
91
92	static int __init softlockup_panic_setup(char *str)
93	{
94	softlockup_panic = simple_strtoul(str, NULL, 0);
95
96	return 1;
97	}
98	__setup("softlockup_panic=", softlockup_panic_setup);
99
100	static int __init nowatchdog_setup(char *str)
101	{
102	watchdog_enabled = 0;
103	return 1;
104	}
105	__setup("nowatchdog", nowatchdog_setup);
106
107	static int __init nosoftlockup_setup(char *str)
108	{
109	watchdog_enabled &= ~SOFT_WATCHDOG_ENABLED;
110	return 1;
111	}
112	__setup("nosoftlockup", nosoftlockup_setup);
113
114	#ifdef CONFIG_SMP
115	static int __init softlockup_all_cpu_backtrace_setup(char *str)
116	{
117	sysctl_softlockup_all_cpu_backtrace =
118	!!simple_strtol(str, NULL, 0);
119	return 1;
120	}
121	__setup("softlockup_all_cpu_backtrace=", softlockup_all_cpu_backtrace_setup);
122	static int __init hardlockup_all_cpu_backtrace_setup(char *str)
123	{
124	sysctl_hardlockup_all_cpu_backtrace =
125	!!simple_strtol(str, NULL, 0);
126	return 1;
127	}
128	__setup("hardlockup_all_cpu_backtrace=", hardlockup_all_cpu_backtrace_setup);
129	#endif
130
131	/*
132	* Hard-lockup warnings should be triggered after just a few seconds. Soft-
133	* lockups can have false positives under extreme conditions. So we generally
134	* want a higher threshold for soft lockups than for hard lockups. So we couple
135	* the thresholds with a factor: we make the soft threshold twice the amount of
136	* time the hard threshold is.
137	*/
138	static int get_softlockup_thresh(void)
139	{
140	return watchdog_thresh * 2;
141	}
142
143	/*
144	* Returns seconds, approximately. We don't need nanosecond
145	* resolution, and we don't need to waste time with a big divide when
146	* 2^30ns == 1.074s.
147	*/
148	static unsigned long get_timestamp(void)
149	{
150	return running_clock() >> 30LL; /* 2^30 ~= 10^9 */
151	}
152
153	static void set_sample_period(void)
154	{
155	/*
156	* convert watchdog_thresh from seconds to ns
157	* the divide by 5 is to give hrtimer several chances (two
158	* or three with the current relation between the soft
159	* and hard thresholds) to increment before the
160	* hardlockup detector generates a warning
161	*/
162	sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5);
163	}
164
165	/* Commands for resetting the watchdog */
166	static void __touch_watchdog(void)
167	{
168	__this_cpu_write(watchdog_touch_ts, get_timestamp());
169	}
170
171	/**
172	* touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
173	*
174	* Call when the scheduler may have stalled for legitimate reasons
175	* preventing the watchdog task from executing - e.g. the scheduler
176	* entering idle state. This should only be used for scheduler events.
177	* Use touch_softlockup_watchdog() for everything else.
178	*/
179	void touch_softlockup_watchdog_sched(void)
180	{
181	/*
182	* Preemption can be enabled. It doesn't matter which CPU's timestamp
183	* gets zeroed here, so use the raw_ operation.
184	*/
185	raw_cpu_write(watchdog_touch_ts, 0);
186	}
187
188	void touch_softlockup_watchdog(void)
189	{
190	touch_softlockup_watchdog_sched();
191	wq_watchdog_touch(raw_smp_processor_id());
192	}
193	EXPORT_SYMBOL(touch_softlockup_watchdog);
194
195	void touch_all_softlockup_watchdogs(void)
196	{
197	int cpu;
198
199	/*
200	* this is done lockless
201	* do we care if a 0 races with a timestamp?
202	* all it means is the softlock check starts one cycle later
203	*/
204	for_each_watchdog_cpu(cpu)
205	per_cpu(watchdog_touch_ts, cpu) = 0;
206	wq_watchdog_touch(-1);
207	}
208
209	void touch_softlockup_watchdog_sync(void)
210	{
211	__this_cpu_write(softlockup_touch_sync, true);
212	__this_cpu_write(watchdog_touch_ts, 0);
213	}
214
215	/* watchdog detector functions */
216	#ifdef CONFIG_HARDLOCKUP_DETECTOR_NMI
217	bool is_hardlockup(void)
218	{
219	unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
220
221	if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
222	return true;
223
224	__this_cpu_write(hrtimer_interrupts_saved, hrint);
225	return false;
226	}
227	#endif
228
229	static int is_softlockup(unsigned long touch_ts)
230	{
231	unsigned long now = get_timestamp();
232
233	if ((watchdog_enabled & SOFT_WATCHDOG_ENABLED) && watchdog_thresh){
234	/* Warn about unreasonable delays. */
235	if (time_after(now, touch_ts + get_softlockup_thresh()))
236	return now - touch_ts;
237	}
238	return 0;
239	}
240
241	static void watchdog_interrupt_count(void)
242	{
243	__this_cpu_inc(hrtimer_interrupts);
244	}
245
246	unsigned long watchdog_get_interrupt_count_cpu(int cpu)
247	{
248	return per_cpu(hrtimer_interrupts, cpu);
249	}
250	/*
251	* These two functions are mostly architecture specific
252	* defining them as weak here.
253	*/
254	int __weak watchdog_nmi_enable(unsigned int cpu)
255	{
256	return 0;
257	}
258	void __weak watchdog_nmi_disable(unsigned int cpu)
259	{
260	}
261
262	static int watchdog_enable_all_cpus(void);
263	static void watchdog_disable_all_cpus(void);
264
265	/* watchdog kicker functions */
266	static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
267	{
268	unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
269	struct pt_regs *regs = get_irq_regs();
270	int duration;
271	int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
272
273	if (atomic_read(&watchdog_park_in_progress) != 0)
274	return HRTIMER_NORESTART;
275
276	/* kick the hardlockup detector */
277	watchdog_interrupt_count();
278
279	/* test for hardlockups on the next cpu */
280	watchdog_check_hardlockup_other_cpu();
281
282	/* kick the softlockup detector */
283	wake_up_process(__this_cpu_read(softlockup_watchdog));
284
285	/* .. and repeat */
286	hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
287
288	if (touch_ts == 0) {
289	if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
290	/*
291	* If the time stamp was touched atomically
292	* make sure the scheduler tick is up to date.
293	*/
294	__this_cpu_write(softlockup_touch_sync, false);
295	sched_clock_tick();
296	}
297
298	/* Clear the guest paused flag on watchdog reset */
299	kvm_check_and_clear_guest_paused();
300	__touch_watchdog();
301	return HRTIMER_RESTART;
302	}
303
304	/* check for a softlockup
305	* This is done by making sure a high priority task is
306	* being scheduled. The task touches the watchdog to
307	* indicate it is getting cpu time. If it hasn't then
308	* this is a good indication some task is hogging the cpu
309	*/
310	duration = is_softlockup(touch_ts);
311	if (unlikely(duration)) {
312	/*
313	* If a virtual machine is stopped by the host it can look to
314	* the watchdog like a soft lockup, check to see if the host
315	* stopped the vm before we issue the warning
316	*/
317	if (kvm_check_and_clear_guest_paused())
318	return HRTIMER_RESTART;
319
320	/* only warn once */
321	if (__this_cpu_read(soft_watchdog_warn) == true) {
322	/*
323	* When multiple processes are causing softlockups the
324	* softlockup detector only warns on the first one
325	* because the code relies on a full quiet cycle to
326	* re-arm. The second process prevents the quiet cycle
327	* and never gets reported. Use task pointers to detect
328	* this.
329	*/
330	if (__this_cpu_read(softlockup_task_ptr_saved) !=
331	current) {
332	__this_cpu_write(soft_watchdog_warn, false);
333	__touch_watchdog();
334	}
335	return HRTIMER_RESTART;
336	}
337
338	if (softlockup_all_cpu_backtrace) {
339	/* Prevent multiple soft-lockup reports if one cpu is already
340	* engaged in dumping cpu back traces
341	*/
342	if (test_and_set_bit(0, &soft_lockup_nmi_warn)) {
343	/* Someone else will report us. Let's give up */
344	__this_cpu_write(soft_watchdog_warn, true);
345	return HRTIMER_RESTART;
346	}
347	}
348
349	pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
350	smp_processor_id(), duration,
351	current->comm, task_pid_nr(current));
352	__this_cpu_write(softlockup_task_ptr_saved, current);
353	print_modules();
354	print_irqtrace_events(current);
355	if (regs)
356	show_regs(regs);
357	else
358	dump_stack();
359
360	if (softlockup_all_cpu_backtrace) {
361	/* Avoid generating two back traces for current
362	* given that one is already made above
363	*/
364	trigger_allbutself_cpu_backtrace();
365
366	clear_bit(0, &soft_lockup_nmi_warn);
367	/* Barrier to sync with other cpus */
368	smp_mb__after_atomic();
369	}
370
371	add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
372	if (softlockup_panic)
373	panic("softlockup: hung tasks");
374	__this_cpu_write(soft_watchdog_warn, true);
375	} else
376	__this_cpu_write(soft_watchdog_warn, false);
377
378	return HRTIMER_RESTART;
379	}
380
381	static void watchdog_set_prio(unsigned int policy, unsigned int prio)
382	{
383	struct sched_param param = { .sched_priority = prio };
384
385	sched_setscheduler(current, policy, &param);
386	}
387
388	static void watchdog_enable(unsigned int cpu)
389	{
390	struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer);
391
392	/* kick off the timer for the hardlockup detector */
393	hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
394	hrtimer->function = watchdog_timer_fn;
395
396	/* Enable the perf event */
397	watchdog_nmi_enable(cpu);
398
399	/* done here because hrtimer_start can only pin to smp_processor_id() */
400	hrtimer_start(hrtimer, ns_to_ktime(sample_period),
401	HRTIMER_MODE_REL_PINNED);
402
403	/* initialize timestamp */
404	watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1);
405	__touch_watchdog();
406	}
407
408	static void watchdog_disable(unsigned int cpu)
409	{
410	struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer);
411
412	watchdog_set_prio(SCHED_NORMAL, 0);
413	hrtimer_cancel(hrtimer);
414	/* disable the perf event */
415	watchdog_nmi_disable(cpu);
416	}
417
418	static void watchdog_cleanup(unsigned int cpu, bool online)
419	{
420	watchdog_disable(cpu);
421	}
422
423	static int watchdog_should_run(unsigned int cpu)
424	{
425	return __this_cpu_read(hrtimer_interrupts) !=
426	__this_cpu_read(soft_lockup_hrtimer_cnt);
427	}
428
429	/*
430	* The watchdog thread function - touches the timestamp.
431	*
432	* It only runs once every sample_period seconds (4 seconds by
433	* default) to reset the softlockup timestamp. If this gets delayed
434	* for more than 2*watchdog_thresh seconds then the debug-printout
435	* triggers in watchdog_timer_fn().
436	*/
437	static void watchdog(unsigned int cpu)
438	{
439	__this_cpu_write(soft_lockup_hrtimer_cnt,
440	__this_cpu_read(hrtimer_interrupts));
441	__touch_watchdog();
442
443	/*
444	* watchdog_nmi_enable() clears the NMI_WATCHDOG_ENABLED bit in the
445	* failure path. Check for failures that can occur asynchronously -
446	* for example, when CPUs are on-lined - and shut down the hardware
447	* perf event on each CPU accordingly.
448	*
449	* The only non-obvious place this bit can be cleared is through
450	* watchdog_nmi_enable(), so a pr_info() is placed there. Placing a
451	* pr_info here would be too noisy as it would result in a message
452	* every few seconds if the hardlockup was disabled but the softlockup
453	* enabled.
454	*/
455	if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
456	watchdog_nmi_disable(cpu);
457	}
458
459	static struct smp_hotplug_thread watchdog_threads = {
460	.store = &softlockup_watchdog,
461	.thread_should_run = watchdog_should_run,
462	.thread_fn = watchdog,
463	.thread_comm = "watchdog/%u",
464	.setup = watchdog_enable,
465	.cleanup = watchdog_cleanup,
466	.park = watchdog_disable,
467	.unpark = watchdog_enable,
468	};
469
470	/*
471	* park all watchdog threads that are specified in 'watchdog_cpumask'
472	*
473	* This function returns an error if kthread_park() of a watchdog thread
474	* fails. In this situation, the watchdog threads of some CPUs can already
475	* be parked and the watchdog threads of other CPUs can still be runnable.
476	* Callers are expected to handle this special condition as appropriate in
477	* their context.
478	*
479	* This function may only be called in a context that is protected against
480	* races with CPU hotplug - for example, via get_online_cpus().
481	*/
482	static int watchdog_park_threads(void)
483	{
484	int cpu, ret = 0;
485
486	atomic_set(&watchdog_park_in_progress, 1);
487
488	for_each_watchdog_cpu(cpu) {
489	ret = kthread_park(per_cpu(softlockup_watchdog, cpu));
490	if (ret)
491	break;
492	}
493
494	atomic_set(&watchdog_park_in_progress, 0);
495
496	return ret;
497	}
498
499	/*
500	* unpark all watchdog threads that are specified in 'watchdog_cpumask'
501	*
502	* This function may only be called in a context that is protected against
503	* races with CPU hotplug - for example, via get_online_cpus().
504	*/
505	static void watchdog_unpark_threads(void)
506	{
507	int cpu;
508
509	for_each_watchdog_cpu(cpu)
510	kthread_unpark(per_cpu(softlockup_watchdog, cpu));
511	}
512
513	/*
514	* Suspend the hard and soft lockup detector by parking the watchdog threads.
515	*/
516	int lockup_detector_suspend(void)
517	{
518	int ret = 0;
519
520	get_online_cpus();
521	mutex_lock(&watchdog_proc_mutex);
522	/*
523	* Multiple suspend requests can be active in parallel (counted by
524	* the 'watchdog_suspended' variable). If the watchdog threads are
525	* running, the first caller takes care that they will be parked.
526	* The state of 'watchdog_running' cannot change while a suspend
527	* request is active (see related code in 'proc' handlers).
528	*/
529	if (watchdog_running && !watchdog_suspended)
530	ret = watchdog_park_threads();
531
532	if (ret == 0)
533	watchdog_suspended++;
534	else {
535	watchdog_disable_all_cpus();
536	pr_err("Failed to suspend lockup detectors, disabled\n");
537	watchdog_enabled = 0;
538	}
539
540	mutex_unlock(&watchdog_proc_mutex);
541
542	return ret;
543	}
544
545	/*
546	* Resume the hard and soft lockup detector by unparking the watchdog threads.
547	*/
548	void lockup_detector_resume(void)
549	{
550	mutex_lock(&watchdog_proc_mutex);
551
552	watchdog_suspended--;
553	/*
554	* The watchdog threads are unparked if they were previously running
555	* and if there is no more active suspend request.
556	*/
557	if (watchdog_running && !watchdog_suspended)
558	watchdog_unpark_threads();
559
560	mutex_unlock(&watchdog_proc_mutex);
561	put_online_cpus();
562	}
563
564	static int update_watchdog_all_cpus(void)
565	{
566	int ret;
567
568	ret = watchdog_park_threads();
569	if (ret)
570	return ret;
571
572	watchdog_unpark_threads();
573
574	return 0;
575	}
576
577	static int watchdog_enable_all_cpus(void)
578	{
579	int err = 0;
580
581	if (!watchdog_running) {
582	err = smpboot_register_percpu_thread_cpumask(&watchdog_threads,
583	&watchdog_cpumask);
584	if (err)
585	pr_err("Failed to create watchdog threads, disabled\n");
586	else
587	watchdog_running = 1;
588	} else {
589	/*
590	* Enable/disable the lockup detectors or
591	* change the sample period 'on the fly'.
592	*/
593	err = update_watchdog_all_cpus();
594
595	if (err) {
596	watchdog_disable_all_cpus();
597	pr_err("Failed to update lockup detectors, disabled\n");
598	}
599	}
600
601	if (err)
602	watchdog_enabled = 0;
603
604	return err;
605	}
606
607	static void watchdog_disable_all_cpus(void)
608	{
609	if (watchdog_running) {
610	watchdog_running = 0;
611	smpboot_unregister_percpu_thread(&watchdog_threads);
612	}
613	}
614
615	#ifdef CONFIG_SYSCTL
616
617	/*
618	* Update the run state of the lockup detectors.
619	*/
620	static int proc_watchdog_update(void)
621	{
622	int err = 0;
623
624	/*
625	* Watchdog threads won't be started if they are already active.
626	* The 'watchdog_running' variable in watchdog_*_all_cpus() takes
627	* care of this. If those threads are already active, the sample
628	* period will be updated and the lockup detectors will be enabled
629	* or disabled 'on the fly'.
630	*/
631	if (watchdog_enabled && watchdog_thresh)
632	err = watchdog_enable_all_cpus();
633	else
634	watchdog_disable_all_cpus();
635
636	return err;
637
638	}
639
640	/*
641	* common function for watchdog, nmi_watchdog and soft_watchdog parameter
642	*
643	* caller | table->data points to | 'which' contains the flag(s)
644	* -------------------|-----------------------|-----------------------------
645	* proc_watchdog | watchdog_user_enabled | NMI_WATCHDOG_ENABLED or'ed
646	* | | with SOFT_WATCHDOG_ENABLED
647	* -------------------|-----------------------|-----------------------------
648	* proc_nmi_watchdog | nmi_watchdog_enabled | NMI_WATCHDOG_ENABLED
649	* -------------------|-----------------------|-----------------------------
650	* proc_soft_watchdog | soft_watchdog_enabled | SOFT_WATCHDOG_ENABLED
651	*/
652	static int proc_watchdog_common(int which, struct ctl_table *table, int write,
653	void __user *buffer, size_t *lenp, loff_t *ppos)
654	{
655	int err, old, new;
656	int *watchdog_param = (int *)table->data;
657
658	get_online_cpus();
659	mutex_lock(&watchdog_proc_mutex);
660
661	if (watchdog_suspended) {
662	/* no parameter changes allowed while watchdog is suspended */
663	err = -EAGAIN;
664	goto out;
665	}
666
667	/*
668	* If the parameter is being read return the state of the corresponding
669	* bit(s) in 'watchdog_enabled', else update 'watchdog_enabled' and the
670	* run state of the lockup detectors.
671	*/
672	if (!write) {
673	*watchdog_param = (watchdog_enabled & which) != 0;
674	err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
675	} else {
676	err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
677	if (err)
678	goto out;
679
680	/*
681	* There is a race window between fetching the current value
682	* from 'watchdog_enabled' and storing the new value. During
683	* this race window, watchdog_nmi_enable() can sneak in and
684	* clear the NMI_WATCHDOG_ENABLED bit in 'watchdog_enabled'.
685	* The 'cmpxchg' detects this race and the loop retries.
686	*/
687	do {
688	old = watchdog_enabled;
689	/*
690	* If the parameter value is not zero set the
691	* corresponding bit(s), else clear it(them).
692	*/
693	if (*watchdog_param)
694	new = old | which;
695	else
696	new = old & ~which;
697	} while (cmpxchg(&watchdog_enabled, old, new) != old);
698
699	/*
700	* Update the run state of the lockup detectors. There is _no_
701	* need to check the value returned by proc_watchdog_update()
702	* and to restore the previous value of 'watchdog_enabled' as
703	* both lockup detectors are disabled if proc_watchdog_update()
704	* returns an error.
705	*/
706	if (old == new)
707	goto out;
708
709	err = proc_watchdog_update();
710	}
711	out:
712	mutex_unlock(&watchdog_proc_mutex);
713	put_online_cpus();
714	return err;
715	}
716
717	/*
718	* /proc/sys/kernel/watchdog
719	*/
720	int proc_watchdog(struct ctl_table *table, int write,
721	void __user *buffer, size_t *lenp, loff_t *ppos)
722	{
723	return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED,
724	table, write, buffer, lenp, ppos);
725	}
726
727	/*
728	* /proc/sys/kernel/nmi_watchdog
729	*/
730	int proc_nmi_watchdog(struct ctl_table *table, int write,
731	void __user *buffer, size_t *lenp, loff_t *ppos)
732	{
733	return proc_watchdog_common(NMI_WATCHDOG_ENABLED,
734	table, write, buffer, lenp, ppos);
735	}
736
737	/*
738	* /proc/sys/kernel/soft_watchdog
739	*/
740	int proc_soft_watchdog(struct ctl_table *table, int write,
741	void __user *buffer, size_t *lenp, loff_t *ppos)
742	{
743	return proc_watchdog_common(SOFT_WATCHDOG_ENABLED,
744	table, write, buffer, lenp, ppos);
745	}
746
747	/*
748	* /proc/sys/kernel/watchdog_thresh
749	*/
750	int proc_watchdog_thresh(struct ctl_table *table, int write,
751	void __user *buffer, size_t *lenp, loff_t *ppos)
752	{
753	int err, old, new;
754
755	get_online_cpus();
756	mutex_lock(&watchdog_proc_mutex);
757
758	if (watchdog_suspended) {
759	/* no parameter changes allowed while watchdog is suspended */
760	err = -EAGAIN;
761	goto out;
762	}
763
764	old = ACCESS_ONCE(watchdog_thresh);
765	err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
766
767	if (err || !write)
768	goto out;
769
770	/*
771	* Update the sample period. Restore on failure.
772	*/
773	new = ACCESS_ONCE(watchdog_thresh);
774	if (old == new)
775	goto out;
776
777	set_sample_period();
778	err = proc_watchdog_update();
779	if (err) {
780	watchdog_thresh = old;
781	set_sample_period();
782	}
783	out:
784	mutex_unlock(&watchdog_proc_mutex);
785	put_online_cpus();
786	return err;
787	}
788
789	/*
790	* The cpumask is the mask of possible cpus that the watchdog can run
791	* on, not the mask of cpus it is actually running on. This allows the
792	* user to specify a mask that will include cpus that have not yet
793	* been brought online, if desired.
794	*/
795	int proc_watchdog_cpumask(struct ctl_table *table, int write,
796	void __user *buffer, size_t *lenp, loff_t *ppos)
797	{
798	int err;
799
800	get_online_cpus();
801	mutex_lock(&watchdog_proc_mutex);
802
803	if (watchdog_suspended) {
804	/* no parameter changes allowed while watchdog is suspended */
805	err = -EAGAIN;
806	goto out;
807	}
808
809	err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
810	if (!err && write) {
811	/* Remove impossible cpus to keep sysctl output cleaner. */
812	cpumask_and(&watchdog_cpumask, &watchdog_cpumask,
813	cpu_possible_mask);
814
815	if (watchdog_running) {
816	/*
817	* Failure would be due to being unable to allocate
818	* a temporary cpumask, so we are likely not in a
819	* position to do much else to make things better.
820	*/
821	if (smpboot_update_cpumask_percpu_thread(
822	&watchdog_threads, &watchdog_cpumask) != 0)
823	pr_err("cpumask update failed\n");
824	}
825	}
826	out:
827	mutex_unlock(&watchdog_proc_mutex);
828	put_online_cpus();
829	return err;
830	}
831
832	#endif /* CONFIG_SYSCTL */
833
834	void __init lockup_detector_init(void)
835	{
836	set_sample_period();
837
838	#ifdef CONFIG_NO_HZ_FULL
839	if (tick_nohz_full_enabled()) {
840	pr_info("Disabling watchdog on nohz_full cores by default\n");
841	cpumask_copy(&watchdog_cpumask, housekeeping_mask);
842	} else
843	cpumask_copy(&watchdog_cpumask, cpu_possible_mask);
844	#else
845	cpumask_copy(&watchdog_cpumask, cpu_possible_mask);
846	#endif
847
848	if (watchdog_enabled)
849	watchdog_enable_all_cpus();
850	}
851