path: root/kernel/exit.c (plain)
blob: a4959643cc98287874e3b48766a2b022810ae3b0

1	/*
2	* linux/kernel/exit.c
3	*
4	* Copyright (C) 1991, 1992 Linus Torvalds
5	*/
6
7	#include <linux/mm.h>
8	#include <linux/slab.h>
9	#include <linux/interrupt.h>
10	#include <linux/module.h>
11	#include <linux/capability.h>
12	#include <linux/completion.h>
13	#include <linux/personality.h>
14	#include <linux/tty.h>
15	#include <linux/iocontext.h>
16	#include <linux/key.h>
17	#include <linux/security.h>
18	#include <linux/cpu.h>
19	#include <linux/acct.h>
20	#include <linux/tsacct_kern.h>
21	#include <linux/file.h>
22	#include <linux/fdtable.h>
23	#include <linux/freezer.h>
24	#include <linux/binfmts.h>
25	#include <linux/nsproxy.h>
26	#include <linux/pid_namespace.h>
27	#include <linux/ptrace.h>
28	#include <linux/profile.h>
29	#include <linux/mount.h>
30	#include <linux/proc_fs.h>
31	#include <linux/kthread.h>
32	#include <linux/mempolicy.h>
33	#include <linux/taskstats_kern.h>
34	#include <linux/delayacct.h>
35	#include <linux/cgroup.h>
36	#include <linux/syscalls.h>
37	#include <linux/signal.h>
38	#include <linux/posix-timers.h>
39	#include <linux/cn_proc.h>
40	#include <linux/mutex.h>
41	#include <linux/futex.h>
42	#include <linux/pipe_fs_i.h>
43	#include <linux/audit.h> /* for audit_free() */
44	#include <linux/resource.h>
45	#include <linux/blkdev.h>
46	#include <linux/task_io_accounting_ops.h>
47	#include <linux/tracehook.h>
48	#include <linux/fs_struct.h>
49	#include <linux/init_task.h>
50	#include <linux/perf_event.h>
51	#include <trace/events/sched.h>
52	#include <linux/hw_breakpoint.h>
53	#include <linux/oom.h>
54	#include <linux/writeback.h>
55	#include <linux/shm.h>
56	#include <linux/kcov.h>
57
58	#include "sched/tune.h"
59
60	#include <asm/uaccess.h>
61	#include <asm/unistd.h>
62	#include <asm/pgtable.h>
63	#include <asm/mmu_context.h>
64
65	static void __unhash_process(struct task_struct *p, bool group_dead)
66	{
67	nr_threads--;
68	detach_pid(p, PIDTYPE_PID);
69	if (group_dead) {
70	detach_pid(p, PIDTYPE_PGID);
71	detach_pid(p, PIDTYPE_SID);
72
73	list_del_rcu(&p->tasks);
74	list_del_init(&p->sibling);
75	__this_cpu_dec(process_counts);
76	}
77	list_del_rcu(&p->thread_group);
78	list_del_rcu(&p->thread_node);
79	}
80
81	/*
82	* This function expects the tasklist_lock write-locked.
83	*/
84	static void __exit_signal(struct task_struct *tsk)
85	{
86	struct signal_struct *sig = tsk->signal;
87	bool group_dead = thread_group_leader(tsk);
88	struct sighand_struct *sighand;
89	struct tty_struct *uninitialized_var(tty);
90	cputime_t utime, stime;
91
92	sighand = rcu_dereference_check(tsk->sighand,
93	lockdep_tasklist_lock_is_held());
94	spin_lock(&sighand->siglock);
95
96	posix_cpu_timers_exit(tsk);
97	if (group_dead) {
98	posix_cpu_timers_exit_group(tsk);
99	tty = sig->tty;
100	sig->tty = NULL;
101	} else {
102	/*
103	* This can only happen if the caller is de_thread().
104	* FIXME: this is the temporary hack, we should teach
105	* posix-cpu-timers to handle this case correctly.
106	*/
107	if (unlikely(has_group_leader_pid(tsk)))
108	posix_cpu_timers_exit_group(tsk);
109
110	/*
111	* If there is any task waiting for the group exit
112	* then notify it:
113	*/
114	if (sig->notify_count > 0 && !--sig->notify_count)
115	wake_up_process(sig->group_exit_task);
116
117	if (tsk == sig->curr_target)
118	sig->curr_target = next_thread(tsk);
119	}
120
121	/*
122	* Accumulate here the counters for all threads as they die. We could
123	* skip the group leader because it is the last user of signal_struct,
124	* but we want to avoid the race with thread_group_cputime() which can
125	* see the empty ->thread_head list.
126	*/
127	task_cputime(tsk, &utime, &stime);
128	write_seqlock(&sig->stats_lock);
129	sig->utime += utime;
130	sig->stime += stime;
131	sig->gtime += task_gtime(tsk);
132	sig->min_flt += tsk->min_flt;
133	sig->maj_flt += tsk->maj_flt;
134	sig->nvcsw += tsk->nvcsw;
135	sig->nivcsw += tsk->nivcsw;
136	sig->inblock += task_io_get_inblock(tsk);
137	sig->oublock += task_io_get_oublock(tsk);
138	task_io_accounting_add(&sig->ioac, &tsk->ioac);
139	sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
140	sig->nr_threads--;
141	__unhash_process(tsk, group_dead);
142	write_sequnlock(&sig->stats_lock);
143
144	/*
145	* Do this under ->siglock, we can race with another thread
146	* doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals.
147	*/
148	flush_sigqueue(&tsk->pending);
149	tsk->sighand = NULL;
150	spin_unlock(&sighand->siglock);
151
152	__cleanup_sighand(sighand);
153	clear_tsk_thread_flag(tsk, TIF_SIGPENDING);
154	if (group_dead) {
155	flush_sigqueue(&sig->shared_pending);
156	tty_kref_put(tty);
157	}
158	}
159
160	static void delayed_put_task_struct(struct rcu_head *rhp)
161	{
162	struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
163
164	perf_event_delayed_put(tsk);
165	trace_sched_process_free(tsk);
166	put_task_struct(tsk);
167	}
168
169
170	void release_task(struct task_struct *p)
171	{
172	struct task_struct *leader;
173	int zap_leader;
174	repeat:
175	/* don't need to get the RCU readlock here - the process is dead and
176	* can't be modifying its own credentials. But shut RCU-lockdep up */
177	rcu_read_lock();
178	atomic_dec(&__task_cred(p)->user->processes);
179	rcu_read_unlock();
180
181	proc_flush_task(p);
182
183	write_lock_irq(&tasklist_lock);
184	ptrace_release_task(p);
185	__exit_signal(p);
186
187	/*
188	* If we are the last non-leader member of the thread
189	* group, and the leader is zombie, then notify the
190	* group leader's parent process. (if it wants notification.)
191	*/
192	zap_leader = 0;
193	leader = p->group_leader;
194	if (leader != p && thread_group_empty(leader)
195	&& leader->exit_state == EXIT_ZOMBIE) {
196	/*
197	* If we were the last child thread and the leader has
198	* exited already, and the leader's parent ignores SIGCHLD,
199	* then we are the one who should release the leader.
200	*/
201	zap_leader = do_notify_parent(leader, leader->exit_signal);
202	if (zap_leader)
203	leader->exit_state = EXIT_DEAD;
204	}
205
206	write_unlock_irq(&tasklist_lock);
207	release_thread(p);
208	call_rcu(&p->rcu, delayed_put_task_struct);
209
210	p = leader;
211	if (unlikely(zap_leader))
212	goto repeat;
213	}
214
215	/*
216	* Note that if this function returns a valid task_struct pointer (!NULL)
217	* task->usage must remain >0 for the duration of the RCU critical section.
218	*/
219	struct task_struct *task_rcu_dereference(struct task_struct **ptask)
220	{
221	struct sighand_struct *sighand;
222	struct task_struct *task;
223
224	/*
225	* We need to verify that release_task() was not called and thus
226	* delayed_put_task_struct() can't run and drop the last reference
227	* before rcu_read_unlock(). We check task->sighand != NULL,
228	* but we can read the already freed and reused memory.
229	*/
230	retry:
231	task = rcu_dereference(*ptask);
232	if (!task)
233	return NULL;
234
235	probe_kernel_address(&task->sighand, sighand);
236
237	/*
238	* Pairs with atomic_dec_and_test() in put_task_struct(). If this task
239	* was already freed we can not miss the preceding update of this
240	* pointer.
241	*/
242	smp_rmb();
243	if (unlikely(task != READ_ONCE(*ptask)))
244	goto retry;
245
246	/*
247	* We've re-checked that "task == *ptask", now we have two different
248	* cases:
249	*
250	* 1. This is actually the same task/task_struct. In this case
251	* sighand != NULL tells us it is still alive.
252	*
253	* 2. This is another task which got the same memory for task_struct.
254	* We can't know this of course, and we can not trust
255	* sighand != NULL.
256	*
257	* In this case we actually return a random value, but this is
258	* correct.
259	*
260	* If we return NULL - we can pretend that we actually noticed that
261	* *ptask was updated when the previous task has exited. Or pretend
262	* that probe_slab_address(&sighand) reads NULL.
263	*
264	* If we return the new task (because sighand is not NULL for any
265	* reason) - this is fine too. This (new) task can't go away before
266	* another gp pass.
267	*
268	* And note: We could even eliminate the false positive if re-read
269	* task->sighand once again to avoid the falsely NULL. But this case
270	* is very unlikely so we don't care.
271	*/
272	if (!sighand)
273	return NULL;
274
275	return task;
276	}
277
278	struct task_struct *try_get_task_struct(struct task_struct **ptask)
279	{
280	struct task_struct *task;
281
282	rcu_read_lock();
283	task = task_rcu_dereference(ptask);
284	if (task)
285	get_task_struct(task);
286	rcu_read_unlock();
287
288	return task;
289	}
290
291	/*
292	* Determine if a process group is "orphaned", according to the POSIX
293	* definition in 2.2.2.52. Orphaned process groups are not to be affected
294	* by terminal-generated stop signals. Newly orphaned process groups are
295	* to receive a SIGHUP and a SIGCONT.
296	*
297	* "I ask you, have you ever known what it is to be an orphan?"
298	*/
299	static int will_become_orphaned_pgrp(struct pid *pgrp,
300	struct task_struct *ignored_task)
301	{
302	struct task_struct *p;
303
304	do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
305	if ((p == ignored_task) ||
306	(p->exit_state && thread_group_empty(p)) ||
307	is_global_init(p->real_parent))
308	continue;
309
310	if (task_pgrp(p->real_parent) != pgrp &&
311	task_session(p->real_parent) == task_session(p))
312	return 0;
313	} while_each_pid_task(pgrp, PIDTYPE_PGID, p);
314
315	return 1;
316	}
317
318	int is_current_pgrp_orphaned(void)
319	{
320	int retval;
321
322	read_lock(&tasklist_lock);
323	retval = will_become_orphaned_pgrp(task_pgrp(current), NULL);
324	read_unlock(&tasklist_lock);
325
326	return retval;
327	}
328
329	static bool has_stopped_jobs(struct pid *pgrp)
330	{
331	struct task_struct *p;
332
333	do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
334	if (p->signal->flags & SIGNAL_STOP_STOPPED)
335	return true;
336	} while_each_pid_task(pgrp, PIDTYPE_PGID, p);
337
338	return false;
339	}
340
341	/*
342	* Check to see if any process groups have become orphaned as
343	* a result of our exiting, and if they have any stopped jobs,
344	* send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2)
345	*/
346	static void
347	kill_orphaned_pgrp(struct task_struct *tsk, struct task_struct *parent)
348	{
349	struct pid *pgrp = task_pgrp(tsk);
350	struct task_struct *ignored_task = tsk;
351
352	if (!parent)
353	/* exit: our father is in a different pgrp than
354	* we are and we were the only connection outside.
355	*/
356	parent = tsk->real_parent;
357	else
358	/* reparent: our child is in a different pgrp than
359	* we are, and it was the only connection outside.
360	*/
361	ignored_task = NULL;
362
363	if (task_pgrp(parent) != pgrp &&
364	task_session(parent) == task_session(tsk) &&
365	will_become_orphaned_pgrp(pgrp, ignored_task) &&
366	has_stopped_jobs(pgrp)) {
367	__kill_pgrp_info(SIGHUP, SEND_SIG_PRIV, pgrp);
368	__kill_pgrp_info(SIGCONT, SEND_SIG_PRIV, pgrp);
369	}
370	}
371
372	#ifdef CONFIG_MEMCG
373	/*
374	* A task is exiting. If it owned this mm, find a new owner for the mm.
375	*/
376	void mm_update_next_owner(struct mm_struct *mm)
377	{
378	struct task_struct *c, *g, *p = current;
379
380	retry:
381	/*
382	* If the exiting or execing task is not the owner, it's
383	* someone else's problem.
384	*/
385	if (mm->owner != p)
386	return;
387	/*
388	* The current owner is exiting/execing and there are no other
389	* candidates. Do not leave the mm pointing to a possibly
390	* freed task structure.
391	*/
392	if (atomic_read(&mm->mm_users) <= 1) {
393	mm->owner = NULL;
394	return;
395	}
396
397	read_lock(&tasklist_lock);
398	/*
399	* Search in the children
400	*/
401	list_for_each_entry(c, &p->children, sibling) {
402	if (c->mm == mm)
403	goto assign_new_owner;
404	}
405
406	/*
407	* Search in the siblings
408	*/
409	list_for_each_entry(c, &p->real_parent->children, sibling) {
410	if (c->mm == mm)
411	goto assign_new_owner;
412	}
413
414	/*
415	* Search through everything else, we should not get here often.
416	*/
417	for_each_process(g) {
418	if (g->flags & PF_KTHREAD)
419	continue;
420	for_each_thread(g, c) {
421	if (c->mm == mm)
422	goto assign_new_owner;
423	if (c->mm)
424	break;
425	}
426	}
427	read_unlock(&tasklist_lock);
428	/*
429	* We found no owner yet mm_users > 1: this implies that we are
430	* most likely racing with swapoff (try_to_unuse()) or /proc or
431	* ptrace or page migration (get_task_mm()). Mark owner as NULL.
432	*/
433	mm->owner = NULL;
434	return;
435
436	assign_new_owner:
437	BUG_ON(c == p);
438	get_task_struct(c);
439	/*
440	* The task_lock protects c->mm from changing.
441	* We always want mm->owner->mm == mm
442	*/
443	task_lock(c);
444	/*
445	* Delay read_unlock() till we have the task_lock()
446	* to ensure that c does not slip away underneath us
447	*/
448	read_unlock(&tasklist_lock);
449	if (c->mm != mm) {
450	task_unlock(c);
451	put_task_struct(c);
452	goto retry;
453	}
454	mm->owner = c;
455	task_unlock(c);
456	put_task_struct(c);
457	}
458	#endif /* CONFIG_MEMCG */
459
460	/*
461	* Turn us into a lazy TLB process if we
462	* aren't already..
463	*/
464	static void exit_mm(struct task_struct *tsk)
465	{
466	struct mm_struct *mm = tsk->mm;
467	struct core_state *core_state;
468
469	mm_release(tsk, mm);
470	if (!mm)
471	return;
472	sync_mm_rss(mm);
473	/*
474	* Serialize with any possible pending coredump.
475	* We must hold mmap_sem around checking core_state
476	* and clearing tsk->mm. The core-inducing thread
477	* will increment ->nr_threads for each thread in the
478	* group with ->mm != NULL.
479	*/
480	down_read(&mm->mmap_sem);
481	core_state = mm->core_state;
482	if (core_state) {
483	struct core_thread self;
484
485	up_read(&mm->mmap_sem);
486
487	self.task = tsk;
488	self.next = xchg(&core_state->dumper.next, &self);
489	/*
490	* Implies mb(), the result of xchg() must be visible
491	* to core_state->dumper.
492	*/
493	if (atomic_dec_and_test(&core_state->nr_threads))
494	complete(&core_state->startup);
495
496	for (;;) {
497	set_task_state(tsk, TASK_UNINTERRUPTIBLE);
498	if (!self.task) /* see coredump_finish() */
499	break;
500	freezable_schedule();
501	}
502	__set_task_state(tsk, TASK_RUNNING);
503	down_read(&mm->mmap_sem);
504	}
505	atomic_inc(&mm->mm_count);
506	BUG_ON(mm != tsk->active_mm);
507	/* more a memory barrier than a real lock */
508	task_lock(tsk);
509	tsk->mm = NULL;
510	up_read(&mm->mmap_sem);
511	enter_lazy_tlb(mm, current);
512	task_unlock(tsk);
513	mm_update_next_owner(mm);
514	mmput(mm);
515	if (test_thread_flag(TIF_MEMDIE))
516	exit_oom_victim();
517	}
518
519	static struct task_struct *find_alive_thread(struct task_struct *p)
520	{
521	struct task_struct *t;
522
523	for_each_thread(p, t) {
524	if (!(t->flags & PF_EXITING))
525	return t;
526	}
527	return NULL;
528	}
529
530	static struct task_struct *find_child_reaper(struct task_struct *father,
531	struct list_head *dead)
532	__releases(&tasklist_lock)
533	__acquires(&tasklist_lock)
534	{
535	struct pid_namespace *pid_ns = task_active_pid_ns(father);
536	struct task_struct *reaper = pid_ns->child_reaper;
537	struct task_struct *p, *n;
538
539	if (likely(reaper != father))
540	return reaper;
541
542	reaper = find_alive_thread(father);
543	if (reaper) {
544	pid_ns->child_reaper = reaper;
545	return reaper;
546	}
547
548	write_unlock_irq(&tasklist_lock);
549	if (unlikely(pid_ns == &init_pid_ns)) {
550	panic("Attempted to kill init! exitcode=0x%08x\n",
551	father->signal->group_exit_code ?: father->exit_code);
552	}
553
554	list_for_each_entry_safe(p, n, dead, ptrace_entry) {
555	list_del_init(&p->ptrace_entry);
556	release_task(p);
557	}
558
559	zap_pid_ns_processes(pid_ns);
560	write_lock_irq(&tasklist_lock);
561
562	return father;
563	}
564
565	/*
566	* When we die, we re-parent all our children, and try to:
567	* 1. give them to another thread in our thread group, if such a member exists
568	* 2. give it to the first ancestor process which prctl'd itself as a
569	* child_subreaper for its children (like a service manager)
570	* 3. give it to the init process (PID 1) in our pid namespace
571	*/
572	static struct task_struct *find_new_reaper(struct task_struct *father,
573	struct task_struct *child_reaper)
574	{
575	struct task_struct *thread, *reaper;
576
577	thread = find_alive_thread(father);
578	if (thread)
579	return thread;
580
581	if (father->signal->has_child_subreaper) {
582	/*
583	* Find the first ->is_child_subreaper ancestor in our pid_ns.
584	* We start from father to ensure we can not look into another
585	* namespace, this is safe because all its threads are dead.
586	*/
587	for (reaper = father;
588	!same_thread_group(reaper, child_reaper);
589	reaper = reaper->real_parent) {
590	/* call_usermodehelper() descendants need this check */
591	if (reaper == &init_task)
592	break;
593	if (!reaper->signal->is_child_subreaper)
594	continue;
595	thread = find_alive_thread(reaper);
596	if (thread)
597	return thread;
598	}
599	}
600
601	return child_reaper;
602	}
603
604	/*
605	* Any that need to be release_task'd are put on the @dead list.
606	*/
607	static void reparent_leader(struct task_struct *father, struct task_struct *p,
608	struct list_head *dead)
609	{
610	if (unlikely(p->exit_state == EXIT_DEAD))
611	return;
612
613	/* We don't want people slaying init. */
614	p->exit_signal = SIGCHLD;
615
616	/* If it has exited notify the new parent about this child's death. */
617	if (!p->ptrace &&
618	p->exit_state == EXIT_ZOMBIE && thread_group_empty(p)) {
619	if (do_notify_parent(p, p->exit_signal)) {
620	p->exit_state = EXIT_DEAD;
621	list_add(&p->ptrace_entry, dead);
622	}
623	}
624
625	kill_orphaned_pgrp(p, father);
626	}
627
628	/*
629	* This does two things:
630	*
631	* A. Make init inherit all the child processes
632	* B. Check to see if any process groups have become orphaned
633	* as a result of our exiting, and if they have any stopped
634	* jobs, send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2)
635	*/
636	static void forget_original_parent(struct task_struct *father,
637	struct list_head *dead)
638	{
639	struct task_struct *p, *t, *reaper;
640
641	if (unlikely(!list_empty(&father->ptraced)))
642	exit_ptrace(father, dead);
643
644	/* Can drop and reacquire tasklist_lock */
645	reaper = find_child_reaper(father, dead);
646	if (list_empty(&father->children))
647	return;
648
649	reaper = find_new_reaper(father, reaper);
650	list_for_each_entry(p, &father->children, sibling) {
651	for_each_thread(p, t) {
652	t->real_parent = reaper;
653	BUG_ON((!t->ptrace) != (t->parent == father));
654	if (likely(!t->ptrace))
655	t->parent = t->real_parent;
656	if (t->pdeath_signal)
657	group_send_sig_info(t->pdeath_signal,
658	SEND_SIG_NOINFO, t);
659	}
660	/*
661	* If this is a threaded reparent there is no need to
662	* notify anyone anything has happened.
663	*/
664	if (!same_thread_group(reaper, father))
665	reparent_leader(father, p, dead);
666	}
667	list_splice_tail_init(&father->children, &reaper->children);
668	}
669
670	/*
671	* Send signals to all our closest relatives so that they know
672	* to properly mourn us..
673	*/
674	static void exit_notify(struct task_struct *tsk, int group_dead)
675	{
676	bool autoreap;
677	struct task_struct *p, *n;
678	LIST_HEAD(dead);
679
680	write_lock_irq(&tasklist_lock);
681	forget_original_parent(tsk, &dead);
682
683	if (group_dead)
684	kill_orphaned_pgrp(tsk->group_leader, NULL);
685
686	if (unlikely(tsk->ptrace)) {
687	int sig = thread_group_leader(tsk) &&
688	thread_group_empty(tsk) &&
689	!ptrace_reparented(tsk) ?
690	tsk->exit_signal : SIGCHLD;
691	autoreap = do_notify_parent(tsk, sig);
692	} else if (thread_group_leader(tsk)) {
693	autoreap = thread_group_empty(tsk) &&
694	do_notify_parent(tsk, tsk->exit_signal);
695	} else {
696	autoreap = true;
697	}
698
699	tsk->exit_state = autoreap ? EXIT_DEAD : EXIT_ZOMBIE;
700	if (tsk->exit_state == EXIT_DEAD)
701	list_add(&tsk->ptrace_entry, &dead);
702
703	/* mt-exec, de_thread() is waiting for group leader */
704	if (unlikely(tsk->signal->notify_count < 0))
705	wake_up_process(tsk->signal->group_exit_task);
706	write_unlock_irq(&tasklist_lock);
707
708	list_for_each_entry_safe(p, n, &dead, ptrace_entry) {
709	list_del_init(&p->ptrace_entry);
710	release_task(p);
711	}
712	}
713
714	#ifdef CONFIG_DEBUG_STACK_USAGE
715	static void check_stack_usage(void)
716	{
717	static DEFINE_SPINLOCK(low_water_lock);
718	static int lowest_to_date = THREAD_SIZE;
719	unsigned long free;
720
721	free = stack_not_used(current);
722
723	if (free >= lowest_to_date)
724	return;
725
726	spin_lock(&low_water_lock);
727	if (free < lowest_to_date) {
728	pr_info("%s (%d) used greatest stack depth: %lu bytes left\n",
729	current->comm, task_pid_nr(current), free);
730	lowest_to_date = free;
731	}
732	spin_unlock(&low_water_lock);
733	}
734	#else
735	static inline void check_stack_usage(void) {}
736	#endif
737
738	void __noreturn do_exit(long code)
739	{
740	struct task_struct *tsk = current;
741	int group_dead;
742	TASKS_RCU(int tasks_rcu_i);
743
744	profile_task_exit(tsk);
745	kcov_task_exit(tsk);
746
747	WARN_ON(blk_needs_flush_plug(tsk));
748
749	if (unlikely(in_interrupt()))
750	panic("Aiee, killing interrupt handler!");
751	if (unlikely(!tsk->pid))
752	panic("Attempted to kill the idle task!");
753
754	/*
755	* If do_exit is called because this processes oopsed, it's possible
756	* that get_fs() was left as KERNEL_DS, so reset it to USER_DS before
757	* continuing. Amongst other possible reasons, this is to prevent
758	* mm_release()->clear_child_tid() from writing to a user-controlled
759	* kernel address.
760	*/
761	set_fs(USER_DS);
762
763	ptrace_event(PTRACE_EVENT_EXIT, code);
764
765	validate_creds_for_do_exit(tsk);
766
767	/*
768	* We're taking recursive faults here in do_exit. Safest is to just
769	* leave this task alone and wait for reboot.
770	*/
771	if (unlikely(tsk->flags & PF_EXITING)) {
772	pr_alert("Fixing recursive fault but reboot is needed!\n");
773	/*
774	* We can do this unlocked here. The futex code uses
775	* this flag just to verify whether the pi state
776	* cleanup has been done or not. In the worst case it
777	* loops once more. We pretend that the cleanup was
778	* done as there is no way to return. Either the
779	* OWNER_DIED bit is set by now or we push the blocked
780	* task into the wait for ever nirwana as well.
781	*/
782	tsk->flags |= PF_EXITPIDONE;
783	set_current_state(TASK_UNINTERRUPTIBLE);
784	schedule();
785	}
786
787	exit_signals(tsk); /* sets PF_EXITING */
788
789	schedtune_exit_task(tsk);
790
791	/*
792	* Ensure that all new tsk->pi_lock acquisitions must observe
793	* PF_EXITING. Serializes against futex.c:attach_to_pi_owner().
794	*/
795	smp_mb();
796	/*
797	* Ensure that we must observe the pi_state in exit_mm() ->
798	* mm_release() -> exit_pi_state_list().
799	*/
800	raw_spin_unlock_wait(&tsk->pi_lock);
801
802	if (unlikely(in_atomic())) {
803	pr_info("note: %s[%d] exited with preempt_count %d\n",
804	current->comm, task_pid_nr(current),
805	preempt_count());
806	preempt_count_set(PREEMPT_ENABLED);
807	}
808
809	/* sync mm's RSS info before statistics gathering */
810	if (tsk->mm)
811	sync_mm_rss(tsk->mm);
812	acct_update_integrals(tsk);
813	group_dead = atomic_dec_and_test(&tsk->signal->live);
814	if (group_dead) {
815	hrtimer_cancel(&tsk->signal->real_timer);
816	exit_itimers(tsk->signal);
817	if (tsk->mm)
818	setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm);
819	}
820	acct_collect(code, group_dead);
821	if (group_dead)
822	tty_audit_exit();
823	audit_free(tsk);
824
825	tsk->exit_code = code;
826	taskstats_exit(tsk, group_dead);
827
828	exit_mm(tsk);
829
830	if (group_dead)
831	acct_process();
832	trace_sched_process_exit(tsk);
833
834	exit_sem(tsk);
835	exit_shm(tsk);
836	exit_files(tsk);
837	exit_fs(tsk);
838	if (group_dead)
839	disassociate_ctty(1);
840	exit_task_namespaces(tsk);
841	exit_task_work(tsk);
842	exit_thread(tsk);
843
844	/*
845	* Flush inherited counters to the parent - before the parent
846	* gets woken up by child-exit notifications.
847	*
848	* because of cgroup mode, must be called before cgroup_exit()
849	*/
850	perf_event_exit_task(tsk);
851
852	sched_autogroup_exit_task(tsk);
853	cgroup_exit(tsk);
854
855	/*
856	* FIXME: do that only when needed, using sched_exit tracepoint
857	*/
858	flush_ptrace_hw_breakpoint(tsk);
859
860	TASKS_RCU(preempt_disable());
861	TASKS_RCU(tasks_rcu_i = __srcu_read_lock(&tasks_rcu_exit_srcu));
862	TASKS_RCU(preempt_enable());
863	exit_notify(tsk, group_dead);
864	proc_exit_connector(tsk);
865	mpol_put_task_policy(tsk);
866	#ifdef CONFIG_FUTEX
867	if (unlikely(current->pi_state_cache))
868	kfree(current->pi_state_cache);
869	#endif
870	/*
871	* Make sure we are holding no locks:
872	*/
873	debug_check_no_locks_held();
874	/*
875	* We can do this unlocked here. The futex code uses this flag
876	* just to verify whether the pi state cleanup has been done
877	* or not. In the worst case it loops once more.
878	*/
879	tsk->flags |= PF_EXITPIDONE;
880
881	if (tsk->io_context)
882	exit_io_context(tsk);
883
884	if (tsk->splice_pipe)
885	free_pipe_info(tsk->splice_pipe);
886
887	if (tsk->task_frag.page)
888	put_page(tsk->task_frag.page);
889
890	validate_creds_for_do_exit(tsk);
891
892	check_stack_usage();
893	preempt_disable();
894	if (tsk->nr_dirtied)
895	__this_cpu_add(dirty_throttle_leaks, tsk->nr_dirtied);
896	exit_rcu();
897	TASKS_RCU(__srcu_read_unlock(&tasks_rcu_exit_srcu, tasks_rcu_i));
898
899	do_task_dead();
900	}
901	EXPORT_SYMBOL_GPL(do_exit);
902
903	void complete_and_exit(struct completion *comp, long code)
904	{
905	if (comp)
906	complete(comp);
907
908	do_exit(code);
909	}
910	EXPORT_SYMBOL(complete_and_exit);
911
912	SYSCALL_DEFINE1(exit, int, error_code)
913	{
914	do_exit((error_code&0xff)<<8);
915	}
916
917	/*
918	* Take down every thread in the group. This is called by fatal signals
919	* as well as by sys_exit_group (below).
920	*/
921	void
922	do_group_exit(int exit_code)
923	{
924	struct signal_struct *sig = current->signal;
925
926	BUG_ON(exit_code & 0x80); /* core dumps don't get here */
927
928	if (signal_group_exit(sig))
929	exit_code = sig->group_exit_code;
930	else if (!thread_group_empty(current)) {
931	struct sighand_struct *const sighand = current->sighand;
932
933	spin_lock_irq(&sighand->siglock);
934	if (signal_group_exit(sig))
935	/* Another thread got here before we took the lock. */
936	exit_code = sig->group_exit_code;
937	else {
938	sig->group_exit_code = exit_code;
939	sig->flags = SIGNAL_GROUP_EXIT;
940	zap_other_threads(current);
941	}
942	spin_unlock_irq(&sighand->siglock);
943	}
944
945	do_exit(exit_code);
946	/* NOTREACHED */
947	}
948
949	/*
950	* this kills every thread in the thread group. Note that any externally
951	* wait4()-ing process will get the correct exit code - even if this
952	* thread is not the thread group leader.
953	*/
954	SYSCALL_DEFINE1(exit_group, int, error_code)
955	{
956	do_group_exit((error_code & 0xff) << 8);
957	/* NOTREACHED */
958	return 0;
959	}
960
961	struct wait_opts {
962	enum pid_type wo_type;
963	int wo_flags;
964	struct pid *wo_pid;
965
966	struct siginfo __user *wo_info;
967	int __user *wo_stat;
968	struct rusage __user *wo_rusage;
969
970	wait_queue_t child_wait;
971	int notask_error;
972	};
973
974	static inline
975	struct pid *task_pid_type(struct task_struct *task, enum pid_type type)
976	{
977	if (type != PIDTYPE_PID)
978	task = task->group_leader;
979	return task->pids[type].pid;
980	}
981
982	static int eligible_pid(struct wait_opts *wo, struct task_struct *p)
983	{
984	return wo->wo_type == PIDTYPE_MAX ||
985	task_pid_type(p, wo->wo_type) == wo->wo_pid;
986	}
987
988	static int
989	eligible_child(struct wait_opts *wo, bool ptrace, struct task_struct *p)
990	{
991	if (!eligible_pid(wo, p))
992	return 0;
993
994	/*
995	* Wait for all children (clone and not) if __WALL is set or
996	* if it is traced by us.
997	*/
998	if (ptrace || (wo->wo_flags & __WALL))
999	return 1;
1000
1001	/*
1002	* Otherwise, wait for clone children *only* if __WCLONE is set;
1003	* otherwise, wait for non-clone children *only*.
1004	*
1005	* Note: a "clone" child here is one that reports to its parent
1006	* using a signal other than SIGCHLD, or a non-leader thread which
1007	* we can only see if it is traced by us.
1008	*/
1009	if ((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE))
1010	return 0;
1011
1012	return 1;
1013	}
1014
1015	static int wait_noreap_copyout(struct wait_opts *wo, struct task_struct *p,
1016	pid_t pid, uid_t uid, int why, int status)
1017	{
1018	struct siginfo __user *infop;
1019	int retval = wo->wo_rusage
1020	? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
1021
1022	put_task_struct(p);
1023	infop = wo->wo_info;
1024	if (infop) {
1025	if (!retval)
1026	retval = put_user(SIGCHLD, &infop->si_signo);
1027	if (!retval)
1028	retval = put_user(0, &infop->si_errno);
1029	if (!retval)
1030	retval = put_user((short)why, &infop->si_code);
1031	if (!retval)
1032	retval = put_user(pid, &infop->si_pid);
1033	if (!retval)
1034	retval = put_user(uid, &infop->si_uid);
1035	if (!retval)
1036	retval = put_user(status, &infop->si_status);
1037	}
1038	if (!retval)
1039	retval = pid;
1040	return retval;
1041	}
1042
1043	/*
1044	* Handle sys_wait4 work for one task in state EXIT_ZOMBIE. We hold
1045	* read_lock(&tasklist_lock) on entry. If we return zero, we still hold
1046	* the lock and this task is uninteresting. If we return nonzero, we have
1047	* released the lock and the system call should return.
1048	*/
1049	static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
1050	{
1051	int state, retval, status;
1052	pid_t pid = task_pid_vnr(p);
1053	uid_t uid = from_kuid_munged(current_user_ns(), task_uid(p));
1054	struct siginfo __user *infop;
1055
1056	if (!likely(wo->wo_flags & WEXITED))
1057	return 0;
1058
1059	if (unlikely(wo->wo_flags & WNOWAIT)) {
1060	int exit_code = p->exit_code;
1061	int why;
1062
1063	get_task_struct(p);
1064	read_unlock(&tasklist_lock);
1065	sched_annotate_sleep();
1066
1067	if ((exit_code & 0x7f) == 0) {
1068	why = CLD_EXITED;
1069	status = exit_code >> 8;
1070	} else {
1071	why = (exit_code & 0x80) ? CLD_DUMPED : CLD_KILLED;
1072	status = exit_code & 0x7f;
1073	}
1074	return wait_noreap_copyout(wo, p, pid, uid, why, status);
1075	}
1076	/*
1077	* Move the task's state to DEAD/TRACE, only one thread can do this.
1078	*/
1079	state = (ptrace_reparented(p) && thread_group_leader(p)) ?
1080	EXIT_TRACE : EXIT_DEAD;
1081	if (cmpxchg(&p->exit_state, EXIT_ZOMBIE, state) != EXIT_ZOMBIE)
1082	return 0;
1083	/*
1084	* We own this thread, nobody else can reap it.
1085	*/
1086	read_unlock(&tasklist_lock);
1087	sched_annotate_sleep();
1088
1089	/*
1090	* Check thread_group_leader() to exclude the traced sub-threads.
1091	*/
1092	if (state == EXIT_DEAD && thread_group_leader(p)) {
1093	struct signal_struct *sig = p->signal;
1094	struct signal_struct *psig = current->signal;
1095	unsigned long maxrss;
1096	cputime_t tgutime, tgstime;
1097
1098	/*
1099	* The resource counters for the group leader are in its
1100	* own task_struct. Those for dead threads in the group
1101	* are in its signal_struct, as are those for the child
1102	* processes it has previously reaped. All these
1103	* accumulate in the parent's signal_struct c* fields.
1104	*
1105	* We don't bother to take a lock here to protect these
1106	* p->signal fields because the whole thread group is dead
1107	* and nobody can change them.
1108	*
1109	* psig->stats_lock also protects us from our sub-theads
1110	* which can reap other children at the same time. Until
1111	* we change k_getrusage()-like users to rely on this lock
1112	* we have to take ->siglock as well.
1113	*
1114	* We use thread_group_cputime_adjusted() to get times for
1115	* the thread group, which consolidates times for all threads
1116	* in the group including the group leader.
1117	*/
1118	thread_group_cputime_adjusted(p, &tgutime, &tgstime);
1119	spin_lock_irq(&current->sighand->siglock);
1120	write_seqlock(&psig->stats_lock);
1121	psig->cutime += tgutime + sig->cutime;
1122	psig->cstime += tgstime + sig->cstime;
1123	psig->cgtime += task_gtime(p) + sig->gtime + sig->cgtime;
1124	psig->cmin_flt +=
1125	p->min_flt + sig->min_flt + sig->cmin_flt;
1126	psig->cmaj_flt +=
1127	p->maj_flt + sig->maj_flt + sig->cmaj_flt;
1128	psig->cnvcsw +=
1129	p->nvcsw + sig->nvcsw + sig->cnvcsw;
1130	psig->cnivcsw +=
1131	p->nivcsw + sig->nivcsw + sig->cnivcsw;
1132	psig->cinblock +=
1133	task_io_get_inblock(p) +
1134	sig->inblock + sig->cinblock;
1135	psig->coublock +=
1136	task_io_get_oublock(p) +
1137	sig->oublock + sig->coublock;
1138	maxrss = max(sig->maxrss, sig->cmaxrss);
1139	if (psig->cmaxrss < maxrss)
1140	psig->cmaxrss = maxrss;
1141	task_io_accounting_add(&psig->ioac, &p->ioac);
1142	task_io_accounting_add(&psig->ioac, &sig->ioac);
1143	write_sequnlock(&psig->stats_lock);
1144	spin_unlock_irq(&current->sighand->siglock);
1145	}
1146
1147	retval = wo->wo_rusage
1148	? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
1149	status = (p->signal->flags & SIGNAL_GROUP_EXIT)
1150	? p->signal->group_exit_code : p->exit_code;
1151	if (!retval && wo->wo_stat)
1152	retval = put_user(status, wo->wo_stat);
1153
1154	infop = wo->wo_info;
1155	if (!retval && infop)
1156	retval = put_user(SIGCHLD, &infop->si_signo);
1157	if (!retval && infop)
1158	retval = put_user(0, &infop->si_errno);
1159	if (!retval && infop) {
1160	int why;
1161
1162	if ((status & 0x7f) == 0) {
1163	why = CLD_EXITED;
1164	status >>= 8;
1165	} else {
1166	why = (status & 0x80) ? CLD_DUMPED : CLD_KILLED;
1167	status &= 0x7f;
1168	}
1169	retval = put_user((short)why, &infop->si_code);
1170	if (!retval)
1171	retval = put_user(status, &infop->si_status);
1172	}
1173	if (!retval && infop)
1174	retval = put_user(pid, &infop->si_pid);
1175	if (!retval && infop)
1176	retval = put_user(uid, &infop->si_uid);
1177	if (!retval)
1178	retval = pid;
1179
1180	if (state == EXIT_TRACE) {
1181	write_lock_irq(&tasklist_lock);
1182	/* We dropped tasklist, ptracer could die and untrace */
1183	ptrace_unlink(p);
1184
1185	/* If parent wants a zombie, don't release it now */
1186	state = EXIT_ZOMBIE;
1187	if (do_notify_parent(p, p->exit_signal))
1188	state = EXIT_DEAD;
1189	p->exit_state = state;
1190	write_unlock_irq(&tasklist_lock);
1191	}
1192	if (state == EXIT_DEAD)
1193	release_task(p);
1194
1195	return retval;
1196	}
1197
1198	static int *task_stopped_code(struct task_struct *p, bool ptrace)
1199	{
1200	if (ptrace) {
1201	if (task_is_traced(p) && !(p->jobctl & JOBCTL_LISTENING))
1202	return &p->exit_code;
1203	} else {
1204	if (p->signal->flags & SIGNAL_STOP_STOPPED)
1205	return &p->signal->group_exit_code;
1206	}
1207	return NULL;
1208	}
1209
1210	/**
1211	* wait_task_stopped - Wait for %TASK_STOPPED or %TASK_TRACED
1212	* @wo: wait options
1213	* @ptrace: is the wait for ptrace
1214	* @p: task to wait for
1215	*
1216	* Handle sys_wait4() work for %p in state %TASK_STOPPED or %TASK_TRACED.
1217	*
1218	* CONTEXT:
1219	* read_lock(&tasklist_lock), which is released if return value is
1220	* non-zero. Also, grabs and releases @p->sighand->siglock.
1221	*
1222	* RETURNS:
1223	* 0 if wait condition didn't exist and search for other wait conditions
1224	* should continue. Non-zero return, -errno on failure and @p's pid on
1225	* success, implies that tasklist_lock is released and wait condition
1226	* search should terminate.
1227	*/
1228	static int wait_task_stopped(struct wait_opts *wo,
1229	int ptrace, struct task_struct *p)
1230	{
1231	struct siginfo __user *infop;
1232	int retval, exit_code, *p_code, why;
1233	uid_t uid = 0; /* unneeded, required by compiler */
1234	pid_t pid;
1235
1236	/*
1237	* Traditionally we see ptrace'd stopped tasks regardless of options.
1238	*/
1239	if (!ptrace && !(wo->wo_flags & WUNTRACED))
1240	return 0;
1241
1242	if (!task_stopped_code(p, ptrace))
1243	return 0;
1244
1245	exit_code = 0;
1246	spin_lock_irq(&p->sighand->siglock);
1247
1248	p_code = task_stopped_code(p, ptrace);
1249	if (unlikely(!p_code))
1250	goto unlock_sig;
1251
1252	exit_code = *p_code;
1253	if (!exit_code)
1254	goto unlock_sig;
1255
1256	if (!unlikely(wo->wo_flags & WNOWAIT))
1257	*p_code = 0;
1258
1259	uid = from_kuid_munged(current_user_ns(), task_uid(p));
1260	unlock_sig:
1261	spin_unlock_irq(&p->sighand->siglock);
1262	if (!exit_code)
1263	return 0;
1264
1265	/*
1266	* Now we are pretty sure this task is interesting.
1267	* Make sure it doesn't get reaped out from under us while we
1268	* give up the lock and then examine it below. We don't want to
1269	* keep holding onto the tasklist_lock while we call getrusage and
1270	* possibly take page faults for user memory.
1271	*/
1272	get_task_struct(p);
1273	pid = task_pid_vnr(p);
1274	why = ptrace ? CLD_TRAPPED : CLD_STOPPED;
1275	read_unlock(&tasklist_lock);
1276	sched_annotate_sleep();
1277
1278	if (unlikely(wo->wo_flags & WNOWAIT))
1279	return wait_noreap_copyout(wo, p, pid, uid, why, exit_code);
1280
1281	retval = wo->wo_rusage
1282	? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
1283	if (!retval && wo->wo_stat)
1284	retval = put_user((exit_code << 8) | 0x7f, wo->wo_stat);
1285
1286	infop = wo->wo_info;
1287	if (!retval && infop)
1288	retval = put_user(SIGCHLD, &infop->si_signo);
1289	if (!retval && infop)
1290	retval = put_user(0, &infop->si_errno);
1291	if (!retval && infop)
1292	retval = put_user((short)why, &infop->si_code);
1293	if (!retval && infop)
1294	retval = put_user(exit_code, &infop->si_status);
1295	if (!retval && infop)
1296	retval = put_user(pid, &infop->si_pid);
1297	if (!retval && infop)
1298	retval = put_user(uid, &infop->si_uid);
1299	if (!retval)
1300	retval = pid;
1301	put_task_struct(p);
1302
1303	BUG_ON(!retval);
1304	return retval;
1305	}
1306
1307	/*
1308	* Handle do_wait work for one task in a live, non-stopped state.
1309	* read_lock(&tasklist_lock) on entry. If we return zero, we still hold
1310	* the lock and this task is uninteresting. If we return nonzero, we have
1311	* released the lock and the system call should return.
1312	*/
1313	static int wait_task_continued(struct wait_opts *wo, struct task_struct *p)
1314	{
1315	int retval;
1316	pid_t pid;
1317	uid_t uid;
1318
1319	if (!unlikely(wo->wo_flags & WCONTINUED))
1320	return 0;
1321
1322	if (!(p->signal->flags & SIGNAL_STOP_CONTINUED))
1323	return 0;
1324
1325	spin_lock_irq(&p->sighand->siglock);
1326	/* Re-check with the lock held. */
1327	if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) {
1328	spin_unlock_irq(&p->sighand->siglock);
1329	return 0;
1330	}
1331	if (!unlikely(wo->wo_flags & WNOWAIT))
1332	p->signal->flags &= ~SIGNAL_STOP_CONTINUED;
1333	uid = from_kuid_munged(current_user_ns(), task_uid(p));
1334	spin_unlock_irq(&p->sighand->siglock);
1335
1336	pid = task_pid_vnr(p);
1337	get_task_struct(p);
1338	read_unlock(&tasklist_lock);
1339	sched_annotate_sleep();
1340
1341	if (!wo->wo_info) {
1342	retval = wo->wo_rusage
1343	? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
1344	put_task_struct(p);
1345	if (!retval && wo->wo_stat)
1346	retval = put_user(0xffff, wo->wo_stat);
1347	if (!retval)
1348	retval = pid;
1349	} else {
1350	retval = wait_noreap_copyout(wo, p, pid, uid,
1351	CLD_CONTINUED, SIGCONT);
1352	BUG_ON(retval == 0);
1353	}
1354
1355	return retval;
1356	}
1357
1358	/*
1359	* Consider @p for a wait by @parent.
1360	*
1361	* -ECHILD should be in ->notask_error before the first call.
1362	* Returns nonzero for a final return, when we have unlocked tasklist_lock.
1363	* Returns zero if the search for a child should continue;
1364	* then ->notask_error is 0 if @p is an eligible child,
1365	* or another error from security_task_wait(), or still -ECHILD.
1366	*/
1367	static int wait_consider_task(struct wait_opts *wo, int ptrace,
1368	struct task_struct *p)
1369	{
1370	/*
1371	* We can race with wait_task_zombie() from another thread.
1372	* Ensure that EXIT_ZOMBIE -> EXIT_DEAD/EXIT_TRACE transition
1373	* can't confuse the checks below.
1374	*/
1375	int exit_state = ACCESS_ONCE(p->exit_state);
1376	int ret;
1377
1378	if (unlikely(exit_state == EXIT_DEAD))
1379	return 0;
1380
1381	ret = eligible_child(wo, ptrace, p);
1382	if (!ret)
1383	return ret;
1384
1385	ret = security_task_wait(p);
1386	if (unlikely(ret < 0)) {
1387	/*
1388	* If we have not yet seen any eligible child,
1389	* then let this error code replace -ECHILD.
1390	* A permission error will give the user a clue
1391	* to look for security policy problems, rather
1392	* than for mysterious wait bugs.
1393	*/
1394	if (wo->notask_error)
1395	wo->notask_error = ret;
1396	return 0;
1397	}
1398
1399	if (unlikely(exit_state == EXIT_TRACE)) {
1400	/*
1401	* ptrace == 0 means we are the natural parent. In this case
1402	* we should clear notask_error, debugger will notify us.
1403	*/
1404	if (likely(!ptrace))
1405	wo->notask_error = 0;
1406	return 0;
1407	}
1408
1409	if (likely(!ptrace) && unlikely(p->ptrace)) {
1410	/*
1411	* If it is traced by its real parent's group, just pretend
1412	* the caller is ptrace_do_wait() and reap this child if it
1413	* is zombie.
1414	*
1415	* This also hides group stop state from real parent; otherwise
1416	* a single stop can be reported twice as group and ptrace stop.
1417	* If a ptracer wants to distinguish these two events for its
1418	* own children it should create a separate process which takes
1419	* the role of real parent.
1420	*/
1421	if (!ptrace_reparented(p))
1422	ptrace = 1;
1423	}
1424
1425	/* slay zombie? */
1426	if (exit_state == EXIT_ZOMBIE) {
1427	/* we don't reap group leaders with subthreads */
1428	if (!delay_group_leader(p)) {
1429	/*
1430	* A zombie ptracee is only visible to its ptracer.
1431	* Notification and reaping will be cascaded to the
1432	* real parent when the ptracer detaches.
1433	*/
1434	if (unlikely(ptrace) || likely(!p->ptrace))
1435	return wait_task_zombie(wo, p);
1436	}
1437
1438	/*
1439	* Allow access to stopped/continued state via zombie by
1440	* falling through. Clearing of notask_error is complex.
1441	*
1442	* When !@ptrace:
1443	*
1444	* If WEXITED is set, notask_error should naturally be
1445	* cleared. If not, subset of WSTOPPED|WCONTINUED is set,
1446	* so, if there are live subthreads, there are events to
1447	* wait for. If all subthreads are dead, it's still safe
1448	* to clear - this function will be called again in finite
1449	* amount time once all the subthreads are released and
1450	* will then return without clearing.
1451	*
1452	* When @ptrace:
1453	*
1454	* Stopped state is per-task and thus can't change once the
1455	* target task dies. Only continued and exited can happen.
1456	* Clear notask_error if WCONTINUED | WEXITED.
1457	*/
1458	if (likely(!ptrace) || (wo->wo_flags & (WCONTINUED | WEXITED)))
1459	wo->notask_error = 0;
1460	} else {
1461	/*
1462	* @p is alive and it's gonna stop, continue or exit, so
1463	* there always is something to wait for.
1464	*/
1465	wo->notask_error = 0;
1466	}
1467
1468	/*
1469	* Wait for stopped. Depending on @ptrace, different stopped state
1470	* is used and the two don't interact with each other.
1471	*/
1472	ret = wait_task_stopped(wo, ptrace, p);
1473	if (ret)
1474	return ret;
1475
1476	/*
1477	* Wait for continued. There's only one continued state and the
1478	* ptracer can consume it which can confuse the real parent. Don't
1479	* use WCONTINUED from ptracer. You don't need or want it.
1480	*/
1481	return wait_task_continued(wo, p);
1482	}
1483
1484	/*
1485	* Do the work of do_wait() for one thread in the group, @tsk.
1486	*
1487	* -ECHILD should be in ->notask_error before the first call.
1488	* Returns nonzero for a final return, when we have unlocked tasklist_lock.
1489	* Returns zero if the search for a child should continue; then
1490	* ->notask_error is 0 if there were any eligible children,
1491	* or another error from security_task_wait(), or still -ECHILD.
1492	*/
1493	static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk)
1494	{
1495	struct task_struct *p;
1496
1497	list_for_each_entry(p, &tsk->children, sibling) {
1498	int ret = wait_consider_task(wo, 0, p);
1499
1500	if (ret)
1501	return ret;
1502	}
1503
1504	return 0;
1505	}
1506
1507	static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk)
1508	{
1509	struct task_struct *p;
1510
1511	list_for_each_entry(p, &tsk->ptraced, ptrace_entry) {
1512	int ret = wait_consider_task(wo, 1, p);
1513
1514	if (ret)
1515	return ret;
1516	}
1517
1518	return 0;
1519	}
1520
1521	static int child_wait_callback(wait_queue_t *wait, unsigned mode,
1522	int sync, void *key)
1523	{
1524	struct wait_opts *wo = container_of(wait, struct wait_opts,
1525	child_wait);
1526	struct task_struct *p = key;
1527
1528	if (!eligible_pid(wo, p))
1529	return 0;
1530
1531	if ((wo->wo_flags & __WNOTHREAD) && wait->private != p->parent)
1532	return 0;
1533
1534	return default_wake_function(wait, mode, sync, key);
1535	}
1536
1537	void __wake_up_parent(struct task_struct *p, struct task_struct *parent)
1538	{
1539	__wake_up_sync_key(&parent->signal->wait_chldexit,
1540	TASK_INTERRUPTIBLE, 1, p);
1541	}
1542
1543	static long do_wait(struct wait_opts *wo)
1544	{
1545	struct task_struct *tsk;
1546	int retval;
1547
1548	trace_sched_process_wait(wo->wo_pid);
1549
1550	init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
1551	wo->child_wait.private = current;
1552	add_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1553	repeat:
1554	/*
1555	* If there is nothing that can match our criteria, just get out.
1556	* We will clear ->notask_error to zero if we see any child that
1557	* might later match our criteria, even if we are not able to reap
1558	* it yet.
1559	*/
1560	wo->notask_error = -ECHILD;
1561	if ((wo->wo_type < PIDTYPE_MAX) &&
1562	(!wo->wo_pid || hlist_empty(&wo->wo_pid->tasks[wo->wo_type])))
1563	goto notask;
1564
1565	set_current_state(TASK_INTERRUPTIBLE);
1566	read_lock(&tasklist_lock);
1567	tsk = current;
1568	do {
1569	retval = do_wait_thread(wo, tsk);
1570	if (retval)
1571	goto end;
1572
1573	retval = ptrace_do_wait(wo, tsk);
1574	if (retval)
1575	goto end;
1576
1577	if (wo->wo_flags & __WNOTHREAD)
1578	break;
1579	} while_each_thread(current, tsk);
1580	read_unlock(&tasklist_lock);
1581
1582	notask:
1583	retval = wo->notask_error;
1584	if (!retval && !(wo->wo_flags & WNOHANG)) {
1585	retval = -ERESTARTSYS;
1586	if (!signal_pending(current)) {
1587	schedule();
1588	goto repeat;
1589	}
1590	}
1591	end:
1592	__set_current_state(TASK_RUNNING);
1593	remove_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1594	return retval;
1595	}
1596
1597	SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
1598	infop, int, options, struct rusage __user *, ru)
1599	{
1600	struct wait_opts wo;
1601	struct pid *pid = NULL;
1602	enum pid_type type;
1603	long ret;
1604
1605	if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED|
1606	__WNOTHREAD|__WCLONE|__WALL))
1607	return -EINVAL;
1608	if (!(options & (WEXITED|WSTOPPED|WCONTINUED)))
1609	return -EINVAL;
1610
1611	switch (which) {
1612	case P_ALL:
1613	type = PIDTYPE_MAX;
1614	break;
1615	case P_PID:
1616	type = PIDTYPE_PID;
1617	if (upid <= 0)
1618	return -EINVAL;
1619	break;
1620	case P_PGID:
1621	type = PIDTYPE_PGID;
1622	if (upid <= 0)
1623	return -EINVAL;
1624	break;
1625	default:
1626	return -EINVAL;
1627	}
1628
1629	if (type < PIDTYPE_MAX)
1630	pid = find_get_pid(upid);
1631
1632	wo.wo_type = type;
1633	wo.wo_pid = pid;
1634	wo.wo_flags = options;
1635	wo.wo_info = infop;
1636	wo.wo_stat = NULL;
1637	wo.wo_rusage = ru;
1638	ret = do_wait(&wo);
1639
1640	if (ret > 0) {
1641	ret = 0;
1642	} else if (infop) {
1643	/*
1644	* For a WNOHANG return, clear out all the fields
1645	* we would set so the user can easily tell the
1646	* difference.
1647	*/
1648	if (!ret)
1649	ret = put_user(0, &infop->si_signo);
1650	if (!ret)
1651	ret = put_user(0, &infop->si_errno);
1652	if (!ret)
1653	ret = put_user(0, &infop->si_code);
1654	if (!ret)
1655	ret = put_user(0, &infop->si_pid);
1656	if (!ret)
1657	ret = put_user(0, &infop->si_uid);
1658	if (!ret)
1659	ret = put_user(0, &infop->si_status);
1660	}
1661
1662	put_pid(pid);
1663	return ret;
1664	}
1665
1666	SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr,
1667	int, options, struct rusage __user *, ru)
1668	{
1669	struct wait_opts wo;
1670	struct pid *pid = NULL;
1671	enum pid_type type;
1672	long ret;
1673
1674	if (options & ~(WNOHANG|WUNTRACED|WCONTINUED|
1675	__WNOTHREAD|__WCLONE|__WALL))
1676	return -EINVAL;
1677
1678	/* -INT_MIN is not defined */
1679	if (upid == INT_MIN)
1680	return -ESRCH;
1681
1682	if (upid == -1)
1683	type = PIDTYPE_MAX;
1684	else if (upid < 0) {
1685	type = PIDTYPE_PGID;
1686	pid = find_get_pid(-upid);
1687	} else if (upid == 0) {
1688	type = PIDTYPE_PGID;
1689	pid = get_task_pid(current, PIDTYPE_PGID);
1690	} else /* upid > 0 */ {
1691	type = PIDTYPE_PID;
1692	pid = find_get_pid(upid);
1693	}
1694
1695	wo.wo_type = type;
1696	wo.wo_pid = pid;
1697	wo.wo_flags = options | WEXITED;
1698	wo.wo_info = NULL;
1699	wo.wo_stat = stat_addr;
1700	wo.wo_rusage = ru;
1701	ret = do_wait(&wo);
1702	put_pid(pid);
1703
1704	return ret;
1705	}
1706
1707	#ifdef __ARCH_WANT_SYS_WAITPID
1708
1709	/*
1710	* sys_waitpid() remains for compatibility. waitpid() should be
1711	* implemented by calling sys_wait4() from libc.a.
1712	*/
1713	SYSCALL_DEFINE3(waitpid, pid_t, pid, int __user *, stat_addr, int, options)
1714	{
1715	return sys_wait4(pid, stat_addr, options, NULL);
1716	}
1717
1718	#endif
1719