![Vegard Nossum](/assets/img/avatar_default.png)
When we get a hung task it can often be valuable to see _all_ the held locks on the system (in case we are being blocked on trying to acquire one), e.g. with this patch we can immediately see where the problem is below: INFO: task trinity-c3:14933 blocked for more than 120 seconds. Not tainted 4.8.0-rc1+ #135 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. trinity-c3 D ffff88010c16fc88 0 14933 1 0x00080004 ffff88010c16fc88 000000003b9aca00 0000000000000000 0000000000000296 00000000776cdf88 ffff88011a520ae0 ffff88011a520b08 ffff88011a520198 ffffffff867d7f00 ffff88011942c080 ffff880116841580 ffff88010c168000 Call Trace: [<ffffffff845e9d37>] schedule+0x77/0x230 [<ffffffff833cb8b9>] __lock_sock+0x129/0x250 [<ffffffff833cb790>] ? __sk_destruct+0x450/0x450 [<ffffffff81408ac0>] ? wake_bit_function+0x2e0/0x2e0 [<ffffffff833d832b>] lock_sock_nested+0xeb/0x120 [<ffffffff83bad815>] irda_setsockopt+0x65/0xb40 [<ffffffff833c6c09>] SyS_setsockopt+0x139/0x230 [<ffffffff833c6ad0>] ? SyS_recv+0x20/0x20 [<ffffffff81004660>] ? trace_event_raw_event_sys_enter+0xb90/0xb90 [<ffffffff823c7023>] ? __this_cpu_preempt_check+0x13/0x20 [<ffffffff8162ee60>] ? __context_tracking_exit.part.3+0x30/0x1b0 [<ffffffff833c6ad0>] ? SyS_recv+0x20/0x20 [<ffffffff81007bd3>] do_syscall_64+0x1b3/0x4b0 [<ffffffff845f84aa>] entry_SYSCALL64_slow_path+0x25/0x25 Showing all locks held in the system: 2 locks held by khungtaskd/563: #0: (rcu_read_lock){......}, at: [<ffffffff81534ce6>] watchdog+0x106/0x910 #1: (tasklist_lock){......}, at: [<ffffffff8141b3c4>] debug_show_all_locks+0x74/0x360 1 lock held by trinity-c0/19280: #0: (sk_lock-AF_IRDA){......}, at: [<ffffffff83bab7c6>] irda_accept+0x176/0x10f0 1 lock held by trinity-c0/12865: #0: (sk_lock-AF_IRDA){......}, at: [<ffffffff83bab7c6>] irda_accept+0x176/0x10f0 Signed-off-by: Vegard Nossum <vegard.nossum@oracle.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mandeep Singh Baines <msb@chromium.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Link: http://lkml.kernel.org/r/1471538460-7505-1-git-send-email-vegard.nossum@oracle.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
257 lines
5.9 KiB
C
257 lines
5.9 KiB
C
/*
|
|
* Detect Hung Task
|
|
*
|
|
* kernel/hung_task.c - kernel thread for detecting tasks stuck in D state
|
|
*
|
|
*/
|
|
|
|
#include <linux/mm.h>
|
|
#include <linux/cpu.h>
|
|
#include <linux/nmi.h>
|
|
#include <linux/init.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/freezer.h>
|
|
#include <linux/kthread.h>
|
|
#include <linux/lockdep.h>
|
|
#include <linux/export.h>
|
|
#include <linux/sysctl.h>
|
|
#include <linux/utsname.h>
|
|
#include <trace/events/sched.h>
|
|
|
|
/*
|
|
* The number of tasks checked:
|
|
*/
|
|
int __read_mostly sysctl_hung_task_check_count = PID_MAX_LIMIT;
|
|
|
|
/*
|
|
* Limit number of tasks checked in a batch.
|
|
*
|
|
* This value controls the preemptibility of khungtaskd since preemption
|
|
* is disabled during the critical section. It also controls the size of
|
|
* the RCU grace period. So it needs to be upper-bound.
|
|
*/
|
|
#define HUNG_TASK_BATCHING 1024
|
|
|
|
/*
|
|
* Zero means infinite timeout - no checking done:
|
|
*/
|
|
unsigned long __read_mostly sysctl_hung_task_timeout_secs = CONFIG_DEFAULT_HUNG_TASK_TIMEOUT;
|
|
|
|
int __read_mostly sysctl_hung_task_warnings = 10;
|
|
|
|
static int __read_mostly did_panic;
|
|
|
|
static struct task_struct *watchdog_task;
|
|
|
|
/*
|
|
* Should we panic (and reboot, if panic_timeout= is set) when a
|
|
* hung task is detected:
|
|
*/
|
|
unsigned int __read_mostly sysctl_hung_task_panic =
|
|
CONFIG_BOOTPARAM_HUNG_TASK_PANIC_VALUE;
|
|
|
|
static int __init hung_task_panic_setup(char *str)
|
|
{
|
|
int rc = kstrtouint(str, 0, &sysctl_hung_task_panic);
|
|
|
|
if (rc)
|
|
return rc;
|
|
return 1;
|
|
}
|
|
__setup("hung_task_panic=", hung_task_panic_setup);
|
|
|
|
static int
|
|
hung_task_panic(struct notifier_block *this, unsigned long event, void *ptr)
|
|
{
|
|
did_panic = 1;
|
|
|
|
return NOTIFY_DONE;
|
|
}
|
|
|
|
static struct notifier_block panic_block = {
|
|
.notifier_call = hung_task_panic,
|
|
};
|
|
|
|
static void check_hung_task(struct task_struct *t, unsigned long timeout)
|
|
{
|
|
unsigned long switch_count = t->nvcsw + t->nivcsw;
|
|
|
|
/*
|
|
* Ensure the task is not frozen.
|
|
* Also, skip vfork and any other user process that freezer should skip.
|
|
*/
|
|
if (unlikely(t->flags & (PF_FROZEN | PF_FREEZER_SKIP)))
|
|
return;
|
|
|
|
/*
|
|
* When a freshly created task is scheduled once, changes its state to
|
|
* TASK_UNINTERRUPTIBLE without having ever been switched out once, it
|
|
* musn't be checked.
|
|
*/
|
|
if (unlikely(!switch_count))
|
|
return;
|
|
|
|
if (switch_count != t->last_switch_count) {
|
|
t->last_switch_count = switch_count;
|
|
return;
|
|
}
|
|
|
|
trace_sched_process_hang(t);
|
|
|
|
if (!sysctl_hung_task_warnings)
|
|
return;
|
|
|
|
if (sysctl_hung_task_warnings > 0)
|
|
sysctl_hung_task_warnings--;
|
|
|
|
/*
|
|
* Ok, the task did not get scheduled for more than 2 minutes,
|
|
* complain:
|
|
*/
|
|
pr_err("INFO: task %s:%d blocked for more than %ld seconds.\n",
|
|
t->comm, t->pid, timeout);
|
|
pr_err(" %s %s %.*s\n",
|
|
print_tainted(), init_utsname()->release,
|
|
(int)strcspn(init_utsname()->version, " "),
|
|
init_utsname()->version);
|
|
pr_err("\"echo 0 > /proc/sys/kernel/hung_task_timeout_secs\""
|
|
" disables this message.\n");
|
|
sched_show_task(t);
|
|
debug_show_all_locks();
|
|
|
|
touch_nmi_watchdog();
|
|
|
|
if (sysctl_hung_task_panic) {
|
|
trigger_all_cpu_backtrace();
|
|
panic("hung_task: blocked tasks");
|
|
}
|
|
}
|
|
|
|
/*
|
|
* To avoid extending the RCU grace period for an unbounded amount of time,
|
|
* periodically exit the critical section and enter a new one.
|
|
*
|
|
* For preemptible RCU it is sufficient to call rcu_read_unlock in order
|
|
* to exit the grace period. For classic RCU, a reschedule is required.
|
|
*/
|
|
static bool rcu_lock_break(struct task_struct *g, struct task_struct *t)
|
|
{
|
|
bool can_cont;
|
|
|
|
get_task_struct(g);
|
|
get_task_struct(t);
|
|
rcu_read_unlock();
|
|
cond_resched();
|
|
rcu_read_lock();
|
|
can_cont = pid_alive(g) && pid_alive(t);
|
|
put_task_struct(t);
|
|
put_task_struct(g);
|
|
|
|
return can_cont;
|
|
}
|
|
|
|
/*
|
|
* Check whether a TASK_UNINTERRUPTIBLE does not get woken up for
|
|
* a really long time (120 seconds). If that happens, print out
|
|
* a warning.
|
|
*/
|
|
static void check_hung_uninterruptible_tasks(unsigned long timeout)
|
|
{
|
|
int max_count = sysctl_hung_task_check_count;
|
|
int batch_count = HUNG_TASK_BATCHING;
|
|
struct task_struct *g, *t;
|
|
|
|
/*
|
|
* If the system crashed already then all bets are off,
|
|
* do not report extra hung tasks:
|
|
*/
|
|
if (test_taint(TAINT_DIE) || did_panic)
|
|
return;
|
|
|
|
rcu_read_lock();
|
|
for_each_process_thread(g, t) {
|
|
if (!max_count--)
|
|
goto unlock;
|
|
if (!--batch_count) {
|
|
batch_count = HUNG_TASK_BATCHING;
|
|
if (!rcu_lock_break(g, t))
|
|
goto unlock;
|
|
}
|
|
/* use "==" to skip the TASK_KILLABLE tasks waiting on NFS */
|
|
if (t->state == TASK_UNINTERRUPTIBLE)
|
|
check_hung_task(t, timeout);
|
|
}
|
|
unlock:
|
|
rcu_read_unlock();
|
|
}
|
|
|
|
static long hung_timeout_jiffies(unsigned long last_checked,
|
|
unsigned long timeout)
|
|
{
|
|
/* timeout of 0 will disable the watchdog */
|
|
return timeout ? last_checked - jiffies + timeout * HZ :
|
|
MAX_SCHEDULE_TIMEOUT;
|
|
}
|
|
|
|
/*
|
|
* Process updating of timeout sysctl
|
|
*/
|
|
int proc_dohung_task_timeout_secs(struct ctl_table *table, int write,
|
|
void __user *buffer,
|
|
size_t *lenp, loff_t *ppos)
|
|
{
|
|
int ret;
|
|
|
|
ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
|
|
|
|
if (ret || !write)
|
|
goto out;
|
|
|
|
wake_up_process(watchdog_task);
|
|
|
|
out:
|
|
return ret;
|
|
}
|
|
|
|
static atomic_t reset_hung_task = ATOMIC_INIT(0);
|
|
|
|
void reset_hung_task_detector(void)
|
|
{
|
|
atomic_set(&reset_hung_task, 1);
|
|
}
|
|
EXPORT_SYMBOL_GPL(reset_hung_task_detector);
|
|
|
|
/*
|
|
* kthread which checks for tasks stuck in D state
|
|
*/
|
|
static int watchdog(void *dummy)
|
|
{
|
|
unsigned long hung_last_checked = jiffies;
|
|
|
|
set_user_nice(current, 0);
|
|
|
|
for ( ; ; ) {
|
|
unsigned long timeout = sysctl_hung_task_timeout_secs;
|
|
long t = hung_timeout_jiffies(hung_last_checked, timeout);
|
|
|
|
if (t <= 0) {
|
|
if (!atomic_xchg(&reset_hung_task, 0))
|
|
check_hung_uninterruptible_tasks(timeout);
|
|
hung_last_checked = jiffies;
|
|
continue;
|
|
}
|
|
schedule_timeout_interruptible(t);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int __init hung_task_init(void)
|
|
{
|
|
atomic_notifier_chain_register(&panic_notifier_list, &panic_block);
|
|
watchdog_task = kthread_run(watchdog, NULL, "khungtaskd");
|
|
|
|
return 0;
|
|
}
|
|
subsys_initcall(hung_task_init);
|