2019-06-04 02:11:33 -06:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2007-07-09 10:52:00 -06:00
|
|
|
/*
|
2011-11-15 09:14:39 -07:00
|
|
|
* kernel/sched/debug.c
|
2007-07-09 10:52:00 -06:00
|
|
|
*
|
2018-03-03 04:20:47 -07:00
|
|
|
* Print the CFS rbtree and other debugging details
|
2007-07-09 10:52:00 -06:00
|
|
|
*
|
|
|
|
* Copyright(C) 2007, Red Hat, Inc., Ingo Molnar
|
|
|
|
*/
|
2011-10-25 02:00:11 -06:00
|
|
|
#include "sched.h"
|
|
|
|
|
2011-01-11 03:11:54 -07:00
|
|
|
static DEFINE_SPINLOCK(sched_debug_lock);
|
|
|
|
|
2007-07-09 10:52:00 -06:00
|
|
|
/*
|
|
|
|
* This allows printing both to /proc/sched_debug and
|
|
|
|
* to the console
|
|
|
|
*/
|
|
|
|
#define SEQ_printf(m, x...) \
|
|
|
|
do { \
|
|
|
|
if (m) \
|
|
|
|
seq_printf(m, x); \
|
|
|
|
else \
|
2018-03-19 12:35:54 -06:00
|
|
|
pr_cont(x); \
|
2007-07-09 10:52:00 -06:00
|
|
|
} while (0)
|
|
|
|
|
2007-10-15 09:00:08 -06:00
|
|
|
/*
|
|
|
|
* Ease the printing of nsec fields:
|
|
|
|
*/
|
2007-12-30 09:24:35 -07:00
|
|
|
static long long nsec_high(unsigned long long nsec)
|
2007-10-15 09:00:08 -06:00
|
|
|
{
|
2007-12-30 09:24:35 -07:00
|
|
|
if ((long long)nsec < 0) {
|
2007-10-15 09:00:08 -06:00
|
|
|
nsec = -nsec;
|
|
|
|
do_div(nsec, 1000000);
|
|
|
|
return -nsec;
|
|
|
|
}
|
|
|
|
do_div(nsec, 1000000);
|
|
|
|
|
|
|
|
return nsec;
|
|
|
|
}
|
|
|
|
|
2007-12-30 09:24:35 -07:00
|
|
|
static unsigned long nsec_low(unsigned long long nsec)
|
2007-10-15 09:00:08 -06:00
|
|
|
{
|
2007-12-30 09:24:35 -07:00
|
|
|
if ((long long)nsec < 0)
|
2007-10-15 09:00:08 -06:00
|
|
|
nsec = -nsec;
|
|
|
|
|
|
|
|
return do_div(nsec, 1000000);
|
|
|
|
}
|
|
|
|
|
|
|
|
#define SPLIT_NS(x) nsec_high(x), nsec_low(x)
|
|
|
|
|
2016-02-22 14:26:50 -07:00
|
|
|
#define SCHED_FEAT(name, enabled) \
|
|
|
|
#name ,
|
|
|
|
|
|
|
|
static const char * const sched_feat_names[] = {
|
|
|
|
#include "features.h"
|
|
|
|
};
|
|
|
|
|
|
|
|
#undef SCHED_FEAT
|
|
|
|
|
|
|
|
static int sched_feat_show(struct seq_file *m, void *v)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < __SCHED_FEAT_NR; i++) {
|
|
|
|
if (!(sysctl_sched_features & (1UL << i)))
|
|
|
|
seq_puts(m, "NO_");
|
|
|
|
seq_printf(m, "%s ", sched_feat_names[i]);
|
|
|
|
}
|
|
|
|
seq_puts(m, "\n");
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-12-30 08:14:15 -07:00
|
|
|
#ifdef CONFIG_JUMP_LABEL
|
2016-02-22 14:26:50 -07:00
|
|
|
|
|
|
|
#define jump_label_key__true STATIC_KEY_INIT_TRUE
|
|
|
|
#define jump_label_key__false STATIC_KEY_INIT_FALSE
|
|
|
|
|
|
|
|
#define SCHED_FEAT(name, enabled) \
|
|
|
|
jump_label_key__##enabled ,
|
|
|
|
|
|
|
|
struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
|
|
|
|
#include "features.h"
|
|
|
|
};
|
|
|
|
|
|
|
|
#undef SCHED_FEAT
|
|
|
|
|
|
|
|
static void sched_feat_disable(int i)
|
|
|
|
{
|
2018-07-31 06:12:22 -06:00
|
|
|
static_key_disable_cpuslocked(&sched_feat_keys[i]);
|
2016-02-22 14:26:50 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static void sched_feat_enable(int i)
|
|
|
|
{
|
2018-07-31 06:12:22 -06:00
|
|
|
static_key_enable_cpuslocked(&sched_feat_keys[i]);
|
2016-02-22 14:26:50 -07:00
|
|
|
}
|
|
|
|
#else
|
|
|
|
static void sched_feat_disable(int i) { };
|
|
|
|
static void sched_feat_enable(int i) { };
|
2018-12-30 08:14:15 -07:00
|
|
|
#endif /* CONFIG_JUMP_LABEL */
|
2016-02-22 14:26:50 -07:00
|
|
|
|
|
|
|
static int sched_feat_set(char *cmp)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
int neg = 0;
|
|
|
|
|
|
|
|
if (strncmp(cmp, "NO_", 3) == 0) {
|
|
|
|
neg = 1;
|
|
|
|
cmp += 3;
|
|
|
|
}
|
|
|
|
|
2018-05-31 05:11:19 -06:00
|
|
|
i = match_string(sched_feat_names, __SCHED_FEAT_NR, cmp);
|
|
|
|
if (i < 0)
|
|
|
|
return i;
|
|
|
|
|
|
|
|
if (neg) {
|
|
|
|
sysctl_sched_features &= ~(1UL << i);
|
|
|
|
sched_feat_disable(i);
|
|
|
|
} else {
|
|
|
|
sysctl_sched_features |= (1UL << i);
|
|
|
|
sched_feat_enable(i);
|
2016-02-22 14:26:50 -07:00
|
|
|
}
|
|
|
|
|
2018-05-31 05:11:19 -06:00
|
|
|
return 0;
|
2016-02-22 14:26:50 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t
|
|
|
|
sched_feat_write(struct file *filp, const char __user *ubuf,
|
|
|
|
size_t cnt, loff_t *ppos)
|
|
|
|
{
|
|
|
|
char buf[64];
|
|
|
|
char *cmp;
|
2018-05-31 05:11:19 -06:00
|
|
|
int ret;
|
2016-02-22 14:26:50 -07:00
|
|
|
struct inode *inode;
|
|
|
|
|
|
|
|
if (cnt > 63)
|
|
|
|
cnt = 63;
|
|
|
|
|
|
|
|
if (copy_from_user(&buf, ubuf, cnt))
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
buf[cnt] = 0;
|
|
|
|
cmp = strstrip(buf);
|
|
|
|
|
|
|
|
/* Ensure the static_key remains in a consistent state */
|
|
|
|
inode = file_inode(filp);
|
2018-07-31 06:12:22 -06:00
|
|
|
cpus_read_lock();
|
2016-02-22 14:26:50 -07:00
|
|
|
inode_lock(inode);
|
2018-05-31 05:11:19 -06:00
|
|
|
ret = sched_feat_set(cmp);
|
2016-02-22 14:26:50 -07:00
|
|
|
inode_unlock(inode);
|
2018-07-31 06:12:22 -06:00
|
|
|
cpus_read_unlock();
|
2018-05-31 05:11:19 -06:00
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
2016-02-22 14:26:50 -07:00
|
|
|
|
|
|
|
*ppos += cnt;
|
|
|
|
|
|
|
|
return cnt;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int sched_feat_open(struct inode *inode, struct file *filp)
|
|
|
|
{
|
|
|
|
return single_open(filp, sched_feat_show, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct file_operations sched_feat_fops = {
|
|
|
|
.open = sched_feat_open,
|
|
|
|
.write = sched_feat_write,
|
|
|
|
.read = seq_read,
|
|
|
|
.llseek = seq_lseek,
|
|
|
|
.release = single_release,
|
|
|
|
};
|
|
|
|
|
2017-09-07 09:03:53 -06:00
|
|
|
__read_mostly bool sched_debug_enabled;
|
|
|
|
|
2016-02-22 14:26:50 -07:00
|
|
|
static __init int sched_init_debug(void)
|
|
|
|
{
|
|
|
|
debugfs_create_file("sched_features", 0644, NULL, NULL,
|
|
|
|
&sched_feat_fops);
|
|
|
|
|
2017-09-07 09:03:53 -06:00
|
|
|
debugfs_create_bool("sched_debug", 0644, NULL,
|
|
|
|
&sched_debug_enabled);
|
|
|
|
|
2016-02-22 14:26:50 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
late_initcall(sched_init_debug);
|
|
|
|
|
2016-02-22 14:26:51 -07:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
|
|
|
#ifdef CONFIG_SYSCTL
|
|
|
|
|
|
|
|
static struct ctl_table sd_ctl_dir[] = {
|
|
|
|
{
|
|
|
|
.procname = "sched_domain",
|
|
|
|
.mode = 0555,
|
|
|
|
},
|
|
|
|
{}
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct ctl_table sd_ctl_root[] = {
|
|
|
|
{
|
|
|
|
.procname = "kernel",
|
|
|
|
.mode = 0555,
|
|
|
|
.child = sd_ctl_dir,
|
|
|
|
},
|
|
|
|
{}
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct ctl_table *sd_alloc_ctl_entry(int n)
|
|
|
|
{
|
|
|
|
struct ctl_table *entry =
|
|
|
|
kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
|
|
|
|
|
|
|
|
return entry;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void sd_free_ctl_entry(struct ctl_table **tablep)
|
|
|
|
{
|
|
|
|
struct ctl_table *entry;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* In the intermediate directories, both the child directory and
|
|
|
|
* procname are dynamically allocated and could fail but the mode
|
|
|
|
* will always be set. In the lowest directory the names are
|
|
|
|
* static strings and all have proc handlers.
|
|
|
|
*/
|
|
|
|
for (entry = *tablep; entry->mode; entry++) {
|
|
|
|
if (entry->child)
|
|
|
|
sd_free_ctl_entry(&entry->child);
|
|
|
|
if (entry->proc_handler == NULL)
|
|
|
|
kfree(entry->procname);
|
|
|
|
}
|
|
|
|
|
|
|
|
kfree(*tablep);
|
|
|
|
*tablep = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
set_table_entry(struct ctl_table *entry,
|
|
|
|
const char *procname, void *data, int maxlen,
|
2019-05-27 00:21:12 -06:00
|
|
|
umode_t mode, proc_handler *proc_handler)
|
2016-02-22 14:26:51 -07:00
|
|
|
{
|
|
|
|
entry->procname = procname;
|
|
|
|
entry->data = data;
|
|
|
|
entry->maxlen = maxlen;
|
|
|
|
entry->mode = mode;
|
|
|
|
entry->proc_handler = proc_handler;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct ctl_table *
|
|
|
|
sd_alloc_ctl_domain_table(struct sched_domain *sd)
|
|
|
|
{
|
2019-05-27 00:21:14 -06:00
|
|
|
struct ctl_table *table = sd_alloc_ctl_entry(9);
|
2016-02-22 14:26:51 -07:00
|
|
|
|
|
|
|
if (table == NULL)
|
|
|
|
return NULL;
|
|
|
|
|
2019-05-27 00:21:14 -06:00
|
|
|
set_table_entry(&table[0], "min_interval", &sd->min_interval, sizeof(long), 0644, proc_doulongvec_minmax);
|
|
|
|
set_table_entry(&table[1], "max_interval", &sd->max_interval, sizeof(long), 0644, proc_doulongvec_minmax);
|
|
|
|
set_table_entry(&table[2], "busy_factor", &sd->busy_factor, sizeof(int), 0644, proc_dointvec_minmax);
|
|
|
|
set_table_entry(&table[3], "imbalance_pct", &sd->imbalance_pct, sizeof(int), 0644, proc_dointvec_minmax);
|
|
|
|
set_table_entry(&table[4], "cache_nice_tries", &sd->cache_nice_tries, sizeof(int), 0644, proc_dointvec_minmax);
|
|
|
|
set_table_entry(&table[5], "flags", &sd->flags, sizeof(int), 0644, proc_dointvec_minmax);
|
|
|
|
set_table_entry(&table[6], "max_newidle_lb_cost", &sd->max_newidle_lb_cost, sizeof(long), 0644, proc_doulongvec_minmax);
|
|
|
|
set_table_entry(&table[7], "name", sd->name, CORENAME_MAX_SIZE, 0444, proc_dostring);
|
|
|
|
/* &table[8] is terminator */
|
2016-02-22 14:26:51 -07:00
|
|
|
|
|
|
|
return table;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
|
|
|
|
{
|
|
|
|
struct ctl_table *entry, *table;
|
|
|
|
struct sched_domain *sd;
|
|
|
|
int domain_num = 0, i;
|
|
|
|
char buf[32];
|
|
|
|
|
|
|
|
for_each_domain(cpu, sd)
|
|
|
|
domain_num++;
|
|
|
|
entry = table = sd_alloc_ctl_entry(domain_num + 1);
|
|
|
|
if (table == NULL)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
i = 0;
|
|
|
|
for_each_domain(cpu, sd) {
|
|
|
|
snprintf(buf, 32, "domain%d", i);
|
|
|
|
entry->procname = kstrdup(buf, GFP_KERNEL);
|
|
|
|
entry->mode = 0555;
|
|
|
|
entry->child = sd_alloc_ctl_domain_table(sd);
|
|
|
|
entry++;
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
return table;
|
|
|
|
}
|
|
|
|
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 06:01:12 -07:00
|
|
|
static cpumask_var_t sd_sysctl_cpus;
|
|
|
|
static struct ctl_table_header *sd_sysctl_header;
|
2017-08-10 09:10:26 -06:00
|
|
|
|
2016-02-22 14:26:51 -07:00
|
|
|
void register_sched_domain_sysctl(void)
|
|
|
|
{
|
2017-08-10 09:10:26 -06:00
|
|
|
static struct ctl_table *cpu_entries;
|
|
|
|
static struct ctl_table **cpu_idx;
|
2019-01-29 08:12:45 -07:00
|
|
|
static bool init_done = false;
|
2016-02-22 14:26:51 -07:00
|
|
|
char buf[32];
|
2017-08-10 09:10:26 -06:00
|
|
|
int i;
|
2016-02-22 14:26:51 -07:00
|
|
|
|
2017-08-10 09:10:26 -06:00
|
|
|
if (!cpu_entries) {
|
|
|
|
cpu_entries = sd_alloc_ctl_entry(num_possible_cpus() + 1);
|
|
|
|
if (!cpu_entries)
|
|
|
|
return;
|
2016-02-22 14:26:51 -07:00
|
|
|
|
2017-08-10 09:10:26 -06:00
|
|
|
WARN_ON(sd_ctl_dir[0].child);
|
|
|
|
sd_ctl_dir[0].child = cpu_entries;
|
|
|
|
}
|
2016-02-22 14:26:51 -07:00
|
|
|
|
2017-08-10 09:10:26 -06:00
|
|
|
if (!cpu_idx) {
|
|
|
|
struct ctl_table *e = cpu_entries;
|
|
|
|
|
|
|
|
cpu_idx = kcalloc(nr_cpu_ids, sizeof(struct ctl_table*), GFP_KERNEL);
|
|
|
|
if (!cpu_idx)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* deal with sparse possible map */
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
cpu_idx[i] = e;
|
|
|
|
e++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!cpumask_available(sd_sysctl_cpus)) {
|
|
|
|
if (!alloc_cpumask_var(&sd_sysctl_cpus, GFP_KERNEL))
|
|
|
|
return;
|
2019-01-29 08:12:45 -07:00
|
|
|
}
|
2017-08-10 09:10:26 -06:00
|
|
|
|
2019-01-29 08:12:45 -07:00
|
|
|
if (!init_done) {
|
|
|
|
init_done = true;
|
2017-08-10 09:10:26 -06:00
|
|
|
/* init to possible to not have holes in @cpu_entries */
|
|
|
|
cpumask_copy(sd_sysctl_cpus, cpu_possible_mask);
|
|
|
|
}
|
|
|
|
|
|
|
|
for_each_cpu(i, sd_sysctl_cpus) {
|
|
|
|
struct ctl_table *e = cpu_idx[i];
|
|
|
|
|
|
|
|
if (e->child)
|
|
|
|
sd_free_ctl_entry(&e->child);
|
|
|
|
|
|
|
|
if (!e->procname) {
|
|
|
|
snprintf(buf, 32, "cpu%d", i);
|
|
|
|
e->procname = kstrdup(buf, GFP_KERNEL);
|
|
|
|
}
|
|
|
|
e->mode = 0555;
|
|
|
|
e->child = sd_alloc_ctl_cpu_table(i);
|
|
|
|
|
|
|
|
__cpumask_clear_cpu(i, sd_sysctl_cpus);
|
2016-02-22 14:26:51 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
WARN_ON(sd_sysctl_header);
|
|
|
|
sd_sysctl_header = register_sysctl_table(sd_ctl_root);
|
|
|
|
}
|
|
|
|
|
2017-08-10 09:10:26 -06:00
|
|
|
void dirty_sched_domain_sysctl(int cpu)
|
|
|
|
{
|
|
|
|
if (cpumask_available(sd_sysctl_cpus))
|
|
|
|
__cpumask_set_cpu(cpu, sd_sysctl_cpus);
|
|
|
|
}
|
|
|
|
|
2016-02-22 14:26:51 -07:00
|
|
|
/* may be called multiple times per register */
|
|
|
|
void unregister_sched_domain_sysctl(void)
|
|
|
|
{
|
|
|
|
unregister_sysctl_table(sd_sysctl_header);
|
|
|
|
sd_sysctl_header = NULL;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_SYSCTL */
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
2008-11-10 09:04:09 -07:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
sched: Add 'autogroup' scheduling feature: automated per session task groups
A recurring complaint from CFS users is that parallel kbuild has
a negative impact on desktop interactivity. This patch
implements an idea from Linus, to automatically create task
groups. Currently, only per session autogroups are implemented,
but the patch leaves the way open for enhancement.
Implementation: each task's signal struct contains an inherited
pointer to a refcounted autogroup struct containing a task group
pointer, the default for all tasks pointing to the
init_task_group. When a task calls setsid(), a new task group
is created, the process is moved into the new task group, and a
reference to the preveious task group is dropped. Child
processes inherit this task group thereafter, and increase it's
refcount. When the last thread of a process exits, the
process's reference is dropped, such that when the last process
referencing an autogroup exits, the autogroup is destroyed.
At runqueue selection time, IFF a task has no cgroup assignment,
its current autogroup is used.
Autogroup bandwidth is controllable via setting it's nice level
through the proc filesystem:
cat /proc/<pid>/autogroup
Displays the task's group and the group's nice level.
echo <nice level> > /proc/<pid>/autogroup
Sets the task group's shares to the weight of nice <level> task.
Setting nice level is rate limited for !admin users due to the
abuse risk of task group locking.
The feature is enabled from boot by default if
CONFIG_SCHED_AUTOGROUP=y is selected, but can be disabled via
the boot option noautogroup, and can also be turned on/off on
the fly via:
echo [01] > /proc/sys/kernel/sched_autogroup_enabled
... which will automatically move tasks to/from the root task group.
Signed-off-by: Mike Galbraith <efault@gmx.de>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Markus Trippelsdorf <markus@trippelsdorf.de>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Paul Turner <pjt@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
[ Removed the task_group_path() debug code, and fixed !EVENTFD build failure. ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
LKML-Reference: <1290281700.28711.9.camel@maggy.simson.net>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-11-30 06:18:03 -07:00
|
|
|
static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
|
2008-11-10 09:04:09 -07:00
|
|
|
{
|
|
|
|
struct sched_entity *se = tg->se[cpu];
|
|
|
|
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 06:01:12 -07:00
|
|
|
#define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
|
|
|
|
#define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)schedstat_val(F))
|
|
|
|
#define PN(F) SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F))
|
|
|
|
#define PN_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)schedstat_val(F)))
|
2008-11-10 09:04:09 -07:00
|
|
|
|
2015-07-14 18:04:36 -06:00
|
|
|
if (!se)
|
2012-10-04 04:51:20 -06:00
|
|
|
return;
|
|
|
|
|
2008-11-10 09:04:09 -07:00
|
|
|
PN(se->exec_start);
|
|
|
|
PN(se->vruntime);
|
|
|
|
PN(se->sum_exec_runtime);
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 06:01:12 -07:00
|
|
|
|
2016-02-05 02:08:36 -07:00
|
|
|
if (schedstat_enabled()) {
|
2016-06-17 11:43:26 -06:00
|
|
|
PN_SCHEDSTAT(se->statistics.wait_start);
|
|
|
|
PN_SCHEDSTAT(se->statistics.sleep_start);
|
|
|
|
PN_SCHEDSTAT(se->statistics.block_start);
|
|
|
|
PN_SCHEDSTAT(se->statistics.sleep_max);
|
|
|
|
PN_SCHEDSTAT(se->statistics.block_max);
|
|
|
|
PN_SCHEDSTAT(se->statistics.exec_max);
|
|
|
|
PN_SCHEDSTAT(se->statistics.slice_max);
|
|
|
|
PN_SCHEDSTAT(se->statistics.wait_max);
|
|
|
|
PN_SCHEDSTAT(se->statistics.wait_sum);
|
|
|
|
P_SCHEDSTAT(se->statistics.wait_count);
|
2016-02-05 02:08:36 -07:00
|
|
|
}
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 06:01:12 -07:00
|
|
|
|
2008-11-10 09:04:09 -07:00
|
|
|
P(se->load.weight);
|
2017-05-06 07:59:54 -06:00
|
|
|
P(se->runnable_weight);
|
2012-10-04 05:18:29 -06:00
|
|
|
#ifdef CONFIG_SMP
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-14 18:04:37 -06:00
|
|
|
P(se->avg.load_avg);
|
|
|
|
P(se->avg.util_avg);
|
2017-05-06 07:59:54 -06:00
|
|
|
P(se->avg.runnable_load_avg);
|
2012-10-04 05:18:29 -06:00
|
|
|
#endif
|
2016-06-17 11:43:26 -06:00
|
|
|
|
|
|
|
#undef PN_SCHEDSTAT
|
2008-11-10 09:04:09 -07:00
|
|
|
#undef PN
|
2016-06-17 11:43:26 -06:00
|
|
|
#undef P_SCHEDSTAT
|
2008-11-10 09:04:09 -07:00
|
|
|
#undef P
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2011-01-11 03:11:54 -07:00
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
|
|
|
static char group_path[PATH_MAX];
|
|
|
|
|
|
|
|
static char *task_group_path(struct task_group *tg)
|
|
|
|
{
|
2011-01-11 03:12:57 -07:00
|
|
|
if (autogroup_path(tg, group_path, PATH_MAX))
|
|
|
|
return group_path;
|
|
|
|
|
2016-08-10 09:23:44 -06:00
|
|
|
cgroup_path(tg->css.cgroup, group_path, PATH_MAX);
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 06:01:12 -07:00
|
|
|
|
2016-08-10 09:23:44 -06:00
|
|
|
return group_path;
|
2011-01-11 03:11:54 -07:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2007-07-09 10:52:00 -06:00
|
|
|
static void
|
2007-08-09 03:16:51 -06:00
|
|
|
print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
|
2007-07-09 10:52:00 -06:00
|
|
|
{
|
2017-08-07 02:44:23 -06:00
|
|
|
if (rq->curr == p)
|
2017-08-07 02:44:22 -06:00
|
|
|
SEQ_printf(m, ">R");
|
2017-08-07 02:44:23 -06:00
|
|
|
else
|
|
|
|
SEQ_printf(m, " %c", task_state_to_char(p));
|
2007-07-09 10:52:00 -06:00
|
|
|
|
2007-10-15 09:00:08 -06:00
|
|
|
SEQ_printf(m, "%15s %5d %9Ld.%06ld %9Ld %5d ",
|
2013-09-09 05:01:41 -06:00
|
|
|
p->comm, task_pid_nr(p),
|
2007-10-15 09:00:08 -06:00
|
|
|
SPLIT_NS(p->se.vruntime),
|
2007-07-09 10:52:00 -06:00
|
|
|
(long long)(p->nvcsw + p->nivcsw),
|
2007-08-05 21:26:59 -06:00
|
|
|
p->prio);
|
2016-06-03 16:58:40 -06:00
|
|
|
|
2015-06-08 02:10:39 -06:00
|
|
|
SEQ_printf(m, "%9Ld.%06ld %9Ld.%06ld %9Ld.%06ld",
|
2016-06-17 11:43:25 -06:00
|
|
|
SPLIT_NS(schedstat_val_or_zero(p->se.statistics.wait_sum)),
|
2015-06-08 02:10:39 -06:00
|
|
|
SPLIT_NS(p->se.sum_exec_runtime),
|
2016-06-17 11:43:25 -06:00
|
|
|
SPLIT_NS(schedstat_val_or_zero(p->se.statistics.sum_sleep_runtime)));
|
2016-06-03 16:58:40 -06:00
|
|
|
|
2013-10-07 04:29:30 -06:00
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
2015-06-25 11:21:42 -06:00
|
|
|
SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
|
2013-10-07 04:29:30 -06:00
|
|
|
#endif
|
2011-01-11 03:11:54 -07:00
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
|
|
|
SEQ_printf(m, " %s", task_group_path(task_group(p)));
|
|
|
|
#endif
|
2008-04-19 11:45:00 -06:00
|
|
|
|
|
|
|
SEQ_printf(m, "\n");
|
2007-07-09 10:52:00 -06:00
|
|
|
}
|
|
|
|
|
2007-08-09 03:16:51 -06:00
|
|
|
static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
|
2007-07-09 10:52:00 -06:00
|
|
|
{
|
|
|
|
struct task_struct *g, *p;
|
|
|
|
|
2018-03-19 12:35:55 -06:00
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
SEQ_printf(m, "runnable tasks:\n");
|
|
|
|
SEQ_printf(m, " S task PID tree-key switches prio"
|
|
|
|
" wait-time sum-exec sum-sleep\n");
|
|
|
|
SEQ_printf(m, "-------------------------------------------------------"
|
|
|
|
"----------------------------------------------------\n");
|
2007-07-09 10:52:00 -06:00
|
|
|
|
2014-09-21 13:33:41 -06:00
|
|
|
rcu_read_lock();
|
2014-08-13 13:19:56 -06:00
|
|
|
for_each_process_thread(g, p) {
|
2013-10-07 04:29:30 -06:00
|
|
|
if (task_cpu(p) != rq_cpu)
|
2007-07-09 10:52:00 -06:00
|
|
|
continue;
|
|
|
|
|
2007-08-09 03:16:51 -06:00
|
|
|
print_task(m, rq, p);
|
2014-08-13 13:19:56 -06:00
|
|
|
}
|
2014-09-21 13:33:41 -06:00
|
|
|
rcu_read_unlock();
|
2007-07-09 10:52:00 -06:00
|
|
|
}
|
|
|
|
|
2007-08-09 03:16:47 -06:00
|
|
|
void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
|
2007-07-09 10:52:00 -06:00
|
|
|
{
|
2007-10-15 09:00:06 -06:00
|
|
|
s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1,
|
|
|
|
spread, rq0_min_vruntime, spread0;
|
2009-06-17 07:20:55 -06:00
|
|
|
struct rq *rq = cpu_rq(cpu);
|
2007-10-15 09:00:05 -06:00
|
|
|
struct sched_entity *last;
|
|
|
|
unsigned long flags;
|
|
|
|
|
2011-01-11 03:11:54 -07:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2018-03-19 12:35:55 -06:00
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
SEQ_printf(m, "cfs_rq[%d]:%s\n", cpu, task_group_path(cfs_rq->tg));
|
2011-01-11 03:11:54 -07:00
|
|
|
#else
|
2018-03-19 12:35:55 -06:00
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
SEQ_printf(m, "cfs_rq[%d]:\n", cpu);
|
2011-01-11 03:11:54 -07:00
|
|
|
#endif
|
2007-10-15 09:00:08 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
|
|
|
|
SPLIT_NS(cfs_rq->exec_clock));
|
2007-10-15 09:00:05 -06:00
|
|
|
|
2009-11-17 06:28:38 -07:00
|
|
|
raw_spin_lock_irqsave(&rq->lock, flags);
|
2017-09-08 17:14:55 -06:00
|
|
|
if (rb_first_cached(&cfs_rq->tasks_timeline))
|
2011-02-01 07:51:03 -07:00
|
|
|
MIN_vruntime = (__pick_first_entity(cfs_rq))->vruntime;
|
2007-10-15 09:00:05 -06:00
|
|
|
last = __pick_last_entity(cfs_rq);
|
|
|
|
if (last)
|
|
|
|
max_vruntime = last->vruntime;
|
2008-11-10 02:46:32 -07:00
|
|
|
min_vruntime = cfs_rq->min_vruntime;
|
2009-06-17 07:20:55 -06:00
|
|
|
rq0_min_vruntime = cpu_rq(0)->cfs.min_vruntime;
|
2009-11-17 06:28:38 -07:00
|
|
|
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
2007-10-15 09:00:08 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime",
|
|
|
|
SPLIT_NS(MIN_vruntime));
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime",
|
|
|
|
SPLIT_NS(min_vruntime));
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "max_vruntime",
|
|
|
|
SPLIT_NS(max_vruntime));
|
2007-10-15 09:00:05 -06:00
|
|
|
spread = max_vruntime - MIN_vruntime;
|
2007-10-15 09:00:08 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread",
|
|
|
|
SPLIT_NS(spread));
|
2007-10-15 09:00:06 -06:00
|
|
|
spread0 = min_vruntime - rq0_min_vruntime;
|
2007-10-15 09:00:08 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread0",
|
|
|
|
SPLIT_NS(spread0));
|
2008-11-10 02:46:32 -07:00
|
|
|
SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over",
|
2007-10-15 09:00:10 -06:00
|
|
|
cfs_rq->nr_spread_over);
|
2012-04-26 05:12:27 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
|
2010-11-15 16:47:00 -07:00
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight);
|
2008-06-27 05:41:14 -06:00
|
|
|
#ifdef CONFIG_SMP
|
2017-05-06 07:59:54 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "runnable_weight", cfs_rq->runnable_weight);
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-14 18:04:37 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", "load_avg",
|
|
|
|
cfs_rq->avg.load_avg);
|
2015-07-14 18:04:41 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", "runnable_load_avg",
|
2017-05-06 07:59:54 -06:00
|
|
|
cfs_rq->avg.runnable_load_avg);
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-14 18:04:37 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", "util_avg",
|
|
|
|
cfs_rq->avg.util_avg);
|
sched/fair: Add util_est on top of PELT
The util_avg signal computed by PELT is too variable for some use-cases.
For example, a big task waking up after a long sleep period will have its
utilization almost completely decayed. This introduces some latency before
schedutil will be able to pick the best frequency to run a task.
The same issue can affect task placement. Indeed, since the task
utilization is already decayed at wakeup, when the task is enqueued in a
CPU, this can result in a CPU running a big task as being temporarily
represented as being almost empty. This leads to a race condition where
other tasks can be potentially allocated on a CPU which just started to run
a big task which slept for a relatively long period.
Moreover, the PELT utilization of a task can be updated every [ms], thus
making it a continuously changing value for certain longer running
tasks. This means that the instantaneous PELT utilization of a RUNNING
task is not really meaningful to properly support scheduler decisions.
For all these reasons, a more stable signal can do a better job of
representing the expected/estimated utilization of a task/cfs_rq.
Such a signal can be easily created on top of PELT by still using it as
an estimator which produces values to be aggregated on meaningful
events.
This patch adds a simple implementation of util_est, a new signal built on
top of PELT's util_avg where:
util_est(task) = max(task::util_avg, f(task::util_avg@dequeue))
This allows to remember how big a task has been reported by PELT in its
previous activations via f(task::util_avg@dequeue), which is the new
_task_util_est(struct task_struct*) function added by this patch.
If a task should change its behavior and it runs longer in a new
activation, after a certain time its util_est will just track the
original PELT signal (i.e. task::util_avg).
The estimated utilization of cfs_rq is defined only for root ones.
That's because the only sensible consumer of this signal are the
scheduler and schedutil when looking for the overall CPU utilization
due to FAIR tasks.
For this reason, the estimated utilization of a root cfs_rq is simply
defined as:
util_est(cfs_rq) = max(cfs_rq::util_avg, cfs_rq::util_est::enqueued)
where:
cfs_rq::util_est::enqueued = sum(_task_util_est(task))
for each RUNNABLE task on that root cfs_rq
It's worth noting that the estimated utilization is tracked only for
objects of interests, specifically:
- Tasks: to better support tasks placement decisions
- root cfs_rqs: to better support both tasks placement decisions as
well as frequencies selection
Signed-off-by: Patrick Bellasi <patrick.bellasi@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Joel Fernandes <joelaf@google.com>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Morten Rasmussen <morten.rasmussen@arm.com>
Cc: Paul Turner <pjt@google.com>
Cc: Rafael J . Wysocki <rafael.j.wysocki@intel.com>
Cc: Steve Muckle <smuckle@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Todd Kjos <tkjos@android.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Cc: Viresh Kumar <viresh.kumar@linaro.org>
Link: http://lkml.kernel.org/r/20180309095245.11071-2-patrick.bellasi@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-09 02:52:42 -07:00
|
|
|
SEQ_printf(m, " .%-30s: %u\n", "util_est_enqueued",
|
|
|
|
cfs_rq->avg.util_est.enqueued);
|
2017-05-08 08:51:41 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "removed.load_avg",
|
|
|
|
cfs_rq->removed.load_avg);
|
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "removed.util_avg",
|
|
|
|
cfs_rq->removed.util_avg);
|
2017-05-08 09:30:46 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "removed.runnable_sum",
|
|
|
|
cfs_rq->removed.runnable_sum);
|
2013-06-28 05:10:35 -06:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-14 18:04:37 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", "tg_load_avg_contrib",
|
|
|
|
cfs_rq->tg_load_avg_contrib);
|
2013-06-28 05:10:35 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg",
|
|
|
|
atomic_long_read(&cfs_rq->tg->load_avg));
|
2008-06-27 05:41:14 -06:00
|
|
|
#endif
|
2013-06-28 05:10:35 -06:00
|
|
|
#endif
|
2013-10-16 12:16:32 -06:00
|
|
|
#ifdef CONFIG_CFS_BANDWIDTH
|
|
|
|
SEQ_printf(m, " .%-30s: %d\n", "throttled",
|
|
|
|
cfs_rq->throttled);
|
|
|
|
SEQ_printf(m, " .%-30s: %d\n", "throttle_count",
|
|
|
|
cfs_rq->throttle_count);
|
|
|
|
#endif
|
2010-11-15 16:47:00 -07:00
|
|
|
|
2013-06-28 05:10:35 -06:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2008-11-10 09:04:09 -07:00
|
|
|
print_cfs_group_stats(m, cpu, cfs_rq->tg);
|
2008-06-27 05:41:14 -06:00
|
|
|
#endif
|
2007-07-09 10:52:00 -06:00
|
|
|
}
|
|
|
|
|
2008-06-19 06:22:24 -06:00
|
|
|
void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
|
|
|
|
{
|
2011-01-11 03:11:54 -07:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
2018-03-19 12:35:55 -06:00
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
SEQ_printf(m, "rt_rq[%d]:%s\n", cpu, task_group_path(rt_rq->tg));
|
2011-01-11 03:11:54 -07:00
|
|
|
#else
|
2018-03-19 12:35:55 -06:00
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
SEQ_printf(m, "rt_rq[%d]:\n", cpu);
|
2011-01-11 03:11:54 -07:00
|
|
|
#endif
|
2008-06-19 06:22:24 -06:00
|
|
|
|
|
|
|
#define P(x) \
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rt_rq->x))
|
2017-06-26 09:07:14 -06:00
|
|
|
#define PU(x) \
|
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", #x, (unsigned long)(rt_rq->x))
|
2008-06-19 06:22:24 -06:00
|
|
|
#define PN(x) \
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
|
|
|
|
|
2017-06-26 09:07:14 -06:00
|
|
|
PU(rt_nr_running);
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
PU(rt_nr_migratory);
|
|
|
|
#endif
|
2008-06-19 06:22:24 -06:00
|
|
|
P(rt_throttled);
|
|
|
|
PN(rt_time);
|
|
|
|
PN(rt_runtime);
|
|
|
|
|
|
|
|
#undef PN
|
2017-06-26 09:07:14 -06:00
|
|
|
#undef PU
|
2008-06-19 06:22:24 -06:00
|
|
|
#undef P
|
|
|
|
}
|
|
|
|
|
2014-10-30 16:39:33 -06:00
|
|
|
void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq)
|
|
|
|
{
|
2016-02-22 14:26:52 -07:00
|
|
|
struct dl_bw *dl_bw;
|
|
|
|
|
2018-03-19 12:35:55 -06:00
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
SEQ_printf(m, "dl_rq[%d]:\n", cpu);
|
2017-06-26 09:07:14 -06:00
|
|
|
|
|
|
|
#define PU(x) \
|
|
|
|
SEQ_printf(m, " .%-30s: %lu\n", #x, (unsigned long)(dl_rq->x))
|
|
|
|
|
|
|
|
PU(dl_nr_running);
|
2016-02-22 14:26:52 -07:00
|
|
|
#ifdef CONFIG_SMP
|
2017-06-26 09:07:14 -06:00
|
|
|
PU(dl_nr_migratory);
|
2016-02-22 14:26:52 -07:00
|
|
|
dl_bw = &cpu_rq(cpu)->rd->dl_bw;
|
|
|
|
#else
|
|
|
|
dl_bw = &dl_rq->dl_bw;
|
|
|
|
#endif
|
|
|
|
SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->bw", dl_bw->bw);
|
|
|
|
SEQ_printf(m, " .%-30s: %lld\n", "dl_bw->total_bw", dl_bw->total_bw);
|
2017-06-26 09:07:14 -06:00
|
|
|
|
|
|
|
#undef PU
|
2014-10-30 16:39:33 -06:00
|
|
|
}
|
|
|
|
|
2007-08-09 03:16:51 -06:00
|
|
|
static void print_cpu(struct seq_file *m, int cpu)
|
2007-07-09 10:52:00 -06:00
|
|
|
{
|
2009-06-17 07:20:55 -06:00
|
|
|
struct rq *rq = cpu_rq(cpu);
|
2011-01-11 03:11:54 -07:00
|
|
|
unsigned long flags;
|
2007-07-09 10:52:00 -06:00
|
|
|
|
|
|
|
#ifdef CONFIG_X86
|
|
|
|
{
|
|
|
|
unsigned int freq = cpu_khz ? : 1;
|
|
|
|
|
2013-02-21 16:15:09 -07:00
|
|
|
SEQ_printf(m, "cpu#%d, %u.%03u MHz\n",
|
2007-07-09 10:52:00 -06:00
|
|
|
cpu, freq / 1000, (freq % 1000));
|
|
|
|
}
|
|
|
|
#else
|
2013-02-21 16:15:09 -07:00
|
|
|
SEQ_printf(m, "cpu#%d\n", cpu);
|
2007-07-09 10:52:00 -06:00
|
|
|
#endif
|
|
|
|
|
2012-05-14 06:34:00 -06:00
|
|
|
#define P(x) \
|
|
|
|
do { \
|
|
|
|
if (sizeof(rq->x) == 4) \
|
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", #x, (long)(rq->x)); \
|
|
|
|
else \
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rq->x));\
|
|
|
|
} while (0)
|
|
|
|
|
2007-10-15 09:00:08 -06:00
|
|
|
#define PN(x) \
|
|
|
|
SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rq->x))
|
2007-07-09 10:52:00 -06:00
|
|
|
|
|
|
|
P(nr_running);
|
|
|
|
P(nr_switches);
|
|
|
|
P(nr_load_updates);
|
|
|
|
P(nr_uninterruptible);
|
2007-10-15 09:00:08 -06:00
|
|
|
PN(next_balance);
|
2013-09-09 05:01:41 -06:00
|
|
|
SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr)));
|
2007-10-15 09:00:08 -06:00
|
|
|
PN(clock);
|
2015-01-05 03:18:12 -07:00
|
|
|
PN(clock_task);
|
2007-07-09 10:52:00 -06:00
|
|
|
#undef P
|
2007-10-15 09:00:08 -06:00
|
|
|
#undef PN
|
2007-07-09 10:52:00 -06:00
|
|
|
|
2009-11-04 09:53:50 -07:00
|
|
|
#ifdef CONFIG_SMP
|
2016-05-02 22:38:25 -06:00
|
|
|
#define P64(n) SEQ_printf(m, " .%-30s: %Ld\n", #n, rq->n);
|
2009-11-04 09:53:50 -07:00
|
|
|
P64(avg_idle);
|
2014-01-23 03:39:54 -07:00
|
|
|
P64(max_idle_balance_cost);
|
2016-05-02 22:38:25 -06:00
|
|
|
#undef P64
|
2009-11-04 09:53:50 -07:00
|
|
|
#endif
|
2008-11-10 02:46:32 -07:00
|
|
|
|
2016-06-17 11:43:26 -06:00
|
|
|
#define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, schedstat_val(rq->n));
|
2016-02-05 02:08:36 -07:00
|
|
|
if (schedstat_enabled()) {
|
|
|
|
P(yld_count);
|
|
|
|
P(sched_count);
|
|
|
|
P(sched_goidle);
|
|
|
|
P(ttwu_count);
|
|
|
|
P(ttwu_local);
|
|
|
|
}
|
2008-11-10 02:46:32 -07:00
|
|
|
#undef P
|
2016-06-17 11:43:26 -06:00
|
|
|
|
2011-01-11 03:11:54 -07:00
|
|
|
spin_lock_irqsave(&sched_debug_lock, flags);
|
2007-08-09 03:16:47 -06:00
|
|
|
print_cfs_stats(m, cpu);
|
2008-06-19 06:22:24 -06:00
|
|
|
print_rt_stats(m, cpu);
|
2014-10-30 16:39:33 -06:00
|
|
|
print_dl_stats(m, cpu);
|
2007-07-09 10:52:00 -06:00
|
|
|
|
2007-08-09 03:16:51 -06:00
|
|
|
print_rq(m, rq, cpu);
|
2011-01-11 03:11:54 -07:00
|
|
|
spin_unlock_irqrestore(&sched_debug_lock, flags);
|
2013-02-21 16:15:09 -07:00
|
|
|
SEQ_printf(m, "\n");
|
2007-07-09 10:52:00 -06:00
|
|
|
}
|
|
|
|
|
2009-11-30 04:16:47 -07:00
|
|
|
static const char *sched_tunable_scaling_names[] = {
|
|
|
|
"none",
|
2018-11-28 08:23:50 -07:00
|
|
|
"logarithmic",
|
2009-11-30 04:16:47 -07:00
|
|
|
"linear"
|
|
|
|
};
|
|
|
|
|
2013-02-21 16:15:09 -07:00
|
|
|
static void sched_debug_header(struct seq_file *m)
|
2007-07-09 10:52:00 -06:00
|
|
|
{
|
2010-11-19 13:11:09 -07:00
|
|
|
u64 ktime, sched_clk, cpu_clk;
|
|
|
|
unsigned long flags;
|
2007-07-09 10:52:00 -06:00
|
|
|
|
2010-11-19 13:11:09 -07:00
|
|
|
local_irq_save(flags);
|
|
|
|
ktime = ktime_to_ns(ktime_get());
|
|
|
|
sched_clk = sched_clock();
|
|
|
|
cpu_clk = local_clock();
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
2013-10-07 04:29:30 -06:00
|
|
|
SEQ_printf(m, "Sched Debug Version: v0.11, %s %.*s\n",
|
2007-07-09 10:52:00 -06:00
|
|
|
init_utsname()->release,
|
|
|
|
(int)strcspn(init_utsname()->version, " "),
|
|
|
|
init_utsname()->version);
|
|
|
|
|
2010-11-19 13:11:09 -07:00
|
|
|
#define P(x) \
|
|
|
|
SEQ_printf(m, "%-40s: %Ld\n", #x, (long long)(x))
|
|
|
|
#define PN(x) \
|
|
|
|
SEQ_printf(m, "%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
|
|
|
|
PN(ktime);
|
|
|
|
PN(sched_clk);
|
|
|
|
PN(cpu_clk);
|
|
|
|
P(jiffies);
|
|
|
|
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
|
2013-11-28 11:38:42 -07:00
|
|
|
P(sched_clock_stable());
|
2010-11-19 13:11:09 -07:00
|
|
|
#endif
|
|
|
|
#undef PN
|
|
|
|
#undef P
|
|
|
|
|
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
SEQ_printf(m, "sysctl_sched\n");
|
2007-07-09 10:52:00 -06:00
|
|
|
|
2007-10-15 09:00:10 -06:00
|
|
|
#define P(x) \
|
2007-10-15 09:00:10 -06:00
|
|
|
SEQ_printf(m, " .%-40s: %Ld\n", #x, (long long)(x))
|
2007-10-15 09:00:10 -06:00
|
|
|
#define PN(x) \
|
2007-10-15 09:00:10 -06:00
|
|
|
SEQ_printf(m, " .%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
|
2007-10-15 09:00:10 -06:00
|
|
|
PN(sysctl_sched_latency);
|
2007-11-09 14:39:37 -07:00
|
|
|
PN(sysctl_sched_min_granularity);
|
2007-10-15 09:00:10 -06:00
|
|
|
PN(sysctl_sched_wakeup_granularity);
|
2010-07-19 13:31:16 -06:00
|
|
|
P(sysctl_sched_child_runs_first);
|
2007-10-15 09:00:10 -06:00
|
|
|
P(sysctl_sched_features);
|
|
|
|
#undef PN
|
|
|
|
#undef P
|
|
|
|
|
2013-02-21 16:15:09 -07:00
|
|
|
SEQ_printf(m, " .%-40s: %d (%s)\n",
|
|
|
|
"sysctl_sched_tunable_scaling",
|
2009-11-30 04:16:47 -07:00
|
|
|
sysctl_sched_tunable_scaling,
|
|
|
|
sched_tunable_scaling_names[sysctl_sched_tunable_scaling]);
|
2013-02-21 16:15:09 -07:00
|
|
|
SEQ_printf(m, "\n");
|
|
|
|
}
|
2009-11-30 04:16:47 -07:00
|
|
|
|
2013-02-21 16:15:09 -07:00
|
|
|
static int sched_debug_show(struct seq_file *m, void *v)
|
|
|
|
{
|
|
|
|
int cpu = (unsigned long)(v - 2);
|
2007-07-09 10:52:00 -06:00
|
|
|
|
2013-02-21 16:15:09 -07:00
|
|
|
if (cpu != -1)
|
|
|
|
print_cpu(m, cpu);
|
|
|
|
else
|
|
|
|
sched_debug_header(m);
|
2007-07-09 10:52:00 -06:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-10-25 02:00:11 -06:00
|
|
|
void sysrq_sched_debug_show(void)
|
2007-07-09 10:52:00 -06:00
|
|
|
{
|
2013-02-21 16:15:09 -07:00
|
|
|
int cpu;
|
|
|
|
|
|
|
|
sched_debug_header(NULL);
|
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
print_cpu(NULL, cpu);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This itererator needs some explanation.
|
|
|
|
* It returns 1 for the header position.
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 06:01:12 -07:00
|
|
|
* This means 2 is CPU 0.
|
|
|
|
* In a hotplugged system some CPUs, including CPU 0, may be missing so we have
|
|
|
|
* to use cpumask_* to iterate over the CPUs.
|
2013-02-21 16:15:09 -07:00
|
|
|
*/
|
|
|
|
static void *sched_debug_start(struct seq_file *file, loff_t *offset)
|
|
|
|
{
|
|
|
|
unsigned long n = *offset;
|
|
|
|
|
|
|
|
if (n == 0)
|
|
|
|
return (void *) 1;
|
|
|
|
|
|
|
|
n--;
|
|
|
|
|
|
|
|
if (n > 0)
|
|
|
|
n = cpumask_next(n - 1, cpu_online_mask);
|
|
|
|
else
|
|
|
|
n = cpumask_first(cpu_online_mask);
|
|
|
|
|
|
|
|
*offset = n + 1;
|
|
|
|
|
|
|
|
if (n < nr_cpu_ids)
|
|
|
|
return (void *)(unsigned long)(n + 2);
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 06:01:12 -07:00
|
|
|
|
2013-02-21 16:15:09 -07:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *sched_debug_next(struct seq_file *file, void *data, loff_t *offset)
|
|
|
|
{
|
|
|
|
(*offset)++;
|
|
|
|
return sched_debug_start(file, offset);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void sched_debug_stop(struct seq_file *file, void *data)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct seq_operations sched_debug_sops = {
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 06:01:12 -07:00
|
|
|
.start = sched_debug_start,
|
|
|
|
.next = sched_debug_next,
|
|
|
|
.stop = sched_debug_stop,
|
|
|
|
.show = sched_debug_show,
|
2013-02-21 16:15:09 -07:00
|
|
|
};
|
|
|
|
|
2007-07-09 10:52:00 -06:00
|
|
|
static int __init init_sched_debug_procfs(void)
|
|
|
|
{
|
2018-04-13 11:44:18 -06:00
|
|
|
if (!proc_create_seq("sched_debug", 0444, NULL, &sched_debug_sops))
|
2007-07-09 10:52:00 -06:00
|
|
|
return -ENOMEM;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
__initcall(init_sched_debug_procfs);
|
|
|
|
|
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-03 06:01:12 -07:00
|
|
|
#define __P(F) SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
|
|
|
|
#define P(F) SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
|
|
|
|
#define __PN(F) SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
|
|
|
|
#define PN(F) SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
|
2013-10-07 04:29:30 -06:00
|
|
|
|
|
|
|
|
2015-06-25 11:21:43 -06:00
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
|
|
void print_numa_stats(struct seq_file *m, int node, unsigned long tsf,
|
|
|
|
unsigned long tpf, unsigned long gsf, unsigned long gpf)
|
|
|
|
{
|
|
|
|
SEQ_printf(m, "numa_faults node=%d ", node);
|
2018-06-20 11:02:47 -06:00
|
|
|
SEQ_printf(m, "task_private=%lu task_shared=%lu ", tpf, tsf);
|
|
|
|
SEQ_printf(m, "group_private=%lu group_shared=%lu\n", gpf, gsf);
|
2015-06-25 11:21:43 -06:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
2013-10-07 04:29:30 -06:00
|
|
|
static void sched_show_numa(struct task_struct *p, struct seq_file *m)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_NUMA_BALANCING
|
|
|
|
struct mempolicy *pol;
|
|
|
|
|
|
|
|
if (p->mm)
|
|
|
|
P(mm->numa_scan_seq);
|
|
|
|
|
|
|
|
task_lock(p);
|
|
|
|
pol = p->mempolicy;
|
|
|
|
if (pol && !(pol->flags & MPOL_F_MORON))
|
|
|
|
pol = NULL;
|
|
|
|
mpol_get(pol);
|
|
|
|
task_unlock(p);
|
|
|
|
|
2015-06-25 11:21:43 -06:00
|
|
|
P(numa_pages_migrated);
|
|
|
|
P(numa_preferred_nid);
|
|
|
|
P(total_numa_faults);
|
|
|
|
SEQ_printf(m, "current_node=%d, numa_group_id=%d\n",
|
|
|
|
task_node(p), task_numa_group_id(p));
|
|
|
|
show_numa_stats(p, m);
|
2013-10-07 04:29:30 -06:00
|
|
|
mpol_put(pol);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2017-08-05 22:41:41 -06:00
|
|
|
void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
|
|
|
|
struct seq_file *m)
|
2007-07-09 10:52:00 -06:00
|
|
|
{
|
2007-10-15 09:00:18 -06:00
|
|
|
unsigned long nr_switches;
|
2007-07-09 10:52:00 -06:00
|
|
|
|
2017-08-05 22:41:41 -06:00
|
|
|
SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr_ns(p, ns),
|
2010-05-26 15:43:22 -06:00
|
|
|
get_nr_threads(p));
|
2007-10-15 09:00:18 -06:00
|
|
|
SEQ_printf(m,
|
2013-06-27 10:50:05 -06:00
|
|
|
"---------------------------------------------------------"
|
|
|
|
"----------\n");
|
2007-10-15 09:00:18 -06:00
|
|
|
#define __P(F) \
|
2013-06-27 10:50:05 -06:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
|
2007-07-09 10:52:00 -06:00
|
|
|
#define P(F) \
|
2013-06-27 10:50:05 -06:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
|
2016-06-17 11:43:26 -06:00
|
|
|
#define P_SCHEDSTAT(F) \
|
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)schedstat_val(p->F))
|
2007-10-15 09:00:18 -06:00
|
|
|
#define __PN(F) \
|
2013-06-27 10:50:05 -06:00
|
|
|
SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
|
2007-10-15 09:00:08 -06:00
|
|
|
#define PN(F) \
|
2013-06-27 10:50:05 -06:00
|
|
|
SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
|
2016-06-17 11:43:26 -06:00
|
|
|
#define PN_SCHEDSTAT(F) \
|
|
|
|
SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)schedstat_val(p->F)))
|
2007-07-09 10:52:00 -06:00
|
|
|
|
2007-10-15 09:00:08 -06:00
|
|
|
PN(se.exec_start);
|
|
|
|
PN(se.vruntime);
|
|
|
|
PN(se.sum_exec_runtime);
|
2007-08-02 09:41:40 -06:00
|
|
|
|
2007-10-15 09:00:18 -06:00
|
|
|
nr_switches = p->nvcsw + p->nivcsw;
|
|
|
|
|
|
|
|
P(se.nr_migrations);
|
|
|
|
|
2016-02-05 02:08:36 -07:00
|
|
|
if (schedstat_enabled()) {
|
2007-10-15 09:00:18 -06:00
|
|
|
u64 avg_atom, avg_per_cpu;
|
|
|
|
|
2016-06-17 11:43:26 -06:00
|
|
|
PN_SCHEDSTAT(se.statistics.sum_sleep_runtime);
|
|
|
|
PN_SCHEDSTAT(se.statistics.wait_start);
|
|
|
|
PN_SCHEDSTAT(se.statistics.sleep_start);
|
|
|
|
PN_SCHEDSTAT(se.statistics.block_start);
|
|
|
|
PN_SCHEDSTAT(se.statistics.sleep_max);
|
|
|
|
PN_SCHEDSTAT(se.statistics.block_max);
|
|
|
|
PN_SCHEDSTAT(se.statistics.exec_max);
|
|
|
|
PN_SCHEDSTAT(se.statistics.slice_max);
|
|
|
|
PN_SCHEDSTAT(se.statistics.wait_max);
|
|
|
|
PN_SCHEDSTAT(se.statistics.wait_sum);
|
|
|
|
P_SCHEDSTAT(se.statistics.wait_count);
|
|
|
|
PN_SCHEDSTAT(se.statistics.iowait_sum);
|
|
|
|
P_SCHEDSTAT(se.statistics.iowait_count);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_migrations_cold);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_failed_migrations_affine);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_failed_migrations_running);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_failed_migrations_hot);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_forced_migrations);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_wakeups);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_wakeups_sync);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_wakeups_migrate);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_wakeups_local);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_wakeups_remote);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_wakeups_affine);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_wakeups_affine_attempts);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_wakeups_passive);
|
|
|
|
P_SCHEDSTAT(se.statistics.nr_wakeups_idle);
|
2016-02-05 02:08:36 -07:00
|
|
|
|
2007-10-15 09:00:18 -06:00
|
|
|
avg_atom = p->se.sum_exec_runtime;
|
|
|
|
if (nr_switches)
|
2014-06-14 07:00:09 -06:00
|
|
|
avg_atom = div64_ul(avg_atom, nr_switches);
|
2007-10-15 09:00:18 -06:00
|
|
|
else
|
|
|
|
avg_atom = -1LL;
|
|
|
|
|
|
|
|
avg_per_cpu = p->se.sum_exec_runtime;
|
2007-11-28 07:52:56 -07:00
|
|
|
if (p->se.nr_migrations) {
|
2008-05-01 05:34:28 -06:00
|
|
|
avg_per_cpu = div64_u64(avg_per_cpu,
|
|
|
|
p->se.nr_migrations);
|
2007-11-28 07:52:56 -07:00
|
|
|
} else {
|
2007-10-15 09:00:18 -06:00
|
|
|
avg_per_cpu = -1LL;
|
2007-11-28 07:52:56 -07:00
|
|
|
}
|
2007-10-15 09:00:18 -06:00
|
|
|
|
|
|
|
__PN(avg_atom);
|
|
|
|
__PN(avg_per_cpu);
|
|
|
|
}
|
2016-06-17 11:43:26 -06:00
|
|
|
|
2007-10-15 09:00:18 -06:00
|
|
|
__P(nr_switches);
|
2013-06-27 10:50:05 -06:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n",
|
2007-10-15 09:00:18 -06:00
|
|
|
"nr_voluntary_switches", (long long)p->nvcsw);
|
2013-06-27 10:50:05 -06:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n",
|
2007-10-15 09:00:18 -06:00
|
|
|
"nr_involuntary_switches", (long long)p->nivcsw);
|
|
|
|
|
2007-07-09 10:52:00 -06:00
|
|
|
P(se.load.weight);
|
2017-05-06 07:59:54 -06:00
|
|
|
P(se.runnable_weight);
|
2013-06-28 05:10:35 -06:00
|
|
|
#ifdef CONFIG_SMP
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-14 18:04:37 -06:00
|
|
|
P(se.avg.load_sum);
|
2017-05-06 07:59:54 -06:00
|
|
|
P(se.avg.runnable_load_sum);
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-14 18:04:37 -06:00
|
|
|
P(se.avg.util_sum);
|
|
|
|
P(se.avg.load_avg);
|
2017-05-06 07:59:54 -06:00
|
|
|
P(se.avg.runnable_load_avg);
|
sched/fair: Rewrite runnable load and utilization average tracking
The idea of runnable load average (let runnable time contribute to weight)
was proposed by Paul Turner and Ben Segall, and it is still followed by
this rewrite. This rewrite aims to solve the following issues:
1. cfs_rq's load average (namely runnable_load_avg and blocked_load_avg) is
updated at the granularity of an entity at a time, which results in the
cfs_rq's load average is stale or partially updated: at any time, only
one entity is up to date, all other entities are effectively lagging
behind. This is undesirable.
To illustrate, if we have n runnable entities in the cfs_rq, as time
elapses, they certainly become outdated:
t0: cfs_rq { e1_old, e2_old, ..., en_old }
and when we update:
t1: update e1, then we have cfs_rq { e1_new, e2_old, ..., en_old }
t2: update e2, then we have cfs_rq { e1_old, e2_new, ..., en_old }
...
We solve this by combining all runnable entities' load averages together
in cfs_rq's avg, and update the cfs_rq's avg as a whole. This is based
on the fact that if we regard the update as a function, then:
w * update(e) = update(w * e) and
update(e1) + update(e2) = update(e1 + e2), then
w1 * update(e1) + w2 * update(e2) = update(w1 * e1 + w2 * e2)
therefore, by this rewrite, we have an entirely updated cfs_rq at the
time we update it:
t1: update cfs_rq { e1_new, e2_new, ..., en_new }
t2: update cfs_rq { e1_new, e2_new, ..., en_new }
...
2. cfs_rq's load average is different between top rq->cfs_rq and other
task_group's per CPU cfs_rqs in whether or not blocked_load_average
contributes to the load.
The basic idea behind runnable load average (the same for utilization)
is that the blocked state is taken into account as opposed to only
accounting for the currently runnable state. Therefore, the average
should include both the runnable/running and blocked load averages.
This rewrite does that.
In addition, we also combine runnable/running and blocked averages
of all entities into the cfs_rq's average, and update it together at
once. This is based on the fact that:
update(runnable) + update(blocked) = update(runnable + blocked)
This significantly reduces the code as we don't need to separately
maintain/update runnable/running load and blocked load.
3. How task_group entities' share is calculated is complex and imprecise.
We reduce the complexity in this rewrite to allow a very simple rule:
the task_group's load_avg is aggregated from its per CPU cfs_rqs's
load_avgs. Then group entity's weight is simply proportional to its
own cfs_rq's load_avg / task_group's load_avg. To illustrate,
if a task_group has { cfs_rq1, cfs_rq2, ..., cfs_rqn }, then,
task_group_avg = cfs_rq1_avg + cfs_rq2_avg + ... + cfs_rqn_avg, then
cfs_rqx's entity's share = cfs_rqx_avg / task_group_avg * task_group's share
To sum up, this rewrite in principle is equivalent to the current one, but
fixes the issues described above. Turns out, it significantly reduces the
code complexity and hence increases clarity and efficiency. In addition,
the new averages are more smooth/continuous (no spurious spikes and valleys)
and updated more consistently and quickly to reflect the load dynamics.
As a result, we have less load tracking overhead, better performance,
and especially better power efficiency due to more balanced load.
Signed-off-by: Yuyang Du <yuyang.du@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: arjan@linux.intel.com
Cc: bsegall@google.com
Cc: dietmar.eggemann@arm.com
Cc: fengguang.wu@intel.com
Cc: len.brown@intel.com
Cc: morten.rasmussen@arm.com
Cc: pjt@google.com
Cc: rafael.j.wysocki@intel.com
Cc: umgwanakikbuti@gmail.com
Cc: vincent.guittot@linaro.org
Link: http://lkml.kernel.org/r/1436918682-4971-3-git-send-email-yuyang.du@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2015-07-14 18:04:37 -06:00
|
|
|
P(se.avg.util_avg);
|
|
|
|
P(se.avg.last_update_time);
|
sched/fair: Add util_est on top of PELT
The util_avg signal computed by PELT is too variable for some use-cases.
For example, a big task waking up after a long sleep period will have its
utilization almost completely decayed. This introduces some latency before
schedutil will be able to pick the best frequency to run a task.
The same issue can affect task placement. Indeed, since the task
utilization is already decayed at wakeup, when the task is enqueued in a
CPU, this can result in a CPU running a big task as being temporarily
represented as being almost empty. This leads to a race condition where
other tasks can be potentially allocated on a CPU which just started to run
a big task which slept for a relatively long period.
Moreover, the PELT utilization of a task can be updated every [ms], thus
making it a continuously changing value for certain longer running
tasks. This means that the instantaneous PELT utilization of a RUNNING
task is not really meaningful to properly support scheduler decisions.
For all these reasons, a more stable signal can do a better job of
representing the expected/estimated utilization of a task/cfs_rq.
Such a signal can be easily created on top of PELT by still using it as
an estimator which produces values to be aggregated on meaningful
events.
This patch adds a simple implementation of util_est, a new signal built on
top of PELT's util_avg where:
util_est(task) = max(task::util_avg, f(task::util_avg@dequeue))
This allows to remember how big a task has been reported by PELT in its
previous activations via f(task::util_avg@dequeue), which is the new
_task_util_est(struct task_struct*) function added by this patch.
If a task should change its behavior and it runs longer in a new
activation, after a certain time its util_est will just track the
original PELT signal (i.e. task::util_avg).
The estimated utilization of cfs_rq is defined only for root ones.
That's because the only sensible consumer of this signal are the
scheduler and schedutil when looking for the overall CPU utilization
due to FAIR tasks.
For this reason, the estimated utilization of a root cfs_rq is simply
defined as:
util_est(cfs_rq) = max(cfs_rq::util_avg, cfs_rq::util_est::enqueued)
where:
cfs_rq::util_est::enqueued = sum(_task_util_est(task))
for each RUNNABLE task on that root cfs_rq
It's worth noting that the estimated utilization is tracked only for
objects of interests, specifically:
- Tasks: to better support tasks placement decisions
- root cfs_rqs: to better support both tasks placement decisions as
well as frequencies selection
Signed-off-by: Patrick Bellasi <patrick.bellasi@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Joel Fernandes <joelaf@google.com>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Morten Rasmussen <morten.rasmussen@arm.com>
Cc: Paul Turner <pjt@google.com>
Cc: Rafael J . Wysocki <rafael.j.wysocki@intel.com>
Cc: Steve Muckle <smuckle@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Todd Kjos <tkjos@android.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Cc: Viresh Kumar <viresh.kumar@linaro.org>
Link: http://lkml.kernel.org/r/20180309095245.11071-2-patrick.bellasi@arm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2018-03-09 02:52:42 -07:00
|
|
|
P(se.avg.util_est.ewma);
|
|
|
|
P(se.avg.util_est.enqueued);
|
2013-06-25 02:03:36 -06:00
|
|
|
#endif
|
2007-07-09 10:52:00 -06:00
|
|
|
P(policy);
|
|
|
|
P(prio);
|
2018-11-05 04:21:55 -07:00
|
|
|
if (task_has_dl_policy(p)) {
|
2016-10-26 03:17:17 -06:00
|
|
|
P(dl.runtime);
|
|
|
|
P(dl.deadline);
|
|
|
|
}
|
2016-06-17 11:43:26 -06:00
|
|
|
#undef PN_SCHEDSTAT
|
2007-10-15 09:00:08 -06:00
|
|
|
#undef PN
|
2007-10-15 09:00:18 -06:00
|
|
|
#undef __PN
|
2016-06-17 11:43:26 -06:00
|
|
|
#undef P_SCHEDSTAT
|
2007-10-15 09:00:18 -06:00
|
|
|
#undef P
|
|
|
|
#undef __P
|
2007-07-09 10:52:00 -06:00
|
|
|
|
|
|
|
{
|
2008-11-16 00:07:15 -07:00
|
|
|
unsigned int this_cpu = raw_smp_processor_id();
|
2007-07-09 10:52:00 -06:00
|
|
|
u64 t0, t1;
|
|
|
|
|
2008-11-16 00:07:15 -07:00
|
|
|
t0 = cpu_clock(this_cpu);
|
|
|
|
t1 = cpu_clock(this_cpu);
|
2013-06-27 10:50:05 -06:00
|
|
|
SEQ_printf(m, "%-45s:%21Ld\n",
|
2007-07-09 10:52:00 -06:00
|
|
|
"clock-delta", (long long)(t1-t0));
|
|
|
|
}
|
2013-10-07 04:29:30 -06:00
|
|
|
|
|
|
|
sched_show_numa(p, m);
|
2007-07-09 10:52:00 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
void proc_sched_set_task(struct task_struct *p)
|
|
|
|
{
|
2007-08-02 09:41:40 -06:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2010-03-10 19:37:45 -07:00
|
|
|
memset(&p->se.statistics, 0, sizeof(p->se.statistics));
|
2007-08-02 09:41:40 -06:00
|
|
|
#endif
|
2007-07-09 10:52:00 -06:00
|
|
|
}
|