sched: Pre-compute cpumask_weight(sched_domain_span(sd))
Dave reported that his large SPARC machines spend lots of time in hweight64(), try and optimize some of those needless cpumask_weight() invocations (esp. with the large offstack cpumasks these are very expensive indeed). Reported-by: David Miller <davem@davemloft.net> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> LKML-Reference: <new-submission> Signed-off-by: Ingo Molnar <mingo@elte.hu>
This commit is contained in:
parent
74f5187ac8
commit
669c55e9f9
|
@ -960,6 +960,7 @@ struct sched_domain {
|
||||||
char *name;
|
char *name;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
unsigned int span_weight;
|
||||||
/*
|
/*
|
||||||
* Span of all CPUs in this domain.
|
* Span of all CPUs in this domain.
|
||||||
*
|
*
|
||||||
|
|
|
@ -6271,6 +6271,9 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
|
||||||
struct rq *rq = cpu_rq(cpu);
|
struct rq *rq = cpu_rq(cpu);
|
||||||
struct sched_domain *tmp;
|
struct sched_domain *tmp;
|
||||||
|
|
||||||
|
for (tmp = sd; tmp; tmp = tmp->parent)
|
||||||
|
tmp->span_weight = cpumask_weight(sched_domain_span(tmp));
|
||||||
|
|
||||||
/* Remove the sched domains which do not contribute to scheduling. */
|
/* Remove the sched domains which do not contribute to scheduling. */
|
||||||
for (tmp = sd; tmp; ) {
|
for (tmp = sd; tmp; ) {
|
||||||
struct sched_domain *parent = tmp->parent;
|
struct sched_domain *parent = tmp->parent;
|
||||||
|
|
|
@ -1508,9 +1508,7 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_
|
||||||
* Pick the largest domain to update shares over
|
* Pick the largest domain to update shares over
|
||||||
*/
|
*/
|
||||||
tmp = sd;
|
tmp = sd;
|
||||||
if (affine_sd && (!tmp ||
|
if (affine_sd && (!tmp || affine_sd->span_weight > sd->span_weight))
|
||||||
cpumask_weight(sched_domain_span(affine_sd)) >
|
|
||||||
cpumask_weight(sched_domain_span(sd))))
|
|
||||||
tmp = affine_sd;
|
tmp = affine_sd;
|
||||||
|
|
||||||
if (tmp) {
|
if (tmp) {
|
||||||
|
@ -1554,10 +1552,10 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_
|
||||||
|
|
||||||
/* Now try balancing at a lower domain level of new_cpu */
|
/* Now try balancing at a lower domain level of new_cpu */
|
||||||
cpu = new_cpu;
|
cpu = new_cpu;
|
||||||
weight = cpumask_weight(sched_domain_span(sd));
|
weight = sd->span_weight;
|
||||||
sd = NULL;
|
sd = NULL;
|
||||||
for_each_domain(cpu, tmp) {
|
for_each_domain(cpu, tmp) {
|
||||||
if (weight <= cpumask_weight(sched_domain_span(tmp)))
|
if (weight <= tmp->span_weight)
|
||||||
break;
|
break;
|
||||||
if (tmp->flags & sd_flag)
|
if (tmp->flags & sd_flag)
|
||||||
sd = tmp;
|
sd = tmp;
|
||||||
|
@ -2243,7 +2241,7 @@ unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
|
||||||
|
|
||||||
unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
|
unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
|
||||||
{
|
{
|
||||||
unsigned long weight = cpumask_weight(sched_domain_span(sd));
|
unsigned long weight = sd->span_weight;
|
||||||
unsigned long smt_gain = sd->smt_gain;
|
unsigned long smt_gain = sd->smt_gain;
|
||||||
|
|
||||||
smt_gain /= weight;
|
smt_gain /= weight;
|
||||||
|
@ -2276,7 +2274,7 @@ unsigned long scale_rt_power(int cpu)
|
||||||
|
|
||||||
static void update_cpu_power(struct sched_domain *sd, int cpu)
|
static void update_cpu_power(struct sched_domain *sd, int cpu)
|
||||||
{
|
{
|
||||||
unsigned long weight = cpumask_weight(sched_domain_span(sd));
|
unsigned long weight = sd->span_weight;
|
||||||
unsigned long power = SCHED_LOAD_SCALE;
|
unsigned long power = SCHED_LOAD_SCALE;
|
||||||
struct sched_group *sdg = sd->groups;
|
struct sched_group *sdg = sd->groups;
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue