![Peter Zijlstra](/assets/img/avatar_default.png)
In commit:3fed382b46
("sched/numa: Implement NUMA node level wake_affine()") Rik changed wake_affine to consider NUMA information when balancing between LLC domains. There are a number of problems here which this patch tries to address: - LLC < NODE; in this case we'd use the wrong information to balance - !NUMA_BALANCING: in this case, the new code doesn't do any balancing at all - re-computes the NUMA data for every wakeup, this can mean iterating up to 64 CPUs for every wakeup. - default affine wakeups inside a cache We address these by saving the load/capacity values for each sched_domain during regular load-balance and using these values in wake_affine_llc(). The obvious down-side to using cached values is that they can be too old and poorly reflect reality. But this way we can use LLC wide information and thus not rely on assuming LLC matches NODE. We also don't rely on NUMA_BALANCING nor do we have to aggegate two nodes (or even cache domains) worth of CPUs for each wakeup. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Josef Bacik <josef@toxicpanda.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Rik van Riel <riel@redhat.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-kernel@vger.kernel.org Fixes:3fed382b46
("sched/numa: Implement NUMA node level wake_affine()") [ Minor readability improvements. ] Signed-off-by: Ingo Molnar <mingo@kernel.org>
235 lines
6.3 KiB
C
235 lines
6.3 KiB
C
#ifndef _LINUX_SCHED_TOPOLOGY_H
|
|
#define _LINUX_SCHED_TOPOLOGY_H
|
|
|
|
#include <linux/topology.h>
|
|
|
|
#include <linux/sched/idle.h>
|
|
|
|
/*
|
|
* sched-domains (multiprocessor balancing) declarations:
|
|
*/
|
|
#ifdef CONFIG_SMP
|
|
|
|
#define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */
|
|
#define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */
|
|
#define SD_BALANCE_EXEC 0x0004 /* Balance on exec */
|
|
#define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */
|
|
#define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */
|
|
#define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */
|
|
#define SD_ASYM_CPUCAPACITY 0x0040 /* Groups have different max cpu capacities */
|
|
#define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share cpu capacity */
|
|
#define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */
|
|
#define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share cpu pkg resources */
|
|
#define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */
|
|
#define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */
|
|
#define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */
|
|
#define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */
|
|
#define SD_NUMA 0x4000 /* cross-node balancing */
|
|
|
|
/*
|
|
* Increase resolution of cpu_capacity calculations
|
|
*/
|
|
#define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT
|
|
#define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT)
|
|
|
|
#ifdef CONFIG_SCHED_SMT
|
|
static inline int cpu_smt_flags(void)
|
|
{
|
|
return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
|
|
}
|
|
#endif
|
|
|
|
#ifdef CONFIG_SCHED_MC
|
|
static inline int cpu_core_flags(void)
|
|
{
|
|
return SD_SHARE_PKG_RESOURCES;
|
|
}
|
|
#endif
|
|
|
|
#ifdef CONFIG_NUMA
|
|
static inline int cpu_numa_flags(void)
|
|
{
|
|
return SD_NUMA;
|
|
}
|
|
#endif
|
|
|
|
extern int arch_asym_cpu_priority(int cpu);
|
|
|
|
struct sched_domain_attr {
|
|
int relax_domain_level;
|
|
};
|
|
|
|
#define SD_ATTR_INIT (struct sched_domain_attr) { \
|
|
.relax_domain_level = -1, \
|
|
}
|
|
|
|
extern int sched_domain_level_max;
|
|
|
|
struct sched_group;
|
|
|
|
struct sched_domain_shared {
|
|
atomic_t ref;
|
|
atomic_t nr_busy_cpus;
|
|
int has_idle_cores;
|
|
|
|
/*
|
|
* Some variables from the most recent sd_lb_stats for this domain,
|
|
* used by wake_affine().
|
|
*/
|
|
unsigned long nr_running;
|
|
unsigned long load;
|
|
unsigned long capacity;
|
|
};
|
|
|
|
struct sched_domain {
|
|
/* These fields must be setup */
|
|
struct sched_domain *parent; /* top domain must be null terminated */
|
|
struct sched_domain *child; /* bottom domain must be null terminated */
|
|
struct sched_group *groups; /* the balancing groups of the domain */
|
|
unsigned long min_interval; /* Minimum balance interval ms */
|
|
unsigned long max_interval; /* Maximum balance interval ms */
|
|
unsigned int busy_factor; /* less balancing by factor if busy */
|
|
unsigned int imbalance_pct; /* No balance until over watermark */
|
|
unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */
|
|
unsigned int busy_idx;
|
|
unsigned int idle_idx;
|
|
unsigned int newidle_idx;
|
|
unsigned int wake_idx;
|
|
unsigned int forkexec_idx;
|
|
unsigned int smt_gain;
|
|
|
|
int nohz_idle; /* NOHZ IDLE status */
|
|
int flags; /* See SD_* */
|
|
int level;
|
|
|
|
/* Runtime fields. */
|
|
unsigned long last_balance; /* init to jiffies. units in jiffies */
|
|
unsigned int balance_interval; /* initialise to 1. units in ms. */
|
|
unsigned int nr_balance_failed; /* initialise to 0 */
|
|
|
|
/* idle_balance() stats */
|
|
u64 max_newidle_lb_cost;
|
|
unsigned long next_decay_max_lb_cost;
|
|
|
|
u64 avg_scan_cost; /* select_idle_sibling */
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
/* load_balance() stats */
|
|
unsigned int lb_count[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_failed[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_balanced[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_gained[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES];
|
|
unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES];
|
|
|
|
/* Active load balancing */
|
|
unsigned int alb_count;
|
|
unsigned int alb_failed;
|
|
unsigned int alb_pushed;
|
|
|
|
/* SD_BALANCE_EXEC stats */
|
|
unsigned int sbe_count;
|
|
unsigned int sbe_balanced;
|
|
unsigned int sbe_pushed;
|
|
|
|
/* SD_BALANCE_FORK stats */
|
|
unsigned int sbf_count;
|
|
unsigned int sbf_balanced;
|
|
unsigned int sbf_pushed;
|
|
|
|
/* try_to_wake_up() stats */
|
|
unsigned int ttwu_wake_remote;
|
|
unsigned int ttwu_move_affine;
|
|
unsigned int ttwu_move_balance;
|
|
#endif
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
char *name;
|
|
#endif
|
|
union {
|
|
void *private; /* used during construction */
|
|
struct rcu_head rcu; /* used during destruction */
|
|
};
|
|
struct sched_domain_shared *shared;
|
|
|
|
unsigned int span_weight;
|
|
/*
|
|
* Span of all CPUs in this domain.
|
|
*
|
|
* NOTE: this field is variable length. (Allocated dynamically
|
|
* by attaching extra space to the end of the structure,
|
|
* depending on how many CPUs the kernel has booted up with)
|
|
*/
|
|
unsigned long span[0];
|
|
};
|
|
|
|
static inline struct cpumask *sched_domain_span(struct sched_domain *sd)
|
|
{
|
|
return to_cpumask(sd->span);
|
|
}
|
|
|
|
extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
|
|
struct sched_domain_attr *dattr_new);
|
|
|
|
/* Allocate an array of sched domains, for partition_sched_domains(). */
|
|
cpumask_var_t *alloc_sched_domains(unsigned int ndoms);
|
|
void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms);
|
|
|
|
bool cpus_share_cache(int this_cpu, int that_cpu);
|
|
|
|
typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);
|
|
typedef int (*sched_domain_flags_f)(void);
|
|
|
|
#define SDTL_OVERLAP 0x01
|
|
|
|
struct sd_data {
|
|
struct sched_domain **__percpu sd;
|
|
struct sched_domain_shared **__percpu sds;
|
|
struct sched_group **__percpu sg;
|
|
struct sched_group_capacity **__percpu sgc;
|
|
};
|
|
|
|
struct sched_domain_topology_level {
|
|
sched_domain_mask_f mask;
|
|
sched_domain_flags_f sd_flags;
|
|
int flags;
|
|
int numa_level;
|
|
struct sd_data data;
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
char *name;
|
|
#endif
|
|
};
|
|
|
|
extern void set_sched_topology(struct sched_domain_topology_level *tl);
|
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
# define SD_INIT_NAME(type) .name = #type
|
|
#else
|
|
# define SD_INIT_NAME(type)
|
|
#endif
|
|
|
|
#else /* CONFIG_SMP */
|
|
|
|
struct sched_domain_attr;
|
|
|
|
static inline void
|
|
partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
|
|
struct sched_domain_attr *dattr_new)
|
|
{
|
|
}
|
|
|
|
static inline bool cpus_share_cache(int this_cpu, int that_cpu)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
#endif /* !CONFIG_SMP */
|
|
|
|
static inline int task_node(const struct task_struct *p)
|
|
{
|
|
return cpu_to_node(task_cpu(p));
|
|
}
|
|
|
|
#endif /* _LINUX_SCHED_TOPOLOGY_H */
|