sched/vtime: Bring up complete kcpustat accessor
Many callsites want to fetch the values of system, user, user_nice, guest or guest_nice kcpustat fields altogether or at least a pair of these. In that case calling kcpustat_field() for each requested field brings unecessary overhead when we could fetch all of them in a row. So provide kcpustat_cpu_fetch() that fetches the whole kcpustat array in a vtime safe way under the same RCU and seqcount block. Signed-off-by: Frederic Weisbecker <frederic@kernel.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Wanpeng Li <wanpengli@tencent.com> Cc: Yauheni Kaliuta <yauheni.kaliuta@redhat.com> Link: https://lkml.kernel.org/r/20191121024430.19938-3-frederic@kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
5a1c95580f
commit
74722bb223
|
@ -81,12 +81,19 @@ static inline unsigned int kstat_cpu_irqs_sum(unsigned int cpu)
|
||||||
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
|
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
|
||||||
extern u64 kcpustat_field(struct kernel_cpustat *kcpustat,
|
extern u64 kcpustat_field(struct kernel_cpustat *kcpustat,
|
||||||
enum cpu_usage_stat usage, int cpu);
|
enum cpu_usage_stat usage, int cpu);
|
||||||
|
extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu);
|
||||||
#else
|
#else
|
||||||
static inline u64 kcpustat_field(struct kernel_cpustat *kcpustat,
|
static inline u64 kcpustat_field(struct kernel_cpustat *kcpustat,
|
||||||
enum cpu_usage_stat usage, int cpu)
|
enum cpu_usage_stat usage, int cpu)
|
||||||
{
|
{
|
||||||
return kcpustat->cpustat[usage];
|
return kcpustat->cpustat[usage];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
|
||||||
|
{
|
||||||
|
*dst = kcpustat_cpu(cpu);
|
||||||
|
}
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
extern void account_user_time(struct task_struct *, u64);
|
extern void account_user_time(struct task_struct *, u64);
|
||||||
|
|
|
@ -912,6 +912,30 @@ void task_cputime(struct task_struct *t, u64 *utime, u64 *stime)
|
||||||
} while (read_seqcount_retry(&vtime->seqcount, seq));
|
} while (read_seqcount_retry(&vtime->seqcount, seq));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int vtime_state_check(struct vtime *vtime, int cpu)
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
* We raced against a context switch, fetch the
|
||||||
|
* kcpustat task again.
|
||||||
|
*/
|
||||||
|
if (vtime->cpu != cpu && vtime->cpu != -1)
|
||||||
|
return -EAGAIN;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Two possible things here:
|
||||||
|
* 1) We are seeing the scheduling out task (prev) or any past one.
|
||||||
|
* 2) We are seeing the scheduling in task (next) but it hasn't
|
||||||
|
* passed though vtime_task_switch() yet so the pending
|
||||||
|
* cputime of the prev task may not be flushed yet.
|
||||||
|
*
|
||||||
|
* Case 1) is ok but 2) is not. So wait for a safe VTIME state.
|
||||||
|
*/
|
||||||
|
if (vtime->state == VTIME_INACTIVE)
|
||||||
|
return -EAGAIN;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
static u64 kcpustat_user_vtime(struct vtime *vtime)
|
static u64 kcpustat_user_vtime(struct vtime *vtime)
|
||||||
{
|
{
|
||||||
if (vtime->state == VTIME_USER)
|
if (vtime->state == VTIME_USER)
|
||||||
|
@ -933,26 +957,9 @@ static int kcpustat_field_vtime(u64 *cpustat,
|
||||||
do {
|
do {
|
||||||
seq = read_seqcount_begin(&vtime->seqcount);
|
seq = read_seqcount_begin(&vtime->seqcount);
|
||||||
|
|
||||||
/*
|
err = vtime_state_check(vtime, cpu);
|
||||||
* We raced against context switch, fetch the
|
if (err < 0)
|
||||||
* kcpustat task again.
|
return err;
|
||||||
*/
|
|
||||||
if (vtime->cpu != cpu && vtime->cpu != -1)
|
|
||||||
return -EAGAIN;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Two possible things here:
|
|
||||||
* 1) We are seeing the scheduling out task (prev) or any past one.
|
|
||||||
* 2) We are seeing the scheduling in task (next) but it hasn't
|
|
||||||
* passed though vtime_task_switch() yet so the pending
|
|
||||||
* cputime of the prev task may not be flushed yet.
|
|
||||||
*
|
|
||||||
* Case 1) is ok but 2) is not. So wait for a safe VTIME state.
|
|
||||||
*/
|
|
||||||
if (vtime->state == VTIME_INACTIVE)
|
|
||||||
return -EAGAIN;
|
|
||||||
|
|
||||||
err = 0;
|
|
||||||
|
|
||||||
*val = cpustat[usage];
|
*val = cpustat[usage];
|
||||||
|
|
||||||
|
@ -1025,4 +1032,93 @@ u64 kcpustat_field(struct kernel_cpustat *kcpustat,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(kcpustat_field);
|
EXPORT_SYMBOL_GPL(kcpustat_field);
|
||||||
|
|
||||||
|
static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
|
||||||
|
const struct kernel_cpustat *src,
|
||||||
|
struct task_struct *tsk, int cpu)
|
||||||
|
{
|
||||||
|
struct vtime *vtime = &tsk->vtime;
|
||||||
|
unsigned int seq;
|
||||||
|
int err;
|
||||||
|
|
||||||
|
do {
|
||||||
|
u64 *cpustat;
|
||||||
|
u64 delta;
|
||||||
|
|
||||||
|
seq = read_seqcount_begin(&vtime->seqcount);
|
||||||
|
|
||||||
|
err = vtime_state_check(vtime, cpu);
|
||||||
|
if (err < 0)
|
||||||
|
return err;
|
||||||
|
|
||||||
|
*dst = *src;
|
||||||
|
cpustat = dst->cpustat;
|
||||||
|
|
||||||
|
/* Task is sleeping, dead or idle, nothing to add */
|
||||||
|
if (vtime->state < VTIME_SYS)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
delta = vtime_delta(vtime);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Task runs either in user (including guest) or kernel space,
|
||||||
|
* add pending nohz time to the right place.
|
||||||
|
*/
|
||||||
|
if (vtime->state == VTIME_SYS) {
|
||||||
|
cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;
|
||||||
|
} else if (vtime->state == VTIME_USER) {
|
||||||
|
if (task_nice(tsk) > 0)
|
||||||
|
cpustat[CPUTIME_NICE] += vtime->utime + delta;
|
||||||
|
else
|
||||||
|
cpustat[CPUTIME_USER] += vtime->utime + delta;
|
||||||
|
} else {
|
||||||
|
WARN_ON_ONCE(vtime->state != VTIME_GUEST);
|
||||||
|
if (task_nice(tsk) > 0) {
|
||||||
|
cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;
|
||||||
|
cpustat[CPUTIME_NICE] += vtime->gtime + delta;
|
||||||
|
} else {
|
||||||
|
cpustat[CPUTIME_GUEST] += vtime->gtime + delta;
|
||||||
|
cpustat[CPUTIME_USER] += vtime->gtime + delta;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} while (read_seqcount_retry(&vtime->seqcount, seq));
|
||||||
|
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
|
void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
|
||||||
|
{
|
||||||
|
const struct kernel_cpustat *src = &kcpustat_cpu(cpu);
|
||||||
|
struct rq *rq;
|
||||||
|
int err;
|
||||||
|
|
||||||
|
if (!vtime_accounting_enabled_cpu(cpu)) {
|
||||||
|
*dst = *src;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
rq = cpu_rq(cpu);
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
struct task_struct *curr;
|
||||||
|
|
||||||
|
rcu_read_lock();
|
||||||
|
curr = rcu_dereference(rq->curr);
|
||||||
|
if (WARN_ON_ONCE(!curr)) {
|
||||||
|
rcu_read_unlock();
|
||||||
|
*dst = *src;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
err = kcpustat_cpu_fetch_vtime(dst, src, curr, cpu);
|
||||||
|
rcu_read_unlock();
|
||||||
|
|
||||||
|
if (!err)
|
||||||
|
return;
|
||||||
|
|
||||||
|
cpu_relax();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EXPORT_SYMBOL_GPL(kcpustat_cpu_fetch);
|
||||||
|
|
||||||
#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
|
#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
|
||||||
|
|
Loading…
Reference in a new issue