2012-10-22 17:28:05 -06:00
|
|
|
/*
|
|
|
|
* drivers/cpufreq/cpufreq_governor.c
|
|
|
|
*
|
|
|
|
* CPUFREQ governors common code
|
|
|
|
*
|
2012-10-25 16:47:42 -06:00
|
|
|
* Copyright (C) 2001 Russell King
|
|
|
|
* (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
|
|
|
|
* (C) 2003 Jun Nakajima <jun.nakajima@intel.com>
|
|
|
|
* (C) 2009 Alexander Clouter <alex@digriz.org.uk>
|
|
|
|
* (c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
|
|
|
|
*
|
2012-10-22 17:28:05 -06:00
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*/
|
|
|
|
|
2012-10-25 16:47:42 -06:00
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
2012-10-22 17:28:05 -06:00
|
|
|
#include <linux/export.h>
|
|
|
|
#include <linux/kernel_stat.h>
|
2013-03-27 09:58:58 -06:00
|
|
|
#include <linux/slab.h>
|
2012-10-25 16:47:42 -06:00
|
|
|
|
|
|
|
#include "cpufreq_governor.h"
|
|
|
|
|
2016-02-07 08:01:31 -07:00
|
|
|
DEFINE_MUTEX(dbs_data_mutex);
|
|
|
|
EXPORT_SYMBOL_GPL(dbs_data_mutex);
|
|
|
|
|
2016-02-07 08:09:51 -07:00
|
|
|
static struct attribute_group *get_sysfs_attr(struct dbs_governor *gov)
|
2013-03-27 09:58:58 -06:00
|
|
|
{
|
2016-02-07 08:09:51 -07:00
|
|
|
return have_governor_per_policy() ?
|
|
|
|
gov->attr_group_gov_pol : gov->attr_group_gov_sys;
|
2013-03-27 09:58:58 -06:00
|
|
|
}
|
|
|
|
|
2016-02-06 05:50:24 -07:00
|
|
|
void dbs_check_cpu(struct cpufreq_policy *policy)
|
2012-10-25 16:47:42 -06:00
|
|
|
{
|
2016-02-06 05:50:24 -07:00
|
|
|
int cpu = policy->cpu;
|
2016-02-07 08:09:51 -07:00
|
|
|
struct dbs_governor *gov = dbs_governor_of(policy);
|
2016-02-07 08:24:26 -07:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
|
|
|
struct dbs_data *dbs_data = policy_dbs->dbs_data;
|
2012-10-25 16:47:42 -06:00
|
|
|
struct od_dbs_tuners *od_tuners = dbs_data->tuners;
|
|
|
|
struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
unsigned int sampling_rate;
|
2012-10-25 16:47:42 -06:00
|
|
|
unsigned int max_load = 0;
|
|
|
|
unsigned int ignore_nice;
|
|
|
|
unsigned int j;
|
|
|
|
|
2016-02-07 08:09:51 -07:00
|
|
|
if (gov->governor == GOV_ONDEMAND) {
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
struct od_cpu_dbs_info_s *od_dbs_info =
|
2016-02-07 08:09:51 -07:00
|
|
|
gov->get_cpu_dbs_info_s(cpu);
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Sometimes, the ondemand governor uses an additional
|
|
|
|
* multiplier to give long delays. So apply this multiplier to
|
|
|
|
* the 'sampling_rate', so as to keep the wake-up-from-idle
|
|
|
|
* detection logic a bit conservative.
|
|
|
|
*/
|
|
|
|
sampling_rate = od_tuners->sampling_rate;
|
|
|
|
sampling_rate *= od_dbs_info->rate_mult;
|
|
|
|
|
2013-08-05 00:58:02 -06:00
|
|
|
ignore_nice = od_tuners->ignore_nice_load;
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
} else {
|
|
|
|
sampling_rate = cs_tuners->sampling_rate;
|
2013-08-05 00:58:02 -06:00
|
|
|
ignore_nice = cs_tuners->ignore_nice_load;
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
}
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2013-06-05 10:01:25 -06:00
|
|
|
/* Get Absolute Load */
|
2012-10-25 16:47:42 -06:00
|
|
|
for_each_cpu(j, policy->cpus) {
|
2015-06-19 05:48:03 -06:00
|
|
|
struct cpu_dbs_info *j_cdbs;
|
2013-02-28 09:57:32 -07:00
|
|
|
u64 cur_wall_time, cur_idle_time;
|
|
|
|
unsigned int idle_time, wall_time;
|
2012-10-25 16:47:42 -06:00
|
|
|
unsigned int load;
|
2013-02-28 09:57:32 -07:00
|
|
|
int io_busy = 0;
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2016-02-07 08:09:51 -07:00
|
|
|
j_cdbs = gov->get_cpu_cdbs(j);
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2013-02-28 09:57:32 -07:00
|
|
|
/*
|
|
|
|
* For the purpose of ondemand, waiting for disk IO is
|
|
|
|
* an indication that you're performance critical, and
|
|
|
|
* not that the system is actually idle. So do not add
|
|
|
|
* the iowait time to the cpu idle time.
|
|
|
|
*/
|
2016-02-07 08:09:51 -07:00
|
|
|
if (gov->governor == GOV_ONDEMAND)
|
2013-02-28 09:57:32 -07:00
|
|
|
io_busy = od_tuners->io_is_busy;
|
|
|
|
cur_idle_time = get_cpu_idle_time(j, &cur_wall_time, io_busy);
|
2012-10-25 16:47:42 -06:00
|
|
|
|
|
|
|
wall_time = (unsigned int)
|
|
|
|
(cur_wall_time - j_cdbs->prev_cpu_wall);
|
|
|
|
j_cdbs->prev_cpu_wall = cur_wall_time;
|
|
|
|
|
cpufreq: governor: Fix negative idle_time when configured with CONFIG_HZ_PERIODIC
It is reported that, with CONFIG_HZ_PERIODIC=y cpu stays at the
lowest frequency even if the usage goes to 100%, neither ondemand
nor conservative governor works, however performance and
userspace work as expected. If set with CONFIG_NO_HZ_FULL=y,
everything goes well.
This problem is caused by improper calculation of the idle_time
when the load is extremely high(near 100%). Firstly, cpufreq_governor
uses get_cpu_idle_time to get the total idle time for specific cpu, then:
1.If the system is configured with CONFIG_NO_HZ_FULL, the idle time is
returned by ktime_get, which is always increasing, it's OK.
2.However, if the system is configured with CONFIG_HZ_PERIODIC,
get_cpu_idle_time might not guarantee to be always increasing,
because it will leverage get_cpu_idle_time_jiffy to calculate the
idle_time, consider the following scenario:
At T1:
idle_tick_1 = total_tick_1 - user_tick_1
sample period(80ms)...
At T2: ( T2 = T1 + 80ms):
idle_tick_2 = total_tick_2 - user_tick_2
Currently the algorithm is using (idle_tick_2 - idle_tick_1) to
get the delta idle_time during the past sample period, however
it CAN NOT guarantee that idle_tick_2 >= idle_tick_1, especially
when cpu load is high.
(Yes, total_tick_2 >= total_tick_1, and user_tick_2 >= user_tick_1,
but how about idle_tick_2 and idle_tick_1? No guarantee.)
So governor might get a negative value of idle_time during the past
sample period, which might mislead the system that the idle time is
very big(converted to unsigned int), and the busy time is nearly zero,
which causes the governor to always choose the lowest cpufreq,
then cause this problem.
In theory there are two solutions:
1.The logic should not rely on the idle tick during every sample period,
but be based on the busy tick directly, as this is how 'top' is
implemented.
2.Or the logic must make sure that the idle_time is strictly increasing
during each sample period, then there would be no negative idle_time
anymore. This solution requires minimum modification to current code
and this patch uses method 2.
Link: https://bugzilla.kernel.org/show_bug.cgi?id=69821
Reported-by: Jan Fikar <j.fikar@gmail.com>
Signed-off-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2015-12-15 21:20:29 -07:00
|
|
|
if (cur_idle_time < j_cdbs->prev_cpu_idle)
|
|
|
|
cur_idle_time = j_cdbs->prev_cpu_idle;
|
|
|
|
|
2012-10-25 16:47:42 -06:00
|
|
|
idle_time = (unsigned int)
|
|
|
|
(cur_idle_time - j_cdbs->prev_cpu_idle);
|
|
|
|
j_cdbs->prev_cpu_idle = cur_idle_time;
|
|
|
|
|
|
|
|
if (ignore_nice) {
|
2016-02-07 08:24:26 -07:00
|
|
|
struct cpu_dbs_info *cdbs = gov->get_cpu_cdbs(cpu);
|
2012-10-25 16:47:42 -06:00
|
|
|
u64 cur_nice;
|
|
|
|
unsigned long cur_nice_jiffies;
|
|
|
|
|
|
|
|
cur_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE] -
|
|
|
|
cdbs->prev_cpu_nice;
|
|
|
|
/*
|
|
|
|
* Assumption: nice time between sampling periods will
|
|
|
|
* be less than 2^32 jiffies for 32 bit sys
|
|
|
|
*/
|
|
|
|
cur_nice_jiffies = (unsigned long)
|
|
|
|
cputime64_to_jiffies64(cur_nice);
|
|
|
|
|
|
|
|
cdbs->prev_cpu_nice =
|
|
|
|
kcpustat_cpu(j).cpustat[CPUTIME_NICE];
|
|
|
|
idle_time += jiffies_to_usecs(cur_nice_jiffies);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (unlikely(!wall_time || wall_time < idle_time))
|
|
|
|
continue;
|
|
|
|
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
/*
|
|
|
|
* If the CPU had gone completely idle, and a task just woke up
|
|
|
|
* on this CPU now, it would be unfair to calculate 'load' the
|
|
|
|
* usual way for this elapsed time-window, because it will show
|
|
|
|
* near-zero load, irrespective of how CPU intensive that task
|
|
|
|
* actually is. This is undesirable for latency-sensitive bursty
|
|
|
|
* workloads.
|
|
|
|
*
|
|
|
|
* To avoid this, we reuse the 'load' from the previous
|
|
|
|
* time-window and give this task a chance to start with a
|
|
|
|
* reasonably high CPU frequency. (However, we shouldn't over-do
|
|
|
|
* this copy, lest we get stuck at a high load (high frequency)
|
|
|
|
* for too long, even when the current system load has actually
|
|
|
|
* dropped down. So we perform the copy only once, upon the
|
|
|
|
* first wake-up from idle.)
|
|
|
|
*
|
2016-02-10 08:53:50 -07:00
|
|
|
* Detecting this situation is easy: the governor's utilization
|
|
|
|
* update handler would not have run during CPU-idle periods.
|
|
|
|
* Hence, an unusually large 'wall_time' (as compared to the
|
|
|
|
* sampling rate) indicates this scenario.
|
2014-06-09 02:51:24 -06:00
|
|
|
*
|
|
|
|
* prev_load can be zero in two cases and we must recalculate it
|
|
|
|
* for both cases:
|
|
|
|
* - during long idle intervals
|
|
|
|
* - explicitly set to zero
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
*/
|
2014-06-09 02:51:24 -06:00
|
|
|
if (unlikely(wall_time > (2 * sampling_rate) &&
|
|
|
|
j_cdbs->prev_load)) {
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
load = j_cdbs->prev_load;
|
2014-06-09 02:51:24 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform a destructive copy, to ensure that we copy
|
|
|
|
* the previous load only once, upon the first wake-up
|
|
|
|
* from idle.
|
|
|
|
*/
|
|
|
|
j_cdbs->prev_load = 0;
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
} else {
|
|
|
|
load = 100 * (wall_time - idle_time) / wall_time;
|
|
|
|
j_cdbs->prev_load = load;
|
|
|
|
}
|
2012-10-25 16:47:42 -06:00
|
|
|
|
|
|
|
if (load > max_load)
|
|
|
|
max_load = load;
|
|
|
|
}
|
|
|
|
|
2016-02-07 08:09:51 -07:00
|
|
|
gov->gov_check_cpu(cpu, max_load);
|
2012-10-25 16:47:42 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dbs_check_cpu);
|
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
void gov_set_update_util(struct policy_dbs_info *policy_dbs,
|
2016-02-10 08:53:50 -07:00
|
|
|
unsigned int delay_us)
|
2012-10-25 16:47:42 -06:00
|
|
|
{
|
2016-02-10 09:07:44 -07:00
|
|
|
struct cpufreq_policy *policy = policy_dbs->policy;
|
2016-02-07 08:09:51 -07:00
|
|
|
struct dbs_governor *gov = dbs_governor_of(policy);
|
2015-12-08 19:04:42 -07:00
|
|
|
int cpu;
|
2013-02-26 23:54:03 -07:00
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
gov_update_sample_delay(policy_dbs, delay_us);
|
|
|
|
policy_dbs->last_sample_time = 0;
|
2016-02-10 08:53:50 -07:00
|
|
|
|
2015-12-08 19:04:42 -07:00
|
|
|
for_each_cpu(cpu, policy->cpus) {
|
2016-02-07 08:09:51 -07:00
|
|
|
struct cpu_dbs_info *cdbs = gov->get_cpu_cdbs(cpu);
|
2016-02-10 08:53:50 -07:00
|
|
|
|
|
|
|
cpufreq_set_update_util_data(cpu, &cdbs->update_util);
|
2013-02-26 23:54:03 -07:00
|
|
|
}
|
|
|
|
}
|
2016-02-10 08:53:50 -07:00
|
|
|
EXPORT_SYMBOL_GPL(gov_set_update_util);
|
2013-02-26 23:54:03 -07:00
|
|
|
|
2016-02-10 08:53:50 -07:00
|
|
|
static inline void gov_clear_update_util(struct cpufreq_policy *policy)
|
2013-02-26 23:54:03 -07:00
|
|
|
{
|
|
|
|
int i;
|
2013-01-30 06:53:37 -07:00
|
|
|
|
2016-02-10 08:53:50 -07:00
|
|
|
for_each_cpu(i, policy->cpus)
|
|
|
|
cpufreq_set_update_util_data(i, NULL);
|
|
|
|
|
|
|
|
synchronize_rcu();
|
2012-10-25 16:47:42 -06:00
|
|
|
}
|
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
static void gov_cancel_work(struct policy_dbs_info *policy_dbs)
|
2015-12-08 19:04:42 -07:00
|
|
|
{
|
2016-02-10 08:53:50 -07:00
|
|
|
/* Tell dbs_update_util_handler() to skip queuing up work items. */
|
2016-02-10 09:07:44 -07:00
|
|
|
atomic_inc(&policy_dbs->skip_work);
|
2015-12-08 19:04:42 -07:00
|
|
|
/*
|
2016-02-10 08:53:50 -07:00
|
|
|
* If dbs_update_util_handler() is already running, it may not notice
|
|
|
|
* the incremented skip_work, so wait for it to complete to prevent its
|
|
|
|
* work item from being queued up after the cancel_work_sync() below.
|
2015-12-08 19:04:42 -07:00
|
|
|
*/
|
2016-02-10 09:07:44 -07:00
|
|
|
gov_clear_update_util(policy_dbs->policy);
|
|
|
|
irq_work_sync(&policy_dbs->irq_work);
|
|
|
|
cancel_work_sync(&policy_dbs->work);
|
|
|
|
atomic_set(&policy_dbs->skip_work, 0);
|
2015-12-08 19:04:42 -07:00
|
|
|
}
|
2015-07-18 00:01:00 -06:00
|
|
|
|
2015-12-08 19:04:42 -07:00
|
|
|
static void dbs_work_handler(struct work_struct *work)
|
2015-07-18 00:01:00 -06:00
|
|
|
{
|
2016-02-10 09:07:44 -07:00
|
|
|
struct policy_dbs_info *policy_dbs;
|
2015-10-28 20:38:38 -06:00
|
|
|
struct cpufreq_policy *policy;
|
2016-02-07 08:09:51 -07:00
|
|
|
struct dbs_governor *gov;
|
2016-02-10 08:53:50 -07:00
|
|
|
unsigned int delay;
|
2015-07-18 00:01:00 -06:00
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
policy_dbs = container_of(work, struct policy_dbs_info, work);
|
|
|
|
policy = policy_dbs->policy;
|
2016-02-07 08:09:51 -07:00
|
|
|
gov = dbs_governor_of(policy);
|
2015-10-28 20:38:38 -06:00
|
|
|
|
2015-12-08 19:04:42 -07:00
|
|
|
/*
|
2016-02-10 08:53:50 -07:00
|
|
|
* Make sure cpufreq_governor_limits() isn't evaluating load or the
|
|
|
|
* ondemand governor isn't updating the sampling rate in parallel.
|
2015-12-08 19:04:42 -07:00
|
|
|
*/
|
2016-02-10 09:07:44 -07:00
|
|
|
mutex_lock(&policy_dbs->timer_mutex);
|
2016-02-07 08:09:51 -07:00
|
|
|
delay = gov->gov_dbs_timer(policy);
|
2016-02-10 09:07:44 -07:00
|
|
|
policy_dbs->sample_delay_ns = jiffies_to_nsecs(delay);
|
|
|
|
mutex_unlock(&policy_dbs->timer_mutex);
|
2015-12-08 19:04:42 -07:00
|
|
|
|
2016-02-10 08:53:50 -07:00
|
|
|
/*
|
|
|
|
* If the atomic operation below is reordered with respect to the
|
|
|
|
* sample delay modification, the utilization update handler may end
|
|
|
|
* up using a stale sample delay value.
|
|
|
|
*/
|
|
|
|
smp_mb__before_atomic();
|
2016-02-10 09:07:44 -07:00
|
|
|
atomic_dec(&policy_dbs->skip_work);
|
2016-02-10 08:53:50 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static void dbs_irq_work(struct irq_work *irq_work)
|
|
|
|
{
|
2016-02-10 09:07:44 -07:00
|
|
|
struct policy_dbs_info *policy_dbs;
|
2015-12-08 19:04:42 -07:00
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
policy_dbs = container_of(irq_work, struct policy_dbs_info, irq_work);
|
|
|
|
schedule_work(&policy_dbs->work);
|
2015-12-08 19:04:42 -07:00
|
|
|
}
|
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
static inline void gov_queue_irq_work(struct policy_dbs_info *policy_dbs)
|
2015-12-08 19:04:42 -07:00
|
|
|
{
|
2016-02-10 08:53:50 -07:00
|
|
|
#ifdef CONFIG_SMP
|
2016-02-10 09:07:44 -07:00
|
|
|
irq_work_queue_on(&policy_dbs->irq_work, smp_processor_id());
|
2016-02-10 08:53:50 -07:00
|
|
|
#else
|
2016-02-10 09:07:44 -07:00
|
|
|
irq_work_queue(&policy_dbs->irq_work);
|
2016-02-10 08:53:50 -07:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dbs_update_util_handler(struct update_util_data *data, u64 time,
|
|
|
|
unsigned long util, unsigned long max)
|
|
|
|
{
|
|
|
|
struct cpu_dbs_info *cdbs = container_of(data, struct cpu_dbs_info, update_util);
|
2016-02-10 09:07:44 -07:00
|
|
|
struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
|
2015-12-08 19:04:42 -07:00
|
|
|
|
|
|
|
/*
|
2016-02-10 08:53:50 -07:00
|
|
|
* The work may not be allowed to be queued up right now.
|
|
|
|
* Possible reasons:
|
|
|
|
* - Work has already been queued up or is in progress.
|
|
|
|
* - The governor is being stopped.
|
|
|
|
* - It is too early (too little time from the previous sample).
|
2015-12-08 19:04:42 -07:00
|
|
|
*/
|
2016-02-10 09:07:44 -07:00
|
|
|
if (atomic_inc_return(&policy_dbs->skip_work) == 1) {
|
2016-02-10 08:53:50 -07:00
|
|
|
u64 delta_ns;
|
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
delta_ns = time - policy_dbs->last_sample_time;
|
|
|
|
if ((s64)delta_ns >= policy_dbs->sample_delay_ns) {
|
|
|
|
policy_dbs->last_sample_time = time;
|
|
|
|
gov_queue_irq_work(policy_dbs);
|
2016-02-10 08:53:50 -07:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2016-02-10 09:07:44 -07:00
|
|
|
atomic_dec(&policy_dbs->skip_work);
|
2015-07-18 00:01:00 -06:00
|
|
|
}
|
2013-01-31 10:28:02 -07:00
|
|
|
|
2013-03-27 09:58:58 -06:00
|
|
|
static void set_sampling_rate(struct dbs_data *dbs_data,
|
2016-02-07 08:09:51 -07:00
|
|
|
struct dbs_governor *gov,
|
|
|
|
unsigned int sampling_rate)
|
2013-03-27 09:58:58 -06:00
|
|
|
{
|
2016-02-07 08:09:51 -07:00
|
|
|
if (gov->governor == GOV_CONSERVATIVE) {
|
2013-03-27 09:58:58 -06:00
|
|
|
struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
|
|
|
|
cs_tuners->sampling_rate = sampling_rate;
|
|
|
|
} else {
|
|
|
|
struct od_dbs_tuners *od_tuners = dbs_data->tuners;
|
|
|
|
od_tuners->sampling_rate = sampling_rate;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-02-07 08:24:26 -07:00
|
|
|
static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *policy,
|
|
|
|
struct dbs_governor *gov)
|
2015-07-18 00:00:59 -06:00
|
|
|
{
|
2016-02-10 09:07:44 -07:00
|
|
|
struct policy_dbs_info *policy_dbs;
|
2015-07-18 00:00:59 -06:00
|
|
|
int j;
|
|
|
|
|
|
|
|
/* Allocate memory for the common information for policy->cpus */
|
2016-02-10 09:07:44 -07:00
|
|
|
policy_dbs = kzalloc(sizeof(*policy_dbs), GFP_KERNEL);
|
|
|
|
if (!policy_dbs)
|
2016-02-07 08:24:26 -07:00
|
|
|
return NULL;
|
2015-07-18 00:00:59 -06:00
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
mutex_init(&policy_dbs->timer_mutex);
|
|
|
|
atomic_set(&policy_dbs->skip_work, 0);
|
|
|
|
init_irq_work(&policy_dbs->irq_work, dbs_irq_work);
|
|
|
|
INIT_WORK(&policy_dbs->work, dbs_work_handler);
|
2016-02-07 08:25:02 -07:00
|
|
|
|
|
|
|
/* Set policy_dbs for all CPUs, online+offline */
|
|
|
|
for_each_cpu(j, policy->related_cpus) {
|
|
|
|
struct cpu_dbs_info *j_cdbs = gov->get_cpu_cdbs(j);
|
|
|
|
|
|
|
|
j_cdbs->policy_dbs = policy_dbs;
|
|
|
|
j_cdbs->update_util.func = dbs_update_util_handler;
|
|
|
|
}
|
2016-02-07 08:24:26 -07:00
|
|
|
return policy_dbs;
|
2015-07-18 00:00:59 -06:00
|
|
|
}
|
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
static void free_policy_dbs_info(struct cpufreq_policy *policy,
|
2016-02-07 08:05:07 -07:00
|
|
|
struct dbs_governor *gov)
|
2015-07-18 00:00:59 -06:00
|
|
|
{
|
2016-02-07 08:05:07 -07:00
|
|
|
struct cpu_dbs_info *cdbs = gov->get_cpu_cdbs(policy->cpu);
|
2016-02-10 09:07:44 -07:00
|
|
|
struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
|
2015-07-18 00:00:59 -06:00
|
|
|
int j;
|
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
mutex_destroy(&policy_dbs->timer_mutex);
|
2015-12-02 21:07:52 -07:00
|
|
|
|
2016-02-07 08:25:02 -07:00
|
|
|
for_each_cpu(j, policy->related_cpus) {
|
|
|
|
struct cpu_dbs_info *j_cdbs = gov->get_cpu_cdbs(j);
|
2015-07-18 00:00:59 -06:00
|
|
|
|
2016-02-07 08:25:02 -07:00
|
|
|
j_cdbs->policy_dbs = NULL;
|
|
|
|
j_cdbs->update_util.func = NULL;
|
|
|
|
}
|
2016-02-10 09:07:44 -07:00
|
|
|
kfree(policy_dbs);
|
2015-07-18 00:00:59 -06:00
|
|
|
}
|
|
|
|
|
2016-02-07 08:07:51 -07:00
|
|
|
static int cpufreq_governor_init(struct cpufreq_policy *policy)
|
2012-10-25 16:47:42 -06:00
|
|
|
{
|
2016-02-07 08:09:51 -07:00
|
|
|
struct dbs_governor *gov = dbs_governor_of(policy);
|
2016-02-07 08:05:07 -07:00
|
|
|
struct dbs_data *dbs_data = gov->gdbs_data;
|
2016-02-07 08:24:26 -07:00
|
|
|
struct policy_dbs_info *policy_dbs;
|
2015-06-04 05:13:27 -06:00
|
|
|
unsigned int latency;
|
|
|
|
int ret;
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2015-07-18 00:01:01 -06:00
|
|
|
/* State should be equivalent to EXIT */
|
|
|
|
if (policy->governor_data)
|
|
|
|
return -EBUSY;
|
|
|
|
|
2016-02-07 08:24:26 -07:00
|
|
|
policy_dbs = alloc_policy_dbs_info(policy, gov);
|
|
|
|
if (!policy_dbs)
|
|
|
|
return -ENOMEM;
|
2015-07-18 00:00:59 -06:00
|
|
|
|
2016-02-07 08:24:26 -07:00
|
|
|
if (dbs_data) {
|
|
|
|
if (WARN_ON(have_governor_per_policy())) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto free_policy_dbs_info;
|
|
|
|
}
|
2015-06-04 05:13:27 -06:00
|
|
|
dbs_data->usage_count++;
|
2016-02-07 08:24:26 -07:00
|
|
|
policy_dbs->dbs_data = dbs_data;
|
|
|
|
policy->governor_data = policy_dbs;
|
2015-06-04 05:13:27 -06:00
|
|
|
return 0;
|
|
|
|
}
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL);
|
2016-02-07 08:24:26 -07:00
|
|
|
if (!dbs_data) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto free_policy_dbs_info;
|
|
|
|
}
|
2015-07-18 00:00:59 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
dbs_data->usage_count = 1;
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2016-02-07 08:05:07 -07:00
|
|
|
ret = gov->init(dbs_data, !policy->governor->initialized);
|
2015-06-04 05:13:27 -06:00
|
|
|
if (ret)
|
2016-02-10 09:07:44 -07:00
|
|
|
goto free_policy_dbs_info;
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
/* policy latency is in ns. Convert it to us first */
|
|
|
|
latency = policy->cpuinfo.transition_latency / 1000;
|
|
|
|
if (latency == 0)
|
|
|
|
latency = 1;
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
/* Bring kernel and HW constraints together */
|
|
|
|
dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate,
|
|
|
|
MIN_LATENCY_MULTIPLIER * latency);
|
2016-02-07 08:09:51 -07:00
|
|
|
set_sampling_rate(dbs_data, gov, max(dbs_data->min_sampling_rate,
|
2015-06-04 05:13:27 -06:00
|
|
|
latency * LATENCY_MULTIPLIER));
|
2013-05-17 04:39:09 -06:00
|
|
|
|
2015-10-15 10:05:22 -06:00
|
|
|
if (!have_governor_per_policy())
|
2016-02-07 08:05:07 -07:00
|
|
|
gov->gdbs_data = dbs_data;
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2016-02-07 08:24:26 -07:00
|
|
|
policy_dbs->dbs_data = dbs_data;
|
|
|
|
policy->governor_data = policy_dbs;
|
2016-01-25 10:03:46 -07:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
ret = sysfs_create_group(get_governor_parent_kobj(policy),
|
2016-02-07 08:09:51 -07:00
|
|
|
get_sysfs_attr(gov));
|
2015-06-04 05:13:27 -06:00
|
|
|
if (ret)
|
2015-10-15 10:05:22 -06:00
|
|
|
goto reset_gdbs_data;
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
return 0;
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2015-10-15 10:05:22 -06:00
|
|
|
reset_gdbs_data:
|
2016-01-25 10:03:46 -07:00
|
|
|
policy->governor_data = NULL;
|
|
|
|
|
2015-10-15 10:05:22 -06:00
|
|
|
if (!have_governor_per_policy())
|
2016-02-07 08:05:07 -07:00
|
|
|
gov->gdbs_data = NULL;
|
|
|
|
gov->exit(dbs_data, !policy->governor->initialized);
|
2016-02-07 08:24:26 -07:00
|
|
|
kfree(dbs_data);
|
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
free_policy_dbs_info:
|
|
|
|
free_policy_dbs_info(policy, gov);
|
2015-06-04 05:13:27 -06:00
|
|
|
return ret;
|
|
|
|
}
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2016-02-04 19:15:24 -07:00
|
|
|
static int cpufreq_governor_exit(struct cpufreq_policy *policy)
|
2015-06-04 05:13:27 -06:00
|
|
|
{
|
2016-02-07 08:09:51 -07:00
|
|
|
struct dbs_governor *gov = dbs_governor_of(policy);
|
2016-02-07 08:24:26 -07:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
|
|
|
struct dbs_data *dbs_data = policy_dbs->dbs_data;
|
2015-07-18 00:01:01 -06:00
|
|
|
|
|
|
|
/* State should be equivalent to INIT */
|
2016-02-07 08:24:26 -07:00
|
|
|
if (policy_dbs->policy)
|
2015-07-18 00:01:01 -06:00
|
|
|
return -EBUSY;
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
if (!--dbs_data->usage_count) {
|
|
|
|
sysfs_remove_group(get_governor_parent_kobj(policy),
|
2016-02-07 08:09:51 -07:00
|
|
|
get_sysfs_attr(gov));
|
2013-05-17 04:39:09 -06:00
|
|
|
|
2016-01-25 10:03:46 -07:00
|
|
|
policy->governor_data = NULL;
|
|
|
|
|
2015-10-15 10:05:22 -06:00
|
|
|
if (!have_governor_per_policy())
|
2016-02-07 08:05:07 -07:00
|
|
|
gov->gdbs_data = NULL;
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2016-02-07 08:05:07 -07:00
|
|
|
gov->exit(dbs_data, policy->governor->initialized == 1);
|
2015-06-04 05:13:27 -06:00
|
|
|
kfree(dbs_data);
|
2016-01-25 10:03:46 -07:00
|
|
|
} else {
|
|
|
|
policy->governor_data = NULL;
|
2013-03-27 09:58:58 -06:00
|
|
|
}
|
2015-07-18 00:00:59 -06:00
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
free_policy_dbs_info(policy, gov);
|
2015-07-18 00:01:01 -06:00
|
|
|
return 0;
|
2015-06-04 05:13:27 -06:00
|
|
|
}
|
2013-03-27 09:58:58 -06:00
|
|
|
|
2016-02-04 19:15:24 -07:00
|
|
|
static int cpufreq_governor_start(struct cpufreq_policy *policy)
|
2015-06-04 05:13:27 -06:00
|
|
|
{
|
2016-02-07 08:09:51 -07:00
|
|
|
struct dbs_governor *gov = dbs_governor_of(policy);
|
2016-02-07 08:24:26 -07:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
|
|
|
struct dbs_data *dbs_data = policy_dbs->dbs_data;
|
2015-06-04 05:13:27 -06:00
|
|
|
unsigned int sampling_rate, ignore_nice, j, cpu = policy->cpu;
|
|
|
|
int io_busy = 0;
|
|
|
|
|
|
|
|
if (!policy->cur)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2015-07-18 00:01:01 -06:00
|
|
|
/* State should be equivalent to INIT */
|
2016-02-07 08:24:26 -07:00
|
|
|
if (policy_dbs->policy)
|
2015-07-18 00:01:01 -06:00
|
|
|
return -EBUSY;
|
|
|
|
|
2016-02-07 08:05:07 -07:00
|
|
|
if (gov->governor == GOV_CONSERVATIVE) {
|
2015-06-04 05:13:27 -06:00
|
|
|
struct cs_dbs_tuners *cs_tuners = dbs_data->tuners;
|
2013-03-27 09:58:58 -06:00
|
|
|
|
|
|
|
sampling_rate = cs_tuners->sampling_rate;
|
2013-08-05 00:58:02 -06:00
|
|
|
ignore_nice = cs_tuners->ignore_nice_load;
|
2012-10-25 16:47:42 -06:00
|
|
|
} else {
|
2015-06-04 05:13:27 -06:00
|
|
|
struct od_dbs_tuners *od_tuners = dbs_data->tuners;
|
|
|
|
|
2013-03-27 09:58:58 -06:00
|
|
|
sampling_rate = od_tuners->sampling_rate;
|
2013-08-05 00:58:02 -06:00
|
|
|
ignore_nice = od_tuners->ignore_nice_load;
|
2013-02-28 09:57:32 -07:00
|
|
|
io_busy = od_tuners->io_is_busy;
|
2012-10-25 16:47:42 -06:00
|
|
|
}
|
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
for_each_cpu(j, policy->cpus) {
|
2016-02-07 08:05:07 -07:00
|
|
|
struct cpu_dbs_info *j_cdbs = gov->get_cpu_cdbs(j);
|
2015-06-04 05:13:27 -06:00
|
|
|
unsigned int prev_load;
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
j_cdbs->prev_cpu_idle =
|
|
|
|
get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy);
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
prev_load = (unsigned int)(j_cdbs->prev_cpu_wall -
|
|
|
|
j_cdbs->prev_cpu_idle);
|
|
|
|
j_cdbs->prev_load = 100 * prev_load /
|
|
|
|
(unsigned int)j_cdbs->prev_cpu_wall;
|
cpufreq: governor: Be friendly towards latency-sensitive bursty workloads
Cpufreq governors like the ondemand governor calculate the load on the CPU
periodically by employing deferrable timers. A deferrable timer won't fire
if the CPU is completely idle (and there are no other timers to be run), in
order to avoid unnecessary wakeups and thus save CPU power.
However, the load calculation logic is agnostic to all this, and this can
lead to the problem described below.
Time (ms) CPU 1
100 Task-A running
110 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
110.5 Task-A running
120 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
125 Task-A went to sleep. With nothing else to do, CPU 1
went completely idle.
200 Task-A woke up and started running again.
200.5 Governor's deferred timer (which was originally programmed
to fire at time 130) fires now. It calculates load for the
time period 120 to 200.5, and finds the load is almost zero.
Hence it decreases the CPU frequency to the minimum.
210 Governor's timer fires, finds load as 100% in the last
10ms interval and increases the CPU frequency.
So, after the workload woke up and started running, the frequency was suddenly
dropped to absolute minimum, and after that, there was an unnecessary delay of
10ms (sampling period) to increase the CPU frequency back to a reasonable value.
And this pattern repeats for every wake-up-from-cpu-idle for that workload.
This can be quite undesirable for latency- or response-time sensitive bursty
workloads. So we need to fix the governor's logic to detect such wake-up-from-
cpu-idle scenarios and start the workload at a reasonably high CPU frequency.
One extreme solution would be to fake a load of 100% in such scenarios. But
that might lead to undesirable side-effects such as frequency spikes (which
might also need voltage changes) especially if the previous frequency happened
to be very low.
We just want to avoid the stupidity of dropping down the frequency to a minimum
and then enduring a needless (and long) delay before ramping it up back again.
So, let us simply carry forward the previous load - that is, let us just pretend
that the 'load' for the current time-window is the same as the load for the
previous window. That way, the frequency and voltage will continue to be set
to whatever values they were set at previously. This means that bursty workloads
will get a chance to influence the CPU frequency at which they wake up from
cpu-idle, based on their past execution history. Thus, they might be able to
avoid suffering from slow wakeups and long response-times.
However, we should take care not to over-do this. For example, such a "copy
previous load" logic will benefit cases like this: (where # represents busy
and . represents idle)
##########.........#########.........###########...........##########........
but it will be detrimental in cases like the one shown below, because it will
retain the high frequency (copied from the previous interval) even in a mostly
idle system:
##########.........#.................#.....................#...............
(i.e., the workload finished and the remaining tasks are such that their busy
periods are smaller than the sampling interval, which causes the timer to
always get deferred. So, this will make the copy-previous-load logic copy
the initial high load to subsequent idle periods over and over again, thus
keeping the frequency high unnecessarily).
So, we modify this copy-previous-load logic such that it is used only once
upon every wakeup-from-idle. Thus if we have 2 consecutive idle periods, the
previous load won't get blindly copied over; cpufreq will freshly evaluate the
load in the second idle interval, thus ensuring that the system comes back to
its normal state.
[ The right way to solve this whole problem is to teach the CPU frequency
governors to also track load on a per-task basis, not just a per-CPU basis,
and then use both the data sources intelligently to set the appropriate
frequency on the CPUs. But that involves redesigning the cpufreq subsystem,
so this patch should make the situation bearable until then. ]
Experimental results:
+-------------------+
I ran a modified version of ebizzy (called 'sleeping-ebizzy') that sleeps in
between its execution such that its total utilization can be a user-defined
value, say 10% or 20% (higher the utilization specified, lesser the amount of
sleeps injected). This ebizzy was run with a single-thread, tied to CPU 8.
Behavior observed with tracing (sample taken from 40% utilization runs):
------------------------------------------------------------------------
Without patch:
~~~~~~~~~~~~~~
kworker/8:2-12137 416.335742: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.335744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.345741: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.345744: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.345746: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.355738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
<...>-40753 416.402202: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 416.502130: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40753 416.505738: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.505739: cpu_frequency: state=2061000 cpu_id=8
kworker/8:2-12137 416.505741: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40753 416.515739: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-12137 416.515742: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-12137 416.515744: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
Observation: Ebizzy went idle at 416.402202, and started running again at
416.502130. But cpufreq noticed the long idle period, and dropped the frequency
at 416.505739, only to increase it back again at 416.515742, realizing that the
workload is in-fact CPU bound. Thus ebizzy needlessly ran at the lowest frequency
for almost 13 milliseconds (almost 1 full sample period), and this pattern
repeats on every sleep-wakeup. This could hurt latency-sensitive workloads quite
a lot.
With patch:
~~~~~~~~~~~
kworker/8:2-29802 464.832535: cpu_frequency: state=2061000 cpu_id=8
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 464.962538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.972533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 464.972536: cpu_frequency: state=4123000 cpu_id=8
kworker/8:2-29802 464.972538: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 464.982531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
<snip> --------------------------------------------------------------------- <snip>
kworker/8:2-29802 465.022533: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.032531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.032532: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.035797: sched_switch: prev_comm=ebizzy ==> next_comm=swapper/8
<idle>-0 465.240178: sched_switch: prev_comm=swapper/8 ==> next_comm=ebizzy
<...>-40738 465.242533: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
kworker/8:2-29802 465.242535: sched_switch: prev_comm=kworker/8:2 ==> next_comm=ebizzy
<...>-40738 465.252531: sched_switch: prev_comm=ebizzy ==> next_comm=kworker/8:2
Observation: Ebizzy went idle at 465.035797, and started running again at
465.240178. Since ebizzy was the only real workload running on this CPU,
cpufreq retained the frequency at 4.1Ghz throughout the run of ebizzy, no
matter how many times ebizzy slept and woke-up in-between. Thus, ebizzy
got the 10ms worth of 4.1 Ghz benefit during every sleep-wakeup (as compared
to the run without the patch) and this boost gave a modest improvement in total
throughput, as shown below.
Sleeping-ebizzy records-per-second:
-----------------------------------
Utilization Without patch With patch Difference (Absolute and % values)
10% 274767 277046 + 2279 (+0.829%)
20% 543429 553484 + 10055 (+1.850%)
40% 1090744 1107959 + 17215 (+1.578%)
60% 1634908 1662018 + 27110 (+1.658%)
A rudimentary and somewhat approximately latency-sensitive workload such as
sleeping-ebizzy itself showed a consistent, noticeable performance improvement
with this patch. Hence, workloads that are truly latency-sensitive will benefit
quite a bit from this change. Moreover, this is an overall win-win since this
patch does not hurt power-savings at all (because, this patch does not reduce
the idle time or idle residency; and the high frequency of the CPU when it goes
to cpu-idle does not affect/hurt the power-savings of deep idle states).
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2014-06-07 14:41:43 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
if (ignore_nice)
|
|
|
|
j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];
|
|
|
|
}
|
2016-02-10 09:07:44 -07:00
|
|
|
policy_dbs->policy = policy;
|
2012-12-27 07:55:38 -07:00
|
|
|
|
2016-02-07 08:05:07 -07:00
|
|
|
if (gov->governor == GOV_CONSERVATIVE) {
|
2015-06-04 05:13:27 -06:00
|
|
|
struct cs_cpu_dbs_info_s *cs_dbs_info =
|
2016-02-07 08:05:07 -07:00
|
|
|
gov->get_cpu_dbs_info_s(cpu);
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
cs_dbs_info->down_skip = 0;
|
|
|
|
cs_dbs_info->requested_freq = policy->cur;
|
|
|
|
} else {
|
2016-02-07 08:05:07 -07:00
|
|
|
struct od_ops *od_ops = gov->gov_ops;
|
|
|
|
struct od_cpu_dbs_info_s *od_dbs_info = gov->get_cpu_dbs_info_s(cpu);
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2015-06-04 05:13:27 -06:00
|
|
|
od_dbs_info->rate_mult = 1;
|
|
|
|
od_dbs_info->sample_type = OD_NORMAL_SAMPLE;
|
|
|
|
od_ops->powersave_bias_init_cpu(cpu);
|
|
|
|
}
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
gov_set_update_util(policy_dbs, sampling_rate);
|
2015-06-04 05:13:27 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-02-04 19:15:24 -07:00
|
|
|
static int cpufreq_governor_stop(struct cpufreq_policy *policy)
|
2015-06-04 05:13:27 -06:00
|
|
|
{
|
2016-02-07 08:24:26 -07:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
2015-07-18 00:00:59 -06:00
|
|
|
|
2015-07-18 00:01:01 -06:00
|
|
|
/* State should be equivalent to START */
|
2016-02-07 08:24:26 -07:00
|
|
|
if (!policy_dbs->policy)
|
2015-07-18 00:01:01 -06:00
|
|
|
return -EBUSY;
|
|
|
|
|
2016-02-10 09:07:44 -07:00
|
|
|
gov_cancel_work(policy_dbs);
|
|
|
|
policy_dbs->policy = NULL;
|
2015-10-28 20:38:38 -06:00
|
|
|
|
2015-07-18 00:01:01 -06:00
|
|
|
return 0;
|
2015-06-04 05:13:27 -06:00
|
|
|
}
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2016-02-04 19:15:24 -07:00
|
|
|
static int cpufreq_governor_limits(struct cpufreq_policy *policy)
|
2015-06-04 05:13:27 -06:00
|
|
|
{
|
2016-02-07 08:24:26 -07:00
|
|
|
struct policy_dbs_info *policy_dbs = policy->governor_data;
|
2013-01-31 10:28:01 -07:00
|
|
|
|
2015-07-18 00:01:01 -06:00
|
|
|
/* State should be equivalent to START */
|
2016-02-07 08:24:26 -07:00
|
|
|
if (!policy_dbs->policy)
|
2015-07-18 00:01:01 -06:00
|
|
|
return -EBUSY;
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2016-02-07 08:23:49 -07:00
|
|
|
mutex_lock(&policy_dbs->timer_mutex);
|
|
|
|
if (policy->max < policy->cur)
|
|
|
|
__cpufreq_driver_target(policy, policy->max, CPUFREQ_RELATION_H);
|
|
|
|
else if (policy->min > policy->cur)
|
|
|
|
__cpufreq_driver_target(policy, policy->min, CPUFREQ_RELATION_L);
|
2016-02-06 05:50:24 -07:00
|
|
|
dbs_check_cpu(policy);
|
2016-02-07 08:23:49 -07:00
|
|
|
mutex_unlock(&policy_dbs->timer_mutex);
|
2015-07-18 00:01:01 -06:00
|
|
|
|
|
|
|
return 0;
|
2015-06-04 05:13:27 -06:00
|
|
|
}
|
2012-10-25 16:47:42 -06:00
|
|
|
|
2016-02-07 08:07:51 -07:00
|
|
|
int cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event)
|
2015-06-04 05:13:27 -06:00
|
|
|
{
|
2016-02-04 19:15:24 -07:00
|
|
|
int ret = -EINVAL;
|
2015-06-04 05:13:27 -06:00
|
|
|
|
cpufreq: governor: Serialize governor callbacks
There are several races reported in cpufreq core around governors (only
ondemand and conservative) by different people.
There are at least two race scenarios present in governor code:
(a) Concurrent access/updates of governor internal structures.
It is possible that fields such as 'dbs_data->usage_count', etc. are
accessed simultaneously for different policies using same governor
structure (i.e. CPUFREQ_HAVE_GOVERNOR_PER_POLICY flag unset). And
because of this we can dereference bad pointers.
For example consider a system with two CPUs with separate 'struct
cpufreq_policy' instances. CPU0 governor: ondemand and CPU1: powersave.
CPU0 switching to powersave and CPU1 to ondemand:
CPU0 CPU1
store* store*
cpufreq_governor_exit() cpufreq_governor_init()
dbs_data = cdata->gdbs_data;
if (!--dbs_data->usage_count)
kfree(dbs_data);
dbs_data->usage_count++;
*Bad pointer dereference*
There are other races possible between EXIT and START/STOP/LIMIT as
well. Its really complicated.
(b) Switching governor state in bad sequence:
For example trying to switch a governor to START state, when the
governor is in EXIT state. There are some checks present in
__cpufreq_governor() but they aren't sufficient as they compare events
against 'policy->governor_enabled', where as we need to take governor's
state into account, which can be used by multiple policies.
These two issues need to be solved separately and the responsibility
should be properly divided between cpufreq and governor core.
The first problem is more about the governor core, as it needs to
protect its structures properly. And the second problem should be fixed
in cpufreq core instead of governor, as its all about sequence of
events.
This patch is trying to solve only the first problem.
There are two types of data we need to protect,
- 'struct common_dbs_data': No matter what, there is going to be a
single copy of this per governor.
- 'struct dbs_data': With CPUFREQ_HAVE_GOVERNOR_PER_POLICY flag set, we
will have per-policy copy of this data, otherwise a single copy.
Because of such complexities, the mutex present in 'struct dbs_data' is
insufficient to solve our problem. For example we need to protect
fetching of 'dbs_data' from different structures at the beginning of
cpufreq_governor_dbs(), to make sure it isn't currently being updated.
This can be fixed if we can guarantee serialization of event parsing
code for an individual governor. This is best solved with a mutex per
governor, and the placeholder for that is 'struct common_dbs_data'.
And so this patch moves the mutex from 'struct dbs_data' to 'struct
common_dbs_data' and takes it at the beginning and drops it at the end
of cpufreq_governor_dbs().
Tested with and without following configuration options:
CONFIG_LOCKDEP_SUPPORT=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_PI_LIST=y
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_DEBUG_LOCK_ALLOC=y
CONFIG_PROVE_LOCKING=y
CONFIG_LOCKDEP=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>
Reviewed-by: Preeti U Murthy <preeti@linux.vnet.ibm.com>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2015-06-03 04:27:13 -06:00
|
|
|
/* Lock governor to block concurrent initialization of governor */
|
2016-02-07 08:01:31 -07:00
|
|
|
mutex_lock(&dbs_data_mutex);
|
cpufreq: governor: Serialize governor callbacks
There are several races reported in cpufreq core around governors (only
ondemand and conservative) by different people.
There are at least two race scenarios present in governor code:
(a) Concurrent access/updates of governor internal structures.
It is possible that fields such as 'dbs_data->usage_count', etc. are
accessed simultaneously for different policies using same governor
structure (i.e. CPUFREQ_HAVE_GOVERNOR_PER_POLICY flag unset). And
because of this we can dereference bad pointers.
For example consider a system with two CPUs with separate 'struct
cpufreq_policy' instances. CPU0 governor: ondemand and CPU1: powersave.
CPU0 switching to powersave and CPU1 to ondemand:
CPU0 CPU1
store* store*
cpufreq_governor_exit() cpufreq_governor_init()
dbs_data = cdata->gdbs_data;
if (!--dbs_data->usage_count)
kfree(dbs_data);
dbs_data->usage_count++;
*Bad pointer dereference*
There are other races possible between EXIT and START/STOP/LIMIT as
well. Its really complicated.
(b) Switching governor state in bad sequence:
For example trying to switch a governor to START state, when the
governor is in EXIT state. There are some checks present in
__cpufreq_governor() but they aren't sufficient as they compare events
against 'policy->governor_enabled', where as we need to take governor's
state into account, which can be used by multiple policies.
These two issues need to be solved separately and the responsibility
should be properly divided between cpufreq and governor core.
The first problem is more about the governor core, as it needs to
protect its structures properly. And the second problem should be fixed
in cpufreq core instead of governor, as its all about sequence of
events.
This patch is trying to solve only the first problem.
There are two types of data we need to protect,
- 'struct common_dbs_data': No matter what, there is going to be a
single copy of this per governor.
- 'struct dbs_data': With CPUFREQ_HAVE_GOVERNOR_PER_POLICY flag set, we
will have per-policy copy of this data, otherwise a single copy.
Because of such complexities, the mutex present in 'struct dbs_data' is
insufficient to solve our problem. For example we need to protect
fetching of 'dbs_data' from different structures at the beginning of
cpufreq_governor_dbs(), to make sure it isn't currently being updated.
This can be fixed if we can guarantee serialization of event parsing
code for an individual governor. This is best solved with a mutex per
governor, and the placeholder for that is 'struct common_dbs_data'.
And so this patch moves the mutex from 'struct dbs_data' to 'struct
common_dbs_data' and takes it at the beginning and drops it at the end
of cpufreq_governor_dbs().
Tested with and without following configuration options:
CONFIG_LOCKDEP_SUPPORT=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_PI_LIST=y
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_DEBUG_LOCK_ALLOC=y
CONFIG_PROVE_LOCKING=y
CONFIG_LOCKDEP=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>
Reviewed-by: Preeti U Murthy <preeti@linux.vnet.ibm.com>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
2015-06-03 04:27:13 -06:00
|
|
|
|
2016-02-04 19:15:24 -07:00
|
|
|
if (event == CPUFREQ_GOV_POLICY_INIT) {
|
2016-02-07 08:07:51 -07:00
|
|
|
ret = cpufreq_governor_init(policy);
|
2016-02-04 19:15:24 -07:00
|
|
|
} else if (policy->governor_data) {
|
|
|
|
switch (event) {
|
|
|
|
case CPUFREQ_GOV_POLICY_EXIT:
|
|
|
|
ret = cpufreq_governor_exit(policy);
|
|
|
|
break;
|
|
|
|
case CPUFREQ_GOV_START:
|
|
|
|
ret = cpufreq_governor_start(policy);
|
|
|
|
break;
|
|
|
|
case CPUFREQ_GOV_STOP:
|
|
|
|
ret = cpufreq_governor_stop(policy);
|
|
|
|
break;
|
|
|
|
case CPUFREQ_GOV_LIMITS:
|
|
|
|
ret = cpufreq_governor_limits(policy);
|
|
|
|
break;
|
|
|
|
}
|
2012-10-25 16:47:42 -06:00
|
|
|
}
|
2015-06-04 05:13:27 -06:00
|
|
|
|
2016-02-07 08:01:31 -07:00
|
|
|
mutex_unlock(&dbs_data_mutex);
|
2015-06-04 05:13:27 -06:00
|
|
|
return ret;
|
2012-10-25 16:47:42 -06:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(cpufreq_governor_dbs);
|