blkcg: cfq doesn't need per-cpu dispatch stats
blkio_group_stats_cpu is used to count dispatch stats using per-cpu counters. This is used by both blk-throtl and cfq-iosched but the sharing is rather silly. * cfq-iosched doesn't need per-cpu dispatch stats. cfq always updates those stats while holding queue_lock. * blk-throtl needs per-cpu dispatch stats but only service_bytes and serviced. It doesn't make use of sectors. This patch makes cfq add and use global stats for service_bytes, serviced and sectors, removes per-cpu sectors counter and moves per-cpu stat printing code to blk-throttle.c. Signed-off-by: Tejun Heo <tj@kernel.org>hifive-unleashed-5.1
parent
629ed0b102
commit
41b38b6d54
|
@ -390,7 +390,6 @@ static void blkio_reset_stats_cpu(struct blkio_group *blkg, int plid)
|
||||||
|
|
||||||
blkg_rwstat_reset(&sc->service_bytes);
|
blkg_rwstat_reset(&sc->service_bytes);
|
||||||
blkg_rwstat_reset(&sc->serviced);
|
blkg_rwstat_reset(&sc->serviced);
|
||||||
blkg_stat_reset(&sc->sectors);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -417,6 +416,8 @@ blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
|
||||||
struct blkio_group_stats *stats = &pd->stats;
|
struct blkio_group_stats *stats = &pd->stats;
|
||||||
|
|
||||||
/* queued stats shouldn't be cleared */
|
/* queued stats shouldn't be cleared */
|
||||||
|
blkg_rwstat_reset(&stats->service_bytes);
|
||||||
|
blkg_rwstat_reset(&stats->serviced);
|
||||||
blkg_rwstat_reset(&stats->merged);
|
blkg_rwstat_reset(&stats->merged);
|
||||||
blkg_rwstat_reset(&stats->service_time);
|
blkg_rwstat_reset(&stats->service_time);
|
||||||
blkg_rwstat_reset(&stats->wait_time);
|
blkg_rwstat_reset(&stats->wait_time);
|
||||||
|
@ -577,66 +578,6 @@ int blkcg_print_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(blkcg_print_rwstat);
|
EXPORT_SYMBOL_GPL(blkcg_print_rwstat);
|
||||||
|
|
||||||
static u64 blkg_prfill_cpu_stat(struct seq_file *sf,
|
|
||||||
struct blkg_policy_data *pd, int off)
|
|
||||||
{
|
|
||||||
u64 v = 0;
|
|
||||||
int cpu;
|
|
||||||
|
|
||||||
for_each_possible_cpu(cpu) {
|
|
||||||
struct blkio_group_stats_cpu *sc =
|
|
||||||
per_cpu_ptr(pd->stats_cpu, cpu);
|
|
||||||
|
|
||||||
v += blkg_stat_read((void *)sc + off);
|
|
||||||
}
|
|
||||||
|
|
||||||
return __blkg_prfill_u64(sf, pd, v);
|
|
||||||
}
|
|
||||||
|
|
||||||
static u64 blkg_prfill_cpu_rwstat(struct seq_file *sf,
|
|
||||||
struct blkg_policy_data *pd, int off)
|
|
||||||
{
|
|
||||||
struct blkg_rwstat rwstat = { }, tmp;
|
|
||||||
int i, cpu;
|
|
||||||
|
|
||||||
for_each_possible_cpu(cpu) {
|
|
||||||
struct blkio_group_stats_cpu *sc =
|
|
||||||
per_cpu_ptr(pd->stats_cpu, cpu);
|
|
||||||
|
|
||||||
tmp = blkg_rwstat_read((void *)sc + off);
|
|
||||||
for (i = 0; i < BLKG_RWSTAT_NR; i++)
|
|
||||||
rwstat.cnt[i] += tmp.cnt[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
return __blkg_prfill_rwstat(sf, pd, &rwstat);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* print per-cpu blkg_stat specified by BLKCG_STAT_PRIV() */
|
|
||||||
int blkcg_print_cpu_stat(struct cgroup *cgrp, struct cftype *cft,
|
|
||||||
struct seq_file *sf)
|
|
||||||
{
|
|
||||||
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
|
||||||
|
|
||||||
blkcg_print_blkgs(sf, blkcg, blkg_prfill_cpu_stat,
|
|
||||||
BLKCG_STAT_POL(cft->private),
|
|
||||||
BLKCG_STAT_OFF(cft->private), false);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL_GPL(blkcg_print_cpu_stat);
|
|
||||||
|
|
||||||
/* print per-cpu blkg_rwstat specified by BLKCG_STAT_PRIV() */
|
|
||||||
int blkcg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
|
||||||
struct seq_file *sf)
|
|
||||||
{
|
|
||||||
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
|
||||||
|
|
||||||
blkcg_print_blkgs(sf, blkcg, blkg_prfill_cpu_rwstat,
|
|
||||||
BLKCG_STAT_POL(cft->private),
|
|
||||||
BLKCG_STAT_OFF(cft->private), true);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL_GPL(blkcg_print_cpu_rwstat);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* blkg_conf_prep - parse and prepare for per-blkg config update
|
* blkg_conf_prep - parse and prepare for per-blkg config update
|
||||||
* @blkcg: target block cgroup
|
* @blkcg: target block cgroup
|
||||||
|
|
|
@ -65,6 +65,10 @@ struct blkg_rwstat {
|
||||||
};
|
};
|
||||||
|
|
||||||
struct blkio_group_stats {
|
struct blkio_group_stats {
|
||||||
|
/* total bytes transferred */
|
||||||
|
struct blkg_rwstat service_bytes;
|
||||||
|
/* total IOs serviced, post merge */
|
||||||
|
struct blkg_rwstat serviced;
|
||||||
/* number of ios merged */
|
/* number of ios merged */
|
||||||
struct blkg_rwstat merged;
|
struct blkg_rwstat merged;
|
||||||
/* total time spent on device in ns, may not be accurate w/ queueing */
|
/* total time spent on device in ns, may not be accurate w/ queueing */
|
||||||
|
@ -73,6 +77,8 @@ struct blkio_group_stats {
|
||||||
struct blkg_rwstat wait_time;
|
struct blkg_rwstat wait_time;
|
||||||
/* number of IOs queued up */
|
/* number of IOs queued up */
|
||||||
struct blkg_rwstat queued;
|
struct blkg_rwstat queued;
|
||||||
|
/* total sectors transferred */
|
||||||
|
struct blkg_stat sectors;
|
||||||
/* total disk time and nr sectors dispatched by this group */
|
/* total disk time and nr sectors dispatched by this group */
|
||||||
struct blkg_stat time;
|
struct blkg_stat time;
|
||||||
#ifdef CONFIG_DEBUG_BLK_CGROUP
|
#ifdef CONFIG_DEBUG_BLK_CGROUP
|
||||||
|
@ -104,8 +110,6 @@ struct blkio_group_stats_cpu {
|
||||||
struct blkg_rwstat service_bytes;
|
struct blkg_rwstat service_bytes;
|
||||||
/* total IOs serviced, post merge */
|
/* total IOs serviced, post merge */
|
||||||
struct blkg_rwstat serviced;
|
struct blkg_rwstat serviced;
|
||||||
/* total sectors transferred */
|
|
||||||
struct blkg_stat sectors;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct blkio_group_conf {
|
struct blkio_group_conf {
|
||||||
|
@ -183,10 +187,6 @@ int blkcg_print_stat(struct cgroup *cgrp, struct cftype *cft,
|
||||||
struct seq_file *sf);
|
struct seq_file *sf);
|
||||||
int blkcg_print_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
int blkcg_print_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
||||||
struct seq_file *sf);
|
struct seq_file *sf);
|
||||||
int blkcg_print_cpu_stat(struct cgroup *cgrp, struct cftype *cft,
|
|
||||||
struct seq_file *sf);
|
|
||||||
int blkcg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
|
||||||
struct seq_file *sf);
|
|
||||||
|
|
||||||
struct blkg_conf_ctx {
|
struct blkg_conf_ctx {
|
||||||
struct gendisk *disk;
|
struct gendisk *disk;
|
||||||
|
|
|
@ -582,7 +582,6 @@ static void throtl_update_dispatch_stats(struct blkio_group *blkg, u64 bytes,
|
||||||
|
|
||||||
stats_cpu = this_cpu_ptr(pd->stats_cpu);
|
stats_cpu = this_cpu_ptr(pd->stats_cpu);
|
||||||
|
|
||||||
blkg_stat_add(&stats_cpu->sectors, bytes >> 9);
|
|
||||||
blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
|
blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
|
||||||
blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
|
blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
|
||||||
|
|
||||||
|
@ -843,6 +842,36 @@ static void throtl_update_blkio_group_common(struct throtl_data *td,
|
||||||
throtl_schedule_delayed_work(td, 0);
|
throtl_schedule_delayed_work(td, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static u64 blkg_prfill_cpu_rwstat(struct seq_file *sf,
|
||||||
|
struct blkg_policy_data *pd, int off)
|
||||||
|
{
|
||||||
|
struct blkg_rwstat rwstat = { }, tmp;
|
||||||
|
int i, cpu;
|
||||||
|
|
||||||
|
for_each_possible_cpu(cpu) {
|
||||||
|
struct blkio_group_stats_cpu *sc =
|
||||||
|
per_cpu_ptr(pd->stats_cpu, cpu);
|
||||||
|
|
||||||
|
tmp = blkg_rwstat_read((void *)sc + off);
|
||||||
|
for (i = 0; i < BLKG_RWSTAT_NR; i++)
|
||||||
|
rwstat.cnt[i] += tmp.cnt[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
return __blkg_prfill_rwstat(sf, pd, &rwstat);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* print per-cpu blkg_rwstat specified by BLKCG_STAT_PRIV() */
|
||||||
|
static int blkcg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
||||||
|
struct seq_file *sf)
|
||||||
|
{
|
||||||
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
||||||
|
|
||||||
|
blkcg_print_blkgs(sf, blkcg, blkg_prfill_cpu_rwstat,
|
||||||
|
BLKCG_STAT_POL(cft->private),
|
||||||
|
BLKCG_STAT_OFF(cft->private), true);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
static u64 blkg_prfill_conf_u64(struct seq_file *sf,
|
static u64 blkg_prfill_conf_u64(struct seq_file *sf,
|
||||||
struct blkg_policy_data *pd, int off)
|
struct blkg_policy_data *pd, int off)
|
||||||
{
|
{
|
||||||
|
|
|
@ -624,29 +624,12 @@ static inline void cfq_blkiocg_update_dispatch_stats(struct blkio_group *blkg,
|
||||||
struct blkio_policy_type *pol, uint64_t bytes,
|
struct blkio_policy_type *pol, uint64_t bytes,
|
||||||
bool direction, bool sync)
|
bool direction, bool sync)
|
||||||
{
|
{
|
||||||
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
||||||
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
|
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
|
||||||
struct blkg_policy_data *pd = blkg->pd[pol->plid];
|
|
||||||
struct blkio_group_stats_cpu *stats_cpu;
|
|
||||||
unsigned long flags;
|
|
||||||
|
|
||||||
/* If per cpu stats are not allocated yet, don't do any accounting. */
|
blkg_stat_add(&stats->sectors, bytes >> 9);
|
||||||
if (pd->stats_cpu == NULL)
|
blkg_rwstat_add(&stats->serviced, rw, 1);
|
||||||
return;
|
blkg_rwstat_add(&stats->service_bytes, rw, bytes);
|
||||||
|
|
||||||
/*
|
|
||||||
* Disabling interrupts to provide mutual exclusion between two
|
|
||||||
* writes on same cpu. It probably is not needed for 64bit. Not
|
|
||||||
* optimizing that case yet.
|
|
||||||
*/
|
|
||||||
local_irq_save(flags);
|
|
||||||
|
|
||||||
stats_cpu = this_cpu_ptr(pd->stats_cpu);
|
|
||||||
|
|
||||||
blkg_stat_add(&stats_cpu->sectors, bytes >> 9);
|
|
||||||
blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
|
|
||||||
blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
|
|
||||||
|
|
||||||
local_irq_restore(flags);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void cfq_blkiocg_update_completion_stats(struct blkio_group *blkg,
|
static inline void cfq_blkiocg_update_completion_stats(struct blkio_group *blkg,
|
||||||
|
@ -1520,20 +1503,20 @@ static struct cftype cfq_blkcg_files[] = {
|
||||||
{
|
{
|
||||||
.name = "sectors",
|
.name = "sectors",
|
||||||
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
||||||
offsetof(struct blkio_group_stats_cpu, sectors)),
|
offsetof(struct blkio_group_stats, sectors)),
|
||||||
.read_seq_string = blkcg_print_cpu_stat,
|
.read_seq_string = blkcg_print_stat,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
.name = "io_service_bytes",
|
.name = "io_service_bytes",
|
||||||
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
||||||
offsetof(struct blkio_group_stats_cpu, service_bytes)),
|
offsetof(struct blkio_group_stats, service_bytes)),
|
||||||
.read_seq_string = blkcg_print_cpu_rwstat,
|
.read_seq_string = blkcg_print_rwstat,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
.name = "io_serviced",
|
.name = "io_serviced",
|
||||||
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
||||||
offsetof(struct blkio_group_stats_cpu, serviced)),
|
offsetof(struct blkio_group_stats, serviced)),
|
||||||
.read_seq_string = blkcg_print_cpu_rwstat,
|
.read_seq_string = blkcg_print_rwstat,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
.name = "io_service_time",
|
.name = "io_service_time",
|
||||||
|
|
Loading…
Reference in New Issue