powerpc/perf: Implement a global lock to avoid races between trace, core and thread imc events.
[ Upstream commit a36e8ba60b ]
IMC(In-memory Collection Counters) does performance monitoring in
two different modes, i.e accumulation mode(core-imc and thread-imc events),
and trace mode(trace-imc events). A cpu thread can either be in
accumulation-mode or trace-mode at a time and this is done via the LDBAR
register in POWER architecture. The current design does not address the
races between thread-imc and trace-imc events.
Patch implements a global id and lock to avoid the races between
core, trace and thread imc events. With this global id-lock
implementation, the system can either run core, thread or trace imc
events at a time. i.e. to run any core-imc events, thread/trace imc events
should not be enabled/monitored.
Signed-off-by: Anju T Sudhakar <anju@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20200313055238.8656-1-anju@linux.vnet.ibm.com
Signed-off-by: Sasha Levin <sashal@kernel.org>
5.4-rM2-2.2.x-imx-squashed
Anju T Sudhakar
Greg Kroah-Hartman
parent
7cafcc37c5
commit
685a1e317a
|
|
@ -44,6 +44,16 @@ static DEFINE_PER_CPU(u64 *, trace_imc_mem);
|
|||
static struct imc_pmu_ref *trace_imc_refc;
|
||||
static int trace_imc_mem_size;
|
||||
|
||||
/*
|
||||
* Global data structure used to avoid races between thread,
|
||||
* core and trace-imc
|
||||
*/
|
||||
static struct imc_pmu_ref imc_global_refc = {
|
||||
.lock = __MUTEX_INITIALIZER(imc_global_refc.lock),
|
||||
.id = 0,
|
||||
.refc = 0,
|
||||
};
|
||||
|
||||
static struct imc_pmu *imc_event_to_pmu(struct perf_event *event)
|
||||
{
|
||||
return container_of(event->pmu, struct imc_pmu, pmu);
|
||||
|
|
@ -698,6 +708,16 @@ static int ppc_core_imc_cpu_offline(unsigned int cpu)
|
|||
return -EINVAL;
|
||||
|
||||
ref->refc = 0;
|
||||
/*
|
||||
* Reduce the global reference count, if this is the
|
||||
* last cpu in this core and core-imc event running
|
||||
* in this cpu.
|
||||
*/
|
||||
mutex_lock(&imc_global_refc.lock);
|
||||
if (imc_global_refc.id == IMC_DOMAIN_CORE)
|
||||
imc_global_refc.refc--;
|
||||
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -710,6 +730,23 @@ static int core_imc_pmu_cpumask_init(void)
|
|||
ppc_core_imc_cpu_offline);
|
||||
}
|
||||
|
||||
static void reset_global_refc(struct perf_event *event)
|
||||
{
|
||||
mutex_lock(&imc_global_refc.lock);
|
||||
imc_global_refc.refc--;
|
||||
|
||||
/*
|
||||
* If no other thread is running any
|
||||
* event for this domain(thread/core/trace),
|
||||
* set the global id to zero.
|
||||
*/
|
||||
if (imc_global_refc.refc <= 0) {
|
||||
imc_global_refc.refc = 0;
|
||||
imc_global_refc.id = 0;
|
||||
}
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
}
|
||||
|
||||
static void core_imc_counters_release(struct perf_event *event)
|
||||
{
|
||||
int rc, core_id;
|
||||
|
|
@ -759,6 +796,8 @@ static void core_imc_counters_release(struct perf_event *event)
|
|||
ref->refc = 0;
|
||||
}
|
||||
mutex_unlock(&ref->lock);
|
||||
|
||||
reset_global_refc(event);
|
||||
}
|
||||
|
||||
static int core_imc_event_init(struct perf_event *event)
|
||||
|
|
@ -819,6 +858,29 @@ static int core_imc_event_init(struct perf_event *event)
|
|||
++ref->refc;
|
||||
mutex_unlock(&ref->lock);
|
||||
|
||||
/*
|
||||
* Since the system can run either in accumulation or trace-mode
|
||||
* of IMC at a time, core-imc events are allowed only if no other
|
||||
* trace/thread imc events are enabled/monitored.
|
||||
*
|
||||
* Take the global lock, and check the refc.id
|
||||
* to know whether any other trace/thread imc
|
||||
* events are running.
|
||||
*/
|
||||
mutex_lock(&imc_global_refc.lock);
|
||||
if (imc_global_refc.id == 0 || imc_global_refc.id == IMC_DOMAIN_CORE) {
|
||||
/*
|
||||
* No other trace/thread imc events are running in
|
||||
* the system, so set the refc.id to core-imc.
|
||||
*/
|
||||
imc_global_refc.id = IMC_DOMAIN_CORE;
|
||||
imc_global_refc.refc++;
|
||||
} else {
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
return -EBUSY;
|
||||
}
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
|
||||
event->hw.event_base = (u64)pcmi->vbase + (config & IMC_EVENT_OFFSET_MASK);
|
||||
event->destroy = core_imc_counters_release;
|
||||
return 0;
|
||||
|
|
@ -877,7 +939,23 @@ static int ppc_thread_imc_cpu_online(unsigned int cpu)
|
|||
|
||||
static int ppc_thread_imc_cpu_offline(unsigned int cpu)
|
||||
{
|
||||
mtspr(SPRN_LDBAR, 0);
|
||||
/*
|
||||
* Set the bit 0 of LDBAR to zero.
|
||||
*
|
||||
* If bit 0 of LDBAR is unset, it will stop posting
|
||||
* the counter data to memory.
|
||||
* For thread-imc, bit 0 of LDBAR will be set to 1 in the
|
||||
* event_add function. So reset this bit here, to stop the updates
|
||||
* to memory in the cpu_offline path.
|
||||
*/
|
||||
mtspr(SPRN_LDBAR, (mfspr(SPRN_LDBAR) & (~(1UL << 63))));
|
||||
|
||||
/* Reduce the refc if thread-imc event running on this cpu */
|
||||
mutex_lock(&imc_global_refc.lock);
|
||||
if (imc_global_refc.id == IMC_DOMAIN_THREAD)
|
||||
imc_global_refc.refc--;
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -916,7 +994,22 @@ static int thread_imc_event_init(struct perf_event *event)
|
|||
if (!target)
|
||||
return -EINVAL;
|
||||
|
||||
mutex_lock(&imc_global_refc.lock);
|
||||
/*
|
||||
* Check if any other trace/core imc events are running in the
|
||||
* system, if not set the global id to thread-imc.
|
||||
*/
|
||||
if (imc_global_refc.id == 0 || imc_global_refc.id == IMC_DOMAIN_THREAD) {
|
||||
imc_global_refc.id = IMC_DOMAIN_THREAD;
|
||||
imc_global_refc.refc++;
|
||||
} else {
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
return -EBUSY;
|
||||
}
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
|
||||
event->pmu->task_ctx_nr = perf_sw_context;
|
||||
event->destroy = reset_global_refc;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1063,10 +1156,12 @@ static void thread_imc_event_del(struct perf_event *event, int flags)
|
|||
int core_id;
|
||||
struct imc_pmu_ref *ref;
|
||||
|
||||
mtspr(SPRN_LDBAR, 0);
|
||||
|
||||
core_id = smp_processor_id() / threads_per_core;
|
||||
ref = &core_imc_refc[core_id];
|
||||
if (!ref) {
|
||||
pr_debug("imc: Failed to get event reference count\n");
|
||||
return;
|
||||
}
|
||||
|
||||
mutex_lock(&ref->lock);
|
||||
ref->refc--;
|
||||
|
|
@ -1082,6 +1177,10 @@ static void thread_imc_event_del(struct perf_event *event, int flags)
|
|||
ref->refc = 0;
|
||||
}
|
||||
mutex_unlock(&ref->lock);
|
||||
|
||||
/* Set bit 0 of LDBAR to zero, to stop posting updates to memory */
|
||||
mtspr(SPRN_LDBAR, (mfspr(SPRN_LDBAR) & (~(1UL << 63))));
|
||||
|
||||
/*
|
||||
* Take a snapshot and calculate the delta and update
|
||||
* the event counter values.
|
||||
|
|
@ -1133,7 +1232,18 @@ static int ppc_trace_imc_cpu_online(unsigned int cpu)
|
|||
|
||||
static int ppc_trace_imc_cpu_offline(unsigned int cpu)
|
||||
{
|
||||
mtspr(SPRN_LDBAR, 0);
|
||||
/*
|
||||
* No need to set bit 0 of LDBAR to zero, as
|
||||
* it is set to zero for imc trace-mode
|
||||
*
|
||||
* Reduce the refc if any trace-imc event running
|
||||
* on this cpu.
|
||||
*/
|
||||
mutex_lock(&imc_global_refc.lock);
|
||||
if (imc_global_refc.id == IMC_DOMAIN_TRACE)
|
||||
imc_global_refc.refc--;
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1226,15 +1336,14 @@ static int trace_imc_event_add(struct perf_event *event, int flags)
|
|||
local_mem = get_trace_imc_event_base_addr();
|
||||
ldbar_value = ((u64)local_mem & THREAD_IMC_LDBAR_MASK) | TRACE_IMC_ENABLE;
|
||||
|
||||
if (core_imc_refc)
|
||||
ref = &core_imc_refc[core_id];
|
||||
/* trace-imc reference count */
|
||||
if (trace_imc_refc)
|
||||
ref = &trace_imc_refc[core_id];
|
||||
if (!ref) {
|
||||
/* If core-imc is not enabled, use trace-imc reference count */
|
||||
if (trace_imc_refc)
|
||||
ref = &trace_imc_refc[core_id];
|
||||
if (!ref)
|
||||
return -EINVAL;
|
||||
pr_debug("imc: Failed to get the event reference count\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
mtspr(SPRN_LDBAR, ldbar_value);
|
||||
mutex_lock(&ref->lock);
|
||||
if (ref->refc == 0) {
|
||||
|
|
@ -1242,13 +1351,11 @@ static int trace_imc_event_add(struct perf_event *event, int flags)
|
|||
get_hard_smp_processor_id(smp_processor_id()))) {
|
||||
mutex_unlock(&ref->lock);
|
||||
pr_err("trace-imc: Unable to start the counters for core %d\n", core_id);
|
||||
mtspr(SPRN_LDBAR, 0);
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
++ref->refc;
|
||||
mutex_unlock(&ref->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1274,16 +1381,13 @@ static void trace_imc_event_del(struct perf_event *event, int flags)
|
|||
int core_id = smp_processor_id() / threads_per_core;
|
||||
struct imc_pmu_ref *ref = NULL;
|
||||
|
||||
if (core_imc_refc)
|
||||
ref = &core_imc_refc[core_id];
|
||||
if (trace_imc_refc)
|
||||
ref = &trace_imc_refc[core_id];
|
||||
if (!ref) {
|
||||
/* If core-imc is not enabled, use trace-imc reference count */
|
||||
if (trace_imc_refc)
|
||||
ref = &trace_imc_refc[core_id];
|
||||
if (!ref)
|
||||
return;
|
||||
pr_debug("imc: Failed to get event reference count\n");
|
||||
return;
|
||||
}
|
||||
mtspr(SPRN_LDBAR, 0);
|
||||
|
||||
mutex_lock(&ref->lock);
|
||||
ref->refc--;
|
||||
if (ref->refc == 0) {
|
||||
|
|
@ -1297,6 +1401,7 @@ static void trace_imc_event_del(struct perf_event *event, int flags)
|
|||
ref->refc = 0;
|
||||
}
|
||||
mutex_unlock(&ref->lock);
|
||||
|
||||
trace_imc_event_stop(event, flags);
|
||||
}
|
||||
|
||||
|
|
@ -1314,10 +1419,30 @@ static int trace_imc_event_init(struct perf_event *event)
|
|||
if (event->attr.sample_period == 0)
|
||||
return -ENOENT;
|
||||
|
||||
/*
|
||||
* Take the global lock, and make sure
|
||||
* no other thread is running any core/thread imc
|
||||
* events
|
||||
*/
|
||||
mutex_lock(&imc_global_refc.lock);
|
||||
if (imc_global_refc.id == 0 || imc_global_refc.id == IMC_DOMAIN_TRACE) {
|
||||
/*
|
||||
* No core/thread imc events are running in the
|
||||
* system, so set the refc.id to trace-imc.
|
||||
*/
|
||||
imc_global_refc.id = IMC_DOMAIN_TRACE;
|
||||
imc_global_refc.refc++;
|
||||
} else {
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
return -EBUSY;
|
||||
}
|
||||
mutex_unlock(&imc_global_refc.lock);
|
||||
|
||||
event->hw.idx = -1;
|
||||
target = event->hw.target;
|
||||
|
||||
event->pmu->task_ctx_nr = perf_hw_context;
|
||||
event->destroy = reset_global_refc;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1429,10 +1554,10 @@ static void cleanup_all_core_imc_memory(void)
|
|||
static void thread_imc_ldbar_disable(void *dummy)
|
||||
{
|
||||
/*
|
||||
* By Zeroing LDBAR, we disable thread-imc
|
||||
* updates.
|
||||
* By setting 0th bit of LDBAR to zero, we disable thread-imc
|
||||
* updates to memory.
|
||||
*/
|
||||
mtspr(SPRN_LDBAR, 0);
|
||||
mtspr(SPRN_LDBAR, (mfspr(SPRN_LDBAR) & (~(1UL << 63))));
|
||||
}
|
||||
|
||||
void thread_imc_disable(void)
|
||||
|
|
|
|||
Loading…
Reference in New Issue