x86/mm: Add tracepoints for TLB flushes
We don't have any good way to figure out what kinds of flushes are being attempted. Right now, we can try to use the vm counters, but those only tell us what we actually did with the hardware (one-by-one vs full) and don't tell us what was actually _requested_. This allows us to select out "interesting" TLB flushes that we might want to optimize (like the ranged ones) and ignore the ones that we have very little control over (the ones at context switch). Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com> Link: http://lkml.kernel.org/r/20140731154059.4C96CBA5@viggo.jf.intel.com Acked-by: Rik van Riel <riel@redhat.com> Cc: Mel Gorman <mgorman@suse.de> Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
This commit is contained in:
parent
a23421f111
commit
d17d8f9ded
|
@ -3,6 +3,10 @@
|
||||||
|
|
||||||
#include <asm/desc.h>
|
#include <asm/desc.h>
|
||||||
#include <linux/atomic.h>
|
#include <linux/atomic.h>
|
||||||
|
#include <linux/mm_types.h>
|
||||||
|
|
||||||
|
#include <trace/events/tlb.h>
|
||||||
|
|
||||||
#include <asm/pgalloc.h>
|
#include <asm/pgalloc.h>
|
||||||
#include <asm/tlbflush.h>
|
#include <asm/tlbflush.h>
|
||||||
#include <asm/paravirt.h>
|
#include <asm/paravirt.h>
|
||||||
|
@ -44,6 +48,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
|
||||||
|
|
||||||
/* Re-load page tables */
|
/* Re-load page tables */
|
||||||
load_cr3(next->pgd);
|
load_cr3(next->pgd);
|
||||||
|
trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
|
||||||
|
|
||||||
/* Stop flush ipis for the previous mm */
|
/* Stop flush ipis for the previous mm */
|
||||||
cpumask_clear_cpu(cpu, mm_cpumask(prev));
|
cpumask_clear_cpu(cpu, mm_cpumask(prev));
|
||||||
|
@ -71,6 +76,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
|
||||||
* to make sure to use no freed page tables.
|
* to make sure to use no freed page tables.
|
||||||
*/
|
*/
|
||||||
load_cr3(next->pgd);
|
load_cr3(next->pgd);
|
||||||
|
trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
|
||||||
load_LDT_nolock(&next->context);
|
load_LDT_nolock(&next->context);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,6 +18,13 @@
|
||||||
#include <asm/dma.h> /* for MAX_DMA_PFN */
|
#include <asm/dma.h> /* for MAX_DMA_PFN */
|
||||||
#include <asm/microcode.h>
|
#include <asm/microcode.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We need to define the tracepoints somewhere, and tlb.c
|
||||||
|
* is only compied when SMP=y.
|
||||||
|
*/
|
||||||
|
#define CREATE_TRACE_POINTS
|
||||||
|
#include <trace/events/tlb.h>
|
||||||
|
|
||||||
#include "mm_internal.h"
|
#include "mm_internal.h"
|
||||||
|
|
||||||
static unsigned long __initdata pgt_buf_start;
|
static unsigned long __initdata pgt_buf_start;
|
||||||
|
|
|
@ -49,6 +49,7 @@ void leave_mm(int cpu)
|
||||||
if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) {
|
if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) {
|
||||||
cpumask_clear_cpu(cpu, mm_cpumask(active_mm));
|
cpumask_clear_cpu(cpu, mm_cpumask(active_mm));
|
||||||
load_cr3(swapper_pg_dir);
|
load_cr3(swapper_pg_dir);
|
||||||
|
trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(leave_mm);
|
EXPORT_SYMBOL_GPL(leave_mm);
|
||||||
|
@ -107,15 +108,19 @@ static void flush_tlb_func(void *info)
|
||||||
|
|
||||||
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
|
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
|
||||||
if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
|
if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
|
||||||
if (f->flush_end == TLB_FLUSH_ALL)
|
if (f->flush_end == TLB_FLUSH_ALL) {
|
||||||
local_flush_tlb();
|
local_flush_tlb();
|
||||||
else {
|
trace_tlb_flush(TLB_REMOTE_SHOOTDOWN, TLB_FLUSH_ALL);
|
||||||
|
} else {
|
||||||
unsigned long addr;
|
unsigned long addr;
|
||||||
|
unsigned long nr_pages =
|
||||||
|
f->flush_end - f->flush_start / PAGE_SIZE;
|
||||||
addr = f->flush_start;
|
addr = f->flush_start;
|
||||||
while (addr < f->flush_end) {
|
while (addr < f->flush_end) {
|
||||||
__flush_tlb_single(addr);
|
__flush_tlb_single(addr);
|
||||||
addr += PAGE_SIZE;
|
addr += PAGE_SIZE;
|
||||||
}
|
}
|
||||||
|
trace_tlb_flush(TLB_REMOTE_SHOOTDOWN, nr_pages);
|
||||||
}
|
}
|
||||||
} else
|
} else
|
||||||
leave_mm(smp_processor_id());
|
leave_mm(smp_processor_id());
|
||||||
|
@ -153,6 +158,7 @@ void flush_tlb_current_task(void)
|
||||||
|
|
||||||
count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
|
count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
|
||||||
local_flush_tlb();
|
local_flush_tlb();
|
||||||
|
trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
|
||||||
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
|
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
|
||||||
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
|
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
|
||||||
preempt_enable();
|
preempt_enable();
|
||||||
|
@ -191,6 +197,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
|
||||||
__flush_tlb_single(addr);
|
__flush_tlb_single(addr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
trace_tlb_flush(TLB_LOCAL_MM_SHOOTDOWN, base_pages_to_flush);
|
||||||
out:
|
out:
|
||||||
if (base_pages_to_flush == TLB_FLUSH_ALL) {
|
if (base_pages_to_flush == TLB_FLUSH_ALL) {
|
||||||
start = 0UL;
|
start = 0UL;
|
||||||
|
|
|
@ -516,4 +516,12 @@ struct vm_special_mapping
|
||||||
struct page **pages;
|
struct page **pages;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
enum tlb_flush_reason {
|
||||||
|
TLB_FLUSH_ON_TASK_SWITCH,
|
||||||
|
TLB_REMOTE_SHOOTDOWN,
|
||||||
|
TLB_LOCAL_SHOOTDOWN,
|
||||||
|
TLB_LOCAL_MM_SHOOTDOWN,
|
||||||
|
NR_TLB_FLUSH_REASONS,
|
||||||
|
};
|
||||||
|
|
||||||
#endif /* _LINUX_MM_TYPES_H */
|
#endif /* _LINUX_MM_TYPES_H */
|
||||||
|
|
40
include/trace/events/tlb.h
Normal file
40
include/trace/events/tlb.h
Normal file
|
@ -0,0 +1,40 @@
|
||||||
|
#undef TRACE_SYSTEM
|
||||||
|
#define TRACE_SYSTEM tlb
|
||||||
|
|
||||||
|
#if !defined(_TRACE_TLB_H) || defined(TRACE_HEADER_MULTI_READ)
|
||||||
|
#define _TRACE_TLB_H
|
||||||
|
|
||||||
|
#include <linux/mm_types.h>
|
||||||
|
#include <linux/tracepoint.h>
|
||||||
|
|
||||||
|
#define TLB_FLUSH_REASON \
|
||||||
|
{ TLB_FLUSH_ON_TASK_SWITCH, "flush on task switch" }, \
|
||||||
|
{ TLB_REMOTE_SHOOTDOWN, "remote shootdown" }, \
|
||||||
|
{ TLB_LOCAL_SHOOTDOWN, "local shootdown" }, \
|
||||||
|
{ TLB_LOCAL_MM_SHOOTDOWN, "local mm shootdown" }
|
||||||
|
|
||||||
|
TRACE_EVENT(tlb_flush,
|
||||||
|
|
||||||
|
TP_PROTO(int reason, unsigned long pages),
|
||||||
|
TP_ARGS(reason, pages),
|
||||||
|
|
||||||
|
TP_STRUCT__entry(
|
||||||
|
__field( int, reason)
|
||||||
|
__field(unsigned long, pages)
|
||||||
|
),
|
||||||
|
|
||||||
|
TP_fast_assign(
|
||||||
|
__entry->reason = reason;
|
||||||
|
__entry->pages = pages;
|
||||||
|
),
|
||||||
|
|
||||||
|
TP_printk("pages:%ld reason:%s (%d)",
|
||||||
|
__entry->pages,
|
||||||
|
__print_symbolic(__entry->reason, TLB_FLUSH_REASON),
|
||||||
|
__entry->reason)
|
||||||
|
);
|
||||||
|
|
||||||
|
#endif /* _TRACE_TLB_H */
|
||||||
|
|
||||||
|
/* This part must be outside protection */
|
||||||
|
#include <trace/define_trace.h>
|
Loading…
Reference in a new issue