![Mel Gorman](/assets/img/avatar_default.png)
An IPI is sent to flush remote TLBs when a page is unmapped that was potentially accesssed by other CPUs. There are many circumstances where this happens but the obvious one is kswapd reclaiming pages belonging to a running process as kswapd and the task are likely running on separate CPUs. On small machines, this is not a significant problem but as machine gets larger with more cores and more memory, the cost of these IPIs can be high. This patch uses a simple structure that tracks CPUs that potentially have TLB entries for pages being unmapped. When the unmapping is complete, the full TLB is flushed on the assumption that a refill cost is lower than flushing individual entries. Architectures wishing to do this must give the following guarantee. If a clean page is unmapped and not immediately flushed, the architecture must guarantee that a write to that linear address from a CPU with a cached TLB entry will trap a page fault. This is essentially what the kernel already depends on but the window is much larger with this patch applied and is worth highlighting. The architecture should consider whether the cost of the full TLB flush is higher than sending an IPI to flush each individual entry. An additional architecture helper called flush_tlb_local is required. It's a trivial wrapper with some accounting in the x86 case. The impact of this patch depends on the workload as measuring any benefit requires both mapped pages co-located on the LRU and memory pressure. The case with the biggest impact is multiple processes reading mapped pages taken from the vm-scalability test suite. The test case uses NR_CPU readers of mapped files that consume 10*RAM. Linear mapped reader on a 4-node machine with 64G RAM and 48 CPUs 4.2.0-rc1 4.2.0-rc1 vanilla flushfull-v7 Ops lru-file-mmap-read-elapsed 159.62 ( 0.00%) 120.68 ( 24.40%) Ops lru-file-mmap-read-time_range 30.59 ( 0.00%) 2.80 ( 90.85%) Ops lru-file-mmap-read-time_stddv 6.70 ( 0.00%) 0.64 ( 90.38%) 4.2.0-rc1 4.2.0-rc1 vanilla flushfull-v7 User 581.00 611.43 System 5804.93 4111.76 Elapsed 161.03 122.12 This is showing that the readers completed 24.40% faster with 29% less system CPU time. From vmstats, it is known that the vanilla kernel was interrupted roughly 900K times per second during the steady phase of the test and the patched kernel was interrupts 180K times per second. The impact is lower on a single socket machine. 4.2.0-rc1 4.2.0-rc1 vanilla flushfull-v7 Ops lru-file-mmap-read-elapsed 25.33 ( 0.00%) 20.38 ( 19.54%) Ops lru-file-mmap-read-time_range 0.91 ( 0.00%) 1.44 (-58.24%) Ops lru-file-mmap-read-time_stddv 0.28 ( 0.00%) 0.47 (-65.34%) 4.2.0-rc1 4.2.0-rc1 vanilla flushfull-v7 User 58.09 57.64 System 111.82 76.56 Elapsed 27.29 22.55 It's still a noticeable improvement with vmstat showing interrupts went from roughly 500K per second to 45K per second. The patch will have no impact on workloads with no memory pressure or have relatively few mapped pages. It will have an unpredictable impact on the workload running on the CPU being flushed as it'll depend on how many TLB entries need to be refilled and how long that takes. Worst case, the TLB will be completely cleared of active entries when the target PFNs were not resident at all. [sasha.levin@oracle.com: trace tlb flush after disabling preemption in try_to_unmap_flush] Signed-off-by: Mel Gorman <mgorman@suse.de> Reviewed-by: Rik van Riel <riel@redhat.com> Cc: Dave Hansen <dave.hansen@intel.com> Acked-by: Ingo Molnar <mingo@kernel.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Sasha Levin <sasha.levin@oracle.com> Cc: Michal Hocko <mhocko@suse.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
441 lines
14 KiB
C
441 lines
14 KiB
C
/* internal.h: mm/ internal definitions
|
|
*
|
|
* Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
|
|
* Written by David Howells (dhowells@redhat.com)
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*/
|
|
#ifndef __MM_INTERNAL_H
|
|
#define __MM_INTERNAL_H
|
|
|
|
#include <linux/fs.h>
|
|
#include <linux/mm.h>
|
|
|
|
void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
|
|
unsigned long floor, unsigned long ceiling);
|
|
|
|
static inline void set_page_count(struct page *page, int v)
|
|
{
|
|
atomic_set(&page->_count, v);
|
|
}
|
|
|
|
extern int __do_page_cache_readahead(struct address_space *mapping,
|
|
struct file *filp, pgoff_t offset, unsigned long nr_to_read,
|
|
unsigned long lookahead_size);
|
|
|
|
/*
|
|
* Submit IO for the read-ahead request in file_ra_state.
|
|
*/
|
|
static inline unsigned long ra_submit(struct file_ra_state *ra,
|
|
struct address_space *mapping, struct file *filp)
|
|
{
|
|
return __do_page_cache_readahead(mapping, filp,
|
|
ra->start, ra->size, ra->async_size);
|
|
}
|
|
|
|
/*
|
|
* Turn a non-refcounted page (->_count == 0) into refcounted with
|
|
* a count of one.
|
|
*/
|
|
static inline void set_page_refcounted(struct page *page)
|
|
{
|
|
VM_BUG_ON_PAGE(PageTail(page), page);
|
|
VM_BUG_ON_PAGE(atomic_read(&page->_count), page);
|
|
set_page_count(page, 1);
|
|
}
|
|
|
|
static inline void __get_page_tail_foll(struct page *page,
|
|
bool get_page_head)
|
|
{
|
|
/*
|
|
* If we're getting a tail page, the elevated page->_count is
|
|
* required only in the head page and we will elevate the head
|
|
* page->_count and tail page->_mapcount.
|
|
*
|
|
* We elevate page_tail->_mapcount for tail pages to force
|
|
* page_tail->_count to be zero at all times to avoid getting
|
|
* false positives from get_page_unless_zero() with
|
|
* speculative page access (like in
|
|
* page_cache_get_speculative()) on tail pages.
|
|
*/
|
|
VM_BUG_ON_PAGE(atomic_read(&page->first_page->_count) <= 0, page);
|
|
if (get_page_head)
|
|
atomic_inc(&page->first_page->_count);
|
|
get_huge_page_tail(page);
|
|
}
|
|
|
|
/*
|
|
* This is meant to be called as the FOLL_GET operation of
|
|
* follow_page() and it must be called while holding the proper PT
|
|
* lock while the pte (or pmd_trans_huge) is still mapping the page.
|
|
*/
|
|
static inline void get_page_foll(struct page *page)
|
|
{
|
|
if (unlikely(PageTail(page)))
|
|
/*
|
|
* This is safe only because
|
|
* __split_huge_page_refcount() can't run under
|
|
* get_page_foll() because we hold the proper PT lock.
|
|
*/
|
|
__get_page_tail_foll(page, true);
|
|
else {
|
|
/*
|
|
* Getting a normal page or the head of a compound page
|
|
* requires to already have an elevated page->_count.
|
|
*/
|
|
VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);
|
|
atomic_inc(&page->_count);
|
|
}
|
|
}
|
|
|
|
extern unsigned long highest_memmap_pfn;
|
|
|
|
/*
|
|
* in mm/vmscan.c:
|
|
*/
|
|
extern int isolate_lru_page(struct page *page);
|
|
extern void putback_lru_page(struct page *page);
|
|
extern bool zone_reclaimable(struct zone *zone);
|
|
|
|
/*
|
|
* in mm/rmap.c:
|
|
*/
|
|
extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);
|
|
|
|
/*
|
|
* in mm/page_alloc.c
|
|
*/
|
|
|
|
/*
|
|
* Structure for holding the mostly immutable allocation parameters passed
|
|
* between functions involved in allocations, including the alloc_pages*
|
|
* family of functions.
|
|
*
|
|
* nodemask, migratetype and high_zoneidx are initialized only once in
|
|
* __alloc_pages_nodemask() and then never change.
|
|
*
|
|
* zonelist, preferred_zone and classzone_idx are set first in
|
|
* __alloc_pages_nodemask() for the fast path, and might be later changed
|
|
* in __alloc_pages_slowpath(). All other functions pass the whole strucure
|
|
* by a const pointer.
|
|
*/
|
|
struct alloc_context {
|
|
struct zonelist *zonelist;
|
|
nodemask_t *nodemask;
|
|
struct zone *preferred_zone;
|
|
int classzone_idx;
|
|
int migratetype;
|
|
enum zone_type high_zoneidx;
|
|
};
|
|
|
|
/*
|
|
* Locate the struct page for both the matching buddy in our
|
|
* pair (buddy1) and the combined O(n+1) page they form (page).
|
|
*
|
|
* 1) Any buddy B1 will have an order O twin B2 which satisfies
|
|
* the following equation:
|
|
* B2 = B1 ^ (1 << O)
|
|
* For example, if the starting buddy (buddy2) is #8 its order
|
|
* 1 buddy is #10:
|
|
* B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
|
|
*
|
|
* 2) Any buddy B will have an order O+1 parent P which
|
|
* satisfies the following equation:
|
|
* P = B & ~(1 << O)
|
|
*
|
|
* Assumption: *_mem_map is contiguous at least up to MAX_ORDER
|
|
*/
|
|
static inline unsigned long
|
|
__find_buddy_index(unsigned long page_idx, unsigned int order)
|
|
{
|
|
return page_idx ^ (1 << order);
|
|
}
|
|
|
|
extern int __isolate_free_page(struct page *page, unsigned int order);
|
|
extern void __free_pages_bootmem(struct page *page, unsigned long pfn,
|
|
unsigned int order);
|
|
extern void prep_compound_page(struct page *page, unsigned long order);
|
|
#ifdef CONFIG_MEMORY_FAILURE
|
|
extern bool is_free_buddy_page(struct page *page);
|
|
#endif
|
|
extern int user_min_free_kbytes;
|
|
|
|
#if defined CONFIG_COMPACTION || defined CONFIG_CMA
|
|
|
|
/*
|
|
* in mm/compaction.c
|
|
*/
|
|
/*
|
|
* compact_control is used to track pages being migrated and the free pages
|
|
* they are being migrated to during memory compaction. The free_pfn starts
|
|
* at the end of a zone and migrate_pfn begins at the start. Movable pages
|
|
* are moved to the end of a zone during a compaction run and the run
|
|
* completes when free_pfn <= migrate_pfn
|
|
*/
|
|
struct compact_control {
|
|
struct list_head freepages; /* List of free pages to migrate to */
|
|
struct list_head migratepages; /* List of pages being migrated */
|
|
unsigned long nr_freepages; /* Number of isolated free pages */
|
|
unsigned long nr_migratepages; /* Number of pages to migrate */
|
|
unsigned long free_pfn; /* isolate_freepages search base */
|
|
unsigned long migrate_pfn; /* isolate_migratepages search base */
|
|
enum migrate_mode mode; /* Async or sync migration mode */
|
|
bool ignore_skip_hint; /* Scan blocks even if marked skip */
|
|
int order; /* order a direct compactor needs */
|
|
const gfp_t gfp_mask; /* gfp mask of a direct compactor */
|
|
const int alloc_flags; /* alloc flags of a direct compactor */
|
|
const int classzone_idx; /* zone index of a direct compactor */
|
|
struct zone *zone;
|
|
int contended; /* Signal need_sched() or lock
|
|
* contention detected during
|
|
* compaction
|
|
*/
|
|
};
|
|
|
|
unsigned long
|
|
isolate_freepages_range(struct compact_control *cc,
|
|
unsigned long start_pfn, unsigned long end_pfn);
|
|
unsigned long
|
|
isolate_migratepages_range(struct compact_control *cc,
|
|
unsigned long low_pfn, unsigned long end_pfn);
|
|
int find_suitable_fallback(struct free_area *area, unsigned int order,
|
|
int migratetype, bool only_stealable, bool *can_steal);
|
|
|
|
#endif
|
|
|
|
/*
|
|
* This function returns the order of a free page in the buddy system. In
|
|
* general, page_zone(page)->lock must be held by the caller to prevent the
|
|
* page from being allocated in parallel and returning garbage as the order.
|
|
* If a caller does not hold page_zone(page)->lock, it must guarantee that the
|
|
* page cannot be allocated or merged in parallel. Alternatively, it must
|
|
* handle invalid values gracefully, and use page_order_unsafe() below.
|
|
*/
|
|
static inline unsigned long page_order(struct page *page)
|
|
{
|
|
/* PageBuddy() must be checked by the caller */
|
|
return page_private(page);
|
|
}
|
|
|
|
/*
|
|
* Like page_order(), but for callers who cannot afford to hold the zone lock.
|
|
* PageBuddy() should be checked first by the caller to minimize race window,
|
|
* and invalid values must be handled gracefully.
|
|
*
|
|
* READ_ONCE is used so that if the caller assigns the result into a local
|
|
* variable and e.g. tests it for valid range before using, the compiler cannot
|
|
* decide to remove the variable and inline the page_private(page) multiple
|
|
* times, potentially observing different values in the tests and the actual
|
|
* use of the result.
|
|
*/
|
|
#define page_order_unsafe(page) READ_ONCE(page_private(page))
|
|
|
|
static inline bool is_cow_mapping(vm_flags_t flags)
|
|
{
|
|
return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
|
|
}
|
|
|
|
/* mm/util.c */
|
|
void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
struct vm_area_struct *prev, struct rb_node *rb_parent);
|
|
|
|
#ifdef CONFIG_MMU
|
|
extern long populate_vma_page_range(struct vm_area_struct *vma,
|
|
unsigned long start, unsigned long end, int *nonblocking);
|
|
extern void munlock_vma_pages_range(struct vm_area_struct *vma,
|
|
unsigned long start, unsigned long end);
|
|
static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
|
|
{
|
|
munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
|
|
}
|
|
|
|
/*
|
|
* must be called with vma's mmap_sem held for read or write, and page locked.
|
|
*/
|
|
extern void mlock_vma_page(struct page *page);
|
|
extern unsigned int munlock_vma_page(struct page *page);
|
|
|
|
/*
|
|
* Clear the page's PageMlocked(). This can be useful in a situation where
|
|
* we want to unconditionally remove a page from the pagecache -- e.g.,
|
|
* on truncation or freeing.
|
|
*
|
|
* It is legal to call this function for any page, mlocked or not.
|
|
* If called for a page that is still mapped by mlocked vmas, all we do
|
|
* is revert to lazy LRU behaviour -- semantics are not broken.
|
|
*/
|
|
extern void clear_page_mlock(struct page *page);
|
|
|
|
/*
|
|
* mlock_migrate_page - called only from migrate_page_copy() to
|
|
* migrate the Mlocked page flag; update statistics.
|
|
*/
|
|
static inline void mlock_migrate_page(struct page *newpage, struct page *page)
|
|
{
|
|
if (TestClearPageMlocked(page)) {
|
|
unsigned long flags;
|
|
int nr_pages = hpage_nr_pages(page);
|
|
|
|
local_irq_save(flags);
|
|
__mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
|
|
SetPageMlocked(newpage);
|
|
__mod_zone_page_state(page_zone(newpage), NR_MLOCK, nr_pages);
|
|
local_irq_restore(flags);
|
|
}
|
|
}
|
|
|
|
extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
|
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
extern unsigned long vma_address(struct page *page,
|
|
struct vm_area_struct *vma);
|
|
#endif
|
|
#else /* !CONFIG_MMU */
|
|
static inline void clear_page_mlock(struct page *page) { }
|
|
static inline void mlock_vma_page(struct page *page) { }
|
|
static inline void mlock_migrate_page(struct page *new, struct page *old) { }
|
|
|
|
#endif /* !CONFIG_MMU */
|
|
|
|
/*
|
|
* Return the mem_map entry representing the 'offset' subpage within
|
|
* the maximally aligned gigantic page 'base'. Handle any discontiguity
|
|
* in the mem_map at MAX_ORDER_NR_PAGES boundaries.
|
|
*/
|
|
static inline struct page *mem_map_offset(struct page *base, int offset)
|
|
{
|
|
if (unlikely(offset >= MAX_ORDER_NR_PAGES))
|
|
return nth_page(base, offset);
|
|
return base + offset;
|
|
}
|
|
|
|
/*
|
|
* Iterator over all subpages within the maximally aligned gigantic
|
|
* page 'base'. Handle any discontiguity in the mem_map.
|
|
*/
|
|
static inline struct page *mem_map_next(struct page *iter,
|
|
struct page *base, int offset)
|
|
{
|
|
if (unlikely((offset & (MAX_ORDER_NR_PAGES - 1)) == 0)) {
|
|
unsigned long pfn = page_to_pfn(base) + offset;
|
|
if (!pfn_valid(pfn))
|
|
return NULL;
|
|
return pfn_to_page(pfn);
|
|
}
|
|
return iter + 1;
|
|
}
|
|
|
|
/*
|
|
* FLATMEM and DISCONTIGMEM configurations use alloc_bootmem_node,
|
|
* so all functions starting at paging_init should be marked __init
|
|
* in those cases. SPARSEMEM, however, allows for memory hotplug,
|
|
* and alloc_bootmem_node is not used.
|
|
*/
|
|
#ifdef CONFIG_SPARSEMEM
|
|
#define __paginginit __meminit
|
|
#else
|
|
#define __paginginit __init
|
|
#endif
|
|
|
|
/* Memory initialisation debug and verification */
|
|
enum mminit_level {
|
|
MMINIT_WARNING,
|
|
MMINIT_VERIFY,
|
|
MMINIT_TRACE
|
|
};
|
|
|
|
#ifdef CONFIG_DEBUG_MEMORY_INIT
|
|
|
|
extern int mminit_loglevel;
|
|
|
|
#define mminit_dprintk(level, prefix, fmt, arg...) \
|
|
do { \
|
|
if (level < mminit_loglevel) { \
|
|
if (level <= MMINIT_WARNING) \
|
|
printk(KERN_WARNING "mminit::" prefix " " fmt, ##arg); \
|
|
else \
|
|
printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \
|
|
} \
|
|
} while (0)
|
|
|
|
extern void mminit_verify_pageflags_layout(void);
|
|
extern void mminit_verify_zonelist(void);
|
|
#else
|
|
|
|
static inline void mminit_dprintk(enum mminit_level level,
|
|
const char *prefix, const char *fmt, ...)
|
|
{
|
|
}
|
|
|
|
static inline void mminit_verify_pageflags_layout(void)
|
|
{
|
|
}
|
|
|
|
static inline void mminit_verify_zonelist(void)
|
|
{
|
|
}
|
|
#endif /* CONFIG_DEBUG_MEMORY_INIT */
|
|
|
|
/* mminit_validate_memmodel_limits is independent of CONFIG_DEBUG_MEMORY_INIT */
|
|
#if defined(CONFIG_SPARSEMEM)
|
|
extern void mminit_validate_memmodel_limits(unsigned long *start_pfn,
|
|
unsigned long *end_pfn);
|
|
#else
|
|
static inline void mminit_validate_memmodel_limits(unsigned long *start_pfn,
|
|
unsigned long *end_pfn)
|
|
{
|
|
}
|
|
#endif /* CONFIG_SPARSEMEM */
|
|
|
|
#define ZONE_RECLAIM_NOSCAN -2
|
|
#define ZONE_RECLAIM_FULL -1
|
|
#define ZONE_RECLAIM_SOME 0
|
|
#define ZONE_RECLAIM_SUCCESS 1
|
|
|
|
extern int hwpoison_filter(struct page *p);
|
|
|
|
extern u32 hwpoison_filter_dev_major;
|
|
extern u32 hwpoison_filter_dev_minor;
|
|
extern u64 hwpoison_filter_flags_mask;
|
|
extern u64 hwpoison_filter_flags_value;
|
|
extern u64 hwpoison_filter_memcg;
|
|
extern u32 hwpoison_filter_enable;
|
|
|
|
extern unsigned long vm_mmap_pgoff(struct file *, unsigned long,
|
|
unsigned long, unsigned long,
|
|
unsigned long, unsigned long);
|
|
|
|
extern void set_pageblock_order(void);
|
|
unsigned long reclaim_clean_pages_from_list(struct zone *zone,
|
|
struct list_head *page_list);
|
|
/* The ALLOC_WMARK bits are used as an index to zone->watermark */
|
|
#define ALLOC_WMARK_MIN WMARK_MIN
|
|
#define ALLOC_WMARK_LOW WMARK_LOW
|
|
#define ALLOC_WMARK_HIGH WMARK_HIGH
|
|
#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */
|
|
|
|
/* Mask to get the watermark bits */
|
|
#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1)
|
|
|
|
#define ALLOC_HARDER 0x10 /* try to alloc harder */
|
|
#define ALLOC_HIGH 0x20 /* __GFP_HIGH set */
|
|
#define ALLOC_CPUSET 0x40 /* check for correct cpuset */
|
|
#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */
|
|
#define ALLOC_FAIR 0x100 /* fair zone allocation */
|
|
|
|
enum ttu_flags;
|
|
struct tlbflush_unmap_batch;
|
|
|
|
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
|
|
void try_to_unmap_flush(void);
|
|
#else
|
|
static inline void try_to_unmap_flush(void)
|
|
{
|
|
}
|
|
|
|
#endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
|
|
#endif /* __MM_INTERNAL_H */
|