redonkable
/
alistair23-linux
1
0
Fork 0
Code Releases Activity

mm, page_alloc: pass preferred nid instead of zonelist to allocator 

The main allocator function __alloc_pages_nodemask() takes a zonelist
pointer as one of its parameters.  All of its callers directly or
indirectly obtain the zonelist via node_zonelist() using a preferred
node id and gfp_mask.  We can make the code a bit simpler by doing the
zonelist lookup in __alloc_pages_nodemask(), passing it a preferred node
id instead (gfp_mask is already another parameter).

There are some code size benefits thanks to removal of inlined
node_zonelist():

  bloat-o-meter add/remove: 2/2 grow/shrink: 4/36 up/down: 399/-1351 (-952)

This will also make things simpler if we proceed with converting cpusets
to zonelists.

Link: http://lkml.kernel.org/r/20170517081140.30654-4-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Reviewed-by: Christoph Lameter <cl@linux.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Cc: Dimitri Sivanich <sivanich@sgi.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Anshuman Khandual <khandual@linux.vnet.ibm.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Li Zefan <lizefan@huawei.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
hifive-unleashed-5.1
Vlastimil Babka 2017-07-06 15:40:03 -07:00 committed by Linus Torvalds
parent 45816682b2
commit 04ec6264f2
6 changed files with 43 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
include/linux/gfp.h
View File

@ -432,14 +432,13 @@ static inline void arch_alloc_page(struct page *page, int order) { }
#endif #endif
struct page * struct page *
__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
struct zonelist *zonelist, nodemask_t *nodemask); nodemask_t *nodemask);
static inline struct page * static inline struct page *
__alloc_pages(gfp_t gfp_mask, unsigned int order, __alloc_pages(gfp_t gfp_mask, unsigned int order, int preferred_nid)
struct zonelist *zonelist)
{ {
return __alloc_pages_nodemask(gfp_mask, order, zonelist, NULL); return __alloc_pages_nodemask(gfp_mask, order, preferred_nid, NULL);
} }
/* /*
@ -452,7 +451,7 @@ __alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order)
VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES);
VM_WARN_ON(!node_online(nid)); VM_WARN_ON(!node_online(nid));
return __alloc_pages(gfp_mask, order, node_zonelist(nid, gfp_mask)); return __alloc_pages(gfp_mask, order, nid);
} }
/* /*

6
include/linux/mempolicy.h
View File

@ -146,7 +146,7 @@ extern void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
enum mpol_rebind_step step); enum mpol_rebind_step step);
extern void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new); extern void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new);
extern struct zonelist *huge_zonelist(struct vm_area_struct *vma, extern int huge_node(struct vm_area_struct *vma,
unsigned long addr, gfp_t gfp_flags, unsigned long addr, gfp_t gfp_flags,
struct mempolicy **mpol, nodemask_t **nodemask); struct mempolicy **mpol, nodemask_t **nodemask);
extern bool init_nodemask_of_mempolicy(nodemask_t *mask); extern bool init_nodemask_of_mempolicy(nodemask_t *mask);
@ -269,13 +269,13 @@ static inline void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
{ {
} }
static inline struct zonelist *huge_zonelist(struct vm_area_struct *vma, static inline int huge_node(struct vm_area_struct *vma,
unsigned long addr, gfp_t gfp_flags, unsigned long addr, gfp_t gfp_flags,
struct mempolicy **mpol, nodemask_t **nodemask) struct mempolicy **mpol, nodemask_t **nodemask)
{ {
*mpol = NULL; *mpol = NULL;
*nodemask = NULL; *nodemask = NULL;
return node_zonelist(0, gfp_flags); return 0;
} }
static inline bool init_nodemask_of_mempolicy(nodemask_t *m) static inline bool init_nodemask_of_mempolicy(nodemask_t *m)

15
mm/hugetlb.c
View File

@ -920,6 +920,8 @@ static struct page *dequeue_huge_page_vma(struct hstate *h,
struct page *page = NULL; struct page *page = NULL;
struct mempolicy *mpol; struct mempolicy *mpol;
nodemask_t *nodemask; nodemask_t *nodemask;
gfp_t gfp_mask;
int nid;
struct zonelist *zonelist; struct zonelist *zonelist;
struct zone *zone; struct zone *zone;
struct zoneref *z; struct zoneref *z;
@ -940,12 +942,13 @@ static struct page *dequeue_huge_page_vma(struct hstate *h,
retry_cpuset: retry_cpuset:
cpuset_mems_cookie = read_mems_allowed_begin(); cpuset_mems_cookie = read_mems_allowed_begin();
zonelist = huge_zonelist(vma, address, gfp_mask = htlb_alloc_mask(h);
htlb_alloc_mask(h), &mpol, &nodemask); nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
zonelist = node_zonelist(nid, gfp_mask);
for_each_zone_zonelist_nodemask(zone, z, zonelist, for_each_zone_zonelist_nodemask(zone, z, zonelist,
MAX_NR_ZONES - 1, nodemask) { MAX_NR_ZONES - 1, nodemask) {
if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) { if (cpuset_zone_allowed(zone, gfp_mask)) {
page = dequeue_huge_page_node(h, zone_to_nid(zone)); page = dequeue_huge_page_node(h, zone_to_nid(zone));
if (page) { if (page) {
if (avoid_reserve) if (avoid_reserve)
@ -1558,13 +1561,13 @@ static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
do { do {
struct page *page; struct page *page;
struct mempolicy *mpol; struct mempolicy *mpol;
struct zonelist *zl; int nid;
nodemask_t *nodemask; nodemask_t *nodemask;
cpuset_mems_cookie = read_mems_allowed_begin(); cpuset_mems_cookie = read_mems_allowed_begin();
zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask); nid = huge_node(vma, addr, gfp, &mpol, &nodemask);
mpol_cond_put(mpol); mpol_cond_put(mpol);
page = __alloc_pages_nodemask(gfp, order, zl, nodemask); page = __alloc_pages_nodemask(gfp, order, nid, nodemask);
if (page) if (page)
return page; return page;
} while (read_mems_allowed_retry(cpuset_mems_cookie)); } while (read_mems_allowed_retry(cpuset_mems_cookie));

6
mm/memory_hotplug.c
View File

@ -1459,11 +1459,9 @@ static struct page *new_node_page(struct page *page, unsigned long private,
gfp_mask |= __GFP_HIGHMEM; gfp_mask |= __GFP_HIGHMEM;
if (!nodes_empty(nmask)) if (!nodes_empty(nmask))
new_page = __alloc_pages_nodemask(gfp_mask, 0, new_page = __alloc_pages_nodemask(gfp_mask, 0, nid, &nmask);
node_zonelist(nid, gfp_mask), &nmask);
if (!new_page) if (!new_page)
new_page = __alloc_pages(gfp_mask, 0, new_page = __alloc_pages(gfp_mask, 0, nid);
node_zonelist(nid, gfp_mask));
return new_page; return new_page;
} }

41
mm/mempolicy.c
View File

@ -1669,9 +1669,9 @@ static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
return NULL; return NULL;
} }
/* Return a zonelist indicated by gfp for node representing a mempolicy */ /* Return the node id preferred by the given mempolicy, or the given id */
static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy, static int policy_node(gfp_t gfp, struct mempolicy *policy,
int nd) int nd)
{ {
if (policy->mode == MPOL_PREFERRED && !(policy->flags & MPOL_F_LOCAL)) if (policy->mode == MPOL_PREFERRED && !(policy->flags & MPOL_F_LOCAL))
nd = policy->v.preferred_node; nd = policy->v.preferred_node;
@ -1684,7 +1684,7 @@ static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy,
WARN_ON_ONCE(policy->mode == MPOL_BIND && (gfp & __GFP_THISNODE)); WARN_ON_ONCE(policy->mode == MPOL_BIND && (gfp & __GFP_THISNODE));
} }
return node_zonelist(nd, gfp); return nd;
} }
/* Do dynamic interleaving for a process */ /* Do dynamic interleaving for a process */
@ -1791,38 +1791,37 @@ static inline unsigned interleave_nid(struct mempolicy *pol,
#ifdef CONFIG_HUGETLBFS #ifdef CONFIG_HUGETLBFS
/* /*
* huge_zonelist(@vma, @addr, @gfp_flags, @mpol) * huge_node(@vma, @addr, @gfp_flags, @mpol)
* @vma: virtual memory area whose policy is sought * @vma: virtual memory area whose policy is sought
* @addr: address in @vma for shared policy lookup and interleave policy * @addr: address in @vma for shared policy lookup and interleave policy
* @gfp_flags: for requested zone * @gfp_flags: for requested zone
* @mpol: pointer to mempolicy pointer for reference counted mempolicy * @mpol: pointer to mempolicy pointer for reference counted mempolicy
* @nodemask: pointer to nodemask pointer for MPOL_BIND nodemask * @nodemask: pointer to nodemask pointer for MPOL_BIND nodemask
* *
* Returns a zonelist suitable for a huge page allocation and a pointer * Returns a nid suitable for a huge page allocation and a pointer
* to the struct mempolicy for conditional unref after allocation. * to the struct mempolicy for conditional unref after allocation.
* If the effective policy is 'BIND, returns a pointer to the mempolicy's * If the effective policy is 'BIND, returns a pointer to the mempolicy's
* @nodemask for filtering the zonelist. * @nodemask for filtering the zonelist.
* *
* Must be protected by read_mems_allowed_begin() * Must be protected by read_mems_allowed_begin()
*/ */
struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr, int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags,
gfp_t gfp_flags, struct mempolicy **mpol, struct mempolicy **mpol, nodemask_t **nodemask)
nodemask_t **nodemask)
{ {
struct zonelist *zl; int nid;
*mpol = get_vma_policy(vma, addr); *mpol = get_vma_policy(vma, addr);
*nodemask = NULL; /* assume !MPOL_BIND */ *nodemask = NULL; /* assume !MPOL_BIND */
if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) { if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
zl = node_zonelist(interleave_nid(*mpol, vma, addr, nid = interleave_nid(*mpol, vma, addr,
huge_page_shift(hstate_vma(vma))), gfp_flags); huge_page_shift(hstate_vma(vma)));
} else { } else {
zl = policy_zonelist(gfp_flags, *mpol, numa_node_id()); nid = policy_node(gfp_flags, *mpol, numa_node_id());
if ((*mpol)->mode == MPOL_BIND) if ((*mpol)->mode == MPOL_BIND)
*nodemask = &(*mpol)->v.nodes; *nodemask = &(*mpol)->v.nodes;
} }
return zl; return nid;
} }
/* /*
@ -1924,12 +1923,10 @@ out:
static struct page *alloc_page_interleave(gfp_t gfp, unsigned order, static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
unsigned nid) unsigned nid)
{ {
struct zonelist *zl;
struct page *page; struct page *page;
zl = node_zonelist(nid, gfp); page = __alloc_pages(gfp, order, nid);
page = __alloc_pages(gfp, order, zl); if (page && page_to_nid(page) == nid)
if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
inc_zone_page_state(page, NUMA_INTERLEAVE_HIT); inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
return page; return page;
} }
@ -1963,8 +1960,8 @@ alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
{ {
struct mempolicy *pol; struct mempolicy *pol;
struct page *page; struct page *page;
int preferred_nid;
unsigned int cpuset_mems_cookie; unsigned int cpuset_mems_cookie;
struct zonelist *zl;
nodemask_t *nmask; nodemask_t *nmask;
retry_cpuset: retry_cpuset:
@ -2007,8 +2004,8 @@ retry_cpuset:
} }
nmask = policy_nodemask(gfp, pol); nmask = policy_nodemask(gfp, pol);
zl = policy_zonelist(gfp, pol, node); preferred_nid = policy_node(gfp, pol, node);
page = __alloc_pages_nodemask(gfp, order, zl, nmask); page = __alloc_pages_nodemask(gfp, order, preferred_nid, nmask);
mpol_cond_put(pol); mpol_cond_put(pol);
out: out:
if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
@ -2055,7 +2052,7 @@ retry_cpuset:
page = alloc_page_interleave(gfp, order, interleave_nodes(pol)); page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
else else
page = __alloc_pages_nodemask(gfp, order, page = __alloc_pages_nodemask(gfp, order,
policy_zonelist(gfp, pol, numa_node_id()), policy_node(gfp, pol, numa_node_id()),
policy_nodemask(gfp, pol)); policy_nodemask(gfp, pol));
if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))

10
mm/page_alloc.c
View File

@ -3980,12 +3980,12 @@ got_pg:
} }
static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order, static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
struct zonelist *zonelist, nodemask_t *nodemask, int preferred_nid, nodemask_t *nodemask,
struct alloc_context *ac, gfp_t *alloc_mask, struct alloc_context *ac, gfp_t *alloc_mask,
unsigned int *alloc_flags) unsigned int *alloc_flags)
{ {
ac->high_zoneidx = gfp_zone(gfp_mask); ac->high_zoneidx = gfp_zone(gfp_mask);
ac->zonelist = zonelist; ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
ac->nodemask = nodemask; ac->nodemask = nodemask;
ac->migratetype = gfpflags_to_migratetype(gfp_mask); ac->migratetype = gfpflags_to_migratetype(gfp_mask);
@ -4030,8 +4030,8 @@ static inline void finalise_ac(gfp_t gfp_mask,
* This is the 'heart' of the zoned buddy allocator. * This is the 'heart' of the zoned buddy allocator.
*/ */
struct page * struct page *
__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
struct zonelist *zonelist, nodemask_t *nodemask) nodemask_t *nodemask)
{ {
struct page *page; struct page *page;
unsigned int alloc_flags = ALLOC_WMARK_LOW; unsigned int alloc_flags = ALLOC_WMARK_LOW;
@ -4039,7 +4039,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
struct alloc_context ac = { }; struct alloc_context ac = { };
gfp_mask &= gfp_allowed_mask; gfp_mask &= gfp_allowed_mask;
if (!prepare_alloc_pages(gfp_mask, order, zonelist, nodemask, &ac, &alloc_mask, &alloc_flags)) if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
return NULL; return NULL;
finalise_ac(gfp_mask, order, &ac); finalise_ac(gfp_mask, order, &ac);