mm: implement new zone specific memblock iterator
Introduce a new iterator for_each_free_mem_pfn_range_in_zone. This iterator will take care of making sure a given memory range provided is in fact contained within a zone. It takes are of all the bounds checking we were doing in deferred_grow_zone, and deferred_init_memmap. In addition it should help to speed up the search a bit by iterating until the end of a range is greater than the start of the zone pfn range, and will exit completely if the start is beyond the end of the zone. Link: http://lkml.kernel.org/r/20190405221225.12227.22573.stgit@localhost.localdomain Signed-off-by: Alexander Duyck <alexander.h.duyck@linux.intel.com> Reviewed-by: Pavel Tatashin <pasha.tatashin@soleen.com> Reviewed-by: Mike Rapoport <rppt@linux.ibm.com> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Dave Jiang <dave.jiang@intel.com> Cc: David S. Miller <davem@davemloft.net> Cc: Ingo Molnar <mingo@kernel.org> Cc: Khalid Aziz <khalid.aziz@oracle.com> Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> Cc: Laurent Dufour <ldufour@linux.vnet.ibm.com> Cc: Matthew Wilcox <willy@infradead.org> Cc: Mel Gorman <mgorman@techsingularity.net> Cc: Michal Hocko <mhocko@suse.com> Cc: Mike Rapoport <rppt@linux.vnet.ibm.com> Cc: Pavel Tatashin <pavel.tatashin@microsoft.com> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: <yi.z.zhang@linux.intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>hifive-unleashed-5.2
parent
56ec43d8b0
commit
837566e7e0
|
@ -240,6 +240,31 @@ void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn,
|
||||||
i >= 0; __next_mem_pfn_range(&i, nid, p_start, p_end, p_nid))
|
i >= 0; __next_mem_pfn_range(&i, nid, p_start, p_end, p_nid))
|
||||||
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
|
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
|
||||||
|
|
||||||
|
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
|
||||||
|
void __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
|
||||||
|
unsigned long *out_spfn,
|
||||||
|
unsigned long *out_epfn);
|
||||||
|
/**
|
||||||
|
* for_each_free_mem_range_in_zone - iterate through zone specific free
|
||||||
|
* memblock areas
|
||||||
|
* @i: u64 used as loop variable
|
||||||
|
* @zone: zone in which all of the memory blocks reside
|
||||||
|
* @p_start: ptr to phys_addr_t for start address of the range, can be %NULL
|
||||||
|
* @p_end: ptr to phys_addr_t for end address of the range, can be %NULL
|
||||||
|
*
|
||||||
|
* Walks over free (memory && !reserved) areas of memblock in a specific
|
||||||
|
* zone. Available once memblock and an empty zone is initialized. The main
|
||||||
|
* assumption is that the zone start, end, and pgdat have been associated.
|
||||||
|
* This way we can use the zone to determine NUMA node, and if a given part
|
||||||
|
* of the memblock is valid for the zone.
|
||||||
|
*/
|
||||||
|
#define for_each_free_mem_pfn_range_in_zone(i, zone, p_start, p_end) \
|
||||||
|
for (i = 0, \
|
||||||
|
__next_mem_pfn_range_in_zone(&i, zone, p_start, p_end); \
|
||||||
|
i != U64_MAX; \
|
||||||
|
__next_mem_pfn_range_in_zone(&i, zone, p_start, p_end))
|
||||||
|
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* for_each_free_mem_range - iterate through free memblock areas
|
* for_each_free_mem_range - iterate through free memblock areas
|
||||||
* @i: u64 used as loop variable
|
* @i: u64 used as loop variable
|
||||||
|
|
|
@ -1255,6 +1255,70 @@ int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
|
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
|
||||||
|
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
|
||||||
|
/**
|
||||||
|
* __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
|
||||||
|
*
|
||||||
|
* @idx: pointer to u64 loop variable
|
||||||
|
* @zone: zone in which all of the memory blocks reside
|
||||||
|
* @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
|
||||||
|
* @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
|
||||||
|
*
|
||||||
|
* This function is meant to be a zone/pfn specific wrapper for the
|
||||||
|
* for_each_mem_range type iterators. Specifically they are used in the
|
||||||
|
* deferred memory init routines and as such we were duplicating much of
|
||||||
|
* this logic throughout the code. So instead of having it in multiple
|
||||||
|
* locations it seemed like it would make more sense to centralize this to
|
||||||
|
* one new iterator that does everything they need.
|
||||||
|
*/
|
||||||
|
void __init_memblock
|
||||||
|
__next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
|
||||||
|
unsigned long *out_spfn, unsigned long *out_epfn)
|
||||||
|
{
|
||||||
|
int zone_nid = zone_to_nid(zone);
|
||||||
|
phys_addr_t spa, epa;
|
||||||
|
int nid;
|
||||||
|
|
||||||
|
__next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
|
||||||
|
&memblock.memory, &memblock.reserved,
|
||||||
|
&spa, &epa, &nid);
|
||||||
|
|
||||||
|
while (*idx != U64_MAX) {
|
||||||
|
unsigned long epfn = PFN_DOWN(epa);
|
||||||
|
unsigned long spfn = PFN_UP(spa);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Verify the end is at least past the start of the zone and
|
||||||
|
* that we have at least one PFN to initialize.
|
||||||
|
*/
|
||||||
|
if (zone->zone_start_pfn < epfn && spfn < epfn) {
|
||||||
|
/* if we went too far just stop searching */
|
||||||
|
if (zone_end_pfn(zone) <= spfn) {
|
||||||
|
*idx = U64_MAX;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (out_spfn)
|
||||||
|
*out_spfn = max(zone->zone_start_pfn, spfn);
|
||||||
|
if (out_epfn)
|
||||||
|
*out_epfn = min(zone_end_pfn(zone), epfn);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
__next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
|
||||||
|
&memblock.memory, &memblock.reserved,
|
||||||
|
&spa, &epa, &nid);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* signal end of iteration */
|
||||||
|
if (out_spfn)
|
||||||
|
*out_spfn = ULONG_MAX;
|
||||||
|
if (out_epfn)
|
||||||
|
*out_epfn = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* memblock_alloc_range_nid - allocate boot memory block
|
* memblock_alloc_range_nid - allocate boot memory block
|
||||||
|
|
|
@ -1631,11 +1631,9 @@ static unsigned long __init deferred_init_pages(struct zone *zone,
|
||||||
static int __init deferred_init_memmap(void *data)
|
static int __init deferred_init_memmap(void *data)
|
||||||
{
|
{
|
||||||
pg_data_t *pgdat = data;
|
pg_data_t *pgdat = data;
|
||||||
int nid = pgdat->node_id;
|
|
||||||
unsigned long start = jiffies;
|
unsigned long start = jiffies;
|
||||||
unsigned long nr_pages = 0;
|
unsigned long nr_pages = 0;
|
||||||
unsigned long spfn, epfn, first_init_pfn, flags;
|
unsigned long spfn, epfn, first_init_pfn, flags;
|
||||||
phys_addr_t spa, epa;
|
|
||||||
int zid;
|
int zid;
|
||||||
struct zone *zone;
|
struct zone *zone;
|
||||||
const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
|
const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
|
||||||
|
@ -1672,14 +1670,12 @@ static int __init deferred_init_memmap(void *data)
|
||||||
* freeing pages we can access pages that are ahead (computing buddy
|
* freeing pages we can access pages that are ahead (computing buddy
|
||||||
* page in __free_one_page()).
|
* page in __free_one_page()).
|
||||||
*/
|
*/
|
||||||
for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
|
for_each_free_mem_pfn_range_in_zone(i, zone, &spfn, &epfn) {
|
||||||
spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
|
spfn = max_t(unsigned long, first_init_pfn, spfn);
|
||||||
epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
|
|
||||||
nr_pages += deferred_init_pages(zone, spfn, epfn);
|
nr_pages += deferred_init_pages(zone, spfn, epfn);
|
||||||
}
|
}
|
||||||
for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
|
for_each_free_mem_pfn_range_in_zone(i, zone, &spfn, &epfn) {
|
||||||
spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
|
spfn = max_t(unsigned long, first_init_pfn, spfn);
|
||||||
epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
|
|
||||||
deferred_free_pages(spfn, epfn);
|
deferred_free_pages(spfn, epfn);
|
||||||
}
|
}
|
||||||
pgdat_resize_unlock(pgdat, &flags);
|
pgdat_resize_unlock(pgdat, &flags);
|
||||||
|
@ -1687,8 +1683,8 @@ static int __init deferred_init_memmap(void *data)
|
||||||
/* Sanity check that the next zone really is unpopulated */
|
/* Sanity check that the next zone really is unpopulated */
|
||||||
WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
|
WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
|
||||||
|
|
||||||
pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
|
pr_info("node %d initialised, %lu pages in %ums\n",
|
||||||
jiffies_to_msecs(jiffies - start));
|
pgdat->node_id, nr_pages, jiffies_to_msecs(jiffies - start));
|
||||||
|
|
||||||
pgdat_init_report_one_done();
|
pgdat_init_report_one_done();
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -1712,13 +1708,11 @@ static int __init deferred_init_memmap(void *data)
|
||||||
static noinline bool __init
|
static noinline bool __init
|
||||||
deferred_grow_zone(struct zone *zone, unsigned int order)
|
deferred_grow_zone(struct zone *zone, unsigned int order)
|
||||||
{
|
{
|
||||||
int nid = zone_to_nid(zone);
|
|
||||||
pg_data_t *pgdat = NODE_DATA(nid);
|
|
||||||
unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
|
unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
|
||||||
|
pg_data_t *pgdat = zone->zone_pgdat;
|
||||||
unsigned long nr_pages = 0;
|
unsigned long nr_pages = 0;
|
||||||
unsigned long first_init_pfn, spfn, epfn, t, flags;
|
unsigned long first_init_pfn, spfn, epfn, t, flags;
|
||||||
unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
|
unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
|
||||||
phys_addr_t spa, epa;
|
|
||||||
u64 i;
|
u64 i;
|
||||||
|
|
||||||
/* Only the last zone may have deferred pages */
|
/* Only the last zone may have deferred pages */
|
||||||
|
@ -1754,9 +1748,8 @@ deferred_grow_zone(struct zone *zone, unsigned int order)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
|
for_each_free_mem_pfn_range_in_zone(i, zone, &spfn, &epfn) {
|
||||||
spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
|
spfn = max_t(unsigned long, first_init_pfn, spfn);
|
||||||
epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
|
|
||||||
|
|
||||||
while (spfn < epfn && nr_pages < nr_pages_needed) {
|
while (spfn < epfn && nr_pages < nr_pages_needed) {
|
||||||
t = ALIGN(spfn + PAGES_PER_SECTION, PAGES_PER_SECTION);
|
t = ALIGN(spfn + PAGES_PER_SECTION, PAGES_PER_SECTION);
|
||||||
|
@ -1770,9 +1763,9 @@ deferred_grow_zone(struct zone *zone, unsigned int order)
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
|
for_each_free_mem_pfn_range_in_zone(i, zone, &spfn, &epfn) {
|
||||||
spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
|
spfn = max_t(unsigned long, first_init_pfn, spfn);
|
||||||
epfn = min_t(unsigned long, first_deferred_pfn, PFN_DOWN(epa));
|
epfn = min_t(unsigned long, first_deferred_pfn, epfn);
|
||||||
deferred_free_pages(spfn, epfn);
|
deferred_free_pages(spfn, epfn);
|
||||||
|
|
||||||
if (first_deferred_pfn == epfn)
|
if (first_deferred_pfn == epfn)
|
||||||
|
|
Loading…
Reference in New Issue