include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 02:04:11 -06:00
|
|
|
#include <linux/gfp.h>
|
2009-04-10 12:33:10 -06:00
|
|
|
#include <linux/initrd.h>
|
2009-03-04 02:46:40 -07:00
|
|
|
#include <linux/ioport.h>
|
2009-03-03 04:15:06 -07:00
|
|
|
#include <linux/swap.h>
|
2010-08-25 14:39:17 -06:00
|
|
|
#include <linux/memblock.h>
|
2011-11-01 07:58:22 -06:00
|
|
|
#include <linux/bootmem.h> /* for max_low_pfn */
|
2009-03-04 02:46:40 -07:00
|
|
|
|
2009-03-03 04:15:06 -07:00
|
|
|
#include <asm/cacheflush.h>
|
2009-03-05 05:55:05 -07:00
|
|
|
#include <asm/e820.h>
|
2009-03-05 05:55:08 -07:00
|
|
|
#include <asm/init.h>
|
2009-03-03 04:15:06 -07:00
|
|
|
#include <asm/page.h>
|
2009-03-04 02:46:40 -07:00
|
|
|
#include <asm/page_types.h>
|
2009-03-03 04:15:06 -07:00
|
|
|
#include <asm/sections.h>
|
2009-05-06 06:06:47 -06:00
|
|
|
#include <asm/setup.h>
|
2009-03-05 05:55:05 -07:00
|
|
|
#include <asm/tlbflush.h>
|
2009-04-28 07:00:50 -06:00
|
|
|
#include <asm/tlb.h>
|
2009-07-01 08:24:23 -06:00
|
|
|
#include <asm/proto.h>
|
2011-11-01 07:58:22 -06:00
|
|
|
#include <asm/dma.h> /* for MAX_DMA_PFN */
|
2012-12-21 00:44:31 -07:00
|
|
|
#include <asm/microcode.h>
|
2009-04-28 07:00:50 -06:00
|
|
|
|
2012-11-16 20:39:01 -07:00
|
|
|
#include "mm_internal.h"
|
|
|
|
|
2012-11-16 20:39:07 -07:00
|
|
|
static unsigned long __initdata pgt_buf_start;
|
|
|
|
static unsigned long __initdata pgt_buf_end;
|
|
|
|
static unsigned long __initdata pgt_buf_top;
|
2009-03-05 05:55:05 -07:00
|
|
|
|
2012-11-16 20:39:02 -07:00
|
|
|
static unsigned long min_pfn_mapped;
|
|
|
|
|
2013-01-24 13:19:42 -07:00
|
|
|
static bool __initdata can_use_brk_pgt = true;
|
|
|
|
|
2012-11-16 20:39:05 -07:00
|
|
|
/*
|
|
|
|
* Pages returned are already directly mapped.
|
|
|
|
*
|
|
|
|
* Changing that is likely to break Xen, see commit:
|
|
|
|
*
|
|
|
|
* 279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve
|
|
|
|
*
|
|
|
|
* for detailed information.
|
|
|
|
*/
|
2012-11-16 20:39:04 -07:00
|
|
|
__ref void *alloc_low_pages(unsigned int num)
|
2012-11-16 20:39:01 -07:00
|
|
|
{
|
|
|
|
unsigned long pfn;
|
2012-11-16 20:39:04 -07:00
|
|
|
int i;
|
2012-11-16 20:39:01 -07:00
|
|
|
|
|
|
|
if (after_bootmem) {
|
2012-11-16 20:39:04 -07:00
|
|
|
unsigned int order;
|
2012-11-16 20:39:01 -07:00
|
|
|
|
2012-11-16 20:39:04 -07:00
|
|
|
order = get_order((unsigned long)num << PAGE_SHIFT);
|
|
|
|
return (void *)__get_free_pages(GFP_ATOMIC | __GFP_NOTRACK |
|
|
|
|
__GFP_ZERO, order);
|
2012-11-16 20:39:01 -07:00
|
|
|
}
|
|
|
|
|
2013-01-24 13:19:42 -07:00
|
|
|
if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
|
2012-11-16 20:39:01 -07:00
|
|
|
unsigned long ret;
|
|
|
|
if (min_pfn_mapped >= max_pfn_mapped)
|
2013-11-12 16:08:28 -07:00
|
|
|
panic("alloc_low_pages: ran out of memory");
|
2012-11-16 20:39:01 -07:00
|
|
|
ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT,
|
|
|
|
max_pfn_mapped << PAGE_SHIFT,
|
2012-11-16 20:39:04 -07:00
|
|
|
PAGE_SIZE * num , PAGE_SIZE);
|
2012-11-16 20:39:01 -07:00
|
|
|
if (!ret)
|
2013-11-12 16:08:28 -07:00
|
|
|
panic("alloc_low_pages: can not alloc memory");
|
2012-11-16 20:39:04 -07:00
|
|
|
memblock_reserve(ret, PAGE_SIZE * num);
|
2012-11-16 20:39:01 -07:00
|
|
|
pfn = ret >> PAGE_SHIFT;
|
2012-11-16 20:39:04 -07:00
|
|
|
} else {
|
|
|
|
pfn = pgt_buf_end;
|
|
|
|
pgt_buf_end += num;
|
2013-01-24 13:19:42 -07:00
|
|
|
printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n",
|
|
|
|
pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1);
|
2012-11-16 20:39:04 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < num; i++) {
|
|
|
|
void *adr;
|
|
|
|
|
|
|
|
adr = __va((pfn + i) << PAGE_SHIFT);
|
|
|
|
clear_page(adr);
|
|
|
|
}
|
2012-11-16 20:39:01 -07:00
|
|
|
|
2012-11-16 20:39:04 -07:00
|
|
|
return __va(pfn << PAGE_SHIFT);
|
2012-11-16 20:39:01 -07:00
|
|
|
}
|
|
|
|
|
x86/mm: Fix boot crash with DEBUG_PAGE_ALLOC=y and more than 512G RAM
Dave Hansen reported that systems between 500G and 600G RAM
crash early if DEBUG_PAGEALLOC is selected.
> [ 0.000000] init_memory_mapping: [mem 0x00000000-0x000fffff]
> [ 0.000000] [mem 0x00000000-0x000fffff] page 4k
> [ 0.000000] BRK [0x02086000, 0x02086fff] PGTABLE
> [ 0.000000] BRK [0x02087000, 0x02087fff] PGTABLE
> [ 0.000000] BRK [0x02088000, 0x02088fff] PGTABLE
> [ 0.000000] init_memory_mapping: [mem 0xe80ee00000-0xe80effffff]
> [ 0.000000] [mem 0xe80ee00000-0xe80effffff] page 4k
> [ 0.000000] BRK [0x02089000, 0x02089fff] PGTABLE
> [ 0.000000] BRK [0x0208a000, 0x0208afff] PGTABLE
> [ 0.000000] Kernel panic - not syncing: alloc_low_page: ran out of memory
It turns out that we missed increasing needed pages in BRK to
mapping initial 2M and [0,1M) when we switched to use the #PF
handler to set memory mappings:
> commit 8170e6bed465b4b0c7687f93e9948aca4358a33b
> Author: H. Peter Anvin <hpa@zytor.com>
> Date: Thu Jan 24 12:19:52 2013 -0800
>
> x86, 64bit: Use a #PF handler to materialize early mappings on demand
Before that, we had the maping from [0,512M) in head_64.S, and we
can spare two pages [0-1M). After that change, we can not reuse
pages anymore.
When we have more than 512M ram, we need an extra page for pgd page
with [512G, 1024g).
Increase pages in BRK for page table to solve the boot crash.
Reported-by: Dave Hansen <dave.hansen@intel.com>
Bisected-by: Dave Hansen <dave.hansen@intel.com>
Tested-by: Dave Hansen <dave.hansen@intel.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Cc: <stable@vger.kernel.org> # v3.9 and later
Link: http://lkml.kernel.org/r/1376351004-4015-1-git-send-email-yinghai@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2013-08-12 17:43:24 -06:00
|
|
|
/* need 3 4k for initial PMD_SIZE, 3 4k for 0-ISA_END_ADDRESS */
|
|
|
|
#define INIT_PGT_BUF_SIZE (6 * PAGE_SIZE)
|
2012-11-16 20:38:58 -07:00
|
|
|
RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
|
|
|
|
void __init early_alloc_pgt_buf(void)
|
|
|
|
{
|
|
|
|
unsigned long tables = INIT_PGT_BUF_SIZE;
|
|
|
|
phys_addr_t base;
|
|
|
|
|
|
|
|
base = __pa(extend_brk(tables, PAGE_SIZE));
|
|
|
|
|
|
|
|
pgt_buf_start = base >> PAGE_SHIFT;
|
|
|
|
pgt_buf_end = pgt_buf_start;
|
|
|
|
pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
|
|
|
|
}
|
|
|
|
|
2009-03-05 05:55:05 -07:00
|
|
|
int after_bootmem;
|
|
|
|
|
|
|
|
int direct_gbpages
|
|
|
|
#ifdef CONFIG_DIRECT_GBPAGES
|
|
|
|
= 1
|
|
|
|
#endif
|
|
|
|
;
|
|
|
|
|
2012-11-16 20:39:08 -07:00
|
|
|
static void __init init_gbpages(void)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
if (direct_gbpages && cpu_has_gbpages)
|
|
|
|
printk(KERN_INFO "Using GB pages for direct mapping\n");
|
|
|
|
else
|
|
|
|
direct_gbpages = 0;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2012-10-24 13:24:44 -06:00
|
|
|
struct map_range {
|
|
|
|
unsigned long start;
|
|
|
|
unsigned long end;
|
|
|
|
unsigned page_size_mask;
|
|
|
|
};
|
|
|
|
|
2012-11-16 20:38:38 -07:00
|
|
|
static int page_size_mask;
|
2009-03-05 05:55:05 -07:00
|
|
|
|
2012-11-16 20:38:41 -07:00
|
|
|
static void __init probe_page_size_mask(void)
|
2012-11-16 20:38:38 -07:00
|
|
|
{
|
2012-11-16 20:39:08 -07:00
|
|
|
init_gbpages();
|
|
|
|
|
2012-11-16 20:38:38 -07:00
|
|
|
#if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK)
|
|
|
|
/*
|
|
|
|
* For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
|
|
|
|
* This will simplify cpa(), which otherwise needs to support splitting
|
|
|
|
* large pages into small in interrupt context, etc.
|
|
|
|
*/
|
|
|
|
if (direct_gbpages)
|
|
|
|
page_size_mask |= 1 << PG_LEVEL_1G;
|
|
|
|
if (cpu_has_pse)
|
|
|
|
page_size_mask |= 1 << PG_LEVEL_2M;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* Enable PSE if available */
|
|
|
|
if (cpu_has_pse)
|
|
|
|
set_in_cr4(X86_CR4_PSE);
|
|
|
|
|
|
|
|
/* Enable PGE if available */
|
|
|
|
if (cpu_has_pge) {
|
|
|
|
set_in_cr4(X86_CR4_PGE);
|
|
|
|
__supported_pte_mask |= _PAGE_GLOBAL;
|
|
|
|
}
|
|
|
|
}
|
2011-04-14 08:49:41 -06:00
|
|
|
|
2009-03-05 05:55:05 -07:00
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
#define NR_RANGE_MR 3
|
|
|
|
#else /* CONFIG_X86_64 */
|
|
|
|
#define NR_RANGE_MR 5
|
|
|
|
#endif
|
|
|
|
|
2009-03-12 06:40:06 -06:00
|
|
|
static int __meminit save_mr(struct map_range *mr, int nr_range,
|
|
|
|
unsigned long start_pfn, unsigned long end_pfn,
|
|
|
|
unsigned long page_size_mask)
|
2009-03-05 05:55:05 -07:00
|
|
|
{
|
|
|
|
if (start_pfn < end_pfn) {
|
|
|
|
if (nr_range >= NR_RANGE_MR)
|
|
|
|
panic("run out of range for init_memory_mapping\n");
|
|
|
|
mr[nr_range].start = start_pfn<<PAGE_SHIFT;
|
|
|
|
mr[nr_range].end = end_pfn<<PAGE_SHIFT;
|
|
|
|
mr[nr_range].page_size_mask = page_size_mask;
|
|
|
|
nr_range++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return nr_range;
|
|
|
|
}
|
|
|
|
|
2012-11-16 20:38:55 -07:00
|
|
|
/*
|
|
|
|
* adjust the page_size_mask for small range to go with
|
|
|
|
* big page size instead small one if nearby are ram too.
|
|
|
|
*/
|
|
|
|
static void __init_refok adjust_range_page_size_mask(struct map_range *mr,
|
|
|
|
int nr_range)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_range; i++) {
|
|
|
|
if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
|
|
|
|
!(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
|
|
|
|
unsigned long start = round_down(mr[i].start, PMD_SIZE);
|
|
|
|
unsigned long end = round_up(mr[i].end, PMD_SIZE);
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
if ((end >> PAGE_SHIFT) > max_low_pfn)
|
|
|
|
continue;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (memblock_is_region_memory(start, end - start))
|
|
|
|
mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
|
|
|
|
}
|
|
|
|
if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
|
|
|
|
!(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
|
|
|
|
unsigned long start = round_down(mr[i].start, PUD_SIZE);
|
|
|
|
unsigned long end = round_up(mr[i].end, PUD_SIZE);
|
|
|
|
|
|
|
|
if (memblock_is_region_memory(start, end - start))
|
|
|
|
mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-11-16 20:38:39 -07:00
|
|
|
static int __meminit split_mem_range(struct map_range *mr, int nr_range,
|
|
|
|
unsigned long start,
|
|
|
|
unsigned long end)
|
2009-03-05 05:55:05 -07:00
|
|
|
{
|
2012-11-16 20:39:15 -07:00
|
|
|
unsigned long start_pfn, end_pfn, limit_pfn;
|
2012-11-16 20:39:14 -07:00
|
|
|
unsigned long pfn;
|
2012-11-16 20:38:39 -07:00
|
|
|
int i;
|
2009-03-05 05:55:05 -07:00
|
|
|
|
2012-11-16 20:39:15 -07:00
|
|
|
limit_pfn = PFN_DOWN(end);
|
|
|
|
|
2009-03-05 05:55:05 -07:00
|
|
|
/* head if not big page alignment ? */
|
2012-11-16 20:39:14 -07:00
|
|
|
pfn = start_pfn = PFN_DOWN(start);
|
2009-03-05 05:55:05 -07:00
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
/*
|
|
|
|
* Don't use a large page for the first 2/4MB of memory
|
|
|
|
* because there are often fixed size MTRRs in there
|
|
|
|
* and overlapping MTRRs into large pages can cause
|
|
|
|
* slowdowns.
|
|
|
|
*/
|
2012-11-16 20:39:14 -07:00
|
|
|
if (pfn == 0)
|
2012-11-16 20:39:13 -07:00
|
|
|
end_pfn = PFN_DOWN(PMD_SIZE);
|
2009-03-05 05:55:05 -07:00
|
|
|
else
|
2012-11-16 20:39:14 -07:00
|
|
|
end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
|
2009-03-05 05:55:05 -07:00
|
|
|
#else /* CONFIG_X86_64 */
|
2012-11-16 20:39:14 -07:00
|
|
|
end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
|
2009-03-05 05:55:05 -07:00
|
|
|
#endif
|
2012-11-16 20:39:15 -07:00
|
|
|
if (end_pfn > limit_pfn)
|
|
|
|
end_pfn = limit_pfn;
|
2009-03-05 05:55:05 -07:00
|
|
|
if (start_pfn < end_pfn) {
|
|
|
|
nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
|
2012-11-16 20:39:14 -07:00
|
|
|
pfn = end_pfn;
|
2009-03-05 05:55:05 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/* big page (2M) range */
|
2012-11-16 20:39:14 -07:00
|
|
|
start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
|
2009-03-05 05:55:05 -07:00
|
|
|
#ifdef CONFIG_X86_32
|
2012-11-16 20:39:15 -07:00
|
|
|
end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
|
2009-03-05 05:55:05 -07:00
|
|
|
#else /* CONFIG_X86_64 */
|
2012-11-16 20:39:14 -07:00
|
|
|
end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
|
2012-11-16 20:39:15 -07:00
|
|
|
if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE)))
|
|
|
|
end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
|
2009-03-05 05:55:05 -07:00
|
|
|
#endif
|
|
|
|
|
|
|
|
if (start_pfn < end_pfn) {
|
|
|
|
nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
|
|
|
|
page_size_mask & (1<<PG_LEVEL_2M));
|
2012-11-16 20:39:14 -07:00
|
|
|
pfn = end_pfn;
|
2009-03-05 05:55:05 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
/* big page (1G) range */
|
2012-11-16 20:39:14 -07:00
|
|
|
start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
|
2012-11-16 20:39:15 -07:00
|
|
|
end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE));
|
2009-03-05 05:55:05 -07:00
|
|
|
if (start_pfn < end_pfn) {
|
|
|
|
nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
|
|
|
|
page_size_mask &
|
|
|
|
((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
|
2012-11-16 20:39:14 -07:00
|
|
|
pfn = end_pfn;
|
2009-03-05 05:55:05 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/* tail is not big page (1G) alignment */
|
2012-11-16 20:39:14 -07:00
|
|
|
start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
|
2012-11-16 20:39:15 -07:00
|
|
|
end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
|
2009-03-05 05:55:05 -07:00
|
|
|
if (start_pfn < end_pfn) {
|
|
|
|
nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
|
|
|
|
page_size_mask & (1<<PG_LEVEL_2M));
|
2012-11-16 20:39:14 -07:00
|
|
|
pfn = end_pfn;
|
2009-03-05 05:55:05 -07:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* tail is not big page (2M) alignment */
|
2012-11-16 20:39:14 -07:00
|
|
|
start_pfn = pfn;
|
2012-11-16 20:39:15 -07:00
|
|
|
end_pfn = limit_pfn;
|
2009-03-05 05:55:05 -07:00
|
|
|
nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
|
|
|
|
|
x86: Fix adjust_range_size_mask calling position
Commit
8d57470d x86, mm: setup page table in top-down
causes a kernel panic while setting mem=2G.
[mem 0x00000000-0x000fffff] page 4k
[mem 0x7fe00000-0x7fffffff] page 1G
[mem 0x7c000000-0x7fdfffff] page 1G
[mem 0x00100000-0x001fffff] page 4k
[mem 0x00200000-0x7bffffff] page 2M
for last entry is not what we want, we should have
[mem 0x00200000-0x3fffffff] page 2M
[mem 0x40000000-0x7bffffff] page 1G
Actually we merge the continuous ranges with same page size too early.
in this case, before merging we have
[mem 0x00200000-0x3fffffff] page 2M
[mem 0x40000000-0x7bffffff] page 2M
after merging them, will get
[mem 0x00200000-0x7bffffff] page 2M
even we can use 1G page to map
[mem 0x40000000-0x7bffffff]
that will cause problem, because we already map
[mem 0x7fe00000-0x7fffffff] page 1G
[mem 0x7c000000-0x7fdfffff] page 1G
with 1G page, aka [0x40000000-0x7fffffff] is mapped with 1G page already.
During phys_pud_init() for [0x40000000-0x7bffffff], it will not
reuse existing that pud page, and allocate new one then try to use
2M page to map it instead, as page_size_mask does not include
PG_LEVEL_1G. At end will have [7c000000-0x7fffffff] not mapped, loop
in phys_pmd_init stop mapping at 0x7bffffff.
That is right behavoir, it maps exact range with exact page size that
we ask, and we should explicitly call it to map [7c000000-0x7fffffff]
before or after mapping 0x40000000-0x7bffffff.
Anyway we need to make sure ranges' page_size_mask correct and consistent
after split_mem_range for each range.
Fix that by calling adjust_range_size_mask before merging range
with same page size.
-v2: update change log.
-v3: add more explanation why [7c000000-0x7fffffff] is not mapped, and
it causes panic.
Bisected-by: "Xie, ChanglongX" <changlongx.xie@intel.com>
Bisected-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
Reported-and-tested-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Link: http://lkml.kernel.org/r/1370015587-20835-1-git-send-email-yinghai@kernel.org
Cc: <stable@vger.kernel.org> v3.9
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2013-05-31 09:53:07 -06:00
|
|
|
if (!after_bootmem)
|
|
|
|
adjust_range_page_size_mask(mr, nr_range);
|
|
|
|
|
2009-03-05 05:55:05 -07:00
|
|
|
/* try to merge same page size and continuous */
|
|
|
|
for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
|
|
|
|
unsigned long old_start;
|
|
|
|
if (mr[i].end != mr[i+1].start ||
|
|
|
|
mr[i].page_size_mask != mr[i+1].page_size_mask)
|
|
|
|
continue;
|
|
|
|
/* move it */
|
|
|
|
old_start = mr[i].start;
|
|
|
|
memmove(&mr[i], &mr[i+1],
|
|
|
|
(nr_range - 1 - i) * sizeof(struct map_range));
|
|
|
|
mr[i--].start = old_start;
|
|
|
|
nr_range--;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < nr_range; i++)
|
2012-05-29 16:06:29 -06:00
|
|
|
printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n",
|
|
|
|
mr[i].start, mr[i].end - 1,
|
2009-03-05 05:55:05 -07:00
|
|
|
(mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
|
|
|
|
(mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
|
|
|
|
|
2012-11-16 20:38:39 -07:00
|
|
|
return nr_range;
|
|
|
|
}
|
|
|
|
|
2013-01-24 13:20:05 -07:00
|
|
|
struct range pfn_mapped[E820_X_MAX];
|
|
|
|
int nr_pfn_mapped;
|
2012-11-16 20:38:52 -07:00
|
|
|
|
|
|
|
static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
|
|
|
|
{
|
|
|
|
nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX,
|
|
|
|
nr_pfn_mapped, start_pfn, end_pfn);
|
|
|
|
nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX);
|
|
|
|
|
|
|
|
max_pfn_mapped = max(max_pfn_mapped, end_pfn);
|
|
|
|
|
|
|
|
if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
|
|
|
|
max_low_pfn_mapped = max(max_low_pfn_mapped,
|
|
|
|
min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
|
|
|
|
}
|
|
|
|
|
|
|
|
bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_pfn_mapped; i++)
|
|
|
|
if ((start_pfn >= pfn_mapped[i].start) &&
|
|
|
|
(end_pfn <= pfn_mapped[i].end))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2012-11-16 20:38:39 -07:00
|
|
|
/*
|
|
|
|
* Setup the direct mapping of the physical memory at PAGE_OFFSET.
|
|
|
|
* This runs before bootmem is initialized and gets pages directly from
|
|
|
|
* the physical memory. To access them they are temporarily mapped.
|
|
|
|
*/
|
|
|
|
unsigned long __init_refok init_memory_mapping(unsigned long start,
|
|
|
|
unsigned long end)
|
|
|
|
{
|
|
|
|
struct map_range mr[NR_RANGE_MR];
|
|
|
|
unsigned long ret = 0;
|
|
|
|
int nr_range, i;
|
|
|
|
|
|
|
|
pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n",
|
|
|
|
start, end - 1);
|
|
|
|
|
|
|
|
memset(mr, 0, sizeof(mr));
|
|
|
|
nr_range = split_mem_range(mr, 0, start, end);
|
|
|
|
|
2009-03-05 05:55:05 -07:00
|
|
|
for (i = 0; i < nr_range; i++)
|
|
|
|
ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
|
|
|
|
mr[i].page_size_mask);
|
|
|
|
|
2012-11-16 20:38:52 -07:00
|
|
|
add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);
|
|
|
|
|
2012-11-16 20:38:44 -07:00
|
|
|
return ret >> PAGE_SHIFT;
|
|
|
|
}
|
|
|
|
|
2012-11-16 20:38:52 -07:00
|
|
|
/*
|
2013-05-09 09:57:42 -06:00
|
|
|
* We need to iterate through the E820 memory map and create direct mappings
|
|
|
|
* for only E820_RAM and E820_KERN_RESERVED regions. We cannot simply
|
|
|
|
* create direct mappings for all pfns from [0 to max_low_pfn) and
|
|
|
|
* [4GB to max_pfn) because of possible memory holes in high addresses
|
|
|
|
* that cannot be marked as UC by fixed/variable range MTRRs.
|
|
|
|
* Depending on the alignment of E820 ranges, this may possibly result
|
|
|
|
* in using smaller size (i.e. 4K instead of 2M or 1G) page tables.
|
|
|
|
*
|
|
|
|
* init_mem_mapping() calls init_range_memory_mapping() with big range.
|
|
|
|
* That range would have hole in the middle or ends, and only ram parts
|
|
|
|
* will be mapped in init_range_memory_mapping().
|
2012-11-16 20:38:52 -07:00
|
|
|
*/
|
2012-11-16 20:38:58 -07:00
|
|
|
static unsigned long __init init_range_memory_mapping(
|
2012-11-16 20:39:18 -07:00
|
|
|
unsigned long r_start,
|
|
|
|
unsigned long r_end)
|
2012-11-16 20:38:52 -07:00
|
|
|
{
|
|
|
|
unsigned long start_pfn, end_pfn;
|
2012-11-16 20:38:58 -07:00
|
|
|
unsigned long mapped_ram_size = 0;
|
2012-11-16 20:38:52 -07:00
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
|
2012-11-16 20:39:18 -07:00
|
|
|
u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end);
|
|
|
|
u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end);
|
|
|
|
if (start >= end)
|
2012-11-16 20:38:52 -07:00
|
|
|
continue;
|
|
|
|
|
2013-01-24 13:19:42 -07:00
|
|
|
/*
|
|
|
|
* if it is overlapping with brk pgt, we need to
|
|
|
|
* alloc pgt buf from memblock instead.
|
|
|
|
*/
|
|
|
|
can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >=
|
|
|
|
min(end, (u64)pgt_buf_top<<PAGE_SHIFT);
|
2012-11-16 20:38:57 -07:00
|
|
|
init_memory_mapping(start, end);
|
2012-11-16 20:38:58 -07:00
|
|
|
mapped_ram_size += end - start;
|
2013-01-24 13:19:42 -07:00
|
|
|
can_use_brk_pgt = true;
|
2012-11-16 20:38:52 -07:00
|
|
|
}
|
2012-11-16 20:38:58 -07:00
|
|
|
|
|
|
|
return mapped_ram_size;
|
2012-11-16 20:38:52 -07:00
|
|
|
}
|
|
|
|
|
2013-09-06 20:07:09 -06:00
|
|
|
static unsigned long __init get_new_step_size(unsigned long step_size)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Explain why we shift by 5 and why we don't have to worry about
|
|
|
|
* 'step_size << 5' overflowing:
|
|
|
|
*
|
|
|
|
* initial mapped size is PMD_SIZE (2M).
|
|
|
|
* We can not set step_size to be PUD_SIZE (1G) yet.
|
|
|
|
* In worse case, when we cross the 1G boundary, and
|
|
|
|
* PG_LEVEL_2M is not set, we will need 1+1+512 pages (2M + 8k)
|
|
|
|
* to map 1G range with PTE. Use 5 as shift for now.
|
|
|
|
*
|
|
|
|
* Don't need to worry about overflow, on 32bit, when step_size
|
|
|
|
* is 0, round_down() returns 0 for start, and that turns it
|
|
|
|
* into 0x100000000ULL.
|
|
|
|
*/
|
|
|
|
return step_size << 5;
|
|
|
|
}
|
|
|
|
|
2013-11-12 16:08:02 -07:00
|
|
|
/**
|
|
|
|
* memory_map_top_down - Map [map_start, map_end) top down
|
|
|
|
* @map_start: start address of the target memory range
|
|
|
|
* @map_end: end address of the target memory range
|
|
|
|
*
|
|
|
|
* This function will setup direct mapping for memory range
|
|
|
|
* [map_start, map_end) in top-down. That said, the page tables
|
|
|
|
* will be allocated at the end of the memory, and we map the
|
|
|
|
* memory in top-down.
|
|
|
|
*/
|
|
|
|
static void __init memory_map_top_down(unsigned long map_start,
|
|
|
|
unsigned long map_end)
|
2012-11-16 20:38:44 -07:00
|
|
|
{
|
2013-11-12 16:08:02 -07:00
|
|
|
unsigned long real_end, start, last_start;
|
2012-11-16 20:38:58 -07:00
|
|
|
unsigned long step_size;
|
|
|
|
unsigned long addr;
|
|
|
|
unsigned long mapped_ram_size = 0;
|
|
|
|
unsigned long new_mapped_ram_size;
|
2012-11-16 20:38:45 -07:00
|
|
|
|
2013-03-06 21:18:21 -07:00
|
|
|
/* xen has big range in reserved near end of ram, skip it at first.*/
|
2013-11-12 16:08:02 -07:00
|
|
|
addr = memblock_find_in_range(map_start, map_end, PMD_SIZE, PMD_SIZE);
|
2012-11-16 20:38:58 -07:00
|
|
|
real_end = addr + PMD_SIZE;
|
|
|
|
|
|
|
|
/* step_size need to be small so pgt_buf from BRK could cover it */
|
|
|
|
step_size = PMD_SIZE;
|
|
|
|
max_pfn_mapped = 0; /* will get exact value next */
|
|
|
|
min_pfn_mapped = real_end >> PAGE_SHIFT;
|
|
|
|
last_start = start = real_end;
|
2013-05-09 09:57:42 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We start from the top (end of memory) and go to the bottom.
|
|
|
|
* The memblock_find_in_range() gets us a block of RAM from the
|
|
|
|
* end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
|
|
|
|
* for page table.
|
|
|
|
*/
|
2013-11-12 16:08:02 -07:00
|
|
|
while (last_start > map_start) {
|
2012-11-16 20:38:58 -07:00
|
|
|
if (last_start > step_size) {
|
|
|
|
start = round_down(last_start - 1, step_size);
|
2013-11-12 16:08:02 -07:00
|
|
|
if (start < map_start)
|
|
|
|
start = map_start;
|
2012-11-16 20:38:58 -07:00
|
|
|
} else
|
2013-11-12 16:08:02 -07:00
|
|
|
start = map_start;
|
2012-11-16 20:38:58 -07:00
|
|
|
new_mapped_ram_size = init_range_memory_mapping(start,
|
|
|
|
last_start);
|
|
|
|
last_start = start;
|
|
|
|
min_pfn_mapped = last_start >> PAGE_SHIFT;
|
|
|
|
/* only increase step_size after big range get mapped */
|
|
|
|
if (new_mapped_ram_size > mapped_ram_size)
|
2013-09-06 20:07:09 -06:00
|
|
|
step_size = get_new_step_size(step_size);
|
2012-11-16 20:38:58 -07:00
|
|
|
mapped_ram_size += new_mapped_ram_size;
|
|
|
|
}
|
|
|
|
|
2013-11-12 16:08:02 -07:00
|
|
|
if (real_end < map_end)
|
|
|
|
init_range_memory_mapping(real_end, map_end);
|
|
|
|
}
|
|
|
|
|
x86/mem-hotplug: support initialize page tables in bottom-up
The Linux kernel cannot migrate pages used by the kernel. As a result,
kernel pages cannot be hot-removed. So we cannot allocate hotpluggable
memory for the kernel.
In a memory hotplug system, any numa node the kernel resides in should be
unhotpluggable. And for a modern server, each node could have at least
16GB memory. So memory around the kernel image is highly likely
unhotpluggable.
ACPI SRAT (System Resource Affinity Table) contains the memory hotplug
info. But before SRAT is parsed, memblock has already started to allocate
memory for the kernel. So we need to prevent memblock from doing this.
So direct memory mapping page tables setup is the case.
init_mem_mapping() is called before SRAT is parsed. To prevent page
tables being allocated within hotpluggable memory, we will use bottom-up
direction to allocate page tables from the end of kernel image to the
higher memory.
Note:
As for allocating page tables in lower memory, TJ said:
: This is an optional behavior which is triggered by a very specific kernel
: boot param, which I suspect is gonna need to stick around to support
: memory hotplug in the current setup unless we add another layer of address
: translation to support memory hotplug.
As for page tables may occupy too much lower memory if using 4K mapping
(CONFIG_DEBUG_PAGEALLOC and CONFIG_KMEMCHECK both disable using >4k
pages), TJ said:
: But as I said in the same paragraph, parsing SRAT earlier doesn't solve
: the problem in itself either. Ignoring the option if 4k mapping is
: required and memory consumption would be prohibitive should work, no?
: Something like that would be necessary if we're gonna worry about cases
: like this no matter how we implement it, but, frankly, I'm not sure this
: is something worth worrying about.
Signed-off-by: Tang Chen <tangchen@cn.fujitsu.com>
Signed-off-by: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Acked-by: Tejun Heo <tj@kernel.org>
Acked-by: Toshi Kani <toshi.kani@hp.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Wanpeng Li <liwanp@linux.vnet.ibm.com>
Cc: Thomas Renninger <trenn@suse.de>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Jiang Liu <jiang.liu@huawei.com>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Lai Jiangshan <laijs@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Taku Izumi <izumi.taku@jp.fujitsu.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-12 16:08:05 -07:00
|
|
|
/**
|
|
|
|
* memory_map_bottom_up - Map [map_start, map_end) bottom up
|
|
|
|
* @map_start: start address of the target memory range
|
|
|
|
* @map_end: end address of the target memory range
|
|
|
|
*
|
|
|
|
* This function will setup direct mapping for memory range
|
|
|
|
* [map_start, map_end) in bottom-up. Since we have limited the
|
|
|
|
* bottom-up allocation above the kernel, the page tables will
|
|
|
|
* be allocated just above the kernel and we map the memory
|
|
|
|
* in [map_start, map_end) in bottom-up.
|
|
|
|
*/
|
|
|
|
static void __init memory_map_bottom_up(unsigned long map_start,
|
|
|
|
unsigned long map_end)
|
|
|
|
{
|
|
|
|
unsigned long next, new_mapped_ram_size, start;
|
|
|
|
unsigned long mapped_ram_size = 0;
|
|
|
|
/* step_size need to be small so pgt_buf from BRK could cover it */
|
|
|
|
unsigned long step_size = PMD_SIZE;
|
|
|
|
|
|
|
|
start = map_start;
|
|
|
|
min_pfn_mapped = start >> PAGE_SHIFT;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We start from the bottom (@map_start) and go to the top (@map_end).
|
|
|
|
* The memblock_find_in_range() gets us a block of RAM from the
|
|
|
|
* end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
|
|
|
|
* for page table.
|
|
|
|
*/
|
|
|
|
while (start < map_end) {
|
|
|
|
if (map_end - start > step_size) {
|
|
|
|
next = round_up(start + 1, step_size);
|
|
|
|
if (next > map_end)
|
|
|
|
next = map_end;
|
|
|
|
} else
|
|
|
|
next = map_end;
|
|
|
|
|
|
|
|
new_mapped_ram_size = init_range_memory_mapping(start, next);
|
|
|
|
start = next;
|
|
|
|
|
|
|
|
if (new_mapped_ram_size > mapped_ram_size)
|
|
|
|
step_size = get_new_step_size(step_size);
|
|
|
|
mapped_ram_size += new_mapped_ram_size;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-11-12 16:08:02 -07:00
|
|
|
void __init init_mem_mapping(void)
|
|
|
|
{
|
|
|
|
unsigned long end;
|
|
|
|
|
|
|
|
probe_page_size_mask();
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
end = max_pfn << PAGE_SHIFT;
|
|
|
|
#else
|
|
|
|
end = max_low_pfn << PAGE_SHIFT;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* the ISA range is always mapped regardless of memory holes */
|
|
|
|
init_memory_mapping(0, ISA_END_ADDRESS);
|
|
|
|
|
x86/mem-hotplug: support initialize page tables in bottom-up
The Linux kernel cannot migrate pages used by the kernel. As a result,
kernel pages cannot be hot-removed. So we cannot allocate hotpluggable
memory for the kernel.
In a memory hotplug system, any numa node the kernel resides in should be
unhotpluggable. And for a modern server, each node could have at least
16GB memory. So memory around the kernel image is highly likely
unhotpluggable.
ACPI SRAT (System Resource Affinity Table) contains the memory hotplug
info. But before SRAT is parsed, memblock has already started to allocate
memory for the kernel. So we need to prevent memblock from doing this.
So direct memory mapping page tables setup is the case.
init_mem_mapping() is called before SRAT is parsed. To prevent page
tables being allocated within hotpluggable memory, we will use bottom-up
direction to allocate page tables from the end of kernel image to the
higher memory.
Note:
As for allocating page tables in lower memory, TJ said:
: This is an optional behavior which is triggered by a very specific kernel
: boot param, which I suspect is gonna need to stick around to support
: memory hotplug in the current setup unless we add another layer of address
: translation to support memory hotplug.
As for page tables may occupy too much lower memory if using 4K mapping
(CONFIG_DEBUG_PAGEALLOC and CONFIG_KMEMCHECK both disable using >4k
pages), TJ said:
: But as I said in the same paragraph, parsing SRAT earlier doesn't solve
: the problem in itself either. Ignoring the option if 4k mapping is
: required and memory consumption would be prohibitive should work, no?
: Something like that would be necessary if we're gonna worry about cases
: like this no matter how we implement it, but, frankly, I'm not sure this
: is something worth worrying about.
Signed-off-by: Tang Chen <tangchen@cn.fujitsu.com>
Signed-off-by: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Acked-by: Tejun Heo <tj@kernel.org>
Acked-by: Toshi Kani <toshi.kani@hp.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Wanpeng Li <liwanp@linux.vnet.ibm.com>
Cc: Thomas Renninger <trenn@suse.de>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Jiang Liu <jiang.liu@huawei.com>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Lai Jiangshan <laijs@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Taku Izumi <izumi.taku@jp.fujitsu.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-12 16:08:05 -07:00
|
|
|
/*
|
|
|
|
* If the allocation is in bottom-up direction, we setup direct mapping
|
|
|
|
* in bottom-up, otherwise we setup direct mapping in top-down.
|
|
|
|
*/
|
|
|
|
if (memblock_bottom_up()) {
|
|
|
|
unsigned long kernel_end = __pa_symbol(_end);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* we need two separate calls here. This is because we want to
|
|
|
|
* allocate page tables above the kernel. So we first map
|
|
|
|
* [kernel_end, end) to make memory above the kernel be mapped
|
|
|
|
* as soon as possible. And then use page tables allocated above
|
|
|
|
* the kernel to map [ISA_END_ADDRESS, kernel_end).
|
|
|
|
*/
|
|
|
|
memory_map_bottom_up(kernel_end, end);
|
|
|
|
memory_map_bottom_up(ISA_END_ADDRESS, kernel_end);
|
|
|
|
} else {
|
|
|
|
memory_map_top_down(ISA_END_ADDRESS, end);
|
|
|
|
}
|
2012-11-16 20:38:58 -07:00
|
|
|
|
2012-11-16 20:38:57 -07:00
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
if (max_pfn > max_low_pfn) {
|
|
|
|
/* can we preseve max_low_pfn ?*/
|
|
|
|
max_low_pfn = max_pfn;
|
|
|
|
}
|
2012-11-16 20:39:06 -07:00
|
|
|
#else
|
|
|
|
early_ioremap_page_table_range_init();
|
x86, 64bit: Use a #PF handler to materialize early mappings on demand
Linear mode (CR0.PG = 0) is mutually exclusive with 64-bit mode; all
64-bit code has to use page tables. This makes it awkward before we
have first set up properly all-covering page tables to access objects
that are outside the static kernel range.
So far we have dealt with that simply by mapping a fixed amount of
low memory, but that fails in at least two upcoming use cases:
1. We will support load and run kernel, struct boot_params, ramdisk,
command line, etc. above the 4 GiB mark.
2. need to access ramdisk early to get microcode to update that as
early possible.
We could use early_iomap to access them too, but it will make code to
messy and hard to be unified with 32 bit.
Hence, set up a #PF table and use a fixed number of buffers to set up
page tables on demand. If the buffers fill up then we simply flush
them and start over. These buffers are all in __initdata, so it does
not increase RAM usage at runtime.
Thus, with the help of the #PF handler, we can set the final kernel
mapping from blank, and switch to init_level4_pgt later.
During the switchover in head_64.S, before #PF handler is available,
we use three pages to handle kernel crossing 1G, 512G boundaries with
sharing page by playing games with page aliasing: the same page is
mapped twice in the higher-level tables with appropriate wraparound.
The kernel region itself will be properly mapped; other mappings may
be spurious.
early_make_pgtable is using kernel high mapping address to access pages
to set page table.
-v4: Add phys_base offset to make kexec happy, and add
init_mapping_kernel() - Yinghai
-v5: fix compiling with xen, and add back ident level3 and level2 for xen
also move back init_level4_pgt from BSS to DATA again.
because we have to clear it anyway. - Yinghai
-v6: switch to init_level4_pgt in init_mem_mapping. - Yinghai
-v7: remove not needed clear_page for init_level4_page
it is with fill 512,8,0 already in head_64.S - Yinghai
-v8: we need to keep that handler alive until init_mem_mapping and don't
let early_trap_init to trash that early #PF handler.
So split early_trap_pf_init out and move it down. - Yinghai
-v9: switchover only cover kernel space instead of 1G so could avoid
touch possible mem holes. - Yinghai
-v11: change far jmp back to far return to initial_code, that is needed
to fix failure that is reported by Konrad on AMD systems. - Yinghai
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Link: http://lkml.kernel.org/r/1359058816-7615-12-git-send-email-yinghai@kernel.org
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2013-01-24 13:19:52 -07:00
|
|
|
#endif
|
|
|
|
|
2012-11-16 20:39:06 -07:00
|
|
|
load_cr3(swapper_pg_dir);
|
|
|
|
__flush_tlb_all();
|
|
|
|
|
2012-11-16 20:38:44 -07:00
|
|
|
early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
|
2012-11-16 20:38:41 -07:00
|
|
|
}
|
2009-03-03 04:15:06 -07:00
|
|
|
|
2009-03-04 02:46:40 -07:00
|
|
|
/*
|
|
|
|
* devmem_is_allowed() checks to see if /dev/mem access to a certain address
|
|
|
|
* is valid. The argument is a physical page number.
|
|
|
|
*
|
|
|
|
*
|
|
|
|
* On x86, access has to be given to the first megabyte of ram because that area
|
|
|
|
* contains bios code and data regions used by X and dosemu and similar apps.
|
|
|
|
* Access has to be given to non-kernel-ram areas as well, these contain the PCI
|
|
|
|
* mmio resources as well as potential bios/acpi data regions.
|
|
|
|
*/
|
|
|
|
int devmem_is_allowed(unsigned long pagenr)
|
|
|
|
{
|
2012-08-28 21:21:43 -06:00
|
|
|
if (pagenr < 256)
|
2009-03-04 02:46:40 -07:00
|
|
|
return 1;
|
|
|
|
if (iomem_is_exclusive(pagenr << PAGE_SHIFT))
|
|
|
|
return 0;
|
|
|
|
if (!page_is_ram(pagenr))
|
|
|
|
return 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-03-03 04:15:06 -07:00
|
|
|
void free_init_pages(char *what, unsigned long begin, unsigned long end)
|
|
|
|
{
|
x86: Make sure free_init_pages() frees pages on page boundary
When CONFIG_NO_BOOTMEM=y, it could use memory more effiently, or
in a more compact fashion.
Example:
Allocated new RAMDISK: 00ec2000 - 0248ce57
Move RAMDISK from 000000002ea04000 - 000000002ffcee56 to 00ec2000 - 0248ce56
The new RAMDISK's end is not page aligned.
Last page could be shared with other users.
When free_init_pages are called for initrd or .init, the page
could be freed and we could corrupt other data.
code segment in free_init_pages():
| for (; addr < end; addr += PAGE_SIZE) {
| ClearPageReserved(virt_to_page(addr));
| init_page_count(virt_to_page(addr));
| memset((void *)(addr & ~(PAGE_SIZE-1)),
| POISON_FREE_INITMEM, PAGE_SIZE);
| free_page(addr);
| totalram_pages++;
| }
last half page could be used as one whole free page.
So page align the boundaries.
-v2: make the original initramdisk to be aligned, according to
Johannes, otherwise we have the chance to lose one page.
we still need to keep initrd_end not aligned, otherwise it could
confuse decompressor.
-v3: change to WARN_ON instead, suggested by Johannes.
-v4: use PAGE_ALIGN, suggested by Johannes.
We may fix that macro name later to PAGE_ALIGN_UP, and PAGE_ALIGN_DOWN
Add comments about assuming ramdisk start is aligned
in relocate_initrd(), change to re get ramdisk_image instead of save it
to make diff smaller. Add warning for wrong range, suggested by Johannes.
-v6: remove one WARN()
We need to align beginning in free_init_pages()
do not copy more than ramdisk_size, noticed by Johannes
Reported-by: Stanislaw Gruszka <sgruszka@redhat.com>
Tested-by: Stanislaw Gruszka <sgruszka@redhat.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: David Miller <davem@davemloft.net>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
LKML-Reference: <1269830604-26214-3-git-send-email-yinghai@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-03-28 20:42:55 -06:00
|
|
|
unsigned long begin_aligned, end_aligned;
|
2009-03-03 04:15:06 -07:00
|
|
|
|
x86: Make sure free_init_pages() frees pages on page boundary
When CONFIG_NO_BOOTMEM=y, it could use memory more effiently, or
in a more compact fashion.
Example:
Allocated new RAMDISK: 00ec2000 - 0248ce57
Move RAMDISK from 000000002ea04000 - 000000002ffcee56 to 00ec2000 - 0248ce56
The new RAMDISK's end is not page aligned.
Last page could be shared with other users.
When free_init_pages are called for initrd or .init, the page
could be freed and we could corrupt other data.
code segment in free_init_pages():
| for (; addr < end; addr += PAGE_SIZE) {
| ClearPageReserved(virt_to_page(addr));
| init_page_count(virt_to_page(addr));
| memset((void *)(addr & ~(PAGE_SIZE-1)),
| POISON_FREE_INITMEM, PAGE_SIZE);
| free_page(addr);
| totalram_pages++;
| }
last half page could be used as one whole free page.
So page align the boundaries.
-v2: make the original initramdisk to be aligned, according to
Johannes, otherwise we have the chance to lose one page.
we still need to keep initrd_end not aligned, otherwise it could
confuse decompressor.
-v3: change to WARN_ON instead, suggested by Johannes.
-v4: use PAGE_ALIGN, suggested by Johannes.
We may fix that macro name later to PAGE_ALIGN_UP, and PAGE_ALIGN_DOWN
Add comments about assuming ramdisk start is aligned
in relocate_initrd(), change to re get ramdisk_image instead of save it
to make diff smaller. Add warning for wrong range, suggested by Johannes.
-v6: remove one WARN()
We need to align beginning in free_init_pages()
do not copy more than ramdisk_size, noticed by Johannes
Reported-by: Stanislaw Gruszka <sgruszka@redhat.com>
Tested-by: Stanislaw Gruszka <sgruszka@redhat.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: David Miller <davem@davemloft.net>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
LKML-Reference: <1269830604-26214-3-git-send-email-yinghai@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-03-28 20:42:55 -06:00
|
|
|
/* Make sure boundaries are page aligned */
|
|
|
|
begin_aligned = PAGE_ALIGN(begin);
|
|
|
|
end_aligned = end & PAGE_MASK;
|
|
|
|
|
|
|
|
if (WARN_ON(begin_aligned != begin || end_aligned != end)) {
|
|
|
|
begin = begin_aligned;
|
|
|
|
end = end_aligned;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (begin >= end)
|
2009-03-03 04:15:06 -07:00
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If debugging page accesses then do not free this memory but
|
|
|
|
* mark them not present - any buggy init-section access will
|
|
|
|
* create a kernel page fault:
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
2012-05-29 16:06:29 -06:00
|
|
|
printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n",
|
|
|
|
begin, end - 1);
|
2009-03-03 04:15:06 -07:00
|
|
|
set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
|
|
|
|
#else
|
|
|
|
/*
|
|
|
|
* We just marked the kernel text read only above, now that
|
|
|
|
* we are going to free part of that, we need to make that
|
2010-11-16 14:31:26 -07:00
|
|
|
* writeable and non-executable first.
|
2009-03-03 04:15:06 -07:00
|
|
|
*/
|
2010-11-16 14:31:26 -07:00
|
|
|
set_memory_nx(begin, (end - begin) >> PAGE_SHIFT);
|
2009-03-03 04:15:06 -07:00
|
|
|
set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);
|
|
|
|
|
2013-07-03 16:02:58 -06:00
|
|
|
free_reserved_area((void *)begin, (void *)end, POISON_FREE_INITMEM, what);
|
2009-03-03 04:15:06 -07:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
void free_initmem(void)
|
|
|
|
{
|
2013-07-03 16:02:58 -06:00
|
|
|
free_init_pages("unused kernel",
|
2009-03-03 04:15:06 -07:00
|
|
|
(unsigned long)(&__init_begin),
|
|
|
|
(unsigned long)(&__init_end));
|
|
|
|
}
|
2009-03-04 02:13:40 -07:00
|
|
|
|
|
|
|
#ifdef CONFIG_BLK_DEV_INITRD
|
2012-06-18 04:30:20 -06:00
|
|
|
void __init free_initrd_mem(unsigned long start, unsigned long end)
|
2009-03-04 02:13:40 -07:00
|
|
|
{
|
2012-12-21 00:44:31 -07:00
|
|
|
#ifdef CONFIG_MICROCODE_EARLY
|
|
|
|
/*
|
|
|
|
* Remember, initrd memory may contain microcode or other useful things.
|
|
|
|
* Before we lose initrd mem, we need to find a place to hold them
|
|
|
|
* now that normal virtual memory is enabled.
|
|
|
|
*/
|
|
|
|
save_microcode_in_initrd();
|
|
|
|
#endif
|
|
|
|
|
x86: Make sure free_init_pages() frees pages on page boundary
When CONFIG_NO_BOOTMEM=y, it could use memory more effiently, or
in a more compact fashion.
Example:
Allocated new RAMDISK: 00ec2000 - 0248ce57
Move RAMDISK from 000000002ea04000 - 000000002ffcee56 to 00ec2000 - 0248ce56
The new RAMDISK's end is not page aligned.
Last page could be shared with other users.
When free_init_pages are called for initrd or .init, the page
could be freed and we could corrupt other data.
code segment in free_init_pages():
| for (; addr < end; addr += PAGE_SIZE) {
| ClearPageReserved(virt_to_page(addr));
| init_page_count(virt_to_page(addr));
| memset((void *)(addr & ~(PAGE_SIZE-1)),
| POISON_FREE_INITMEM, PAGE_SIZE);
| free_page(addr);
| totalram_pages++;
| }
last half page could be used as one whole free page.
So page align the boundaries.
-v2: make the original initramdisk to be aligned, according to
Johannes, otherwise we have the chance to lose one page.
we still need to keep initrd_end not aligned, otherwise it could
confuse decompressor.
-v3: change to WARN_ON instead, suggested by Johannes.
-v4: use PAGE_ALIGN, suggested by Johannes.
We may fix that macro name later to PAGE_ALIGN_UP, and PAGE_ALIGN_DOWN
Add comments about assuming ramdisk start is aligned
in relocate_initrd(), change to re get ramdisk_image instead of save it
to make diff smaller. Add warning for wrong range, suggested by Johannes.
-v6: remove one WARN()
We need to align beginning in free_init_pages()
do not copy more than ramdisk_size, noticed by Johannes
Reported-by: Stanislaw Gruszka <sgruszka@redhat.com>
Tested-by: Stanislaw Gruszka <sgruszka@redhat.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Cc: David Miller <davem@davemloft.net>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
LKML-Reference: <1269830604-26214-3-git-send-email-yinghai@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-03-28 20:42:55 -06:00
|
|
|
/*
|
|
|
|
* end could be not aligned, and We can not align that,
|
|
|
|
* decompresser could be confused by aligned initrd_end
|
|
|
|
* We already reserve the end partial page before in
|
|
|
|
* - i386_start_kernel()
|
|
|
|
* - x86_64_start_kernel()
|
|
|
|
* - relocate_initrd()
|
|
|
|
* So here We can do PAGE_ALIGN() safely to get partial page to be freed
|
|
|
|
*/
|
2013-07-03 16:02:58 -06:00
|
|
|
free_init_pages("initrd", start, PAGE_ALIGN(end));
|
2009-03-04 02:13:40 -07:00
|
|
|
}
|
|
|
|
#endif
|
2011-11-01 07:58:22 -06:00
|
|
|
|
|
|
|
void __init zone_sizes_init(void)
|
|
|
|
{
|
|
|
|
unsigned long max_zone_pfns[MAX_NR_ZONES];
|
|
|
|
|
|
|
|
memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
|
|
|
|
|
|
|
|
#ifdef CONFIG_ZONE_DMA
|
|
|
|
max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_ZONE_DMA32
|
|
|
|
max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
|
|
|
|
#endif
|
|
|
|
max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
|
|
|
|
#ifdef CONFIG_HIGHMEM
|
|
|
|
max_zone_pfns[ZONE_HIGHMEM] = max_pfn;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
free_area_init_nodes(max_zone_pfns);
|
|
|
|
}
|
|
|
|
|