x86/mm: Replace compile-time checks for 5-level paging with runtime-time checks
This patch converts the of CONFIG_X86_5LEVEL check to runtime checks for p4d folding. Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Arjan van de Ven <arjan@linux.intel.com> Cc: Borislav Petkov <bp@suse.de> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: David Woodhouse <dwmw2@infradead.org> Cc: Josh Poimboeuf <jpoimboe@redhat.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-mm@kvack.org Link: http://lkml.kernel.org/r/20180214182542.69302-9-kirill.shutemov@linux.intel.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
98219dda2a
commit
91f606a8fa
|
@ -217,29 +217,26 @@ static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd)
|
|||
|
||||
static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
|
||||
{
|
||||
#if defined(CONFIG_PAGE_TABLE_ISOLATION) && !defined(CONFIG_X86_5LEVEL)
|
||||
p4dp->pgd = pti_set_user_pgd(&p4dp->pgd, p4d.pgd);
|
||||
#else
|
||||
*p4dp = p4d;
|
||||
#endif
|
||||
pgd_t pgd;
|
||||
|
||||
if (pgtable_l5_enabled || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) {
|
||||
*p4dp = p4d;
|
||||
return;
|
||||
}
|
||||
|
||||
pgd = native_make_pgd(p4d_val(p4d));
|
||||
pgd = pti_set_user_pgd((pgd_t *)p4dp, pgd);
|
||||
*p4dp = native_make_p4d(pgd_val(pgd));
|
||||
}
|
||||
|
||||
static inline void native_p4d_clear(p4d_t *p4d)
|
||||
{
|
||||
#ifdef CONFIG_X86_5LEVEL
|
||||
native_set_p4d(p4d, native_make_p4d(0));
|
||||
#else
|
||||
native_set_p4d(p4d, (p4d_t) { .pgd = native_make_pgd(0)});
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
|
||||
{
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
*pgdp = pti_set_user_pgd(pgdp, pgd);
|
||||
#else
|
||||
*pgdp = pgd;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void native_pgd_clear(pgd_t *pgd)
|
||||
|
|
|
@ -348,9 +348,7 @@ static inline bool kasan_page_table(struct seq_file *m, struct pg_state *st,
|
|||
void *pt)
|
||||
{
|
||||
if (__pa(pt) == __pa(kasan_zero_pmd) ||
|
||||
#ifdef CONFIG_X86_5LEVEL
|
||||
__pa(pt) == __pa(kasan_zero_p4d) ||
|
||||
#endif
|
||||
(pgtable_l5_enabled && __pa(pt) == __pa(kasan_zero_p4d)) ||
|
||||
__pa(pt) == __pa(kasan_zero_pud)) {
|
||||
pgprotval_t prot = pte_flags(kasan_zero_pte[0]);
|
||||
note_page(m, st, __pgprot(prot), 5);
|
||||
|
|
|
@ -439,7 +439,7 @@ static noinline int vmalloc_fault(unsigned long address)
|
|||
if (pgd_none(*pgd_ref))
|
||||
return -1;
|
||||
|
||||
if (CONFIG_PGTABLE_LEVELS > 4) {
|
||||
if (pgtable_l5_enabled) {
|
||||
if (pgd_none(*pgd)) {
|
||||
set_pgd(pgd, *pgd_ref);
|
||||
arch_flush_lazy_mmu_mode();
|
||||
|
@ -454,7 +454,7 @@ static noinline int vmalloc_fault(unsigned long address)
|
|||
if (p4d_none(*p4d_ref))
|
||||
return -1;
|
||||
|
||||
if (p4d_none(*p4d) && CONFIG_PGTABLE_LEVELS == 4) {
|
||||
if (p4d_none(*p4d) && !pgtable_l5_enabled) {
|
||||
set_p4d(p4d, *p4d_ref);
|
||||
arch_flush_lazy_mmu_mode();
|
||||
} else {
|
||||
|
|
|
@ -120,7 +120,7 @@ int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
|
|||
result = ident_p4d_init(info, p4d, addr, next);
|
||||
if (result)
|
||||
return result;
|
||||
if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
|
||||
if (pgtable_l5_enabled) {
|
||||
set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag));
|
||||
} else {
|
||||
/*
|
||||
|
|
|
@ -88,12 +88,7 @@ static int __init nonx32_setup(char *str)
|
|||
}
|
||||
__setup("noexec32=", nonx32_setup);
|
||||
|
||||
/*
|
||||
* When memory was added make sure all the processes MM have
|
||||
* suitable PGD entries in the local PGD level page.
|
||||
*/
|
||||
#ifdef CONFIG_X86_5LEVEL
|
||||
void sync_global_pgds(unsigned long start, unsigned long end)
|
||||
static void sync_global_pgds_l5(unsigned long start, unsigned long end)
|
||||
{
|
||||
unsigned long addr;
|
||||
|
||||
|
@ -129,8 +124,8 @@ void sync_global_pgds(unsigned long start, unsigned long end)
|
|||
spin_unlock(&pgd_lock);
|
||||
}
|
||||
}
|
||||
#else
|
||||
void sync_global_pgds(unsigned long start, unsigned long end)
|
||||
|
||||
static void sync_global_pgds_l4(unsigned long start, unsigned long end)
|
||||
{
|
||||
unsigned long addr;
|
||||
|
||||
|
@ -173,7 +168,18 @@ void sync_global_pgds(unsigned long start, unsigned long end)
|
|||
spin_unlock(&pgd_lock);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* When memory was added make sure all the processes MM have
|
||||
* suitable PGD entries in the local PGD level page.
|
||||
*/
|
||||
void sync_global_pgds(unsigned long start, unsigned long end)
|
||||
{
|
||||
if (pgtable_l5_enabled)
|
||||
sync_global_pgds_l5(start, end);
|
||||
else
|
||||
sync_global_pgds_l4(start, end);
|
||||
}
|
||||
|
||||
/*
|
||||
* NOTE: This function is marked __ref because it calls __init function
|
||||
|
@ -632,7 +638,7 @@ phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end,
|
|||
unsigned long vaddr = (unsigned long)__va(paddr);
|
||||
int i = p4d_index(vaddr);
|
||||
|
||||
if (!IS_ENABLED(CONFIG_X86_5LEVEL))
|
||||
if (!pgtable_l5_enabled)
|
||||
return phys_pud_init((pud_t *) p4d_page, paddr, paddr_end, page_size_mask);
|
||||
|
||||
for (; i < PTRS_PER_P4D; i++, paddr = paddr_next) {
|
||||
|
@ -712,7 +718,7 @@ kernel_physical_mapping_init(unsigned long paddr_start,
|
|||
page_size_mask);
|
||||
|
||||
spin_lock(&init_mm.page_table_lock);
|
||||
if (IS_ENABLED(CONFIG_X86_5LEVEL))
|
||||
if (pgtable_l5_enabled)
|
||||
pgd_populate(&init_mm, pgd, p4d);
|
||||
else
|
||||
p4d_populate(&init_mm, p4d_offset(pgd, vaddr), (pud_t *) p4d);
|
||||
|
@ -1093,7 +1099,7 @@ remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end,
|
|||
* 5-level case we should free them. This code will have to change
|
||||
* to adapt for boot-time switching between 4 and 5 level page tables.
|
||||
*/
|
||||
if (CONFIG_PGTABLE_LEVELS == 5)
|
||||
if (pgtable_l5_enabled)
|
||||
free_pud_table(pud_base, p4d, altmap);
|
||||
}
|
||||
|
||||
|
|
|
@ -176,10 +176,10 @@ static void __init clear_pgds(unsigned long start,
|
|||
* With folded p4d, pgd_clear() is nop, use p4d_clear()
|
||||
* instead.
|
||||
*/
|
||||
if (CONFIG_PGTABLE_LEVELS < 5)
|
||||
p4d_clear(p4d_offset(pgd, start));
|
||||
else
|
||||
if (pgtable_l5_enabled)
|
||||
pgd_clear(pgd);
|
||||
else
|
||||
p4d_clear(p4d_offset(pgd, start));
|
||||
}
|
||||
|
||||
pgd = pgd_offset_k(start);
|
||||
|
@ -191,7 +191,7 @@ static inline p4d_t *early_p4d_offset(pgd_t *pgd, unsigned long addr)
|
|||
{
|
||||
unsigned long p4d;
|
||||
|
||||
if (!IS_ENABLED(CONFIG_X86_5LEVEL))
|
||||
if (!pgtable_l5_enabled)
|
||||
return (p4d_t *)pgd;
|
||||
|
||||
p4d = __pa_nodebug(pgd_val(*pgd)) & PTE_PFN_MASK;
|
||||
|
@ -272,7 +272,7 @@ void __init kasan_early_init(void)
|
|||
for (i = 0; i < PTRS_PER_PUD; i++)
|
||||
kasan_zero_pud[i] = __pud(pud_val);
|
||||
|
||||
for (i = 0; IS_ENABLED(CONFIG_X86_5LEVEL) && i < PTRS_PER_P4D; i++)
|
||||
for (i = 0; pgtable_l5_enabled && i < PTRS_PER_P4D; i++)
|
||||
kasan_zero_p4d[i] = __p4d(p4d_val);
|
||||
|
||||
kasan_map_early_shadow(early_top_pgt);
|
||||
|
@ -303,7 +303,7 @@ void __init kasan_init(void)
|
|||
* bunch of things like kernel code, modules, EFI mapping, etc.
|
||||
* We need to take extra steps to not overwrite them.
|
||||
*/
|
||||
if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
|
||||
if (pgtable_l5_enabled) {
|
||||
void *ptr;
|
||||
|
||||
ptr = (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END));
|
||||
|
|
|
@ -124,7 +124,7 @@ void __init kernel_randomize_memory(void)
|
|||
*/
|
||||
entropy = remain_entropy / (ARRAY_SIZE(kaslr_regions) - i);
|
||||
prandom_bytes_state(&rand_state, &rand, sizeof(rand));
|
||||
if (IS_ENABLED(CONFIG_X86_5LEVEL))
|
||||
if (pgtable_l5_enabled)
|
||||
entropy = (rand % (entropy + 1)) & P4D_MASK;
|
||||
else
|
||||
entropy = (rand % (entropy + 1)) & PUD_MASK;
|
||||
|
@ -136,7 +136,7 @@ void __init kernel_randomize_memory(void)
|
|||
* randomization alignment.
|
||||
*/
|
||||
vaddr += get_padding(&kaslr_regions[i]);
|
||||
if (IS_ENABLED(CONFIG_X86_5LEVEL))
|
||||
if (pgtable_l5_enabled)
|
||||
vaddr = round_up(vaddr + 1, P4D_SIZE);
|
||||
else
|
||||
vaddr = round_up(vaddr + 1, PUD_SIZE);
|
||||
|
@ -212,7 +212,7 @@ void __meminit init_trampoline(void)
|
|||
return;
|
||||
}
|
||||
|
||||
if (IS_ENABLED(CONFIG_X86_5LEVEL))
|
||||
if (pgtable_l5_enabled)
|
||||
init_trampoline_p4d();
|
||||
else
|
||||
init_trampoline_pud();
|
||||
|
|
|
@ -157,7 +157,7 @@ static void sync_current_stack_to_mm(struct mm_struct *mm)
|
|||
unsigned long sp = current_stack_pointer;
|
||||
pgd_t *pgd = pgd_offset(mm, sp);
|
||||
|
||||
if (CONFIG_PGTABLE_LEVELS > 4) {
|
||||
if (pgtable_l5_enabled) {
|
||||
if (unlikely(pgd_none(*pgd))) {
|
||||
pgd_t *pgd_ref = pgd_offset_k(sp);
|
||||
|
||||
|
|
|
@ -227,7 +227,7 @@ int __init efi_alloc_page_tables(void)
|
|||
|
||||
pud = pud_alloc(&init_mm, p4d, EFI_VA_END);
|
||||
if (!pud) {
|
||||
if (CONFIG_PGTABLE_LEVELS > 4)
|
||||
if (pgtable_l5_enabled)
|
||||
free_page((unsigned long) pgd_page_vaddr(*pgd));
|
||||
free_page((unsigned long)efi_pgd);
|
||||
return -ENOMEM;
|
||||
|
|
|
@ -50,7 +50,7 @@ static int set_up_temporary_text_mapping(pgd_t *pgd)
|
|||
{
|
||||
pmd_t *pmd;
|
||||
pud_t *pud;
|
||||
p4d_t *p4d;
|
||||
p4d_t *p4d = NULL;
|
||||
|
||||
/*
|
||||
* The new mapping only has to cover the page containing the image
|
||||
|
@ -66,7 +66,7 @@ static int set_up_temporary_text_mapping(pgd_t *pgd)
|
|||
* tables used by the image kernel.
|
||||
*/
|
||||
|
||||
if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
|
||||
if (pgtable_l5_enabled) {
|
||||
p4d = (p4d_t *)get_safe_page(GFP_ATOMIC);
|
||||
if (!p4d)
|
||||
return -ENOMEM;
|
||||
|
@ -84,7 +84,7 @@ static int set_up_temporary_text_mapping(pgd_t *pgd)
|
|||
__pmd((jump_address_phys & PMD_MASK) | __PAGE_KERNEL_LARGE_EXEC));
|
||||
set_pud(pud + pud_index(restore_jump_address),
|
||||
__pud(__pa(pmd) | _KERNPG_TABLE));
|
||||
if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
|
||||
if (p4d) {
|
||||
set_p4d(p4d + p4d_index(restore_jump_address), __p4d(__pa(pud) | _KERNPG_TABLE));
|
||||
set_pgd(pgd + pgd_index(restore_jump_address), __pgd(__pa(p4d) | _KERNPG_TABLE));
|
||||
} else {
|
||||
|
|
Loading…
Reference in a new issue