![Xunlei Pang](/assets/img/avatar_default.png)
Kernel identity mappings on x86-64 kernels are created in two ways: by the early x86 boot code, or by kernel_ident_mapping_init(). Native kernels (which is the dominant usecase) use the former, but the kexec and the hibernation code uses kernel_ident_mapping_init(). There's a subtle difference between these two ways of how identity mappings are created, the current kernel_ident_mapping_init() code creates identity mappings always using 2MB page(PMD level) - while the native kernel boot path also utilizes gbpages where available. This difference is suboptimal both for performance and for memory usage: kernel_ident_mapping_init() needs to allocate pages for the page tables when creating the new identity mappings. This patch adds 1GB page(PUD level) support to kernel_ident_mapping_init() to address these concerns. The primary advantage would be better TLB coverage/performance, because we'd utilize 1GB TLBs instead of 2MB ones. It is also useful for machines with large number of memory to save paging structure allocations(around 4MB/TB using 2MB page) when setting identity mappings for all the memory, after using 1GB page it will consume only 8KB/TB. ( Note that this change alone does not activate gbpages in kexec, we are doing that in a separate patch. ) Signed-off-by: Xunlei Pang <xlpang@redhat.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Borislav Petkov <bp@alien8.de> Cc: Brian Gerst <brgerst@gmail.com> Cc: Dave Young <dyoung@redhat.com> Cc: Denys Vlasenko <dvlasenk@redhat.com> Cc: Eric Biederman <ebiederm@xmission.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Josh Poimboeuf <jpoimboe@redhat.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Yinghai Lu <yinghai@kernel.org> Cc: akpm@linux-foundation.org Cc: kexec@lists.infradead.org Link: http://lkml.kernel.org/r/1493862171-8799-1-git-send-email-xlpang@redhat.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
132 lines
2.9 KiB
C
132 lines
2.9 KiB
C
/*
|
|
* Helper routines for building identity mapping page tables. This is
|
|
* included by both the compressed kernel and the regular kernel.
|
|
*/
|
|
|
|
static void ident_pmd_init(struct x86_mapping_info *info, pmd_t *pmd_page,
|
|
unsigned long addr, unsigned long end)
|
|
{
|
|
addr &= PMD_MASK;
|
|
for (; addr < end; addr += PMD_SIZE) {
|
|
pmd_t *pmd = pmd_page + pmd_index(addr);
|
|
|
|
if (pmd_present(*pmd))
|
|
continue;
|
|
|
|
set_pmd(pmd, __pmd((addr - info->offset) | info->page_flag));
|
|
}
|
|
}
|
|
|
|
static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
|
|
unsigned long addr, unsigned long end)
|
|
{
|
|
unsigned long next;
|
|
|
|
for (; addr < end; addr = next) {
|
|
pud_t *pud = pud_page + pud_index(addr);
|
|
pmd_t *pmd;
|
|
|
|
next = (addr & PUD_MASK) + PUD_SIZE;
|
|
if (next > end)
|
|
next = end;
|
|
|
|
if (info->direct_gbpages) {
|
|
pud_t pudval;
|
|
|
|
if (pud_present(*pud))
|
|
continue;
|
|
|
|
addr &= PUD_MASK;
|
|
pudval = __pud((addr - info->offset) | info->page_flag);
|
|
set_pud(pud, pudval);
|
|
continue;
|
|
}
|
|
|
|
if (pud_present(*pud)) {
|
|
pmd = pmd_offset(pud, 0);
|
|
ident_pmd_init(info, pmd, addr, next);
|
|
continue;
|
|
}
|
|
pmd = (pmd_t *)info->alloc_pgt_page(info->context);
|
|
if (!pmd)
|
|
return -ENOMEM;
|
|
ident_pmd_init(info, pmd, addr, next);
|
|
set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page,
|
|
unsigned long addr, unsigned long end)
|
|
{
|
|
unsigned long next;
|
|
|
|
for (; addr < end; addr = next) {
|
|
p4d_t *p4d = p4d_page + p4d_index(addr);
|
|
pud_t *pud;
|
|
|
|
next = (addr & P4D_MASK) + P4D_SIZE;
|
|
if (next > end)
|
|
next = end;
|
|
|
|
if (p4d_present(*p4d)) {
|
|
pud = pud_offset(p4d, 0);
|
|
ident_pud_init(info, pud, addr, next);
|
|
continue;
|
|
}
|
|
pud = (pud_t *)info->alloc_pgt_page(info->context);
|
|
if (!pud)
|
|
return -ENOMEM;
|
|
ident_pud_init(info, pud, addr, next);
|
|
set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
|
|
unsigned long pstart, unsigned long pend)
|
|
{
|
|
unsigned long addr = pstart + info->offset;
|
|
unsigned long end = pend + info->offset;
|
|
unsigned long next;
|
|
int result;
|
|
|
|
for (; addr < end; addr = next) {
|
|
pgd_t *pgd = pgd_page + pgd_index(addr);
|
|
p4d_t *p4d;
|
|
|
|
next = (addr & PGDIR_MASK) + PGDIR_SIZE;
|
|
if (next > end)
|
|
next = end;
|
|
|
|
if (pgd_present(*pgd)) {
|
|
p4d = p4d_offset(pgd, 0);
|
|
result = ident_p4d_init(info, p4d, addr, next);
|
|
if (result)
|
|
return result;
|
|
continue;
|
|
}
|
|
|
|
p4d = (p4d_t *)info->alloc_pgt_page(info->context);
|
|
if (!p4d)
|
|
return -ENOMEM;
|
|
result = ident_p4d_init(info, p4d, addr, next);
|
|
if (result)
|
|
return result;
|
|
if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
|
|
set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE));
|
|
} else {
|
|
/*
|
|
* With p4d folded, pgd is equal to p4d.
|
|
* The pgd entry has to point to the pud page table in this case.
|
|
*/
|
|
pud_t *pud = pud_offset(p4d, 0);
|
|
set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|