![Zhimin Gu](/assets/img/avatar_default.png)
Set up the temporary text mapping for the final jump address
so that the system could jump to the right address after all
the pages have been copied back to their original address -
otherwise the final mapping for the jump address is invalid.
Analogous changes were made for 64-bit in commit 65c0554b73
(x86/power/64: Fix kernel text mapping corruption during image
restoration).
Signed-off-by: Zhimin Gu <kookoo.gu@intel.com>
Acked-by: Pavel Machek <pavel@ucw.cz>
Signed-off-by: Chen Yu <yu.c.chen@intel.com>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
249 lines
5.9 KiB
C
249 lines
5.9 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Hibernation support for x86
|
|
*
|
|
* Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
|
|
* Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
|
|
* Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
|
|
*/
|
|
#include <linux/gfp.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/suspend.h>
|
|
#include <linux/scatterlist.h>
|
|
#include <linux/kdebug.h>
|
|
|
|
#include <crypto/hash.h>
|
|
|
|
#include <asm/e820/api.h>
|
|
#include <asm/init.h>
|
|
#include <asm/proto.h>
|
|
#include <asm/page.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/mtrr.h>
|
|
#include <asm/sections.h>
|
|
#include <asm/suspend.h>
|
|
#include <asm/tlbflush.h>
|
|
|
|
/*
|
|
* Address to jump to in the last phase of restore in order to get to the image
|
|
* kernel's text (this value is passed in the image header).
|
|
*/
|
|
unsigned long restore_jump_address __visible;
|
|
unsigned long jump_address_phys;
|
|
|
|
/*
|
|
* Value of the cr3 register from before the hibernation (this value is passed
|
|
* in the image header).
|
|
*/
|
|
unsigned long restore_cr3 __visible;
|
|
unsigned long temp_pgt __visible;
|
|
unsigned long relocated_restore_code __visible;
|
|
|
|
/**
|
|
* pfn_is_nosave - check if given pfn is in the 'nosave' section
|
|
*/
|
|
int pfn_is_nosave(unsigned long pfn)
|
|
{
|
|
unsigned long nosave_begin_pfn;
|
|
unsigned long nosave_end_pfn;
|
|
|
|
nosave_begin_pfn = __pa_symbol(&__nosave_begin) >> PAGE_SHIFT;
|
|
nosave_end_pfn = PAGE_ALIGN(__pa_symbol(&__nosave_end)) >> PAGE_SHIFT;
|
|
|
|
return pfn >= nosave_begin_pfn && pfn < nosave_end_pfn;
|
|
}
|
|
|
|
|
|
#define MD5_DIGEST_SIZE 16
|
|
|
|
struct restore_data_record {
|
|
unsigned long jump_address;
|
|
unsigned long jump_address_phys;
|
|
unsigned long cr3;
|
|
unsigned long magic;
|
|
u8 e820_digest[MD5_DIGEST_SIZE];
|
|
};
|
|
|
|
#if IS_BUILTIN(CONFIG_CRYPTO_MD5)
|
|
/**
|
|
* get_e820_md5 - calculate md5 according to given e820 table
|
|
*
|
|
* @table: the e820 table to be calculated
|
|
* @buf: the md5 result to be stored to
|
|
*/
|
|
static int get_e820_md5(struct e820_table *table, void *buf)
|
|
{
|
|
struct crypto_shash *tfm;
|
|
struct shash_desc *desc;
|
|
int size;
|
|
int ret = 0;
|
|
|
|
tfm = crypto_alloc_shash("md5", 0, 0);
|
|
if (IS_ERR(tfm))
|
|
return -ENOMEM;
|
|
|
|
desc = kmalloc(sizeof(struct shash_desc) + crypto_shash_descsize(tfm),
|
|
GFP_KERNEL);
|
|
if (!desc) {
|
|
ret = -ENOMEM;
|
|
goto free_tfm;
|
|
}
|
|
|
|
desc->tfm = tfm;
|
|
desc->flags = 0;
|
|
|
|
size = offsetof(struct e820_table, entries) +
|
|
sizeof(struct e820_entry) * table->nr_entries;
|
|
|
|
if (crypto_shash_digest(desc, (u8 *)table, size, buf))
|
|
ret = -EINVAL;
|
|
|
|
kzfree(desc);
|
|
|
|
free_tfm:
|
|
crypto_free_shash(tfm);
|
|
return ret;
|
|
}
|
|
|
|
static int hibernation_e820_save(void *buf)
|
|
{
|
|
return get_e820_md5(e820_table_firmware, buf);
|
|
}
|
|
|
|
static bool hibernation_e820_mismatch(void *buf)
|
|
{
|
|
int ret;
|
|
u8 result[MD5_DIGEST_SIZE];
|
|
|
|
memset(result, 0, MD5_DIGEST_SIZE);
|
|
/* If there is no digest in suspend kernel, let it go. */
|
|
if (!memcmp(result, buf, MD5_DIGEST_SIZE))
|
|
return false;
|
|
|
|
ret = get_e820_md5(e820_table_firmware, result);
|
|
if (ret)
|
|
return true;
|
|
|
|
return memcmp(result, buf, MD5_DIGEST_SIZE) ? true : false;
|
|
}
|
|
#else
|
|
static int hibernation_e820_save(void *buf)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static bool hibernation_e820_mismatch(void *buf)
|
|
{
|
|
/* If md5 is not builtin for restore kernel, let it go. */
|
|
return false;
|
|
}
|
|
#endif
|
|
|
|
#ifdef CONFIG_X86_64
|
|
#define RESTORE_MAGIC 0x23456789ABCDEF01UL
|
|
#else
|
|
#define RESTORE_MAGIC 0x12345678UL
|
|
#endif
|
|
|
|
/**
|
|
* arch_hibernation_header_save - populate the architecture specific part
|
|
* of a hibernation image header
|
|
* @addr: address to save the data at
|
|
*/
|
|
int arch_hibernation_header_save(void *addr, unsigned int max_size)
|
|
{
|
|
struct restore_data_record *rdr = addr;
|
|
|
|
if (max_size < sizeof(struct restore_data_record))
|
|
return -EOVERFLOW;
|
|
rdr->magic = RESTORE_MAGIC;
|
|
rdr->jump_address = (unsigned long)restore_registers;
|
|
rdr->jump_address_phys = __pa_symbol(restore_registers);
|
|
|
|
/*
|
|
* The restore code fixes up CR3 and CR4 in the following sequence:
|
|
*
|
|
* [in hibernation asm]
|
|
* 1. CR3 <= temporary page tables
|
|
* 2. CR4 <= mmu_cr4_features (from the kernel that restores us)
|
|
* 3. CR3 <= rdr->cr3
|
|
* 4. CR4 <= mmu_cr4_features (from us, i.e. the image kernel)
|
|
* [in restore_processor_state()]
|
|
* 5. CR4 <= saved CR4
|
|
* 6. CR3 <= saved CR3
|
|
*
|
|
* Our mmu_cr4_features has CR4.PCIDE=0, and toggling
|
|
* CR4.PCIDE while CR3's PCID bits are nonzero is illegal, so
|
|
* rdr->cr3 needs to point to valid page tables but must not
|
|
* have any of the PCID bits set.
|
|
*/
|
|
rdr->cr3 = restore_cr3 & ~CR3_PCID_MASK;
|
|
|
|
return hibernation_e820_save(rdr->e820_digest);
|
|
}
|
|
|
|
/**
|
|
* arch_hibernation_header_restore - read the architecture specific data
|
|
* from the hibernation image header
|
|
* @addr: address to read the data from
|
|
*/
|
|
int arch_hibernation_header_restore(void *addr)
|
|
{
|
|
struct restore_data_record *rdr = addr;
|
|
|
|
if (rdr->magic != RESTORE_MAGIC) {
|
|
pr_crit("Unrecognized hibernate image header format!\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
restore_jump_address = rdr->jump_address;
|
|
jump_address_phys = rdr->jump_address_phys;
|
|
restore_cr3 = rdr->cr3;
|
|
|
|
if (hibernation_e820_mismatch(rdr->e820_digest)) {
|
|
pr_crit("Hibernate inconsistent memory map detected!\n");
|
|
return -ENODEV;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int relocate_restore_code(void)
|
|
{
|
|
pgd_t *pgd;
|
|
p4d_t *p4d;
|
|
pud_t *pud;
|
|
pmd_t *pmd;
|
|
pte_t *pte;
|
|
|
|
relocated_restore_code = get_safe_page(GFP_ATOMIC);
|
|
if (!relocated_restore_code)
|
|
return -ENOMEM;
|
|
|
|
memcpy((void *)relocated_restore_code, core_restore_code, PAGE_SIZE);
|
|
|
|
/* Make the page containing the relocated code executable */
|
|
pgd = (pgd_t *)__va(read_cr3_pa()) +
|
|
pgd_index(relocated_restore_code);
|
|
p4d = p4d_offset(pgd, relocated_restore_code);
|
|
if (p4d_large(*p4d)) {
|
|
set_p4d(p4d, __p4d(p4d_val(*p4d) & ~_PAGE_NX));
|
|
goto out;
|
|
}
|
|
pud = pud_offset(p4d, relocated_restore_code);
|
|
if (pud_large(*pud)) {
|
|
set_pud(pud, __pud(pud_val(*pud) & ~_PAGE_NX));
|
|
goto out;
|
|
}
|
|
pmd = pmd_offset(pud, relocated_restore_code);
|
|
if (pmd_large(*pmd)) {
|
|
set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_NX));
|
|
goto out;
|
|
}
|
|
pte = pte_offset_kernel(pmd, relocated_restore_code);
|
|
set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_NX));
|
|
out:
|
|
__flush_tlb_all();
|
|
return 0;
|
|
}
|