15385dfe7e
Pull x86/smap support from Ingo Molnar: "This adds support for the SMAP (Supervisor Mode Access Prevention) CPU feature on Intel CPUs: a hardware feature that prevents unintended user-space data access from kernel privileged code. It's turned on automatically when possible. This, in combination with SMEP, makes it even harder to exploit kernel bugs such as NULL pointer dereferences." Fix up trivial conflict in arch/x86/kernel/entry_64.S due to newly added includes right next to each other. * 'x86-smap-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86, smep, smap: Make the switching functions one-way x86, suspend: On wakeup always initialize cr4 and EFER x86-32: Start out eflags and cr4 clean x86, smap: Do not abuse the [f][x]rstor_checking() functions for user space x86-32, smap: Add STAC/CLAC instructions to 32-bit kernel entry x86, smap: Reduce the SMAP overhead for signal handling x86, smap: A page fault due to SMAP is an oops x86, smap: Turn on Supervisor Mode Access Prevention x86, smap: Add STAC and CLAC instructions to control user space access x86, uaccess: Merge prototypes for clear_user/__clear_user x86, smap: Add a header file with macros for STAC/CLAC x86, alternative: Add header guards to <asm/alternative-asm.h> x86, alternative: Use .pushsection/.popsection x86, smap: Add CR4 bit for SMAP x86-32, mm: The WP test should be done on a kernel page
240 lines
8.3 KiB
C
240 lines
8.3 KiB
C
#ifndef _ASM_X86_ALTERNATIVE_H
|
|
#define _ASM_X86_ALTERNATIVE_H
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/stddef.h>
|
|
#include <linux/stringify.h>
|
|
#include <asm/asm.h>
|
|
|
|
/*
|
|
* Alternative inline assembly for SMP.
|
|
*
|
|
* The LOCK_PREFIX macro defined here replaces the LOCK and
|
|
* LOCK_PREFIX macros used everywhere in the source tree.
|
|
*
|
|
* SMP alternatives use the same data structures as the other
|
|
* alternatives and the X86_FEATURE_UP flag to indicate the case of a
|
|
* UP system running a SMP kernel. The existing apply_alternatives()
|
|
* works fine for patching a SMP kernel for UP.
|
|
*
|
|
* The SMP alternative tables can be kept after boot and contain both
|
|
* UP and SMP versions of the instructions to allow switching back to
|
|
* SMP at runtime, when hotplugging in a new CPU, which is especially
|
|
* useful in virtualized environments.
|
|
*
|
|
* The very common lock prefix is handled as special case in a
|
|
* separate table which is a pure address list without replacement ptr
|
|
* and size information. That keeps the table sizes small.
|
|
*/
|
|
|
|
#ifdef CONFIG_SMP
|
|
#define LOCK_PREFIX_HERE \
|
|
".pushsection .smp_locks,\"a\"\n" \
|
|
".balign 4\n" \
|
|
".long 671f - .\n" /* offset */ \
|
|
".popsection\n" \
|
|
"671:"
|
|
|
|
#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
|
|
|
|
#else /* ! CONFIG_SMP */
|
|
#define LOCK_PREFIX_HERE ""
|
|
#define LOCK_PREFIX ""
|
|
#endif
|
|
|
|
struct alt_instr {
|
|
s32 instr_offset; /* original instruction */
|
|
s32 repl_offset; /* offset to replacement instruction */
|
|
u16 cpuid; /* cpuid bit set for replacement */
|
|
u8 instrlen; /* length of original instruction */
|
|
u8 replacementlen; /* length of new instruction, <= instrlen */
|
|
};
|
|
|
|
extern void alternative_instructions(void);
|
|
extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
|
|
|
|
struct module;
|
|
|
|
#ifdef CONFIG_SMP
|
|
extern void alternatives_smp_module_add(struct module *mod, char *name,
|
|
void *locks, void *locks_end,
|
|
void *text, void *text_end);
|
|
extern void alternatives_smp_module_del(struct module *mod);
|
|
extern void alternatives_enable_smp(void);
|
|
extern int alternatives_text_reserved(void *start, void *end);
|
|
extern bool skip_smp_alternatives;
|
|
#else
|
|
static inline void alternatives_smp_module_add(struct module *mod, char *name,
|
|
void *locks, void *locks_end,
|
|
void *text, void *text_end) {}
|
|
static inline void alternatives_smp_module_del(struct module *mod) {}
|
|
static inline void alternatives_enable_smp(void) {}
|
|
static inline int alternatives_text_reserved(void *start, void *end)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_SMP */
|
|
|
|
#define OLDINSTR(oldinstr) "661:\n\t" oldinstr "\n662:\n"
|
|
|
|
#define b_replacement(number) "663"#number
|
|
#define e_replacement(number) "664"#number
|
|
|
|
#define alt_slen "662b-661b"
|
|
#define alt_rlen(number) e_replacement(number)"f-"b_replacement(number)"f"
|
|
|
|
#define ALTINSTR_ENTRY(feature, number) \
|
|
" .long 661b - .\n" /* label */ \
|
|
" .long " b_replacement(number)"f - .\n" /* new instruction */ \
|
|
" .word " __stringify(feature) "\n" /* feature bit */ \
|
|
" .byte " alt_slen "\n" /* source len */ \
|
|
" .byte " alt_rlen(number) "\n" /* replacement len */
|
|
|
|
#define DISCARD_ENTRY(number) /* rlen <= slen */ \
|
|
" .byte 0xff + (" alt_rlen(number) ") - (" alt_slen ")\n"
|
|
|
|
#define ALTINSTR_REPLACEMENT(newinstr, feature, number) /* replacement */ \
|
|
b_replacement(number)":\n\t" newinstr "\n" e_replacement(number) ":\n\t"
|
|
|
|
/* alternative assembly primitive: */
|
|
#define ALTERNATIVE(oldinstr, newinstr, feature) \
|
|
OLDINSTR(oldinstr) \
|
|
".pushsection .altinstructions,\"a\"\n" \
|
|
ALTINSTR_ENTRY(feature, 1) \
|
|
".popsection\n" \
|
|
".pushsection .discard,\"aw\",@progbits\n" \
|
|
DISCARD_ENTRY(1) \
|
|
".popsection\n" \
|
|
".pushsection .altinstr_replacement, \"ax\"\n" \
|
|
ALTINSTR_REPLACEMENT(newinstr, feature, 1) \
|
|
".popsection"
|
|
|
|
#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
|
|
OLDINSTR(oldinstr) \
|
|
".pushsection .altinstructions,\"a\"\n" \
|
|
ALTINSTR_ENTRY(feature1, 1) \
|
|
ALTINSTR_ENTRY(feature2, 2) \
|
|
".popsection\n" \
|
|
".pushsection .discard,\"aw\",@progbits\n" \
|
|
DISCARD_ENTRY(1) \
|
|
DISCARD_ENTRY(2) \
|
|
".popsection\n" \
|
|
".pushsection .altinstr_replacement, \"ax\"\n" \
|
|
ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \
|
|
ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
|
|
".popsection"
|
|
|
|
/*
|
|
* This must be included *after* the definition of ALTERNATIVE due to
|
|
* <asm/arch_hweight.h>
|
|
*/
|
|
#include <asm/cpufeature.h>
|
|
|
|
/*
|
|
* Alternative instructions for different CPU types or capabilities.
|
|
*
|
|
* This allows to use optimized instructions even on generic binary
|
|
* kernels.
|
|
*
|
|
* length of oldinstr must be longer or equal the length of newinstr
|
|
* It can be padded with nops as needed.
|
|
*
|
|
* For non barrier like inlines please define new variants
|
|
* without volatile and memory clobber.
|
|
*/
|
|
#define alternative(oldinstr, newinstr, feature) \
|
|
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
|
|
|
|
/*
|
|
* Alternative inline assembly with input.
|
|
*
|
|
* Pecularities:
|
|
* No memory clobber here.
|
|
* Argument numbers start with 1.
|
|
* Best is to use constraints that are fixed size (like (%1) ... "r")
|
|
* If you use variable sized constraints like "m" or "g" in the
|
|
* replacement make sure to pad to the worst case length.
|
|
* Leaving an unused argument 0 to keep API compatibility.
|
|
*/
|
|
#define alternative_input(oldinstr, newinstr, feature, input...) \
|
|
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
|
|
: : "i" (0), ## input)
|
|
|
|
/* Like alternative_input, but with a single output argument */
|
|
#define alternative_io(oldinstr, newinstr, feature, output, input...) \
|
|
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
|
|
: output : "i" (0), ## input)
|
|
|
|
/* Like alternative_io, but for replacing a direct call with another one. */
|
|
#define alternative_call(oldfunc, newfunc, feature, output, input...) \
|
|
asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
|
|
: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
|
|
|
|
/*
|
|
* Like alternative_call, but there are two features and respective functions.
|
|
* If CPU has feature2, function2 is used.
|
|
* Otherwise, if CPU has feature1, function1 is used.
|
|
* Otherwise, old function is used.
|
|
*/
|
|
#define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \
|
|
output, input...) \
|
|
asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
|
|
"call %P[new2]", feature2) \
|
|
: output : [old] "i" (oldfunc), [new1] "i" (newfunc1), \
|
|
[new2] "i" (newfunc2), ## input)
|
|
|
|
/*
|
|
* use this macro(s) if you need more than one output parameter
|
|
* in alternative_io
|
|
*/
|
|
#define ASM_OUTPUT2(a...) a
|
|
|
|
/*
|
|
* use this macro if you need clobbers but no inputs in
|
|
* alternative_{input,io,call}()
|
|
*/
|
|
#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
|
|
|
|
struct paravirt_patch_site;
|
|
#ifdef CONFIG_PARAVIRT
|
|
void apply_paravirt(struct paravirt_patch_site *start,
|
|
struct paravirt_patch_site *end);
|
|
#else
|
|
static inline void apply_paravirt(struct paravirt_patch_site *start,
|
|
struct paravirt_patch_site *end)
|
|
{}
|
|
#define __parainstructions NULL
|
|
#define __parainstructions_end NULL
|
|
#endif
|
|
|
|
extern void *text_poke_early(void *addr, const void *opcode, size_t len);
|
|
|
|
/*
|
|
* Clear and restore the kernel write-protection flag on the local CPU.
|
|
* Allows the kernel to edit read-only pages.
|
|
* Side-effect: any interrupt handler running between save and restore will have
|
|
* the ability to write to read-only pages.
|
|
*
|
|
* Warning:
|
|
* Code patching in the UP case is safe if NMIs and MCE handlers are stopped and
|
|
* no thread can be preempted in the instructions being modified (no iret to an
|
|
* invalid instruction possible) or if the instructions are changed from a
|
|
* consistent state to another consistent state atomically.
|
|
* More care must be taken when modifying code in the SMP case because of
|
|
* Intel's errata. text_poke_smp() takes care that errata, but still
|
|
* doesn't support NMI/MCE handler code modifying.
|
|
* On the local CPU you need to be protected again NMI or MCE handlers seeing an
|
|
* inconsistent instruction while you patch.
|
|
*/
|
|
struct text_poke_param {
|
|
void *addr;
|
|
const void *opcode;
|
|
size_t len;
|
|
};
|
|
|
|
extern void *text_poke(void *addr, const void *opcode, size_t len);
|
|
extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
|
|
extern void text_poke_smp_batch(struct text_poke_param *params, int n);
|
|
|
|
#endif /* _ASM_X86_ALTERNATIVE_H */
|