2019-05-27 00:55:01 -06:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
2008-06-23 19:32:39 -06:00
|
|
|
#ifndef __ASM_POWERPC_FEATURE_FIXUPS_H
|
|
|
|
|
#define __ASM_POWERPC_FEATURE_FIXUPS_H
|
|
|
|
|
|
2018-07-05 10:24:57 -06:00
|
|
|
#include <asm/asm-const.h>
|
|
|
|
|
|
2008-06-23 19:32:39 -06:00
|
|
|
/*
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Feature section common macros
|
|
|
|
|
*
|
|
|
|
|
* Note that the entries now contain offsets between the table entry
|
|
|
|
|
* and the code rather than absolute code pointers in order to be
|
|
|
|
|
* useable with the vdso shared library. There is also an assumption
|
|
|
|
|
* that values will be negative, that is, the fixup table has to be
|
|
|
|
|
* located after the code it fixes up.
|
|
|
|
|
*/
|
2008-06-23 19:32:48 -06:00
|
|
|
#if defined(CONFIG_PPC64) && !defined(__powerpc64__)
|
2008-06-23 19:32:39 -06:00
|
|
|
/* 64 bits kernel, 32 bits code (ie. vdso32) */
|
2017-03-08 22:42:12 -07:00
|
|
|
#define FTR_ENTRY_LONG .8byte
|
2008-06-23 19:32:48 -06:00
|
|
|
#define FTR_ENTRY_OFFSET .long 0xffffffff; .long
|
2009-04-29 14:58:01 -06:00
|
|
|
#elif defined(CONFIG_PPC64)
|
2017-03-08 22:42:12 -07:00
|
|
|
#define FTR_ENTRY_LONG .8byte
|
|
|
|
|
#define FTR_ENTRY_OFFSET .8byte
|
2008-06-23 19:32:48 -06:00
|
|
|
#else
|
2009-04-29 14:58:01 -06:00
|
|
|
#define FTR_ENTRY_LONG .long
|
|
|
|
|
#define FTR_ENTRY_OFFSET .long
|
2008-06-23 19:32:48 -06:00
|
|
|
#endif
|
|
|
|
|
|
powerpc: Introduce infrastructure for feature sections with alternatives
The current feature section logic only supports nop'ing out code, this means
if you want to choose at runtime between instruction sequences, one or both
cases will have to execute the nop'ed out contents of the other section, eg:
BEGIN_FTR_SECTION
or 1,1,1
END_FTR_SECTION_IFSET(FOO)
BEGIN_FTR_SECTION
or 2,2,2
END_FTR_SECTION_IFCLR(FOO)
and the resulting code will be either,
or 1,1,1
nop
or,
nop
or 2,2,2
For small code segments this is fine, but for larger code blocks and in
performance criticial code segments, it would be nice to avoid the nops.
This commit starts to implement logic to allow the following:
BEGIN_FTR_SECTION
or 1,1,1
FTR_SECTION_ELSE
or 2,2,2
ALT_FTR_SECTION_END_IFSET(FOO)
and the resulting code will be:
or 1,1,1
or,
or 2,2,2
We achieve this by extending the existing FTR macros. The current feature
section semantic just becomes a special case, ie. if the else case is empty
we nop out the default case.
The key limitation is that the size of the else case must be less than or
equal to the size of the default case. If the else case is smaller the
remainder of the section is nop'ed.
We let the linker put the else case code in with the rest of the text,
so that relative branches from the else case are more likley to link,
this has the disadvantage that we can't free the unused else cases.
This commit introduces the required macro and linker script changes, but
does not enable the patching of the alternative sections.
We also need to update two hand-made section entries in reg.h and timex.h
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-06-23 19:32:54 -06:00
|
|
|
#define START_FTR_SECTION(label) label##1:
|
|
|
|
|
|
|
|
|
|
#define FTR_SECTION_ELSE_NESTED(label) \
|
|
|
|
|
label##2: \
|
|
|
|
|
.pushsection __ftr_alt_##label,"a"; \
|
|
|
|
|
.align 2; \
|
|
|
|
|
label##3:
|
|
|
|
|
|
2010-11-07 11:22:29 -07:00
|
|
|
#define MAKE_FTR_SECTION_ENTRY(msk, val, label, sect) \
|
|
|
|
|
label##4: \
|
|
|
|
|
.popsection; \
|
|
|
|
|
.pushsection sect,"a"; \
|
|
|
|
|
.align 3; \
|
|
|
|
|
label##5: \
|
|
|
|
|
FTR_ENTRY_LONG msk; \
|
|
|
|
|
FTR_ENTRY_LONG val; \
|
|
|
|
|
FTR_ENTRY_OFFSET label##1b-label##5b; \
|
|
|
|
|
FTR_ENTRY_OFFSET label##2b-label##5b; \
|
|
|
|
|
FTR_ENTRY_OFFSET label##3b-label##5b; \
|
|
|
|
|
FTR_ENTRY_OFFSET label##4b-label##5b; \
|
2011-04-04 22:27:11 -06:00
|
|
|
.ifgt (label##4b- label##3b)-(label##2b- label##1b); \
|
2010-11-07 11:22:29 -07:00
|
|
|
.error "Feature section else case larger than body"; \
|
|
|
|
|
.endif; \
|
powerpc: Introduce infrastructure for feature sections with alternatives
The current feature section logic only supports nop'ing out code, this means
if you want to choose at runtime between instruction sequences, one or both
cases will have to execute the nop'ed out contents of the other section, eg:
BEGIN_FTR_SECTION
or 1,1,1
END_FTR_SECTION_IFSET(FOO)
BEGIN_FTR_SECTION
or 2,2,2
END_FTR_SECTION_IFCLR(FOO)
and the resulting code will be either,
or 1,1,1
nop
or,
nop
or 2,2,2
For small code segments this is fine, but for larger code blocks and in
performance criticial code segments, it would be nice to avoid the nops.
This commit starts to implement logic to allow the following:
BEGIN_FTR_SECTION
or 1,1,1
FTR_SECTION_ELSE
or 2,2,2
ALT_FTR_SECTION_END_IFSET(FOO)
and the resulting code will be:
or 1,1,1
or,
or 2,2,2
We achieve this by extending the existing FTR macros. The current feature
section semantic just becomes a special case, ie. if the else case is empty
we nop out the default case.
The key limitation is that the size of the else case must be less than or
equal to the size of the default case. If the else case is smaller the
remainder of the section is nop'ed.
We let the linker put the else case code in with the rest of the text,
so that relative branches from the else case are more likley to link,
this has the disadvantage that we can't free the unused else cases.
This commit introduces the required macro and linker script changes, but
does not enable the patching of the alternative sections.
We also need to update two hand-made section entries in reg.h and timex.h
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-06-23 19:32:54 -06:00
|
|
|
.popsection;
|
2008-06-23 19:32:39 -06:00
|
|
|
|
|
|
|
|
|
|
|
|
|
/* CPU feature dependent sections */
|
powerpc: Introduce infrastructure for feature sections with alternatives
The current feature section logic only supports nop'ing out code, this means
if you want to choose at runtime between instruction sequences, one or both
cases will have to execute the nop'ed out contents of the other section, eg:
BEGIN_FTR_SECTION
or 1,1,1
END_FTR_SECTION_IFSET(FOO)
BEGIN_FTR_SECTION
or 2,2,2
END_FTR_SECTION_IFCLR(FOO)
and the resulting code will be either,
or 1,1,1
nop
or,
nop
or 2,2,2
For small code segments this is fine, but for larger code blocks and in
performance criticial code segments, it would be nice to avoid the nops.
This commit starts to implement logic to allow the following:
BEGIN_FTR_SECTION
or 1,1,1
FTR_SECTION_ELSE
or 2,2,2
ALT_FTR_SECTION_END_IFSET(FOO)
and the resulting code will be:
or 1,1,1
or,
or 2,2,2
We achieve this by extending the existing FTR macros. The current feature
section semantic just becomes a special case, ie. if the else case is empty
we nop out the default case.
The key limitation is that the size of the else case must be less than or
equal to the size of the default case. If the else case is smaller the
remainder of the section is nop'ed.
We let the linker put the else case code in with the rest of the text,
so that relative branches from the else case are more likley to link,
this has the disadvantage that we can't free the unused else cases.
This commit introduces the required macro and linker script changes, but
does not enable the patching of the alternative sections.
We also need to update two hand-made section entries in reg.h and timex.h
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-06-23 19:32:54 -06:00
|
|
|
#define BEGIN_FTR_SECTION_NESTED(label) START_FTR_SECTION(label)
|
|
|
|
|
#define BEGIN_FTR_SECTION START_FTR_SECTION(97)
|
2008-06-23 19:32:39 -06:00
|
|
|
|
powerpc: Introduce infrastructure for feature sections with alternatives
The current feature section logic only supports nop'ing out code, this means
if you want to choose at runtime between instruction sequences, one or both
cases will have to execute the nop'ed out contents of the other section, eg:
BEGIN_FTR_SECTION
or 1,1,1
END_FTR_SECTION_IFSET(FOO)
BEGIN_FTR_SECTION
or 2,2,2
END_FTR_SECTION_IFCLR(FOO)
and the resulting code will be either,
or 1,1,1
nop
or,
nop
or 2,2,2
For small code segments this is fine, but for larger code blocks and in
performance criticial code segments, it would be nice to avoid the nops.
This commit starts to implement logic to allow the following:
BEGIN_FTR_SECTION
or 1,1,1
FTR_SECTION_ELSE
or 2,2,2
ALT_FTR_SECTION_END_IFSET(FOO)
and the resulting code will be:
or 1,1,1
or,
or 2,2,2
We achieve this by extending the existing FTR macros. The current feature
section semantic just becomes a special case, ie. if the else case is empty
we nop out the default case.
The key limitation is that the size of the else case must be less than or
equal to the size of the default case. If the else case is smaller the
remainder of the section is nop'ed.
We let the linker put the else case code in with the rest of the text,
so that relative branches from the else case are more likley to link,
this has the disadvantage that we can't free the unused else cases.
This commit introduces the required macro and linker script changes, but
does not enable the patching of the alternative sections.
We also need to update two hand-made section entries in reg.h and timex.h
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-06-23 19:32:54 -06:00
|
|
|
#define END_FTR_SECTION_NESTED(msk, val, label) \
|
|
|
|
|
FTR_SECTION_ELSE_NESTED(label) \
|
2008-06-23 19:32:39 -06:00
|
|
|
MAKE_FTR_SECTION_ENTRY(msk, val, label, __ftr_fixup)
|
|
|
|
|
|
|
|
|
|
#define END_FTR_SECTION(msk, val) \
|
|
|
|
|
END_FTR_SECTION_NESTED(msk, val, 97)
|
|
|
|
|
|
2017-04-19 07:05:51 -06:00
|
|
|
#define END_FTR_SECTION_NESTED_IFSET(msk, label) \
|
|
|
|
|
END_FTR_SECTION_NESTED((msk), (msk), label)
|
|
|
|
|
|
2008-06-23 19:32:39 -06:00
|
|
|
#define END_FTR_SECTION_IFSET(msk) END_FTR_SECTION((msk), (msk))
|
|
|
|
|
#define END_FTR_SECTION_IFCLR(msk) END_FTR_SECTION((msk), 0)
|
|
|
|
|
|
powerpc: Introduce infrastructure for feature sections with alternatives
The current feature section logic only supports nop'ing out code, this means
if you want to choose at runtime between instruction sequences, one or both
cases will have to execute the nop'ed out contents of the other section, eg:
BEGIN_FTR_SECTION
or 1,1,1
END_FTR_SECTION_IFSET(FOO)
BEGIN_FTR_SECTION
or 2,2,2
END_FTR_SECTION_IFCLR(FOO)
and the resulting code will be either,
or 1,1,1
nop
or,
nop
or 2,2,2
For small code segments this is fine, but for larger code blocks and in
performance criticial code segments, it would be nice to avoid the nops.
This commit starts to implement logic to allow the following:
BEGIN_FTR_SECTION
or 1,1,1
FTR_SECTION_ELSE
or 2,2,2
ALT_FTR_SECTION_END_IFSET(FOO)
and the resulting code will be:
or 1,1,1
or,
or 2,2,2
We achieve this by extending the existing FTR macros. The current feature
section semantic just becomes a special case, ie. if the else case is empty
we nop out the default case.
The key limitation is that the size of the else case must be less than or
equal to the size of the default case. If the else case is smaller the
remainder of the section is nop'ed.
We let the linker put the else case code in with the rest of the text,
so that relative branches from the else case are more likley to link,
this has the disadvantage that we can't free the unused else cases.
This commit introduces the required macro and linker script changes, but
does not enable the patching of the alternative sections.
We also need to update two hand-made section entries in reg.h and timex.h
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-06-23 19:32:54 -06:00
|
|
|
/* CPU feature sections with alternatives, use BEGIN_FTR_SECTION to start */
|
|
|
|
|
#define FTR_SECTION_ELSE FTR_SECTION_ELSE_NESTED(97)
|
|
|
|
|
#define ALT_FTR_SECTION_END_NESTED(msk, val, label) \
|
|
|
|
|
MAKE_FTR_SECTION_ENTRY(msk, val, label, __ftr_fixup)
|
|
|
|
|
#define ALT_FTR_SECTION_END_NESTED_IFSET(msk, label) \
|
|
|
|
|
ALT_FTR_SECTION_END_NESTED(msk, msk, label)
|
|
|
|
|
#define ALT_FTR_SECTION_END_NESTED_IFCLR(msk, label) \
|
|
|
|
|
ALT_FTR_SECTION_END_NESTED(msk, 0, label)
|
|
|
|
|
#define ALT_FTR_SECTION_END(msk, val) \
|
|
|
|
|
ALT_FTR_SECTION_END_NESTED(msk, val, 97)
|
|
|
|
|
#define ALT_FTR_SECTION_END_IFSET(msk) \
|
|
|
|
|
ALT_FTR_SECTION_END_NESTED_IFSET(msk, 97)
|
|
|
|
|
#define ALT_FTR_SECTION_END_IFCLR(msk) \
|
|
|
|
|
ALT_FTR_SECTION_END_NESTED_IFCLR(msk, 97)
|
2008-06-23 19:32:39 -06:00
|
|
|
|
2008-12-18 12:13:32 -07:00
|
|
|
/* MMU feature dependent sections */
|
|
|
|
|
#define BEGIN_MMU_FTR_SECTION_NESTED(label) START_FTR_SECTION(label)
|
|
|
|
|
#define BEGIN_MMU_FTR_SECTION START_FTR_SECTION(97)
|
|
|
|
|
|
|
|
|
|
#define END_MMU_FTR_SECTION_NESTED(msk, val, label) \
|
|
|
|
|
FTR_SECTION_ELSE_NESTED(label) \
|
|
|
|
|
MAKE_FTR_SECTION_ENTRY(msk, val, label, __mmu_ftr_fixup)
|
|
|
|
|
|
|
|
|
|
#define END_MMU_FTR_SECTION(msk, val) \
|
|
|
|
|
END_MMU_FTR_SECTION_NESTED(msk, val, 97)
|
|
|
|
|
|
2019-04-18 00:51:24 -06:00
|
|
|
#define END_MMU_FTR_SECTION_NESTED_IFSET(msk, label) \
|
|
|
|
|
END_MMU_FTR_SECTION_NESTED((msk), (msk), label)
|
|
|
|
|
|
2008-12-18 12:13:32 -07:00
|
|
|
#define END_MMU_FTR_SECTION_IFSET(msk) END_MMU_FTR_SECTION((msk), (msk))
|
|
|
|
|
#define END_MMU_FTR_SECTION_IFCLR(msk) END_MMU_FTR_SECTION((msk), 0)
|
|
|
|
|
|
|
|
|
|
/* MMU feature sections with alternatives, use BEGIN_FTR_SECTION to start */
|
|
|
|
|
#define MMU_FTR_SECTION_ELSE_NESTED(label) FTR_SECTION_ELSE_NESTED(label)
|
|
|
|
|
#define MMU_FTR_SECTION_ELSE MMU_FTR_SECTION_ELSE_NESTED(97)
|
|
|
|
|
#define ALT_MMU_FTR_SECTION_END_NESTED(msk, val, label) \
|
|
|
|
|
MAKE_FTR_SECTION_ENTRY(msk, val, label, __mmu_ftr_fixup)
|
|
|
|
|
#define ALT_MMU_FTR_SECTION_END_NESTED_IFSET(msk, label) \
|
|
|
|
|
ALT_MMU_FTR_SECTION_END_NESTED(msk, msk, label)
|
|
|
|
|
#define ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(msk, label) \
|
|
|
|
|
ALT_MMU_FTR_SECTION_END_NESTED(msk, 0, label)
|
|
|
|
|
#define ALT_MMU_FTR_SECTION_END(msk, val) \
|
|
|
|
|
ALT_MMU_FTR_SECTION_END_NESTED(msk, val, 97)
|
|
|
|
|
#define ALT_MMU_FTR_SECTION_END_IFSET(msk) \
|
|
|
|
|
ALT_MMU_FTR_SECTION_END_NESTED_IFSET(msk, 97)
|
|
|
|
|
#define ALT_MMU_FTR_SECTION_END_IFCLR(msk) \
|
|
|
|
|
ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(msk, 97)
|
|
|
|
|
|
2008-06-23 19:32:39 -06:00
|
|
|
/* Firmware feature dependent sections */
|
powerpc: Introduce infrastructure for feature sections with alternatives
The current feature section logic only supports nop'ing out code, this means
if you want to choose at runtime between instruction sequences, one or both
cases will have to execute the nop'ed out contents of the other section, eg:
BEGIN_FTR_SECTION
or 1,1,1
END_FTR_SECTION_IFSET(FOO)
BEGIN_FTR_SECTION
or 2,2,2
END_FTR_SECTION_IFCLR(FOO)
and the resulting code will be either,
or 1,1,1
nop
or,
nop
or 2,2,2
For small code segments this is fine, but for larger code blocks and in
performance criticial code segments, it would be nice to avoid the nops.
This commit starts to implement logic to allow the following:
BEGIN_FTR_SECTION
or 1,1,1
FTR_SECTION_ELSE
or 2,2,2
ALT_FTR_SECTION_END_IFSET(FOO)
and the resulting code will be:
or 1,1,1
or,
or 2,2,2
We achieve this by extending the existing FTR macros. The current feature
section semantic just becomes a special case, ie. if the else case is empty
we nop out the default case.
The key limitation is that the size of the else case must be less than or
equal to the size of the default case. If the else case is smaller the
remainder of the section is nop'ed.
We let the linker put the else case code in with the rest of the text,
so that relative branches from the else case are more likley to link,
this has the disadvantage that we can't free the unused else cases.
This commit introduces the required macro and linker script changes, but
does not enable the patching of the alternative sections.
We also need to update two hand-made section entries in reg.h and timex.h
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-06-23 19:32:54 -06:00
|
|
|
#define BEGIN_FW_FTR_SECTION_NESTED(label) START_FTR_SECTION(label)
|
|
|
|
|
#define BEGIN_FW_FTR_SECTION START_FTR_SECTION(97)
|
2008-06-23 19:32:39 -06:00
|
|
|
|
powerpc: Introduce infrastructure for feature sections with alternatives
The current feature section logic only supports nop'ing out code, this means
if you want to choose at runtime between instruction sequences, one or both
cases will have to execute the nop'ed out contents of the other section, eg:
BEGIN_FTR_SECTION
or 1,1,1
END_FTR_SECTION_IFSET(FOO)
BEGIN_FTR_SECTION
or 2,2,2
END_FTR_SECTION_IFCLR(FOO)
and the resulting code will be either,
or 1,1,1
nop
or,
nop
or 2,2,2
For small code segments this is fine, but for larger code blocks and in
performance criticial code segments, it would be nice to avoid the nops.
This commit starts to implement logic to allow the following:
BEGIN_FTR_SECTION
or 1,1,1
FTR_SECTION_ELSE
or 2,2,2
ALT_FTR_SECTION_END_IFSET(FOO)
and the resulting code will be:
or 1,1,1
or,
or 2,2,2
We achieve this by extending the existing FTR macros. The current feature
section semantic just becomes a special case, ie. if the else case is empty
we nop out the default case.
The key limitation is that the size of the else case must be less than or
equal to the size of the default case. If the else case is smaller the
remainder of the section is nop'ed.
We let the linker put the else case code in with the rest of the text,
so that relative branches from the else case are more likley to link,
this has the disadvantage that we can't free the unused else cases.
This commit introduces the required macro and linker script changes, but
does not enable the patching of the alternative sections.
We also need to update two hand-made section entries in reg.h and timex.h
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-06-23 19:32:54 -06:00
|
|
|
#define END_FW_FTR_SECTION_NESTED(msk, val, label) \
|
|
|
|
|
FTR_SECTION_ELSE_NESTED(label) \
|
2008-06-23 19:32:39 -06:00
|
|
|
MAKE_FTR_SECTION_ENTRY(msk, val, label, __fw_ftr_fixup)
|
|
|
|
|
|
|
|
|
|
#define END_FW_FTR_SECTION(msk, val) \
|
|
|
|
|
END_FW_FTR_SECTION_NESTED(msk, val, 97)
|
|
|
|
|
|
|
|
|
|
#define END_FW_FTR_SECTION_IFSET(msk) END_FW_FTR_SECTION((msk), (msk))
|
|
|
|
|
#define END_FW_FTR_SECTION_IFCLR(msk) END_FW_FTR_SECTION((msk), 0)
|
|
|
|
|
|
powerpc: Introduce infrastructure for feature sections with alternatives
The current feature section logic only supports nop'ing out code, this means
if you want to choose at runtime between instruction sequences, one or both
cases will have to execute the nop'ed out contents of the other section, eg:
BEGIN_FTR_SECTION
or 1,1,1
END_FTR_SECTION_IFSET(FOO)
BEGIN_FTR_SECTION
or 2,2,2
END_FTR_SECTION_IFCLR(FOO)
and the resulting code will be either,
or 1,1,1
nop
or,
nop
or 2,2,2
For small code segments this is fine, but for larger code blocks and in
performance criticial code segments, it would be nice to avoid the nops.
This commit starts to implement logic to allow the following:
BEGIN_FTR_SECTION
or 1,1,1
FTR_SECTION_ELSE
or 2,2,2
ALT_FTR_SECTION_END_IFSET(FOO)
and the resulting code will be:
or 1,1,1
or,
or 2,2,2
We achieve this by extending the existing FTR macros. The current feature
section semantic just becomes a special case, ie. if the else case is empty
we nop out the default case.
The key limitation is that the size of the else case must be less than or
equal to the size of the default case. If the else case is smaller the
remainder of the section is nop'ed.
We let the linker put the else case code in with the rest of the text,
so that relative branches from the else case are more likley to link,
this has the disadvantage that we can't free the unused else cases.
This commit introduces the required macro and linker script changes, but
does not enable the patching of the alternative sections.
We also need to update two hand-made section entries in reg.h and timex.h
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-06-23 19:32:54 -06:00
|
|
|
/* Firmware feature sections with alternatives */
|
|
|
|
|
#define FW_FTR_SECTION_ELSE_NESTED(label) FTR_SECTION_ELSE_NESTED(label)
|
|
|
|
|
#define FW_FTR_SECTION_ELSE FTR_SECTION_ELSE_NESTED(97)
|
|
|
|
|
#define ALT_FW_FTR_SECTION_END_NESTED(msk, val, label) \
|
|
|
|
|
MAKE_FTR_SECTION_ENTRY(msk, val, label, __fw_ftr_fixup)
|
|
|
|
|
#define ALT_FW_FTR_SECTION_END_NESTED_IFSET(msk, label) \
|
|
|
|
|
ALT_FW_FTR_SECTION_END_NESTED(msk, msk, label)
|
|
|
|
|
#define ALT_FW_FTR_SECTION_END_NESTED_IFCLR(msk, label) \
|
|
|
|
|
ALT_FW_FTR_SECTION_END_NESTED(msk, 0, label)
|
|
|
|
|
#define ALT_FW_FTR_SECTION_END(msk, val) \
|
|
|
|
|
ALT_FW_FTR_SECTION_END_NESTED(msk, val, 97)
|
|
|
|
|
#define ALT_FW_FTR_SECTION_END_IFSET(msk) \
|
|
|
|
|
ALT_FW_FTR_SECTION_END_NESTED_IFSET(msk, 97)
|
|
|
|
|
#define ALT_FW_FTR_SECTION_END_IFCLR(msk) \
|
|
|
|
|
ALT_FW_FTR_SECTION_END_NESTED_IFCLR(msk, 97)
|
|
|
|
|
|
2009-04-29 14:58:01 -06:00
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
2011-04-06 12:23:29 -06:00
|
|
|
#define ASM_FTR_IF(section_if, section_else, msk, val) \
|
|
|
|
|
stringify_in_c(BEGIN_FTR_SECTION) \
|
|
|
|
|
section_if "; " \
|
|
|
|
|
stringify_in_c(FTR_SECTION_ELSE) \
|
|
|
|
|
section_else "; " \
|
|
|
|
|
stringify_in_c(ALT_FTR_SECTION_END((msk), (val)))
|
|
|
|
|
|
|
|
|
|
#define ASM_FTR_IFSET(section_if, section_else, msk) \
|
|
|
|
|
ASM_FTR_IF(section_if, section_else, (msk), (msk))
|
|
|
|
|
|
|
|
|
|
#define ASM_FTR_IFCLR(section_if, section_else, msk) \
|
|
|
|
|
ASM_FTR_IF(section_if, section_else, (msk), 0)
|
|
|
|
|
|
2009-04-29 14:58:01 -06:00
|
|
|
#define ASM_MMU_FTR_IF(section_if, section_else, msk, val) \
|
|
|
|
|
stringify_in_c(BEGIN_MMU_FTR_SECTION) \
|
|
|
|
|
section_if "; " \
|
|
|
|
|
stringify_in_c(MMU_FTR_SECTION_ELSE) \
|
|
|
|
|
section_else "; " \
|
|
|
|
|
stringify_in_c(ALT_MMU_FTR_SECTION_END((msk), (val)))
|
|
|
|
|
|
|
|
|
|
#define ASM_MMU_FTR_IFSET(section_if, section_else, msk) \
|
|
|
|
|
ASM_MMU_FTR_IF(section_if, section_else, (msk), (msk))
|
|
|
|
|
|
|
|
|
|
#define ASM_MMU_FTR_IFCLR(section_if, section_else, msk) \
|
|
|
|
|
ASM_MMU_FTR_IF(section_if, section_else, (msk), 0)
|
|
|
|
|
|
2008-06-23 19:32:39 -06:00
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
|
|
2008-07-01 09:16:40 -06:00
|
|
|
/* LWSYNC feature sections */
|
|
|
|
|
#define START_LWSYNC_SECTION(label) label##1:
|
|
|
|
|
#define MAKE_LWSYNC_SECTION_ENTRY(label, sect) \
|
|
|
|
|
label##2: \
|
|
|
|
|
.pushsection sect,"a"; \
|
|
|
|
|
.align 2; \
|
|
|
|
|
label##3: \
|
2010-02-26 00:29:17 -07:00
|
|
|
FTR_ENTRY_OFFSET label##1b-label##3b; \
|
2008-07-01 09:16:40 -06:00
|
|
|
.popsection;
|
|
|
|
|
|
2018-05-21 17:00:00 -06:00
|
|
|
#define STF_ENTRY_BARRIER_FIXUP_SECTION \
|
|
|
|
|
953: \
|
|
|
|
|
.pushsection __stf_entry_barrier_fixup,"a"; \
|
|
|
|
|
.align 2; \
|
|
|
|
|
954: \
|
|
|
|
|
FTR_ENTRY_OFFSET 953b-954b; \
|
|
|
|
|
.popsection;
|
|
|
|
|
|
|
|
|
|
#define STF_EXIT_BARRIER_FIXUP_SECTION \
|
|
|
|
|
955: \
|
|
|
|
|
.pushsection __stf_exit_barrier_fixup,"a"; \
|
|
|
|
|
.align 2; \
|
|
|
|
|
956: \
|
|
|
|
|
FTR_ENTRY_OFFSET 955b-956b; \
|
|
|
|
|
.popsection;
|
|
|
|
|
|
powerpc/64s: Add support for RFI flush of L1-D cache
On some CPUs we can prevent the Meltdown vulnerability by flushing the
L1-D cache on exit from kernel to user mode, and from hypervisor to
guest.
This is known to be the case on at least Power7, Power8 and Power9. At
this time we do not know the status of the vulnerability on other CPUs
such as the 970 (Apple G5), pasemi CPUs (AmigaOne X1000) or Freescale
CPUs. As more information comes to light we can enable this, or other
mechanisms on those CPUs.
The vulnerability occurs when the load of an architecturally
inaccessible memory region (eg. userspace load of kernel memory) is
speculatively executed to the point where its result can influence the
address of a subsequent speculatively executed load.
In order for that to happen, the first load must hit in the L1,
because before the load is sent to the L2 the permission check is
performed. Therefore if no kernel addresses hit in the L1 the
vulnerability can not occur. We can ensure that is the case by
flushing the L1 whenever we return to userspace. Similarly for
hypervisor vs guest.
In order to flush the L1-D cache on exit, we add a section of nops at
each (h)rfi location that returns to a lower privileged context, and
patch that with some sequence. Newer firmwares are able to advertise
to us that there is a special nop instruction that flushes the L1-D.
If we do not see that advertised, we fall back to doing a displacement
flush in software.
For guest kernels we support migration between some CPU versions, and
different CPUs may use different flush instructions. So that we are
prepared to migrate to a machine with a different flush instruction
activated, we may have to patch more than one flush instruction at
boot if the hypervisor tells us to.
In the end this patch is mostly the work of Nicholas Piggin and
Michael Ellerman. However a cast of thousands contributed to analysis
of the issue, earlier versions of the patch, back ports testing etc.
Many thanks to all of them.
Tested-by: Jon Masters <jcm@redhat.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2018-01-09 09:07:15 -07:00
|
|
|
#define RFI_FLUSH_FIXUP_SECTION \
|
|
|
|
|
951: \
|
|
|
|
|
.pushsection __rfi_flush_fixup,"a"; \
|
|
|
|
|
.align 2; \
|
|
|
|
|
952: \
|
|
|
|
|
FTR_ENTRY_OFFSET 951b-952b; \
|
|
|
|
|
.popsection;
|
|
|
|
|
|
2018-04-23 22:15:55 -06:00
|
|
|
#define NOSPEC_BARRIER_FIXUP_SECTION \
|
|
|
|
|
953: \
|
|
|
|
|
.pushsection __barrier_nospec_fixup,"a"; \
|
|
|
|
|
.align 2; \
|
|
|
|
|
954: \
|
|
|
|
|
FTR_ENTRY_OFFSET 953b-954b; \
|
|
|
|
|
.popsection;
|
|
|
|
|
|
2018-12-12 07:03:00 -07:00
|
|
|
#define START_BTB_FLUSH_SECTION \
|
|
|
|
|
955: \
|
|
|
|
|
|
|
|
|
|
#define END_BTB_FLUSH_SECTION \
|
|
|
|
|
956: \
|
|
|
|
|
.pushsection __btb_flush_fixup,"a"; \
|
|
|
|
|
.align 2; \
|
|
|
|
|
957: \
|
|
|
|
|
FTR_ENTRY_OFFSET 955b-957b; \
|
|
|
|
|
FTR_ENTRY_OFFSET 956b-957b; \
|
|
|
|
|
.popsection;
|
powerpc/64s: Add support for RFI flush of L1-D cache
On some CPUs we can prevent the Meltdown vulnerability by flushing the
L1-D cache on exit from kernel to user mode, and from hypervisor to
guest.
This is known to be the case on at least Power7, Power8 and Power9. At
this time we do not know the status of the vulnerability on other CPUs
such as the 970 (Apple G5), pasemi CPUs (AmigaOne X1000) or Freescale
CPUs. As more information comes to light we can enable this, or other
mechanisms on those CPUs.
The vulnerability occurs when the load of an architecturally
inaccessible memory region (eg. userspace load of kernel memory) is
speculatively executed to the point where its result can influence the
address of a subsequent speculatively executed load.
In order for that to happen, the first load must hit in the L1,
because before the load is sent to the L2 the permission check is
performed. Therefore if no kernel addresses hit in the L1 the
vulnerability can not occur. We can ensure that is the case by
flushing the L1 whenever we return to userspace. Similarly for
hypervisor vs guest.
In order to flush the L1-D cache on exit, we add a section of nops at
each (h)rfi location that returns to a lower privileged context, and
patch that with some sequence. Newer firmwares are able to advertise
to us that there is a special nop instruction that flushes the L1-D.
If we do not see that advertised, we fall back to doing a displacement
flush in software.
For guest kernels we support migration between some CPU versions, and
different CPUs may use different flush instructions. So that we are
prepared to migrate to a machine with a different flush instruction
activated, we may have to patch more than one flush instruction at
boot if the hypervisor tells us to.
In the end this patch is mostly the work of Nicholas Piggin and
Michael Ellerman. However a cast of thousands contributed to analysis
of the issue, earlier versions of the patch, back ports testing etc.
Many thanks to all of them.
Tested-by: Jon Masters <jcm@redhat.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2018-01-09 09:07:15 -07:00
|
|
|
|
2016-07-04 23:03:41 -06:00
|
|
|
#ifndef __ASSEMBLY__
|
powerpc/64s: Add support for RFI flush of L1-D cache
On some CPUs we can prevent the Meltdown vulnerability by flushing the
L1-D cache on exit from kernel to user mode, and from hypervisor to
guest.
This is known to be the case on at least Power7, Power8 and Power9. At
this time we do not know the status of the vulnerability on other CPUs
such as the 970 (Apple G5), pasemi CPUs (AmigaOne X1000) or Freescale
CPUs. As more information comes to light we can enable this, or other
mechanisms on those CPUs.
The vulnerability occurs when the load of an architecturally
inaccessible memory region (eg. userspace load of kernel memory) is
speculatively executed to the point where its result can influence the
address of a subsequent speculatively executed load.
In order for that to happen, the first load must hit in the L1,
because before the load is sent to the L2 the permission check is
performed. Therefore if no kernel addresses hit in the L1 the
vulnerability can not occur. We can ensure that is the case by
flushing the L1 whenever we return to userspace. Similarly for
hypervisor vs guest.
In order to flush the L1-D cache on exit, we add a section of nops at
each (h)rfi location that returns to a lower privileged context, and
patch that with some sequence. Newer firmwares are able to advertise
to us that there is a special nop instruction that flushes the L1-D.
If we do not see that advertised, we fall back to doing a displacement
flush in software.
For guest kernels we support migration between some CPU versions, and
different CPUs may use different flush instructions. So that we are
prepared to migrate to a machine with a different flush instruction
activated, we may have to patch more than one flush instruction at
boot if the hypervisor tells us to.
In the end this patch is mostly the work of Nicholas Piggin and
Michael Ellerman. However a cast of thousands contributed to analysis
of the issue, earlier versions of the patch, back ports testing etc.
Many thanks to all of them.
Tested-by: Jon Masters <jcm@redhat.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2018-01-09 09:07:15 -07:00
|
|
|
#include <linux/types.h>
|
|
|
|
|
|
2018-05-21 17:00:00 -06:00
|
|
|
extern long stf_barrier_fallback;
|
|
|
|
|
extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup;
|
|
|
|
|
extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup;
|
powerpc/64s: Add support for RFI flush of L1-D cache
On some CPUs we can prevent the Meltdown vulnerability by flushing the
L1-D cache on exit from kernel to user mode, and from hypervisor to
guest.
This is known to be the case on at least Power7, Power8 and Power9. At
this time we do not know the status of the vulnerability on other CPUs
such as the 970 (Apple G5), pasemi CPUs (AmigaOne X1000) or Freescale
CPUs. As more information comes to light we can enable this, or other
mechanisms on those CPUs.
The vulnerability occurs when the load of an architecturally
inaccessible memory region (eg. userspace load of kernel memory) is
speculatively executed to the point where its result can influence the
address of a subsequent speculatively executed load.
In order for that to happen, the first load must hit in the L1,
because before the load is sent to the L2 the permission check is
performed. Therefore if no kernel addresses hit in the L1 the
vulnerability can not occur. We can ensure that is the case by
flushing the L1 whenever we return to userspace. Similarly for
hypervisor vs guest.
In order to flush the L1-D cache on exit, we add a section of nops at
each (h)rfi location that returns to a lower privileged context, and
patch that with some sequence. Newer firmwares are able to advertise
to us that there is a special nop instruction that flushes the L1-D.
If we do not see that advertised, we fall back to doing a displacement
flush in software.
For guest kernels we support migration between some CPU versions, and
different CPUs may use different flush instructions. So that we are
prepared to migrate to a machine with a different flush instruction
activated, we may have to patch more than one flush instruction at
boot if the hypervisor tells us to.
In the end this patch is mostly the work of Nicholas Piggin and
Michael Ellerman. However a cast of thousands contributed to analysis
of the issue, earlier versions of the patch, back ports testing etc.
Many thanks to all of them.
Tested-by: Jon Masters <jcm@redhat.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2018-01-09 09:07:15 -07:00
|
|
|
extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;
|
2018-04-23 22:15:55 -06:00
|
|
|
extern long __start___barrier_nospec_fixup, __stop___barrier_nospec_fixup;
|
2018-12-12 07:03:00 -07:00
|
|
|
extern long __start__btb_flush_fixup, __stop__btb_flush_fixup;
|
powerpc/64s: Add support for RFI flush of L1-D cache
On some CPUs we can prevent the Meltdown vulnerability by flushing the
L1-D cache on exit from kernel to user mode, and from hypervisor to
guest.
This is known to be the case on at least Power7, Power8 and Power9. At
this time we do not know the status of the vulnerability on other CPUs
such as the 970 (Apple G5), pasemi CPUs (AmigaOne X1000) or Freescale
CPUs. As more information comes to light we can enable this, or other
mechanisms on those CPUs.
The vulnerability occurs when the load of an architecturally
inaccessible memory region (eg. userspace load of kernel memory) is
speculatively executed to the point where its result can influence the
address of a subsequent speculatively executed load.
In order for that to happen, the first load must hit in the L1,
because before the load is sent to the L2 the permission check is
performed. Therefore if no kernel addresses hit in the L1 the
vulnerability can not occur. We can ensure that is the case by
flushing the L1 whenever we return to userspace. Similarly for
hypervisor vs guest.
In order to flush the L1-D cache on exit, we add a section of nops at
each (h)rfi location that returns to a lower privileged context, and
patch that with some sequence. Newer firmwares are able to advertise
to us that there is a special nop instruction that flushes the L1-D.
If we do not see that advertised, we fall back to doing a displacement
flush in software.
For guest kernels we support migration between some CPU versions, and
different CPUs may use different flush instructions. So that we are
prepared to migrate to a machine with a different flush instruction
activated, we may have to patch more than one flush instruction at
boot if the hypervisor tells us to.
In the end this patch is mostly the work of Nicholas Piggin and
Michael Ellerman. However a cast of thousands contributed to analysis
of the issue, earlier versions of the patch, back ports testing etc.
Many thanks to all of them.
Tested-by: Jon Masters <jcm@redhat.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2018-01-09 09:07:15 -07:00
|
|
|
|
2016-07-04 23:03:41 -06:00
|
|
|
void apply_feature_fixups(void);
|
2016-08-10 01:27:34 -06:00
|
|
|
void setup_feature_keys(void);
|
2016-07-04 23:03:41 -06:00
|
|
|
#endif
|
|
|
|
|
|
2008-06-23 19:32:39 -06:00
|
|
|
#endif /* __ASM_POWERPC_FEATURE_FIXUPS_H */
|