![Nadav Amit](/assets/img/avatar_default.png)
As described in:
77b0bf55bc
: ("kbuild/Makefile: Prepare for using macros in inline assembly code to work around asm() related GCC inlining bugs")
GCC's inlining heuristics are broken with common asm() patterns used in
kernel code, resulting in the effective disabling of inlining.
The workaround is to set an assembly macro and call it from the inline
assembly block. As a result GCC considers the inline assembly block as
a single instruction. (Which it isn't, but that's the best we can get.)
This patch increases the kernel size:
text data bss dec hex filename
18146889 10225380 2957312 31329581 1de0d2d ./vmlinux before
18147336 10226688 2957312 31331336 1de1408 ./vmlinux after (+1755)
But enables more aggressive inlining (and probably better branch decisions).
The number of static text symbols in vmlinux is much lower:
Before: 40218
After: 40053 (-165)
The assembly code gets harder to read due to the extra macro layer.
[ mingo: Rewrote the changelog. ]
Tested-by: Kees Cook <keescook@chromium.org>
Signed-off-by: Nadav Amit <namit@vmware.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/20181003213100.189959-7-namit@vmware.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
253 lines
7.1 KiB
C
253 lines
7.1 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_GENERIC_BUG_H
|
|
#define _ASM_GENERIC_BUG_H
|
|
|
|
#include <linux/compiler.h>
|
|
|
|
#define CUT_HERE "------------[ cut here ]------------\n"
|
|
|
|
#ifdef CONFIG_GENERIC_BUG
|
|
#define BUGFLAG_WARNING (1 << 0)
|
|
#define BUGFLAG_ONCE (1 << 1)
|
|
#define BUGFLAG_DONE (1 << 2)
|
|
#define BUGFLAG_TAINT(taint) ((taint) << 8)
|
|
#define BUG_GET_TAINT(bug) ((bug)->flags >> 8)
|
|
#endif
|
|
|
|
#ifndef __ASSEMBLY__
|
|
#include <linux/kernel.h>
|
|
|
|
struct bug_entry {
|
|
#ifdef CONFIG_GENERIC_BUG
|
|
#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
|
|
unsigned long bug_addr;
|
|
#else
|
|
signed int bug_addr_disp;
|
|
#endif
|
|
#ifdef CONFIG_DEBUG_BUGVERBOSE
|
|
#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
|
|
const char *file;
|
|
#else
|
|
signed int file_disp;
|
|
#endif
|
|
unsigned short line;
|
|
#endif
|
|
unsigned short flags;
|
|
#endif /* CONFIG_GENERIC_BUG */
|
|
};
|
|
|
|
#ifdef CONFIG_BUG
|
|
|
|
/*
|
|
* Don't use BUG() or BUG_ON() unless there's really no way out; one
|
|
* example might be detecting data structure corruption in the middle
|
|
* of an operation that can't be backed out of. If the (sub)system
|
|
* can somehow continue operating, perhaps with reduced functionality,
|
|
* it's probably not BUG-worthy.
|
|
*
|
|
* If you're tempted to BUG(), think again: is completely giving up
|
|
* really the *only* solution? There are usually better options, where
|
|
* users don't need to reboot ASAP and can mostly shut down cleanly.
|
|
*/
|
|
#ifndef HAVE_ARCH_BUG
|
|
#define BUG() do { \
|
|
printk("BUG: failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__); \
|
|
barrier_before_unreachable(); \
|
|
panic("BUG!"); \
|
|
} while (0)
|
|
#endif
|
|
|
|
#ifndef HAVE_ARCH_BUG_ON
|
|
#define BUG_ON(condition) do { if (unlikely(condition)) BUG(); } while (0)
|
|
#endif
|
|
|
|
#ifdef __WARN_FLAGS
|
|
#define __WARN_TAINT(taint) __WARN_FLAGS(BUGFLAG_TAINT(taint))
|
|
#define __WARN_ONCE_TAINT(taint) __WARN_FLAGS(BUGFLAG_ONCE|BUGFLAG_TAINT(taint))
|
|
|
|
#define WARN_ON_ONCE(condition) ({ \
|
|
int __ret_warn_on = !!(condition); \
|
|
if (unlikely(__ret_warn_on)) \
|
|
__WARN_ONCE_TAINT(TAINT_WARN); \
|
|
unlikely(__ret_warn_on); \
|
|
})
|
|
#endif
|
|
|
|
/*
|
|
* WARN(), WARN_ON(), WARN_ON_ONCE, and so on can be used to report
|
|
* significant kernel issues that need prompt attention if they should ever
|
|
* appear at runtime.
|
|
*
|
|
* Do not use these macros when checking for invalid external inputs
|
|
* (e.g. invalid system call arguments, or invalid data coming from
|
|
* network/devices), and on transient conditions like ENOMEM or EAGAIN.
|
|
* These macros should be used for recoverable kernel issues only.
|
|
* For invalid external inputs, transient conditions, etc use
|
|
* pr_err[_once/_ratelimited]() followed by dump_stack(), if necessary.
|
|
* Do not include "BUG"/"WARNING" in format strings manually to make these
|
|
* conditions distinguishable from kernel issues.
|
|
*
|
|
* Use the versions with printk format strings to provide better diagnostics.
|
|
*/
|
|
#ifndef __WARN_TAINT
|
|
extern __printf(3, 4)
|
|
void warn_slowpath_fmt(const char *file, const int line,
|
|
const char *fmt, ...);
|
|
extern __printf(4, 5)
|
|
void warn_slowpath_fmt_taint(const char *file, const int line, unsigned taint,
|
|
const char *fmt, ...);
|
|
extern void warn_slowpath_null(const char *file, const int line);
|
|
#define WANT_WARN_ON_SLOWPATH
|
|
#define __WARN() warn_slowpath_null(__FILE__, __LINE__)
|
|
#define __WARN_printf(arg...) warn_slowpath_fmt(__FILE__, __LINE__, arg)
|
|
#define __WARN_printf_taint(taint, arg...) \
|
|
warn_slowpath_fmt_taint(__FILE__, __LINE__, taint, arg)
|
|
#else
|
|
extern __printf(1, 2) void __warn_printk(const char *fmt, ...);
|
|
#define __WARN() __WARN_TAINT(TAINT_WARN)
|
|
#define __WARN_printf(arg...) do { __warn_printk(arg); __WARN(); } while (0)
|
|
#define __WARN_printf_taint(taint, arg...) \
|
|
do { __warn_printk(arg); __WARN_TAINT(taint); } while (0)
|
|
#endif
|
|
|
|
/* used internally by panic.c */
|
|
struct warn_args;
|
|
struct pt_regs;
|
|
|
|
void __warn(const char *file, int line, void *caller, unsigned taint,
|
|
struct pt_regs *regs, struct warn_args *args);
|
|
|
|
#ifndef WARN_ON
|
|
#define WARN_ON(condition) ({ \
|
|
int __ret_warn_on = !!(condition); \
|
|
if (unlikely(__ret_warn_on)) \
|
|
__WARN(); \
|
|
unlikely(__ret_warn_on); \
|
|
})
|
|
#endif
|
|
|
|
#ifndef WARN
|
|
#define WARN(condition, format...) ({ \
|
|
int __ret_warn_on = !!(condition); \
|
|
if (unlikely(__ret_warn_on)) \
|
|
__WARN_printf(format); \
|
|
unlikely(__ret_warn_on); \
|
|
})
|
|
#endif
|
|
|
|
#define WARN_TAINT(condition, taint, format...) ({ \
|
|
int __ret_warn_on = !!(condition); \
|
|
if (unlikely(__ret_warn_on)) \
|
|
__WARN_printf_taint(taint, format); \
|
|
unlikely(__ret_warn_on); \
|
|
})
|
|
|
|
#ifndef WARN_ON_ONCE
|
|
#define WARN_ON_ONCE(condition) ({ \
|
|
static bool __section(.data.once) __warned; \
|
|
int __ret_warn_once = !!(condition); \
|
|
\
|
|
if (unlikely(__ret_warn_once && !__warned)) { \
|
|
__warned = true; \
|
|
WARN_ON(1); \
|
|
} \
|
|
unlikely(__ret_warn_once); \
|
|
})
|
|
#endif
|
|
|
|
#define WARN_ONCE(condition, format...) ({ \
|
|
static bool __section(.data.once) __warned; \
|
|
int __ret_warn_once = !!(condition); \
|
|
\
|
|
if (unlikely(__ret_warn_once && !__warned)) { \
|
|
__warned = true; \
|
|
WARN(1, format); \
|
|
} \
|
|
unlikely(__ret_warn_once); \
|
|
})
|
|
|
|
#define WARN_TAINT_ONCE(condition, taint, format...) ({ \
|
|
static bool __section(.data.once) __warned; \
|
|
int __ret_warn_once = !!(condition); \
|
|
\
|
|
if (unlikely(__ret_warn_once && !__warned)) { \
|
|
__warned = true; \
|
|
WARN_TAINT(1, taint, format); \
|
|
} \
|
|
unlikely(__ret_warn_once); \
|
|
})
|
|
|
|
#else /* !CONFIG_BUG */
|
|
#ifndef HAVE_ARCH_BUG
|
|
#define BUG() do {} while (1)
|
|
#endif
|
|
|
|
#ifndef HAVE_ARCH_BUG_ON
|
|
#define BUG_ON(condition) do { if (condition) BUG(); } while (0)
|
|
#endif
|
|
|
|
#ifndef HAVE_ARCH_WARN_ON
|
|
#define WARN_ON(condition) ({ \
|
|
int __ret_warn_on = !!(condition); \
|
|
unlikely(__ret_warn_on); \
|
|
})
|
|
#endif
|
|
|
|
#ifndef WARN
|
|
#define WARN(condition, format...) ({ \
|
|
int __ret_warn_on = !!(condition); \
|
|
no_printk(format); \
|
|
unlikely(__ret_warn_on); \
|
|
})
|
|
#endif
|
|
|
|
#define WARN_ON_ONCE(condition) WARN_ON(condition)
|
|
#define WARN_ONCE(condition, format...) WARN(condition, format)
|
|
#define WARN_TAINT(condition, taint, format...) WARN(condition, format)
|
|
#define WARN_TAINT_ONCE(condition, taint, format...) WARN(condition, format)
|
|
|
|
#endif
|
|
|
|
/*
|
|
* WARN_ON_SMP() is for cases that the warning is either
|
|
* meaningless for !SMP or may even cause failures.
|
|
* This is usually used for cases that we have
|
|
* WARN_ON(!spin_is_locked(&lock)) checks, as spin_is_locked()
|
|
* returns 0 for uniprocessor settings.
|
|
* It can also be used with values that are only defined
|
|
* on SMP:
|
|
*
|
|
* struct foo {
|
|
* [...]
|
|
* #ifdef CONFIG_SMP
|
|
* int bar;
|
|
* #endif
|
|
* };
|
|
*
|
|
* void func(struct foo *zoot)
|
|
* {
|
|
* WARN_ON_SMP(!zoot->bar);
|
|
*
|
|
* For CONFIG_SMP, WARN_ON_SMP() should act the same as WARN_ON(),
|
|
* and should be a nop and return false for uniprocessor.
|
|
*
|
|
* if (WARN_ON_SMP(x)) returns true only when CONFIG_SMP is set
|
|
* and x is true.
|
|
*/
|
|
#ifdef CONFIG_SMP
|
|
# define WARN_ON_SMP(x) WARN_ON(x)
|
|
#else
|
|
/*
|
|
* Use of ({0;}) because WARN_ON_SMP(x) may be used either as
|
|
* a stand alone line statement or as a condition in an if ()
|
|
* statement.
|
|
* A simple "0" would cause gcc to give a "statement has no effect"
|
|
* warning.
|
|
*/
|
|
# define WARN_ON_SMP(x) ({0;})
|
|
#endif
|
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif
|