268bb0ce3e
Commit e66eed651f ("list: remove prefetching from regular list
iterators") removed the include of prefetch.h from list.h, which
uncovered several cases that had apparently relied on that rather
obscure header file dependency.
So this fixes things up a bit, using
grep -L linux/prefetch.h $(git grep -l '[^a-z_]prefetchw*(' -- '*.[ch]')
grep -L 'prefetchw*(' $(git grep -l 'linux/prefetch.h' -- '*.[ch]')
to guide us in finding files that either need <linux/prefetch.h>
inclusion, or have it despite not needing it.
There are more of them around (mostly network drivers), but this gets
many core ones.
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
254 lines
6.8 KiB
C
254 lines
6.8 KiB
C
#ifndef _ASM_X86_UACCESS_64_H
|
|
#define _ASM_X86_UACCESS_64_H
|
|
|
|
/*
|
|
* User space memory access functions
|
|
*/
|
|
#include <linux/compiler.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/lockdep.h>
|
|
#include <asm/alternative.h>
|
|
#include <asm/cpufeature.h>
|
|
#include <asm/page.h>
|
|
|
|
/*
|
|
* Copy To/From Userspace
|
|
*/
|
|
|
|
/* Handles exceptions in both to and from, but doesn't do access_ok */
|
|
__must_check unsigned long
|
|
copy_user_generic_string(void *to, const void *from, unsigned len);
|
|
__must_check unsigned long
|
|
copy_user_generic_unrolled(void *to, const void *from, unsigned len);
|
|
|
|
static __always_inline __must_check unsigned long
|
|
copy_user_generic(void *to, const void *from, unsigned len)
|
|
{
|
|
unsigned ret;
|
|
|
|
alternative_call(copy_user_generic_unrolled,
|
|
copy_user_generic_string,
|
|
X86_FEATURE_REP_GOOD,
|
|
ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
|
|
"=d" (len)),
|
|
"1" (to), "2" (from), "3" (len)
|
|
: "memory", "rcx", "r8", "r9", "r10", "r11");
|
|
return ret;
|
|
}
|
|
|
|
__must_check unsigned long
|
|
_copy_to_user(void __user *to, const void *from, unsigned len);
|
|
__must_check unsigned long
|
|
_copy_from_user(void *to, const void __user *from, unsigned len);
|
|
__must_check unsigned long
|
|
copy_in_user(void __user *to, const void __user *from, unsigned len);
|
|
|
|
static inline unsigned long __must_check copy_from_user(void *to,
|
|
const void __user *from,
|
|
unsigned long n)
|
|
{
|
|
int sz = __compiletime_object_size(to);
|
|
|
|
might_fault();
|
|
if (likely(sz == -1 || sz >= n))
|
|
n = _copy_from_user(to, from, n);
|
|
#ifdef CONFIG_DEBUG_VM
|
|
else
|
|
WARN(1, "Buffer overflow detected!\n");
|
|
#endif
|
|
return n;
|
|
}
|
|
|
|
static __always_inline __must_check
|
|
int copy_to_user(void __user *dst, const void *src, unsigned size)
|
|
{
|
|
might_fault();
|
|
|
|
return _copy_to_user(dst, src, size);
|
|
}
|
|
|
|
static __always_inline __must_check
|
|
int __copy_from_user(void *dst, const void __user *src, unsigned size)
|
|
{
|
|
int ret = 0;
|
|
|
|
might_fault();
|
|
if (!__builtin_constant_p(size))
|
|
return copy_user_generic(dst, (__force void *)src, size);
|
|
switch (size) {
|
|
case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
|
|
ret, "b", "b", "=q", 1);
|
|
return ret;
|
|
case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
|
|
ret, "w", "w", "=r", 2);
|
|
return ret;
|
|
case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
|
|
ret, "l", "k", "=r", 4);
|
|
return ret;
|
|
case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
|
|
ret, "q", "", "=r", 8);
|
|
return ret;
|
|
case 10:
|
|
__get_user_asm(*(u64 *)dst, (u64 __user *)src,
|
|
ret, "q", "", "=r", 10);
|
|
if (unlikely(ret))
|
|
return ret;
|
|
__get_user_asm(*(u16 *)(8 + (char *)dst),
|
|
(u16 __user *)(8 + (char __user *)src),
|
|
ret, "w", "w", "=r", 2);
|
|
return ret;
|
|
case 16:
|
|
__get_user_asm(*(u64 *)dst, (u64 __user *)src,
|
|
ret, "q", "", "=r", 16);
|
|
if (unlikely(ret))
|
|
return ret;
|
|
__get_user_asm(*(u64 *)(8 + (char *)dst),
|
|
(u64 __user *)(8 + (char __user *)src),
|
|
ret, "q", "", "=r", 8);
|
|
return ret;
|
|
default:
|
|
return copy_user_generic(dst, (__force void *)src, size);
|
|
}
|
|
}
|
|
|
|
static __always_inline __must_check
|
|
int __copy_to_user(void __user *dst, const void *src, unsigned size)
|
|
{
|
|
int ret = 0;
|
|
|
|
might_fault();
|
|
if (!__builtin_constant_p(size))
|
|
return copy_user_generic((__force void *)dst, src, size);
|
|
switch (size) {
|
|
case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
|
|
ret, "b", "b", "iq", 1);
|
|
return ret;
|
|
case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
|
|
ret, "w", "w", "ir", 2);
|
|
return ret;
|
|
case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
|
|
ret, "l", "k", "ir", 4);
|
|
return ret;
|
|
case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
|
|
ret, "q", "", "er", 8);
|
|
return ret;
|
|
case 10:
|
|
__put_user_asm(*(u64 *)src, (u64 __user *)dst,
|
|
ret, "q", "", "er", 10);
|
|
if (unlikely(ret))
|
|
return ret;
|
|
asm("":::"memory");
|
|
__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
|
|
ret, "w", "w", "ir", 2);
|
|
return ret;
|
|
case 16:
|
|
__put_user_asm(*(u64 *)src, (u64 __user *)dst,
|
|
ret, "q", "", "er", 16);
|
|
if (unlikely(ret))
|
|
return ret;
|
|
asm("":::"memory");
|
|
__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
|
|
ret, "q", "", "er", 8);
|
|
return ret;
|
|
default:
|
|
return copy_user_generic((__force void *)dst, src, size);
|
|
}
|
|
}
|
|
|
|
static __always_inline __must_check
|
|
int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
|
|
{
|
|
int ret = 0;
|
|
|
|
might_fault();
|
|
if (!__builtin_constant_p(size))
|
|
return copy_user_generic((__force void *)dst,
|
|
(__force void *)src, size);
|
|
switch (size) {
|
|
case 1: {
|
|
u8 tmp;
|
|
__get_user_asm(tmp, (u8 __user *)src,
|
|
ret, "b", "b", "=q", 1);
|
|
if (likely(!ret))
|
|
__put_user_asm(tmp, (u8 __user *)dst,
|
|
ret, "b", "b", "iq", 1);
|
|
return ret;
|
|
}
|
|
case 2: {
|
|
u16 tmp;
|
|
__get_user_asm(tmp, (u16 __user *)src,
|
|
ret, "w", "w", "=r", 2);
|
|
if (likely(!ret))
|
|
__put_user_asm(tmp, (u16 __user *)dst,
|
|
ret, "w", "w", "ir", 2);
|
|
return ret;
|
|
}
|
|
|
|
case 4: {
|
|
u32 tmp;
|
|
__get_user_asm(tmp, (u32 __user *)src,
|
|
ret, "l", "k", "=r", 4);
|
|
if (likely(!ret))
|
|
__put_user_asm(tmp, (u32 __user *)dst,
|
|
ret, "l", "k", "ir", 4);
|
|
return ret;
|
|
}
|
|
case 8: {
|
|
u64 tmp;
|
|
__get_user_asm(tmp, (u64 __user *)src,
|
|
ret, "q", "", "=r", 8);
|
|
if (likely(!ret))
|
|
__put_user_asm(tmp, (u64 __user *)dst,
|
|
ret, "q", "", "er", 8);
|
|
return ret;
|
|
}
|
|
default:
|
|
return copy_user_generic((__force void *)dst,
|
|
(__force void *)src, size);
|
|
}
|
|
}
|
|
|
|
__must_check long
|
|
strncpy_from_user(char *dst, const char __user *src, long count);
|
|
__must_check long
|
|
__strncpy_from_user(char *dst, const char __user *src, long count);
|
|
__must_check long strnlen_user(const char __user *str, long n);
|
|
__must_check long __strnlen_user(const char __user *str, long n);
|
|
__must_check long strlen_user(const char __user *str);
|
|
__must_check unsigned long clear_user(void __user *mem, unsigned long len);
|
|
__must_check unsigned long __clear_user(void __user *mem, unsigned long len);
|
|
|
|
static __must_check __always_inline int
|
|
__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
|
|
{
|
|
return copy_user_generic(dst, (__force const void *)src, size);
|
|
}
|
|
|
|
static __must_check __always_inline int
|
|
__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
|
|
{
|
|
return copy_user_generic((__force void *)dst, src, size);
|
|
}
|
|
|
|
extern long __copy_user_nocache(void *dst, const void __user *src,
|
|
unsigned size, int zerorest);
|
|
|
|
static inline int
|
|
__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
|
|
{
|
|
might_sleep();
|
|
return __copy_user_nocache(dst, src, size, 1);
|
|
}
|
|
|
|
static inline int
|
|
__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
|
|
unsigned size)
|
|
{
|
|
return __copy_user_nocache(dst, src, size, 0);
|
|
}
|
|
|
|
unsigned long
|
|
copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest);
|
|
|
|
#endif /* _ASM_X86_UACCESS_64_H */
|