2019-05-28 11:10:25 -06:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2007-07-21 09:10:01 -06:00
|
|
|
/*
|
|
|
|
* Copyright 2006 Andi Kleen, SUSE Labs.
|
|
|
|
*
|
2011-05-23 07:31:30 -06:00
|
|
|
* Fast user context implementation of clock_gettime, gettimeofday, and time.
|
2007-07-21 09:10:01 -06:00
|
|
|
*
|
2014-03-17 16:22:09 -06:00
|
|
|
* 32 Bit compat layer by Stefani Seibold <stefani@seibold.net>
|
|
|
|
* sponsored by Rohde & Schwarz GmbH & Co. KG Munich/Germany
|
|
|
|
*
|
2007-07-21 09:10:01 -06:00
|
|
|
* The code should have no internal unresolved relocations.
|
|
|
|
* Check with readelf after changing.
|
|
|
|
*/
|
|
|
|
|
2014-03-17 16:22:09 -06:00
|
|
|
#include <uapi/linux/time.h>
|
2007-07-21 09:10:01 -06:00
|
|
|
#include <asm/vgtod.h>
|
2014-03-17 16:22:10 -06:00
|
|
|
#include <asm/vvar.h>
|
2007-07-21 09:10:01 -06:00
|
|
|
#include <asm/unistd.h>
|
2014-03-17 16:22:10 -06:00
|
|
|
#include <asm/msr.h>
|
2015-12-10 20:20:22 -07:00
|
|
|
#include <asm/pvclock.h>
|
2017-03-03 06:21:42 -07:00
|
|
|
#include <asm/mshyperv.h>
|
2014-03-17 16:22:10 -06:00
|
|
|
#include <linux/math64.h>
|
|
|
|
#include <linux/time.h>
|
2015-12-10 20:20:22 -07:00
|
|
|
#include <linux/kernel.h>
|
2007-07-21 09:10:01 -06:00
|
|
|
|
2011-05-23 07:31:24 -06:00
|
|
|
#define gtod (&VVAR(vsyscall_gtod_data))
|
2007-07-21 09:10:01 -06:00
|
|
|
|
2014-03-17 16:22:09 -06:00
|
|
|
extern int __vdso_clock_gettime(clockid_t clock, struct timespec *ts);
|
|
|
|
extern int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz);
|
|
|
|
extern time_t __vdso_time(time_t *t);
|
|
|
|
|
2015-12-10 20:20:20 -07:00
|
|
|
#ifdef CONFIG_PARAVIRT_CLOCK
|
2019-05-01 12:20:53 -06:00
|
|
|
extern u8 pvclock_page[PAGE_SIZE]
|
2015-12-10 20:20:20 -07:00
|
|
|
__attribute__((visibility("hidden")));
|
|
|
|
#endif
|
|
|
|
|
2017-03-03 06:21:42 -07:00
|
|
|
#ifdef CONFIG_HYPERV_TSCPAGE
|
2019-05-01 12:20:53 -06:00
|
|
|
extern u8 hvclock_page[PAGE_SIZE]
|
2017-03-03 06:21:42 -07:00
|
|
|
__attribute__((visibility("hidden")));
|
|
|
|
#endif
|
|
|
|
|
2014-03-17 16:22:09 -06:00
|
|
|
#ifndef BUILD_VDSO32
|
|
|
|
|
2014-03-17 16:22:03 -06:00
|
|
|
notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
|
|
|
|
{
|
|
|
|
long ret;
|
x86/vdso: Fix asm constraints on vDSO syscall fallbacks
The syscall fallbacks in the vDSO have incorrect asm constraints.
They are not marked as writing to their outputs -- instead, they are
marked as clobbering "memory", which is useless. In particular, gcc
is smart enough to know that the timespec parameter hasn't escaped,
so a memory clobber doesn't clobber it. And passing a pointer as an
asm *input* does not tell gcc that the pointed-to value is changed.
Add in the fact that the asm instructions weren't volatile, and gcc
was free to omit them entirely unless their sole output (the return
value) is used. Which it is (phew!), but that stops happening with
some upcoming patches.
As a trivial example, the following code:
void test_fallback(struct timespec *ts)
{
vdso_fallback_gettime(CLOCK_MONOTONIC, ts);
}
compiles to:
00000000000000c0 <test_fallback>:
c0: c3 retq
To add insult to injury, the RCX and R11 clobbers on 64-bit
builds were missing.
The "memory" clobber is also unnecessary -- no ordering with respect to
other memory operations is needed, but that's going to be fixed in a
separate not-for-stable patch.
Fixes: 2aae950b21e4 ("x86_64: Add vDSO for x86-64 with gettimeofday/clock_gettime/getcpu")
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: stable@vger.kernel.org
Link: https://lkml.kernel.org/r/2c0231690551989d2fafa60ed0e7b5cc8b403908.1538422295.git.luto@kernel.org
2018-10-01 13:52:15 -06:00
|
|
|
asm ("syscall" : "=a" (ret), "=m" (*ts) :
|
|
|
|
"0" (__NR_clock_gettime), "D" (clock), "S" (ts) :
|
2018-10-04 15:44:43 -06:00
|
|
|
"rcx", "r11");
|
2014-03-17 16:22:03 -06:00
|
|
|
return ret;
|
2011-07-14 04:47:22 -06:00
|
|
|
}
|
|
|
|
|
2015-12-10 20:20:22 -07:00
|
|
|
#else
|
|
|
|
|
|
|
|
notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
|
|
|
|
{
|
|
|
|
long ret;
|
|
|
|
|
x86/vdso: Fix asm constraints on vDSO syscall fallbacks
The syscall fallbacks in the vDSO have incorrect asm constraints.
They are not marked as writing to their outputs -- instead, they are
marked as clobbering "memory", which is useless. In particular, gcc
is smart enough to know that the timespec parameter hasn't escaped,
so a memory clobber doesn't clobber it. And passing a pointer as an
asm *input* does not tell gcc that the pointed-to value is changed.
Add in the fact that the asm instructions weren't volatile, and gcc
was free to omit them entirely unless their sole output (the return
value) is used. Which it is (phew!), but that stops happening with
some upcoming patches.
As a trivial example, the following code:
void test_fallback(struct timespec *ts)
{
vdso_fallback_gettime(CLOCK_MONOTONIC, ts);
}
compiles to:
00000000000000c0 <test_fallback>:
c0: c3 retq
To add insult to injury, the RCX and R11 clobbers on 64-bit
builds were missing.
The "memory" clobber is also unnecessary -- no ordering with respect to
other memory operations is needed, but that's going to be fixed in a
separate not-for-stable patch.
Fixes: 2aae950b21e4 ("x86_64: Add vDSO for x86-64 with gettimeofday/clock_gettime/getcpu")
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: stable@vger.kernel.org
Link: https://lkml.kernel.org/r/2c0231690551989d2fafa60ed0e7b5cc8b403908.1538422295.git.luto@kernel.org
2018-10-01 13:52:15 -06:00
|
|
|
asm (
|
2015-12-10 20:20:22 -07:00
|
|
|
"mov %%ebx, %%edx \n"
|
2018-10-03 17:23:49 -06:00
|
|
|
"mov %[clock], %%ebx \n"
|
2015-12-10 20:20:22 -07:00
|
|
|
"call __kernel_vsyscall \n"
|
|
|
|
"mov %%edx, %%ebx \n"
|
x86/vdso: Fix asm constraints on vDSO syscall fallbacks
The syscall fallbacks in the vDSO have incorrect asm constraints.
They are not marked as writing to their outputs -- instead, they are
marked as clobbering "memory", which is useless. In particular, gcc
is smart enough to know that the timespec parameter hasn't escaped,
so a memory clobber doesn't clobber it. And passing a pointer as an
asm *input* does not tell gcc that the pointed-to value is changed.
Add in the fact that the asm instructions weren't volatile, and gcc
was free to omit them entirely unless their sole output (the return
value) is used. Which it is (phew!), but that stops happening with
some upcoming patches.
As a trivial example, the following code:
void test_fallback(struct timespec *ts)
{
vdso_fallback_gettime(CLOCK_MONOTONIC, ts);
}
compiles to:
00000000000000c0 <test_fallback>:
c0: c3 retq
To add insult to injury, the RCX and R11 clobbers on 64-bit
builds were missing.
The "memory" clobber is also unnecessary -- no ordering with respect to
other memory operations is needed, but that's going to be fixed in a
separate not-for-stable patch.
Fixes: 2aae950b21e4 ("x86_64: Add vDSO for x86-64 with gettimeofday/clock_gettime/getcpu")
Signed-off-by: Andy Lutomirski <luto@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: stable@vger.kernel.org
Link: https://lkml.kernel.org/r/2c0231690551989d2fafa60ed0e7b5cc8b403908.1538422295.git.luto@kernel.org
2018-10-01 13:52:15 -06:00
|
|
|
: "=a" (ret), "=m" (*ts)
|
2018-10-03 17:23:49 -06:00
|
|
|
: "0" (__NR_clock_gettime), [clock] "g" (clock), "c" (ts)
|
2018-10-04 15:44:43 -06:00
|
|
|
: "edx");
|
2015-12-10 20:20:22 -07:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_PARAVIRT_CLOCK
|
2015-12-10 20:20:20 -07:00
|
|
|
static notrace const struct pvclock_vsyscall_time_info *get_pvti0(void)
|
2012-11-27 18:28:57 -07:00
|
|
|
{
|
2015-12-10 20:20:20 -07:00
|
|
|
return (const struct pvclock_vsyscall_time_info *)&pvclock_page;
|
2012-11-27 18:28:57 -07:00
|
|
|
}
|
|
|
|
|
2018-09-17 06:45:42 -06:00
|
|
|
static notrace u64 vread_pvclock(void)
|
2012-11-27 18:28:57 -07:00
|
|
|
{
|
2015-12-10 20:20:20 -07:00
|
|
|
const struct pvclock_vcpu_time_info *pvti = &get_pvti0()->pvti;
|
2016-06-09 12:12:34 -06:00
|
|
|
u32 version;
|
2018-09-17 06:45:43 -06:00
|
|
|
u64 ret;
|
2012-11-27 18:28:57 -07:00
|
|
|
|
|
|
|
/*
|
2015-12-10 20:20:19 -07:00
|
|
|
* Note: The kernel and hypervisor must guarantee that cpu ID
|
|
|
|
* number maps 1:1 to per-CPU pvclock time info.
|
|
|
|
*
|
|
|
|
* Because the hypervisor is entirely unaware of guest userspace
|
|
|
|
* preemption, it cannot guarantee that per-CPU pvclock time
|
|
|
|
* info is updated if the underlying CPU changes or that that
|
|
|
|
* version is increased whenever underlying CPU changes.
|
|
|
|
*
|
|
|
|
* On KVM, we are guaranteed that pvti updates for any vCPU are
|
|
|
|
* atomic as seen by *all* vCPUs. This is an even stronger
|
|
|
|
* guarantee than we get with a normal seqlock.
|
2015-04-23 05:20:18 -06:00
|
|
|
*
|
2015-12-10 20:20:19 -07:00
|
|
|
* On Xen, we don't appear to have that guarantee, but Xen still
|
|
|
|
* supplies a valid seqlock using the version field.
|
2016-01-04 16:14:28 -07:00
|
|
|
*
|
2015-12-10 20:20:19 -07:00
|
|
|
* We only do pvclock vdso timing at all if
|
|
|
|
* PVCLOCK_TSC_STABLE_BIT is set, and we interpret that bit to
|
|
|
|
* mean that all vCPUs have matching pvti and that the TSC is
|
|
|
|
* synced, so we can just look at vCPU 0's pvti.
|
2012-11-27 18:28:57 -07:00
|
|
|
*/
|
2015-12-10 20:20:19 -07:00
|
|
|
|
|
|
|
do {
|
2016-06-09 05:06:08 -06:00
|
|
|
version = pvclock_read_begin(pvti);
|
2015-12-10 20:20:19 -07:00
|
|
|
|
2018-09-17 06:45:42 -06:00
|
|
|
if (unlikely(!(pvti->flags & PVCLOCK_TSC_STABLE_BIT)))
|
|
|
|
return U64_MAX;
|
2016-01-04 16:14:28 -07:00
|
|
|
|
2016-09-01 06:21:03 -06:00
|
|
|
ret = __pvclock_read_cycles(pvti, rdtsc_ordered());
|
2016-06-09 05:06:08 -06:00
|
|
|
} while (pvclock_read_retry(pvti, version));
|
2015-12-10 20:20:19 -07:00
|
|
|
|
2018-09-17 06:45:43 -06:00
|
|
|
return ret;
|
2012-11-27 18:28:57 -07:00
|
|
|
}
|
|
|
|
#endif
|
2017-03-03 06:21:42 -07:00
|
|
|
#ifdef CONFIG_HYPERV_TSCPAGE
|
2018-09-17 06:45:42 -06:00
|
|
|
static notrace u64 vread_hvclock(void)
|
2017-03-03 06:21:42 -07:00
|
|
|
{
|
|
|
|
const struct ms_hyperv_tsc_page *tsc_pg =
|
|
|
|
(const struct ms_hyperv_tsc_page *)&hvclock_page;
|
|
|
|
|
2018-09-17 06:45:42 -06:00
|
|
|
return hv_read_tsc_page(tsc_pg);
|
2017-03-03 06:21:42 -07:00
|
|
|
}
|
|
|
|
#endif
|
2012-11-27 18:28:57 -07:00
|
|
|
|
2018-09-17 06:45:42 -06:00
|
|
|
notrace static inline u64 vgetcyc(int mode)
|
2007-07-21 09:10:01 -06:00
|
|
|
{
|
2018-09-17 06:45:42 -06:00
|
|
|
if (mode == VCLOCK_TSC)
|
2018-09-17 06:45:43 -06:00
|
|
|
return (u64)rdtsc_ordered();
|
2019-06-21 09:43:04 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* For any memory-mapped vclock type, we need to make sure that gcc
|
|
|
|
* doesn't cleverly hoist a load before the mode check. Otherwise we
|
|
|
|
* might end up touching the memory-mapped page even if the vclock in
|
|
|
|
* question isn't enabled, which will segfault. Hence the barriers.
|
|
|
|
*/
|
2012-11-27 18:28:57 -07:00
|
|
|
#ifdef CONFIG_PARAVIRT_CLOCK
|
2019-06-21 09:43:04 -06:00
|
|
|
if (mode == VCLOCK_PVCLOCK) {
|
|
|
|
barrier();
|
2018-09-17 06:45:42 -06:00
|
|
|
return vread_pvclock();
|
2019-06-21 09:43:04 -06:00
|
|
|
}
|
2017-03-03 06:21:42 -07:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_HYPERV_TSCPAGE
|
2019-06-21 09:43:04 -06:00
|
|
|
if (mode == VCLOCK_HVCLOCK) {
|
|
|
|
barrier();
|
2018-09-17 06:45:42 -06:00
|
|
|
return vread_hvclock();
|
2019-06-21 09:43:04 -06:00
|
|
|
}
|
2012-11-27 18:28:57 -07:00
|
|
|
#endif
|
2018-09-17 06:45:42 -06:00
|
|
|
return U64_MAX;
|
2007-07-21 09:10:01 -06:00
|
|
|
}
|
|
|
|
|
2018-09-17 06:45:39 -06:00
|
|
|
notrace static int do_hres(clockid_t clk, struct timespec *ts)
|
2007-07-21 09:10:01 -06:00
|
|
|
{
|
2018-09-17 06:45:39 -06:00
|
|
|
struct vgtod_ts *base = >od->basetime[clk];
|
2018-10-05 12:02:43 -06:00
|
|
|
u64 cycles, last, sec, ns;
|
2018-09-17 06:45:37 -06:00
|
|
|
unsigned int seq;
|
2012-03-01 23:11:09 -07:00
|
|
|
|
2007-07-21 09:10:01 -06:00
|
|
|
do {
|
2014-03-17 16:22:10 -06:00
|
|
|
seq = gtod_read_begin(gtod);
|
2018-10-05 12:02:43 -06:00
|
|
|
cycles = vgetcyc(gtod->vclock_mode);
|
2018-09-17 06:45:38 -06:00
|
|
|
ns = base->nsec;
|
2018-09-17 06:45:43 -06:00
|
|
|
last = gtod->cycle_last;
|
2018-09-17 06:45:42 -06:00
|
|
|
if (unlikely((s64)cycles < 0))
|
|
|
|
return vdso_fallback_gettime(clk, ts);
|
2018-09-17 06:45:43 -06:00
|
|
|
if (cycles > last)
|
|
|
|
ns += (cycles - last) * gtod->mult;
|
2014-03-17 16:22:10 -06:00
|
|
|
ns >>= gtod->shift;
|
2018-10-05 12:02:43 -06:00
|
|
|
sec = base->sec;
|
2014-03-17 16:22:10 -06:00
|
|
|
} while (unlikely(gtod_read_retry(gtod, seq)));
|
|
|
|
|
2018-10-05 12:02:43 -06:00
|
|
|
/*
|
|
|
|
* Do this outside the loop: a race inside the loop could result
|
|
|
|
* in __iter_div_u64_rem() being extremely slow.
|
|
|
|
*/
|
|
|
|
ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
|
2014-03-17 16:22:10 -06:00
|
|
|
ts->tv_nsec = ns;
|
2011-05-23 07:31:27 -06:00
|
|
|
|
2018-09-17 06:45:42 -06:00
|
|
|
return 0;
|
2007-07-21 09:10:01 -06:00
|
|
|
}
|
|
|
|
|
2018-09-17 06:45:40 -06:00
|
|
|
notrace static void do_coarse(clockid_t clk, struct timespec *ts)
|
2009-08-19 20:13:34 -06:00
|
|
|
{
|
2018-09-17 06:45:40 -06:00
|
|
|
struct vgtod_ts *base = >od->basetime[clk];
|
2018-09-17 06:45:37 -06:00
|
|
|
unsigned int seq;
|
2018-09-17 06:45:38 -06:00
|
|
|
|
2009-08-19 20:13:34 -06:00
|
|
|
do {
|
2014-03-17 16:22:10 -06:00
|
|
|
seq = gtod_read_begin(gtod);
|
2018-09-17 06:45:38 -06:00
|
|
|
ts->tv_sec = base->sec;
|
|
|
|
ts->tv_nsec = base->nsec;
|
2014-03-17 16:22:10 -06:00
|
|
|
} while (unlikely(gtod_read_retry(gtod, seq)));
|
2009-08-19 20:13:34 -06:00
|
|
|
}
|
|
|
|
|
2008-05-12 13:20:41 -06:00
|
|
|
notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
|
2007-07-21 09:10:01 -06:00
|
|
|
{
|
2018-09-17 06:45:41 -06:00
|
|
|
unsigned int msk;
|
2011-06-05 11:50:20 -06:00
|
|
|
|
2018-09-17 06:45:41 -06:00
|
|
|
/* Sort out negative (CPU/FD) and invalid clocks */
|
|
|
|
if (unlikely((unsigned int) clock >= MAX_CLOCKS))
|
|
|
|
return vdso_fallback_gettime(clock, ts);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Convert the clockid to a bitmask and use it to check which
|
|
|
|
* clocks are handled in the VDSO directly.
|
|
|
|
*/
|
|
|
|
msk = 1U << clock;
|
|
|
|
if (likely(msk & VGTOD_HRES)) {
|
2018-09-17 06:45:42 -06:00
|
|
|
return do_hres(clock, ts);
|
2018-09-17 06:45:41 -06:00
|
|
|
} else if (msk & VGTOD_COARSE) {
|
|
|
|
do_coarse(clock, ts);
|
|
|
|
return 0;
|
|
|
|
}
|
2014-03-17 16:22:04 -06:00
|
|
|
return vdso_fallback_gettime(clock, ts);
|
2007-07-21 09:10:01 -06:00
|
|
|
}
|
2018-09-17 06:45:41 -06:00
|
|
|
|
2007-07-21 09:10:01 -06:00
|
|
|
int clock_gettime(clockid_t, struct timespec *)
|
|
|
|
__attribute__((weak, alias("__vdso_clock_gettime")));
|
|
|
|
|
2008-05-12 13:20:41 -06:00
|
|
|
notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
|
2007-07-21 09:10:01 -06:00
|
|
|
{
|
2012-03-01 23:11:09 -07:00
|
|
|
if (likely(tv != NULL)) {
|
2018-09-17 06:45:39 -06:00
|
|
|
struct timespec *ts = (struct timespec *) tv;
|
|
|
|
|
2018-09-17 06:45:42 -06:00
|
|
|
do_hres(CLOCK_REALTIME, ts);
|
2012-03-01 23:11:09 -07:00
|
|
|
tv->tv_usec /= 1000;
|
2007-07-21 09:10:01 -06:00
|
|
|
}
|
2012-03-01 23:11:09 -07:00
|
|
|
if (unlikely(tz != NULL)) {
|
2014-03-17 16:22:10 -06:00
|
|
|
tz->tz_minuteswest = gtod->tz_minuteswest;
|
|
|
|
tz->tz_dsttime = gtod->tz_dsttime;
|
2012-03-01 23:11:09 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
2007-07-21 09:10:01 -06:00
|
|
|
}
|
|
|
|
int gettimeofday(struct timeval *, struct timezone *)
|
|
|
|
__attribute__((weak, alias("__vdso_gettimeofday")));
|
2011-05-23 07:31:30 -06:00
|
|
|
|
2011-06-05 11:50:20 -06:00
|
|
|
/*
|
|
|
|
* This will break when the xtime seconds get inaccurate, but that is
|
|
|
|
* unlikely
|
|
|
|
*/
|
2011-05-23 07:31:30 -06:00
|
|
|
notrace time_t __vdso_time(time_t *t)
|
|
|
|
{
|
2014-03-17 16:22:09 -06:00
|
|
|
/* This is atomic on x86 so we don't need any locks. */
|
2018-09-17 06:45:38 -06:00
|
|
|
time_t result = READ_ONCE(gtod->basetime[CLOCK_REALTIME].sec);
|
2011-05-23 07:31:30 -06:00
|
|
|
|
|
|
|
if (t)
|
|
|
|
*t = result;
|
|
|
|
return result;
|
|
|
|
}
|
2017-12-04 08:01:55 -07:00
|
|
|
time_t time(time_t *t)
|
2011-05-23 07:31:30 -06:00
|
|
|
__attribute__((weak, alias("__vdso_time")));
|