2005-04-16 16:20:36 -06:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
|
|
|
|
* Copyright 2003 Andi Kleen, SuSE Labs.
|
|
|
|
*
|
|
|
|
* Thanks to hpa@transmeta.com for some useful hint.
|
|
|
|
* Special thanks to Ingo Molnar for his early experience with
|
|
|
|
* a different vsyscall implementation for Linux/IA32 and for the name.
|
|
|
|
*
|
|
|
|
* vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
|
|
|
|
* at virtual address -10Mbyte+1024bytes etc... There are at max 4
|
|
|
|
* vsyscalls. One vsyscall can reserve more than 1 slot to avoid
|
|
|
|
* jumping out of line if necessary. We cannot add more with this
|
|
|
|
* mechanism because older kernels won't return -ENOSYS.
|
|
|
|
* If we want more than four we need a vDSO.
|
|
|
|
*
|
|
|
|
* Note: the concept clashes with user mode linux. If you use UML and
|
|
|
|
* want per guest time just set the kernel.vsyscall64 sysctl to 0.
|
|
|
|
*/
|
|
|
|
|
2008-11-12 05:17:38 -07:00
|
|
|
/* Disable profiling for userspace code: */
|
2008-11-12 13:24:24 -07:00
|
|
|
#define DISABLE_BRANCH_PROFILING
|
2008-11-11 22:14:39 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/time.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/timer.h>
|
|
|
|
#include <linux/seqlock.h>
|
|
|
|
#include <linux/jiffies.h>
|
|
|
|
#include <linux/sysctl.h>
|
2007-02-16 02:28:21 -07:00
|
|
|
#include <linux/clocksource.h>
|
2006-09-26 02:52:28 -06:00
|
|
|
#include <linux/getcpu.h>
|
2006-11-14 08:57:46 -07:00
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/smp.h>
|
|
|
|
#include <linux/notifier.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
#include <asm/vsyscall.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/page.h>
|
2007-02-16 02:28:21 -07:00
|
|
|
#include <asm/unistd.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <asm/fixmap.h>
|
|
|
|
#include <asm/errno.h>
|
|
|
|
#include <asm/io.h>
|
2006-09-26 02:52:28 -06:00
|
|
|
#include <asm/segment.h>
|
|
|
|
#include <asm/desc.h>
|
|
|
|
#include <asm/topology.h>
|
2007-07-21 09:10:01 -06:00
|
|
|
#include <asm/vgtod.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-05-12 13:20:41 -06:00
|
|
|
#define __vsyscall(nr) \
|
|
|
|
__attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
|
2008-01-30 05:30:56 -07:00
|
|
|
#define __syscall_clobber "r11","cx","memory"
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-05-02 11:27:11 -06:00
|
|
|
/*
|
|
|
|
* vsyscall_gtod_data contains data that is :
|
|
|
|
* - readonly from vsyscalls
|
2007-10-19 17:25:36 -06:00
|
|
|
* - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
|
2007-05-02 11:27:11 -06:00
|
|
|
* Try to keep this structure as small as possible to avoid cache line ping pongs
|
|
|
|
*/
|
2006-09-26 02:52:28 -06:00
|
|
|
int __vgetcpu_mode __section_vgetcpu_mode;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-07-21 09:10:01 -06:00
|
|
|
struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-02-16 02:28:21 -07:00
|
|
|
.lock = SEQLOCK_UNLOCKED,
|
|
|
|
.sysctl_enabled = 1,
|
|
|
|
};
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-10-18 04:04:57 -06:00
|
|
|
void update_vsyscall_tz(void)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
|
|
|
|
/* sys_tz has changed */
|
|
|
|
vsyscall_gtod_data.sys_tz = sys_tz;
|
|
|
|
write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
|
|
|
|
}
|
|
|
|
|
2007-02-16 02:28:21 -07:00
|
|
|
void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-02-16 02:28:21 -07:00
|
|
|
unsigned long flags;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-02-16 02:28:21 -07:00
|
|
|
write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
|
|
|
|
/* copy vsyscall data */
|
2007-05-02 11:27:11 -06:00
|
|
|
vsyscall_gtod_data.clock.vread = clock->vread;
|
|
|
|
vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
|
|
|
|
vsyscall_gtod_data.clock.mask = clock->mask;
|
|
|
|
vsyscall_gtod_data.clock.mult = clock->mult;
|
|
|
|
vsyscall_gtod_data.clock.shift = clock->shift;
|
|
|
|
vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
|
|
|
|
vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
|
2007-07-21 09:10:01 -06:00
|
|
|
vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
|
2009-08-19 20:13:34 -06:00
|
|
|
vsyscall_gtod_data.wall_time_coarse = __current_kernel_time();
|
2007-02-16 02:28:21 -07:00
|
|
|
write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2007-02-16 02:28:21 -07:00
|
|
|
/* RED-PEN may want to readd seq locking, but then the variable should be
|
|
|
|
* write-once.
|
|
|
|
*/
|
2006-01-11 14:45:30 -07:00
|
|
|
static __always_inline void do_get_tz(struct timezone * tz)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-02-16 02:28:21 -07:00
|
|
|
*tz = __vsyscall_gtod_data.sys_tz;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-01-11 14:45:30 -07:00
|
|
|
static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
int ret;
|
2008-02-20 15:57:30 -07:00
|
|
|
asm volatile("syscall"
|
2005-04-16 16:20:36 -06:00
|
|
|
: "=a" (ret)
|
2007-02-16 02:28:21 -07:00
|
|
|
: "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
|
|
|
|
: __syscall_clobber );
|
2005-04-16 16:20:36 -06:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2006-01-11 14:45:30 -07:00
|
|
|
static __always_inline long time_syscall(long *t)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
long secs;
|
2008-02-20 15:57:30 -07:00
|
|
|
asm volatile("syscall"
|
2005-04-16 16:20:36 -06:00
|
|
|
: "=a" (secs)
|
|
|
|
: "0" (__NR_time),"D" (t) : __syscall_clobber);
|
|
|
|
return secs;
|
|
|
|
}
|
|
|
|
|
2007-02-16 02:28:21 -07:00
|
|
|
static __always_inline void do_vgettimeofday(struct timeval * tv)
|
|
|
|
{
|
|
|
|
cycle_t now, base, mask, cycle_delta;
|
2007-05-02 11:27:11 -06:00
|
|
|
unsigned seq;
|
|
|
|
unsigned long mult, shift, nsec;
|
2007-02-16 02:28:21 -07:00
|
|
|
cycle_t (*vread)(void);
|
|
|
|
do {
|
|
|
|
seq = read_seqbegin(&__vsyscall_gtod_data.lock);
|
|
|
|
|
|
|
|
vread = __vsyscall_gtod_data.clock.vread;
|
|
|
|
if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
|
2007-03-14 03:17:59 -06:00
|
|
|
gettimeofday(tv,NULL);
|
2007-02-16 02:28:21 -07:00
|
|
|
return;
|
|
|
|
}
|
2008-11-08 12:27:00 -07:00
|
|
|
|
2007-02-16 02:28:21 -07:00
|
|
|
now = vread();
|
|
|
|
base = __vsyscall_gtod_data.clock.cycle_last;
|
|
|
|
mask = __vsyscall_gtod_data.clock.mask;
|
|
|
|
mult = __vsyscall_gtod_data.clock.mult;
|
|
|
|
shift = __vsyscall_gtod_data.clock.shift;
|
|
|
|
|
2007-05-02 11:27:11 -06:00
|
|
|
tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
|
|
|
|
nsec = __vsyscall_gtod_data.wall_time_nsec;
|
2007-02-16 02:28:21 -07:00
|
|
|
} while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
|
|
|
|
|
|
|
|
/* calculate interval: */
|
|
|
|
cycle_delta = (now - base) & mask;
|
|
|
|
/* convert to nsecs: */
|
2007-05-02 11:27:11 -06:00
|
|
|
nsec += (cycle_delta * mult) >> shift;
|
2007-02-16 02:28:21 -07:00
|
|
|
|
2007-05-02 11:27:11 -06:00
|
|
|
while (nsec >= NSEC_PER_SEC) {
|
2007-02-16 02:28:21 -07:00
|
|
|
tv->tv_sec += 1;
|
2007-05-02 11:27:11 -06:00
|
|
|
nsec -= NSEC_PER_SEC;
|
2007-02-16 02:28:21 -07:00
|
|
|
}
|
2007-05-02 11:27:11 -06:00
|
|
|
tv->tv_usec = nsec / NSEC_PER_USEC;
|
2007-02-16 02:28:21 -07:00
|
|
|
}
|
|
|
|
|
2005-09-12 10:49:24 -06:00
|
|
|
int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
if (tv)
|
|
|
|
do_vgettimeofday(tv);
|
|
|
|
if (tz)
|
|
|
|
do_get_tz(tz);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* This will break when the xtime seconds get inaccurate, but that is
|
|
|
|
* unlikely */
|
2005-09-12 10:49:24 -06:00
|
|
|
time_t __vsyscall(1) vtime(time_t *t)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-05-21 06:31:52 -06:00
|
|
|
struct timeval tv;
|
2007-05-02 11:27:11 -06:00
|
|
|
time_t result;
|
2007-02-16 02:28:21 -07:00
|
|
|
if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
|
2005-04-16 16:20:36 -06:00
|
|
|
return time_syscall(t);
|
2007-05-21 06:31:52 -06:00
|
|
|
|
2007-10-18 04:07:05 -06:00
|
|
|
vgettimeofday(&tv, NULL);
|
2007-05-21 06:31:52 -06:00
|
|
|
result = tv.tv_sec;
|
2007-05-02 11:27:11 -06:00
|
|
|
if (t)
|
|
|
|
*t = result;
|
|
|
|
return result;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2006-09-26 02:52:28 -06:00
|
|
|
/* Fast way to get current CPU and node.
|
|
|
|
This helps to do per node and per CPU caches in user space.
|
|
|
|
The result is not guaranteed without CPU affinity, but usually
|
|
|
|
works out because the scheduler tries to keep a thread on the same
|
|
|
|
CPU.
|
|
|
|
|
|
|
|
tcache must point to a two element sized long array.
|
|
|
|
All arguments can be NULL. */
|
|
|
|
long __vsyscall(2)
|
|
|
|
vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-01-30 05:31:06 -07:00
|
|
|
unsigned int p;
|
2006-09-26 02:52:28 -06:00
|
|
|
unsigned long j = 0;
|
|
|
|
|
|
|
|
/* Fast cache - only recompute value once per jiffies and avoid
|
|
|
|
relatively costly rdtscp/cpuid otherwise.
|
|
|
|
This works because the scheduler usually keeps the process
|
|
|
|
on the same CPU and this syscall doesn't guarantee its
|
|
|
|
results anyways.
|
|
|
|
We do this here because otherwise user space would do it on
|
|
|
|
its own in a likely inferior way (no access to jiffies).
|
|
|
|
If you don't like it pass NULL. */
|
2006-09-29 17:47:55 -06:00
|
|
|
if (tcache && tcache->blob[0] == (j = __jiffies)) {
|
|
|
|
p = tcache->blob[1];
|
2006-09-26 02:52:28 -06:00
|
|
|
} else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
|
|
|
|
/* Load per CPU data from RDTSCP */
|
2008-01-30 05:31:06 -07:00
|
|
|
native_read_tscp(&p);
|
2006-09-26 02:52:28 -06:00
|
|
|
} else {
|
|
|
|
/* Load per CPU data from GDT */
|
|
|
|
asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
|
|
|
|
}
|
|
|
|
if (tcache) {
|
2006-09-29 17:47:55 -06:00
|
|
|
tcache->blob[0] = j;
|
|
|
|
tcache->blob[1] = p;
|
2006-09-26 02:52:28 -06:00
|
|
|
}
|
|
|
|
if (cpu)
|
|
|
|
*cpu = p & 0xfff;
|
|
|
|
if (node)
|
|
|
|
*node = p >> 12;
|
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2008-04-23 05:20:56 -06:00
|
|
|
static long __vsyscall(3) venosys_1(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
return -ENOSYS;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_SYSCTL
|
|
|
|
static ctl_table kernel_table2[] = {
|
2007-10-18 04:05:27 -06:00
|
|
|
{ .procname = "vsyscall64",
|
2007-02-16 02:28:21 -07:00
|
|
|
.data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
|
2008-02-27 01:39:52 -07:00
|
|
|
.mode = 0644,
|
2009-09-23 16:57:19 -06:00
|
|
|
.proc_handler = proc_dointvec },
|
2007-02-14 01:33:50 -07:00
|
|
|
{}
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
static ctl_table kernel_root_table2[] = {
|
|
|
|
{ .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
|
|
|
|
.child = kernel_table2 },
|
2007-02-14 01:33:50 -07:00
|
|
|
{}
|
2005-04-16 16:20:36 -06:00
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
2006-11-14 08:57:46 -07:00
|
|
|
/* Assume __initcall executes before all user space. Hopefully kmod
|
|
|
|
doesn't violate that. We'll find out if it does. */
|
|
|
|
static void __cpuinit vsyscall_set_cpu(int cpu)
|
2006-09-26 02:52:28 -06:00
|
|
|
{
|
2008-06-24 22:19:01 -06:00
|
|
|
unsigned long d;
|
2006-09-26 02:52:28 -06:00
|
|
|
unsigned long node = 0;
|
|
|
|
#ifdef CONFIG_NUMA
|
x86: fix cpu_to_node references
In x86_64 and i386 architectures most arrays that are sized using
NR_CPUS lay in local memory on node 0. Not only will most (99%?) of the
systems not use all the slots in these arrays, particularly when NR_CPUS
is increased to accommodate future very high cpu count systems, but a
number of cache lines are passed unnecessarily on the system bus when
these arrays are referenced by cpus on other nodes.
Typically, the values in these arrays are referenced by the cpu
accessing it's own values, though when passing IPI interrupts, the cpu
does access the data relevant to the targeted cpu/node. Of course, if
the referencing cpu is not on node 0, then the reference will still
require cross node exchanges of cache lines. A common use of this is
for an interrupt service routine to pass the interrupt to other cpus
local to that node.
Ideally, all the elements in these arrays should be moved to the per_cpu
data area. In some cases (such as x86_cpu_to_apicid) the array is
referenced before the per_cpu data areas are setup. In this case, a
static array is declared in the __initdata area and initialized by the
booting cpu (BSP). The values are then moved to the per_cpu area after
it is initialized and the original static array is freed with the rest
of the __initdata.
This patch:
Fix four instances where cpu_to_node is referenced by array instead of
via the cpu_to_node macro. This is preparation to moving it to the
per_cpu data area.
Signed-off-by: Mike Travis <travis@sgi.com>
Cc: Andi Kleen <ak@suse.de>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: "Siddha, Suresh B" <suresh.b.siddha@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2007-10-17 10:04:39 -06:00
|
|
|
node = cpu_to_node(cpu);
|
2006-09-26 02:52:28 -06:00
|
|
|
#endif
|
2007-10-19 12:35:04 -06:00
|
|
|
if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
|
2006-11-14 08:57:46 -07:00
|
|
|
write_rdtscp_aux((node << 12) | cpu);
|
2006-09-26 02:52:28 -06:00
|
|
|
|
|
|
|
/* Store cpu number in limit so that it can be loaded quickly
|
|
|
|
in user space in vgetcpu.
|
|
|
|
12 bits for the CPU and 8 bits for the node. */
|
2008-06-24 22:19:01 -06:00
|
|
|
d = 0x0f40000000000ULL;
|
|
|
|
d |= cpu;
|
|
|
|
d |= (node & 0xf) << 12;
|
|
|
|
d |= (node >> 4) << 48;
|
|
|
|
write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
|
2006-09-26 02:52:28 -06:00
|
|
|
}
|
|
|
|
|
2006-11-14 08:57:46 -07:00
|
|
|
static void __cpuinit cpu_vsyscall_init(void *arg)
|
|
|
|
{
|
|
|
|
/* preemption should be already off */
|
|
|
|
vsyscall_set_cpu(raw_smp_processor_id());
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __cpuinit
|
|
|
|
cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
|
|
|
|
{
|
|
|
|
long cpu = (long)arg;
|
2007-05-09 03:35:10 -06:00
|
|
|
if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
|
2008-06-06 03:18:06 -06:00
|
|
|
smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
|
2006-11-14 08:57:46 -07:00
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2008-01-30 05:32:39 -07:00
|
|
|
void __init map_vsyscall(void)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
|
|
|
extern char __vsyscall_0;
|
|
|
|
unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
|
|
|
|
|
2006-12-06 18:14:09 -07:00
|
|
|
/* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
|
2005-04-16 16:20:36 -06:00
|
|
|
__set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init vsyscall_init(void)
|
|
|
|
{
|
|
|
|
BUG_ON(((unsigned long) &vgettimeofday !=
|
|
|
|
VSYSCALL_ADDR(__NR_vgettimeofday)));
|
|
|
|
BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
|
|
|
|
BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
|
2006-09-26 02:52:28 -06:00
|
|
|
BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
|
2005-05-16 22:53:33 -06:00
|
|
|
#ifdef CONFIG_SYSCTL
|
2007-02-14 01:34:09 -07:00
|
|
|
register_sysctl_table(kernel_root_table2);
|
2005-05-16 22:53:33 -06:00
|
|
|
#endif
|
2008-05-09 01:39:44 -06:00
|
|
|
on_each_cpu(cpu_vsyscall_init, NULL, 1);
|
2006-11-14 08:57:46 -07:00
|
|
|
hotcpu_notifier(cpu_vsyscall_notifier, 0);
|
2005-04-16 16:20:36 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
__initcall(vsyscall_init);
|