![Linus Torvalds](/assets/img/avatar_default.png)
In order for ntpd to correctly synchronize the clocks, the frequency of the system clock must not be off by more than 500 ppm (or, put another way, 1:2000), or ntpd will end up giving up on trying to synchronize properly, and ends up reseting the clock in jumps instead. The fast TSC PIT calibration sometimes failed this test - it was assuming that the PIT reads always took about one microsecond each (2us for the two reads to get a 16-bit timer), and that calibrating TSC to the PIT over 15ms should thus be sufficient to get much closer than 500ppm (max 2us error on both sides giving 4us over 15ms: a 270 ppm error value). However, that assumption does not always hold: apparently some hardware is either very much slower at reading the PIT registers, or there was other noise causing at least one machine to get 700+ ppm errors. So instead of using a fixed 15ms timing loop, this changes the fast PIT calibration to read the TSC delta over the individual PIT timer reads, and use the result to calculate the error bars on the PIT read timing properly. We then successfully calibrate the TSC only if the maximum error bars fall below 500ppm. In the process, we also relax the timing to allow up to 25ms for the calibration, although it can happen much faster depending on hardware. Reported-and-tested-by: Jesper Krogh <jesper@krogh.cc> Cc: john stultz <johnstul@us.ibm.com> Cc: Thomas Gleixner <tglx@linutronix.de> Acked-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
886 lines
22 KiB
C
886 lines
22 KiB
C
#include <linux/kernel.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/init.h>
|
|
#include <linux/module.h>
|
|
#include <linux/timer.h>
|
|
#include <linux/acpi_pmtmr.h>
|
|
#include <linux/cpufreq.h>
|
|
#include <linux/dmi.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/clocksource.h>
|
|
#include <linux/percpu.h>
|
|
|
|
#include <asm/hpet.h>
|
|
#include <asm/timer.h>
|
|
#include <asm/vgtod.h>
|
|
#include <asm/time.h>
|
|
#include <asm/delay.h>
|
|
#include <asm/hypervisor.h>
|
|
|
|
unsigned int cpu_khz; /* TSC clocks / usec, not used here */
|
|
EXPORT_SYMBOL(cpu_khz);
|
|
unsigned int tsc_khz;
|
|
EXPORT_SYMBOL(tsc_khz);
|
|
|
|
/*
|
|
* TSC can be unstable due to cpufreq or due to unsynced TSCs
|
|
*/
|
|
static int tsc_unstable;
|
|
|
|
/* native_sched_clock() is called before tsc_init(), so
|
|
we must start with the TSC soft disabled to prevent
|
|
erroneous rdtsc usage on !cpu_has_tsc processors */
|
|
static int tsc_disabled = -1;
|
|
|
|
static int tsc_clocksource_reliable;
|
|
/*
|
|
* Scheduler clock - returns current time in nanosec units.
|
|
*/
|
|
u64 native_sched_clock(void)
|
|
{
|
|
u64 this_offset;
|
|
|
|
/*
|
|
* Fall back to jiffies if there's no TSC available:
|
|
* ( But note that we still use it if the TSC is marked
|
|
* unstable. We do this because unlike Time Of Day,
|
|
* the scheduler clock tolerates small errors and it's
|
|
* very important for it to be as fast as the platform
|
|
* can achive it. )
|
|
*/
|
|
if (unlikely(tsc_disabled)) {
|
|
/* No locking but a rare wrong value is not a big deal: */
|
|
return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
|
|
}
|
|
|
|
/* read the Time Stamp Counter: */
|
|
rdtscll(this_offset);
|
|
|
|
/* return the value in ns */
|
|
return __cycles_2_ns(this_offset);
|
|
}
|
|
|
|
/* We need to define a real function for sched_clock, to override the
|
|
weak default version */
|
|
#ifdef CONFIG_PARAVIRT
|
|
unsigned long long sched_clock(void)
|
|
{
|
|
return paravirt_sched_clock();
|
|
}
|
|
#else
|
|
unsigned long long
|
|
sched_clock(void) __attribute__((alias("native_sched_clock")));
|
|
#endif
|
|
|
|
int check_tsc_unstable(void)
|
|
{
|
|
return tsc_unstable;
|
|
}
|
|
EXPORT_SYMBOL_GPL(check_tsc_unstable);
|
|
|
|
#ifdef CONFIG_X86_TSC
|
|
int __init notsc_setup(char *str)
|
|
{
|
|
printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
|
|
"cannot disable TSC completely.\n");
|
|
tsc_disabled = 1;
|
|
return 1;
|
|
}
|
|
#else
|
|
/*
|
|
* disable flag for tsc. Takes effect by clearing the TSC cpu flag
|
|
* in cpu/common.c
|
|
*/
|
|
int __init notsc_setup(char *str)
|
|
{
|
|
setup_clear_cpu_cap(X86_FEATURE_TSC);
|
|
return 1;
|
|
}
|
|
#endif
|
|
|
|
__setup("notsc", notsc_setup);
|
|
|
|
static int __init tsc_setup(char *str)
|
|
{
|
|
if (!strcmp(str, "reliable"))
|
|
tsc_clocksource_reliable = 1;
|
|
return 1;
|
|
}
|
|
|
|
__setup("tsc=", tsc_setup);
|
|
|
|
#define MAX_RETRIES 5
|
|
#define SMI_TRESHOLD 50000
|
|
|
|
/*
|
|
* Read TSC and the reference counters. Take care of SMI disturbance
|
|
*/
|
|
static u64 tsc_read_refs(u64 *p, int hpet)
|
|
{
|
|
u64 t1, t2;
|
|
int i;
|
|
|
|
for (i = 0; i < MAX_RETRIES; i++) {
|
|
t1 = get_cycles();
|
|
if (hpet)
|
|
*p = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
|
|
else
|
|
*p = acpi_pm_read_early();
|
|
t2 = get_cycles();
|
|
if ((t2 - t1) < SMI_TRESHOLD)
|
|
return t2;
|
|
}
|
|
return ULLONG_MAX;
|
|
}
|
|
|
|
/*
|
|
* Calculate the TSC frequency from HPET reference
|
|
*/
|
|
static unsigned long calc_hpet_ref(u64 deltatsc, u64 hpet1, u64 hpet2)
|
|
{
|
|
u64 tmp;
|
|
|
|
if (hpet2 < hpet1)
|
|
hpet2 += 0x100000000ULL;
|
|
hpet2 -= hpet1;
|
|
tmp = ((u64)hpet2 * hpet_readl(HPET_PERIOD));
|
|
do_div(tmp, 1000000);
|
|
do_div(deltatsc, tmp);
|
|
|
|
return (unsigned long) deltatsc;
|
|
}
|
|
|
|
/*
|
|
* Calculate the TSC frequency from PMTimer reference
|
|
*/
|
|
static unsigned long calc_pmtimer_ref(u64 deltatsc, u64 pm1, u64 pm2)
|
|
{
|
|
u64 tmp;
|
|
|
|
if (!pm1 && !pm2)
|
|
return ULONG_MAX;
|
|
|
|
if (pm2 < pm1)
|
|
pm2 += (u64)ACPI_PM_OVRRUN;
|
|
pm2 -= pm1;
|
|
tmp = pm2 * 1000000000LL;
|
|
do_div(tmp, PMTMR_TICKS_PER_SEC);
|
|
do_div(deltatsc, tmp);
|
|
|
|
return (unsigned long) deltatsc;
|
|
}
|
|
|
|
#define CAL_MS 10
|
|
#define CAL_LATCH (CLOCK_TICK_RATE / (1000 / CAL_MS))
|
|
#define CAL_PIT_LOOPS 1000
|
|
|
|
#define CAL2_MS 50
|
|
#define CAL2_LATCH (CLOCK_TICK_RATE / (1000 / CAL2_MS))
|
|
#define CAL2_PIT_LOOPS 5000
|
|
|
|
|
|
/*
|
|
* Try to calibrate the TSC against the Programmable
|
|
* Interrupt Timer and return the frequency of the TSC
|
|
* in kHz.
|
|
*
|
|
* Return ULONG_MAX on failure to calibrate.
|
|
*/
|
|
static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin)
|
|
{
|
|
u64 tsc, t1, t2, delta;
|
|
unsigned long tscmin, tscmax;
|
|
int pitcnt;
|
|
|
|
/* Set the Gate high, disable speaker */
|
|
outb((inb(0x61) & ~0x02) | 0x01, 0x61);
|
|
|
|
/*
|
|
* Setup CTC channel 2* for mode 0, (interrupt on terminal
|
|
* count mode), binary count. Set the latch register to 50ms
|
|
* (LSB then MSB) to begin countdown.
|
|
*/
|
|
outb(0xb0, 0x43);
|
|
outb(latch & 0xff, 0x42);
|
|
outb(latch >> 8, 0x42);
|
|
|
|
tsc = t1 = t2 = get_cycles();
|
|
|
|
pitcnt = 0;
|
|
tscmax = 0;
|
|
tscmin = ULONG_MAX;
|
|
while ((inb(0x61) & 0x20) == 0) {
|
|
t2 = get_cycles();
|
|
delta = t2 - tsc;
|
|
tsc = t2;
|
|
if ((unsigned long) delta < tscmin)
|
|
tscmin = (unsigned int) delta;
|
|
if ((unsigned long) delta > tscmax)
|
|
tscmax = (unsigned int) delta;
|
|
pitcnt++;
|
|
}
|
|
|
|
/*
|
|
* Sanity checks:
|
|
*
|
|
* If we were not able to read the PIT more than loopmin
|
|
* times, then we have been hit by a massive SMI
|
|
*
|
|
* If the maximum is 10 times larger than the minimum,
|
|
* then we got hit by an SMI as well.
|
|
*/
|
|
if (pitcnt < loopmin || tscmax > 10 * tscmin)
|
|
return ULONG_MAX;
|
|
|
|
/* Calculate the PIT value */
|
|
delta = t2 - t1;
|
|
do_div(delta, ms);
|
|
return delta;
|
|
}
|
|
|
|
/*
|
|
* This reads the current MSB of the PIT counter, and
|
|
* checks if we are running on sufficiently fast and
|
|
* non-virtualized hardware.
|
|
*
|
|
* Our expectations are:
|
|
*
|
|
* - the PIT is running at roughly 1.19MHz
|
|
*
|
|
* - each IO is going to take about 1us on real hardware,
|
|
* but we allow it to be much faster (by a factor of 10) or
|
|
* _slightly_ slower (ie we allow up to a 2us read+counter
|
|
* update - anything else implies a unacceptably slow CPU
|
|
* or PIT for the fast calibration to work.
|
|
*
|
|
* - with 256 PIT ticks to read the value, we have 214us to
|
|
* see the same MSB (and overhead like doing a single TSC
|
|
* read per MSB value etc).
|
|
*
|
|
* - We're doing 2 reads per loop (LSB, MSB), and we expect
|
|
* them each to take about a microsecond on real hardware.
|
|
* So we expect a count value of around 100. But we'll be
|
|
* generous, and accept anything over 50.
|
|
*
|
|
* - if the PIT is stuck, and we see *many* more reads, we
|
|
* return early (and the next caller of pit_expect_msb()
|
|
* then consider it a failure when they don't see the
|
|
* next expected value).
|
|
*
|
|
* These expectations mean that we know that we have seen the
|
|
* transition from one expected value to another with a fairly
|
|
* high accuracy, and we didn't miss any events. We can thus
|
|
* use the TSC value at the transitions to calculate a pretty
|
|
* good value for the TSC frequencty.
|
|
*/
|
|
static inline int pit_expect_msb(unsigned char val, u64 *tscp, unsigned long *deltap)
|
|
{
|
|
int count;
|
|
u64 tsc = 0;
|
|
|
|
for (count = 0; count < 50000; count++) {
|
|
/* Ignore LSB */
|
|
inb(0x42);
|
|
if (inb(0x42) != val)
|
|
break;
|
|
tsc = get_cycles();
|
|
}
|
|
*deltap = get_cycles() - tsc;
|
|
*tscp = tsc;
|
|
|
|
/*
|
|
* We require _some_ success, but the quality control
|
|
* will be based on the error terms on the TSC values.
|
|
*/
|
|
return count > 5;
|
|
}
|
|
|
|
/*
|
|
* How many MSB values do we want to see? We aim for
|
|
* a maximum error rate of 500ppm (in practice the
|
|
* real error is much smaller), but refuse to spend
|
|
* more than 25ms on it.
|
|
*/
|
|
#define MAX_QUICK_PIT_MS 25
|
|
#define MAX_QUICK_PIT_ITERATIONS (MAX_QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
|
|
|
|
static unsigned long quick_pit_calibrate(void)
|
|
{
|
|
int i;
|
|
u64 tsc, delta;
|
|
unsigned long d1, d2;
|
|
|
|
/* Set the Gate high, disable speaker */
|
|
outb((inb(0x61) & ~0x02) | 0x01, 0x61);
|
|
|
|
/*
|
|
* Counter 2, mode 0 (one-shot), binary count
|
|
*
|
|
* NOTE! Mode 2 decrements by two (and then the
|
|
* output is flipped each time, giving the same
|
|
* final output frequency as a decrement-by-one),
|
|
* so mode 0 is much better when looking at the
|
|
* individual counts.
|
|
*/
|
|
outb(0xb0, 0x43);
|
|
|
|
/* Start at 0xffff */
|
|
outb(0xff, 0x42);
|
|
outb(0xff, 0x42);
|
|
|
|
/*
|
|
* The PIT starts counting at the next edge, so we
|
|
* need to delay for a microsecond. The easiest way
|
|
* to do that is to just read back the 16-bit counter
|
|
* once from the PIT.
|
|
*/
|
|
inb(0x42);
|
|
inb(0x42);
|
|
|
|
if (pit_expect_msb(0xff, &tsc, &d1)) {
|
|
for (i = 1; i <= MAX_QUICK_PIT_ITERATIONS; i++) {
|
|
if (!pit_expect_msb(0xff-i, &delta, &d2))
|
|
break;
|
|
|
|
/*
|
|
* Iterate until the error is less than 500 ppm
|
|
*/
|
|
delta -= tsc;
|
|
if (d1+d2 < delta >> 11)
|
|
goto success;
|
|
}
|
|
}
|
|
printk("Fast TSC calibration failed\n");
|
|
return 0;
|
|
|
|
success:
|
|
/*
|
|
* Ok, if we get here, then we've seen the
|
|
* MSB of the PIT decrement 'i' times, and the
|
|
* error has shrunk to less than 500 ppm.
|
|
*
|
|
* As a result, we can depend on there not being
|
|
* any odd delays anywhere, and the TSC reads are
|
|
* reliable (within the error). We also adjust the
|
|
* delta to the middle of the error bars, just
|
|
* because it looks nicer.
|
|
*
|
|
* kHz = ticks / time-in-seconds / 1000;
|
|
* kHz = (t2 - t1) / (I * 256 / PIT_TICK_RATE) / 1000
|
|
* kHz = ((t2 - t1) * PIT_TICK_RATE) / (I * 256 * 1000)
|
|
*/
|
|
delta += (long)(d2 - d1)/2;
|
|
delta *= PIT_TICK_RATE;
|
|
do_div(delta, i*256*1000);
|
|
printk("Fast TSC calibration using PIT\n");
|
|
return delta;
|
|
}
|
|
|
|
/**
|
|
* native_calibrate_tsc - calibrate the tsc on boot
|
|
*/
|
|
unsigned long native_calibrate_tsc(void)
|
|
{
|
|
u64 tsc1, tsc2, delta, ref1, ref2;
|
|
unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX;
|
|
unsigned long flags, latch, ms, fast_calibrate, tsc_khz;
|
|
int hpet = is_hpet_enabled(), i, loopmin;
|
|
|
|
tsc_khz = get_hypervisor_tsc_freq();
|
|
if (tsc_khz) {
|
|
printk(KERN_INFO "TSC: Frequency read from the hypervisor\n");
|
|
return tsc_khz;
|
|
}
|
|
|
|
local_irq_save(flags);
|
|
fast_calibrate = quick_pit_calibrate();
|
|
local_irq_restore(flags);
|
|
if (fast_calibrate)
|
|
return fast_calibrate;
|
|
|
|
/*
|
|
* Run 5 calibration loops to get the lowest frequency value
|
|
* (the best estimate). We use two different calibration modes
|
|
* here:
|
|
*
|
|
* 1) PIT loop. We set the PIT Channel 2 to oneshot mode and
|
|
* load a timeout of 50ms. We read the time right after we
|
|
* started the timer and wait until the PIT count down reaches
|
|
* zero. In each wait loop iteration we read the TSC and check
|
|
* the delta to the previous read. We keep track of the min
|
|
* and max values of that delta. The delta is mostly defined
|
|
* by the IO time of the PIT access, so we can detect when a
|
|
* SMI/SMM disturbance happend between the two reads. If the
|
|
* maximum time is significantly larger than the minimum time,
|
|
* then we discard the result and have another try.
|
|
*
|
|
* 2) Reference counter. If available we use the HPET or the
|
|
* PMTIMER as a reference to check the sanity of that value.
|
|
* We use separate TSC readouts and check inside of the
|
|
* reference read for a SMI/SMM disturbance. We dicard
|
|
* disturbed values here as well. We do that around the PIT
|
|
* calibration delay loop as we have to wait for a certain
|
|
* amount of time anyway.
|
|
*/
|
|
|
|
/* Preset PIT loop values */
|
|
latch = CAL_LATCH;
|
|
ms = CAL_MS;
|
|
loopmin = CAL_PIT_LOOPS;
|
|
|
|
for (i = 0; i < 3; i++) {
|
|
unsigned long tsc_pit_khz;
|
|
|
|
/*
|
|
* Read the start value and the reference count of
|
|
* hpet/pmtimer when available. Then do the PIT
|
|
* calibration, which will take at least 50ms, and
|
|
* read the end value.
|
|
*/
|
|
local_irq_save(flags);
|
|
tsc1 = tsc_read_refs(&ref1, hpet);
|
|
tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin);
|
|
tsc2 = tsc_read_refs(&ref2, hpet);
|
|
local_irq_restore(flags);
|
|
|
|
/* Pick the lowest PIT TSC calibration so far */
|
|
tsc_pit_min = min(tsc_pit_min, tsc_pit_khz);
|
|
|
|
/* hpet or pmtimer available ? */
|
|
if (!hpet && !ref1 && !ref2)
|
|
continue;
|
|
|
|
/* Check, whether the sampling was disturbed by an SMI */
|
|
if (tsc1 == ULLONG_MAX || tsc2 == ULLONG_MAX)
|
|
continue;
|
|
|
|
tsc2 = (tsc2 - tsc1) * 1000000LL;
|
|
if (hpet)
|
|
tsc2 = calc_hpet_ref(tsc2, ref1, ref2);
|
|
else
|
|
tsc2 = calc_pmtimer_ref(tsc2, ref1, ref2);
|
|
|
|
tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2);
|
|
|
|
/* Check the reference deviation */
|
|
delta = ((u64) tsc_pit_min) * 100;
|
|
do_div(delta, tsc_ref_min);
|
|
|
|
/*
|
|
* If both calibration results are inside a 10% window
|
|
* then we can be sure, that the calibration
|
|
* succeeded. We break out of the loop right away. We
|
|
* use the reference value, as it is more precise.
|
|
*/
|
|
if (delta >= 90 && delta <= 110) {
|
|
printk(KERN_INFO
|
|
"TSC: PIT calibration matches %s. %d loops\n",
|
|
hpet ? "HPET" : "PMTIMER", i + 1);
|
|
return tsc_ref_min;
|
|
}
|
|
|
|
/*
|
|
* Check whether PIT failed more than once. This
|
|
* happens in virtualized environments. We need to
|
|
* give the virtual PC a slightly longer timeframe for
|
|
* the HPET/PMTIMER to make the result precise.
|
|
*/
|
|
if (i == 1 && tsc_pit_min == ULONG_MAX) {
|
|
latch = CAL2_LATCH;
|
|
ms = CAL2_MS;
|
|
loopmin = CAL2_PIT_LOOPS;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Now check the results.
|
|
*/
|
|
if (tsc_pit_min == ULONG_MAX) {
|
|
/* PIT gave no useful value */
|
|
printk(KERN_WARNING "TSC: Unable to calibrate against PIT\n");
|
|
|
|
/* We don't have an alternative source, disable TSC */
|
|
if (!hpet && !ref1 && !ref2) {
|
|
printk("TSC: No reference (HPET/PMTIMER) available\n");
|
|
return 0;
|
|
}
|
|
|
|
/* The alternative source failed as well, disable TSC */
|
|
if (tsc_ref_min == ULONG_MAX) {
|
|
printk(KERN_WARNING "TSC: HPET/PMTIMER calibration "
|
|
"failed.\n");
|
|
return 0;
|
|
}
|
|
|
|
/* Use the alternative source */
|
|
printk(KERN_INFO "TSC: using %s reference calibration\n",
|
|
hpet ? "HPET" : "PMTIMER");
|
|
|
|
return tsc_ref_min;
|
|
}
|
|
|
|
/* We don't have an alternative source, use the PIT calibration value */
|
|
if (!hpet && !ref1 && !ref2) {
|
|
printk(KERN_INFO "TSC: Using PIT calibration value\n");
|
|
return tsc_pit_min;
|
|
}
|
|
|
|
/* The alternative source failed, use the PIT calibration value */
|
|
if (tsc_ref_min == ULONG_MAX) {
|
|
printk(KERN_WARNING "TSC: HPET/PMTIMER calibration failed. "
|
|
"Using PIT calibration\n");
|
|
return tsc_pit_min;
|
|
}
|
|
|
|
/*
|
|
* The calibration values differ too much. In doubt, we use
|
|
* the PIT value as we know that there are PMTIMERs around
|
|
* running at double speed. At least we let the user know:
|
|
*/
|
|
printk(KERN_WARNING "TSC: PIT calibration deviates from %s: %lu %lu.\n",
|
|
hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min);
|
|
printk(KERN_INFO "TSC: Using PIT calibration value\n");
|
|
return tsc_pit_min;
|
|
}
|
|
|
|
#ifdef CONFIG_X86_32
|
|
/* Only called from the Powernow K7 cpu freq driver */
|
|
int recalibrate_cpu_khz(void)
|
|
{
|
|
#ifndef CONFIG_SMP
|
|
unsigned long cpu_khz_old = cpu_khz;
|
|
|
|
if (cpu_has_tsc) {
|
|
tsc_khz = calibrate_tsc();
|
|
cpu_khz = tsc_khz;
|
|
cpu_data(0).loops_per_jiffy =
|
|
cpufreq_scale(cpu_data(0).loops_per_jiffy,
|
|
cpu_khz_old, cpu_khz);
|
|
return 0;
|
|
} else
|
|
return -ENODEV;
|
|
#else
|
|
return -ENODEV;
|
|
#endif
|
|
}
|
|
|
|
EXPORT_SYMBOL(recalibrate_cpu_khz);
|
|
|
|
#endif /* CONFIG_X86_32 */
|
|
|
|
/* Accelerators for sched_clock()
|
|
* convert from cycles(64bits) => nanoseconds (64bits)
|
|
* basic equation:
|
|
* ns = cycles / (freq / ns_per_sec)
|
|
* ns = cycles * (ns_per_sec / freq)
|
|
* ns = cycles * (10^9 / (cpu_khz * 10^3))
|
|
* ns = cycles * (10^6 / cpu_khz)
|
|
*
|
|
* Then we use scaling math (suggested by george@mvista.com) to get:
|
|
* ns = cycles * (10^6 * SC / cpu_khz) / SC
|
|
* ns = cycles * cyc2ns_scale / SC
|
|
*
|
|
* And since SC is a constant power of two, we can convert the div
|
|
* into a shift.
|
|
*
|
|
* We can use khz divisor instead of mhz to keep a better precision, since
|
|
* cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
|
|
* (mathieu.desnoyers@polymtl.ca)
|
|
*
|
|
* -johnstul@us.ibm.com "math is hard, lets go shopping!"
|
|
*/
|
|
|
|
DEFINE_PER_CPU(unsigned long, cyc2ns);
|
|
|
|
static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
|
|
{
|
|
unsigned long long tsc_now, ns_now;
|
|
unsigned long flags, *scale;
|
|
|
|
local_irq_save(flags);
|
|
sched_clock_idle_sleep_event();
|
|
|
|
scale = &per_cpu(cyc2ns, cpu);
|
|
|
|
rdtscll(tsc_now);
|
|
ns_now = __cycles_2_ns(tsc_now);
|
|
|
|
if (cpu_khz)
|
|
*scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
|
|
|
|
sched_clock_idle_wakeup_event(0);
|
|
local_irq_restore(flags);
|
|
}
|
|
|
|
#ifdef CONFIG_CPU_FREQ
|
|
|
|
/* Frequency scaling support. Adjust the TSC based timer when the cpu frequency
|
|
* changes.
|
|
*
|
|
* RED-PEN: On SMP we assume all CPUs run with the same frequency. It's
|
|
* not that important because current Opteron setups do not support
|
|
* scaling on SMP anyroads.
|
|
*
|
|
* Should fix up last_tsc too. Currently gettimeofday in the
|
|
* first tick after the change will be slightly wrong.
|
|
*/
|
|
|
|
static unsigned int ref_freq;
|
|
static unsigned long loops_per_jiffy_ref;
|
|
static unsigned long tsc_khz_ref;
|
|
|
|
static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
|
|
void *data)
|
|
{
|
|
struct cpufreq_freqs *freq = data;
|
|
unsigned long *lpj, dummy;
|
|
|
|
if (cpu_has(&cpu_data(freq->cpu), X86_FEATURE_CONSTANT_TSC))
|
|
return 0;
|
|
|
|
lpj = &dummy;
|
|
if (!(freq->flags & CPUFREQ_CONST_LOOPS))
|
|
#ifdef CONFIG_SMP
|
|
lpj = &cpu_data(freq->cpu).loops_per_jiffy;
|
|
#else
|
|
lpj = &boot_cpu_data.loops_per_jiffy;
|
|
#endif
|
|
|
|
if (!ref_freq) {
|
|
ref_freq = freq->old;
|
|
loops_per_jiffy_ref = *lpj;
|
|
tsc_khz_ref = tsc_khz;
|
|
}
|
|
if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
|
|
(val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
|
|
(val == CPUFREQ_RESUMECHANGE)) {
|
|
*lpj = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
|
|
|
|
tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);
|
|
if (!(freq->flags & CPUFREQ_CONST_LOOPS))
|
|
mark_tsc_unstable("cpufreq changes");
|
|
}
|
|
|
|
set_cyc2ns_scale(tsc_khz, freq->cpu);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct notifier_block time_cpufreq_notifier_block = {
|
|
.notifier_call = time_cpufreq_notifier
|
|
};
|
|
|
|
static int __init cpufreq_tsc(void)
|
|
{
|
|
if (!cpu_has_tsc)
|
|
return 0;
|
|
if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
|
|
return 0;
|
|
cpufreq_register_notifier(&time_cpufreq_notifier_block,
|
|
CPUFREQ_TRANSITION_NOTIFIER);
|
|
return 0;
|
|
}
|
|
|
|
core_initcall(cpufreq_tsc);
|
|
|
|
#endif /* CONFIG_CPU_FREQ */
|
|
|
|
/* clocksource code */
|
|
|
|
static struct clocksource clocksource_tsc;
|
|
|
|
/*
|
|
* We compare the TSC to the cycle_last value in the clocksource
|
|
* structure to avoid a nasty time-warp. This can be observed in a
|
|
* very small window right after one CPU updated cycle_last under
|
|
* xtime/vsyscall_gtod lock and the other CPU reads a TSC value which
|
|
* is smaller than the cycle_last reference value due to a TSC which
|
|
* is slighty behind. This delta is nowhere else observable, but in
|
|
* that case it results in a forward time jump in the range of hours
|
|
* due to the unsigned delta calculation of the time keeping core
|
|
* code, which is necessary to support wrapping clocksources like pm
|
|
* timer.
|
|
*/
|
|
static cycle_t read_tsc(void)
|
|
{
|
|
cycle_t ret = (cycle_t)get_cycles();
|
|
|
|
return ret >= clocksource_tsc.cycle_last ?
|
|
ret : clocksource_tsc.cycle_last;
|
|
}
|
|
|
|
#ifdef CONFIG_X86_64
|
|
static cycle_t __vsyscall_fn vread_tsc(void)
|
|
{
|
|
cycle_t ret = (cycle_t)vget_cycles();
|
|
|
|
return ret >= __vsyscall_gtod_data.clock.cycle_last ?
|
|
ret : __vsyscall_gtod_data.clock.cycle_last;
|
|
}
|
|
#endif
|
|
|
|
static struct clocksource clocksource_tsc = {
|
|
.name = "tsc",
|
|
.rating = 300,
|
|
.read = read_tsc,
|
|
.mask = CLOCKSOURCE_MASK(64),
|
|
.shift = 22,
|
|
.flags = CLOCK_SOURCE_IS_CONTINUOUS |
|
|
CLOCK_SOURCE_MUST_VERIFY,
|
|
#ifdef CONFIG_X86_64
|
|
.vread = vread_tsc,
|
|
#endif
|
|
};
|
|
|
|
void mark_tsc_unstable(char *reason)
|
|
{
|
|
if (!tsc_unstable) {
|
|
tsc_unstable = 1;
|
|
printk("Marking TSC unstable due to %s\n", reason);
|
|
/* Change only the rating, when not registered */
|
|
if (clocksource_tsc.mult)
|
|
clocksource_change_rating(&clocksource_tsc, 0);
|
|
else
|
|
clocksource_tsc.rating = 0;
|
|
}
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(mark_tsc_unstable);
|
|
|
|
static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d)
|
|
{
|
|
printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
|
|
d->ident);
|
|
tsc_unstable = 1;
|
|
return 0;
|
|
}
|
|
|
|
/* List of systems that have known TSC problems */
|
|
static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
|
|
{
|
|
.callback = dmi_mark_tsc_unstable,
|
|
.ident = "IBM Thinkpad 380XD",
|
|
.matches = {
|
|
DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
|
|
DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
|
|
},
|
|
},
|
|
{}
|
|
};
|
|
|
|
static void __init check_system_tsc_reliable(void)
|
|
{
|
|
#ifdef CONFIG_MGEODE_LX
|
|
/* RTSC counts during suspend */
|
|
#define RTSC_SUSP 0x100
|
|
unsigned long res_low, res_high;
|
|
|
|
rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
|
|
/* Geode_LX - the OLPC CPU has a possibly a very reliable TSC */
|
|
if (res_low & RTSC_SUSP)
|
|
tsc_clocksource_reliable = 1;
|
|
#endif
|
|
if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
|
|
tsc_clocksource_reliable = 1;
|
|
}
|
|
|
|
/*
|
|
* Make an educated guess if the TSC is trustworthy and synchronized
|
|
* over all CPUs.
|
|
*/
|
|
__cpuinit int unsynchronized_tsc(void)
|
|
{
|
|
if (!cpu_has_tsc || tsc_unstable)
|
|
return 1;
|
|
|
|
#ifdef CONFIG_X86_SMP
|
|
if (apic_is_clustered_box())
|
|
return 1;
|
|
#endif
|
|
|
|
if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
|
|
return 0;
|
|
/*
|
|
* Intel systems are normally all synchronized.
|
|
* Exceptions must mark TSC as unstable:
|
|
*/
|
|
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
|
|
/* assume multi socket systems are not synchronized: */
|
|
if (num_possible_cpus() > 1)
|
|
tsc_unstable = 1;
|
|
}
|
|
|
|
return tsc_unstable;
|
|
}
|
|
|
|
static void __init init_tsc_clocksource(void)
|
|
{
|
|
clocksource_tsc.mult = clocksource_khz2mult(tsc_khz,
|
|
clocksource_tsc.shift);
|
|
if (tsc_clocksource_reliable)
|
|
clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
|
|
/* lower the rating if we already know its unstable: */
|
|
if (check_tsc_unstable()) {
|
|
clocksource_tsc.rating = 0;
|
|
clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
|
|
}
|
|
clocksource_register(&clocksource_tsc);
|
|
}
|
|
|
|
void __init tsc_init(void)
|
|
{
|
|
u64 lpj;
|
|
int cpu;
|
|
|
|
if (!cpu_has_tsc)
|
|
return;
|
|
|
|
tsc_khz = calibrate_tsc();
|
|
cpu_khz = tsc_khz;
|
|
|
|
if (!tsc_khz) {
|
|
mark_tsc_unstable("could not calculate TSC khz");
|
|
return;
|
|
}
|
|
|
|
#ifdef CONFIG_X86_64
|
|
if (cpu_has(&boot_cpu_data, X86_FEATURE_CONSTANT_TSC) &&
|
|
(boot_cpu_data.x86_vendor == X86_VENDOR_AMD))
|
|
cpu_khz = calibrate_cpu();
|
|
#endif
|
|
|
|
printk("Detected %lu.%03lu MHz processor.\n",
|
|
(unsigned long)cpu_khz / 1000,
|
|
(unsigned long)cpu_khz % 1000);
|
|
|
|
/*
|
|
* Secondary CPUs do not run through tsc_init(), so set up
|
|
* all the scale factors for all CPUs, assuming the same
|
|
* speed as the bootup CPU. (cpufreq notifiers will fix this
|
|
* up if their speed diverges)
|
|
*/
|
|
for_each_possible_cpu(cpu)
|
|
set_cyc2ns_scale(cpu_khz, cpu);
|
|
|
|
if (tsc_disabled > 0)
|
|
return;
|
|
|
|
/* now allow native_sched_clock() to use rdtsc */
|
|
tsc_disabled = 0;
|
|
|
|
lpj = ((u64)tsc_khz * 1000);
|
|
do_div(lpj, HZ);
|
|
lpj_fine = lpj;
|
|
|
|
use_tsc_delay();
|
|
/* Check and install the TSC clocksource */
|
|
dmi_check_system(bad_tsc_dmi_table);
|
|
|
|
if (unsynchronized_tsc())
|
|
mark_tsc_unstable("TSCs unsynchronized");
|
|
|
|
check_system_tsc_reliable();
|
|
init_tsc_clocksource();
|
|
}
|
|
|