2008-03-19 11:26:14 -06:00
|
|
|
/*
|
|
|
|
* x86 SMP booting functions
|
|
|
|
*
|
2009-01-05 07:08:04 -07:00
|
|
|
* (c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
|
2009-01-30 18:03:42 -07:00
|
|
|
* (c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
|
2008-03-19 11:26:14 -06:00
|
|
|
* Copyright 2001 Andi Kleen, SuSE Labs.
|
|
|
|
*
|
|
|
|
* Much of the core SMP work is based on previous work by Thomas Radke, to
|
|
|
|
* whom a great many thanks are extended.
|
|
|
|
*
|
|
|
|
* Thanks to Intel for making available several different Pentium,
|
|
|
|
* Pentium Pro and Pentium-II/Xeon MP machines.
|
|
|
|
* Original development of Linux SMP code supported by Caldera.
|
|
|
|
*
|
|
|
|
* This code is released under the GNU General Public License version 2 or
|
|
|
|
* later.
|
|
|
|
*
|
|
|
|
* Fixes
|
|
|
|
* Felix Koop : NR_CPUS used properly
|
|
|
|
* Jose Renau : Handle single CPU case.
|
|
|
|
* Alan Cox : By repeated request 8) - Total BogoMIPS report.
|
|
|
|
* Greg Wright : Fix for kernel stacks panic.
|
|
|
|
* Erich Boleyn : MP v1.4 and additional changes.
|
|
|
|
* Matthias Sattler : Changes for 2.1 kernel map.
|
|
|
|
* Michel Lespinasse : Changes for 2.1 kernel map.
|
|
|
|
* Michael Chastain : Change trampoline.S to gnu as.
|
|
|
|
* Alan Cox : Dumb bug: 'B' step PPro's are fine
|
|
|
|
* Ingo Molnar : Added APIC timers, based on code
|
|
|
|
* from Jose Renau
|
|
|
|
* Ingo Molnar : various cleanups and rewrites
|
|
|
|
* Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug.
|
|
|
|
* Maciej W. Rozycki : Bits for genuine 82489DX APICs
|
|
|
|
* Andi Kleen : Changed for SMP boot into long mode.
|
|
|
|
* Martin J. Bligh : Added support for multi-quad systems
|
|
|
|
* Dave Jones : Report invalid combinations of Athlon CPUs.
|
|
|
|
* Rusty Russell : Hacked into shape for new "hotplug" boot process.
|
|
|
|
* Andi Kleen : Converted to new state machine.
|
|
|
|
* Ashok Raj : CPU hotplug support
|
|
|
|
* Glauber Costa : i386 and x86_64 integration
|
|
|
|
*/
|
|
|
|
|
2008-03-03 10:12:42 -07:00
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/smp.h>
|
2008-03-03 10:12:58 -07:00
|
|
|
#include <linux/module.h>
|
2008-03-03 10:13:03 -07:00
|
|
|
#include <linux/sched.h>
|
2008-03-03 10:13:07 -07:00
|
|
|
#include <linux/percpu.h>
|
2008-03-03 10:13:12 -07:00
|
|
|
#include <linux/bootmem.h>
|
2008-03-19 11:25:59 -06:00
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/nmi.h>
|
2009-09-01 19:25:07 -06:00
|
|
|
#include <linux/tboot.h>
|
2008-03-03 10:13:07 -07:00
|
|
|
|
2008-03-19 11:26:11 -06:00
|
|
|
#include <asm/acpi.h>
|
2008-03-19 11:25:59 -06:00
|
|
|
#include <asm/desc.h>
|
2008-03-03 10:13:07 -07:00
|
|
|
#include <asm/nmi.h>
|
|
|
|
#include <asm/irq.h>
|
2008-09-23 15:26:42 -06:00
|
|
|
#include <asm/idle.h>
|
2008-04-10 15:28:10 -06:00
|
|
|
#include <asm/trampoline.h>
|
2008-03-03 10:13:07 -07:00
|
|
|
#include <asm/cpu.h>
|
|
|
|
#include <asm/numa.h>
|
2008-03-19 11:25:59 -06:00
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <asm/mtrr.h>
|
2008-03-19 11:26:00 -06:00
|
|
|
#include <asm/vmi.h>
|
2009-02-17 05:58:15 -07:00
|
|
|
#include <asm/apic.h>
|
x86: fix wakeup_cpu with numaq/es7000, v2
Impact: fix secondary-CPU wakeup/init path with numaq and es7000
While looking at wakeup_secondary_cpu for WAKE_SECONDARY_VIA_NMI:
|#ifdef WAKE_SECONDARY_VIA_NMI
|/*
| * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
| * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
| * won't ... remember to clear down the APIC, etc later.
| */
|static int __devinit
|wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
|{
| unsigned long send_status, accept_status = 0;
| int maxlvt;
|...
| if (APIC_INTEGRATED(apic_version[phys_apicid])) {
| maxlvt = lapic_get_maxlvt();
I noticed that there is no warning about undefined phys_apicid...
because WAKE_SECONDARY_VIA_NMI and WAKE_SECONDARY_VIA_INIT can not be
defined at the same time. So NUMAQ is using wrong wakeup_secondary_cpu.
WAKE_SECONDARY_VIA_NMI, WAKE_SECONDARY_VIA_INIT and
WAKE_SECONDARY_VIA_MIP are variants of a weird and fragile
preprocessor-driven "HAL" mechanisms to specify the kind of secondary-CPU
wakeup strategy a given x86 kernel will use.
The vast majority of systems want to use INIT for secondary wakeup - NUMAQ
uses an NMI, (old-style-) ES7000 uses 'MIP' (a firmware driven in-memory
flag to let secondaries continue).
So convert these mechanisms to x86_quirks and add a
->wakeup_secondary_cpu() method to specify the rare exception
to the sane default.
Extend genapic accordingly as well, for 32-bit.
While looking further, I noticed that functions in wakecup.h for numaq
and es7000 are different to the default in mach_wakecpu.h - but smpboot.c
will only use default mach_wakecpu.h with smphook.h.
So we need to add mach_wakecpu.h for mach_generic, to properly support
numaq and es7000, and vectorize the following SMP init methods:
int trampoline_phys_low;
int trampoline_phys_high;
void (*wait_for_init_deassert)(atomic_t *deassert);
void (*smp_callin_clear_local_apic)(void);
void (*store_NMI_vector)(unsigned short *high, unsigned short *low);
void (*restore_NMI_vector)(unsigned short *high, unsigned short *low);
void (*inquire_remote_apic)(int apicid);
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-16 04:12:49 -07:00
|
|
|
#include <asm/setup.h>
|
2009-01-21 01:26:06 -07:00
|
|
|
#include <asm/uv/uv.h>
|
2008-03-19 11:25:59 -06:00
|
|
|
#include <linux/mc146818rtc.h>
|
2008-03-03 10:12:42 -07:00
|
|
|
|
2009-01-28 11:34:09 -07:00
|
|
|
#include <asm/smpboot_hooks.h>
|
2008-03-19 11:25:59 -06:00
|
|
|
|
2008-04-04 13:41:00 -06:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-03-19 11:26:14 -06:00
|
|
|
u8 apicid_2_node[MAX_APICID];
|
x86: fix app crashes after SMP resume
After resume on a 2cpu laptop, kernel builds collapse with a sed hang,
sh or make segfault (often on 20295564), real-time signal to cc1 etc.
Several hurdles to jump, but a manually-assisted bisect led to -rc1's
d2bcbad5f3ad38a1c09861bca7e252dde7bb8259 x86: do not zap_low_mappings
in __smp_prepare_cpus. Though the low mappings were removed at bootup,
they were left behind (with Global flags helping to keep them in TLB)
after resume or cpu online, causing the crashes seen.
Reinstate zap_low_mappings (with local __flush_tlb_all) for each cpu_up
on x86_32. This used to be serialized by smp_commenced_mask: that's now
gone, but a low_mappings flag will do. No need for native_smp_cpus_done
to repeat the zap: let mem_init zap BSP's low mappings just like on UP.
(In passing, fix error code from native_cpu_up: do_boot_cpu returns a
variety of diagnostic values, Dprintk what it says but convert to -EIO.
And save_pg_dir separately before zap_low_mappings: doesn't matter now,
but zapping twice in succession wiped out resume's swsusp_pg_dir.)
That worked well on the duo and one quad, but wouldn't boot 3rd or 4th
cpu on P4 Xeon, oopsing just after unlock_ipi_call_lock. The TLB flush
IPI now being sent reveals a long-standing bug: the booting cpu has its
APIC readied in smp_callin at the top of start_secondary, but isn't put
into the cpu_online_map until just before that unlock_ipi_call_lock.
So native_smp_call_function_mask to online cpus would send_IPI_allbutself,
including the cpu just coming up, though it has been excluded from the
count to wait for: by the time it handles the IPI, the call data on
native_smp_call_function_mask's stack may well have been overwritten.
So fall back to send_IPI_mask while cpu_online_map does not match
cpu_callout_map: perhaps there's a better APICological fix to be
made at the start_secondary end, but I wouldn't know that.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-05-13 07:26:57 -06:00
|
|
|
static int low_mappings;
|
2008-03-19 11:26:13 -06:00
|
|
|
#endif
|
|
|
|
|
2008-03-19 11:26:01 -06:00
|
|
|
/* State of each CPU */
|
|
|
|
DEFINE_PER_CPU(int, cpu_state) = { 0 };
|
|
|
|
|
2008-03-19 11:25:59 -06:00
|
|
|
/* Store all idle threads, this can be reused instead of creating
|
|
|
|
* a new thread. Also avoids complicated thread destroy functionality
|
|
|
|
* for idle threads.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
/*
|
|
|
|
* Needed only for CONFIG_HOTPLUG_CPU because __cpuinitdata is
|
|
|
|
* removed after init for !CONFIG_HOTPLUG_CPU.
|
|
|
|
*/
|
|
|
|
static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
|
|
|
|
#define get_idle_for_cpu(x) (per_cpu(idle_thread_array, x))
|
|
|
|
#define set_idle_for_cpu(x, p) (per_cpu(idle_thread_array, x) = (p))
|
|
|
|
#else
|
2008-07-24 23:22:53 -06:00
|
|
|
static struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
|
2008-03-19 11:25:59 -06:00
|
|
|
#define get_idle_for_cpu(x) (idle_thread_array[(x)])
|
|
|
|
#define set_idle_for_cpu(x, p) (idle_thread_array[(x)] = (p))
|
|
|
|
#endif
|
2008-03-19 11:25:53 -06:00
|
|
|
|
2008-03-03 10:12:58 -07:00
|
|
|
/* Number of siblings per CPU package */
|
|
|
|
int smp_num_siblings = 1;
|
|
|
|
EXPORT_SYMBOL(smp_num_siblings);
|
|
|
|
|
|
|
|
/* Last level cache ID of each logical CPU */
|
|
|
|
DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
|
|
|
|
|
|
|
|
/* representing HT siblings of each logical CPU */
|
2009-03-12 22:19:50 -06:00
|
|
|
DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
|
2008-03-03 10:12:58 -07:00
|
|
|
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
|
|
|
|
|
|
|
|
/* representing HT and core siblings of each logical CPU */
|
2009-03-12 22:19:50 -06:00
|
|
|
DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
|
2008-03-03 10:12:58 -07:00
|
|
|
EXPORT_PER_CPU_SYMBOL(cpu_core_map);
|
|
|
|
|
|
|
|
/* Per CPU bogomips and other parameters */
|
|
|
|
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
|
|
|
|
EXPORT_PER_CPU_SYMBOL(cpu_info);
|
2008-03-03 10:13:02 -07:00
|
|
|
|
2009-02-25 21:50:49 -07:00
|
|
|
atomic_t init_deasserted;
|
2008-03-19 11:25:59 -06:00
|
|
|
|
2008-03-19 11:25:56 -06:00
|
|
|
#if defined(CONFIG_NUMA) && defined(CONFIG_X86_32)
|
|
|
|
/* which node each logical CPU is on */
|
|
|
|
int cpu_to_node_map[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = 0 };
|
|
|
|
EXPORT_SYMBOL(cpu_to_node_map);
|
|
|
|
|
|
|
|
/* set up a mapping between cpu and node. */
|
|
|
|
static void map_cpu_to_node(int cpu, int node)
|
|
|
|
{
|
|
|
|
printk(KERN_INFO "Mapping cpu %d to node %d\n", cpu, node);
|
2009-03-12 22:19:53 -06:00
|
|
|
cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
|
2008-03-19 11:25:56 -06:00
|
|
|
cpu_to_node_map[cpu] = node;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* undo a mapping between cpu and node. */
|
|
|
|
static void unmap_cpu_to_node(int cpu)
|
|
|
|
{
|
|
|
|
int node;
|
|
|
|
|
|
|
|
printk(KERN_INFO "Unmapping cpu %d from all nodes\n", cpu);
|
|
|
|
for (node = 0; node < MAX_NUMNODES; node++)
|
2009-03-12 22:19:53 -06:00
|
|
|
cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
|
2008-03-19 11:25:56 -06:00
|
|
|
cpu_to_node_map[cpu] = 0;
|
|
|
|
}
|
|
|
|
#else /* !(CONFIG_NUMA && CONFIG_X86_32) */
|
|
|
|
#define map_cpu_to_node(cpu, node) ({})
|
|
|
|
#define unmap_cpu_to_node(cpu) ({})
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
2008-07-10 12:16:49 -06:00
|
|
|
static int boot_cpu_logical_apicid;
|
|
|
|
|
2008-03-19 11:25:56 -06:00
|
|
|
u8 cpu_2_logical_apicid[NR_CPUS] __read_mostly =
|
|
|
|
{ [0 ... NR_CPUS-1] = BAD_APICID };
|
|
|
|
|
2008-04-23 05:20:56 -06:00
|
|
|
static void map_cpu_to_logical_apicid(void)
|
2008-03-19 11:25:56 -06:00
|
|
|
{
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
int apicid = logical_smp_processor_id();
|
2009-01-27 22:50:47 -07:00
|
|
|
int node = apic->apicid_to_node(apicid);
|
2008-03-19 11:25:56 -06:00
|
|
|
|
|
|
|
if (!node_online(node))
|
|
|
|
node = first_online_node;
|
|
|
|
|
|
|
|
cpu_2_logical_apicid[cpu] = apicid;
|
|
|
|
map_cpu_to_node(cpu, node);
|
|
|
|
}
|
|
|
|
|
2008-06-04 12:35:03 -06:00
|
|
|
void numa_remove_cpu(int cpu)
|
2008-03-19 11:25:56 -06:00
|
|
|
{
|
|
|
|
cpu_2_logical_apicid[cpu] = BAD_APICID;
|
|
|
|
unmap_cpu_to_node(cpu);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
#define map_cpu_to_logical_apicid() do {} while (0)
|
|
|
|
#endif
|
|
|
|
|
2008-03-19 11:25:59 -06:00
|
|
|
/*
|
|
|
|
* Report back to the Boot Processor.
|
|
|
|
* Running on AP.
|
|
|
|
*/
|
2008-04-23 05:20:56 -06:00
|
|
|
static void __cpuinit smp_callin(void)
|
2008-03-19 11:25:59 -06:00
|
|
|
{
|
|
|
|
int cpuid, phys_id;
|
|
|
|
unsigned long timeout;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If waken up by an INIT in an 82489DX configuration
|
|
|
|
* we may get here before an INIT-deassert IPI reaches
|
|
|
|
* our local APIC. We have to wait for the IPI or we'll
|
|
|
|
* lock up on an APIC access.
|
|
|
|
*/
|
2009-01-28 08:21:32 -07:00
|
|
|
if (apic->wait_for_init_deassert)
|
|
|
|
apic->wait_for_init_deassert(&init_deasserted);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* (This works even if the APIC is not enabled.)
|
|
|
|
*/
|
2008-07-11 19:44:16 -06:00
|
|
|
phys_id = read_apic_id();
|
2008-03-19 11:25:59 -06:00
|
|
|
cpuid = smp_processor_id();
|
2009-01-04 06:18:03 -07:00
|
|
|
if (cpumask_test_cpu(cpuid, cpu_callin_mask)) {
|
2008-03-19 11:25:59 -06:00
|
|
|
panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__,
|
|
|
|
phys_id, cpuid);
|
|
|
|
}
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* STARTUP IPIs are fragile beasts as they might sometimes
|
|
|
|
* trigger some glue motherboard logic. Complete APIC bus
|
|
|
|
* silence for 1 second, this overestimates the time the
|
|
|
|
* boot CPU is spending to send the up to 2 STARTUP IPIs
|
|
|
|
* by a factor of two. This should be enough.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Waiting 2s total for startup (udelay is not yet working)
|
|
|
|
*/
|
|
|
|
timeout = jiffies + 2*HZ;
|
|
|
|
while (time_before(jiffies, timeout)) {
|
|
|
|
/*
|
|
|
|
* Has the boot CPU finished it's STARTUP sequence?
|
|
|
|
*/
|
2009-01-04 06:18:03 -07:00
|
|
|
if (cpumask_test_cpu(cpuid, cpu_callout_mask))
|
2008-03-19 11:25:59 -06:00
|
|
|
break;
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!time_before(jiffies, timeout)) {
|
|
|
|
panic("%s: CPU%d started up but did not get a callout!\n",
|
|
|
|
__func__, cpuid);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* the boot CPU has finished the init stage and is spinning
|
|
|
|
* on callin_map until we finish. We are free to set up this
|
|
|
|
* CPU, first the APIC. (this is probably redundant on most
|
|
|
|
* boards)
|
|
|
|
*/
|
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("CALLIN, before setup_local_APIC().\n");
|
2009-01-28 08:31:52 -07:00
|
|
|
if (apic->smp_callin_clear_local_apic)
|
|
|
|
apic->smp_callin_clear_local_apic();
|
2008-03-19 11:25:59 -06:00
|
|
|
setup_local_APIC();
|
|
|
|
end_local_APIC_setup();
|
|
|
|
map_cpu_to_logical_apicid();
|
|
|
|
|
2008-09-07 08:57:22 -06:00
|
|
|
notify_cpu_starting(cpuid);
|
2008-03-19 11:25:59 -06:00
|
|
|
/*
|
|
|
|
* Get our bogomips.
|
|
|
|
*
|
|
|
|
* Need to enable IRQs because it can take longer and then
|
|
|
|
* the NMI watchdog might kill us.
|
|
|
|
*/
|
|
|
|
local_irq_enable();
|
|
|
|
calibrate_delay();
|
|
|
|
local_irq_disable();
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Stack at about %p\n", &cpuid);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Save our processor parameters
|
|
|
|
*/
|
|
|
|
smp_store_cpu_info(cpuid);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Allow the master to continue.
|
|
|
|
*/
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_set_cpu(cpuid, cpu_callin_mask);
|
2008-03-19 11:25:59 -06:00
|
|
|
}
|
|
|
|
|
2008-03-19 11:26:00 -06:00
|
|
|
/*
|
|
|
|
* Activate a secondary processor.
|
|
|
|
*/
|
2008-12-24 15:30:02 -07:00
|
|
|
notrace static void __cpuinit start_secondary(void *unused)
|
2008-03-19 11:26:00 -06:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Don't put *anything* before cpu_init(), SMP booting is too
|
|
|
|
* fragile that we want to limit the things done here to the
|
|
|
|
* most necessary things.
|
|
|
|
*/
|
|
|
|
vmi_bringup();
|
|
|
|
cpu_init();
|
|
|
|
preempt_disable();
|
|
|
|
smp_callin();
|
|
|
|
|
|
|
|
/* otherwise gcc will move up smp_processor_id before the cpu_init */
|
|
|
|
barrier();
|
|
|
|
/*
|
|
|
|
* Check TSC synchronization with the BP:
|
|
|
|
*/
|
|
|
|
check_tsc_sync_target();
|
|
|
|
|
|
|
|
if (nmi_watchdog == NMI_IO_APIC) {
|
|
|
|
disable_8259A_irq(0);
|
|
|
|
enable_NMI_through_LVT0();
|
|
|
|
enable_8259A_irq(0);
|
|
|
|
}
|
|
|
|
|
x86: fix app crashes after SMP resume
After resume on a 2cpu laptop, kernel builds collapse with a sed hang,
sh or make segfault (often on 20295564), real-time signal to cc1 etc.
Several hurdles to jump, but a manually-assisted bisect led to -rc1's
d2bcbad5f3ad38a1c09861bca7e252dde7bb8259 x86: do not zap_low_mappings
in __smp_prepare_cpus. Though the low mappings were removed at bootup,
they were left behind (with Global flags helping to keep them in TLB)
after resume or cpu online, causing the crashes seen.
Reinstate zap_low_mappings (with local __flush_tlb_all) for each cpu_up
on x86_32. This used to be serialized by smp_commenced_mask: that's now
gone, but a low_mappings flag will do. No need for native_smp_cpus_done
to repeat the zap: let mem_init zap BSP's low mappings just like on UP.
(In passing, fix error code from native_cpu_up: do_boot_cpu returns a
variety of diagnostic values, Dprintk what it says but convert to -EIO.
And save_pg_dir separately before zap_low_mappings: doesn't matter now,
but zapping twice in succession wiped out resume's swsusp_pg_dir.)
That worked well on the duo and one quad, but wouldn't boot 3rd or 4th
cpu on P4 Xeon, oopsing just after unlock_ipi_call_lock. The TLB flush
IPI now being sent reveals a long-standing bug: the booting cpu has its
APIC readied in smp_callin at the top of start_secondary, but isn't put
into the cpu_online_map until just before that unlock_ipi_call_lock.
So native_smp_call_function_mask to online cpus would send_IPI_allbutself,
including the cpu just coming up, though it has been excluded from the
count to wait for: by the time it handles the IPI, the call data on
native_smp_call_function_mask's stack may well have been overwritten.
So fall back to send_IPI_mask while cpu_online_map does not match
cpu_callout_map: perhaps there's a better APICological fix to be
made at the start_secondary end, but I wouldn't know that.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-05-13 07:26:57 -06:00
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
while (low_mappings)
|
|
|
|
cpu_relax();
|
|
|
|
__flush_tlb_all();
|
|
|
|
#endif
|
|
|
|
|
2009-03-12 22:19:54 -06:00
|
|
|
/* This must be done before setting cpu_online_mask */
|
2008-03-19 11:26:00 -06:00
|
|
|
set_cpu_sibling_map(raw_smp_processor_id());
|
|
|
|
wmb();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We need to hold call_lock, so there is no inconsistency
|
|
|
|
* between the time smp_call_function() determines number of
|
|
|
|
* IPI recipients, and the time when the determination is made
|
|
|
|
* for which cpus receive the IPI. Holding this
|
|
|
|
* lock helps us to not include this cpu in a currently in progress
|
|
|
|
* smp_call_function().
|
2008-08-09 16:09:02 -06:00
|
|
|
*
|
|
|
|
* We need to hold vector_lock so there the set of online cpus
|
|
|
|
* does not change while we are assigning vectors to cpus. Holding
|
|
|
|
* this lock ensures we don't half assign or remove an irq from a cpu.
|
2008-03-19 11:26:00 -06:00
|
|
|
*/
|
2008-09-07 03:29:58 -06:00
|
|
|
ipi_call_lock();
|
2008-08-09 16:09:02 -06:00
|
|
|
lock_vector_lock();
|
|
|
|
__setup_vector_irq(smp_processor_id());
|
2009-01-04 06:18:03 -07:00
|
|
|
set_cpu_online(smp_processor_id(), true);
|
2008-08-09 16:09:02 -06:00
|
|
|
unlock_vector_lock();
|
2008-09-07 03:29:58 -06:00
|
|
|
ipi_call_unlock();
|
2008-03-19 11:26:00 -06:00
|
|
|
per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
|
|
|
|
|
2008-09-07 03:29:58 -06:00
|
|
|
/* enable local interrupts */
|
|
|
|
local_irq_enable();
|
|
|
|
|
2009-08-19 04:35:53 -06:00
|
|
|
x86_cpuinit.setup_percpu_clockev();
|
2008-03-19 11:26:00 -06:00
|
|
|
|
|
|
|
wmb();
|
|
|
|
cpu_idle();
|
|
|
|
}
|
|
|
|
|
2009-03-12 22:19:53 -06:00
|
|
|
#ifdef CONFIG_CPUMASK_OFFSTACK
|
|
|
|
/* In this case, llc_shared_map is a pointer to a cpumask. */
|
|
|
|
static inline void copy_cpuinfo_x86(struct cpuinfo_x86 *dst,
|
|
|
|
const struct cpuinfo_x86 *src)
|
|
|
|
{
|
|
|
|
struct cpumask *llc = dst->llc_shared_map;
|
|
|
|
*dst = *src;
|
|
|
|
dst->llc_shared_map = llc;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline void copy_cpuinfo_x86(struct cpuinfo_x86 *dst,
|
|
|
|
const struct cpuinfo_x86 *src)
|
|
|
|
{
|
|
|
|
*dst = *src;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_CPUMASK_OFFSTACK */
|
|
|
|
|
2008-03-19 11:25:05 -06:00
|
|
|
/*
|
|
|
|
* The bootstrap kernel entry code has set these up. Save them for
|
|
|
|
* a given CPU
|
|
|
|
*/
|
|
|
|
|
|
|
|
void __cpuinit smp_store_cpu_info(int id)
|
|
|
|
{
|
|
|
|
struct cpuinfo_x86 *c = &cpu_data(id);
|
|
|
|
|
2009-03-12 22:19:53 -06:00
|
|
|
copy_cpuinfo_x86(c, &boot_cpu_data);
|
2008-03-19 11:25:05 -06:00
|
|
|
c->cpu_index = id;
|
|
|
|
if (id != 0)
|
|
|
|
identify_secondary_cpu(c);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2008-03-03 10:13:02 -07:00
|
|
|
void __cpuinit set_cpu_sibling_map(int cpu)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct cpuinfo_x86 *c = &cpu_data(cpu);
|
|
|
|
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
|
2008-03-03 10:13:02 -07:00
|
|
|
|
|
|
|
if (smp_num_siblings > 1) {
|
2009-01-04 06:18:03 -07:00
|
|
|
for_each_cpu(i, cpu_sibling_setup_mask) {
|
|
|
|
struct cpuinfo_x86 *o = &cpu_data(i);
|
|
|
|
|
|
|
|
if (c->phys_proc_id == o->phys_proc_id &&
|
|
|
|
c->cpu_core_id == o->cpu_core_id) {
|
|
|
|
cpumask_set_cpu(i, cpu_sibling_mask(cpu));
|
|
|
|
cpumask_set_cpu(cpu, cpu_sibling_mask(i));
|
|
|
|
cpumask_set_cpu(i, cpu_core_mask(cpu));
|
|
|
|
cpumask_set_cpu(cpu, cpu_core_mask(i));
|
2009-03-12 22:19:53 -06:00
|
|
|
cpumask_set_cpu(i, c->llc_shared_map);
|
|
|
|
cpumask_set_cpu(cpu, o->llc_shared_map);
|
2008-03-03 10:13:02 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
|
2008-03-03 10:13:02 -07:00
|
|
|
}
|
|
|
|
|
2009-03-12 22:19:53 -06:00
|
|
|
cpumask_set_cpu(cpu, c->llc_shared_map);
|
2008-03-03 10:13:02 -07:00
|
|
|
|
|
|
|
if (current_cpu_data.x86_max_cores == 1) {
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_copy(cpu_core_mask(cpu), cpu_sibling_mask(cpu));
|
2008-03-03 10:13:02 -07:00
|
|
|
c->booted_cores = 1;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2009-01-04 06:18:03 -07:00
|
|
|
for_each_cpu(i, cpu_sibling_setup_mask) {
|
2008-03-03 10:13:02 -07:00
|
|
|
if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
|
|
|
|
per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
|
2009-03-12 22:19:53 -06:00
|
|
|
cpumask_set_cpu(i, c->llc_shared_map);
|
|
|
|
cpumask_set_cpu(cpu, cpu_data(i).llc_shared_map);
|
2008-03-03 10:13:02 -07:00
|
|
|
}
|
|
|
|
if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_set_cpu(i, cpu_core_mask(cpu));
|
|
|
|
cpumask_set_cpu(cpu, cpu_core_mask(i));
|
2008-03-03 10:13:02 -07:00
|
|
|
/*
|
|
|
|
* Does this new cpu bringup a new core?
|
|
|
|
*/
|
2009-01-04 06:18:03 -07:00
|
|
|
if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) {
|
2008-03-03 10:13:02 -07:00
|
|
|
/*
|
|
|
|
* for each core in package, increment
|
|
|
|
* the booted_cores for this new cpu
|
|
|
|
*/
|
2009-01-04 06:18:03 -07:00
|
|
|
if (cpumask_first(cpu_sibling_mask(i)) == i)
|
2008-03-03 10:13:02 -07:00
|
|
|
c->booted_cores++;
|
|
|
|
/*
|
|
|
|
* increment the core count for all
|
|
|
|
* the other cpus in this package
|
|
|
|
*/
|
|
|
|
if (i != cpu)
|
|
|
|
cpu_data(i).booted_cores++;
|
|
|
|
} else if (i != cpu && !c->booted_cores)
|
|
|
|
c->booted_cores = cpu_data(i).booted_cores;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-03-03 10:13:03 -07:00
|
|
|
/* maps the cpu to the sched domain representing multi-core */
|
2008-12-26 04:53:41 -07:00
|
|
|
const struct cpumask *cpu_coregroup_mask(int cpu)
|
2008-03-03 10:13:03 -07:00
|
|
|
{
|
|
|
|
struct cpuinfo_x86 *c = &cpu_data(cpu);
|
|
|
|
/*
|
|
|
|
* For perf, we return last level cache shared map.
|
|
|
|
* And for power savings, we return cpu_core_map
|
|
|
|
*/
|
2009-09-03 01:44:28 -06:00
|
|
|
if ((sched_mc_power_savings || sched_smt_power_savings) &&
|
|
|
|
!(cpu_has(c, X86_FEATURE_AMD_DCM)))
|
2009-01-04 06:18:03 -07:00
|
|
|
return cpu_core_mask(cpu);
|
2008-03-03 10:13:03 -07:00
|
|
|
else
|
2009-03-12 22:19:53 -06:00
|
|
|
return c->llc_shared_map;
|
2008-12-26 04:53:41 -07:00
|
|
|
}
|
|
|
|
|
2008-04-23 05:20:56 -06:00
|
|
|
static void impress_friends(void)
|
2008-03-19 11:25:27 -06:00
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
unsigned long bogosum = 0;
|
|
|
|
/*
|
|
|
|
* Allow the user to impress friends.
|
|
|
|
*/
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Before bogomips.\n");
|
2008-03-19 11:25:27 -06:00
|
|
|
for_each_possible_cpu(cpu)
|
2009-01-04 06:18:03 -07:00
|
|
|
if (cpumask_test_cpu(cpu, cpu_callout_mask))
|
2008-03-19 11:25:27 -06:00
|
|
|
bogosum += cpu_data(cpu).loops_per_jiffy;
|
|
|
|
printk(KERN_INFO
|
|
|
|
"Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
|
2008-03-19 11:25:29 -06:00
|
|
|
num_online_cpus(),
|
2008-03-19 11:25:27 -06:00
|
|
|
bogosum/(500000/HZ),
|
|
|
|
(bogosum/(5000/HZ))%100);
|
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Before bogocount - setting activated=1.\n");
|
2008-03-19 11:25:27 -06:00
|
|
|
}
|
|
|
|
|
x86: fix wakeup_cpu with numaq/es7000, v2
Impact: fix secondary-CPU wakeup/init path with numaq and es7000
While looking at wakeup_secondary_cpu for WAKE_SECONDARY_VIA_NMI:
|#ifdef WAKE_SECONDARY_VIA_NMI
|/*
| * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
| * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
| * won't ... remember to clear down the APIC, etc later.
| */
|static int __devinit
|wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
|{
| unsigned long send_status, accept_status = 0;
| int maxlvt;
|...
| if (APIC_INTEGRATED(apic_version[phys_apicid])) {
| maxlvt = lapic_get_maxlvt();
I noticed that there is no warning about undefined phys_apicid...
because WAKE_SECONDARY_VIA_NMI and WAKE_SECONDARY_VIA_INIT can not be
defined at the same time. So NUMAQ is using wrong wakeup_secondary_cpu.
WAKE_SECONDARY_VIA_NMI, WAKE_SECONDARY_VIA_INIT and
WAKE_SECONDARY_VIA_MIP are variants of a weird and fragile
preprocessor-driven "HAL" mechanisms to specify the kind of secondary-CPU
wakeup strategy a given x86 kernel will use.
The vast majority of systems want to use INIT for secondary wakeup - NUMAQ
uses an NMI, (old-style-) ES7000 uses 'MIP' (a firmware driven in-memory
flag to let secondaries continue).
So convert these mechanisms to x86_quirks and add a
->wakeup_secondary_cpu() method to specify the rare exception
to the sane default.
Extend genapic accordingly as well, for 32-bit.
While looking further, I noticed that functions in wakecup.h for numaq
and es7000 are different to the default in mach_wakecpu.h - but smpboot.c
will only use default mach_wakecpu.h with smphook.h.
So we need to add mach_wakecpu.h for mach_generic, to properly support
numaq and es7000, and vectorize the following SMP init methods:
int trampoline_phys_low;
int trampoline_phys_high;
void (*wait_for_init_deassert)(atomic_t *deassert);
void (*smp_callin_clear_local_apic)(void);
void (*store_NMI_vector)(unsigned short *high, unsigned short *low);
void (*restore_NMI_vector)(unsigned short *high, unsigned short *low);
void (*inquire_remote_apic)(int apicid);
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-16 04:12:49 -07:00
|
|
|
void __inquire_remote_apic(int apicid)
|
2008-03-19 11:25:59 -06:00
|
|
|
{
|
|
|
|
unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
|
|
|
|
char *names[] = { "ID", "VERSION", "SPIV" };
|
|
|
|
int timeout;
|
|
|
|
u32 status;
|
|
|
|
|
2008-09-10 22:56:46 -06:00
|
|
|
printk(KERN_INFO "Inquiring remote APIC 0x%x...\n", apicid);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
for (i = 0; i < ARRAY_SIZE(regs); i++) {
|
2008-09-10 22:56:46 -06:00
|
|
|
printk(KERN_INFO "... APIC 0x%x %s: ", apicid, names[i]);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Wait for idle.
|
|
|
|
*/
|
|
|
|
status = safe_apic_wait_icr_idle();
|
|
|
|
if (status)
|
|
|
|
printk(KERN_CONT
|
|
|
|
"a previous APIC delivery may have failed\n");
|
|
|
|
|
2008-07-10 12:16:49 -06:00
|
|
|
apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
timeout = 0;
|
|
|
|
do {
|
|
|
|
udelay(100);
|
|
|
|
status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
|
|
|
|
} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
|
|
|
|
|
|
|
|
switch (status) {
|
|
|
|
case APIC_ICR_RR_VALID:
|
|
|
|
status = apic_read(APIC_RRR);
|
|
|
|
printk(KERN_CONT "%08x\n", status);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
printk(KERN_CONT "failed\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
|
|
|
|
* INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
|
|
|
|
* won't ... remember to clear down the APIC, etc later.
|
|
|
|
*/
|
2009-04-18 13:45:28 -06:00
|
|
|
int __cpuinit
|
x86: fix wakeup_cpu with numaq/es7000, v2
Impact: fix secondary-CPU wakeup/init path with numaq and es7000
While looking at wakeup_secondary_cpu for WAKE_SECONDARY_VIA_NMI:
|#ifdef WAKE_SECONDARY_VIA_NMI
|/*
| * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
| * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
| * won't ... remember to clear down the APIC, etc later.
| */
|static int __devinit
|wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
|{
| unsigned long send_status, accept_status = 0;
| int maxlvt;
|...
| if (APIC_INTEGRATED(apic_version[phys_apicid])) {
| maxlvt = lapic_get_maxlvt();
I noticed that there is no warning about undefined phys_apicid...
because WAKE_SECONDARY_VIA_NMI and WAKE_SECONDARY_VIA_INIT can not be
defined at the same time. So NUMAQ is using wrong wakeup_secondary_cpu.
WAKE_SECONDARY_VIA_NMI, WAKE_SECONDARY_VIA_INIT and
WAKE_SECONDARY_VIA_MIP are variants of a weird and fragile
preprocessor-driven "HAL" mechanisms to specify the kind of secondary-CPU
wakeup strategy a given x86 kernel will use.
The vast majority of systems want to use INIT for secondary wakeup - NUMAQ
uses an NMI, (old-style-) ES7000 uses 'MIP' (a firmware driven in-memory
flag to let secondaries continue).
So convert these mechanisms to x86_quirks and add a
->wakeup_secondary_cpu() method to specify the rare exception
to the sane default.
Extend genapic accordingly as well, for 32-bit.
While looking further, I noticed that functions in wakecup.h for numaq
and es7000 are different to the default in mach_wakecpu.h - but smpboot.c
will only use default mach_wakecpu.h with smphook.h.
So we need to add mach_wakecpu.h for mach_generic, to properly support
numaq and es7000, and vectorize the following SMP init methods:
int trampoline_phys_low;
int trampoline_phys_high;
void (*wait_for_init_deassert)(atomic_t *deassert);
void (*smp_callin_clear_local_apic)(void);
void (*store_NMI_vector)(unsigned short *high, unsigned short *low);
void (*restore_NMI_vector)(unsigned short *high, unsigned short *low);
void (*inquire_remote_apic)(int apicid);
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-16 04:12:49 -07:00
|
|
|
wakeup_secondary_cpu_via_nmi(int logical_apicid, unsigned long start_eip)
|
2008-03-19 11:25:59 -06:00
|
|
|
{
|
|
|
|
unsigned long send_status, accept_status = 0;
|
|
|
|
int maxlvt;
|
|
|
|
|
|
|
|
/* Target chip */
|
|
|
|
/* Boot on the stack */
|
|
|
|
/* Kick the second */
|
2009-01-27 21:29:25 -07:00
|
|
|
apic_icr_write(APIC_DM_NMI | apic->dest_logical, logical_apicid);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Waiting for send to finish...\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
send_status = safe_apic_wait_icr_idle();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Give the other CPU some time to accept the IPI.
|
|
|
|
*/
|
|
|
|
udelay(200);
|
x86: fix wakeup_cpu with numaq/es7000, v2
Impact: fix secondary-CPU wakeup/init path with numaq and es7000
While looking at wakeup_secondary_cpu for WAKE_SECONDARY_VIA_NMI:
|#ifdef WAKE_SECONDARY_VIA_NMI
|/*
| * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
| * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
| * won't ... remember to clear down the APIC, etc later.
| */
|static int __devinit
|wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
|{
| unsigned long send_status, accept_status = 0;
| int maxlvt;
|...
| if (APIC_INTEGRATED(apic_version[phys_apicid])) {
| maxlvt = lapic_get_maxlvt();
I noticed that there is no warning about undefined phys_apicid...
because WAKE_SECONDARY_VIA_NMI and WAKE_SECONDARY_VIA_INIT can not be
defined at the same time. So NUMAQ is using wrong wakeup_secondary_cpu.
WAKE_SECONDARY_VIA_NMI, WAKE_SECONDARY_VIA_INIT and
WAKE_SECONDARY_VIA_MIP are variants of a weird and fragile
preprocessor-driven "HAL" mechanisms to specify the kind of secondary-CPU
wakeup strategy a given x86 kernel will use.
The vast majority of systems want to use INIT for secondary wakeup - NUMAQ
uses an NMI, (old-style-) ES7000 uses 'MIP' (a firmware driven in-memory
flag to let secondaries continue).
So convert these mechanisms to x86_quirks and add a
->wakeup_secondary_cpu() method to specify the rare exception
to the sane default.
Extend genapic accordingly as well, for 32-bit.
While looking further, I noticed that functions in wakecup.h for numaq
and es7000 are different to the default in mach_wakecpu.h - but smpboot.c
will only use default mach_wakecpu.h with smphook.h.
So we need to add mach_wakecpu.h for mach_generic, to properly support
numaq and es7000, and vectorize the following SMP init methods:
int trampoline_phys_low;
int trampoline_phys_high;
void (*wait_for_init_deassert)(atomic_t *deassert);
void (*smp_callin_clear_local_apic)(void);
void (*store_NMI_vector)(unsigned short *high, unsigned short *low);
void (*restore_NMI_vector)(unsigned short *high, unsigned short *low);
void (*inquire_remote_apic)(int apicid);
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-16 04:12:49 -07:00
|
|
|
if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
|
2008-09-14 11:58:49 -06:00
|
|
|
maxlvt = lapic_get_maxlvt();
|
|
|
|
if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
|
|
|
|
apic_write(APIC_ESR, 0);
|
|
|
|
accept_status = (apic_read(APIC_ESR) & 0xEF);
|
|
|
|
}
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("NMI sent.\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
if (send_status)
|
|
|
|
printk(KERN_ERR "APIC never delivered???\n");
|
|
|
|
if (accept_status)
|
|
|
|
printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
|
|
|
|
|
|
|
|
return (send_status | accept_status);
|
|
|
|
}
|
|
|
|
|
2009-04-18 13:45:28 -06:00
|
|
|
static int __cpuinit
|
x86: fix wakeup_cpu with numaq/es7000, v2
Impact: fix secondary-CPU wakeup/init path with numaq and es7000
While looking at wakeup_secondary_cpu for WAKE_SECONDARY_VIA_NMI:
|#ifdef WAKE_SECONDARY_VIA_NMI
|/*
| * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
| * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
| * won't ... remember to clear down the APIC, etc later.
| */
|static int __devinit
|wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
|{
| unsigned long send_status, accept_status = 0;
| int maxlvt;
|...
| if (APIC_INTEGRATED(apic_version[phys_apicid])) {
| maxlvt = lapic_get_maxlvt();
I noticed that there is no warning about undefined phys_apicid...
because WAKE_SECONDARY_VIA_NMI and WAKE_SECONDARY_VIA_INIT can not be
defined at the same time. So NUMAQ is using wrong wakeup_secondary_cpu.
WAKE_SECONDARY_VIA_NMI, WAKE_SECONDARY_VIA_INIT and
WAKE_SECONDARY_VIA_MIP are variants of a weird and fragile
preprocessor-driven "HAL" mechanisms to specify the kind of secondary-CPU
wakeup strategy a given x86 kernel will use.
The vast majority of systems want to use INIT for secondary wakeup - NUMAQ
uses an NMI, (old-style-) ES7000 uses 'MIP' (a firmware driven in-memory
flag to let secondaries continue).
So convert these mechanisms to x86_quirks and add a
->wakeup_secondary_cpu() method to specify the rare exception
to the sane default.
Extend genapic accordingly as well, for 32-bit.
While looking further, I noticed that functions in wakecup.h for numaq
and es7000 are different to the default in mach_wakecpu.h - but smpboot.c
will only use default mach_wakecpu.h with smphook.h.
So we need to add mach_wakecpu.h for mach_generic, to properly support
numaq and es7000, and vectorize the following SMP init methods:
int trampoline_phys_low;
int trampoline_phys_high;
void (*wait_for_init_deassert)(atomic_t *deassert);
void (*smp_callin_clear_local_apic)(void);
void (*store_NMI_vector)(unsigned short *high, unsigned short *low);
void (*restore_NMI_vector)(unsigned short *high, unsigned short *low);
void (*inquire_remote_apic)(int apicid);
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-11-16 04:12:49 -07:00
|
|
|
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
|
2008-03-19 11:25:59 -06:00
|
|
|
{
|
|
|
|
unsigned long send_status, accept_status = 0;
|
|
|
|
int maxlvt, num_starts, j;
|
|
|
|
|
x86: APIC: remove apic_write_around(); use alternatives
Use alternatives to select the workaround for the 11AP Pentium erratum
for the affected steppings on the fly rather than build time. Remove the
X86_GOOD_APIC configuration option and replace all the calls to
apic_write_around() with plain apic_write(), protecting accesses to the
ESR as appropriate due to the 3AP Pentium erratum. Remove
apic_read_around() and all its invocations altogether as not needed.
Remove apic_write_atomic() and all its implementing backends. The use of
ASM_OUTPUT2() is not strictly needed for input constraints, but I have
used it for readability's sake.
I had the feeling no one else was brave enough to do it, so I went ahead
and here it is. Verified by checking the generated assembly and tested
with both a 32-bit and a 64-bit configuration, also with the 11AP
"feature" forced on and verified with gdb on /proc/kcore to work as
expected (as an 11AP machines are quite hard to get hands on these days).
Some script complained about the use of "volatile", but apic_write() needs
it for the same reason and is effectively a replacement for writel(), so I
have disregarded it.
I am not sure what the policy wrt defconfig files is, they are generated
and there is risk of a conflict resulting from an unrelated change, so I
have left changes to them out. The option will get removed from them at
the next run.
Some testing with machines other than mine will be needed to avoid some
stupid mistake, but despite its volume, the change is not really that
intrusive, so I am fairly confident that because it works for me, it will
everywhere.
Signed-off-by: Maciej W. Rozycki <macro@linux-mips.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-07-16 12:15:30 -06:00
|
|
|
maxlvt = lapic_get_maxlvt();
|
|
|
|
|
2008-03-19 11:25:59 -06:00
|
|
|
/*
|
|
|
|
* Be paranoid about clearing APIC errors.
|
|
|
|
*/
|
|
|
|
if (APIC_INTEGRATED(apic_version[phys_apicid])) {
|
x86: APIC: remove apic_write_around(); use alternatives
Use alternatives to select the workaround for the 11AP Pentium erratum
for the affected steppings on the fly rather than build time. Remove the
X86_GOOD_APIC configuration option and replace all the calls to
apic_write_around() with plain apic_write(), protecting accesses to the
ESR as appropriate due to the 3AP Pentium erratum. Remove
apic_read_around() and all its invocations altogether as not needed.
Remove apic_write_atomic() and all its implementing backends. The use of
ASM_OUTPUT2() is not strictly needed for input constraints, but I have
used it for readability's sake.
I had the feeling no one else was brave enough to do it, so I went ahead
and here it is. Verified by checking the generated assembly and tested
with both a 32-bit and a 64-bit configuration, also with the 11AP
"feature" forced on and verified with gdb on /proc/kcore to work as
expected (as an 11AP machines are quite hard to get hands on these days).
Some script complained about the use of "volatile", but apic_write() needs
it for the same reason and is effectively a replacement for writel(), so I
have disregarded it.
I am not sure what the policy wrt defconfig files is, they are generated
and there is risk of a conflict resulting from an unrelated change, so I
have left changes to them out. The option will get removed from them at
the next run.
Some testing with machines other than mine will be needed to avoid some
stupid mistake, but despite its volume, the change is not really that
intrusive, so I am fairly confident that because it works for me, it will
everywhere.
Signed-off-by: Maciej W. Rozycki <macro@linux-mips.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-07-16 12:15:30 -06:00
|
|
|
if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
|
|
|
|
apic_write(APIC_ESR, 0);
|
2008-03-19 11:25:59 -06:00
|
|
|
apic_read(APIC_ESR);
|
|
|
|
}
|
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Asserting INIT.\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Turn INIT on target chip
|
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* Send IPI
|
|
|
|
*/
|
2008-07-10 12:16:49 -06:00
|
|
|
apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
|
|
|
|
phys_apicid);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Waiting for send to finish...\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
send_status = safe_apic_wait_icr_idle();
|
|
|
|
|
|
|
|
mdelay(10);
|
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Deasserting INIT.\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/* Target chip */
|
|
|
|
/* Send IPI */
|
2008-07-10 12:16:49 -06:00
|
|
|
apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Waiting for send to finish...\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
send_status = safe_apic_wait_icr_idle();
|
|
|
|
|
|
|
|
mb();
|
|
|
|
atomic_set(&init_deasserted, 1);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Should we send STARTUP IPIs ?
|
|
|
|
*
|
|
|
|
* Determine this based on the APIC version.
|
|
|
|
* If we don't have an integrated APIC, don't send the STARTUP IPIs.
|
|
|
|
*/
|
|
|
|
if (APIC_INTEGRATED(apic_version[phys_apicid]))
|
|
|
|
num_starts = 2;
|
|
|
|
else
|
|
|
|
num_starts = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Paravirt / VMI wants a startup IPI hook here to set up the
|
|
|
|
* target processor state.
|
|
|
|
*/
|
|
|
|
startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
|
|
|
|
(unsigned long)stack_start.sp);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Run STARTUP IPI loop.
|
|
|
|
*/
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("#startup loops: %d.\n", num_starts);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
for (j = 1; j <= num_starts; j++) {
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Sending STARTUP #%d.\n", j);
|
x86: APIC: remove apic_write_around(); use alternatives
Use alternatives to select the workaround for the 11AP Pentium erratum
for the affected steppings on the fly rather than build time. Remove the
X86_GOOD_APIC configuration option and replace all the calls to
apic_write_around() with plain apic_write(), protecting accesses to the
ESR as appropriate due to the 3AP Pentium erratum. Remove
apic_read_around() and all its invocations altogether as not needed.
Remove apic_write_atomic() and all its implementing backends. The use of
ASM_OUTPUT2() is not strictly needed for input constraints, but I have
used it for readability's sake.
I had the feeling no one else was brave enough to do it, so I went ahead
and here it is. Verified by checking the generated assembly and tested
with both a 32-bit and a 64-bit configuration, also with the 11AP
"feature" forced on and verified with gdb on /proc/kcore to work as
expected (as an 11AP machines are quite hard to get hands on these days).
Some script complained about the use of "volatile", but apic_write() needs
it for the same reason and is effectively a replacement for writel(), so I
have disregarded it.
I am not sure what the policy wrt defconfig files is, they are generated
and there is risk of a conflict resulting from an unrelated change, so I
have left changes to them out. The option will get removed from them at
the next run.
Some testing with machines other than mine will be needed to avoid some
stupid mistake, but despite its volume, the change is not really that
intrusive, so I am fairly confident that because it works for me, it will
everywhere.
Signed-off-by: Maciej W. Rozycki <macro@linux-mips.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-07-16 12:15:30 -06:00
|
|
|
if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
|
|
|
|
apic_write(APIC_ESR, 0);
|
2008-03-19 11:25:59 -06:00
|
|
|
apic_read(APIC_ESR);
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("After apic_write.\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* STARTUP IPI
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* Target chip */
|
|
|
|
/* Boot on the stack */
|
|
|
|
/* Kick the second */
|
2008-07-10 12:16:49 -06:00
|
|
|
apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
|
|
|
|
phys_apicid);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Give the other CPU some time to accept the IPI.
|
|
|
|
*/
|
|
|
|
udelay(300);
|
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Startup point 1.\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Waiting for send to finish...\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
send_status = safe_apic_wait_icr_idle();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Give the other CPU some time to accept the IPI.
|
|
|
|
*/
|
|
|
|
udelay(200);
|
x86: APIC: remove apic_write_around(); use alternatives
Use alternatives to select the workaround for the 11AP Pentium erratum
for the affected steppings on the fly rather than build time. Remove the
X86_GOOD_APIC configuration option and replace all the calls to
apic_write_around() with plain apic_write(), protecting accesses to the
ESR as appropriate due to the 3AP Pentium erratum. Remove
apic_read_around() and all its invocations altogether as not needed.
Remove apic_write_atomic() and all its implementing backends. The use of
ASM_OUTPUT2() is not strictly needed for input constraints, but I have
used it for readability's sake.
I had the feeling no one else was brave enough to do it, so I went ahead
and here it is. Verified by checking the generated assembly and tested
with both a 32-bit and a 64-bit configuration, also with the 11AP
"feature" forced on and verified with gdb on /proc/kcore to work as
expected (as an 11AP machines are quite hard to get hands on these days).
Some script complained about the use of "volatile", but apic_write() needs
it for the same reason and is effectively a replacement for writel(), so I
have disregarded it.
I am not sure what the policy wrt defconfig files is, they are generated
and there is risk of a conflict resulting from an unrelated change, so I
have left changes to them out. The option will get removed from them at
the next run.
Some testing with machines other than mine will be needed to avoid some
stupid mistake, but despite its volume, the change is not really that
intrusive, so I am fairly confident that because it works for me, it will
everywhere.
Signed-off-by: Maciej W. Rozycki <macro@linux-mips.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-07-16 12:15:30 -06:00
|
|
|
if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
|
2008-03-19 11:25:59 -06:00
|
|
|
apic_write(APIC_ESR, 0);
|
|
|
|
accept_status = (apic_read(APIC_ESR) & 0xEF);
|
|
|
|
if (send_status || accept_status)
|
|
|
|
break;
|
|
|
|
}
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("After Startup.\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
if (send_status)
|
|
|
|
printk(KERN_ERR "APIC never delivered???\n");
|
|
|
|
if (accept_status)
|
|
|
|
printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
|
|
|
|
|
|
|
|
return (send_status | accept_status);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct create_idle {
|
|
|
|
struct work_struct work;
|
|
|
|
struct task_struct *idle;
|
|
|
|
struct completion done;
|
|
|
|
int cpu;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void __cpuinit do_fork_idle(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct create_idle *c_idle =
|
|
|
|
container_of(work, struct create_idle, work);
|
|
|
|
|
|
|
|
c_idle->idle = fork_idle(c_idle->cpu);
|
|
|
|
complete(&c_idle->done);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
|
|
|
|
* (ie clustered apic addressing mode), this is a LOGICAL apic ID.
|
2009-02-26 05:51:40 -07:00
|
|
|
* Returns zero if CPU booted OK, else error code from
|
|
|
|
* ->wakeup_secondary_cpu.
|
2008-03-19 11:25:59 -06:00
|
|
|
*/
|
2009-02-17 08:22:09 -07:00
|
|
|
static int __cpuinit do_boot_cpu(int apicid, int cpu)
|
2008-03-19 11:25:59 -06:00
|
|
|
{
|
|
|
|
unsigned long boot_error = 0;
|
|
|
|
unsigned long start_ip;
|
2009-02-17 08:22:09 -07:00
|
|
|
int timeout;
|
2008-03-19 11:25:59 -06:00
|
|
|
struct create_idle c_idle = {
|
2009-02-17 08:22:09 -07:00
|
|
|
.cpu = cpu,
|
|
|
|
.done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
|
2008-03-19 11:25:59 -06:00
|
|
|
};
|
2009-02-17 08:22:09 -07:00
|
|
|
|
2008-03-19 11:25:59 -06:00
|
|
|
INIT_WORK(&c_idle.work, do_fork_idle);
|
|
|
|
|
|
|
|
alternatives_smp_switch(1);
|
|
|
|
|
|
|
|
c_idle.idle = get_idle_for_cpu(cpu);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We can't use kernel_thread since we must avoid to
|
|
|
|
* reschedule the child.
|
|
|
|
*/
|
|
|
|
if (c_idle.idle) {
|
|
|
|
c_idle.idle->thread.sp = (unsigned long) (((struct pt_regs *)
|
|
|
|
(THREAD_SIZE + task_stack_page(c_idle.idle))) - 1);
|
|
|
|
init_idle(c_idle.idle, cpu);
|
|
|
|
goto do_rest;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!keventd_up() || current_is_keventd())
|
|
|
|
c_idle.work.func(&c_idle.work);
|
|
|
|
else {
|
|
|
|
schedule_work(&c_idle.work);
|
|
|
|
wait_for_completion(&c_idle.done);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (IS_ERR(c_idle.idle)) {
|
|
|
|
printk("failed fork for CPU %d\n", cpu);
|
|
|
|
return PTR_ERR(c_idle.idle);
|
|
|
|
}
|
|
|
|
|
|
|
|
set_idle_for_cpu(cpu, c_idle.idle);
|
|
|
|
do_rest:
|
|
|
|
per_cpu(current_task, cpu) = c_idle.idle;
|
2009-01-18 08:38:58 -07:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-03-19 11:25:59 -06:00
|
|
|
/* Stack for startup_32 can be just as for start_secondary onwards */
|
|
|
|
irq_ctx_init(cpu);
|
|
|
|
#else
|
|
|
|
clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
|
2009-01-13 04:41:35 -07:00
|
|
|
initial_gs = per_cpu_offset(cpu);
|
2009-01-18 08:38:58 -07:00
|
|
|
per_cpu(kernel_stack, cpu) =
|
|
|
|
(unsigned long)task_stack_page(c_idle.idle) -
|
|
|
|
KERNEL_STACK_OFFSET + THREAD_SIZE;
|
2008-03-19 11:25:59 -06:00
|
|
|
#endif
|
2008-05-28 17:19:53 -06:00
|
|
|
early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
|
2008-05-28 10:01:54 -06:00
|
|
|
initial_code = (unsigned long)start_secondary;
|
2008-05-27 19:22:54 -06:00
|
|
|
stack_start.sp = (void *) c_idle.idle->thread.sp;
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/* start_ip had better be page-aligned! */
|
|
|
|
start_ip = setup_trampoline();
|
|
|
|
|
|
|
|
/* So we see what's up */
|
2008-09-10 22:56:46 -06:00
|
|
|
printk(KERN_INFO "Booting processor %d APIC 0x%x ip 0x%lx\n",
|
2008-03-19 11:25:59 -06:00
|
|
|
cpu, apicid, start_ip);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This grunge runs the startup process for
|
|
|
|
* the targeted processor.
|
|
|
|
*/
|
|
|
|
|
|
|
|
atomic_set(&init_deasserted, 0);
|
|
|
|
|
2008-04-16 10:45:15 -06:00
|
|
|
if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
|
2008-03-19 11:25:59 -06:00
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Setting warm reset code and vector.\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
|
2008-04-16 10:45:15 -06:00
|
|
|
smpboot_setup_warm_reset_vector(start_ip);
|
|
|
|
/*
|
|
|
|
* Be paranoid about clearing APIC errors.
|
2008-10-22 08:00:09 -06:00
|
|
|
*/
|
|
|
|
if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
|
|
|
|
apic_write(APIC_ESR, 0);
|
|
|
|
apic_read(APIC_ESR);
|
|
|
|
}
|
2008-04-16 10:45:15 -06:00
|
|
|
}
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/*
|
2009-02-26 05:51:40 -07:00
|
|
|
* Kick the secondary CPU. Use the method in the APIC driver
|
|
|
|
* if it's defined - or use an INIT boot APIC message otherwise:
|
2008-03-19 11:25:59 -06:00
|
|
|
*/
|
2009-02-26 05:51:40 -07:00
|
|
|
if (apic->wakeup_secondary_cpu)
|
|
|
|
boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
|
|
|
|
else
|
|
|
|
boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
if (!boot_error) {
|
|
|
|
/*
|
|
|
|
* allow APs to start initializing.
|
|
|
|
*/
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Before Callout %d.\n", cpu);
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_set_cpu(cpu, cpu_callout_mask);
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("After Callout %d.\n", cpu);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Wait 5s total for a response
|
|
|
|
*/
|
|
|
|
for (timeout = 0; timeout < 50000; timeout++) {
|
2009-01-04 06:18:03 -07:00
|
|
|
if (cpumask_test_cpu(cpu, cpu_callin_mask))
|
2008-03-19 11:25:59 -06:00
|
|
|
break; /* It has booted */
|
|
|
|
udelay(100);
|
|
|
|
}
|
|
|
|
|
2009-01-04 06:18:03 -07:00
|
|
|
if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
|
2008-03-19 11:25:59 -06:00
|
|
|
/* number CPUs logically, starting from 1 (BSP is 0) */
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("OK.\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
printk(KERN_INFO "CPU%d: ", cpu);
|
|
|
|
print_cpu_info(&cpu_data(cpu));
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("CPU has booted.\n");
|
2008-03-19 11:25:59 -06:00
|
|
|
} else {
|
|
|
|
boot_error = 1;
|
|
|
|
if (*((volatile unsigned char *)trampoline_base)
|
|
|
|
== 0xA5)
|
|
|
|
/* trampoline started but...? */
|
|
|
|
printk(KERN_ERR "Stuck ??\n");
|
|
|
|
else
|
|
|
|
/* trampoline code not run */
|
|
|
|
printk(KERN_ERR "Not responding.\n");
|
2009-01-28 08:31:52 -07:00
|
|
|
if (apic->inquire_remote_apic)
|
|
|
|
apic->inquire_remote_apic(apicid);
|
2008-03-19 11:25:59 -06:00
|
|
|
}
|
|
|
|
}
|
2009-01-13 04:41:35 -07:00
|
|
|
|
2008-03-19 11:25:59 -06:00
|
|
|
if (boot_error) {
|
|
|
|
/* Try to put things back the way they were before ... */
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 13:21:12 -06:00
|
|
|
numa_remove_cpu(cpu); /* was set by numa_add_cpu */
|
2009-01-04 06:18:03 -07:00
|
|
|
|
|
|
|
/* was set by do_boot_cpu() */
|
|
|
|
cpumask_clear_cpu(cpu, cpu_callout_mask);
|
|
|
|
|
|
|
|
/* was set by cpu_init() */
|
|
|
|
cpumask_clear_cpu(cpu, cpu_initialized_mask);
|
|
|
|
|
|
|
|
set_cpu_present(cpu, false);
|
2008-03-19 11:25:59 -06:00
|
|
|
per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* mark "stuck" area as not stuck */
|
|
|
|
*((volatile unsigned long *)trampoline_base) = 0;
|
|
|
|
|
2009-04-03 18:15:53 -06:00
|
|
|
if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
|
|
|
|
/*
|
|
|
|
* Cleanup possible dangling ends...
|
|
|
|
*/
|
|
|
|
smpboot_restore_warm_reset_vector();
|
|
|
|
}
|
2008-04-07 12:38:33 -06:00
|
|
|
|
2008-03-19 11:25:59 -06:00
|
|
|
return boot_error;
|
|
|
|
}
|
|
|
|
|
|
|
|
int __cpuinit native_cpu_up(unsigned int cpu)
|
|
|
|
{
|
2009-01-27 22:50:47 -07:00
|
|
|
int apicid = apic->cpu_present_to_apicid(cpu);
|
2008-03-19 11:25:59 -06:00
|
|
|
unsigned long flags;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
WARN_ON(irqs_disabled());
|
|
|
|
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("++++++++++++++++++++=_---CPU UP %u\n", cpu);
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
if (apicid == BAD_APICID || apicid == boot_cpu_physical_apicid ||
|
|
|
|
!physid_isset(apicid, phys_cpu_present_map)) {
|
|
|
|
printk(KERN_ERR "%s: bad cpu %d\n", __func__, cpu);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Already booted CPU?
|
|
|
|
*/
|
2009-01-04 06:18:03 -07:00
|
|
|
if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("do_boot_cpu %d Already started\n", cpu);
|
2008-03-19 11:25:59 -06:00
|
|
|
return -ENOSYS;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Save current MTRR state in case it was changed since early boot
|
|
|
|
* (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
|
|
|
|
*/
|
|
|
|
mtrr_save_state();
|
|
|
|
|
|
|
|
per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_32
|
|
|
|
/* init low mem mapping */
|
2008-03-17 17:37:13 -06:00
|
|
|
clone_pgd_range(swapper_pg_dir, swapper_pg_dir + KERNEL_PGD_BOUNDARY,
|
x86: fix app crashes after SMP resume
After resume on a 2cpu laptop, kernel builds collapse with a sed hang,
sh or make segfault (often on 20295564), real-time signal to cc1 etc.
Several hurdles to jump, but a manually-assisted bisect led to -rc1's
d2bcbad5f3ad38a1c09861bca7e252dde7bb8259 x86: do not zap_low_mappings
in __smp_prepare_cpus. Though the low mappings were removed at bootup,
they were left behind (with Global flags helping to keep them in TLB)
after resume or cpu online, causing the crashes seen.
Reinstate zap_low_mappings (with local __flush_tlb_all) for each cpu_up
on x86_32. This used to be serialized by smp_commenced_mask: that's now
gone, but a low_mappings flag will do. No need for native_smp_cpus_done
to repeat the zap: let mem_init zap BSP's low mappings just like on UP.
(In passing, fix error code from native_cpu_up: do_boot_cpu returns a
variety of diagnostic values, Dprintk what it says but convert to -EIO.
And save_pg_dir separately before zap_low_mappings: doesn't matter now,
but zapping twice in succession wiped out resume's swsusp_pg_dir.)
That worked well on the duo and one quad, but wouldn't boot 3rd or 4th
cpu on P4 Xeon, oopsing just after unlock_ipi_call_lock. The TLB flush
IPI now being sent reveals a long-standing bug: the booting cpu has its
APIC readied in smp_callin at the top of start_secondary, but isn't put
into the cpu_online_map until just before that unlock_ipi_call_lock.
So native_smp_call_function_mask to online cpus would send_IPI_allbutself,
including the cpu just coming up, though it has been excluded from the
count to wait for: by the time it handles the IPI, the call data on
native_smp_call_function_mask's stack may well have been overwritten.
So fall back to send_IPI_mask while cpu_online_map does not match
cpu_callout_map: perhaps there's a better APICological fix to be
made at the start_secondary end, but I wouldn't know that.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-05-13 07:26:57 -06:00
|
|
|
min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
|
2008-03-19 11:25:59 -06:00
|
|
|
flush_tlb_all();
|
x86: fix app crashes after SMP resume
After resume on a 2cpu laptop, kernel builds collapse with a sed hang,
sh or make segfault (often on 20295564), real-time signal to cc1 etc.
Several hurdles to jump, but a manually-assisted bisect led to -rc1's
d2bcbad5f3ad38a1c09861bca7e252dde7bb8259 x86: do not zap_low_mappings
in __smp_prepare_cpus. Though the low mappings were removed at bootup,
they were left behind (with Global flags helping to keep them in TLB)
after resume or cpu online, causing the crashes seen.
Reinstate zap_low_mappings (with local __flush_tlb_all) for each cpu_up
on x86_32. This used to be serialized by smp_commenced_mask: that's now
gone, but a low_mappings flag will do. No need for native_smp_cpus_done
to repeat the zap: let mem_init zap BSP's low mappings just like on UP.
(In passing, fix error code from native_cpu_up: do_boot_cpu returns a
variety of diagnostic values, Dprintk what it says but convert to -EIO.
And save_pg_dir separately before zap_low_mappings: doesn't matter now,
but zapping twice in succession wiped out resume's swsusp_pg_dir.)
That worked well on the duo and one quad, but wouldn't boot 3rd or 4th
cpu on P4 Xeon, oopsing just after unlock_ipi_call_lock. The TLB flush
IPI now being sent reveals a long-standing bug: the booting cpu has its
APIC readied in smp_callin at the top of start_secondary, but isn't put
into the cpu_online_map until just before that unlock_ipi_call_lock.
So native_smp_call_function_mask to online cpus would send_IPI_allbutself,
including the cpu just coming up, though it has been excluded from the
count to wait for: by the time it handles the IPI, the call data on
native_smp_call_function_mask's stack may well have been overwritten.
So fall back to send_IPI_mask while cpu_online_map does not match
cpu_callout_map: perhaps there's a better APICological fix to be
made at the start_secondary end, but I wouldn't know that.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-05-13 07:26:57 -06:00
|
|
|
low_mappings = 1;
|
2008-03-19 11:25:59 -06:00
|
|
|
|
|
|
|
err = do_boot_cpu(apicid, cpu);
|
x86: fix app crashes after SMP resume
After resume on a 2cpu laptop, kernel builds collapse with a sed hang,
sh or make segfault (often on 20295564), real-time signal to cc1 etc.
Several hurdles to jump, but a manually-assisted bisect led to -rc1's
d2bcbad5f3ad38a1c09861bca7e252dde7bb8259 x86: do not zap_low_mappings
in __smp_prepare_cpus. Though the low mappings were removed at bootup,
they were left behind (with Global flags helping to keep them in TLB)
after resume or cpu online, causing the crashes seen.
Reinstate zap_low_mappings (with local __flush_tlb_all) for each cpu_up
on x86_32. This used to be serialized by smp_commenced_mask: that's now
gone, but a low_mappings flag will do. No need for native_smp_cpus_done
to repeat the zap: let mem_init zap BSP's low mappings just like on UP.
(In passing, fix error code from native_cpu_up: do_boot_cpu returns a
variety of diagnostic values, Dprintk what it says but convert to -EIO.
And save_pg_dir separately before zap_low_mappings: doesn't matter now,
but zapping twice in succession wiped out resume's swsusp_pg_dir.)
That worked well on the duo and one quad, but wouldn't boot 3rd or 4th
cpu on P4 Xeon, oopsing just after unlock_ipi_call_lock. The TLB flush
IPI now being sent reveals a long-standing bug: the booting cpu has its
APIC readied in smp_callin at the top of start_secondary, but isn't put
into the cpu_online_map until just before that unlock_ipi_call_lock.
So native_smp_call_function_mask to online cpus would send_IPI_allbutself,
including the cpu just coming up, though it has been excluded from the
count to wait for: by the time it handles the IPI, the call data on
native_smp_call_function_mask's stack may well have been overwritten.
So fall back to send_IPI_mask while cpu_online_map does not match
cpu_callout_map: perhaps there's a better APICological fix to be
made at the start_secondary end, but I wouldn't know that.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-05-13 07:26:57 -06:00
|
|
|
|
2009-06-12 02:36:52 -06:00
|
|
|
zap_low_mappings(false);
|
x86: fix app crashes after SMP resume
After resume on a 2cpu laptop, kernel builds collapse with a sed hang,
sh or make segfault (often on 20295564), real-time signal to cc1 etc.
Several hurdles to jump, but a manually-assisted bisect led to -rc1's
d2bcbad5f3ad38a1c09861bca7e252dde7bb8259 x86: do not zap_low_mappings
in __smp_prepare_cpus. Though the low mappings were removed at bootup,
they were left behind (with Global flags helping to keep them in TLB)
after resume or cpu online, causing the crashes seen.
Reinstate zap_low_mappings (with local __flush_tlb_all) for each cpu_up
on x86_32. This used to be serialized by smp_commenced_mask: that's now
gone, but a low_mappings flag will do. No need for native_smp_cpus_done
to repeat the zap: let mem_init zap BSP's low mappings just like on UP.
(In passing, fix error code from native_cpu_up: do_boot_cpu returns a
variety of diagnostic values, Dprintk what it says but convert to -EIO.
And save_pg_dir separately before zap_low_mappings: doesn't matter now,
but zapping twice in succession wiped out resume's swsusp_pg_dir.)
That worked well on the duo and one quad, but wouldn't boot 3rd or 4th
cpu on P4 Xeon, oopsing just after unlock_ipi_call_lock. The TLB flush
IPI now being sent reveals a long-standing bug: the booting cpu has its
APIC readied in smp_callin at the top of start_secondary, but isn't put
into the cpu_online_map until just before that unlock_ipi_call_lock.
So native_smp_call_function_mask to online cpus would send_IPI_allbutself,
including the cpu just coming up, though it has been excluded from the
count to wait for: by the time it handles the IPI, the call data on
native_smp_call_function_mask's stack may well have been overwritten.
So fall back to send_IPI_mask while cpu_online_map does not match
cpu_callout_map: perhaps there's a better APICological fix to be
made at the start_secondary end, but I wouldn't know that.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-05-13 07:26:57 -06:00
|
|
|
low_mappings = 0;
|
|
|
|
#else
|
|
|
|
err = do_boot_cpu(apicid, cpu);
|
|
|
|
#endif
|
|
|
|
if (err) {
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("do_boot_cpu failed %d\n", err);
|
x86: fix app crashes after SMP resume
After resume on a 2cpu laptop, kernel builds collapse with a sed hang,
sh or make segfault (often on 20295564), real-time signal to cc1 etc.
Several hurdles to jump, but a manually-assisted bisect led to -rc1's
d2bcbad5f3ad38a1c09861bca7e252dde7bb8259 x86: do not zap_low_mappings
in __smp_prepare_cpus. Though the low mappings were removed at bootup,
they were left behind (with Global flags helping to keep them in TLB)
after resume or cpu online, causing the crashes seen.
Reinstate zap_low_mappings (with local __flush_tlb_all) for each cpu_up
on x86_32. This used to be serialized by smp_commenced_mask: that's now
gone, but a low_mappings flag will do. No need for native_smp_cpus_done
to repeat the zap: let mem_init zap BSP's low mappings just like on UP.
(In passing, fix error code from native_cpu_up: do_boot_cpu returns a
variety of diagnostic values, Dprintk what it says but convert to -EIO.
And save_pg_dir separately before zap_low_mappings: doesn't matter now,
but zapping twice in succession wiped out resume's swsusp_pg_dir.)
That worked well on the duo and one quad, but wouldn't boot 3rd or 4th
cpu on P4 Xeon, oopsing just after unlock_ipi_call_lock. The TLB flush
IPI now being sent reveals a long-standing bug: the booting cpu has its
APIC readied in smp_callin at the top of start_secondary, but isn't put
into the cpu_online_map until just before that unlock_ipi_call_lock.
So native_smp_call_function_mask to online cpus would send_IPI_allbutself,
including the cpu just coming up, though it has been excluded from the
count to wait for: by the time it handles the IPI, the call data on
native_smp_call_function_mask's stack may well have been overwritten.
So fall back to send_IPI_mask while cpu_online_map does not match
cpu_callout_map: perhaps there's a better APICological fix to be
made at the start_secondary end, but I wouldn't know that.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-05-13 07:26:57 -06:00
|
|
|
return -EIO;
|
2008-03-19 11:25:59 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check TSC synchronization with the AP (keep irqs disabled
|
|
|
|
* while doing so):
|
|
|
|
*/
|
|
|
|
local_irq_save(flags);
|
|
|
|
check_tsc_sync_source(cpu);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
2008-04-19 08:55:17 -06:00
|
|
|
while (!cpu_online(cpu)) {
|
2008-03-19 11:25:59 -06:00
|
|
|
cpu_relax();
|
|
|
|
touch_nmi_watchdog();
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-03-19 11:26:11 -06:00
|
|
|
/*
|
|
|
|
* Fall back to non SMP mode after errors.
|
|
|
|
*
|
|
|
|
* RED-PEN audit/test this more. I bet there is more state messed up here.
|
|
|
|
*/
|
|
|
|
static __init void disable_smp(void)
|
|
|
|
{
|
2009-03-12 22:19:54 -06:00
|
|
|
init_cpu_present(cpumask_of(0));
|
|
|
|
init_cpu_possible(cpumask_of(0));
|
2008-03-19 11:26:11 -06:00
|
|
|
smpboot_clear_io_apic_irqs();
|
2008-05-28 18:09:53 -06:00
|
|
|
|
2008-03-19 11:26:11 -06:00
|
|
|
if (smp_found_config)
|
2008-06-19 20:51:05 -06:00
|
|
|
physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
|
2008-03-19 11:26:11 -06:00
|
|
|
else
|
2008-06-19 20:51:05 -06:00
|
|
|
physid_set_mask_of_physid(0, &phys_cpu_present_map);
|
2008-03-19 11:26:11 -06:00
|
|
|
map_cpu_to_logical_apicid();
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_set_cpu(0, cpu_sibling_mask(0));
|
|
|
|
cpumask_set_cpu(0, cpu_core_mask(0));
|
2008-03-19 11:26:11 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Various sanity checks.
|
|
|
|
*/
|
|
|
|
static int __init smp_sanity_check(unsigned max_cpus)
|
|
|
|
{
|
2008-03-28 13:12:16 -06:00
|
|
|
preempt_disable();
|
2008-08-14 03:16:30 -06:00
|
|
|
|
2009-01-29 20:30:04 -07:00
|
|
|
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
|
2008-08-14 03:16:30 -06:00
|
|
|
if (def_to_bigsmp && nr_cpu_ids > 8) {
|
|
|
|
unsigned int cpu;
|
|
|
|
unsigned nr;
|
|
|
|
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"More than 8 CPUs detected - skipping them.\n"
|
2009-01-29 15:19:22 -07:00
|
|
|
"Use CONFIG_X86_BIGSMP.\n");
|
2008-08-14 03:16:30 -06:00
|
|
|
|
|
|
|
nr = 0;
|
|
|
|
for_each_present_cpu(cpu) {
|
|
|
|
if (nr >= 8)
|
2009-01-04 06:18:03 -07:00
|
|
|
set_cpu_present(cpu, false);
|
2008-08-14 03:16:30 -06:00
|
|
|
nr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
nr = 0;
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
if (nr >= 8)
|
2009-01-04 06:18:03 -07:00
|
|
|
set_cpu_possible(cpu, false);
|
2008-08-14 03:16:30 -06:00
|
|
|
nr++;
|
|
|
|
}
|
|
|
|
|
|
|
|
nr_cpu_ids = 8;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2008-03-19 11:26:11 -06:00
|
|
|
if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
|
2008-12-05 04:42:20 -07:00
|
|
|
printk(KERN_WARNING
|
|
|
|
"weird, boot CPU (#%d) not listed by the BIOS.\n",
|
|
|
|
hard_smp_processor_id());
|
|
|
|
|
2008-03-19 11:26:11 -06:00
|
|
|
physid_set(hard_smp_processor_id(), phys_cpu_present_map);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we couldn't find an SMP configuration at boot time,
|
|
|
|
* get out of here now!
|
|
|
|
*/
|
|
|
|
if (!smp_found_config && !acpi_lapic) {
|
2008-03-28 13:12:16 -06:00
|
|
|
preempt_enable();
|
2008-03-19 11:26:11 -06:00
|
|
|
printk(KERN_NOTICE "SMP motherboard not detected.\n");
|
|
|
|
disable_smp();
|
|
|
|
if (APIC_init_uniprocessor())
|
|
|
|
printk(KERN_NOTICE "Local APIC not detected."
|
|
|
|
" Using dummy APIC emulation.\n");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Should not be necessary because the MP table should list the boot
|
|
|
|
* CPU too, but we do it for the sake of robustness anyway.
|
|
|
|
*/
|
2009-01-28 04:43:18 -07:00
|
|
|
if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
|
2008-03-19 11:26:11 -06:00
|
|
|
printk(KERN_NOTICE
|
|
|
|
"weird, boot CPU (#%d) not listed by the BIOS.\n",
|
|
|
|
boot_cpu_physical_apicid);
|
|
|
|
physid_set(hard_smp_processor_id(), phys_cpu_present_map);
|
|
|
|
}
|
2008-03-28 13:12:16 -06:00
|
|
|
preempt_enable();
|
2008-03-19 11:26:11 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we couldn't find a local APIC, then get out of here now!
|
|
|
|
*/
|
|
|
|
if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
|
|
|
|
!cpu_has_apic) {
|
2009-06-07 06:48:40 -06:00
|
|
|
if (!disable_apic) {
|
|
|
|
pr_err("BIOS bug, local APIC #%d not detected!...\n",
|
|
|
|
boot_cpu_physical_apicid);
|
|
|
|
pr_err("... forcing use of dummy APIC emulation."
|
2008-03-19 11:26:11 -06:00
|
|
|
"(tell your hw vendor)\n");
|
2009-06-07 06:48:40 -06:00
|
|
|
}
|
2008-03-19 11:26:11 -06:00
|
|
|
smpboot_clear_io_apic();
|
2009-01-30 19:36:17 -07:00
|
|
|
arch_disable_smp_support();
|
2008-03-19 11:26:11 -06:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
verify_local_APIC();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If SMP should be disabled, then really disable it!
|
|
|
|
*/
|
|
|
|
if (!max_cpus) {
|
2008-05-21 15:09:43 -06:00
|
|
|
printk(KERN_INFO "SMP mode deactivated.\n");
|
2008-03-19 11:26:11 -06:00
|
|
|
smpboot_clear_io_apic();
|
2008-06-05 20:28:02 -06:00
|
|
|
|
|
|
|
localise_nmi_watchdog();
|
|
|
|
|
2008-04-21 15:14:44 -06:00
|
|
|
connect_bsp_APIC();
|
|
|
|
setup_local_APIC();
|
|
|
|
end_local_APIC_setup();
|
2008-03-19 11:26:11 -06:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __init smp_cpu_index_default(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct cpuinfo_x86 *c;
|
|
|
|
|
2008-04-19 08:55:17 -06:00
|
|
|
for_each_possible_cpu(i) {
|
2008-03-19 11:26:11 -06:00
|
|
|
c = &cpu_data(i);
|
|
|
|
/* mark all to hotplug */
|
2008-12-31 19:08:46 -07:00
|
|
|
c->cpu_index = nr_cpu_ids;
|
2008-03-19 11:26:11 -06:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Prepare for SMP bootup. The MP table or ACPI has been read
|
|
|
|
* earlier. Just do some sanity checking here and enable APIC mode.
|
|
|
|
*/
|
|
|
|
void __init native_smp_prepare_cpus(unsigned int max_cpus)
|
|
|
|
{
|
2009-03-12 22:19:50 -06:00
|
|
|
unsigned int i;
|
|
|
|
|
2008-05-12 07:44:38 -06:00
|
|
|
preempt_disable();
|
2008-03-19 11:26:11 -06:00
|
|
|
smp_cpu_index_default();
|
|
|
|
current_cpu_data = boot_cpu_data;
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_copy(cpu_callin_mask, cpumask_of(0));
|
2008-03-19 11:26:11 -06:00
|
|
|
mb();
|
|
|
|
/*
|
|
|
|
* Setup boot CPU information
|
|
|
|
*/
|
|
|
|
smp_store_cpu_info(0); /* Final full version of the data */
|
2008-07-10 12:16:49 -06:00
|
|
|
#ifdef CONFIG_X86_32
|
2008-03-19 11:26:11 -06:00
|
|
|
boot_cpu_logical_apicid = logical_smp_processor_id();
|
2008-07-10 12:16:49 -06:00
|
|
|
#endif
|
2008-03-19 11:26:11 -06:00
|
|
|
current_thread_info()->cpu = 0; /* needed? */
|
2009-03-12 22:19:50 -06:00
|
|
|
for_each_possible_cpu(i) {
|
2009-06-15 00:58:26 -06:00
|
|
|
zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
|
|
|
|
zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
|
|
|
|
zalloc_cpumask_var(&cpu_data(i).llc_shared_map, GFP_KERNEL);
|
2009-03-12 22:19:50 -06:00
|
|
|
}
|
2008-03-19 11:26:11 -06:00
|
|
|
set_cpu_sibling_map(0);
|
|
|
|
|
2008-07-10 12:16:58 -06:00
|
|
|
enable_IR_x2apic();
|
2009-02-16 18:29:58 -07:00
|
|
|
#ifdef CONFIG_X86_64
|
2009-01-27 22:50:47 -07:00
|
|
|
default_setup_apic_routing();
|
2008-07-10 12:16:58 -06:00
|
|
|
#endif
|
|
|
|
|
2008-03-19 11:26:11 -06:00
|
|
|
if (smp_sanity_check(max_cpus) < 0) {
|
|
|
|
printk(KERN_INFO "SMP disabled\n");
|
|
|
|
disable_smp();
|
2008-05-12 07:44:38 -06:00
|
|
|
goto out;
|
2008-03-19 11:26:11 -06:00
|
|
|
}
|
|
|
|
|
2008-03-28 13:12:16 -06:00
|
|
|
preempt_disable();
|
2008-07-11 19:44:16 -06:00
|
|
|
if (read_apic_id() != boot_cpu_physical_apicid) {
|
2008-03-19 11:26:11 -06:00
|
|
|
panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
|
2008-07-11 19:44:16 -06:00
|
|
|
read_apic_id(), boot_cpu_physical_apicid);
|
2008-03-19 11:26:11 -06:00
|
|
|
/* Or can we switch back to PIC here? */
|
|
|
|
}
|
2008-03-28 13:12:16 -06:00
|
|
|
preempt_enable();
|
2008-03-19 11:26:11 -06:00
|
|
|
|
|
|
|
connect_bsp_APIC();
|
2008-05-28 10:38:28 -06:00
|
|
|
|
2008-03-19 11:26:11 -06:00
|
|
|
/*
|
|
|
|
* Switch from PIC to APIC mode.
|
|
|
|
*/
|
|
|
|
setup_local_APIC();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Enable IO APIC before setting up error vector
|
|
|
|
*/
|
|
|
|
if (!skip_ioapic_setup && nr_ioapics)
|
|
|
|
enable_IO_APIC();
|
2009-02-15 00:57:28 -07:00
|
|
|
|
2008-03-19 11:26:11 -06:00
|
|
|
end_local_APIC_setup();
|
|
|
|
|
|
|
|
map_cpu_to_logical_apicid();
|
|
|
|
|
2009-01-28 04:43:18 -07:00
|
|
|
if (apic->setup_portio_remap)
|
|
|
|
apic->setup_portio_remap();
|
2008-03-19 11:26:11 -06:00
|
|
|
|
|
|
|
smpboot_setup_io_apic();
|
|
|
|
/*
|
|
|
|
* Set up local APIC timer on boot CPU.
|
|
|
|
*/
|
|
|
|
|
|
|
|
printk(KERN_INFO "CPU%d: ", 0);
|
|
|
|
print_cpu_info(&cpu_data(0));
|
2009-08-19 04:35:53 -06:00
|
|
|
x86_init.timers.setup_percpu_clockev();
|
2008-08-21 12:49:05 -06:00
|
|
|
|
|
|
|
if (is_uv_system())
|
|
|
|
uv_system_init();
|
2009-08-19 19:05:36 -06:00
|
|
|
|
|
|
|
set_mtrr_aps_delayed_init();
|
2008-05-12 07:44:38 -06:00
|
|
|
out:
|
|
|
|
preempt_enable();
|
2008-03-19 11:26:11 -06:00
|
|
|
}
|
2009-08-19 19:05:36 -06:00
|
|
|
|
|
|
|
void arch_enable_nonboot_cpus_begin(void)
|
|
|
|
{
|
|
|
|
set_mtrr_aps_delayed_init();
|
|
|
|
}
|
|
|
|
|
|
|
|
void arch_enable_nonboot_cpus_end(void)
|
|
|
|
{
|
|
|
|
mtrr_aps_init();
|
|
|
|
}
|
|
|
|
|
2008-03-19 11:26:01 -06:00
|
|
|
/*
|
|
|
|
* Early setup to make printk work.
|
|
|
|
*/
|
|
|
|
void __init native_smp_prepare_boot_cpu(void)
|
|
|
|
{
|
|
|
|
int me = smp_processor_id();
|
2009-01-30 01:47:53 -07:00
|
|
|
switch_to_new_gdt(me);
|
2009-01-04 06:18:03 -07:00
|
|
|
/* already set me in cpu_online_mask in boot_cpu_init() */
|
|
|
|
cpumask_set_cpu(me, cpu_callout_mask);
|
2008-03-19 11:26:01 -06:00
|
|
|
per_cpu(cpu_state, me) = CPU_ONLINE;
|
|
|
|
}
|
|
|
|
|
2008-03-19 11:26:02 -06:00
|
|
|
void __init native_smp_cpus_done(unsigned int max_cpus)
|
|
|
|
{
|
2008-07-21 13:35:38 -06:00
|
|
|
pr_debug("Boot done.\n");
|
2008-03-19 11:26:02 -06:00
|
|
|
|
|
|
|
impress_friends();
|
|
|
|
#ifdef CONFIG_X86_IO_APIC
|
|
|
|
setup_ioapic_dest();
|
|
|
|
#endif
|
|
|
|
check_nmi_watchdog();
|
2009-08-19 19:05:36 -06:00
|
|
|
mtrr_aps_init();
|
2008-03-19 11:26:02 -06:00
|
|
|
}
|
|
|
|
|
2008-12-17 16:21:39 -07:00
|
|
|
static int __initdata setup_possible_cpus = -1;
|
|
|
|
static int __init _setup_possible_cpus(char *str)
|
|
|
|
{
|
|
|
|
get_option(&str, &setup_possible_cpus);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("possible_cpus", _setup_possible_cpus);
|
|
|
|
|
|
|
|
|
2008-03-03 10:12:42 -07:00
|
|
|
/*
|
2009-03-12 22:19:54 -06:00
|
|
|
* cpu_possible_mask should be static, it cannot change as cpu's
|
2008-03-03 10:12:42 -07:00
|
|
|
* are onlined, or offlined. The reason is per-cpu data-structures
|
|
|
|
* are allocated by some modules at init time, and dont expect to
|
|
|
|
* do this dynamically on cpu arrival/departure.
|
2009-03-12 22:19:54 -06:00
|
|
|
* cpu_present_mask on the other hand can change dynamically.
|
2008-03-03 10:12:42 -07:00
|
|
|
* In case when cpu_hotplug is not compiled, then we resort to current
|
|
|
|
* behaviour, which is cpu_possible == cpu_present.
|
|
|
|
* - Ashok Raj
|
|
|
|
*
|
|
|
|
* Three ways to find out the number of additional hotplug CPUs:
|
|
|
|
* - If the BIOS specified disabled CPUs in ACPI/mptables use that.
|
2008-12-17 16:21:39 -07:00
|
|
|
* - The user can overwrite it with possible_cpus=NUM
|
2008-03-03 10:12:42 -07:00
|
|
|
* - Otherwise don't reserve additional CPUs.
|
|
|
|
* We do this because additional CPUs waste a lot of memory.
|
|
|
|
* -AK
|
|
|
|
*/
|
|
|
|
__init void prefill_possible_map(void)
|
|
|
|
{
|
2008-10-05 09:51:52 -06:00
|
|
|
int i, possible;
|
2008-03-03 10:12:42 -07:00
|
|
|
|
2008-07-02 19:54:40 -06:00
|
|
|
/* no processor from mptable or madt */
|
|
|
|
if (!num_processors)
|
|
|
|
num_processors = 1;
|
|
|
|
|
2008-12-17 16:21:39 -07:00
|
|
|
if (setup_possible_cpus == -1)
|
|
|
|
possible = num_processors + disabled_cpus;
|
|
|
|
else
|
|
|
|
possible = setup_possible_cpus;
|
|
|
|
|
2008-12-31 19:08:45 -07:00
|
|
|
total_cpus = max_t(int, possible, num_processors + disabled_cpus);
|
|
|
|
|
2008-12-17 16:21:39 -07:00
|
|
|
if (possible > CONFIG_NR_CPUS) {
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"%d Processors exceeds NR_CPUS limit of %d\n",
|
|
|
|
possible, CONFIG_NR_CPUS);
|
|
|
|
possible = CONFIG_NR_CPUS;
|
|
|
|
}
|
2008-03-03 10:12:42 -07:00
|
|
|
|
|
|
|
printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
|
|
|
|
possible, max_t(int, possible - num_processors, 0));
|
|
|
|
|
|
|
|
for (i = 0; i < possible; i++)
|
2009-01-04 06:18:03 -07:00
|
|
|
set_cpu_possible(i, true);
|
2008-05-12 13:21:13 -06:00
|
|
|
|
|
|
|
nr_cpu_ids = possible;
|
2008-03-03 10:12:42 -07:00
|
|
|
}
|
2008-03-03 10:13:07 -07:00
|
|
|
|
2008-09-29 16:29:42 -06:00
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
|
|
|
|
static void remove_siblinginfo(int cpu)
|
|
|
|
{
|
|
|
|
int sibling;
|
|
|
|
struct cpuinfo_x86 *c = &cpu_data(cpu);
|
|
|
|
|
2009-01-04 06:18:03 -07:00
|
|
|
for_each_cpu(sibling, cpu_core_mask(cpu)) {
|
|
|
|
cpumask_clear_cpu(cpu, cpu_core_mask(sibling));
|
2008-09-29 16:29:42 -06:00
|
|
|
/*/
|
|
|
|
* last thread sibling in this cpu core going down
|
|
|
|
*/
|
2009-01-04 06:18:03 -07:00
|
|
|
if (cpumask_weight(cpu_sibling_mask(cpu)) == 1)
|
2008-09-29 16:29:42 -06:00
|
|
|
cpu_data(sibling).booted_cores--;
|
|
|
|
}
|
|
|
|
|
2009-01-04 06:18:03 -07:00
|
|
|
for_each_cpu(sibling, cpu_sibling_mask(cpu))
|
|
|
|
cpumask_clear_cpu(cpu, cpu_sibling_mask(sibling));
|
|
|
|
cpumask_clear(cpu_sibling_mask(cpu));
|
|
|
|
cpumask_clear(cpu_core_mask(cpu));
|
2008-09-29 16:29:42 -06:00
|
|
|
c->phys_proc_id = 0;
|
|
|
|
c->cpu_core_id = 0;
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
|
2008-09-29 16:29:42 -06:00
|
|
|
}
|
|
|
|
|
2008-03-03 10:13:07 -07:00
|
|
|
static void __ref remove_cpu_from_maps(int cpu)
|
|
|
|
{
|
2009-01-04 06:18:03 -07:00
|
|
|
set_cpu_online(cpu, false);
|
|
|
|
cpumask_clear_cpu(cpu, cpu_callout_mask);
|
|
|
|
cpumask_clear_cpu(cpu, cpu_callin_mask);
|
2008-03-03 10:13:07 -07:00
|
|
|
/* was set by cpu_init() */
|
2009-01-04 06:18:03 -07:00
|
|
|
cpumask_clear_cpu(cpu, cpu_initialized_mask);
|
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2008-05-12 13:21:12 -06:00
|
|
|
numa_remove_cpu(cpu);
|
2008-03-03 10:13:07 -07:00
|
|
|
}
|
|
|
|
|
2008-08-22 04:52:14 -06:00
|
|
|
void cpu_disable_common(void)
|
2008-03-03 10:13:07 -07:00
|
|
|
{
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
/*
|
|
|
|
* HACK:
|
|
|
|
* Allow any queued timer interrupts to get serviced
|
|
|
|
* This is only a temporary solution until we cleanup
|
|
|
|
* fixup_irqs as we do for IA64.
|
|
|
|
*/
|
|
|
|
local_irq_enable();
|
|
|
|
mdelay(1);
|
|
|
|
|
|
|
|
local_irq_disable();
|
|
|
|
remove_siblinginfo(cpu);
|
|
|
|
|
|
|
|
/* It's now safe to remove this processor from the online map */
|
2008-08-09 16:09:02 -06:00
|
|
|
lock_vector_lock();
|
2008-03-03 10:13:07 -07:00
|
|
|
remove_cpu_from_maps(cpu);
|
2008-08-09 16:09:02 -06:00
|
|
|
unlock_vector_lock();
|
2008-12-16 18:33:58 -07:00
|
|
|
fixup_irqs();
|
2008-08-22 04:52:14 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
int native_cpu_disable(void)
|
|
|
|
{
|
|
|
|
int cpu = smp_processor_id();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perhaps use cpufreq to drop frequency, but that could go
|
|
|
|
* into generic code.
|
|
|
|
*
|
|
|
|
* We won't take down the boot processor on i386 due to some
|
|
|
|
* interrupts only being able to be serviced by the BSP.
|
|
|
|
* Especially so if we're not using an IOAPIC -zwane
|
|
|
|
*/
|
|
|
|
if (cpu == 0)
|
|
|
|
return -EBUSY;
|
|
|
|
|
|
|
|
if (nmi_watchdog == NMI_LOCAL_APIC)
|
|
|
|
stop_apic_nmi_watchdog(NULL);
|
|
|
|
clear_local_APIC();
|
|
|
|
|
|
|
|
cpu_disable_common();
|
2008-03-03 10:13:07 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-08-22 04:52:11 -06:00
|
|
|
void native_cpu_die(unsigned int cpu)
|
2008-03-03 10:13:07 -07:00
|
|
|
{
|
|
|
|
/* We don't do anything here: idle task is faking death itself. */
|
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
for (i = 0; i < 10; i++) {
|
|
|
|
/* They ack this in play_dead by setting CPU_DEAD */
|
|
|
|
if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
|
|
|
|
printk(KERN_INFO "CPU %d is now offline\n", cpu);
|
|
|
|
if (1 == num_online_cpus())
|
|
|
|
alternatives_smp_switch(0);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
msleep(100);
|
|
|
|
}
|
|
|
|
printk(KERN_ERR "CPU %u didn't die...\n", cpu);
|
|
|
|
}
|
2008-08-22 04:52:13 -06:00
|
|
|
|
|
|
|
void play_dead_common(void)
|
|
|
|
{
|
|
|
|
idle_task_exit();
|
|
|
|
reset_lazy_tlbstate();
|
|
|
|
irq_ctx_exit(raw_smp_processor_id());
|
2008-09-23 15:26:42 -06:00
|
|
|
c1e_remove_cpu(raw_smp_processor_id());
|
2008-08-22 04:52:13 -06:00
|
|
|
|
|
|
|
mb();
|
|
|
|
/* Ack it */
|
|
|
|
__get_cpu_var(cpu_state) = CPU_DEAD;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* With physical CPU hotplug, we should halt the cpu
|
|
|
|
*/
|
|
|
|
local_irq_disable();
|
|
|
|
}
|
|
|
|
|
|
|
|
void native_play_dead(void)
|
|
|
|
{
|
|
|
|
play_dead_common();
|
2009-06-30 20:31:07 -06:00
|
|
|
tboot_shutdown(TB_SHUTDOWN_WFS);
|
2008-08-22 04:52:13 -06:00
|
|
|
wbinvd_halt();
|
|
|
|
}
|
|
|
|
|
2008-03-03 10:13:07 -07:00
|
|
|
#else /* ... !CONFIG_HOTPLUG_CPU */
|
2008-08-22 04:52:11 -06:00
|
|
|
int native_cpu_disable(void)
|
2008-03-03 10:13:07 -07:00
|
|
|
{
|
|
|
|
return -ENOSYS;
|
|
|
|
}
|
|
|
|
|
2008-08-22 04:52:11 -06:00
|
|
|
void native_cpu_die(unsigned int cpu)
|
2008-03-03 10:13:07 -07:00
|
|
|
{
|
|
|
|
/* We said "no" in __cpu_disable */
|
|
|
|
BUG();
|
|
|
|
}
|
2008-08-22 04:52:13 -06:00
|
|
|
|
|
|
|
void native_play_dead(void)
|
|
|
|
{
|
|
|
|
BUG();
|
|
|
|
}
|
|
|
|
|
2008-03-03 10:12:42 -07:00
|
|
|
#endif
|