![Yinghai Lu](/assets/img/avatar_default.png)
Tim found: WARNING: at arch/x86/kernel/smpboot.c:324 topology_sane.isra.2+0x6f/0x80() Hardware name: S2600CP sched: CPU #1's llc-sibling CPU #0 is not on the same node! [node: 1 != 0]. Ignoring dependency. smpboot: Booting Node 1, Processors #1 Modules linked in: Pid: 0, comm: swapper/1 Not tainted 3.9.0-0-generic #1 Call Trace: set_cpu_sibling_map+0x279/0x449 start_secondary+0x11d/0x1e5 Don Morris reproduced on a HP z620 workstation, and bisected it to commite8d1955258
("acpi, memory-hotplug: parse SRAT before memblock is ready") It turns out movable_map has some problems, and it breaks several things 1. numa_init is called several times, NOT just for srat. so those nodes_clear(numa_nodes_parsed) memset(&numa_meminfo, 0, sizeof(numa_meminfo)) can not be just removed. Need to consider sequence is: numaq, srat, amd, dummy. and make fall back path working. 2. simply split acpi_numa_init to early_parse_srat. a. that early_parse_srat is NOT called for ia64, so you break ia64. b. for (i = 0; i < MAX_LOCAL_APIC; i++) set_apicid_to_node(i, NUMA_NO_NODE) still left in numa_init. So it will just clear result from early_parse_srat. it should be moved before that.... c. it breaks ACPI_TABLE_OVERIDE...as the acpi table scan is moved early before override from INITRD is settled. 3. that patch TITLE is total misleading, there is NO x86 in the title, but it changes critical x86 code. It caused x86 guys did not pay attention to find the problem early. Those patches really should be routed via tip/x86/mm. 4. after that commit, following range can not use movable ram: a. real_mode code.... well..funny, legacy Node0 [0,1M) could be hot-removed? b. initrd... it will be freed after booting, so it could be on movable... c. crashkernel for kdump...: looks like we can not put kdump kernel above 4G anymore. d. init_mem_mapping: can not put page table high anymore. e. initmem_init: vmemmap can not be high local node anymore. That is not good. If node is hotplugable, the mem related range like page table and vmemmap could be on the that node without problem and should be on that node. We have workaround patch that could fix some problems, but some can not be fixed. So just remove that offending commit and related ones including:f7210e6c4a
("mm/memblock.c: use CONFIG_HAVE_MEMBLOCK_NODE_MAP to protect movablecore_map in memblock_overlaps_region().")01a178a94e
("acpi, memory-hotplug: support getting hotplug info from SRAT")27168d38fa
("acpi, memory-hotplug: extend movablemem_map ranges to the end of node")e8d1955258
("acpi, memory-hotplug: parse SRAT before memblock is ready")fb06bc8e5f
("page_alloc: bootmem limit with movablecore_map")42f47e27e7
("page_alloc: make movablemem_map have higher priority")6981ec3114
("page_alloc: introduce zone_movable_limit[] to keep movable limit for nodes")34b71f1e04
("page_alloc: add movable_memmap kernel parameter")4d59a75125
("x86: get pg_data_t's memory from other node") Later we should have patches that will make sure kernel put page table and vmemmap on local node ram instead of push them down to node0. Also need to find way to put other kernel used ram to local node ram. Reported-by: Tim Gardner <tim.gardner@canonical.com> Reported-by: Don Morris <don.morris@hp.com> Bisected-by: Don Morris <don.morris@hp.com> Tested-by: Don Morris <don.morris@hp.com> Signed-off-by: Yinghai Lu <yinghai@kernel.org> Cc: Tony Luck <tony.luck@intel.com> Cc: Thomas Renninger <trenn@suse.de> Cc: Tejun Heo <tj@kernel.org> Cc: Tang Chen <tangchen@cn.fujitsu.com> Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
199 lines
4.7 KiB
C
199 lines
4.7 KiB
C
/*
|
|
* ACPI 3.0 based NUMA setup
|
|
* Copyright 2004 Andi Kleen, SuSE Labs.
|
|
*
|
|
* Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
|
|
*
|
|
* Called from acpi_numa_init while reading the SRAT and SLIT tables.
|
|
* Assumes all memory regions belonging to a single proximity domain
|
|
* are in one chunk. Holes between them will be included in the node.
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/acpi.h>
|
|
#include <linux/mmzone.h>
|
|
#include <linux/bitmap.h>
|
|
#include <linux/module.h>
|
|
#include <linux/topology.h>
|
|
#include <linux/bootmem.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/mm.h>
|
|
#include <asm/proto.h>
|
|
#include <asm/numa.h>
|
|
#include <asm/e820.h>
|
|
#include <asm/apic.h>
|
|
#include <asm/uv/uv.h>
|
|
|
|
int acpi_numa __initdata;
|
|
|
|
static __init int setup_node(int pxm)
|
|
{
|
|
return acpi_map_pxm_to_node(pxm);
|
|
}
|
|
|
|
static __init void bad_srat(void)
|
|
{
|
|
printk(KERN_ERR "SRAT: SRAT not used.\n");
|
|
acpi_numa = -1;
|
|
}
|
|
|
|
static __init inline int srat_disabled(void)
|
|
{
|
|
return acpi_numa < 0;
|
|
}
|
|
|
|
/* Callback for SLIT parsing */
|
|
void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
|
|
{
|
|
int i, j;
|
|
|
|
for (i = 0; i < slit->locality_count; i++)
|
|
for (j = 0; j < slit->locality_count; j++)
|
|
numa_set_distance(pxm_to_node(i), pxm_to_node(j),
|
|
slit->entry[slit->locality_count * i + j]);
|
|
}
|
|
|
|
/* Callback for Proximity Domain -> x2APIC mapping */
|
|
void __init
|
|
acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
|
|
{
|
|
int pxm, node;
|
|
int apic_id;
|
|
|
|
if (srat_disabled())
|
|
return;
|
|
if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
|
|
bad_srat();
|
|
return;
|
|
}
|
|
if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
|
|
return;
|
|
pxm = pa->proximity_domain;
|
|
apic_id = pa->apic_id;
|
|
if (!apic->apic_id_valid(apic_id)) {
|
|
printk(KERN_INFO "SRAT: PXM %u -> X2APIC 0x%04x ignored\n",
|
|
pxm, apic_id);
|
|
return;
|
|
}
|
|
node = setup_node(pxm);
|
|
if (node < 0) {
|
|
printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
|
|
bad_srat();
|
|
return;
|
|
}
|
|
|
|
if (apic_id >= MAX_LOCAL_APIC) {
|
|
printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
|
|
return;
|
|
}
|
|
set_apicid_to_node(apic_id, node);
|
|
node_set(node, numa_nodes_parsed);
|
|
acpi_numa = 1;
|
|
printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
|
|
pxm, apic_id, node);
|
|
}
|
|
|
|
/* Callback for Proximity Domain -> LAPIC mapping */
|
|
void __init
|
|
acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
|
|
{
|
|
int pxm, node;
|
|
int apic_id;
|
|
|
|
if (srat_disabled())
|
|
return;
|
|
if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
|
|
bad_srat();
|
|
return;
|
|
}
|
|
if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
|
|
return;
|
|
pxm = pa->proximity_domain_lo;
|
|
if (acpi_srat_revision >= 2)
|
|
pxm |= *((unsigned int*)pa->proximity_domain_hi) << 8;
|
|
node = setup_node(pxm);
|
|
if (node < 0) {
|
|
printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
|
|
bad_srat();
|
|
return;
|
|
}
|
|
|
|
if (get_uv_system_type() >= UV_X2APIC)
|
|
apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
|
|
else
|
|
apic_id = pa->apic_id;
|
|
|
|
if (apic_id >= MAX_LOCAL_APIC) {
|
|
printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
|
|
return;
|
|
}
|
|
|
|
set_apicid_to_node(apic_id, node);
|
|
node_set(node, numa_nodes_parsed);
|
|
acpi_numa = 1;
|
|
printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
|
|
pxm, apic_id, node);
|
|
}
|
|
|
|
#ifdef CONFIG_MEMORY_HOTPLUG
|
|
static inline int save_add_info(void) {return 1;}
|
|
#else
|
|
static inline int save_add_info(void) {return 0;}
|
|
#endif
|
|
|
|
/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
|
|
int __init
|
|
acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
|
|
{
|
|
u64 start, end;
|
|
int node, pxm;
|
|
|
|
if (srat_disabled())
|
|
goto out_err;
|
|
if (ma->header.length != sizeof(struct acpi_srat_mem_affinity))
|
|
goto out_err_bad_srat;
|
|
if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
|
|
goto out_err;
|
|
if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
|
|
goto out_err;
|
|
|
|
start = ma->base_address;
|
|
end = start + ma->length;
|
|
pxm = ma->proximity_domain;
|
|
if (acpi_srat_revision <= 1)
|
|
pxm &= 0xff;
|
|
|
|
node = setup_node(pxm);
|
|
if (node < 0) {
|
|
printk(KERN_ERR "SRAT: Too many proximity domains.\n");
|
|
goto out_err_bad_srat;
|
|
}
|
|
|
|
if (numa_add_memblk(node, start, end) < 0)
|
|
goto out_err_bad_srat;
|
|
|
|
node_set(node, numa_nodes_parsed);
|
|
|
|
printk(KERN_INFO "SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]\n",
|
|
node, pxm,
|
|
(unsigned long long) start, (unsigned long long) end - 1);
|
|
|
|
return 0;
|
|
out_err_bad_srat:
|
|
bad_srat();
|
|
out_err:
|
|
return -1;
|
|
}
|
|
|
|
void __init acpi_numa_arch_fixup(void) {}
|
|
|
|
int __init x86_acpi_numa_init(void)
|
|
{
|
|
int ret;
|
|
|
|
ret = acpi_numa_init();
|
|
if (ret < 0)
|
|
return ret;
|
|
return srat_disabled() ? -EINVAL : 0;
|
|
}
|