2008-01-30 05:30:37 -07:00
|
|
|
/*
|
2005-04-16 16:20:36 -06:00
|
|
|
* Generic VM initialization for x86-64 NUMA setups.
|
|
|
|
* Copyright 2002,2003 Andi Kleen, SuSE Labs.
|
2008-01-30 05:30:37 -07:00
|
|
|
*/
|
2005-04-16 16:20:36 -06:00
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/bootmem.h>
|
|
|
|
#include <linux/mmzone.h>
|
|
|
|
#include <linux/ctype.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/nodemask.h>
|
2008-01-30 05:33:11 -07:00
|
|
|
#include <linux/sched.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
#include <asm/e820.h>
|
|
|
|
#include <asm/proto.h>
|
|
|
|
#include <asm/dma.h>
|
|
|
|
#include <asm/numa.h>
|
|
|
|
#include <asm/acpi.h>
|
2008-01-30 05:30:16 -07:00
|
|
|
#include <asm/k8.h>
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
#ifndef Dprintk
|
|
|
|
#define Dprintk(x...)
|
|
|
|
#endif
|
|
|
|
|
2005-09-06 16:17:45 -06:00
|
|
|
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
|
2008-01-30 05:30:37 -07:00
|
|
|
EXPORT_SYMBOL(node_data);
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
bootmem_data_t plat_node_bdata[MAX_NUMNODES];
|
|
|
|
|
2006-03-25 08:31:46 -07:00
|
|
|
struct memnode memnode;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-03-25 16:06:51 -06:00
|
|
|
#ifdef CONFIG_SMP
|
2008-01-30 05:33:25 -07:00
|
|
|
int x86_cpu_to_node_map_init[NR_CPUS] = {
|
2005-09-12 10:49:24 -06:00
|
|
|
[0 ... NR_CPUS-1] = NUMA_NO_NODE
|
2005-09-12 10:49:24 -06:00
|
|
|
};
|
2008-01-30 05:33:11 -07:00
|
|
|
void *x86_cpu_to_node_map_early_ptr;
|
2008-03-25 16:06:51 -06:00
|
|
|
EXPORT_SYMBOL(x86_cpu_to_node_map_early_ptr);
|
|
|
|
#endif
|
2008-01-30 05:33:25 -07:00
|
|
|
DEFINE_PER_CPU(int, x86_cpu_to_node_map) = NUMA_NO_NODE;
|
2008-01-30 05:33:11 -07:00
|
|
|
EXPORT_PER_CPU_SYMBOL(x86_cpu_to_node_map);
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2008-01-30 05:33:25 -07:00
|
|
|
s16 apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
|
2008-01-30 05:30:37 -07:00
|
|
|
[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
|
2005-09-12 10:49:24 -06:00
|
|
|
};
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2008-01-30 05:30:38 -07:00
|
|
|
cpumask_t node_to_cpumask_map[MAX_NUMNODES] __read_mostly;
|
|
|
|
EXPORT_SYMBOL(node_to_cpumask_map);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
int numa_off __initdata;
|
2007-02-13 05:26:19 -07:00
|
|
|
unsigned long __initdata nodemap_addr;
|
|
|
|
unsigned long __initdata nodemap_size;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2005-11-05 09:25:54 -07:00
|
|
|
/*
|
|
|
|
* Given a shift value, try to populate memnodemap[]
|
|
|
|
* Returns :
|
|
|
|
* 1 if OK
|
|
|
|
* 0 if memnodmap[] too small (of shift too small)
|
|
|
|
* -1 if node overlap or lost ram (shift too big)
|
|
|
|
*/
|
2008-01-30 05:30:37 -07:00
|
|
|
static int __init populate_memnodemap(const struct bootnode *nodes,
|
2008-03-25 11:14:35 -06:00
|
|
|
int numnodes, int shift, int *nodeids)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2005-11-05 09:25:54 -07:00
|
|
|
unsigned long addr, end;
|
2008-01-30 05:30:37 -07:00
|
|
|
int i, res = -1;
|
2005-07-28 22:15:38 -06:00
|
|
|
|
2008-01-30 05:33:25 -07:00
|
|
|
memset(memnodemap, 0xff, sizeof(s16)*memnodemapsize);
|
2005-07-28 22:15:38 -06:00
|
|
|
for (i = 0; i < numnodes; i++) {
|
2005-11-05 09:25:54 -07:00
|
|
|
addr = nodes[i].start;
|
|
|
|
end = nodes[i].end;
|
|
|
|
if (addr >= end)
|
2005-07-28 22:15:38 -06:00
|
|
|
continue;
|
2007-02-13 05:26:19 -07:00
|
|
|
if ((end >> shift) >= memnodemapsize)
|
2005-11-05 09:25:54 -07:00
|
|
|
return 0;
|
|
|
|
do {
|
2008-01-30 05:33:25 -07:00
|
|
|
if (memnodemap[addr >> shift] != NUMA_NO_NODE)
|
2005-07-28 22:15:38 -06:00
|
|
|
return -1;
|
2008-03-25 11:14:35 -06:00
|
|
|
|
|
|
|
if (!nodeids)
|
|
|
|
memnodemap[addr >> shift] = i;
|
|
|
|
else
|
|
|
|
memnodemap[addr >> shift] = nodeids[i];
|
|
|
|
|
2007-02-13 05:26:19 -07:00
|
|
|
addr += (1UL << shift);
|
2005-11-05 09:25:54 -07:00
|
|
|
} while (addr < end);
|
|
|
|
res = 1;
|
2008-01-30 05:30:37 -07:00
|
|
|
}
|
2005-11-05 09:25:54 -07:00
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2007-02-13 05:26:19 -07:00
|
|
|
static int __init allocate_cachealigned_memnodemap(void)
|
|
|
|
{
|
2008-02-01 09:49:41 -07:00
|
|
|
unsigned long addr;
|
2007-02-13 05:26:19 -07:00
|
|
|
|
|
|
|
memnodemap = memnode.embedded_map;
|
2008-01-30 05:33:15 -07:00
|
|
|
if (memnodemapsize <= ARRAY_SIZE(memnode.embedded_map))
|
2007-02-13 05:26:19 -07:00
|
|
|
return 0;
|
|
|
|
|
2008-02-01 09:49:41 -07:00
|
|
|
addr = 0x8000;
|
|
|
|
nodemap_size = round_up(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES);
|
|
|
|
nodemap_addr = find_e820_area(addr, end_pfn<<PAGE_SHIFT,
|
|
|
|
nodemap_size, L1_CACHE_BYTES);
|
2007-02-13 05:26:19 -07:00
|
|
|
if (nodemap_addr == -1UL) {
|
|
|
|
printk(KERN_ERR
|
|
|
|
"NUMA: Unable to allocate Memory to Node hash map\n");
|
|
|
|
nodemap_addr = nodemap_size = 0;
|
|
|
|
return -1;
|
|
|
|
}
|
2008-02-01 09:49:41 -07:00
|
|
|
memnodemap = phys_to_virt(nodemap_addr);
|
2008-02-01 09:49:41 -07:00
|
|
|
reserve_early(nodemap_addr, nodemap_addr + nodemap_size, "MEMNODEMAP");
|
2007-02-13 05:26:19 -07:00
|
|
|
|
|
|
|
printk(KERN_DEBUG "NUMA: Allocated memnodemap from %lx - %lx\n",
|
|
|
|
nodemap_addr, nodemap_addr + nodemap_size);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The LSB of all start and end addresses in the node map is the value of the
|
|
|
|
* maximum possible shift.
|
|
|
|
*/
|
2008-01-30 05:30:37 -07:00
|
|
|
static int __init extract_lsb_from_nodes(const struct bootnode *nodes,
|
|
|
|
int numnodes)
|
2005-11-05 09:25:54 -07:00
|
|
|
{
|
2007-02-13 05:26:20 -07:00
|
|
|
int i, nodes_used = 0;
|
2007-02-13 05:26:19 -07:00
|
|
|
unsigned long start, end;
|
|
|
|
unsigned long bitfield = 0, memtop = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < numnodes; i++) {
|
|
|
|
start = nodes[i].start;
|
|
|
|
end = nodes[i].end;
|
|
|
|
if (start >= end)
|
|
|
|
continue;
|
2007-02-13 05:26:20 -07:00
|
|
|
bitfield |= start;
|
|
|
|
nodes_used++;
|
2007-02-13 05:26:19 -07:00
|
|
|
if (end > memtop)
|
|
|
|
memtop = end;
|
|
|
|
}
|
2007-02-13 05:26:20 -07:00
|
|
|
if (nodes_used <= 1)
|
|
|
|
i = 63;
|
|
|
|
else
|
|
|
|
i = find_first_bit(&bitfield, sizeof(unsigned long)*8);
|
2007-02-13 05:26:19 -07:00
|
|
|
memnodemapsize = (memtop >> i)+1;
|
|
|
|
return i;
|
|
|
|
}
|
2005-11-05 09:25:54 -07:00
|
|
|
|
2008-03-25 11:14:35 -06:00
|
|
|
int __init compute_hash_shift(struct bootnode *nodes, int numnodes,
|
|
|
|
int *nodeids)
|
2007-02-13 05:26:19 -07:00
|
|
|
{
|
|
|
|
int shift;
|
2005-11-05 09:25:54 -07:00
|
|
|
|
2007-02-13 05:26:19 -07:00
|
|
|
shift = extract_lsb_from_nodes(nodes, numnodes);
|
|
|
|
if (allocate_cachealigned_memnodemap())
|
|
|
|
return -1;
|
2006-01-11 14:44:33 -07:00
|
|
|
printk(KERN_DEBUG "NUMA: Using %d for the hash shift.\n",
|
2005-11-05 09:25:54 -07:00
|
|
|
shift);
|
|
|
|
|
2008-03-25 11:14:35 -06:00
|
|
|
if (populate_memnodemap(nodes, numnodes, shift, nodeids) != 1) {
|
2008-01-30 05:30:37 -07:00
|
|
|
printk(KERN_INFO "Your memory is not aligned you need to "
|
|
|
|
"rebuild your kernel with a bigger NODEMAPSIZE "
|
|
|
|
"shift=%d\n", shift);
|
2005-11-05 09:25:54 -07:00
|
|
|
return -1;
|
|
|
|
}
|
2005-07-28 22:15:38 -06:00
|
|
|
return shift;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
|
|
|
|
2005-06-23 01:08:07 -06:00
|
|
|
int early_pfn_to_nid(unsigned long pfn)
|
|
|
|
{
|
|
|
|
return phys_to_nid(pfn << PAGE_SHIFT);
|
|
|
|
}
|
|
|
|
|
2008-01-30 05:30:37 -07:00
|
|
|
static void * __init early_node_mem(int nodeid, unsigned long start,
|
2008-02-01 09:49:41 -07:00
|
|
|
unsigned long end, unsigned long size,
|
|
|
|
unsigned long align)
|
2006-04-07 11:49:21 -06:00
|
|
|
{
|
2008-02-01 09:49:41 -07:00
|
|
|
unsigned long mem = find_e820_area(start, end, size, align);
|
2006-04-07 11:49:21 -06:00
|
|
|
void *ptr;
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2008-02-01 09:49:42 -07:00
|
|
|
if (mem != -1L)
|
2006-04-07 11:49:21 -06:00
|
|
|
return __va(mem);
|
2008-02-01 09:49:42 -07:00
|
|
|
|
2008-02-01 09:49:41 -07:00
|
|
|
ptr = __alloc_bootmem_nopanic(size, align, __pa(MAX_DMA_ADDRESS));
|
2007-10-17 10:04:35 -06:00
|
|
|
if (ptr == NULL) {
|
2006-04-07 11:49:21 -06:00
|
|
|
printk(KERN_ERR "Cannot find %lu bytes in node %d\n",
|
2008-01-30 05:30:37 -07:00
|
|
|
size, nodeid);
|
2006-04-07 11:49:21 -06:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
return ptr;
|
|
|
|
}
|
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
/* Initialize bootmem allocator for a node */
|
2008-01-30 05:30:37 -07:00
|
|
|
void __init setup_node_bootmem(int nodeid, unsigned long start,
|
|
|
|
unsigned long end)
|
|
|
|
{
|
|
|
|
unsigned long start_pfn, end_pfn, bootmap_pages, bootmap_size;
|
|
|
|
unsigned long bootmap_start, nodedata_phys;
|
2006-04-07 11:49:21 -06:00
|
|
|
void *bootmap;
|
2005-04-16 16:20:36 -06:00
|
|
|
const int pgdat_size = round_up(sizeof(pg_data_t), PAGE_SIZE);
|
2008-03-18 13:52:37 -06:00
|
|
|
int nid;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-01-30 05:30:37 -07:00
|
|
|
start = round_up(start, ZONE_ALIGN);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-01-30 05:30:37 -07:00
|
|
|
printk(KERN_INFO "Bootmem setup node %d %016lx-%016lx\n", nodeid,
|
|
|
|
start, end);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
start_pfn = start >> PAGE_SHIFT;
|
|
|
|
end_pfn = end >> PAGE_SHIFT;
|
|
|
|
|
2008-02-01 09:49:41 -07:00
|
|
|
node_data[nodeid] = early_node_mem(nodeid, start, end, pgdat_size,
|
|
|
|
SMP_CACHE_BYTES);
|
2006-04-07 11:49:21 -06:00
|
|
|
if (node_data[nodeid] == NULL)
|
|
|
|
return;
|
|
|
|
nodedata_phys = __pa(node_data[nodeid]);
|
2008-02-04 08:47:56 -07:00
|
|
|
printk(KERN_INFO " NODE_DATA [%016lx - %016lx]\n", nodedata_phys,
|
|
|
|
nodedata_phys + pgdat_size - 1);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
memset(NODE_DATA(nodeid), 0, sizeof(pg_data_t));
|
|
|
|
NODE_DATA(nodeid)->bdata = &plat_node_bdata[nodeid];
|
|
|
|
NODE_DATA(nodeid)->node_start_pfn = start_pfn;
|
|
|
|
NODE_DATA(nodeid)->node_spanned_pages = end_pfn - start_pfn;
|
|
|
|
|
2008-03-18 13:52:37 -06:00
|
|
|
/*
|
|
|
|
* Find a place for the bootmem map
|
|
|
|
* nodedata_phys could be on other nodes by alloc_bootmem,
|
|
|
|
* so need to sure bootmap_start not to be small, otherwise
|
|
|
|
* early_node_mem will get that with find_e820_area instead
|
|
|
|
* of alloc_bootmem, that could clash with reserved range
|
|
|
|
*/
|
2008-01-30 05:30:37 -07:00
|
|
|
bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn);
|
2008-03-18 13:52:37 -06:00
|
|
|
nid = phys_to_nid(nodedata_phys);
|
|
|
|
if (nid == nodeid)
|
|
|
|
bootmap_start = round_up(nodedata_phys + pgdat_size, PAGE_SIZE);
|
|
|
|
else
|
|
|
|
bootmap_start = round_up(start, PAGE_SIZE);
|
2008-02-01 09:49:41 -07:00
|
|
|
/*
|
2008-05-14 09:15:10 -06:00
|
|
|
* SMP_CACHE_BYTES could be enough, but init_bootmem_node like
|
2008-02-01 09:49:41 -07:00
|
|
|
* to use that to align to PAGE_SIZE
|
|
|
|
*/
|
2006-04-07 11:49:21 -06:00
|
|
|
bootmap = early_node_mem(nodeid, bootmap_start, end,
|
2008-02-01 09:49:41 -07:00
|
|
|
bootmap_pages<<PAGE_SHIFT, PAGE_SIZE);
|
2006-04-07 11:49:21 -06:00
|
|
|
if (bootmap == NULL) {
|
|
|
|
if (nodedata_phys < start || nodedata_phys >= end)
|
2008-03-18 13:40:04 -06:00
|
|
|
free_bootmem(nodedata_phys, pgdat_size);
|
2006-04-07 11:49:21 -06:00
|
|
|
node_data[nodeid] = NULL;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
bootmap_start = __pa(bootmap);
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
bootmap_size = init_bootmem_node(NODE_DATA(nodeid),
|
2008-01-30 05:30:37 -07:00
|
|
|
bootmap_start >> PAGE_SHIFT,
|
|
|
|
start_pfn, end_pfn);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-02-04 08:47:56 -07:00
|
|
|
printk(KERN_INFO " bootmap [%016lx - %016lx] pages %lx\n",
|
|
|
|
bootmap_start, bootmap_start + bootmap_size - 1,
|
|
|
|
bootmap_pages);
|
|
|
|
|
2006-09-27 02:49:52 -06:00
|
|
|
free_bootmem_with_active_regions(nodeid, end);
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-03-18 13:52:37 -06:00
|
|
|
/*
|
|
|
|
* convert early reserve to bootmem reserve earlier
|
|
|
|
* otherwise early_node_mem could use early reserved mem
|
|
|
|
* on previous node
|
|
|
|
*/
|
|
|
|
early_res_to_bootmem(start, end);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* in some case early_node_mem could use alloc_bootmem
|
|
|
|
* to get range on other node, don't reserve that again
|
|
|
|
*/
|
|
|
|
if (nid != nodeid)
|
|
|
|
printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nodeid, nid);
|
|
|
|
else
|
|
|
|
reserve_bootmem_node(NODE_DATA(nodeid), nodedata_phys,
|
|
|
|
pgdat_size, BOOTMEM_DEFAULT);
|
|
|
|
nid = phys_to_nid(bootmap_start);
|
|
|
|
if (nid != nodeid)
|
|
|
|
printk(KERN_INFO " bootmap(%d) on node %d\n", nodeid, nid);
|
|
|
|
else
|
|
|
|
reserve_bootmem_node(NODE_DATA(nodeid), bootmap_start,
|
|
|
|
bootmap_pages<<PAGE_SHIFT, BOOTMEM_DEFAULT);
|
|
|
|
|
2006-04-07 11:49:18 -06:00
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
|
|
|
srat_reserve_add_area(nodeid);
|
|
|
|
#endif
|
2005-04-16 16:20:36 -06:00
|
|
|
node_set_online(nodeid);
|
2008-01-30 05:30:37 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2008-01-30 05:30:37 -07:00
|
|
|
/*
|
|
|
|
* There are unfortunately some poorly designed mainboards around that
|
|
|
|
* only connect memory to a single CPU. This breaks the 1:1 cpu->node
|
|
|
|
* mapping. To avoid this fill in the mapping for all possible CPUs,
|
|
|
|
* as the number of CPUs is not known yet. We round robin the existing
|
|
|
|
* nodes.
|
|
|
|
*/
|
2005-04-16 16:20:36 -06:00
|
|
|
void __init numa_init_array(void)
|
|
|
|
{
|
|
|
|
int rr, i;
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2005-09-30 12:59:22 -06:00
|
|
|
rr = first_node(node_online_map);
|
2005-04-16 16:20:36 -06:00
|
|
|
for (i = 0; i < NR_CPUS; i++) {
|
2008-01-30 05:33:33 -07:00
|
|
|
if (early_cpu_to_node(i) != NUMA_NO_NODE)
|
2005-04-16 16:20:36 -06:00
|
|
|
continue;
|
2008-01-30 05:30:37 -07:00
|
|
|
numa_set_node(i, rr);
|
2005-04-16 16:20:36 -06:00
|
|
|
rr = next_node(rr, node_online_map);
|
|
|
|
if (rr == MAX_NUMNODES)
|
|
|
|
rr = first_node(node_online_map);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_NUMA_EMU
|
2007-02-13 05:26:22 -07:00
|
|
|
/* Numa emulation */
|
2007-05-02 11:27:09 -06:00
|
|
|
char *cmdline __initdata;
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2007-02-13 05:26:22 -07:00
|
|
|
/*
|
2008-01-30 05:30:37 -07:00
|
|
|
* Setups up nid to range from addr to addr + size. If the end
|
|
|
|
* boundary is greater than max_addr, then max_addr is used instead.
|
|
|
|
* The return value is 0 if there is additional memory left for
|
|
|
|
* allocation past addr and -1 otherwise. addr is adjusted to be at
|
|
|
|
* the end of the node.
|
2007-02-13 05:26:22 -07:00
|
|
|
*/
|
2007-05-02 11:27:09 -06:00
|
|
|
static int __init setup_node_range(int nid, struct bootnode *nodes, u64 *addr,
|
|
|
|
u64 size, u64 max_addr)
|
2007-02-13 05:26:22 -07:00
|
|
|
{
|
2007-05-02 11:27:09 -06:00
|
|
|
int ret = 0;
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2007-05-02 11:27:09 -06:00
|
|
|
nodes[nid].start = *addr;
|
|
|
|
*addr += size;
|
|
|
|
if (*addr >= max_addr) {
|
|
|
|
*addr = max_addr;
|
|
|
|
ret = -1;
|
|
|
|
}
|
|
|
|
nodes[nid].end = *addr;
|
2007-05-02 11:27:20 -06:00
|
|
|
node_set(nid, node_possible_map);
|
2007-05-02 11:27:09 -06:00
|
|
|
printk(KERN_INFO "Faking node %d at %016Lx-%016Lx (%LuMB)\n", nid,
|
|
|
|
nodes[nid].start, nodes[nid].end,
|
|
|
|
(nodes[nid].end - nodes[nid].start) >> 20);
|
|
|
|
return ret;
|
2007-02-13 05:26:22 -07:00
|
|
|
}
|
|
|
|
|
2007-05-02 11:27:09 -06:00
|
|
|
/*
|
|
|
|
* Splits num_nodes nodes up equally starting at node_start. The return value
|
|
|
|
* is the number of nodes split up and addr is adjusted to be at the end of the
|
|
|
|
* last node allocated.
|
|
|
|
*/
|
|
|
|
static int __init split_nodes_equally(struct bootnode *nodes, u64 *addr,
|
|
|
|
u64 max_addr, int node_start,
|
|
|
|
int num_nodes)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2007-05-02 11:27:09 -06:00
|
|
|
unsigned int big;
|
|
|
|
u64 size;
|
|
|
|
int i;
|
2007-02-13 05:26:22 -07:00
|
|
|
|
2007-05-02 11:27:09 -06:00
|
|
|
if (num_nodes <= 0)
|
|
|
|
return -1;
|
|
|
|
if (num_nodes > MAX_NUMNODES)
|
|
|
|
num_nodes = MAX_NUMNODES;
|
2007-07-21 09:11:29 -06:00
|
|
|
size = (max_addr - *addr - e820_hole_size(*addr, max_addr)) /
|
2007-05-02 11:27:09 -06:00
|
|
|
num_nodes;
|
2007-02-13 05:26:22 -07:00
|
|
|
/*
|
2007-05-02 11:27:09 -06:00
|
|
|
* Calculate the number of big nodes that can be allocated as a result
|
|
|
|
* of consolidating the leftovers.
|
2007-02-13 05:26:22 -07:00
|
|
|
*/
|
2007-05-02 11:27:09 -06:00
|
|
|
big = ((size & ~FAKE_NODE_MIN_HASH_MASK) * num_nodes) /
|
|
|
|
FAKE_NODE_MIN_SIZE;
|
|
|
|
|
|
|
|
/* Round down to nearest FAKE_NODE_MIN_SIZE. */
|
|
|
|
size &= FAKE_NODE_MIN_HASH_MASK;
|
|
|
|
if (!size) {
|
|
|
|
printk(KERN_ERR "Not enough memory for each node. "
|
|
|
|
"NUMA emulation disabled.\n");
|
|
|
|
return -1;
|
2007-02-13 05:26:22 -07:00
|
|
|
}
|
2007-05-02 11:27:09 -06:00
|
|
|
|
|
|
|
for (i = node_start; i < num_nodes + node_start; i++) {
|
|
|
|
u64 end = *addr + size;
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2007-02-13 05:26:22 -07:00
|
|
|
if (i < big)
|
|
|
|
end += FAKE_NODE_MIN_SIZE;
|
|
|
|
/*
|
2007-05-02 11:27:09 -06:00
|
|
|
* The final node can have the remaining system RAM. Other
|
|
|
|
* nodes receive roughly the same amount of available pages.
|
2007-02-13 05:26:22 -07:00
|
|
|
*/
|
2007-05-02 11:27:09 -06:00
|
|
|
if (i == num_nodes + node_start - 1)
|
|
|
|
end = max_addr;
|
|
|
|
else
|
2007-07-21 09:11:29 -06:00
|
|
|
while (end - *addr - e820_hole_size(*addr, end) <
|
2007-05-02 11:27:09 -06:00
|
|
|
size) {
|
|
|
|
end += FAKE_NODE_MIN_SIZE;
|
|
|
|
if (end > max_addr) {
|
|
|
|
end = max_addr;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (setup_node_range(i, nodes, addr, end - *addr, max_addr) < 0)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return i - node_start + 1;
|
|
|
|
}
|
|
|
|
|
2007-05-02 11:27:09 -06:00
|
|
|
/*
|
|
|
|
* Splits the remaining system RAM into chunks of size. The remaining memory is
|
|
|
|
* always assigned to a final node and can be asymmetric. Returns the number of
|
|
|
|
* nodes split.
|
|
|
|
*/
|
|
|
|
static int __init split_nodes_by_size(struct bootnode *nodes, u64 *addr,
|
|
|
|
u64 max_addr, int node_start, u64 size)
|
|
|
|
{
|
|
|
|
int i = node_start;
|
|
|
|
size = (size << 20) & FAKE_NODE_MIN_HASH_MASK;
|
|
|
|
while (!setup_node_range(i++, nodes, addr, size, max_addr))
|
|
|
|
;
|
|
|
|
return i - node_start;
|
|
|
|
}
|
|
|
|
|
2007-05-02 11:27:09 -06:00
|
|
|
/*
|
|
|
|
* Sets up the system RAM area from start_pfn to end_pfn according to the
|
|
|
|
* numa=fake command-line option.
|
|
|
|
*/
|
2008-04-04 19:11:09 -06:00
|
|
|
static struct bootnode nodes[MAX_NUMNODES] __initdata;
|
|
|
|
|
2007-05-02 11:27:09 -06:00
|
|
|
static int __init numa_emulation(unsigned long start_pfn, unsigned long end_pfn)
|
|
|
|
{
|
2008-01-30 05:30:37 -07:00
|
|
|
u64 size, addr = start_pfn << PAGE_SHIFT;
|
2007-05-02 11:27:09 -06:00
|
|
|
u64 max_addr = end_pfn << PAGE_SHIFT;
|
2008-01-30 05:30:37 -07:00
|
|
|
int num_nodes = 0, num = 0, coeff_flag, coeff = -1, i;
|
2007-05-02 11:27:09 -06:00
|
|
|
|
|
|
|
memset(&nodes, 0, sizeof(nodes));
|
|
|
|
/*
|
|
|
|
* If the numa=fake command-line is just a single number N, split the
|
|
|
|
* system RAM into N fake nodes.
|
|
|
|
*/
|
|
|
|
if (!strchr(cmdline, '*') && !strchr(cmdline, ',')) {
|
2008-01-30 05:30:37 -07:00
|
|
|
long n = simple_strtol(cmdline, NULL, 0);
|
|
|
|
|
|
|
|
num_nodes = split_nodes_equally(nodes, &addr, max_addr, 0, n);
|
2007-05-02 11:27:09 -06:00
|
|
|
if (num_nodes < 0)
|
|
|
|
return num_nodes;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Parse the command line. */
|
2007-05-02 11:27:09 -06:00
|
|
|
for (coeff_flag = 0; ; cmdline++) {
|
2007-05-02 11:27:09 -06:00
|
|
|
if (*cmdline && isdigit(*cmdline)) {
|
|
|
|
num = num * 10 + *cmdline - '0';
|
|
|
|
continue;
|
2007-02-13 05:26:22 -07:00
|
|
|
}
|
2007-05-02 11:27:09 -06:00
|
|
|
if (*cmdline == '*') {
|
|
|
|
if (num > 0)
|
|
|
|
coeff = num;
|
|
|
|
coeff_flag = 1;
|
|
|
|
}
|
2007-05-02 11:27:09 -06:00
|
|
|
if (!*cmdline || *cmdline == ',') {
|
2007-05-02 11:27:09 -06:00
|
|
|
if (!coeff_flag)
|
|
|
|
coeff = 1;
|
2007-05-02 11:27:09 -06:00
|
|
|
/*
|
|
|
|
* Round down to the nearest FAKE_NODE_MIN_SIZE.
|
|
|
|
* Command-line coefficients are in megabytes.
|
|
|
|
*/
|
|
|
|
size = ((u64)num << 20) & FAKE_NODE_MIN_HASH_MASK;
|
2007-05-02 11:27:09 -06:00
|
|
|
if (size)
|
2007-05-02 11:27:09 -06:00
|
|
|
for (i = 0; i < coeff; i++, num_nodes++)
|
|
|
|
if (setup_node_range(num_nodes, nodes,
|
|
|
|
&addr, size, max_addr) < 0)
|
|
|
|
goto done;
|
2007-05-02 11:27:09 -06:00
|
|
|
if (!*cmdline)
|
|
|
|
break;
|
|
|
|
coeff_flag = 0;
|
|
|
|
coeff = -1;
|
2007-02-13 05:26:22 -07:00
|
|
|
}
|
2007-05-02 11:27:09 -06:00
|
|
|
num = 0;
|
|
|
|
}
|
|
|
|
done:
|
|
|
|
if (!num_nodes)
|
|
|
|
return -1;
|
2007-05-02 11:27:09 -06:00
|
|
|
/* Fill remainder of system RAM, if appropriate. */
|
2007-05-02 11:27:09 -06:00
|
|
|
if (addr < max_addr) {
|
2007-05-02 11:27:09 -06:00
|
|
|
if (coeff_flag && coeff < 0) {
|
|
|
|
/* Split remaining nodes into num-sized chunks */
|
|
|
|
num_nodes += split_nodes_by_size(nodes, &addr, max_addr,
|
|
|
|
num_nodes, num);
|
|
|
|
goto out;
|
|
|
|
}
|
2007-05-02 11:27:09 -06:00
|
|
|
switch (*(cmdline - 1)) {
|
|
|
|
case '*':
|
|
|
|
/* Split remaining nodes into coeff chunks */
|
|
|
|
if (coeff <= 0)
|
|
|
|
break;
|
|
|
|
num_nodes += split_nodes_equally(nodes, &addr, max_addr,
|
|
|
|
num_nodes, coeff);
|
|
|
|
break;
|
|
|
|
case ',':
|
|
|
|
/* Do not allocate remaining system RAM */
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
/* Give one final node */
|
|
|
|
setup_node_range(num_nodes, nodes, &addr,
|
|
|
|
max_addr - addr, max_addr);
|
|
|
|
num_nodes++;
|
|
|
|
}
|
2007-05-02 11:27:09 -06:00
|
|
|
}
|
|
|
|
out:
|
2008-03-25 11:14:35 -06:00
|
|
|
memnode_shift = compute_hash_shift(nodes, num_nodes, NULL);
|
2007-05-02 11:27:09 -06:00
|
|
|
if (memnode_shift < 0) {
|
|
|
|
memnode_shift = 0;
|
|
|
|
printk(KERN_ERR "No NUMA hash function found. NUMA emulation "
|
|
|
|
"disabled.\n");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We need to vacate all active ranges that may have been registered by
|
2007-07-21 09:11:30 -06:00
|
|
|
* SRAT and set acpi_numa to -1 so that srat_disabled() always returns
|
|
|
|
* true. NUMA emulation has succeeded so we will not scan ACPI nodes.
|
2007-05-02 11:27:09 -06:00
|
|
|
*/
|
|
|
|
remove_all_active_ranges();
|
2007-07-21 09:11:30 -06:00
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
|
|
|
acpi_numa = -1;
|
|
|
|
#endif
|
2007-05-02 11:27:20 -06:00
|
|
|
for_each_node_mask(i, node_possible_map) {
|
2006-09-27 02:49:52 -06:00
|
|
|
e820_register_active_regions(i, nodes[i].start >> PAGE_SHIFT,
|
|
|
|
nodes[i].end >> PAGE_SHIFT);
|
2008-01-30 05:30:37 -07:00
|
|
|
setup_node_bootmem(i, nodes[i].start, nodes[i].end);
|
2006-09-27 02:49:52 -06:00
|
|
|
}
|
2007-07-21 09:10:32 -06:00
|
|
|
acpi_fake_nodes(nodes, num_nodes);
|
2008-01-30 05:30:37 -07:00
|
|
|
numa_init_array();
|
|
|
|
return 0;
|
2005-04-16 16:20:36 -06:00
|
|
|
}
|
2007-05-02 11:27:09 -06:00
|
|
|
#endif /* CONFIG_NUMA_EMU */
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
void __init numa_initmem_init(unsigned long start_pfn, unsigned long end_pfn)
|
2008-01-30 05:30:37 -07:00
|
|
|
{
|
2005-04-16 16:20:36 -06:00
|
|
|
int i;
|
|
|
|
|
2007-05-02 11:27:20 -06:00
|
|
|
nodes_clear(node_possible_map);
|
2008-02-17 03:02:21 -07:00
|
|
|
nodes_clear(node_online_map);
|
2007-05-02 11:27:20 -06:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
#ifdef CONFIG_NUMA_EMU
|
2007-05-02 11:27:09 -06:00
|
|
|
if (cmdline && !numa_emulation(start_pfn, end_pfn))
|
2008-01-30 05:30:37 -07:00
|
|
|
return;
|
2007-05-02 11:27:20 -06:00
|
|
|
nodes_clear(node_possible_map);
|
2008-02-17 03:02:21 -07:00
|
|
|
nodes_clear(node_online_map);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
|
|
|
if (!numa_off && !acpi_scan_nodes(start_pfn << PAGE_SHIFT,
|
|
|
|
end_pfn << PAGE_SHIFT))
|
2008-01-30 05:30:37 -07:00
|
|
|
return;
|
2007-05-02 11:27:20 -06:00
|
|
|
nodes_clear(node_possible_map);
|
2008-02-17 03:02:21 -07:00
|
|
|
nodes_clear(node_online_map);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef CONFIG_K8_NUMA
|
2008-01-30 05:30:37 -07:00
|
|
|
if (!numa_off && !k8_scan_nodes(start_pfn<<PAGE_SHIFT,
|
|
|
|
end_pfn<<PAGE_SHIFT))
|
2005-04-16 16:20:36 -06:00
|
|
|
return;
|
2007-05-02 11:27:20 -06:00
|
|
|
nodes_clear(node_possible_map);
|
2008-02-17 03:02:21 -07:00
|
|
|
nodes_clear(node_online_map);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
printk(KERN_INFO "%s\n",
|
|
|
|
numa_off ? "NUMA turned off" : "No NUMA configuration found");
|
|
|
|
|
2008-01-30 05:30:37 -07:00
|
|
|
printk(KERN_INFO "Faking a node at %016lx-%016lx\n",
|
2005-04-16 16:20:36 -06:00
|
|
|
start_pfn << PAGE_SHIFT,
|
2008-01-30 05:30:37 -07:00
|
|
|
end_pfn << PAGE_SHIFT);
|
|
|
|
/* setup dummy node covering all memory */
|
|
|
|
memnode_shift = 63;
|
2007-02-13 05:26:19 -07:00
|
|
|
memnodemap = memnode.embedded_map;
|
2005-04-16 16:20:36 -06:00
|
|
|
memnodemap[0] = 0;
|
|
|
|
node_set_online(0);
|
2007-05-02 11:27:20 -06:00
|
|
|
node_set(0, node_possible_map);
|
2005-04-16 16:20:36 -06:00
|
|
|
for (i = 0; i < NR_CPUS; i++)
|
2005-11-05 09:25:53 -07:00
|
|
|
numa_set_node(i, 0);
|
2008-01-30 05:33:11 -07:00
|
|
|
/* cpumask_of_cpu() may not be available during early startup */
|
2008-01-30 05:33:11 -07:00
|
|
|
memset(&node_to_cpumask_map[0], 0, sizeof(node_to_cpumask_map[0]));
|
|
|
|
cpu_set(0, node_to_cpumask_map[0]);
|
2006-09-27 02:49:52 -06:00
|
|
|
e820_register_active_regions(0, start_pfn, end_pfn);
|
2005-04-16 16:20:36 -06:00
|
|
|
setup_node_bootmem(0, start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
|
|
|
|
}
|
|
|
|
|
[PATCH] x86_64: Change init sections for CPU hotplug support
This patch adds __cpuinit and __cpuinitdata sections that need to exist past
boot to support cpu hotplug.
Caveat: This is done *only* for EM64T CPU Hotplug support, on request from
Andi Kleen. Much of the generic hotplug code in kernel, and none of the other
archs that support CPU hotplug today, i386, ia64, ppc64, s390 and parisc dont
mark sections with __cpuinit, but only mark them as __devinit, and
__devinitdata.
If someone is motivated to change generic code, we need to make sure all
existing hotplug code does not break, on other arch's that dont use __cpuinit,
and __cpudevinit.
Signed-off-by: Ashok Raj <ashok.raj@intel.com>
Acked-by: Andi Kleen <ak@muc.de>
Acked-by: Zwane Mwaikambo <zwane@arm.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-06-25 15:54:58 -06:00
|
|
|
__cpuinit void numa_add_cpu(int cpu)
|
2005-04-16 16:20:36 -06:00
|
|
|
{
|
2008-01-30 05:33:33 -07:00
|
|
|
set_bit(cpu,
|
|
|
|
(unsigned long *)&node_to_cpumask_map[early_cpu_to_node(cpu)]);
|
2008-01-30 05:30:37 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2005-11-05 09:25:53 -07:00
|
|
|
void __cpuinit numa_set_node(int cpu, int node)
|
|
|
|
{
|
2008-01-30 05:33:25 -07:00
|
|
|
int *cpu_to_node_map = x86_cpu_to_node_map_early_ptr;
|
2008-01-30 05:33:11 -07:00
|
|
|
|
|
|
|
if(cpu_to_node_map)
|
|
|
|
cpu_to_node_map[cpu] = node;
|
|
|
|
else if(per_cpu_offset(cpu))
|
|
|
|
per_cpu(x86_cpu_to_node_map, cpu) = node;
|
|
|
|
else
|
|
|
|
Dprintk(KERN_INFO "Setting node for non-present cpu %d\n", cpu);
|
2005-11-05 09:25:53 -07:00
|
|
|
}
|
|
|
|
|
2008-01-30 05:30:37 -07:00
|
|
|
unsigned long __init numa_free_all_bootmem(void)
|
|
|
|
{
|
2005-04-16 16:20:36 -06:00
|
|
|
unsigned long pages = 0;
|
2008-01-30 05:30:37 -07:00
|
|
|
int i;
|
|
|
|
|
|
|
|
for_each_online_node(i)
|
2005-04-16 16:20:36 -06:00
|
|
|
pages += free_all_bootmem_node(NODE_DATA(i));
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2005-04-16 16:20:36 -06:00
|
|
|
return pages;
|
2008-01-30 05:30:37 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
|
|
|
void __init paging_init(void)
|
2008-01-30 05:30:37 -07:00
|
|
|
{
|
2006-10-11 02:20:39 -06:00
|
|
|
unsigned long max_zone_pfns[MAX_NR_ZONES];
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2006-10-11 02:20:39 -06:00
|
|
|
memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
|
|
|
|
max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
|
|
|
|
max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
|
|
|
|
max_zone_pfns[ZONE_NORMAL] = end_pfn;
|
2005-11-05 09:25:54 -07:00
|
|
|
|
2007-02-13 05:26:25 -07:00
|
|
|
sparse_memory_present_with_active_regions(MAX_NUMNODES);
|
|
|
|
sparse_init();
|
2005-11-05 09:25:54 -07:00
|
|
|
|
2006-09-27 02:49:52 -06:00
|
|
|
free_area_init_nodes(max_zone_pfns);
|
2008-01-30 05:30:37 -07:00
|
|
|
}
|
2005-04-16 16:20:36 -06:00
|
|
|
|
2006-09-26 02:52:32 -06:00
|
|
|
static __init int numa_setup(char *opt)
|
2008-01-30 05:30:37 -07:00
|
|
|
{
|
2006-09-26 02:52:32 -06:00
|
|
|
if (!opt)
|
|
|
|
return -EINVAL;
|
2008-01-30 05:30:37 -07:00
|
|
|
if (!strncmp(opt, "off", 3))
|
2005-04-16 16:20:36 -06:00
|
|
|
numa_off = 1;
|
|
|
|
#ifdef CONFIG_NUMA_EMU
|
2007-05-02 11:27:09 -06:00
|
|
|
if (!strncmp(opt, "fake=", 5))
|
|
|
|
cmdline = opt + 5;
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
2008-01-30 05:30:37 -07:00
|
|
|
if (!strncmp(opt, "noacpi", 6))
|
|
|
|
acpi_numa = -1;
|
|
|
|
if (!strncmp(opt, "hotadd=", 7))
|
2006-04-07 11:49:18 -06:00
|
|
|
hotadd_percent = simple_strtoul(opt+7, NULL, 10);
|
2005-04-16 16:20:36 -06:00
|
|
|
#endif
|
2006-09-26 02:52:32 -06:00
|
|
|
return 0;
|
2008-01-30 05:30:37 -07:00
|
|
|
}
|
2006-09-26 02:52:32 -06:00
|
|
|
early_param("numa", numa_setup);
|
|
|
|
|
2006-01-11 14:45:36 -07:00
|
|
|
/*
|
|
|
|
* Setup early cpu_to_node.
|
|
|
|
*
|
|
|
|
* Populate cpu_to_node[] only if x86_cpu_to_apicid[],
|
|
|
|
* and apicid_to_node[] tables have valid entries for a CPU.
|
|
|
|
* This means we skip cpu_to_node[] initialisation for NUMA
|
|
|
|
* emulation and faking node case (when running a kernel compiled
|
|
|
|
* for NUMA on a non NUMA box), which is OK as cpu_to_node[]
|
|
|
|
* is already initialized in a round robin manner at numa_init_array,
|
|
|
|
* prior to this call, and this initialization is good enough
|
|
|
|
* for the fake NUMA cases.
|
|
|
|
*/
|
|
|
|
void __init init_cpu_to_node(void)
|
|
|
|
{
|
|
|
|
int i;
|
2008-01-30 05:30:37 -07:00
|
|
|
|
|
|
|
for (i = 0; i < NR_CPUS; i++) {
|
2008-02-19 16:35:54 -07:00
|
|
|
int node;
|
2008-01-30 05:33:10 -07:00
|
|
|
u16 apicid = x86_cpu_to_apicid_init[i];
|
2008-01-30 05:30:37 -07:00
|
|
|
|
2006-01-11 14:45:36 -07:00
|
|
|
if (apicid == BAD_APICID)
|
|
|
|
continue;
|
2008-02-19 16:35:54 -07:00
|
|
|
node = apicid_to_node[apicid];
|
|
|
|
if (node == NUMA_NO_NODE)
|
2006-01-11 14:45:36 -07:00
|
|
|
continue;
|
2008-02-19 16:35:54 -07:00
|
|
|
if (!node_online(node))
|
|
|
|
continue;
|
|
|
|
numa_set_node(i, node);
|
2006-01-11 14:45:36 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2006-01-11 14:46:27 -07:00
|
|
|
|