Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu
* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu: module: add stub for is_module_percpu_address percpu, module: implement and use is_kernel/module_percpu_address() module: encapsulate percpu handling better and record percpu_size
This commit is contained in:
Linus Torvalds
commit
9e74e7c81a
|
|
@ -330,8 +330,11 @@ struct module
|
|||
struct module_notes_attrs *notes_attrs;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
/* Per-cpu data. */
|
||||
void *percpu;
|
||||
void __percpu *percpu;
|
||||
unsigned int percpu_size;
|
||||
#endif
|
||||
|
||||
/* The command line arguments (may be mangled). People like
|
||||
keeping pointers to this stuff */
|
||||
|
|
@ -392,6 +395,7 @@ static inline int module_is_live(struct module *mod)
|
|||
struct module *__module_text_address(unsigned long addr);
|
||||
struct module *__module_address(unsigned long addr);
|
||||
bool is_module_address(unsigned long addr);
|
||||
bool is_module_percpu_address(unsigned long addr);
|
||||
bool is_module_text_address(unsigned long addr);
|
||||
|
||||
static inline int within_module_core(unsigned long addr, struct module *mod)
|
||||
|
|
@ -563,6 +567,11 @@ static inline bool is_module_address(unsigned long addr)
|
|||
return false;
|
||||
}
|
||||
|
||||
static inline bool is_module_percpu_address(unsigned long addr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline bool is_module_text_address(unsigned long addr)
|
||||
{
|
||||
return false;
|
||||
|
|
|
|||
|
|
@ -137,6 +137,7 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
|
|||
extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
|
||||
extern void __percpu *__alloc_percpu(size_t size, size_t align);
|
||||
extern void free_percpu(void __percpu *__pdata);
|
||||
extern bool is_kernel_percpu_address(unsigned long addr);
|
||||
extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
|
||||
|
||||
#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
|
||||
|
|
@ -163,6 +164,12 @@ static inline void free_percpu(void __percpu *p)
|
|||
kfree(p);
|
||||
}
|
||||
|
||||
/* can't distinguish from other static vars, always false */
|
||||
static inline bool is_kernel_percpu_address(unsigned long addr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline phys_addr_t per_cpu_ptr_to_phys(void *addr)
|
||||
{
|
||||
return __pa(addr);
|
||||
|
|
|
|||
|
|
@ -582,9 +582,6 @@ static int static_obj(void *obj)
|
|||
unsigned long start = (unsigned long) &_stext,
|
||||
end = (unsigned long) &_end,
|
||||
addr = (unsigned long) obj;
|
||||
#ifdef CONFIG_SMP
|
||||
int i;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* static variable?
|
||||
|
|
@ -595,24 +592,16 @@ static int static_obj(void *obj)
|
|||
if (arch_is_kernel_data(addr))
|
||||
return 1;
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
/*
|
||||
* percpu var?
|
||||
* in-kernel percpu var?
|
||||
*/
|
||||
for_each_possible_cpu(i) {
|
||||
start = (unsigned long) &__per_cpu_start + per_cpu_offset(i);
|
||||
end = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM
|
||||
+ per_cpu_offset(i);
|
||||
|
||||
if ((addr >= start) && (addr < end))
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
if (is_kernel_percpu_address(addr))
|
||||
return 1;
|
||||
|
||||
/*
|
||||
* module var?
|
||||
* module static or percpu var?
|
||||
*/
|
||||
return is_module_address(addr);
|
||||
return is_module_address(addr) || is_module_percpu_address(addr);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
104
kernel/module.c
104
kernel/module.c
|
|
@ -370,27 +370,33 @@ EXPORT_SYMBOL_GPL(find_module);
|
|||
|
||||
#ifdef CONFIG_SMP
|
||||
|
||||
static void *percpu_modalloc(unsigned long size, unsigned long align,
|
||||
const char *name)
|
||||
static inline void __percpu *mod_percpu(struct module *mod)
|
||||
{
|
||||
void *ptr;
|
||||
return mod->percpu;
|
||||
}
|
||||
|
||||
static int percpu_modalloc(struct module *mod,
|
||||
unsigned long size, unsigned long align)
|
||||
{
|
||||
if (align > PAGE_SIZE) {
|
||||
printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",
|
||||
name, align, PAGE_SIZE);
|
||||
mod->name, align, PAGE_SIZE);
|
||||
align = PAGE_SIZE;
|
||||
}
|
||||
|
||||
ptr = __alloc_reserved_percpu(size, align);
|
||||
if (!ptr)
|
||||
mod->percpu = __alloc_reserved_percpu(size, align);
|
||||
if (!mod->percpu) {
|
||||
printk(KERN_WARNING
|
||||
"Could not allocate %lu bytes percpu data\n", size);
|
||||
return ptr;
|
||||
return -ENOMEM;
|
||||
}
|
||||
mod->percpu_size = size;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void percpu_modfree(void *freeme)
|
||||
static void percpu_modfree(struct module *mod)
|
||||
{
|
||||
free_percpu(freeme);
|
||||
free_percpu(mod->percpu);
|
||||
}
|
||||
|
||||
static unsigned int find_pcpusec(Elf_Ehdr *hdr,
|
||||
|
|
@ -400,24 +406,62 @@ static unsigned int find_pcpusec(Elf_Ehdr *hdr,
|
|||
return find_sec(hdr, sechdrs, secstrings, ".data.percpu");
|
||||
}
|
||||
|
||||
static void percpu_modcopy(void *pcpudest, const void *from, unsigned long size)
|
||||
static void percpu_modcopy(struct module *mod,
|
||||
const void *from, unsigned long size)
|
||||
{
|
||||
int cpu;
|
||||
|
||||
for_each_possible_cpu(cpu)
|
||||
memcpy(pcpudest + per_cpu_offset(cpu), from, size);
|
||||
memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
|
||||
}
|
||||
|
||||
/**
|
||||
* is_module_percpu_address - test whether address is from module static percpu
|
||||
* @addr: address to test
|
||||
*
|
||||
* Test whether @addr belongs to module static percpu area.
|
||||
*
|
||||
* RETURNS:
|
||||
* %true if @addr is from module static percpu area
|
||||
*/
|
||||
bool is_module_percpu_address(unsigned long addr)
|
||||
{
|
||||
struct module *mod;
|
||||
unsigned int cpu;
|
||||
|
||||
preempt_disable();
|
||||
|
||||
list_for_each_entry_rcu(mod, &modules, list) {
|
||||
if (!mod->percpu_size)
|
||||
continue;
|
||||
for_each_possible_cpu(cpu) {
|
||||
void *start = per_cpu_ptr(mod->percpu, cpu);
|
||||
|
||||
if ((void *)addr >= start &&
|
||||
(void *)addr < start + mod->percpu_size) {
|
||||
preempt_enable();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
preempt_enable();
|
||||
return false;
|
||||
}
|
||||
|
||||
#else /* ... !CONFIG_SMP */
|
||||
|
||||
static inline void *percpu_modalloc(unsigned long size, unsigned long align,
|
||||
const char *name)
|
||||
static inline void __percpu *mod_percpu(struct module *mod)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
static inline void percpu_modfree(void *pcpuptr)
|
||||
static inline int percpu_modalloc(struct module *mod,
|
||||
unsigned long size, unsigned long align)
|
||||
{
|
||||
return -ENOMEM;
|
||||
}
|
||||
static inline void percpu_modfree(struct module *mod)
|
||||
{
|
||||
BUG();
|
||||
}
|
||||
static inline unsigned int find_pcpusec(Elf_Ehdr *hdr,
|
||||
Elf_Shdr *sechdrs,
|
||||
|
|
@ -425,12 +469,16 @@ static inline unsigned int find_pcpusec(Elf_Ehdr *hdr,
|
|||
{
|
||||
return 0;
|
||||
}
|
||||
static inline void percpu_modcopy(void *pcpudst, const void *src,
|
||||
unsigned long size)
|
||||
static inline void percpu_modcopy(struct module *mod,
|
||||
const void *from, unsigned long size)
|
||||
{
|
||||
/* pcpusec should be 0, and size of that section should be 0. */
|
||||
BUG_ON(size != 0);
|
||||
}
|
||||
bool is_module_percpu_address(unsigned long addr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
|
|
@ -1400,8 +1448,7 @@ static void free_module(struct module *mod)
|
|||
/* This may be NULL, but that's OK */
|
||||
module_free(mod, mod->module_init);
|
||||
kfree(mod->args);
|
||||
if (mod->percpu)
|
||||
percpu_modfree(mod->percpu);
|
||||
percpu_modfree(mod);
|
||||
#if defined(CONFIG_MODULE_UNLOAD)
|
||||
if (mod->refptr)
|
||||
free_percpu(mod->refptr);
|
||||
|
|
@ -1520,7 +1567,7 @@ static int simplify_symbols(Elf_Shdr *sechdrs,
|
|||
default:
|
||||
/* Divert to percpu allocation if a percpu var. */
|
||||
if (sym[i].st_shndx == pcpuindex)
|
||||
secbase = (unsigned long)mod->percpu;
|
||||
secbase = (unsigned long)mod_percpu(mod);
|
||||
else
|
||||
secbase = sechdrs[sym[i].st_shndx].sh_addr;
|
||||
sym[i].st_value += secbase;
|
||||
|
|
@ -1954,7 +2001,7 @@ static noinline struct module *load_module(void __user *umod,
|
|||
unsigned int modindex, versindex, infoindex, pcpuindex;
|
||||
struct module *mod;
|
||||
long err = 0;
|
||||
void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */
|
||||
void *ptr = NULL; /* Stops spurious gcc warning */
|
||||
unsigned long symoffs, stroffs, *strmap;
|
||||
|
||||
mm_segment_t old_fs;
|
||||
|
|
@ -2094,15 +2141,11 @@ static noinline struct module *load_module(void __user *umod,
|
|||
|
||||
if (pcpuindex) {
|
||||
/* We have a special allocation for this section. */
|
||||
percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size,
|
||||
sechdrs[pcpuindex].sh_addralign,
|
||||
mod->name);
|
||||
if (!percpu) {
|
||||
err = -ENOMEM;
|
||||
err = percpu_modalloc(mod, sechdrs[pcpuindex].sh_size,
|
||||
sechdrs[pcpuindex].sh_addralign);
|
||||
if (err)
|
||||
goto free_mod;
|
||||
}
|
||||
sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
|
||||
mod->percpu = percpu;
|
||||
}
|
||||
|
||||
/* Determine total sizes, and put offsets in sh_entsize. For now
|
||||
|
|
@ -2317,7 +2360,7 @@ static noinline struct module *load_module(void __user *umod,
|
|||
sort_extable(mod->extable, mod->extable + mod->num_exentries);
|
||||
|
||||
/* Finally, copy percpu area over. */
|
||||
percpu_modcopy(mod->percpu, (void *)sechdrs[pcpuindex].sh_addr,
|
||||
percpu_modcopy(mod, (void *)sechdrs[pcpuindex].sh_addr,
|
||||
sechdrs[pcpuindex].sh_size);
|
||||
|
||||
add_kallsyms(mod, sechdrs, hdr->e_shnum, symindex, strindex,
|
||||
|
|
@ -2409,8 +2452,7 @@ static noinline struct module *load_module(void __user *umod,
|
|||
module_free(mod, mod->module_core);
|
||||
/* mod will be freed with core. Don't access it beyond this line! */
|
||||
free_percpu:
|
||||
if (percpu)
|
||||
percpu_modfree(percpu);
|
||||
percpu_modfree(mod);
|
||||
free_mod:
|
||||
kfree(args);
|
||||
kfree(strmap);
|
||||
|
|
|
|||
26
mm/percpu.c
26
mm/percpu.c
|
|
@ -1303,6 +1303,32 @@ void free_percpu(void __percpu *ptr)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(free_percpu);
|
||||
|
||||
/**
|
||||
* is_kernel_percpu_address - test whether address is from static percpu area
|
||||
* @addr: address to test
|
||||
*
|
||||
* Test whether @addr belongs to in-kernel static percpu area. Module
|
||||
* static percpu areas are not considered. For those, use
|
||||
* is_module_percpu_address().
|
||||
*
|
||||
* RETURNS:
|
||||
* %true if @addr is from in-kernel static percpu area, %false otherwise.
|
||||
*/
|
||||
bool is_kernel_percpu_address(unsigned long addr)
|
||||
{
|
||||
const size_t static_size = __per_cpu_end - __per_cpu_start;
|
||||
void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
|
||||
unsigned int cpu;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
void *start = per_cpu_ptr(base, cpu);
|
||||
|
||||
if ((void *)addr >= start && (void *)addr < start + static_size)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* per_cpu_ptr_to_phys - convert translated percpu address to physical address
|
||||
* @addr: the address to be converted to physical address
|
||||
|
|
|
|||
Loading…
Reference in a new issue