c0ebfdc3fe
Originally, the first chunk was served by one or two chunks, each given a region they are responsible for. Despite this, the arithmetic was based off of the true base_addr of the chunk making it be overly inclusive. This patch moves the base_addr of chunks that are responsible for the first chunk. The base_addr must remain page aligned to keep the address alignment correct, so it is the beginning of the region served page aligned down. start_offset holds where the region served begins from this new base_addr. The corresponding percpu address checks are modified to be more specific as a result. The first chunk considers only the dynamic region and both first chunk and reserved chunk checks ignore the static region. The static region addresses should never be passed into the allocator. There is no impact here besides distinguishing the first chunk and making the checks specific. The percpu pointer to physical address is left intact as addresses are not given out in the non-allocated portion of percpu memory. nr_pages is added to pcpu_chunk to keep track of the size of the entire region served containing both start_offset and end_offset. This variable will be used to manage the bitmap allocator. Signed-off-by: Dennis Zhou <dennisszhou@gmail.com> Reviewed-by: Josef Bacik <jbacik@fb.com> Signed-off-by: Tejun Heo <tj@kernel.org>
171 lines
4.2 KiB
C
171 lines
4.2 KiB
C
#ifndef _MM_PERCPU_INTERNAL_H
|
|
#define _MM_PERCPU_INTERNAL_H
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/percpu.h>
|
|
|
|
struct pcpu_chunk {
|
|
#ifdef CONFIG_PERCPU_STATS
|
|
int nr_alloc; /* # of allocations */
|
|
size_t max_alloc_size; /* largest allocation size */
|
|
#endif
|
|
|
|
struct list_head list; /* linked to pcpu_slot lists */
|
|
int free_size; /* free bytes in the chunk */
|
|
int contig_hint; /* max contiguous size hint */
|
|
void *base_addr; /* base address of this chunk */
|
|
|
|
int map_used; /* # of map entries used before the sentry */
|
|
int map_alloc; /* # of map entries allocated */
|
|
int *map; /* allocation map */
|
|
struct list_head map_extend_list;/* on pcpu_map_extend_chunks */
|
|
|
|
void *data; /* chunk data */
|
|
int first_free; /* no free below this */
|
|
bool immutable; /* no [de]population allowed */
|
|
int start_offset; /* the overlap with the previous
|
|
region to have a page aligned
|
|
base_addr */
|
|
int end_offset; /* additional area required to
|
|
have the region end page
|
|
aligned */
|
|
|
|
int nr_pages; /* # of pages served by this chunk */
|
|
int nr_populated; /* # of populated pages */
|
|
unsigned long populated[]; /* populated bitmap */
|
|
};
|
|
|
|
extern spinlock_t pcpu_lock;
|
|
|
|
extern struct list_head *pcpu_slot;
|
|
extern int pcpu_nr_slots;
|
|
extern int pcpu_nr_empty_pop_pages;
|
|
|
|
extern struct pcpu_chunk *pcpu_first_chunk;
|
|
extern struct pcpu_chunk *pcpu_reserved_chunk;
|
|
|
|
#ifdef CONFIG_PERCPU_STATS
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
struct percpu_stats {
|
|
u64 nr_alloc; /* lifetime # of allocations */
|
|
u64 nr_dealloc; /* lifetime # of deallocations */
|
|
u64 nr_cur_alloc; /* current # of allocations */
|
|
u64 nr_max_alloc; /* max # of live allocations */
|
|
u32 nr_chunks; /* current # of live chunks */
|
|
u32 nr_max_chunks; /* max # of live chunks */
|
|
size_t min_alloc_size; /* min allocaiton size */
|
|
size_t max_alloc_size; /* max allocation size */
|
|
};
|
|
|
|
extern struct percpu_stats pcpu_stats;
|
|
extern struct pcpu_alloc_info pcpu_stats_ai;
|
|
|
|
/*
|
|
* For debug purposes. We don't care about the flexible array.
|
|
*/
|
|
static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
|
|
{
|
|
memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info));
|
|
|
|
/* initialize min_alloc_size to unit_size */
|
|
pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size;
|
|
}
|
|
|
|
/*
|
|
* pcpu_stats_area_alloc - increment area allocation stats
|
|
* @chunk: the location of the area being allocated
|
|
* @size: size of area to allocate in bytes
|
|
*
|
|
* CONTEXT:
|
|
* pcpu_lock.
|
|
*/
|
|
static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
|
|
{
|
|
lockdep_assert_held(&pcpu_lock);
|
|
|
|
pcpu_stats.nr_alloc++;
|
|
pcpu_stats.nr_cur_alloc++;
|
|
pcpu_stats.nr_max_alloc =
|
|
max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc);
|
|
pcpu_stats.min_alloc_size =
|
|
min(pcpu_stats.min_alloc_size, size);
|
|
pcpu_stats.max_alloc_size =
|
|
max(pcpu_stats.max_alloc_size, size);
|
|
|
|
chunk->nr_alloc++;
|
|
chunk->max_alloc_size = max(chunk->max_alloc_size, size);
|
|
}
|
|
|
|
/*
|
|
* pcpu_stats_area_dealloc - decrement allocation stats
|
|
* @chunk: the location of the area being deallocated
|
|
*
|
|
* CONTEXT:
|
|
* pcpu_lock.
|
|
*/
|
|
static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
|
|
{
|
|
lockdep_assert_held(&pcpu_lock);
|
|
|
|
pcpu_stats.nr_dealloc++;
|
|
pcpu_stats.nr_cur_alloc--;
|
|
|
|
chunk->nr_alloc--;
|
|
}
|
|
|
|
/*
|
|
* pcpu_stats_chunk_alloc - increment chunk stats
|
|
*/
|
|
static inline void pcpu_stats_chunk_alloc(void)
|
|
{
|
|
unsigned long flags;
|
|
spin_lock_irqsave(&pcpu_lock, flags);
|
|
|
|
pcpu_stats.nr_chunks++;
|
|
pcpu_stats.nr_max_chunks =
|
|
max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks);
|
|
|
|
spin_unlock_irqrestore(&pcpu_lock, flags);
|
|
}
|
|
|
|
/*
|
|
* pcpu_stats_chunk_dealloc - decrement chunk stats
|
|
*/
|
|
static inline void pcpu_stats_chunk_dealloc(void)
|
|
{
|
|
unsigned long flags;
|
|
spin_lock_irqsave(&pcpu_lock, flags);
|
|
|
|
pcpu_stats.nr_chunks--;
|
|
|
|
spin_unlock_irqrestore(&pcpu_lock, flags);
|
|
}
|
|
|
|
#else
|
|
|
|
static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
|
|
{
|
|
}
|
|
|
|
static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
|
|
{
|
|
}
|
|
|
|
static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
|
|
{
|
|
}
|
|
|
|
static inline void pcpu_stats_chunk_alloc(void)
|
|
{
|
|
}
|
|
|
|
static inline void pcpu_stats_chunk_dealloc(void)
|
|
{
|
|
}
|
|
|
|
#endif /* !CONFIG_PERCPU_STATS */
|
|
|
|
#endif
|