redonkable
/
alistair23-linux
1
0
Fork 0
Code Releases Activity

Merge branch 'irq/for-block' into irq/core 

Add the new irq spreading infrastructure.
hifive-unleashed-5.1
Thomas Gleixner 2016-09-15 20:54:40 +02:00
parent 16217dc79d ee8d41e53e
commit 0a30d69195
9 changed files with 295 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
drivers/base/platform-msi.c
View File

@ -142,13 +142,12 @@ static int platform_msi_alloc_descs_with_irq(struct device *dev, int virq,
}
for (i = 0; i < nvec; i++) {
desc = alloc_msi_entry(dev);
desc = alloc_msi_entry(dev, 1, NULL);
if (!desc)
break;
desc->platform.msi_priv_data = data;
desc->platform.msi_index = base + i;
desc->nvec_used = 1;
desc->irq = virq ? virq + i : 0;
list_add_tail(&desc->list, dev_to_msi_list(dev));

159
drivers/pci/msi.c
View File

@ -550,15 +550,23 @@ error_attrs:
return ret;
}
static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec)
static struct msi_desc *
msi_setup_entry(struct pci_dev *dev, int nvec, bool affinity)
{
u16 control;
struct cpumask *masks = NULL;
struct msi_desc *entry;
u16 control;
if (affinity) {
masks = irq_create_affinity_masks(dev->irq_affinity, nvec);
if (!masks)
pr_err("Unable to allocate affinity masks, ignoring\n");
}
/* MSI Entry Initialization */
entry = alloc_msi_entry(&dev->dev);
entry = alloc_msi_entry(&dev->dev, nvec, masks);
if (!entry)
return NULL;
goto out;
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
@ -569,8 +577,6 @@ static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec)
entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */
entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1;
entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec));
entry->nvec_used = nvec;
entry->affinity = dev->irq_affinity;
if (control & PCI_MSI_FLAGS_64BIT)
entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64;
@ -581,6 +587,8 @@ static struct msi_desc *msi_setup_entry(struct pci_dev *dev, int nvec)
if (entry->msi_attrib.maskbit)
pci_read_config_dword(dev, entry->mask_pos, &entry->masked);
out:
kfree(masks);
return entry;
}
@ -609,7 +617,7 @@ static int msi_verify_entries(struct pci_dev *dev)
* an error, and a positive return value indicates the number of interrupts
* which could have been allocated.
*/
static int msi_capability_init(struct pci_dev *dev, int nvec)
static int msi_capability_init(struct pci_dev *dev, int nvec, bool affinity)
{
struct msi_desc *entry;
int ret;
@ -617,7 +625,7 @@ static int msi_capability_init(struct pci_dev *dev, int nvec)
pci_msi_set_enable(dev, 0); /* Disable MSI during set up */
entry = msi_setup_entry(dev, nvec);
entry = msi_setup_entry(dev, nvec, affinity);
if (!entry)
return -ENOMEM;
@ -680,28 +688,29 @@ static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries)
}
static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
struct msix_entry *entries, int nvec)
struct msix_entry *entries, int nvec,
bool affinity)
{
const struct cpumask *mask = NULL;
struct cpumask *curmsk, *masks = NULL;
struct msi_desc *entry;
int cpu = -1, i;
int ret, i;
for (i = 0; i < nvec; i++) {
if (dev->irq_affinity) {
cpu = cpumask_next(cpu, dev->irq_affinity);
if (cpu >= nr_cpu_ids)
cpu = cpumask_first(dev->irq_affinity);
mask = cpumask_of(cpu);
}
if (affinity) {
masks = irq_create_affinity_masks(dev->irq_affinity, nvec);
if (!masks)
pr_err("Unable to allocate affinity masks, ignoring\n");
}
entry = alloc_msi_entry(&dev->dev);
for (i = 0, curmsk = masks; i < nvec; i++) {
entry = alloc_msi_entry(&dev->dev, 1, curmsk);
if (!entry) {
if (!i)
iounmap(base);
else
free_msi_irqs(dev);
/* No enough memory. Don't try again */
return -ENOMEM;
ret = -ENOMEM;
goto out;
}
entry->msi_attrib.is_msix = 1;
@ -712,12 +721,14 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
entry->msi_attrib.entry_nr = i;
entry->msi_attrib.default_irq = dev->irq;
entry->mask_base = base;
entry->nvec_used = 1;
entry->affinity = mask;
list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
if (masks)
curmsk++;
}
ret = 0;
out:
kfree(masks);
return 0;
}
@ -746,8 +757,8 @@ static void msix_program_entries(struct pci_dev *dev,
* single MSI-X irq. A return of zero indicates the successful setup of
* requested MSI-X entries with allocated irqs or non-zero for otherwise.
**/
static int msix_capability_init(struct pci_dev *dev,
struct msix_entry *entries, int nvec)
static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
int nvec, bool affinity)
{
int ret;
u16 control;
@ -762,7 +773,7 @@ static int msix_capability_init(struct pci_dev *dev,
if (!base)
return -ENOMEM;
ret = msix_setup_entries(dev, base, entries, nvec);
ret = msix_setup_entries(dev, base, entries, nvec, affinity);
if (ret)
return ret;
@ -942,22 +953,8 @@ int pci_msix_vec_count(struct pci_dev *dev)
}
EXPORT_SYMBOL(pci_msix_vec_count);
/**
* pci_enable_msix - configure device's MSI-X capability structure
* @dev: pointer to the pci_dev data structure of MSI-X device function
* @entries: pointer to an array of MSI-X entries (optional)
* @nvec: number of MSI-X irqs requested for allocation by device driver
*
* Setup the MSI-X capability structure of device function with the number
* of requested irqs upon its software driver call to request for
* MSI-X mode enabled on its hardware device function. A return of zero
* indicates the successful configuration of MSI-X capability structure
* with new allocated MSI-X irqs. A return of < 0 indicates a failure.
* Or a return of > 0 indicates that driver request is exceeding the number
* of irqs or MSI-X vectors available. Driver should use the returned value to
* re-send its request.
**/
int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec)
static int __pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries,
int nvec, bool affinity)
{
int nr_entries;
int i, j;
@ -989,7 +986,27 @@ int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec)
dev_info(&dev->dev, "can't enable MSI-X (MSI IRQ already assigned)\n");
return -EINVAL;
}
return msix_capability_init(dev, entries, nvec);
return msix_capability_init(dev, entries, nvec, affinity);
}
/**
* pci_enable_msix - configure device's MSI-X capability structure
* @dev: pointer to the pci_dev data structure of MSI-X device function
* @entries: pointer to an array of MSI-X entries (optional)
* @nvec: number of MSI-X irqs requested for allocation by device driver
*
* Setup the MSI-X capability structure of device function with the number
* of requested irqs upon its software driver call to request for
* MSI-X mode enabled on its hardware device function. A return of zero
* indicates the successful configuration of MSI-X capability structure
* with new allocated MSI-X irqs. A return of < 0 indicates a failure.
* Or a return of > 0 indicates that driver request is exceeding the number
* of irqs or MSI-X vectors available. Driver should use the returned value to
* re-send its request.
**/
int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec)
{
return __pci_enable_msix(dev, entries, nvec, false);
}
EXPORT_SYMBOL(pci_enable_msix);
@ -1042,6 +1059,7 @@ EXPORT_SYMBOL(pci_msi_enabled);
static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
unsigned int flags)
{
bool affinity = flags & PCI_IRQ_AFFINITY;
int nvec;
int rc;
@ -1070,19 +1088,17 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
nvec = maxvec;
for (;;) {
if (flags & PCI_IRQ_AFFINITY) {
dev->irq_affinity = irq_create_affinity_mask(&nvec);
if (affinity) {
nvec = irq_calc_affinity_vectors(dev->irq_affinity,
nvec);
if (nvec < minvec)
return -ENOSPC;
}
rc = msi_capability_init(dev, nvec);
rc = msi_capability_init(dev, nvec, affinity);
if (rc == 0)
return nvec;
kfree(dev->irq_affinity);
dev->irq_affinity = NULL;
if (rc < 0)
return rc;
if (rc < minvec)
@ -1114,26 +1130,24 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
struct msix_entry *entries, int minvec, int maxvec,
unsigned int flags)
{
int nvec = maxvec;
int rc;
bool affinity = flags & PCI_IRQ_AFFINITY;
int rc, nvec = maxvec;
if (maxvec < minvec)
return -ERANGE;
for (;;) {
if (flags & PCI_IRQ_AFFINITY) {
dev->irq_affinity = irq_create_affinity_mask(&nvec);
if (affinity) {
nvec = irq_calc_affinity_vectors(dev->irq_affinity,
nvec);
if (nvec < minvec)
return -ENOSPC;
}
rc = pci_enable_msix(dev, entries, nvec);
rc = __pci_enable_msix(dev, entries, nvec, affinity);
if (rc == 0)
return nvec;
kfree(dev->irq_affinity);
dev->irq_affinity = NULL;
if (rc < 0)
return rc;
if (rc < minvec)
@ -1257,6 +1271,37 @@ int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
}
EXPORT_SYMBOL(pci_irq_vector);
/**
* pci_irq_get_affinity - return the affinity of a particular msi vector
* @dev: PCI device to operate on
* @nr: device-relative interrupt vector index (0-based).
*/
const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
{
if (dev->msix_enabled) {
struct msi_desc *entry;
int i = 0;
for_each_pci_msi_entry(entry, dev) {
if (i == nr)
return entry->affinity;
i++;
}
WARN_ON_ONCE(1);
return NULL;
} else if (dev->msi_enabled) {
struct msi_desc *entry = first_pci_msi_entry(dev);
if (WARN_ON_ONCE(!entry || nr >= entry->nvec_used))
return NULL;
return &entry->affinity[nr];
} else {
return cpu_possible_mask;
}
}
EXPORT_SYMBOL(pci_irq_get_affinity);
struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc)
{
return to_pci_dev(desc->dev);

3
drivers/staging/fsl-mc/bus/mc-msi.c
View File

@ -213,7 +213,7 @@ static int fsl_mc_msi_alloc_descs(struct device *dev, unsigned int irq_count)
struct msi_desc *msi_desc;
for (i = 0; i < irq_count; i++) {
msi_desc = alloc_msi_entry(dev);
msi_desc = alloc_msi_entry(dev, 1, NULL);
if (!msi_desc) {
dev_err(dev, "Failed to allocate msi entry\n");
error = -ENOMEM;
@ -221,7 +221,6 @@ static int fsl_mc_msi_alloc_descs(struct device *dev, unsigned int irq_count)
}
msi_desc->fsl_mc.msi_index = i;
msi_desc->nvec_used = 1;
INIT_LIST_HEAD(&msi_desc->list);
list_add_tail(&msi_desc->list, dev_to_msi_list(dev));
}

14
include/linux/interrupt.h
View File

@ -278,7 +278,8 @@ extern int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m);
extern int
irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify);
struct cpumask *irq_create_affinity_mask(unsigned int *nr_vecs);
struct cpumask *irq_create_affinity_masks(const struct cpumask *affinity, int nvec);
int irq_calc_affinity_vectors(const struct cpumask *affinity, int maxvec);
#else /* CONFIG_SMP */
@ -311,11 +312,18 @@ irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
return 0;
}
static inline struct cpumask *irq_create_affinity_mask(unsigned int *nr_vecs)
static inline struct cpumask *
irq_create_affinity_masks(const struct cpumask *affinity, int nvec)
{
*nr_vecs = 1;
return NULL;
}
static inline int
irq_calc_affinity_vectors(const struct cpumask *affinity, int maxvec)
{
return maxvec;
}
#endif /* CONFIG_SMP */
/*

5
include/linux/msi.h
View File

@ -68,7 +68,7 @@ struct msi_desc {
unsigned int nvec_used;
struct device *dev;
struct msi_msg msg;
const struct cpumask *affinity;
struct cpumask *affinity;
union {
/* PCI MSI/X specific data */
@ -123,7 +123,8 @@ static inline void *msi_desc_to_pci_sysdata(struct msi_desc *desc)
}
#endif /* CONFIG_PCI_MSI */
struct msi_desc *alloc_msi_entry(struct device *dev);
struct msi_desc *alloc_msi_entry(struct device *dev, int nvec,
const struct cpumask *affinity);
void free_msi_entry(struct msi_desc *entry);
void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg);
void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg);

6
include/linux/pci.h
View File

@ -1300,6 +1300,7 @@ int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs,
unsigned int max_vecs, unsigned int flags);
void pci_free_irq_vectors(struct pci_dev *dev);
int pci_irq_vector(struct pci_dev *dev, unsigned int nr);
const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev, int vec);
#else
static inline int pci_msi_vec_count(struct pci_dev *dev) { return -ENOSYS; }
@ -1342,6 +1343,11 @@ static inline int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
return -EINVAL;
return dev->irq;
}
static inline const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev,
int vec)
{
return cpu_possible_mask;
}
#endif
#ifdef CONFIG_PCIEPORTBUS

173
kernel/irq/affinity.c
View File

@ -4,60 +4,151 @@
#include <linux/slab.h>
#include <linux/cpu.h>
static int get_first_sibling(unsigned int cpu)
static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk,
int cpus_per_vec)
{
unsigned int ret;
const struct cpumask *siblmsk;
int cpu, sibl;
ret = cpumask_first(topology_sibling_cpumask(cpu));
if (ret < nr_cpu_ids)
return ret;
return cpu;
for ( ; cpus_per_vec > 0; ) {
cpu = cpumask_first(nmsk);
/* Should not happen, but I'm too lazy to think about it */
if (cpu >= nr_cpu_ids)
return;
cpumask_clear_cpu(cpu, nmsk);
cpumask_set_cpu(cpu, irqmsk);
cpus_per_vec--;
/* If the cpu has siblings, use them first */
siblmsk = topology_sibling_cpumask(cpu);
for (sibl = -1; cpus_per_vec > 0; ) {
sibl = cpumask_next(sibl, siblmsk);
if (sibl >= nr_cpu_ids)
break;
if (!cpumask_test_and_clear_cpu(sibl, nmsk))
continue;
cpumask_set_cpu(sibl, irqmsk);
cpus_per_vec--;
}
}
}
/*
* Take a map of online CPUs and the number of available interrupt vectors
* and generate an output cpumask suitable for spreading MSI/MSI-X vectors
* so that they are distributed as good as possible around the CPUs. If
* more vectors than CPUs are available we'll map one to each CPU,
* otherwise we map one to the first sibling of each socket.
*
* If there are more vectors than CPUs we will still only have one bit
* set per CPU, but interrupt code will keep on assigning the vectors from
* the start of the bitmap until we run out of vectors.
*/
struct cpumask *irq_create_affinity_mask(unsigned int *nr_vecs)
static int get_nodes_in_cpumask(const struct cpumask *mask, nodemask_t *nodemsk)
{
struct cpumask *affinity_mask;
unsigned int max_vecs = *nr_vecs;
int n, nodes;
if (max_vecs == 1)
return NULL;
affinity_mask = kzalloc(cpumask_size(), GFP_KERNEL);
if (!affinity_mask) {
*nr_vecs = 1;
return NULL;
/* Calculate the number of nodes in the supplied affinity mask */
for (n = 0, nodes = 0; n < num_online_nodes(); n++) {
if (cpumask_intersects(mask, cpumask_of_node(n))) {
node_set(n, *nodemsk);
nodes++;
}
}
return nodes;
}
/**
* irq_create_affinity_masks - Create affinity masks for multiqueue spreading
* @affinity: The affinity mask to spread. If NULL cpu_online_mask
* is used
* @nvecs: The number of vectors
*
* Returns the masks pointer or NULL if allocation failed.
*/
struct cpumask *irq_create_affinity_masks(const struct cpumask *affinity,
int nvec)
{
int n, nodes, vecs_per_node, cpus_per_vec, extra_vecs, curvec = 0;
nodemask_t nodemsk = NODE_MASK_NONE;
struct cpumask *masks;
cpumask_var_t nmsk;
if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
return NULL;
masks = kzalloc(nvec * sizeof(*masks), GFP_KERNEL);
if (!masks)
goto out;
/* Stabilize the cpumasks */
get_online_cpus();
if (max_vecs >= num_online_cpus()) {
cpumask_copy(affinity_mask, cpu_online_mask);
*nr_vecs = num_online_cpus();
} else {
unsigned int vecs = 0, cpu;
/* If the supplied affinity mask is NULL, use cpu online mask */
if (!affinity)
affinity = cpu_online_mask;
for_each_online_cpu(cpu) {
if (cpu == get_first_sibling(cpu)) {
cpumask_set_cpu(cpu, affinity_mask);
vecs++;
}
nodes = get_nodes_in_cpumask(affinity, &nodemsk);
if (--max_vecs == 0)
/*
* If the number of nodes in the mask is less than or equal the
* number of vectors we just spread the vectors across the nodes.
*/
if (nvec <= nodes) {
for_each_node_mask(n, nodemsk) {
cpumask_copy(masks + curvec, cpumask_of_node(n));
if (++curvec == nvec)
break;
}
*nr_vecs = vecs;
goto outonl;
}
put_online_cpus();
return affinity_mask;
/* Spread the vectors per node */
vecs_per_node = nvec / nodes;
/* Account for rounding errors */
extra_vecs = nvec - (nodes * vecs_per_node);
for_each_node_mask(n, nodemsk) {
int ncpus, v, vecs_to_assign = vecs_per_node;
/* Get the cpus on this node which are in the mask */
cpumask_and(nmsk, affinity, cpumask_of_node(n));
/* Calculate the number of cpus per vector */
ncpus = cpumask_weight(nmsk);
for (v = 0; curvec < nvec && v < vecs_to_assign; curvec++, v++) {
cpus_per_vec = ncpus / vecs_to_assign;
/* Account for extra vectors to compensate rounding errors */
if (extra_vecs) {
cpus_per_vec++;
if (!--extra_vecs)
vecs_per_node++;
}
irq_spread_init_one(masks + curvec, nmsk, cpus_per_vec);
}
if (curvec >= nvec)
break;
}
outonl:
put_online_cpus();
out:
free_cpumask_var(nmsk);
return masks;
}
/**
* irq_calc_affinity_vectors - Calculate to optimal number of vectors for a given affinity mask
* @affinity: The affinity mask to spread. If NULL cpu_online_mask
* is used
* @maxvec: The maximum number of vectors available
*/
int irq_calc_affinity_vectors(const struct cpumask *affinity, int maxvec)
{
int cpus, ret;
/* Stabilize the cpumasks */
get_online_cpus();
/* If the supplied affinity mask is NULL, use cpu online mask */
if (!affinity)
affinity = cpu_online_mask;
cpus = cpumask_weight(affinity);
ret = (cpus < maxvec) ? cpus : maxvec;
put_online_cpus();
return ret;
}

31
kernel/irq/irqdesc.c
View File

@ -424,25 +424,24 @@ static int alloc_descs(unsigned int start, unsigned int cnt, int node,
const struct cpumask *mask = NULL;
struct irq_desc *desc;
unsigned int flags;
int i, cpu = -1;
int i;
if (affinity && cpumask_empty(affinity))
return -EINVAL;
/* Validate affinity mask(s) */
if (affinity) {
for (i = 0, mask = affinity; i < cnt; i++, mask++) {
if (cpumask_empty(mask))
return -EINVAL;
}
}
flags = affinity ? IRQD_AFFINITY_MANAGED : 0;
mask = NULL;
for (i = 0; i < cnt; i++) {
if (affinity) {
cpu = cpumask_next(cpu, affinity);
if (cpu >= nr_cpu_ids)
cpu = cpumask_first(affinity);
node = cpu_to_node(cpu);
/*
* For single allocations we use the caller provided
* mask otherwise we use the mask of the target cpu
*/
mask = cnt == 1 ? affinity : cpumask_of(cpu);
node = cpu_to_node(cpumask_first(affinity));
mask = affinity;
affinity++;
}
desc = alloc_desc(start + i, node, flags, mask, owner);
if (!desc)
@ -670,9 +669,9 @@ EXPORT_SYMBOL_GPL(irq_free_descs);
* @cnt: Number of consecutive irqs to allocate.
* @node: Preferred node on which the irq descriptor should be allocated
* @owner: Owning module (can be NULL)
* @affinity: Optional pointer to an affinity mask which hints where the
* irq descriptors should be allocated and which default
* affinities to use
* @affinity: Optional pointer to an affinity mask array of size @cnt which
* hints where the irq descriptors should be allocated and which
* default affinities to use
*
* Returns the first irq number or error code
*/

26
kernel/irq/msi.c
View File

@ -18,20 +18,42 @@
/* Temparory solution for building, will be removed later */
#include <linux/pci.h>
struct msi_desc *alloc_msi_entry(struct device *dev)
/**
* alloc_msi_entry - Allocate an initialize msi_entry
* @dev: Pointer to the device for which this is allocated
* @nvec: The number of vectors used in this entry
* @affinity: Optional pointer to an affinity mask array size of @nvec
*
* If @affinity is not NULL then a an affinity array[@nvec] is allocated
* and the affinity masks from @affinity are copied.
*/
struct msi_desc *
alloc_msi_entry(struct device *dev, int nvec, const struct cpumask *affinity)
{
struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL);
struct msi_desc *desc;
desc = kzalloc(sizeof(*desc), GFP_KERNEL);
if (!desc)
return NULL;
INIT_LIST_HEAD(&desc->list);
desc->dev = dev;
desc->nvec_used = nvec;
if (affinity) {
desc->affinity = kmemdup(affinity,
nvec * sizeof(*desc->affinity), GFP_KERNEL);
if (!desc->affinity) {
kfree(desc);
return NULL;
}
}
return desc;
}
void free_msi_entry(struct msi_desc *entry)
{
kfree(entry->affinity);
kfree(entry);
}