Re: [PATCH 17/25] x86_64 irq: Remove the msi assumption that irq == vector

Re: [PATCH 17/25] x86_64 irq: Remove the msi assumption that irq == vector

[Dave Olson]

Eric and I had a brief side discussion, which we thought should
be shared with the other folks interested in this issue.

On Thu, 22 Jun 2006, Eric W. Biederman wrote:
| > Will there any (exported) way for a driver to find out the actual vector
| > corresponding
| > to it's irq?
|
| Not in a good way.  The vector changes at runtime, when the irq
| is migrated between cpus.

Since this has to be handled for MSI in general, it should be possible
to handle it for HyperTransport as well.

Our InfiniPath HT chip has an additional constraint that it needs a callback if
the vector changes, because a register past the config space also
has to change.

Or we have to have a way to disable migration from the driver (probably
not a bad idea anyway, for performance sensitive drivers), or both.

| > If not, this will break the infinipath HTX driver, which needs to
| > program the vector into the HT config space of our chip (something the
| > MSI infrastructure does for PCI and PCI-e, but which doesn't exist for
| > HT, so I do it myself).
|
| Right we need a cousin of msi layer to implement generic HT interrupt
| support.  Otherwise whatever we implement will be prone to break.
|
| The good news is this should be much easier to implement now than
| it was earlier.
|
| > Before the MSI support changed what request_irq() returned, I either
| > had to do some heuristics to figure out the vector, or hack a way into
| > the kernel so I could get the vector.
|
| So far I see what your driver is doing as a hack, who's only excuse

Agreed.

| is that there isn't generic code to do better.  However now that
| this code isn't scheduled to go in before 2.6.19 I expect we can
| have a generic HT layer before this code reaches a stable kernel.
|
| Looking at the ipath_ht400.c driver your code appears to have the side
| effect of always binding to cpu 0, and always in fix physical mode.
| Ok on small systems but probably not what you want on systems with
| multiple instances of this chip.

Definitely not what's wanted for multiple instances on large systems in
all cases (from a cache perspective, multiple adapters interrupting on
the same processor isn't necessarily bad, but beyond 2 per cpu is
a real problem with openib, in terms of available cpu cycles).

| I was thinking earlier because of the on the wire similarities I could
| share msi_ops between the implementations.  But the HT code really
| implements this quite differently much more like an io_apic in it's
| register layout so I guess a ht_ops is needed.

Probably.

Dave Olson
olson@unixfolk.com
http://www.unixfolk.com/dave

Re: [PATCH 17/25] x86_64 irq: Remove the msi assumption that irq == vector

[Eric W. Biederman]

Dave Olson <olson@unixfolk.com> writes:

> Eric and I had a brief side discussion, which we thought should
> be shared with the other folks interested in this issue.

Dave.  Can you play with the patch below?
It is against 2.6.17-mm6.

It is just about a working version of a generic ht irq patch.

No callbacks yet but that should be terribly hard to add.

If you could adapt the ipath driver and give me some feedback
that would be great.

I have another more standard test case that I will be looking at
in just a bit.

Eric


diff --git a/arch/i386/kernel/io_apic.c b/arch/i386/kernel/io_apic.c
index 70b5962..6ab4a33 100644
--- a/arch/i386/kernel/io_apic.c
+++ b/arch/i386/kernel/io_apic.c
@@ -2532,6 +2532,96 @@ struct msi_ops arch_msi_ops = {
 
 #endif /* CONFIG_PCI_MSI */
 
+/*
+ * Hypertransport interrupt support
+ */
+#ifdef CONFIG_HT_IRQ
+
+#ifdef CONFIG_SMP
+
+static void target_ht_irq(unsigned int irq, unsigned int dest)
+{
+	u32 low, high;
+	low  = read_ht_irq_low(irq);
+	high = read_ht_irq_high(irq);
+
+	low  &= ~(HT_IRQ_LOW_DEST_ID_MASK);
+	high &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
+
+	low  |= HT_IRQ_LOW_DEST_ID(dest);
+	high |= HT_IRQ_HIGH_DEST_ID(dest);
+
+	write_ht_irq_low(irq, low);
+	write_ht_irq_high(irq, high);
+}
+
+static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
+{
+	unsigned int dest;
+	cpumask_t tmp;
+	int vector;
+
+	cpus_and(tmp, mask, cpu_online_map);
+	if (cpus_empty(tmp))
+		tmp = TARGET_CPUS;
+
+	cpus_and(mask, tmp, CPU_MASK_ALL);
+
+	dest = cpu_mask_to_apicid(mask);
+
+	target_ht_irq(irq, dest);
+	set_native_irq_info(irq, mask);
+}
+#endif
+
+static struct hw_interrupt_type ht_irq_chip = {
+	.name		= "PCI-HT",
+	.mask		= mask_ht_irq,
+	.unmask		= unmask_ht_irq,
+	.ack		= ack_apic_edge,
+#ifdef CONFIG_SMP
+	.set_affinity	= set_ht_irq_affinity,
+#endif	
+	.retrigger	= ioapic_retrigger_irq,
+};
+
+int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
+{
+	int vector;
+
+	vector = assign_irq_vector(irq, TARGET_CPUS);
+	if (vector >= 0) {
+		u32 low, high;
+		unsigned dest;
+		cpumask_t tmp;
+
+		cpus_clear(tmp);
+		cpu_set(vector >> 8, tmp);
+		dest = cpu_mask_to_apicid(tmp);
+
+		high = 	HT_IRQ_HIGH_DEST_ID(dest);
+
+		low =	HT_IRQ_LOW_BASE | 
+			HT_IRQ_LOW_DEST_ID(dest) | 
+			HT_IRQ_LOW_VECTOR(vector) |
+			((INT_DEST_MODE == 0) ?
+				HT_IRQ_LOW_DM_PHYSICAL :
+				HT_IRQ_LOW_DM_PHYSICAL) |
+			HT_IRQ_LOW_RQEOI_EDGE |
+			((INT_DELIVERY_MODE != dest_LowestPrio) ?
+				HT_IRQ_LOW_MT_FIXED :
+				HT_IRQ_LOW_MT_ARBITRATED) |
+			HT_IRQ_LOW_IRQ_MASKED;
+
+		write_ht_irq_low(irq, low);
+		write_ht_irq_high(irq, high);
+
+		set_irq_chip_and_handler(irq, &ht_irq_chip, handle_edge_irq);
+	}
+	return vector;
+}
+#endif /* CONFIG_HT_IRQ */
+
 /* --------------------------------------------------------------------------
                           ACPI-based IOAPIC Configuration
    -------------------------------------------------------------------------- */
diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c
index 7e46935..9f3e167 100644
--- a/arch/powerpc/sysdev/mpic.c
+++ b/arch/powerpc/sysdev/mpic.c
@@ -249,9 +249,9 @@ static void __init mpic_scan_ht_pic(stru
 	for (pos = readb(devbase + PCI_CAPABILITY_LIST); pos != 0;
 	     pos = readb(devbase + pos + PCI_CAP_LIST_NEXT)) {
 		u8 id = readb(devbase + pos + PCI_CAP_LIST_ID);
-		if (id == PCI_CAP_ID_HT_IRQCONF) {
+		if (id == PCI_CAP_ID_HT) {
 			id = readb(devbase + pos + 3);
-			if (id == 0x80)
+			if (id == HT_CAPTYPE_IRQ)
 				break;
 		}
 	}
diff --git a/arch/x86_64/kernel/io_apic.c b/arch/x86_64/kernel/io_apic.c
index 8d030b4..84eeaf8 100644
--- a/arch/x86_64/kernel/io_apic.c
+++ b/arch/x86_64/kernel/io_apic.c
@@ -42,6 +42,7 @@ #include <asm/acpi.h>
 #include <asm/dma.h>
 #include <asm/nmi.h>
 #include <asm/msidef.h>
+#include <asm/hypertransport.h>
 
 static int assign_irq_vector(int irq, cpumask_t mask);
 
@@ -2093,6 +2094,98 @@ struct msi_ops arch_msi_ops = {
 
 #endif
 
+/*
+ * Hypertransport interrupt support
+ */
+#ifdef CONFIG_HT_IRQ
+
+#ifdef CONFIG_SMP
+
+static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
+{
+	u32 low, high;
+	low  = read_ht_irq_low(irq);
+	high = read_ht_irq_high(irq);
+
+	low  &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
+	high &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
+
+	low  |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
+	high |= HT_IRQ_HIGH_DEST_ID(dest);
+
+	write_ht_irq_low(irq, low);
+	write_ht_irq_high(irq, high);
+}
+
+static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
+{
+	unsigned int dest;
+	cpumask_t tmp;
+	int vector;
+
+	cpus_and(tmp, mask, cpu_online_map);
+	if (cpus_empty(tmp))
+		tmp = TARGET_CPUS;
+
+	cpus_and(mask, tmp, CPU_MASK_ALL);
+
+	vector = assign_irq_vector(irq, mask);
+	if (vector < 0)
+		return;
+
+	cpus_clear(tmp);
+	cpu_set(vector >> 8, tmp);
+	dest = cpu_mask_to_apicid(tmp);
+
+	target_ht_irq(irq, dest, vector & 0xff);
+	set_native_irq_info(irq, mask);
+}
+#endif
+
+static struct hw_interrupt_type ht_irq_chip = {
+	.name		= "PCI-HT",
+	.mask		= mask_ht_irq,
+	.unmask		= unmask_ht_irq,
+	.ack		= ack_apic_edge,
+#ifdef CONFIG_SMP
+	.set_affinity	= set_ht_irq_affinity,
+#endif	
+	.retrigger	= ioapic_retrigger_irq,
+};
+
+int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
+{
+	int vector;
+
+	vector = assign_irq_vector(irq, TARGET_CPUS);
+	if (vector >= 0) {
+		u32 low, high;
+		unsigned dest;
+		cpumask_t tmp;
+
+		cpus_clear(tmp);
+		cpu_set(vector >> 8, tmp);
+		dest = cpu_mask_to_apicid(tmp);
+
+		high = 	HT_IRQ_HIGH_DEST_ID(dest);
+
+		low =	HT_IRQ_LOW_BASE | 
+			HT_IRQ_LOW_DEST_ID(dest) | 
+			HT_IRQ_LOW_VECTOR(vector) |
+			((INT_DEST_MODE == 0) ?
+				HT_IRQ_LOW_DM_PHYSICAL :
+				HT_IRQ_LOW_DM_PHYSICAL) |
+			HT_IRQ_LOW_RQEOI_EDGE |
+			((INT_DELIVERY_MODE != dest_LowestPrio) ?
+				HT_IRQ_LOW_MT_FIXED :
+				HT_IRQ_LOW_MT_ARBITRATED);
+
+		set_irq_chip_and_handler(irq, &ht_irq_chip, handle_edge_irq);
+	}
+	return vector;
+}
+#endif /* CONFIG_HT_IRQ */
+
 /* --------------------------------------------------------------------------
                           ACPI-based IOAPIC Configuration
    -------------------------------------------------------------------------- */
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index 4d762fc..bcae7f3 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -27,3 +27,12 @@ config PCI_DEBUG
 
 	  When in doubt, say N.
 
+config HT_IRQ
+	bool "Interrupts on hypertransport devices"
+	depends on PCI
+	depends on X86_LOCAL_APIC && X86_IO_APIC
+	help
+	   This allows native hypertransport devices to use interrupts.
+
+	   If unsure say Y.
+           
\ No newline at end of file
diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile
index 983d0f8..2752c57 100644
--- a/drivers/pci/Makefile
+++ b/drivers/pci/Makefile
@@ -26,6 +26,7 @@ obj-$(CONFIG_PPC32) += setup-irq.o
 obj-$(CONFIG_PPC64) += setup-bus.o
 obj-$(CONFIG_MIPS) += setup-bus.o setup-irq.o
 obj-$(CONFIG_X86_VISWS) += setup-irq.o
+obj-$(CONFIG_HT_IRQ) += htirq.o
 
 msiobj-y := msi.o
 msiobj-$(CONFIG_IA64) += msi-apic.o
diff --git a/drivers/pci/htirq.c b/drivers/pci/htirq.c
new file mode 100644
index 0000000..56fbab6
--- /dev/null
+++ b/drivers/pci/htirq.c
@@ -0,0 +1,188 @@
+/*
+ * File:	htirq.c
+ * Purpose:	Hypertransport Interrupt Capability
+ *
+ * Copyright (C) 2006 Linux Networx
+ * Copyright (C) Eric Biederman <ebiederman@lnxi.com>
+ */
+
+#include <linux/irq.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/gfp.h>
+
+/* Global ht irq lock.  
+ *
+ * This is needed to serialize access to the data port in hypertransport
+ * irq capability.
+ *
+ * With multiple simultaneous hypertransport irq devices it might pay
+ * to make this more fine grained.  But start with simple, stupid, and correct.
+ */
+static DEFINE_SPINLOCK(ht_irq_lock);
+
+struct ht_irq_cfg {
+	struct pci_dev *dev;
+	unsigned pos;
+	unsigned idx;
+};
+
+void write_ht_irq_low(unsigned int irq, u32 data)
+{
+	struct ht_irq_cfg *cfg = get_irq_data(irq);
+	unsigned long flags;
+	spin_lock_irqsave(&ht_irq_lock, flags);
+	pci_write_config_byte(cfg->dev, cfg->pos + 2, cfg->idx);
+	pci_write_config_dword(cfg->dev, cfg->pos + 4, data);
+	spin_unlock_irqrestore(&ht_irq_lock, flags);
+}
+
+void write_ht_irq_high(unsigned int irq, u32 data)
+{
+	struct ht_irq_cfg *cfg = get_irq_data(irq);
+	unsigned long flags;
+	spin_lock_irqsave(&ht_irq_lock, flags);
+	pci_write_config_byte(cfg->dev, cfg->pos + 2, cfg->idx + 1);
+	pci_write_config_dword(cfg->dev, cfg->pos + 4, data);
+	spin_unlock_irqrestore(&ht_irq_lock, flags);
+}
+
+u32 read_ht_irq_low(unsigned int irq)
+{
+	struct ht_irq_cfg *cfg = get_irq_data(irq);
+	unsigned long flags;
+	u32 data;
+	spin_lock_irqsave(&ht_irq_lock, flags);
+	pci_write_config_byte(cfg->dev, cfg->pos + 2, cfg->idx);
+	pci_read_config_dword(cfg->dev, cfg->pos + 4, &data);
+	spin_unlock_irqrestore(&ht_irq_lock, flags);
+	return data;
+}
+
+
+u32 read_ht_irq_high(unsigned int irq)
+{
+	struct ht_irq_cfg *cfg = get_irq_data(irq);
+	unsigned long flags;
+	u32 data;
+	spin_lock_irqsave(&ht_irq_lock, flags);
+	pci_write_config_byte(cfg->dev, cfg->pos + 2, cfg->idx + 1);
+	pci_read_config_dword(cfg->dev, cfg->pos + 4, &data);
+	spin_unlock_irqrestore(&ht_irq_lock, flags);
+	return data;
+}
+
+void mask_ht_irq(unsigned int irq)
+{
+	struct ht_irq_cfg *cfg;
+	unsigned long flags;
+	u32 data;
+
+	cfg = get_irq_data(irq);
+
+	spin_lock_irqsave(&ht_irq_lock, flags);
+	pci_write_config_byte(cfg->dev, cfg->pos + 2, cfg->idx);
+	pci_read_config_dword(cfg->dev, cfg->pos + 4, &data);
+	data |= 1;
+	pci_write_config_dword(cfg->dev, cfg->pos + 4, data);
+	spin_unlock_irqrestore(&ht_irq_lock, flags);
+}
+
+void unmask_ht_irq(unsigned int irq)
+{
+	struct ht_irq_cfg *cfg;
+	unsigned long flags;
+	u32 data;
+
+	cfg = get_irq_data(irq);
+
+	spin_lock_irqsave(&ht_irq_lock, flags);
+	pci_write_config_byte(cfg->dev, cfg->pos + 2, cfg->idx);
+	pci_read_config_dword(cfg->dev, cfg->pos + 4, &data);
+	data &= ~1;
+	pci_write_config_dword(cfg->dev, cfg->pos + 4, data);
+	spin_unlock_irqrestore(&ht_irq_lock, flags);
+}
+
+/**
+ * ht_create_irq - create an irq and attach it to a device.
+ * @dev: The hypertransport device to find the irq capability on.
+ * @idx: Which of the possible irqs to attach to.
+ *
+ * ht_create_irq is needs to be called for all hypertransport devices
+ * that generate irqs.
+ * 
+ * The irq number of the new irq or a negative error value is returned.
+ */
+int ht_create_irq(struct pci_dev *dev, int idx)
+{
+	struct ht_irq_cfg *cfg;
+	unsigned long flags;
+	u32 data;
+	int max_irq;
+	int pos;
+	int irq;
+	pos = pci_find_capability(dev, PCI_CAP_ID_HT);
+	while (pos) {
+		u8 subtype;
+		pci_read_config_byte(dev, pos + 3, &subtype);
+		if (subtype != HT_CAPTYPE_IRQ)
+			pci_find_next_capability(dev, pos, PCI_CAP_ID_HT);
+	}
+	if (!pos)
+		return -EINVAL;
+
+	/* Verify the idx I want to use is in range */
+	spin_lock_irqsave(&ht_irq_lock, flags);
+	pci_write_config_byte(dev, pos + 2, 1);
+	pci_read_config_dword(dev, pos + 4, &data);
+	spin_unlock_irqrestore(&ht_irq_lock, flags);
+
+	max_irq = (data >> 16) & 0xff;
+	if ( idx > max_irq) 
+		return -EINVAL;
+
+	cfg = kmalloc(sizeof(*cfg), GFP_KERNEL);
+	if (!cfg)
+		return -ENOMEM;
+
+	cfg->dev = dev;
+	cfg->pos = pos;
+	cfg->idx = 0x10 + (idx * 2);
+		
+	irq = create_irq();
+	if (irq < 0) {
+		kfree(cfg);
+		return -EBUSY;
+	}
+	set_irq_data(irq, cfg);
+
+	if (arch_setup_ht_irq(irq, dev) < 0) {
+		ht_destroy_irq(irq);
+		return -EBUSY;
+	}
+
+	return irq;
+}
+
+/**
+ * ht_destroy_irq - destroy an irq created with ht_create_irq
+ *
+ * This reverses ht_create_irq removing the specified irq from
+ * existence.  The irq should be free before this happens.
+ */
+void ht_destroy_irq(unsigned int irq)
+{
+	struct ht_irq_cfg *cfg;
+
+	cfg = get_irq_data(irq);
+	set_irq_chip(irq, NULL);
+	set_irq_data(irq, NULL);
+	destroy_irq(irq);
+
+	kfree(cfg);
+}
+
+EXPORT_SYMBOL(ht_create_irq);
+EXPORT_SYMBOL(ht_destroy_irq);
diff --git a/include/asm-i386/hypertransport.h b/include/asm-i386/hypertransport.h
new file mode 100644
index 0000000..c16c6ff
--- /dev/null
+++ b/include/asm-i386/hypertransport.h
@@ -0,0 +1,42 @@
+#ifndef ASM_HYPERTRANSPORT_H
+#define ASM_HYPERTRANSPORT_H
+
+/*
+ * Constants for x86 Hypertransport Interrupts.
+ */
+
+#define HT_IRQ_LOW_BASE			0xf8000000
+
+#define HT_IRQ_LOW_VECTOR_SHIFT		16
+#define  HT_IRQ_LOW_VECTOR_MASK		0x00ff0000
+#define  HT_IRQ_LOW_VECTOR(v)		(((v) << HT_IRQ_LOW_VECTOR_SHIFT) & HT_IRQ_LOW_VECTOR_MASK)
+
+#define HT_IRQ_LOW_DEST_ID_SHIFT	8
+#define  HT_IRQ_LOW_DEST_ID_MASK	0x0000ff00
+#define  HT_IRQ_LOW_DEST_ID(v)		(((v) << HT_IRQ_LOW_DEST_ID_SHIFT) & HT_IRQ_LOW_DEST_ID_MASK)
+
+#define HT_IRQ_LOW_DM_PHYSICAL		0x0000000
+#define HT_IRQ_LOW_DM_LOGICAL		0x0000040
+
+#define HT_IRQ_LOW_RQEOI_EDGE		0x0000000
+#define HT_IRQ_LOW_RQEOI_LEVEL		0x0000020
+
+
+#define HT_IRQ_LOW_MT_FIXED		0x0000000
+#define HT_IRQ_LOW_MT_ARBITRATED	0x0000004
+#define HT_IRQ_LOW_MT_SMI		0x0000008
+#define HT_IRQ_LOW_MT_NMI		0x000000c
+#define HT_IRQ_LOW_MT_INIT		0x0000010
+#define HT_IRQ_LOW_MT_STARTUP		0x0000014
+#define HT_IRQ_LOW_MT_EXTINT		0x0000018
+#define HT_IRQ_LOW_MT_LINT1		0x000008c
+#define HT_IRQ_LOW_MT_LINT0		0x0000098
+
+#define HT_IRQ_LOW_IRQ_MASKED		0x0000001
+
+
+#define HT_IRQ_HIGH_DEST_ID_SHIFT	0
+#define  HT_IRQ_HIGH_DEST_ID_MASK	0x00ffffff
+#define  HT_IRQ_HIGH_DEST_ID(v)		((((v) >> 8) << HT_IRQ_HIGH_DEST_ID_SHIFT) & HT_IRQ_HIGH_DEST_ID_MASK)
+
+#endif /* ASM_HYPERTRANSPORT_H */
diff --git a/include/asm-x86_64/hypertransport.h b/include/asm-x86_64/hypertransport.h
new file mode 100644
index 0000000..c16c6ff
--- /dev/null
+++ b/include/asm-x86_64/hypertransport.h
@@ -0,0 +1,42 @@
+#ifndef ASM_HYPERTRANSPORT_H
+#define ASM_HYPERTRANSPORT_H
+
+/*
+ * Constants for x86 Hypertransport Interrupts.
+ */
+
+#define HT_IRQ_LOW_BASE			0xf8000000
+
+#define HT_IRQ_LOW_VECTOR_SHIFT		16
+#define  HT_IRQ_LOW_VECTOR_MASK		0x00ff0000
+#define  HT_IRQ_LOW_VECTOR(v)		(((v) << HT_IRQ_LOW_VECTOR_SHIFT) & HT_IRQ_LOW_VECTOR_MASK)
+
+#define HT_IRQ_LOW_DEST_ID_SHIFT	8
+#define  HT_IRQ_LOW_DEST_ID_MASK	0x0000ff00
+#define  HT_IRQ_LOW_DEST_ID(v)		(((v) << HT_IRQ_LOW_DEST_ID_SHIFT) & HT_IRQ_LOW_DEST_ID_MASK)
+
+#define HT_IRQ_LOW_DM_PHYSICAL		0x0000000
+#define HT_IRQ_LOW_DM_LOGICAL		0x0000040
+
+#define HT_IRQ_LOW_RQEOI_EDGE		0x0000000
+#define HT_IRQ_LOW_RQEOI_LEVEL		0x0000020
+
+
+#define HT_IRQ_LOW_MT_FIXED		0x0000000
+#define HT_IRQ_LOW_MT_ARBITRATED	0x0000004
+#define HT_IRQ_LOW_MT_SMI		0x0000008
+#define HT_IRQ_LOW_MT_NMI		0x000000c
+#define HT_IRQ_LOW_MT_INIT		0x0000010
+#define HT_IRQ_LOW_MT_STARTUP		0x0000014
+#define HT_IRQ_LOW_MT_EXTINT		0x0000018
+#define HT_IRQ_LOW_MT_LINT1		0x000008c
+#define HT_IRQ_LOW_MT_LINT0		0x0000098
+
+#define HT_IRQ_LOW_IRQ_MASKED		0x0000001
+
+
+#define HT_IRQ_HIGH_DEST_ID_SHIFT	0
+#define  HT_IRQ_HIGH_DEST_ID_MASK	0x00ffffff
+#define  HT_IRQ_HIGH_DEST_ID(v)		((((v) >> 8) << HT_IRQ_HIGH_DEST_ID_SHIFT) & HT_IRQ_HIGH_DEST_ID_MASK)
+
+#endif /* ASM_HYPERTRANSPORT_H */
diff --git a/include/linux/pci.h b/include/linux/pci.h
index aeebe36..69c55a9 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -676,6 +676,23 @@ extern int msi_register(struct msi_ops *
 
 #endif
 
+#ifdef CONFIG_HT_IRQ
+/* Helper functions.. */
+void write_ht_irq_low(unsigned int irq, u32 data);
+void write_ht_irq_high(unsigned int irq, u32 data);
+u32  read_ht_irq_low(unsigned int irq);
+u32  read_ht_irq_high(unsigned int irq);
+void mask_ht_irq(unsigned int irq);
+void unmask_ht_irq(unsigned int irq);
+
+/* The functions a driver should call */
+int  ht_create_irq(struct pci_dev *dev, int idx);
+void ht_destroy_irq(unsigned int irq);
+
+/* The arch hook for getting things started */
+int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev);
+#endif /* CONFIG_HT_IRQ */
+
 extern void pci_block_user_cfg_access(struct pci_dev *dev);
 extern void pci_unblock_user_cfg_access(struct pci_dev *dev);
 
diff --git a/include/linux/pci_regs.h b/include/linux/pci_regs.h
index 6bce4a2..7a5dcc2 100644
--- a/include/linux/pci_regs.h
+++ b/include/linux/pci_regs.h
@@ -12,6 +12,11 @@
  *	PCI Local Bus Specification
  *	PCI to PCI Bridge Specification
  *	PCI System Design Guide
+ *
+ * 	For hypertransport information, please consult the following manuals 
+ * 	from http://www.hypertranposrt.org
+ *
+ *	The Hypertransport I/O Link Specification
  */
 
 #ifndef LINUX_PCI_REGS_H
@@ -196,7 +201,7 @@ #define  PCI_CAP_ID_SLOTID	0x04	/* Slot 
 #define  PCI_CAP_ID_MSI		0x05	/* Message Signalled Interrupts */
 #define  PCI_CAP_ID_CHSWP	0x06	/* CompactPCI HotSwap */
 #define  PCI_CAP_ID_PCIX	0x07	/* PCI-X */
-#define  PCI_CAP_ID_HT_IRQCONF	0x08	/* HyperTransport IRQ Configuration */
+#define  PCI_CAP_ID_HT		0x08	/* HyperTransport Capability */
 #define  PCI_CAP_ID_VNDR	0x09	/* Vendor specific capability */
 #define  PCI_CAP_ID_SHPC 	0x0C	/* PCI Standard Hot-Plug Controller */
 #define  PCI_CAP_ID_EXP 	0x10	/* PCI Express */
@@ -447,4 +452,20 @@ #define  PCI_PWR_DATA_RAIL(x)	(((x) >> 1
 #define PCI_PWR_CAP		12	/* Capability */
 #define  PCI_PWR_CAP_BUDGET(x)	((x) & 1)	/* Included in system budget */
 
+/* Hypertransport sub capability types */
+#define HT_CAPTYPE_SLAVE	0x00	/* Slave/Primary link configuration */
+#define HT_CAPTYPE_HOST		0x20	/* Host/Secondary link configuration */
+#define HT_CAPTYPE_IRQ		0x80	/* IRQ Configuration */
+#define HT_CAPTYPE_REMAPPING_40	0xA0	/* 40 bit address remapping */
+#define HT_CAPTYPE_REMAPPING_64 0xA2	/* 64 bit address remapping */
+#define HT_CAPTYPE_UNITID_CLUMP	0x90	/* Unit ID clumping */
+#define HT_CAPTYPE_EXTCONF	0x98	/* Extended Configuration Space Access */
+#define HT_CAPTYPE_MSI_MAPPING	0xA8	/* MSI Mapping Capability */
+#define HT_CAPTYPE_DIRECT_ROUTE	0xB0	/* Direct routing configuration */
+#define HT_CAPTYPE_VCSET	0xB8	/* Virtual Channel configuration */
+#define HT_CAPTYPE_ERROR_RETRY	0xC0	/* Retry on error configuration */
+#define HT_CAPTYPE_GEN3		0xD0	/* Generation 3 hypertransport configuration */
+#define HT_CAPTYPE_PM		0xE0	/* Hypertransport powermanagement configuration */
+
+
 #endif /* LINUX_PCI_REGS_H */

Re: [PATCH 17/25] x86_64 irq: Remove the msi assumption that irq == vector

[Dave Olson]

On Fri, 7 Jul 2006, Eric W. Biederman wrote:

| Dave Olson <olson@unixfolk.com> writes:
| 
| > Eric and I had a brief side discussion, which we thought should
| > be shared with the other folks interested in this issue.
| 
| Dave.  Can you play with the patch below?
| It is against 2.6.17-mm6.
| 
| It is just about a working version of a generic ht irq patch.

I'll try to do this by Monday.

Dave Olson
olson@unixfolk.com
http://www.unixfolk.com/dave

[PATCH 0/25] Decouple IRQ issues (MSI, i386, x86_64, ia64)

[Eric W. Biederman]

The following patchset is against 2.6.17-rc6-mm2.
It was the only easy place I could get everyones work who has been
touching relevant code.

The primary aim of this patch is to remove maintenances problems caused
by the irq infrastructure.  The two big issues I address are an
artificially small cap on the number of irqs, and that MSI assumes
vector == irq.  My primary focus is on x86_64 but I have touched
other architectures where necessary to keep them from breaking.

- To increase the number of irqs I modify the code to look at
  the (cpu, vector) pair instead of just  looking at the vector.

  With a large number of irqs available systems with a large irq
  count no longer need to compress their irq numbers to fit.
  Removing a lot of brittle special cases.

  For acpi guys the result is that irq == gsi.

- Addressing the fact that MSI assumes irq == vector takes a few more
  patches.  But suffice it to say when I am done none of the generic
  irq code even knows what a vector is.

In quick testing on a large Unisys x86_64 machine we stumbled over at
least one driver that assumed that NR_IRQS could always fit into an 8
bit number.  This driver is clearly buggy today.  But this has become
a class of bugs that it is now much easier to hit. 

I've done my best but if this patchset wasn't perfect it won't
surprise me.  But I'm pretty certain I have succeeded in decoupling
any fixes should be small and well contained.

Eric

[PATCH 1/25] irq: Convert the move_irq flag from a 32bit word to a single bit

[Eric W. Biederman]

This is a minor space optimization.  In practice I don't think
this has any affect because of our alignment constraints and
the other fields but there is not point in chewing up an
uncessary word and since we already read the flag field
this should improve the cache hit ratio of the irq handler.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 include/linux/irq.h    |    5 +++--
 kernel/irq/migration.c |    6 +++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/include/linux/irq.h b/include/linux/irq.h
index c684bab..1ad1acb 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -45,6 +45,9 @@ #define IRQ_NOREQUEST	1024	/* IRQ cannot
 #define IRQ_NOAUTOEN	2048	/* IRQ will not be enabled on request irq */
 #define IRQ_DELAYED_DISABLE \
 			4096	/* IRQ disable (masking) happens delayed. */
+#if defined(CONFIG_GENERIC_PENDING_IRQ) || defined(CONFIG_IRQBALANCE)
+# define IRQ_MOVE_PENDING 8192	/* need to re-target IRQ destination */
+#endif
 
 /*
  * IRQ types, see also include/linux/interrupt.h
@@ -130,7 +133,6 @@ #endif
  * @affinity:		IRQ affinity on SMP
  * @cpu:		cpu index useful for balancing
  * @pending_mask:	pending rebalanced interrupts
- * @move_irq:		need to re-target IRQ destination
  * @dir:		/proc/irq/ procfs entry
  * @affinity_entry:	/proc/irq/smp_affinity procfs entry on SMP
  *
@@ -156,7 +158,6 @@ #ifdef CONFIG_SMP
 #endif
 #if defined(CONFIG_GENERIC_PENDING_IRQ) || defined(CONFIG_IRQBALANCE)
 	cpumask_t		pending_mask;
-	unsigned int		move_irq;	/* need to re-target IRQ dest */
 #endif
 #ifdef CONFIG_PROC_FS
 	struct proc_dir_entry *dir;
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c
index a57ebe9..9b234df 100644
--- a/kernel/irq/migration.c
+++ b/kernel/irq/migration.c
@@ -7,7 +7,7 @@ void set_pending_irq(unsigned int irq, c
 	unsigned long flags;
 
 	spin_lock_irqsave(&desc->lock, flags);
-	desc->move_irq = 1;
+	desc->status |= IRQ_MOVE_PENDING;
 	irq_desc[irq].pending_mask = mask;
 	spin_unlock_irqrestore(&desc->lock, flags);
 }
@@ -17,7 +17,7 @@ void move_native_irq(int irq)
 	struct irq_desc *desc = irq_desc + irq;
 	cpumask_t tmp;
 
-	if (likely(!desc->move_irq))
+	if (likely(!(desc->status & IRQ_MOVE_PENDING)))
 		return;
 
 	/*
@@ -28,7 +28,7 @@ void move_native_irq(int irq)
 		return;
 	}
 
-	desc->move_irq = 0;
+	desc->status &= ~IRQ_MOVE_PENDING;
 
 	if (unlikely(cpus_empty(irq_desc[irq].pending_mask)))
 		return;
-- 
1.4.0.gc07e

[PATCH 2/25] irq: Add moved_masked_irq

[Eric W. Biederman]

Currently move_native_irq disables and renables the irq we are
migrating to ensure we don't take that irq when we are actually
doing the migration operation.  Disabling the irq needs to
happen but sometimes doing the work is move_native_irq is too late.

On x86 with ioapics the irq move sequences needs to be:
edge_triggered:
  mask irq.
  move irq.
  unmask irq.
  ack irq.
level_triggered:
  mask irq.
  ack irq.
  move irq.
  unmask irq.

We can esasily perform the edge triggered sequence, with the current
defintion of move_native_irq.  However the level triggered case does
not map well.  For that I have added move_masked_irq, to allow
me to disable the irqs around both the ack and the move.

Q: Why have we not seen this problem earlier?

A: The only symptom I have been able to reproduce is that if we change
   the vector before acknowleding an irq the wrong irq is acknowledged.
   Since we currently are not reprogramming the irq vector during
   migration no problems show up.

   We have to mask the irq before we acknowledge the irq or else we could
   hit a window where an irq is asserted just before we acknowledge it.

   Edge triggered irqs do not have this problem because acknowledgements
   do not propogate in the same way.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 include/linux/irq.h    |    6 ++++++
 kernel/irq/migration.c |   28 +++++++++++++++++++++-------
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/include/linux/irq.h b/include/linux/irq.h
index 1ad1acb..b79d178 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -200,6 +200,7 @@ #if defined(CONFIG_GENERIC_PENDING_IRQ) 
 
 void set_pending_irq(unsigned int irq, cpumask_t mask);
 void move_native_irq(int irq);
+void move_masked_irq(int irq);
 
 #ifdef CONFIG_PCI_MSI
 /*
@@ -241,6 +242,10 @@ static inline void move_native_irq(int i
 {
 }
 
+static inline void move_masked_irq(int irq)
+{
+}
+
 static inline void set_pending_irq(unsigned int irq, cpumask_t mask)
 {
 }
@@ -256,6 +261,7 @@ #else /* CONFIG_SMP */
 
 #define move_irq(x)
 #define move_native_irq(x)
+#define move_masked_irq(x)
 
 #endif /* CONFIG_SMP */
 
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c
index 9b234df..4baa3bb 100644
--- a/kernel/irq/migration.c
+++ b/kernel/irq/migration.c
@@ -12,7 +12,7 @@ void set_pending_irq(unsigned int irq, c
 	spin_unlock_irqrestore(&desc->lock, flags);
 }
 
-void move_native_irq(int irq)
+void move_masked_irq(int irq)
 {
 	struct irq_desc *desc = irq_desc + irq;
 	cpumask_t tmp;
@@ -48,15 +48,29 @@ void move_native_irq(int irq)
 	 * when an active trigger is comming in. This could
 	 * cause some ioapics to mal-function.
 	 * Being paranoid i guess!
+	 *
+	 * For correct operation this depends on the caller
+	 * masking the irqs.
 	 */
 	if (likely(!cpus_empty(tmp))) {
-		if (likely(!(desc->status & IRQ_DISABLED)))
-			desc->chip->disable(irq);
-
 		desc->chip->set_affinity(irq,tmp);
-
-		if (likely(!(desc->status & IRQ_DISABLED)))
-			desc->chip->enable(irq);
 	}
 	cpus_clear(irq_desc[irq].pending_mask);
 }
+
+void move_native_irq(int irq)
+{
+	struct irq_desc *desc = irq_desc + irq;
+
+	if (likely(!(desc->status & IRQ_MOVE_PENDING)))
+		return;
+
+	if (likely(!(desc->status & IRQ_DISABLED)))
+		desc->chip->disable(irq);
+
+	move_masked_irq(irq);
+
+	if (likely(!(desc->status & IRQ_DISABLED)))
+		desc->chip->enable(irq);
+}
+
-- 
1.4.0.gc07e

[PATCH 3/25] x86_64 irq: Reenable migrating irqs to other cpus.

[Eric W. Biederman]

In the latest changes the code for migrating x86_64 irqs was dropped.
This reads it in a fashion that will work even if we change the
vector on level triggered irqs when we migrate them.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 arch/x86_64/kernel/io_apic.c |   31 ++++++++++++++++++++++++++++---
 1 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/arch/x86_64/kernel/io_apic.c b/arch/x86_64/kernel/io_apic.c
index 2e9f1cf..f50be45 100644
--- a/arch/x86_64/kernel/io_apic.c
+++ b/arch/x86_64/kernel/io_apic.c
@@ -1572,18 +1572,43 @@ static int ioapic_retrigger_vector(unsig
  * races.
  */
 
-static void ack_apic(unsigned int vector)
+static void ack_apic(unsigned int irq)
 {
 	ack_APIC_irq();
 }
 
+static void ack_apic_edge(unsigned int irq)
+{
+	move_native_irq(irq);
+	ack_APIC_irq();
+}
+
+static void ack_apic_level(unsigned int irq)
+{
+	int do_unmask_irq = 0;
+	/* If we are moving the irq we need to mask it */
+	if (unlikely(irq_desc[irq].status & IRQ_MOVE_PENDING)) {
+		do_unmask_irq = 1;
+		mask_IO_APIC_irq(irq);
+	}
+	/* We must acknowledge the irq before we move it
+	 * or the acknowledge will not propogate properly.
+	 */
+	ack_APIC_irq();
+
+	/* Now we can move and renable the irq */
+	move_masked_irq(irq);
+	if (unlikely(do_unmask_irq))
+		unmask_IO_APIC_irq(irq);
+}
+
 static struct irq_chip ioapic_chip __read_mostly = {
 	.name 		= "IO-APIC",
 	.startup 	= startup_ioapic_vector,
 	.mask	 	= mask_ioapic_vector,
 	.unmask	 	= unmask_ioapic_vector,
-	.ack 		= ack_apic,
-	.eoi 		= ack_apic,
+	.ack 		= ack_apic_edge,
+	.eoi 		= ack_apic_level,
 #ifdef CONFIG_SMP
 	.set_affinity 	= set_ioapic_affinity_vector,
 #endif
-- 
1.4.0.gc07e

[PATCH 4/25] msi: Simplify msi enable and disable.

[Eric W. Biederman]

The problem.  Because the disable routines leave the msi interrupts in all
sorts of half enabled states the enable routines become impossible
to implement correctly, and almost impossible to understand.

Simplifing this allows me to simply kill the buggy reroute_msix_table,
and generally makes the code more maintainable.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 drivers/pci/msi.c |  122 +++++++----------------------------------------------
 1 files changed, 16 insertions(+), 106 deletions(-)

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 76d023d..c1c93f0 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -915,7 +915,6 @@ int pci_enable_msi(struct pci_dev* dev)
 {
 	struct pci_bus *bus;
 	int pos, temp, status = -EINVAL;
-	u16 control;
 
 	if (!pci_msi_enable || !dev)
  		return status;
@@ -937,27 +936,8 @@ int pci_enable_msi(struct pci_dev* dev)
 	if (!pos)
 		return -EINVAL;
 
-	if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
-		/* Lookup Sucess */
-		unsigned long flags;
+	BUG_ON(!msi_lookup_vector(dev, PCI_CAP_ID_MSI));
 
-		pci_read_config_word(dev, msi_control_reg(pos), &control);
-		if (control & PCI_MSI_FLAGS_ENABLE)
-			return 0;	/* Already in MSI mode */
-		spin_lock_irqsave(&msi_lock, flags);
-		if (!vector_irq[dev->irq]) {
-			msi_desc[dev->irq]->msi_attrib.state = 0;
-			vector_irq[dev->irq] = -1;
-			nr_released_vectors--;
-			spin_unlock_irqrestore(&msi_lock, flags);
-			status = msi_register_init(dev, msi_desc[dev->irq]);
-			if (status == 0)
-				enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
-			return status;
-		}
-		spin_unlock_irqrestore(&msi_lock, flags);
-		dev->irq = temp;
-	}
 	/* Check whether driver already requested for MSI-X vectors */
 	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
 	if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
@@ -999,6 +979,8 @@ void pci_disable_msi(struct pci_dev* dev
 	if (!(control & PCI_MSI_FLAGS_ENABLE))
 		return;
 
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+
 	spin_lock_irqsave(&msi_lock, flags);
 	entry = msi_desc[dev->irq];
 	if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) {
@@ -1012,14 +994,12 @@ void pci_disable_msi(struct pci_dev* dev
 		       pci_name(dev), dev->irq);
 		BUG_ON(entry->msi_attrib.state > 0);
 	} else {
-		vector_irq[dev->irq] = 0; /* free it */
-		nr_released_vectors++;
 		default_vector = entry->msi_attrib.default_vector;
 		spin_unlock_irqrestore(&msi_lock, flags);
+		msi_free_vector(dev, dev->irq, 0);
+
 		/* Restore dev->irq to its default pin-assertion vector */
 		dev->irq = default_vector;
-		disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI),
-					PCI_CAP_ID_MSI);
 	}
 }
 
@@ -1067,57 +1047,6 @@ static int msi_free_vector(struct pci_de
 	return 0;
 }
 
-static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec)
-{
-	int vector = head, tail = 0;
-	int i, j = 0, nr_entries = 0;
-	void __iomem *base;
-	unsigned long flags;
-
-	spin_lock_irqsave(&msi_lock, flags);
-	while (head != tail) {
-		nr_entries++;
-		tail = msi_desc[vector]->link.tail;
-		if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr)
-			j = vector;
-		vector = tail;
-	}
-	if (*nvec > nr_entries) {
-		spin_unlock_irqrestore(&msi_lock, flags);
-		*nvec = nr_entries;
-		return -EINVAL;
-	}
-	vector = ((j > 0) ? j : head);
-	for (i = 0; i < *nvec; i++) {
-		j = msi_desc[vector]->msi_attrib.entry_nr;
-		msi_desc[vector]->msi_attrib.state = 0;	/* Mark it not active */
-		vector_irq[vector] = -1;		/* Mark it busy */
-		nr_released_vectors--;
-		entries[i].vector = vector;
-		if (j != (entries + i)->entry) {
-			base = msi_desc[vector]->mask_base;
-			msi_desc[vector]->msi_attrib.entry_nr =
-				(entries + i)->entry;
-			writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
-				PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base +
-				(entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
-				PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
-			writel(	readl(base + j * PCI_MSIX_ENTRY_SIZE +
-				PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base +
-				(entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
-				PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
-			writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE +
-				PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector,
-				base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE +
-				PCI_MSIX_ENTRY_DATA_OFFSET);
-		}
-		vector = msi_desc[vector]->link.tail;
-	}
-	spin_unlock_irqrestore(&msi_lock, flags);
-
-	return 0;
-}
-
 /**
  * pci_enable_msix - configure device's MSI-X capability structure
  * @dev: pointer to the pci_dev data structure of MSI-X device function
@@ -1160,9 +1089,6 @@ int pci_enable_msix(struct pci_dev* dev,
  		return -EINVAL;
 
 	pci_read_config_word(dev, msi_control_reg(pos), &control);
-	if (control & PCI_MSIX_FLAGS_ENABLE)
-		return -EINVAL;			/* Already in MSI-X mode */
-
 	nr_entries = multi_msix_capable(control);
 	if (nvec > nr_entries)
 		return -EINVAL;
@@ -1177,19 +1103,8 @@ int pci_enable_msix(struct pci_dev* dev,
 		}
 	}
 	temp = dev->irq;
-	if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
-		/* Lookup Sucess */
-		nr_entries = nvec;
-		/* Reroute MSI-X table */
-		if (reroute_msix_table(dev->irq, entries, &nr_entries)) {
-			/* #requested > #previous-assigned */
-			dev->irq = temp;
-			return nr_entries;
-		}
-		dev->irq = temp;
-		enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
-		return 0;
-	}
+	BUG_ON(!msi_lookup_vector(dev, PCI_CAP_ID_MSIX));
+
 	/* Check whether driver already requested for MSI vector */
    	if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 &&
 		!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
@@ -1248,37 +1163,32 @@ void pci_disable_msix(struct pci_dev* de
 	if (!(control & PCI_MSIX_FLAGS_ENABLE))
 		return;
 
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+
 	temp = dev->irq;
 	if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
 		int state, vector, head, tail = 0, warning = 0;
 		unsigned long flags;
 
 		vector = head = dev->irq;
-		spin_lock_irqsave(&msi_lock, flags);
+		dev->irq = temp;			/* Restore pin IRQ */
 		while (head != tail) {
+			spin_lock_irqsave(&msi_lock, flags);
 			state = msi_desc[vector]->msi_attrib.state;
+			tail = msi_desc[vector]->link.tail;
+			spin_unlock_irqrestore(&msi_lock, flags);
 			if (state)
 				warning = 1;
-			else {
-				vector_irq[vector] = 0; /* free it */
-				nr_released_vectors++;
-			}
-			tail = msi_desc[vector]->link.tail;
+			else if (vector != head)	/* Release MSI-X vector */
+				msi_free_vector(dev, vector, 0);
 			vector = tail;
 		}
-		spin_unlock_irqrestore(&msi_lock, flags);
+		msi_free_vector(dev, vector, 0);
 		if (warning) {
-			dev->irq = temp;
 			printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without "
 			       "free_irq() on all MSI-X vectors\n",
 			       pci_name(dev));
 			BUG_ON(warning > 0);
-		} else {
-			dev->irq = temp;
-			disable_msi_mode(dev,
-				pci_find_capability(dev, PCI_CAP_ID_MSIX),
-				PCI_CAP_ID_MSIX);
-
 		}
 	}
 }
-- 
1.4.0.gc07e

[PATCH 5/25] msi: Make the msi boolean tests return either 0 or 1.

[Eric W. Biederman]

This allows the output of the msi tests to be stored directly
in a bit field.  If you don't do this a value greater than
one will be truncated and become 0.  Changing true to false
with bizare consequences.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 drivers/pci/msi.h |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/pci/msi.h b/drivers/pci/msi.h
index 56951c3..9b31d4c 100644
--- a/drivers/pci/msi.h
+++ b/drivers/pci/msi.h
@@ -110,8 +110,8 @@ #define multi_msi_capable(control) \
 	(1 << ((control & PCI_MSI_FLAGS_QMASK) >> 1))
 #define multi_msi_enable(control, num) \
 	control |= (((num >> 1) << 4) & PCI_MSI_FLAGS_QSIZE);
-#define is_64bit_address(control)	(control & PCI_MSI_FLAGS_64BIT)
-#define is_mask_bit_support(control)	(control & PCI_MSI_FLAGS_MASKBIT)
+#define is_64bit_address(control)	(!!(control & PCI_MSI_FLAGS_64BIT))
+#define is_mask_bit_support(control)	(!!(control & PCI_MSI_FLAGS_MASKBIT))
 #define msi_enable(control, num) multi_msi_enable(control, num); \
 	control |= PCI_MSI_FLAGS_ENABLE
 
-- 
1.4.0.gc07e

[PATCH 6/25] msi: Implement helper functions read_msi_msg and write_msi_msg.

[Eric W. Biederman]

In support of this I also add a struct msi_msg that captures
the the two address and one data field ina typical msi message,
and I remember the pos and if the address is 64bit in
struct msi_desc.

This makes the code a little more readable and easier to maintain,
and paves the way to further simplfications.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 drivers/pci/msi.c   |  195 +++++++++++++++++++++++++--------------------------
 drivers/pci/msi.h   |    9 +-
 include/linux/pci.h |    6 ++
 3 files changed, 104 insertions(+), 106 deletions(-)

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index c1c93f0..e9db6c5 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -94,63 +94,100 @@ static void msi_set_mask_bit(unsigned in
 	}
 }
 
-#ifdef CONFIG_SMP
-static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask)
+static void read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
 {
-	struct msi_desc *entry;
-	u32 address_hi, address_lo;
-	unsigned int irq = vector;
-	unsigned int dest_cpu = first_cpu(cpu_mask);
-
-	entry = (struct msi_desc *)msi_desc[vector];
-	if (!entry || !entry->dev)
-		return;
+	switch(entry->msi_attrib.type) {
+	case PCI_CAP_ID_MSI:
+	{
+		struct pci_dev *dev = entry->dev;
+		int pos = entry->msi_attrib.pos;
+		u16 data;
+
+		pci_read_config_dword(dev, msi_lower_address_reg(pos),
+					&msg->address_lo);
+		if (entry->msi_attrib.is_64) {
+			pci_read_config_dword(dev, msi_upper_address_reg(pos),
+						&msg->address_hi);
+			pci_read_config_word(dev, msi_data_reg(pos, 1), &data);
+		} else {
+			msg->address_hi = 0;
+			pci_read_config_word(dev, msi_data_reg(pos, 1), &data);
+		}
+		msg->data = data;
+		break;
+	}
+	case PCI_CAP_ID_MSIX:
+	{
+		void __iomem *base;
+		base = entry->mask_base + 
+			entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
+ 
+		msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
+		msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
+		msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET);
+ 		break;
+ 	}
+ 	default:
+		BUG();
+	}
+}
 
+static void write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
+{
 	switch (entry->msi_attrib.type) {
 	case PCI_CAP_ID_MSI:
 	{
-		int pos = pci_find_capability(entry->dev, PCI_CAP_ID_MSI);
-
-		if (!pos)
-			return;
-
-		pci_read_config_dword(entry->dev, msi_upper_address_reg(pos),
-			&address_hi);
-		pci_read_config_dword(entry->dev, msi_lower_address_reg(pos),
-			&address_lo);
-
-		msi_ops->target(vector, dest_cpu, &address_hi, &address_lo);
-
-		pci_write_config_dword(entry->dev, msi_upper_address_reg(pos),
-			address_hi);
-		pci_write_config_dword(entry->dev, msi_lower_address_reg(pos),
-			address_lo);
-		set_native_irq_info(irq, cpu_mask);
+		struct pci_dev *dev = entry->dev;
+		int pos = entry->msi_attrib.pos;
+
+		pci_write_config_dword(dev, msi_lower_address_reg(pos), 
+					msg->address_lo);
+		if (entry->msi_attrib.is_64) {
+			pci_write_config_dword(dev, msi_upper_address_reg(pos), 
+						msg->address_hi);
+			pci_write_config_word(dev, msi_data_reg(pos, 1), 
+						msg->data);
+		} else {
+			pci_write_config_word(dev, msi_data_reg(pos, 0), 
+						msg->data);
+		}
 		break;
 	}
 	case PCI_CAP_ID_MSIX:
 	{
-		int offset_hi =
-			entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
-				PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET;
-		int offset_lo =
-			entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
-				PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET;
-
-		address_hi = readl(entry->mask_base + offset_hi);
-		address_lo = readl(entry->mask_base + offset_lo);
-
-		msi_ops->target(vector, dest_cpu, &address_hi, &address_lo);
-
-		writel(address_hi, entry->mask_base + offset_hi);
-		writel(address_lo, entry->mask_base + offset_lo);
-		set_native_irq_info(irq, cpu_mask);
+		void __iomem *base;
+		base = entry->mask_base + 
+			entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
+
+		writel(msg->address_lo,
+			base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
+		writel(msg->address_hi,
+			base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
+		writel(msg->data, base + PCI_MSIX_ENTRY_DATA_OFFSET);
 		break;
 	}
 	default:
-		break;
+		BUG();
 	}
 }
+
+#ifdef CONFIG_SMP
+static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask)
+{
+	struct msi_desc *entry;
+	struct msi_msg msg;
+	unsigned int irq = vector;
+	unsigned int dest_cpu = first_cpu(cpu_mask);
+
+	entry = (struct msi_desc *)msi_desc[vector];
+	if (!entry || !entry->dev)
+		return;
+
+	read_msi_msg(entry, &msg);
+	msi_ops->target(vector, dest_cpu, &msg.address_hi, &msg.address_lo);
+	write_msi_msg(entry, &msg);
+	set_native_irq_info(irq, cpu_mask);
+}
 #else
 #define set_msi_affinity NULL
 #endif /* CONFIG_SMP */
@@ -614,23 +651,10 @@ int pci_save_msix_state(struct pci_dev *
 
 	vector = head = dev->irq;
 	while (head != tail) {
-		int j;
-		void __iomem *base;
 		struct msi_desc *entry;
 
 		entry = msi_desc[vector];
-		base = entry->mask_base;
-		j = entry->msi_attrib.entry_nr;
-
-		entry->address_lo_save =
-			readl(base + j * PCI_MSIX_ENTRY_SIZE +
-			      PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
-		entry->address_hi_save =
-			readl(base + j * PCI_MSIX_ENTRY_SIZE +
-			      PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
-		entry->data_save =
-			readl(base + j * PCI_MSIX_ENTRY_SIZE +
-			      PCI_MSIX_ENTRY_DATA_OFFSET);
+		read_msi_msg(entry, &entry->msg_save);
 
 		tail = msi_desc[vector]->link.tail;
 		vector = tail;
@@ -647,8 +671,6 @@ void pci_restore_msix_state(struct pci_d
 	u16 save;
 	int pos;
 	int vector, head, tail = 0;
-	void __iomem *base;
-	int j;
 	struct msi_desc *entry;
 	int temp;
 	struct pci_cap_saved_state *save_state;
@@ -671,18 +693,7 @@ void pci_restore_msix_state(struct pci_d
 	vector = head = dev->irq;
 	while (head != tail) {
 		entry = msi_desc[vector];
-		base = entry->mask_base;
-		j = entry->msi_attrib.entry_nr;
-
-		writel(entry->address_lo_save,
-			base + j * PCI_MSIX_ENTRY_SIZE +
-			PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
-		writel(entry->address_hi_save,
-			base + j * PCI_MSIX_ENTRY_SIZE +
-			PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
-		writel(entry->data_save,
-			base + j * PCI_MSIX_ENTRY_SIZE +
-			PCI_MSIX_ENTRY_DATA_OFFSET);
+		write_msi_msg(entry, &entry->msg_save);
 
 		tail = msi_desc[vector]->link.tail;
 		vector = tail;
@@ -697,29 +708,19 @@ #endif
 static int msi_register_init(struct pci_dev *dev, struct msi_desc *entry)
 {
 	int status;
-	u32 address_hi;
-	u32 address_lo;
-	u32 data;
+	struct msi_msg msg;
 	int pos, vector = dev->irq;
 	u16 control;
 
-   	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	pos = entry->msi_attrib.pos;
 	pci_read_config_word(dev, msi_control_reg(pos), &control);
 
 	/* Configure MSI capability structure */
-	status = msi_ops->setup(dev, vector, &address_hi, &address_lo, &data);
+	status = msi_ops->setup(dev, vector, &msg.address_hi, &msg.address_lo, &msg.data);
 	if (status < 0)
 		return status;
 
-	pci_write_config_dword(dev, msi_lower_address_reg(pos), address_lo);
-	if (is_64bit_address(control)) {
-		pci_write_config_dword(dev,
-			msi_upper_address_reg(pos), address_hi);
-		pci_write_config_word(dev,
-			msi_data_reg(pos, 1), data);
-	} else
-		pci_write_config_word(dev,
-			msi_data_reg(pos, 0), data);
+	write_msi_msg(entry, &msg);
 	if (entry->msi_attrib.maskbit) {
 		unsigned int maskbits, temp;
 		/* All MSIs are unmasked by default, Mask them all */
@@ -769,9 +770,11 @@ static int msi_capability_init(struct pc
 	entry->link.tail = vector;
 	entry->msi_attrib.type = PCI_CAP_ID_MSI;
 	entry->msi_attrib.state = 0;			/* Mark it not active */
+	entry->msi_attrib.is_64 = is_64bit_address(control);
 	entry->msi_attrib.entry_nr = 0;
 	entry->msi_attrib.maskbit = is_mask_bit_support(control);
 	entry->msi_attrib.default_vector = dev->irq;	/* Save IOAPIC IRQ */
+	entry->msi_attrib.pos = pos;
 	dev->irq = vector;
 	entry->dev = dev;
 	if (is_mask_bit_support(control)) {
@@ -809,9 +812,7 @@ static int msix_capability_init(struct p
 				struct msix_entry *entries, int nvec)
 {
 	struct msi_desc *head = NULL, *tail = NULL, *entry = NULL;
-	u32 address_hi;
-	u32 address_lo;
-	u32 data;
+	struct msi_msg msg;
 	int status;
 	int vector, pos, i, j, nr_entries, temp = 0;
 	unsigned long phys_addr;
@@ -848,9 +849,11 @@ static int msix_capability_init(struct p
  		entries[i].vector = vector;
 		entry->msi_attrib.type = PCI_CAP_ID_MSIX;
  		entry->msi_attrib.state = 0;		/* Mark it not active */
+		entry->msi_attrib.is_64 = 1;
 		entry->msi_attrib.entry_nr = j;
 		entry->msi_attrib.maskbit = 1;
 		entry->msi_attrib.default_vector = dev->irq;
+		entry->msi_attrib.pos = pos;
 		entry->dev = dev;
 		entry->mask_base = base;
 		if (!head) {
@@ -869,21 +872,13 @@ static int msix_capability_init(struct p
 		irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
 		/* Configure MSI-X capability structure */
 		status = msi_ops->setup(dev, vector,
-					&address_hi,
-					&address_lo,
-					&data);
+					&msg.address_hi,
+					&msg.address_lo,
+					&msg.data);
 		if (status < 0)
 			break;
 
-		writel(address_lo,
-			base + j * PCI_MSIX_ENTRY_SIZE +
-			PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
-		writel(address_hi,
-			base + j * PCI_MSIX_ENTRY_SIZE +
-			PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
-		writel(data,
-			base + j * PCI_MSIX_ENTRY_SIZE +
-			PCI_MSIX_ENTRY_DATA_OFFSET);
+		write_msi_msg(entry, &msg);
 		attach_msi_entry(entry, vector);
 	}
 	if (i != nvec) {
diff --git a/drivers/pci/msi.h b/drivers/pci/msi.h
index 9b31d4c..62f61b6 100644
--- a/drivers/pci/msi.h
+++ b/drivers/pci/msi.h
@@ -130,10 +130,10 @@ struct msi_desc {
 		__u8	type	: 5; 	/* {0: unused, 5h:MSI, 11h:MSI-X} */
 		__u8	maskbit	: 1; 	/* mask-pending bit supported ?   */
 		__u8	state	: 1; 	/* {0: free, 1: busy}		  */
-		__u8	reserved: 1; 	/* reserved			  */
+		__u8	is_64	: 1;	/* Address size: 0=32bit 1=64bit  */
 		__u8	entry_nr;    	/* specific enabled entry 	  */
 		__u8	default_vector; /* default pre-assigned vector    */
-		__u8	unused; 	/* formerly unused destination cpu*/
+		__u8	pos;	 	/* Location of the msi capability */
 	}msi_attrib;
 
 	struct {
@@ -146,10 +146,7 @@ struct msi_desc {
 
 #ifdef CONFIG_PM
 	/* PM save area for MSIX address/data */
-
-	u32	address_hi_save;
-	u32	address_lo_save;
-	u32	data_save;
+	struct msi_msg msg_save;
 #endif
 };
 
diff --git a/include/linux/pci.h b/include/linux/pci.h
index e36e3d5..c7be27b 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -591,6 +591,12 @@ struct msix_entry {
 	u16	entry;	/* driver uses to specify entry, OS writes */
 };
 
+struct msi_msg {
+	u32	address_lo;	/* low 32 bits of msi message address */
+	u32	address_hi;	/* high 32 bits of msi message address */
+	u32	data;		/* 16 bits of msi message data */
+};
+
 #ifndef CONFIG_PCI_MSI
 static inline void pci_scan_msi_device(struct pci_dev *dev) {}
 static inline int pci_enable_msi(struct pci_dev *dev) {return -1;}
-- 
1.4.0.gc07e

[PATCH 7/25] msi: Refactor the msi_ops.

[Eric W. Biederman]

The current msi_ops are short sighted in a number of ways, this
patch attempts to fix the glaring deficiences.

- Report in msi_ops if a 64bit address is needed in the msi message,
  so we can fail 32bit only msi structures.

- Send and receive a full struct msi_msg in both setup and target.
  This is a little cleaner and allows for architectures that need
  to modify the data to retarget the msi interrupt to a different cpu.

- In target pass in the full cpu mask instead of just the first
  cpu in case we can make use of the full cpu mask.

- Operate in terms of irqs and not vectors, currently there
  is still a 1-1 relationship but on architectures other than
  ia64 I expect this will change.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 drivers/pci/msi-altix.c |   49 +++++++++++++++++++------------------
 drivers/pci/msi-apic.c  |   36 ++++++++++++++-------------
 drivers/pci/msi.c       |   22 ++++++++---------
 drivers/pci/msi.h       |   62 -----------------------------------------------
 include/linux/pci.h     |   62 +++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 116 insertions(+), 115 deletions(-)

diff --git a/drivers/pci/msi-altix.c b/drivers/pci/msi-altix.c
index bed4183..7aedc2a 100644
--- a/drivers/pci/msi-altix.c
+++ b/drivers/pci/msi-altix.c
@@ -26,7 +26,7 @@ struct sn_msi_info {
 static struct sn_msi_info *sn_msi_info;
 
 static void
-sn_msi_teardown(unsigned int vector)
+sn_msi_teardown(unsigned int irq)
 {
 	nasid_t nasid;
 	int widget;
@@ -36,7 +36,7 @@ sn_msi_teardown(unsigned int vector)
 	struct pcibus_bussoft *bussoft;
 	struct sn_pcibus_provider *provider;
 
-	sn_irq_info = sn_msi_info[vector].sn_irq_info;
+	sn_irq_info = sn_msi_info[irq].sn_irq_info;
 	if (sn_irq_info == NULL || sn_irq_info->irq_int_bit >= 0)
 		return;
 
@@ -45,9 +45,9 @@ sn_msi_teardown(unsigned int vector)
 	provider = SN_PCIDEV_BUSPROVIDER(pdev);
 
 	(*provider->dma_unmap)(pdev,
-			       sn_msi_info[vector].pci_addr,
+			       sn_msi_info[irq].pci_addr,
 			       PCI_DMA_FROMDEVICE);
-	sn_msi_info[vector].pci_addr = 0;
+	sn_msi_info[irq].pci_addr = 0;
 
 	bussoft = SN_PCIDEV_BUSSOFT(pdev);
 	nasid = NASID_GET(bussoft->bs_base);
@@ -56,14 +56,13 @@ sn_msi_teardown(unsigned int vector)
 			SWIN_WIDGETNUM(bussoft->bs_base);
 
 	sn_intr_free(nasid, widget, sn_irq_info);
-	sn_msi_info[vector].sn_irq_info = NULL;
+	sn_msi_info[irq].sn_irq_info = NULL;
 
 	return;
 }
 
 int
-sn_msi_setup(struct pci_dev *pdev, unsigned int vector,
-	     u32 *addr_hi, u32 *addr_lo, u32 *data)
+sn_msi_setup(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
 {
 	int widget;
 	int status;
@@ -93,7 +92,7 @@ sn_msi_setup(struct pci_dev *pdev, unsig
 	if (! sn_irq_info)
 		return -ENOMEM;
 
-	status = sn_intr_alloc(nasid, widget, sn_irq_info, vector, -1, -1);
+	status = sn_intr_alloc(nasid, widget, sn_irq_info, irq, -1, -1);
 	if (status) {
 		kfree(sn_irq_info);
 		return -ENOMEM;
@@ -119,28 +118,27 @@ sn_msi_setup(struct pci_dev *pdev, unsig
 		return -ENOMEM;
 	}
 
-	sn_msi_info[vector].sn_irq_info = sn_irq_info;
-	sn_msi_info[vector].pci_addr = bus_addr;
+	sn_msi_info[irq].sn_irq_info = sn_irq_info;
+	sn_msi_info[irq].pci_addr = bus_addr;
 
-	*addr_hi = (u32)(bus_addr >> 32);
-	*addr_lo = (u32)(bus_addr & 0x00000000ffffffff);
+	msg->address_hi = (u32)(bus_addr >> 32);
+	msg->address_lo = (u32)(bus_addr & 0x00000000ffffffff);
 
 	/*
 	 * In the SN platform, bit 16 is a "send vector" bit which
 	 * must be present in order to move the vector through the system.
 	 */
-	*data = 0x100 + (unsigned int)vector;
+	msg->data = 0x100 + irq;
 
 #ifdef CONFIG_SMP
-	set_irq_affinity_info((vector & 0xff), sn_irq_info->irq_cpuid, 0);
+	set_irq_affinity_info(irq, sn_irq_info->irq_cpuid, 0);
 #endif
 
 	return 0;
 }
 
 static void
-sn_msi_target(unsigned int vector, unsigned int cpu,
-	      u32 *addr_hi, u32 *addr_lo)
+sn_msi_target(unsigned int irq, cpumask_t cpu_mask, struct msi_msg *msg)
 {
 	int slice;
 	nasid_t nasid;
@@ -150,8 +148,10 @@ sn_msi_target(unsigned int vector, unsig
 	struct sn_irq_info *sn_irq_info;
 	struct sn_irq_info *new_irq_info;
 	struct sn_pcibus_provider *provider;
+	unsigned int cpu;
 
-	sn_irq_info = sn_msi_info[vector].sn_irq_info;
+	cpu = first_cpu(cpu_mask);
+	sn_irq_info = sn_msi_info[irq].sn_irq_info;
 	if (sn_irq_info == NULL || sn_irq_info->irq_int_bit >= 0)
 		return;
 
@@ -163,15 +163,15 @@ sn_msi_target(unsigned int vector, unsig
 	pdev = sn_pdev->pdi_linux_pcidev;
 	provider = SN_PCIDEV_BUSPROVIDER(pdev);
 
-	bus_addr = (u64)(*addr_hi) << 32 | (u64)(*addr_lo);
+	bus_addr = (u64)(msg->address_hi) << 32 | (u64)(msg->address_lo);
 	(*provider->dma_unmap)(pdev, bus_addr, PCI_DMA_FROMDEVICE);
-	sn_msi_info[vector].pci_addr = 0;
+	sn_msi_info[irq].pci_addr = 0;
 
 	nasid = cpuid_to_nasid(cpu);
 	slice = cpuid_to_slice(cpu);
 
 	new_irq_info = sn_retarget_vector(sn_irq_info, nasid, slice);
-	sn_msi_info[vector].sn_irq_info = new_irq_info;
+	sn_msi_info[irq].sn_irq_info = new_irq_info;
 	if (new_irq_info == NULL)
 		return;
 
@@ -184,12 +184,13 @@ sn_msi_target(unsigned int vector, unsig
 					sizeof(new_irq_info->irq_xtalkaddr),
 					SN_DMA_MSI|SN_DMA_ADDR_XIO);
 
-	sn_msi_info[vector].pci_addr = bus_addr;
-	*addr_hi = (u32)(bus_addr >> 32);
-	*addr_lo = (u32)(bus_addr & 0x00000000ffffffff);
+	sn_msi_info[irq].pci_addr = bus_addr;
+	msg->address_hi = (u32)(bus_addr >> 32);
+	msg->address_lo = (u32)(bus_addr & 0x00000000ffffffff);
 }
 
 struct msi_ops sn_msi_ops = {
+	.needs_64bit_address = 1,
 	.setup = sn_msi_setup,
 	.teardown = sn_msi_teardown,
 #ifdef CONFIG_SMP
@@ -201,7 +202,7 @@ int
 sn_msi_init(void)
 {
 	sn_msi_info =
-		kzalloc(sizeof(struct sn_msi_info) * NR_VECTORS, GFP_KERNEL);
+		kzalloc(sizeof(struct sn_msi_info) * NR_IRQS, GFP_KERNEL);
 	if (! sn_msi_info)
 		return -ENOMEM;
 
diff --git a/drivers/pci/msi-apic.c b/drivers/pci/msi-apic.c
index 5ed798b..1ce2589 100644
--- a/drivers/pci/msi-apic.c
+++ b/drivers/pci/msi-apic.c
@@ -46,37 +46,36 @@ #define     MSI_ADDR_REDIRECTION_LOWPRI	
 
 
 static void
-msi_target_apic(unsigned int vector,
-		unsigned int dest_cpu,
-		u32 *address_hi,	/* in/out */
-		u32 *address_lo)	/* in/out */
+msi_target_apic(unsigned int irq, cpumask_t cpu_mask, struct msi_msg *msg)
 {
-	u32 addr = *address_lo;
+	u32 addr = msg->address_lo;
 
 	addr &= MSI_ADDR_DESTID_MASK;
-	addr |= MSI_ADDR_DESTID_CPU(cpu_physical_id(dest_cpu));
+	addr |= MSI_ADDR_DESTID_CPU(cpu_physical_id(first_cpu(cpu_mask)));
 
-	*address_lo = addr;
+	msg->address_lo = addr;
 }
 
 static int
 msi_setup_apic(struct pci_dev *pdev,	/* unused in generic */
-		unsigned int vector,
-		u32 *address_hi,
-		u32 *address_lo,
-		u32 *data)
+		unsigned int irq,
+		struct msi_msg *msg)
 {
 	unsigned long	dest_phys_id;
+	unsigned int	vector;
 
 	dest_phys_id = cpu_physical_id(first_cpu(cpu_online_map));
+	vector = irq;
 
-	*address_hi = 0;
-	*address_lo =	MSI_ADDR_HEADER |
-			MSI_ADDR_DESTMODE_PHYS |
-			MSI_ADDR_REDIRECTION_CPU |
-			MSI_ADDR_DESTID_CPU(dest_phys_id);
+	msg->address_hi = 0;
+	msg->address_lo =
+		MSI_ADDR_HEADER |
+		MSI_ADDR_DESTMODE_PHYS |
+		MSI_ADDR_REDIRECTION_CPU |
+		MSI_ADDR_DESTID_CPU(dest_phys_id);
 
-	*data = MSI_DATA_TRIGGER_EDGE |
+	msg->data = 
+		MSI_DATA_TRIGGER_EDGE |
 		MSI_DATA_LEVEL_ASSERT |
 		MSI_DATA_DELIVERY_FIXED |
 		MSI_DATA_VECTOR(vector);
@@ -85,7 +84,7 @@ msi_setup_apic(struct pci_dev *pdev,	/* 
 }
 
 static void
-msi_teardown_apic(unsigned int vector)
+msi_teardown_apic(unsigned int irq)
 {
 	return;		/* no-op */
 }
@@ -95,6 +94,7 @@ msi_teardown_apic(unsigned int vector)
  */
 
 struct msi_ops msi_apic_ops = {
+	.needs_64bit_address = 0,
 	.setup = msi_setup_apic,
 	.teardown = msi_teardown_apic,
 	.target = msi_target_apic,
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index e9db6c5..40499c0 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -172,19 +172,17 @@ static void write_msi_msg(struct msi_des
 }
 
 #ifdef CONFIG_SMP
-static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask)
+static void set_msi_affinity(unsigned int irq, cpumask_t cpu_mask)
 {
 	struct msi_desc *entry;
 	struct msi_msg msg;
-	unsigned int irq = vector;
-	unsigned int dest_cpu = first_cpu(cpu_mask);
 
-	entry = (struct msi_desc *)msi_desc[vector];
+	entry = msi_desc[irq];
 	if (!entry || !entry->dev)
 		return;
 
 	read_msi_msg(entry, &msg);
-	msi_ops->target(vector, dest_cpu, &msg.address_hi, &msg.address_lo);
+	msi_ops->target(irq, cpu_mask, &msg);
 	write_msi_msg(entry, &msg);
 	set_native_irq_info(irq, cpu_mask);
 }
@@ -709,14 +707,14 @@ static int msi_register_init(struct pci_
 {
 	int status;
 	struct msi_msg msg;
-	int pos, vector = dev->irq;
+	int pos;
 	u16 control;
 
 	pos = entry->msi_attrib.pos;
 	pci_read_config_word(dev, msi_control_reg(pos), &control);
 
 	/* Configure MSI capability structure */
-	status = msi_ops->setup(dev, vector, &msg.address_hi, &msg.address_lo, &msg.data);
+	status = msi_ops->setup(dev, dev->irq, &msg);
 	if (status < 0)
 		return status;
 
@@ -871,10 +869,7 @@ static int msix_capability_init(struct p
 		/* Replace with MSI-X handler */
 		irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
 		/* Configure MSI-X capability structure */
-		status = msi_ops->setup(dev, vector,
-					&msg.address_hi,
-					&msg.address_lo,
-					&msg.data);
+		status = msi_ops->setup(dev, vector, &msg);
 		if (status < 0)
 			break;
 
@@ -910,6 +905,7 @@ int pci_enable_msi(struct pci_dev* dev)
 {
 	struct pci_bus *bus;
 	int pos, temp, status = -EINVAL;
+	u16 control;
 
 	if (!pci_msi_enable || !dev)
  		return status;
@@ -931,6 +927,10 @@ int pci_enable_msi(struct pci_dev* dev)
 	if (!pos)
 		return -EINVAL;
 
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (!is_64bit_address(control) && msi_ops->needs_64bit_address)
+		return -EINVAL;
+
 	BUG_ON(!msi_lookup_vector(dev, PCI_CAP_ID_MSI));
 
 	/* Check whether driver already requested for MSI-X vectors */
diff --git a/drivers/pci/msi.h b/drivers/pci/msi.h
index 62f61b6..3519eca 100644
--- a/drivers/pci/msi.h
+++ b/drivers/pci/msi.h
@@ -6,68 +6,6 @@
 #ifndef MSI_H
 #define MSI_H
 
-/*
- * MSI operation vector.  Used by the msi core code (drivers/pci/msi.c)
- * to abstract platform-specific tasks relating to MSI address generation
- * and resource management.
- */
-struct msi_ops {
-	/**
-	 * setup - generate an MSI bus address and data for a given vector
-	 * @pdev: PCI device context (in)
-	 * @vector: vector allocated by the msi core (in)
-	 * @addr_hi: upper 32 bits of PCI bus MSI address (out)
-	 * @addr_lo: lower 32 bits of PCI bus MSI address (out)
-	 * @data: MSI data payload (out)
-	 *
-	 * Description: The setup op is used to generate a PCI bus addres and
-	 * data which the msi core will program into the card MSI capability
-	 * registers.  The setup routine is responsible for picking an initial
-	 * cpu to target the MSI at.  The setup routine is responsible for
-	 * examining pdev to determine the MSI capabilities of the card and
-	 * generating a suitable address/data.  The setup routine is
-	 * responsible for allocating and tracking any system resources it
-	 * needs to route the MSI to the cpu it picks, and for associating
-	 * those resources with the passed in vector.
-	 *
-	 * Returns 0 if the MSI address/data was successfully setup.
-	 **/
-
-	int	(*setup)    (struct pci_dev *pdev, unsigned int vector,
-			     u32 *addr_hi, u32 *addr_lo, u32 *data);
-
-	/**
-	 * teardown - release resources allocated by setup
-	 * @vector: vector context for resources (in)
-	 *
-	 * Description:  The teardown op is used to release any resources
-	 * that were allocated in the setup routine associated with the passed
-	 * in vector.
-	 **/
-
-	void	(*teardown) (unsigned int vector);
-
-	/**
-	 * target - retarget an MSI at a different cpu
-	 * @vector: vector context for resources (in)
-	 * @cpu:  new cpu to direct vector at (in)
-	 * @addr_hi: new value of PCI bus upper 32 bits (in/out)
-	 * @addr_lo: new value of PCI bus lower 32 bits (in/out)
-	 *
-	 * Description:  The target op is used to redirect an MSI vector
-	 * at a different cpu.  addr_hi/addr_lo coming in are the existing
-	 * values that the MSI core has programmed into the card.  The
-	 * target code is responsible for freeing any resources (if any)
-	 * associated with the old address, and generating a new PCI bus
-	 * addr_hi/addr_lo that will redirect the vector at the indicated cpu.
-	 **/
-
-	void	(*target)   (unsigned int vector, unsigned int cpu,
-			     u32 *addr_hi, u32 *addr_lo);
-};
-
-extern int msi_register(struct msi_ops *ops);
-
 #include <asm/msi.h>
 
 /*
diff --git a/include/linux/pci.h b/include/linux/pci.h
index c7be27b..1aa01aa 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -613,6 +613,68 @@ extern int pci_enable_msix(struct pci_de
 	struct msix_entry *entries, int nvec);
 extern void pci_disable_msix(struct pci_dev *dev);
 extern void msi_remove_pci_irq_vectors(struct pci_dev *dev);
+
+/*
+ * MSI operation vector.  Used by the msi core code (drivers/pci/msi.c)
+ * to abstract platform-specific tasks relating to MSI address generation
+ * and resource management.
+ */
+struct msi_ops {
+	int needs_64bit_address;
+	/**
+	 * setup - generate an MSI bus address and data for a given vector
+	 * @pdev: PCI device context (in)
+	 * @irq: irq allocated by the msi core (in)
+	 * @msg: PCI bus address and data for msi message (out)
+	 *
+	 * Description: The setup op is used to generate a PCI bus addres and
+	 * data which the msi core will program into the card MSI capability
+	 * registers.  The setup routine is responsible for picking an initial
+	 * cpu to target the MSI at.  The setup routine is responsible for
+	 * examining pdev to determine the MSI capabilities of the card and
+	 * generating a suitable address/data.  The setup routine is
+	 * responsible for allocating and tracking any system resources it
+	 * needs to route the MSI to the cpu it picks, and for associating
+	 * those resources with the passed in vector.
+	 *
+	 * Returns 0 if the MSI address/data was successfully setup.
+	 **/
+
+	int	(*setup)    (struct pci_dev *pdev, unsigned int irq,
+			     struct msi_msg *msg);
+
+	/**
+	 * teardown - release resources allocated by setup
+	 * @vector: vector context for resources (in)
+	 *
+	 * Description:  The teardown op is used to release any resources
+	 * that were allocated in the setup routine associated with the passed
+	 * in vector.
+	 **/
+
+	void	(*teardown) (unsigned int irq);
+
+	/**
+	 * target - retarget an MSI at a different cpu
+	 * @vector: vector context for resources (in)
+	 * @cpu:  new cpu to direct vector at (in)
+	 * @addr_hi: new value of PCI bus upper 32 bits (in/out)
+	 * @addr_lo: new value of PCI bus lower 32 bits (in/out)
+	 *
+	 * Description:  The target op is used to redirect an MSI vector
+	 * at a different cpu.  addr_hi/addr_lo coming in are the existing
+	 * values that the MSI core has programmed into the card.  The
+	 * target code is responsible for freeing any resources (if any)
+	 * associated with the old address, and generating a new PCI bus
+	 * addr_hi/addr_lo that will redirect the vector at the indicated cpu.
+	 **/
+
+	void	(*target)   (unsigned int irq, cpumask_t cpumask,
+			     struct msi_msg *msg);
+};
+
+extern int msi_register(struct msi_ops *ops);
+
 #endif
 
 extern void pci_block_user_cfg_access(struct pci_dev *dev);
-- 
1.4.0.gc07e

[PATCH 8/25] msi: Simplify the msi irq limit policy.

[Eric W. Biederman]

Currently we attempt to predict how many irqs we will be
able to allocate with msi using pci_vector_resources and some
complicated accounting, and then we only allow each device
as many irqs as we think are available on average.

Only the s2io driver even takes advantage of this feature
all other drivers have a fixed number of irqs they need and
bail if they can't get them.

pci_vector_resources is inaccurate if anyone ever frees an irq.
The whole implmentation is racy.  The current irq limit policy
does not appear to make sense with current drivers.  So I have
simplified things.  We can revisit this we we need a more sophisticated
policy.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 arch/i386/pci/irq.c |   30 -----------------------------
 arch/ia64/pci/pci.c |    9 ---------
 drivers/pci/msi.c   |   53 ++++++++-------------------------------------------
 drivers/pci/msi.h   |   11 -----------
 4 files changed, 8 insertions(+), 95 deletions(-)

diff --git a/arch/i386/pci/irq.c b/arch/i386/pci/irq.c
index 8ce6950..768584d 100644
--- a/arch/i386/pci/irq.c
+++ b/arch/i386/pci/irq.c
@@ -1170,33 +1170,3 @@ #endif
 	}
 	return 0;
 }
-
-int pci_vector_resources(int last, int nr_released)
-{
-	int count = nr_released;
-
-	int next = last;
-	int offset = (last % 8);
-
-	while (next < FIRST_SYSTEM_VECTOR) {
-		next += 8;
-#ifdef CONFIG_X86_64
-		if (next == IA32_SYSCALL_VECTOR)
-			continue;
-#else
-		if (next == SYSCALL_VECTOR)
-			continue;
-#endif
-		count++;
-		if (next >= FIRST_SYSTEM_VECTOR) {
-			if (offset%8) {
-				next = FIRST_DEVICE_VECTOR + offset;
-				offset++;
-				continue;
-			}
-			count--;
-		}
-	}
-
-	return count;
-}
diff --git a/arch/ia64/pci/pci.c b/arch/ia64/pci/pci.c
index 5bef0e3..b0028fd 100644
--- a/arch/ia64/pci/pci.c
+++ b/arch/ia64/pci/pci.c
@@ -810,12 +810,3 @@ pcibios_prep_mwi (struct pci_dev *dev)
 	}
 	return rc;
 }
-
-int pci_vector_resources(int last, int nr_released)
-{
-	int count = nr_released;
-
-	count += (IA64_LAST_DEVICE_VECTOR - last);
-
-	return count;
-}
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 40499c0..772f5b6 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -30,8 +30,6 @@ static kmem_cache_t* msi_cachep;
 static int pci_msi_enable = 1;
 static int last_alloc_vector;
 static int nr_released_vectors;
-static int nr_reserved_vectors = NR_HP_RESERVED_VECTORS;
-static int nr_msix_devices;
 
 #ifndef CONFIG_X86_IO_APIC
 int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1};
@@ -542,11 +540,6 @@ void pci_scan_msi_device(struct pci_dev 
 {
 	if (!dev)
 		return;
-
-   	if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0)
-		nr_msix_devices++;
-	else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0)
-		nr_reserved_vectors++;
 }
 
 #ifdef CONFIG_PM
@@ -877,13 +870,19 @@ static int msix_capability_init(struct p
 		attach_msi_entry(entry, vector);
 	}
 	if (i != nvec) {
+		int avail = i - 1;
 		i--;
 		for (; i >= 0; i--) {
 			vector = (entries + i)->vector;
 			msi_free_vector(dev, vector, 0);
 			(entries + i)->vector = 0;
 		}
-		return -EBUSY;
+		/* If we had some success report the number of irqs
+		 * we succeeded in setting up.
+		 */
+		if (avail <= 0)
+			avail = -EBUSY;
+		return avail;
 	}
 	/* Set MSI-X enabled bits */
 	enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
@@ -943,14 +942,6 @@ int pci_enable_msi(struct pci_dev* dev)
 			return -EINVAL;
 	}
 	status = msi_capability_init(dev);
-	if (!status) {
-   		if (!pos)
-			nr_reserved_vectors--;	/* Only MSI capable */
-		else if (nr_msix_devices > 0)
-			nr_msix_devices--;	/* Both MSI and MSI-X capable,
-						   but choose enabling MSI */
-	}
-
 	return status;
 }
 
@@ -1060,10 +1051,9 @@ static int msi_free_vector(struct pci_de
 int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
 {
 	struct pci_bus *bus;
-	int status, pos, nr_entries, free_vectors;
+	int status, pos, nr_entries;
 	int i, j, temp;
 	u16 control;
-	unsigned long flags;
 
 	if (!pci_msi_enable || !dev || !entries)
  		return -EINVAL;
@@ -1109,34 +1099,7 @@ int pci_enable_msix(struct pci_dev* dev,
 		dev->irq = temp;
 		return -EINVAL;
 	}
-
-	spin_lock_irqsave(&msi_lock, flags);
-	/*
-	 * msi_lock is provided to ensure that enough vectors resources are
-	 * available before granting.
-	 */
-	free_vectors = pci_vector_resources(last_alloc_vector,
-				nr_released_vectors);
-	/* Ensure that each MSI/MSI-X device has one vector reserved by
-	   default to avoid any MSI-X driver to take all available
- 	   resources */
-	free_vectors -= nr_reserved_vectors;
-	/* Find the average of free vectors among MSI-X devices */
-	if (nr_msix_devices > 0)
-		free_vectors /= nr_msix_devices;
-	spin_unlock_irqrestore(&msi_lock, flags);
-
-	if (nvec > free_vectors) {
-		if (free_vectors > 0)
-			return free_vectors;
-		else
-			return -EBUSY;
-	}
-
 	status = msix_capability_init(dev, entries, nvec);
-	if (!status && nr_msix_devices > 0)
-		nr_msix_devices--;
-
 	return status;
 }
 
diff --git a/drivers/pci/msi.h b/drivers/pci/msi.h
index 3519eca..6793241 100644
--- a/drivers/pci/msi.h
+++ b/drivers/pci/msi.h
@@ -8,19 +8,8 @@ #define MSI_H
 
 #include <asm/msi.h>
 
-/*
- * Assume the maximum number of hot plug slots supported by the system is about
- * ten. The worstcase is that each of these slots is hot-added with a device,
- * which has two MSI/MSI-X capable functions. To avoid any MSI-X driver, which
- * attempts to request all available vectors, NR_HP_RESERVED_VECTORS is defined
- * as below to ensure at least one message is assigned to each detected MSI/
- * MSI-X device function.
- */
-#define NR_HP_RESERVED_VECTORS 	20
-
 extern int vector_irq[NR_VECTORS];
 extern void (*interrupt[NR_IRQS])(void);
-extern int pci_vector_resources(int last, int nr_released);
 
 /*
  * MSI-X Address Register
-- 
1.4.0.gc07e

[PATCH 9/25] irq: Add a dynamic irq creation API

[Eric W. Biederman]

With the msi support comes a new concept in irq handling,
irqs that are created dynamically at run time.

Currently the msi code allocates irqs backwards.  First it
allocates a platform dependent routing value for an
interrupt the ``vector'' and then it figures out from the
vector which irq you are on.

This msi backwards allocator suffers from two basic
problems.  The allocator suffers because it is trying
to do something that is architecture specific in a generic
way making it brittle, inflexible, and tied to tightly
to the architecture implementation.  The alloctor also
suffers from it's very backwards nature as it has tied
things together that should have no dependencies.

To solve the basic dynamic irq allocation problem two
new architecture specific functions are added:
create_irq and destroy_irq.

create_irq takes no input and returns an unused irq number,
that won't be reused until it is returned to the free
poll with destroy_irq.  The irq then can be used for
any purpose although the only initial consumer is
the msi code.

destroy_irq takes an irq number allocated with create_irq
and returns it to the free pool.

Making this functionality per architecture increases
the simplicity of the irq allocation code and increases
it's flexibility.

dynamic_irq_init() and dynamic_irq_cleanup() are added
to automate the irq_desc initializtion that should happen
for dynamic irqs.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 include/linux/irq.h |    9 +++++++-
 kernel/irq/chip.c   |   56 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 1 deletions(-)

diff --git a/include/linux/irq.h b/include/linux/irq.h
index b79d178..6d1ad88 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -392,8 +392,15 @@ set_irq_chained_handler(unsigned int irq
 	__set_irq_handler(irq, handle, 1);
 }
 
-/* Set/get chip/data for an IRQ: */
+/* Handle dynamic irq creation and destruction */
+extern int create_irq(void);
+extern void destroy_irq(unsigned int irq);
+
+/* Dynamic irq helper functions */
+extern void dynamic_irq_init(unsigned int irq);
+extern void dynamic_irq_cleanup(unsigned int irq);
 
+/* Set/get chip/data for an IRQ: */
 extern int set_irq_chip(unsigned int irq, struct irq_chip *chip);
 extern int set_irq_data(unsigned int irq, void *data);
 extern int set_irq_chip_data(unsigned int irq, void *data);
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 431e9d5..9c01e48 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -18,6 +18,62 @@ #include <linux/kernel_stat.h>
 #include "internals.h"
 
 /**
+ *	dynamic_irq_init - initialize a dynamically allocated irq
+ *	@irq:	irq number to initialize
+ */
+void dynamic_irq_init(unsigned int irq)
+{
+	struct irq_desc *desc;
+	unsigned long flags;
+
+	if (irq >= NR_IRQS) {
+		printk(KERN_ERR "Trying to initialize invalid IRQ%d\n", irq);
+		WARN_ON(1);
+		return;
+	}
+
+	/* Ensure we don't have left over values from a previous use of this irq */
+	desc = irq_desc + irq;
+	spin_lock_irqsave(&desc->lock, flags);
+	desc->status = IRQ_DISABLED;
+	desc->chip = &no_irq_chip;
+	desc->handle_irq = handle_bad_irq;
+	desc->depth = 1;
+	desc->handler_data = NULL;
+	desc->chip_data = NULL;
+	desc->action = NULL;
+	desc->irq_count = 0;
+	desc->irqs_unhandled = 0;
+#ifdef CONFIG_SMP
+	desc->affinity = CPU_MASK_ALL;
+#endif
+	spin_unlock_irqrestore(&desc->lock, flags);
+}
+
+/**
+ *	dynamic_irq_cleanup - cleanup a dynamically allocated irq
+ *	@irq:	irq number to initialize
+ */
+void dynamic_irq_cleanup(unsigned int irq)
+{
+	struct irq_desc *desc;
+	unsigned long flags;
+
+	if (irq >= NR_IRQS) {
+		printk(KERN_ERR "Trying to cleanup invalid IRQ%d\n", irq);
+		WARN_ON(1);
+		return;
+	}
+	
+	desc = irq_desc + irq;
+	spin_lock_irqsave(&desc->lock, flags);
+	desc->handle_irq = handle_bad_irq;
+	desc->chip = &no_irq_chip;
+	spin_unlock_irqrestore(&desc->lock, flags);
+}
+
+
+/**
  *	set_irq_chip - set the irq chip for an irq
  *	@irq:	irq number
  *	@chip:	pointer to irq chip description structure
-- 
1.4.0.gc07e

[PATCH 10/25] ia64 irq: Dynamic irq support

[Eric W. Biederman]

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 arch/ia64/kernel/irq_ia64.c |   23 +++++++++++++++++++++++
 1 files changed, 23 insertions(+), 0 deletions(-)

diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c
index f503530..b0189c7 100644
--- a/arch/ia64/kernel/irq_ia64.c
+++ b/arch/ia64/kernel/irq_ia64.c
@@ -31,6 +31,7 @@ #include <linux/smp.h>
 #include <linux/smp_lock.h>
 #include <linux/threads.h>
 #include <linux/bitops.h>
+#include <linux/irq.h>
 
 #include <asm/delay.h>
 #include <asm/intrinsics.h>
@@ -106,6 +107,28 @@ reserve_irq_vector (int vector)
 	return test_and_set_bit(pos, ia64_vector_mask);
 }
 
+/*
+ * Dynamic irq allocate and deallocation for MSI
+ */
+int create_irq(void)
+{
+	int vector;
+	unsigned long flags;
+
+	vector = assign_irq_vector(AUTO_ASSIGN);
+
+	if (vector >= 0)
+		dynamic_irq_init(irq);
+
+	return vector;
+}
+
+void destroy_irq(unsigned int irq)
+{
+	dynamic_irq_cleanup(irq);
+	free_irq_vector(irq);
+}
+
 #ifdef CONFIG_SMP
 #	define IS_RESCHEDULE(vec)	(vec == IA64_IPI_RESCHEDULE)
 #else
-- 
1.4.0.gc07e

[PATCH 11/25] i386 irq: Dynamic irq support

[Eric W. Biederman]

The current implementation of create_irq() is a hack but it is the
current hack that msi.c uses, and unfortunately the ``generic'' apic
msi ops depend on this hack.  Thus we are stuck this hack of assuming
irq == vector until the depencencies in the generic msi code are removed.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 arch/i386/kernel/io_apic.c |   48 ++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 48 insertions(+), 0 deletions(-)

diff --git a/arch/i386/kernel/io_apic.c b/arch/i386/kernel/io_apic.c
index 16966f4..04f78ff 100644
--- a/arch/i386/kernel/io_apic.c
+++ b/arch/i386/kernel/io_apic.c
@@ -2497,6 +2497,54 @@ static int __init ioapic_init_sysfs(void
 
 device_initcall(ioapic_init_sysfs);
 
+#ifdef CONFIG_PCI_MSI
+/*
+ * Dynamic irq allocate and deallocation for MSI
+ */
+int create_irq(void)
+{
+	/* Hack of the day: irq == vector.
+	 *
+	 * Ultimately this will be be more general,
+	 * and not depend on the irq to vector identity mapping.
+	 * But this version is needed until msi.c can cope with
+	 * the more general form.
+	 */
+	int irq, vector;
+	unsigned long flags;
+	vector = assign_irq_vector(AUTO_ASSIGN);
+	irq = vector;
+
+	if (vector >= 0) {
+		struct irq_desc *desc;
+
+		spin_lock_irqsave(&vector_lock, flags);
+		vector_irq[vector] = irq;
+		irq_vector[irq] = vector;
+		spin_unlock_irqrestore(&vector_lock, flags);
+		
+		set_intr_gate(vector, interrupt[irq]);
+
+		dynamic_irq_init(irq);
+	}
+	return irq;
+}
+
+void destroy_irq(unsigned int irq)
+{
+	unsigned long flags;
+	unsigned int vector;
+
+	dynmic_irq_cleanup(irq);
+
+	spin_lock_irqsave(&vector_lock, flags);
+	vector = irq_vector[irq];
+	vector_irq[vector] = -1;
+	irq_vector[irq] = 0;
+	spin_unlock_irqrestore(&vector_lock, flags);
+}
+#endif /* CONFIG_PCI_MSI */
+
 /* --------------------------------------------------------------------------
                           ACPI-based IOAPIC Configuration
    -------------------------------------------------------------------------- */
-- 
1.4.0.gc07e

[PATCH 12/25] x86_64 irq: Dynamic irq support

[Eric W. Biederman]

The current implementation of create_irq() is a hack
but it is the current hack that msi.c uses, and unfortunately
the ``generic'' apic msi ops depend on this hack.  Thus
we are this hack of assuming irq == vector until the depencencies
in the generic irq code are removed.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 arch/x86_64/kernel/io_apic.c |   48 ++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 48 insertions(+), 0 deletions(-)

diff --git a/arch/x86_64/kernel/io_apic.c b/arch/x86_64/kernel/io_apic.c
index f50be45..ae64a63 100644
--- a/arch/x86_64/kernel/io_apic.c
+++ b/arch/x86_64/kernel/io_apic.c
@@ -2013,6 +2013,54 @@ static int __init ioapic_init_sysfs(void
 
 device_initcall(ioapic_init_sysfs);
 
+#ifdef CONFIG_PCI_MSI
+/*
+ * Dynamic irq allocate and deallocation for MSI
+ */
+int create_irq(void)
+{
+	/* Hack of the day: irq == vector.
+	 *
+	 * Ultimately this will be be more general,
+	 * and not depend on the irq to vector identity mapping.
+	 * But this version is needed until msi.c can cope with
+	 * the more general form.
+	 */
+	int irq, vector;
+	unsigned long flags;
+	vector = assign_irq_vector(AUTO_ASSIGN);
+	irq = vector;
+
+	if (vector >= 0) {
+		struct irq_desc *desc;
+
+		spin_lock_irqsave(&vector_lock, flags);
+		vector_irq[vector] = irq;
+		irq_vector[irq] = vector;
+		spin_unlock_irqrestore(&vector_lock, flags);
+		
+		set_intr_gate(vector, interrupt[irq]);
+
+		dynamic_irq_init(irq);
+	}
+	return irq;
+}
+
+void destroy_irq(unsigned int irq)
+{
+	unsigned long flags;
+	unsigned int vector;
+
+	dynamic_irq_cleanup(irq);
+
+	spin_lock_irqsave(&vector_lock, flags);
+	vector = irq_vector[irq];
+	vector_irq[vector] = -1;
+	irq_vector[irq] = 0;
+	spin_unlock_irqrestore(&vector_lock, flags);
+}
+#endif
+
 /* --------------------------------------------------------------------------
                           ACPI-based IOAPIC Configuration
    -------------------------------------------------------------------------- */
-- 
1.4.0.gc07e

[PATCH 13/25] msi: Make the msi code irq based and not vector based.

[Eric W. Biederman]

The msi currently allocates irqs backwards.  First it
allocates a platform dependent routing value for an
interrupt the ``vector'' and then it figures out from the
vector which irq you are on.

For ia64 this is fine.  For x86 and x86_64 this is complete
nonsense and makes an enourmous mess of the irq handling
code and prevents some pretty significant cleanups in
the code for handling large numbers of irqs.

This patch refactors msi.c to work in terms of irqs and
create_irq/destroy_irq for dynamically managing irqs.

Hopefully this is finally a version of msi.c that
is useful on more than just x86 derivatives.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 drivers/pci/msi.c |  425 +++++++++++++++++++++--------------------------------
 drivers/pci/msi.h |    7 -
 2 files changed, 168 insertions(+), 264 deletions(-)

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 772f5b6..a5d3685 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -6,6 +6,7 @@
  * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
  */
 
+#include <linux/err.h>
 #include <linux/mm.h>
 #include <linux/irq.h>
 #include <linux/interrupt.h>
@@ -28,12 +29,6 @@ static struct msi_desc* msi_desc[NR_IRQS
 static kmem_cache_t* msi_cachep;
 
 static int pci_msi_enable = 1;
-static int last_alloc_vector;
-static int nr_released_vectors;
-
-#ifndef CONFIG_X86_IO_APIC
-int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1};
-#endif
 
 static struct msi_ops *msi_ops;
 
@@ -60,11 +55,11 @@ static int msi_cache_init(void)
 	return 0;
 }
 
-static void msi_set_mask_bit(unsigned int vector, int flag)
+static void msi_set_mask_bit(unsigned int irq, int flag)
 {
 	struct msi_desc *entry;
 
-	entry = (struct msi_desc *)msi_desc[vector];
+	entry = msi_desc[irq];
 	if (!entry || !entry->dev || !entry->mask_base)
 		return;
 	switch (entry->msi_attrib.type) {
@@ -188,23 +183,23 @@ #else
 #define set_msi_affinity NULL
 #endif /* CONFIG_SMP */
 
-static void mask_MSI_irq(unsigned int vector)
+static void mask_MSI_irq(unsigned int irq)
 {
-	msi_set_mask_bit(vector, 1);
+	msi_set_mask_bit(irq, 1);
 }
 
-static void unmask_MSI_irq(unsigned int vector)
+static void unmask_MSI_irq(unsigned int irq)
 {
-	msi_set_mask_bit(vector, 0);
+	msi_set_mask_bit(irq, 0);
 }
 
-static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector)
+static unsigned int startup_msi_irq_wo_maskbit(unsigned int irq)
 {
 	struct msi_desc *entry;
 	unsigned long flags;
 
 	spin_lock_irqsave(&msi_lock, flags);
-	entry = msi_desc[vector];
+	entry = msi_desc[irq];
 	if (!entry || !entry->dev) {
 		spin_unlock_irqrestore(&msi_lock, flags);
 		return 0;
@@ -215,39 +210,39 @@ static unsigned int startup_msi_irq_wo_m
 	return 0;	/* never anything pending */
 }
 
-static unsigned int startup_msi_irq_w_maskbit(unsigned int vector)
+static unsigned int startup_msi_irq_w_maskbit(unsigned int irq)
 {
-	startup_msi_irq_wo_maskbit(vector);
-	unmask_MSI_irq(vector);
+	startup_msi_irq_wo_maskbit(irq);
+	unmask_MSI_irq(irq);
 	return 0;	/* never anything pending */
 }
 
-static void shutdown_msi_irq(unsigned int vector)
+static void shutdown_msi_irq(unsigned int irq)
 {
 	struct msi_desc *entry;
 	unsigned long flags;
 
 	spin_lock_irqsave(&msi_lock, flags);
-	entry = msi_desc[vector];
+	entry = msi_desc[irq];
 	if (entry && entry->dev)
 		entry->msi_attrib.state = 0;	/* Mark it not active */
 	spin_unlock_irqrestore(&msi_lock, flags);
 }
 
-static void end_msi_irq_wo_maskbit(unsigned int vector)
+static void end_msi_irq_wo_maskbit(unsigned int irq)
 {
-	move_native_irq(vector);
+	move_native_irq(irq);
 	ack_APIC_irq();
 }
 
-static void end_msi_irq_w_maskbit(unsigned int vector)
+static void end_msi_irq_w_maskbit(unsigned int irq)
 {
-	move_native_irq(vector);
-	unmask_MSI_irq(vector);
+	move_native_irq(irq);
+	unmask_MSI_irq(irq);
 	ack_APIC_irq();
 }
 
-static void do_nothing(unsigned int vector)
+static void do_nothing(unsigned int irq)
 {
 }
 
@@ -298,86 +293,7 @@ static struct hw_interrupt_type msi_irq_
 	.set_affinity	= set_msi_affinity
 };
 
-static int msi_free_vector(struct pci_dev* dev, int vector, int reassign);
-static int assign_msi_vector(void)
-{
-	static int new_vector_avail = 1;
-	int vector;
-	unsigned long flags;
-
-	/*
-	 * msi_lock is provided to ensure that successful allocation of MSI
-	 * vector is assigned unique among drivers.
-	 */
-	spin_lock_irqsave(&msi_lock, flags);
-
-	if (!new_vector_avail) {
-		int free_vector = 0;
-
-		/*
-	 	 * vector_irq[] = -1 indicates that this specific vector is:
-	 	 * - assigned for MSI (since MSI have no associated IRQ) or
-	 	 * - assigned for legacy if less than 16, or
-	 	 * - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping
-	 	 * vector_irq[] = 0 indicates that this vector, previously
-		 * assigned for MSI, is freed by hotplug removed operations.
-		 * This vector will be reused for any subsequent hotplug added
-		 * operations.
-	 	 * vector_irq[] > 0 indicates that this vector is assigned for
-		 * IOxAPIC IRQs. This vector and its value provides a 1-to-1
-		 * vector-to-IOxAPIC IRQ mapping.
-	 	 */
-		for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
-			if (vector_irq[vector] != 0)
-				continue;
-			free_vector = vector;
-			if (!msi_desc[vector])
-			      	break;
-			else
-				continue;
-		}
-		if (!free_vector) {
-			spin_unlock_irqrestore(&msi_lock, flags);
-			return -EBUSY;
-		}
-		vector_irq[free_vector] = -1;
-		nr_released_vectors--;
-		spin_unlock_irqrestore(&msi_lock, flags);
-		if (msi_desc[free_vector] != NULL) {
-			struct pci_dev *dev;
-			int tail;
-
-			/* free all linked vectors before re-assign */
-			do {
-				spin_lock_irqsave(&msi_lock, flags);
-				dev = msi_desc[free_vector]->dev;
-				tail = msi_desc[free_vector]->link.tail;
-				spin_unlock_irqrestore(&msi_lock, flags);
-				msi_free_vector(dev, tail, 1);
-			} while (free_vector != tail);
-		}
-
-		return free_vector;
-	}
-	vector = assign_irq_vector(AUTO_ASSIGN);
-	last_alloc_vector = vector;
-	if (vector  == LAST_DEVICE_VECTOR)
-		new_vector_avail = 0;
-
-	spin_unlock_irqrestore(&msi_lock, flags);
-	return vector;
-}
-
-static int get_new_vector(void)
-{
-	int vector = assign_msi_vector();
-
-	if (vector > 0)
-		set_intr_gate(vector, interrupt[vector]);
-
-	return vector;
-}
-
+static int msi_free_irq(struct pci_dev* dev, int irq);
 static int msi_init(void)
 {
 	static int status = -ENOMEM;
@@ -401,13 +317,13 @@ static int msi_init(void)
 	}
 
 	if (! msi_ops) {
+		pci_msi_enable = 0;
 		printk(KERN_WARNING
 		       "PCI: MSI ops not registered. MSI disabled.\n");
 		status = -EINVAL;
 		return status;
 	}
 
-	last_alloc_vector = assign_irq_vector(AUTO_ASSIGN);
 	status = msi_cache_init();
 	if (status < 0) {
 		pci_msi_enable = 0;
@@ -415,23 +331,9 @@ static int msi_init(void)
 		return status;
 	}
 
-	if (last_alloc_vector < 0) {
-		pci_msi_enable = 0;
-		printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n");
-		status = -EBUSY;
-		return status;
-	}
-	vector_irq[last_alloc_vector] = 0;
-	nr_released_vectors++;
-
 	return status;
 }
 
-static int get_msi_vector(struct pci_dev *dev)
-{
-	return get_new_vector();
-}
-
 static struct msi_desc* alloc_msi_entry(void)
 {
 	struct msi_desc *entry;
@@ -447,29 +349,45 @@ static struct msi_desc* alloc_msi_entry(
 	return entry;
 }
 
-static void attach_msi_entry(struct msi_desc *entry, int vector)
+static void attach_msi_entry(struct msi_desc *entry, int irq)
 {
 	unsigned long flags;
 
 	spin_lock_irqsave(&msi_lock, flags);
-	msi_desc[vector] = entry;
+	msi_desc[irq] = entry;
 	spin_unlock_irqrestore(&msi_lock, flags);
 }
 
-static void irq_handler_init(int cap_id, int pos, int mask)
+static int create_msi_irq(struct hw_interrupt_type *handler)
 {
-	unsigned long flags;
+	struct msi_desc *entry;
+	int irq;
 
-	spin_lock_irqsave(&irq_desc[pos].lock, flags);
-	if (cap_id == PCI_CAP_ID_MSIX)
-		irq_desc[pos].chip = &msix_irq_type;
-	else {
-		if (!mask)
-			irq_desc[pos].chip = &msi_irq_wo_maskbit_type;
-		else
-			irq_desc[pos].chip = &msi_irq_w_maskbit_type;
+	entry = alloc_msi_entry();
+	if (!entry)
+		return -ENOMEM;
+	
+	irq = create_irq();
+	if (irq < 0) {
+		kmem_cache_free(msi_cachep, entry);
+		return -EBUSY;
 	}
-	spin_unlock_irqrestore(&irq_desc[pos].lock, flags);
+	
+	set_irq_chip(irq, handler);
+	set_irq_data(irq, entry);
+
+	return irq;
+}
+
+static void destroy_msi_irq(unsigned int irq)
+{
+	struct msi_desc *entry;
+
+	entry = get_irq_data(irq);
+	set_irq_chip(irq, NULL);
+	set_irq_data(irq, NULL);
+	destroy_irq(irq);
+	kmem_cache_free(msi_cachep, entry);
 }
 
 static void enable_msi_mode(struct pci_dev *dev, int pos, int type)
@@ -514,21 +432,21 @@ void disable_msi_mode(struct pci_dev *de
 	}
 }
 
-static int msi_lookup_vector(struct pci_dev *dev, int type)
+static int msi_lookup_irq(struct pci_dev *dev, int type)
 {
-	int vector;
+	int irq;
 	unsigned long flags;
 
 	spin_lock_irqsave(&msi_lock, flags);
-	for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
-		if (!msi_desc[vector] || msi_desc[vector]->dev != dev ||
-			msi_desc[vector]->msi_attrib.type != type ||
-			msi_desc[vector]->msi_attrib.default_vector != dev->irq)
+	for (irq = 0; irq < NR_IRQS; irq++) {
+		if (!msi_desc[irq] || msi_desc[irq]->dev != dev ||
+			msi_desc[irq]->msi_attrib.type != type ||
+			msi_desc[irq]->msi_attrib.default_irq != dev->irq)
 			continue;
 		spin_unlock_irqrestore(&msi_lock, flags);
-		/* This pre-assigned MSI vector for this device
-		   already exits. Override dev->irq with this vector */
-		dev->irq = vector;
+		/* This pre-assigned MSI irq for this device
+		   already exits. Override dev->irq with this irq */
+		dev->irq = irq;
 		return 0;
 	}
 	spin_unlock_irqrestore(&msi_lock, flags);
@@ -613,7 +531,7 @@ int pci_save_msix_state(struct pci_dev *
 {
 	int pos;
 	int temp;
-	int vector, head, tail = 0;
+	int irq, head, tail = 0;
 	u16 control;
 	struct pci_cap_saved_state *save_state;
 
@@ -635,20 +553,20 @@ int pci_save_msix_state(struct pci_dev *
 
 	/* save the table */
 	temp = dev->irq;
-	if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
+	if (msi_lookup_irq(dev, PCI_CAP_ID_MSIX)) {
 		kfree(save_state);
 		return -EINVAL;
 	}
 
-	vector = head = dev->irq;
+	irq = head = dev->irq;
 	while (head != tail) {
 		struct msi_desc *entry;
 
-		entry = msi_desc[vector];
+		entry = msi_desc[irq];
 		read_msi_msg(entry, &entry->msg_save);
 
-		tail = msi_desc[vector]->link.tail;
-		vector = tail;
+		tail = msi_desc[irq]->link.tail;
+		irq = tail;
 	}
 	dev->irq = temp;
 
@@ -661,7 +579,7 @@ void pci_restore_msix_state(struct pci_d
 {
 	u16 save;
 	int pos;
-	int vector, head, tail = 0;
+	int irq, head, tail = 0;
 	struct msi_desc *entry;
 	int temp;
 	struct pci_cap_saved_state *save_state;
@@ -679,15 +597,15 @@ void pci_restore_msix_state(struct pci_d
 
 	/* route the table */
 	temp = dev->irq;
-	if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX))
+	if (msi_lookup_irq(dev, PCI_CAP_ID_MSIX))
 		return;
-	vector = head = dev->irq;
+	irq = head = dev->irq;
 	while (head != tail) {
-		entry = msi_desc[vector];
+		entry = msi_desc[irq];
 		write_msi_msg(entry, &entry->msg_save);
 
-		tail = msi_desc[vector]->link.tail;
-		vector = tail;
+		tail = msi_desc[irq]->link.tail;
+		irq = tail;
 	}
 	dev->irq = temp;
 
@@ -734,55 +652,54 @@ static int msi_register_init(struct pci_
  * @dev: pointer to the pci_dev data structure of MSI device function
  *
  * Setup the MSI capability structure of device function with a single
- * MSI vector, regardless of device function is capable of handling
+ * MSI irq, regardless of device function is capable of handling
  * multiple messages. A return of zero indicates the successful setup
- * of an entry zero with the new MSI vector or non-zero for otherwise.
+ * of an entry zero with the new MSI irq or non-zero for otherwise.
  **/
 static int msi_capability_init(struct pci_dev *dev)
 {
 	int status;
 	struct msi_desc *entry;
-	int pos, vector;
+	int pos, irq;
 	u16 control;
+	struct hw_interrupt_type *handler;
 
    	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
 	pci_read_config_word(dev, msi_control_reg(pos), &control);
 	/* MSI Entry Initialization */
-	entry = alloc_msi_entry();
-	if (!entry)
-		return -ENOMEM;
+	handler = &msi_irq_wo_maskbit_type;
+	if (is_mask_bit_support(control))
+		handler = &msi_irq_w_maskbit_type;
 
-	vector = get_msi_vector(dev);
-	if (vector < 0) {
-		kmem_cache_free(msi_cachep, entry);
-		return -EBUSY;
-	}
-	entry->link.head = vector;
-	entry->link.tail = vector;
+	irq = create_msi_irq(handler);
+	if (irq < 0)
+		return irq;
+
+	entry = get_irq_data(irq);
+	entry->link.head = irq;
+	entry->link.tail = irq;
 	entry->msi_attrib.type = PCI_CAP_ID_MSI;
 	entry->msi_attrib.state = 0;			/* Mark it not active */
 	entry->msi_attrib.is_64 = is_64bit_address(control);
 	entry->msi_attrib.entry_nr = 0;
 	entry->msi_attrib.maskbit = is_mask_bit_support(control);
-	entry->msi_attrib.default_vector = dev->irq;	/* Save IOAPIC IRQ */
+	entry->msi_attrib.default_irq = dev->irq;	/* Save IOAPIC IRQ */
 	entry->msi_attrib.pos = pos;
-	dev->irq = vector;
+	dev->irq = irq;
 	entry->dev = dev;
 	if (is_mask_bit_support(control)) {
 		entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos,
 				is_64bit_address(control));
 	}
-	/* Replace with MSI handler */
-	irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit);
 	/* Configure MSI capability structure */
 	status = msi_register_init(dev, entry);
 	if (status != 0) {
-		dev->irq = entry->msi_attrib.default_vector;
-		kmem_cache_free(msi_cachep, entry);
+		dev->irq = entry->msi_attrib.default_irq;
+		destroy_msi_irq(irq);
 		return status;
 	}
 
-	attach_msi_entry(entry, vector);
+	attach_msi_entry(entry, irq);
 	/* Set MSI enabled bits	 */
 	enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
 
@@ -796,8 +713,8 @@ static int msi_capability_init(struct pc
  * @nvec: number of @entries
  *
  * Setup the MSI-X capability structure of device function with a
- * single MSI-X vector. A return of zero indicates the successful setup of
- * requested MSI-X entries with allocated vectors or non-zero for otherwise.
+ * single MSI-X irq. A return of zero indicates the successful setup of
+ * requested MSI-X entries with allocated irqs or non-zero for otherwise.
  **/
 static int msix_capability_init(struct pci_dev *dev,
 				struct msix_entry *entries, int nvec)
@@ -805,7 +722,7 @@ static int msix_capability_init(struct p
 	struct msi_desc *head = NULL, *tail = NULL, *entry = NULL;
 	struct msi_msg msg;
 	int status;
-	int vector, pos, i, j, nr_entries, temp = 0;
+	int irq, pos, i, j, nr_entries, temp = 0;
 	unsigned long phys_addr;
 	u32 table_offset;
  	u16 control;
@@ -827,54 +744,50 @@ static int msix_capability_init(struct p
 
 	/* MSI-X Table Initialization */
 	for (i = 0; i < nvec; i++) {
-		entry = alloc_msi_entry();
-		if (!entry)
-			break;
-		vector = get_msi_vector(dev);
-		if (vector < 0) {
-			kmem_cache_free(msi_cachep, entry);
+		irq = create_msi_irq(&msix_irq_type);
+		if (irq < 0)
 			break;
-		}
 
+		entry = get_irq_data(irq);
  		j = entries[i].entry;
- 		entries[i].vector = vector;
+ 		entries[i].vector = irq;
 		entry->msi_attrib.type = PCI_CAP_ID_MSIX;
  		entry->msi_attrib.state = 0;		/* Mark it not active */
 		entry->msi_attrib.is_64 = 1;
 		entry->msi_attrib.entry_nr = j;
 		entry->msi_attrib.maskbit = 1;
-		entry->msi_attrib.default_vector = dev->irq;
+		entry->msi_attrib.default_irq = dev->irq;
 		entry->msi_attrib.pos = pos;
 		entry->dev = dev;
 		entry->mask_base = base;
 		if (!head) {
-			entry->link.head = vector;
-			entry->link.tail = vector;
+			entry->link.head = irq;
+			entry->link.tail = irq;
 			head = entry;
 		} else {
 			entry->link.head = temp;
 			entry->link.tail = tail->link.tail;
-			tail->link.tail = vector;
-			head->link.head = vector;
+			tail->link.tail = irq;
+			head->link.head = irq;
 		}
-		temp = vector;
+		temp = irq;
 		tail = entry;
-		/* Replace with MSI-X handler */
-		irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
 		/* Configure MSI-X capability structure */
-		status = msi_ops->setup(dev, vector, &msg);
-		if (status < 0)
+		status = msi_ops->setup(dev, irq, &msg);
+		if (status < 0) {
+			destroy_msi_irq(irq);
 			break;
+		}
 
 		write_msi_msg(entry, &msg);
-		attach_msi_entry(entry, vector);
+		attach_msi_entry(entry, irq);
 	}
 	if (i != nvec) {
 		int avail = i - 1;
 		i--;
 		for (; i >= 0; i--) {
-			vector = (entries + i)->vector;
-			msi_free_vector(dev, vector, 0);
+			irq = (entries + i)->vector;
+			msi_free_irq(dev, irq);
 			(entries + i)->vector = 0;
 		}
 		/* If we had some success report the number of irqs
@@ -895,10 +808,10 @@ static int msix_capability_init(struct p
  * @dev: pointer to the pci_dev data structure of MSI device function
  *
  * Setup the MSI capability structure of device function with
- * a single MSI vector upon its software driver call to request for
+ * a single MSI irq upon its software driver call to request for
  * MSI mode enabled on its hardware device function. A return of zero
  * indicates the successful setup of an entry zero with the new MSI
- * vector or non-zero for otherwise.
+ * irq or non-zero for otherwise.
  **/
 int pci_enable_msi(struct pci_dev* dev)
 {
@@ -930,13 +843,13 @@ int pci_enable_msi(struct pci_dev* dev)
 	if (!is_64bit_address(control) && msi_ops->needs_64bit_address)
 		return -EINVAL;
 
-	BUG_ON(!msi_lookup_vector(dev, PCI_CAP_ID_MSI));
+	BUG_ON(!msi_lookup_irq(dev, PCI_CAP_ID_MSI));
 
-	/* Check whether driver already requested for MSI-X vectors */
+	/* Check whether driver already requested for MSI-X irqs */
 	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
-	if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
+	if (pos > 0 && !msi_lookup_irq(dev, PCI_CAP_ID_MSIX)) {
 			printk(KERN_INFO "PCI: %s: Can't enable MSI.  "
-			       "Device already has MSI-X vectors assigned\n",
+			       "Device already has MSI-X irq assigned\n",
 			       pci_name(dev));
 			dev->irq = temp;
 			return -EINVAL;
@@ -948,7 +861,7 @@ int pci_enable_msi(struct pci_dev* dev)
 void pci_disable_msi(struct pci_dev* dev)
 {
 	struct msi_desc *entry;
-	int pos, default_vector;
+	int pos, default_irq;
 	u16 control;
 	unsigned long flags;
 
@@ -976,30 +889,30 @@ void pci_disable_msi(struct pci_dev* dev
 	if (entry->msi_attrib.state) {
 		spin_unlock_irqrestore(&msi_lock, flags);
 		printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without "
-		       "free_irq() on MSI vector %d\n",
+		       "free_irq() on MSI irq %d\n",
 		       pci_name(dev), dev->irq);
 		BUG_ON(entry->msi_attrib.state > 0);
 	} else {
-		default_vector = entry->msi_attrib.default_vector;
+		default_irq = entry->msi_attrib.default_irq;
 		spin_unlock_irqrestore(&msi_lock, flags);
-		msi_free_vector(dev, dev->irq, 0);
+		msi_free_irq(dev, dev->irq);
 
-		/* Restore dev->irq to its default pin-assertion vector */
-		dev->irq = default_vector;
+		/* Restore dev->irq to its default pin-assertion irq */
+		dev->irq = default_irq;
 	}
 }
 
-static int msi_free_vector(struct pci_dev* dev, int vector, int reassign)
+static int msi_free_irq(struct pci_dev* dev, int irq)
 {
 	struct msi_desc *entry;
 	int head, entry_nr, type;
 	void __iomem *base;
 	unsigned long flags;
 
-	msi_ops->teardown(vector);
+	msi_ops->teardown(irq);
 
 	spin_lock_irqsave(&msi_lock, flags);
-	entry = msi_desc[vector];
+	entry = msi_desc[irq];
 	if (!entry || entry->dev != dev) {
 		spin_unlock_irqrestore(&msi_lock, flags);
 		return -EINVAL;
@@ -1011,22 +924,16 @@ static int msi_free_vector(struct pci_de
 	msi_desc[entry->link.head]->link.tail = entry->link.tail;
 	msi_desc[entry->link.tail]->link.head = entry->link.head;
 	entry->dev = NULL;
-	if (!reassign) {
-		vector_irq[vector] = 0;
-		nr_released_vectors++;
-	}
-	msi_desc[vector] = NULL;
+	msi_desc[irq] = NULL;
 	spin_unlock_irqrestore(&msi_lock, flags);
 
-	kmem_cache_free(msi_cachep, entry);
+	destroy_msi_irq(irq);
 
 	if (type == PCI_CAP_ID_MSIX) {
-		if (!reassign)
-			writel(1, base +
-				entry_nr * PCI_MSIX_ENTRY_SIZE +
-				PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
+		writel(1, base + entry_nr * PCI_MSIX_ENTRY_SIZE +
+			PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
 
-		if (head == vector)
+		if (head == irq)
 			iounmap(base);
 	}
 
@@ -1037,15 +944,15 @@ static int msi_free_vector(struct pci_de
  * pci_enable_msix - configure device's MSI-X capability structure
  * @dev: pointer to the pci_dev data structure of MSI-X device function
  * @entries: pointer to an array of MSI-X entries
- * @nvec: number of MSI-X vectors requested for allocation by device driver
+ * @nvec: number of MSI-X irqs requested for allocation by device driver
  *
  * Setup the MSI-X capability structure of device function with the number
- * of requested vectors upon its software driver call to request for
+ * of requested irqs upon its software driver call to request for
  * MSI-X mode enabled on its hardware device function. A return of zero
  * indicates the successful configuration of MSI-X capability structure
- * with new allocated MSI-X vectors. A return of < 0 indicates a failure.
+ * with new allocated MSI-X irqs. A return of < 0 indicates a failure.
  * Or a return of > 0 indicates that driver request is exceeding the number
- * of vectors available. Driver should use the returned value to re-send
+ * of irqs available. Driver should use the returned value to re-send
  * its request.
  **/
 int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
@@ -1088,13 +995,13 @@ int pci_enable_msix(struct pci_dev* dev,
 		}
 	}
 	temp = dev->irq;
-	BUG_ON(!msi_lookup_vector(dev, PCI_CAP_ID_MSIX));
+	BUG_ON(!msi_lookup_irq(dev, PCI_CAP_ID_MSIX));
 
-	/* Check whether driver already requested for MSI vector */
+	/* Check whether driver already requested for MSI irq */
    	if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 &&
-		!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
+		!msi_lookup_irq(dev, PCI_CAP_ID_MSI)) {
 		printk(KERN_INFO "PCI: %s: Can't enable MSI-X.  "
-		       "Device already has an MSI vector assigned\n",
+		       "Device already has an MSI irq assigned\n",
 		       pci_name(dev));
 		dev->irq = temp;
 		return -EINVAL;
@@ -1124,27 +1031,27 @@ void pci_disable_msix(struct pci_dev* de
 	disable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
 
 	temp = dev->irq;
-	if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
-		int state, vector, head, tail = 0, warning = 0;
+	if (!msi_lookup_irq(dev, PCI_CAP_ID_MSIX)) {
+		int state, irq, head, tail = 0, warning = 0;
 		unsigned long flags;
 
-		vector = head = dev->irq;
+		irq = head = dev->irq;
 		dev->irq = temp;			/* Restore pin IRQ */
 		while (head != tail) {
 			spin_lock_irqsave(&msi_lock, flags);
-			state = msi_desc[vector]->msi_attrib.state;
-			tail = msi_desc[vector]->link.tail;
+			state = msi_desc[irq]->msi_attrib.state;
+			tail = msi_desc[irq]->link.tail;
 			spin_unlock_irqrestore(&msi_lock, flags);
 			if (state)
 				warning = 1;
-			else if (vector != head)	/* Release MSI-X vector */
-				msi_free_vector(dev, vector, 0);
-			vector = tail;
+			else if (irq != head)	/* Release MSI-X irq */
+				msi_free_irq(dev, irq);
+			irq = tail;
 		}
-		msi_free_vector(dev, vector, 0);
+		msi_free_irq(dev, irq);
 		if (warning) {
 			printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without "
-			       "free_irq() on all MSI-X vectors\n",
+			       "free_irq() on all MSI-X irqs\n",
 			       pci_name(dev));
 			BUG_ON(warning > 0);
 		}
@@ -1152,11 +1059,11 @@ void pci_disable_msix(struct pci_dev* de
 }
 
 /**
- * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state
+ * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state
  * @dev: pointer to the pci_dev data structure of MSI(X) device function
  *
  * Being called during hotplug remove, from which the device function
- * is hot-removed. All previous assigned MSI/MSI-X vectors, if
+ * is hot-removed. All previous assigned MSI/MSI-X irqs, if
  * allocated for this device function, are reclaimed to unused state,
  * which may be used later on.
  **/
@@ -1170,42 +1077,42 @@ void msi_remove_pci_irq_vectors(struct p
 
 	temp = dev->irq;		/* Save IOAPIC IRQ */
 	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
-	if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
+	if (pos > 0 && !msi_lookup_irq(dev, PCI_CAP_ID_MSI)) {
 		spin_lock_irqsave(&msi_lock, flags);
 		state = msi_desc[dev->irq]->msi_attrib.state;
 		spin_unlock_irqrestore(&msi_lock, flags);
 		if (state) {
 			printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
-			       "called without free_irq() on MSI vector %d\n",
+			       "called without free_irq() on MSI irq %d\n",
 			       pci_name(dev), dev->irq);
 			BUG_ON(state > 0);
-		} else /* Release MSI vector assigned to this device */
-			msi_free_vector(dev, dev->irq, 0);
+		} else /* Release MSI irq assigned to this device */
+			msi_free_irq(dev, dev->irq);
 		dev->irq = temp;		/* Restore IOAPIC IRQ */
 	}
 	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
-	if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
-		int vector, head, tail = 0, warning = 0;
+	if (pos > 0 && !msi_lookup_irq(dev, PCI_CAP_ID_MSIX)) {
+		int irq, head, tail = 0, warning = 0;
 		void __iomem *base = NULL;
 
-		vector = head = dev->irq;
+		irq = head = dev->irq;
 		while (head != tail) {
 			spin_lock_irqsave(&msi_lock, flags);
-			state = msi_desc[vector]->msi_attrib.state;
-			tail = msi_desc[vector]->link.tail;
-			base = msi_desc[vector]->mask_base;
+			state = msi_desc[irq]->msi_attrib.state;
+			tail = msi_desc[irq]->link.tail;
+			base = msi_desc[irq]->mask_base;
 			spin_unlock_irqrestore(&msi_lock, flags);
 			if (state)
 				warning = 1;
-			else if (vector != head) /* Release MSI-X vector */
-				msi_free_vector(dev, vector, 0);
-			vector = tail;
+			else if (irq != head) /* Release MSI-X irq */
+				msi_free_irq(dev, irq);
+			irq = tail;
 		}
-		msi_free_vector(dev, vector, 0);
+		msi_free_irq(dev, irq);
 		if (warning) {
 			iounmap(base);
 			printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
-			       "called without free_irq() on all MSI-X vectors\n",
+			       "called without free_irq() on all MSI-X irqs\n",
 			       pci_name(dev));
 			BUG_ON(warning > 0);
 		}
diff --git a/drivers/pci/msi.h b/drivers/pci/msi.h
index 6793241..435d05a 100644
--- a/drivers/pci/msi.h
+++ b/drivers/pci/msi.h
@@ -8,9 +8,6 @@ #define MSI_H
 
 #include <asm/msi.h>
 
-extern int vector_irq[NR_VECTORS];
-extern void (*interrupt[NR_IRQS])(void);
-
 /*
  * MSI-X Address Register
  */
@@ -58,9 +55,9 @@ struct msi_desc {
 		__u8	maskbit	: 1; 	/* mask-pending bit supported ?   */
 		__u8	state	: 1; 	/* {0: free, 1: busy}		  */
 		__u8	is_64	: 1;	/* Address size: 0=32bit 1=64bit  */
-		__u8	entry_nr;    	/* specific enabled entry 	  */
-		__u8	default_vector; /* default pre-assigned vector    */
 		__u8	pos;	 	/* Location of the msi capability */
+		__u16	entry_nr;    	/* specific enabled entry 	  */
+		unsigned default_irq;	/* default pre-assigned irq	  */
 	}msi_attrib;
 
 	struct {
-- 
1.4.0.gc07e

[PATCH 14/25] x86_64 irq: Move msi message composition into io_apic.c

[Eric W. Biederman]

This removes the hardcoded assumption that irq == vector in the
msi composition code, and it allows the msi message composition
to setup logical mode, or lowest priorirty delivery mode as we
do for other apic interrupts, and with the same selection criteria.

Basically this moves the problem of what is in the msi message into
the architecture irq management code where it belongs.  Not in
a generic layer that doesn't have enough information to compose
msi messages properly.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 arch/x86_64/kernel/io_apic.c |   73 ++++++++++++++++++++++++++++++++++++++++++
 include/asm-x86_64/msi.h     |    7 +---
 include/asm-x86_64/msidef.h  |   47 +++++++++++++++++++++++++++
 3 files changed, 122 insertions(+), 5 deletions(-)

diff --git a/arch/x86_64/kernel/io_apic.c b/arch/x86_64/kernel/io_apic.c
index ae64a63..7ad0980 100644
--- a/arch/x86_64/kernel/io_apic.c
+++ b/arch/x86_64/kernel/io_apic.c
@@ -42,6 +42,7 @@ #include <asm/mach_apic.h>
 #include <asm/acpi.h>
 #include <asm/dma.h>
 #include <asm/nmi.h>
+#include <asm/msidef.h>
 
 #define __apicdebuginit  __init
 
@@ -2061,6 +2062,78 @@ void destroy_irq(unsigned int irq)
 }
 #endif
 
+/*
+ * MSI mesage composition
+ */
+#ifdef CONFIG_PCI_MSI
+static int msi_msg_setup(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
+{
+	/* For now always this code always uses physical delivery
+	 * mode.
+	 */
+	int vector;
+	unsigned dest;
+
+	vector = assign_irq_vector(irq);
+	if (vector >= 0) {
+		cpumask_t tmp;
+
+		cpus_clear(tmp);
+		cpu_set(first_cpu(cpu_online_map), tmp);
+		dest = cpu_mask_to_apicid(tmp);
+
+		msg->address_hi = MSI_ADDR_BASE_HI;
+		msg->address_lo =
+			MSI_ADDR_BASE_LO |
+			((INT_DEST_MODE == 0) ?
+				MSI_ADDR_DEST_MODE_PHYSICAL:
+				MSI_ADDR_DEST_MODE_LOGICAL) |
+			((INT_DELIVERY_MODE != dest_LowestPrio) ?
+				MSI_ADDR_REDIRECTION_CPU:
+				MSI_ADDR_REDIRECTION_LOWPRI) |
+			MSI_ADDR_DEST_ID(dest);
+
+		msg->data = 
+			MSI_DATA_TRIGGER_EDGE |	
+			MSI_DATA_LEVEL_ASSERT |
+			((INT_DELIVERY_MODE != dest_LowestPrio) ?
+				MSI_DATA_DELIVERY_FIXED:
+				MSI_DATA_DELIVERY_LOWPRI) |
+			MSI_DATA_VECTOR(vector); 
+	}
+	return vector;
+}
+
+static void msi_msg_teardown(unsigned int irq)
+{
+	return;
+}
+
+static void msi_msg_set_affinity(unsigned int irq, cpumask_t mask, struct msi_msg *msg)
+{
+	int vector;
+	unsigned dest;
+
+	vector = assign_irq_vector(irq);
+	if (vector > 0) {
+		dest = cpu_mask_to_apicid(mask);
+
+		msg->data &= ~MSI_DATA_VECTOR_MASK;
+		msg->data |= MSI_DATA_VECTOR(vector);
+		msg->address_lo &= ~MSI_ADDR_DEST_ID_MASK;
+		msg->address_lo |= MSI_ADDR_DEST_ID(dest);
+	}
+}
+
+struct msi_ops arch_msi_ops = {
+	.needs_64bit_address = 0,
+	.setup = msi_msg_setup,
+	.teardown = msi_msg_teardown,
+	.target = msi_msg_set_affinity,
+};
+
+#endif
+
 /* --------------------------------------------------------------------------
                           ACPI-based IOAPIC Configuration
    -------------------------------------------------------------------------- */
diff --git a/include/asm-x86_64/msi.h b/include/asm-x86_64/msi.h
index 3ad2346..1876fda 100644
--- a/include/asm-x86_64/msi.h
+++ b/include/asm-x86_64/msi.h
@@ -10,14 +10,11 @@ #include <asm/desc.h>
 #include <asm/mach_apic.h>
 #include <asm/smp.h>
 
-#define LAST_DEVICE_VECTOR	(FIRST_SYSTEM_VECTOR - 1)
-#define MSI_TARGET_CPU_SHIFT	12
-
-extern struct msi_ops msi_apic_ops;
+extern struct msi_ops arch_msi_ops;
 
 static inline int msi_arch_init(void)
 {
-	msi_register(&msi_apic_ops);
+	msi_register(&arch_msi_ops);
 	return 0;
 }
 
diff --git a/include/asm-x86_64/msidef.h b/include/asm-x86_64/msidef.h
new file mode 100644
index 0000000..4667f1a
--- /dev/null
+++ b/include/asm-x86_64/msidef.h
@@ -0,0 +1,47 @@
+#ifndef ASM_MSIDEF_H
+#define ASM_MSIDEF_H
+
+/*
+ * Constants for Intel APIC based MSI messages.
+ */
+
+/*
+ * Shifts for MSI data
+ */
+
+#define MSI_DATA_VECTOR_SHIFT		0
+#define  MSI_DATA_VECTOR_MASK		0x000000ff
+#define	 MSI_DATA_VECTOR(v)		(((v) << MSI_DATA_VECTOR_SHIFT) & MSI_DATA_VECTOR_MASK)
+
+#define MSI_DATA_DELIVERY_MODE_SHIFT	8
+#define  MSI_DATA_DELIVERY_FIXED	(0 << MSI_DATA_DELIVERY_MODE_SHIFT)
+#define  MSI_DATA_DELIVERY_LOWPRI	(1 << MSI_DATA_DELIVERY_MODE_SHIFT)
+
+#define MSI_DATA_LEVEL_SHIFT		14
+#define	 MSI_DATA_LEVEL_DEASSERT	(0 << MSI_DATA_LEVEL_SHIFT)
+#define	 MSI_DATA_LEVEL_ASSERT		(1 << MSI_DATA_LEVEL_SHIFT)
+
+#define MSI_DATA_TRIGGER_SHIFT		15
+#define  MSI_DATA_TRIGGER_EDGE		(0 << MSI_DATA_TRIGGER_SHIFT)
+#define  MSI_DATA_TRIGGER_LEVEL		(1 << MSI_DATA_TRIGGER_SHIFT)
+
+/*
+ * Shift/mask fields for msi address
+ */
+
+#define MSI_ADDR_BASE_HI		0
+#define MSI_ADDR_BASE_LO		0xfee00000
+
+#define MSI_ADDR_DEST_MODE_SHIFT	2
+#define  MSI_ADDR_DEST_MODE_PHYSICAL	(0 << MSI_ADDR_DEST_MODE_SHIFT)
+#define	 MSI_ADDR_DEST_MODE_LOGICAL	(1 << MSI_ADDR_DEST_MODE_SHIFT)
+	
+#define MSI_ADDR_REDIRECTION_SHIFT	3
+#define  MSI_ADDR_REDIRECTION_CPU	(0 << MSI_ADDR_REDIRECTION_SHIFT) /* dedicated cpu */
+#define  MSI_ADDR_REDIRECTION_LOWPRI	(1 << MSI_ADDR_REDIRECTION_SHIFT) /* lowest priority */
+
+#define MSI_ADDR_DEST_ID_SHIFT		12
+#define	 MSI_ADDR_DEST_ID_MASK		0x00ffff0
+#define  MSI_ADDR_DEST_ID(dest)		(((dest) << MSI_ADDR_DEST_ID_SHIFT) & MSI_ADDR_DEST_ID_MASK)
+
+#endif /* ASM_MSIDEF_H */
-- 
1.4.0.gc07e

[PATCH 15/25] i386 irq: Move msi message composition into io_apic.c

[Eric W. Biederman]

This removes the hardcoded assumption that irq == vector in the
msi composition code, and it allows the msi message composition
to setup logical mode, or lowest priorirty delivery mode as we
do for other apic interrupts, and with the same selection criteria.

Basically this moves the problem of what is in the msi message into
the architecture irq management code where it belongs.  Not in
a generic layer that doesn't have enough information to compose
msi messages properly.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 arch/i386/kernel/io_apic.c |   70 ++++++++++++++++++++++++++++++++++++++++++++
 include/asm-i386/msi.h     |    7 +---
 include/asm-i386/msidef.h  |   47 ++++++++++++++++++++++++++++++
 3 files changed, 119 insertions(+), 5 deletions(-)

diff --git a/arch/i386/kernel/io_apic.c b/arch/i386/kernel/io_apic.c
index 04f78ff..68af125 100644
--- a/arch/i386/kernel/io_apic.c
+++ b/arch/i386/kernel/io_apic.c
@@ -32,6 +32,7 @@ #include <linux/compiler.h>
 #include <linux/acpi.h>
 #include <linux/module.h>
 #include <linux/sysdev.h>
+#include <linux/pci.h>
 
 #include <asm/io.h>
 #include <asm/smp.h>
@@ -39,6 +40,7 @@ #include <asm/desc.h>
 #include <asm/timer.h>
 #include <asm/i8259.h>
 #include <asm/nmi.h>
+#include <asm/msidef.h>
 
 #include <mach_apic.h>
 #include <mach_apicdef.h>
@@ -2545,6 +2547,74 @@ void destroy_irq(unsigned int irq)
 }
 #endif /* CONFIG_PCI_MSI */
 
+/*
+ * MSI mesage composition
+ */
+#ifdef CONFIG_PCI_MSI
+static int msi_msg_setup(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
+{
+	/* For now always this code always uses physical delivery
+	 * mode.
+	 */
+	int vector;
+	unsigned dest;
+
+	vector = assign_irq_vector(irq);
+	if (vector >= 0) {
+		dest = cpu_mask_to_apicid(TARGET_CPUS);
+
+		msg->address_hi = MSI_ADDR_BASE_HI;
+		msg->address_lo =
+			MSI_ADDR_BASE_LO |
+			((INT_DEST_MODE == 0) ?
+				MSI_ADDR_DEST_MODE_PHYSICAL:
+				MSI_ADDR_DEST_MODE_LOGICAL) |
+			((INT_DELIVERY_MODE != dest_LowestPrio) ?
+				MSI_ADDR_REDIRECTION_CPU:
+				MSI_ADDR_REDIRECTION_LOWPRI) |
+			MSI_ADDR_DEST_ID(dest);
+
+		msg->data = 
+			MSI_DATA_TRIGGER_EDGE |	
+			MSI_DATA_LEVEL_ASSERT |
+			((INT_DELIVERY_MODE != dest_LowestPrio) ?
+				MSI_DATA_DELIVERY_FIXED:
+				MSI_DATA_DELIVERY_LOWPRI) |
+			MSI_DATA_VECTOR(vector); 
+	}
+	return vector;
+}
+
+static void msi_msg_teardown(unsigned int irq)
+{
+	return;
+}
+
+static void msi_msg_set_affinity(unsigned int irq, cpumask_t mask, struct msi_msg *msg)
+{
+	int vector;
+	unsigned dest;
+
+	vector = assign_irq_vector(irq);
+	if (vector > 0) {
+		dest = cpu_mask_to_apicid(mask);
+
+		msg->data &= ~MSI_DATA_VECTOR_MASK;
+		msg->data |= MSI_DATA_VECTOR(vector);
+		msg->address_lo &= ~MSI_ADDR_DEST_ID_MASK;
+		msg->address_lo |= MSI_ADDR_DEST_ID(dest);
+	}
+}
+
+struct msi_ops arch_msi_ops = {
+	.needs_64bit_address = 0,
+	.setup = msi_msg_setup,
+	.teardown = msi_msg_teardown,
+	.target = msi_msg_set_affinity,
+};
+
+#endif /* CONFIG_PCI_MSI */
+
 /* --------------------------------------------------------------------------
                           ACPI-based IOAPIC Configuration
    -------------------------------------------------------------------------- */
diff --git a/include/asm-i386/msi.h b/include/asm-i386/msi.h
index b11c4b7..7368a89 100644
--- a/include/asm-i386/msi.h
+++ b/include/asm-i386/msi.h
@@ -9,14 +9,11 @@ #define ASM_MSI_H
 #include <asm/desc.h>
 #include <mach_apic.h>
 
-#define LAST_DEVICE_VECTOR	(FIRST_SYSTEM_VECTOR - 1)
-#define MSI_TARGET_CPU_SHIFT	12
-
-extern struct msi_ops msi_apic_ops;
+extern struct msi_ops arch_msi_ops;
 
 static inline int msi_arch_init(void)
 {
-	msi_register(&msi_apic_ops);
+	msi_register(&arch_msi_ops);
 	return 0;
 }
 
diff --git a/include/asm-i386/msidef.h b/include/asm-i386/msidef.h
new file mode 100644
index 0000000..4667f1a
--- /dev/null
+++ b/include/asm-i386/msidef.h
@@ -0,0 +1,47 @@
+#ifndef ASM_MSIDEF_H
+#define ASM_MSIDEF_H
+
+/*
+ * Constants for Intel APIC based MSI messages.
+ */
+
+/*
+ * Shifts for MSI data
+ */
+
+#define MSI_DATA_VECTOR_SHIFT		0
+#define  MSI_DATA_VECTOR_MASK		0x000000ff
+#define	 MSI_DATA_VECTOR(v)		(((v) << MSI_DATA_VECTOR_SHIFT) & MSI_DATA_VECTOR_MASK)
+
+#define MSI_DATA_DELIVERY_MODE_SHIFT	8
+#define  MSI_DATA_DELIVERY_FIXED	(0 << MSI_DATA_DELIVERY_MODE_SHIFT)
+#define  MSI_DATA_DELIVERY_LOWPRI	(1 << MSI_DATA_DELIVERY_MODE_SHIFT)
+
+#define MSI_DATA_LEVEL_SHIFT		14
+#define	 MSI_DATA_LEVEL_DEASSERT	(0 << MSI_DATA_LEVEL_SHIFT)
+#define	 MSI_DATA_LEVEL_ASSERT		(1 << MSI_DATA_LEVEL_SHIFT)
+
+#define MSI_DATA_TRIGGER_SHIFT		15
+#define  MSI_DATA_TRIGGER_EDGE		(0 << MSI_DATA_TRIGGER_SHIFT)
+#define  MSI_DATA_TRIGGER_LEVEL		(1 << MSI_DATA_TRIGGER_SHIFT)
+
+/*
+ * Shift/mask fields for msi address
+ */
+
+#define MSI_ADDR_BASE_HI		0
+#define MSI_ADDR_BASE_LO		0xfee00000
+
+#define MSI_ADDR_DEST_MODE_SHIFT	2
+#define  MSI_ADDR_DEST_MODE_PHYSICAL	(0 << MSI_ADDR_DEST_MODE_SHIFT)
+#define	 MSI_ADDR_DEST_MODE_LOGICAL	(1 << MSI_ADDR_DEST_MODE_SHIFT)
+	
+#define MSI_ADDR_REDIRECTION_SHIFT	3
+#define  MSI_ADDR_REDIRECTION_CPU	(0 << MSI_ADDR_REDIRECTION_SHIFT) /* dedicated cpu */
+#define  MSI_ADDR_REDIRECTION_LOWPRI	(1 << MSI_ADDR_REDIRECTION_SHIFT) /* lowest priority */
+
+#define MSI_ADDR_DEST_ID_SHIFT		12
+#define	 MSI_ADDR_DEST_ID_MASK		0x00ffff0
+#define  MSI_ADDR_DEST_ID(dest)		(((dest) << MSI_ADDR_DEST_ID_SHIFT) & MSI_ADDR_DEST_ID_MASK)
+
+#endif /* ASM_MSIDEF_H */
-- 
1.4.0.gc07e

[PATCH 16/25] msi: Only build msi-apic.c on ia64

[Eric W. Biederman]

After the previous changes ia64 is the only architecture useing msi-apic.c

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 drivers/pci/Makefile |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile
index f2d152b..f4c7a4b 100644
--- a/drivers/pci/Makefile
+++ b/drivers/pci/Makefile
@@ -27,7 +27,8 @@ obj-$(CONFIG_PPC64) += setup-bus.o
 obj-$(CONFIG_MIPS) += setup-bus.o setup-irq.o
 obj-$(CONFIG_X86_VISWS) += setup-irq.o
 
-msiobj-y := msi.o msi-apic.o
+msiobj-y := msi.o
+msiobj-$(CONFIG_IA64) := msi-apic.o
 msiobj-$(CONFIG_IA64_GENERIC) += msi-altix.o
 msiobj-$(CONFIG_IA64_SGI_SN2) += msi-altix.o
 obj-$(CONFIG_PCI_MSI) += $(msiobj-y)
-- 
1.4.0.gc07e

[PATCH 17/25] x86_64 irq: Remove the msi assumption that irq == vector

[Eric W. Biederman]

This patch removes the change in behavior of the irq allocation
code when CONFIG_PCI_MSI is defined.  Removing all instances
of the assumption that irq == vector.

create_irq is rewritten to first allocate a free irq and then to
assign that irq a vector.

assign_irq_vector is made static and the AUTO_ASSIGN case which
allocates an vector not bound to an irq is removed.

The ioapic vector methods are removed, and everything now
works with irqs.

The definition of NR_IRQS no longer depends on CONFIG_PCI_MSI

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
---
 arch/x86_64/kernel/io_apic.c |  147 +++++++++++++-----------------------------
 include/asm-x86_64/hw_irq.h  |    1 
 include/asm-x86_64/io_apic.h |   40 -----------
 include/asm-x86_64/irq.h     |    5 -
 4 files changed, 45 insertions(+), 148 deletions(-)

diff --git a/arch/x86_64/kernel/io_apic.c b/arch/x86_64/kernel/io_apic.c
index 7ad0980..1a63a0e 100644
--- a/arch/x86_64/kernel/io_apic.c
+++ b/arch/x86_64/kernel/io_apic.c
@@ -44,6 +44,8 @@ #include <asm/dma.h>
 #include <asm/nmi.h>
 #include <asm/msidef.h>
 
+static int assign_irq_vector(int irq);
+
 #define __apicdebuginit  __init
 
 int sis_apic_bug; /* not actually supported, dummy for compile */
@@ -83,14 +85,6 @@ static struct irq_pin_list {
 	short apic, pin, next;
 } irq_2_pin[PIN_MAP_SIZE];
 
-int vector_irq[NR_VECTORS] __read_mostly = { [0 ... NR_VECTORS - 1] = -1};
-#ifdef CONFIG_PCI_MSI
-#define vector_to_irq(vector) 	\
-	(platform_legacy_irq(vector) ? vector : vector_irq[vector])
-#else
-#define vector_to_irq(vector)	(vector)
-#endif
-
 #define __DO_ACTION(R, ACTION, FINAL)					\
 									\
 {									\
@@ -135,7 +129,7 @@ static void set_ioapic_affinity_irq(unsi
 
 	spin_lock_irqsave(&ioapic_lock, flags);
 	__DO_ACTION(1, = dest, )
-	set_irq_info(irq, mask);
+	set_native_irq_info(irq, mask);
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 }
 #endif
@@ -834,18 +828,14 @@ static inline int IO_APIC_irq_trigger(in
 /* irq_vectors is indexed by the sum of all RTEs in all I/O APICs. */
 u8 irq_vector[NR_IRQ_VECTORS] __read_mostly = { FIRST_DEVICE_VECTOR , 0 };
 
-int assign_irq_vector(int irq)
+static int __assign_irq_vector(int irq)
 {
 	static int current_vector = FIRST_DEVICE_VECTOR, offset = 0;
-	unsigned long flags;
 	int vector;
 
-	BUG_ON(irq != AUTO_ASSIGN && (unsigned)irq >= NR_IRQ_VECTORS);
-
-	spin_lock_irqsave(&vector_lock, flags);
+	BUG_ON((unsigned)irq >= NR_IRQ_VECTORS);
 
-	if (irq != AUTO_ASSIGN && IO_APIC_VECTOR(irq) > 0) {
-		spin_unlock_irqrestore(&vector_lock, flags);
+	if (IO_APIC_VECTOR(irq) > 0) {
 		return IO_APIC_VECTOR(irq);
 	}
 next:
@@ -860,10 +850,18 @@ next:
 	}
 
 	vector = current_vector;
-	vector_irq[vector] = irq;
-	if (irq != AUTO_ASSIGN)
-		IO_APIC_VECTOR(irq) = vector;
+	IO_APIC_VECTOR(irq) = vector;
 
+	return vector;
+}
+
+static int assign_irq_vector(int irq)
+{
+	int vector;
+	unsigned long flags;
+
+	spin_lock_irqsave(&vector_lock, flags);
+	vector = __assign_irq_vector(irq);
 	spin_unlock_irqrestore(&vector_lock, flags);
 
 	return vector;
@@ -879,18 +877,14 @@ #define IOAPIC_LEVEL	1
 
 static void ioapic_register_intr(int irq, int vector, unsigned long trigger)
 {
-	unsigned idx;
-
-	idx = use_pci_vector() && !platform_legacy_irq(irq) ? vector : irq;
-
 	if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
 			trigger == IOAPIC_LEVEL)
-		set_irq_chip_and_handler(idx, &ioapic_chip,
+		set_irq_chip_and_handler(irq, &ioapic_chip,
 					 handle_fasteoi_irq);
 	else
-		set_irq_chip_and_handler(idx, &ioapic_chip,
+		set_irq_chip_and_handler(irq, &ioapic_chip,
 					 handle_edge_irq);
-	set_intr_gate(vector, interrupt[idx]);
+	set_intr_gate(vector, interrupt[irq]);
 }
 
 static void __init setup_IO_APIC_irqs(void)
@@ -1110,17 +1104,12 @@ void __apicdebuginit print_IO_APIC(void)
 		);
 	}
 	}
-	if (use_pci_vector())
-		printk(KERN_INFO "Using vector-based indexing\n");
 	printk(KERN_DEBUG "IRQ to pin mappings:\n");
 	for (i = 0; i < NR_IRQS; i++) {
 		struct irq_pin_list *entry = irq_2_pin + i;
 		if (entry->pin < 0)
 			continue;
- 		if (use_pci_vector() && !platform_legacy_irq(i))
-			printk(KERN_DEBUG "IRQ%d ", IO_APIC_VECTOR(i));
-		else
-			printk(KERN_DEBUG "IRQ%d ", i);
+		printk(KERN_DEBUG "IRQ%d ", i);
 		for (;;) {
 			printk("-> %d:%d", entry->apic, entry->pin);
 			if (!entry->next)
@@ -1523,42 +1512,8 @@ static unsigned int startup_ioapic_irq(u
 	return was_pending;
 }
 
-static unsigned int startup_ioapic_vector(unsigned int vector)
-{
-	int irq = vector_to_irq(vector);
-
-	return startup_ioapic_irq(irq);
-}
-
-static void mask_ioapic_vector (unsigned int vector)
+static int ioapic_retrigger_irq(unsigned int irq)
 {
-	int irq = vector_to_irq(vector);
-
-	mask_IO_APIC_irq(irq);
-}
-
-static void unmask_ioapic_vector (unsigned int vector)
-{
-	int irq = vector_to_irq(vector);
-
-	unmask_IO_APIC_irq(irq);
-}
-
-#ifdef CONFIG_SMP
-static void set_ioapic_affinity_vector (unsigned int vector,
-					cpumask_t cpu_mask)
-{
-	int irq = vector_to_irq(vector);
-
-	set_native_irq_info(vector, cpu_mask);
-	set_ioapic_affinity_irq(irq, cpu_mask);
-}
-#endif // CONFIG_SMP
-
-static int ioapic_retrigger_vector(unsigned int vector)
-{
-	int irq = vector_to_irq(vector);
-
 	send_IPI_self(IO_APIC_VECTOR(irq));
 
 	return 1;
@@ -1605,15 +1560,15 @@ static void ack_apic_level(unsigned int 
 
 static struct irq_chip ioapic_chip __read_mostly = {
 	.name 		= "IO-APIC",
-	.startup 	= startup_ioapic_vector,
-	.mask	 	= mask_ioapic_vector,
-	.unmask	 	= unmask_ioapic_vector,
+	.startup 	= startup_ioapic_irq,
+	.mask	 	= mask_IO_APIC_irq,
+	.unmask	 	= unmask_IO_APIC_irq,
 	.ack 		= ack_apic_edge,
 	.eoi 		= ack_apic_level,
 #ifdef CONFIG_SMP
-	.set_affinity 	= set_ioapic_affinity_vector,
+	.set_affinity 	= set_ioapic_affinity_irq,
 #endif
-	.retrigger	= ioapic_retrigger_vector,
+	.retrigger	= ioapic_retrigger_irq,
 };
 
 static inline void init_IO_APIC_traps(void)
@@ -1633,11 +1588,6 @@ static inline void init_IO_APIC_traps(vo
 	 */
 	for (irq = 0; irq < NR_IRQS ; irq++) {
 		int tmp = irq;
-		if (use_pci_vector()) {
-			if (!platform_legacy_irq(tmp))
-				if ((tmp = vector_to_irq(tmp)) == -1)
-					continue;
-		}
 		if (IO_APIC_IRQ(tmp) && !IO_APIC_VECTOR(tmp)) {
 			/*
 			 * Hmm.. We don't have an entry for this,
@@ -2014,34 +1964,31 @@ static int __init ioapic_init_sysfs(void
 
 device_initcall(ioapic_init_sysfs);
 
-#ifdef CONFIG_PCI_MSI
 /*
- * Dynamic irq allocate and deallocation for MSI
+ * Dynamic irq allocate and deallocation
  */
 int create_irq(void)
 {
-	/* Hack of the day: irq == vector.
-	 *
-	 * Ultimately this will be be more general,
-	 * and not depend on the irq to vector identity mapping.
-	 * But this version is needed until msi.c can cope with
-	 * the more general form.
-	 */
-	int irq, vector;
+	/* Allocate an unused irq */
+	int irq, new, vector;
 	unsigned long flags;
-	vector = assign_irq_vector(AUTO_ASSIGN);
-	irq = vector;
 
-	if (vector >= 0) {
-		struct irq_desc *desc;
+	irq = -ENOSPC;
+	spin_lock_irqsave(&vector_lock, flags);
+	for (new = (NR_IRQS - 1); new >= 0; new--) {
+		if (platform_legacy_irq(new))
+			continue;
+		if (irq_vector[new] != 0)
+			continue;
+		vector = __assign_irq_vector(new);
+		if (likely(vector > 0))
+			irq = new;
+		break;
+	}
+	spin_unlock_irqrestore(&vector_lock, flags);
 
-		spin_lock_irqsave(&vector_lock, flags);
-		vector_irq[vector] = irq;
-		irq_vector[irq] = vector;
-		spin_unlock_irqrestore(&vector_lock, flags);
-		
+	if (irq >= 0) {
 		set_intr_gate(vector, interrupt[irq]);
-
 		dynamic_irq_init(irq);
 	}
 	return irq;
@@ -2050,17 +1997,13 @@ int create_irq(void)
 void destroy_irq(unsigned int irq)
 {
 	unsigned long flags;
-	unsigned int vector;
 
 	dynamic_irq_cleanup(irq);
 
 	spin_lock_irqsave(&vector_lock, flags);
-	vector = irq_vector[irq];
-	vector_irq[vector] = -1;
 	irq_vector[irq] = 0;
 	spin_unlock_irqrestore(&vector_lock, flags);
 }
-#endif
 
 /*
  * MSI mesage composition
@@ -2216,7 +2159,7 @@ int io_apic_set_pci_routing (int ioapic,
 	spin_lock_irqsave(&ioapic_lock, flags);
 	io_apic_write(ioapic, 0x11+2*pin, *(((int *)&entry)+1));
 	io_apic_write(ioapic, 0x10+2*pin, *(((int *)&entry)+0));
-	set_native_irq_info(use_pci_vector() ?  entry.vector : irq, TARGET_CPUS);
+	set_native_irq_info(irq, TARGET_CPUS);
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 
 	return 0;
diff --git a/include/asm-x86_64/hw_irq.h b/include/asm-x86_64/hw_irq.h
index f5da94a..1a8dc18 100644
--- a/include/asm-x86_64/hw_irq.h
+++ b/include/asm-x86_64/hw_irq.h
@@ -75,7 +75,6 @@ #define FIRST_SYSTEM_VECTOR	0xef   /* du
 #ifndef __ASSEMBLY__
 extern u8 irq_vector[NR_IRQ_VECTORS];
 #define IO_APIC_VECTOR(irq)	(irq_vector[irq])
-#define AUTO_ASSIGN		-1
 
 /*
  * Various low-level irq details needed by irq.c, process.c,
diff --git a/include/asm-x86_64/io_apic.h b/include/asm-x86_64/io_apic.h
index fb7a090..2885bea 100644
--- a/include/asm-x86_64/io_apic.h
+++ b/include/asm-x86_64/io_apic.h
@@ -12,45 +12,7 @@ #include <asm/mpspec.h>
 
 #ifdef CONFIG_X86_IO_APIC
 
-#ifdef CONFIG_PCI_MSI
-static inline int use_pci_vector(void)	{return 1;}
-static inline void disable_edge_ioapic_vector(unsigned int vector) { }
-static inline void mask_and_ack_level_ioapic_vector(unsigned int vector) { }
-static inline void end_edge_ioapic_vector (unsigned int vector) { }
-#define startup_level_ioapic	startup_level_ioapic_vector
-#define shutdown_level_ioapic	mask_IO_APIC_vector
-#define enable_level_ioapic	unmask_IO_APIC_vector
-#define disable_level_ioapic	mask_IO_APIC_vector
-#define mask_and_ack_level_ioapic mask_and_ack_level_ioapic_vector
-#define end_level_ioapic	end_level_ioapic_vector
-#define set_ioapic_affinity	set_ioapic_affinity_vector
-
-#define startup_edge_ioapic 	startup_edge_ioapic_vector
-#define shutdown_edge_ioapic 	disable_edge_ioapic_vector
-#define enable_edge_ioapic 	unmask_IO_APIC_vector
-#define disable_edge_ioapic 	disable_edge_ioapic_vector
-#define ack_edge_ioapic 	ack_edge_ioapic_vector
-#define end_edge_ioapic 	end_edge_ioapic_vector
-#else
 static inline int use_pci_vector(void)	{return 0;}
-static inline void disable_edge_ioapic_irq(unsigned int irq) { }
-static inline void mask_and_ack_level_ioapic_irq(unsigned int irq) { }
-static inline void end_edge_ioapic_irq (unsigned int irq) { }
-#define startup_level_ioapic	startup_level_ioapic_irq
-#define shutdown_level_ioapic	mask_IO_APIC_irq
-#define enable_level_ioapic	unmask_IO_APIC_irq
-#define disable_level_ioapic	mask_IO_APIC_irq
-#define mask_and_ack_level_ioapic mask_and_ack_level_ioapic_irq
-#define end_level_ioapic	end_level_ioapic_irq
-#define set_ioapic_affinity	set_ioapic_affinity_irq
-
-#define startup_edge_ioapic 	startup_edge_ioapic_irq
-#define shutdown_edge_ioapic 	disable_edge_ioapic_irq
-#define enable_edge_ioapic 	unmask_IO_APIC_irq
-#define disable_edge_ioapic 	disable_edge_ioapic_irq
-#define ack_edge_ioapic 	ack_edge_ioapic_irq
-#define end_edge_ioapic 	end_edge_ioapic_irq
-#endif
 
 #define APIC_MISMATCH_DEBUG
 
@@ -213,8 +175,6 @@ #else  /* !CONFIG_X86_IO_APIC */
 #define io_apic_assign_pci_irqs 0
 #endif
 
-extern int assign_irq_vector(int irq);
-
 void enable_NMI_through_LVT0 (void * dummy);
 
 extern spinlock_t i8259A_lock;
diff --git a/include/asm-x86_64/irq.h b/include/asm-x86_64/irq.h
index 9db5a1b..0c8d570 100644
--- a/include/asm-x86_64/irq.h
+++ b/include/asm-x86_64/irq.h
@@ -31,13 +31,8 @@ #define NR_VECTORS 256
 
 #define FIRST_SYSTEM_VECTOR	0xef   /* duplicated in hw_irq.h */
 
-#ifdef CONFIG_PCI_MSI
-#define NR_IRQS FIRST_SYSTEM_VECTOR
-#define NR_IRQ_VECTORS NR_IRQS
-#else
 #define NR_IRQS 224
 #define NR_IRQ_VECTORS (32 * NR_CPUS)
-#endif
 
 static __inline__ int irq_canonicalize(int irq)
 {
-- 
1.4.0.gc07e