diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
index 8059531bf0ac..9a785293203f 100644
--- a/arch/sparc64/kernel/entry.S
+++ b/arch/sparc64/kernel/entry.S
@@ -430,15 +430,15 @@ do_ivec:
 	membar		#Sync
 
 	sethi		%hi(ivector_table), %g2
-	sllx		%g3, 3, %g3
+	sllx		%g3, 4, %g3
 	or		%g2, %lo(ivector_table), %g2
 	add		%g2, %g3, %g3
 
 	TRAP_LOAD_IRQ_WORK(%g6, %g1)
 
-	lduw		[%g6], %g5		/* g5 = irq_work(cpu) */
-	stw		%g5, [%g3 + 0x00]	/* bucket->irq_chain = g5 */
-	stw		%g3, [%g6]		/* irq_work(cpu) = bucket */
+	ldx		[%g6], %g5		/* g5 = irq_work(cpu) */
+	stx		%g5, [%g3 + 0x00]	/* bucket->irq_chain = g5 */
+	stx		%g3, [%g6]		/* irq_work(cpu) = bucket */
 	wr		%g0, 1 << PIL_DEVICE_IRQ, %set_softint
 	retry
 do_ivec_xcall:
diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c
index 045ab27d4271..4db4dd576210 100644
--- a/arch/sparc64/kernel/irq.c
+++ b/arch/sparc64/kernel/irq.c
@@ -59,23 +59,20 @@
  * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S
  */
 struct ino_bucket {
-	/* Next handler in per-CPU IRQ worklist.  We know that
-	 * bucket pointers have the high 32-bits clear, so to
-	 * save space we only store the bits we need.
-	 */
-/*0x00*/unsigned int irq_chain;
+/*0x00*/unsigned long irq_chain;
 
 	/* Virtual interrupt number assigned to this INO.  */
-/*0x04*/unsigned int virt_irq;
+/*0x08*/unsigned int virt_irq;
+/*0x0c*/unsigned int __pad;
 };
 
 #define NUM_IVECS	(IMAP_INR + 1)
 struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES)));
 
 #define __irq_ino(irq) \
-        (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0])
-#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq))
-#define __irq(bucket) ((unsigned int)(unsigned long)(bucket))
+        (((struct ino_bucket *)(irq)) - &ivector_table[0])
+#define __bucket(irq) ((struct ino_bucket *)(irq))
+#define __irq(bucket) ((unsigned long)(bucket))
 
 /* This has to be in the main kernel image, it cannot be
  * turned into per-cpu data.  The reason is that the main
@@ -87,13 +84,13 @@ struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BY
 #define irq_work(__cpu)	&(trap_block[(__cpu)].irq_worklist)
 
 static struct {
-	unsigned int irq;
+	unsigned long irq;
 	unsigned int dev_handle;
 	unsigned int dev_ino;
 } virt_to_real_irq_table[NR_IRQS];
 static DEFINE_SPINLOCK(virt_irq_alloc_lock);
 
-unsigned char virt_irq_alloc(unsigned int real_irq)
+unsigned char virt_irq_alloc(unsigned long real_irq)
 {
 	unsigned long flags;
 	unsigned char ent;
@@ -134,7 +131,7 @@ void virt_irq_free(unsigned int virt_irq)
 }
 #endif
 
-static unsigned int virt_to_real_irq(unsigned char virt_irq)
+static unsigned long virt_to_real_irq(unsigned char virt_irq)
 {
 	return virt_to_real_irq_table[virt_irq].irq;
 }
@@ -227,7 +224,7 @@ struct irq_handler_data {
 
 static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq)
 {
-	unsigned int real_irq = virt_to_real_irq(virt_irq);
+	unsigned long real_irq = virt_to_real_irq(virt_irq);
 	struct ino_bucket *bucket = NULL;
 
 	if (likely(real_irq))
@@ -694,18 +691,28 @@ void handler_irq(int irq, struct pt_regs *regs)
 {
 	struct ino_bucket *bucket;
 	struct pt_regs *old_regs;
+	unsigned long pstate;
 
 	clear_softint(1 << irq);
 
 	old_regs = set_irq_regs(regs);
 	irq_enter();
 
-	/* Sliiiick... */
-	bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0));
+	/* Grab an atomic snapshot of the pending IVECs.  */
+	__asm__ __volatile__("rdpr	%%pstate, %0\n\t"
+			     "wrpr	%0, %3, %%pstate\n\t"
+			     "ldx	[%2], %1\n\t"
+			     "stx	%%g0, [%2]\n\t"
+			     "wrpr	%0, 0x0, %%pstate\n\t"
+			     : "=&r" (pstate), "=&r" (bucket)
+			     : "r" (irq_work(smp_processor_id())),
+			       "i" (PSTATE_IE)
+			     : "memory");
+
 	while (bucket) {
 		struct ino_bucket *next = __bucket(bucket->irq_chain);
 
-		bucket->irq_chain = 0;
+		bucket->irq_chain = 0UL;
 		__do_IRQ(bucket->virt_irq);
 
 		bucket = next;
@@ -808,7 +815,7 @@ void init_irqwork_curcpu(void)
 {
 	int cpu = hard_smp_processor_id();
 
-	trap_block[cpu].irq_worklist = 0;
+	trap_block[cpu].irq_worklist = 0UL;
 }
 
 /* Please be very careful with register_one_mondo() and
diff --git a/arch/sparc64/kernel/sun4v_ivec.S b/arch/sparc64/kernel/sun4v_ivec.S
index 574bc248bca6..e3e9d4c1574b 100644
--- a/arch/sparc64/kernel/sun4v_ivec.S
+++ b/arch/sparc64/kernel/sun4v_ivec.S
@@ -101,14 +101,14 @@ sun4v_dev_mondo:
 
 	/* Get &ivector_table[IVEC] into %g4.  */
 	sethi	%hi(ivector_table), %g4
-	sllx	%g3, 3, %g3
+	sllx	%g3, 4, %g3
 	or	%g4, %lo(ivector_table), %g4
 	add	%g4, %g3, %g4
 
 	/* Insert ivector_table[] entry into __irq_work[] queue.  */
-	lduw	[%g1], %g2		/* g2 = irq_work(cpu) */
-	stw	%g2, [%g4 + 0x00]	/* bucket->irq_chain = g2 */
-	stw	%g4, [%g1]		/* irq_work(cpu) = bucket */
+	ldx	[%g1], %g2		/* g2 = irq_work(cpu) */
+	stx	%g2, [%g4 + 0x00]	/* bucket->irq_chain = g2 */
+	stx	%g4, [%g1]		/* irq_work(cpu) = bucket */
 
 	/* Signal the interrupt by setting (1 << pil) in %softint.  */
 	wr	%g0, 1 << PIL_DEVICE_IRQ, %set_softint
diff --git a/include/asm-sparc64/cpudata.h b/include/asm-sparc64/cpudata.h
index 98a6e609163e..379c219c3b7a 100644
--- a/include/asm-sparc64/cpudata.h
+++ b/include/asm-sparc64/cpudata.h
@@ -75,12 +75,11 @@ struct trap_per_cpu {
 	unsigned long		tsb_huge_temp;
 
 /* Dcache line 8: IRQ work list, and keep trap_block a power-of-2 in size.  */
-	unsigned int		irq_worklist;
+	unsigned long		irq_worklist;
 	unsigned int		cpu_mondo_qmask;
 	unsigned int		dev_mondo_qmask;
 	unsigned int		resum_qmask;
 	unsigned int		nonresum_qmask;
-	unsigned int		__pad2[1];
 	void			*hdesc;
 } __attribute__((aligned(64)));
 extern struct trap_per_cpu trap_block[NR_CPUS];
@@ -129,10 +128,10 @@ extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch,
 #define TRAP_PER_CPU_TSB_HUGE		0xd0
 #define TRAP_PER_CPU_TSB_HUGE_TEMP	0xd8
 #define TRAP_PER_CPU_IRQ_WORKLIST	0xe0
-#define TRAP_PER_CPU_CPU_MONDO_QMASK	0xe4
-#define TRAP_PER_CPU_DEV_MONDO_QMASK	0xe8
-#define TRAP_PER_CPU_RESUM_QMASK	0xec
-#define TRAP_PER_CPU_NONRESUM_QMASK	0xf0
+#define TRAP_PER_CPU_CPU_MONDO_QMASK	0xe8
+#define TRAP_PER_CPU_DEV_MONDO_QMASK	0xec
+#define TRAP_PER_CPU_RESUM_QMASK	0xf0
+#define TRAP_PER_CPU_NONRESUM_QMASK	0xf4
 
 #define TRAP_BLOCK_SZ_SHIFT		8
 
diff --git a/include/asm-sparc64/irq.h b/include/asm-sparc64/irq.h
index bad3c28cdee8..24841c22e81b 100644
--- a/include/asm-sparc64/irq.h
+++ b/include/asm-sparc64/irq.h
@@ -59,7 +59,7 @@ extern unsigned int sun4u_build_msi(u32 portid, unsigned int *virt_irq_p,
 extern void sun4u_destroy_msi(unsigned int virt_irq);
 extern unsigned int sbus_build_irq(void *sbus, unsigned int ino);
 
-extern unsigned char virt_irq_alloc(unsigned int real_irq);
+extern unsigned char virt_irq_alloc(unsigned long real_irq);
 #ifdef CONFIG_PCI_MSI
 extern void virt_irq_free(unsigned int virt_irq);
 #endif