diff --git a/drivers/usb/gadget/lh7a40x_udc.c b/drivers/usb/gadget/lh7a40x_udc.c
index e02fea5a5433..1a362c5e7f3d 100644
--- a/drivers/usb/gadget/lh7a40x_udc.c
+++ b/drivers/usb/gadget/lh7a40x_udc.c
@@ -1062,11 +1062,11 @@ static int lh7a40x_ep_enable(struct usb_ep *_ep,
 	ep->pio_irqs = 0;
 	ep->ep.maxpacket = le16_to_cpu(desc->wMaxPacketSize);
 
+	spin_unlock_irqrestore(&ep->dev->lock, flags);
+
 	/* Reset halt state (does flush) */
 	lh7a40x_set_halt(_ep, 0);
 
-	spin_unlock_irqrestore(&ep->dev->lock, flags);
-
 	DEBUG("%s: enabled %s\n", __FUNCTION__, _ep->name);
 	return 0;
 }
@@ -1775,6 +1775,7 @@ static void lh7a40x_ep0_setup(struct lh7a40x_udc *dev, u32 csr)
 					break;
 
 				qep = &dev->ep[ep_num];
+				spin_unlock(&dev->lock);
 				if (ctrl.bRequest == USB_REQ_SET_FEATURE) {
 					DEBUG_SETUP("SET_FEATURE (%d)\n",
 						    ep_num);
@@ -1784,6 +1785,7 @@ static void lh7a40x_ep0_setup(struct lh7a40x_udc *dev, u32 csr)
 						    ep_num);
 					lh7a40x_set_halt(&qep->ep, 0);
 				}
+				spin_lock(&dev->lock);
 				usb_set_index(0);
 
 				/* Reply with a ZLP on next IN token */