diff --git a/drivers/usb/serial/visor.c b/drivers/usb/serial/visor.c
index b9341f0e452b..ad1f9232292d 100644
--- a/drivers/usb/serial/visor.c
+++ b/drivers/usb/serial/visor.c
@@ -595,20 +595,23 @@ static void visor_unthrottle(struct tty_struct *tty)
 {
 	struct usb_serial_port *port = tty->driver_data;
 	struct visor_private *priv = usb_get_serial_port_data(port);
-	int result;
+	int result, was_throttled;
 
 	dbg("%s - port %d", __func__, port->number);
 	spin_lock_irq(&priv->lock);
 	priv->throttled = 0;
+	was_throttled = priv->actually_throttled;
 	priv->actually_throttled = 0;
 	spin_unlock_irq(&priv->lock);
 
-	port->read_urb->dev = port->serial->dev;
-	result = usb_submit_urb(port->read_urb, GFP_KERNEL);
-	if (result)
-		dev_err(&port->dev,
-			"%s - failed submitting read urb, error %d\n",
+	if (was_throttled) {
+		port->read_urb->dev = port->serial->dev;
+		result = usb_submit_urb(port->read_urb, GFP_KERNEL);
+		if (result)
+			dev_err(&port->dev,
+				"%s - failed submitting read urb, error %d\n",
 							__func__, result);
+	}
 }
 
 static int palm_os_3_probe(struct usb_serial *serial,