From 31d6b4036098f6b59bcfa20375626b500c7d7417 Mon Sep 17 00:00:00 2001 From: Dany Madden Date: Wed, 25 Nov 2020 18:04:24 -0600 Subject: [PATCH 1/9] ibmvnic: handle inconsistent login with reset Inconsistent login with the vnicserver is causing the device to be removed. This does not give the device a chance to recover from error state. This patch schedules a FATAL reset instead to bring the adapter up. Fixes: 032c5e82847a2 ("Driver for IBM System i/p VNIC protocol") Signed-off-by: Dany Madden Signed-off-by: Lijun Pan Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 2491ebc97871..b28a979061dc 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -4414,7 +4414,7 @@ static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq, adapter->req_rx_add_queues != be32_to_cpu(login_rsp->num_rxadd_subcrqs))) { dev_err(dev, "FATAL: Inconsistent login and login rsp\n"); - ibmvnic_remove(adapter->vdev); + ibmvnic_reset(adapter, VNIC_RESET_FATAL); return -EIO; } size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) + From 18f141bf97d42f65abfdf17fd93fb3a0dac100e7 Mon Sep 17 00:00:00 2001 From: Dany Madden Date: Wed, 25 Nov 2020 18:04:25 -0600 Subject: [PATCH 2/9] ibmvnic: stop free_all_rwi on failed reset When ibmvnic fails to reset, it breaks out of the reset loop and frees all of the remaining resets from the workqueue. Doing so prevents the adapter from recovering if no reset is scheduled after that. Instead, have the driver continue to process resets on the workqueue. Remove the no longer need free_all_rwi(). Fixes: ed651a10875f1 ("ibmvnic: Updated reset handling") Signed-off-by: Dany Madden Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index b28a979061dc..e9243d8c5abc 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -2173,17 +2173,6 @@ static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter) return rwi; } -static void free_all_rwi(struct ibmvnic_adapter *adapter) -{ - struct ibmvnic_rwi *rwi; - - rwi = get_next_rwi(adapter); - while (rwi) { - kfree(rwi); - rwi = get_next_rwi(adapter); - } -} - static void __ibmvnic_reset(struct work_struct *work) { struct ibmvnic_rwi *rwi; @@ -2252,9 +2241,9 @@ static void __ibmvnic_reset(struct work_struct *work) else adapter->state = reset_state; rc = 0; - } else if (rc && rc != IBMVNIC_INIT_FAILED && - !adapter->force_reset_recovery) - break; + } + if (rc) + netdev_dbg(adapter->netdev, "Reset failed, rc=%d\n", rc); rwi = get_next_rwi(adapter); @@ -2268,11 +2257,6 @@ static void __ibmvnic_reset(struct work_struct *work) complete(&adapter->reset_done); } - if (rc) { - netdev_dbg(adapter->netdev, "Reset failed\n"); - free_all_rwi(adapter); - } - clear_bit_unlock(0, &adapter->resetting); } From 9281cf2d584083a450fd65fd27cc5f0e692f6e30 Mon Sep 17 00:00:00 2001 From: Dany Madden Date: Wed, 25 Nov 2020 18:04:26 -0600 Subject: [PATCH 3/9] ibmvnic: avoid memset null scrq msgs scrq->msgs could be NULL during device reset, causing Linux to crash. So, check before memset scrq->msgs. Fixes: c8b2ad0a4a901 ("ibmvnic: Sanitize entire SCRQ buffer on reset") Signed-off-by: Dany Madden Signed-off-by: Lijun Pan Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index e9243d8c5abc..939a276dae7d 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -2844,15 +2844,26 @@ static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter, { int rc; + if (!scrq) { + netdev_dbg(adapter->netdev, + "Invalid scrq reset. irq (%d) or msgs (%p).\n", + scrq->irq, scrq->msgs); + return -EINVAL; + } + if (scrq->irq) { free_irq(scrq->irq, scrq); irq_dispose_mapping(scrq->irq); scrq->irq = 0; } - - memset(scrq->msgs, 0, 4 * PAGE_SIZE); - atomic_set(&scrq->used, 0); - scrq->cur = 0; + if (scrq->msgs) { + memset(scrq->msgs, 0, 4 * PAGE_SIZE); + atomic_set(&scrq->used, 0); + scrq->cur = 0; + } else { + netdev_dbg(adapter->netdev, "Invalid scrq reset\n"); + return -EINVAL; + } rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token, 4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq); From 0cb4bc66ba5ea2d3b94ec2a00775888130db628a Mon Sep 17 00:00:00 2001 From: Dany Madden Date: Wed, 25 Nov 2020 18:04:27 -0600 Subject: [PATCH 4/9] ibmvnic: restore adapter state on failed reset In a failed reset, driver could end up in VNIC_PROBED or VNIC_CLOSED state and cannot recover in subsequent resets, leaving it offline. This patch restores the adapter state to reset_state, the original state when reset was called. Fixes: b27507bb59ed5 ("net/ibmvnic: unlock rtnl_lock in reset so linkwatch_event can run") Fixes: 2770a7984db58 ("ibmvnic: Introduce hard reset recovery") Signed-off-by: Dany Madden Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 67 ++++++++++++++++-------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 939a276dae7d..5ce9f70984ee 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -1857,7 +1857,7 @@ static int do_change_param_reset(struct ibmvnic_adapter *adapter, if (reset_state == VNIC_OPEN) { rc = __ibmvnic_close(netdev); if (rc) - return rc; + goto out; } release_resources(adapter); @@ -1875,24 +1875,25 @@ static int do_change_param_reset(struct ibmvnic_adapter *adapter, } rc = ibmvnic_reset_init(adapter, true); - if (rc) - return IBMVNIC_INIT_FAILED; + if (rc) { + rc = IBMVNIC_INIT_FAILED; + goto out; + } /* If the adapter was in PROBE state prior to the reset, * exit here. */ if (reset_state == VNIC_PROBED) - return 0; + goto out; rc = ibmvnic_login(netdev); if (rc) { - adapter->state = reset_state; - return rc; + goto out; } rc = init_resources(adapter); if (rc) - return rc; + goto out; ibmvnic_disable_irqs(adapter); @@ -1902,8 +1903,10 @@ static int do_change_param_reset(struct ibmvnic_adapter *adapter, return 0; rc = __ibmvnic_open(netdev); - if (rc) - return IBMVNIC_OPEN_FAILED; + if (rc) { + rc = IBMVNIC_OPEN_FAILED; + goto out; + } /* refresh device's multicast list */ ibmvnic_set_multi(netdev); @@ -1912,7 +1915,10 @@ static int do_change_param_reset(struct ibmvnic_adapter *adapter, for (i = 0; i < adapter->req_rx_queues; i++) napi_schedule(&adapter->napi[i]); - return 0; +out: + if (rc) + adapter->state = reset_state; + return rc; } /** @@ -2015,7 +2021,6 @@ static int do_reset(struct ibmvnic_adapter *adapter, rc = ibmvnic_login(netdev); if (rc) { - adapter->state = reset_state; goto out; } @@ -2083,6 +2088,9 @@ static int do_reset(struct ibmvnic_adapter *adapter, rc = 0; out: + /* restore the adapter state if reset failed */ + if (rc) + adapter->state = reset_state; rtnl_unlock(); return rc; @@ -2115,43 +2123,46 @@ static int do_hard_reset(struct ibmvnic_adapter *adapter, if (rc) { netdev_err(adapter->netdev, "Couldn't initialize crq. rc=%d\n", rc); - return rc; + goto out; } rc = ibmvnic_reset_init(adapter, false); if (rc) - return rc; + goto out; /* If the adapter was in PROBE state prior to the reset, * exit here. */ if (reset_state == VNIC_PROBED) - return 0; + goto out; rc = ibmvnic_login(netdev); - if (rc) { - adapter->state = VNIC_PROBED; - return 0; - } + if (rc) + goto out; rc = init_resources(adapter); if (rc) - return rc; + goto out; ibmvnic_disable_irqs(adapter); adapter->state = VNIC_CLOSED; if (reset_state == VNIC_CLOSED) - return 0; + goto out; rc = __ibmvnic_open(netdev); - if (rc) - return IBMVNIC_OPEN_FAILED; + if (rc) { + rc = IBMVNIC_OPEN_FAILED; + goto out; + } call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, netdev); call_netdevice_notifiers(NETDEV_RESEND_IGMP, netdev); - - return 0; +out: + /* restore adapter state if reset failed */ + if (rc) + adapter->state = reset_state; + return rc; } static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter) @@ -2235,13 +2246,7 @@ static void __ibmvnic_reset(struct work_struct *work) rc = do_reset(adapter, rwi, reset_state); } kfree(rwi); - if (rc == IBMVNIC_OPEN_FAILED) { - if (list_empty(&adapter->rwi_list)) - adapter->state = VNIC_CLOSED; - else - adapter->state = reset_state; - rc = 0; - } + if (rc) netdev_dbg(adapter->netdev, "Reset failed, rc=%d\n", rc); From f15fde9d47b887b406f5e76490d601cfc26643c9 Mon Sep 17 00:00:00 2001 From: Sukadev Bhattiprolu Date: Wed, 25 Nov 2020 18:04:28 -0600 Subject: [PATCH 5/9] ibmvnic: delay next reset if hard reset fails If auto-priority failover is enabled, the backing device needs time to settle if hard resetting fails for any reason. Add a delay of 60 seconds before retrying the hard-reset. Fixes: 2770a7984db5 ("ibmvnic: Introduce hard reset recovery") Signed-off-by: Sukadev Bhattiprolu Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 5ce9f70984ee..d4d40b349788 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -2241,6 +2241,14 @@ static void __ibmvnic_reset(struct work_struct *work) rc = do_hard_reset(adapter, rwi, reset_state); rtnl_unlock(); } + if (rc) { + /* give backing device time to settle down */ + netdev_dbg(adapter->netdev, + "[S:%d] Hard reset failed, waiting 60 secs\n", + adapter->state); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(60 * HZ); + } } else if (!(rwi->reset_reason == VNIC_RESET_FATAL && adapter->from_passive_init)) { rc = do_reset(adapter, rwi, reset_state); From 76cdc5c5d99ce4856ad0ac38facc33b52fa64f77 Mon Sep 17 00:00:00 2001 From: Sukadev Bhattiprolu Date: Wed, 25 Nov 2020 18:04:29 -0600 Subject: [PATCH 6/9] ibmvnic: track pending login If after ibmvnic sends a LOGIN it gets a FAILOVER, it is possible that the worker thread will start reset process and free the login response buffer before it gets a (now stale) LOGIN_RSP. The ibmvnic tasklet will then try to access the login response buffer and crash. Have ibmvnic track pending logins and discard any stale login responses. Fixes: 032c5e82847a ("Driver for IBM System i/p VNIC protocol") Signed-off-by: Sukadev Bhattiprolu Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 17 +++++++++++++++++ drivers/net/ethernet/ibm/ibmvnic.h | 1 + 2 files changed, 18 insertions(+) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index d4d40b349788..13eae1666477 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -3841,6 +3841,8 @@ static int send_login(struct ibmvnic_adapter *adapter) crq.login.cmd = LOGIN; crq.login.ioba = cpu_to_be32(buffer_token); crq.login.len = cpu_to_be32(buffer_size); + + adapter->login_pending = true; ibmvnic_send_crq(adapter, &crq); return 0; @@ -4393,6 +4395,15 @@ static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq, u64 *size_array; int i; + /* CHECK: Test/set of login_pending does not need to be atomic + * because only ibmvnic_tasklet tests/clears this. + */ + if (!adapter->login_pending) { + netdev_warn(netdev, "Ignoring unexpected login response\n"); + return 0; + } + adapter->login_pending = false; + dma_unmap_single(dev, adapter->login_buf_token, adapter->login_buf_sz, DMA_TO_DEVICE); dma_unmap_single(dev, adapter->login_rsp_buf_token, @@ -4764,6 +4775,11 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq, case IBMVNIC_CRQ_INIT: dev_info(dev, "Partner initialized\n"); adapter->from_passive_init = true; + /* Discard any stale login responses from prev reset. + * CHECK: should we clear even on INIT_COMPLETE? + */ + adapter->login_pending = false; + if (!completion_done(&adapter->init_done)) { complete(&adapter->init_done); adapter->init_done_rc = -EIO; @@ -5196,6 +5212,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) dev_set_drvdata(&dev->dev, netdev); adapter->vdev = dev; adapter->netdev = netdev; + adapter->login_pending = false; ether_addr_copy(adapter->mac_addr, mac_addr_p); ether_addr_copy(netdev->dev_addr, adapter->mac_addr); diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h index 47a3fd71c96f..bffa7f939ee1 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.h +++ b/drivers/net/ethernet/ibm/ibmvnic.h @@ -1086,6 +1086,7 @@ struct ibmvnic_adapter { struct delayed_work ibmvnic_delayed_reset; unsigned long resetting; bool napi_enabled, from_passive_init; + bool login_pending; bool failover_pending; bool force_reset_recovery; From c98d9cc4170da7e16a1012563d0f9fbe1c7cfe27 Mon Sep 17 00:00:00 2001 From: Dany Madden Date: Wed, 25 Nov 2020 18:04:30 -0600 Subject: [PATCH 7/9] ibmvnic: send_login should check for crq errors send_login() does not check for the result of ibmvnic_send_crq() of the login request. This results in the driver needlessly retrying the login 10 times even when CRQ is no longer active. Check the return code and give up in case of errors in sending the CRQ. The only time we want to retry is if we get a PARITALSUCCESS response from the partner. Fixes: 032c5e82847a2 ("Driver for IBM System i/p VNIC protocol") Signed-off-by: Dany Madden Signed-off-by: Sukadev Bhattiprolu Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 13eae1666477..5154e0421175 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -850,10 +850,8 @@ static int ibmvnic_login(struct net_device *netdev) adapter->init_done_rc = 0; reinit_completion(&adapter->init_done); rc = send_login(adapter); - if (rc) { - netdev_warn(netdev, "Unable to login\n"); + if (rc) return rc; - } if (!wait_for_completion_timeout(&adapter->init_done, timeout)) { @@ -3729,15 +3727,16 @@ static int send_login(struct ibmvnic_adapter *adapter) struct ibmvnic_login_rsp_buffer *login_rsp_buffer; struct ibmvnic_login_buffer *login_buffer; struct device *dev = &adapter->vdev->dev; + struct vnic_login_client_data *vlcd; dma_addr_t rsp_buffer_token; dma_addr_t buffer_token; size_t rsp_buffer_size; union ibmvnic_crq crq; + int client_data_len; size_t buffer_size; __be64 *tx_list_p; __be64 *rx_list_p; - int client_data_len; - struct vnic_login_client_data *vlcd; + int rc; int i; if (!adapter->tx_scrq || !adapter->rx_scrq) { @@ -3843,16 +3842,23 @@ static int send_login(struct ibmvnic_adapter *adapter) crq.login.len = cpu_to_be32(buffer_size); adapter->login_pending = true; - ibmvnic_send_crq(adapter, &crq); + rc = ibmvnic_send_crq(adapter, &crq); + if (rc) { + adapter->login_pending = false; + netdev_err(adapter->netdev, "Failed to send login, rc=%d\n", rc); + goto buf_rsp_map_failed; + } return 0; buf_rsp_map_failed: kfree(login_rsp_buffer); + adapter->login_rsp_buf = NULL; buf_rsp_alloc_failed: dma_unmap_single(dev, buffer_token, buffer_size, DMA_TO_DEVICE); buf_map_failed: kfree(login_buffer); + adapter->login_buf = NULL; buf_alloc_failed: return -1; } From a86d5c682b798b2dadaa4171c1d124cf3c45a17c Mon Sep 17 00:00:00 2001 From: Dany Madden Date: Wed, 25 Nov 2020 18:04:31 -0600 Subject: [PATCH 8/9] ibmvnic: no reset timeout for 5 seconds after reset Reset timeout is going off right after adapter reset. This patch ensures that timeout is scheduled if it has been 5 seconds since the last reset. 5 seconds is the default watchdog timeout. Fixes: ed651a10875f1 ("ibmvnic: Updated reset handling") Signed-off-by: Dany Madden Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 11 +++++++++-- drivers/net/ethernet/ibm/ibmvnic.h | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 5154e0421175..d48a694b92c9 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -2252,6 +2252,7 @@ static void __ibmvnic_reset(struct work_struct *work) rc = do_reset(adapter, rwi, reset_state); } kfree(rwi); + adapter->last_reset_time = jiffies; if (rc) netdev_dbg(adapter->netdev, "Reset failed, rc=%d\n", rc); @@ -2355,7 +2356,13 @@ static void ibmvnic_tx_timeout(struct net_device *dev, unsigned int txqueue) "Adapter is resetting, skip timeout reset\n"); return; } - + /* No queuing up reset until at least 5 seconds (default watchdog val) + * after last reset + */ + if (time_before(jiffies, (adapter->last_reset_time + dev->watchdog_timeo))) { + netdev_dbg(dev, "Not yet time to tx timeout.\n"); + return; + } ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT); } @@ -5282,7 +5289,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) adapter->state = VNIC_PROBED; adapter->wait_for_reset = false; - + adapter->last_reset_time = jiffies; return 0; ibmvnic_register_fail: diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h index bffa7f939ee1..21e7ea858cda 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.h +++ b/drivers/net/ethernet/ibm/ibmvnic.h @@ -1087,6 +1087,8 @@ struct ibmvnic_adapter { unsigned long resetting; bool napi_enabled, from_passive_init; bool login_pending; + /* last device reset time */ + unsigned long last_reset_time; bool failover_pending; bool force_reset_recovery; From 98c41f04a67abf5e7f7191d55d286e905d1430ef Mon Sep 17 00:00:00 2001 From: Dany Madden Date: Wed, 25 Nov 2020 18:04:32 -0600 Subject: [PATCH 9/9] ibmvnic: reduce wait for completion time Reduce the wait time for Command Response Queue response from 30 seconds to 20 seconds, as recommended by VIOS and Power Hypervisor teams. Fixes: bd0b672313941 ("ibmvnic: Move login and queue negotiation into ibmvnic_open") Fixes: 53da09e92910f ("ibmvnic: Add set_link_state routine for setting adapter link state") Signed-off-by: Dany Madden Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index d48a694b92c9..bca1becd33f0 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -834,7 +834,7 @@ static void release_napi(struct ibmvnic_adapter *adapter) static int ibmvnic_login(struct net_device *netdev) { struct ibmvnic_adapter *adapter = netdev_priv(netdev); - unsigned long timeout = msecs_to_jiffies(30000); + unsigned long timeout = msecs_to_jiffies(20000); int retry_count = 0; int retries = 10; bool retry; @@ -938,7 +938,7 @@ static void release_resources(struct ibmvnic_adapter *adapter) static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state) { struct net_device *netdev = adapter->netdev; - unsigned long timeout = msecs_to_jiffies(30000); + unsigned long timeout = msecs_to_jiffies(20000); union ibmvnic_crq crq; bool resend; int rc; @@ -5130,7 +5130,7 @@ map_failed: static int ibmvnic_reset_init(struct ibmvnic_adapter *adapter, bool reset) { struct device *dev = &adapter->vdev->dev; - unsigned long timeout = msecs_to_jiffies(30000); + unsigned long timeout = msecs_to_jiffies(20000); u64 old_num_rx_queues, old_num_tx_queues; int rc;