b952b2befb
[ 60.988363] ====================================================== [ 60.988754] [ INFO: possible circular locking dependency detected ] [ 60.989152] 3.19.0+ #194 Not tainted [ 60.989377] ------------------------------------------------------- [ 60.989781] swapper/3/0 is trying to acquire lock: [ 60.990079] (&(&n_ptr->lock)->rlock){+.-...}, at: [<ffffffffa0006dca>] tipc_link_retransmit+0x1aa/0x240 [tipc] [ 60.990743] [ 60.990743] but task is already holding lock: [ 60.991106] (&(&bclink->lock)->rlock){+.-...}, at: [<ffffffffa00004be>] tipc_bclink_lock+0x8e/0xa0 [tipc] [ 60.991738] [ 60.991738] which lock already depends on the new lock. [ 60.991738] [ 60.992174] [ 60.992174] the existing dependency chain (in reverse order) is: [ 60.992174] -> #1 (&(&bclink->lock)->rlock){+.-...}: [ 60.992174] [<ffffffff810a9c0c>] lock_acquire+0x9c/0x140 [ 60.992174] [<ffffffff8179c41f>] _raw_spin_lock_bh+0x3f/0x50 [ 60.992174] [<ffffffffa00004be>] tipc_bclink_lock+0x8e/0xa0 [tipc] [ 60.992174] [<ffffffffa0000f57>] tipc_bclink_add_node+0x97/0xf0 [tipc] [ 60.992174] [<ffffffffa0011815>] tipc_node_link_up+0xf5/0x110 [tipc] [ 60.992174] [<ffffffffa0007783>] link_state_event+0x2b3/0x4f0 [tipc] [ 60.992174] [<ffffffffa00193c0>] tipc_link_proto_rcv+0x24c/0x418 [tipc] [ 60.992174] [<ffffffffa0008857>] tipc_rcv+0x827/0xac0 [tipc] [ 60.992174] [<ffffffffa0002ca3>] tipc_l2_rcv_msg+0x73/0xd0 [tipc] [ 60.992174] [<ffffffff81646e66>] __netif_receive_skb_core+0x746/0x980 [ 60.992174] [<ffffffff816470c1>] __netif_receive_skb+0x21/0x70 [ 60.992174] [<ffffffff81647295>] netif_receive_skb_internal+0x35/0x130 [ 60.992174] [<ffffffff81648218>] napi_gro_receive+0x158/0x1d0 [ 60.992174] [<ffffffff81559e05>] e1000_clean_rx_irq+0x155/0x490 [ 60.992174] [<ffffffff8155c1b7>] e1000_clean+0x267/0x990 [ 60.992174] [<ffffffff81647b60>] net_rx_action+0x150/0x360 [ 60.992174] [<ffffffff8105ec43>] __do_softirq+0x123/0x360 [ 60.992174] [<ffffffff8105f12e>] irq_exit+0x8e/0xb0 [ 60.992174] [<ffffffff8179f9f5>] do_IRQ+0x65/0x110 [ 60.992174] [<ffffffff8179da6f>] ret_from_intr+0x0/0x13 [ 60.992174] [<ffffffff8100de9f>] arch_cpu_idle+0xf/0x20 [ 60.992174] [<ffffffff8109dfa6>] cpu_startup_entry+0x2f6/0x3f0 [ 60.992174] [<ffffffff81033cda>] start_secondary+0x13a/0x150 [ 60.992174] -> #0 (&(&n_ptr->lock)->rlock){+.-...}: [ 60.992174] [<ffffffff810a8f7d>] __lock_acquire+0x163d/0x1ca0 [ 60.992174] [<ffffffff810a9c0c>] lock_acquire+0x9c/0x140 [ 60.992174] [<ffffffff8179c41f>] _raw_spin_lock_bh+0x3f/0x50 [ 60.992174] [<ffffffffa0006dca>] tipc_link_retransmit+0x1aa/0x240 [tipc] [ 60.992174] [<ffffffffa0001e11>] tipc_bclink_rcv+0x611/0x640 [tipc] [ 60.992174] [<ffffffffa0008646>] tipc_rcv+0x616/0xac0 [tipc] [ 60.992174] [<ffffffffa0002ca3>] tipc_l2_rcv_msg+0x73/0xd0 [tipc] [ 60.992174] [<ffffffff81646e66>] __netif_receive_skb_core+0x746/0x980 [ 60.992174] [<ffffffff816470c1>] __netif_receive_skb+0x21/0x70 [ 60.992174] [<ffffffff81647295>] netif_receive_skb_internal+0x35/0x130 [ 60.992174] [<ffffffff81648218>] napi_gro_receive+0x158/0x1d0 [ 60.992174] [<ffffffff81559e05>] e1000_clean_rx_irq+0x155/0x490 [ 60.992174] [<ffffffff8155c1b7>] e1000_clean+0x267/0x990 [ 60.992174] [<ffffffff81647b60>] net_rx_action+0x150/0x360 [ 60.992174] [<ffffffff8105ec43>] __do_softirq+0x123/0x360 [ 60.992174] [<ffffffff8105f12e>] irq_exit+0x8e/0xb0 [ 60.992174] [<ffffffff8179f9f5>] do_IRQ+0x65/0x110 [ 60.992174] [<ffffffff8179da6f>] ret_from_intr+0x0/0x13 [ 60.992174] [<ffffffff8100de9f>] arch_cpu_idle+0xf/0x20 [ 60.992174] [<ffffffff8109dfa6>] cpu_startup_entry+0x2f6/0x3f0 [ 60.992174] [<ffffffff81033cda>] start_secondary+0x13a/0x150 [ 60.992174] [ 60.992174] other info that might help us debug this: [ 60.992174] [ 60.992174] Possible unsafe locking scenario: [ 60.992174] [ 60.992174] CPU0 CPU1 [ 60.992174] ---- ---- [ 60.992174] lock(&(&bclink->lock)->rlock); [ 60.992174] lock(&(&n_ptr->lock)->rlock); [ 60.992174] lock(&(&bclink->lock)->rlock); [ 60.992174] lock(&(&n_ptr->lock)->rlock); [ 60.992174] [ 60.992174] *** DEADLOCK *** [ 60.992174] [ 60.992174] 3 locks held by swapper/3/0: [ 60.992174] #0: (rcu_read_lock){......}, at: [<ffffffff81646791>] __netif_receive_skb_core+0x71/0x980 [ 60.992174] #1: (rcu_read_lock){......}, at: [<ffffffffa0002c35>] tipc_l2_rcv_msg+0x5/0xd0 [tipc] [ 60.992174] #2: (&(&bclink->lock)->rlock){+.-...}, at: [<ffffffffa00004be>] tipc_bclink_lock+0x8e/0xa0 [tipc] [ 60.992174] The correct the sequence of grabbing n_ptr->lock and bclink->lock should be that the former is first held and the latter is then taken, which exactly happened on CPU1. But especially when the retransmission of broadcast link is failed, bclink->lock is first held in tipc_bclink_rcv(), and n_ptr->lock is taken in link_retransmit_failure() called by tipc_link_retransmit() subsequently, which is demonstrated on CPU0. As a result, deadlock occurs. If the order of holding the two locks happening on CPU0 is reversed, the deadlock risk will be relieved. Therefore, the node lock taken in link_retransmit_failure() originally is moved to tipc_bclink_rcv() so that it's obtained before bclink lock. But the precondition of the adjustment of node lock is that responding to bclink reset event must be moved from tipc_bclink_unlock() to tipc_node_unlock(). Reviewed-by: Erik Hugne <erik.hugne@ericsson.com> Signed-off-by: Ying Xue <ying.xue@windriver.com> Signed-off-by: David S. Miller <davem@davemloft.net>
137 lines
5.1 KiB
C
137 lines
5.1 KiB
C
/*
|
|
* net/tipc/bcast.h: Include file for TIPC broadcast code
|
|
*
|
|
* Copyright (c) 2003-2006, 2014-2015, Ericsson AB
|
|
* Copyright (c) 2005, 2010-2011, Wind River Systems
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are met:
|
|
*
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* 3. Neither the names of the copyright holders nor the names of its
|
|
* contributors may be used to endorse or promote products derived from
|
|
* this software without specific prior written permission.
|
|
*
|
|
* Alternatively, this software may be distributed under the terms of the
|
|
* GNU General Public License ("GPL") version 2 as published by the Free
|
|
* Software Foundation.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
* POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#ifndef _TIPC_BCAST_H
|
|
#define _TIPC_BCAST_H
|
|
|
|
#include <linux/tipc_config.h>
|
|
#include "link.h"
|
|
#include "node.h"
|
|
|
|
/**
|
|
* struct tipc_bcbearer_pair - a pair of bearers used by broadcast link
|
|
* @primary: pointer to primary bearer
|
|
* @secondary: pointer to secondary bearer
|
|
*
|
|
* Bearers must have same priority and same set of reachable destinations
|
|
* to be paired.
|
|
*/
|
|
|
|
struct tipc_bcbearer_pair {
|
|
struct tipc_bearer *primary;
|
|
struct tipc_bearer *secondary;
|
|
};
|
|
|
|
#define BCBEARER MAX_BEARERS
|
|
|
|
/**
|
|
* struct tipc_bcbearer - bearer used by broadcast link
|
|
* @bearer: (non-standard) broadcast bearer structure
|
|
* @media: (non-standard) broadcast media structure
|
|
* @bpairs: array of bearer pairs
|
|
* @bpairs_temp: temporary array of bearer pairs used by tipc_bcbearer_sort()
|
|
* @remains: temporary node map used by tipc_bcbearer_send()
|
|
* @remains_new: temporary node map used tipc_bcbearer_send()
|
|
*
|
|
* Note: The fields labelled "temporary" are incorporated into the bearer
|
|
* to avoid consuming potentially limited stack space through the use of
|
|
* large local variables within multicast routines. Concurrent access is
|
|
* prevented through use of the spinlock "bclink_lock".
|
|
*/
|
|
struct tipc_bcbearer {
|
|
struct tipc_bearer bearer;
|
|
struct tipc_media media;
|
|
struct tipc_bcbearer_pair bpairs[MAX_BEARERS];
|
|
struct tipc_bcbearer_pair bpairs_temp[TIPC_MAX_LINK_PRI + 1];
|
|
struct tipc_node_map remains;
|
|
struct tipc_node_map remains_new;
|
|
};
|
|
|
|
/**
|
|
* struct tipc_bclink - link used for broadcast messages
|
|
* @lock: spinlock governing access to structure
|
|
* @link: (non-standard) broadcast link structure
|
|
* @node: (non-standard) node structure representing b'cast link's peer node
|
|
* @bcast_nodes: map of broadcast-capable nodes
|
|
* @retransmit_to: node that most recently requested a retransmit
|
|
*
|
|
* Handles sequence numbering, fragmentation, bundling, etc.
|
|
*/
|
|
struct tipc_bclink {
|
|
spinlock_t lock;
|
|
struct tipc_link link;
|
|
struct tipc_node node;
|
|
struct sk_buff_head arrvq;
|
|
struct sk_buff_head inputq;
|
|
struct tipc_node_map bcast_nodes;
|
|
struct tipc_node *retransmit_to;
|
|
};
|
|
|
|
struct tipc_node;
|
|
extern const char tipc_bclink_name[];
|
|
|
|
/**
|
|
* tipc_nmap_equal - test for equality of node maps
|
|
*/
|
|
static inline int tipc_nmap_equal(struct tipc_node_map *nm_a,
|
|
struct tipc_node_map *nm_b)
|
|
{
|
|
return !memcmp(nm_a, nm_b, sizeof(*nm_a));
|
|
}
|
|
|
|
int tipc_bclink_init(struct net *net);
|
|
void tipc_bclink_stop(struct net *net);
|
|
void tipc_bclink_add_node(struct net *net, u32 addr);
|
|
void tipc_bclink_remove_node(struct net *net, u32 addr);
|
|
struct tipc_node *tipc_bclink_retransmit_to(struct net *tn);
|
|
void tipc_bclink_acknowledge(struct tipc_node *n_ptr, u32 acked);
|
|
void tipc_bclink_rcv(struct net *net, struct sk_buff *buf);
|
|
u32 tipc_bclink_get_last_sent(struct net *net);
|
|
u32 tipc_bclink_acks_missing(struct tipc_node *n_ptr);
|
|
void tipc_bclink_update_link_state(struct tipc_node *node,
|
|
u32 last_sent);
|
|
int tipc_bclink_reset_stats(struct net *net);
|
|
int tipc_bclink_set_queue_limits(struct net *net, u32 limit);
|
|
void tipc_bcbearer_sort(struct net *net, struct tipc_node_map *nm_ptr,
|
|
u32 node, bool action);
|
|
uint tipc_bclink_get_mtu(void);
|
|
int tipc_bclink_xmit(struct net *net, struct sk_buff_head *list);
|
|
void tipc_bclink_wakeup_users(struct net *net);
|
|
int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg);
|
|
void tipc_bclink_input(struct net *net);
|
|
|
|
#endif
|