![Alexander Duyck](/assets/img/avatar_default.png)
This patch addresses a kernel unaligned access bug seen on a sparc64 system with an igb adapter. Specifically the __skb_flow_get_ports was returning a be32 pointer which was then having the value directly returned. In order to prevent this it is actually easier to simply not populate the ports or address values when an skb is not present. In this case the assumption is that the data isn't needed and rather than slow down the faster aligned accesses by making them have to assume the unaligned path on architectures that don't support efficent unaligned access it makes more sense to simply switch off the bits that were copying the source and destination address/port for the case where we only care about the protocol types and lengths which are normally 16 bit fields anyway. Reported-by: David S. Miller <davem@davemloft.net> Signed-off-by: Alexander Duyck <alexander.h.duyck@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net>
472 lines
11 KiB
C
472 lines
11 KiB
C
#include <linux/skbuff.h>
|
|
#include <linux/export.h>
|
|
#include <linux/ip.h>
|
|
#include <linux/ipv6.h>
|
|
#include <linux/if_vlan.h>
|
|
#include <net/ip.h>
|
|
#include <net/ipv6.h>
|
|
#include <linux/igmp.h>
|
|
#include <linux/icmp.h>
|
|
#include <linux/sctp.h>
|
|
#include <linux/dccp.h>
|
|
#include <linux/if_tunnel.h>
|
|
#include <linux/if_pppox.h>
|
|
#include <linux/ppp_defs.h>
|
|
#include <net/flow_keys.h>
|
|
#include <scsi/fc/fc_fcoe.h>
|
|
|
|
/* copy saddr & daddr, possibly using 64bit load/store
|
|
* Equivalent to : flow->src = iph->saddr;
|
|
* flow->dst = iph->daddr;
|
|
*/
|
|
static void iph_to_flow_copy_addrs(struct flow_keys *flow, const struct iphdr *iph)
|
|
{
|
|
BUILD_BUG_ON(offsetof(typeof(*flow), dst) !=
|
|
offsetof(typeof(*flow), src) + sizeof(flow->src));
|
|
memcpy(&flow->src, &iph->saddr, sizeof(flow->src) + sizeof(flow->dst));
|
|
}
|
|
|
|
/**
|
|
* __skb_flow_get_ports - extract the upper layer ports and return them
|
|
* @skb: sk_buff to extract the ports from
|
|
* @thoff: transport header offset
|
|
* @ip_proto: protocol for which to get port offset
|
|
* @data: raw buffer pointer to the packet, if NULL use skb->data
|
|
* @hlen: packet header length, if @data is NULL use skb_headlen(skb)
|
|
*
|
|
* The function will try to retrieve the ports at offset thoff + poff where poff
|
|
* is the protocol port offset returned from proto_ports_offset
|
|
*/
|
|
__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
|
|
void *data, int hlen)
|
|
{
|
|
int poff = proto_ports_offset(ip_proto);
|
|
|
|
if (!data) {
|
|
data = skb->data;
|
|
hlen = skb_headlen(skb);
|
|
}
|
|
|
|
if (poff >= 0) {
|
|
__be32 *ports, _ports;
|
|
|
|
ports = __skb_header_pointer(skb, thoff + poff,
|
|
sizeof(_ports), data, hlen, &_ports);
|
|
if (ports)
|
|
return *ports;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(__skb_flow_get_ports);
|
|
|
|
/**
|
|
* __skb_flow_dissect - extract the flow_keys struct and return it
|
|
* @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
|
|
* @data: raw buffer pointer to the packet, if NULL use skb->data
|
|
* @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
|
|
* @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
|
|
* @hlen: packet header length, if @data is NULL use skb_headlen(skb)
|
|
*
|
|
* The function will try to retrieve the struct flow_keys from either the skbuff
|
|
* or a raw buffer specified by the rest parameters
|
|
*/
|
|
bool __skb_flow_dissect(const struct sk_buff *skb, struct flow_keys *flow,
|
|
void *data, __be16 proto, int nhoff, int hlen)
|
|
{
|
|
u8 ip_proto;
|
|
|
|
if (!data) {
|
|
data = skb->data;
|
|
proto = skb->protocol;
|
|
nhoff = skb_network_offset(skb);
|
|
hlen = skb_headlen(skb);
|
|
}
|
|
|
|
memset(flow, 0, sizeof(*flow));
|
|
|
|
again:
|
|
switch (proto) {
|
|
case htons(ETH_P_IP): {
|
|
const struct iphdr *iph;
|
|
struct iphdr _iph;
|
|
ip:
|
|
iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
|
|
if (!iph || iph->ihl < 5)
|
|
return false;
|
|
nhoff += iph->ihl * 4;
|
|
|
|
ip_proto = iph->protocol;
|
|
if (ip_is_fragment(iph))
|
|
ip_proto = 0;
|
|
|
|
/* skip the address processing if skb is NULL. The assumption
|
|
* here is that if there is no skb we are not looking for flow
|
|
* info but lengths and protocols.
|
|
*/
|
|
if (!skb)
|
|
break;
|
|
|
|
iph_to_flow_copy_addrs(flow, iph);
|
|
break;
|
|
}
|
|
case htons(ETH_P_IPV6): {
|
|
const struct ipv6hdr *iph;
|
|
struct ipv6hdr _iph;
|
|
__be32 flow_label;
|
|
|
|
ipv6:
|
|
iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
|
|
if (!iph)
|
|
return false;
|
|
|
|
ip_proto = iph->nexthdr;
|
|
nhoff += sizeof(struct ipv6hdr);
|
|
|
|
/* see comment above in IPv4 section */
|
|
if (!skb)
|
|
break;
|
|
|
|
flow->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
|
|
flow->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
|
|
|
|
flow_label = ip6_flowlabel(iph);
|
|
if (flow_label) {
|
|
/* Awesome, IPv6 packet has a flow label so we can
|
|
* use that to represent the ports without any
|
|
* further dissection.
|
|
*/
|
|
flow->n_proto = proto;
|
|
flow->ip_proto = ip_proto;
|
|
flow->ports = flow_label;
|
|
flow->thoff = (u16)nhoff;
|
|
|
|
return true;
|
|
}
|
|
|
|
break;
|
|
}
|
|
case htons(ETH_P_8021AD):
|
|
case htons(ETH_P_8021Q): {
|
|
const struct vlan_hdr *vlan;
|
|
struct vlan_hdr _vlan;
|
|
|
|
vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
|
|
if (!vlan)
|
|
return false;
|
|
|
|
proto = vlan->h_vlan_encapsulated_proto;
|
|
nhoff += sizeof(*vlan);
|
|
goto again;
|
|
}
|
|
case htons(ETH_P_PPP_SES): {
|
|
struct {
|
|
struct pppoe_hdr hdr;
|
|
__be16 proto;
|
|
} *hdr, _hdr;
|
|
hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
|
|
if (!hdr)
|
|
return false;
|
|
proto = hdr->proto;
|
|
nhoff += PPPOE_SES_HLEN;
|
|
switch (proto) {
|
|
case htons(PPP_IP):
|
|
goto ip;
|
|
case htons(PPP_IPV6):
|
|
goto ipv6;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
case htons(ETH_P_FCOE):
|
|
flow->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
|
|
/* fall through */
|
|
default:
|
|
return false;
|
|
}
|
|
|
|
switch (ip_proto) {
|
|
case IPPROTO_GRE: {
|
|
struct gre_hdr {
|
|
__be16 flags;
|
|
__be16 proto;
|
|
} *hdr, _hdr;
|
|
|
|
hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
|
|
if (!hdr)
|
|
return false;
|
|
/*
|
|
* Only look inside GRE if version zero and no
|
|
* routing
|
|
*/
|
|
if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) {
|
|
proto = hdr->proto;
|
|
nhoff += 4;
|
|
if (hdr->flags & GRE_CSUM)
|
|
nhoff += 4;
|
|
if (hdr->flags & GRE_KEY)
|
|
nhoff += 4;
|
|
if (hdr->flags & GRE_SEQ)
|
|
nhoff += 4;
|
|
if (proto == htons(ETH_P_TEB)) {
|
|
const struct ethhdr *eth;
|
|
struct ethhdr _eth;
|
|
|
|
eth = __skb_header_pointer(skb, nhoff,
|
|
sizeof(_eth),
|
|
data, hlen, &_eth);
|
|
if (!eth)
|
|
return false;
|
|
proto = eth->h_proto;
|
|
nhoff += sizeof(*eth);
|
|
}
|
|
goto again;
|
|
}
|
|
break;
|
|
}
|
|
case IPPROTO_IPIP:
|
|
proto = htons(ETH_P_IP);
|
|
goto ip;
|
|
case IPPROTO_IPV6:
|
|
proto = htons(ETH_P_IPV6);
|
|
goto ipv6;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
flow->n_proto = proto;
|
|
flow->ip_proto = ip_proto;
|
|
flow->thoff = (u16) nhoff;
|
|
|
|
/* unless skb is set we don't need to record port info */
|
|
if (skb)
|
|
flow->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
|
|
data, hlen);
|
|
|
|
return true;
|
|
}
|
|
EXPORT_SYMBOL(__skb_flow_dissect);
|
|
|
|
static u32 hashrnd __read_mostly;
|
|
static __always_inline void __flow_hash_secret_init(void)
|
|
{
|
|
net_get_random_once(&hashrnd, sizeof(hashrnd));
|
|
}
|
|
|
|
static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c)
|
|
{
|
|
__flow_hash_secret_init();
|
|
return jhash_3words(a, b, c, hashrnd);
|
|
}
|
|
|
|
static inline u32 __flow_hash_from_keys(struct flow_keys *keys)
|
|
{
|
|
u32 hash;
|
|
|
|
/* get a consistent hash (same value on both flow directions) */
|
|
if (((__force u32)keys->dst < (__force u32)keys->src) ||
|
|
(((__force u32)keys->dst == (__force u32)keys->src) &&
|
|
((__force u16)keys->port16[1] < (__force u16)keys->port16[0]))) {
|
|
swap(keys->dst, keys->src);
|
|
swap(keys->port16[0], keys->port16[1]);
|
|
}
|
|
|
|
hash = __flow_hash_3words((__force u32)keys->dst,
|
|
(__force u32)keys->src,
|
|
(__force u32)keys->ports);
|
|
if (!hash)
|
|
hash = 1;
|
|
|
|
return hash;
|
|
}
|
|
|
|
u32 flow_hash_from_keys(struct flow_keys *keys)
|
|
{
|
|
return __flow_hash_from_keys(keys);
|
|
}
|
|
EXPORT_SYMBOL(flow_hash_from_keys);
|
|
|
|
/*
|
|
* __skb_get_hash: calculate a flow hash based on src/dst addresses
|
|
* and src/dst port numbers. Sets hash in skb to non-zero hash value
|
|
* on success, zero indicates no valid hash. Also, sets l4_hash in skb
|
|
* if hash is a canonical 4-tuple hash over transport ports.
|
|
*/
|
|
void __skb_get_hash(struct sk_buff *skb)
|
|
{
|
|
struct flow_keys keys;
|
|
|
|
if (!skb_flow_dissect(skb, &keys))
|
|
return;
|
|
|
|
if (keys.ports)
|
|
skb->l4_hash = 1;
|
|
|
|
skb->sw_hash = 1;
|
|
|
|
skb->hash = __flow_hash_from_keys(&keys);
|
|
}
|
|
EXPORT_SYMBOL(__skb_get_hash);
|
|
|
|
/*
|
|
* Returns a Tx hash based on the given packet descriptor a Tx queues' number
|
|
* to be used as a distribution range.
|
|
*/
|
|
u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
|
|
unsigned int num_tx_queues)
|
|
{
|
|
u32 hash;
|
|
u16 qoffset = 0;
|
|
u16 qcount = num_tx_queues;
|
|
|
|
if (skb_rx_queue_recorded(skb)) {
|
|
hash = skb_get_rx_queue(skb);
|
|
while (unlikely(hash >= num_tx_queues))
|
|
hash -= num_tx_queues;
|
|
return hash;
|
|
}
|
|
|
|
if (dev->num_tc) {
|
|
u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
|
|
qoffset = dev->tc_to_txq[tc].offset;
|
|
qcount = dev->tc_to_txq[tc].count;
|
|
}
|
|
|
|
return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
|
|
}
|
|
EXPORT_SYMBOL(__skb_tx_hash);
|
|
|
|
u32 __skb_get_poff(const struct sk_buff *skb, void *data,
|
|
const struct flow_keys *keys, int hlen)
|
|
{
|
|
u32 poff = keys->thoff;
|
|
|
|
switch (keys->ip_proto) {
|
|
case IPPROTO_TCP: {
|
|
/* access doff as u8 to avoid unaligned access */
|
|
const u8 *doff;
|
|
u8 _doff;
|
|
|
|
doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
|
|
data, hlen, &_doff);
|
|
if (!doff)
|
|
return poff;
|
|
|
|
poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
|
|
break;
|
|
}
|
|
case IPPROTO_UDP:
|
|
case IPPROTO_UDPLITE:
|
|
poff += sizeof(struct udphdr);
|
|
break;
|
|
/* For the rest, we do not really care about header
|
|
* extensions at this point for now.
|
|
*/
|
|
case IPPROTO_ICMP:
|
|
poff += sizeof(struct icmphdr);
|
|
break;
|
|
case IPPROTO_ICMPV6:
|
|
poff += sizeof(struct icmp6hdr);
|
|
break;
|
|
case IPPROTO_IGMP:
|
|
poff += sizeof(struct igmphdr);
|
|
break;
|
|
case IPPROTO_DCCP:
|
|
poff += sizeof(struct dccp_hdr);
|
|
break;
|
|
case IPPROTO_SCTP:
|
|
poff += sizeof(struct sctphdr);
|
|
break;
|
|
}
|
|
|
|
return poff;
|
|
}
|
|
|
|
/* skb_get_poff() returns the offset to the payload as far as it could
|
|
* be dissected. The main user is currently BPF, so that we can dynamically
|
|
* truncate packets without needing to push actual payload to the user
|
|
* space and can analyze headers only, instead.
|
|
*/
|
|
u32 skb_get_poff(const struct sk_buff *skb)
|
|
{
|
|
struct flow_keys keys;
|
|
|
|
if (!skb_flow_dissect(skb, &keys))
|
|
return 0;
|
|
|
|
return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
|
|
}
|
|
|
|
static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
|
|
{
|
|
#ifdef CONFIG_XPS
|
|
struct xps_dev_maps *dev_maps;
|
|
struct xps_map *map;
|
|
int queue_index = -1;
|
|
|
|
rcu_read_lock();
|
|
dev_maps = rcu_dereference(dev->xps_maps);
|
|
if (dev_maps) {
|
|
map = rcu_dereference(
|
|
dev_maps->cpu_map[raw_smp_processor_id()]);
|
|
if (map) {
|
|
if (map->len == 1)
|
|
queue_index = map->queues[0];
|
|
else
|
|
queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
|
|
map->len)];
|
|
if (unlikely(queue_index >= dev->real_num_tx_queues))
|
|
queue_index = -1;
|
|
}
|
|
}
|
|
rcu_read_unlock();
|
|
|
|
return queue_index;
|
|
#else
|
|
return -1;
|
|
#endif
|
|
}
|
|
|
|
static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
|
|
{
|
|
struct sock *sk = skb->sk;
|
|
int queue_index = sk_tx_queue_get(sk);
|
|
|
|
if (queue_index < 0 || skb->ooo_okay ||
|
|
queue_index >= dev->real_num_tx_queues) {
|
|
int new_index = get_xps_queue(dev, skb);
|
|
if (new_index < 0)
|
|
new_index = skb_tx_hash(dev, skb);
|
|
|
|
if (queue_index != new_index && sk &&
|
|
rcu_access_pointer(sk->sk_dst_cache))
|
|
sk_tx_queue_set(sk, new_index);
|
|
|
|
queue_index = new_index;
|
|
}
|
|
|
|
return queue_index;
|
|
}
|
|
|
|
struct netdev_queue *netdev_pick_tx(struct net_device *dev,
|
|
struct sk_buff *skb,
|
|
void *accel_priv)
|
|
{
|
|
int queue_index = 0;
|
|
|
|
if (dev->real_num_tx_queues != 1) {
|
|
const struct net_device_ops *ops = dev->netdev_ops;
|
|
if (ops->ndo_select_queue)
|
|
queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
|
|
__netdev_pick_tx);
|
|
else
|
|
queue_index = __netdev_pick_tx(dev, skb);
|
|
|
|
if (!accel_priv)
|
|
queue_index = netdev_cap_txqueue(dev, queue_index);
|
|
}
|
|
|
|
skb_set_queue_mapping(skb, queue_index);
|
|
return netdev_get_tx_queue(dev, queue_index);
|
|
}
|