7400b06396
If the AVX2 set is available, we can exploit the repetitive
characteristic of this algorithm to provide a fast, vectorised
version by using 256-bit wide AVX2 operations for bucket loads and
bitwise intersections.
In most cases, this implementation consistently outperforms rbtree
set instances despite the fact they are configured to use a given,
single, ranged data type out of the ones used for performance
measurements by the nft_concat_range.sh kselftest.
That script, injecting packets directly on the ingoing device path
with pktgen, reports, averaged over five runs on a single AMD Epyc
7402 thread (3.35GHz, 768 KiB L1D$, 12 MiB L2$), the figures below.
CONFIG_RETPOLINE was not set here.
Note that this is not a fair comparison over hash and rbtree set
types: non-ranged entries (used to have a reference for hash types)
would be matched faster than this, and matching on a single field
only (which is the case for rbtree) is also significantly faster.
However, it's not possible at the moment to choose this set type
for non-ranged entries, and the current implementation also needs
a few minor adjustments in order to match on less than two fields.
---------------.-----------------------------------.------------.
AMD Epyc 7402 | baselines, Mpps | this patch |
1 thread |___________________________________|____________|
3.35GHz | | | | | |
768KiB L1D$ | netdev | hash | rbtree | | |
---------------| hook | no | single | | pipapo |
type entries | drop | ranges | field | pipapo | AVX2 |
---------------|--------|--------|--------|--------|------------|
net,port | | | | | |
1000 | 19.0 | 10.4 | 3.8 | 4.0 | 7.5 +87% |
---------------|--------|--------|--------|--------|------------|
port,net | | | | | |
100 | 18.8 | 10.3 | 5.8 | 6.3 | 8.1 +29% |
---------------|--------|--------|--------|--------|------------|
net6,port | | | | | |
1000 | 16.4 | 7.6 | 1.8 | 2.1 | 4.8 +128% |
---------------|--------|--------|--------|--------|------------|
port,proto | | | | | |
30000 | 19.6 | 11.6 | 3.9 | 0.5 | 2.6 +420% |
---------------|--------|--------|--------|--------|------------|
net6,port,mac | | | | | |
10 | 16.5 | 5.4 | 4.3 | 3.4 | 4.7 +38% |
---------------|--------|--------|--------|--------|------------|
net6,port,mac, | | | | | |
proto 1000 | 16.5 | 5.7 | 1.9 | 1.4 | 3.6 +26% |
---------------|--------|--------|--------|--------|------------|
net,mac | | | | | |
1000 | 19.0 | 8.4 | 3.9 | 2.5 | 6.4 +156% |
---------------'--------'--------'--------'--------'------------'
A similar strategy could be easily reused to implement specialised
versions for other SIMD sets, and I plan to post at least a NEON
version at a later time.
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
104 lines
3.4 KiB
C
104 lines
3.4 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _NET_NF_TABLES_CORE_H
|
|
#define _NET_NF_TABLES_CORE_H
|
|
|
|
#include <net/netfilter/nf_tables.h>
|
|
|
|
extern struct nft_expr_type nft_imm_type;
|
|
extern struct nft_expr_type nft_cmp_type;
|
|
extern struct nft_expr_type nft_lookup_type;
|
|
extern struct nft_expr_type nft_bitwise_type;
|
|
extern struct nft_expr_type nft_byteorder_type;
|
|
extern struct nft_expr_type nft_payload_type;
|
|
extern struct nft_expr_type nft_dynset_type;
|
|
extern struct nft_expr_type nft_range_type;
|
|
extern struct nft_expr_type nft_meta_type;
|
|
extern struct nft_expr_type nft_rt_type;
|
|
extern struct nft_expr_type nft_exthdr_type;
|
|
|
|
#ifdef CONFIG_NETWORK_SECMARK
|
|
extern struct nft_object_type nft_secmark_obj_type;
|
|
#endif
|
|
|
|
int nf_tables_core_module_init(void);
|
|
void nf_tables_core_module_exit(void);
|
|
|
|
struct nft_cmp_fast_expr {
|
|
u32 data;
|
|
enum nft_registers sreg:8;
|
|
u8 len;
|
|
};
|
|
|
|
struct nft_immediate_expr {
|
|
struct nft_data data;
|
|
enum nft_registers dreg:8;
|
|
u8 dlen;
|
|
};
|
|
|
|
/* Calculate the mask for the nft_cmp_fast expression. On big endian the
|
|
* mask needs to include the *upper* bytes when interpreting that data as
|
|
* something smaller than the full u32, therefore a cpu_to_le32 is done.
|
|
*/
|
|
static inline u32 nft_cmp_fast_mask(unsigned int len)
|
|
{
|
|
return cpu_to_le32(~0U >> (sizeof_field(struct nft_cmp_fast_expr,
|
|
data) * BITS_PER_BYTE - len));
|
|
}
|
|
|
|
extern const struct nft_expr_ops nft_cmp_fast_ops;
|
|
|
|
struct nft_payload {
|
|
enum nft_payload_bases base:8;
|
|
u8 offset;
|
|
u8 len;
|
|
enum nft_registers dreg:8;
|
|
};
|
|
|
|
struct nft_payload_set {
|
|
enum nft_payload_bases base:8;
|
|
u8 offset;
|
|
u8 len;
|
|
enum nft_registers sreg:8;
|
|
u8 csum_type;
|
|
u8 csum_offset;
|
|
u8 csum_flags;
|
|
};
|
|
|
|
extern const struct nft_expr_ops nft_payload_fast_ops;
|
|
|
|
extern struct static_key_false nft_counters_enabled;
|
|
extern struct static_key_false nft_trace_enabled;
|
|
|
|
extern const struct nft_set_type nft_set_rhash_type;
|
|
extern const struct nft_set_type nft_set_hash_type;
|
|
extern const struct nft_set_type nft_set_hash_fast_type;
|
|
extern const struct nft_set_type nft_set_rbtree_type;
|
|
extern const struct nft_set_type nft_set_bitmap_type;
|
|
extern const struct nft_set_type nft_set_pipapo_type;
|
|
extern const struct nft_set_type nft_set_pipapo_avx2_type;
|
|
|
|
struct nft_expr;
|
|
struct nft_regs;
|
|
struct nft_pktinfo;
|
|
void nft_meta_get_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
void nft_cmp_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
void nft_lookup_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
void nft_payload_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
void nft_immediate_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
void nft_bitwise_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
void nft_range_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
void nft_byteorder_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
void nft_dynset_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
void nft_rt_get_eval(const struct nft_expr *expr,
|
|
struct nft_regs *regs, const struct nft_pktinfo *pkt);
|
|
#endif /* _NET_NF_TABLES_CORE_H */
|