c5815ac7e2
This commit refactors the existing cgroup programs with libbpf
bpf loader. Since bpf_program__attach doesn't support cgroup program
attachment, this explicitly attaches cgroup bpf program with
bpf_program__attach_cgroup(bpf_prog, cg1).
Also, to change attach_type of bpf program, this uses libbpf's
bpf_program__set_expected_attach_type helper to switch EGRESS to
INGRESS. To keep bpf program attached to the cgroup hierarchy even
after the exit, this commit uses the BPF_LINK_PINNING to pin the link
attachment even after it is closed.
Besides, this program was broken due to the typo of BPF MAP definition.
But this commit solves the problem by fixing this from 'queue_stats' map
struct hvm_queue_stats -> hbm_queue_stats.
Fixes: 36b5d47113 ("selftests/bpf: samples/bpf: Split off legacy stuff from bpf_helpers.h")
Signed-off-by: Daniel T. Lee <danieltimlee@gmail.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20201124090310.24374-2-danieltimlee@gmail.com
218 lines
5.7 KiB
C
218 lines
5.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0
|
|
*
|
|
* Copyright (c) 2019 Facebook
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of version 2 of the GNU General Public
|
|
* License as published by the Free Software Foundation.
|
|
*
|
|
* Include file for sample Host Bandwidth Manager (HBM) BPF programs
|
|
*/
|
|
#define KBUILD_MODNAME "foo"
|
|
#include <stddef.h>
|
|
#include <stdbool.h>
|
|
#include <uapi/linux/bpf.h>
|
|
#include <uapi/linux/if_ether.h>
|
|
#include <uapi/linux/if_packet.h>
|
|
#include <uapi/linux/ip.h>
|
|
#include <uapi/linux/ipv6.h>
|
|
#include <uapi/linux/in.h>
|
|
#include <uapi/linux/tcp.h>
|
|
#include <uapi/linux/filter.h>
|
|
#include <uapi/linux/pkt_cls.h>
|
|
#include <net/ipv6.h>
|
|
#include <net/inet_ecn.h>
|
|
#include <bpf/bpf_endian.h>
|
|
#include <bpf/bpf_helpers.h>
|
|
#include "hbm.h"
|
|
|
|
#define DROP_PKT 0
|
|
#define ALLOW_PKT 1
|
|
#define TCP_ECN_OK 1
|
|
#define CWR 2
|
|
|
|
#ifndef HBM_DEBUG // Define HBM_DEBUG to enable debugging
|
|
#undef bpf_printk
|
|
#define bpf_printk(fmt, ...)
|
|
#endif
|
|
|
|
#define INITIAL_CREDIT_PACKETS 100
|
|
#define MAX_BYTES_PER_PACKET 1500
|
|
#define MARK_THRESH (40 * MAX_BYTES_PER_PACKET)
|
|
#define DROP_THRESH (80 * 5 * MAX_BYTES_PER_PACKET)
|
|
#define LARGE_PKT_DROP_THRESH (DROP_THRESH - (15 * MAX_BYTES_PER_PACKET))
|
|
#define MARK_REGION_SIZE (LARGE_PKT_DROP_THRESH - MARK_THRESH)
|
|
#define LARGE_PKT_THRESH 120
|
|
#define MAX_CREDIT (100 * MAX_BYTES_PER_PACKET)
|
|
#define INIT_CREDIT (INITIAL_CREDIT_PACKETS * MAX_BYTES_PER_PACKET)
|
|
|
|
// Time base accounting for fq's EDT
|
|
#define BURST_SIZE_NS 100000 // 100us
|
|
#define MARK_THRESH_NS 50000 // 50us
|
|
#define DROP_THRESH_NS 500000 // 500us
|
|
// Reserve 20us of queuing for small packets (less than 120 bytes)
|
|
#define LARGE_PKT_DROP_THRESH_NS (DROP_THRESH_NS - 20000)
|
|
#define MARK_REGION_SIZE_NS (LARGE_PKT_DROP_THRESH_NS - MARK_THRESH_NS)
|
|
|
|
// rate in bytes per ns << 20
|
|
#define CREDIT_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
|
|
#define BYTES_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
|
|
#define BYTES_TO_NS(bytes, rate) div64_u64(((u64)(bytes)) << 20, (u64)(rate))
|
|
|
|
struct {
|
|
__uint(type, BPF_MAP_TYPE_CGROUP_STORAGE);
|
|
__type(key, struct bpf_cgroup_storage_key);
|
|
__type(value, struct hbm_vqueue);
|
|
} queue_state SEC(".maps");
|
|
|
|
struct {
|
|
__uint(type, BPF_MAP_TYPE_ARRAY);
|
|
__uint(max_entries, 1);
|
|
__type(key, u32);
|
|
__type(value, struct hbm_queue_stats);
|
|
} queue_stats SEC(".maps");
|
|
|
|
struct hbm_pkt_info {
|
|
int cwnd;
|
|
int rtt;
|
|
int packets_out;
|
|
bool is_ip;
|
|
bool is_tcp;
|
|
short ecn;
|
|
};
|
|
|
|
static int get_tcp_info(struct __sk_buff *skb, struct hbm_pkt_info *pkti)
|
|
{
|
|
struct bpf_sock *sk;
|
|
struct bpf_tcp_sock *tp;
|
|
|
|
sk = skb->sk;
|
|
if (sk) {
|
|
sk = bpf_sk_fullsock(sk);
|
|
if (sk) {
|
|
if (sk->protocol == IPPROTO_TCP) {
|
|
tp = bpf_tcp_sock(sk);
|
|
if (tp) {
|
|
pkti->cwnd = tp->snd_cwnd;
|
|
pkti->rtt = tp->srtt_us >> 3;
|
|
pkti->packets_out = tp->packets_out;
|
|
return 0;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
pkti->cwnd = 0;
|
|
pkti->rtt = 0;
|
|
pkti->packets_out = 0;
|
|
return 1;
|
|
}
|
|
|
|
static void hbm_get_pkt_info(struct __sk_buff *skb,
|
|
struct hbm_pkt_info *pkti)
|
|
{
|
|
struct iphdr iph;
|
|
struct ipv6hdr *ip6h;
|
|
|
|
pkti->cwnd = 0;
|
|
pkti->rtt = 0;
|
|
bpf_skb_load_bytes(skb, 0, &iph, 12);
|
|
if (iph.version == 6) {
|
|
ip6h = (struct ipv6hdr *)&iph;
|
|
pkti->is_ip = true;
|
|
pkti->is_tcp = (ip6h->nexthdr == 6);
|
|
pkti->ecn = (ip6h->flow_lbl[0] >> 4) & INET_ECN_MASK;
|
|
} else if (iph.version == 4) {
|
|
pkti->is_ip = true;
|
|
pkti->is_tcp = (iph.protocol == 6);
|
|
pkti->ecn = iph.tos & INET_ECN_MASK;
|
|
} else {
|
|
pkti->is_ip = false;
|
|
pkti->is_tcp = false;
|
|
pkti->ecn = 0;
|
|
}
|
|
if (pkti->is_tcp)
|
|
get_tcp_info(skb, pkti);
|
|
}
|
|
|
|
static __always_inline void hbm_init_vqueue(struct hbm_vqueue *qdp, int rate)
|
|
{
|
|
bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
|
|
qdp->lasttime = bpf_ktime_get_ns();
|
|
qdp->credit = INIT_CREDIT;
|
|
qdp->rate = rate * 128;
|
|
}
|
|
|
|
static __always_inline void hbm_init_edt_vqueue(struct hbm_vqueue *qdp,
|
|
int rate)
|
|
{
|
|
unsigned long long curtime;
|
|
|
|
curtime = bpf_ktime_get_ns();
|
|
bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
|
|
qdp->lasttime = curtime - BURST_SIZE_NS; // support initial burst
|
|
qdp->credit = 0; // not used
|
|
qdp->rate = rate * 128;
|
|
}
|
|
|
|
static __always_inline void hbm_update_stats(struct hbm_queue_stats *qsp,
|
|
int len,
|
|
unsigned long long curtime,
|
|
bool congestion_flag,
|
|
bool drop_flag,
|
|
bool cwr_flag,
|
|
bool ecn_ce_flag,
|
|
struct hbm_pkt_info *pkti,
|
|
int credit)
|
|
{
|
|
int rv = ALLOW_PKT;
|
|
|
|
if (qsp != NULL) {
|
|
// Following is needed for work conserving
|
|
__sync_add_and_fetch(&(qsp->bytes_total), len);
|
|
if (qsp->stats) {
|
|
// Optionally update statistics
|
|
if (qsp->firstPacketTime == 0)
|
|
qsp->firstPacketTime = curtime;
|
|
qsp->lastPacketTime = curtime;
|
|
__sync_add_and_fetch(&(qsp->pkts_total), 1);
|
|
if (congestion_flag) {
|
|
__sync_add_and_fetch(&(qsp->pkts_marked), 1);
|
|
__sync_add_and_fetch(&(qsp->bytes_marked), len);
|
|
}
|
|
if (drop_flag) {
|
|
__sync_add_and_fetch(&(qsp->pkts_dropped), 1);
|
|
__sync_add_and_fetch(&(qsp->bytes_dropped),
|
|
len);
|
|
}
|
|
if (ecn_ce_flag)
|
|
__sync_add_and_fetch(&(qsp->pkts_ecn_ce), 1);
|
|
if (pkti->cwnd) {
|
|
__sync_add_and_fetch(&(qsp->sum_cwnd),
|
|
pkti->cwnd);
|
|
__sync_add_and_fetch(&(qsp->sum_cwnd_cnt), 1);
|
|
}
|
|
if (pkti->rtt)
|
|
__sync_add_and_fetch(&(qsp->sum_rtt),
|
|
pkti->rtt);
|
|
__sync_add_and_fetch(&(qsp->sum_credit), credit);
|
|
|
|
if (drop_flag)
|
|
rv = DROP_PKT;
|
|
if (cwr_flag)
|
|
rv |= 2;
|
|
if (rv == DROP_PKT)
|
|
__sync_add_and_fetch(&(qsp->returnValCount[0]),
|
|
1);
|
|
else if (rv == ALLOW_PKT)
|
|
__sync_add_and_fetch(&(qsp->returnValCount[1]),
|
|
1);
|
|
else if (rv == 2)
|
|
__sync_add_and_fetch(&(qsp->returnValCount[2]),
|
|
1);
|
|
else if (rv == 3)
|
|
__sync_add_and_fetch(&(qsp->returnValCount[3]),
|
|
1);
|
|
}
|
|
}
|
|
}
|