![Florian Westphal](/assets/img/avatar_default.png)
Split assignment and initialization from one into two functions. This is required by followup patches that add Datacenter TCP (DCTCP) congestion control algorithm - we need to be able to determine if the connection is moderated by DCTCP before the 3WHS has finished. As we walk the available congestion control list during the assignment, we are always guaranteed to have Reno present as it's fixed compiled-in. Therefore, since we're doing the early assignment, we don't have a real use for the Reno alias tcp_init_congestion_ops anymore and can thus remove it. Actual usage of the congestion control operations are being made after the 3WHS has finished, in some cases however we can access get_info() via diag if implemented, therefore we need to zero out the private area for those modules. Joint work with Daniel Borkmann and Glenn Judd. Signed-off-by: Florian Westphal <fw@strlen.de> Signed-off-by: Daniel Borkmann <dborkman@redhat.com> Signed-off-by: Glenn Judd <glenn.judd@morganstanley.com> Acked-by: Stephen Hemminger <stephen@networkplumber.org> Signed-off-by: David S. Miller <davem@davemloft.net>
358 lines
8.7 KiB
C
358 lines
8.7 KiB
C
/*
|
|
* Plugable TCP congestion control support and newReno
|
|
* congestion control.
|
|
* Based on ideas from I/O scheduler support and Web100.
|
|
*
|
|
* Copyright (C) 2005 Stephen Hemminger <shemminger@osdl.org>
|
|
*/
|
|
|
|
#define pr_fmt(fmt) "TCP: " fmt
|
|
|
|
#include <linux/module.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/types.h>
|
|
#include <linux/list.h>
|
|
#include <linux/gfp.h>
|
|
#include <net/tcp.h>
|
|
|
|
static DEFINE_SPINLOCK(tcp_cong_list_lock);
|
|
static LIST_HEAD(tcp_cong_list);
|
|
|
|
/* Simple linear search, don't expect many entries! */
|
|
static struct tcp_congestion_ops *tcp_ca_find(const char *name)
|
|
{
|
|
struct tcp_congestion_ops *e;
|
|
|
|
list_for_each_entry_rcu(e, &tcp_cong_list, list) {
|
|
if (strcmp(e->name, name) == 0)
|
|
return e;
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/*
|
|
* Attach new congestion control algorithm to the list
|
|
* of available options.
|
|
*/
|
|
int tcp_register_congestion_control(struct tcp_congestion_ops *ca)
|
|
{
|
|
int ret = 0;
|
|
|
|
/* all algorithms must implement ssthresh and cong_avoid ops */
|
|
if (!ca->ssthresh || !ca->cong_avoid) {
|
|
pr_err("%s does not implement required ops\n", ca->name);
|
|
return -EINVAL;
|
|
}
|
|
|
|
spin_lock(&tcp_cong_list_lock);
|
|
if (tcp_ca_find(ca->name)) {
|
|
pr_notice("%s already registered\n", ca->name);
|
|
ret = -EEXIST;
|
|
} else {
|
|
list_add_tail_rcu(&ca->list, &tcp_cong_list);
|
|
pr_info("%s registered\n", ca->name);
|
|
}
|
|
spin_unlock(&tcp_cong_list_lock);
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(tcp_register_congestion_control);
|
|
|
|
/*
|
|
* Remove congestion control algorithm, called from
|
|
* the module's remove function. Module ref counts are used
|
|
* to ensure that this can't be done till all sockets using
|
|
* that method are closed.
|
|
*/
|
|
void tcp_unregister_congestion_control(struct tcp_congestion_ops *ca)
|
|
{
|
|
spin_lock(&tcp_cong_list_lock);
|
|
list_del_rcu(&ca->list);
|
|
spin_unlock(&tcp_cong_list_lock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(tcp_unregister_congestion_control);
|
|
|
|
/* Assign choice of congestion control. */
|
|
void tcp_assign_congestion_control(struct sock *sk)
|
|
{
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
|
struct tcp_congestion_ops *ca;
|
|
|
|
rcu_read_lock();
|
|
list_for_each_entry_rcu(ca, &tcp_cong_list, list) {
|
|
if (likely(try_module_get(ca->owner))) {
|
|
icsk->icsk_ca_ops = ca;
|
|
goto out;
|
|
}
|
|
/* Fallback to next available. The last really
|
|
* guaranteed fallback is Reno from this list.
|
|
*/
|
|
}
|
|
out:
|
|
rcu_read_unlock();
|
|
|
|
/* Clear out private data before diag gets it and
|
|
* the ca has not been initialized.
|
|
*/
|
|
if (ca->get_info)
|
|
memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
|
|
}
|
|
|
|
void tcp_init_congestion_control(struct sock *sk)
|
|
{
|
|
const struct inet_connection_sock *icsk = inet_csk(sk);
|
|
|
|
if (icsk->icsk_ca_ops->init)
|
|
icsk->icsk_ca_ops->init(sk);
|
|
}
|
|
|
|
/* Manage refcounts on socket close. */
|
|
void tcp_cleanup_congestion_control(struct sock *sk)
|
|
{
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
|
|
|
if (icsk->icsk_ca_ops->release)
|
|
icsk->icsk_ca_ops->release(sk);
|
|
module_put(icsk->icsk_ca_ops->owner);
|
|
}
|
|
|
|
/* Used by sysctl to change default congestion control */
|
|
int tcp_set_default_congestion_control(const char *name)
|
|
{
|
|
struct tcp_congestion_ops *ca;
|
|
int ret = -ENOENT;
|
|
|
|
spin_lock(&tcp_cong_list_lock);
|
|
ca = tcp_ca_find(name);
|
|
#ifdef CONFIG_MODULES
|
|
if (!ca && capable(CAP_NET_ADMIN)) {
|
|
spin_unlock(&tcp_cong_list_lock);
|
|
|
|
request_module("tcp_%s", name);
|
|
spin_lock(&tcp_cong_list_lock);
|
|
ca = tcp_ca_find(name);
|
|
}
|
|
#endif
|
|
|
|
if (ca) {
|
|
ca->flags |= TCP_CONG_NON_RESTRICTED; /* default is always allowed */
|
|
list_move(&ca->list, &tcp_cong_list);
|
|
ret = 0;
|
|
}
|
|
spin_unlock(&tcp_cong_list_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/* Set default value from kernel configuration at bootup */
|
|
static int __init tcp_congestion_default(void)
|
|
{
|
|
return tcp_set_default_congestion_control(CONFIG_DEFAULT_TCP_CONG);
|
|
}
|
|
late_initcall(tcp_congestion_default);
|
|
|
|
/* Build string with list of available congestion control values */
|
|
void tcp_get_available_congestion_control(char *buf, size_t maxlen)
|
|
{
|
|
struct tcp_congestion_ops *ca;
|
|
size_t offs = 0;
|
|
|
|
rcu_read_lock();
|
|
list_for_each_entry_rcu(ca, &tcp_cong_list, list) {
|
|
offs += snprintf(buf + offs, maxlen - offs,
|
|
"%s%s",
|
|
offs == 0 ? "" : " ", ca->name);
|
|
}
|
|
rcu_read_unlock();
|
|
}
|
|
|
|
/* Get current default congestion control */
|
|
void tcp_get_default_congestion_control(char *name)
|
|
{
|
|
struct tcp_congestion_ops *ca;
|
|
/* We will always have reno... */
|
|
BUG_ON(list_empty(&tcp_cong_list));
|
|
|
|
rcu_read_lock();
|
|
ca = list_entry(tcp_cong_list.next, struct tcp_congestion_ops, list);
|
|
strncpy(name, ca->name, TCP_CA_NAME_MAX);
|
|
rcu_read_unlock();
|
|
}
|
|
|
|
/* Built list of non-restricted congestion control values */
|
|
void tcp_get_allowed_congestion_control(char *buf, size_t maxlen)
|
|
{
|
|
struct tcp_congestion_ops *ca;
|
|
size_t offs = 0;
|
|
|
|
*buf = '\0';
|
|
rcu_read_lock();
|
|
list_for_each_entry_rcu(ca, &tcp_cong_list, list) {
|
|
if (!(ca->flags & TCP_CONG_NON_RESTRICTED))
|
|
continue;
|
|
offs += snprintf(buf + offs, maxlen - offs,
|
|
"%s%s",
|
|
offs == 0 ? "" : " ", ca->name);
|
|
}
|
|
rcu_read_unlock();
|
|
}
|
|
|
|
/* Change list of non-restricted congestion control */
|
|
int tcp_set_allowed_congestion_control(char *val)
|
|
{
|
|
struct tcp_congestion_ops *ca;
|
|
char *saved_clone, *clone, *name;
|
|
int ret = 0;
|
|
|
|
saved_clone = clone = kstrdup(val, GFP_USER);
|
|
if (!clone)
|
|
return -ENOMEM;
|
|
|
|
spin_lock(&tcp_cong_list_lock);
|
|
/* pass 1 check for bad entries */
|
|
while ((name = strsep(&clone, " ")) && *name) {
|
|
ca = tcp_ca_find(name);
|
|
if (!ca) {
|
|
ret = -ENOENT;
|
|
goto out;
|
|
}
|
|
}
|
|
|
|
/* pass 2 clear old values */
|
|
list_for_each_entry_rcu(ca, &tcp_cong_list, list)
|
|
ca->flags &= ~TCP_CONG_NON_RESTRICTED;
|
|
|
|
/* pass 3 mark as allowed */
|
|
while ((name = strsep(&val, " ")) && *name) {
|
|
ca = tcp_ca_find(name);
|
|
WARN_ON(!ca);
|
|
if (ca)
|
|
ca->flags |= TCP_CONG_NON_RESTRICTED;
|
|
}
|
|
out:
|
|
spin_unlock(&tcp_cong_list_lock);
|
|
kfree(saved_clone);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/* Change congestion control for socket */
|
|
int tcp_set_congestion_control(struct sock *sk, const char *name)
|
|
{
|
|
struct inet_connection_sock *icsk = inet_csk(sk);
|
|
struct tcp_congestion_ops *ca;
|
|
int err = 0;
|
|
|
|
rcu_read_lock();
|
|
ca = tcp_ca_find(name);
|
|
|
|
/* no change asking for existing value */
|
|
if (ca == icsk->icsk_ca_ops)
|
|
goto out;
|
|
|
|
#ifdef CONFIG_MODULES
|
|
/* not found attempt to autoload module */
|
|
if (!ca && capable(CAP_NET_ADMIN)) {
|
|
rcu_read_unlock();
|
|
request_module("tcp_%s", name);
|
|
rcu_read_lock();
|
|
ca = tcp_ca_find(name);
|
|
}
|
|
#endif
|
|
if (!ca)
|
|
err = -ENOENT;
|
|
|
|
else if (!((ca->flags & TCP_CONG_NON_RESTRICTED) ||
|
|
ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)))
|
|
err = -EPERM;
|
|
|
|
else if (!try_module_get(ca->owner))
|
|
err = -EBUSY;
|
|
|
|
else {
|
|
tcp_cleanup_congestion_control(sk);
|
|
icsk->icsk_ca_ops = ca;
|
|
|
|
if (sk->sk_state != TCP_CLOSE && icsk->icsk_ca_ops->init)
|
|
icsk->icsk_ca_ops->init(sk);
|
|
}
|
|
out:
|
|
rcu_read_unlock();
|
|
return err;
|
|
}
|
|
|
|
/* Slow start is used when congestion window is no greater than the slow start
|
|
* threshold. We base on RFC2581 and also handle stretch ACKs properly.
|
|
* We do not implement RFC3465 Appropriate Byte Counting (ABC) per se but
|
|
* something better;) a packet is only considered (s)acked in its entirety to
|
|
* defend the ACK attacks described in the RFC. Slow start processes a stretch
|
|
* ACK of degree N as if N acks of degree 1 are received back to back except
|
|
* ABC caps N to 2. Slow start exits when cwnd grows over ssthresh and
|
|
* returns the leftover acks to adjust cwnd in congestion avoidance mode.
|
|
*/
|
|
int tcp_slow_start(struct tcp_sock *tp, u32 acked)
|
|
{
|
|
u32 cwnd = tp->snd_cwnd + acked;
|
|
|
|
if (cwnd > tp->snd_ssthresh)
|
|
cwnd = tp->snd_ssthresh + 1;
|
|
acked -= cwnd - tp->snd_cwnd;
|
|
tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp);
|
|
return acked;
|
|
}
|
|
EXPORT_SYMBOL_GPL(tcp_slow_start);
|
|
|
|
/* In theory this is tp->snd_cwnd += 1 / tp->snd_cwnd (or alternative w) */
|
|
void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w)
|
|
{
|
|
if (tp->snd_cwnd_cnt >= w) {
|
|
if (tp->snd_cwnd < tp->snd_cwnd_clamp)
|
|
tp->snd_cwnd++;
|
|
tp->snd_cwnd_cnt = 0;
|
|
} else {
|
|
tp->snd_cwnd_cnt++;
|
|
}
|
|
}
|
|
EXPORT_SYMBOL_GPL(tcp_cong_avoid_ai);
|
|
|
|
/*
|
|
* TCP Reno congestion control
|
|
* This is special case used for fallback as well.
|
|
*/
|
|
/* This is Jacobson's slow start and congestion avoidance.
|
|
* SIGCOMM '88, p. 328.
|
|
*/
|
|
void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked)
|
|
{
|
|
struct tcp_sock *tp = tcp_sk(sk);
|
|
|
|
if (!tcp_is_cwnd_limited(sk))
|
|
return;
|
|
|
|
/* In "safe" area, increase. */
|
|
if (tp->snd_cwnd <= tp->snd_ssthresh)
|
|
tcp_slow_start(tp, acked);
|
|
/* In dangerous area, increase slowly. */
|
|
else
|
|
tcp_cong_avoid_ai(tp, tp->snd_cwnd);
|
|
}
|
|
EXPORT_SYMBOL_GPL(tcp_reno_cong_avoid);
|
|
|
|
/* Slow start threshold is half the congestion window (min 2) */
|
|
u32 tcp_reno_ssthresh(struct sock *sk)
|
|
{
|
|
const struct tcp_sock *tp = tcp_sk(sk);
|
|
|
|
return max(tp->snd_cwnd >> 1U, 2U);
|
|
}
|
|
EXPORT_SYMBOL_GPL(tcp_reno_ssthresh);
|
|
|
|
struct tcp_congestion_ops tcp_reno = {
|
|
.flags = TCP_CONG_NON_RESTRICTED,
|
|
.name = "reno",
|
|
.owner = THIS_MODULE,
|
|
.ssthresh = tcp_reno_ssthresh,
|
|
.cong_avoid = tcp_reno_cong_avoid,
|
|
};
|