Commit 3cd3399d authored by Eric Dumazet's avatar Eric Dumazet Committed by Jakub Kicinski

net: implement per-cpu reserves for memory_allocated

We plan keeping sk->sk_forward_alloc as small as possible
in future patches.

This means we are going to call sk_memory_allocated_add()
and sk_memory_allocated_sub() more often.

Implement a per-cpu cache of +1/-1 MB, to reduce number
of changes to sk->sk_prot->memory_allocated, which
would otherwise be cause of false sharing.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarSoheil Hassas Yeganeh <soheil@google.com>
Reviewed-by: default avatarShakeel Butt <shakeelb@google.com>
Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 0defbb0a
...@@ -1397,22 +1397,48 @@ static inline bool sk_under_memory_pressure(const struct sock *sk) ...@@ -1397,22 +1397,48 @@ static inline bool sk_under_memory_pressure(const struct sock *sk)
return !!*sk->sk_prot->memory_pressure; return !!*sk->sk_prot->memory_pressure;
} }
static inline long
proto_memory_allocated(const struct proto *prot)
{
return max(0L, atomic_long_read(prot->memory_allocated));
}
static inline long static inline long
sk_memory_allocated(const struct sock *sk) sk_memory_allocated(const struct sock *sk)
{ {
return atomic_long_read(sk->sk_prot->memory_allocated); return proto_memory_allocated(sk->sk_prot);
} }
/* 1 MB per cpu, in page units */
#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
static inline long static inline long
sk_memory_allocated_add(struct sock *sk, int amt) sk_memory_allocated_add(struct sock *sk, int amt)
{ {
return atomic_long_add_return(amt, sk->sk_prot->memory_allocated); int local_reserve;
preempt_disable();
local_reserve = __this_cpu_add_return(*sk->sk_prot->per_cpu_fw_alloc, amt);
if (local_reserve >= SK_MEMORY_PCPU_RESERVE) {
__this_cpu_sub(*sk->sk_prot->per_cpu_fw_alloc, local_reserve);
atomic_long_add(local_reserve, sk->sk_prot->memory_allocated);
}
preempt_enable();
return sk_memory_allocated(sk);
} }
static inline void static inline void
sk_memory_allocated_sub(struct sock *sk, int amt) sk_memory_allocated_sub(struct sock *sk, int amt)
{ {
atomic_long_sub(amt, sk->sk_prot->memory_allocated); int local_reserve;
preempt_disable();
local_reserve = __this_cpu_sub_return(*sk->sk_prot->per_cpu_fw_alloc, amt);
if (local_reserve <= -SK_MEMORY_PCPU_RESERVE) {
__this_cpu_sub(*sk->sk_prot->per_cpu_fw_alloc, local_reserve);
atomic_long_add(local_reserve, sk->sk_prot->memory_allocated);
}
preempt_enable();
} }
#define SK_ALLOC_PERCPU_COUNTER_BATCH 16 #define SK_ALLOC_PERCPU_COUNTER_BATCH 16
...@@ -1441,12 +1467,6 @@ proto_sockets_allocated_sum_positive(struct proto *prot) ...@@ -1441,12 +1467,6 @@ proto_sockets_allocated_sum_positive(struct proto *prot)
return percpu_counter_sum_positive(prot->sockets_allocated); return percpu_counter_sum_positive(prot->sockets_allocated);
} }
static inline long
proto_memory_allocated(struct proto *prot)
{
return atomic_long_read(prot->memory_allocated);
}
static inline bool static inline bool
proto_memory_pressure(struct proto *prot) proto_memory_pressure(struct proto *prot)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment