Commit 0927f71d authored by Doron Roberts-Kedes's avatar Doron Roberts-Kedes Committed by David S. Miller

net/tls: Calculate nsg for zerocopy path without skb_cow_data.

decrypt_skb fails if the number of sg elements required to map it
is greater than MAX_SKB_FRAGS. nsg must always be calculated, but
skb_cow_data adds unnecessary memcpy's for the zerocopy case.

The new function skb_nsg calculates the number of scatterlist elements
required to map the skb without the extra overhead of skb_cow_data.
This patch reduces memcpy by 50% on my encrypted NBD benchmarks.
Reported-by: default avatarVakul Garg <Vakul.garg@nxp.com>
Reviewed-by: default avatarVakul Garg <Vakul.garg@nxp.com>
Tested-by: default avatarVakul Garg <Vakul.garg@nxp.com>
Signed-off-by: default avatarDoron Roberts-Kedes <doronrk@fb.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 7e8d5755
...@@ -43,6 +43,82 @@ ...@@ -43,6 +43,82 @@
#define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE #define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE
static int __skb_nsg(struct sk_buff *skb, int offset, int len,
unsigned int recursion_level)
{
int start = skb_headlen(skb);
int i, chunk = start - offset;
struct sk_buff *frag_iter;
int elt = 0;
if (unlikely(recursion_level >= 24))
return -EMSGSIZE;
if (chunk > 0) {
if (chunk > len)
chunk = len;
elt++;
len -= chunk;
if (len == 0)
return elt;
offset += chunk;
}
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
int end;
WARN_ON(start > offset + len);
end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
chunk = end - offset;
if (chunk > 0) {
if (chunk > len)
chunk = len;
elt++;
len -= chunk;
if (len == 0)
return elt;
offset += chunk;
}
start = end;
}
if (unlikely(skb_has_frag_list(skb))) {
skb_walk_frags(skb, frag_iter) {
int end, ret;
WARN_ON(start > offset + len);
end = start + frag_iter->len;
chunk = end - offset;
if (chunk > 0) {
if (chunk > len)
chunk = len;
ret = __skb_nsg(frag_iter, offset - start, chunk,
recursion_level + 1);
if (unlikely(ret < 0))
return ret;
elt += ret;
len -= chunk;
if (len == 0)
return elt;
offset += chunk;
}
start = end;
}
}
BUG_ON(len);
return elt;
}
/* Return the number of scatterlist elements required to completely map the
* skb, or -EMSGSIZE if the recursion depth is exceeded.
*/
static int skb_nsg(struct sk_buff *skb, int offset, int len)
{
return __skb_nsg(skb, offset, len, 0);
}
static int tls_do_decryption(struct sock *sk, static int tls_do_decryption(struct sock *sk,
struct scatterlist *sgin, struct scatterlist *sgin,
struct scatterlist *sgout, struct scatterlist *sgout,
...@@ -678,12 +754,14 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb, ...@@ -678,12 +754,14 @@ static int decrypt_internal(struct sock *sk, struct sk_buff *skb,
n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1; n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1;
else else
n_sgout = sg_nents(out_sg); n_sgout = sg_nents(out_sg);
n_sgin = skb_nsg(skb, rxm->offset + tls_ctx->rx.prepend_size,
rxm->full_len - tls_ctx->rx.prepend_size);
} else { } else {
n_sgout = 0; n_sgout = 0;
*zc = false; *zc = false;
n_sgin = skb_cow_data(skb, 0, &unused);
} }
n_sgin = skb_cow_data(skb, 0, &unused);
if (n_sgin < 1) if (n_sgin < 1)
return -EBADMSG; return -EBADMSG;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment