[TCP]: Calculate SKB tso factor more accurately.

Eliminate tp->mss_tso_factor.  Instead, we calculate the
SKB tso factor as we walk the write queue for initial
transmit or fragment SKBs.
Signed-off-by: David S. Miller <davem@davemloft.net>

include/linux/tcp.h

@@ -254,7 +254,6 @@ struct tcp_opt {
 	__u32	max_window;	/* Maximal window ever seen from peer	*/
 	__u32	pmtu_cookie;	/* Last pmtu seen by socket		*/
 	__u32	mss_cache;	/* Cached effective mss, not including SACKS */
-	__u32	mss_tso_factor;	/* Real packets per TSO packet */
 	__u16	mss_cache_std;	/* Like mss_cache, but without TSO */
 	__u16	mss_clamp;	/* Maximal mss, negotiated at connection setup */
 	__u16	ext_header_len;	/* Network protocol overhead (IP/IPv6 options) */

include/net/tcp.h

@@ -1047,7 +1047,7 @@ static inline void tcp_reset_xmit_timer(struct sock *sk, int what, unsigned long
  * is not a big flaw.
  */
 
-static inline unsigned int tcp_current_mss(struct sock *sk, int large, int *factor)
+static inline unsigned int tcp_current_mss(struct sock *sk, int large)
 {
 	struct tcp_opt *tp = tcp_sk(sk);
 	struct dst_entry *dst = __sk_dst_get(sk);
@@ -1057,8 +1057,6 @@ static inline unsigned int tcp_current_mss(struct sock *sk, int large, int *fact
 		    (sk->sk_route_caps & NETIF_F_TSO) &&
 		    !tp->urg_mode);
 	mss_now = do_large ? tp->mss_cache : tp->mss_cache_std;
-	if (factor)
-		*factor = do_large ? tp->mss_tso_factor : 1;
 
 	if (dst) {
 		u32 mtu = dst_pmtu(dst);
@@ -1545,7 +1543,7 @@ static __inline__ void __tcp_push_pending_frames(struct sock *sk,
 static __inline__ void tcp_push_pending_frames(struct sock *sk,
 					       struct tcp_opt *tp)
 {
-	__tcp_push_pending_frames(sk, tp, tcp_current_mss(sk, 1, NULL), tp->nonagle);
+	__tcp_push_pending_frames(sk, tp, tcp_current_mss(sk, 1), tp->nonagle);
 }
 
 static __inline__ int tcp_may_send_now(struct sock *sk, struct tcp_opt *tp)
@@ -1553,7 +1551,7 @@ static __inline__ int tcp_may_send_now(struct sock *sk, struct tcp_opt *tp)
 	struct sk_buff *skb = sk->sk_send_head;
 
 	return (skb &&
-		tcp_snd_test(tp, skb, tcp_current_mss(sk, 1, NULL),
+		tcp_snd_test(tp, skb, tcp_current_mss(sk, 1),
 			     tcp_skb_is_last(sk, skb) ? TCP_NAGLE_PUSH : tp->nonagle));
 }
 

net/ipv4/tcp.c

@@ -590,14 +590,13 @@ static inline int forced_push(struct tcp_opt *tp)
 }
 
 static inline void skb_entail(struct sock *sk, struct tcp_opt *tp,
-			      struct sk_buff *skb, int tso_factor)
+			      struct sk_buff *skb)
 {
 	skb->csum = 0;
 	TCP_SKB_CB(skb)->seq = tp->write_seq;
 	TCP_SKB_CB(skb)->end_seq = tp->write_seq;
 	TCP_SKB_CB(skb)->flags = TCPCB_FLAG_ACK;
 	TCP_SKB_CB(skb)->sacked = 0;
-	TCP_SKB_CB(skb)->tso_factor = tso_factor;
 	__skb_queue_tail(&sk->sk_write_queue, skb);
 	sk_charge_skb(sk, skb);
 	if (!sk->sk_send_head)
@@ -633,7 +632,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse
 			 size_t psize, int flags)
 {
 	struct tcp_opt *tp = tcp_sk(sk);
-	int mss_now, mss_factor_now;
+	int mss_now;
 	int err;
 	ssize_t copied;
 	long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
@@ -645,7 +644,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse
 
 	clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 
-	mss_now = tcp_current_mss(sk, !(flags&MSG_OOB), &mss_factor_now);
+	mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 	copied = 0;
 
 	err = -EPIPE;
@@ -669,7 +668,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse
 			if (!skb)
 				goto wait_for_memory;
 
-			skb_entail(sk, tp, skb, mss_factor_now);
+			skb_entail(sk, tp, skb);
 			copy = mss_now;
 		}
 
@@ -720,8 +719,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse
 		if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
 			goto do_error;
 
-		mss_now = tcp_current_mss(sk, !(flags&MSG_OOB),
-					  &mss_factor_now);
+		mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 	}
 
 out:
@@ -782,7 +780,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 	struct tcp_opt *tp = tcp_sk(sk);
 	struct sk_buff *skb;
 	int iovlen, flags;
-	int mss_now, mss_factor_now;
+	int mss_now;
 	int err, copied;
 	long timeo;
 
@@ -800,7 +798,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 	/* This should be in poll */
 	clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 
-	mss_now = tcp_current_mss(sk, !(flags&MSG_OOB), &mss_factor_now);
+	mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 
 	/* Ok commence sending. */
 	iovlen = msg->msg_iovlen;
@@ -845,7 +843,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 				     NETIF_F_HW_CSUM))
 					skb->ip_summed = CHECKSUM_HW;
 
-				skb_entail(sk, tp, skb, mss_factor_now);
+				skb_entail(sk, tp, skb);
 				copy = mss_now;
 			}
 
@@ -964,8 +962,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 			if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
 				goto do_error;
 
-			mss_now = tcp_current_mss(sk, !(flags&MSG_OOB),
-						  &mss_factor_now);
+			mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 		}
 	}
 

net/ipv4/tcp_input.c

@@ -974,7 +974,6 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_
 		sk->sk_route_caps &= ~NETIF_F_TSO;
 		sk->sk_no_largesend = 1;
 		tp->mss_cache = tp->mss_cache_std;
-		tp->mss_tso_factor = 1;
 	}
 
 	if (!tcp_get_pcount(&tp->sacked_out))

net/ipv4/tcp_ipv4.c

@@ -2076,7 +2076,6 @@ static int tcp_v4_init_sock(struct sock *sk)
 	tp->snd_ssthresh = 0x7fffffff;	/* Infinity */
 	tp->snd_cwnd_clamp = ~0;
 	tp->mss_cache_std = tp->mss_cache = 536;
-	tp->mss_tso_factor = 1;
 
 	tp->reordering = sysctl_tcp_reordering;
 

net/ipv4/tcp_output.c

@@ -422,6 +422,23 @@ void tcp_push_one(struct sock *sk, unsigned cur_mss)
 	}
 }
 
+static void tcp_set_skb_tso_factor(struct sk_buff *skb, unsigned int mss,
+				   unsigned int mss_std)
+{
+	if (skb->len <= mss_std) {
+		/* Avoid the costly divide in the normal
+		 * non-TSO case.
+		 */
+		TCP_SKB_CB(skb)->tso_factor = 1;
+	} else {
+		unsigned int factor;
+
+		factor = skb->len + (mss_std - 1);
+		factor /= mss;
+		TCP_SKB_CB(skb)->tso_factor = factor;
+	}
+}
+
 /* Function to create two new TCP segments.  Shrinks the given segment
  * to the specified size and appends a new segment with the rest of the
  * packet to the list.  This won't be called frequently, I hope. 
@@ -457,7 +474,6 @@ static int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len)
 	TCP_SKB_CB(buff)->sacked =
 		(TCP_SKB_CB(skb)->sacked &
 		 (TCPCB_LOST | TCPCB_EVER_RETRANS | TCPCB_AT_TAIL));
-	TCP_SKB_CB(buff)->tso_factor = tp->mss_tso_factor;
 	if (TCP_SKB_CB(buff)->sacked&TCPCB_LOST) {
 		tcp_inc_pcount(&tp->lost_out, buff);
 		tcp_inc_pcount(&tp->left_out, buff);
@@ -484,6 +500,10 @@ static int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len)
 	 */
 	TCP_SKB_CB(buff)->when = TCP_SKB_CB(skb)->when;
 
+	/* Fix up tso_factor for both original and new SKB.  */
+	tcp_set_skb_tso_factor(skb, tp->mss_cache, tp->mss_cache_std);
+	tcp_set_skb_tso_factor(buff, tp->mss_cache, tp->mss_cache_std);
+
 	/* Link BUFF into the send queue. */
 	__skb_append(skb, buff);
 
@@ -598,7 +618,6 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu)
 	/* And store cached results */
 	tp->pmtu_cookie = pmtu;
 	tp->mss_cache = tp->mss_cache_std = mss_now;
-	tp->mss_tso_factor = 1;
 
 	if (sk->sk_route_caps & NETIF_F_TSO) {
 		int large_mss, factor;
@@ -617,7 +636,6 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu)
 			factor = tp->snd_cwnd;
 
 		tp->mss_cache = mss_now * factor;
-		tp->mss_tso_factor = factor;
 	}
 
 	return mss_now;
@@ -634,7 +652,7 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu)
 int tcp_write_xmit(struct sock *sk, int nonagle)
 {
 	struct tcp_opt *tp = tcp_sk(sk);
-	unsigned int mss_now;
+	unsigned int mss_now, mss_std;
 
 	/* If we are closed, the bytes will have to remain here.
 	 * In time closedown will finish, we empty the write queue and all
@@ -649,7 +667,8 @@ int tcp_write_xmit(struct sock *sk, int nonagle)
 		 * We also handle things correctly when the user adds some
 		 * IP options mid-stream.  Silly to do, but cover it.
 		 */
-		mss_now = tcp_current_mss(sk, 1, NULL);
+		mss_now = tcp_current_mss(sk, 1);
+		mss_std = tp->mss_cache_std;
 
 		while ((skb = sk->sk_send_head) &&
 		       tcp_snd_test(tp, skb, mss_now,
@@ -658,7 +677,8 @@ int tcp_write_xmit(struct sock *sk, int nonagle)
 			if (skb->len > mss_now) {
 				if (tcp_fragment(sk, skb, mss_now))
 					break;
-			}
+			} else
+				tcp_set_skb_tso_factor(skb, mss_now, mss_std);
 
 			TCP_SKB_CB(skb)->when = tcp_time_stamp;
 			if (tcp_transmit_skb(sk, skb_clone(skb, GFP_ATOMIC)))
@@ -871,7 +891,7 @@ void tcp_simple_retransmit(struct sock *sk)
 {
 	struct tcp_opt *tp = tcp_sk(sk);
 	struct sk_buff *skb;
-	unsigned int mss = tcp_current_mss(sk, 0, NULL);
+	unsigned int mss = tcp_current_mss(sk, 0);
 	int lost = 0;
 
 	sk_stream_for_retrans_queue(skb, sk) {
@@ -916,7 +936,7 @@ void tcp_simple_retransmit(struct sock *sk)
 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
 {
 	struct tcp_opt *tp = tcp_sk(sk);
- 	unsigned int cur_mss = tcp_current_mss(sk, 0, NULL);
+ 	unsigned int cur_mss = tcp_current_mss(sk, 0);
 	int err;
 
 	/* Do not sent more than we queued. 1/4 is reserved for possible
@@ -934,7 +954,6 @@ int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
 			sk->sk_route_caps &= ~NETIF_F_TSO;
 			sk->sk_no_largesend = 1;
 			tp->mss_cache = tp->mss_cache_std;
-			tp->mss_tso_factor = 1;
 		}
 
 		if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
@@ -1125,7 +1144,7 @@ void tcp_send_fin(struct sock *sk)
 	 * unsent frames.  But be careful about outgoing SACKS
 	 * and IP options.
 	 */
-	mss_now = tcp_current_mss(sk, 1, NULL);
+	mss_now = tcp_current_mss(sk, 1);
 
 	if (sk->sk_send_head != NULL) {
 		TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN;
@@ -1516,8 +1535,9 @@ int tcp_write_wakeup(struct sock *sk)
 		if ((skb = sk->sk_send_head) != NULL &&
 		    before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)) {
 			int err;
-			int mss = tcp_current_mss(sk, 0, NULL);
-			int seg_size = tp->snd_una+tp->snd_wnd-TCP_SKB_CB(skb)->seq;
+			unsigned int mss = tcp_current_mss(sk, 0);
+			unsigned int mss_std = tp->mss_cache_std;
+			unsigned int seg_size = tp->snd_una+tp->snd_wnd-TCP_SKB_CB(skb)->seq;
 
 			if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
 				tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
@@ -1538,9 +1558,10 @@ int tcp_write_wakeup(struct sock *sk)
 					sk->sk_no_largesend = 1;
 					sk->sk_route_caps &= ~NETIF_F_TSO;
 					tp->mss_cache = tp->mss_cache_std;
-					tp->mss_tso_factor = 1;
 				}
-			}
+			} else
+				tcp_set_skb_tso_factor(skb, mss, mss_std);
+
 			TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
 			TCP_SKB_CB(skb)->when = tcp_time_stamp;
 			err = tcp_transmit_skb(sk, skb_clone(skb, GFP_ATOMIC));

net/ipv6/tcp_ipv6.c

@@ -1930,7 +1930,6 @@ static int tcp_v6_init_sock(struct sock *sk)
 	tp->snd_ssthresh = 0x7fffffff;
 	tp->snd_cwnd_clamp = ~0;
 	tp->mss_cache_std = tp->mss_cache = 536;
-	tp->mss_tso_factor = 1;
 
 	tp->reordering = sysctl_tcp_reordering;