netvsc_drv.c 62.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * Copyright (c) 2009, Microsoft Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
14
 * this program; if not, see <http://www.gnu.org/licenses/>.
15 16
 *
 * Authors:
17
 *   Haiyang Zhang <haiyangz@microsoft.com>
18 19
 *   Hank Janssen  <hjanssen@microsoft.com>
 */
20 21
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

22
#include <linux/init.h>
23
#include <linux/atomic.h>
24 25 26 27 28 29 30 31
#include <linux/module.h>
#include <linux/highmem.h>
#include <linux/device.h>
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/etherdevice.h>
32
#include <linux/pci.h>
33
#include <linux/skbuff.h>
34
#include <linux/if_vlan.h>
35
#include <linux/in.h>
36
#include <linux/slab.h>
37
#include <linux/rtnetlink.h>
38
#include <linux/netpoll.h>
39

40 41 42 43
#include <net/arp.h>
#include <net/route.h>
#include <net/sock.h>
#include <net/pkt_sched.h>
44 45
#include <net/checksum.h>
#include <net/ip6_checksum.h>
46

47
#include "hyperv_net.h"
48

49 50 51 52
#define RING_SIZE_MIN	64
#define RETRY_US_LO	5000
#define RETRY_US_HI	10000
#define RETRY_MAX	2000	/* >10 sec */
53

54
#define LINKCHANGE_INT (2 * HZ)
55
#define VF_TAKEOVER_INT (HZ / 10)
56

57
static unsigned int ring_size __ro_after_init = 128;
58
module_param(ring_size, uint, 0444);
59
MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
60
unsigned int netvsc_ring_bytes __ro_after_init;
61

62 63 64 65 66 67
static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
				NETIF_MSG_LINK | NETIF_MSG_IFUP |
				NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR |
				NETIF_MSG_TX_ERR;

static int debug = -1;
68
module_param(debug, int, 0444);
69 70
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

71 72
static LIST_HEAD(netvsc_dev_list);

73
static void netvsc_change_rx_flags(struct net_device *net, int change)
74
{
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
	struct net_device_context *ndev_ctx = netdev_priv(net);
	struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
	int inc;

	if (!vf_netdev)
		return;

	if (change & IFF_PROMISC) {
		inc = (net->flags & IFF_PROMISC) ? 1 : -1;
		dev_set_promiscuity(vf_netdev, inc);
	}

	if (change & IFF_ALLMULTI) {
		inc = (net->flags & IFF_ALLMULTI) ? 1 : -1;
		dev_set_allmulti(vf_netdev, inc);
	}
}

static void netvsc_set_rx_mode(struct net_device *net)
{
	struct net_device_context *ndev_ctx = netdev_priv(net);
96 97
	struct net_device *vf_netdev;
	struct netvsc_device *nvdev;
98

99 100
	rcu_read_lock();
	vf_netdev = rcu_dereference(ndev_ctx->vf_netdev);
101 102 103 104
	if (vf_netdev) {
		dev_uc_sync(vf_netdev, net);
		dev_mc_sync(vf_netdev, net);
	}
105

106 107 108 109
	nvdev = rcu_dereference(ndev_ctx->nvdev);
	if (nvdev)
		rndis_filter_update(nvdev);
	rcu_read_unlock();
110 111
}

112 113 114 115 116 117 118 119 120
static void netvsc_tx_enable(struct netvsc_device *nvscdev,
			     struct net_device *ndev)
{
	nvscdev->tx_disable = false;
	virt_wmb(); /* ensure queue wake up mechanism is on */

	netif_tx_wake_all_queues(ndev);
}

121 122
static int netvsc_open(struct net_device *net)
{
123
	struct net_device_context *ndev_ctx = netdev_priv(net);
124
	struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
125
	struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev);
126
	struct rndis_device *rdev;
127
	int ret = 0;
128

129 130
	netif_carrier_off(net);

131
	/* Open up the device */
132
	ret = rndis_filter_open(nvdev);
133 134 135
	if (ret != 0) {
		netdev_err(net, "unable to open device (ret %d).\n", ret);
		return ret;
136 137
	}

138
	rdev = nvdev->extension;
139
	if (!rdev->link_state) {
140
		netif_carrier_on(net);
141
		netvsc_tx_enable(nvdev, net);
142
	}
143

144 145 146 147 148 149 150 151 152 153 154 155
	if (vf_netdev) {
		/* Setting synthetic device up transparently sets
		 * slave as up. If open fails, then slave will be
		 * still be offline (and not used).
		 */
		ret = dev_open(vf_netdev);
		if (ret)
			netdev_warn(net,
				    "unable to open slave: %s: %d\n",
				    vf_netdev->name, ret);
	}
	return 0;
156 157
}

158
static int netvsc_wait_until_empty(struct netvsc_device *nvdev)
159
{
160 161
	unsigned int retry = 0;
	int i;
162 163

	/* Ensure pending bytes in ring are read */
164 165 166
	for (;;) {
		u32 aread = 0;

167
		for (i = 0; i < nvdev->num_chn; i++) {
168 169 170
			struct vmbus_channel *chn
				= nvdev->chan_table[i].channel;

171 172 173
			if (!chn)
				continue;

174 175 176
			/* make sure receive not running now */
			napi_synchronize(&nvdev->chan_table[i].napi);

177
			aread = hv_get_bytes_to_read(&chn->inbound);
178 179 180
			if (aread)
				break;

181
			aread = hv_get_bytes_to_read(&chn->outbound);
182 183 184 185
			if (aread)
				break;
		}

186 187
		if (aread == 0)
			return 0;
188

189 190
		if (++retry > RETRY_MAX)
			return -ETIMEDOUT;
191

192
		usleep_range(RETRY_US_LO, RETRY_US_HI);
193
	}
194
}
195

196 197 198 199 200 201 202 203 204 205 206
static void netvsc_tx_disable(struct netvsc_device *nvscdev,
			      struct net_device *ndev)
{
	if (nvscdev) {
		nvscdev->tx_disable = true;
		virt_wmb(); /* ensure txq will not wake up after stop */
	}

	netif_tx_disable(ndev);
}

207 208 209 210 211 212 213 214
static int netvsc_close(struct net_device *net)
{
	struct net_device_context *net_device_ctx = netdev_priv(net);
	struct net_device *vf_netdev
		= rtnl_dereference(net_device_ctx->vf_netdev);
	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
	int ret;

215
	netvsc_tx_disable(nvdev, net);
216 217 218 219 220 221 222 223 224

	/* No need to close rndis filter if it is removed already */
	if (!nvdev)
		return 0;

	ret = rndis_filter_close(nvdev);
	if (ret != 0) {
		netdev_err(net, "unable to close device (ret %d).\n", ret);
		return ret;
225
	}
226

227 228 229 230
	ret = netvsc_wait_until_empty(nvdev);
	if (ret)
		netdev_err(net, "Ring buffer not empty after closing rndis\n");

231 232 233
	if (vf_netdev)
		dev_close(vf_netdev);

234 235 236
	return ret;
}

237 238
static inline void *init_ppi_data(struct rndis_message *msg,
				  u32 ppi_size, u32 pkt_type)
239
{
240
	struct rndis_packet *rndis_pkt = &msg->msg.pkt;
241 242 243
	struct rndis_per_packet_info *ppi;

	rndis_pkt->data_offset += ppi_size;
244 245
	ppi = (void *)rndis_pkt + rndis_pkt->per_pkt_info_offset
		+ rndis_pkt->per_pkt_info_len;
246 247 248 249 250 251 252

	ppi->size = ppi_size;
	ppi->type = pkt_type;
	ppi->ppi_offset = sizeof(struct rndis_per_packet_info);

	rndis_pkt->per_pkt_info_len += ppi_size;

253
	return ppi + 1;
254 255
}

256 257
/* Azure hosts don't support non-TCP port numbers in hashing for fragmented
 * packets. We can use ethtool to change UDP hash level when necessary.
258
 */
259 260 261
static inline u32 netvsc_get_hash(
	struct sk_buff *skb,
	const struct net_device_context *ndc)
262 263
{
	struct flow_keys flow;
264
	u32 hash, pkt_proto = 0;
265 266 267 268 269 270 271
	static u32 hashrnd __read_mostly;

	net_get_random_once(&hashrnd, sizeof(hashrnd));

	if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
		return 0;

272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
	switch (flow.basic.ip_proto) {
	case IPPROTO_TCP:
		if (flow.basic.n_proto == htons(ETH_P_IP))
			pkt_proto = HV_TCP4_L4HASH;
		else if (flow.basic.n_proto == htons(ETH_P_IPV6))
			pkt_proto = HV_TCP6_L4HASH;

		break;

	case IPPROTO_UDP:
		if (flow.basic.n_proto == htons(ETH_P_IP))
			pkt_proto = HV_UDP4_L4HASH;
		else if (flow.basic.n_proto == htons(ETH_P_IPV6))
			pkt_proto = HV_UDP6_L4HASH;

		break;
	}

	if (pkt_proto & ndc->l4_hash) {
291 292 293 294 295 296 297
		return skb_get_hash(skb);
	} else {
		if (flow.basic.n_proto == htons(ETH_P_IP))
			hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd);
		else if (flow.basic.n_proto == htons(ETH_P_IPV6))
			hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd);
		else
298
			return 0;
299

300
		__skb_set_sw_hash(skb, hash, false);
301 302 303 304 305
	}

	return hash;
}

306 307 308 309 310 311 312
static inline int netvsc_get_tx_queue(struct net_device *ndev,
				      struct sk_buff *skb, int old_idx)
{
	const struct net_device_context *ndc = netdev_priv(ndev);
	struct sock *sk = skb->sk;
	int q_idx;

313 314
	q_idx = ndc->tx_table[netvsc_get_hash(skb, ndc) &
			      (VRSS_SEND_TAB_SIZE - 1)];
315 316 317 318 319 320 321 322 323

	/* If queue index changed record the new value */
	if (q_idx != old_idx &&
	    sk && sk_fullsock(sk) && rcu_access_pointer(sk->sk_dst_cache))
		sk_tx_queue_set(sk, q_idx);

	return q_idx;
}

324 325 326 327 328 329 330 331 332 333 334
/*
 * Select queue for transmit.
 *
 * If a valid queue has already been assigned, then use that.
 * Otherwise compute tx queue based on hash and the send table.
 *
 * This is basically similar to default (__netdev_pick_tx) with the added step
 * of using the host send_table when no other queue has been assigned.
 *
 * TODO support XPS - but get_xps_queue not exported
 */
335
static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb)
336
{
337 338
	int q_idx = sk_tx_queue_get(skb->sk);

339
	if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) {
340 341 342 343 344 345 346
		/* If forwarding a packet, we use the recorded queue when
		 * available for better cache locality.
		 */
		if (skb_rx_queue_recorded(skb))
			q_idx = skb_get_rx_queue(skb);
		else
			q_idx = netvsc_get_tx_queue(ndev, skb, q_idx);
347
	}
348 349 350 351

	return q_idx;
}

352
static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
353
			       struct net_device *sb_dev,
354 355 356 357 358 359 360 361 362
			       select_queue_fallback_t fallback)
{
	struct net_device_context *ndc = netdev_priv(ndev);
	struct net_device *vf_netdev;
	u16 txq;

	rcu_read_lock();
	vf_netdev = rcu_dereference(ndc->vf_netdev);
	if (vf_netdev) {
363 364 365 366
		const struct net_device_ops *vf_ops = vf_netdev->netdev_ops;

		if (vf_ops->ndo_select_queue)
			txq = vf_ops->ndo_select_queue(vf_netdev, skb,
367
						       sb_dev, fallback);
368
		else
369
			txq = fallback(vf_netdev, skb, NULL);
370 371 372 373 374 375

		/* Record the queue selected by VF so that it can be
		 * used for common case where VF has more queues than
		 * the synthetic device.
		 */
		qdisc_skb_cb(skb)->slave_dev_queue_mapping = txq;
376 377 378 379 380 381 382 383 384 385 386
	} else {
		txq = netvsc_pick_tx(ndev, skb);
	}
	rcu_read_unlock();

	while (unlikely(txq >= ndev->real_num_tx_queues))
		txq -= ndev->real_num_tx_queues;

	return txq;
}

387
static u32 fill_pg_buf(struct page *page, u32 offset, u32 len,
388
		       struct hv_page_buffer *pb)
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
{
	int j = 0;

	/* Deal with compund pages by ignoring unused part
	 * of the page.
	 */
	page += (offset >> PAGE_SHIFT);
	offset &= ~PAGE_MASK;

	while (len > 0) {
		unsigned long bytes;

		bytes = PAGE_SIZE - offset;
		if (bytes > len)
			bytes = len;
		pb[j].pfn = page_to_pfn(page);
		pb[j].offset = offset;
		pb[j].len = bytes;

		offset += bytes;
		len -= bytes;

		if (offset == PAGE_SIZE && len) {
			page++;
			offset = 0;
			j++;
		}
	}

	return j + 1;
}

421
static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
422
			   struct hv_netvsc_packet *packet,
423
			   struct hv_page_buffer *pb)
424 425 426 427 428 429 430
{
	u32 slots_used = 0;
	char *data = skb->data;
	int frags = skb_shinfo(skb)->nr_frags;
	int i;

	/* The packet is laid out thus:
431
	 * 1. hdr: RNDIS header and PPI
432 433 434
	 * 2. skb linear data
	 * 3. skb fragment data
	 */
435 436 437
	slots_used += fill_pg_buf(virt_to_page(hdr),
				  offset_in_page(hdr),
				  len, &pb[slots_used]);
438

439 440 441
	packet->rmsg_size = len;
	packet->rmsg_pgcnt = slots_used;

442 443 444 445 446 447 448 449 450 451 452
	slots_used += fill_pg_buf(virt_to_page(data),
				offset_in_page(data),
				skb_headlen(skb), &pb[slots_used]);

	for (i = 0; i < frags; i++) {
		skb_frag_t *frag = skb_shinfo(skb)->frags + i;

		slots_used += fill_pg_buf(skb_frag_page(frag),
					frag->page_offset,
					skb_frag_size(frag), &pb[slots_used]);
	}
453
	return slots_used;
454 455
}

456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
static int count_skb_frag_slots(struct sk_buff *skb)
{
	int i, frags = skb_shinfo(skb)->nr_frags;
	int pages = 0;

	for (i = 0; i < frags; i++) {
		skb_frag_t *frag = skb_shinfo(skb)->frags + i;
		unsigned long size = skb_frag_size(frag);
		unsigned long offset = frag->page_offset;

		/* Skip unused frames from start of page */
		offset &= ~PAGE_MASK;
		pages += PFN_UP(offset + size);
	}
	return pages;
}

static int netvsc_get_slots(struct sk_buff *skb)
474
{
475 476 477 478 479 480 481 482 483
	char *data = skb->data;
	unsigned int offset = offset_in_page(data);
	unsigned int len = skb_headlen(skb);
	int slots;
	int frag_slots;

	slots = DIV_ROUND_UP(offset + len, PAGE_SIZE);
	frag_slots = count_skb_frag_slots(skb);
	return slots + frag_slots;
484 485
}

486
static u32 net_checksum_info(struct sk_buff *skb)
487
{
488 489
	if (skb->protocol == htons(ETH_P_IP)) {
		struct iphdr *ip = ip_hdr(skb);
490

491 492 493 494
		if (ip->protocol == IPPROTO_TCP)
			return TRANSPORT_INFO_IPV4_TCP;
		else if (ip->protocol == IPPROTO_UDP)
			return TRANSPORT_INFO_IPV4_UDP;
495
	} else {
496 497 498 499
		struct ipv6hdr *ip6 = ipv6_hdr(skb);

		if (ip6->nexthdr == IPPROTO_TCP)
			return TRANSPORT_INFO_IPV6_TCP;
500
		else if (ip6->nexthdr == IPPROTO_UDP)
501
			return TRANSPORT_INFO_IPV6_UDP;
502 503
	}

504
	return TRANSPORT_INFO_NOT_IP;
505 506
}

507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
/* Send skb on the slave VF device. */
static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev,
			  struct sk_buff *skb)
{
	struct net_device_context *ndev_ctx = netdev_priv(net);
	unsigned int len = skb->len;
	int rc;

	skb->dev = vf_netdev;
	skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;

	rc = dev_queue_xmit(skb);
	if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) {
		struct netvsc_vf_pcpu_stats *pcpu_stats
			= this_cpu_ptr(ndev_ctx->vf_stats);

		u64_stats_update_begin(&pcpu_stats->syncp);
		pcpu_stats->tx_packets++;
		pcpu_stats->tx_bytes += len;
		u64_stats_update_end(&pcpu_stats->syncp);
	} else {
		this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
	}

	return rc;
}

534
static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
535 536
{
	struct net_device_context *net_device_ctx = netdev_priv(net);
537
	struct hv_netvsc_packet *packet = NULL;
538
	int ret;
539 540
	unsigned int num_data_pgs;
	struct rndis_message *rndis_msg;
541
	struct net_device *vf_netdev;
542
	u32 rndis_msg_size;
543
	u32 hash;
544
	struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT];
545

546 547 548
	/* If VF is present and up then redirect packets to it.
	 * Skip the VF if it is marked down or has no carrier.
	 * If netpoll is in uses, then VF can not be used either.
549 550 551
	 */
	vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev);
	if (vf_netdev && netif_running(vf_netdev) &&
552
	    netif_carrier_ok(vf_netdev) && !netpoll_tx_running(net))
553 554
		return netvsc_vf_xmit(net, vf_netdev, skb);

555 556
	/* We will atmost need two pages to describe the rndis
	 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
557 558
	 * of pages in a single packet. If skb is scattered around
	 * more pages we try linearizing it.
559
	 */
560 561 562

	num_data_pgs = netvsc_get_slots(skb) + 2;

563
	if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
564 565 566 567
		++net_device_ctx->eth_stats.tx_scattered;

		if (skb_linearize(skb))
			goto no_memory;
568

569
		num_data_pgs = netvsc_get_slots(skb) + 2;
570
		if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
571
			++net_device_ctx->eth_stats.tx_too_big;
572 573
			goto drop;
		}
574
	}
575

576 577 578 579 580 581
	/*
	 * Place the rndis header in the skb head room and
	 * the skb->cb will be used for hv_netvsc_packet
	 * structure.
	 */
	ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
582 583 584
	if (ret)
		goto no_memory;

585 586 587 588
	/* Use the skb control buffer for building up the packet */
	BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
			FIELD_SIZEOF(struct sk_buff, cb));
	packet = (struct hv_netvsc_packet *)skb->cb;
589

590 591
	packet->q_idx = skb_get_queue_mapping(skb);

592
	packet->total_data_buflen = skb->len;
593 594
	packet->total_bytes = skb->len;
	packet->total_packets = 1;
595

596
	rndis_msg = (struct rndis_message *)skb->head;
597

598 599 600
	/* Add the rndis header */
	rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
	rndis_msg->msg_len = packet->total_data_buflen;
601 602 603 604 605 606

	rndis_msg->msg.pkt = (struct rndis_packet) {
		.data_offset = sizeof(struct rndis_packet),
		.data_len = packet->total_data_buflen,
		.per_pkt_info_offset = sizeof(struct rndis_packet),
	};
607 608 609

	rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);

610 611
	hash = skb_get_hash_raw(skb);
	if (hash != 0 && net->real_num_tx_queues > 1) {
612 613
		u32 *hash_info;

614
		rndis_msg_size += NDIS_HASH_PPI_SIZE;
615 616 617
		hash_info = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
					  NBL_HASH_VALUE);
		*hash_info = hash;
618 619
	}

620
	if (skb_vlan_tag_present(skb)) {
621 622 623
		struct ndis_pkt_8021q_info *vlan;

		rndis_msg_size += NDIS_VLAN_PPI_SIZE;
624 625
		vlan = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
				     IEEE_8021Q_INFO);
626

627
		vlan->value = 0;
628 629
		vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
		vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
630 631 632
				VLAN_PRIO_SHIFT;
	}

633
	if (skb_is_gso(skb)) {
634 635 636
		struct ndis_tcp_lso_info *lso_info;

		rndis_msg_size += NDIS_LSO_PPI_SIZE;
637 638
		lso_info = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
					 TCP_LARGESEND_PKTINFO);
639

640
		lso_info->value = 0;
641
		lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
642
		if (skb->protocol == htons(ETH_P_IP)) {
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
			lso_info->lso_v2_transmit.ip_version =
				NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
			ip_hdr(skb)->tot_len = 0;
			ip_hdr(skb)->check = 0;
			tcp_hdr(skb)->check =
				~csum_tcpudp_magic(ip_hdr(skb)->saddr,
						   ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
		} else {
			lso_info->lso_v2_transmit.ip_version =
				NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
			ipv6_hdr(skb)->payload_len = 0;
			tcp_hdr(skb)->check =
				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
						 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
		}
658
		lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb);
659
		lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
660
	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
661 662 663
		if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) {
			struct ndis_tcp_ip_checksum_info *csum_info;

664
			rndis_msg_size += NDIS_CSUM_PPI_SIZE;
665 666
			csum_info = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
						  TCPIP_CHKSUM_PKTINFO);
667

668
			csum_info->value = 0;
669 670 671
			csum_info->transmit.tcp_header_offset = skb_transport_offset(skb);

			if (skb->protocol == htons(ETH_P_IP)) {
672
				csum_info->transmit.is_ipv4 = 1;
673 674 675 676 677 678

				if (ip_hdr(skb)->protocol == IPPROTO_TCP)
					csum_info->transmit.tcp_checksum = 1;
				else
					csum_info->transmit.udp_checksum = 1;
			} else {
679 680
				csum_info->transmit.is_ipv6 = 1;

681 682 683 684 685
				if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
					csum_info->transmit.tcp_checksum = 1;
				else
					csum_info->transmit.udp_checksum = 1;
			}
686
		} else {
687
			/* Can't do offload of this type of checksum */
688 689 690
			if (skb_checksum_help(skb))
				goto drop;
		}
691 692
	}

693 694
	/* Start filling in the page buffers with the rndis hdr */
	rndis_msg->msg_len += rndis_msg_size;
695
	packet->total_data_buflen = rndis_msg->msg_len;
696
	packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
697
					       skb, packet, pb);
698

699 700
	/* timestamp packet in software */
	skb_tx_timestamp(skb);
701

702
	ret = netvsc_send(net, packet, rndis_msg, pb, skb);
703
	if (likely(ret == 0))
704
		return NETDEV_TX_OK;
705 706 707

	if (ret == -EAGAIN) {
		++net_device_ctx->eth_stats.tx_busy;
708
		return NETDEV_TX_BUSY;
709 710 711 712
	}

	if (ret == -ENOSPC)
		++net_device_ctx->eth_stats.tx_no_space;
713 714 715 716

drop:
	dev_kfree_skb_any(skb);
	net->stats.tx_dropped++;
717

718
	return NETDEV_TX_OK;
719 720 721 722

no_memory:
	++net_device_ctx->eth_stats.tx_no_memory;
	goto drop;
723
}
724

725
/*
726 727
 * netvsc_linkstatus_callback - Link up/down notification
 */
728
void netvsc_linkstatus_callback(struct net_device *net,
729
				struct rndis_message *resp)
730
{
731
	struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
732
	struct net_device_context *ndev_ctx = netdev_priv(net);
733 734
	struct netvsc_reconfig *event;
	unsigned long flags;
735

736 737 738 739
	/* Update the physical link speed when changing to another vSwitch */
	if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
		u32 speed;

740 741
		speed = *(u32 *)((void *)indicate
				 + indicate->status_buf_offset) / 10000;
742 743 744 745 746
		ndev_ctx->speed = speed;
		return;
	}

	/* Handle these link change statuses below */
747 748 749
	if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE &&
	    indicate->status != RNDIS_STATUS_MEDIA_CONNECT &&
	    indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT)
750
		return;
751

752
	if (net->reg_state != NETREG_REGISTERED)
753 754
		return;

755 756 757 758 759 760 761 762 763 764
	event = kzalloc(sizeof(*event), GFP_ATOMIC);
	if (!event)
		return;
	event->event = indicate->status;

	spin_lock_irqsave(&ndev_ctx->lock, flags);
	list_add_tail(&event->list, &ndev_ctx->reconfig_events);
	spin_unlock_irqrestore(&ndev_ctx->lock, flags);

	schedule_delayed_work(&ndev_ctx->dwork, 0);
765 766
}

767 768 769 770 771 772 773 774
static void netvsc_comp_ipcsum(struct sk_buff *skb)
{
	struct iphdr *iph = (struct iphdr *)skb->data;

	iph->check = 0;
	iph->check = ip_fast_csum(iph, iph->ihl);
}

775
static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
776
					     struct napi_struct *napi,
777 778 779
					     const struct ndis_tcp_ip_checksum_info *csum_info,
					     const struct ndis_pkt_8021q_info *vlan,
					     void *data, u32 buflen)
780 781 782
{
	struct sk_buff *skb;

783
	skb = napi_alloc_skb(napi, buflen);
784 785
	if (!skb)
		return skb;
786

787 788 789 790
	/*
	 * Copy to skb. This copy is needed here since the memory pointed by
	 * hv_netvsc_packet cannot be deallocated
	 */
791
	skb_put_data(skb, data, buflen);
792 793

	skb->protocol = eth_type_trans(skb, net);
794 795 796 797

	/* skb is already created with CHECKSUM_NONE */
	skb_checksum_none_assert(skb);

798 799 800 801 802 803 804 805 806 807
	/* Incoming packets may have IP header checksum verified by the host.
	 * They may not have IP header checksum computed after coalescing.
	 * We compute it here if the flags are set, because on Linux, the IP
	 * checksum is always checked.
	 */
	if (csum_info && csum_info->receive.ip_checksum_value_invalid &&
	    csum_info->receive.ip_checksum_succeeded &&
	    skb->protocol == htons(ETH_P_IP))
		netvsc_comp_ipcsum(skb);

808
	/* Do L4 checksum offload if enabled and present. */
809 810 811
	if (csum_info && (net->features & NETIF_F_RXCSUM)) {
		if (csum_info->receive.tcp_checksum_succeeded ||
		    csum_info->receive.udp_checksum_succeeded)
812 813 814
			skb->ip_summed = CHECKSUM_UNNECESSARY;
	}

815 816 817
	if (vlan) {
		u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT);

818
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
819
				       vlan_tci);
820
	}
821

822 823 824 825 826 827 828
	return skb;
}

/*
 * netvsc_recv_callback -  Callback when we receive a packet from the
 * "wire" on the specified device.
 */
829
int netvsc_recv_callback(struct net_device *net,
830
			 struct netvsc_device *net_device,
831 832 833 834
			 struct vmbus_channel *channel,
			 void  *data, u32 len,
			 const struct ndis_tcp_ip_checksum_info *csum_info,
			 const struct ndis_pkt_8021q_info *vlan)
835
{
836
	struct net_device_context *net_device_ctx = netdev_priv(net);
stephen hemminger's avatar
stephen hemminger committed
837
	u16 q_idx = channel->offermsg.offer.sub_channel_index;
838
	struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];
839 840 841
	struct sk_buff *skb;
	struct netvsc_stats *rx_stats;

842
	if (net->reg_state != NETREG_REGISTERED)
843 844 845
		return NVSP_STAT_FAIL;

	/* Allocate a skb - TODO direct I/O to pages? */
846 847
	skb = netvsc_alloc_recv_skb(net, &nvchan->napi,
				    csum_info, vlan, data, len);
848
	if (unlikely(!skb)) {
849
		++net_device_ctx->eth_stats.rx_no_memory;
850 851
		return NVSP_STAT_FAIL;
	}
852

853
	skb_record_rx_queue(skb, q_idx);
854 855 856 857 858 859

	/*
	 * Even if injecting the packet, record the statistics
	 * on the synthetic device because modifying the VF device
	 * statistics will not work correctly.
	 */
stephen hemminger's avatar
stephen hemminger committed
860
	rx_stats = &nvchan->rx_stats;
861
	u64_stats_update_begin(&rx_stats->syncp);
862
	rx_stats->packets++;
863
	rx_stats->bytes += len;
864 865 866 867 868

	if (skb->pkt_type == PACKET_BROADCAST)
		++rx_stats->broadcast;
	else if (skb->pkt_type == PACKET_MULTICAST)
		++rx_stats->multicast;
869
	u64_stats_update_end(&rx_stats->syncp);
870

stephen hemminger's avatar
stephen hemminger committed
871
	napi_gro_receive(&nvchan->napi, skb);
872
	return NVSP_STAT_SUCCESS;
873 874
}

875 876 877
static void netvsc_get_drvinfo(struct net_device *net,
			       struct ethtool_drvinfo *info)
{
878 879
	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
	strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
880 881
}

882 883 884 885
static void netvsc_get_channels(struct net_device *net,
				struct ethtool_channels *channel)
{
	struct net_device_context *net_device_ctx = netdev_priv(net);
886
	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
887 888 889 890 891 892 893

	if (nvdev) {
		channel->max_combined	= nvdev->max_chn;
		channel->combined_count = nvdev->num_chn;
	}
}

894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
/* Alloc struct netvsc_device_info, and initialize it from either existing
 * struct netvsc_device, or from default values.
 */
static struct netvsc_device_info *netvsc_devinfo_get
			(struct netvsc_device *nvdev)
{
	struct netvsc_device_info *dev_info;

	dev_info = kzalloc(sizeof(*dev_info), GFP_ATOMIC);

	if (!dev_info)
		return NULL;

	if (nvdev) {
		dev_info->num_chn = nvdev->num_chn;
		dev_info->send_sections = nvdev->send_section_cnt;
		dev_info->send_section_size = nvdev->send_section_size;
		dev_info->recv_sections = nvdev->recv_section_cnt;
		dev_info->recv_section_size = nvdev->recv_section_size;
913 914 915

		memcpy(dev_info->rss_key, nvdev->extension->rss_key,
		       NETVSC_HASH_KEYLEN);
916 917 918 919 920 921 922 923 924 925 926
	} else {
		dev_info->num_chn = VRSS_CHANNEL_DEFAULT;
		dev_info->send_sections = NETVSC_DEFAULT_TX;
		dev_info->send_section_size = NETVSC_SEND_SECTION_SIZE;
		dev_info->recv_sections = NETVSC_DEFAULT_RX;
		dev_info->recv_section_size = NETVSC_RECV_SECTION_SIZE;
	}

	return dev_info;
}

927 928 929 930 931 932 933 934 935 936 937 938 939
static int netvsc_detach(struct net_device *ndev,
			 struct netvsc_device *nvdev)
{
	struct net_device_context *ndev_ctx = netdev_priv(ndev);
	struct hv_device *hdev = ndev_ctx->device_ctx;
	int ret;

	/* Don't try continuing to try and setup sub channels */
	if (cancel_work_sync(&nvdev->subchan_work))
		nvdev->num_chn = 1;

	/* If device was up (receiving) then shutdown */
	if (netif_running(ndev)) {
940
		netvsc_tx_disable(nvdev, ndev);
941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976

		ret = rndis_filter_close(nvdev);
		if (ret) {
			netdev_err(ndev,
				   "unable to close device (ret %d).\n", ret);
			return ret;
		}

		ret = netvsc_wait_until_empty(nvdev);
		if (ret) {
			netdev_err(ndev,
				   "Ring buffer not empty after closing rndis\n");
			return ret;
		}
	}

	netif_device_detach(ndev);

	rndis_filter_device_remove(hdev, nvdev);

	return 0;
}

static int netvsc_attach(struct net_device *ndev,
			 struct netvsc_device_info *dev_info)
{
	struct net_device_context *ndev_ctx = netdev_priv(ndev);
	struct hv_device *hdev = ndev_ctx->device_ctx;
	struct netvsc_device *nvdev;
	struct rndis_device *rdev;
	int ret;

	nvdev = rndis_filter_device_add(hdev, dev_info);
	if (IS_ERR(nvdev))
		return PTR_ERR(nvdev);

977
	if (nvdev->num_chn > 1) {
978
		ret = rndis_set_subchannel(ndev, nvdev, dev_info);
979 980 981 982 983 984 985 986 987

		/* if unavailable, just proceed with one queue */
		if (ret) {
			nvdev->max_chn = 1;
			nvdev->num_chn = 1;
		}
	}

	/* In any case device is now ready */
988
	nvdev->tx_disable = false;
989
	netif_device_attach(ndev);
990

991
	/* Note: enable and attach happen when sub-channels setup */
992 993 994 995 996
	netif_carrier_off(ndev);

	if (netif_running(ndev)) {
		ret = rndis_filter_open(nvdev);
		if (ret)
997
			goto err;
998 999 1000 1001 1002 1003 1004

		rdev = nvdev->extension;
		if (!rdev->link_state)
			netif_carrier_on(ndev);
	}

	return 0;
1005 1006 1007 1008 1009 1010 1011

err:
	netif_device_detach(ndev);

	rndis_filter_device_remove(hdev, nvdev);

	return ret;
1012 1013
}

1014 1015 1016 1017
static int netvsc_set_channels(struct net_device *net,
			       struct ethtool_channels *channels)
{
	struct net_device_context *net_device_ctx = netdev_priv(net);
1018
	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
1019
	unsigned int orig, count = channels->combined_count;
1020
	struct netvsc_device_info *device_info;
1021
	int ret;
1022 1023 1024 1025 1026 1027

	/* We do not support separate count for rx, tx, or other */
	if (count == 0 ||
	    channels->rx_count || channels->tx_count || channels->other_count)
		return -EINVAL;

1028
	if (!nvdev || nvdev->destroy)
1029 1030
		return -ENODEV;

1031
	if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5)
1032 1033
		return -EINVAL;

1034
	if (count > nvdev->max_chn)
1035 1036
		return -EINVAL;

1037
	orig = nvdev->num_chn;
1038

1039 1040 1041 1042 1043 1044
	device_info = netvsc_devinfo_get(nvdev);

	if (!device_info)
		return -ENOMEM;

	device_info->num_chn = count;
1045

1046 1047
	ret = netvsc_detach(net, nvdev);
	if (ret)
1048
		goto out;
1049

1050
	ret = netvsc_attach(net, device_info);
1051
	if (ret) {
1052 1053
		device_info->num_chn = orig;
		if (netvsc_attach(net, device_info))
1054
			netdev_err(net, "restoring channel setting failed\n");
1055
	}
1056

1057 1058
out:
	kfree(device_info);
1059 1060 1061
	return ret;
}

1062 1063
static bool
netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
1064
{
1065 1066
	struct ethtool_link_ksettings diff1 = *cmd;
	struct ethtool_link_ksettings diff2 = {};
1067

1068 1069
	diff1.base.speed = 0;
	diff1.base.duplex = 0;
1070
	/* advertising and cmd are usually set */
1071 1072
	ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
	diff1.base.cmd = 0;
1073
	/* We set port to PORT_OTHER */
1074
	diff2.base.port = PORT_OTHER;
1075 1076 1077 1078 1079 1080 1081 1082

	return !memcmp(&diff1, &diff2, sizeof(diff1));
}

static void netvsc_init_settings(struct net_device *dev)
{
	struct net_device_context *ndc = netdev_priv(dev);

1083
	ndc->l4_hash = HV_DEFAULT_L4HASH;
1084

1085
	ndc->speed = SPEED_UNKNOWN;
1086
	ndc->duplex = DUPLEX_FULL;
1087 1088
}

1089 1090
static int netvsc_get_link_ksettings(struct net_device *dev,
				     struct ethtool_link_ksettings *cmd)
1091 1092 1093
{
	struct net_device_context *ndc = netdev_priv(dev);

1094 1095 1096
	cmd->base.speed = ndc->speed;
	cmd->base.duplex = ndc->duplex;
	cmd->base.port = PORT_OTHER;
1097 1098 1099 1100

	return 0;
}

1101 1102
static int netvsc_set_link_ksettings(struct net_device *dev,
				     const struct ethtool_link_ksettings *cmd)
1103 1104 1105 1106
{
	struct net_device_context *ndc = netdev_priv(dev);
	u32 speed;

1107
	speed = cmd->base.speed;
1108
	if (!ethtool_validate_speed(speed) ||
1109
	    !ethtool_validate_duplex(cmd->base.duplex) ||
1110 1111 1112 1113
	    !netvsc_validate_ethtool_ss_cmd(cmd))
		return -EINVAL;

	ndc->speed = speed;
1114
	ndc->duplex = cmd->base.duplex;
1115 1116 1117 1118

	return 0;
}

1119 1120 1121
static int netvsc_change_mtu(struct net_device *ndev, int mtu)
{
	struct net_device_context *ndevctx = netdev_priv(ndev);
1122
	struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
1123
	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1124
	int orig_mtu = ndev->mtu;
1125
	struct netvsc_device_info *device_info;
1126
	int ret = 0;
1127

1128
	if (!nvdev || nvdev->destroy)
1129 1130
		return -ENODEV;

1131 1132 1133 1134 1135
	device_info = netvsc_devinfo_get(nvdev);

	if (!device_info)
		return -ENOMEM;

1136 1137 1138 1139
	/* Change MTU of underlying VF netdev first. */
	if (vf_netdev) {
		ret = dev_set_mtu(vf_netdev, mtu);
		if (ret)
1140
			goto out;
1141 1142
	}

1143 1144 1145
	ret = netvsc_detach(ndev, nvdev);
	if (ret)
		goto rollback_vf;
1146 1147 1148

	ndev->mtu = mtu;

1149 1150 1151
	ret = netvsc_attach(ndev, device_info);
	if (!ret)
		goto out;
1152

1153 1154
	/* Attempt rollback to original MTU */
	ndev->mtu = orig_mtu;
1155

1156
	if (netvsc_attach(ndev, device_info))
1157 1158 1159 1160
		netdev_err(ndev, "restoring mtu failed\n");
rollback_vf:
	if (vf_netdev)
		dev_set_mtu(vf_netdev, orig_mtu);
1161

1162 1163
out:
	kfree(device_info);
1164
	return ret;
1165 1166
}

1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
static void netvsc_get_vf_stats(struct net_device *net,
				struct netvsc_vf_pcpu_stats *tot)
{
	struct net_device_context *ndev_ctx = netdev_priv(net);
	int i;

	memset(tot, 0, sizeof(*tot));

	for_each_possible_cpu(i) {
		const struct netvsc_vf_pcpu_stats *stats
			= per_cpu_ptr(ndev_ctx->vf_stats, i);
		u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
		unsigned int start;

		do {
			start = u64_stats_fetch_begin_irq(&stats->syncp);
			rx_packets = stats->rx_packets;
			tx_packets = stats->tx_packets;
			rx_bytes = stats->rx_bytes;
			tx_bytes = stats->tx_bytes;
		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));

		tot->rx_packets += rx_packets;
		tot->tx_packets += tx_packets;
		tot->rx_bytes   += rx_bytes;
		tot->tx_bytes   += tx_bytes;
		tot->tx_dropped += stats->tx_dropped;
	}
}

1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254
static void netvsc_get_pcpu_stats(struct net_device *net,
				  struct netvsc_ethtool_pcpu_stats *pcpu_tot)
{
	struct net_device_context *ndev_ctx = netdev_priv(net);
	struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev);
	int i;

	/* fetch percpu stats of vf */
	for_each_possible_cpu(i) {
		const struct netvsc_vf_pcpu_stats *stats =
			per_cpu_ptr(ndev_ctx->vf_stats, i);
		struct netvsc_ethtool_pcpu_stats *this_tot = &pcpu_tot[i];
		unsigned int start;

		do {
			start = u64_stats_fetch_begin_irq(&stats->syncp);
			this_tot->vf_rx_packets = stats->rx_packets;
			this_tot->vf_tx_packets = stats->tx_packets;
			this_tot->vf_rx_bytes = stats->rx_bytes;
			this_tot->vf_tx_bytes = stats->tx_bytes;
		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
		this_tot->rx_packets = this_tot->vf_rx_packets;
		this_tot->tx_packets = this_tot->vf_tx_packets;
		this_tot->rx_bytes   = this_tot->vf_rx_bytes;
		this_tot->tx_bytes   = this_tot->vf_tx_bytes;
	}

	/* fetch percpu stats of netvsc */
	for (i = 0; i < nvdev->num_chn; i++) {
		const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
		const struct netvsc_stats *stats;
		struct netvsc_ethtool_pcpu_stats *this_tot =
			&pcpu_tot[nvchan->channel->target_cpu];
		u64 packets, bytes;
		unsigned int start;

		stats = &nvchan->tx_stats;
		do {
			start = u64_stats_fetch_begin_irq(&stats->syncp);
			packets = stats->packets;
			bytes = stats->bytes;
		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));

		this_tot->tx_bytes	+= bytes;
		this_tot->tx_packets	+= packets;

		stats = &nvchan->rx_stats;
		do {
			start = u64_stats_fetch_begin_irq(&stats->syncp);
			packets = stats->packets;
			bytes = stats->bytes;
		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));

		this_tot->rx_bytes	+= bytes;
		this_tot->rx_packets	+= packets;
	}
}

1255 1256
static void netvsc_get_stats64(struct net_device *net,
			       struct rtnl_link_stats64 *t)
1257 1258
{
	struct net_device_context *ndev_ctx = netdev_priv(net);
1259
	struct netvsc_device *nvdev;
1260
	struct netvsc_vf_pcpu_stats vf_tot;
1261
	int i;
1262

1263 1264 1265
	rcu_read_lock();

	nvdev = rcu_dereference(ndev_ctx->nvdev);
1266
	if (!nvdev)
1267
		goto out;
1268

1269 1270 1271 1272 1273 1274 1275 1276 1277
	netdev_stats_to_stats64(t, &net->stats);

	netvsc_get_vf_stats(net, &vf_tot);
	t->rx_packets += vf_tot.rx_packets;
	t->tx_packets += vf_tot.tx_packets;
	t->rx_bytes   += vf_tot.rx_bytes;
	t->tx_bytes   += vf_tot.tx_bytes;
	t->tx_dropped += vf_tot.tx_dropped;

1278 1279 1280 1281
	for (i = 0; i < nvdev->num_chn; i++) {
		const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
		const struct netvsc_stats *stats;
		u64 packets, bytes, multicast;
1282 1283
		unsigned int start;

1284
		stats = &nvchan->tx_stats;
1285
		do {
1286 1287 1288 1289 1290 1291 1292
			start = u64_stats_fetch_begin_irq(&stats->syncp);
			packets = stats->packets;
			bytes = stats->bytes;
		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));

		t->tx_bytes	+= bytes;
		t->tx_packets	+= packets;
1293

1294
		stats = &nvchan->rx_stats;
1295
		do {
1296 1297 1298 1299 1300 1301 1302 1303 1304
			start = u64_stats_fetch_begin_irq(&stats->syncp);
			packets = stats->packets;
			bytes = stats->bytes;
			multicast = stats->multicast + stats->broadcast;
		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));

		t->rx_bytes	+= bytes;
		t->rx_packets	+= packets;
		t->multicast	+= multicast;
1305
	}
1306 1307
out:
	rcu_read_unlock();
1308
}
1309 1310 1311

static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
{
1312
	struct net_device_context *ndc = netdev_priv(ndev);
1313
	struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev);
1314
	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1315 1316 1317
	struct sockaddr *addr = p;
	int err;

1318 1319
	err = eth_prepare_mac_addr_change(ndev, p);
	if (err)
1320 1321
		return err;

1322 1323 1324
	if (!nvdev)
		return -ENODEV;

1325 1326 1327 1328 1329 1330
	if (vf_netdev) {
		err = dev_set_mac_address(vf_netdev, addr);
		if (err)
			return err;
	}

1331
	err = rndis_filter_set_device_mac(nvdev, addr->sa_data);
1332 1333 1334 1335 1336 1337
	if (!err) {
		eth_commit_mac_addr_change(ndev, p);
	} else if (vf_netdev) {
		/* rollback change on VF */
		memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN);
		dev_set_mac_address(vf_netdev, addr);
1338 1339 1340 1341 1342
	}

	return err;
}

1343 1344 1345 1346 1347
static const struct {
	char name[ETH_GSTRING_LEN];
	u16 offset;
} netvsc_stats[] = {
	{ "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) },
1348
	{ "tx_no_memory", offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
1349 1350 1351
	{ "tx_no_space",  offsetof(struct netvsc_ethtool_stats, tx_no_space) },
	{ "tx_too_big",	  offsetof(struct netvsc_ethtool_stats, tx_too_big) },
	{ "tx_busy",	  offsetof(struct netvsc_ethtool_stats, tx_busy) },
1352 1353
	{ "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) },
	{ "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) },
1354
	{ "rx_no_memory", offsetof(struct netvsc_ethtool_stats, rx_no_memory) },
1355 1356
	{ "stop_queue", offsetof(struct netvsc_ethtool_stats, stop_queue) },
	{ "wake_queue", offsetof(struct netvsc_ethtool_stats, wake_queue) },
1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
}, pcpu_stats[] = {
	{ "cpu%u_rx_packets",
		offsetof(struct netvsc_ethtool_pcpu_stats, rx_packets) },
	{ "cpu%u_rx_bytes",
		offsetof(struct netvsc_ethtool_pcpu_stats, rx_bytes) },
	{ "cpu%u_tx_packets",
		offsetof(struct netvsc_ethtool_pcpu_stats, tx_packets) },
	{ "cpu%u_tx_bytes",
		offsetof(struct netvsc_ethtool_pcpu_stats, tx_bytes) },
	{ "cpu%u_vf_rx_packets",
		offsetof(struct netvsc_ethtool_pcpu_stats, vf_rx_packets) },
	{ "cpu%u_vf_rx_bytes",
		offsetof(struct netvsc_ethtool_pcpu_stats, vf_rx_bytes) },
	{ "cpu%u_vf_tx_packets",
		offsetof(struct netvsc_ethtool_pcpu_stats, vf_tx_packets) },
	{ "cpu%u_vf_tx_bytes",
		offsetof(struct netvsc_ethtool_pcpu_stats, vf_tx_bytes) },
1374 1375 1376 1377 1378 1379
}, vf_stats[] = {
	{ "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) },
	{ "vf_rx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) },
	{ "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) },
	{ "vf_tx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) },
	{ "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) },
1380 1381
};

1382
#define NETVSC_GLOBAL_STATS_LEN	ARRAY_SIZE(netvsc_stats)
1383
#define NETVSC_VF_STATS_LEN	ARRAY_SIZE(vf_stats)
1384

1385 1386 1387
/* statistics per queue (rx/tx packets/bytes) */
#define NETVSC_PCPU_STATS_LEN (num_present_cpus() * ARRAY_SIZE(pcpu_stats))

1388 1389 1390
/* 4 statistics per queue (rx/tx packets/bytes) */
#define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)

1391 1392
static int netvsc_get_sset_count(struct net_device *dev, int string_set)
{
1393
	struct net_device_context *ndc = netdev_priv(dev);
1394
	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1395 1396 1397

	if (!nvdev)
		return -ENODEV;
1398

1399 1400
	switch (string_set) {
	case ETH_SS_STATS:
1401 1402
		return NETVSC_GLOBAL_STATS_LEN
			+ NETVSC_VF_STATS_LEN
1403 1404
			+ NETVSC_QUEUE_STATS_LEN(nvdev)
			+ NETVSC_PCPU_STATS_LEN;
1405 1406 1407 1408 1409 1410 1411 1412 1413
	default:
		return -EINVAL;
	}
}

static void netvsc_get_ethtool_stats(struct net_device *dev,
				     struct ethtool_stats *stats, u64 *data)
{
	struct net_device_context *ndc = netdev_priv(dev);
1414
	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1415
	const void *nds = &ndc->eth_stats;
1416
	const struct netvsc_stats *qstats;
1417
	struct netvsc_vf_pcpu_stats sum;
1418
	struct netvsc_ethtool_pcpu_stats *pcpu_sum;
1419 1420
	unsigned int start;
	u64 packets, bytes;
1421
	int i, j, cpu;
1422

1423 1424 1425
	if (!nvdev)
		return;

1426
	for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
1427
		data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
1428

1429 1430 1431 1432
	netvsc_get_vf_stats(dev, &sum);
	for (j = 0; j < NETVSC_VF_STATS_LEN; j++)
		data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset);

1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
	for (j = 0; j < nvdev->num_chn; j++) {
		qstats = &nvdev->chan_table[j].tx_stats;

		do {
			start = u64_stats_fetch_begin_irq(&qstats->syncp);
			packets = qstats->packets;
			bytes = qstats->bytes;
		} while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
		data[i++] = packets;
		data[i++] = bytes;

		qstats = &nvdev->chan_table[j].rx_stats;
		do {
			start = u64_stats_fetch_begin_irq(&qstats->syncp);
			packets = qstats->packets;
			bytes = qstats->bytes;
		} while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
		data[i++] = packets;
		data[i++] = bytes;
	}
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465

	pcpu_sum = kvmalloc_array(num_possible_cpus(),
				  sizeof(struct netvsc_ethtool_pcpu_stats),
				  GFP_KERNEL);
	netvsc_get_pcpu_stats(dev, pcpu_sum);
	for_each_present_cpu(cpu) {
		struct netvsc_ethtool_pcpu_stats *this_sum = &pcpu_sum[cpu];

		for (j = 0; j < ARRAY_SIZE(pcpu_stats); j++)
			data[i++] = *(u64 *)((void *)this_sum
					     + pcpu_stats[j].offset);
	}
	kvfree(pcpu_sum);
1466 1467 1468 1469
}

static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
{
1470
	struct net_device_context *ndc = netdev_priv(dev);
1471
	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1472
	u8 *p = data;
1473
	int i, cpu;
1474

1475 1476 1477
	if (!nvdev)
		return;

1478 1479
	switch (stringset) {
	case ETH_SS_STATS:
1480 1481 1482 1483 1484 1485 1486 1487 1488
		for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) {
			memcpy(p, netvsc_stats[i].name, ETH_GSTRING_LEN);
			p += ETH_GSTRING_LEN;
		}

		for (i = 0; i < ARRAY_SIZE(vf_stats); i++) {
			memcpy(p, vf_stats[i].name, ETH_GSTRING_LEN);
			p += ETH_GSTRING_LEN;
		}
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500

		for (i = 0; i < nvdev->num_chn; i++) {
			sprintf(p, "tx_queue_%u_packets", i);
			p += ETH_GSTRING_LEN;
			sprintf(p, "tx_queue_%u_bytes", i);
			p += ETH_GSTRING_LEN;
			sprintf(p, "rx_queue_%u_packets", i);
			p += ETH_GSTRING_LEN;
			sprintf(p, "rx_queue_%u_bytes", i);
			p += ETH_GSTRING_LEN;
		}

1501 1502 1503 1504 1505 1506 1507
		for_each_present_cpu(cpu) {
			for (i = 0; i < ARRAY_SIZE(pcpu_stats); i++) {
				sprintf(p, pcpu_stats[i].name, cpu);
				p += ETH_GSTRING_LEN;
			}
		}

1508 1509 1510 1511
		break;
	}
}

1512
static int
1513 1514
netvsc_get_rss_hash_opts(struct net_device_context *ndc,
			 struct ethtool_rxnfc *info)
1515
{
1516 1517
	const u32 l4_flag = RXH_L4_B_0_1 | RXH_L4_B_2_3;

1518 1519 1520 1521
	info->data = RXH_IP_SRC | RXH_IP_DST;

	switch (info->flow_type) {
	case TCP_V4_FLOW:
1522 1523 1524 1525 1526
		if (ndc->l4_hash & HV_TCP4_L4HASH)
			info->data |= l4_flag;

		break;

1527
	case TCP_V6_FLOW:
1528 1529 1530
		if (ndc->l4_hash & HV_TCP6_L4HASH)
			info->data |= l4_flag;

1531 1532
		break;

1533
	case UDP_V4_FLOW:
1534 1535
		if (ndc->l4_hash & HV_UDP4_L4HASH)
			info->data |= l4_flag;
1536 1537 1538

		break;

1539
	case UDP_V6_FLOW:
1540 1541
		if (ndc->l4_hash & HV_UDP6_L4HASH)
			info->data |= l4_flag;
1542 1543 1544

		break;

1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
	case IPV4_FLOW:
	case IPV6_FLOW:
		break;
	default:
		info->data = 0;
		break;
	}

	return 0;
}

1556 1557 1558 1559 1560
static int
netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
		 u32 *rules)
{
	struct net_device_context *ndc = netdev_priv(dev);
1561
	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1562 1563 1564

	if (!nvdev)
		return -ENODEV;
1565 1566 1567 1568 1569

	switch (info->cmd) {
	case ETHTOOL_GRXRINGS:
		info->data = nvdev->num_chn;
		return 0;
1570 1571

	case ETHTOOL_GRXFH:
1572
		return netvsc_get_rss_hash_opts(ndc, info);
1573 1574 1575 1576
	}
	return -EOPNOTSUPP;
}

1577 1578 1579 1580 1581
static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
				    struct ethtool_rxnfc *info)
{
	if (info->data == (RXH_IP_SRC | RXH_IP_DST |
			   RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
1582
		switch (info->flow_type) {
1583 1584 1585 1586 1587 1588 1589 1590
		case TCP_V4_FLOW:
			ndc->l4_hash |= HV_TCP4_L4HASH;
			break;

		case TCP_V6_FLOW:
			ndc->l4_hash |= HV_TCP6_L4HASH;
			break;

1591 1592 1593 1594 1595 1596 1597 1598 1599
		case UDP_V4_FLOW:
			ndc->l4_hash |= HV_UDP4_L4HASH;
			break;

		case UDP_V6_FLOW:
			ndc->l4_hash |= HV_UDP6_L4HASH;
			break;

		default:
1600
			return -EOPNOTSUPP;
1601
		}
1602 1603 1604 1605 1606

		return 0;
	}

	if (info->data == (RXH_IP_SRC | RXH_IP_DST)) {
1607
		switch (info->flow_type) {
1608 1609 1610 1611 1612 1613 1614 1615
		case TCP_V4_FLOW:
			ndc->l4_hash &= ~HV_TCP4_L4HASH;
			break;

		case TCP_V6_FLOW:
			ndc->l4_hash &= ~HV_TCP6_L4HASH;
			break;

1616 1617 1618 1619 1620 1621 1622 1623 1624
		case UDP_V4_FLOW:
			ndc->l4_hash &= ~HV_UDP4_L4HASH;
			break;

		case UDP_V6_FLOW:
			ndc->l4_hash &= ~HV_UDP6_L4HASH;
			break;

		default:
1625
			return -EOPNOTSUPP;
1626
		}
1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644

		return 0;
	}

	return -EOPNOTSUPP;
}

static int
netvsc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info)
{
	struct net_device_context *ndc = netdev_priv(ndev);

	if (info->cmd == ETHTOOL_SRXFH)
		return netvsc_set_rss_hash_opts(ndc, info);

	return -EOPNOTSUPP;
}

1645
#ifdef CONFIG_NET_POLL_CONTROLLER
1646
static void netvsc_poll_controller(struct net_device *dev)
1647
{
1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
	struct net_device_context *ndc = netdev_priv(dev);
	struct netvsc_device *ndev;
	int i;

	rcu_read_lock();
	ndev = rcu_dereference(ndc->nvdev);
	if (ndev) {
		for (i = 0; i < ndev->num_chn; i++) {
			struct netvsc_channel *nvchan = &ndev->chan_table[i];

			napi_schedule(&nvchan->napi);
		}
	}
	rcu_read_unlock();
1662 1663
}
#endif
1664

1665 1666 1667 1668 1669 1670 1671
static u32 netvsc_get_rxfh_key_size(struct net_device *dev)
{
	return NETVSC_HASH_KEYLEN;
}

static u32 netvsc_rss_indir_size(struct net_device *dev)
{
1672
	return ITAB_NUM;
1673 1674 1675 1676 1677 1678
}

static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
			   u8 *hfunc)
{
	struct net_device_context *ndc = netdev_priv(dev);
1679
	struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1680
	struct rndis_device *rndis_dev;
1681
	int i;
1682

1683 1684 1685
	if (!ndev)
		return -ENODEV;

1686 1687 1688
	if (hfunc)
		*hfunc = ETH_RSS_HASH_TOP;	/* Toeplitz */

1689
	rndis_dev = ndev->extension;
1690 1691
	if (indir) {
		for (i = 0; i < ITAB_NUM; i++)
1692
			indir[i] = ndc->rx_table[i];
1693 1694
	}

1695 1696 1697 1698 1699 1700 1701 1702 1703 1704
	if (key)
		memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN);

	return 0;
}

static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir,
			   const u8 *key, const u8 hfunc)
{
	struct net_device_context *ndc = netdev_priv(dev);
1705
	struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1706
	struct rndis_device *rndis_dev;
1707
	int i;
1708

1709 1710 1711
	if (!ndev)
		return -ENODEV;

1712 1713 1714
	if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
		return -EOPNOTSUPP;

1715
	rndis_dev = ndev->extension;
1716 1717
	if (indir) {
		for (i = 0; i < ITAB_NUM; i++)
1718
			if (indir[i] >= ndev->num_chn)
1719 1720 1721
				return -EINVAL;

		for (i = 0; i < ITAB_NUM; i++)
1722
			ndc->rx_table[i] = indir[i];
1723 1724 1725 1726 1727 1728 1729 1730
	}

	if (!key) {
		if (!indir)
			return 0;

		key = rndis_dev->rss_key;
	}
1731

1732
	return rndis_filter_set_rss_param(rndis_dev, key);
1733 1734
}

1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772
/* Hyper-V RNDIS protocol does not have ring in the HW sense.
 * It does have pre-allocated receive area which is divided into sections.
 */
static void __netvsc_get_ringparam(struct netvsc_device *nvdev,
				   struct ethtool_ringparam *ring)
{
	u32 max_buf_size;

	ring->rx_pending = nvdev->recv_section_cnt;
	ring->tx_pending = nvdev->send_section_cnt;

	if (nvdev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
		max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
	else
		max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;

	ring->rx_max_pending = max_buf_size / nvdev->recv_section_size;
	ring->tx_max_pending = NETVSC_SEND_BUFFER_SIZE
		/ nvdev->send_section_size;
}

static void netvsc_get_ringparam(struct net_device *ndev,
				 struct ethtool_ringparam *ring)
{
	struct net_device_context *ndevctx = netdev_priv(ndev);
	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);

	if (!nvdev)
		return;

	__netvsc_get_ringparam(nvdev, ring);
}

static int netvsc_set_ringparam(struct net_device *ndev,
				struct ethtool_ringparam *ring)
{
	struct net_device_context *ndevctx = netdev_priv(ndev);
	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1773
	struct netvsc_device_info *device_info;
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
	struct ethtool_ringparam orig;
	u32 new_tx, new_rx;
	int ret = 0;

	if (!nvdev || nvdev->destroy)
		return -ENODEV;

	memset(&orig, 0, sizeof(orig));
	__netvsc_get_ringparam(nvdev, &orig);

	new_tx = clamp_t(u32, ring->tx_pending,
			 NETVSC_MIN_TX_SECTIONS, orig.tx_max_pending);
	new_rx = clamp_t(u32, ring->rx_pending,
			 NETVSC_MIN_RX_SECTIONS, orig.rx_max_pending);

	if (new_tx == orig.tx_pending &&
	    new_rx == orig.rx_pending)
		return 0;	 /* no change */

1793 1794 1795 1796 1797 1798 1799
	device_info = netvsc_devinfo_get(nvdev);

	if (!device_info)
		return -ENOMEM;

	device_info->send_sections = new_tx;
	device_info->recv_sections = new_rx;
1800

1801 1802
	ret = netvsc_detach(ndev, nvdev);
	if (ret)
1803
		goto out;
1804

1805
	ret = netvsc_attach(ndev, device_info);
1806
	if (ret) {
1807 1808
		device_info->send_sections = orig.tx_pending;
		device_info->recv_sections = orig.rx_pending;
1809

1810
		if (netvsc_attach(ndev, device_info))
1811 1812
			netdev_err(ndev, "restoring ringparam failed");
	}
1813

1814 1815
out:
	kfree(device_info);
1816 1817 1818
	return ret;
}

1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
static u32 netvsc_get_msglevel(struct net_device *ndev)
{
	struct net_device_context *ndev_ctx = netdev_priv(ndev);

	return ndev_ctx->msg_enable;
}

static void netvsc_set_msglevel(struct net_device *ndev, u32 val)
{
	struct net_device_context *ndev_ctx = netdev_priv(ndev);

	ndev_ctx->msg_enable = val;
}

1833 1834
static const struct ethtool_ops ethtool_ops = {
	.get_drvinfo	= netvsc_get_drvinfo,
1835 1836
	.get_msglevel	= netvsc_get_msglevel,
	.set_msglevel	= netvsc_set_msglevel,
1837
	.get_link	= ethtool_op_get_link,
1838 1839 1840
	.get_ethtool_stats = netvsc_get_ethtool_stats,
	.get_sset_count = netvsc_get_sset_count,
	.get_strings	= netvsc_get_strings,
1841
	.get_channels   = netvsc_get_channels,
1842
	.set_channels   = netvsc_set_channels,
1843
	.get_ts_info	= ethtool_op_get_ts_info,
1844
	.get_rxnfc	= netvsc_get_rxnfc,
1845
	.set_rxnfc	= netvsc_set_rxnfc,
1846 1847 1848 1849
	.get_rxfh_key_size = netvsc_get_rxfh_key_size,
	.get_rxfh_indir_size = netvsc_rss_indir_size,
	.get_rxfh	= netvsc_get_rxfh,
	.set_rxfh	= netvsc_set_rxfh,
1850 1851
	.get_link_ksettings = netvsc_get_link_ksettings,
	.set_link_ksettings = netvsc_set_link_ksettings,
1852 1853
	.get_ringparam	= netvsc_get_ringparam,
	.set_ringparam	= netvsc_set_ringparam,
1854 1855
};

1856 1857 1858 1859
static const struct net_device_ops device_ops = {
	.ndo_open =			netvsc_open,
	.ndo_stop =			netvsc_close,
	.ndo_start_xmit =		netvsc_start_xmit,
1860 1861
	.ndo_change_rx_flags =		netvsc_change_rx_flags,
	.ndo_set_rx_mode =		netvsc_set_rx_mode,
1862
	.ndo_change_mtu =		netvsc_change_mtu,
1863
	.ndo_validate_addr =		eth_validate_addr,
1864
	.ndo_set_mac_address =		netvsc_set_mac_addr,
1865
	.ndo_select_queue =		netvsc_select_queue,
1866
	.ndo_get_stats64 =		netvsc_get_stats64,
1867 1868 1869
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller =		netvsc_poll_controller,
#endif
1870 1871
};

1872
/*
1873 1874 1875
 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
 * present send GARP packet to network peers with netif_notify_peers().
1876
 */
1877
static void netvsc_link_change(struct work_struct *w)
1878
{
1879 1880 1881 1882
	struct net_device_context *ndev_ctx =
		container_of(w, struct net_device_context, dwork.work);
	struct hv_device *device_obj = ndev_ctx->device_ctx;
	struct net_device *net = hv_get_drvdata(device_obj);
1883
	struct netvsc_device *net_device;
1884
	struct rndis_device *rdev;
1885 1886 1887
	struct netvsc_reconfig *event = NULL;
	bool notify = false, reschedule = false;
	unsigned long flags, next_reconfig, delay;
1888

1889 1890 1891 1892 1893 1894
	/* if changes are happening, comeback later */
	if (!rtnl_trylock()) {
		schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
		return;
	}

1895 1896
	net_device = rtnl_dereference(ndev_ctx->nvdev);
	if (!net_device)
1897 1898
		goto out_unlock;

1899 1900
	rdev = net_device->extension;

1901 1902 1903 1904 1905 1906 1907 1908 1909
	next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT;
	if (time_is_after_jiffies(next_reconfig)) {
		/* link_watch only sends one notification with current state
		 * per second, avoid doing reconfig more frequently. Handle
		 * wrap around.
		 */
		delay = next_reconfig - jiffies;
		delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT;
		schedule_delayed_work(&ndev_ctx->dwork, delay);
1910
		goto out_unlock;
1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923
	}
	ndev_ctx->last_reconfig = jiffies;

	spin_lock_irqsave(&ndev_ctx->lock, flags);
	if (!list_empty(&ndev_ctx->reconfig_events)) {
		event = list_first_entry(&ndev_ctx->reconfig_events,
					 struct netvsc_reconfig, list);
		list_del(&event->list);
		reschedule = !list_empty(&ndev_ctx->reconfig_events);
	}
	spin_unlock_irqrestore(&ndev_ctx->lock, flags);

	if (!event)
1924
		goto out_unlock;
1925 1926 1927 1928 1929 1930 1931 1932

	switch (event->event) {
		/* Only the following events are possible due to the check in
		 * netvsc_linkstatus_callback()
		 */
	case RNDIS_STATUS_MEDIA_CONNECT:
		if (rdev->link_state) {
			rdev->link_state = false;
1933
			netif_carrier_on(net);
1934
			netvsc_tx_enable(net_device, net);
1935 1936 1937 1938 1939 1940 1941 1942 1943
		} else {
			notify = true;
		}
		kfree(event);
		break;
	case RNDIS_STATUS_MEDIA_DISCONNECT:
		if (!rdev->link_state) {
			rdev->link_state = true;
			netif_carrier_off(net);
1944
			netvsc_tx_disable(net_device, net);
1945 1946 1947 1948 1949 1950 1951 1952
		}
		kfree(event);
		break;
	case RNDIS_STATUS_NETWORK_CHANGE:
		/* Only makes sense if carrier is present */
		if (!rdev->link_state) {
			rdev->link_state = true;
			netif_carrier_off(net);
1953
			netvsc_tx_disable(net_device, net);
1954 1955
			event->event = RNDIS_STATUS_MEDIA_CONNECT;
			spin_lock_irqsave(&ndev_ctx->lock, flags);
1956
			list_add(&event->list, &ndev_ctx->reconfig_events);
1957 1958
			spin_unlock_irqrestore(&ndev_ctx->lock, flags);
			reschedule = true;
1959
		}
1960
		break;
1961 1962 1963 1964 1965 1966
	}

	rtnl_unlock();

	if (notify)
		netdev_notify_peers(net);
1967 1968 1969 1970 1971 1972

	/* link_watch only sends one notification with current state per
	 * second, handle next reconfig event in 2 seconds.
	 */
	if (reschedule)
		schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1973 1974 1975 1976 1977

	return;

out_unlock:
	rtnl_unlock();
1978 1979
}

1980 1981
static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
{
1982
	struct net_device_context *net_device_ctx;
1983 1984
	struct net_device *dev;

1985 1986 1987
	dev = netdev_master_upper_dev_get(vf_netdev);
	if (!dev || dev->netdev_ops != &device_ops)
		return NULL;	/* not a netvsc device */
1988

1989 1990 1991
	net_device_ctx = netdev_priv(dev);
	if (!rtnl_dereference(net_device_ctx->nvdev))
		return NULL;	/* device is removed */
1992

1993
	return dev;
1994 1995
}

1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007
/* Called when VF is injecting data into network stack.
 * Change the associated network device from VF to netvsc.
 * note: already called with rcu_read_lock
 */
static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb)
{
	struct sk_buff *skb = *pskb;
	struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data);
	struct net_device_context *ndev_ctx = netdev_priv(ndev);
	struct netvsc_vf_pcpu_stats *pcpu_stats
		 = this_cpu_ptr(ndev_ctx->vf_stats);

2008 2009 2010 2011 2012 2013
	skb = skb_share_check(skb, GFP_ATOMIC);
	if (unlikely(!skb))
		return RX_HANDLER_CONSUMED;

	*pskb = skb;

2014 2015 2016 2017 2018 2019 2020 2021 2022 2023
	skb->dev = ndev;

	u64_stats_update_begin(&pcpu_stats->syncp);
	pcpu_stats->rx_packets++;
	pcpu_stats->rx_bytes += skb->len;
	u64_stats_update_end(&pcpu_stats->syncp);

	return RX_HANDLER_ANOTHER;
}

2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
static int netvsc_vf_join(struct net_device *vf_netdev,
			  struct net_device *ndev)
{
	struct net_device_context *ndev_ctx = netdev_priv(ndev);
	int ret;

	ret = netdev_rx_handler_register(vf_netdev,
					 netvsc_vf_handle_frame, ndev);
	if (ret != 0) {
		netdev_err(vf_netdev,
			   "can not register netvsc VF receive handler (err = %d)\n",
			   ret);
		goto rx_handler_failed;
	}

	ret = netdev_master_upper_dev_link(vf_netdev, ndev,
					   NULL, NULL, NULL);
	if (ret != 0) {
		netdev_err(vf_netdev,
			   "can not set master device %s (err = %d)\n",
			   ndev->name, ret);
		goto upper_link_failed;
	}

	/* set slave flag before open to prevent IPv6 addrconf */
	vf_netdev->flags |= IFF_SLAVE;

	schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);

	call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);

	netdev_info(vf_netdev, "joined to %s\n", ndev->name);
	return 0;

upper_link_failed:
	netdev_rx_handler_unregister(vf_netdev);
rx_handler_failed:
	return ret;
}

2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074
static void __netvsc_vf_setup(struct net_device *ndev,
			      struct net_device *vf_netdev)
{
	int ret;

	/* Align MTU of VF with master */
	ret = dev_set_mtu(vf_netdev, ndev->mtu);
	if (ret)
		netdev_warn(vf_netdev,
			    "unable to change mtu to %u\n", ndev->mtu);

2075 2076
	/* set multicast etc flags on VF */
	dev_change_flags(vf_netdev, ndev->flags | IFF_SLAVE);
2077 2078 2079

	/* sync address list from ndev to VF */
	netif_addr_lock_bh(ndev);
2080 2081
	dev_uc_sync(vf_netdev, ndev);
	dev_mc_sync(vf_netdev, ndev);
2082
	netif_addr_unlock_bh(ndev);
2083

2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097
	if (netif_running(ndev)) {
		ret = dev_open(vf_netdev);
		if (ret)
			netdev_warn(vf_netdev,
				    "unable to open: %d\n", ret);
	}
}

/* Setup VF as slave of the synthetic device.
 * Runs in workqueue to avoid recursion in netlink callbacks.
 */
static void netvsc_vf_setup(struct work_struct *w)
{
	struct net_device_context *ndev_ctx
2098
		= container_of(w, struct net_device_context, vf_takeover.work);
2099 2100 2101
	struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx);
	struct net_device *vf_netdev;

2102
	if (!rtnl_trylock()) {
2103
		schedule_delayed_work(&ndev_ctx->vf_takeover, 0);
2104 2105 2106
		return;
	}

2107 2108 2109 2110 2111 2112 2113
	vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
	if (vf_netdev)
		__netvsc_vf_setup(ndev, vf_netdev);

	rtnl_unlock();
}

2114 2115
/* Find netvsc by VF serial number.
 * The PCI hyperv controller records the serial number as the slot kobj name.
2116 2117 2118 2119 2120 2121
 */
static struct net_device *get_netvsc_byslot(const struct net_device *vf_netdev)
{
	struct device *parent = vf_netdev->dev.parent;
	struct net_device_context *ndev_ctx;
	struct pci_dev *pdev;
2122
	u32 serial;
2123 2124 2125 2126 2127 2128 2129 2130 2131 2132

	if (!parent || !dev_is_pci(parent))
		return NULL; /* not a PCI device */

	pdev = to_pci_dev(parent);
	if (!pdev->slot) {
		netdev_notice(vf_netdev, "no PCI slot information\n");
		return NULL;
	}

2133 2134 2135 2136 2137 2138
	if (kstrtou32(pci_slot_name(pdev->slot), 10, &serial)) {
		netdev_notice(vf_netdev, "Invalid vf serial:%s\n",
			      pci_slot_name(pdev->slot));
		return NULL;
	}

2139 2140 2141 2142
	list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) {
		if (!ndev_ctx->vf_alloc)
			continue;

2143
		if (ndev_ctx->vf_serial == serial)
2144 2145 2146 2147
			return hv_get_drvdata(ndev_ctx->device_ctx);
	}

	netdev_notice(vf_netdev,
2148
		      "no netdev found for vf serial:%u\n", serial);
2149 2150 2151
	return NULL;
}

2152
static int netvsc_register_vf(struct net_device *vf_netdev)
2153
{
2154
	struct net_device_context *net_device_ctx;
2155
	struct netvsc_device *netvsc_dev;
2156
	struct net_device *ndev;
2157
	int ret;
2158

2159 2160 2161
	if (vf_netdev->addr_len != ETH_ALEN)
		return NOTIFY_DONE;

2162
	ndev = get_netvsc_byslot(vf_netdev);
2163 2164 2165
	if (!ndev)
		return NOTIFY_DONE;

2166
	net_device_ctx = netdev_priv(ndev);
2167
	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
2168
	if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
2169
		return NOTIFY_DONE;
2170

2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185
	/* if syntihetic interface is a different namespace,
	 * then move the VF to that namespace; join will be
	 * done again in that context.
	 */
	if (!net_eq(dev_net(ndev), dev_net(vf_netdev))) {
		ret = dev_change_net_namespace(vf_netdev,
					       dev_net(ndev), "eth%d");
		if (ret)
			netdev_err(vf_netdev,
				   "could not move to same namespace as %s: %d\n",
				   ndev->name, ret);
		else
			netdev_info(vf_netdev,
				    "VF moved to namespace with: %s\n",
				    ndev->name);
2186
		return NOTIFY_DONE;
2187
	}
2188

2189
	netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
2190

2191 2192 2193
	if (netvsc_vf_join(vf_netdev, ndev) != 0)
		return NOTIFY_DONE;

2194
	dev_hold(vf_netdev);
2195 2196
	rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
	return NOTIFY_OK;
2197 2198
}

2199
/* VF up/down change detected, schedule to change data path */
2200
static int netvsc_vf_changed(struct net_device *vf_netdev)
2201
{
2202
	struct net_device_context *net_device_ctx;
2203
	struct netvsc_device *netvsc_dev;
2204
	struct net_device *ndev;
2205
	bool vf_is_up = netif_running(vf_netdev);
2206

2207 2208 2209 2210
	ndev = get_netvsc_byref(vf_netdev);
	if (!ndev)
		return NOTIFY_DONE;

2211 2212
	net_device_ctx = netdev_priv(ndev);
	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
2213
	if (!netvsc_dev)
2214
		return NOTIFY_DONE;
2215

2216 2217 2218
	netvsc_switch_datapath(ndev, vf_is_up);
	netdev_info(ndev, "Data path switched %s VF: %s\n",
		    vf_is_up ? "to" : "from", vf_netdev->name);
2219

2220
	return NOTIFY_OK;
2221 2222
}

2223
static int netvsc_unregister_vf(struct net_device *vf_netdev)
2224
{
2225
	struct net_device *ndev;
2226
	struct net_device_context *net_device_ctx;
2227

2228 2229 2230
	ndev = get_netvsc_byref(vf_netdev);
	if (!ndev)
		return NOTIFY_DONE;
2231 2232

	net_device_ctx = netdev_priv(ndev);
2233
	cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
2234

2235
	netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
2236

2237 2238
	netdev_rx_handler_unregister(vf_netdev);
	netdev_upper_dev_unlink(vf_netdev, ndev);
2239
	RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
2240
	dev_put(vf_netdev);
2241

2242
	return NOTIFY_OK;
2243 2244
}

2245 2246
static int netvsc_probe(struct hv_device *dev,
			const struct hv_vmbus_device_id *dev_id)
2247 2248 2249
{
	struct net_device *net = NULL;
	struct net_device_context *net_device_ctx;
2250
	struct netvsc_device_info *device_info = NULL;
2251
	struct netvsc_device *nvdev;
2252
	int ret = -ENOMEM;
2253

2254
	net = alloc_etherdev_mq(sizeof(struct net_device_context),
2255
				VRSS_CHANNEL_MAX);
2256
	if (!net)
2257
		goto no_net;
2258

2259 2260
	netif_carrier_off(net);

2261 2262
	netvsc_init_settings(net);

2263
	net_device_ctx = netdev_priv(net);
2264
	net_device_ctx->device_ctx = dev;
2265 2266 2267 2268 2269
	net_device_ctx->msg_enable = netif_msg_init(debug, default_msg);
	if (netif_msg_probe(net_device_ctx))
		netdev_dbg(net, "netvsc msg_enable: %d\n",
			   net_device_ctx->msg_enable);

2270
	hv_set_drvdata(dev, net);
2271

2272
	INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change);
2273

2274 2275
	spin_lock_init(&net_device_ctx->lock);
	INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
2276
	INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup);
2277 2278 2279 2280 2281

	net_device_ctx->vf_stats
		= netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats);
	if (!net_device_ctx->vf_stats)
		goto no_stats;
2282

2283
	net->netdev_ops = &device_ops;
2284
	net->ethtool_ops = &ethtool_ops;
2285
	SET_NETDEV_DEV(net, &dev->device);
2286

2287 2288 2289
	/* We always need headroom for rndis header */
	net->needed_headroom = RNDIS_AND_PPI_SIZE;

2290 2291 2292 2293 2294 2295
	/* Initialize the number of queues to be 1, we may change it if more
	 * channels are offered later.
	 */
	netif_set_real_num_tx_queues(net, 1);
	netif_set_real_num_rx_queues(net, 1);

2296
	/* Notify the netvsc driver of the new device */
2297 2298 2299 2300 2301 2302 2303 2304
	device_info = netvsc_devinfo_get(NULL);

	if (!device_info) {
		ret = -ENOMEM;
		goto devinfo_failed;
	}

	nvdev = rndis_filter_device_add(dev, device_info);
2305 2306
	if (IS_ERR(nvdev)) {
		ret = PTR_ERR(nvdev);
2307
		netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
2308
		goto rndis_failed;
2309
	}
2310

2311
	memcpy(net->dev_addr, device_info->mac_adr, ETH_ALEN);
2312

2313 2314 2315 2316 2317 2318 2319 2320 2321 2322
	/* We must get rtnl lock before scheduling nvdev->subchan_work,
	 * otherwise netvsc_subchan_work() can get rtnl lock first and wait
	 * all subchannels to show up, but that may not happen because
	 * netvsc_probe() can't get rtnl lock and as a result vmbus_onoffer()
	 * -> ... -> device_add() -> ... -> __device_attach() can't get
	 * the device lock, so all the subchannels can't be processed --
	 * finally netvsc_subchan_work() hangs for ever.
	 */
	rtnl_lock();

2323 2324 2325
	if (nvdev->num_chn > 1)
		schedule_work(&nvdev->subchan_work);

2326
	/* hw_features computed in rndis_netdev_set_hwcaps() */
2327 2328 2329 2330 2331
	net->features = net->hw_features |
		NETIF_F_HIGHDMA | NETIF_F_SG |
		NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
	net->vlan_features = net->features;

2332 2333
	netdev_lockdep_set_classes(net);

2334 2335 2336 2337 2338 2339 2340
	/* MTU range: 68 - 1500 or 65521 */
	net->min_mtu = NETVSC_MTU_MIN;
	if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
		net->max_mtu = NETVSC_MTU - ETH_HLEN;
	else
		net->max_mtu = ETH_DATA_LEN;

2341 2342
	nvdev->tx_disable = false;

2343
	ret = register_netdevice(net);
2344 2345
	if (ret != 0) {
		pr_err("Unable to register netdev.\n");
2346
		goto register_failed;
2347 2348
	}

2349 2350
	list_add(&net_device_ctx->list, &netvsc_dev_list);
	rtnl_unlock();
2351 2352

	kfree(device_info);
2353
	return 0;
2354 2355

register_failed:
2356
	rtnl_unlock();
2357 2358
	rndis_filter_device_remove(dev, nvdev);
rndis_failed:
2359 2360
	kfree(device_info);
devinfo_failed:
2361 2362 2363 2364 2365 2366
	free_percpu(net_device_ctx->vf_stats);
no_stats:
	hv_set_drvdata(dev, NULL);
	free_netdev(net);
no_net:
	return ret;
2367 2368
}

2369
static int netvsc_remove(struct hv_device *dev)
2370
{
2371
	struct net_device_context *ndev_ctx;
2372 2373
	struct net_device *vf_netdev, *net;
	struct netvsc_device *nvdev;
2374

2375
	net = hv_get_drvdata(dev);
2376
	if (net == NULL) {
2377
		dev_err(&dev->device, "No net device to remove\n");
2378 2379 2380
		return 0;
	}

2381
	ndev_ctx = netdev_priv(net);
2382

2383 2384
	cancel_delayed_work_sync(&ndev_ctx->dwork);

2385 2386 2387
	rtnl_lock();
	nvdev = rtnl_dereference(ndev_ctx->nvdev);
	if (nvdev)
2388 2389
		cancel_work_sync(&nvdev->subchan_work);

2390 2391
	/*
	 * Call to the vsc driver to let it know that the device is being
2392
	 * removed. Also blocks mtu and channel changes.
2393
	 */
2394 2395
	vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
	if (vf_netdev)
2396
		netvsc_unregister_vf(vf_netdev);
2397

2398 2399 2400
	if (nvdev)
		rndis_filter_device_remove(dev, nvdev);

2401
	unregister_netdevice(net);
2402
	list_del(&ndev_ctx->list);
2403

2404 2405
	rtnl_unlock();

2406 2407
	hv_set_drvdata(dev, NULL);

2408
	free_percpu(ndev_ctx->vf_stats);
2409
	free_netdev(net);
2410
	return 0;
2411 2412
}

2413
static const struct hv_vmbus_device_id id_table[] = {
2414
	/* Network guid */
2415
	{ HV_NIC_GUID, },
2416
	{ },
2417 2418 2419 2420
};

MODULE_DEVICE_TABLE(vmbus, id_table);

2421
/* The one and only one */
2422
static struct  hv_driver netvsc_drv = {
2423
	.name = KBUILD_MODNAME,
2424
	.id_table = id_table,
2425 2426
	.probe = netvsc_probe,
	.remove = netvsc_remove,
2427
	.driver = {
2428
		.probe_type = PROBE_FORCE_SYNCHRONOUS,
2429
	},
2430
};
2431

2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476
/*
 * On Hyper-V, every VF interface is matched with a corresponding
 * synthetic interface. The synthetic interface is presented first
 * to the guest. When the corresponding VF instance is registered,
 * we will take care of switching the data path.
 */
static int netvsc_netdev_event(struct notifier_block *this,
			       unsigned long event, void *ptr)
{
	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);

	/* Skip our own events */
	if (event_dev->netdev_ops == &device_ops)
		return NOTIFY_DONE;

	/* Avoid non-Ethernet type devices */
	if (event_dev->type != ARPHRD_ETHER)
		return NOTIFY_DONE;

	/* Avoid Vlan dev with same MAC registering as VF */
	if (is_vlan_dev(event_dev))
		return NOTIFY_DONE;

	/* Avoid Bonding master dev with same MAC registering as VF */
	if ((event_dev->priv_flags & IFF_BONDING) &&
	    (event_dev->flags & IFF_MASTER))
		return NOTIFY_DONE;

	switch (event) {
	case NETDEV_REGISTER:
		return netvsc_register_vf(event_dev);
	case NETDEV_UNREGISTER:
		return netvsc_unregister_vf(event_dev);
	case NETDEV_UP:
	case NETDEV_DOWN:
		return netvsc_vf_changed(event_dev);
	default:
		return NOTIFY_DONE;
	}
}

static struct notifier_block netvsc_netdev_notifier = {
	.notifier_call = netvsc_netdev_event,
};

2477
static void __exit netvsc_drv_exit(void)
2478
{
2479
	unregister_netdevice_notifier(&netvsc_netdev_notifier);
2480
	vmbus_driver_unregister(&netvsc_drv);
2481 2482
}

2483
static int __init netvsc_drv_init(void)
2484
{
2485 2486
	int ret;

2487 2488
	if (ring_size < RING_SIZE_MIN) {
		ring_size = RING_SIZE_MIN;
2489
		pr_info("Increased ring_size to %u (min allowed)\n",
2490 2491
			ring_size);
	}
2492
	netvsc_ring_bytes = ring_size * PAGE_SIZE;
2493

2494
	ret = vmbus_driver_register(&netvsc_drv);
2495 2496 2497
	if (ret)
		return ret;

2498
	register_netdevice_notifier(&netvsc_netdev_notifier);
2499
	return 0;
2500 2501
}

2502
MODULE_LICENSE("GPL");
2503
MODULE_DESCRIPTION("Microsoft Hyper-V network driver");
2504

2505
module_init(netvsc_drv_init);
2506
module_exit(netvsc_drv_exit);