ixgbe_main.c 217 KB
Newer Older
1 2 3
/*******************************************************************************

  Intel 10 Gigabit PCI Express Linux driver
4
  Copyright(c) 1999 - 2011 Intel Corporation.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
  this program; if not, write to the Free Software Foundation, Inc.,
  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Contact Information:
  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

*******************************************************************************/

#include <linux/types.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/vmalloc.h>
#include <linux/string.h>
#include <linux/in.h>
35
#include <linux/interrupt.h>
36 37
#include <linux/ip.h>
#include <linux/tcp.h>
38
#include <linux/sctp.h>
39
#include <linux/pkt_sched.h>
40
#include <linux/ipv6.h>
41
#include <linux/slab.h>
42 43 44
#include <net/checksum.h>
#include <net/ip6_checksum.h>
#include <linux/ethtool.h>
45
#include <linux/if.h>
46
#include <linux/if_vlan.h>
47
#include <linux/prefetch.h>
48
#include <scsi/fc/fc_fcoe.h>
49 50 51

#include "ixgbe.h"
#include "ixgbe_common.h"
52
#include "ixgbe_dcb_82599.h"
53
#include "ixgbe_sriov.h"
54 55

char ixgbe_driver_name[] = "ixgbe";
56
static const char ixgbe_driver_string[] =
57
			      "Intel(R) 10 Gigabit PCI Express Network Driver";
58
#define MAJ 3
59
#define MIN 4
Don Skidmore's avatar
Don Skidmore committed
60
#define BUILD 8
61
#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
62
	__stringify(BUILD) "-k"
63
const char ixgbe_driver_version[] = DRV_VERSION;
64 65
static const char ixgbe_copyright[] =
				"Copyright (c) 1999-2011 Intel Corporation.";
66 67

static const struct ixgbe_info *ixgbe_info_tbl[] = {
68
	[board_82598] = &ixgbe_82598_info,
69
	[board_82599] = &ixgbe_82599_info,
70
	[board_X540] = &ixgbe_X540_info,
71 72 73 74 75 76 77 78 79 80
};

/* ixgbe_pci_tbl - PCI Device ID Table
 *
 * Wildcard entries (PCI_ANY_ID) should come last
 * Last entry must be all 0s
 *
 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
 *   Class, Class Mask, private data (not used) }
 */
81
static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
82 83
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598),
	 board_82598 },
84
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT),
85
	 board_82598 },
86
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT),
87
	 board_82598 },
88 89
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT),
	 board_82598 },
90 91
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2),
	 board_82598 },
92
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4),
93
	 board_82598 },
Jesse Brandeburg's avatar
Jesse Brandeburg committed
94 95
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT),
	 board_82598 },
96 97 98 99
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT),
	 board_82598 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM),
	 board_82598 },
100 101
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR),
	 board_82598 },
102 103
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM),
	 board_82598 },
104 105
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX),
	 board_82598 },
106 107
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4),
	 board_82599 },
108 109
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM),
	 board_82599 },
110 111
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR),
	 board_82599 },
112 113
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP),
	 board_82599 },
114 115
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM),
	 board_82599 },
116 117
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ),
	 board_82599 },
118 119
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4),
	 board_82599 },
120 121 122 123
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE),
	 board_82599 },
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE),
	 board_82599 },
124 125
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM),
	 board_82599 },
126 127
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE),
	 board_82599 },
128
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T),
129
	 board_X540 },
130 131
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2),
	 board_82599 },
132 133
	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS),
	 board_82599 },
134 135 136 137 138 139

	/* required last entry */
	{0, }
};
MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);

140
#ifdef CONFIG_IXGBE_DCA
141
static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
142
			    void *p);
143 144 145 146 147 148 149
static struct notifier_block dca_notifier = {
	.notifier_call = ixgbe_notify_dca,
	.next          = NULL,
	.priority      = 0
};
#endif

150 151 152
#ifdef CONFIG_PCI_IOV
static unsigned int max_vfs;
module_param(max_vfs, uint, 0);
153 154
MODULE_PARM_DESC(max_vfs,
		 "Maximum number of virtual functions to allocate per physical function");
155 156
#endif /* CONFIG_PCI_IOV */

157 158 159 160 161 162 163
MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRV_VERSION);

#define DEFAULT_DEBUG_LEVEL_SHIFT 3

164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
static inline void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 gcr;
	u32 gpie;
	u32 vmdctl;

#ifdef CONFIG_PCI_IOV
	/* disable iov and allow time for transactions to clear */
	pci_disable_sriov(adapter->pdev);
#endif

	/* turn off device IOV mode */
	gcr = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
	gcr &= ~(IXGBE_GCR_EXT_SRIOV);
	IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr);
	gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
	gpie &= ~IXGBE_GPIE_VTMODE_MASK;
	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);

	/* set default pool back to 0 */
	vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
	vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
	IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
188
	IXGBE_WRITE_FLUSH(hw);
189 190 191

	/* take a breather then clean up driver data */
	msleep(100);
192 193

	kfree(adapter->vfinfo);
194 195 196 197 198 199
	adapter->vfinfo = NULL;

	adapter->num_vfs = 0;
	adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
}

200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
{
	if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
	    !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
		schedule_work(&adapter->service_task);
}

static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
{
	BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));

	/* flush memory to make sure state is correct before next watchog */
	smp_mb__before_clear_bit();
	clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
}

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
struct ixgbe_reg_info {
	u32 ofs;
	char *name;
};

static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {

	/* General Registers */
	{IXGBE_CTRL, "CTRL"},
	{IXGBE_STATUS, "STATUS"},
	{IXGBE_CTRL_EXT, "CTRL_EXT"},

	/* Interrupt Registers */
	{IXGBE_EICR, "EICR"},

	/* RX Registers */
	{IXGBE_SRRCTL(0), "SRRCTL"},
	{IXGBE_DCA_RXCTRL(0), "DRXCTL"},
	{IXGBE_RDLEN(0), "RDLEN"},
	{IXGBE_RDH(0), "RDH"},
	{IXGBE_RDT(0), "RDT"},
	{IXGBE_RXDCTL(0), "RXDCTL"},
	{IXGBE_RDBAL(0), "RDBAL"},
	{IXGBE_RDBAH(0), "RDBAH"},

	/* TX Registers */
	{IXGBE_TDBAL(0), "TDBAL"},
	{IXGBE_TDBAH(0), "TDBAH"},
	{IXGBE_TDLEN(0), "TDLEN"},
	{IXGBE_TDH(0), "TDH"},
	{IXGBE_TDT(0), "TDT"},
	{IXGBE_TXDCTL(0), "TXDCTL"},

	/* List Terminator */
	{}
};


/*
 * ixgbe_regdump - register printout routine
 */
static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
{
	int i = 0, j = 0;
	char rname[16];
	u32 regs[64];

	switch (reginfo->ofs) {
	case IXGBE_SRRCTL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
		break;
	case IXGBE_DCA_RXCTRL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
		break;
	case IXGBE_RDLEN(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
		break;
	case IXGBE_RDH(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
		break;
	case IXGBE_RDT(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
		break;
	case IXGBE_RXDCTL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
		break;
	case IXGBE_RDBAL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
		break;
	case IXGBE_RDBAH(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
		break;
	case IXGBE_TDBAL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
		break;
	case IXGBE_TDBAH(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
		break;
	case IXGBE_TDLEN(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
		break;
	case IXGBE_TDH(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
		break;
	case IXGBE_TDT(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
		break;
	case IXGBE_TXDCTL(0):
		for (i = 0; i < 64; i++)
			regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
		break;
	default:
321
		pr_info("%-15s %08x\n", reginfo->name,
322 323 324 325 326 327
			IXGBE_READ_REG(hw, reginfo->ofs));
		return;
	}

	for (i = 0; i < 8; i++) {
		snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
328
		pr_err("%-15s", rname);
329
		for (j = 0; j < 8; j++)
330 331
			pr_cont(" %08x", regs[i*8+j]);
		pr_cont("\n");
332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
	}

}

/*
 * ixgbe_dump - Print registers, tx-rings and rx-rings
 */
static void ixgbe_dump(struct ixgbe_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	struct ixgbe_hw *hw = &adapter->hw;
	struct ixgbe_reg_info *reginfo;
	int n = 0;
	struct ixgbe_ring *tx_ring;
	struct ixgbe_tx_buffer *tx_buffer_info;
	union ixgbe_adv_tx_desc *tx_desc;
	struct my_u0 { u64 a; u64 b; } *u0;
	struct ixgbe_ring *rx_ring;
	union ixgbe_adv_rx_desc *rx_desc;
	struct ixgbe_rx_buffer *rx_buffer_info;
	u32 staterr;
	int i = 0;

	if (!netif_msg_hw(adapter))
		return;

	/* Print netdevice Info */
	if (netdev) {
		dev_info(&adapter->pdev->dev, "Net device Info\n");
361
		pr_info("Device Name     state            "
362
			"trans_start      last_rx\n");
363 364 365 366 367
		pr_info("%-15s %016lX %016lX %016lX\n",
			netdev->name,
			netdev->state,
			netdev->trans_start,
			netdev->last_rx);
368 369 370 371
	}

	/* Print Registers */
	dev_info(&adapter->pdev->dev, "Register Dump\n");
372
	pr_info(" Register Name   Value\n");
373 374 375 376 377 378 379 380 381 382
	for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
	     reginfo->name; reginfo++) {
		ixgbe_regdump(hw, reginfo);
	}

	/* Print TX Ring Summary */
	if (!netdev || !netif_running(netdev))
		goto exit;

	dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
383
	pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
384 385 386 387
	for (n = 0; n < adapter->num_tx_queues; n++) {
		tx_ring = adapter->tx_ring[n];
		tx_buffer_info =
			&tx_ring->tx_buffer_info[tx_ring->next_to_clean];
388
		pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414
			   n, tx_ring->next_to_use, tx_ring->next_to_clean,
			   (u64)tx_buffer_info->dma,
			   tx_buffer_info->length,
			   tx_buffer_info->next_to_watch,
			   (u64)tx_buffer_info->time_stamp);
	}

	/* Print TX Rings */
	if (!netif_msg_tx_done(adapter))
		goto rx_ring_summary;

	dev_info(&adapter->pdev->dev, "TX Rings Dump\n");

	/* Transmit Descriptor Formats
	 *
	 * Advanced Transmit Descriptor
	 *   +--------------------------------------------------------------+
	 * 0 |         Buffer Address [63:0]                                |
	 *   +--------------------------------------------------------------+
	 * 8 |  PAYLEN  | PORTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
	 *   +--------------------------------------------------------------+
	 *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
	 */

	for (n = 0; n < adapter->num_tx_queues; n++) {
		tx_ring = adapter->tx_ring[n];
415 416 417 418
		pr_info("------------------------------------\n");
		pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
		pr_info("------------------------------------\n");
		pr_info("T [desc]     [address 63:0  ] "
419 420 421 422
			"[PlPOIdStDDt Ln] [bi->dma       ] "
			"leng  ntw timestamp        bi->skb\n");

		for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
423
			tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
424 425
			tx_buffer_info = &tx_ring->tx_buffer_info[i];
			u0 = (struct my_u0 *)tx_desc;
426
			pr_info("T [0x%03X]    %016llX %016llX %016llX"
427
				" %04X  %p %016llX %p", i,
428 429 430 431 432 433 434 435 436
				le64_to_cpu(u0->a),
				le64_to_cpu(u0->b),
				(u64)tx_buffer_info->dma,
				tx_buffer_info->length,
				tx_buffer_info->next_to_watch,
				(u64)tx_buffer_info->time_stamp,
				tx_buffer_info->skb);
			if (i == tx_ring->next_to_use &&
				i == tx_ring->next_to_clean)
437
				pr_cont(" NTC/U\n");
438
			else if (i == tx_ring->next_to_use)
439
				pr_cont(" NTU\n");
440
			else if (i == tx_ring->next_to_clean)
441
				pr_cont(" NTC\n");
442
			else
443
				pr_cont("\n");
444 445 446 447 448 449 450 451 452 453 454 455 456

			if (netif_msg_pktdata(adapter) &&
				tx_buffer_info->dma != 0)
				print_hex_dump(KERN_INFO, "",
					DUMP_PREFIX_ADDRESS, 16, 1,
					phys_to_virt(tx_buffer_info->dma),
					tx_buffer_info->length, true);
		}
	}

	/* Print RX Rings Summary */
rx_ring_summary:
	dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
457
	pr_info("Queue [NTU] [NTC]\n");
458 459
	for (n = 0; n < adapter->num_rx_queues; n++) {
		rx_ring = adapter->rx_ring[n];
460 461
		pr_info("%5d %5X %5X\n",
			n, rx_ring->next_to_use, rx_ring->next_to_clean);
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
	}

	/* Print RX Rings */
	if (!netif_msg_rx_status(adapter))
		goto exit;

	dev_info(&adapter->pdev->dev, "RX Rings Dump\n");

	/* Advanced Receive Descriptor (Read) Format
	 *    63                                           1        0
	 *    +-----------------------------------------------------+
	 *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
	 *    +----------------------------------------------+------+
	 *  8 |       Header Buffer Address [63:1]           |  DD  |
	 *    +-----------------------------------------------------+
	 *
	 *
	 * Advanced Receive Descriptor (Write-Back) Format
	 *
	 *   63       48 47    32 31  30      21 20 16 15   4 3     0
	 *   +------------------------------------------------------+
	 * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
	 *   | Checksum   Ident  |   |           |    | Type | Type |
	 *   +------------------------------------------------------+
	 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
	 *   +------------------------------------------------------+
	 *   63       48 47    32 31            20 19               0
	 */
	for (n = 0; n < adapter->num_rx_queues; n++) {
		rx_ring = adapter->rx_ring[n];
492 493 494 495
		pr_info("------------------------------------\n");
		pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
		pr_info("------------------------------------\n");
		pr_info("R  [desc]      [ PktBuf     A0] "
496 497
			"[  HeadBuf   DD] [bi->dma       ] [bi->skb] "
			"<-- Adv Rx Read format\n");
498
		pr_info("RWB[desc]      [PcsmIpSHl PtRs] "
499 500 501 502 503
			"[vl er S cks ln] ---------------- [bi->skb] "
			"<-- Adv Rx Write-Back format\n");

		for (i = 0; i < rx_ring->count; i++) {
			rx_buffer_info = &rx_ring->rx_buffer_info[i];
504
			rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
505 506 507 508
			u0 = (struct my_u0 *)rx_desc;
			staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
			if (staterr & IXGBE_RXD_STAT_DD) {
				/* Descriptor Done */
509
				pr_info("RWB[0x%03X]     %016llX "
510 511 512 513 514
					"%016llX ---------------- %p", i,
					le64_to_cpu(u0->a),
					le64_to_cpu(u0->b),
					rx_buffer_info->skb);
			} else {
515
				pr_info("R  [0x%03X]     %016llX "
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
					"%016llX %016llX %p", i,
					le64_to_cpu(u0->a),
					le64_to_cpu(u0->b),
					(u64)rx_buffer_info->dma,
					rx_buffer_info->skb);

				if (netif_msg_pktdata(adapter)) {
					print_hex_dump(KERN_INFO, "",
					   DUMP_PREFIX_ADDRESS, 16, 1,
					   phys_to_virt(rx_buffer_info->dma),
					   rx_ring->rx_buf_len, true);

					if (rx_ring->rx_buf_len
						< IXGBE_RXBUFFER_2048)
						print_hex_dump(KERN_INFO, "",
						  DUMP_PREFIX_ADDRESS, 16, 1,
						  phys_to_virt(
						    rx_buffer_info->page_dma +
						    rx_buffer_info->page_offset
						  ),
						  PAGE_SIZE/2, true);
				}
			}

			if (i == rx_ring->next_to_use)
541
				pr_cont(" NTU\n");
542
			else if (i == rx_ring->next_to_clean)
543
				pr_cont(" NTC\n");
544
			else
545
				pr_cont("\n");
546 547 548 549 550 551 552 553

		}
	}

exit:
	return;
}

554 555 556 557 558 559 560
static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
{
	u32 ctrl_ext;

	/* Let firmware take over control of h/w */
	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
561
			ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
562 563 564 565 566 567 568 569 570
}

static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
{
	u32 ctrl_ext;

	/* Let firmware know the driver has taken over */
	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
571
			ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
572
}
573

574 575 576 577 578 579 580 581 582
/*
 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
 * @adapter: pointer to adapter struct
 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 * @queue: queue to map the corresponding interrupt to
 * @msix_vector: the vector to map to the corresponding queue
 *
 */
static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
583
			   u8 queue, u8 msix_vector)
584 585
{
	u32 ivar, index;
586 587 588 589 590 591 592 593 594 595 596 597 598
	struct ixgbe_hw *hw = &adapter->hw;
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
		msix_vector |= IXGBE_IVAR_ALLOC_VAL;
		if (direction == -1)
			direction = 0;
		index = (((direction * 64) + queue) >> 2) & 0x1F;
		ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
		ivar &= ~(0xFF << (8 * (queue & 0x3)));
		ivar |= (msix_vector << (8 * (queue & 0x3)));
		IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
		break;
	case ixgbe_mac_82599EB:
599
	case ixgbe_mac_X540:
600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
		if (direction == -1) {
			/* other causes */
			msix_vector |= IXGBE_IVAR_ALLOC_VAL;
			index = ((queue & 1) * 8);
			ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
			ivar &= ~(0xFF << index);
			ivar |= (msix_vector << index);
			IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
			break;
		} else {
			/* tx or rx causes */
			msix_vector |= IXGBE_IVAR_ALLOC_VAL;
			index = ((16 * (queue & 1)) + (8 * direction));
			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
			ivar &= ~(0xFF << index);
			ivar |= (msix_vector << index);
			IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
			break;
		}
	default:
		break;
	}
622 623
}

624
static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
625
					  u64 qmask)
626 627 628
{
	u32 mask;

629 630
	switch (adapter->hw.mac.type) {
	case ixgbe_mac_82598EB:
631 632
		mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
633 634
		break;
	case ixgbe_mac_82599EB:
635
	case ixgbe_mac_X540:
636 637 638 639
		mask = (qmask & 0xFFFFFFFF);
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
		mask = (qmask >> 32);
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
640 641 642
		break;
	default:
		break;
643 644 645
	}
}

646 647
static inline void ixgbe_unmap_tx_resource(struct ixgbe_ring *ring,
					   struct ixgbe_tx_buffer *tx_buffer)
648
{
649 650 651 652 653 654
	if (tx_buffer->dma) {
		if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_MAPPED_AS_PAGE)
			dma_unmap_page(ring->dev,
			               tx_buffer->dma,
			               tx_buffer->length,
			               DMA_TO_DEVICE);
655
		else
656 657 658 659
			dma_unmap_single(ring->dev,
			                 tx_buffer->dma,
			                 tx_buffer->length,
			                 DMA_TO_DEVICE);
660
	}
661 662 663 664 665 666 667 668
	tx_buffer->dma = 0;
}

void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *tx_ring,
				      struct ixgbe_tx_buffer *tx_buffer_info)
{
	ixgbe_unmap_tx_resource(tx_ring, tx_buffer_info);
	if (tx_buffer_info->skb)
669
		dev_kfree_skb_any(tx_buffer_info->skb);
670
	tx_buffer_info->skb = NULL;
671 672 673
	/* tx_buffer_info must be completely set up in the transmit path */
}

674 675 676 677 678 679 680 681 682 683 684 685 686
static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	struct ixgbe_hw_stats *hwstats = &adapter->stats;
	u32 data = 0;
	u32 xoff[8] = {0};
	int i;

	if ((hw->fc.current_mode == ixgbe_fc_full) ||
	    (hw->fc.current_mode == ixgbe_fc_rx_pause)) {
		switch (hw->mac.type) {
		case ixgbe_mac_82598EB:
			data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
687 688
			break;
		default:
689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
			data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
		}
		hwstats->lxoffrxc += data;

		/* refill credits (no tx hang) if we received xoff */
		if (!data)
			return;

		for (i = 0; i < adapter->num_tx_queues; i++)
			clear_bit(__IXGBE_HANG_CHECK_ARMED,
				  &adapter->tx_ring[i]->state);
		return;
	} else if (!(adapter->dcb_cfg.pfc_mode_enable))
		return;

	/* update stats for each tc, only valid with PFC enabled */
	for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
		switch (hw->mac.type) {
		case ixgbe_mac_82598EB:
			xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
709
			break;
710 711
		default:
			xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
712
		}
713 714 715 716 717 718
		hwstats->pxoffrxc[i] += xoff[i];
	}

	/* disarm tx queues that have received xoff frames */
	for (i = 0; i < adapter->num_tx_queues; i++) {
		struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
719
		u8 tc = tx_ring->dcb_tc;
720 721 722

		if (xoff[tc])
			clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
723 724 725
	}
}

726
static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
727
{
728 729 730 731 732 733
	return ring->tx_stats.completed;
}

static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
{
	struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
734 735
	struct ixgbe_hw *hw = &adapter->hw;

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
	u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
	u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));

	if (head != tail)
		return (head < tail) ?
			tail - head : (tail + ring->count - head);

	return 0;
}

static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
{
	u32 tx_done = ixgbe_get_tx_completed(tx_ring);
	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
	u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
	bool ret = false;

753
	clear_check_for_tx_hang(tx_ring);
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775

	/*
	 * Check for a hung queue, but be thorough. This verifies
	 * that a transmit has been completed since the previous
	 * check AND there is at least one packet pending. The
	 * ARMED bit is set to indicate a potential hang. The
	 * bit is cleared if a pause frame is received to remove
	 * false hang detection due to PFC or 802.3x frames. By
	 * requiring this to fail twice we avoid races with
	 * pfc clearing the ARMED bit and conditions where we
	 * run the check_tx_hang logic with a transmit completion
	 * pending but without time to complete it yet.
	 */
	if ((tx_done_old == tx_done) && tx_pending) {
		/* make sure it is true for two checks in a row */
		ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
				       &tx_ring->state);
	} else {
		/* update completed stats and continue */
		tx_ring->tx_stats.tx_done_old = tx_done;
		/* reset the countdown */
		clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
776 777
	}

778
	return ret;
779 780
}

781 782 783 784 785 786 787 788 789 790 791 792 793
/**
 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
 * @adapter: driver private struct
 **/
static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
{

	/* Do the reset outside of interrupt context */
	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
		adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
		ixgbe_service_event_schedule(adapter);
	}
}
794

795 796
/**
 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
797
 * @q_vector: structure containing interrupt and ring information
798
 * @tx_ring: tx ring to clean
799
 **/
800
static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
801
			       struct ixgbe_ring *tx_ring)
802
{
803
	struct ixgbe_adapter *adapter = q_vector->adapter;
804 805
	struct ixgbe_tx_buffer *tx_buffer;
	union ixgbe_adv_tx_desc *tx_desc;
806
	unsigned int total_bytes = 0, total_packets = 0;
807 808
	u16 i = tx_ring->next_to_clean;
	u16 count;
809

810 811
	tx_buffer = &tx_ring->tx_buffer_info[i];
	tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
812

813 814 815 816 817 818 819 820 821 822
	for (count = 0; count < q_vector->tx.work_limit; count++) {
		union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;

		/* if next_to_watch is not set then there is no work pending */
		if (!eop_desc)
			break;

		/* if DD is not set pending work has not been completed */
		if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
			break;
823

824 825 826 827 828
		/* count the packet as being completed */
		tx_ring->tx_stats.completed++;

		/* clear next_to_watch to prevent false hangs */
		tx_buffer->next_to_watch = NULL;
829

830 831 832 833 834
		/* prevent any other reads prior to eop_desc being verified */
		rmb();

		do {
			ixgbe_unmap_tx_resource(tx_ring, tx_buffer);
835
			tx_desc->wb.status = 0;
836 837 838 839 840 841 842 843
			if (likely(tx_desc == eop_desc)) {
				eop_desc = NULL;
				dev_kfree_skb_any(tx_buffer->skb);
				tx_buffer->skb = NULL;

				total_bytes += tx_buffer->bytecount;
				total_packets += tx_buffer->gso_segs;
			}
844

845 846
			tx_buffer++;
			tx_desc++;
847
			i++;
848
			if (unlikely(i == tx_ring->count)) {
849
				i = 0;
850

851 852
				tx_buffer = tx_ring->tx_buffer_info;
				tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
853
			}
854

855
		} while (eop_desc);
856 857
	}

858
	tx_ring->next_to_clean = i;
859
	u64_stats_update_begin(&tx_ring->syncp);
860
	tx_ring->stats.bytes += total_bytes;
861
	tx_ring->stats.packets += total_packets;
862
	u64_stats_update_end(&tx_ring->syncp);
863 864
	q_vector->tx.total_bytes += total_bytes;
	q_vector->tx.total_packets += total_packets;
865

866 867 868
	if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
		/* schedule immediate reset if we believe we hung */
		struct ixgbe_hw *hw = &adapter->hw;
869
		tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
870 871 872 873 874 875 876 877 878 879 880
		e_err(drv, "Detected Tx Unit Hang\n"
			"  Tx Queue             <%d>\n"
			"  TDH, TDT             <%x>, <%x>\n"
			"  next_to_use          <%x>\n"
			"  next_to_clean        <%x>\n"
			"tx_buffer_info[next_to_clean]\n"
			"  time_stamp           <%lx>\n"
			"  jiffies              <%lx>\n",
			tx_ring->queue_index,
			IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
			IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
881 882
			tx_ring->next_to_use, i,
			tx_ring->tx_buffer_info[i].time_stamp, jiffies);
883 884 885 886 887 888 889

		netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);

		e_info(probe,
		       "tx hang %d detected on queue %d, resetting adapter\n",
			adapter->tx_timeout_count + 1, tx_ring->queue_index);

890
		/* schedule immediate reset if we believe we hung */
891
		ixgbe_tx_timeout_reset(adapter);
892 893 894 895

		/* the adapter is about to reset, no point in enabling stuff */
		return true;
	}
896

897
#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
898
	if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
899
		     (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
900 901 902 903
		/* Make sure that anybody stopping the queue after this
		 * sees the new next_to_clean.
		 */
		smp_mb();
904
		if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) &&
905
		    !test_bit(__IXGBE_DOWN, &adapter->state)) {
906
			netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index);
907
			++tx_ring->tx_stats.restart_queue;
908
		}
909
	}
910

911
	return count < q_vector->tx.work_limit;
912 913
}

914
#ifdef CONFIG_IXGBE_DCA
915
static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
916 917
				struct ixgbe_ring *rx_ring,
				int cpu)
918
{
919
	struct ixgbe_hw *hw = &adapter->hw;
920
	u32 rxctrl;
921 922 923 924 925 926 927 928 929
	u8 reg_idx = rx_ring->reg_idx;

	rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx));
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
		rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK;
		rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
		break;
	case ixgbe_mac_82599EB:
930
	case ixgbe_mac_X540:
931 932 933 934 935 936
		rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599;
		rxctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) <<
			   IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599);
		break;
	default:
		break;
937
	}
938 939 940 941
	rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN;
	rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN;
	rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN);
	IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
942 943 944
}

static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
945 946
				struct ixgbe_ring *tx_ring,
				int cpu)
947
{
948
	struct ixgbe_hw *hw = &adapter->hw;
949
	u32 txctrl;
950 951 952 953 954 955 956 957 958 959 960
	u8 reg_idx = tx_ring->reg_idx;

	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
		txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx));
		txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK;
		txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
		txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
		IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl);
		break;
	case ixgbe_mac_82599EB:
961
	case ixgbe_mac_X540:
962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
		txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx));
		txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599;
		txctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) <<
			   IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599);
		txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
		IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl);
		break;
	default:
		break;
	}
}

static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
{
	struct ixgbe_adapter *adapter = q_vector->adapter;
977
	int cpu = get_cpu();
978 979
	long r_idx;
	int i;
980

981 982 983
	if (q_vector->cpu == cpu)
		goto out_no_update;

984 985
	r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
	for (i = 0; i < q_vector->tx.count; i++) {
986
		ixgbe_update_tx_dca(adapter, adapter->tx_ring[r_idx], cpu);
987
		r_idx = find_next_bit(q_vector->tx.idx, adapter->num_tx_queues,
988
				      r_idx + 1);
989
	}
990

991 992
	r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
	for (i = 0; i < q_vector->rx.count; i++) {
993
		ixgbe_update_rx_dca(adapter, adapter->rx_ring[r_idx], cpu);
994
		r_idx = find_next_bit(q_vector->rx.idx, adapter->num_rx_queues,
995 996 997 998 999
				      r_idx + 1);
	}

	q_vector->cpu = cpu;
out_no_update:
1000 1001 1002 1003 1004
	put_cpu();
}

static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
{
1005
	int num_q_vectors;
1006 1007 1008 1009 1010
	int i;

	if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
		return;

1011 1012 1013
	/* always use CB2 mode, difference is masked in the CB driver */
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);

1014 1015 1016 1017 1018 1019 1020 1021
	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
		num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
	else
		num_q_vectors = 1;

	for (i = 0; i < num_q_vectors; i++) {
		adapter->q_vector[i]->cpu = -1;
		ixgbe_update_dca(adapter->q_vector[i]);
1022 1023 1024 1025 1026
	}
}

static int __ixgbe_notify_dca(struct device *dev, void *data)
{
1027
	struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
1028 1029
	unsigned long event = *(unsigned long *)data;

1030
	if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
1031 1032
		return 0;

1033 1034
	switch (event) {
	case DCA_PROVIDER_ADD:
1035 1036 1037
		/* if we're already enabled, don't do it again */
		if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
			break;
1038
		if (dca_add_requester(dev) == 0) {
1039
			adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
			ixgbe_setup_dca(adapter);
			break;
		}
		/* Fall Through since DCA is disabled. */
	case DCA_PROVIDER_REMOVE:
		if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
			dca_remove_requester(dev);
			adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
			IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
		}
		break;
	}

1053
	return 0;
1054
}
1055
#endif /* CONFIG_IXGBE_DCA */
Emil Tantilov's avatar
Emil Tantilov committed
1056 1057 1058 1059 1060 1061 1062

static inline void ixgbe_rx_hash(union ixgbe_adv_rx_desc *rx_desc,
				 struct sk_buff *skb)
{
	skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
}

1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080
/**
 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
 * @adapter: address of board private structure
 * @rx_desc: advanced rx descriptor
 *
 * Returns : true if it is FCoE pkt
 */
static inline bool ixgbe_rx_is_fcoe(struct ixgbe_adapter *adapter,
				    union ixgbe_adv_rx_desc *rx_desc)
{
	__le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;

	return (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
	       ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
		(cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
			     IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
}

1081 1082 1083 1084
/**
 * ixgbe_receive_skb - Send a completed packet up the stack
 * @adapter: board private structure
 * @skb: packet to send up
1085 1086 1087
 * @status: hardware indication of status of receive
 * @rx_ring: rx descriptor ring (for a specific queue) to setup
 * @rx_desc: rx descriptor
1088
 **/
Herbert Xu's avatar
Herbert Xu committed
1089
static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector,
1090 1091 1092
			      struct sk_buff *skb, u8 status,
			      struct ixgbe_ring *ring,
			      union ixgbe_adv_rx_desc *rx_desc)
1093
{
Herbert Xu's avatar
Herbert Xu committed
1094 1095
	struct ixgbe_adapter *adapter = q_vector->adapter;
	struct napi_struct *napi = &q_vector->napi;
1096 1097
	bool is_vlan = (status & IXGBE_RXD_STAT_VP);
	u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
1098

1099 1100 1101 1102 1103 1104 1105
	if (is_vlan && (tag & VLAN_VID_MASK))
		__vlan_hwaccel_put_tag(skb, tag);

	if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
		napi_gro_receive(napi, skb);
	else
		netif_rx(skb);
1106 1107
}

1108 1109 1110 1111 1112
/**
 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
 * @adapter: address of board private structure
 * @status_err: hardware indication of status of receive
 * @skb: skb currently being received and modified
1113
 * @status_err: status error value of last descriptor in packet
1114
 **/
1115
static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter,
1116
				     union ixgbe_adv_rx_desc *rx_desc,
1117 1118
				     struct sk_buff *skb,
				     u32 status_err)
1119
{
1120
	skb->ip_summed = CHECKSUM_NONE;
1121

1122 1123
	/* Rx csum disabled */
	if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
1124
		return;
1125 1126 1127 1128

	/* if IP and error */
	if ((status_err & IXGBE_RXD_STAT_IPCS) &&
	    (status_err & IXGBE_RXDADV_ERR_IPE)) {
1129 1130 1131
		adapter->hw_csum_rx_error++;
		return;
	}
1132 1133 1134 1135 1136

	if (!(status_err & IXGBE_RXD_STAT_L4CS))
		return;

	if (status_err & IXGBE_RXDADV_ERR_TCPE) {
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
		u16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;

		/*
		 * 82599 errata, UDP frames with a 0 checksum can be marked as
		 * checksum errors.
		 */
		if ((pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
		    (adapter->hw.mac.type == ixgbe_mac_82599EB))
			return;

1147 1148 1149 1150
		adapter->hw_csum_rx_error++;
		return;
	}

1151
	/* It must be a TCP or UDP packet with a valid checksum */
1152
	skb->ip_summed = CHECKSUM_UNNECESSARY;
1153 1154
}

1155
static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
1156 1157 1158 1159 1160 1161 1162 1163
{
	/*
	 * Force memory writes to complete before letting h/w
	 * know there are new descriptors to fetch.  (Only
	 * applicable for weak-ordered memory model archs,
	 * such as IA-64).
	 */
	wmb();
1164
	writel(val, rx_ring->tail);
1165 1166
}

1167 1168
/**
 * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split
1169 1170
 * @rx_ring: ring to place buffers on
 * @cleaned_count: number of buffers to replace
1171
 **/
1172
void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1173 1174
{
	union ixgbe_adv_rx_desc *rx_desc;
1175
	struct ixgbe_rx_buffer *bi;
1176 1177
	struct sk_buff *skb;
	u16 i = rx_ring->next_to_use;
1178

1179 1180 1181 1182
	/* do nothing if no valid netdev defined */
	if (!rx_ring->netdev)
		return;

1183
	while (cleaned_count--) {
1184
		rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1185 1186
		bi = &rx_ring->rx_buffer_info[i];
		skb = bi->skb;
1187

1188
		if (!skb) {
1189
			skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1190
							rx_ring->rx_buf_len);
1191
			if (!skb) {
1192
				rx_ring->rx_stats.alloc_rx_buff_failed++;
1193 1194
				goto no_buffers;
			}
1195 1196
			/* initialize queue mapping */
			skb_record_rx_queue(skb, rx_ring->queue_index);
1197
			bi->skb = skb;
1198
		}
1199

1200
		if (!bi->dma) {
1201
			bi->dma = dma_map_single(rx_ring->dev,
1202
						 skb->data,
1203
						 rx_ring->rx_buf_len,
1204
						 DMA_FROM_DEVICE);
1205
			if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1206
				rx_ring->rx_stats.alloc_rx_buff_failed++;
1207 1208 1209
				bi->dma = 0;
				goto no_buffers;
			}
1210
		}
1211

1212
		if (ring_is_ps_enabled(rx_ring)) {
1213
			if (!bi->page) {
1214
				bi->page = netdev_alloc_page(rx_ring->netdev);
1215
				if (!bi->page) {
1216
					rx_ring->rx_stats.alloc_rx_page_failed++;
1217 1218 1219 1220 1221 1222 1223
					goto no_buffers;
				}
			}

			if (!bi->page_dma) {
				/* use a half page if we're re-using */
				bi->page_offset ^= PAGE_SIZE / 2;
1224
				bi->page_dma = dma_map_page(rx_ring->dev,
1225 1226 1227 1228
							    bi->page,
							    bi->page_offset,
							    PAGE_SIZE / 2,
							    DMA_FROM_DEVICE);
1229
				if (dma_mapping_error(rx_ring->dev,
1230
						      bi->page_dma)) {
1231
					rx_ring->rx_stats.alloc_rx_page_failed++;
1232 1233 1234 1235 1236 1237 1238
					bi->page_dma = 0;
					goto no_buffers;
				}
			}

			/* Refresh the desc even if buffer_addrs didn't change
			 * because each write-back erases this info. */
1239 1240
			rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
			rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1241
		} else {
1242
			rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1243
			rx_desc->read.hdr_addr = 0;
1244 1245 1246 1247 1248 1249
		}

		i++;
		if (i == rx_ring->count)
			i = 0;
	}
1250

1251 1252 1253
no_buffers:
	if (rx_ring->next_to_use != i) {
		rx_ring->next_to_use = i;
1254
		ixgbe_release_rx_desc(rx_ring, i);
1255 1256 1257
	}
}

1258
static inline u16 ixgbe_get_hlen(union ixgbe_adv_rx_desc *rx_desc)
1259
{
1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
	/* HW will not DMA in data larger than the given buffer, even if it
	 * parses the (NFS, of course) header to be larger.  In that case, it
	 * fills the header buffer and spills the rest into the page.
	 */
	u16 hdr_info = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info);
	u16 hlen = (hdr_info &  IXGBE_RXDADV_HDRBUFLEN_MASK) >>
		    IXGBE_RXDADV_HDRBUFLEN_SHIFT;
	if (hlen > IXGBE_RX_HDR_SIZE)
		hlen = IXGBE_RX_HDR_SIZE;
	return hlen;
1270 1271
}

1272 1273 1274 1275 1276 1277 1278 1279
/**
 * ixgbe_transform_rsc_queue - change rsc queue into a full packet
 * @skb: pointer to the last skb in the rsc queue
 *
 * This function changes a queue full of hw rsc buffers into a completed
 * packet.  It uses the ->prev pointers to find the first packet and then
 * turns it into the frag list owner.
 **/
1280
static inline struct sk_buff *ixgbe_transform_rsc_queue(struct sk_buff *skb)
1281 1282
{
	unsigned int frag_list_size = 0;
1283
	unsigned int skb_cnt = 1;
1284 1285 1286 1287 1288 1289

	while (skb->prev) {
		struct sk_buff *prev = skb->prev;
		frag_list_size += skb->len;
		skb->prev = NULL;
		skb = prev;
1290
		skb_cnt++;
1291 1292 1293 1294 1295 1296 1297
	}

	skb_shinfo(skb)->frag_list = skb->next;
	skb->next = NULL;
	skb->len += frag_list_size;
	skb->data_len += frag_list_size;
	skb->truesize += frag_list_size;
1298 1299
	IXGBE_RSC_CB(skb)->skb_cnt = skb_cnt;

1300 1301 1302
	return skb;
}

1303 1304 1305 1306 1307
static inline bool ixgbe_get_rsc_state(union ixgbe_adv_rx_desc *rx_desc)
{
	return !!(le32_to_cpu(rx_desc->wb.lower.lo_dword.data) &
		IXGBE_RXDADV_RSCCNT_MASK);
}
1308

1309
static void ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
1310 1311
			       struct ixgbe_ring *rx_ring,
			       int *work_done, int work_to_do)
1312
{
Herbert Xu's avatar
Herbert Xu committed
1313
	struct ixgbe_adapter *adapter = q_vector->adapter;
1314 1315 1316
	union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
	struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer;
	struct sk_buff *skb;
1317
	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1318
	const int current_node = numa_node_id();
1319 1320 1321
#ifdef IXGBE_FCOE
	int ddp_bytes = 0;
#endif /* IXGBE_FCOE */
1322 1323 1324
	u32 staterr;
	u16 i;
	u16 cleaned_count = 0;
1325
	bool pkt_is_rsc = false;
1326 1327

	i = rx_ring->next_to_clean;
1328
	rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1329 1330 1331
	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);

	while (staterr & IXGBE_RXD_STAT_DD) {
1332
		u32 upper_len = 0;
1333

1334
		rmb(); /* read descriptor and rx_buffer_info after status DD */
1335

1336 1337
		rx_buffer_info = &rx_ring->rx_buffer_info[i];

1338 1339
		skb = rx_buffer_info->skb;
		rx_buffer_info->skb = NULL;
1340
		prefetch(skb->data);
1341

1342
		if (ring_is_rsc_enabled(rx_ring))
1343
			pkt_is_rsc = ixgbe_get_rsc_state(rx_desc);
1344 1345

		/* if this is a skb from previous receive DMA will be 0 */
1346
		if (rx_buffer_info->dma) {
1347
			u16 hlen;
1348
			if (pkt_is_rsc &&
1349 1350
			    !(staterr & IXGBE_RXD_STAT_EOP) &&
			    !skb->prev) {
1351 1352 1353 1354 1355 1356 1357
				/*
				 * When HWRSC is enabled, delay unmapping
				 * of the first packet. It carries the
				 * header information, HW may still
				 * access the header after the writeback.
				 * Only unmap it when EOP is reached
				 */
1358
				IXGBE_RSC_CB(skb)->delay_unmap = true;
1359
				IXGBE_RSC_CB(skb)->dma = rx_buffer_info->dma;
1360
			} else {
1361
				dma_unmap_single(rx_ring->dev,
1362 1363 1364
						 rx_buffer_info->dma,
						 rx_ring->rx_buf_len,
						 DMA_FROM_DEVICE);
1365
			}
1366
			rx_buffer_info->dma = 0;
1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378

			if (ring_is_ps_enabled(rx_ring)) {
				hlen = ixgbe_get_hlen(rx_desc);
				upper_len = le16_to_cpu(rx_desc->wb.upper.length);
			} else {
				hlen = le16_to_cpu(rx_desc->wb.upper.length);
			}

			skb_put(skb, hlen);
		} else {
			/* assume packet split since header is unmapped */
			upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1379 1380 1381
		}

		if (upper_len) {
1382 1383 1384 1385
			dma_unmap_page(rx_ring->dev,
				       rx_buffer_info->page_dma,
				       PAGE_SIZE / 2,
				       DMA_FROM_DEVICE);
1386 1387
			rx_buffer_info->page_dma = 0;
			skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1388 1389 1390
					   rx_buffer_info->page,
					   rx_buffer_info->page_offset,
					   upper_len);
1391

1392 1393
			if ((page_count(rx_buffer_info->page) == 1) &&
			    (page_to_nid(rx_buffer_info->page) == current_node))
1394
				get_page(rx_buffer_info->page);
1395 1396
			else
				rx_buffer_info->page = NULL;
1397 1398 1399 1400 1401 1402 1403 1404 1405 1406

			skb->len += upper_len;
			skb->data_len += upper_len;
			skb->truesize += upper_len;
		}

		i++;
		if (i == rx_ring->count)
			i = 0;

1407
		next_rxd = IXGBE_RX_DESC_ADV(rx_ring, i);
1408 1409
		prefetch(next_rxd);
		cleaned_count++;
1410

1411
		if (pkt_is_rsc) {
1412 1413 1414 1415 1416 1417 1418
			u32 nextp = (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
				     IXGBE_RXDADV_NEXTP_SHIFT;
			next_buffer = &rx_ring->rx_buffer_info[nextp];
		} else {
			next_buffer = &rx_ring->rx_buffer_info[i];
		}

1419
		if (!(staterr & IXGBE_RXD_STAT_EOP)) {
1420
			if (ring_is_ps_enabled(rx_ring)) {
1421 1422 1423 1424 1425 1426 1427 1428
				rx_buffer_info->skb = next_buffer->skb;
				rx_buffer_info->dma = next_buffer->dma;
				next_buffer->skb = skb;
				next_buffer->dma = 0;
			} else {
				skb->next = next_buffer->skb;
				skb->next->prev = skb;
			}
1429
			rx_ring->rx_stats.non_eop_descs++;
1430 1431 1432
			goto next_desc;
		}

1433 1434 1435 1436 1437 1438 1439 1440 1441
		if (skb->prev) {
			skb = ixgbe_transform_rsc_queue(skb);
			/* if we got here without RSC the packet is invalid */
			if (!pkt_is_rsc) {
				__pskb_trim(skb, 0);
				rx_buffer_info->skb = skb;
				goto next_desc;
			}
		}
1442 1443 1444 1445 1446 1447 1448 1449 1450 1451

		if (ring_is_rsc_enabled(rx_ring)) {
			if (IXGBE_RSC_CB(skb)->delay_unmap) {
				dma_unmap_single(rx_ring->dev,
						 IXGBE_RSC_CB(skb)->dma,
						 rx_ring->rx_buf_len,
						 DMA_FROM_DEVICE);
				IXGBE_RSC_CB(skb)->dma = 0;
				IXGBE_RSC_CB(skb)->delay_unmap = false;
			}
1452 1453
		}
		if (pkt_is_rsc) {
1454 1455
			if (ring_is_ps_enabled(rx_ring))
				rx_ring->rx_stats.rsc_count +=
1456
					skb_shinfo(skb)->nr_frags;
1457
			else
1458 1459
				rx_ring->rx_stats.rsc_count +=
					IXGBE_RSC_CB(skb)->skb_cnt;
1460 1461 1462 1463
			rx_ring->rx_stats.rsc_flush++;
		}

		/* ERR_MASK will only have valid bits if EOP set */
1464 1465
		if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
			dev_kfree_skb_any(skb);
1466 1467 1468
			goto next_desc;
		}

1469
		ixgbe_rx_checksum(adapter, rx_desc, skb, staterr);
Emil Tantilov's avatar
Emil Tantilov committed
1470 1471
		if (adapter->netdev->features & NETIF_F_RXHASH)
			ixgbe_rx_hash(rx_desc, skb);
1472 1473 1474 1475 1476

		/* probably a little skewed due to removing CRC */
		total_rx_bytes += skb->len;
		total_rx_packets++;

1477
		skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1478 1479
#ifdef IXGBE_FCOE
		/* if ddp, not passing to ULD unless for FCP_RSP or error */
1480 1481 1482
		if (ixgbe_rx_is_fcoe(adapter, rx_desc)) {
			ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb,
						   staterr);
1483 1484
			if (!ddp_bytes) {
				dev_kfree_skb_any(skb);
1485
				goto next_desc;
1486
			}
1487
		}
1488
#endif /* IXGBE_FCOE */
1489
		ixgbe_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
1490 1491 1492 1493

next_desc:
		rx_desc->wb.upper.status_error = 0;

1494 1495 1496 1497
		(*work_done)++;
		if (*work_done >= work_to_do)
			break;

1498 1499
		/* return some buffers to hardware, one at a time is too slow */
		if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
1500
			ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1501 1502 1503 1504 1505 1506
			cleaned_count = 0;
		}

		/* use prefetched values */
		rx_desc = next_rxd;
		staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1507 1508
	}

1509
	rx_ring->next_to_clean = i;
1510
	cleaned_count = ixgbe_desc_unused(rx_ring);
1511 1512

	if (cleaned_count)
1513
		ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1514

1515 1516 1517 1518 1519
#ifdef IXGBE_FCOE
	/* include DDPed FCoE data */
	if (ddp_bytes > 0) {
		unsigned int mss;

1520
		mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) -
1521 1522 1523 1524 1525 1526 1527 1528 1529
			sizeof(struct fc_frame_header) -
			sizeof(struct fcoe_crc_eof);
		if (mss > 512)
			mss &= ~511;
		total_rx_bytes += ddp_bytes;
		total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss);
	}
#endif /* IXGBE_FCOE */

1530 1531 1532 1533
	u64_stats_update_begin(&rx_ring->syncp);
	rx_ring->stats.packets += total_rx_packets;
	rx_ring->stats.bytes += total_rx_bytes;
	u64_stats_update_end(&rx_ring->syncp);
1534 1535
	q_vector->rx.total_packets += total_rx_packets;
	q_vector->rx.total_bytes += total_rx_bytes;
1536 1537
}

1538
static int ixgbe_clean_rxonly(struct napi_struct *, int);
1539 1540 1541 1542 1543 1544 1545 1546 1547
/**
 * ixgbe_configure_msix - Configure MSI-X hardware
 * @adapter: board private structure
 *
 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
 * interrupts.
 **/
static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
{
1548
	struct ixgbe_q_vector *q_vector;
1549
	int i, q_vectors, v_idx, r_idx;
1550
	u32 mask;
1551

1552
	q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1553

1554 1555
	/*
	 * Populate the IVAR table and set the ITR values to the
1556 1557 1558
	 * corresponding register.
	 */
	for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1559
		q_vector = adapter->q_vector[v_idx];
1560
		/* XXX for_each_set_bit(...) */
1561
		r_idx = find_first_bit(q_vector->rx.idx,
1562
				       adapter->num_rx_queues);
1563

1564
		for (i = 0; i < q_vector->rx.count; i++) {
1565 1566
			u8 reg_idx = adapter->rx_ring[r_idx]->reg_idx;
			ixgbe_set_ivar(adapter, 0, reg_idx, v_idx);
1567
			r_idx = find_next_bit(q_vector->rx.idx,
1568 1569
					      adapter->num_rx_queues,
					      r_idx + 1);
1570
		}
1571
		r_idx = find_first_bit(q_vector->tx.idx,
1572
				       adapter->num_tx_queues);
1573

1574
		for (i = 0; i < q_vector->tx.count; i++) {
1575 1576
			u8 reg_idx = adapter->tx_ring[r_idx]->reg_idx;
			ixgbe_set_ivar(adapter, 1, reg_idx, v_idx);
1577
			r_idx = find_next_bit(q_vector->tx.idx,
1578 1579
					      adapter->num_tx_queues,
					      r_idx + 1);
1580 1581
		}

1582
		if (q_vector->tx.count && !q_vector->rx.count)
1583 1584
			/* tx only */
			q_vector->eitr = adapter->tx_eitr_param;
1585
		else if (q_vector->rx.count)
1586 1587
			/* rx or mixed */
			q_vector->eitr = adapter->rx_eitr_param;
1588

1589
		ixgbe_write_eitr(q_vector);
1590 1591
		/* If ATR is enabled, set interrupt affinity */
		if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
			/*
			 * Allocate the affinity_hint cpumask, assign the mask
			 * for this vector, and set our affinity_hint for
			 * this irq.
			 */
			if (!alloc_cpumask_var(&q_vector->affinity_mask,
			                       GFP_KERNEL))
				return;
			cpumask_set_cpu(v_idx, q_vector->affinity_mask);
			irq_set_affinity_hint(adapter->msix_entries[v_idx].vector,
			                      q_vector->affinity_mask);
		}
1604 1605
	}

1606 1607
	switch (adapter->hw.mac.type) {
	case ixgbe_mac_82598EB:
1608
		ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
1609
			       v_idx);
1610 1611
		break;
	case ixgbe_mac_82599EB:
1612
	case ixgbe_mac_X540:
1613
		ixgbe_set_ivar(adapter, -1, 1, v_idx);
1614 1615 1616 1617 1618
		break;

	default:
		break;
	}
1619 1620
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);

1621
	/* set up to autoclear timer, and the vectors */
1622
	mask = IXGBE_EIMS_ENABLE_MASK;
1623 1624 1625 1626 1627 1628
	if (adapter->num_vfs)
		mask &= ~(IXGBE_EIMS_OTHER |
			  IXGBE_EIMS_MAILBOX |
			  IXGBE_EIMS_LSC);
	else
		mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
1629
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
1630 1631
}

1632 1633 1634 1635 1636 1637 1638 1639 1640
enum latency_range {
	lowest_latency = 0,
	low_latency = 1,
	bulk_latency = 2,
	latency_invalid = 255
};

/**
 * ixgbe_update_itr - update the dynamic ITR value based on statistics
1641 1642
 * @q_vector: structure containing interrupt and ring information
 * @ring_container: structure containing ring performance data
1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653
 *
 *      Stores a new ITR value based on packets and byte
 *      counts during the last interrupt.  The advantage of per interrupt
 *      computation is faster updates and more accurate ITR for the current
 *      traffic pattern.  Constants in this function were computed
 *      based on theoretical maximum wire speed and thresholds were set based
 *      on testing data as well as attempting to minimize response time
 *      while increasing bulk throughput.
 *      this functionality is controlled by the InterruptThrottleRate module
 *      parameter (see ixgbe_param.c)
 **/
1654 1655
static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
			     struct ixgbe_ring_container *ring_container)
1656 1657
{
	u64 bytes_perint;
1658 1659 1660 1661 1662
	struct ixgbe_adapter *adapter = q_vector->adapter;
	int bytes = ring_container->total_bytes;
	int packets = ring_container->total_packets;
	u32 timepassed_us;
	u8 itr_setting = ring_container->itr;
1663 1664

	if (packets == 0)
1665
		return;
1666 1667 1668 1669 1670 1671 1672

	/* simple throttlerate management
	 *    0-20MB/s lowest (100000 ints/s)
	 *   20-100MB/s low   (20000 ints/s)
	 *  100-1249MB/s bulk (8000 ints/s)
	 */
	/* what was last interrupt timeslice? */
1673
	timepassed_us = 1000000/q_vector->eitr;
1674 1675 1676 1677 1678
	bytes_perint = bytes / timepassed_us; /* bytes/usec */

	switch (itr_setting) {
	case lowest_latency:
		if (bytes_perint > adapter->eitr_low)
1679
			itr_setting = low_latency;
1680 1681 1682
		break;
	case low_latency:
		if (bytes_perint > adapter->eitr_high)
1683
			itr_setting = bulk_latency;
1684
		else if (bytes_perint <= adapter->eitr_low)
1685
			itr_setting = lowest_latency;
1686 1687 1688
		break;
	case bulk_latency:
		if (bytes_perint <= adapter->eitr_high)
1689
			itr_setting = low_latency;
1690 1691 1692
		break;
	}

1693 1694 1695 1696 1697 1698
	/* clear work counters since we have the values we need */
	ring_container->total_bytes = 0;
	ring_container->total_packets = 0;

	/* write updated itr to ring container */
	ring_container->itr = itr_setting;
1699 1700
}

1701 1702
/**
 * ixgbe_write_eitr - write EITR register in hardware specific way
1703
 * @q_vector: structure containing interrupt and ring information
1704 1705 1706 1707 1708
 *
 * This function is made to be called by ethtool and by the driver
 * when it needs to update EITR registers at runtime.  Hardware
 * specific quirks/differences are taken care of here.
 */
1709
void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
1710
{
1711
	struct ixgbe_adapter *adapter = q_vector->adapter;
1712
	struct ixgbe_hw *hw = &adapter->hw;
1713 1714 1715
	int v_idx = q_vector->v_idx;
	u32 itr_reg = EITR_INTS_PER_SEC_TO_REG(q_vector->eitr);

1716 1717
	switch (adapter->hw.mac.type) {
	case ixgbe_mac_82598EB:
1718 1719
		/* must write high and low 16 bits to reset counter */
		itr_reg |= (itr_reg << 16);
1720 1721
		break;
	case ixgbe_mac_82599EB:
1722
	case ixgbe_mac_X540:
1723
		/*
1724
		 * 82599 and X540 can support a value of zero, so allow it for
1725 1726 1727 1728 1729 1730 1731
		 * max interrupt rate, but there is an errata where it can
		 * not be zero with RSC
		 */
		if (itr_reg == 8 &&
		    !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))
			itr_reg = 0;

1732 1733 1734 1735 1736
		/*
		 * set the WDIS bit to not clear the timer bits and cause an
		 * immediate assertion of the interrupt
		 */
		itr_reg |= IXGBE_EITR_CNT_WDIS;
1737 1738 1739
		break;
	default:
		break;
1740 1741 1742 1743
	}
	IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
}

1744
static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
1745
{
1746 1747
	u32 new_itr = q_vector->eitr;
	u8 current_itr;
1748

1749 1750
	ixgbe_update_itr(q_vector, &q_vector->tx);
	ixgbe_update_itr(q_vector, &q_vector->rx);
1751

1752
	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764

	switch (current_itr) {
	/* counts and packets in update_itr are dependent on these numbers */
	case lowest_latency:
		new_itr = 100000;
		break;
	case low_latency:
		new_itr = 20000; /* aka hwitr = ~200 */
		break;
	case bulk_latency:
		new_itr = 8000;
		break;
1765 1766
	default:
		break;
1767 1768 1769
	}

	if (new_itr != q_vector->eitr) {
1770
		/* do an exponential smoothing */
1771
		new_itr = ((q_vector->eitr * 9) + new_itr)/10;
1772

1773
		/* save the algorithm value here */
1774
		q_vector->eitr = new_itr;
1775 1776

		ixgbe_write_eitr(q_vector);
1777 1778 1779
	}
}

1780
/**
1781 1782
 * ixgbe_check_overtemp_subtask - check for over tempurature
 * @adapter: pointer to adapter
1783
 **/
1784
static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
1785 1786 1787 1788
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 eicr = adapter->interrupt_event;

1789
	if (test_bit(__IXGBE_DOWN, &adapter->state))
1790 1791
		return;

1792 1793 1794 1795 1796 1797
	if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
	    !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
		return;

	adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;

1798
	switch (hw->device_id) {
1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813
	case IXGBE_DEV_ID_82599_T3_LOM:
		/*
		 * Since the warning interrupt is for both ports
		 * we don't have to check if:
		 *  - This interrupt wasn't for our port.
		 *  - We may have missed the interrupt so always have to
		 *    check if we  got a LSC
		 */
		if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
		    !(eicr & IXGBE_EICR_LSC))
			return;

		if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
			u32 autoneg;
			bool link_up = false;
1814 1815 1816

			hw->mac.ops.check_link(hw, &autoneg, &link_up, false);

1817 1818 1819 1820 1821 1822 1823 1824 1825
			if (link_up)
				return;
		}

		/* Check if this is not due to overtemp */
		if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
			return;

		break;
1826 1827
	default:
		if (!(eicr & IXGBE_EICR_GPI_SDP0))
1828
			return;
1829
		break;
1830
	}
1831 1832 1833 1834
	e_crit(drv,
	       "Network adapter has been stopped because it has over heated. "
	       "Restart the computer. If the problem persists, "
	       "power off the system and replace the adapter\n");
1835 1836

	adapter->interrupt_event = 0;
1837 1838
}

1839 1840 1841 1842 1843 1844
static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
{
	struct ixgbe_hw *hw = &adapter->hw;

	if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
	    (eicr & IXGBE_EICR_GPI_SDP1)) {
1845
		e_crit(probe, "Fan has stopped, replace the adapter\n");
1846 1847 1848 1849
		/* write to clear the interrupt */
		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
	}
}
1850

1851 1852 1853 1854
static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
{
	struct ixgbe_hw *hw = &adapter->hw;

1855 1856 1857
	if (eicr & IXGBE_EICR_GPI_SDP2) {
		/* Clear the interrupt */
		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1858 1859 1860 1861
		if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
			adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
			ixgbe_service_event_schedule(adapter);
		}
1862 1863
	}

1864 1865 1866
	if (eicr & IXGBE_EICR_GPI_SDP1) {
		/* Clear the interrupt */
		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1867 1868 1869 1870
		if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
			adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
			ixgbe_service_event_schedule(adapter);
		}
1871 1872 1873
	}
}

1874 1875 1876 1877 1878 1879 1880 1881 1882
static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;

	adapter->lsc_int++;
	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
	adapter->link_check_timeout = jiffies;
	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
		IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
1883
		IXGBE_WRITE_FLUSH(hw);
1884
		ixgbe_service_event_schedule(adapter);
1885 1886 1887
	}
}

1888 1889
static irqreturn_t ixgbe_msix_lsc(int irq, void *data)
{
1890
	struct ixgbe_adapter *adapter = data;
1891
	struct ixgbe_hw *hw = &adapter->hw;
1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
	u32 eicr;

	/*
	 * Workaround for Silicon errata.  Use clear-by-write instead
	 * of clear-by-read.  Reading with EICS will return the
	 * interrupt causes without clearing, which later be done
	 * with the write to EICR.
	 */
	eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
	IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
1902

1903 1904
	if (eicr & IXGBE_EICR_LSC)
		ixgbe_check_lsc(adapter);
1905

1906 1907 1908
	if (eicr & IXGBE_EICR_MAILBOX)
		ixgbe_msg_task(adapter);

1909 1910
	switch (hw->mac.type) {
	case ixgbe_mac_82599EB:
1911
	case ixgbe_mac_X540:
1912 1913
		/* Handle Flow Director Full threshold interrupt */
		if (eicr & IXGBE_EICR_FLOW_DIR) {
1914
			int reinit_count = 0;
1915 1916
			int i;
			for (i = 0; i < adapter->num_tx_queues; i++) {
1917
				struct ixgbe_ring *ring = adapter->tx_ring[i];
1918
				if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
1919 1920 1921 1922 1923 1924 1925 1926 1927
						       &ring->state))
					reinit_count++;
			}
			if (reinit_count) {
				/* no more flow director interrupts until after init */
				IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
				eicr &= ~IXGBE_EICR_FLOW_DIR;
				adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
				ixgbe_service_event_schedule(adapter);
1928 1929
			}
		}
1930 1931 1932 1933 1934 1935 1936
		ixgbe_check_sfp_event(adapter, eicr);
		if ((adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
		    ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
			if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
				adapter->interrupt_event = eicr;
				adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
				ixgbe_service_event_schedule(adapter);
1937 1938
			}
		}
1939 1940 1941
		break;
	default:
		break;
1942
	}
1943 1944 1945

	ixgbe_check_fan_failure(adapter, eicr);

1946
	/* re-enable the original interrupt state, no lsc, no queues */
1947
	if (!test_bit(__IXGBE_DOWN, &adapter->state))
1948 1949
		IXGBE_WRITE_REG(hw, IXGBE_EIMS, eicr &
		                ~(IXGBE_EIMS_LSC | IXGBE_EIMS_RTX_QUEUE));
1950 1951 1952 1953

	return IRQ_HANDLED;
}

1954 1955 1956 1957
static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
					   u64 qmask)
{
	u32 mask;
1958
	struct ixgbe_hw *hw = &adapter->hw;
1959

1960 1961
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
1962
		mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1963 1964 1965
		IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
		break;
	case ixgbe_mac_82599EB:
1966
	case ixgbe_mac_X540:
1967
		mask = (qmask & 0xFFFFFFFF);
1968 1969
		if (mask)
			IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1970
		mask = (qmask >> 32);
1971 1972 1973 1974 1975
		if (mask)
			IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
		break;
	default:
		break;
1976 1977 1978 1979 1980
	}
	/* skip the flush */
}

static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
1981
					    u64 qmask)
1982 1983
{
	u32 mask;
1984
	struct ixgbe_hw *hw = &adapter->hw;
1985

1986 1987
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
1988
		mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1989 1990 1991
		IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
		break;
	case ixgbe_mac_82599EB:
1992
	case ixgbe_mac_X540:
1993
		mask = (qmask & 0xFFFFFFFF);
1994 1995
		if (mask)
			IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1996
		mask = (qmask >> 32);
1997 1998 1999 2000 2001
		if (mask)
			IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
		break;
	default:
		break;
2002 2003 2004 2005
	}
	/* skip the flush */
}

2006 2007
static irqreturn_t ixgbe_msix_clean_tx(int irq, void *data)
{
2008 2009
	struct ixgbe_q_vector *q_vector = data;
	struct ixgbe_adapter  *adapter = q_vector->adapter;
2010
	struct ixgbe_ring     *tx_ring;
2011 2012
	int i, r_idx;

2013
	if (!q_vector->tx.count)
2014 2015
		return IRQ_HANDLED;

2016 2017
	r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
	for (i = 0; i < q_vector->tx.count; i++) {
2018
		tx_ring = adapter->tx_ring[r_idx];
2019
		r_idx = find_next_bit(q_vector->tx.idx, adapter->num_tx_queues,
2020
				      r_idx + 1);
2021
	}
2022

2023
	/* EIAM disabled interrupts (on this vector) for us */
2024 2025
	napi_schedule(&q_vector->napi);

2026 2027 2028
	return IRQ_HANDLED;
}

2029 2030 2031 2032 2033
/**
 * ixgbe_msix_clean_rx - single unshared vector rx clean (all queues)
 * @irq: unused
 * @data: pointer to our q_vector struct for this interrupt vector
 **/
2034 2035
static irqreturn_t ixgbe_msix_clean_rx(int irq, void *data)
{
2036 2037
	struct ixgbe_q_vector *q_vector = data;
	struct ixgbe_adapter  *adapter = q_vector->adapter;
2038
	struct ixgbe_ring  *rx_ring;
2039
	int r_idx;
2040
	int i;
2041

2042 2043 2044 2045 2046
#ifdef CONFIG_IXGBE_DCA
	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
		ixgbe_update_dca(q_vector);
#endif

2047 2048
	r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
	for (i = 0; i < q_vector->rx.count; i++) {
2049
		rx_ring = adapter->rx_ring[r_idx];
2050
		r_idx = find_next_bit(q_vector->rx.idx, adapter->num_rx_queues,
2051
				      r_idx + 1);
2052 2053
	}

2054
	if (!q_vector->rx.count)
2055 2056
		return IRQ_HANDLED;

2057
	/* EIAM disabled interrupts (on this vector) for us */
2058
	napi_schedule(&q_vector->napi);
2059 2060 2061 2062 2063 2064

	return IRQ_HANDLED;
}

static irqreturn_t ixgbe_msix_clean_many(int irq, void *data)
{
2065 2066 2067 2068 2069 2070
	struct ixgbe_q_vector *q_vector = data;
	struct ixgbe_adapter  *adapter = q_vector->adapter;
	struct ixgbe_ring  *ring;
	int r_idx;
	int i;

2071
	if (!q_vector->tx.count && !q_vector->rx.count)
2072 2073
		return IRQ_HANDLED;

2074 2075
	r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
	for (i = 0; i < q_vector->tx.count; i++) {
2076
		ring = adapter->tx_ring[r_idx];
2077
		r_idx = find_next_bit(q_vector->tx.idx, adapter->num_tx_queues,
2078
				      r_idx + 1);
2079 2080
	}

2081 2082
	r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
	for (i = 0; i < q_vector->rx.count; i++) {
2083
		ring = adapter->rx_ring[r_idx];
2084
		r_idx = find_next_bit(q_vector->rx.idx, adapter->num_rx_queues,
2085
				      r_idx + 1);
2086 2087
	}

2088
	/* EIAM disabled interrupts (on this vector) for us */
2089
	napi_schedule(&q_vector->napi);
2090 2091 2092 2093

	return IRQ_HANDLED;
}

2094 2095 2096 2097 2098
/**
 * ixgbe_clean_rxonly - msix (aka one shot) rx clean routine
 * @napi: napi struct with our devices info in it
 * @budget: amount of work driver is allowed to do this pass, in packets
 *
2099 2100
 * This function is optimized for cleaning one queue only on a single
 * q_vector!!!
2101
 **/
2102 2103
static int ixgbe_clean_rxonly(struct napi_struct *napi, int budget)
{
2104
	struct ixgbe_q_vector *q_vector =
2105
			       container_of(napi, struct ixgbe_q_vector, napi);
2106
	struct ixgbe_adapter *adapter = q_vector->adapter;
2107
	struct ixgbe_ring *rx_ring = NULL;
2108
	int work_done = 0;
2109
	long r_idx;
2110

2111
#ifdef CONFIG_IXGBE_DCA
2112
	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2113
		ixgbe_update_dca(q_vector);
2114
#endif
2115

2116
	r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
2117 2118
	rx_ring = adapter->rx_ring[r_idx];

Herbert Xu's avatar
Herbert Xu committed
2119
	ixgbe_clean_rx_irq(q_vector, rx_ring, &work_done, budget);
2120

2121 2122
	/* If all Rx work done, exit the polling mode */
	if (work_done < budget) {
2123
		napi_complete(napi);
2124
		if (adapter->rx_itr_setting & 1)
2125
			ixgbe_set_itr(q_vector);
2126
		if (!test_bit(__IXGBE_DOWN, &adapter->state))
2127
			ixgbe_irq_enable_queues(adapter,
2128
						((u64)1 << q_vector->v_idx));
2129 2130 2131 2132 2133
	}

	return work_done;
}

2134
/**
2135
 * ixgbe_clean_rxtx_many - msix (aka one shot) rx clean routine
2136 2137 2138 2139 2140 2141
 * @napi: napi struct with our devices info in it
 * @budget: amount of work driver is allowed to do this pass, in packets
 *
 * This function will clean more than one rx queue associated with a
 * q_vector.
 **/
2142
static int ixgbe_clean_rxtx_many(struct napi_struct *napi, int budget)
2143 2144
{
	struct ixgbe_q_vector *q_vector =
2145
			       container_of(napi, struct ixgbe_q_vector, napi);
2146
	struct ixgbe_adapter *adapter = q_vector->adapter;
2147
	struct ixgbe_ring *ring = NULL;
2148 2149
	int work_done = 0, i;
	long r_idx;
2150 2151
	bool tx_clean_complete = true;

2152 2153 2154 2155 2156
#ifdef CONFIG_IXGBE_DCA
	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
		ixgbe_update_dca(q_vector);
#endif

2157 2158
	r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
	for (i = 0; i < q_vector->tx.count; i++) {
2159
		ring = adapter->tx_ring[r_idx];
2160
		tx_clean_complete &= ixgbe_clean_tx_irq(q_vector, ring);
2161
		r_idx = find_next_bit(q_vector->tx.idx, adapter->num_tx_queues,
2162
				      r_idx + 1);
2163
	}
2164 2165 2166

	/* attempt to distribute budget to each queue fairly, but don't allow
	 * the budget to go below 1 because we'll exit polling */
2167
	budget /= (q_vector->rx.count ?: 1);
2168
	budget = max(budget, 1);
2169 2170
	r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
	for (i = 0; i < q_vector->rx.count; i++) {
2171
		ring = adapter->rx_ring[r_idx];
2172
		ixgbe_clean_rx_irq(q_vector, ring, &work_done, budget);
2173
		r_idx = find_next_bit(q_vector->rx.idx, adapter->num_rx_queues,
2174
				      r_idx + 1);
2175 2176
	}

2177
	r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
2178
	ring = adapter->rx_ring[r_idx];
2179
	/* If all Rx work done, exit the polling mode */
2180
	if (work_done < budget) {
2181
		napi_complete(napi);
2182
		if (adapter->rx_itr_setting & 1)
2183
			ixgbe_set_itr(q_vector);
2184
		if (!test_bit(__IXGBE_DOWN, &adapter->state))
2185
			ixgbe_irq_enable_queues(adapter,
2186
						((u64)1 << q_vector->v_idx));
2187 2188 2189 2190 2191
		return 0;
	}

	return work_done;
}
2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203

/**
 * ixgbe_clean_txonly - msix (aka one shot) tx clean routine
 * @napi: napi struct with our devices info in it
 * @budget: amount of work driver is allowed to do this pass, in packets
 *
 * This function is optimized for cleaning one queue only on a single
 * q_vector!!!
 **/
static int ixgbe_clean_txonly(struct napi_struct *napi, int budget)
{
	struct ixgbe_q_vector *q_vector =
2204
			       container_of(napi, struct ixgbe_q_vector, napi);
2205 2206 2207 2208 2209 2210 2211
	struct ixgbe_adapter *adapter = q_vector->adapter;
	struct ixgbe_ring *tx_ring = NULL;
	int work_done = 0;
	long r_idx;

#ifdef CONFIG_IXGBE_DCA
	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2212
		ixgbe_update_dca(q_vector);
2213 2214
#endif

2215
	r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
2216 2217
	tx_ring = adapter->tx_ring[r_idx];

2218 2219 2220
	if (!ixgbe_clean_tx_irq(q_vector, tx_ring))
		work_done = budget;

2221
	/* If all Tx work done, exit the polling mode */
2222 2223
	if (work_done < budget) {
		napi_complete(napi);
2224
		if (adapter->tx_itr_setting & 1)
2225
			ixgbe_set_itr(q_vector);
2226
		if (!test_bit(__IXGBE_DOWN, &adapter->state))
2227 2228
			ixgbe_irq_enable_queues(adapter,
						((u64)1 << q_vector->v_idx));
2229 2230 2231 2232 2233
	}

	return work_done;
}

2234
static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx,
2235
				     int r_idx)
2236
{
2237
	struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2238
	struct ixgbe_ring *rx_ring = a->rx_ring[r_idx];
2239

2240 2241
	set_bit(r_idx, q_vector->rx.idx);
	q_vector->rx.count++;
2242
	rx_ring->q_vector = q_vector;
2243 2244 2245
}

static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx,
2246
				     int t_idx)
2247
{
2248
	struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2249
	struct ixgbe_ring *tx_ring = a->tx_ring[t_idx];
2250

2251 2252
	set_bit(t_idx, q_vector->tx.idx);
	q_vector->tx.count++;
2253
	tx_ring->q_vector = q_vector;
2254
	q_vector->tx.work_limit = a->tx_work_limit;
2255 2256
}

2257
/**
2258 2259
 * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors
 * @adapter: board private structure to initialize
2260
 *
2261 2262 2263 2264 2265
 * This function maps descriptor rings to the queue-specific vectors
 * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 * one vector per ring/queue, but on a constrained vector budget, we
 * group the rings as "efficiently" as possible.  You would add new
 * mapping configurations in here.
2266
 **/
2267
static int ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter)
2268
{
2269
	int q_vectors;
2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280
	int v_start = 0;
	int rxr_idx = 0, txr_idx = 0;
	int rxr_remaining = adapter->num_rx_queues;
	int txr_remaining = adapter->num_tx_queues;
	int i, j;
	int rqpv, tqpv;
	int err = 0;

	/* No mapping required if MSI-X is disabled. */
	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
		goto out;
2281

2282 2283
	q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;

2284 2285 2286 2287
	/*
	 * The ideal configuration...
	 * We have enough vectors to map one per queue.
	 */
2288
	if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
2289 2290
		for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
			map_vector_to_rxq(adapter, v_start, rxr_idx);
2291

2292 2293
		for (; txr_idx < txr_remaining; v_start++, txr_idx++)
			map_vector_to_txq(adapter, v_start, txr_idx);
2294 2295

		goto out;
2296
	}
2297

2298 2299 2300 2301 2302 2303
	/*
	 * If we don't have enough vectors for a 1-to-1
	 * mapping, we'll have to group them so there are
	 * multiple queues per vector.
	 */
	/* Re-adjusting *qpv takes care of the remainder. */
2304 2305
	for (i = v_start; i < q_vectors; i++) {
		rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
2306 2307 2308 2309 2310
		for (j = 0; j < rqpv; j++) {
			map_vector_to_rxq(adapter, i, rxr_idx);
			rxr_idx++;
			rxr_remaining--;
		}
2311
		tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
2312 2313 2314 2315
		for (j = 0; j < tqpv; j++) {
			map_vector_to_txq(adapter, i, txr_idx);
			txr_idx++;
			txr_remaining--;
2316 2317
		}
	}
2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333
out:
	return err;
}

/**
 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
 * @adapter: board private structure
 *
 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
 * interrupts from the kernel.
 **/
static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	irqreturn_t (*handler)(int, void *);
	int i, vector, q_vectors, err;
2334
	int ri = 0, ti = 0;
2335 2336 2337 2338

	/* Decrement for Other and TCP Timer vectors */
	q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;

2339
	err = ixgbe_map_rings_to_vectors(adapter);
2340
	if (err)
2341
		return err;
2342

2343
#define SET_HANDLER(_v) (((_v)->rx.count && (_v)->tx.count)        \
2344
					  ? &ixgbe_msix_clean_many : \
2345 2346
			  (_v)->rx.count ? &ixgbe_msix_clean_rx   : \
			  (_v)->tx.count ? &ixgbe_msix_clean_tx   : \
2347
			  NULL)
2348
	for (vector = 0; vector < q_vectors; vector++) {
2349 2350
		struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
		handler = SET_HANDLER(q_vector);
2351

2352
		if (handler == &ixgbe_msix_clean_rx) {
2353 2354
			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
			         "%s-%s-%d", netdev->name, "rx", ri++);
2355
		} else if (handler == &ixgbe_msix_clean_tx) {
2356 2357
			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
			         "%s-%s-%d", netdev->name, "tx", ti++);
2358
		} else if (handler == &ixgbe_msix_clean_many) {
2359 2360
			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
			         "%s-%s-%d", netdev->name, "TxRx", ri++);
2361
			ti++;
2362 2363 2364
		} else {
			/* skip this unused q_vector */
			continue;
2365
		}
2366
		err = request_irq(adapter->msix_entries[vector].vector,
2367 2368
				  handler, 0, q_vector->name,
				  q_vector);
2369
		if (err) {
2370
			e_err(probe, "request_irq failed for MSIX interrupt "
2371
			      "Error: %d\n", err);
2372
			goto free_queue_irqs;
2373 2374 2375
		}
	}

2376
	sprintf(adapter->lsc_int_name, "%s:lsc", netdev->name);
2377
	err = request_irq(adapter->msix_entries[vector].vector,
2378
			  ixgbe_msix_lsc, 0, adapter->lsc_int_name, adapter);
2379
	if (err) {
2380
		e_err(probe, "request_irq for msix_lsc failed: %d\n", err);
2381
		goto free_queue_irqs;
2382 2383 2384 2385
	}

	return 0;

2386 2387 2388
free_queue_irqs:
	for (i = vector - 1; i >= 0; i--)
		free_irq(adapter->msix_entries[--vector].vector,
2389
			 adapter->q_vector[i]);
2390 2391
	adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
	pci_disable_msix(adapter->pdev);
2392 2393 2394 2395 2396
	kfree(adapter->msix_entries);
	adapter->msix_entries = NULL;
	return err;
}

2397 2398 2399 2400
/**
 * ixgbe_irq_enable - Enable default interrupt generation settings
 * @adapter: board private structure
 **/
2401 2402
static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
				    bool flush)
2403 2404
{
	u32 mask;
2405 2406

	mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
2407 2408
	if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
		mask |= IXGBE_EIMS_GPI_SDP0;
2409 2410
	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
		mask |= IXGBE_EIMS_GPI_SDP1;
2411 2412
	switch (adapter->hw.mac.type) {
	case ixgbe_mac_82599EB:
2413
	case ixgbe_mac_X540:
2414
		mask |= IXGBE_EIMS_ECC;
2415 2416
		mask |= IXGBE_EIMS_GPI_SDP1;
		mask |= IXGBE_EIMS_GPI_SDP2;
2417 2418
		if (adapter->num_vfs)
			mask |= IXGBE_EIMS_MAILBOX;
2419 2420 2421
		break;
	default:
		break;
2422
	}
2423
	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
2424
		mask |= IXGBE_EIMS_FLOW_DIR;
2425

2426
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
2427 2428 2429 2430
	if (queues)
		ixgbe_irq_enable_queues(adapter, ~0);
	if (flush)
		IXGBE_WRITE_FLUSH(&adapter->hw);
2431 2432 2433 2434 2435

	if (adapter->num_vfs > 32) {
		u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
	}
2436
}
2437

2438
/**
2439
 * ixgbe_intr - legacy mode Interrupt Handler
2440 2441 2442 2443 2444
 * @irq: interrupt number
 * @data: pointer to a network interface device structure
 **/
static irqreturn_t ixgbe_intr(int irq, void *data)
{
2445
	struct ixgbe_adapter *adapter = data;
2446
	struct ixgbe_hw *hw = &adapter->hw;
2447
	struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2448 2449
	u32 eicr;

2450
	/*
2451
	 * Workaround for silicon errata on 82598.  Mask the interrupts
2452 2453 2454 2455
	 * before the read of EICR.
	 */
	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);

2456 2457 2458
	/* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
	 * therefore no explict interrupt disable is necessary */
	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2459
	if (!eicr) {
2460 2461
		/*
		 * shared interrupt alert!
2462
		 * make sure interrupts are enabled because the read will
2463 2464 2465 2466 2467 2468
		 * have disabled interrupts due to EIAM
		 * finish the workaround of silicon errata on 82598.  Unmask
		 * the interrupt that we masked before the EICR read.
		 */
		if (!test_bit(__IXGBE_DOWN, &adapter->state))
			ixgbe_irq_enable(adapter, true, true);
2469
		return IRQ_NONE;	/* Not our interrupt */
2470
	}
2471

2472 2473
	if (eicr & IXGBE_EICR_LSC)
		ixgbe_check_lsc(adapter);
2474

2475 2476
	switch (hw->mac.type) {
	case ixgbe_mac_82599EB:
2477
		ixgbe_check_sfp_event(adapter, eicr);
2478 2479
		if ((adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
		    ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
2480 2481 2482 2483 2484
			if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
				adapter->interrupt_event = eicr;
				adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
				ixgbe_service_event_schedule(adapter);
			}
2485 2486 2487 2488 2489
		}
		break;
	default:
		break;
	}
2490

2491 2492
	ixgbe_check_fan_failure(adapter, eicr);

2493
	if (napi_schedule_prep(&(q_vector->napi))) {
2494
		/* would disable interrupts here but EIAM disabled it */
2495
		__napi_schedule(&(q_vector->napi));
2496 2497
	}

2498 2499 2500 2501 2502 2503 2504 2505
	/*
	 * re-enable link(maybe) and non-queue interrupts, no flush.
	 * ixgbe_poll will re-enable the queue interrupts
	 */

	if (!test_bit(__IXGBE_DOWN, &adapter->state))
		ixgbe_irq_enable(adapter, false, false);

2506 2507 2508
	return IRQ_HANDLED;
}

2509 2510 2511 2512 2513
static inline void ixgbe_reset_q_vectors(struct ixgbe_adapter *adapter)
{
	int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;

	for (i = 0; i < q_vectors; i++) {
2514
		struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
2515 2516 2517 2518
		bitmap_zero(q_vector->rx.idx, MAX_RX_QUEUES);
		bitmap_zero(q_vector->tx.idx, MAX_TX_QUEUES);
		q_vector->rx.count = 0;
		q_vector->tx.count = 0;
2519 2520 2521
	}
}

2522 2523 2524 2525 2526 2527 2528
/**
 * ixgbe_request_irq - initialize interrupts
 * @adapter: board private structure
 *
 * Attempts to configure interrupts using the best available
 * capabilities of the hardware and kernel.
 **/
2529
static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
2530 2531
{
	struct net_device *netdev = adapter->netdev;
2532
	int err;
2533

2534 2535 2536
	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
		err = ixgbe_request_msix_irqs(adapter);
	} else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) {
2537
		err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
2538
				  netdev->name, adapter);
2539
	} else {
2540
		err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
2541
				  netdev->name, adapter);
2542 2543 2544
	}

	if (err)
2545
		e_err(probe, "request_irq failed, Error %d\n", err);
2546 2547 2548 2549 2550 2551 2552

	return err;
}

static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
{
	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2553
		int i, q_vectors;
2554

2555 2556 2557
		q_vectors = adapter->num_msix_vectors;

		i = q_vectors - 1;
2558
		free_irq(adapter->msix_entries[i].vector, adapter);
2559

2560 2561
		i--;
		for (; i >= 0; i--) {
2562
			/* free only the irqs that were actually requested */
2563 2564
			if (!adapter->q_vector[i]->rx.count &&
			    !adapter->q_vector[i]->tx.count)
2565 2566
				continue;

2567
			free_irq(adapter->msix_entries[i].vector,
2568
				 adapter->q_vector[i]);
2569 2570 2571 2572
		}

		ixgbe_reset_q_vectors(adapter);
	} else {
2573
		free_irq(adapter->pdev->irq, adapter);
2574 2575 2576
	}
}

2577 2578 2579 2580 2581 2582
/**
 * ixgbe_irq_disable - Mask off interrupt generation on the NIC
 * @adapter: board private structure
 **/
static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
{
2583 2584
	switch (adapter->hw.mac.type) {
	case ixgbe_mac_82598EB:
2585
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
2586 2587
		break;
	case ixgbe_mac_82599EB:
2588
	case ixgbe_mac_X540:
2589 2590
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
2591
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
2592 2593
		if (adapter->num_vfs > 32)
			IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
2594 2595 2596
		break;
	default:
		break;
2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607
	}
	IXGBE_WRITE_FLUSH(&adapter->hw);
	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
		int i;
		for (i = 0; i < adapter->num_msix_vectors; i++)
			synchronize_irq(adapter->msix_entries[i].vector);
	} else {
		synchronize_irq(adapter->pdev->irq);
	}
}

2608 2609 2610 2611 2612 2613 2614 2615
/**
 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
 *
 **/
static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;

2616
	IXGBE_WRITE_REG(hw, IXGBE_EITR(0),
2617
			EITR_INTS_PER_SEC_TO_REG(adapter->rx_eitr_param));
2618

2619 2620
	ixgbe_set_ivar(adapter, 0, 0, 0);
	ixgbe_set_ivar(adapter, 1, 0, 0);
2621 2622 2623 2624

	map_vector_to_rxq(adapter, 0, 0);
	map_vector_to_txq(adapter, 0, 0);

2625
	e_info(hw, "Legacy interrupt IVAR setup done\n");
2626 2627
}

2628 2629 2630 2631 2632 2633 2634
/**
 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
 * @adapter: board private structure
 * @ring: structure containing ring specific data
 *
 * Configure the Tx descriptor ring after a reset.
 **/
2635 2636
void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
			     struct ixgbe_ring *ring)
2637 2638 2639
{
	struct ixgbe_hw *hw = &adapter->hw;
	u64 tdba = ring->dma;
2640 2641
	int wait_loop = 10;
	u32 txdctl;
2642
	u8 reg_idx = ring->reg_idx;
2643

2644 2645 2646 2647 2648 2649
	/* disable queue to avoid issues while updating state */
	txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx),
			txdctl & ~IXGBE_TXDCTL_ENABLE);
	IXGBE_WRITE_FLUSH(hw);

2650
	IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
2651
			(tdba & DMA_BIT_MASK(32)));
2652 2653 2654 2655 2656
	IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
	IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
			ring->count * sizeof(union ixgbe_adv_tx_desc));
	IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
	IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
2657
	ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
2658

2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672
	/* configure fetching thresholds */
	if (adapter->rx_itr_setting == 0) {
		/* cannot set wthresh when itr==0 */
		txdctl &= ~0x007F0000;
	} else {
		/* enable WTHRESH=8 descriptors, to encourage burst writeback */
		txdctl |= (8 << 16);
	}
	if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
		/* PThresh workaround for Tx hang with DFP enabled. */
		txdctl |= 32;
	}

	/* reinitialize flowdirector state */
2673 2674 2675 2676 2677 2678 2679 2680
	if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
	    adapter->atr_sample_rate) {
		ring->atr_sample_rate = adapter->atr_sample_rate;
		ring->atr_count = 0;
		set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
	} else {
		ring->atr_sample_rate = 0;
	}
2681

2682 2683
	clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);

2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694
	/* enable queue */
	txdctl |= IXGBE_TXDCTL_ENABLE;
	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);

	/* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
	if (hw->mac.type == ixgbe_mac_82598EB &&
	    !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
		return;

	/* poll to verify queue is enabled */
	do {
2695
		usleep_range(1000, 2000);
2696 2697 2698 2699
		txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
	} while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
	if (!wait_loop)
		e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
2700 2701
}

2702 2703 2704 2705
static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 rttdcs;
2706
	u32 reg;
2707
	u8 tcs = netdev_get_num_tc(adapter->netdev);
2708 2709 2710 2711 2712 2713 2714 2715 2716 2717

	if (hw->mac.type == ixgbe_mac_82598EB)
		return;

	/* disable the arbiter while setting MTQC */
	rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
	rttdcs |= IXGBE_RTTDCS_ARBDIS;
	IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);

	/* set transmit pool layout */
2718
	switch (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
2719 2720 2721 2722
	case (IXGBE_FLAG_SRIOV_ENABLED):
		IXGBE_WRITE_REG(hw, IXGBE_MTQC,
				(IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF));
		break;
2723 2724 2725 2726 2727 2728 2729
	default:
		if (!tcs)
			reg = IXGBE_MTQC_64Q_1PB;
		else if (tcs <= 4)
			reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
		else
			reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
2730

2731
		IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
2732

2733 2734 2735 2736 2737 2738
		/* Enable Security TX Buffer IFG for multiple pb */
		if (tcs) {
			reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
			reg |= IXGBE_SECTX_DCB;
			IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
		}
2739 2740 2741 2742 2743 2744 2745 2746
		break;
	}

	/* re-enable the arbiter */
	rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
	IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
}

2747
/**
2748
 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
2749 2750 2751 2752 2753 2754
 * @adapter: board private structure
 *
 * Configure the Tx unit of the MAC after a reset.
 **/
static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
{
2755 2756
	struct ixgbe_hw *hw = &adapter->hw;
	u32 dmatxctl;
2757
	u32 i;
2758

2759 2760 2761 2762 2763 2764 2765 2766 2767
	ixgbe_setup_mtqc(adapter);

	if (hw->mac.type != ixgbe_mac_82598EB) {
		/* DMATXCTL.EN must be before Tx queues are enabled */
		dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
		dmatxctl |= IXGBE_DMATXCTL_TE;
		IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
	}

2768
	/* Setup the HW Tx Head and Tail descriptor pointers */
2769 2770
	for (i = 0; i < adapter->num_tx_queues; i++)
		ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
2771 2772
}

2773
#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2774

2775
static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
2776
				   struct ixgbe_ring *rx_ring)
2777 2778
{
	u32 srrctl;
2779
	u8 reg_idx = rx_ring->reg_idx;
2780

2781 2782 2783 2784
	switch (adapter->hw.mac.type) {
	case ixgbe_mac_82598EB: {
		struct ixgbe_ring_feature *feature = adapter->ring_feature;
		const int mask = feature[RING_F_RSS].mask;
2785
		reg_idx = reg_idx & mask;
2786
	}
2787 2788
		break;
	case ixgbe_mac_82599EB:
2789
	case ixgbe_mac_X540:
2790 2791 2792 2793
	default:
		break;
	}

2794
	srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx));
2795 2796 2797

	srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
	srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2798 2799
	if (adapter->num_vfs)
		srrctl |= IXGBE_SRRCTL_DROP_EN;
2800

2801 2802 2803
	srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
		  IXGBE_SRRCTL_BSIZEHDR_MASK;

2804
	if (ring_is_ps_enabled(rx_ring)) {
2805 2806 2807 2808 2809
#if (PAGE_SIZE / 2) > IXGBE_MAX_RXBUFFER
		srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
#else
		srrctl |= (PAGE_SIZE / 2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
#endif
2810 2811
		srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
	} else {
2812 2813
		srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
			  IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2814 2815
		srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
	}
2816

2817
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx), srrctl);
2818
}
2819

2820
static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
2821
{
2822 2823
	struct ixgbe_hw *hw = &adapter->hw;
	static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
2824 2825
			  0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
			  0x6A3E67EA, 0x14364D17, 0x3BED200D};
2826 2827 2828
	u32 mrqc = 0, reta = 0;
	u32 rxcsum;
	int i, j;
2829
	u8 tcs = netdev_get_num_tc(adapter->netdev);
2830 2831 2832 2833
	int maxq = adapter->ring_feature[RING_F_RSS].indices;

	if (tcs)
		maxq = min(maxq, adapter->num_tx_queues / tcs);
2834

2835 2836 2837 2838 2839 2840
	/* Fill out hash function seeds */
	for (i = 0; i < 10; i++)
		IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);

	/* Fill out redirection table */
	for (i = 0, j = 0; i < 128; i++, j++) {
2841
		if (j == maxq)
2842 2843 2844 2845 2846 2847 2848
			j = 0;
		/* reta = 4-byte sliding window of
		 * 0x00..(indices-1)(indices-1)00..etc. */
		reta = (reta << 8) | (j * 0x11);
		if ((i & 3) == 3)
			IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
	}
2849

2850 2851 2852 2853 2854
	/* Disable indicating checksum in descriptor, enables RSS hash */
	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
	rxcsum |= IXGBE_RXCSUM_PCSD;
	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);

2855 2856
	if (adapter->hw.mac.type == ixgbe_mac_82598EB &&
	    (adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
2857
		mrqc = IXGBE_MRQC_RSSEN;
2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876
	} else {
		int mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED
					     | IXGBE_FLAG_SRIOV_ENABLED);

		switch (mask) {
		case (IXGBE_FLAG_RSS_ENABLED):
			if (!tcs)
				mrqc = IXGBE_MRQC_RSSEN;
			else if (tcs <= 4)
				mrqc = IXGBE_MRQC_RTRSS4TCEN;
			else
				mrqc = IXGBE_MRQC_RTRSS8TCEN;
			break;
		case (IXGBE_FLAG_SRIOV_ENABLED):
			mrqc = IXGBE_MRQC_VMDQEN;
			break;
		default:
			break;
		}
2877 2878
	}

2879 2880 2881 2882 2883 2884 2885
	/* Perform hash on these packet types */
	mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4
	      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
	      | IXGBE_MRQC_RSS_FIELD_IPV6
	      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP;

	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2886 2887
}

2888 2889 2890 2891 2892
/**
 * ixgbe_configure_rscctl - enable RSC for the indicated ring
 * @adapter:    address of board private structure
 * @index:      index of ring to set
 **/
2893
static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
2894
				   struct ixgbe_ring *ring)
2895 2896 2897
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 rscctrl;
2898
	int rx_buf_len;
2899
	u8 reg_idx = ring->reg_idx;
2900

2901
	if (!ring_is_rsc_enabled(ring))
2902
		return;
2903

2904 2905
	rx_buf_len = ring->rx_buf_len;
	rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
2906 2907 2908 2909 2910 2911
	rscctrl |= IXGBE_RSCCTL_RSCEN;
	/*
	 * we must limit the number of descriptors so that the
	 * total size of max desc * buf_len is not greater
	 * than 65535
	 */
2912
	if (ring_is_ps_enabled(ring)) {
2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
#if (MAX_SKB_FRAGS > 16)
		rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
#elif (MAX_SKB_FRAGS > 8)
		rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
#elif (MAX_SKB_FRAGS > 4)
		rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
#else
		rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
#endif
	} else {
		if (rx_buf_len < IXGBE_RXBUFFER_4096)
			rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
		else if (rx_buf_len < IXGBE_RXBUFFER_8192)
			rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
		else
			rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
	}
2930
	IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
2931 2932
}

2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966
/**
 *  ixgbe_set_uta - Set unicast filter table address
 *  @adapter: board private structure
 *
 *  The unicast table address is a register array of 32-bit registers.
 *  The table is meant to be used in a way similar to how the MTA is used
 *  however due to certain limitations in the hardware it is necessary to
 *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
 *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
 **/
static void ixgbe_set_uta(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	int i;

	/* The UTA table only exists on 82599 hardware and newer */
	if (hw->mac.type < ixgbe_mac_82599EB)
		return;

	/* we only need to do this if VMDq is enabled */
	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
		return;

	for (i = 0; i < 128; i++)
		IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
}

#define IXGBE_MAX_RX_DESC_POLL 10
static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
				       struct ixgbe_ring *ring)
{
	struct ixgbe_hw *hw = &adapter->hw;
	int wait_loop = IXGBE_MAX_RX_DESC_POLL;
	u32 rxdctl;
2967
	u8 reg_idx = ring->reg_idx;
2968 2969 2970 2971 2972 2973 2974

	/* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
	if (hw->mac.type == ixgbe_mac_82598EB &&
	    !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
		return;

	do {
2975
		usleep_range(1000, 2000);
2976 2977 2978 2979 2980 2981 2982 2983 2984
		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
	} while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));

	if (!wait_loop) {
		e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
		      "the polling period\n", reg_idx);
	}
}

2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014
void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
			    struct ixgbe_ring *ring)
{
	struct ixgbe_hw *hw = &adapter->hw;
	int wait_loop = IXGBE_MAX_RX_DESC_POLL;
	u32 rxdctl;
	u8 reg_idx = ring->reg_idx;

	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
	rxdctl &= ~IXGBE_RXDCTL_ENABLE;

	/* write value back with RXDCTL.ENABLE bit cleared */
	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);

	if (hw->mac.type == ixgbe_mac_82598EB &&
	    !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
		return;

	/* the hardware may take up to 100us to really disable the rx queue */
	do {
		udelay(10);
		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
	} while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));

	if (!wait_loop) {
		e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
		      "the polling period\n", reg_idx);
	}
}

3015 3016
void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
			     struct ixgbe_ring *ring)
3017 3018 3019
{
	struct ixgbe_hw *hw = &adapter->hw;
	u64 rdba = ring->dma;
3020
	u32 rxdctl;
3021
	u8 reg_idx = ring->reg_idx;
3022

3023 3024
	/* disable queue to avoid issues while updating state */
	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3025
	ixgbe_disable_rx_queue(adapter, ring);
3026

3027 3028 3029 3030 3031 3032
	IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
	IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
	IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
			ring->count * sizeof(union ixgbe_adv_rx_desc));
	IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
	IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
3033
	ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
3034 3035 3036 3037

	ixgbe_configure_srrctl(adapter, ring);
	ixgbe_configure_rscctl(adapter, ring);

3038 3039 3040 3041 3042 3043 3044 3045
	/* If operating in IOV mode set RLPML for X540 */
	if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
	    hw->mac.type == ixgbe_mac_X540) {
		rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
		rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
			    ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN);
	}

3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062
	if (hw->mac.type == ixgbe_mac_82598EB) {
		/*
		 * enable cache line friendly hardware writes:
		 * PTHRESH=32 descriptors (half the internal cache),
		 * this also removes ugly rx_no_buffer_count increment
		 * HTHRESH=4 descriptors (to minimize latency on fetch)
		 * WTHRESH=8 burst writeback up to two cache lines
		 */
		rxdctl &= ~0x3FFFFF;
		rxdctl |=  0x080420;
	}

	/* enable receive descriptor ring */
	rxdctl |= IXGBE_RXDCTL_ENABLE;
	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);

	ixgbe_rx_desc_queue_enable(adapter, ring);
3063
	ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
3064 3065
}

3066 3067 3068 3069 3070 3071 3072
static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	int p;

	/* PSRTYPE must be initialized in non 82598 adapters */
	u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
3073 3074
		      IXGBE_PSRTYPE_UDPHDR |
		      IXGBE_PSRTYPE_IPV4HDR |
3075
		      IXGBE_PSRTYPE_L2HDR |
3076
		      IXGBE_PSRTYPE_IPV6HDR;
3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088

	if (hw->mac.type == ixgbe_mac_82598EB)
		return;

	if (adapter->flags & IXGBE_FLAG_RSS_ENABLED)
		psrtype |= (adapter->num_rx_queues_per_pool << 29);

	for (p = 0; p < adapter->num_rx_pools; p++)
		IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(adapter->num_vfs + p),
				psrtype);
}

3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128
static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 gcr_ext;
	u32 vt_reg_bits;
	u32 reg_offset, vf_shift;
	u32 vmdctl;

	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
		return;

	vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
	vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN | IXGBE_VT_CTL_REPLEN;
	vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT);
	IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl | vt_reg_bits);

	vf_shift = adapter->num_vfs % 32;
	reg_offset = (adapter->num_vfs > 32) ? 1 : 0;

	/* Enable only the PF's pool for Tx/Rx */
	IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (1 << vf_shift));
	IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), 0);
	IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (1 << vf_shift));
	IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), 0);
	IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);

	/* Map PF MAC address in RAR Entry 0 to first pool following VFs */
	hw->mac.ops.set_vmdq(hw, 0, adapter->num_vfs);

	/*
	 * Set up VF register offsets for selected VT Mode,
	 * i.e. 32 or 64 VFs for SR-IOV
	 */
	gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
	gcr_ext |= IXGBE_GCR_EXT_MSIX_EN;
	gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64;
	IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);

	/* enable Tx loopback for VF/PF communication */
	IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
3129
	/* Enable MAC Anti-Spoofing */
3130 3131 3132
	hw->mac.ops.set_mac_anti_spoofing(hw,
					  (adapter->antispoofing_enabled =
					   (adapter->num_vfs != 0)),
3133
					  adapter->num_vfs);
3134 3135
}

3136
static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
3137 3138 3139 3140
{
	struct ixgbe_hw *hw = &adapter->hw;
	struct net_device *netdev = adapter->netdev;
	int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3141
	int rx_buf_len;
3142 3143 3144
	struct ixgbe_ring *rx_ring;
	int i;
	u32 mhadd, hlreg0;
3145

3146
	/* Decide whether to use packet split mode or not */
3147 3148 3149
	/* On by default */
	adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;

3150
	/* Do not use packet split if we're in SR-IOV Mode */
3151 3152 3153 3154 3155 3156
	if (adapter->num_vfs)
		adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;

	/* Disable packet split due to 82599 erratum #45 */
	if (hw->mac.type == ixgbe_mac_82599EB)
		adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
3157 3158 3159

	/* Set the RX buffer length according to the mode */
	if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
3160
		rx_buf_len = IXGBE_RX_HDR_SIZE;
3161
	} else {
3162
		if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
3163
		    (netdev->mtu <= ETH_DATA_LEN))
3164
			rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3165
		else
3166
			rx_buf_len = ALIGN(max_frame + VLAN_HLEN, 1024);
3167 3168
	}

3169
#ifdef IXGBE_FCOE
3170 3171 3172 3173
	/* adjust max frame to be able to do baby jumbo for FCoE */
	if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
	    (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
		max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
3174

3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187
#endif /* IXGBE_FCOE */
	mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
	if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
		mhadd &= ~IXGBE_MHADD_MFS_MASK;
		mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;

		IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
	}

	hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
	/* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
	hlreg0 |= IXGBE_HLREG0_JUMBOEN;
	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3188

3189 3190 3191 3192
	/*
	 * Setup the HW Rx Head and Tail Descriptor Pointers and
	 * the Base and Length of the Rx Descriptor Ring
	 */
3193
	for (i = 0; i < adapter->num_rx_queues; i++) {
3194
		rx_ring = adapter->rx_ring[i];
3195
		rx_ring->rx_buf_len = rx_buf_len;
3196

3197
		if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)
3198 3199 3200 3201 3202 3203
			set_ring_ps_enabled(rx_ring);
		else
			clear_ring_ps_enabled(rx_ring);

		if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
			set_ring_rsc_enabled(rx_ring);
3204
		else
3205
			clear_ring_rsc_enabled(rx_ring);
3206

3207
#ifdef IXGBE_FCOE
3208
		if (netdev->features & NETIF_F_FCOE_MTU) {
3209 3210
			struct ixgbe_ring_feature *f;
			f = &adapter->ring_feature[RING_F_FCOE];
3211
			if ((i >= f->mask) && (i < f->mask + f->indices)) {
3212
				clear_ring_ps_enabled(rx_ring);
3213 3214
				if (rx_buf_len < IXGBE_FCOE_JUMBO_FRAME_SIZE)
					rx_ring->rx_buf_len =
3215
						IXGBE_FCOE_JUMBO_FRAME_SIZE;
3216 3217 3218 3219
			} else if (!ring_is_rsc_enabled(rx_ring) &&
				   !ring_is_ps_enabled(rx_ring)) {
				rx_ring->rx_buf_len =
						IXGBE_FCOE_JUMBO_FRAME_SIZE;
3220
			}
3221 3222
		}
#endif /* IXGBE_FCOE */
3223 3224 3225
	}
}

3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245
static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);

	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
		/*
		 * For VMDq support of different descriptor types or
		 * buffer sizes through the use of multiple SRRCTL
		 * registers, RDRXCTL.MVMEN must be set to 1
		 *
		 * also, the manual doesn't mention it clearly but DCA hints
		 * will only use queue 0's tags unless this bit is set.  Side
		 * effects of setting this bit are only that SRRCTL must be
		 * fully programmed [0..15]
		 */
		rdrxctl |= IXGBE_RDRXCTL_MVMEN;
		break;
	case ixgbe_mac_82599EB:
3246
	case ixgbe_mac_X540:
3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262
		/* Disable RSC for ACK packets */
		IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
		   (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
		rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
		/* hardware requires some bits to be set by default */
		rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
		rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
		break;
	default:
		/* We should do nothing since we don't know this hardware */
		return;
	}

	IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
}

3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279
/**
 * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
 * @adapter: board private structure
 *
 * Configure the Rx unit of the MAC after a reset.
 **/
static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	int i;
	u32 rxctrl;

	/* disable receives while setting up the descriptors */
	rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
	IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);

	ixgbe_setup_psrtype(adapter);
3280
	ixgbe_setup_rdrxctl(adapter);
3281

3282
	/* Program registers for the distribution of queues */
3283 3284
	ixgbe_setup_mrqc(adapter);

3285 3286
	ixgbe_set_uta(adapter);

3287 3288 3289 3290 3291 3292 3293
	/* set_rx_buffer_len must be called before ring initialization */
	ixgbe_set_rx_buffer_len(adapter);

	/*
	 * Setup the HW Rx Head and Tail Descriptor Pointers and
	 * the Base and Length of the Rx Descriptor Ring
	 */
3294 3295
	for (i = 0; i < adapter->num_rx_queues; i++)
		ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);
3296

3297 3298 3299 3300 3301 3302 3303
	/* disable drop enable for 82598 parts */
	if (hw->mac.type == ixgbe_mac_82598EB)
		rxctrl |= IXGBE_RXCTRL_DMBYPS;

	/* enable all receives */
	rxctrl |= IXGBE_RXCTRL_RXEN;
	hw->mac.ops.enable_rx_dma(hw, rxctrl);
3304 3305
}

3306 3307 3308 3309
static void ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	struct ixgbe_hw *hw = &adapter->hw;
3310
	int pool_ndx = adapter->num_vfs;
3311 3312

	/* add VID to filter table */
3313
	hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, true);
3314
	set_bit(vid, adapter->active_vlans);
3315 3316 3317 3318 3319 3320
}

static void ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	struct ixgbe_hw *hw = &adapter->hw;
3321
	int pool_ndx = adapter->num_vfs;
3322 3323

	/* remove VID from filter table */
3324
	hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, false);
3325
	clear_bit(vid, adapter->active_vlans);
3326 3327
}

3328 3329 3330 3331 3332 3333 3334
/**
 * ixgbe_vlan_filter_disable - helper to disable hw vlan filtering
 * @adapter: driver data
 */
static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364
	u32 vlnctrl;

	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
	vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
}

/**
 * ixgbe_vlan_filter_enable - helper to enable hw vlan filtering
 * @adapter: driver data
 */
static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 vlnctrl;

	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
	vlnctrl |= IXGBE_VLNCTRL_VFE;
	vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
}

/**
 * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
 * @adapter: driver data
 */
static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 vlnctrl;
3365 3366 3367 3368
	int i, j;

	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
3369 3370
		vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
		vlnctrl &= ~IXGBE_VLNCTRL_VME;
3371 3372 3373
		IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
		break;
	case ixgbe_mac_82599EB:
3374
	case ixgbe_mac_X540:
3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387
		for (i = 0; i < adapter->num_rx_queues; i++) {
			j = adapter->rx_ring[i]->reg_idx;
			vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
			vlnctrl &= ~IXGBE_RXDCTL_VME;
			IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
		}
		break;
	default:
		break;
	}
}

/**
3388
 * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
3389 3390
 * @adapter: driver data
 */
3391
static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
3392 3393
{
	struct ixgbe_hw *hw = &adapter->hw;
3394
	u32 vlnctrl;
3395 3396 3397 3398
	int i, j;

	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
3399 3400
		vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
		vlnctrl |= IXGBE_VLNCTRL_VME;
3401 3402 3403
		IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
		break;
	case ixgbe_mac_82599EB:
3404
	case ixgbe_mac_X540:
3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416
		for (i = 0; i < adapter->num_rx_queues; i++) {
			j = adapter->rx_ring[i]->reg_idx;
			vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
			vlnctrl |= IXGBE_RXDCTL_VME;
			IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
		}
		break;
	default:
		break;
	}
}

3417 3418
static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
{
3419
	u16 vid;
3420

3421 3422 3423 3424
	ixgbe_vlan_rx_add_vid(adapter->netdev, 0);

	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
		ixgbe_vlan_rx_add_vid(adapter->netdev, vid);
3425 3426
}

3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440
/**
 * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
 * @netdev: network interface device structure
 *
 * Writes unicast address list to the RAR table.
 * Returns: -ENOMEM on failure/insufficient address space
 *                0 on no addresses written
 *                X on writing X addresses to the RAR table
 **/
static int ixgbe_write_uc_addr_list(struct net_device *netdev)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	struct ixgbe_hw *hw = &adapter->hw;
	unsigned int vfn = adapter->num_vfs;
3441
	unsigned int rar_entries = IXGBE_MAX_PF_MACVLANS;
3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468
	int count = 0;

	/* return ENOMEM indicating insufficient memory for addresses */
	if (netdev_uc_count(netdev) > rar_entries)
		return -ENOMEM;

	if (!netdev_uc_empty(netdev) && rar_entries) {
		struct netdev_hw_addr *ha;
		/* return error if we do not support writing to RAR table */
		if (!hw->mac.ops.set_rar)
			return -ENOMEM;

		netdev_for_each_uc_addr(ha, netdev) {
			if (!rar_entries)
				break;
			hw->mac.ops.set_rar(hw, rar_entries--, ha->addr,
					    vfn, IXGBE_RAH_AV);
			count++;
		}
	}
	/* write the addresses in reverse order to avoid write combining */
	for (; rar_entries > 0 ; rar_entries--)
		hw->mac.ops.clear_rar(hw, rar_entries);

	return count;
}

3469
/**
3470
 * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
3471 3472
 * @netdev: network interface device structure
 *
3473 3474 3475 3476
 * The set_rx_method entry point is called whenever the unicast/multicast
 * address list or the network interface flags are updated.  This routine is
 * responsible for configuring the hardware for proper unicast, multicast and
 * promiscuous mode.
3477
 **/
3478
void ixgbe_set_rx_mode(struct net_device *netdev)
3479 3480 3481
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	struct ixgbe_hw *hw = &adapter->hw;
3482 3483
	u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
	int count;
3484 3485 3486 3487 3488

	/* Check for Promiscuous and All Multicast modes */

	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);

3489 3490 3491 3492 3493
	/* set all bits that we expect to always be set */
	fctrl |= IXGBE_FCTRL_BAM;
	fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
	fctrl |= IXGBE_FCTRL_PMCF;

3494 3495 3496
	/* clear the bits we are changing the status of */
	fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);

3497
	if (netdev->flags & IFF_PROMISC) {
3498
		hw->addr_ctrl.user_set_promisc = true;
3499
		fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3500
		vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE);
3501 3502
		/* don't hardware filter vlans in promisc mode */
		ixgbe_vlan_filter_disable(adapter);
3503
	} else {
3504 3505
		if (netdev->flags & IFF_ALLMULTI) {
			fctrl |= IXGBE_FCTRL_MPE;
3506 3507 3508 3509
			vmolr |= IXGBE_VMOLR_MPE;
		} else {
			/*
			 * Write addresses to the MTA, if the attempt fails
Lucas De Marchi's avatar
Lucas De Marchi committed
3510
			 * then we should just turn on promiscuous mode so
3511 3512 3513 3514
			 * that we can at least receive multicast traffic
			 */
			hw->mac.ops.update_mc_addr_list(hw, netdev);
			vmolr |= IXGBE_VMOLR_ROMPE;
3515
		}
3516
		ixgbe_vlan_filter_enable(adapter);
3517
		hw->addr_ctrl.user_set_promisc = false;
3518 3519 3520
		/*
		 * Write addresses to available RAR registers, if there is not
		 * sufficient space to store all the addresses then enable
Lucas De Marchi's avatar
Lucas De Marchi committed
3521
		 * unicast promiscuous mode
3522 3523 3524 3525 3526 3527
		 */
		count = ixgbe_write_uc_addr_list(netdev);
		if (count < 0) {
			fctrl |= IXGBE_FCTRL_UPE;
			vmolr |= IXGBE_VMOLR_ROPE;
		}
3528 3529
	}

3530
	if (adapter->num_vfs) {
3531
		ixgbe_restore_vf_multicasts(adapter);
3532 3533 3534 3535 3536 3537 3538
		vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(adapter->num_vfs)) &
			 ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
			   IXGBE_VMOLR_ROPE);
		IXGBE_WRITE_REG(hw, IXGBE_VMOLR(adapter->num_vfs), vmolr);
	}

	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3539 3540 3541 3542 3543

	if (netdev->features & NETIF_F_HW_VLAN_RX)
		ixgbe_vlan_strip_enable(adapter);
	else
		ixgbe_vlan_strip_disable(adapter);
3544 3545
}

3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556
static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
{
	int q_idx;
	struct ixgbe_q_vector *q_vector;
	int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;

	/* legacy and MSI only use one vector */
	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
		q_vectors = 1;

	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
3557
		struct napi_struct *napi;
3558
		q_vector = adapter->q_vector[q_idx];
3559
		napi = &q_vector->napi;
3560
		if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3561 3562
			if (!q_vector->rx.count || !q_vector->tx.count) {
				if (q_vector->tx.count == 1)
3563
					napi->poll = &ixgbe_clean_txonly;
3564
				else if (q_vector->rx.count == 1)
3565 3566 3567
					napi->poll = &ixgbe_clean_rxonly;
			}
		}
3568 3569

		napi_enable(napi);
3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583
	}
}

static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
{
	int q_idx;
	struct ixgbe_q_vector *q_vector;
	int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;

	/* legacy and MSI only use one vector */
	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
		q_vectors = 1;

	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
3584
		q_vector = adapter->q_vector[q_idx];
3585 3586 3587 3588
		napi_disable(&q_vector->napi);
	}
}

Jeff Kirsher's avatar
Jeff Kirsher committed
3589
#ifdef CONFIG_IXGBE_DCB
3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600
/*
 * ixgbe_configure_dcb - Configure DCB hardware
 * @adapter: ixgbe adapter struct
 *
 * This is called by the driver on open to configure the DCB hardware.
 * This is also called by the gennetlink interface when reconfiguring
 * the DCB state.
 */
static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
3601
	int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3602

3603 3604 3605 3606 3607 3608 3609 3610 3611
	if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
		if (hw->mac.type == ixgbe_mac_82598EB)
			netif_set_gso_max_size(adapter->netdev, 65536);
		return;
	}

	if (hw->mac.type == ixgbe_mac_82598EB)
		netif_set_gso_max_size(adapter->netdev, 32768);

3612 3613

	/* Enable VLAN tag insert/strip */
3614
	adapter->netdev->features |= NETIF_F_HW_VLAN_RX;
3615

3616
	hw->mac.ops.set_vfta(&adapter->hw, 0, 0, true);
3617 3618

	/* reconfigure the hardware */
3619
	if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
3620
#ifdef IXGBE_FCOE
3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638
		if (adapter->netdev->features & NETIF_F_FCOE_MTU)
			max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
#endif
		ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
						DCB_TX_CONFIG);
		ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
						DCB_RX_CONFIG);
		ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
	} else {
		struct net_device *dev = adapter->netdev;

		if (adapter->ixgbe_ieee_ets)
			dev->dcbnl_ops->ieee_setets(dev,
						    adapter->ixgbe_ieee_ets);
		if (adapter->ixgbe_ieee_pfc)
			dev->dcbnl_ops->ieee_setpfc(dev,
						    adapter->ixgbe_ieee_pfc);
	}
3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655

	/* Enable RSS Hash per TC */
	if (hw->mac.type != ixgbe_mac_82598EB) {
		int i;
		u32 reg = 0;

		for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
			u8 msb = 0;
			u8 cnt = adapter->netdev->tc_to_txq[i].count;

			while (cnt >>= 1)
				msb++;

			reg |= msb << IXGBE_RQTC_SHIFT_TC(i);
		}
		IXGBE_WRITE_REG(hw, IXGBE_RQTC, reg);
	}
3656 3657 3658
}

#endif
3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672

static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
{
	int hdrm = 0;
	int num_tc = netdev_get_num_tc(adapter->netdev);
	struct ixgbe_hw *hw = &adapter->hw;

	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
	    adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
		hdrm = 64 << adapter->fdir_pballoc;

	hw->mac.ops.set_rxpba(&adapter->hw, num_tc, hdrm, PBA_STRATEGY_EQUAL);
}

3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686
static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	struct hlist_node *node, *node2;
	struct ixgbe_fdir_filter *filter;

	spin_lock(&adapter->fdir_perfect_lock);

	if (!hlist_empty(&adapter->fdir_filter_list))
		ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask);

	hlist_for_each_entry_safe(filter, node, node2,
				  &adapter->fdir_filter_list, fdir_node) {
		ixgbe_fdir_write_perfect_filter_82599(hw,
3687 3688 3689 3690 3691
				&filter->filter,
				filter->sw_idx,
				(filter->action == IXGBE_FDIR_DROP_QUEUE) ?
				IXGBE_FDIR_DROP_QUEUE :
				adapter->rx_ring[filter->action]->reg_idx);
3692 3693 3694 3695 3696
	}

	spin_unlock(&adapter->fdir_perfect_lock);
}

3697 3698 3699
static void ixgbe_configure(struct ixgbe_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
3700
	struct ixgbe_hw *hw = &adapter->hw;
3701 3702
	int i;

3703
	ixgbe_configure_pb(adapter);
Jeff Kirsher's avatar
Jeff Kirsher committed
3704
#ifdef CONFIG_IXGBE_DCB
3705
	ixgbe_configure_dcb(adapter);
3706
#endif
3707

3708 3709 3710
	ixgbe_set_rx_mode(netdev);
	ixgbe_restore_vlan(adapter);

3711 3712 3713 3714 3715
#ifdef IXGBE_FCOE
	if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
		ixgbe_configure_fcoe(adapter);

#endif /* IXGBE_FCOE */
3716 3717
	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
		for (i = 0; i < adapter->num_tx_queues; i++)
3718
			adapter->tx_ring[i]->atr_sample_rate =
3719
						       adapter->atr_sample_rate;
3720
		ixgbe_init_fdir_signature_82599(hw, adapter->fdir_pballoc);
3721 3722 3723 3724
	} else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
		ixgbe_init_fdir_perfect_82599(&adapter->hw,
					      adapter->fdir_pballoc);
		ixgbe_fdir_filter_restore(adapter);
3725
	}
3726
	ixgbe_configure_virtualization(adapter);
3727

3728 3729 3730 3731
	ixgbe_configure_tx(adapter);
	ixgbe_configure_rx(adapter);
}

3732 3733 3734 3735 3736 3737 3738
static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
{
	switch (hw->phy.type) {
	case ixgbe_phy_sfp_avago:
	case ixgbe_phy_sfp_ftl:
	case ixgbe_phy_sfp_intel:
	case ixgbe_phy_sfp_unknown:
3739 3740 3741 3742
	case ixgbe_phy_sfp_passive_tyco:
	case ixgbe_phy_sfp_passive_unknown:
	case ixgbe_phy_sfp_active_unknown:
	case ixgbe_phy_sfp_ftl_active:
3743 3744 3745 3746 3747 3748
		return true;
	default:
		return false;
	}
}

3749
/**
3750 3751 3752 3753 3754
 * ixgbe_sfp_link_config - set up SFP+ link
 * @adapter: pointer to private adapter struct
 **/
static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
{
3755 3756 3757 3758 3759 3760 3761 3762
	/*
	 * We are assuming the worst case scenerio here, and that
	 * is that an SFP was inserted/removed after the reset
	 * but before SFP detection was enabled.  As such the best
	 * solution is to just start searching as soon as we start
	 */
	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
		adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
3763

3764
	adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
3765 3766 3767 3768
}

/**
 * ixgbe_non_sfp_link_config - set up non-SFP+ link
3769 3770 3771 3772
 * @hw: pointer to private hardware struct
 *
 * Returns 0 on success, negative on failure
 **/
3773
static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
3774 3775
{
	u32 autoneg;
3776
	bool negotiation, link_up = false;
3777 3778 3779 3780 3781 3782 3783 3784
	u32 ret = IXGBE_ERR_LINK_SETUP;

	if (hw->mac.ops.check_link)
		ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false);

	if (ret)
		goto link_cfg_out;

3785 3786
	autoneg = hw->phy.autoneg_advertised;
	if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
3787 3788
		ret = hw->mac.ops.get_link_capabilities(hw, &autoneg,
							&negotiation);
3789 3790 3791
	if (ret)
		goto link_cfg_out;

3792 3793
	if (hw->mac.ops.setup_link)
		ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up);
3794 3795 3796 3797
link_cfg_out:
	return ret;
}

3798
static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
3799 3800
{
	struct ixgbe_hw *hw = &adapter->hw;
3801
	u32 gpie = 0;
3802

3803
	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3804 3805 3806
		gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
		       IXGBE_GPIE_OCD;
		gpie |= IXGBE_GPIE_EIAME;
3807 3808 3809 3810 3811 3812 3813 3814 3815
		/*
		 * use EIAM to auto-mask when MSI-X interrupt is asserted
		 * this saves a register write for every interrupt
		 */
		switch (hw->mac.type) {
		case ixgbe_mac_82598EB:
			IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
			break;
		case ixgbe_mac_82599EB:
3816 3817
		case ixgbe_mac_X540:
		default:
3818 3819 3820 3821 3822
			IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
			IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
			break;
		}
	} else {
3823 3824 3825 3826
		/* legacy interrupts, use EIAM to auto-mask when reading EICR,
		 * specifically only auto mask tx and rx interrupts */
		IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
	}
3827

3828 3829 3830 3831 3832 3833
	/* XXX: to interrupt immediately for EICS writes, enable this */
	/* gpie |= IXGBE_GPIE_EIMEN; */

	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
		gpie &= ~IXGBE_GPIE_VTMODE_MASK;
		gpie |= IXGBE_GPIE_VTMODE_64;
3834 3835
	}

3836 3837
	/* Enable fan failure interrupt */
	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
3838 3839
		gpie |= IXGBE_SDP1_GPIEN;

3840
	if (hw->mac.type == ixgbe_mac_82599EB) {
3841 3842
		gpie |= IXGBE_SDP1_GPIEN;
		gpie |= IXGBE_SDP2_GPIEN;
3843
	}
3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855

	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
}

static int ixgbe_up_complete(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	int err;
	u32 ctrl_ext;

	ixgbe_get_hw_control(adapter);
	ixgbe_setup_gpie(adapter);
3856

3857 3858 3859 3860 3861
	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
		ixgbe_configure_msix(adapter);
	else
		ixgbe_configure_msi_and_legacy(adapter);

3862 3863 3864
	/* enable the optics for both mult-speed fiber and 82599 SFP+ fiber */
	if (hw->mac.ops.enable_tx_laser &&
	    ((hw->phy.multispeed_fiber) ||
3865
	     ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
3866
	      (hw->mac.type == ixgbe_mac_82599EB))))
3867 3868
		hw->mac.ops.enable_tx_laser(hw);

3869
	clear_bit(__IXGBE_DOWN, &adapter->state);
3870 3871
	ixgbe_napi_enable_all(adapter);

3872 3873 3874 3875 3876 3877 3878 3879
	if (ixgbe_is_sfp(hw)) {
		ixgbe_sfp_link_config(adapter);
	} else {
		err = ixgbe_non_sfp_link_config(hw);
		if (err)
			e_err(probe, "link_config FAILED %d\n", err);
	}

3880 3881
	/* clear any pending interrupts, may auto mask */
	IXGBE_READ_REG(hw, IXGBE_EICR);
3882
	ixgbe_irq_enable(adapter, true, true);
3883

3884 3885 3886 3887 3888 3889 3890
	/*
	 * If this adapter has a fan, check to see if we had a failure
	 * before we enabled the interrupt.
	 */
	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
		u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
		if (esdp & IXGBE_ESDP_SDP1)
3891
			e_crit(drv, "Fan has stopped, replace the adapter\n");
3892 3893
	}

3894
	/* enable transmits */
3895
	netif_tx_start_all_queues(adapter->netdev);
3896

3897 3898
	/* bring the link up in the watchdog, this could race with our first
	 * link up interrupt but shouldn't be a problem */
3899 3900
	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
	adapter->link_check_timeout = jiffies;
3901
	mod_timer(&adapter->service_timer, jiffies);
3902 3903 3904 3905 3906 3907

	/* Set PF Reset Done bit so PF/VF Mail Ops can work */
	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
	ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);

3908 3909 3910
	return 0;
}

3911 3912 3913
void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
{
	WARN_ON(in_interrupt());
3914 3915 3916
	/* put off any impending NetWatchDogTimeout */
	adapter->netdev->trans_start = jiffies;

3917
	while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
3918
		usleep_range(1000, 2000);
3919
	ixgbe_down(adapter);
3920 3921 3922 3923 3924 3925 3926 3927
	/*
	 * If SR-IOV enabled then wait a bit before bringing the adapter
	 * back up to give the VFs time to respond to the reset.  The
	 * two second wait is based upon the watchdog timer cycle in
	 * the VF driver.
	 */
	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
		msleep(2000);
3928 3929 3930 3931
	ixgbe_up(adapter);
	clear_bit(__IXGBE_RESETTING, &adapter->state);
}

3932 3933 3934 3935 3936 3937 3938 3939 3940 3941
int ixgbe_up(struct ixgbe_adapter *adapter)
{
	/* hardware has been reset, we need to reload some things */
	ixgbe_configure(adapter);

	return ixgbe_up_complete(adapter);
}

void ixgbe_reset(struct ixgbe_adapter *adapter)
{
3942
	struct ixgbe_hw *hw = &adapter->hw;
3943 3944
	int err;

3945 3946 3947 3948 3949 3950 3951 3952 3953
	/* lock SFP init bit to prevent race conditions with the watchdog */
	while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
		usleep_range(1000, 2000);

	/* clear all SFP and link config related flags while holding SFP_INIT */
	adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
			     IXGBE_FLAG2_SFP_NEEDS_RESET);
	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;

3954
	err = hw->mac.ops.init_hw(hw);
3955 3956 3957
	switch (err) {
	case 0:
	case IXGBE_ERR_SFP_NOT_PRESENT:
3958
	case IXGBE_ERR_SFP_NOT_SUPPORTED:
3959 3960
		break;
	case IXGBE_ERR_MASTER_REQUESTS_PENDING:
3961
		e_dev_err("master disable timed out\n");
3962
		break;
3963 3964
	case IXGBE_ERR_EEPROM_VERSION:
		/* We are running on a pre-production device, log a warning */
3965 3966 3967 3968 3969 3970
		e_dev_warn("This device is a pre-production adapter/LOM. "
			   "Please be aware there may be issuesassociated with "
			   "your hardware.  If you are experiencing problems "
			   "please contact your Intel or hardware "
			   "representative who provided you with this "
			   "hardware.\n");
3971
		break;
3972
	default:
3973
		e_dev_err("Hardware Error: %d\n", err);
3974
	}
3975

3976 3977
	clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);

3978
	/* reprogram the RAR[0] in case user changed it. */
3979 3980
	hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
			    IXGBE_RAH_AV);
3981 3982 3983 3984 3985 3986
}

/**
 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
 * @rx_ring: ring to free buffers from
 **/
3987
static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
3988
{
3989
	struct device *dev = rx_ring->dev;
3990
	unsigned long size;
3991
	u16 i;
3992

3993 3994 3995
	/* ring already cleared, nothing to do */
	if (!rx_ring->rx_buffer_info)
		return;
3996

3997
	/* Free all the Rx ring sk_buffs */
3998 3999 4000 4001 4002
	for (i = 0; i < rx_ring->count; i++) {
		struct ixgbe_rx_buffer *rx_buffer_info;

		rx_buffer_info = &rx_ring->rx_buffer_info[i];
		if (rx_buffer_info->dma) {
4003
			dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
4004
					 rx_ring->rx_buf_len,
4005
					 DMA_FROM_DEVICE);
4006 4007 4008
			rx_buffer_info->dma = 0;
		}
		if (rx_buffer_info->skb) {
4009
			struct sk_buff *skb = rx_buffer_info->skb;
4010
			rx_buffer_info->skb = NULL;
4011 4012
			do {
				struct sk_buff *this = skb;
4013
				if (IXGBE_RSC_CB(this)->delay_unmap) {
4014
					dma_unmap_single(dev,
4015
							 IXGBE_RSC_CB(this)->dma,
4016
							 rx_ring->rx_buf_len,
4017
							 DMA_FROM_DEVICE);
4018
					IXGBE_RSC_CB(this)->dma = 0;
4019
					IXGBE_RSC_CB(skb)->delay_unmap = false;
4020
				}
4021 4022 4023
				skb = skb->prev;
				dev_kfree_skb(this);
			} while (skb);
4024 4025 4026
		}
		if (!rx_buffer_info->page)
			continue;
4027
		if (rx_buffer_info->page_dma) {
4028
			dma_unmap_page(dev, rx_buffer_info->page_dma,
4029
				       PAGE_SIZE / 2, DMA_FROM_DEVICE);
4030 4031
			rx_buffer_info->page_dma = 0;
		}
4032 4033
		put_page(rx_buffer_info->page);
		rx_buffer_info->page = NULL;
4034
		rx_buffer_info->page_offset = 0;
4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050
	}

	size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
	memset(rx_ring->rx_buffer_info, 0, size);

	/* Zero out the descriptor ring */
	memset(rx_ring->desc, 0, rx_ring->size);

	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;
}

/**
 * ixgbe_clean_tx_ring - Free Tx Buffers
 * @tx_ring: ring to be cleaned
 **/
4051
static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
4052 4053 4054
{
	struct ixgbe_tx_buffer *tx_buffer_info;
	unsigned long size;
4055
	u16 i;
4056

4057 4058 4059
	/* ring already cleared, nothing to do */
	if (!tx_ring->tx_buffer_info)
		return;
4060

4061
	/* Free all the Tx ring sk_buffs */
4062 4063
	for (i = 0; i < tx_ring->count; i++) {
		tx_buffer_info = &tx_ring->tx_buffer_info[i];
4064
		ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077
	}

	size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
	memset(tx_ring->tx_buffer_info, 0, size);

	/* Zero out the descriptor ring */
	memset(tx_ring->desc, 0, tx_ring->size);

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
}

/**
4078
 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
4079 4080
 * @adapter: board private structure
 **/
4081
static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
4082 4083 4084
{
	int i;

4085
	for (i = 0; i < adapter->num_rx_queues; i++)
4086
		ixgbe_clean_rx_ring(adapter->rx_ring[i]);
4087 4088 4089
}

/**
4090
 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
4091 4092
 * @adapter: board private structure
 **/
4093
static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
4094 4095 4096
{
	int i;

4097
	for (i = 0; i < adapter->num_tx_queues; i++)
4098
		ixgbe_clean_tx_ring(adapter->tx_ring[i]);
4099 4100
}

4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117
static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
{
	struct hlist_node *node, *node2;
	struct ixgbe_fdir_filter *filter;

	spin_lock(&adapter->fdir_perfect_lock);

	hlist_for_each_entry_safe(filter, node, node2,
				  &adapter->fdir_filter_list, fdir_node) {
		hlist_del(&filter->fdir_node);
		kfree(filter);
	}
	adapter->fdir_filter_count = 0;

	spin_unlock(&adapter->fdir_perfect_lock);
}

4118 4119 4120
void ixgbe_down(struct ixgbe_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
4121
	struct ixgbe_hw *hw = &adapter->hw;
4122
	u32 rxctrl;
4123
	int i;
4124
	int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
4125 4126 4127 4128 4129

	/* signal that we are down to the interrupt handler */
	set_bit(__IXGBE_DOWN, &adapter->state);

	/* disable receives */
4130 4131
	rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
	IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
4132

4133 4134 4135 4136 4137
	/* disable all enabled rx queues */
	for (i = 0; i < adapter->num_rx_queues; i++)
		/* this call also flushes the previous write */
		ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);

4138
	usleep_range(10000, 20000);
4139

4140 4141
	netif_tx_stop_all_queues(netdev);

4142
	/* call carrier off first to avoid false dev_watchdog timeouts */
4143 4144 4145 4146 4147 4148 4149
	netif_carrier_off(netdev);
	netif_tx_disable(netdev);

	ixgbe_irq_disable(adapter);

	ixgbe_napi_disable_all(adapter);

4150 4151
	adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT |
			     IXGBE_FLAG2_RESET_REQUESTED);
4152 4153 4154 4155
	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;

	del_timer_sync(&adapter->service_timer);

4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168
	/* disable receive for all VFs and wait one second */
	if (adapter->num_vfs) {
		/* ping all the active vfs to let them know we are going down */
		ixgbe_ping_all_vfs(adapter);

		/* Disable all VFTE/VFRE TX/RX */
		ixgbe_disable_tx_rx(adapter);

		/* Mark all the VFs as inactive */
		for (i = 0 ; i < adapter->num_vfs; i++)
			adapter->vfinfo[i].clear_to_send = 0;
	}

4169 4170 4171 4172 4173 4174 4175 4176 4177
	/* Cleanup the affinity_hint CPU mask memory and callback */
	for (i = 0; i < num_q_vectors; i++) {
		struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
		/* clear the affinity_mask in the IRQ descriptor */
		irq_set_affinity_hint(adapter->msix_entries[i]. vector, NULL);
		/* release the CPU mask memory */
		free_cpumask_var(q_vector->affinity_mask);
	}

4178 4179
	/* disable transmits in the hardware now that interrupts are off */
	for (i = 0; i < adapter->num_tx_queues; i++) {
4180
		u8 reg_idx = adapter->tx_ring[i]->reg_idx;
4181
		IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
4182
	}
4183 4184

	/* Disable the Tx DMA engine on 82599 and X540 */
4185 4186
	switch (hw->mac.type) {
	case ixgbe_mac_82599EB:
4187
	case ixgbe_mac_X540:
4188
		IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
4189 4190
				(IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
				 ~IXGBE_DMATXCTL_TE));
4191 4192 4193 4194
		break;
	default:
		break;
	}
4195

4196 4197
	if (!pci_channel_offline(adapter->pdev))
		ixgbe_reset(adapter);
4198 4199 4200 4201

	/* power down the optics for multispeed fiber and 82599 SFP+ fiber */
	if (hw->mac.ops.disable_tx_laser &&
	    ((hw->phy.multispeed_fiber) ||
4202
	     ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
4203 4204 4205
	      (hw->mac.type == ixgbe_mac_82599EB))))
		hw->mac.ops.disable_tx_laser(hw);

4206 4207 4208
	ixgbe_clean_all_tx_rings(adapter);
	ixgbe_clean_all_rx_rings(adapter);

4209
#ifdef CONFIG_IXGBE_DCA
4210
	/* since we reset the hardware DCA settings were cleared */
4211
	ixgbe_setup_dca(adapter);
4212
#endif
4213 4214 4215
}

/**
4216 4217 4218 4219 4220
 * ixgbe_poll - NAPI Rx polling callback
 * @napi: structure for representing this polling device
 * @budget: how many packets driver is allowed to clean
 *
 * This function is used for legacy and MSI, NAPI mode
4221
 **/
4222
static int ixgbe_poll(struct napi_struct *napi, int budget)
4223
{
4224
	struct ixgbe_q_vector *q_vector =
4225
				container_of(napi, struct ixgbe_q_vector, napi);
4226
	struct ixgbe_adapter *adapter = q_vector->adapter;
4227
	int tx_clean_complete, work_done = 0;
4228

4229
#ifdef CONFIG_IXGBE_DCA
4230 4231
	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
		ixgbe_update_dca(q_vector);
4232 4233
#endif

4234 4235
	tx_clean_complete = ixgbe_clean_tx_irq(q_vector, adapter->tx_ring[0]);
	ixgbe_clean_rx_irq(q_vector, adapter->rx_ring[0], &work_done, budget);
4236

4237
	if (!tx_clean_complete)
4238 4239
		work_done = budget;

4240 4241
	/* If budget not fully consumed, exit the polling mode */
	if (work_done < budget) {
4242
		napi_complete(napi);
4243
		if (adapter->rx_itr_setting & 1)
4244
			ixgbe_set_itr(q_vector);
4245
		if (!test_bit(__IXGBE_DOWN, &adapter->state))
4246
			ixgbe_irq_enable_queues(adapter, IXGBE_EIMS_RTX_QUEUE);
4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259
	}
	return work_done;
}

/**
 * ixgbe_tx_timeout - Respond to a Tx Hang
 * @netdev: network interface device structure
 **/
static void ixgbe_tx_timeout(struct net_device *netdev)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);

	/* Do the reset outside of interrupt context */
4260
	ixgbe_tx_timeout_reset(adapter);
4261 4262
}

4263 4264 4265 4266 4267 4268 4269 4270
/**
 * ixgbe_set_rss_queues: Allocate queues for RSS
 * @adapter: board private structure to initialize
 *
 * This is our "base" multiqueue mode.  RSS (Receive Side Scaling) will try
 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
 *
 **/
4271 4272 4273
static inline bool ixgbe_set_rss_queues(struct ixgbe_adapter *adapter)
{
	bool ret = false;
4274
	struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_RSS];
4275 4276

	if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
4277 4278 4279
		f->mask = 0xF;
		adapter->num_rx_queues = f->indices;
		adapter->num_tx_queues = f->indices;
4280 4281 4282
		ret = true;
	} else {
		ret = false;
4283 4284
	}

4285 4286 4287
	return ret;
}

4288 4289 4290 4291 4292 4293 4294 4295 4296 4297
/**
 * ixgbe_set_fdir_queues: Allocate queues for Flow Director
 * @adapter: board private structure to initialize
 *
 * Flow Director is an advanced Rx filter, attempting to get Rx flows back
 * to the original CPU that initiated the Tx session.  This runs in addition
 * to RSS, so if a packet doesn't match an FDIR filter, we can still spread the
 * Rx load across CPUs using RSS.
 *
 **/
4298
static inline bool ixgbe_set_fdir_queues(struct ixgbe_adapter *adapter)
4299 4300 4301 4302 4303 4304 4305 4306
{
	bool ret = false;
	struct ixgbe_ring_feature *f_fdir = &adapter->ring_feature[RING_F_FDIR];

	f_fdir->indices = min((int)num_online_cpus(), f_fdir->indices);
	f_fdir->mask = 0;

	/* Flow Director must have RSS enabled */
4307 4308
	if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
	    (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) {
4309 4310 4311 4312 4313 4314 4315 4316 4317
		adapter->num_tx_queues = f_fdir->indices;
		adapter->num_rx_queues = f_fdir->indices;
		ret = true;
	} else {
		adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
	}
	return ret;
}

4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332
#ifdef IXGBE_FCOE
/**
 * ixgbe_set_fcoe_queues: Allocate queues for Fiber Channel over Ethernet (FCoE)
 * @adapter: board private structure to initialize
 *
 * FCoE RX FCRETA can use up to 8 rx queues for up to 8 different exchanges.
 * The ring feature mask is not used as a mask for FCoE, as it can take any 8
 * rx queues out of the max number of rx queues, instead, it is used as the
 * index of the first rx queue used by FCoE.
 *
 **/
static inline bool ixgbe_set_fcoe_queues(struct ixgbe_adapter *adapter)
{
	struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];

4333 4334 4335
	if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
		return false;

4336
	f->indices = min((int)num_online_cpus(), f->indices);
4337

4338 4339
	adapter->num_rx_queues = 1;
	adapter->num_tx_queues = 1;
4340

4341 4342
	if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
		e_info(probe, "FCoE enabled with RSS\n");
4343
		if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
4344 4345 4346
			ixgbe_set_fdir_queues(adapter);
		else
			ixgbe_set_rss_queues(adapter);
4347
	}
4348

4349 4350 4351 4352
	/* adding FCoE rx rings to the end */
	f->mask = adapter->num_rx_queues;
	adapter->num_rx_queues += f->indices;
	adapter->num_tx_queues += f->indices;
4353

4354 4355 4356 4357
	return true;
}
#endif /* IXGBE_FCOE */

4358 4359 4360
/* Artificial max queue cap per traffic class in DCB mode */
#define DCB_QUEUE_CAP 8

4361 4362 4363
#ifdef CONFIG_IXGBE_DCB
static inline bool ixgbe_set_dcb_queues(struct ixgbe_adapter *adapter)
{
4364 4365 4366
	int per_tc_q, q, i, offset = 0;
	struct net_device *dev = adapter->netdev;
	int tcs = netdev_get_num_tc(dev);
4367

4368 4369
	if (!tcs)
		return false;
4370

4371 4372 4373
	/* Map queue offset and counts onto allocated tx queues */
	per_tc_q = min(dev->num_tx_queues / tcs, (unsigned int)DCB_QUEUE_CAP);
	q = min((int)num_online_cpus(), per_tc_q);
4374 4375

	for (i = 0; i < tcs; i++) {
4376 4377 4378
		netdev_set_prio_tc_map(dev, i, i);
		netdev_set_tc_queue(dev, i, q, offset);
		offset += q;
4379 4380
	}

4381 4382
	adapter->num_tx_queues = q * tcs;
	adapter->num_rx_queues = q * tcs;
4383 4384

#ifdef IXGBE_FCOE
4385 4386 4387 4388
	/* FCoE enabled queues require special configuration indexed
	 * by feature specific indices and mask. Here we map FCoE
	 * indices onto the DCB queue pairs allowing FCoE to own
	 * configuration later.
4389
	 */
4390 4391 4392 4393 4394 4395 4396 4397 4398
	if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) {
		int tc;
		struct ixgbe_ring_feature *f =
					&adapter->ring_feature[RING_F_FCOE];

		tc = netdev_get_prio_tc_map(dev, adapter->fcoe.up);
		f->indices = dev->tc_to_txq[tc].count;
		f->mask = dev->tc_to_txq[tc].offset;
	}
4399 4400
#endif

4401
	return true;
4402
}
4403
#endif
4404

4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417
/**
 * ixgbe_set_sriov_queues: Allocate queues for IOV use
 * @adapter: board private structure to initialize
 *
 * IOV doesn't actually use anything, so just NAK the
 * request for now and let the other queue routines
 * figure out what to do.
 */
static inline bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter)
{
	return false;
}

4418
/*
Lucas De Marchi's avatar
Lucas De Marchi committed
4419
 * ixgbe_set_num_queues: Allocate queues for device, feature dependent
4420 4421 4422 4423 4424 4425 4426 4427 4428
 * @adapter: board private structure to initialize
 *
 * This is the top level queue allocation routine.  The order here is very
 * important, starting with the "most" number of features turned on at once,
 * and ending with the smallest set of features.  This way large combinations
 * can be allocated if they're turned on, and smaller combinations are the
 * fallthrough conditions.
 *
 **/
4429
static int ixgbe_set_num_queues(struct ixgbe_adapter *adapter)
4430
{
4431 4432 4433 4434 4435 4436 4437
	/* Start with base case */
	adapter->num_rx_queues = 1;
	adapter->num_tx_queues = 1;
	adapter->num_rx_pools = adapter->num_rx_queues;
	adapter->num_rx_queues_per_pool = 1;

	if (ixgbe_set_sriov_queues(adapter))
4438
		goto done;
4439

4440 4441
#ifdef CONFIG_IXGBE_DCB
	if (ixgbe_set_dcb_queues(adapter))
4442
		goto done;
4443 4444

#endif
4445 4446 4447 4448 4449
#ifdef IXGBE_FCOE
	if (ixgbe_set_fcoe_queues(adapter))
		goto done;

#endif /* IXGBE_FCOE */
4450 4451 4452
	if (ixgbe_set_fdir_queues(adapter))
		goto done;

4453
	if (ixgbe_set_rss_queues(adapter))
4454 4455 4456 4457 4458 4459 4460
		goto done;

	/* fallback to base case */
	adapter->num_rx_queues = 1;
	adapter->num_tx_queues = 1;

done:
4461
	/* Notify the stack of the (possibly) reduced queue counts. */
4462
	netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
4463 4464
	return netif_set_real_num_rx_queues(adapter->netdev,
					    adapter->num_rx_queues);
4465 4466
}

4467
static void ixgbe_acquire_msix_vectors(struct ixgbe_adapter *adapter,
4468
				       int vectors)
4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486
{
	int err, vector_threshold;

	/* We'll want at least 3 (vector_threshold):
	 * 1) TxQ[0] Cleanup
	 * 2) RxQ[0] Cleanup
	 * 3) Other (Link Status Change, etc.)
	 * 4) TCP Timer (optional)
	 */
	vector_threshold = MIN_MSIX_COUNT;

	/* The more we get, the more we will assign to Tx/Rx Cleanup
	 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
	 * Right now, we simply care about how many we'll get; we'll
	 * set them up later while requesting irq's.
	 */
	while (vectors >= vector_threshold) {
		err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
4487
				      vectors);
4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500
		if (!err) /* Success in acquiring all requested vectors. */
			break;
		else if (err < 0)
			vectors = 0; /* Nasty failure, quit now */
		else /* err == number of vectors we should try again with */
			vectors = err;
	}

	if (vectors < vector_threshold) {
		/* Can't allocate enough MSI-X interrupts?  Oh well.
		 * This just means we'll go with either a single MSI
		 * vector or fall back to legacy interrupts.
		 */
4501 4502
		netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
			     "Unable to allocate MSI-X interrupts\n");
4503 4504 4505 4506 4507
		adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
		kfree(adapter->msix_entries);
		adapter->msix_entries = NULL;
	} else {
		adapter->flags |= IXGBE_FLAG_MSIX_ENABLED; /* Woot! */
4508 4509 4510 4511 4512 4513
		/*
		 * Adjust for only the vectors we'll use, which is minimum
		 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
		 * vectors we were allocated.
		 */
		adapter->num_msix_vectors = min(vectors,
4514
				   adapter->max_msix_q_vectors + NON_Q_VECTORS);
4515 4516 4517 4518
	}
}

/**
4519
 * ixgbe_cache_ring_rss - Descriptor ring to register mapping for RSS
4520 4521
 * @adapter: board private structure to initialize
 *
4522 4523
 * Cache the descriptor ring offsets for RSS to the assigned rings.
 *
4524
 **/
4525
static inline bool ixgbe_cache_ring_rss(struct ixgbe_adapter *adapter)
4526
{
4527 4528
	int i;

4529 4530
	if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED))
		return false;
4531

4532 4533 4534 4535 4536 4537
	for (i = 0; i < adapter->num_rx_queues; i++)
		adapter->rx_ring[i]->reg_idx = i;
	for (i = 0; i < adapter->num_tx_queues; i++)
		adapter->tx_ring[i]->reg_idx = i;

	return true;
4538 4539 4540
}

#ifdef CONFIG_IXGBE_DCB
4541 4542

/* ixgbe_get_first_reg_idx - Return first register index associated with ring */
John Fastabend's avatar
John Fastabend committed
4543 4544
static void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc,
				    unsigned int *tx, unsigned int *rx)
4545 4546 4547 4548 4549 4550 4551 4552 4553 4554
{
	struct net_device *dev = adapter->netdev;
	struct ixgbe_hw *hw = &adapter->hw;
	u8 num_tcs = netdev_get_num_tc(dev);

	*tx = 0;
	*rx = 0;

	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
4555 4556
		*tx = tc << 2;
		*rx = tc << 3;
4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595
		break;
	case ixgbe_mac_82599EB:
	case ixgbe_mac_X540:
		if (num_tcs == 8) {
			if (tc < 3) {
				*tx = tc << 5;
				*rx = tc << 4;
			} else if (tc <  5) {
				*tx = ((tc + 2) << 4);
				*rx = tc << 4;
			} else if (tc < num_tcs) {
				*tx = ((tc + 8) << 3);
				*rx = tc << 4;
			}
		} else if (num_tcs == 4) {
			*rx =  tc << 5;
			switch (tc) {
			case 0:
				*tx =  0;
				break;
			case 1:
				*tx = 64;
				break;
			case 2:
				*tx = 96;
				break;
			case 3:
				*tx = 112;
				break;
			default:
				break;
			}
		}
		break;
	default:
		break;
	}
}

4596 4597 4598 4599 4600 4601 4602 4603 4604
/**
 * ixgbe_cache_ring_dcb - Descriptor ring to register mapping for DCB
 * @adapter: board private structure to initialize
 *
 * Cache the descriptor ring offsets for DCB to the assigned rings.
 *
 **/
static inline bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter)
{
4605 4606 4607
	struct net_device *dev = adapter->netdev;
	int i, j, k;
	u8 num_tcs = netdev_get_num_tc(dev);
4608

4609
	if (!num_tcs)
4610
		return false;
4611

4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
	for (i = 0, k = 0; i < num_tcs; i++) {
		unsigned int tx_s, rx_s;
		u16 count = dev->tc_to_txq[i].count;

		ixgbe_get_first_reg_idx(adapter, i, &tx_s, &rx_s);
		for (j = 0; j < count; j++, k++) {
			adapter->tx_ring[k]->reg_idx = tx_s + j;
			adapter->rx_ring[k]->reg_idx = rx_s + j;
			adapter->tx_ring[k]->dcb_tc = i;
			adapter->rx_ring[k]->dcb_tc = i;
4622 4623
		}
	}
4624 4625

	return true;
4626 4627 4628
}
#endif

4629 4630 4631 4632 4633 4634 4635
/**
 * ixgbe_cache_ring_fdir - Descriptor ring to register mapping for Flow Director
 * @adapter: board private structure to initialize
 *
 * Cache the descriptor ring offsets for Flow Director to the assigned rings.
 *
 **/
4636
static inline bool ixgbe_cache_ring_fdir(struct ixgbe_adapter *adapter)
4637 4638 4639 4640
{
	int i;
	bool ret = false;

4641 4642
	if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
	    (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) {
4643
		for (i = 0; i < adapter->num_rx_queues; i++)
4644
			adapter->rx_ring[i]->reg_idx = i;
4645
		for (i = 0; i < adapter->num_tx_queues; i++)
4646
			adapter->tx_ring[i]->reg_idx = i;
4647 4648 4649 4650 4651 4652
		ret = true;
	}

	return ret;
}

4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663
#ifdef IXGBE_FCOE
/**
 * ixgbe_cache_ring_fcoe - Descriptor ring to register mapping for the FCoE
 * @adapter: board private structure to initialize
 *
 * Cache the descriptor ring offsets for FCoE mode to the assigned rings.
 *
 */
static inline bool ixgbe_cache_ring_fcoe(struct ixgbe_adapter *adapter)
{
	struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];
4664 4665 4666 4667 4668
	int i;
	u8 fcoe_rx_i = 0, fcoe_tx_i = 0;

	if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
		return false;
4669

4670
	if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
4671
		if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
4672 4673 4674
			ixgbe_cache_ring_fdir(adapter);
		else
			ixgbe_cache_ring_rss(adapter);
4675

4676 4677
		fcoe_rx_i = f->mask;
		fcoe_tx_i = f->mask;
4678
	}
4679 4680 4681 4682 4683
	for (i = 0; i < f->indices; i++, fcoe_rx_i++, fcoe_tx_i++) {
		adapter->rx_ring[f->mask + i]->reg_idx = fcoe_rx_i;
		adapter->tx_ring[f->mask + i]->reg_idx = fcoe_tx_i;
	}
	return true;
4684 4685 4686
}

#endif /* IXGBE_FCOE */
4687 4688 4689 4690 4691 4692 4693 4694 4695 4696
/**
 * ixgbe_cache_ring_sriov - Descriptor ring to register mapping for sriov
 * @adapter: board private structure to initialize
 *
 * SR-IOV doesn't use any descriptor rings but changes the default if
 * no other mapping is used.
 *
 */
static inline bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter)
{
4697 4698
	adapter->rx_ring[0]->reg_idx = adapter->num_vfs * 2;
	adapter->tx_ring[0]->reg_idx = adapter->num_vfs * 2;
4699 4700 4701 4702 4703 4704
	if (adapter->num_vfs)
		return true;
	else
		return false;
}

4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718
/**
 * ixgbe_cache_ring_register - Descriptor ring to register mapping
 * @adapter: board private structure to initialize
 *
 * Once we know the feature-set enabled for the device, we'll cache
 * the register offset the descriptor ring is assigned to.
 *
 * Note, the order the various feature calls is important.  It must start with
 * the "most" features enabled at the same time, then trickle down to the
 * least amount of features turned on at once.
 **/
static void ixgbe_cache_ring_register(struct ixgbe_adapter *adapter)
{
	/* start with default case */
4719 4720
	adapter->rx_ring[0]->reg_idx = 0;
	adapter->tx_ring[0]->reg_idx = 0;
4721

4722 4723 4724
	if (ixgbe_cache_ring_sriov(adapter))
		return;

4725 4726 4727 4728 4729
#ifdef CONFIG_IXGBE_DCB
	if (ixgbe_cache_ring_dcb(adapter))
		return;
#endif

4730 4731 4732 4733
#ifdef IXGBE_FCOE
	if (ixgbe_cache_ring_fcoe(adapter))
		return;
#endif /* IXGBE_FCOE */
4734

4735 4736 4737
	if (ixgbe_cache_ring_fdir(adapter))
		return;

4738 4739
	if (ixgbe_cache_ring_rss(adapter))
		return;
4740 4741
}

4742 4743 4744 4745 4746
/**
 * ixgbe_alloc_queues - Allocate memory for all rings
 * @adapter: board private structure to initialize
 *
 * We allocate one ring per queue at run-time since we don't know the
4747 4748
 * number of queues at compile-time.  The polling_netdev array is
 * intended for Multiqueue, but should work fine with a single queue.
4749
 **/
4750
static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter)
4751
{
4752
	int rx = 0, tx = 0, nid = adapter->node;
4753

4754 4755 4756 4757 4758 4759 4760
	if (nid < 0 || !node_online(nid))
		nid = first_online_node;

	for (; tx < adapter->num_tx_queues; tx++) {
		struct ixgbe_ring *ring;

		ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
4761
		if (!ring)
4762
			ring = kzalloc(sizeof(*ring), GFP_KERNEL);
4763
		if (!ring)
4764
			goto err_allocation;
4765
		ring->count = adapter->tx_ring_count;
4766 4767
		ring->queue_index = tx;
		ring->numa_node = nid;
4768
		ring->dev = &adapter->pdev->dev;
4769
		ring->netdev = adapter->netdev;
4770

4771
		adapter->tx_ring[tx] = ring;
4772
	}
4773

4774 4775
	for (; rx < adapter->num_rx_queues; rx++) {
		struct ixgbe_ring *ring;
4776

4777
		ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
4778
		if (!ring)
4779
			ring = kzalloc(sizeof(*ring), GFP_KERNEL);
4780
		if (!ring)
4781 4782 4783 4784
			goto err_allocation;
		ring->count = adapter->rx_ring_count;
		ring->queue_index = rx;
		ring->numa_node = nid;
4785
		ring->dev = &adapter->pdev->dev;
4786
		ring->netdev = adapter->netdev;
4787

4788
		adapter->rx_ring[rx] = ring;
4789 4790 4791 4792 4793 4794
	}

	ixgbe_cache_ring_register(adapter);

	return 0;

4795 4796 4797 4798 4799 4800
err_allocation:
	while (tx)
		kfree(adapter->tx_ring[--tx]);

	while (rx)
		kfree(adapter->rx_ring[--rx]);
4801 4802 4803 4804 4805 4806 4807 4808 4809 4810
	return -ENOMEM;
}

/**
 * ixgbe_set_interrupt_capability - set MSI-X or MSI if supported
 * @adapter: board private structure to initialize
 *
 * Attempt to configure the interrupts using the best available
 * capabilities of the hardware and the kernel.
 **/
Al Viro's avatar
Al Viro committed
4811
static int ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter)
4812
{
4813
	struct ixgbe_hw *hw = &adapter->hw;
4814 4815 4816 4817 4818 4819 4820
	int err = 0;
	int vector, v_budget;

	/*
	 * It's easy to be greedy for MSI-X vectors, but it really
	 * doesn't do us much good if we have a lot more vectors
	 * than CPU's.  So let's be conservative and only ask for
4821
	 * (roughly) the same number of vectors as there are CPU's.
4822 4823
	 */
	v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues,
4824
		       (int)num_online_cpus()) + NON_Q_VECTORS;
4825 4826 4827

	/*
	 * At the same time, hardware can only support a maximum of
4828 4829 4830 4831
	 * hw.mac->max_msix_vectors vectors.  With features
	 * such as RSS and VMDq, we can easily surpass the number of Rx and Tx
	 * descriptor queues supported by our device.  Thus, we cap it off in
	 * those rare cases where the cpu count also exceeds our vector limit.
4832
	 */
4833
	v_budget = min(v_budget, (int)hw->mac.max_msix_vectors);
4834 4835 4836 4837

	/* A failure in MSI-X entry allocation isn't fatal, but it does
	 * mean we disable MSI-X capabilities of the adapter. */
	adapter->msix_entries = kcalloc(v_budget,
4838
					sizeof(struct msix_entry), GFP_KERNEL);
4839 4840 4841
	if (adapter->msix_entries) {
		for (vector = 0; vector < v_budget; vector++)
			adapter->msix_entries[vector].entry = vector;
4842

4843
		ixgbe_acquire_msix_vectors(adapter, v_budget);
4844

4845 4846 4847
		if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
			goto out;
	}
4848

4849 4850
	adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
	adapter->flags &= ~IXGBE_FLAG_RSS_ENABLED;
4851
	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
4852
		e_err(probe,
4853
		      "ATR is not supported while multiple "
4854 4855
		      "queues are disabled.  Disabling Flow Director\n");
	}
4856 4857
	adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
	adapter->atr_sample_rate = 0;
4858 4859 4860
	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
		ixgbe_disable_sriov(adapter);

4861 4862 4863
	err = ixgbe_set_num_queues(adapter);
	if (err)
		return err;
4864 4865 4866 4867 4868

	err = pci_enable_msi(adapter->pdev);
	if (!err) {
		adapter->flags |= IXGBE_FLAG_MSI_ENABLED;
	} else {
4869 4870 4871
		netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
			     "Unable to allocate MSI interrupt, "
			     "falling back to legacy.  Error: %d\n", err);
4872 4873 4874 4875 4876 4877 4878 4879
		/* reset err */
		err = 0;
	}

out:
	return err;
}

4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894
/**
 * ixgbe_alloc_q_vectors - Allocate memory for interrupt vectors
 * @adapter: board private structure to initialize
 *
 * We allocate one q_vector per queue interrupt.  If allocation fails we
 * return -ENOMEM.
 **/
static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter)
{
	int q_idx, num_q_vectors;
	struct ixgbe_q_vector *q_vector;
	int (*poll)(struct napi_struct *, int);

	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
		num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
4895
		poll = &ixgbe_clean_rxtx_many;
4896 4897 4898 4899 4900 4901
	} else {
		num_q_vectors = 1;
		poll = &ixgbe_poll;
	}

	for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
4902
		q_vector = kzalloc_node(sizeof(struct ixgbe_q_vector),
4903
					GFP_KERNEL, adapter->node);
4904 4905
		if (!q_vector)
			q_vector = kzalloc(sizeof(struct ixgbe_q_vector),
4906
					   GFP_KERNEL);
4907 4908 4909
		if (!q_vector)
			goto err_out;
		q_vector->adapter = adapter;
4910
		if (q_vector->tx.count && !q_vector->rx.count)
4911 4912 4913
			q_vector->eitr = adapter->tx_eitr_param;
		else
			q_vector->eitr = adapter->rx_eitr_param;
4914
		q_vector->v_idx = q_idx;
4915
		netif_napi_add(adapter->netdev, &q_vector->napi, (*poll), 64);
4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943
		adapter->q_vector[q_idx] = q_vector;
	}

	return 0;

err_out:
	while (q_idx) {
		q_idx--;
		q_vector = adapter->q_vector[q_idx];
		netif_napi_del(&q_vector->napi);
		kfree(q_vector);
		adapter->q_vector[q_idx] = NULL;
	}
	return -ENOMEM;
}

/**
 * ixgbe_free_q_vectors - Free memory allocated for interrupt vectors
 * @adapter: board private structure to initialize
 *
 * This function frees the memory allocated to the q_vectors.  In addition if
 * NAPI is enabled it will delete any references to the NAPI struct prior
 * to freeing the q_vector.
 **/
static void ixgbe_free_q_vectors(struct ixgbe_adapter *adapter)
{
	int q_idx, num_q_vectors;

4944
	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
4945
		num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
4946
	else
4947 4948 4949 4950 4951
		num_q_vectors = 1;

	for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
		struct ixgbe_q_vector *q_vector = adapter->q_vector[q_idx];
		adapter->q_vector[q_idx] = NULL;
4952
		netif_napi_del(&q_vector->napi);
4953 4954 4955 4956
		kfree(q_vector);
	}
}

4957
static void ixgbe_reset_interrupt_capability(struct ixgbe_adapter *adapter)
4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979
{
	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
		adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
		pci_disable_msix(adapter->pdev);
		kfree(adapter->msix_entries);
		adapter->msix_entries = NULL;
	} else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) {
		adapter->flags &= ~IXGBE_FLAG_MSI_ENABLED;
		pci_disable_msi(adapter->pdev);
	}
}

/**
 * ixgbe_init_interrupt_scheme - Determine proper interrupt scheme
 * @adapter: board private structure to initialize
 *
 * We determine which interrupt scheme to use based on...
 * - Kernel support (MSI, MSI-X)
 *   - which can be user-defined (via MODULE_PARAM)
 * - Hardware queue count (num_*_queues)
 *   - defined by miscellaneous hardware support/features (RSS, etc.)
 **/
4980
int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter)
4981 4982 4983 4984
{
	int err;

	/* Number of supported queues */
4985 4986 4987
	err = ixgbe_set_num_queues(adapter);
	if (err)
		return err;
4988 4989 4990

	err = ixgbe_set_interrupt_capability(adapter);
	if (err) {
4991
		e_dev_err("Unable to setup interrupt capabilities\n");
4992
		goto err_set_interrupt;
4993 4994
	}

4995 4996
	err = ixgbe_alloc_q_vectors(adapter);
	if (err) {
4997
		e_dev_err("Unable to allocate memory for queue vectors\n");
4998 4999 5000 5001 5002
		goto err_alloc_q_vectors;
	}

	err = ixgbe_alloc_queues(adapter);
	if (err) {
5003
		e_dev_err("Unable to allocate memory for queues\n");
5004 5005 5006
		goto err_alloc_queues;
	}

5007
	e_dev_info("Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n",
5008 5009
		   (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled",
		   adapter->num_rx_queues, adapter->num_tx_queues);
5010 5011 5012

	set_bit(__IXGBE_DOWN, &adapter->state);

5013
	return 0;
5014

5015 5016 5017 5018
err_alloc_queues:
	ixgbe_free_q_vectors(adapter);
err_alloc_q_vectors:
	ixgbe_reset_interrupt_capability(adapter);
5019
err_set_interrupt:
5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031
	return err;
}

/**
 * ixgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings
 * @adapter: board private structure to clear interrupt scheme on
 *
 * We go through and clear interrupt specific resources and reset the structure
 * to pre-load conditions
 **/
void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter)
{
5032 5033 5034 5035 5036 5037 5038
	int i;

	for (i = 0; i < adapter->num_tx_queues; i++) {
		kfree(adapter->tx_ring[i]);
		adapter->tx_ring[i] = NULL;
	}
	for (i = 0; i < adapter->num_rx_queues; i++) {
Eric Dumazet's avatar
Eric Dumazet committed
5039 5040 5041 5042 5043
		struct ixgbe_ring *ring = adapter->rx_ring[i];

		/* ixgbe_get_stats64() might access this ring, we must wait
		 * a grace period before freeing it.
		 */
5044
		kfree_rcu(ring, rcu);
5045 5046
		adapter->rx_ring[i] = NULL;
	}
5047

5048 5049 5050
	adapter->num_tx_queues = 0;
	adapter->num_rx_queues = 0;

5051 5052
	ixgbe_free_q_vectors(adapter);
	ixgbe_reset_interrupt_capability(adapter);
5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066
}

/**
 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
 * @adapter: board private structure to initialize
 *
 * ixgbe_sw_init initializes the Adapter private data structure.
 * Fields are initialized based on PCI device information and
 * OS network device settings (MTU size).
 **/
static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	struct pci_dev *pdev = adapter->pdev;
5067
	struct net_device *dev = adapter->netdev;
5068
	unsigned int rss;
Jeff Kirsher's avatar
Jeff Kirsher committed
5069
#ifdef CONFIG_IXGBE_DCB
5070 5071 5072
	int j;
	struct tc_configuration *tc;
#endif
5073
	int max_frame = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
5074

5075 5076 5077 5078 5079 5080 5081 5082
	/* PCI config space info */

	hw->vendor_id = pdev->vendor;
	hw->device_id = pdev->device;
	hw->revision_id = pdev->revision;
	hw->subsystem_vendor_id = pdev->subsystem_vendor;
	hw->subsystem_device_id = pdev->subsystem_device;

5083 5084 5085 5086
	/* Set capability flags */
	rss = min(IXGBE_MAX_RSS_INDICES, (int)num_online_cpus());
	adapter->ring_feature[RING_F_RSS].indices = rss;
	adapter->flags |= IXGBE_FLAG_RSS_ENABLED;
5087 5088
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
5089 5090
		if (hw->device_id == IXGBE_DEV_ID_82598AT)
			adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
5091
		adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82598;
5092 5093
		break;
	case ixgbe_mac_82599EB:
5094
	case ixgbe_mac_X540:
5095
		adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82599;
5096 5097
		adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
		adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
5098 5099
		if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
			adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
5100 5101 5102
		/* Flow Director hash filters enabled */
		adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
		adapter->atr_sample_rate = 20;
5103
		adapter->ring_feature[RING_F_FDIR].indices =
5104
							 IXGBE_MAX_FDIR_INDICES;
5105
		adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
5106
#ifdef IXGBE_FCOE
5107 5108 5109
		adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
		adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
		adapter->ring_feature[RING_F_FCOE].indices = 0;
5110
#ifdef CONFIG_IXGBE_DCB
5111
		/* Default traffic class to use for FCoE */
5112
		adapter->fcoe.up = IXGBE_FCOE_DEFTC;
5113
#endif
5114
#endif /* IXGBE_FCOE */
5115 5116 5117
		break;
	default:
		break;
5118
	}
5119

5120 5121 5122
	/* n-tuple support exists, always init our spinlock */
	spin_lock_init(&adapter->fdir_perfect_lock);

Jeff Kirsher's avatar
Jeff Kirsher committed
5123
#ifdef CONFIG_IXGBE_DCB
5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134
	/* Configure DCB traffic classes */
	for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
		tc = &adapter->dcb_cfg.tc_config[j];
		tc->path[DCB_TX_CONFIG].bwg_id = 0;
		tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
		tc->path[DCB_RX_CONFIG].bwg_id = 0;
		tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
		tc->dcb_pfc = pfc_disabled;
	}
	adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
	adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
5135
	adapter->dcb_cfg.pfc_mode_enable = false;
5136
	adapter->dcb_set_bitmap = 0x00;
5137
	adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
5138
	ixgbe_copy_dcb_cfg(&adapter->dcb_cfg, &adapter->temp_dcb_cfg,
5139
			   MAX_TRAFFIC_CLASS);
5140 5141

#endif
5142 5143

	/* default flow control settings */
5144
	hw->fc.requested_mode = ixgbe_fc_full;
5145
	hw->fc.current_mode = ixgbe_fc_full;	/* init for ethtool output */
5146 5147 5148
#ifdef CONFIG_DCB
	adapter->last_lfc_mode = hw->fc.current_mode;
#endif
5149 5150
	hw->fc.high_water = FC_HIGH_WATER(max_frame);
	hw->fc.low_water = FC_LOW_WATER(max_frame);
5151 5152
	hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
	hw->fc.send_xon = true;
5153
	hw->fc.disable_fc_autoneg = false;
5154

5155
	/* enable itr by default in dynamic mode */
5156 5157 5158 5159
	adapter->rx_itr_setting = 1;
	adapter->rx_eitr_param = 20000;
	adapter->tx_itr_setting = 1;
	adapter->tx_eitr_param = 10000;
5160 5161 5162 5163 5164 5165 5166 5167 5168

	/* set defaults for eitr in MegaBytes */
	adapter->eitr_low = 10;
	adapter->eitr_high = 20;

	/* set default ring sizes */
	adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
	adapter->rx_ring_count = IXGBE_DEFAULT_RXD;

5169 5170 5171
	/* set default work limits */
	adapter->tx_work_limit = adapter->tx_ring_count;

5172
	/* initialize eeprom parameters */
5173
	if (ixgbe_init_eeprom_params_generic(hw)) {
5174
		e_dev_err("EEPROM initialization failed\n");
5175 5176 5177
		return -EIO;
	}

5178
	/* enable rx csum by default */
5179 5180
	adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;

5181 5182 5183
	/* get assigned NUMA node */
	adapter->node = dev_to_node(&pdev->dev);

5184 5185 5186 5187 5188 5189 5190
	set_bit(__IXGBE_DOWN, &adapter->state);

	return 0;
}

/**
 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
5191
 * @tx_ring:    tx descriptor ring (for a specific queue) to setup
5192 5193 5194
 *
 * Return 0 on success, negative on failure
 **/
5195
int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
5196
{
5197
	struct device *dev = tx_ring->dev;
5198 5199
	int size;

5200
	size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
Eric Dumazet's avatar
Eric Dumazet committed
5201
	tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
5202
	if (!tx_ring->tx_buffer_info)
Eric Dumazet's avatar
Eric Dumazet committed
5203
		tx_ring->tx_buffer_info = vzalloc(size);
5204 5205
	if (!tx_ring->tx_buffer_info)
		goto err;
5206 5207

	/* round up to nearest 4K */
5208
	tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
5209
	tx_ring->size = ALIGN(tx_ring->size, 4096);
5210

5211
	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
5212
					   &tx_ring->dma, GFP_KERNEL);
5213 5214
	if (!tx_ring->desc)
		goto err;
5215

5216 5217
	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
5218
	return 0;
5219 5220 5221 5222

err:
	vfree(tx_ring->tx_buffer_info);
	tx_ring->tx_buffer_info = NULL;
5223
	dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
5224
	return -ENOMEM;
5225 5226
}

5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241
/**
 * ixgbe_setup_all_tx_resources - allocate all queues Tx resources
 * @adapter: board private structure
 *
 * If this function returns with an error, then it's possible one or
 * more of the rings is populated (while the rest are not).  It is the
 * callers duty to clean those orphaned rings.
 *
 * Return 0 on success, negative on failure
 **/
static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
{
	int i, err = 0;

	for (i = 0; i < adapter->num_tx_queues; i++) {
5242
		err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
5243 5244
		if (!err)
			continue;
5245
		e_err(probe, "Allocation for Tx Queue %u failed\n", i);
5246 5247 5248 5249 5250 5251
		break;
	}

	return err;
}

5252 5253
/**
 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
5254
 * @rx_ring:    rx descriptor ring (for a specific queue) to setup
5255 5256 5257
 *
 * Returns 0 on success, negative on failure
 **/
5258
int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
5259
{
5260
	struct device *dev = rx_ring->dev;
5261
	int size;
5262

5263
	size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
Eric Dumazet's avatar
Eric Dumazet committed
5264
	rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
5265
	if (!rx_ring->rx_buffer_info)
Eric Dumazet's avatar
Eric Dumazet committed
5266
		rx_ring->rx_buffer_info = vzalloc(size);
5267 5268
	if (!rx_ring->rx_buffer_info)
		goto err;
5269 5270

	/* Round up to nearest 4K */
5271 5272
	rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
	rx_ring->size = ALIGN(rx_ring->size, 4096);
5273

5274
	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
5275
					   &rx_ring->dma, GFP_KERNEL);
5276

5277 5278
	if (!rx_ring->desc)
		goto err;
5279

5280 5281
	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;
5282 5283

	return 0;
5284 5285 5286 5287
err:
	vfree(rx_ring->rx_buffer_info);
	rx_ring->rx_buffer_info = NULL;
	dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
5288
	return -ENOMEM;
5289 5290
}

5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305
/**
 * ixgbe_setup_all_rx_resources - allocate all queues Rx resources
 * @adapter: board private structure
 *
 * If this function returns with an error, then it's possible one or
 * more of the rings is populated (while the rest are not).  It is the
 * callers duty to clean those orphaned rings.
 *
 * Return 0 on success, negative on failure
 **/
static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
{
	int i, err = 0;

	for (i = 0; i < adapter->num_rx_queues; i++) {
5306
		err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
5307 5308
		if (!err)
			continue;
5309
		e_err(probe, "Allocation for Rx Queue %u failed\n", i);
5310 5311 5312 5313 5314 5315
		break;
	}

	return err;
}

5316 5317 5318 5319 5320 5321
/**
 * ixgbe_free_tx_resources - Free Tx Resources per Queue
 * @tx_ring: Tx descriptor ring for a specific queue
 *
 * Free all transmit software resources
 **/
5322
void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
5323
{
5324
	ixgbe_clean_tx_ring(tx_ring);
5325 5326 5327 5328

	vfree(tx_ring->tx_buffer_info);
	tx_ring->tx_buffer_info = NULL;

5329 5330 5331 5332 5333 5334
	/* if not set, then don't free */
	if (!tx_ring->desc)
		return;

	dma_free_coherent(tx_ring->dev, tx_ring->size,
			  tx_ring->desc, tx_ring->dma);
5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349

	tx_ring->desc = NULL;
}

/**
 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
 * @adapter: board private structure
 *
 * Free all transmit software resources
 **/
static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_tx_queues; i++)
5350
		if (adapter->tx_ring[i]->desc)
5351
			ixgbe_free_tx_resources(adapter->tx_ring[i]);
5352 5353 5354
}

/**
5355
 * ixgbe_free_rx_resources - Free Rx Resources
5356 5357 5358 5359
 * @rx_ring: ring to clean the resources from
 *
 * Free all receive software resources
 **/
5360
void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
5361
{
5362
	ixgbe_clean_rx_ring(rx_ring);
5363 5364 5365 5366

	vfree(rx_ring->rx_buffer_info);
	rx_ring->rx_buffer_info = NULL;

5367 5368 5369 5370 5371 5372
	/* if not set, then don't free */
	if (!rx_ring->desc)
		return;

	dma_free_coherent(rx_ring->dev, rx_ring->size,
			  rx_ring->desc, rx_ring->dma);
5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387

	rx_ring->desc = NULL;
}

/**
 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
 * @adapter: board private structure
 *
 * Free all receive software resources
 **/
static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_rx_queues; i++)
5388
		if (adapter->rx_ring[i]->desc)
5389
			ixgbe_free_rx_resources(adapter->rx_ring[i]);
5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401
}

/**
 * ixgbe_change_mtu - Change the Maximum Transfer Unit
 * @netdev: network interface device structure
 * @new_mtu: new value for maximum frame size
 *
 * Returns 0 on success, negative on failure
 **/
static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
5402
	struct ixgbe_hw *hw = &adapter->hw;
5403 5404
	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;

5405
	/* MTU < 68 is an error and causes problems on some kernels */
5406 5407 5408 5409 5410 5411 5412 5413
	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED &&
	    hw->mac.type != ixgbe_mac_X540) {
		if ((new_mtu < 68) || (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE))
			return -EINVAL;
	} else {
		if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE))
			return -EINVAL;
	}
5414

5415
	e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5416
	/* must set new MTU before calling down or up */
5417 5418
	netdev->mtu = new_mtu;

5419 5420 5421
	hw->fc.high_water = FC_HIGH_WATER(max_frame);
	hw->fc.low_water = FC_LOW_WATER(max_frame);

5422 5423
	if (netif_running(netdev))
		ixgbe_reinit_locked(adapter);
5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443

	return 0;
}

/**
 * ixgbe_open - Called when a network interface is made active
 * @netdev: network interface device structure
 *
 * Returns 0 on success, negative value on failure
 *
 * The open entry point is called when a network interface is made
 * active by the system (IFF_UP).  At this point all resources needed
 * for transmit and receive operations are allocated, the interrupt
 * handler is registered with the OS, the watchdog timer is started,
 * and the stack is notified that the interface is ready.
 **/
static int ixgbe_open(struct net_device *netdev)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	int err;
5444 5445 5446 5447

	/* disallow open during test */
	if (test_bit(__IXGBE_TESTING, &adapter->state))
		return -EBUSY;
5448

5449 5450
	netif_carrier_off(netdev);

5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462
	/* allocate transmit descriptors */
	err = ixgbe_setup_all_tx_resources(adapter);
	if (err)
		goto err_setup_tx;

	/* allocate receive descriptors */
	err = ixgbe_setup_all_rx_resources(adapter);
	if (err)
		goto err_setup_rx;

	ixgbe_configure(adapter);

5463
	err = ixgbe_request_irq(adapter);
5464 5465 5466 5467 5468 5469 5470
	if (err)
		goto err_req_irq;

	err = ixgbe_up_complete(adapter);
	if (err)
		goto err_up;

5471 5472
	netif_tx_start_all_queues(netdev);

5473 5474 5475
	return 0;

err_up:
5476
	ixgbe_release_hw_control(adapter);
5477 5478 5479
	ixgbe_free_irq(adapter);
err_req_irq:
err_setup_rx:
5480
	ixgbe_free_all_rx_resources(adapter);
5481
err_setup_tx:
5482
	ixgbe_free_all_tx_resources(adapter);
5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505
	ixgbe_reset(adapter);

	return err;
}

/**
 * ixgbe_close - Disables a network interface
 * @netdev: network interface device structure
 *
 * Returns 0, this is not allowed to fail
 *
 * The close entry point is called when an interface is de-activated
 * by the OS.  The hardware is still under the drivers control, but
 * needs to be disabled.  A global MAC reset is issued to stop the
 * hardware, and all transmit and receive resources are freed.
 **/
static int ixgbe_close(struct net_device *netdev)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);

	ixgbe_down(adapter);
	ixgbe_free_irq(adapter);

5506 5507
	ixgbe_fdir_filter_exit(adapter);

5508 5509 5510
	ixgbe_free_all_tx_resources(adapter);
	ixgbe_free_all_rx_resources(adapter);

5511
	ixgbe_release_hw_control(adapter);
5512 5513 5514 5515

	return 0;
}

5516 5517 5518
#ifdef CONFIG_PM
static int ixgbe_resume(struct pci_dev *pdev)
{
5519 5520
	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
	struct net_device *netdev = adapter->netdev;
5521 5522 5523 5524
	u32 err;

	pci_set_power_state(pdev, PCI_D0);
	pci_restore_state(pdev);
5525 5526 5527 5528 5529
	/*
	 * pci_restore_state clears dev->state_saved so call
	 * pci_save_state to restore it.
	 */
	pci_save_state(pdev);
5530 5531

	err = pci_enable_device_mem(pdev);
5532
	if (err) {
5533
		e_dev_err("Cannot enable PCI device from suspend\n");
5534 5535 5536 5537
		return err;
	}
	pci_set_master(pdev);

5538
	pci_wake_from_d3(pdev, false);
5539 5540 5541

	err = ixgbe_init_interrupt_scheme(adapter);
	if (err) {
5542
		e_dev_err("Cannot initialize interrupts for device\n");
5543 5544 5545 5546 5547
		return err;
	}

	ixgbe_reset(adapter);

5548 5549
	IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);

5550
	if (netif_running(netdev)) {
5551
		err = ixgbe_open(netdev);
5552 5553 5554 5555 5556 5557 5558 5559 5560
		if (err)
			return err;
	}

	netif_device_attach(netdev);

	return 0;
}
#endif /* CONFIG_PM */
5561 5562

static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
5563
{
5564 5565
	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
	struct net_device *netdev = adapter->netdev;
5566 5567 5568
	struct ixgbe_hw *hw = &adapter->hw;
	u32 ctrl, fctrl;
	u32 wufc = adapter->wol;
5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581
#ifdef CONFIG_PM
	int retval = 0;
#endif

	netif_device_detach(netdev);

	if (netif_running(netdev)) {
		ixgbe_down(adapter);
		ixgbe_free_irq(adapter);
		ixgbe_free_all_tx_resources(adapter);
		ixgbe_free_all_rx_resources(adapter);
	}

5582
	ixgbe_clear_interrupt_scheme(adapter);
5583 5584 5585 5586
#ifdef CONFIG_DCB
	kfree(adapter->ixgbe_ieee_pfc);
	kfree(adapter->ixgbe_ieee_ets);
#endif
5587

5588 5589 5590 5591
#ifdef CONFIG_PM
	retval = pci_save_state(pdev);
	if (retval)
		return retval;
5592

5593
#endif
5594 5595
	if (wufc) {
		ixgbe_set_rx_mode(netdev);
5596

5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613
		/* turn on all-multi mode if wake on multicast is enabled */
		if (wufc & IXGBE_WUFC_MC) {
			fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
			fctrl |= IXGBE_FCTRL_MPE;
			IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
		}

		ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
		ctrl |= IXGBE_CTRL_GIO_DIS;
		IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);

		IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
	} else {
		IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
		IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
	}

5614 5615
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
5616
		pci_wake_from_d3(pdev, false);
5617 5618
		break;
	case ixgbe_mac_82599EB:
5619
	case ixgbe_mac_X540:
5620 5621 5622 5623 5624
		pci_wake_from_d3(pdev, !!wufc);
		break;
	default:
		break;
	}
5625

5626 5627
	*enable_wake = !!wufc;

5628 5629 5630 5631
	ixgbe_release_hw_control(adapter);

	pci_disable_device(pdev);

5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650
	return 0;
}

#ifdef CONFIG_PM
static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
{
	int retval;
	bool wake;

	retval = __ixgbe_shutdown(pdev, &wake);
	if (retval)
		return retval;

	if (wake) {
		pci_prepare_to_sleep(pdev);
	} else {
		pci_wake_from_d3(pdev, false);
		pci_set_power_state(pdev, PCI_D3hot);
	}
5651 5652 5653

	return 0;
}
5654
#endif /* CONFIG_PM */
5655 5656 5657

static void ixgbe_shutdown(struct pci_dev *pdev)
{
5658 5659 5660 5661 5662 5663 5664 5665
	bool wake;

	__ixgbe_shutdown(pdev, &wake);

	if (system_state == SYSTEM_POWER_OFF) {
		pci_wake_from_d3(pdev, wake);
		pci_set_power_state(pdev, PCI_D3hot);
	}
5666 5667
}

5668 5669 5670 5671 5672 5673
/**
 * ixgbe_update_stats - Update the board statistics counters.
 * @adapter: board private structure
 **/
void ixgbe_update_stats(struct ixgbe_adapter *adapter)
{
5674
	struct net_device *netdev = adapter->netdev;
5675
	struct ixgbe_hw *hw = &adapter->hw;
5676
	struct ixgbe_hw_stats *hwstats = &adapter->stats;
5677 5678
	u64 total_mpc = 0;
	u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
5679 5680 5681
	u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
	u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
	u64 bytes = 0, packets = 0;
5682

5683 5684 5685 5686
	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
	    test_bit(__IXGBE_RESETTING, &adapter->state))
		return;

5687
	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
5688
		u64 rsc_count = 0;
5689
		u64 rsc_flush = 0;
5690 5691
		for (i = 0; i < 16; i++)
			adapter->hw_rx_no_dma_resources +=
5692
				IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5693
		for (i = 0; i < adapter->num_rx_queues; i++) {
5694 5695
			rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
			rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
5696 5697 5698
		}
		adapter->rsc_total_count = rsc_count;
		adapter->rsc_total_flush = rsc_flush;
5699 5700
	}

5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716
	for (i = 0; i < adapter->num_rx_queues; i++) {
		struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
		non_eop_descs += rx_ring->rx_stats.non_eop_descs;
		alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
		alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
		bytes += rx_ring->stats.bytes;
		packets += rx_ring->stats.packets;
	}
	adapter->non_eop_descs = non_eop_descs;
	adapter->alloc_rx_page_failed = alloc_rx_page_failed;
	adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
	netdev->stats.rx_bytes = bytes;
	netdev->stats.rx_packets = packets;

	bytes = 0;
	packets = 0;
Jesse Brandeburg's avatar
Jesse Brandeburg committed
5717
	/* gather some stats to the adapter struct that are per queue */
5718 5719 5720 5721 5722 5723 5724
	for (i = 0; i < adapter->num_tx_queues; i++) {
		struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
		restart_queue += tx_ring->tx_stats.restart_queue;
		tx_busy += tx_ring->tx_stats.tx_busy;
		bytes += tx_ring->stats.bytes;
		packets += tx_ring->stats.packets;
	}
5725
	adapter->restart_queue = restart_queue;
5726 5727 5728
	adapter->tx_busy = tx_busy;
	netdev->stats.tx_bytes = bytes;
	netdev->stats.tx_packets = packets;
Jesse Brandeburg's avatar
Jesse Brandeburg committed
5729

5730
	hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5731 5732 5733 5734
	for (i = 0; i < 8; i++) {
		/* for packet buffers not used, the register should read 0 */
		mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
		missed_rx += mpc;
5735 5736
		hwstats->mpc[i] += mpc;
		total_mpc += hwstats->mpc[i];
5737
		if (hw->mac.type == ixgbe_mac_82598EB)
5738 5739 5740 5741 5742
			hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
		hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
		hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
		hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
		hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5743 5744
		switch (hw->mac.type) {
		case ixgbe_mac_82598EB:
5745 5746
			hwstats->pxonrxc[i] +=
				IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5747 5748
			break;
		case ixgbe_mac_82599EB:
5749
		case ixgbe_mac_X540:
5750 5751 5752 5753 5754
			hwstats->pxonrxc[i] +=
				IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
			break;
		default:
			break;
5755
		}
5756 5757
		hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
		hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5758
	}
5759
	hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5760
	/* work around hardware counting issue */
5761
	hwstats->gprc -= missed_rx;
5762

5763 5764
	ixgbe_update_xoff_received(adapter);

5765
	/* 82598 hardware only has a 32 bit counter in the high register */
5766 5767 5768 5769 5770 5771 5772
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB:
		hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
		hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
		hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
		hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
		break;
5773
	case ixgbe_mac_X540:
5774 5775 5776 5777 5778 5779
		/* OS2BMC stats are X540 only*/
		hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
		hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
		hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
		hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
	case ixgbe_mac_82599EB:
5780
		hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
5781
		IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
5782
		hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
5783
		IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
5784
		hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
5785
		IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
5786 5787 5788
		hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
		hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
		hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
5789
#ifdef IXGBE_FCOE
5790 5791 5792 5793 5794 5795
		hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
		hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
		hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
		hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
		hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
		hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5796
#endif /* IXGBE_FCOE */
5797 5798 5799
		break;
	default:
		break;
5800
	}
5801
	bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5802 5803
	hwstats->bprc += bprc;
	hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5804
	if (hw->mac.type == ixgbe_mac_82598EB)
5805 5806 5807 5808 5809 5810 5811 5812 5813
		hwstats->mprc -= bprc;
	hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
	hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
	hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
	hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
	hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
	hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
	hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
	hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5814
	lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5815
	hwstats->lxontxc += lxon;
5816
	lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5817 5818 5819 5820
	hwstats->lxofftxc += lxoff;
	hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
	hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
	hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5821 5822 5823 5824
	/*
	 * 82598 errata - tx of flow control packets is included in tx counters
	 */
	xon_off_tot = lxon + lxoff;
5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839
	hwstats->gptc -= xon_off_tot;
	hwstats->mptc -= xon_off_tot;
	hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
	hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
	hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
	hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
	hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
	hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
	hwstats->ptc64 -= xon_off_tot;
	hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
	hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
	hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
	hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
	hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
	hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5840 5841

	/* Fill out the OS statistics structure */
5842
	netdev->stats.multicast = hwstats->mprc;
5843 5844

	/* Rx Errors */
5845
	netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
5846
	netdev->stats.rx_dropped = 0;
5847 5848
	netdev->stats.rx_length_errors = hwstats->rlec;
	netdev->stats.rx_crc_errors = hwstats->crcerrs;
5849
	netdev->stats.rx_missed_errors = total_mpc;
5850 5851 5852
}

/**
5853 5854
 * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table
 * @adapter - pointer to the device adapter structure
5855
 **/
5856
static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
5857
{
5858
	struct ixgbe_hw *hw = &adapter->hw;
5859
	int i;
5860

5861 5862 5863 5864
	if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
		return;

	adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
5865

5866
	/* if interface is down do nothing */
5867
	if (test_bit(__IXGBE_DOWN, &adapter->state))
5868 5869 5870 5871 5872 5873 5874 5875
		return;

	/* do nothing if we are not using signature filters */
	if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
		return;

	adapter->fdir_overflow++;

5876 5877 5878
	if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
		for (i = 0; i < adapter->num_tx_queues; i++)
			set_bit(__IXGBE_TX_FDIR_INIT_DONE,
5879
			        &(adapter->tx_ring[i]->state));
5880 5881
		/* re-enable flow director interrupts */
		IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897
	} else {
		e_err(probe, "failed to finish FDIR re-initialization, "
		      "ignored adding FDIR ATR filters\n");
	}
}

/**
 * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts
 * @adapter - pointer to the device adapter structure
 *
 * This function serves two purposes.  First it strobes the interrupt lines
 * in order to make certain interrupts are occuring.  Secondly it sets the
 * bits needed to check for TX hangs.  As a result we should immediately
 * determine if a hang has occured.
 */
static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
5898
{
5899
	struct ixgbe_hw *hw = &adapter->hw;
5900 5901
	u64 eics = 0;
	int i;
5902

5903 5904 5905 5906
	/* If we're down or resetting, just bail */
	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
	    test_bit(__IXGBE_RESETTING, &adapter->state))
		return;
5907

5908 5909 5910 5911 5912
	/* Force detection of hung controller */
	if (netif_carrier_ok(adapter->netdev)) {
		for (i = 0; i < adapter->num_tx_queues; i++)
			set_check_for_tx_hang(adapter->tx_ring[i]);
	}
5913

5914 5915 5916 5917 5918 5919 5920 5921
	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
		/*
		 * for legacy and MSI interrupts don't set any bits
		 * that are enabled for EIAM, because this operation
		 * would set *both* EIMS and EICS for any bit in EIAM
		 */
		IXGBE_WRITE_REG(hw, IXGBE_EICS,
			(IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
5922 5923 5924 5925
	} else {
		/* get one bit for every active tx/rx interrupt vector */
		for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
			struct ixgbe_q_vector *qv = adapter->q_vector[i];
5926
			if (qv->rx.count || qv->tx.count)
5927 5928
				eics |= ((u64)1 << i);
		}
5929
	}
5930

5931
	/* Cause software interrupt to ensure rings are cleaned */
5932 5933
	ixgbe_irq_rearm_queues(adapter, eics);

5934 5935
}

5936
/**
5937 5938 5939
 * ixgbe_watchdog_update_link - update the link status
 * @adapter - pointer to the device adapter structure
 * @link_speed - pointer to a u32 to store the link_speed
5940
 **/
5941
static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
5942 5943
{
	struct ixgbe_hw *hw = &adapter->hw;
5944 5945
	u32 link_speed = adapter->link_speed;
	bool link_up = adapter->link_up;
5946
	int i;
5947

5948 5949 5950 5951 5952
	if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
		return;

	if (hw->mac.ops.check_link) {
		hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
5953
	} else {
5954 5955 5956
		/* always assume link is up, if no check link function */
		link_speed = IXGBE_LINK_SPEED_10GB_FULL;
		link_up = true;
5957
	}
5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976
	if (link_up) {
		if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
			for (i = 0; i < MAX_TRAFFIC_CLASS; i++)
				hw->mac.ops.fc_enable(hw, i);
		} else {
			hw->mac.ops.fc_enable(hw, 0);
		}
	}

	if (link_up ||
	    time_after(jiffies, (adapter->link_check_timeout +
				 IXGBE_TRY_LINK_TIMEOUT))) {
		adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
		IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
		IXGBE_WRITE_FLUSH(hw);
	}

	adapter->link_up = link_up;
	adapter->link_speed = link_speed;
5977 5978 5979
}

/**
5980 5981 5982
 * ixgbe_watchdog_link_is_up - update netif_carrier status and
 *                             print link up message
 * @adapter - pointer to the device adapter structure
5983
 **/
5984
static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
5985
{
5986
	struct net_device *netdev = adapter->netdev;
5987
	struct ixgbe_hw *hw = &adapter->hw;
5988 5989
	u32 link_speed = adapter->link_speed;
	bool flow_rx, flow_tx;
5990

5991 5992
	/* only continue if link was previously down */
	if (netif_carrier_ok(netdev))
5993
		return;
5994

5995
	adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
5996

5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016
	switch (hw->mac.type) {
	case ixgbe_mac_82598EB: {
		u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
		u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
		flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
		flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
	}
		break;
	case ixgbe_mac_X540:
	case ixgbe_mac_82599EB: {
		u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
		u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
		flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
		flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
	}
		break;
	default:
		flow_tx = false;
		flow_rx = false;
		break;
6017
	}
6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028
	e_info(drv, "NIC Link is Up %s, Flow Control: %s\n",
	       (link_speed == IXGBE_LINK_SPEED_10GB_FULL ?
	       "10 Gbps" :
	       (link_speed == IXGBE_LINK_SPEED_1GB_FULL ?
	       "1 Gbps" :
	       (link_speed == IXGBE_LINK_SPEED_100_FULL ?
	       "100 Mbps" :
	       "unknown speed"))),
	       ((flow_rx && flow_tx) ? "RX/TX" :
	       (flow_rx ? "RX" :
	       (flow_tx ? "TX" : "None"))));
6029

6030 6031
	netif_carrier_on(netdev);
	ixgbe_check_vf_rate_limit(adapter);
6032 6033
}

6034
/**
6035 6036 6037
 * ixgbe_watchdog_link_is_down - update netif_carrier status and
 *                               print link down message
 * @adapter - pointer to the adapter structure
6038
 **/
6039
static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter* adapter)
6040
{
6041
	struct net_device *netdev = adapter->netdev;
6042
	struct ixgbe_hw *hw = &adapter->hw;
6043

6044 6045
	adapter->link_up = false;
	adapter->link_speed = 0;
6046

6047 6048 6049
	/* only continue if link was up previously */
	if (!netif_carrier_ok(netdev))
		return;
6050

6051 6052 6053
	/* poll for SFP+ cable when link is down */
	if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
		adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
6054

6055 6056 6057
	e_info(drv, "NIC Link is Down\n");
	netif_carrier_off(netdev);
}
6058

6059 6060 6061 6062 6063 6064
/**
 * ixgbe_watchdog_flush_tx - flush queues on link down
 * @adapter - pointer to the device adapter structure
 **/
static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
{
6065
	int i;
6066
	int some_tx_pending = 0;
6067

6068
	if (!netif_carrier_ok(adapter->netdev)) {
6069
		for (i = 0; i < adapter->num_tx_queues; i++) {
6070
			struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082
			if (tx_ring->next_to_use != tx_ring->next_to_clean) {
				some_tx_pending = 1;
				break;
			}
		}

		if (some_tx_pending) {
			/* We've lost link, so the controller stops DMA,
			 * but we've got queued Tx work that's never going
			 * to get done, so reset controller to flush Tx.
			 * (Do the reset outside of interrupt context).
			 */
6083
			adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
6084
		}
6085 6086 6087
	}
}

6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107
static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
{
	u32 ssvpc;

	/* Do not perform spoof check for 82598 */
	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
		return;

	ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);

	/*
	 * ssvpc register is cleared on read, if zero then no
	 * spoofed packets in the last interval.
	 */
	if (!ssvpc)
		return;

	e_warn(drv, "%d Spoofed packets detected\n", ssvpc);
}

6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123
/**
 * ixgbe_watchdog_subtask - check and bring link up
 * @adapter - pointer to the device adapter structure
 **/
static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
{
	/* if interface is down do nothing */
	if (test_bit(__IXGBE_DOWN, &adapter->state))
		return;

	ixgbe_watchdog_update_link(adapter);

	if (adapter->link_up)
		ixgbe_watchdog_link_is_up(adapter);
	else
		ixgbe_watchdog_link_is_down(adapter);
6124

6125
	ixgbe_spoof_check(adapter);
6126
	ixgbe_update_stats(adapter);
6127 6128

	ixgbe_watchdog_flush_tx(adapter);
6129
}
6130

6131
/**
6132 6133
 * ixgbe_sfp_detection_subtask - poll for SFP+ cable
 * @adapter - the ixgbe adapter structure
6134
 **/
6135
static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
6136 6137
{
	struct ixgbe_hw *hw = &adapter->hw;
6138
	s32 err;
6139

6140 6141 6142 6143
	/* not searching for SFP so there is nothing to do here */
	if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
	    !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
		return;
6144

6145 6146 6147
	/* someone else is in init, wait until next service event */
	if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
		return;
6148

6149 6150 6151
	err = hw->phy.ops.identify_sfp(hw);
	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
		goto sfp_out;
6152

6153 6154 6155 6156
	if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
		/* If no cable is present, then we need to reset
		 * the next time we find a good cable. */
		adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
6157
	}
6158

6159 6160 6161
	/* exit on error */
	if (err)
		goto sfp_out;
6162

6163 6164 6165
	/* exit if reset not needed */
	if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
		goto sfp_out;
6166

6167
	adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
6168

6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194
	/*
	 * A module may be identified correctly, but the EEPROM may not have
	 * support for that module.  setup_sfp() will fail in that case, so
	 * we should not allow that module to load.
	 */
	if (hw->mac.type == ixgbe_mac_82598EB)
		err = hw->phy.ops.reset(hw);
	else
		err = hw->mac.ops.setup_sfp(hw);

	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
		goto sfp_out;

	adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
	e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);

sfp_out:
	clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);

	if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) &&
	    (adapter->netdev->reg_state == NETREG_REGISTERED)) {
		e_dev_err("failed to initialize because an unsupported "
			  "SFP+ module type was detected.\n");
		e_dev_err("Reload the driver after installing a "
			  "supported module.\n");
		unregister_netdev(adapter->netdev);
6195
	}
6196
}
6197

6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249
/**
 * ixgbe_sfp_link_config_subtask - set up link SFP after module install
 * @adapter - the ixgbe adapter structure
 **/
static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 autoneg;
	bool negotiation;

	if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
		return;

	/* someone else is in init, wait until next service event */
	if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
		return;

	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;

	autoneg = hw->phy.autoneg_advertised;
	if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
		hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation);
	hw->mac.autotry_restart = false;
	if (hw->mac.ops.setup_link)
		hw->mac.ops.setup_link(hw, autoneg, negotiation, true);

	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
	adapter->link_check_timeout = jiffies;
	clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
}

/**
 * ixgbe_service_timer - Timer Call-back
 * @data: pointer to adapter cast into an unsigned long
 **/
static void ixgbe_service_timer(unsigned long data)
{
	struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
	unsigned long next_event_offset;

	/* poll faster when waiting for link */
	if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
		next_event_offset = HZ / 10;
	else
		next_event_offset = HZ * 2;

	/* Reset the timer */
	mod_timer(&adapter->service_timer, next_event_offset + jiffies);

	ixgbe_service_event_schedule(adapter);
}

6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268
static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
{
	if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED))
		return;

	adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED;

	/* If we're already down or resetting, just bail */
	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
	    test_bit(__IXGBE_RESETTING, &adapter->state))
		return;

	ixgbe_dump(adapter);
	netdev_err(adapter->netdev, "Reset adapter\n");
	adapter->tx_timeout_count++;

	ixgbe_reinit_locked(adapter);
}

6269 6270 6271 6272 6273 6274 6275 6276 6277 6278
/**
 * ixgbe_service_task - manages and runs subtasks
 * @work: pointer to work_struct containing our data
 **/
static void ixgbe_service_task(struct work_struct *work)
{
	struct ixgbe_adapter *adapter = container_of(work,
						     struct ixgbe_adapter,
						     service_task);

6279
	ixgbe_reset_subtask(adapter);
6280 6281
	ixgbe_sfp_detection_subtask(adapter);
	ixgbe_sfp_link_config_subtask(adapter);
6282
	ixgbe_check_overtemp_subtask(adapter);
6283
	ixgbe_watchdog_subtask(adapter);
6284
	ixgbe_fdir_reinit_subtask(adapter);
6285
	ixgbe_check_hang_subtask(adapter);
6286 6287

	ixgbe_service_event_complete(adapter);
6288 6289
}

6290 6291
void ixgbe_tx_ctxtdesc(struct ixgbe_ring *tx_ring, u32 vlan_macip_lens,
		       u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
6292 6293
{
	struct ixgbe_adv_tx_context_desc *context_desc;
6294
	u16 i = tx_ring->next_to_use;
6295

6296
	context_desc = IXGBE_TX_CTXTDESC_ADV(tx_ring, i);
6297

6298 6299
	i++;
	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
6300

6301 6302
	/* set bits to identify this as an advanced context descriptor */
	type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
6303

6304 6305 6306 6307 6308
	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
	context_desc->seqnum_seed	= cpu_to_le32(fcoe_sof_eof);
	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
}
6309

6310 6311 6312 6313 6314 6315
static int ixgbe_tso(struct ixgbe_ring *tx_ring, struct sk_buff *skb,
		     u32 tx_flags, __be16 protocol, u8 *hdr_len)
{
	int err;
	u32 vlan_macip_lens, type_tucmd;
	u32 mss_l4len_idx, l4len;
6316

6317 6318
	if (!skb_is_gso(skb))
		return 0;
6319

6320 6321 6322 6323
	if (skb_header_cloned(skb)) {
		err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
		if (err)
			return err;
6324 6325
	}

6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367
	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
	type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;

	if (protocol == __constant_htons(ETH_P_IP)) {
		struct iphdr *iph = ip_hdr(skb);
		iph->tot_len = 0;
		iph->check = 0;
		tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
							 iph->daddr, 0,
							 IPPROTO_TCP,
							 0);
		type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
	} else if (skb_is_gso_v6(skb)) {
		ipv6_hdr(skb)->payload_len = 0;
		tcp_hdr(skb)->check =
		    ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
				     &ipv6_hdr(skb)->daddr,
				     0, IPPROTO_TCP, 0);
	}

	l4len = tcp_hdrlen(skb);
	*hdr_len = skb_transport_offset(skb) + l4len;

	/* mss_l4len_id: use 1 as index for TSO */
	mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
	mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
	mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;

	/* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
	vlan_macip_lens = skb_network_header_len(skb);
	vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
	vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;

	ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd,
	                  mss_l4len_idx);

	return 1;
}

static bool ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
			  struct sk_buff *skb, u32 tx_flags,
			  __be16 protocol)
6368
{
6369 6370 6371
	u32 vlan_macip_lens = 0;
	u32 mss_l4len_idx = 0;
	u32 type_tucmd = 0;
6372

6373
	if (skb->ip_summed != CHECKSUM_PARTIAL) {
6374
	    if (!(tx_flags & IXGBE_TX_FLAGS_HW_VLAN))
6375 6376 6377 6378 6379 6380 6381 6382
			return false;
	} else {
		u8 l4_hdr = 0;
		switch (protocol) {
		case __constant_htons(ETH_P_IP):
			vlan_macip_lens |= skb_network_header_len(skb);
			type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
			l4_hdr = ip_hdr(skb)->protocol;
6383
			break;
6384 6385 6386 6387 6388 6389 6390 6391 6392 6393
		case __constant_htons(ETH_P_IPV6):
			vlan_macip_lens |= skb_network_header_len(skb);
			l4_hdr = ipv6_hdr(skb)->nexthdr;
			break;
		default:
			if (unlikely(net_ratelimit())) {
				dev_warn(tx_ring->dev,
				 "partial checksum but proto=%x!\n",
				 skb->protocol);
			}
6394 6395
			break;
		}
6396 6397

		switch (l4_hdr) {
6398
		case IPPROTO_TCP:
6399 6400 6401
			type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
			mss_l4len_idx = tcp_hdrlen(skb) <<
					IXGBE_ADVTXD_L4LEN_SHIFT;
6402 6403
			break;
		case IPPROTO_SCTP:
6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417
			type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
			mss_l4len_idx = sizeof(struct sctphdr) <<
					IXGBE_ADVTXD_L4LEN_SHIFT;
			break;
		case IPPROTO_UDP:
			mss_l4len_idx = sizeof(struct udphdr) <<
					IXGBE_ADVTXD_L4LEN_SHIFT;
			break;
		default:
			if (unlikely(net_ratelimit())) {
				dev_warn(tx_ring->dev,
				 "partial checksum but l4 proto=%x!\n",
				 skb->protocol);
			}
6418 6419 6420 6421
			break;
		}
	}

6422 6423
	vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
	vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
6424

6425 6426
	ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0,
			  type_tucmd, mss_l4len_idx);
6427

6428
	return (skb->ip_summed == CHECKSUM_PARTIAL);
6429 6430
}

6431
static __le32 ixgbe_tx_cmd_type(u32 tx_flags)
6432
{
6433 6434 6435 6436
	/* set type for advanced descriptor with frame checksum insertion */
	__le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
				      IXGBE_ADVTXD_DCMD_IFCS |
				      IXGBE_ADVTXD_DCMD_DEXT);
6437

6438
	/* set HW vlan bit if vlan is present */
6439
	if (tx_flags & IXGBE_TX_FLAGS_HW_VLAN)
6440
		cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
6441

6442 6443 6444 6445 6446 6447 6448
	/* set segmentation enable bits for TSO/FSO */
#ifdef IXGBE_FCOE
	if ((tx_flags & IXGBE_TX_FLAGS_TSO) || (tx_flags & IXGBE_TX_FLAGS_FSO))
#else
	if (tx_flags & IXGBE_TX_FLAGS_TSO)
#endif
		cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
6449

6450 6451
	return cmd_type;
}
6452

6453 6454 6455 6456
static __le32 ixgbe_tx_olinfo_status(u32 tx_flags, unsigned int paylen)
{
	__le32 olinfo_status =
		cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
6457

6458 6459 6460 6461 6462 6463
	if (tx_flags & IXGBE_TX_FLAGS_TSO) {
		olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM |
					    (1 << IXGBE_ADVTXD_IDX_SHIFT));
		/* enble IPv4 checksum for TSO */
		if (tx_flags & IXGBE_TX_FLAGS_IPV4)
			olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
6464 6465
	}

6466 6467 6468
	/* enable L4 checksum for TSO and TX checksum offload */
	if (tx_flags & IXGBE_TX_FLAGS_CSUM)
		olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
6469

6470 6471 6472 6473 6474
#ifdef IXGBE_FCOE
	/* use index 1 context for FCOE/FSO */
	if (tx_flags & IXGBE_TX_FLAGS_FCOE)
		olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC |
					    (1 << IXGBE_ADVTXD_IDX_SHIFT));
6475

6476 6477 6478
#endif
	return olinfo_status;
}
6479

6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509
#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \
		       IXGBE_TXD_CMD_RS)

static void ixgbe_tx_map(struct ixgbe_ring *tx_ring,
			 struct sk_buff *skb,
			 struct ixgbe_tx_buffer *first,
			 u32 tx_flags,
			 const u8 hdr_len)
{
	struct device *dev = tx_ring->dev;
	struct ixgbe_tx_buffer *tx_buffer_info;
	union ixgbe_adv_tx_desc *tx_desc;
	dma_addr_t dma;
	__le32 cmd_type, olinfo_status;
	struct skb_frag_struct *frag;
	unsigned int f = 0;
	unsigned int data_len = skb->data_len;
	unsigned int size = skb_headlen(skb);
	u32 offset = 0;
	u32 paylen = skb->len - hdr_len;
	u16 i = tx_ring->next_to_use;
	u16 gso_segs;

#ifdef IXGBE_FCOE
	if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
		if (data_len >= sizeof(struct fcoe_crc_eof)) {
			data_len -= sizeof(struct fcoe_crc_eof);
		} else {
			size -= sizeof(struct fcoe_crc_eof) - data_len;
			data_len = 0;
6510 6511
		}
	}
6512

6513 6514 6515 6516
#endif
	dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
	if (dma_mapping_error(dev, dma))
		goto dma_error;
6517

6518 6519
	cmd_type = ixgbe_tx_cmd_type(tx_flags);
	olinfo_status = ixgbe_tx_olinfo_status(tx_flags, paylen);
6520

6521
	tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
6522

6523 6524 6525 6526 6527 6528
	for (;;) {
		while (size > IXGBE_MAX_DATA_PER_TXD) {
			tx_desc->read.buffer_addr = cpu_to_le64(dma + offset);
			tx_desc->read.cmd_type_len =
				cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
			tx_desc->read.olinfo_status = olinfo_status;
6529

6530 6531
			offset += IXGBE_MAX_DATA_PER_TXD;
			size -= IXGBE_MAX_DATA_PER_TXD;
6532

6533 6534 6535 6536 6537 6538 6539
			tx_desc++;
			i++;
			if (i == tx_ring->count) {
				tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
				i = 0;
			}
		}
6540 6541

		tx_buffer_info = &tx_ring->tx_buffer_info[i];
6542 6543 6544
		tx_buffer_info->length = offset + size;
		tx_buffer_info->tx_flags = tx_flags;
		tx_buffer_info->dma = dma;
6545

6546 6547 6548
		tx_desc->read.buffer_addr = cpu_to_le64(dma + offset);
		tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
		tx_desc->read.olinfo_status = olinfo_status;
6549

6550 6551
		if (!data_len)
			break;
6552

6553 6554 6555 6556 6557 6558 6559 6560
		frag = &skb_shinfo(skb)->frags[f];
#ifdef IXGBE_FCOE
		size = min_t(unsigned int, data_len, frag->size);
#else
		size = frag->size;
#endif
		data_len -= size;
		f++;
6561

6562 6563
		offset = 0;
		tx_flags |= IXGBE_TX_FLAGS_MAPPED_AS_PAGE;
6564

6565 6566 6567 6568
		dma = dma_map_page(dev, frag->page, frag->page_offset,
				   size, DMA_TO_DEVICE);
		if (dma_mapping_error(dev, dma))
			goto dma_error;
6569

6570 6571 6572 6573 6574 6575 6576
		tx_desc++;
		i++;
		if (i == tx_ring->count) {
			tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
			i = 0;
		}
	}
6577

6578
	tx_desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD);
6579

6580 6581 6582
	i++;
	if (i == tx_ring->count)
		i = 0;
6583

6584
	tx_ring->next_to_use = i;
6585

6586 6587 6588 6589 6590 6591 6592 6593 6594 6595
	if (tx_flags & IXGBE_TX_FLAGS_TSO)
		gso_segs = skb_shinfo(skb)->gso_segs;
#ifdef IXGBE_FCOE
	/* adjust for FCoE Sequence Offload */
	else if (tx_flags & IXGBE_TX_FLAGS_FSO)
		gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
					skb_shinfo(skb)->gso_size);
#endif /* IXGBE_FCOE */
	else
		gso_segs = 1;
6596

6597 6598 6599 6600
	/* multiply data chunks by size of headers */
	tx_buffer_info->bytecount = paylen + (gso_segs * hdr_len);
	tx_buffer_info->gso_segs = gso_segs;
	tx_buffer_info->skb = skb;
6601

6602 6603
	/* set the timestamp */
	first->time_stamp = jiffies;
6604 6605 6606 6607 6608 6609 6610 6611 6612

	/*
	 * Force memory writes to complete before letting h/w
	 * know there are new descriptors to fetch.  (Only
	 * applicable for weak-ordered memory model archs,
	 * such as IA-64).
	 */
	wmb();

6613 6614 6615 6616
	/* set next_to_watch value indicating a packet is present */
	first->next_to_watch = tx_desc;

	/* notify HW of packet */
6617
	writel(i, tx_ring->tail);
6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636

	return;
dma_error:
	dev_err(dev, "TX DMA map failed\n");

	/* clear dma mappings for failed tx_buffer_info map */
	for (;;) {
		tx_buffer_info = &tx_ring->tx_buffer_info[i];
		ixgbe_unmap_tx_resource(tx_ring, tx_buffer_info);
		if (tx_buffer_info == first)
			break;
		if (i == 0)
			i = tx_ring->count;
		i--;
	}

	dev_kfree_skb_any(skb);

	tx_ring->next_to_use = i;
6637 6638
}

6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649
static void ixgbe_atr(struct ixgbe_ring *ring, struct sk_buff *skb,
		      u32 tx_flags, __be16 protocol)
{
	struct ixgbe_q_vector *q_vector = ring->q_vector;
	union ixgbe_atr_hash_dword input = { .dword = 0 };
	union ixgbe_atr_hash_dword common = { .dword = 0 };
	union {
		unsigned char *network;
		struct iphdr *ipv4;
		struct ipv6hdr *ipv6;
	} hdr;
6650
	struct tcphdr *th;
6651
	__be16 vlan_id;
6652

6653 6654 6655 6656 6657 6658
	/* if ring doesn't have a interrupt vector, cannot perform ATR */
	if (!q_vector)
		return;

	/* do nothing if sampling is disabled */
	if (!ring->atr_sample_rate)
6659
		return;
6660

6661
	ring->atr_count++;
6662

6663 6664 6665 6666 6667 6668 6669 6670 6671
	/* snag network header to get L4 type and address */
	hdr.network = skb_network_header(skb);

	/* Currently only IPv4/IPv6 with TCP is supported */
	if ((protocol != __constant_htons(ETH_P_IPV6) ||
	     hdr.ipv6->nexthdr != IPPROTO_TCP) &&
	    (protocol != __constant_htons(ETH_P_IP) ||
	     hdr.ipv4->protocol != IPPROTO_TCP))
		return;
6672 6673

	th = tcp_hdr(skb);
6674

6675 6676
	/* skip this packet since it is invalid or the socket is closing */
	if (!th || th->fin)
6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700
		return;

	/* sample on all syn packets or once every atr sample count */
	if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
		return;

	/* reset sample count */
	ring->atr_count = 0;

	vlan_id = htons(tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);

	/*
	 * src and dst are inverted, think how the receiver sees them
	 *
	 * The input is broken into two sections, a non-compressed section
	 * containing vm_pool, vlan_id, and flow_type.  The rest of the data
	 * is XORed together and stored in the compressed dword.
	 */
	input.formatted.vlan_id = vlan_id;

	/*
	 * since src port and flex bytes occupy the same word XOR them together
	 * and write the value to source port portion of compressed dword
	 */
6701
	if (tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN))
6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720
		common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q);
	else
		common.port.src ^= th->dest ^ protocol;
	common.port.dst ^= th->source;

	if (protocol == __constant_htons(ETH_P_IP)) {
		input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
		common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
	} else {
		input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
		common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
			     hdr.ipv6->saddr.s6_addr32[1] ^
			     hdr.ipv6->saddr.s6_addr32[2] ^
			     hdr.ipv6->saddr.s6_addr32[3] ^
			     hdr.ipv6->daddr.s6_addr32[0] ^
			     hdr.ipv6->daddr.s6_addr32[1] ^
			     hdr.ipv6->daddr.s6_addr32[2] ^
			     hdr.ipv6->daddr.s6_addr32[3];
	}
6721 6722

	/* This assumes the Rx queue and Tx queue are bound to the same CPU */
6723 6724
	ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw,
					      input, common, ring->queue_index);
6725 6726
}

6727
static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
6728
{
6729
	netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
6730 6731 6732 6733 6734 6735 6736
	/* Herbert's original patch had:
	 *  smp_mb__after_netif_stop_queue();
	 * but since that doesn't exist yet, just open code it. */
	smp_mb();

	/* We need to check again in a case another CPU has just
	 * made room available. */
6737
	if (likely(ixgbe_desc_unused(tx_ring) < size))
6738 6739 6740
		return -EBUSY;

	/* A reprieve! - use start_queue because it doesn't call schedule */
6741
	netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
6742
	++tx_ring->tx_stats.restart_queue;
6743 6744 6745
	return 0;
}

6746
static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
6747
{
6748
	if (likely(ixgbe_desc_unused(tx_ring) >= size))
6749
		return 0;
6750
	return __ixgbe_maybe_stop_tx(tx_ring, size);
6751 6752
}

6753 6754 6755
static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb)
{
	struct ixgbe_adapter *adapter = netdev_priv(dev);
6756 6757
	int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :
					       smp_processor_id();
6758
#ifdef IXGBE_FCOE
6759
	__be16 protocol = vlan_get_protocol(skb);
6760

6761 6762 6763 6764 6765 6766
	if (((protocol == htons(ETH_P_FCOE)) ||
	    (protocol == htons(ETH_P_FIP))) &&
	    (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) {
		txq &= (adapter->ring_feature[RING_F_FCOE].indices - 1);
		txq += adapter->ring_feature[RING_F_FCOE].mask;
		return txq;
6767 6768 6769
	}
#endif

6770 6771 6772
	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
		while (unlikely(txq >= dev->real_num_tx_queues))
			txq -= dev->real_num_tx_queues;
6773
		return txq;
6774
	}
6775

6776 6777 6778
	return skb_tx_hash(dev, skb);
}

6779
netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
6780 6781
			  struct ixgbe_adapter *adapter,
			  struct ixgbe_ring *tx_ring)
6782
{
6783
	struct ixgbe_tx_buffer *first;
6784
	int tso;
6785
	u32 tx_flags = 0;
6786 6787 6788 6789
#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
	unsigned short f;
#endif
	u16 count = TXD_USE_COUNT(skb_headlen(skb));
6790
	__be16 protocol = skb->protocol;
6791
	u8 hdr_len = 0;
6792

6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810
	/*
	 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
	 *       + 1 desc for skb_head_len/IXGBE_MAX_DATA_PER_TXD,
	 *       + 2 desc gap to keep tail from touching head,
	 *       + 1 desc for context descriptor,
	 * otherwise try next time
	 */
#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
		count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
#else
	count += skb_shinfo(skb)->nr_frags;
#endif
	if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) {
		tx_ring->tx_stats.tx_busy++;
		return NETDEV_TX_BUSY;
	}

6811
	/* if we have a HW VLAN tag being added default to the HW one */
6812
	if (vlan_tx_tag_present(skb)) {
6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841
		tx_flags |= vlan_tx_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT;
		tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
	/* else if it is a SW VLAN check the next protocol and store the tag */
	} else if (protocol == __constant_htons(ETH_P_8021Q)) {
		struct vlan_hdr *vhdr, _vhdr;
		vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
		if (!vhdr)
			goto out_drop;

		protocol = vhdr->h_vlan_encapsulated_proto;
		tx_flags |= ntohs(vhdr->h_vlan_TCI) << IXGBE_TX_FLAGS_VLAN_SHIFT;
		tx_flags |= IXGBE_TX_FLAGS_SW_VLAN;
	}

	if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
	    skb->priority != TC_PRIO_CONTROL) {
		tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
		tx_flags |= tx_ring->dcb_tc <<
			    IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT;
		if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) {
			struct vlan_ethhdr *vhdr;
			if (skb_header_cloned(skb) &&
			    pskb_expand_head(skb, 0, 0, GFP_ATOMIC))
				goto out_drop;
			vhdr = (struct vlan_ethhdr *)skb->data;
			vhdr->h_vlan_TCI = htons(tx_flags >>
						 IXGBE_TX_FLAGS_VLAN_SHIFT);
		} else {
			tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
6842
		}
6843
	}
6844

6845
	/* record the location of the first descriptor for this packet */
6846
	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
6847

6848
#ifdef IXGBE_FCOE
6849 6850 6851
	/* setup tx offload for FCoE */
	if ((protocol == __constant_htons(ETH_P_FCOE)) &&
	    (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) {
6852 6853 6854 6855
		tso = ixgbe_fso(tx_ring, skb, tx_flags, &hdr_len);
		if (tso < 0)
			goto out_drop;
		else if (tso)
6856 6857 6858 6859
			tx_flags |= IXGBE_TX_FLAGS_FSO |
				    IXGBE_TX_FLAGS_FCOE;
		else
			tx_flags |= IXGBE_TX_FLAGS_FCOE;
6860

6861
		goto xmit_fcoe;
6862
	}
6863

6864 6865 6866 6867
#endif /* IXGBE_FCOE */
	/* setup IPv4/IPv6 offloads */
	if (protocol == __constant_htons(ETH_P_IP))
		tx_flags |= IXGBE_TX_FLAGS_IPV4;
6868

6869 6870
	tso = ixgbe_tso(tx_ring, skb, tx_flags, protocol, &hdr_len);
	if (tso < 0)
6871
		goto out_drop;
6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883
	else if (tso)
		tx_flags |= IXGBE_TX_FLAGS_TSO;
	else if (ixgbe_tx_csum(tx_ring, skb, tx_flags, protocol))
		tx_flags |= IXGBE_TX_FLAGS_CSUM;

	/* add the ATR filter if ATR is on */
	if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
		ixgbe_atr(tx_ring, skb, tx_flags, protocol);

#ifdef IXGBE_FCOE
xmit_fcoe:
#endif /* IXGBE_FCOE */
6884 6885 6886
	ixgbe_tx_map(tx_ring, skb, first, tx_flags, hdr_len);

	ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
6887 6888

	return NETDEV_TX_OK;
6889 6890 6891 6892

out_drop:
	dev_kfree_skb_any(skb);
	return NETDEV_TX_OK;
6893 6894
}

6895 6896 6897 6898 6899 6900
static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	struct ixgbe_ring *tx_ring;

	tx_ring = adapter->tx_ring[skb->queue_mapping];
6901
	return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
6902 6903
}

6904 6905 6906 6907 6908 6909 6910 6911 6912 6913
/**
 * ixgbe_set_mac - Change the Ethernet Address of the NIC
 * @netdev: network interface device structure
 * @p: pointer to an address structure
 *
 * Returns 0 on success, negative on failure
 **/
static int ixgbe_set_mac(struct net_device *netdev, void *p)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
6914
	struct ixgbe_hw *hw = &adapter->hw;
6915 6916 6917 6918 6919 6920
	struct sockaddr *addr = p;

	if (!is_valid_ether_addr(addr->sa_data))
		return -EADDRNOTAVAIL;

	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
6921
	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
6922

6923 6924
	hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
			    IXGBE_RAH_AV);
6925 6926 6927 6928

	return 0;
}

6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962
static int
ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	struct ixgbe_hw *hw = &adapter->hw;
	u16 value;
	int rc;

	if (prtad != hw->phy.mdio.prtad)
		return -EINVAL;
	rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
	if (!rc)
		rc = value;
	return rc;
}

static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
			    u16 addr, u16 value)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	struct ixgbe_hw *hw = &adapter->hw;

	if (prtad != hw->phy.mdio.prtad)
		return -EINVAL;
	return hw->phy.ops.write_reg(hw, addr, devad, value);
}

static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);

	return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd);
}

6963 6964
/**
 * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
6965
 * netdev->dev_addrs
6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985
 * @netdev: network interface device structure
 *
 * Returns non-zero on failure
 **/
static int ixgbe_add_sanmac_netdev(struct net_device *dev)
{
	int err = 0;
	struct ixgbe_adapter *adapter = netdev_priv(dev);
	struct ixgbe_mac_info *mac = &adapter->hw.mac;

	if (is_valid_ether_addr(mac->san_addr)) {
		rtnl_lock();
		err = dev_addr_add(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
		rtnl_unlock();
	}
	return err;
}

/**
 * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
6986
 * netdev->dev_addrs
6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004
 * @netdev: network interface device structure
 *
 * Returns non-zero on failure
 **/
static int ixgbe_del_sanmac_netdev(struct net_device *dev)
{
	int err = 0;
	struct ixgbe_adapter *adapter = netdev_priv(dev);
	struct ixgbe_mac_info *mac = &adapter->hw.mac;

	if (is_valid_ether_addr(mac->san_addr)) {
		rtnl_lock();
		err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
		rtnl_unlock();
	}
	return err;
}

7005 7006 7007 7008 7009 7010 7011 7012 7013
#ifdef CONFIG_NET_POLL_CONTROLLER
/*
 * Polling 'interrupt' - used by things like netconsole to send skbs
 * without having to re-enable interrupts. It's not called while
 * the interrupt routine is executing.
 */
static void ixgbe_netpoll(struct net_device *netdev)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
7014
	int i;
7015

7016 7017 7018 7019
	/* if interface is down do nothing */
	if (test_bit(__IXGBE_DOWN, &adapter->state))
		return;

7020
	adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
7021 7022 7023 7024 7025 7026 7027 7028 7029
	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
		int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
		for (i = 0; i < num_q_vectors; i++) {
			struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
			ixgbe_msix_clean_many(0, q_vector);
		}
	} else {
		ixgbe_intr(adapter->pdev->irq, netdev);
	}
7030 7031 7032 7033
	adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
}
#endif

Eric Dumazet's avatar
Eric Dumazet committed
7034 7035 7036 7037 7038 7039
static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev,
						   struct rtnl_link_stats64 *stats)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	int i;

Eric Dumazet's avatar
Eric Dumazet committed
7040
	rcu_read_lock();
Eric Dumazet's avatar
Eric Dumazet committed
7041
	for (i = 0; i < adapter->num_rx_queues; i++) {
Eric Dumazet's avatar
Eric Dumazet committed
7042
		struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]);
Eric Dumazet's avatar
Eric Dumazet committed
7043 7044 7045
		u64 bytes, packets;
		unsigned int start;

Eric Dumazet's avatar
Eric Dumazet committed
7046 7047 7048 7049 7050 7051 7052 7053 7054
		if (ring) {
			do {
				start = u64_stats_fetch_begin_bh(&ring->syncp);
				packets = ring->stats.packets;
				bytes   = ring->stats.bytes;
			} while (u64_stats_fetch_retry_bh(&ring->syncp, start));
			stats->rx_packets += packets;
			stats->rx_bytes   += bytes;
		}
Eric Dumazet's avatar
Eric Dumazet committed
7055
	}
7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071

	for (i = 0; i < adapter->num_tx_queues; i++) {
		struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]);
		u64 bytes, packets;
		unsigned int start;

		if (ring) {
			do {
				start = u64_stats_fetch_begin_bh(&ring->syncp);
				packets = ring->stats.packets;
				bytes   = ring->stats.bytes;
			} while (u64_stats_fetch_retry_bh(&ring->syncp, start));
			stats->tx_packets += packets;
			stats->tx_bytes   += bytes;
		}
	}
Eric Dumazet's avatar
Eric Dumazet committed
7072
	rcu_read_unlock();
Eric Dumazet's avatar
Eric Dumazet committed
7073 7074 7075 7076 7077 7078 7079 7080 7081
	/* following stats updated by ixgbe_watchdog_task() */
	stats->multicast	= netdev->stats.multicast;
	stats->rx_errors	= netdev->stats.rx_errors;
	stats->rx_length_errors	= netdev->stats.rx_length_errors;
	stats->rx_crc_errors	= netdev->stats.rx_crc_errors;
	stats->rx_missed_errors	= netdev->stats.rx_missed_errors;
	return stats;
}

7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160
/* ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.
 * #adapter: pointer to ixgbe_adapter
 * @tc: number of traffic classes currently enabled
 *
 * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm
 * 802.1Q priority maps to a packet buffer that exists.
 */
static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)
{
	struct ixgbe_hw *hw = &adapter->hw;
	u32 reg, rsave;
	int i;

	/* 82598 have a static priority to TC mapping that can not
	 * be changed so no validation is needed.
	 */
	if (hw->mac.type == ixgbe_mac_82598EB)
		return;

	reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);
	rsave = reg;

	for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
		u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);

		/* If up2tc is out of bounds default to zero */
		if (up2tc > tc)
			reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);
	}

	if (reg != rsave)
		IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);

	return;
}


/* ixgbe_setup_tc - routine to configure net_device for multiple traffic
 * classes.
 *
 * @netdev: net device to configure
 * @tc: number of traffic classes to enable
 */
int ixgbe_setup_tc(struct net_device *dev, u8 tc)
{
	struct ixgbe_adapter *adapter = netdev_priv(dev);
	struct ixgbe_hw *hw = &adapter->hw;

	/* If DCB is anabled do not remove traffic classes, multiple
	 * traffic classes are required to implement DCB
	 */
	if (!tc && (adapter->flags & IXGBE_FLAG_DCB_ENABLED))
		return 0;

	/* Hardware supports up to 8 traffic classes */
	if (tc > MAX_TRAFFIC_CLASS ||
	    (hw->mac.type == ixgbe_mac_82598EB && tc < MAX_TRAFFIC_CLASS))
		return -EINVAL;

	/* Hardware has to reinitialize queues and interrupts to
	 * match packet buffer alignment. Unfortunantly, the
	 * hardware is not flexible enough to do this dynamically.
	 */
	if (netif_running(dev))
		ixgbe_close(dev);
	ixgbe_clear_interrupt_scheme(adapter);

	if (tc)
		netdev_set_num_tc(dev, tc);
	else
		netdev_reset_tc(dev);

	ixgbe_init_interrupt_scheme(adapter);
	ixgbe_validate_rtr(adapter, tc);
	if (netif_running(dev))
		ixgbe_open(dev);

	return 0;
}
Eric Dumazet's avatar
Eric Dumazet committed
7161

7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253
void ixgbe_do_reset(struct net_device *netdev)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);

	if (netif_running(netdev))
		ixgbe_reinit_locked(adapter);
	else
		ixgbe_reset(adapter);
}

static u32 ixgbe_fix_features(struct net_device *netdev, u32 data)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);

#ifdef CONFIG_DCB
	if (adapter->flags & IXGBE_FLAG_DCB_ENABLED)
		data &= ~NETIF_F_HW_VLAN_RX;
#endif

	/* return error if RXHASH is being enabled when RSS is not supported */
	if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED))
		data &= ~NETIF_F_RXHASH;

	/* If Rx checksum is disabled, then RSC/LRO should also be disabled */
	if (!(data & NETIF_F_RXCSUM))
		data &= ~NETIF_F_LRO;

	/* Turn off LRO if not RSC capable or invalid ITR settings */
	if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)) {
		data &= ~NETIF_F_LRO;
	} else if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
		   (adapter->rx_itr_setting != 1 &&
		    adapter->rx_itr_setting > IXGBE_MAX_RSC_INT_RATE)) {
		data &= ~NETIF_F_LRO;
		e_info(probe, "rx-usecs set too low, not enabling RSC\n");
	}

	return data;
}

static int ixgbe_set_features(struct net_device *netdev, u32 data)
{
	struct ixgbe_adapter *adapter = netdev_priv(netdev);
	bool need_reset = false;

	/* If Rx checksum is disabled, then RSC/LRO should also be disabled */
	if (!(data & NETIF_F_RXCSUM))
		adapter->flags &= ~IXGBE_FLAG_RX_CSUM_ENABLED;
	else
		adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;

	/* Make sure RSC matches LRO, reset if change */
	if (!!(data & NETIF_F_LRO) !=
	     !!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {
		adapter->flags2 ^= IXGBE_FLAG2_RSC_ENABLED;
		switch (adapter->hw.mac.type) {
		case ixgbe_mac_X540:
		case ixgbe_mac_82599EB:
			need_reset = true;
			break;
		default:
			break;
		}
	}

	/*
	 * Check if Flow Director n-tuple support was enabled or disabled.  If
	 * the state changed, we need to reset.
	 */
	if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) {
		/* turn off ATR, enable perfect filters and reset */
		if (data & NETIF_F_NTUPLE) {
			adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
			adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
			need_reset = true;
		}
	} else if (!(data & NETIF_F_NTUPLE)) {
		/* turn off Flow Director, set ATR and reset */
		adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
		if ((adapter->flags &  IXGBE_FLAG_RSS_ENABLED) &&
		    !(adapter->flags &  IXGBE_FLAG_DCB_ENABLED))
			adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
		need_reset = true;
	}

	if (need_reset)
		ixgbe_do_reset(netdev);

	return 0;

}

7254
static const struct net_device_ops ixgbe_netdev_ops = {
7255
	.ndo_open		= ixgbe_open,
7256
	.ndo_stop		= ixgbe_close,
7257
	.ndo_start_xmit		= ixgbe_xmit_frame,
7258
	.ndo_select_queue	= ixgbe_select_queue,
7259
	.ndo_set_rx_mode        = ixgbe_set_rx_mode,
7260 7261 7262 7263 7264 7265
	.ndo_validate_addr	= eth_validate_addr,
	.ndo_set_mac_address	= ixgbe_set_mac,
	.ndo_change_mtu		= ixgbe_change_mtu,
	.ndo_tx_timeout		= ixgbe_tx_timeout,
	.ndo_vlan_rx_add_vid	= ixgbe_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid	= ixgbe_vlan_rx_kill_vid,
7266
	.ndo_do_ioctl		= ixgbe_ioctl,
7267 7268 7269 7270
	.ndo_set_vf_mac		= ixgbe_ndo_set_vf_mac,
	.ndo_set_vf_vlan	= ixgbe_ndo_set_vf_vlan,
	.ndo_set_vf_tx_rate	= ixgbe_ndo_set_vf_bw,
	.ndo_get_vf_config	= ixgbe_ndo_get_vf_config,
Eric Dumazet's avatar
Eric Dumazet committed
7271
	.ndo_get_stats64	= ixgbe_get_stats64,
John Fastabend's avatar
John Fastabend committed
7272
	.ndo_setup_tc		= ixgbe_setup_tc,
7273 7274 7275
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller	= ixgbe_netpoll,
#endif
7276 7277
#ifdef IXGBE_FCOE
	.ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
7278
	.ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
7279
	.ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
7280 7281
	.ndo_fcoe_enable = ixgbe_fcoe_enable,
	.ndo_fcoe_disable = ixgbe_fcoe_disable,
7282
	.ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
7283
#endif /* IXGBE_FCOE */
7284 7285
	.ndo_set_features = ixgbe_set_features,
	.ndo_fix_features = ixgbe_fix_features,
7286 7287
};

7288 7289 7290 7291 7292 7293
static void __devinit ixgbe_probe_vf(struct ixgbe_adapter *adapter,
			   const struct ixgbe_info *ii)
{
#ifdef CONFIG_PCI_IOV
	struct ixgbe_hw *hw = &adapter->hw;
	int err;
7294 7295
	int num_vf_macvlans, i;
	struct vf_macvlans *mv_list;
7296

7297
	if (hw->mac.type == ixgbe_mac_82598EB || !max_vfs)
7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308
		return;

	/* The 82599 supports up to 64 VFs per physical function
	 * but this implementation limits allocation to 63 so that
	 * basic networking resources are still available to the
	 * physical function
	 */
	adapter->num_vfs = (max_vfs > 63) ? 63 : max_vfs;
	adapter->flags |= IXGBE_FLAG_SRIOV_ENABLED;
	err = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
	if (err) {
7309
		e_err(probe, "Failed to enable PCI sriov: %d\n", err);
7310 7311
		goto err_novfs;
	}
7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331

	num_vf_macvlans = hw->mac.num_rar_entries -
		(IXGBE_MAX_PF_MACVLANS + 1 + adapter->num_vfs);

	adapter->mv_list = mv_list = kcalloc(num_vf_macvlans,
					     sizeof(struct vf_macvlans),
					     GFP_KERNEL);
	if (mv_list) {
		/* Initialize list of VF macvlans */
		INIT_LIST_HEAD(&adapter->vf_mvs.l);
		for (i = 0; i < num_vf_macvlans; i++) {
			mv_list->vf = -1;
			mv_list->free = true;
			mv_list->rar_entry = hw->mac.num_rar_entries -
				(i + adapter->num_vfs + 1);
			list_add(&mv_list->l, &adapter->vf_mvs.l);
			mv_list++;
		}
	}

7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352
	/* If call to enable VFs succeeded then allocate memory
	 * for per VF control structures.
	 */
	adapter->vfinfo =
		kcalloc(adapter->num_vfs,
			sizeof(struct vf_data_storage), GFP_KERNEL);
	if (adapter->vfinfo) {
		/* Now that we're sure SR-IOV is enabled
		 * and memory allocated set up the mailbox parameters
		 */
		ixgbe_init_mbx_params_pf(hw);
		memcpy(&hw->mbx.ops, ii->mbx_ops,
		       sizeof(hw->mbx.ops));

		/* Disable RSC when in SR-IOV mode */
		adapter->flags2 &= ~(IXGBE_FLAG2_RSC_CAPABLE |
				     IXGBE_FLAG2_RSC_ENABLED);
		return;
	}

	/* Oh oh */
7353 7354
	e_err(probe, "Unable to allocate memory for VF Data Storage - "
	      "SRIOV disabled\n");
7355 7356 7357 7358 7359 7360 7361 7362
	pci_disable_sriov(adapter->pdev);

err_novfs:
	adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
	adapter->num_vfs = 0;
#endif /* CONFIG_PCI_IOV */
}

7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374
/**
 * ixgbe_probe - Device Initialization Routine
 * @pdev: PCI device information struct
 * @ent: entry in ixgbe_pci_tbl
 *
 * Returns 0 on success, negative on failure
 *
 * ixgbe_probe initializes an adapter identified by a pci_dev structure.
 * The OS initialization, configuring of the adapter private structure,
 * and a hardware reset occur.
 **/
static int __devinit ixgbe_probe(struct pci_dev *pdev,
7375
				 const struct pci_device_id *ent)
7376 7377 7378 7379 7380 7381 7382
{
	struct net_device *netdev;
	struct ixgbe_adapter *adapter = NULL;
	struct ixgbe_hw *hw;
	const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
	static int cards_found;
	int i, err, pci_using_dac;
7383
	u8 part_str[IXGBE_PBANUM_LENGTH];
7384
	unsigned int indices = num_possible_cpus();
7385 7386 7387
#ifdef IXGBE_FCOE
	u16 device_caps;
#endif
7388
	u32 eec;
7389

7390 7391 7392 7393 7394 7395 7396 7397 7398
	/* Catch broken hardware that put the wrong VF device ID in
	 * the PCIe SR-IOV capability.
	 */
	if (pdev->is_virtfn) {
		WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
		     pci_name(pdev), pdev->vendor, pdev->device);
		return -EINVAL;
	}

7399
	err = pci_enable_device_mem(pdev);
7400 7401 7402
	if (err)
		return err;

7403 7404
	if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
	    !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
7405 7406
		pci_using_dac = 1;
	} else {
7407
		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
7408
		if (err) {
7409 7410
			err = dma_set_coherent_mask(&pdev->dev,
						    DMA_BIT_MASK(32));
7411
			if (err) {
7412 7413
				dev_err(&pdev->dev,
					"No usable DMA configuration, aborting\n");
7414 7415 7416 7417 7418 7419
				goto err_dma;
			}
		}
		pci_using_dac = 0;
	}

7420
	err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
7421
					   IORESOURCE_MEM), ixgbe_driver_name);
7422
	if (err) {
7423 7424
		dev_err(&pdev->dev,
			"pci_request_selected_regions failed 0x%x\n", err);
7425 7426 7427
		goto err_pci_reg;
	}

7428
	pci_enable_pcie_error_reporting(pdev);
7429

7430
	pci_set_master(pdev);
7431
	pci_save_state(pdev);
7432

7433 7434 7435 7436
#ifdef CONFIG_IXGBE_DCB
	indices *= MAX_TRAFFIC_CLASS;
#endif

7437 7438 7439 7440 7441
	if (ii->mac == ixgbe_mac_82598EB)
		indices = min_t(unsigned int, indices, IXGBE_MAX_RSS_INDICES);
	else
		indices = min_t(unsigned int, indices, IXGBE_MAX_FDIR_INDICES);

7442
#ifdef IXGBE_FCOE
7443 7444 7445 7446
	indices += min_t(unsigned int, num_possible_cpus(),
			 IXGBE_MAX_FCOE_INDICES);
#endif
	netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices);
7447 7448 7449 7450 7451 7452 7453 7454
	if (!netdev) {
		err = -ENOMEM;
		goto err_alloc_etherdev;
	}

	SET_NETDEV_DEV(netdev, &pdev->dev);

	adapter = netdev_priv(netdev);
7455
	pci_set_drvdata(pdev, adapter);
7456 7457 7458 7459 7460 7461 7462

	adapter->netdev = netdev;
	adapter->pdev = pdev;
	hw = &adapter->hw;
	hw->back = adapter;
	adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1;

7463
	hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
7464
			      pci_resource_len(pdev, 0));
7465 7466 7467 7468 7469 7470 7471 7472 7473 7474
	if (!hw->hw_addr) {
		err = -EIO;
		goto err_ioremap;
	}

	for (i = 1; i <= 5; i++) {
		if (pci_resource_len(pdev, i) == 0)
			continue;
	}

7475
	netdev->netdev_ops = &ixgbe_netdev_ops;
7476 7477
	ixgbe_set_ethtool_ops(netdev);
	netdev->watchdog_timeo = 5 * HZ;
7478
	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
7479 7480 7481 7482 7483

	adapter->bd_number = cards_found;

	/* Setup hw api */
	memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
7484
	hw->mac.type  = ii->mac;
7485

7486 7487 7488 7489 7490 7491 7492 7493 7494
	/* EEPROM */
	memcpy(&hw->eeprom.ops, ii->eeprom_ops, sizeof(hw->eeprom.ops));
	eec = IXGBE_READ_REG(hw, IXGBE_EEC);
	/* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
	if (!(eec & (1 << 8)))
		hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;

	/* PHY */
	memcpy(&hw->phy.ops, ii->phy_ops, sizeof(hw->phy.ops));
7495
	hw->phy.sfp_type = ixgbe_sfp_type_unknown;
7496 7497 7498 7499 7500 7501 7502
	/* ixgbe_identify_phy_generic will set prtad and mmds properly */
	hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
	hw->phy.mdio.mmds = 0;
	hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
	hw->phy.mdio.dev = netdev;
	hw->phy.mdio.mdio_read = ixgbe_mdio_read;
	hw->phy.mdio.mdio_write = ixgbe_mdio_write;
7503

7504
	ii->get_invariants(hw);
7505 7506 7507 7508 7509 7510

	/* setup the private structure */
	err = ixgbe_sw_init(adapter);
	if (err)
		goto err_sw_init;

7511
	/* Make it possible the adapter to be woken up via WOL */
7512 7513 7514
	switch (adapter->hw.mac.type) {
	case ixgbe_mac_82599EB:
	case ixgbe_mac_X540:
7515
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
7516 7517 7518 7519
		break;
	default:
		break;
	}
7520

7521 7522 7523 7524 7525 7526 7527
	/*
	 * If there is a fan on this device and it has failed log the
	 * failure.
	 */
	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
		u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
		if (esdp & IXGBE_ESDP_SDP1)
7528
			e_crit(probe, "Fan has stopped, replace the adapter\n");
7529 7530
	}

7531
	/* reset_hw fills in the perm_addr as well */
7532
	hw->phy.reset_if_overtemp = true;
7533
	err = hw->mac.ops.reset_hw(hw);
7534
	hw->phy.reset_if_overtemp = false;
7535 7536 7537 7538
	if (err == IXGBE_ERR_SFP_NOT_PRESENT &&
	    hw->mac.type == ixgbe_mac_82598EB) {
		err = 0;
	} else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
7539
		e_dev_err("failed to load because an unsupported SFP+ "
7540 7541 7542
			  "module type was detected.\n");
		e_dev_err("Reload the driver after installing a supported "
			  "module.\n");
7543 7544
		goto err_sw_init;
	} else if (err) {
7545
		e_dev_err("HW Init failed: %d\n", err);
7546 7547 7548
		goto err_sw_init;
	}

7549 7550
	ixgbe_probe_vf(adapter, ii);

7551
	netdev->features = NETIF_F_SG |
7552
			   NETIF_F_IP_CSUM |
7553
			   NETIF_F_IPV6_CSUM |
7554 7555
			   NETIF_F_HW_VLAN_TX |
			   NETIF_F_HW_VLAN_RX |
7556 7557 7558 7559 7560 7561
			   NETIF_F_HW_VLAN_FILTER |
			   NETIF_F_TSO |
			   NETIF_F_TSO6 |
			   NETIF_F_GRO |
			   NETIF_F_RXHASH |
			   NETIF_F_RXCSUM;
7562

7563
	netdev->hw_features = netdev->features;
7564

7565 7566 7567
	switch (adapter->hw.mac.type) {
	case ixgbe_mac_82599EB:
	case ixgbe_mac_X540:
7568
		netdev->features |= NETIF_F_SCTP_CSUM;
7569 7570
		netdev->hw_features |= NETIF_F_SCTP_CSUM |
				       NETIF_F_NTUPLE;
7571 7572 7573 7574
		break;
	default:
		break;
	}
7575

7576 7577
	netdev->vlan_features |= NETIF_F_TSO;
	netdev->vlan_features |= NETIF_F_TSO6;
7578
	netdev->vlan_features |= NETIF_F_IP_CSUM;
7579
	netdev->vlan_features |= NETIF_F_IPV6_CSUM;
7580 7581
	netdev->vlan_features |= NETIF_F_SG;

7582 7583
	netdev->priv_flags |= IFF_UNICAST_FLT;

7584 7585 7586
	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
		adapter->flags &= ~(IXGBE_FLAG_RSS_ENABLED |
				    IXGBE_FLAG_DCB_ENABLED);
7587

Jeff Kirsher's avatar
Jeff Kirsher committed
7588
#ifdef CONFIG_IXGBE_DCB
7589 7590 7591
	netdev->dcbnl_ops = &dcbnl_ops;
#endif

7592
#ifdef IXGBE_FCOE
7593
	if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
7594 7595
		if (hw->mac.ops.get_device_caps) {
			hw->mac.ops.get_device_caps(hw, &device_caps);
7596 7597
			if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
				adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
7598 7599
		}
	}
7600 7601 7602 7603 7604
	if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
		netdev->vlan_features |= NETIF_F_FCOE_CRC;
		netdev->vlan_features |= NETIF_F_FSO;
		netdev->vlan_features |= NETIF_F_FCOE_MTU;
	}
7605
#endif /* IXGBE_FCOE */
7606
	if (pci_using_dac) {
7607
		netdev->features |= NETIF_F_HIGHDMA;
7608 7609
		netdev->vlan_features |= NETIF_F_HIGHDMA;
	}
7610

7611 7612
	if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
		netdev->hw_features |= NETIF_F_LRO;
7613
	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
7614 7615
		netdev->features |= NETIF_F_LRO;

7616
	/* make sure the EEPROM is good */
7617
	if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
7618
		e_dev_err("The EEPROM Checksum Is Not Valid\n");
7619 7620 7621 7622 7623 7624 7625
		err = -EIO;
		goto err_eeprom;
	}

	memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len);
	memcpy(netdev->perm_addr, hw->mac.perm_addr, netdev->addr_len);

7626
	if (ixgbe_validate_mac_addr(netdev->perm_addr)) {
7627
		e_dev_err("invalid MAC address\n");
7628 7629 7630 7631
		err = -EIO;
		goto err_eeprom;
	}

7632 7633 7634
	/* power down the optics for multispeed fiber and 82599 SFP+ fiber */
	if (hw->mac.ops.disable_tx_laser &&
	    ((hw->phy.multispeed_fiber) ||
7635
	     ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
7636
	      (hw->mac.type == ixgbe_mac_82599EB))))
7637 7638
		hw->mac.ops.disable_tx_laser(hw);

7639 7640
	setup_timer(&adapter->service_timer, &ixgbe_service_timer,
	            (unsigned long) adapter);
7641

7642 7643
	INIT_WORK(&adapter->service_task, ixgbe_service_task);
	clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
7644

7645 7646 7647
	err = ixgbe_init_interrupt_scheme(adapter);
	if (err)
		goto err_sw_init;
7648

7649 7650
	if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
		netdev->hw_features &= ~NETIF_F_RXHASH;
Emil Tantilov's avatar
Emil Tantilov committed
7651
		netdev->features &= ~NETIF_F_RXHASH;
7652
	}
Emil Tantilov's avatar
Emil Tantilov committed
7653

7654
	switch (pdev->device) {
7655 7656 7657
	case IXGBE_DEV_ID_82599_SFP:
		/* Only this subdevice supports WOL */
		if (pdev->subsystem_device == IXGBE_SUBDEV_ID_82599_SFP)
7658
			adapter->wol = IXGBE_WUFC_MAG;
7659
		break;
7660 7661
	case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
		/* All except this subdevice support WOL */
7662
		if (pdev->subsystem_device != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
7663
			adapter->wol = IXGBE_WUFC_MAG;
7664
		break;
7665
	case IXGBE_DEV_ID_82599_KX4:
7666
		adapter->wol = IXGBE_WUFC_MAG;
7667 7668 7669 7670 7671 7672 7673
		break;
	default:
		adapter->wol = 0;
		break;
	}
	device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);

7674 7675 7676
	/* pick up the PCI bus settings for reporting later */
	hw->mac.ops.get_bus_info(hw);

7677
	/* print bus type/speed/width info */
7678
	e_dev_info("(PCI Express:%s:%s) %pM\n",
7679 7680
		   (hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" :
		    hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" :
7681 7682 7683 7684 7685 7686
		    "Unknown"),
		   (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" :
		    hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" :
		    hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" :
		    "Unknown"),
		   netdev->dev_addr);
7687 7688 7689

	err = ixgbe_read_pba_string_generic(hw, part_str, IXGBE_PBANUM_LENGTH);
	if (err)
7690
		strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH);
7691
	if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
7692
		e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
7693
			   hw->mac.type, hw->phy.type, hw->phy.sfp_type,
7694
		           part_str);
7695
	else
7696 7697
		e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
			   hw->mac.type, hw->phy.type, part_str);
7698

7699
	if (hw->bus.width <= ixgbe_bus_width_pcie_x4) {
7700 7701 7702 7703
		e_dev_warn("PCI-Express bandwidth available for this card is "
			   "not sufficient for optimal performance.\n");
		e_dev_warn("For optimal performance a x8 PCI-Express slot "
			   "is required.\n");
7704 7705
	}

7706 7707 7708
	/* save off EEPROM version number */
	hw->eeprom.ops.read(hw, 0x29, &adapter->eeprom_version);

7709
	/* reset the hardware with the new settings */
7710
	err = hw->mac.ops.start_hw(hw);
7711

7712 7713
	if (err == IXGBE_ERR_EEPROM_VERSION) {
		/* We are running on a pre-production device, log a warning */
7714 7715 7716 7717 7718 7719
		e_dev_warn("This device is a pre-production adapter/LOM. "
			   "Please be aware there may be issues associated "
			   "with your hardware.  If you are experiencing "
			   "problems please contact your Intel or hardware "
			   "representative who provided you with this "
			   "hardware.\n");
7720
	}
7721 7722 7723 7724 7725
	strcpy(netdev->name, "eth%d");
	err = register_netdev(netdev);
	if (err)
		goto err_register;

7726 7727 7728
	/* carrier off reporting is important to ethtool even BEFORE open */
	netif_carrier_off(netdev);

7729
#ifdef CONFIG_IXGBE_DCA
7730
	if (dca_add_requester(&pdev->dev) == 0) {
7731 7732 7733 7734
		adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
		ixgbe_setup_dca(adapter);
	}
#endif
7735
	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
7736
		e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
7737 7738 7739 7740
		for (i = 0; i < adapter->num_vfs; i++)
			ixgbe_vf_configuration(pdev, (i | 0x10000000));
	}

7741 7742
	/* Inform firmware of driver version */
	if (hw->mac.ops.set_fw_drv_ver)
7743 7744
		hw->mac.ops.set_fw_drv_ver(hw, MAJ, MIN, BUILD,
					   FW_CEM_UNUSED_VER);
7745

7746 7747
	/* add san mac addr to netdev */
	ixgbe_add_sanmac_netdev(netdev);
7748

7749
	e_dev_info("Intel(R) 10 Gigabit Network Connection\n");
7750 7751 7752 7753
	cards_found++;
	return 0;

err_register:
7754
	ixgbe_release_hw_control(adapter);
7755
	ixgbe_clear_interrupt_scheme(adapter);
7756 7757
err_sw_init:
err_eeprom:
7758 7759
	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
		ixgbe_disable_sriov(adapter);
7760
	adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
7761 7762 7763 7764
	iounmap(hw->hw_addr);
err_ioremap:
	free_netdev(netdev);
err_alloc_etherdev:
7765 7766
	pci_release_selected_regions(pdev,
				     pci_select_bars(pdev, IORESOURCE_MEM));
7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783
err_pci_reg:
err_dma:
	pci_disable_device(pdev);
	return err;
}

/**
 * ixgbe_remove - Device Removal Routine
 * @pdev: PCI device information struct
 *
 * ixgbe_remove is called by the PCI subsystem to alert the driver
 * that it should release a PCI device.  The could be caused by a
 * Hot-Plug event, or because the driver is going to be removed from
 * memory.
 **/
static void __devexit ixgbe_remove(struct pci_dev *pdev)
{
7784 7785
	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
	struct net_device *netdev = adapter->netdev;
7786 7787

	set_bit(__IXGBE_DOWN, &adapter->state);
7788
	cancel_work_sync(&adapter->service_task);
7789

7790
#ifdef CONFIG_IXGBE_DCA
7791 7792 7793 7794 7795 7796 7797
	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
		adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
		dca_remove_requester(&pdev->dev);
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
	}

#endif
7798 7799 7800 7801 7802
#ifdef IXGBE_FCOE
	if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
		ixgbe_cleanup_fcoe(adapter);

#endif /* IXGBE_FCOE */
7803 7804 7805 7806

	/* remove the added san mac */
	ixgbe_del_sanmac_netdev(netdev);

7807 7808
	if (netdev->reg_state == NETREG_REGISTERED)
		unregister_netdev(netdev);
7809

7810 7811 7812
	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
		ixgbe_disable_sriov(adapter);

7813
	ixgbe_clear_interrupt_scheme(adapter);
7814

7815
	ixgbe_release_hw_control(adapter);
7816 7817

	iounmap(adapter->hw.hw_addr);
7818
	pci_release_selected_regions(pdev, pci_select_bars(pdev,
7819
				     IORESOURCE_MEM));
7820

7821
	e_dev_info("complete\n");
7822

7823 7824
	free_netdev(netdev);

7825
	pci_disable_pcie_error_reporting(pdev);
7826

7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838
	pci_disable_device(pdev);
}

/**
 * ixgbe_io_error_detected - called when PCI error is detected
 * @pdev: Pointer to PCI device
 * @state: The current pci connection state
 *
 * This function is called after a PCI bus error affecting
 * this device has been detected.
 */
static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
7839
						pci_channel_state_t state)
7840
{
7841 7842
	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
	struct net_device *netdev = adapter->netdev;
7843 7844 7845

	netif_device_detach(netdev);

7846 7847 7848
	if (state == pci_channel_io_perm_failure)
		return PCI_ERS_RESULT_DISCONNECT;

7849 7850 7851 7852
	if (netif_running(netdev))
		ixgbe_down(adapter);
	pci_disable_device(pdev);

7853
	/* Request a slot reset. */
7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864
	return PCI_ERS_RESULT_NEED_RESET;
}

/**
 * ixgbe_io_slot_reset - called after the pci bus has been reset.
 * @pdev: Pointer to PCI device
 *
 * Restart the card from scratch, as if from a cold-boot.
 */
static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
{
7865
	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7866 7867
	pci_ers_result_t result;
	int err;
7868

7869
	if (pci_enable_device_mem(pdev)) {
7870
		e_err(probe, "Cannot re-enable PCI device after reset.\n");
7871 7872 7873 7874
		result = PCI_ERS_RESULT_DISCONNECT;
	} else {
		pci_set_master(pdev);
		pci_restore_state(pdev);
7875
		pci_save_state(pdev);
7876

7877
		pci_wake_from_d3(pdev, false);
7878

7879
		ixgbe_reset(adapter);
7880
		IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
7881 7882 7883 7884 7885
		result = PCI_ERS_RESULT_RECOVERED;
	}

	err = pci_cleanup_aer_uncorrect_error_status(pdev);
	if (err) {
7886 7887
		e_dev_err("pci_cleanup_aer_uncorrect_error_status "
			  "failed 0x%0x\n", err);
7888 7889
		/* non-fatal, continue */
	}
7890

7891
	return result;
7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902
}

/**
 * ixgbe_io_resume - called when traffic can start flowing again.
 * @pdev: Pointer to PCI device
 *
 * This callback is called when the error recovery driver tells us that
 * its OK to resume normal operation.
 */
static void ixgbe_io_resume(struct pci_dev *pdev)
{
7903 7904
	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
	struct net_device *netdev = adapter->netdev;
7905 7906 7907

	if (netif_running(netdev)) {
		if (ixgbe_up(adapter)) {
7908
			e_info(probe, "ixgbe_up failed after reset\n");
7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943
			return;
		}
	}

	netif_device_attach(netdev);
}

static struct pci_error_handlers ixgbe_err_handler = {
	.error_detected = ixgbe_io_error_detected,
	.slot_reset = ixgbe_io_slot_reset,
	.resume = ixgbe_io_resume,
};

static struct pci_driver ixgbe_driver = {
	.name     = ixgbe_driver_name,
	.id_table = ixgbe_pci_tbl,
	.probe    = ixgbe_probe,
	.remove   = __devexit_p(ixgbe_remove),
#ifdef CONFIG_PM
	.suspend  = ixgbe_suspend,
	.resume   = ixgbe_resume,
#endif
	.shutdown = ixgbe_shutdown,
	.err_handler = &ixgbe_err_handler
};

/**
 * ixgbe_init_module - Driver Registration Routine
 *
 * ixgbe_init_module is the first routine called when the driver is
 * loaded. All it does is register with the PCI subsystem.
 **/
static int __init ixgbe_init_module(void)
{
	int ret;
7944
	pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version);
7945
	pr_info("%s\n", ixgbe_copyright);
7946

7947
#ifdef CONFIG_IXGBE_DCA
7948 7949
	dca_register_notify(&dca_notifier);
#endif
7950

7951 7952 7953
	ret = pci_register_driver(&ixgbe_driver);
	return ret;
}
7954

7955 7956 7957 7958 7959 7960 7961 7962 7963 7964
module_init(ixgbe_init_module);

/**
 * ixgbe_exit_module - Driver Exit Cleanup Routine
 *
 * ixgbe_exit_module is called just before the driver is removed
 * from memory.
 **/
static void __exit ixgbe_exit_module(void)
{
7965
#ifdef CONFIG_IXGBE_DCA
7966 7967
	dca_unregister_notify(&dca_notifier);
#endif
7968
	pci_unregister_driver(&ixgbe_driver);
Eric Dumazet's avatar
Eric Dumazet committed
7969
	rcu_barrier(); /* Wait for completion of call_rcu()'s */
7970
}
7971

7972
#ifdef CONFIG_IXGBE_DCA
7973
static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
7974
			    void *p)
7975 7976 7977 7978
{
	int ret_val;

	ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event,
7979
					 __ixgbe_notify_dca);
7980 7981 7982

	return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
}
7983

7984
#endif /* CONFIG_IXGBE_DCA */
7985

7986 7987 7988
module_exit(ixgbe_exit_module);

/* ixgbe_main.c */