ixgbe: DCB, use multiple Tx rings per traffic class

This enables multiple {Tx|Rx} rings per traffic class while in DCB
mode. In order to get this working as expected the tc_to_tx net
device mapping is configured as well as the prio_tc_map.

skb priorities are mapped across a range of queue pairs to get
a distribution per traffic class. The maximum number of
queue pairs used while in DCB mode is capped at 64. The hardware
max is actually 128 queues but 64 is sufficient for now and
allocating more seemed a bit excessive. It is easy enough to
increase the cap later if need be.

To get the 802.1Q priority tags inserted correctly ixgbe was
previously using the skb queue_mapping field to directly set
the 802.1Q priority. This no longer works because we have removed
the 1:1 mapping between queues and traffic class. Each ring
is aligned with an 802.1Qaz traffic class so here we add an
extra field to the ring struct to identify the 802.1Q traffic
class. This uses an extra byte of the ixgbe_ring struct
fortunately there was a 2byte hole,

struct ixgbe_ring {
        void *                     desc;                 /*     0     8 */
        struct device *            dev;                  /*     8     8 */
        struct net_device *        netdev;               /*    16     8 */
        union {
                struct ixgbe_tx_buffer * tx_buffer_info; /*           8 */
                struct ixgbe_rx_buffer * rx_buffer_info; /*           8 */
        };                                               /*    24     8 */
        long unsigned int          state;                /*    32     8 */
        u8                         atr_sample_rate;      /*    40     1 */
        u8                         atr_count;            /*    41     1 */
        u16                        count;                /*    42     2 */
        u16                        rx_buf_len;           /*    44     2 */
        u16                        next_to_use;          /*    46     2 */
        u16                        next_to_clean;        /*    48     2 */
        u8                         queue_index;          /*    50     1 */
        u8                         reg_idx;              /*    51     1 */
        u16                        work_limit;           /*    52     2 */

        /* XXX 2 bytes hole, try to pack */

        u8 *                       tail;                 /*    56     8 */
        /* --- cacheline 1 boundary (64 bytes) --- */

Now we can set the VLAN priority directly and it will be
correct. User space can indicate the 802.1Qaz priority
using the SO_PRIORITY setsocket() option and QOS layer will
steer the skb to the correct rings. Additionally using
the multiq qdisc with a queue_mapping action works as
well.
Signed-off-by: John Fastabend <john.r.fastabend@intel.com>
Tested-by: Ross Brattain <ross.b.brattain@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>

drivers/net/ixgbe/ixgbe.h

@@ -209,6 +209,7 @@ struct ixgbe_ring {
 					 * associated with this ring, which is
 					 * different for DCB and RSS modes
 					 */
+	u8 dcb_tc;
 
 	u16 work_limit;			/* max work per interrupt */
 
@@ -243,7 +244,7 @@ enum ixgbe_ring_f_enum {
 	RING_F_ARRAY_SIZE      /* must be last in enum set */
 };
 
-#define IXGBE_MAX_DCB_INDICES   8
+#define IXGBE_MAX_DCB_INDICES  64
 #define IXGBE_MAX_RSS_INDICES  16
 #define IXGBE_MAX_VMDQ_INDICES 64
 #define IXGBE_MAX_FDIR_INDICES 64
@@ -542,6 +543,7 @@ extern void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
 extern void ixgbe_clear_rscctl(struct ixgbe_adapter *adapter,
                                struct ixgbe_ring *ring);
 extern void ixgbe_set_rx_mode(struct net_device *netdev);
+extern int ixgbe_setup_tc(struct net_device *dev, u8 tc);
 #ifdef IXGBE_FCOE
 extern void ixgbe_configure_fcoe(struct ixgbe_adapter *adapter);
 extern int ixgbe_fso(struct ixgbe_adapter *adapter,

drivers/net/ixgbe/ixgbe_dcb_nl.c

@@ -145,6 +145,9 @@ static u8 ixgbe_dcbnl_set_state(struct net_device *netdev, u8 state)
 		}
 
 		adapter->flags |= IXGBE_FLAG_DCB_ENABLED;
+		if (!netdev_get_num_tc(netdev))
+			ixgbe_setup_tc(netdev, MAX_TRAFFIC_CLASS);
+
 		ixgbe_init_interrupt_scheme(adapter);
 		if (netif_running(netdev))
 			netdev->netdev_ops->ndo_open(netdev);
@@ -169,6 +172,8 @@ static u8 ixgbe_dcbnl_set_state(struct net_device *netdev, u8 state)
 				break;
 			}
 
+			ixgbe_setup_tc(netdev, 0);
+
 			ixgbe_init_interrupt_scheme(adapter);
 			if (netif_running(netdev))
 				netdev->netdev_ops->ndo_open(netdev);
@@ -351,7 +356,7 @@ static u8 ixgbe_dcbnl_set_all(struct net_device *netdev)
 		return DCB_NO_HW_CHG;
 
 	ret = ixgbe_copy_dcb_cfg(&adapter->temp_dcb_cfg, &adapter->dcb_cfg,
-				 adapter->ring_feature[RING_F_DCB].indices);
+				 MAX_TRAFFIC_CLASS);
 
 	if (ret)
 		return DCB_NO_HW_CHG;