scsi: mpt3sas: Handling HBA vSES device

Each direct attached device will have a unique Port ID, but with an
exception. HBA vSES may use the same Port ID of another direct attached
device Port's ID. As a result, special handling is needed for vSES.

Create a virtual_phy object when a new HBA vSES device is detected and add
this virtual_phy object to vphys_list of port ID's hba_port object.  When
the HBA vSES device is removed then remove the corresponding virtual_phy
object from its parent's hba_port's vphy_list and free this virtual_vphy
object.

In hba_port object add vphy_mask field to hold the list of HBA phy bits
which are assigned to vSES devices. Also add vphy_list list to hold list of
virtual_phy objects which holds the same portID of current hba_port's
portID.

Also, add a hba_vphy field in _sas_phy object to determine whether this
_sas_phy object belongs to vSES device or not.

 - Allocate a virtual_phy object whenever a virtual phy is detected while
   processing the SASIOUnitPage0's phy data. And this allocated virtual_phy
   object to corresponding PortID's hba_port's vphy_list.

 - When a vSES device is added to the SML then initialize the corresponding
   virtual_phy objects's sas_address field with vSES device's SAS Address.

 - Free this virtual_phy object during driver unload time and when this
   vSES device is removed.

Link: https://lore.kernel.org/r/20201027130847.9962-11-sreekanth.reddy@broadcom.comSigned-off-by: Sreekanth Reddy <sreekanth.reddy@broadcom.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

drivers/scsi/mpt3sas/mpt3sas_base.h

@@ -771,6 +771,7 @@ struct _sas_phy {
 	u16	handle;
 	u16	attached_handle;
 	u8	phy_belongs_to_port;
+	u8	hba_vphy;
 	struct hba_port *port;
 };
 
@@ -1023,12 +1024,29 @@ struct reply_post_struct {
 	dma_addr_t			reply_post_free_dma;
 };
 
+/**
+ * struct virtual_phy - vSES phy structure
+ * sas_address: SAS Address of vSES device
+ * phy_mask: vSES device's phy number
+ * flags: flags used to manage this structure
+ */
+struct virtual_phy {
+	struct	list_head list;
+	u64	sas_address;
+	u32	phy_mask;
+	u8	flags;
+};
+
+#define MPT_VPHY_FLAG_DIRTY_PHY	0x01
+
 /**
  * struct hba_port - Saves each HBA's Wide/Narrow port info
  * @sas_address: sas address of this wide/narrow port's attached device
  * @phy_mask: HBA PHY's belonging to this port
  * @port_id: port number
  * @flags: hba port flags
+ * @vphys_mask : mask of vSES devices Phy number
+ * @vphys_list : list containing vSES device structures
  */
 struct hba_port {
 	struct list_head list;
@@ -1036,6 +1054,8 @@ struct hba_port {
 	u32	phy_mask;
 	u8      port_id;
 	u8	flags;
+	u32	vphys_mask;
+	struct list_head vphys_list;
 };
 
 /* hba port flags */
@@ -1688,6 +1708,9 @@ mpt3sas_raid_device_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle);
 void mpt3sas_scsih_change_queue_depth(struct scsi_device *sdev, int qdepth);
 struct _sas_device *
 __mpt3sas_get_sdev_by_rphy(struct MPT3SAS_ADAPTER *ioc, struct sas_rphy *rphy);
+struct virtual_phy *
+mpt3sas_get_vphy_by_phy(struct MPT3SAS_ADAPTER *ioc,
+	struct hba_port *port, u32 phy);
 
 /* config shared API */
 u8 mpt3sas_config_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,

drivers/scsi/mpt3sas/mpt3sas_scsih.c

@@ -380,6 +380,30 @@ mpt3sas_get_port_by_id(struct MPT3SAS_ADAPTER *ioc, u8 port_id)
 	return NULL;
 }
 
+/**
+ * mpt3sas_get_vphy_by_phy - get virtual_phy object corresponding to phy number
+ * @ioc: per adapter object
+ * @port: hba_port object
+ * @phy: phy number
+ *
+ * Return virtual_phy object corresponding to phy number.
+ */
+struct virtual_phy *
+mpt3sas_get_vphy_by_phy(struct MPT3SAS_ADAPTER *ioc,
+	struct hba_port *port, u32 phy)
+{
+	struct virtual_phy *vphy, *vphy_next;
+
+	if (!port->vphys_mask)
+		return NULL;
+
+	list_for_each_entry_safe(vphy, vphy_next, &port->vphys_list, list) {
+		if (vphy->phy_mask & (1 << phy))
+			return vphy;
+	}
+	return NULL;
+}
+
 /**
  * _scsih_is_boot_device - search for matching boot device.
  * @sas_address: sas address
@@ -6152,6 +6176,47 @@ _scsih_sas_port_refresh(struct MPT3SAS_ADAPTER *ioc)
 	port_table_entry = NULL;
 }
 
+/**
+ * _scsih_alloc_vphy - allocate virtual_phy object
+ * @ioc: per adapter object
+ * @port_id: Port ID number
+ * @phy_num: HBA Phy number
+ *
+ * Returns allocated virtual_phy object.
+ */
+static struct virtual_phy *
+_scsih_alloc_vphy(struct MPT3SAS_ADAPTER *ioc, u8 port_id, u8 phy_num)
+{
+	struct virtual_phy *vphy;
+	struct hba_port *port;
+
+	port = mpt3sas_get_port_by_id(ioc, port_id);
+	if (!port)
+		return NULL;
+
+	vphy = mpt3sas_get_vphy_by_phy(ioc, port, phy_num);
+	if (!vphy) {
+		vphy = kzalloc(sizeof(struct virtual_phy), GFP_KERNEL);
+		if (!vphy)
+			return NULL;
+
+		/*
+		 * Enable bit corresponding to HBA phy number on its
+		 * parent hba_port object's vphys_mask field.
+		 */
+		port->vphys_mask |= (1 << phy_num);
+		vphy->phy_mask |= (1 << phy_num);
+
+		INIT_LIST_HEAD(&port->vphys_list);
+		list_add_tail(&vphy->list, &port->vphys_list);
+
+		ioc_info(ioc,
+		    "vphy entry: %p, port id: %d, phy:%d is added to port's vphys_list\n",
+		    vphy, port->port_id, phy_num);
+	}
+	return vphy;
+}
+
 /**
  * _scsih_sas_host_refresh - refreshing sas host object contents
  * @ioc: per adapter object
@@ -6172,6 +6237,7 @@ _scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)
 	u16 attached_handle;
 	u8 link_rate, port_id;
 	struct hba_port *port;
+	Mpi2SasPhyPage0_t phy_pg0;
 
 	dtmprintk(ioc,
 		  ioc_info(ioc, "updating handles for sas_host(0x%016llx)\n",
@@ -6211,6 +6277,31 @@ _scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)
 				port->flags = HBA_PORT_FLAG_NEW_PORT;
 			list_add_tail(&port->list, &ioc->port_table_list);
 		}
+		/*
+		 * Check whether current Phy belongs to HBA vSES device or not.
+		 */
+		if (le32_to_cpu(sas_iounit_pg0->PhyData[i].ControllerPhyDeviceInfo) &
+		    MPI2_SAS_DEVICE_INFO_SEP &&
+		    (link_rate >=  MPI2_SAS_NEG_LINK_RATE_1_5)) {
+			if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply,
+			    &phy_pg0, i))) {
+				ioc_err(ioc,
+				    "failure at %s:%d/%s()!\n",
+				     __FILE__, __LINE__, __func__);
+				goto out;
+			}
+			if (!(le32_to_cpu(phy_pg0.PhyInfo) &
+			    MPI2_SAS_PHYINFO_VIRTUAL_PHY))
+				continue;
+			/*
+			 * Allocate a virtual_phy object for vSES device, if
+			 * this vSES device is hot added.
+			 */
+			if (!_scsih_alloc_vphy(ioc, port_id, i))
+				goto out;
+			ioc->sas_hba.phy[i].hba_vphy = 1;
+		}
+
 		ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;
 		attached_handle = le16_to_cpu(sas_iounit_pg0->PhyData[i].
 		    AttachedDevHandle);
@@ -6353,6 +6444,21 @@ _scsih_sas_host_add(struct MPT3SAS_ADAPTER *ioc)
 			    &ioc->port_table_list);
 		}
 
+		/*
+		 * Check whether current Phy belongs to HBA vSES device or not.
+		 */
+		if ((le32_to_cpu(phy_pg0.PhyInfo) &
+		    MPI2_SAS_PHYINFO_VIRTUAL_PHY) &&
+		    (phy_pg0.NegotiatedLinkRate >> 4) >=
+		    MPI2_SAS_NEG_LINK_RATE_1_5) {
+			/*
+			 * Allocate a virtual_phy object for vSES device.
+			 */
+			if (!_scsih_alloc_vphy(ioc, port_id, i))
+				goto out;
+			ioc->sas_hba.phy[i].hba_vphy = 1;
+		}
+
 		ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;
 		ioc->sas_hba.phy[i].phy_id = i;
 		ioc->sas_hba.phy[i].port = mpt3sas_get_port_by_id(ioc, port_id);

drivers/scsi/mpt3sas/mpt3sas_transport.c

@@ -690,6 +690,7 @@ mpt3sas_transport_port_add(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 	struct _sas_device *sas_device = NULL;
 	int i;
 	struct sas_port *port;
+	struct virtual_phy *vphy = NULL;
 
 	if (!hba_port) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
@@ -743,9 +744,20 @@ mpt3sas_transport_port_add(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 			continue;
 		list_add_tail(&sas_node->phy[i].port_siblings,
 		    &mpt3sas_port->phy_list);
-		if (sas_node->handle <= ioc->sas_hba.num_phys)
-			hba_port->phy_mask |= (1 << i);
 		mpt3sas_port->num_phys++;
+		if (sas_node->handle <= ioc->sas_hba.num_phys) {
+			if (!sas_node->phy[i].hba_vphy) {
+				hba_port->phy_mask |= (1 << i);
+				continue;
+			}
+
+			vphy = mpt3sas_get_vphy_by_phy(ioc, hba_port, i);
+			if (!vphy) {
+				ioc_err(ioc, "failure at %s:%d/%s()!\n",
+				    __FILE__, __LINE__, __func__);
+				goto out_fail;
+			}
+		}
 	}
 
 	if (!mpt3sas_port->num_phys) {
@@ -795,8 +807,14 @@ mpt3sas_transport_port_add(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 	if (mpt3sas_port->remote_identify.device_type == SAS_END_DEVICE) {
 		rphy = sas_end_device_alloc(port);
 		sas_device->rphy = rphy;
-		if (sas_node->handle <= ioc->sas_hba.num_phys)
-			hba_port->sas_address = sas_device->sas_address;
+		if (sas_node->handle <= ioc->sas_hba.num_phys) {
+			if (!vphy)
+				hba_port->sas_address =
+				    sas_device->sas_address;
+			else
+				vphy->sas_address =
+				    sas_device->sas_address;
+		}
 	} else {
 		rphy = sas_expander_alloc(port,
 		    mpt3sas_port->remote_identify.device_type);
@@ -866,6 +884,7 @@ mpt3sas_transport_port_remove(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
 	u8 found = 0;
 	struct _sas_phy *mpt3sas_phy, *next_phy;
 	struct hba_port *hba_port_next, *hba_port = NULL;
+	struct virtual_phy *vphy, *vphy_next = NULL;
 
 	if (!port)
 		return;
@@ -894,17 +913,56 @@ mpt3sas_transport_port_remove(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
 	}
 
 	if (sas_node->handle <= ioc->sas_hba.num_phys) {
+		if (port->vphys_mask) {
+			list_for_each_entry_safe(vphy, vphy_next,
+			    &port->vphys_list, list) {
+				if (vphy->sas_address != sas_address)
+					continue;
+				ioc_info(ioc,
+				    "remove vphy entry: %p of port:%p,from %d port's vphys list\n",
+				    vphy, port, port->port_id);
+				port->vphys_mask &= ~vphy->phy_mask;
+				list_del(&vphy->list);
+				kfree(vphy);
+			}
+		}
+
 		list_for_each_entry_safe(hba_port, hba_port_next,
 		    &ioc->port_table_list, list) {
 			if (hba_port != port)
 				continue;
-			if (hba_port->sas_address != sas_address)
-				continue;
-			ioc_info(ioc,
-			    "remove hba_port entry: %p port: %d from hba_port list\n",
-			    hba_port, hba_port->port_id);
-			list_del(&hba_port->list);
-			kfree(hba_port);
+			/*
+			 * Delete hba_port object if
+			 *  - hba_port object's sas address matches with current
+			 *    removed device's sas address and no vphy's
+			 *    associated with it.
+			 *  - Current removed device is a vSES device and
+			 *    none of the other direct attached device have
+			 *    this vSES device's port number (hence hba_port
+			 *    object sas_address field will be zero).
+			 */
+			if ((hba_port->sas_address == sas_address ||
+			    !hba_port->sas_address) && !hba_port->vphys_mask) {
+				ioc_info(ioc,
+				    "remove hba_port entry: %p port: %d from hba_port list\n",
+				    hba_port, hba_port->port_id);
+				list_del(&hba_port->list);
+				kfree(hba_port);
+			} else if (hba_port->sas_address == sas_address &&
+			    hba_port->vphys_mask) {
+				/*
+				 * Current removed device is a non vSES device
+				 * and a vSES device has the same port number
+				 * as of current device's port number. Hence
+				 * only clear the sas_address filed, don't
+				 * delete the hba_port object.
+				 */
+				ioc_info(ioc,
+				    "clearing sas_address from hba_port entry: %p port: %d from hba_port list\n",
+				    hba_port, hba_port->port_id);
+				port->sas_address = 0;
+			}
+			break;
 		}
 	}