Commit f5c57710 authored by Gavin Shan's avatar Gavin Shan Committed by Benjamin Herrenschmidt

powerpc/eeh: Use partial hotplug for EEH unaware drivers

When EEH error happens to one specific PE, some devices with drivers
supporting EEH won't except hotplug on the device. However, there
might have other deivces without driver, or with driver without EEH
support. For the case, we need do partial hotplug in order to make
sure that the PE becomes absolutely quite during reset. Otherise,
the PE reset might fail and leads to failure of error recovery.

The current code doesn't handle that 'mixed' case properly, it either
uses the error callbacks to the drivers, or tries hotplug, but doesn't
handle a PE (EEH domain) composed of a combination of the two.

The patch intends to support so-called "partial" hotplug for EEH:
Before we do reset, we stop and remove those PCI devices without
EEH sensitive driver. The corresponding EEH devices are not detached
from its PE, but with special flag. After the reset is done, those
EEH devices with the special flag will be scanned one by one.
Signed-off-by: default avatarGavin Shan <shangw@linux.vnet.ibm.com>
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
parent ab444ec9
...@@ -84,7 +84,8 @@ struct eeh_pe { ...@@ -84,7 +84,8 @@ struct eeh_pe {
* another tree except the currently existing tree of PCI * another tree except the currently existing tree of PCI
* buses and PCI devices * buses and PCI devices
*/ */
#define EEH_DEV_IRQ_DISABLED (1<<0) /* Interrupt disabled */ #define EEH_DEV_IRQ_DISABLED (1 << 0) /* Interrupt disabled */
#define EEH_DEV_DISCONNECTED (1 << 1) /* Removing from PE */
struct eeh_dev { struct eeh_dev {
int mode; /* EEH mode */ int mode; /* EEH mode */
...@@ -97,6 +98,7 @@ struct eeh_dev { ...@@ -97,6 +98,7 @@ struct eeh_dev {
struct pci_controller *phb; /* Associated PHB */ struct pci_controller *phb; /* Associated PHB */
struct device_node *dn; /* Associated device node */ struct device_node *dn; /* Associated device node */
struct pci_dev *pdev; /* Associated PCI device */ struct pci_dev *pdev; /* Associated PCI device */
struct pci_bus *bus; /* PCI bus for partial hotplug */
}; };
static inline struct device_node *eeh_dev_to_of_node(struct eeh_dev *edev) static inline struct device_node *eeh_dev_to_of_node(struct eeh_dev *edev)
...@@ -197,6 +199,8 @@ struct eeh_pe *eeh_pe_get(struct eeh_dev *edev); ...@@ -197,6 +199,8 @@ struct eeh_pe *eeh_pe_get(struct eeh_dev *edev);
int eeh_add_to_parent_pe(struct eeh_dev *edev); int eeh_add_to_parent_pe(struct eeh_dev *edev);
int eeh_rmv_from_parent_pe(struct eeh_dev *edev); int eeh_rmv_from_parent_pe(struct eeh_dev *edev);
void eeh_pe_update_time_stamp(struct eeh_pe *pe); void eeh_pe_update_time_stamp(struct eeh_pe *pe);
void *eeh_pe_traverse(struct eeh_pe *root,
eeh_traverse_func fn, void *flag);
void *eeh_pe_dev_traverse(struct eeh_pe *root, void *eeh_pe_dev_traverse(struct eeh_pe *root,
eeh_traverse_func fn, void *flag); eeh_traverse_func fn, void *flag);
void eeh_pe_restore_bars(struct eeh_pe *pe); void eeh_pe_restore_bars(struct eeh_pe *pe);
......
...@@ -900,7 +900,21 @@ void eeh_add_device_late(struct pci_dev *dev) ...@@ -900,7 +900,21 @@ void eeh_add_device_late(struct pci_dev *dev)
pr_debug("EEH: Already referenced !\n"); pr_debug("EEH: Already referenced !\n");
return; return;
} }
WARN_ON(edev->pdev);
/*
* The EEH cache might not be removed correctly because of
* unbalanced kref to the device during unplug time, which
* relies on pcibios_release_device(). So we have to remove
* that here explicitly.
*/
if (edev->pdev) {
eeh_rmv_from_parent_pe(edev);
eeh_addr_cache_rmv_dev(edev->pdev);
eeh_sysfs_remove_device(edev->pdev);
edev->pdev = NULL;
dev->dev.archdata.edev = NULL;
}
edev->pdev = dev; edev->pdev = dev;
dev->dev.archdata.edev = edev; dev->dev.archdata.edev = edev;
...@@ -982,14 +996,24 @@ void eeh_remove_device(struct pci_dev *dev) ...@@ -982,14 +996,24 @@ void eeh_remove_device(struct pci_dev *dev)
/* Unregister the device with the EEH/PCI address search system */ /* Unregister the device with the EEH/PCI address search system */
pr_debug("EEH: Removing device %s\n", pci_name(dev)); pr_debug("EEH: Removing device %s\n", pci_name(dev));
if (!edev || !edev->pdev) { if (!edev || !edev->pdev || !edev->pe) {
pr_debug("EEH: Not referenced !\n"); pr_debug("EEH: Not referenced !\n");
return; return;
} }
/*
* During the hotplug for EEH error recovery, we need the EEH
* device attached to the parent PE in order for BAR restore
* a bit later. So we keep it for BAR restore and remove it
* from the parent PE during the BAR resotre.
*/
edev->pdev = NULL; edev->pdev = NULL;
dev->dev.archdata.edev = NULL; dev->dev.archdata.edev = NULL;
if (!(edev->pe->state & EEH_PE_KEEP))
eeh_rmv_from_parent_pe(edev); eeh_rmv_from_parent_pe(edev);
else
edev->mode |= EEH_DEV_DISCONNECTED;
eeh_addr_cache_rmv_dev(dev); eeh_addr_cache_rmv_dev(dev);
eeh_sysfs_remove_device(dev); eeh_sysfs_remove_device(dev);
} }
......
...@@ -338,6 +338,54 @@ static void *eeh_report_failure(void *data, void *userdata) ...@@ -338,6 +338,54 @@ static void *eeh_report_failure(void *data, void *userdata)
return NULL; return NULL;
} }
static void *eeh_rmv_device(void *data, void *userdata)
{
struct pci_driver *driver;
struct eeh_dev *edev = (struct eeh_dev *)data;
struct pci_dev *dev = eeh_dev_to_pci_dev(edev);
int *removed = (int *)userdata;
/*
* Actually, we should remove the PCI bridges as well.
* However, that's lots of complexity to do that,
* particularly some of devices under the bridge might
* support EEH. So we just care about PCI devices for
* simplicity here.
*/
if (!dev || (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE))
return NULL;
driver = eeh_pcid_get(dev);
if (driver && driver->err_handler)
return NULL;
/* Remove it from PCI subsystem */
pr_debug("EEH: Removing %s without EEH sensitive driver\n",
pci_name(dev));
edev->bus = dev->bus;
edev->mode |= EEH_DEV_DISCONNECTED;
(*removed)++;
pci_stop_and_remove_bus_device(dev);
return NULL;
}
static void *eeh_pe_detach_dev(void *data, void *userdata)
{
struct eeh_pe *pe = (struct eeh_pe *)data;
struct eeh_dev *edev, *tmp;
eeh_pe_for_each_dev(pe, edev, tmp) {
if (!(edev->mode & EEH_DEV_DISCONNECTED))
continue;
edev->mode &= ~(EEH_DEV_DISCONNECTED | EEH_DEV_IRQ_DISABLED);
eeh_rmv_from_parent_pe(edev);
}
return NULL;
}
/** /**
* eeh_reset_device - Perform actual reset of a pci slot * eeh_reset_device - Perform actual reset of a pci slot
* @pe: EEH PE * @pe: EEH PE
...@@ -349,8 +397,9 @@ static void *eeh_report_failure(void *data, void *userdata) ...@@ -349,8 +397,9 @@ static void *eeh_report_failure(void *data, void *userdata)
*/ */
static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus) static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus)
{ {
struct pci_bus *frozen_bus = eeh_pe_bus_get(pe);
struct timeval tstamp; struct timeval tstamp;
int cnt, rc; int cnt, rc, removed = 0;
/* pcibios will clear the counter; save the value */ /* pcibios will clear the counter; save the value */
cnt = pe->freeze_count; cnt = pe->freeze_count;
...@@ -362,10 +411,11 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus) ...@@ -362,10 +411,11 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus)
* devices are expected to be attached soon when calling * devices are expected to be attached soon when calling
* into pcibios_add_pci_devices(). * into pcibios_add_pci_devices().
*/ */
if (bus) {
eeh_pe_state_mark(pe, EEH_PE_KEEP); eeh_pe_state_mark(pe, EEH_PE_KEEP);
if (bus)
pcibios_remove_pci_devices(bus); pcibios_remove_pci_devices(bus);
} else if (frozen_bus)
eeh_pe_dev_traverse(pe, eeh_rmv_device, &removed);
/* Reset the pci controller. (Asserts RST#; resets config space). /* Reset the pci controller. (Asserts RST#; resets config space).
* Reconfigure bridges and devices. Don't try to bring the system * Reconfigure bridges and devices. Don't try to bring the system
...@@ -386,10 +436,24 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus) ...@@ -386,10 +436,24 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus)
* potentially weird things happen. * potentially weird things happen.
*/ */
if (bus) { if (bus) {
pr_info("EEH: Sleep 5s ahead of complete hotplug\n");
ssleep(5); ssleep(5);
/*
* The EEH device is still connected with its parent
* PE. We should disconnect it so the binding can be
* rebuilt when adding PCI devices.
*/
eeh_pe_traverse(pe, eeh_pe_detach_dev, NULL);
pcibios_add_pci_devices(bus); pcibios_add_pci_devices(bus);
eeh_pe_state_clear(pe, EEH_PE_KEEP); } else if (frozen_bus && removed) {
pr_info("EEH: Sleep 5s ahead of partial hotplug\n");
ssleep(5);
eeh_pe_traverse(pe, eeh_pe_detach_dev, NULL);
pcibios_add_pci_devices(frozen_bus);
} }
eeh_pe_state_clear(pe, EEH_PE_KEEP);
pe->tstamp = tstamp; pe->tstamp = tstamp;
pe->freeze_count = cnt; pe->freeze_count = cnt;
......
...@@ -149,7 +149,7 @@ static struct eeh_pe *eeh_pe_next(struct eeh_pe *pe, ...@@ -149,7 +149,7 @@ static struct eeh_pe *eeh_pe_next(struct eeh_pe *pe,
* callback returns something other than NULL, or no more PEs * callback returns something other than NULL, or no more PEs
* to be traversed. * to be traversed.
*/ */
static void *eeh_pe_traverse(struct eeh_pe *root, void *eeh_pe_traverse(struct eeh_pe *root,
eeh_traverse_func fn, void *flag) eeh_traverse_func fn, void *flag)
{ {
struct eeh_pe *pe; struct eeh_pe *pe;
...@@ -409,7 +409,7 @@ int eeh_rmv_from_parent_pe(struct eeh_dev *edev) ...@@ -409,7 +409,7 @@ int eeh_rmv_from_parent_pe(struct eeh_dev *edev)
int cnt; int cnt;
if (!edev->pe) { if (!edev->pe) {
pr_warning("%s: No PE found for EEH device %s\n", pr_debug("%s: No PE found for EEH device %s\n",
__func__, edev->dn->full_name); __func__, edev->dn->full_name);
return -EEXIST; return -EEXIST;
} }
...@@ -728,18 +728,12 @@ static void eeh_restore_device_bars(struct eeh_dev *edev, ...@@ -728,18 +728,12 @@ static void eeh_restore_device_bars(struct eeh_dev *edev,
*/ */
static void *eeh_restore_one_device_bars(void *data, void *flag) static void *eeh_restore_one_device_bars(void *data, void *flag)
{ {
struct pci_dev *pdev = NULL;
struct eeh_dev *edev = (struct eeh_dev *)data; struct eeh_dev *edev = (struct eeh_dev *)data;
struct pci_dev *pdev = eeh_dev_to_pci_dev(edev);
struct device_node *dn = eeh_dev_to_of_node(edev); struct device_node *dn = eeh_dev_to_of_node(edev);
/* Trace the PCI bridge */ /* Do special restore for bridges */
if (eeh_probe_mode_dev()) { if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
pdev = eeh_dev_to_pci_dev(edev);
if (pdev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
pdev = NULL;
}
if (pdev)
eeh_restore_bridge_bars(pdev, edev, dn); eeh_restore_bridge_bars(pdev, edev, dn);
else else
eeh_restore_device_bars(edev, dn); eeh_restore_device_bars(edev, dn);
......
...@@ -68,6 +68,13 @@ void eeh_sysfs_add_device(struct pci_dev *pdev) ...@@ -68,6 +68,13 @@ void eeh_sysfs_add_device(struct pci_dev *pdev)
void eeh_sysfs_remove_device(struct pci_dev *pdev) void eeh_sysfs_remove_device(struct pci_dev *pdev)
{ {
/*
* The parent directory might have been removed. We needn't
* continue for that case.
*/
if (!pdev->dev.kobj.sd)
return;
device_remove_file(&pdev->dev, &dev_attr_eeh_mode); device_remove_file(&pdev->dev, &dev_attr_eeh_mode);
device_remove_file(&pdev->dev, &dev_attr_eeh_config_addr); device_remove_file(&pdev->dev, &dev_attr_eeh_config_addr);
device_remove_file(&pdev->dev, &dev_attr_eeh_pe_config_addr); device_remove_file(&pdev->dev, &dev_attr_eeh_pe_config_addr);
......
...@@ -114,7 +114,7 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag) ...@@ -114,7 +114,7 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag)
* the root bridge. So it's not reasonable to continue * the root bridge. So it's not reasonable to continue
* the probing. * the probing.
*/ */
if (!dn || !edev) if (!dn || !edev || edev->pe)
return 0; return 0;
/* Skip for PCI-ISA bridge */ /* Skip for PCI-ISA bridge */
......
...@@ -153,7 +153,7 @@ static void *pseries_eeh_of_probe(struct device_node *dn, void *flag) ...@@ -153,7 +153,7 @@ static void *pseries_eeh_of_probe(struct device_node *dn, void *flag)
/* Retrieve OF node and eeh device */ /* Retrieve OF node and eeh device */
edev = of_node_to_eeh_dev(dn); edev = of_node_to_eeh_dev(dn);
if (!of_device_is_available(dn)) if (edev->pe || !of_device_is_available(dn))
return NULL; return NULL;
/* Retrieve class/vendor/device IDs */ /* Retrieve class/vendor/device IDs */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment