From e8b908146d44310473e43b3382eca126e12d279c Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Tue, 4 Apr 2023 08:27:13 +0300 Subject: [PATCH 1/3] PCI/PM: Increase wait time after resume PCIe r6.0 sec 6.6.1 prescribes that a device must be able to respond to config requests within 1.0 s (PCI_RESET_WAIT) after exiting conventional reset and this same delay is prescribed when coming out of D3cold (as that involves reset too). A device that requires more than 1 second to initialize after reset may respond to config requests with Request Retry Status completions (sec 2.3.1), and we accommodate that in Linux with a 60 second cap (PCIE_RESET_READY_POLL_MS). Previously we waited up to PCIE_RESET_READY_POLL_MS only in the reset code path, not in the resume path. However, a device has surfaced, namely Intel Titan Ridge xHCI, which requires a longer delay also in the resume code path. Make the resume code path to use this same extended delay as the reset path. Link: https://bugzilla.kernel.org/show_bug.cgi?id=216728 Link: https://lore.kernel.org/r/20230404052714.51315-2-mika.westerberg@linux.intel.com Reported-by: Chris Chiu Signed-off-by: Mika Westerberg Signed-off-by: Bjorn Helgaas Cc: Lukas Wunner --- drivers/pci/pci-driver.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 57ddcc59af30..6b5b2a818e65 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -572,7 +572,8 @@ static void pci_pm_default_resume_early(struct pci_dev *pci_dev) static void pci_pm_bridge_power_up_actions(struct pci_dev *pci_dev) { - pci_bridge_wait_for_secondary_bus(pci_dev, "resume", PCI_RESET_WAIT); + pci_bridge_wait_for_secondary_bus(pci_dev, "resume", + PCIE_RESET_READY_POLL_MS); /* * When powering on a bridge from D3cold, the whole hierarchy may be * powered on into D0uninitialized state, resume them to give them a From e74b2b58ff715ca17bb49c3ad89f665a7150e14b Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Tue, 4 Apr 2023 15:32:55 -0500 Subject: [PATCH 2/3] PCI/PM: Drop pci_bridge_wait_for_secondary_bus() timeout parameter All callers of pci_bridge_wait_for_secondary_bus() supply a timeout of PCIE_RESET_READY_POLL_MS, so drop the parameter. Move the definition of PCIE_RESET_READY_POLL_MS into pci.c, the only user. [bhelgaas: extracted from https://lore.kernel.org/r/20230404052714.51315-3-mika.westerberg@linux.intel.com] Signed-off-by: Mika Westerberg Signed-off-by: Bjorn Helgaas --- drivers/pci/pci-driver.c | 4 ++-- drivers/pci/pci.c | 18 ++++++++++++------ drivers/pci/pci.h | 9 +-------- drivers/pci/pcie/dpc.c | 3 +-- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 6b5b2a818e65..ae9baf801681 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -572,8 +572,8 @@ static void pci_pm_default_resume_early(struct pci_dev *pci_dev) static void pci_pm_bridge_power_up_actions(struct pci_dev *pci_dev) { - pci_bridge_wait_for_secondary_bus(pci_dev, "resume", - PCIE_RESET_READY_POLL_MS); + pci_bridge_wait_for_secondary_bus(pci_dev, "resume"); + /* * When powering on a bridge from D3cold, the whole hierarchy may be * powered on into D0uninitialized state, resume them to give them a diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 7a67611dc5f4..0b4f3b08f780 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -64,6 +64,14 @@ struct pci_pme_device { #define PME_TIMEOUT 1000 /* How long between PME checks */ +/* + * Devices may extend the 1 sec period through Request Retry Status + * completions (PCIe r6.0 sec 2.3.1). The spec does not provide an upper + * limit, but 60 sec ought to be enough for any device to become + * responsive. + */ +#define PCIE_RESET_READY_POLL_MS 60000 /* msec */ + static void pci_dev_d3_sleep(struct pci_dev *dev) { unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay); @@ -4939,7 +4947,6 @@ static int pci_bus_max_d3cold_delay(const struct pci_bus *bus) * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible * @dev: PCI bridge * @reset_type: reset type in human-readable form - * @timeout: maximum time to wait for devices on secondary bus (milliseconds) * * Handle necessary delays before access to the devices on the secondary * side of the bridge are permitted after D3cold to D0 transition @@ -4952,8 +4959,7 @@ static int pci_bus_max_d3cold_delay(const struct pci_bus *bus) * Return 0 on success or -ENOTTY if the first device on the secondary bus * failed to become accessible. */ -int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type, - int timeout) +int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type) { struct pci_dev *child; int delay; @@ -5031,7 +5037,8 @@ int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type, } } - return pci_dev_wait(child, reset_type, timeout - delay); + return pci_dev_wait(child, reset_type, + PCIE_RESET_READY_POLL_MS - delay); } void pci_reset_secondary_bus(struct pci_dev *dev) @@ -5068,8 +5075,7 @@ int pci_bridge_secondary_bus_reset(struct pci_dev *dev) { pcibios_reset_secondary_bus(dev); - return pci_bridge_wait_for_secondary_bus(dev, "bus reset", - PCIE_RESET_READY_POLL_MS); + return pci_bridge_wait_for_secondary_bus(dev, "bus reset"); } EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset); diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index d2c08670a20e..022da58afb33 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -70,12 +70,6 @@ struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, * Reset (PCIe r6.0 sec 5.8). */ #define PCI_RESET_WAIT 1000 /* msec */ -/* - * Devices may extend the 1 sec period through Request Retry Status completions - * (PCIe r6.0 sec 2.3.1). The spec does not provide an upper limit, but 60 sec - * ought to be enough for any device to become responsive. - */ -#define PCIE_RESET_READY_POLL_MS 60000 /* msec */ void pci_update_current_state(struct pci_dev *dev, pci_power_t state); void pci_refresh_power_state(struct pci_dev *dev); @@ -100,8 +94,7 @@ void pci_msix_init(struct pci_dev *dev); bool pci_bridge_d3_possible(struct pci_dev *dev); void pci_bridge_d3_update(struct pci_dev *dev); void pci_bridge_reconfigure_ltr(struct pci_dev *dev); -int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type, - int timeout); +int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type); static inline void pci_wakeup_event(struct pci_dev *dev) { diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c index a5d7c69b764e..3ceed8e3de41 100644 --- a/drivers/pci/pcie/dpc.c +++ b/drivers/pci/pcie/dpc.c @@ -170,8 +170,7 @@ pci_ers_result_t dpc_reset_link(struct pci_dev *pdev) pci_write_config_word(pdev, cap + PCI_EXP_DPC_STATUS, PCI_EXP_DPC_STATUS_TRIGGER); - if (pci_bridge_wait_for_secondary_bus(pdev, "DPC", - PCIE_RESET_READY_POLL_MS)) { + if (pci_bridge_wait_for_secondary_bus(pdev, "DPC")) { clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags); ret = PCI_ERS_RESULT_DISCONNECT; } else { From a5a6dd2624698b6e3045c3a1450874d8c790d5d9 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Thu, 13 Apr 2023 13:40:42 -0600 Subject: [PATCH 3/3] PCI/PM: Extend D3hot delay for NVIDIA HDA controllers Assignment of NVIDIA Ampere-based GPUs have seen a regression since the below referenced commit, where the reduced D3hot transition delay appears to introduce a small window where a D3hot->D0 transition followed by a bus reset can wedge the device. The entire device is subsequently unavailable, returning -1 on config space read and is unrecoverable without a host reset. This has been observed with RTX A2000 and A5000 GPU and audio functions assigned to a Windows VM, where shutdown of the VM places the devices in D3hot prior to vfio-pci performing a bus reset when userspace releases the devices. The issue has roughly a 2-3% chance of occurring per shutdown. Restoring the HDA controller d3hot_delay to the effective value before the below commit has been shown to resolve the issue. NVIDIA confirms this change should be safe for all of their HDA controllers. Fixes: 3e347969a577 ("PCI/PM: Reduce D3hot delay with usleep_range()") Link: https://lore.kernel.org/r/20230413194042.605768-1-alex.williamson@redhat.com Reported-by: Zhiyi Guo Signed-off-by: Alex Williamson Signed-off-by: Bjorn Helgaas Reviewed-by: Tarun Gupta Cc: Abhishek Sahu Cc: Tarun Gupta --- drivers/pci/quirks.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 44cab813bf95..f4e2a88729fd 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -1939,6 +1939,19 @@ static void quirk_radeon_pm(struct pci_dev *dev) } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x6741, quirk_radeon_pm); +/* + * NVIDIA Ampere-based HDA controllers can wedge the whole device if a bus + * reset is performed too soon after transition to D0, extend d3hot_delay + * to previous effective default for all NVIDIA HDA controllers. + */ +static void quirk_nvidia_hda_pm(struct pci_dev *dev) +{ + quirk_d3hot_delay(dev, 20); +} +DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, + PCI_CLASS_MULTIMEDIA_HD_AUDIO, 8, + quirk_nvidia_hda_pm); + /* * Ryzen5/7 XHCI controllers fail upon resume from runtime suspend or s2idle. * https://bugzilla.kernel.org/show_bug.cgi?id=205587