2018-05-06 21:58:06 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0+
|
2015-11-29 20:17:49 +00:00
|
|
|
/*
|
|
|
|
* PCI autoconfiguration library
|
|
|
|
*
|
|
|
|
* Author: Matt Porter <mporter@mvista.com>
|
|
|
|
*
|
|
|
|
* Copyright 2000 MontaVista Software Inc.
|
pci: Work around PCIe link training failures
Attempt to handle cases with a downstream port of a PCIe switch where
link training never completes and the link continues switching between
speeds indefinitely with the data link layer never reaching the active
state.
It has been observed with a downstream port of the ASMedia ASM2824 Gen 3
switch wired to the upstream port of the Pericom PI7C9X2G304 Gen 2
switch, using a Delock Riser Card PCI Express x1 > 2 x PCIe x1 device,
P/N 41433, wired to a SiFive HiFive Unmatched board. In this setup the
switches are supposed to negotiate the link speed of preferably 5.0GT/s,
falling back to 2.5GT/s.
However the link continues oscillating between the two speeds, at the
rate of 34-35 times per second, with link training reported repeatedly
active ~84% of the time, e.g.:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=05, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 5GT/s (downgraded), Width x1 (ok)
TrErr- Train+ SlotClk+ DLActive- BWMgmt+ ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 8GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Forcibly limiting the target link speed to 2.5GT/s with the upstream
ASM2824 device makes the two switches communicate correctly however:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=09, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (ok)
TrErr- Train- SlotClk+ DLActive+ BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 2.5GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
and then:
05:00.0 PCI bridge [0604]: Pericom Semiconductor PI7C9X2G304 EL/SL PCIe2 3-Port/4-Lane Packet Switch [12d8:2304] (rev 05) (prog-if 00 [Normal decode])
[...]
Bus: primary=05, secondary=06, subordinate=09, sec-latency=0
[...]
Capabilities: [c0] Express (v2) Upstream Port, MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (downgraded)
TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Make use of this observation then and attempt to detect the inability to
negotiate the link speed automatically, and then handle it by hand. Use
the Data Link Layer Link Active status flag as the primary indicator of
successful link speed negotiation, but given that the flag is optional
by hardware to implement (the ASM2824 does have it though), resort to
checking for the mandatory Link Bandwidth Management Status flag showing
that the link speed or width has been changed in an attempt to correct
unreliable link operation (the ASM2824 does set it too).
If these checks indicate that link may not operate correctly, then poll
the Data Link Layer Link Active status flag along with the Link Training
flag for the duration of 200ms to see if the link has stabilised, that
is either that the Data Link Layer Link Active status flag has been set
or that Link Training has been inactive during at least the second half
of the interval.
If that has indicated failure, restrict the target speed to 2.5GT/s,
request a link retrain and check again if the link has stabilised. If
that does not work either, then restore the original speed setting and
claim defeat, otherwise we are done.
NB interestingly enough with the ASM2824 vs PI7C9X2G304 configuration
referred above asking the ASM2824 to retrain with a higher target link
speed once the 2.5GT/s speed has been negotiated makes the two devices
successfully negotiate 5.0GT/s. Lifting the 2.5GT/s speed restriction
would however prevent our workaround from working with an OS that issues
a reset and that is unaware of the problem. This is because the devices
would then try to negotiate a higher link speed from scratch and fail,
while the sticky property of the Target Link Speed setting will keep the
2.5GT/s speed restriction across a reset.
Keep the 2.5GT/s speed restriction then, conservatively, if functional
once applied.
Signed-off-by: Maciej W. Rozycki <macro@orcam.me.uk>
Reviewed-by: Stefan Roese <sr@denx.de>
2021-11-20 23:03:30 +00:00
|
|
|
* Copyright (c) 2021 Maciej W. Rozycki <macro@orcam.me.uk>
|
2015-11-29 20:17:49 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <common.h>
|
2016-01-19 03:19:16 +00:00
|
|
|
#include <dm.h>
|
2015-11-29 20:17:49 +00:00
|
|
|
#include <errno.h>
|
2020-05-10 17:40:05 +00:00
|
|
|
#include <log.h>
|
2015-11-29 20:17:49 +00:00
|
|
|
#include <pci.h>
|
pci: Work around PCIe link training failures
Attempt to handle cases with a downstream port of a PCIe switch where
link training never completes and the link continues switching between
speeds indefinitely with the data link layer never reaching the active
state.
It has been observed with a downstream port of the ASMedia ASM2824 Gen 3
switch wired to the upstream port of the Pericom PI7C9X2G304 Gen 2
switch, using a Delock Riser Card PCI Express x1 > 2 x PCIe x1 device,
P/N 41433, wired to a SiFive HiFive Unmatched board. In this setup the
switches are supposed to negotiate the link speed of preferably 5.0GT/s,
falling back to 2.5GT/s.
However the link continues oscillating between the two speeds, at the
rate of 34-35 times per second, with link training reported repeatedly
active ~84% of the time, e.g.:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=05, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 5GT/s (downgraded), Width x1 (ok)
TrErr- Train+ SlotClk+ DLActive- BWMgmt+ ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 8GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Forcibly limiting the target link speed to 2.5GT/s with the upstream
ASM2824 device makes the two switches communicate correctly however:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=09, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (ok)
TrErr- Train- SlotClk+ DLActive+ BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 2.5GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
and then:
05:00.0 PCI bridge [0604]: Pericom Semiconductor PI7C9X2G304 EL/SL PCIe2 3-Port/4-Lane Packet Switch [12d8:2304] (rev 05) (prog-if 00 [Normal decode])
[...]
Bus: primary=05, secondary=06, subordinate=09, sec-latency=0
[...]
Capabilities: [c0] Express (v2) Upstream Port, MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (downgraded)
TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Make use of this observation then and attempt to detect the inability to
negotiate the link speed automatically, and then handle it by hand. Use
the Data Link Layer Link Active status flag as the primary indicator of
successful link speed negotiation, but given that the flag is optional
by hardware to implement (the ASM2824 does have it though), resort to
checking for the mandatory Link Bandwidth Management Status flag showing
that the link speed or width has been changed in an attempt to correct
unreliable link operation (the ASM2824 does set it too).
If these checks indicate that link may not operate correctly, then poll
the Data Link Layer Link Active status flag along with the Link Training
flag for the duration of 200ms to see if the link has stabilised, that
is either that the Data Link Layer Link Active status flag has been set
or that Link Training has been inactive during at least the second half
of the interval.
If that has indicated failure, restrict the target speed to 2.5GT/s,
request a link retrain and check again if the link has stabilised. If
that does not work either, then restore the original speed setting and
claim defeat, otherwise we are done.
NB interestingly enough with the ASM2824 vs PI7C9X2G304 configuration
referred above asking the ASM2824 to retrain with a higher target link
speed once the 2.5GT/s speed has been negotiated makes the two devices
successfully negotiate 5.0GT/s. Lifting the 2.5GT/s speed restriction
would however prevent our workaround from working with an OS that issues
a reset and that is unaware of the problem. This is because the devices
would then try to negotiate a higher link speed from scratch and fail,
while the sticky property of the Target Link Speed setting will keep the
2.5GT/s speed restriction across a reset.
Keep the 2.5GT/s speed restriction then, conservatively, if functional
once applied.
Signed-off-by: Maciej W. Rozycki <macro@orcam.me.uk>
Reviewed-by: Stefan Roese <sr@denx.de>
2021-11-20 23:03:30 +00:00
|
|
|
#include <time.h>
|
2021-09-17 12:11:21 +00:00
|
|
|
#include "pci_internal.h"
|
2015-11-29 20:17:49 +00:00
|
|
|
|
2022-11-16 18:10:33 +00:00
|
|
|
/* the user can define CFG_SYS_PCI_CACHE_LINE_SIZE to avoid problems */
|
|
|
|
#ifndef CFG_SYS_PCI_CACHE_LINE_SIZE
|
|
|
|
#define CFG_SYS_PCI_CACHE_LINE_SIZE 8
|
2015-11-29 20:17:49 +00:00
|
|
|
#endif
|
|
|
|
|
2021-10-07 12:50:59 +00:00
|
|
|
static void dm_pciauto_setup_device(struct udevice *dev,
|
2021-01-12 11:03:43 +00:00
|
|
|
struct pci_region *mem,
|
|
|
|
struct pci_region *prefetch,
|
|
|
|
struct pci_region *io)
|
2015-11-29 20:17:49 +00:00
|
|
|
{
|
|
|
|
u32 bar_response;
|
|
|
|
pci_size_t bar_size;
|
|
|
|
u16 cmdstat = 0;
|
|
|
|
int bar, bar_nr = 0;
|
2021-10-07 12:50:59 +00:00
|
|
|
int bars_num;
|
2015-11-29 20:17:49 +00:00
|
|
|
u8 header_type;
|
|
|
|
int rom_addr;
|
|
|
|
pci_addr_t bar_value;
|
2016-02-18 07:14:47 +00:00
|
|
|
struct pci_region *bar_res = NULL;
|
2015-11-29 20:17:49 +00:00
|
|
|
int found_mem64 = 0;
|
|
|
|
u16 class;
|
|
|
|
|
|
|
|
dm_pci_read_config16(dev, PCI_COMMAND, &cmdstat);
|
|
|
|
cmdstat = (cmdstat & ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY)) |
|
|
|
|
PCI_COMMAND_MASTER;
|
|
|
|
|
2021-10-07 12:50:59 +00:00
|
|
|
dm_pci_read_config8(dev, PCI_HEADER_TYPE, &header_type);
|
|
|
|
header_type &= 0x7f;
|
|
|
|
|
|
|
|
switch (header_type) {
|
|
|
|
case PCI_HEADER_TYPE_NORMAL:
|
|
|
|
bars_num = 6;
|
|
|
|
break;
|
|
|
|
case PCI_HEADER_TYPE_BRIDGE:
|
|
|
|
bars_num = 2;
|
|
|
|
break;
|
|
|
|
case PCI_HEADER_TYPE_CARDBUS:
|
|
|
|
/* CardBus header does not have any BAR */
|
|
|
|
bars_num = 0;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
/* Skip configuring BARs for unknown header types */
|
|
|
|
bars_num = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2015-11-29 20:17:49 +00:00
|
|
|
for (bar = PCI_BASE_ADDRESS_0;
|
|
|
|
bar < PCI_BASE_ADDRESS_0 + (bars_num * 4); bar += 4) {
|
2019-09-25 14:56:16 +00:00
|
|
|
int ret = 0;
|
|
|
|
|
2015-11-29 20:17:49 +00:00
|
|
|
/* Tickle the BAR and get the response */
|
2021-01-12 11:03:43 +00:00
|
|
|
dm_pci_write_config32(dev, bar, 0xffffffff);
|
2015-11-29 20:17:49 +00:00
|
|
|
dm_pci_read_config32(dev, bar, &bar_response);
|
|
|
|
|
2021-01-03 22:06:45 +00:00
|
|
|
/* If BAR is not implemented (or invalid) go to the next BAR */
|
|
|
|
if (!bar_response || bar_response == 0xffffffff)
|
2015-11-29 20:17:49 +00:00
|
|
|
continue;
|
|
|
|
|
|
|
|
found_mem64 = 0;
|
|
|
|
|
|
|
|
/* Check the BAR type and set our address mask */
|
|
|
|
if (bar_response & PCI_BASE_ADDRESS_SPACE) {
|
2021-01-03 22:06:45 +00:00
|
|
|
bar_size = bar_response & PCI_BASE_ADDRESS_IO_MASK;
|
|
|
|
bar_size &= ~(bar_size - 1);
|
|
|
|
|
2021-01-12 11:03:43 +00:00
|
|
|
bar_res = io;
|
2015-11-29 20:17:49 +00:00
|
|
|
|
|
|
|
debug("PCI Autoconfig: BAR %d, I/O, size=0x%llx, ",
|
|
|
|
bar_nr, (unsigned long long)bar_size);
|
|
|
|
} else {
|
|
|
|
if ((bar_response & PCI_BASE_ADDRESS_MEM_TYPE_MASK) ==
|
|
|
|
PCI_BASE_ADDRESS_MEM_TYPE_64) {
|
|
|
|
u32 bar_response_upper;
|
|
|
|
u64 bar64;
|
|
|
|
|
2021-01-12 11:03:43 +00:00
|
|
|
dm_pci_write_config32(dev, bar + 4, 0xffffffff);
|
2015-11-29 20:17:49 +00:00
|
|
|
dm_pci_read_config32(dev, bar + 4,
|
|
|
|
&bar_response_upper);
|
|
|
|
|
|
|
|
bar64 = ((u64)bar_response_upper << 32) |
|
|
|
|
bar_response;
|
|
|
|
|
|
|
|
bar_size = ~(bar64 & PCI_BASE_ADDRESS_MEM_MASK)
|
|
|
|
+ 1;
|
2021-01-12 11:03:43 +00:00
|
|
|
found_mem64 = 1;
|
2015-11-29 20:17:49 +00:00
|
|
|
} else {
|
|
|
|
bar_size = (u32)(~(bar_response &
|
|
|
|
PCI_BASE_ADDRESS_MEM_MASK) + 1);
|
|
|
|
}
|
2021-01-12 11:03:43 +00:00
|
|
|
|
|
|
|
if (prefetch &&
|
|
|
|
(bar_response & PCI_BASE_ADDRESS_MEM_PREFETCH))
|
|
|
|
bar_res = prefetch;
|
|
|
|
else
|
|
|
|
bar_res = mem;
|
2015-11-29 20:17:49 +00:00
|
|
|
|
2021-03-03 00:57:35 +00:00
|
|
|
debug("PCI Autoconfig: BAR %d, %s%s, size=0x%llx, ",
|
2015-11-29 20:17:49 +00:00
|
|
|
bar_nr, bar_res == prefetch ? "Prf" : "Mem",
|
2021-03-03 00:57:35 +00:00
|
|
|
found_mem64 ? "64" : "",
|
2015-11-29 20:17:49 +00:00
|
|
|
(unsigned long long)bar_size);
|
|
|
|
}
|
|
|
|
|
2021-01-12 11:03:43 +00:00
|
|
|
ret = pciauto_region_allocate(bar_res, bar_size,
|
|
|
|
&bar_value, found_mem64);
|
|
|
|
if (ret)
|
|
|
|
printf("PCI: Failed autoconfig bar %x\n", bar);
|
|
|
|
|
|
|
|
if (!ret) {
|
2015-11-29 20:17:49 +00:00
|
|
|
/* Write it out and update our limit */
|
|
|
|
dm_pci_write_config32(dev, bar, (u32)bar_value);
|
|
|
|
|
|
|
|
if (found_mem64) {
|
|
|
|
bar += 4;
|
|
|
|
#ifdef CONFIG_SYS_PCI_64BIT
|
|
|
|
dm_pci_write_config32(dev, bar,
|
|
|
|
(u32)(bar_value >> 32));
|
|
|
|
#else
|
|
|
|
/*
|
|
|
|
* If we are a 64-bit decoder then increment to
|
|
|
|
* the upper 32 bits of the bar and force it to
|
|
|
|
* locate in the lower 4GB of memory.
|
|
|
|
*/
|
|
|
|
dm_pci_write_config32(dev, bar, 0x00000000);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
cmdstat |= (bar_response & PCI_BASE_ADDRESS_SPACE) ?
|
|
|
|
PCI_COMMAND_IO : PCI_COMMAND_MEMORY;
|
|
|
|
|
|
|
|
debug("\n");
|
|
|
|
|
|
|
|
bar_nr++;
|
|
|
|
}
|
|
|
|
|
2021-01-12 11:03:43 +00:00
|
|
|
/* Configure the expansion ROM address */
|
2021-10-07 12:50:57 +00:00
|
|
|
if (header_type == PCI_HEADER_TYPE_NORMAL ||
|
|
|
|
header_type == PCI_HEADER_TYPE_BRIDGE) {
|
2021-01-12 11:03:43 +00:00
|
|
|
rom_addr = (header_type == PCI_HEADER_TYPE_NORMAL) ?
|
|
|
|
PCI_ROM_ADDRESS : PCI_ROM_ADDRESS1;
|
|
|
|
dm_pci_write_config32(dev, rom_addr, 0xfffffffe);
|
|
|
|
dm_pci_read_config32(dev, rom_addr, &bar_response);
|
|
|
|
if (bar_response) {
|
|
|
|
bar_size = -(bar_response & ~1);
|
|
|
|
debug("PCI Autoconfig: ROM, size=%#x, ",
|
|
|
|
(unsigned int)bar_size);
|
|
|
|
if (pciauto_region_allocate(mem, bar_size, &bar_value,
|
|
|
|
false) == 0) {
|
|
|
|
dm_pci_write_config32(dev, rom_addr, bar_value);
|
2015-11-29 20:17:49 +00:00
|
|
|
}
|
2021-01-12 11:03:43 +00:00
|
|
|
cmdstat |= PCI_COMMAND_MEMORY;
|
|
|
|
debug("\n");
|
2015-11-29 20:17:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* PCI_COMMAND_IO must be set for VGA device */
|
|
|
|
dm_pci_read_config16(dev, PCI_CLASS_DEVICE, &class);
|
|
|
|
if (class == PCI_CLASS_DISPLAY_VGA)
|
|
|
|
cmdstat |= PCI_COMMAND_IO;
|
|
|
|
|
|
|
|
dm_pci_write_config16(dev, PCI_COMMAND, cmdstat);
|
|
|
|
dm_pci_write_config8(dev, PCI_CACHE_LINE_SIZE,
|
2022-11-16 18:10:33 +00:00
|
|
|
CFG_SYS_PCI_CACHE_LINE_SIZE);
|
2015-11-29 20:17:49 +00:00
|
|
|
dm_pci_write_config8(dev, PCI_LATENCY_TIMER, 0x80);
|
|
|
|
}
|
|
|
|
|
pci: Work around PCIe link training failures
Attempt to handle cases with a downstream port of a PCIe switch where
link training never completes and the link continues switching between
speeds indefinitely with the data link layer never reaching the active
state.
It has been observed with a downstream port of the ASMedia ASM2824 Gen 3
switch wired to the upstream port of the Pericom PI7C9X2G304 Gen 2
switch, using a Delock Riser Card PCI Express x1 > 2 x PCIe x1 device,
P/N 41433, wired to a SiFive HiFive Unmatched board. In this setup the
switches are supposed to negotiate the link speed of preferably 5.0GT/s,
falling back to 2.5GT/s.
However the link continues oscillating between the two speeds, at the
rate of 34-35 times per second, with link training reported repeatedly
active ~84% of the time, e.g.:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=05, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 5GT/s (downgraded), Width x1 (ok)
TrErr- Train+ SlotClk+ DLActive- BWMgmt+ ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 8GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Forcibly limiting the target link speed to 2.5GT/s with the upstream
ASM2824 device makes the two switches communicate correctly however:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=09, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (ok)
TrErr- Train- SlotClk+ DLActive+ BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 2.5GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
and then:
05:00.0 PCI bridge [0604]: Pericom Semiconductor PI7C9X2G304 EL/SL PCIe2 3-Port/4-Lane Packet Switch [12d8:2304] (rev 05) (prog-if 00 [Normal decode])
[...]
Bus: primary=05, secondary=06, subordinate=09, sec-latency=0
[...]
Capabilities: [c0] Express (v2) Upstream Port, MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (downgraded)
TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Make use of this observation then and attempt to detect the inability to
negotiate the link speed automatically, and then handle it by hand. Use
the Data Link Layer Link Active status flag as the primary indicator of
successful link speed negotiation, but given that the flag is optional
by hardware to implement (the ASM2824 does have it though), resort to
checking for the mandatory Link Bandwidth Management Status flag showing
that the link speed or width has been changed in an attempt to correct
unreliable link operation (the ASM2824 does set it too).
If these checks indicate that link may not operate correctly, then poll
the Data Link Layer Link Active status flag along with the Link Training
flag for the duration of 200ms to see if the link has stabilised, that
is either that the Data Link Layer Link Active status flag has been set
or that Link Training has been inactive during at least the second half
of the interval.
If that has indicated failure, restrict the target speed to 2.5GT/s,
request a link retrain and check again if the link has stabilised. If
that does not work either, then restore the original speed setting and
claim defeat, otherwise we are done.
NB interestingly enough with the ASM2824 vs PI7C9X2G304 configuration
referred above asking the ASM2824 to retrain with a higher target link
speed once the 2.5GT/s speed has been negotiated makes the two devices
successfully negotiate 5.0GT/s. Lifting the 2.5GT/s speed restriction
would however prevent our workaround from working with an OS that issues
a reset and that is unaware of the problem. This is because the devices
would then try to negotiate a higher link speed from scratch and fail,
while the sticky property of the Target Link Speed setting will keep the
2.5GT/s speed restriction across a reset.
Keep the 2.5GT/s speed restriction then, conservatively, if functional
once applied.
Signed-off-by: Maciej W. Rozycki <macro@orcam.me.uk>
Reviewed-by: Stefan Roese <sr@denx.de>
2021-11-20 23:03:30 +00:00
|
|
|
/*
|
|
|
|
* Check if the link of a downstream PCIe port operates correctly.
|
|
|
|
*
|
|
|
|
* For that check if the optional Data Link Layer Link Active status gets
|
|
|
|
* on within a 200ms period or failing that wait until the completion of
|
|
|
|
* that period and check if link training has shown the completed status
|
|
|
|
* continuously throughout the second half of that period.
|
|
|
|
*
|
|
|
|
* Observation with the ASMedia ASM2824 Gen 3 switch indicates it takes
|
|
|
|
* 11-44ms to indicate the Data Link Layer Link Active status at 2.5GT/s,
|
|
|
|
* though it may take a couple of link training iterations.
|
|
|
|
*/
|
|
|
|
static bool dm_pciauto_exp_link_stable(struct udevice *dev, int pcie_off)
|
|
|
|
{
|
|
|
|
u64 loops = 0, trcount = 0, ntrcount = 0, flips = 0;
|
|
|
|
bool dllla, lnktr, plnktr;
|
|
|
|
u16 exp_lnksta;
|
|
|
|
pci_dev_t bdf;
|
|
|
|
u64 end;
|
|
|
|
|
|
|
|
dm_pci_read_config16(dev, pcie_off + PCI_EXP_LNKSTA, &exp_lnksta);
|
|
|
|
plnktr = !!(exp_lnksta & PCI_EXP_LNKSTA_LT);
|
|
|
|
|
|
|
|
end = get_ticks() + usec_to_tick(200000);
|
|
|
|
do {
|
|
|
|
dm_pci_read_config16(dev, pcie_off + PCI_EXP_LNKSTA,
|
|
|
|
&exp_lnksta);
|
|
|
|
dllla = !!(exp_lnksta & PCI_EXP_LNKSTA_DLLLA);
|
|
|
|
lnktr = !!(exp_lnksta & PCI_EXP_LNKSTA_LT);
|
|
|
|
|
|
|
|
flips += plnktr ^ lnktr;
|
|
|
|
if (lnktr) {
|
|
|
|
ntrcount = 0;
|
|
|
|
trcount++;
|
|
|
|
} else {
|
|
|
|
ntrcount++;
|
|
|
|
}
|
|
|
|
loops++;
|
|
|
|
|
|
|
|
plnktr = lnktr;
|
|
|
|
} while (!dllla && get_ticks() < end);
|
|
|
|
|
|
|
|
bdf = dm_pci_get_bdf(dev);
|
|
|
|
debug("PCI Autoconfig: %02x.%02x.%02x: Fixup link: DL active: %u; "
|
|
|
|
"%3llu flips, %6llu loops of which %6llu while training, "
|
|
|
|
"final %6llu stable\n",
|
|
|
|
PCI_BUS(bdf), PCI_DEV(bdf), PCI_FUNC(bdf),
|
|
|
|
(unsigned int)dllla,
|
|
|
|
(unsigned long long)flips, (unsigned long long)loops,
|
|
|
|
(unsigned long long)trcount, (unsigned long long)ntrcount);
|
|
|
|
|
|
|
|
return dllla || ntrcount >= loops / 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Retrain the link of a downstream PCIe port by hand if necessary.
|
|
|
|
*
|
|
|
|
* This is needed at least where a downstream port of the ASMedia ASM2824
|
|
|
|
* Gen 3 switch is wired to the upstream port of the Pericom PI7C9X2G304
|
|
|
|
* Gen 2 switch, and observed with the Delock Riser Card PCI Express x1 >
|
|
|
|
* 2 x PCIe x1 device, P/N 41433, plugged into the SiFive HiFive Unmatched
|
|
|
|
* board.
|
|
|
|
*
|
|
|
|
* In such a configuration the switches are supposed to negotiate the link
|
|
|
|
* speed of preferably 5.0GT/s, falling back to 2.5GT/s. However the link
|
|
|
|
* continues switching between the two speeds indefinitely and the data
|
|
|
|
* link layer never reaches the active state, with link training reported
|
|
|
|
* repeatedly active ~84% of the time. Forcing the target link speed to
|
|
|
|
* 2.5GT/s with the upstream ASM2824 device makes the two switches talk to
|
|
|
|
* each other correctly however. And more interestingly retraining with a
|
|
|
|
* higher target link speed afterwards lets the two successfully negotiate
|
|
|
|
* 5.0GT/s.
|
|
|
|
*
|
|
|
|
* As this can potentially happen with any device and is cheap in the case
|
|
|
|
* of correctly operating hardware, let's do it for all downstream ports,
|
|
|
|
* for root complexes, PCIe switches and PCI/PCI-X to PCIe bridges.
|
|
|
|
*
|
|
|
|
* First check if automatic link training may have failed to complete, as
|
|
|
|
* indicated by the optional Data Link Layer Link Active status being off
|
|
|
|
* and the Link Bandwidth Management Status indicating that hardware has
|
|
|
|
* changed the link speed or width in an attempt to correct unreliable
|
|
|
|
* link operation. If this is the case, then check if the link operates
|
|
|
|
* correctly by seeing whether it is being trained excessively. If it is,
|
|
|
|
* then conclude the link is broken.
|
|
|
|
*
|
|
|
|
* In that case restrict the speed to 2.5GT/s, observing that the Target
|
|
|
|
* Link Speed field is sticky and therefore the link will stay restricted
|
|
|
|
* even after a device reset is later made by an OS that is unaware of the
|
|
|
|
* problem. With the speed restricted request that the link be retrained
|
|
|
|
* and check again if the link operates correctly. If not, then set the
|
|
|
|
* Target Link Speed back to the original value.
|
|
|
|
*
|
|
|
|
* This requires the presence of the Link Control 2 register, so make sure
|
|
|
|
* the PCI Express Capability Version is at least 2. Also don't try, for
|
|
|
|
* obvious reasons, to limit the speed if 2.5GT/s is the only link speed
|
|
|
|
* supported.
|
|
|
|
*/
|
|
|
|
static void dm_pciauto_exp_fixup_link(struct udevice *dev, int pcie_off)
|
|
|
|
{
|
|
|
|
u16 exp_lnksta, exp_lnkctl, exp_lnkctl2;
|
|
|
|
u16 exp_flags, exp_type, exp_version;
|
|
|
|
u32 exp_lnkcap;
|
|
|
|
pci_dev_t bdf;
|
|
|
|
|
|
|
|
dm_pci_read_config16(dev, pcie_off + PCI_EXP_FLAGS, &exp_flags);
|
|
|
|
exp_version = exp_flags & PCI_EXP_FLAGS_VERS;
|
|
|
|
if (exp_version < 2)
|
|
|
|
return;
|
|
|
|
|
|
|
|
exp_type = (exp_flags & PCI_EXP_FLAGS_TYPE) >> 4;
|
|
|
|
switch (exp_type) {
|
|
|
|
case PCI_EXP_TYPE_ROOT_PORT:
|
|
|
|
case PCI_EXP_TYPE_DOWNSTREAM:
|
|
|
|
case PCI_EXP_TYPE_PCIE_BRIDGE:
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
dm_pci_read_config32(dev, pcie_off + PCI_EXP_LNKCAP, &exp_lnkcap);
|
|
|
|
if ((exp_lnkcap & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB)
|
|
|
|
return;
|
|
|
|
|
|
|
|
dm_pci_read_config16(dev, pcie_off + PCI_EXP_LNKSTA, &exp_lnksta);
|
|
|
|
if ((exp_lnksta & (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_DLLLA)) !=
|
|
|
|
PCI_EXP_LNKSTA_LBMS)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (dm_pciauto_exp_link_stable(dev, pcie_off))
|
|
|
|
return;
|
|
|
|
|
|
|
|
bdf = dm_pci_get_bdf(dev);
|
|
|
|
printf("PCI Autoconfig: %02x.%02x.%02x: "
|
|
|
|
"Downstream link non-functional\n",
|
|
|
|
PCI_BUS(bdf), PCI_DEV(bdf), PCI_FUNC(bdf));
|
|
|
|
printf("PCI Autoconfig: %02x.%02x.%02x: "
|
|
|
|
"Retrying with speed restricted to 2.5GT/s...\n",
|
|
|
|
PCI_BUS(bdf), PCI_DEV(bdf), PCI_FUNC(bdf));
|
|
|
|
|
|
|
|
dm_pci_read_config16(dev, pcie_off + PCI_EXP_LNKCTL, &exp_lnkctl);
|
|
|
|
dm_pci_read_config16(dev, pcie_off + PCI_EXP_LNKCTL2, &exp_lnkctl2);
|
|
|
|
|
|
|
|
dm_pci_write_config16(dev, pcie_off + PCI_EXP_LNKCTL2,
|
|
|
|
(exp_lnkctl2 & ~PCI_EXP_LNKCTL2_TLS) |
|
|
|
|
PCI_EXP_LNKCTL2_TLS_2_5GT);
|
|
|
|
dm_pci_write_config16(dev, pcie_off + PCI_EXP_LNKCTL,
|
|
|
|
exp_lnkctl | PCI_EXP_LNKCTL_RL);
|
|
|
|
|
|
|
|
if (dm_pciauto_exp_link_stable(dev, pcie_off)) {
|
|
|
|
printf("PCI Autoconfig: %02x.%02x.%02x: Succeeded!\n",
|
|
|
|
PCI_BUS(bdf), PCI_DEV(bdf), PCI_FUNC(bdf));
|
|
|
|
} else {
|
|
|
|
printf("PCI Autoconfig: %02x.%02x.%02x: Failed!\n",
|
|
|
|
PCI_BUS(bdf), PCI_DEV(bdf), PCI_FUNC(bdf));
|
|
|
|
|
|
|
|
dm_pci_write_config16(dev, pcie_off + PCI_EXP_LNKCTL2,
|
|
|
|
exp_lnkctl2);
|
|
|
|
dm_pci_write_config16(dev, pcie_off + PCI_EXP_LNKCTL,
|
|
|
|
exp_lnkctl | PCI_EXP_LNKCTL_RL);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-11-29 20:17:49 +00:00
|
|
|
void dm_pciauto_prescan_setup_bridge(struct udevice *dev, int sub_bus)
|
|
|
|
{
|
|
|
|
struct pci_region *pci_mem;
|
|
|
|
struct pci_region *pci_prefetch;
|
|
|
|
struct pci_region *pci_io;
|
|
|
|
u16 cmdstat, prefechable_64;
|
2021-09-10 11:33:35 +00:00
|
|
|
u8 io_32;
|
2016-01-19 03:19:16 +00:00
|
|
|
struct udevice *ctlr = pci_get_controller(dev);
|
|
|
|
struct pci_controller *ctlr_hose = dev_get_uclass_priv(ctlr);
|
pci: Work around PCIe link training failures
Attempt to handle cases with a downstream port of a PCIe switch where
link training never completes and the link continues switching between
speeds indefinitely with the data link layer never reaching the active
state.
It has been observed with a downstream port of the ASMedia ASM2824 Gen 3
switch wired to the upstream port of the Pericom PI7C9X2G304 Gen 2
switch, using a Delock Riser Card PCI Express x1 > 2 x PCIe x1 device,
P/N 41433, wired to a SiFive HiFive Unmatched board. In this setup the
switches are supposed to negotiate the link speed of preferably 5.0GT/s,
falling back to 2.5GT/s.
However the link continues oscillating between the two speeds, at the
rate of 34-35 times per second, with link training reported repeatedly
active ~84% of the time, e.g.:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=05, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 5GT/s (downgraded), Width x1 (ok)
TrErr- Train+ SlotClk+ DLActive- BWMgmt+ ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 8GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Forcibly limiting the target link speed to 2.5GT/s with the upstream
ASM2824 device makes the two switches communicate correctly however:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=09, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (ok)
TrErr- Train- SlotClk+ DLActive+ BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 2.5GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
and then:
05:00.0 PCI bridge [0604]: Pericom Semiconductor PI7C9X2G304 EL/SL PCIe2 3-Port/4-Lane Packet Switch [12d8:2304] (rev 05) (prog-if 00 [Normal decode])
[...]
Bus: primary=05, secondary=06, subordinate=09, sec-latency=0
[...]
Capabilities: [c0] Express (v2) Upstream Port, MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (downgraded)
TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Make use of this observation then and attempt to detect the inability to
negotiate the link speed automatically, and then handle it by hand. Use
the Data Link Layer Link Active status flag as the primary indicator of
successful link speed negotiation, but given that the flag is optional
by hardware to implement (the ASM2824 does have it though), resort to
checking for the mandatory Link Bandwidth Management Status flag showing
that the link speed or width has been changed in an attempt to correct
unreliable link operation (the ASM2824 does set it too).
If these checks indicate that link may not operate correctly, then poll
the Data Link Layer Link Active status flag along with the Link Training
flag for the duration of 200ms to see if the link has stabilised, that
is either that the Data Link Layer Link Active status flag has been set
or that Link Training has been inactive during at least the second half
of the interval.
If that has indicated failure, restrict the target speed to 2.5GT/s,
request a link retrain and check again if the link has stabilised. If
that does not work either, then restore the original speed setting and
claim defeat, otherwise we are done.
NB interestingly enough with the ASM2824 vs PI7C9X2G304 configuration
referred above asking the ASM2824 to retrain with a higher target link
speed once the 2.5GT/s speed has been negotiated makes the two devices
successfully negotiate 5.0GT/s. Lifting the 2.5GT/s speed restriction
would however prevent our workaround from working with an OS that issues
a reset and that is unaware of the problem. This is because the devices
would then try to negotiate a higher link speed from scratch and fail,
while the sticky property of the Target Link Speed setting will keep the
2.5GT/s speed restriction across a reset.
Keep the 2.5GT/s speed restriction then, conservatively, if functional
once applied.
Signed-off-by: Maciej W. Rozycki <macro@orcam.me.uk>
Reviewed-by: Stefan Roese <sr@denx.de>
2021-11-20 23:03:30 +00:00
|
|
|
int pcie_off;
|
2015-11-29 20:17:49 +00:00
|
|
|
|
|
|
|
pci_mem = ctlr_hose->pci_mem;
|
|
|
|
pci_prefetch = ctlr_hose->pci_prefetch;
|
|
|
|
pci_io = ctlr_hose->pci_io;
|
|
|
|
|
|
|
|
dm_pci_read_config16(dev, PCI_COMMAND, &cmdstat);
|
|
|
|
dm_pci_read_config16(dev, PCI_PREF_MEMORY_BASE, &prefechable_64);
|
|
|
|
prefechable_64 &= PCI_PREF_RANGE_TYPE_MASK;
|
2021-11-25 10:30:58 +00:00
|
|
|
dm_pci_read_config8(dev, PCI_IO_BASE, &io_32);
|
2021-09-10 11:33:35 +00:00
|
|
|
io_32 &= PCI_IO_RANGE_TYPE_MASK;
|
2015-11-29 20:17:49 +00:00
|
|
|
|
|
|
|
/* Configure bus number registers */
|
|
|
|
dm_pci_write_config8(dev, PCI_PRIMARY_BUS,
|
2020-12-17 04:20:07 +00:00
|
|
|
PCI_BUS(dm_pci_get_bdf(dev)) - dev_seq(ctlr));
|
|
|
|
dm_pci_write_config8(dev, PCI_SECONDARY_BUS, sub_bus - dev_seq(ctlr));
|
2015-11-29 20:17:49 +00:00
|
|
|
dm_pci_write_config8(dev, PCI_SUBORDINATE_BUS, 0xff);
|
|
|
|
|
|
|
|
if (pci_mem) {
|
|
|
|
/* Round memory allocator to 1MB boundary */
|
|
|
|
pciauto_region_align(pci_mem, 0x100000);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set up memory and I/O filter limits, assume 32-bit
|
|
|
|
* I/O space
|
|
|
|
*/
|
|
|
|
dm_pci_write_config16(dev, PCI_MEMORY_BASE,
|
2021-09-10 11:33:35 +00:00
|
|
|
((pci_mem->bus_lower & 0xfff00000) >> 16) &
|
|
|
|
PCI_MEMORY_RANGE_MASK);
|
2015-11-29 20:17:49 +00:00
|
|
|
|
|
|
|
cmdstat |= PCI_COMMAND_MEMORY;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pci_prefetch) {
|
|
|
|
/* Round memory allocator to 1MB boundary */
|
|
|
|
pciauto_region_align(pci_prefetch, 0x100000);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set up memory and I/O filter limits, assume 32-bit
|
|
|
|
* I/O space
|
|
|
|
*/
|
|
|
|
dm_pci_write_config16(dev, PCI_PREF_MEMORY_BASE,
|
2021-09-10 11:33:35 +00:00
|
|
|
(((pci_prefetch->bus_lower & 0xfff00000) >> 16) &
|
|
|
|
PCI_PREF_RANGE_MASK) | prefechable_64);
|
2015-11-29 20:17:49 +00:00
|
|
|
if (prefechable_64 == PCI_PREF_RANGE_TYPE_64)
|
|
|
|
#ifdef CONFIG_SYS_PCI_64BIT
|
|
|
|
dm_pci_write_config32(dev, PCI_PREF_BASE_UPPER32,
|
|
|
|
pci_prefetch->bus_lower >> 32);
|
|
|
|
#else
|
|
|
|
dm_pci_write_config32(dev, PCI_PREF_BASE_UPPER32, 0x0);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
cmdstat |= PCI_COMMAND_MEMORY;
|
|
|
|
} else {
|
|
|
|
/* We don't support prefetchable memory for now, so disable */
|
2021-11-25 10:34:37 +00:00
|
|
|
dm_pci_write_config16(dev, PCI_PREF_MEMORY_BASE, 0xfff0 |
|
2021-09-10 11:33:35 +00:00
|
|
|
prefechable_64);
|
|
|
|
dm_pci_write_config16(dev, PCI_PREF_MEMORY_LIMIT, 0x0 |
|
|
|
|
prefechable_64);
|
2015-11-29 20:17:49 +00:00
|
|
|
if (prefechable_64 == PCI_PREF_RANGE_TYPE_64) {
|
|
|
|
dm_pci_write_config16(dev, PCI_PREF_BASE_UPPER32, 0x0);
|
|
|
|
dm_pci_write_config16(dev, PCI_PREF_LIMIT_UPPER32, 0x0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pci_io) {
|
|
|
|
/* Round I/O allocator to 4KB boundary */
|
|
|
|
pciauto_region_align(pci_io, 0x1000);
|
|
|
|
|
|
|
|
dm_pci_write_config8(dev, PCI_IO_BASE,
|
2021-09-10 11:33:35 +00:00
|
|
|
(((pci_io->bus_lower & 0x0000f000) >> 8) &
|
|
|
|
PCI_IO_RANGE_MASK) | io_32);
|
|
|
|
if (io_32 == PCI_IO_RANGE_TYPE_32)
|
|
|
|
dm_pci_write_config16(dev, PCI_IO_BASE_UPPER16,
|
2015-11-29 20:17:49 +00:00
|
|
|
(pci_io->bus_lower & 0xffff0000) >> 16);
|
|
|
|
|
|
|
|
cmdstat |= PCI_COMMAND_IO;
|
2021-11-25 10:32:43 +00:00
|
|
|
} else {
|
|
|
|
/* Disable I/O if unsupported */
|
|
|
|
dm_pci_write_config8(dev, PCI_IO_BASE, 0xf0 | io_32);
|
|
|
|
dm_pci_write_config8(dev, PCI_IO_LIMIT, 0x0 | io_32);
|
|
|
|
if (io_32 == PCI_IO_RANGE_TYPE_32) {
|
|
|
|
dm_pci_write_config16(dev, PCI_IO_BASE_UPPER16, 0x0);
|
|
|
|
dm_pci_write_config16(dev, PCI_IO_LIMIT_UPPER16, 0x0);
|
|
|
|
}
|
2015-11-29 20:17:49 +00:00
|
|
|
}
|
|
|
|
|
pci: Work around PCIe link training failures
Attempt to handle cases with a downstream port of a PCIe switch where
link training never completes and the link continues switching between
speeds indefinitely with the data link layer never reaching the active
state.
It has been observed with a downstream port of the ASMedia ASM2824 Gen 3
switch wired to the upstream port of the Pericom PI7C9X2G304 Gen 2
switch, using a Delock Riser Card PCI Express x1 > 2 x PCIe x1 device,
P/N 41433, wired to a SiFive HiFive Unmatched board. In this setup the
switches are supposed to negotiate the link speed of preferably 5.0GT/s,
falling back to 2.5GT/s.
However the link continues oscillating between the two speeds, at the
rate of 34-35 times per second, with link training reported repeatedly
active ~84% of the time, e.g.:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=05, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 5GT/s (downgraded), Width x1 (ok)
TrErr- Train+ SlotClk+ DLActive- BWMgmt+ ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 8GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Forcibly limiting the target link speed to 2.5GT/s with the upstream
ASM2824 device makes the two switches communicate correctly however:
02:03.0 PCI bridge [0604]: ASMedia Technology Inc. ASM2824 PCIe Gen3 Packet Switch [1b21:2824] (rev 01) (prog-if 00 [Normal decode])
[...]
Bus: primary=02, secondary=05, subordinate=09, sec-latency=0
[...]
Capabilities: [80] Express (v2) Downstream Port (Slot+), MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (ok)
TrErr- Train- SlotClk+ DLActive+ BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 2.5GT/s, EnterCompliance- SpeedDis+, Selectable De-emphasis: -3.5dB
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
and then:
05:00.0 PCI bridge [0604]: Pericom Semiconductor PI7C9X2G304 EL/SL PCIe2 3-Port/4-Lane Packet Switch [12d8:2304] (rev 05) (prog-if 00 [Normal decode])
[...]
Bus: primary=05, secondary=06, subordinate=09, sec-latency=0
[...]
Capabilities: [c0] Express (v2) Upstream Port, MSI 00
[...]
LnkSta: Speed 2.5GT/s (downgraded), Width x1 (downgraded)
TrErr- Train- SlotClk+ DLActive- BWMgmt- ABWMgmt-
[...]
LnkCtl2: Target Link Speed: 5GT/s, EnterCompliance- SpeedDis-
Transmit Margin: Normal Operating Range, EnterModifiedCompliance- ComplianceSOS-
Compliance De-emphasis: -6dB
[...]
Make use of this observation then and attempt to detect the inability to
negotiate the link speed automatically, and then handle it by hand. Use
the Data Link Layer Link Active status flag as the primary indicator of
successful link speed negotiation, but given that the flag is optional
by hardware to implement (the ASM2824 does have it though), resort to
checking for the mandatory Link Bandwidth Management Status flag showing
that the link speed or width has been changed in an attempt to correct
unreliable link operation (the ASM2824 does set it too).
If these checks indicate that link may not operate correctly, then poll
the Data Link Layer Link Active status flag along with the Link Training
flag for the duration of 200ms to see if the link has stabilised, that
is either that the Data Link Layer Link Active status flag has been set
or that Link Training has been inactive during at least the second half
of the interval.
If that has indicated failure, restrict the target speed to 2.5GT/s,
request a link retrain and check again if the link has stabilised. If
that does not work either, then restore the original speed setting and
claim defeat, otherwise we are done.
NB interestingly enough with the ASM2824 vs PI7C9X2G304 configuration
referred above asking the ASM2824 to retrain with a higher target link
speed once the 2.5GT/s speed has been negotiated makes the two devices
successfully negotiate 5.0GT/s. Lifting the 2.5GT/s speed restriction
would however prevent our workaround from working with an OS that issues
a reset and that is unaware of the problem. This is because the devices
would then try to negotiate a higher link speed from scratch and fail,
while the sticky property of the Target Link Speed setting will keep the
2.5GT/s speed restriction across a reset.
Keep the 2.5GT/s speed restriction then, conservatively, if functional
once applied.
Signed-off-by: Maciej W. Rozycki <macro@orcam.me.uk>
Reviewed-by: Stefan Roese <sr@denx.de>
2021-11-20 23:03:30 +00:00
|
|
|
/* For PCIe devices see if we need to retrain the link by hand */
|
|
|
|
pcie_off = dm_pci_find_capability(dev, PCI_CAP_ID_EXP);
|
|
|
|
if (pcie_off)
|
|
|
|
dm_pciauto_exp_fixup_link(dev, pcie_off);
|
|
|
|
|
2015-11-29 20:17:49 +00:00
|
|
|
/* Enable memory and I/O accesses, enable bus master */
|
|
|
|
dm_pci_write_config16(dev, PCI_COMMAND, cmdstat | PCI_COMMAND_MASTER);
|
|
|
|
}
|
|
|
|
|
|
|
|
void dm_pciauto_postscan_setup_bridge(struct udevice *dev, int sub_bus)
|
|
|
|
{
|
|
|
|
struct pci_region *pci_mem;
|
|
|
|
struct pci_region *pci_prefetch;
|
|
|
|
struct pci_region *pci_io;
|
2016-01-19 03:19:16 +00:00
|
|
|
struct udevice *ctlr = pci_get_controller(dev);
|
|
|
|
struct pci_controller *ctlr_hose = dev_get_uclass_priv(ctlr);
|
2015-11-29 20:17:49 +00:00
|
|
|
|
|
|
|
pci_mem = ctlr_hose->pci_mem;
|
|
|
|
pci_prefetch = ctlr_hose->pci_prefetch;
|
|
|
|
pci_io = ctlr_hose->pci_io;
|
|
|
|
|
|
|
|
/* Configure bus number registers */
|
2020-12-17 04:20:07 +00:00
|
|
|
dm_pci_write_config8(dev, PCI_SUBORDINATE_BUS, sub_bus - dev_seq(ctlr));
|
2015-11-29 20:17:49 +00:00
|
|
|
|
|
|
|
if (pci_mem) {
|
|
|
|
/* Round memory allocator to 1MB boundary */
|
|
|
|
pciauto_region_align(pci_mem, 0x100000);
|
|
|
|
|
|
|
|
dm_pci_write_config16(dev, PCI_MEMORY_LIMIT,
|
2021-09-10 11:33:35 +00:00
|
|
|
((pci_mem->bus_lower - 1) >> 16) &
|
|
|
|
PCI_MEMORY_RANGE_MASK);
|
2015-11-29 20:17:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (pci_prefetch) {
|
|
|
|
u16 prefechable_64;
|
|
|
|
|
|
|
|
dm_pci_read_config16(dev, PCI_PREF_MEMORY_LIMIT,
|
|
|
|
&prefechable_64);
|
|
|
|
prefechable_64 &= PCI_PREF_RANGE_TYPE_MASK;
|
|
|
|
|
|
|
|
/* Round memory allocator to 1MB boundary */
|
|
|
|
pciauto_region_align(pci_prefetch, 0x100000);
|
|
|
|
|
|
|
|
dm_pci_write_config16(dev, PCI_PREF_MEMORY_LIMIT,
|
2021-09-10 11:33:35 +00:00
|
|
|
(((pci_prefetch->bus_lower - 1) >> 16) &
|
|
|
|
PCI_PREF_RANGE_MASK) | prefechable_64);
|
2015-11-29 20:17:49 +00:00
|
|
|
if (prefechable_64 == PCI_PREF_RANGE_TYPE_64)
|
|
|
|
#ifdef CONFIG_SYS_PCI_64BIT
|
|
|
|
dm_pci_write_config32(dev, PCI_PREF_LIMIT_UPPER32,
|
|
|
|
(pci_prefetch->bus_lower - 1) >> 32);
|
|
|
|
#else
|
|
|
|
dm_pci_write_config32(dev, PCI_PREF_LIMIT_UPPER32, 0x0);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pci_io) {
|
2021-09-10 11:33:35 +00:00
|
|
|
u8 io_32;
|
|
|
|
|
|
|
|
dm_pci_read_config8(dev, PCI_IO_LIMIT,
|
|
|
|
&io_32);
|
|
|
|
io_32 &= PCI_IO_RANGE_TYPE_MASK;
|
|
|
|
|
2015-11-29 20:17:49 +00:00
|
|
|
/* Round I/O allocator to 4KB boundary */
|
|
|
|
pciauto_region_align(pci_io, 0x1000);
|
|
|
|
|
|
|
|
dm_pci_write_config8(dev, PCI_IO_LIMIT,
|
2021-09-10 11:33:35 +00:00
|
|
|
((((pci_io->bus_lower - 1) & 0x0000f000) >> 8) &
|
|
|
|
PCI_IO_RANGE_MASK) | io_32);
|
|
|
|
if (io_32 == PCI_IO_RANGE_TYPE_32)
|
|
|
|
dm_pci_write_config16(dev, PCI_IO_LIMIT_UPPER16,
|
2015-11-29 20:17:49 +00:00
|
|
|
((pci_io->bus_lower - 1) & 0xffff0000) >> 16);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* HJF: Changed this to return int. I think this is required
|
|
|
|
* to get the correct result when scanning bridges
|
|
|
|
*/
|
|
|
|
int dm_pciauto_config_device(struct udevice *dev)
|
|
|
|
{
|
|
|
|
struct pci_region *pci_mem;
|
|
|
|
struct pci_region *pci_prefetch;
|
|
|
|
struct pci_region *pci_io;
|
|
|
|
unsigned int sub_bus = PCI_BUS(dm_pci_get_bdf(dev));
|
|
|
|
unsigned short class;
|
2016-01-19 03:19:16 +00:00
|
|
|
struct udevice *ctlr = pci_get_controller(dev);
|
|
|
|
struct pci_controller *ctlr_hose = dev_get_uclass_priv(ctlr);
|
2020-12-17 04:20:18 +00:00
|
|
|
int ret;
|
2015-11-29 20:17:49 +00:00
|
|
|
|
|
|
|
pci_mem = ctlr_hose->pci_mem;
|
|
|
|
pci_prefetch = ctlr_hose->pci_prefetch;
|
|
|
|
pci_io = ctlr_hose->pci_io;
|
|
|
|
|
|
|
|
dm_pci_read_config16(dev, PCI_CLASS_DEVICE, &class);
|
|
|
|
|
|
|
|
switch (class) {
|
|
|
|
case PCI_CLASS_BRIDGE_PCI:
|
|
|
|
debug("PCI Autoconfig: Found P2P bridge, device %d\n",
|
|
|
|
PCI_DEV(dm_pci_get_bdf(dev)));
|
|
|
|
|
2021-10-07 12:50:59 +00:00
|
|
|
dm_pciauto_setup_device(dev, pci_mem, pci_prefetch, pci_io);
|
2015-11-29 20:17:49 +00:00
|
|
|
|
2020-12-17 04:20:18 +00:00
|
|
|
ret = dm_pci_hose_probe_bus(dev);
|
|
|
|
if (ret < 0)
|
|
|
|
return log_msg_ret("probe", ret);
|
|
|
|
sub_bus = ret;
|
2015-11-29 20:17:49 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case PCI_CLASS_BRIDGE_CARDBUS:
|
|
|
|
/*
|
|
|
|
* just do a minimal setup of the bridge,
|
|
|
|
* let the OS take care of the rest
|
|
|
|
*/
|
2021-10-07 12:50:59 +00:00
|
|
|
dm_pciauto_setup_device(dev, pci_mem, pci_prefetch, pci_io);
|
2015-11-29 20:17:49 +00:00
|
|
|
|
|
|
|
debug("PCI Autoconfig: Found P2CardBus bridge, device %d\n",
|
|
|
|
PCI_DEV(dm_pci_get_bdf(dev)));
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
#if defined(CONFIG_PCIAUTO_SKIP_HOST_BRIDGE)
|
|
|
|
case PCI_CLASS_BRIDGE_OTHER:
|
|
|
|
debug("PCI Autoconfig: Skipping bridge device %d\n",
|
|
|
|
PCI_DEV(dm_pci_get_bdf(dev)));
|
|
|
|
break;
|
|
|
|
#endif
|
2021-05-15 01:34:17 +00:00
|
|
|
#if defined(CONFIG_ARCH_MPC834X)
|
2015-11-29 20:17:49 +00:00
|
|
|
case PCI_CLASS_BRIDGE_OTHER:
|
|
|
|
/*
|
|
|
|
* The host/PCI bridge 1 seems broken in 8349 - it presents
|
|
|
|
* itself as 'PCI_CLASS_BRIDGE_OTHER' and appears as an _agent_
|
|
|
|
* device claiming resources io/mem/irq.. we only allow for
|
|
|
|
* the PIMMR window to be allocated (BAR0 - 1MB size)
|
|
|
|
*/
|
|
|
|
debug("PCI Autoconfig: Broken bridge found, only minimal config\n");
|
|
|
|
dm_pciauto_setup_device(dev, 0, hose->pci_mem,
|
2021-01-12 11:03:43 +00:00
|
|
|
hose->pci_prefetch, hose->pci_io);
|
2015-11-29 20:17:49 +00:00
|
|
|
break;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
default:
|
2021-10-07 12:50:59 +00:00
|
|
|
dm_pciauto_setup_device(dev, pci_mem, pci_prefetch, pci_io);
|
2015-11-29 20:17:49 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return sub_bus;
|
|
|
|
}
|