From 25118cce6627ecceabd29d6795d7af16729a35dd Mon Sep 17 00:00:00 2001 From: Joe Damato Date: Wed, 9 Oct 2024 17:55:08 +0000 Subject: [PATCH 1/2] tg3: Link IRQs to NAPI instances Link IRQs to NAPI instances with netif_napi_set_irq. This information can be queried with the netdev-genl API. Begin by testing my tg3 device in its default state: 1 TX queue and 4 RX queues. Compare the output of /proc/interrupts for my tg3 device with the output of netdev-genl after applying this patch: $ cat /proc/interrupts | grep eth0 343: [...] eth0-tx-0 344: [...] eth0-rx-1 345: [...] eth0-rx-2 346: [...] eth0-rx-3 347: [...] eth0-rx-4 As you can see above, tg3 has named the IRQs such that there is a dedicated tx IRQ and 4 dedicated rx IRQs, for a total of 5 IRQs. $ ./tools/net/ynl/cli.py --spec Documentation/netlink/specs/netdev.yaml \ --dump napi-get --json='{"ifindex": 2}' [{'id': 8197, 'ifindex': 2, 'irq': 347}, {'id': 8196, 'ifindex': 2, 'irq': 346}, {'id': 8195, 'ifindex': 2, 'irq': 345}, {'id': 8194, 'ifindex': 2, 'irq': 344}, {'id': 8193, 'ifindex': 2, 'irq': 343}] Netlink displays the same IRQs as above, noting that each is mapped to a unique NAPI instance. Now, reconfigure the NIC to have 4 TX queues and 4 RX queues: $ sudo ethtool -L eth0 rx 4 tx 4 $ sudo ethtool -l eth0 | tail -5 Current hardware settings: RX: 4 TX: 4 Other: n/a Combined: n/a Examine /proc/interrupts once again, noting that tg3 will now rename the IRQs to suggest that they are combined tx and rx without allocating additional IRQs, so the total IRQ count in /proc/interrupts is unchanged: 343: [...] eth0-0 344: [...] eth0-txrx-1 345: [...] eth0-txrx-2 346: [...] eth0-txrx-3 347: [...] eth0-txrx-4 Check the output from netlink again: $ ./tools/net/ynl/cli.py --spec Documentation/netlink/specs/netdev.yaml \ --dump napi-get --json='{"ifindex": 2}' [{'id': 8973, 'ifindex': 2, 'irq': 347}, {'id': 8972, 'ifindex': 2, 'irq': 346}, {'id': 8971, 'ifindex': 2, 'irq': 345}, {'id': 8970, 'ifindex': 2, 'irq': 344}, {'id': 8969, 'ifindex': 2, 'irq': 343}] Signed-off-by: Joe Damato Reviewed-by: Pavan Chebbi Reviewed-by: Michael Chan Link: https://patch.msgid.link/20241009175509.31753-2-jdamato@fastly.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/tg3.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 378815917741..6564072b47ba 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -7413,9 +7413,11 @@ static void tg3_napi_init(struct tg3 *tp) { int i; - netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll); - for (i = 1; i < tp->irq_cnt; i++) - netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix); + for (i = 0; i < tp->irq_cnt; i++) { + netif_napi_add(tp->dev, &tp->napi[i].napi, + i ? tg3_poll_msix : tg3_poll); + netif_napi_set_irq(&tp->napi[i].napi, tp->napi[i].irq_vec); + } } static void tg3_napi_fini(struct tg3 *tp) From aec5514d739fad575069a43832cb11c066992057 Mon Sep 17 00:00:00 2001 From: Joe Damato Date: Wed, 9 Oct 2024 17:55:09 +0000 Subject: [PATCH 2/2] tg3: Link queues to NAPIs Link queues to NAPIs using the netdev-genl API so this information is queryable. First, test with the default setting on my tg3 NIC at boot with 1 TX queue: $ ./tools/net/ynl/cli.py --spec Documentation/netlink/specs/netdev.yaml \ --dump queue-get --json='{"ifindex": 2}' [{'id': 0, 'ifindex': 2, 'napi-id': 8194, 'type': 'rx'}, {'id': 1, 'ifindex': 2, 'napi-id': 8195, 'type': 'rx'}, {'id': 2, 'ifindex': 2, 'napi-id': 8196, 'type': 'rx'}, {'id': 3, 'ifindex': 2, 'napi-id': 8197, 'type': 'rx'}, {'id': 0, 'ifindex': 2, 'napi-id': 8193, 'type': 'tx'}] Now, adjust the number of TX queues to be 4 via ethtool: $ sudo ethtool -L eth0 tx 4 $ sudo ethtool -l eth0 | tail -5 Current hardware settings: RX: 4 TX: 4 Other: n/a Combined: n/a Despite "Combined: n/a" in the ethtool output, /proc/interrupts shows the tg3 has renamed the IRQs to be combined: 343: [...] eth0-0 344: [...] eth0-txrx-1 345: [...] eth0-txrx-2 346: [...] eth0-txrx-3 347: [...] eth0-txrx-4 Now query this via netlink to ensure the queues are linked properly to their NAPIs: $ ./tools/net/ynl/cli.py --spec Documentation/netlink/specs/netdev.yaml \ --dump queue-get --json='{"ifindex": 2}' [{'id': 0, 'ifindex': 2, 'napi-id': 8960, 'type': 'rx'}, {'id': 1, 'ifindex': 2, 'napi-id': 8961, 'type': 'rx'}, {'id': 2, 'ifindex': 2, 'napi-id': 8962, 'type': 'rx'}, {'id': 3, 'ifindex': 2, 'napi-id': 8963, 'type': 'rx'}, {'id': 0, 'ifindex': 2, 'napi-id': 8960, 'type': 'tx'}, {'id': 1, 'ifindex': 2, 'napi-id': 8961, 'type': 'tx'}, {'id': 2, 'ifindex': 2, 'napi-id': 8962, 'type': 'tx'}, {'id': 3, 'ifindex': 2, 'napi-id': 8963, 'type': 'tx'}] As you can see above, id 0 for both TX and RX share a NAPI, NAPI ID 8960, and so on for each queue index up to 3. Signed-off-by: Joe Damato Reviewed-by: Michael Chan Link: https://patch.msgid.link/20241009175509.31753-3-jdamato@fastly.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/tg3.c | 39 ++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 6564072b47ba..c4bce6493afa 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -7395,18 +7395,49 @@ tx_recovery: static void tg3_napi_disable(struct tg3 *tp) { + int txq_idx = tp->txq_cnt - 1; + int rxq_idx = tp->rxq_cnt - 1; + struct tg3_napi *tnapi; int i; - for (i = tp->irq_cnt - 1; i >= 0; i--) - napi_disable(&tp->napi[i].napi); + for (i = tp->irq_cnt - 1; i >= 0; i--) { + tnapi = &tp->napi[i]; + if (tnapi->tx_buffers) { + netif_queue_set_napi(tp->dev, txq_idx, + NETDEV_QUEUE_TYPE_TX, NULL); + txq_idx--; + } + if (tnapi->rx_rcb) { + netif_queue_set_napi(tp->dev, rxq_idx, + NETDEV_QUEUE_TYPE_RX, NULL); + rxq_idx--; + } + napi_disable(&tnapi->napi); + } } static void tg3_napi_enable(struct tg3 *tp) { + int txq_idx = 0, rxq_idx = 0; + struct tg3_napi *tnapi; int i; - for (i = 0; i < tp->irq_cnt; i++) - napi_enable(&tp->napi[i].napi); + for (i = 0; i < tp->irq_cnt; i++) { + tnapi = &tp->napi[i]; + napi_enable(&tnapi->napi); + if (tnapi->tx_buffers) { + netif_queue_set_napi(tp->dev, txq_idx, + NETDEV_QUEUE_TYPE_TX, + &tnapi->napi); + txq_idx++; + } + if (tnapi->rx_rcb) { + netif_queue_set_napi(tp->dev, rxq_idx, + NETDEV_QUEUE_TYPE_RX, + &tnapi->napi); + rxq_idx++; + } + } } static void tg3_napi_init(struct tg3 *tp)