Re: [PATCH net-next v1 0/5] net: lan743x: automatically recover from TX hang

[Chris Babroski <cbabroski@nvidia.com>]

Hi Nicolai,

> What do TX_ERR_STS0, DMAC_INT_STS and FCT_INT_STS show while it hangs?

No errors; the following was captured during a hang:
* TX_ERR_STS0=0x0
* RX_ERR_STS0-3=0x0
* DMAC_INT_STS=0xF100
* FCT_INT_STS=0x100000F1
* DMAC_CFG=0x10060
* DMAC_CMD=0x1000F0
* FCT_TX_USED=0x34
* TX_HEAD=0x3C
* TX_TAIL=0x38

> DMAC_CFG MAX_READ_REQ is not a size either. It is the number of
> outstanding TX data reads, 1 to 6, and the driver sets 6. Does the
> hang survive MAX_READ_REQ = 1?

So far I have not been able to reproduce with MAX_READ_REQ = 1, but I
have reproduced with MAX_READ_REQ = 3+. It's possible I have not run
long enough and the issue would still eventually reproduce with
MAX_READ_REQ = 1 though.

> If you can share the reproducer I can run it here.

I have been able to reproduce consistently just by starting an infinite
scp loop to copy a random 50MB file off of the SoC onto a Windows
laptop. Usually will reproduce within a few minutes when on the same
local network. Something about Windows scp/ssh significantly increases
repro rate; we have only reproduced once this way when doing repeated
scp from a Linux server.

Regarding the test data I mentioned: I do have a debug kernel module
that can reproduce consistently on our custom HW and I included it below
for reference.

The test configuration is based on some data we captured while
reproducing with the scp loop. It requires that SG and TSO are enabled.
See comment at top of file for details and change IFNAME to your
device's interface name.

I have not reproduced the TX hang with a LAN743x EVB connected directly
to a server using this test module or the scp loop method.

> So not the request size. What is left in the driver is the descriptor
> prefetch. lan743x_tx_open() sets TX_PF_THRES to 0x10 and
> TX_PF_PRI_THRES to 4, and the prefetcher works hardest with a full
> ring, which is when you see the most hangs. Does TX_PF_THRES 1 change
> the rate?

Unfortunately, no. I can still reproduce with TX_PF_THRES set to 1 and
TX_PF_PRI_THRES set to 1 as well (tried various combinations of values).
With the test module all reproduced on first NAK.

I also tried enabling DMA_CMPL_RETRY_EN and increasing the completion
timeout to 17-64s with setpci and could still reproduce quickly with the
test module.

Thanks,
Chris

Test module (debug only)
-------------------------

// SPDX-License-Identifier: GPL-2.0-only
/*
 * LAN743x TX hang reproducer. DEBUG ONLY.
 *
 * Sends the same TSO frame to TX queue 0 of IFNAME as fast as the driver
 * accepts it, until the module is removed. Each frame is a 54 byte
 * Ethernet/IPv4/TCP header in the skb linear area plus one 23560 byte
 * payload fragment at page offset 14335 (0x37FF), segmented by the NIC into
 * 1240 byte TCP segments. The driver maps this to three TX descriptors:
 * header, TSO extension and payload.
 *
 * Frames are addressed to the interface's own MAC and to 192.0.2.1
 * (TEST-NET-1). Needs scatter-gather, TSO and TX checksum offload enabled.
 *
 * If the queue stays stopped for STOP_MS, the sender reports it and stops;
 * the module then idles until it is removed. Remove this module before
 * unloading the network driver, it holds a reference on the netdev.
 */

#include <linux/delay.h>
#include <linux/etherdevice.h>
#include <linux/ip.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/tcp.h>
#include <net/ip.h>
#include <net/net_namespace.h>
#include <net/tcp.h>

#define IFNAME		"eth0"

#define HEADER_LEN	(ETH_HLEN + sizeof(struct iphdr) + sizeof(struct tcphdr))
#define PAYLOAD_LEN	23560
#define PAYLOAD_OFFSET	14335
#define GSO_SIZE	1240
#define STOP_MS		1000

static struct task_struct *test_task;
static struct net_device *test_dev;

static struct sk_buff *tx_test_build_skb(struct net_device *dev, u32 sequence)
{
	struct ethhdr *eth;
	struct tcphdr *th;
	struct sk_buff *skb;
	struct iphdr *iph;
	struct page *page;
	unsigned int order;

	/* headers in the linear area */
	skb = alloc_skb(LL_RESERVED_SPACE(dev) + HEADER_LEN, GFP_KERNEL);
	if (!skb)
		return NULL;

	skb_reserve(skb, LL_RESERVED_SPACE(dev));
	skb_put_zero(skb, HEADER_LEN);
	skb_reset_mac_header(skb);
	skb_set_network_header(skb, ETH_HLEN);
	skb_set_transport_header(skb, ETH_HLEN + sizeof(*iph));

	eth = eth_hdr(skb);
	ether_addr_copy(eth->h_source, dev->dev_addr);
	ether_addr_copy(eth->h_dest, dev->dev_addr);
	eth->h_proto = htons(ETH_P_IP);

	iph = ip_hdr(skb);
	iph->version = 4;
	iph->ihl = sizeof(*iph) / 4;
	iph->tot_len = htons(sizeof(*iph) + sizeof(*th) + PAYLOAD_LEN);
	iph->id = htons((u16)sequence);
	iph->frag_off = htons(IP_DF);
	iph->ttl = 64;
	iph->protocol = IPPROTO_TCP;
	iph->saddr = htonl(0xc0000202); /* 192.0.2.2 */
	iph->daddr = htonl(0xc0000201); /* 192.0.2.1 */
	ip_send_check(iph);

	th = tcp_hdr(skb);
	th->source = htons(12345);
	th->dest = htons(22);
	th->seq = htonl(sequence * PAYLOAD_LEN);
	th->ack_seq = htonl(1);
	th->doff = sizeof(*th) / 4;
	th->ack = 1;
	th->window = htons(U16_MAX);
	th->check = ~tcp_v4_check(sizeof(*th) + PAYLOAD_LEN,
				  iph->saddr, iph->daddr, 0);

	/* payload in one page fragment, starting at an unaligned offset */
	order = get_order(PAYLOAD_OFFSET + PAYLOAD_LEN);
	page = alloc_pages(GFP_KERNEL | __GFP_COMP | __GFP_ZERO, order);
	if (!page) {
		kfree_skb(skb);
		return NULL;
	}
	skb_add_rx_frag(skb, 0, page, PAYLOAD_OFFSET, PAYLOAD_LEN,
			PAGE_SIZE << order);

	/* TCP checksum and segmentation offloaded to the NIC */
	skb->dev = dev;
	skb->protocol = htons(ETH_P_IP);
	skb->ip_summed = CHECKSUM_PARTIAL;
	skb->csum_start = skb_transport_header(skb) - skb->head;
	skb->csum_offset = offsetof(struct tcphdr, check);
	skb_set_queue_mapping(skb, 0);
	skb_shinfo(skb)->gso_size = GSO_SIZE;
	skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(PAYLOAD_LEN, GSO_SIZE);
	skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;

	return skb;
}

static int tx_test_thread(void *data)
{
	struct netdev_queue *txq = netdev_get_tx_queue(test_dev, 0);
	unsigned long stopped_since = 0;
	struct sk_buff *skb;
	u32 sequence = 1;
	int ret;

	while (!kthread_should_stop()) {
		/* the driver stops the queue while its TX ring is full */
		if (netif_tx_queue_stopped(txq)) {
			if (!stopped_since) {
				stopped_since = jiffies;
			} else if (time_after(jiffies, stopped_since +
					      msecs_to_jiffies(STOP_MS))) {
				netdev_warn(test_dev,
					    "tx test: queue 0 remained stopped for %u ms after %u packets\n",
					    STOP_MS, sequence - 1);
				break;
			}
			usleep_range(1000, 2000);
			continue;
		}
		stopped_since = 0;

		skb = tx_test_build_skb(test_dev, sequence);
		if (!skb) {
			netdev_err(test_dev, "tx test: skb allocation failed\n");
			break;
		}

		ret = dev_queue_xmit(skb);
		if (ret != NET_XMIT_SUCCESS && ret != NET_XMIT_CN) {
			netdev_err(test_dev,
				   "tx test: dev_queue_xmit failed after %u packets: %d\n",
				   sequence - 1, ret);
			break;
		}

		sequence++;
		cond_resched();
	}

	netdev_info(test_dev, "tx test: sender stopped after %u packets\n",
		    sequence - 1);
	while (!kthread_should_stop())
		msleep_interruptible(1000);
	return 0;
}

static int __init tx_test_init(void)
{
	int ret;

	test_dev = dev_get_by_name(&init_net, IFNAME);
	if (!test_dev) {
		pr_err("lan743x_tx_test: %s not found\n", IFNAME);
		return -ENODEV;
	}

	if (!netif_running(test_dev) || !netif_carrier_ok(test_dev)) {
		netdev_err(test_dev, "tx test: interface must be up with carrier\n");
		ret = -ENETDOWN;
		goto put_dev;
	}

	if (!(test_dev->features & NETIF_F_SG) ||
	    !(test_dev->features & NETIF_F_TSO) ||
	    !(test_dev->features & NETIF_F_HW_CSUM)) {
		netdev_err(test_dev,
			   "tx test: needs scatter-gather, TSO and TX checksum offload (ethtool -K %s sg on tso on tx on)\n",
			   IFNAME);
		ret = -EOPNOTSUPP;
		goto put_dev;
	}

	netdev_info(test_dev,
		    "tx test: starting header=%zu payload=%u@%u gso_size=%u\n",
		    HEADER_LEN, PAYLOAD_LEN, PAYLOAD_OFFSET, GSO_SIZE);

	test_task = kthread_run(tx_test_thread, NULL, "lan743x-tx-test");
	if (IS_ERR(test_task)) {
		ret = PTR_ERR(test_task);
		test_task = NULL;
		goto put_dev;
	}

	return 0;

put_dev:
	dev_put(test_dev);
	test_dev = NULL;
	return ret;
}

static void __exit tx_test_exit(void)
{
	if (test_task)
		kthread_stop(test_task);
	if (test_dev)
		dev_put(test_dev);
}

module_init(tx_test_init);
module_exit(tx_test_exit);

MODULE_DESCRIPTION("LAN743x TX hang reproducer (DEBUG ONLY)");
MODULE_LICENSE("GPL");