* Re: [PATCH net-next 1/2] dt-bindings: net: econet: add EN751221 ethernet
2026-09-09 21:21 ` [PATCH net-next 1/2] dt-bindings: net: econet: add EN751221 ethernet Caleb James DeLisle
@ 2026-09-09 23:09 ` Matheus Sampaio Queiroga
2026-09-10 0:40 ` Caleb James DeLisle
2026-09-10 2:11 ` Andrew Lunn
1 sibling, 1 reply; 9+ messages in thread
From: Matheus Sampaio Queiroga @ 2026-09-09 23:09 UTC (permalink / raw)
To: cjd
Cc: 25181214217, andrew+netdev, angelogioacchino.delregno, b.larsson,
conor+dt, davem, devicetree, dong100, edumazet, enelsonmoore,
han.junyang, hkallweit1, jes, krzk+dt, kuba, linux-arm-kernel,
linux-kernel, linux-mediatek, matthias.bgg, maxime.chevallier,
naseefkm, netdev, p.zabel, pabeni, robh, vadim.fedorenko,
wangruikang, xuanzhuo
Hello everyone, please excuse this email.
In my opinion, the Ethernet driver for the Econet/Airoha MIPS SoCs
should be merged with the existing Airoha driver.
i get PPE and xPON to start and work on the en7523, en751221, and en7528.
With a discussion on the Openwrt forum,
it is said that integrating the en751627, en7528, and en7580 Ethernet
drivers into the new EcoNet Ethernet driver.
These families are closer to what we already have in the Aiorha driver.
These are my local modifications
---
diff --git a/drivers/net/ethernet/airoha/Kconfig
b/drivers/net/ethernet/airoha/Kconfig
index 789906516bf8..66a7ddfed7f9 100644
--- a/drivers/net/ethernet/airoha/Kconfig
+++ b/drivers/net/ethernet/airoha/Kconfig
@@ -1,29 +1,45 @@
# SPDX-License-Identifier: GPL-2.0-only
config NET_VENDOR_AIROHA
- bool "Airoha devices"
- depends on ARCH_AIROHA || COMPILE_TEST
+ bool "Airoha/EcoNet devices"
+ depends on ARCH_AIROHA || ECONET || COMPILE_TEST
help
- If you have a Airoha SoC with ethernet, say Y.
+ Enable Ethernet drivers for the Airoha ARM and EcoNet MIPS frame
+ engines.
if NET_VENDOR_AIROHA
config NET_AIROHA_NPU
tristate "Airoha NPU support"
+ depends on ARCH_AIROHA || COMPILE_TEST
select WANT_DEV_COREDUMP
select REGMAP_MMIO
help
- This driver supports Airoha Network Processor (NPU) available
- on the Airoha Soc family.
+ Support the Network Processor Unit available on newer Airoha SoCs.
+ The Ethernet and PPE drivers also work without this module by using
+ the CPU-direct PPE path. Enable this option when NPU firmware is
+ available and PPE statistics or firmware-assisted commits are needed.
config NET_AIROHA
- tristate "Airoha SoC Gigabit Ethernet support"
- depends on NET_DSA || !NET_DSA
- select NET_AIROHA_NPU
- select PAGE_POOL
+ tristate "Airoha/EcoNet frame-engine Ethernet support"
+ depends on OF
select PHYLINK
+ select PAGE_POOL
+ imply NET_AIROHA_NPU if ARCH_AIROHA
+ help
+ Support the shared Airoha/EcoNet frame-engine Ethernet datapath,
+ including generation-1 EN751221/EN7528 and generation-2
+ EN7523/EN7581/AN7583 SoCs. The hardware generation is selected
+ from the device-tree compatible and SoC data at runtime. QDMA, GDM,
+ PPE and phylink infrastructure are built as one driver so both
+ generations follow the same lifecycle and common interfaces.
+
+config NET_AIROHA_PPE_DEBUGFS
+ bool
+ default y
+ depends on NET_AIROHA && DEBUG_FS
help
- This driver supports the gigabit ethernet MACs in the
- Airoha SoC family.
+ Build the PPE debugfs interface for the shared Airoha/EcoNet
+ Ethernet driver.
config NET_AIROHA_FLOW_STATS
default y
@@ -32,4 +48,35 @@ config NET_AIROHA_FLOW_STATS
help
Enable Airoha flowtable statistic counters.
-endif #NET_VENDOR_AIROHA
+config NET_AIROHA_SOC_WED
+ bool "Airoha WED v1 support"
+ depends on NET_AIROHA && PCI
+ select NET_MEDIATEK_WED_COMMON
+ help
+ Enable WiFi Ethernet Datapath (WED) hardware offload for the
+ EN751221, EN7528 and EN7523 SoCs. When enabled, mt76 WiFi drivers can
+ offload wireless transmit processing through the WED engine rather
+ than taking the CPU path.
+
+ If unsure, say N.
+
+config AIROHA_XPON_V1
+ tristate "Airoha/EcoNet xPON MAC driver"
+ depends on ARCH_AIROHA || ECONET || COMPILE_TEST
+ depends on NET_AIROHA
+ depends on OF
+ depends on SFP
+ depends on OPTICAL_FRONTEND || OPTICAL_FRONTEND=n
+ depends on MFD_SYSCON
+ select PHY_AIROHA_XPON
+ select XPON
+ select XPON_OMCI
+ select XPON_OAM
+ help
+ Driver for the shared xPON MAC complex found in Airoha/EcoNet SoCs.
+ GPON and EPON are implemented by one platform driver and selected by
+ the airoha,pon-mode device-tree property. GPON includes ITU-T G.984.3
+ activation/PLOAM and uses the in-kernel OMCI agent; EPON includes
+ IEEE 802.3 MPCP LLID registration and security support.
+
+endif # NET_VENDOR_AIROHA
diff --git a/drivers/net/ethernet/airoha/Makefile
b/drivers/net/ethernet/airoha/Makefile
index 94468053e34b..0d4ad98d9133 100644
--- a/drivers/net/ethernet/airoha/Makefile
+++ b/drivers/net/ethernet/airoha/Makefile
@@ -1,9 +1,15 @@
# SPDX-License-Identifier: GPL-2.0-only
#
-# Airoha for the Mediatek SoCs built-in ethernet macs
+# Airoha and EcoNet frame-engine Ethernet drivers
#
obj-$(CONFIG_NET_AIROHA) += airoha-eth.o
airoha-eth-y := airoha_eth.o airoha_ppe.o
-airoha-eth-$(CONFIG_DEBUG_FS) += airoha_ppe_debugfs.o
+airoha-eth-$(CONFIG_NET_AIROHA_PPE_DEBUGFS) += airoha_ppe_debugfs.o
+airoha-eth-$(CONFIG_NET_AIROHA_SOC_WED) += airoha_wed.o
airoha_wed_debugfs.o
+
obj-$(CONFIG_NET_AIROHA_NPU) += airoha_npu.o
+
+# Airoha/EcoNet shared GPON + EPON MAC complex.
+obj-$(CONFIG_AIROHA_XPON_V1) += airoha-xpon.o
+airoha-xpon-y := airoha_xpon.o airoha_ploam.o airoha_gpon_omci.o
diff --git a/drivers/net/ethernet/airoha/airoha_eth.c
b/drivers/net/ethernet/airoha/airoha_eth.c
index b37b42592f40..688beff747cd 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.c
+++ b/drivers/net/ethernet/airoha/airoha_eth.c
@@ -2,1380 +2,1969 @@
/*
* Copyright (c) 2024 AIROHA Inc
* Author: Lorenzo Bianconi <lorenzo@kernel.org>
+ * Author: Caleb James DeLisle <cjd@cjdns.fr>
+ * Author: Matheus Sampaio Queiroga <srherobrine20@gmail.com>
*/
+
+#include <linux/bitmap.h>
+#include <linux/bits.h>
+#include <linux/bitops.h>
+#include <linux/debugfs.h>
+#include <linux/dev_printk.h>
+#include <linux/dma-mapping.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/export.h>
+#include <linux/if_vlan.h>
+#include <linux/in.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/ioport.h>
+#include <linux/ipv6.h>
+#include <linux/mdio.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/netdevice.h>
#include <linux/of.h>
#include <linux/of_net.h>
#include <linux/of_reserved_mem.h>
+#include <linux/pcs/pcs.h>
#include <linux/platform_device.h>
+#include <linux/phylink.h>
+#include <linux/printk.h>
+#include <linux/ratelimit.h>
+#include <linux/rcupdate.h>
+#include <linux/reset.h>
+#include <linux/rtnetlink.h>
+#include <linux/seq_file.h>
+#include <linux/skbuff.h>
+#include <linux/slab.h>
+#include <linux/sizes.h>
+#include <linux/spinlock.h>
#include <linux/tcp.h>
-#include <linux/if_vlan.h>
-#include <linux/pcs/pcs.h>
+#include <linux/types.h>
#include <linux/u64_stats_sync.h>
+#include <linux/udp.h>
+#include <linux/unaligned.h>
+#include <net/dsa.h>
#include <net/dst_metadata.h>
#include <net/ip6_checksum.h>
+#include <net/net_namespace.h>
#include <net/page_pool/helpers.h>
#include <net/pkt_cls.h>
#include <net/tcp.h>
#include <uapi/linux/ppp_defs.h>
-#include "airoha_regs.h"
#include "airoha_eth.h"
+#include "airoha_regs.h"
+#include "airoha_wed.h"
+
+static void econet_prepare_qdma_cfg(struct airoha_qdma_mips_cfg *cfg,
+ const struct airoha_eth_soc_data *soc,
+ int id);
+
+/*
------------------------------------------------------------------------- */
+/* Common Ethernet helpers shared by all supported frame engines. */
u32 airoha_rr(void __iomem *base, u32 offset)
{
return readl(base + offset);
}
+EXPORT_SYMBOL_GPL(airoha_rr);
void airoha_wr(void __iomem *base, u32 offset, u32 val)
{
writel(val, base + offset);
}
+EXPORT_SYMBOL_GPL(airoha_wr);
u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val)
{
- val |= (airoha_rr(base, offset) & ~mask);
+ val |= airoha_rr(base, offset) & ~mask;
airoha_wr(base, offset, val);
return val;
}
+EXPORT_SYMBOL_GPL(airoha_rmw);
-static void airoha_qdma_set_irqmask(struct airoha_irq_bank *irq_bank,
- int index, u32 clear, u32 set)
+int airoha_eth_set_dma_mask(struct device *dev)
{
- struct airoha_qdma *qdma = irq_bank->qdma;
- int bank = irq_bank - &qdma->irq_banks[0];
- unsigned long flags;
-
- if (WARN_ON_ONCE(index >= ARRAY_SIZE(irq_bank->irqmask)))
- return;
-
- spin_lock_irqsave(&irq_bank->irq_lock, flags);
+ int err;
- irq_bank->irqmask[index] &= ~clear;
- irq_bank->irqmask[index] |= set;
- airoha_qdma_wr(qdma, REG_INT_ENABLE(bank, index),
- irq_bank->irqmask[index]);
- /* Read irq_enable register in order to guarantee the update above
- * completes in the spinlock critical section.
- */
- airoha_qdma_rr(qdma, REG_INT_ENABLE(bank, index));
+ err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+ if (err)
+ return dev_err_probe(dev, err,
+ "failed configuring 32-bit DMA mask\n");
- spin_unlock_irqrestore(&irq_bank->irq_lock, flags);
+ return 0;
}
+EXPORT_SYMBOL_GPL(airoha_eth_set_dma_mask);
-static void airoha_qdma_irq_enable(struct airoha_irq_bank *irq_bank,
- int index, u32 mask)
+int airoha_eth_get_port_id(struct device *dev, struct device_node *np,
+ u32 min, u32 max, u32 *id)
{
- airoha_qdma_set_irqmask(irq_bank, index, 0, mask);
-}
+ int err;
-static void airoha_qdma_irq_disable(struct airoha_irq_bank *irq_bank,
- int index, u32 mask)
-{
- airoha_qdma_set_irqmask(irq_bank, index, mask, 0);
+ err = of_property_read_u32(np, "reg", id);
+ if (err)
+ return dev_err_probe(dev, err, "missing port id\n");
+ if (*id < min || *id > max)
+ return dev_err_probe(dev, -EINVAL,
+ "invalid port id: %u\n", *id);
+
+ return 0;
}
+EXPORT_SYMBOL_GPL(airoha_eth_get_port_id);
-static int airoha_set_macaddr(struct airoha_gdm_dev *dev, const u8 *addr)
+struct net_device *airoha_eth_alloc_napi_dev(const char *name)
{
- u8 ref_addr[ETH_ALEN] __aligned(2);
- struct airoha_eth *eth = dev->eth;
- u32 reg, val, lmin, lmax;
- int i;
-
- eth_zero_addr(ref_addr);
- lmin = (addr[3] << 16) | (addr[4] << 8) | addr[5];
- lmax = lmin;
-
- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
- struct airoha_gdm_port *port = eth->ports[i];
- int j;
-
- if (!port)
- continue;
-
- for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
- struct airoha_gdm_dev *iter_dev;
- struct net_device *netdev;
-
- iter_dev = port->devs[j];
- if (!iter_dev || iter_dev == dev)
- continue;
+ struct net_device *netdev;
- if (airoha_is_lan_gdm_dev(iter_dev) !=
- airoha_is_lan_gdm_dev(dev))
- continue;
+ netdev = alloc_netdev_dummy(0);
+ if (!netdev)
+ return NULL;
- netdev = netdev_from_priv(iter_dev);
- if (netdev->reg_state != NETREG_REGISTERED)
- continue;
+ netdev->threaded = true;
+ strscpy(netdev->name, name, sizeof(netdev->name));
- ether_addr_copy(ref_addr, netdev->dev_addr);
- val = (netdev->dev_addr[3] << 16) |
- (netdev->dev_addr[4] << 8) | netdev->dev_addr[5];
- if (val < lmin)
- lmin = val;
- if (val > lmax)
- lmax = val;
- }
- }
+ return netdev;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_alloc_napi_dev);
- if (!is_zero_ether_addr(ref_addr) && memcmp(ref_addr, addr, 3)) {
- /* According to the HW design, hw mac address MSBs must be
- * the same for each net_device with the same LAN/WAN
- * configuration.
- */
- struct net_device *netdev = netdev_from_priv(dev);
+int airoha_eth_init_mac_address(struct device *dev, struct device_node *np,
+ struct net_device *netdev)
+{
+ int err;
- dev_warn(eth->dev,
- "%s: wrong mac addr, MSBs must be %02x:%02x:%02x\n",
- netdev->name, ref_addr[0], ref_addr[1],
- ref_addr[2]);
- dev_warn(eth->dev, "FE hw forwarding won't work properly\n");
+ err = of_get_ethdev_address(np, netdev);
+ if (!err)
+ return 0;
+ if (err == -EPROBE_DEFER)
+ return err;
- return -EINVAL;
- }
+ eth_hw_addr_random(netdev);
+ dev_info(dev, "generated random MAC address %pM\n", netdev->dev_addr);
- reg = airoha_is_lan_gdm_dev(dev) ? REG_FE_LAN_MAC_H : REG_FE_WAN_MAC_H;
- val = (addr[0] << 16) | (addr[1] << 8) | addr[2];
- airoha_fe_wr(eth, reg, val);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_init_mac_address);
- airoha_fe_wr(eth, REG_FE_MAC_LMIN(reg), lmin);
- airoha_fe_wr(eth, REG_FE_MAC_LMAX(reg), lmax);
+void airoha_eth_get_drvinfo(struct net_device *netdev,
+ struct ethtool_drvinfo *info)
+{
+ struct device *parent = netdev->dev.parent;
- airoha_ppe_init_upd_mem(dev, addr);
+ if (!parent)
+ return;
- return 0;
+ if (parent->driver)
+ strscpy(info->driver, parent->driver->name,
+ sizeof(info->driver));
+ strscpy(info->bus_info, dev_name(parent), sizeof(info->bus_info));
}
+EXPORT_SYMBOL_GPL(airoha_eth_get_drvinfo);
-static void airoha_set_gdm_port_fwd_cfg(struct airoha_eth *eth, u32 addr,
- u32 val)
+static void airoha_gdm_mac_config(struct phylink_config *config,
+ unsigned int mode,
+ const struct phylink_link_state *state)
{
- airoha_fe_rmw(eth, addr, GDM_OCFQ_MASK,
- FIELD_PREP(GDM_OCFQ_MASK, val));
- airoha_fe_rmw(eth, addr, GDM_MCFQ_MASK,
- FIELD_PREP(GDM_MCFQ_MASK, val));
- airoha_fe_rmw(eth, addr, GDM_BCFQ_MASK,
- FIELD_PREP(GDM_BCFQ_MASK, val));
- airoha_fe_rmw(eth, addr, GDM_UCFQ_MASK,
- FIELD_PREP(GDM_UCFQ_MASK, val));
+ struct airoha_gdm_common *gdm;
+
+ gdm = container_of(config, struct airoha_gdm_common, phylink_config);
+ if (gdm->mac_ops && gdm->mac_ops->mac_config)
+ gdm->mac_ops->mac_config(gdm->priv, mode, state);
}
-static int airoha_set_vip_for_gdm_port(struct airoha_gdm_dev *dev, bool
enable)
+static void airoha_gdm_mac_link_up(struct phylink_config *config,
+ struct phy_device *phy,
+ unsigned int mode,
+ phy_interface_t interface,
+ int speed, int duplex,
+ bool tx_pause, bool rx_pause)
{
- struct airoha_gdm_port *port = dev->port;
- struct airoha_eth *eth = dev->eth;
- u32 vip_port;
+ struct airoha_gdm_common *gdm;
- vip_port = eth->soc->ops.get_vip_port(port, dev->nbq);
- if (enable) {
- airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port);
- airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port);
- } else {
- airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, vip_port);
- airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, vip_port);
- }
-
- return 0;
+ gdm = container_of(config, struct airoha_gdm_common, phylink_config);
+ if (gdm->mac_ops && gdm->mac_ops->mac_link_up)
+ gdm->mac_ops->mac_link_up(gdm->priv, phy, mode, interface,
+ speed, duplex, tx_pause, rx_pause);
}
-static void airoha_fe_maccr_init(struct airoha_eth *eth)
+static void airoha_gdm_mac_link_down(struct phylink_config *config,
+ unsigned int mode,
+ phy_interface_t interface)
{
- int p;
+ struct airoha_gdm_common *gdm;
- for (p = 1; p <= ARRAY_SIZE(eth->ports); p++) {
- airoha_fe_set(eth, REG_GDM_FWD_CFG(p),
- GDM_TCP_CKSUM_MASK | GDM_UDP_CKSUM_MASK |
- GDM_IP4_CKSUM_MASK | GDM_DROP_CRC_ERR_MASK);
- airoha_fe_rmw(eth, REG_GDM_LEN_CFG(p),
- GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK,
- FIELD_PREP(GDM_SHORT_LEN_MASK, 60) |
- FIELD_PREP(GDM_LONG_LEN_MASK, AIROHA_MAX_RX_SIZE));
- }
+ gdm = container_of(config, struct airoha_gdm_common, phylink_config);
+ if (gdm->mac_ops && gdm->mac_ops->mac_link_down)
+ gdm->mac_ops->mac_link_down(gdm->priv, mode, interface);
+}
- airoha_fe_rmw(eth, REG_CDM_VLAN_CTRL(1), CDM_VLAN_MASK,
- FIELD_PREP(CDM_VLAN_MASK, 0x8100));
+static const struct phylink_mac_ops airoha_gdm_phylink_ops = {
+ .mac_config = airoha_gdm_mac_config,
+ .mac_link_up = airoha_gdm_mac_link_up,
+ .mac_link_down = airoha_gdm_mac_link_down,
+};
- airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PAD);
+void airoha_gdm_common_init(struct airoha_gdm_common *gdm,
+ struct airoha_eth *eth, struct net_device *netdev,
+ enum airoha_eth_family family, u8 id,
+ u8 pse_port, void *priv,
+ const struct airoha_gdm_mac_ops *mac_ops)
+{
+ memset(gdm, 0, sizeof(*gdm));
+ gdm->magic = AIROHA_GDM_COMMON_MAGIC;
+ gdm->family = family;
+ gdm->id = id;
+ gdm->pse_port = pse_port;
+ gdm->eth = eth;
+ gdm->netdev = netdev;
+ gdm->priv = priv;
+ gdm->mac_ops = mac_ops;
+ gdm->phylink_config.dev = &netdev->dev;
+ gdm->phylink_config.type = PHYLINK_NETDEV;
}
-static void airoha_fe_vip_setup(struct airoha_eth *eth)
+int airoha_gdm_phylink_create(struct airoha_gdm_common *gdm,
+ struct device_node *np,
+ phy_interface_t phy_mode)
{
- airoha_fe_wr(eth, REG_FE_VIP_PATN(3), ETH_P_PPP_DISC);
- airoha_fe_wr(eth, REG_FE_VIP_EN(3), PATN_FCPU_EN_MASK | PATN_EN_MASK);
-
- airoha_fe_wr(eth, REG_FE_VIP_PATN(4), PPP_LCP);
- airoha_fe_wr(eth, REG_FE_VIP_EN(4),
- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
- PATN_EN_MASK);
-
- airoha_fe_wr(eth, REG_FE_VIP_PATN(6), PPP_IPCP);
- airoha_fe_wr(eth, REG_FE_VIP_EN(6),
- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
- PATN_EN_MASK);
-
- airoha_fe_wr(eth, REG_FE_VIP_PATN(7), PPP_CHAP);
- airoha_fe_wr(eth, REG_FE_VIP_EN(7),
- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
- PATN_EN_MASK);
+ struct phylink *phylink;
- /* BOOTP (0x43) */
- airoha_fe_wr(eth, REG_FE_VIP_PATN(8), 0x43);
- airoha_fe_wr(eth, REG_FE_VIP_EN(8),
- PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
- FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+ phylink = phylink_create(&gdm->phylink_config,
+ of_fwnode_handle(np), phy_mode,
+ &airoha_gdm_phylink_ops);
+ if (IS_ERR(phylink))
+ return PTR_ERR(phylink);
- /* BOOTP (0x44) */
- airoha_fe_wr(eth, REG_FE_VIP_PATN(9), 0x44);
- airoha_fe_wr(eth, REG_FE_VIP_EN(9),
- PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
- FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+ gdm->phylink = phylink;
+ return 0;
+}
- /* ISAKMP */
- airoha_fe_wr(eth, REG_FE_VIP_PATN(10), 0x1f401f4);
- airoha_fe_wr(eth, REG_FE_VIP_EN(10),
- PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
- FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+int airoha_gdm_phylink_connect(struct airoha_gdm_common *gdm,
+ bool allow_no_phy)
+{
+ int err;
- airoha_fe_wr(eth, REG_FE_VIP_PATN(11), PPP_IPV6CP);
- airoha_fe_wr(eth, REG_FE_VIP_EN(11),
- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
- PATN_EN_MASK);
+ if (!gdm->phylink)
+ return -ENODEV;
- /* DHCPv6 */
- airoha_fe_wr(eth, REG_FE_VIP_PATN(12), 0x2220223);
- airoha_fe_wr(eth, REG_FE_VIP_EN(12),
- PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
- FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+ err = phylink_of_phy_connect(gdm->phylink,
+ gdm->netdev->dev.of_node, 0);
+ if (err) {
+ if (allow_no_phy && err == -ENODEV)
+ return 0;
+ return err;
+ }
- airoha_fe_wr(eth, REG_FE_VIP_PATN(19), PPP_PAP);
- airoha_fe_wr(eth, REG_FE_VIP_EN(19),
- PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
- PATN_EN_MASK);
+ phylink_start(gdm->phylink);
+ gdm->phylink_started = true;
+ return 0;
+}
- /* ETH->ETH_P_1905 (0x893a) */
- airoha_fe_wr(eth, REG_FE_VIP_PATN(20), 0x893a);
- airoha_fe_wr(eth, REG_FE_VIP_EN(20),
- PATN_FCPU_EN_MASK | PATN_EN_MASK);
+void airoha_gdm_phylink_disconnect(struct airoha_gdm_common *gdm)
+{
+ if (!gdm->phylink_started)
+ return;
- airoha_fe_wr(eth, REG_FE_VIP_PATN(21), ETH_P_LLDP);
- airoha_fe_wr(eth, REG_FE_VIP_EN(21),
- PATN_FCPU_EN_MASK | PATN_EN_MASK);
+ phylink_stop(gdm->phylink);
+ phylink_disconnect_phy(gdm->phylink);
+ gdm->phylink_started = false;
}
-static u32 airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth *eth,
- u32 port, u32 queue)
+void airoha_gdm_phylink_destroy(struct airoha_gdm_common *gdm)
{
- airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
- PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK,
- FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
- FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue));
+ airoha_gdm_phylink_disconnect(gdm);
+ if (!gdm->phylink)
+ return;
- return airoha_fe_get(eth, REG_FE_PSE_QUEUE_CFG_VAL,
- PSE_CFG_OQ_RSV_MASK);
+ phylink_destroy(gdm->phylink);
+ gdm->phylink = NULL;
}
-static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth,
- u32 port, u32 queue, u32 val)
+
+/*
------------------------------------------------------------------------- */
+/* QDMA common helpers and hardware-specific implementations. */
+
+void airoha_qdma_setup(struct airoha_qdma *qdma, struct airoha_eth *eth,
+ void __iomem *regs, u8 id, u8 num_channels)
{
- airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_VAL, PSE_CFG_OQ_RSV_MASK,
- FIELD_PREP(PSE_CFG_OQ_RSV_MASK, val));
- airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
- PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK |
- PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK,
- FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
- FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue) |
- PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK);
+ qdma->eth = eth;
+ qdma->regs = regs;
+ qdma->id = id;
+ qdma->num_channels = num_channels;
}
-static u32 airoha_fe_get_pse_all_rsv(struct airoha_eth *eth)
+
+static void airoha_qdma_skb_meta_init(struct airoha_qdma_skb_meta *meta)
{
- return airoha_fe_get(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK);
+ memset(meta, 0, sizeof(*meta));
+ meta->channel = AIROHA_MTK_INVALID_CHANNEL;
}
-static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth,
- u32 port, u32 queue, u32 val)
+void airoha_qdma_skb_get_mtk_meta(struct sk_buff *skb,
+ struct net_device *netdev,
+ enum airoha_mtk_tag_mode mode,
+ struct airoha_qdma_skb_meta *meta)
{
- u32 orig_val = airoha_fe_get_pse_queue_rsv_pages(eth, port, queue);
- u32 tmp, all_rsv, fq_limit;
+#if IS_ENABLED(CONFIG_NET_DSA)
+ struct ethhdr *ehdr;
+ u8 xmit_tpid;
+ u16 tag;
- airoha_fe_set_pse_queue_rsv_pages(eth, port, queue, val);
+ airoha_qdma_skb_meta_init(meta);
- /* modify all rsv */
- all_rsv = airoha_fe_get_pse_all_rsv(eth);
- all_rsv += (val - orig_val);
- airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK,
- FIELD_PREP(PSE_ALLRSV_MASK, all_rsv));
+ if (!netdev_uses_dsa(netdev) ||
+ netdev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK ||
+ skb_headlen(skb) < ETH_HLEN + AIROHA_MTK_HDR_LEN)
+ return;
- /* modify hthd */
- fq_limit = airoha_fe_get(eth, PSE_FQ_CFG, PSE_FQ_LIMIT_MASK);
- tmp = fq_limit - all_rsv - 0x20;
- airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
- PSE_SHARE_USED_HTHD_MASK,
- FIELD_PREP(PSE_SHARE_USED_HTHD_MASK, tmp));
+ tag = get_unaligned_be16(skb->data + 2 * ETH_ALEN);
+ meta->has_mtk_tag = true;
+ meta->mtk_tag = tag;
+ meta->port_mask = tag & AIROHA_MTK_STAG_PORT_MASK;
+ if (meta->port_mask)
+ meta->channel = __ffs(meta->port_mask);
- tmp = fq_limit - all_rsv - 0x100;
- airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
- PSE_SHARE_USED_MTHD_MASK,
- FIELD_PREP(PSE_SHARE_USED_MTHD_MASK, tmp));
- tmp = (3 * tmp) >> 2;
- airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET,
- PSE_SHARE_USED_LTHD_MASK,
- FIELD_PREP(PSE_SHARE_USED_LTHD_MASK, tmp));
+ if (mode == AIROHA_MTK_TAG_IN_SKB)
+ return;
- return 0;
+ if (skb_cow_head(skb, 0)) {
+ airoha_qdma_skb_meta_init(meta);
+ return;
+ }
+
+ ehdr = (struct ethhdr *)skb->data;
+ xmit_tpid = tag >> 8;
+
+ switch (xmit_tpid) {
+ case AIROHA_MTK_HDR_XMIT_TAGGED_TPID_8100:
+ ehdr->h_proto = cpu_to_be16(ETH_P_8021Q);
+ tag &= ~(AIROHA_MTK_HDR_XMIT_TAGGED_TPID_8100 << 8);
+ break;
+ case AIROHA_MTK_HDR_XMIT_TAGGED_TPID_88A8:
+ ehdr->h_proto = cpu_to_be16(ETH_P_8021AD);
+ tag &= ~(AIROHA_MTK_HDR_XMIT_TAGGED_TPID_88A8 << 8);
+ break;
+ default:
+ /* Newer Airoha QDMA carries an untagged MTK DSA header in
+ * descriptor metadata so the PPE sees the original Ethernet
+ * header. EcoNet selects AIROHA_MTK_TAG_IN_SKB instead.
+ */
+ memmove(skb->data + AIROHA_MTK_HDR_LEN, skb->data,
+ 2 * ETH_ALEN);
+ __skb_pull(skb, AIROHA_MTK_HDR_LEN);
+ break;
+ }
+
+ meta->mtk_tag = tag;
+#else
+ airoha_qdma_skb_meta_init(meta);
+#endif
}
-static void airoha_fe_pse_ports_init(struct airoha_eth *eth)
+/* QDMA datapath. */
+/* The non-dma part of RX packet descriptor */
+struct econet_q_rx_ent {
+ void *buf;
+ dma_addr_t dma_addr;
+ u16 dma_len;
+};
+
+struct econet_q_rx {
+ /* No lock, access only in NAPI, or else when NAPI is disabled
+ * and qdma->lock is held */
+ struct econet_q_rx_ent *entry;
+ struct desc *desc;
+ u16 cpu_i;
+
+ /* Not modified after init */
+ struct airoha_qdma_mips *qdma;
+ struct qchain_regs __iomem *qchain_regs;
+ int ndesc;
+ int buf_size;
+ struct napi_struct napi;
+ struct page_pool *page_pool;
+};
+
+/* The non-dma part of TX packet descriptor */
+struct econet_q_tx_ent {
+ struct sk_buff *skb;
+ dma_addr_t dma_addr;
+ u16 dma_len;
+ u16 freelist_next;
+};
+
+struct econet_q_tx {
+ /* protect concurrent queue accesses
+ * use _bh unless in napi poll */
+ spinlock_t lock_bh;
+ struct econet_q_tx_ent *entry;
+ struct desc *desc;
+
+ /* FIFO of free entries because they complete out of order. */
+ u16 freelist_head;
+ u16 freelist_tail;
+
+ /* Not modified after init */
+ struct airoha_qdma_mips *qdma;
+ struct qchain_regs __iomem *qchain_regs;
+ int ndesc;
+ struct napi_struct napi;
+};
+
+struct econet_irq {
+ /* protect concurrent irqmask accesses
+ * use _irqsave unless in irq handler */
+ spinlock_t lock_irq;
+ u32 irqmask[ECONET_QDMA_IRQ_REGS];
+ u32 __iomem *mask_reg[ECONET_QDMA_IRQ_REGS];
+ u32 __iomem *status_reg[ECONET_QDMA_IRQ_REGS];
+
+ /* Not modified after init */
+ struct airoha_qdma_mips *qdma;
+ int irq;
+};
+
+struct econet_tx_doneq {
+ /* No lock, access only in NAPI */
+ u32 *q;
+ struct qregs_doneq __iomem *regs;
+
+ /* Not modified after init */
+ struct airoha_qdma_mips *qdma;
+ int size;
+ struct napi_struct napi;
+};
+
+
+struct airoha_qdma_mips {
+ struct airoha_qdma *qdma;
+ struct mutex lock;
+ struct qregs __iomem *regs;
+
+ struct econet_irq irqs[ECONET_MAX_QDMA_IRQS];
+ struct econet_tx_doneq q_tx_done[QDMA_NUM_TX_DONE];
+ struct econet_q_tx q_tx[QDMA_NUM_CHAINS];
+ struct econet_q_rx q_rx[QDMA_NUM_CHAINS];
+
+ struct fwdesc *hwf_desc;
+ int num_fwd_descs;
+ u32 fwd_buf_size;
+ struct airoha_qdma_slm slm;
+ struct airoha_qdma_mips_cfg cfg;
+};
+
+static void econet_fill_rx_queue(struct econet_q_rx *q, u32 end_i)
{
- const u32 pse_port_num_queues[] = {
- [FE_PSE_PORT_CDM1] = 6,
- [FE_PSE_PORT_GDM1] = 6,
- [FE_PSE_PORT_GDM2] = 32,
- [FE_PSE_PORT_GDM3] = 6,
- [FE_PSE_PORT_PPE1] = 4,
- [FE_PSE_PORT_CDM2] = 6,
- [FE_PSE_PORT_CDM3] = 8,
- [FE_PSE_PORT_CDM4] = 10,
- [FE_PSE_PORT_PPE2] = 4,
- [FE_PSE_PORT_GDM4] = 2,
- [FE_PSE_PORT_CDM5] = 2,
- };
- int q;
+ u32 ndesc = q->ndesc;
+ u32 cpu_i = (q->cpu_i + 1) % ndesc;
+ int rx_offset = q->qdma->cfg.rx_2b_offset ? NET_IP_ALIGN : 0;
- if (airoha_ppe_is_enabled(eth, 1)) {
- u32 all_rsv;
+ for (; cpu_i != end_i; cpu_i = (cpu_i + 1) % ndesc) {
+ struct econet_q_rx_ent *e = &q->entry[cpu_i];
+ struct desc *pdesc = &q->desc[cpu_i];
+ struct page *page;
+ u16 pkt_len;
+ int offset;
+ int i;
- /* hw misses PPE2 oq rsv */
- all_rsv = airoha_fe_get_pse_all_rsv(eth);
- all_rsv += PSE_RSV_PAGES *
- pse_port_num_queues[FE_PSE_PORT_PPE2];
- airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK,
- FIELD_PREP(PSE_ALLRSV_MASK, all_rsv));
- }
+ page = page_pool_dev_alloc_frag(q->page_pool, &offset,
+ q->buf_size);
- /* CDM1 */
- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM1, q,
- PSE_QUEUE_RSV_PAGES);
- /* GDM1 */
- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM1]; q++)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM1, q,
- PSE_QUEUE_RSV_PAGES);
- /* GDM2 */
- for (q = 6; q < pse_port_num_queues[FE_PSE_PORT_GDM2]; q++)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM2, q, 0);
- /* GDM3 */
- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM3]; q++)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM3, q,
- PSE_QUEUE_RSV_PAGES);
- /* PPE1 */
- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE1]; q++) {
- if (q < pse_port_num_queues[FE_PSE_PORT_PPE1] / 2)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q,
- PSE_QUEUE_RSV_PAGES);
- else
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 0);
- }
- /* CDM2 */
- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM2]; q++)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM2, q,
- PSE_QUEUE_RSV_PAGES);
- /* CDM3 */
- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM3] - 1; q++)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM3, q, 0);
- /* CDM4 */
- for (q = 4; q < pse_port_num_queues[FE_PSE_PORT_CDM4]; q++)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM4, q,
- PSE_QUEUE_RSV_PAGES);
- if (airoha_ppe_is_enabled(eth, 1)) {
- /* PPE2 */
- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE2]; q++) {
- if (q < pse_port_num_queues[FE_PSE_PORT_PPE2] / 2)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2,
- q,
- PSE_QUEUE_RSV_PAGES);
- else
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2,
- q, 0);
- }
+ if (!page)
+ break;
+
+ WARN_ON_ONCE(e->dma_addr);
+ e->buf = page_address(page) + offset;
+ e->dma_addr = page_pool_get_dma_addr(page) + offset;
+ pkt_len = SKB_WITH_OVERHEAD(q->buf_size) - rx_offset;
+ e->dma_len = pkt_len + rx_offset;
+
+ WRITE_ONCE(pdesc->info, ((struct desc_info) {
+ .word = FIELD_PREP(DESC_INFO_PKT_LEN_MASK, pkt_len)
+ }));
+ WRITE_ONCE(pdesc->pkt_addr, e->dma_addr);
+ for (i = 0; i < ARRAY_SIZE(pdesc->msg.raw); i++)
+ WRITE_ONCE(pdesc->msg.raw[i], 0);
}
- /* GDM4 */
- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM4]; q++)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM4, q,
- PSE_QUEUE_RSV_PAGES);
- /* CDM5 */
- for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM5]; q++)
- airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM5, q,
- PSE_QUEUE_RSV_PAGES);
+
+ cpu_i = (cpu_i - 1) % ndesc;
+
+ q->cpu_i = cpu_i;
+ /* Publish every refilled descriptor before returning ownership to
DMA. */
+ dma_wmb();
+ econet_wreg(cpu_i, &q->qchain_regs->rx_cpui);
}
-static int airoha_fe_mc_vlan_clear(struct airoha_eth *eth)
+static void econet_qdma_rx_process_one(struct econet_q_rx *q, u32 cpu_i,
+ enum dma_data_direction dir)
{
- int i;
+ struct econet_q_rx_ent *e = &q->entry[cpu_i];
+ struct sk_buff *skb;
+ struct page *page;
+ struct desc desc;
+ u32 hash;
+ u8 sport;
+ int len;
- for (i = 0; i < AIROHA_FE_MC_MAX_VLAN_TABLE; i++) {
- int err, j;
- u32 val;
+ memcpy(&desc, &q->desc[cpu_i], sizeof(desc));
- airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
+ dma_sync_single_for_cpu(q->qdma->qdma->eth->dev, e->dma_addr,
e->dma_len, dir);
- val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
- MC_VLAN_CFG_TABLE_SEL_MASK | MC_VLAN_CFG_RW_MASK;
- airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
- err = read_poll_timeout(airoha_fe_rr, val,
- val & MC_VLAN_CFG_CMD_DONE_MASK,
- USEC_PER_MSEC, 5 * USEC_PER_MSEC,
- false, eth, REG_MC_VLAN_CFG);
- if (err)
- return err;
+ /* Not needed but a couple of WARN_ON_ONCE() check this later */
+ e->dma_addr = 0;
- for (j = 0; j < AIROHA_FE_MC_MAX_VLAN_PORT; j++) {
- airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
+ /* Make debug more readable */
+ WRITE_ONCE(q->desc[cpu_i].pkt_addr, 0);
- val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
- FIELD_PREP(MC_VLAN_CFG_PORT_ID_MASK, j) |
- MC_VLAN_CFG_RW_MASK;
- airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
- err = read_poll_timeout(airoha_fe_rr, val,
- val & MC_VLAN_CFG_CMD_DONE_MASK,
- USEC_PER_MSEC,
- 5 * USEC_PER_MSEC, false, eth,
- REG_MC_VLAN_CFG);
- if (err)
- return err;
- }
+ len = get_desc_info_pkt_len(&desc.info);
+ if (!len || len > SKB_WITH_OVERHEAD(q->buf_size) -
+ (q->qdma->cfg.rx_2b_offset ? NET_IP_ALIGN : 0))
+ goto return_page;
+
+ if (WARN_ON_ONCE(is_desc_info_nls(&desc.info)))
+ goto return_page;
+
+ skb = napi_build_skb(e->buf, q->buf_size);
+ if (!skb)
+ goto return_page;
+
+ if (q->qdma->cfg.rx_2b_offset)
+ skb_reserve(skb, NET_IP_ALIGN);
+ __skb_put(skb, len);
+ skb_mark_for_recycle(skb);
+ skb->ip_summed = CHECKSUM_NONE;
+
+ /* EN751221 QDMA1 uses PWAN_FERxMsg_T for GPON/EPON OAM. Word 0
+ * carries OAM/channel/GEM metadata while words 1..3 retain the regular
+ * FE RX layout. Divert management frames before interpreting them as
+ * Ethernet/PPE traffic.
+ */
+ if (econet_rx_xpon_oam(q->qdma->qdma->eth,
+ q->qdma->qdma->id, skb, &desc.msg))
+ return;
+
+ hash = get_erx_ppe_entry(&desc.msg.erx);
+ skb_set_hash(skb, jhash_1word(hash, 0),
+ PKT_HASH_TYPE_L4);
+ airoha_ppe_dev_check_skb_reason(q->qdma->qdma->eth->ppe_dev, skb, hash,
+ get_erx_crsn(&desc.msg.erx));
+
+ if (airoha_is(q->qdma->qdma->eth, econet_en7528))
+ sport = FIELD_GET(EN7528_ERX_SPORT_MASK,
+ desc.msg.erx.bitfield_0);
+ else
+ sport = get_erx_sport(&desc.msg.erx);
+
+ if (econet_rx_before_recv(q->qdma->qdma->eth, skb, sport)) {
+ dev_kfree_skb(skb);
+ } else {
+ if ((skb->dev->features & NETIF_F_RXCSUM) &&
+ sport != ETX_FPORT_QDMA0_CPU &&
+ sport != ETX_FPORT_QDMA1_CPU &&
+ (is_erx_ip4(&desc.msg.erx) ||
+ is_erx_ip6(&desc.msg.erx)) &&
+ !is_erx_ip4f(&desc.msg.erx) &&
+ !is_erx_l4f(&desc.msg.erx))
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+ napi_gro_receive(&q->napi, skb);
}
- return 0;
+ return;
+
+return_page:
+ page = virt_to_head_page(e->buf);
+ page_pool_put_full_page(q->page_pool, page, false);
}
-static void airoha_fe_crsn_qsel_init(struct airoha_eth *eth)
+static int econet_qdma_rx_process(struct econet_q_rx *q, int budget)
{
- /* CDM1_CRSN_QSEL */
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_22 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_22),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22),
- CDM_CRSN_QSEL_Q1));
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_08 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_08),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08),
- CDM_CRSN_QSEL_Q1));
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_21 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_21),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21),
- CDM_CRSN_QSEL_Q1));
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_24 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_24),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24),
- CDM_CRSN_QSEL_Q6));
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(1, CRSN_25 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_25),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25),
- CDM_CRSN_QSEL_Q1));
- /* CDM2_CRSN_QSEL */
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_08 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_08),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_08),
- CDM_CRSN_QSEL_Q1));
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_21 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_21),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_21),
- CDM_CRSN_QSEL_Q1));
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_22 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_22),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_22),
- CDM_CRSN_QSEL_Q1));
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_24 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_24),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_24),
- CDM_CRSN_QSEL_Q6));
- airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(2, CRSN_25 >> 2),
- CDM_CRSN_QSEL_REASON_MASK(CRSN_25),
- FIELD_PREP(CDM_CRSN_QSEL_REASON_MASK(CRSN_25),
- CDM_CRSN_QSEL_Q1));
+ enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
+ u32 hardware_i = econet_rreg(&q->qchain_regs->rx_hwi);
+ int ndesc = q->ndesc;
+ u32 cpu_i;
+ int done;
+
+ /*
+ * The device publishes descriptors before advancing RX_HWI. Pair the
+ * producer-side ordering with a DMA read barrier before consuming
them.
+ */
+ dma_rmb();
+
+ /* The stored value of cpu_i is the last entry actually handled.
+ * Whereas hardware_i is the next entry that has not yet been received.
+ */
+ cpu_i = (q->cpu_i + 1) % ndesc;
+
+ for (done = 0; done < budget && cpu_i != hardware_i; done++) {
+ econet_qdma_rx_process_one(q, cpu_i, dir);
+ cpu_i = (cpu_i + 1) % ndesc;
+ }
+
+ econet_fill_rx_queue(q, cpu_i);
+
+ return done;
}
-static void airoha_fe_lro_rxq_enable(struct airoha_eth *eth, int qdma_id,
- int lro_queue_index, int qid,
- int buf_size)
-{
- int id = qdma_id + 1;
+#define IRQ_PURPOSE(t, s, c) \
+ ((union econet_irq_purpose){ .type = IPS_ ## t, .source = IPSC_ ##
s, .chain = (c) })
- airoha_fe_rmw(eth, REG_CDM_LRO_LIMIT(id),
- CDM_LRO_AGG_NUM_MASK | CDM_LRO_AGG_SIZE_MASK,
- FIELD_PREP(CDM_LRO_AGG_SIZE_MASK, buf_size) |
- FIELD_PREP(CDM_LRO_AGG_NUM_MASK,
- AIROHA_RXQ_LRO_MAX_AGG_COUNT));
- airoha_fe_rmw(eth, REG_CDM_LRO_AGE_TIME(id),
- CDM_LRO_AGE_TIME_MASK | CDM_LRO_AGG_TIME_MASK,
- FIELD_PREP(CDM_LRO_AGE_TIME_MASK,
- AIROHA_RXQ_LRO_MAX_AGE_TIME) |
- FIELD_PREP(CDM_LRO_AGG_TIME_MASK,
- AIROHA_RXQ_LRO_MAX_AGG_TIME));
- airoha_fe_rmw(eth, REG_CDM_LRO_RXQ(id, lro_queue_index),
- LRO_RXQ_MASK(lro_queue_index),
- __field_prep(LRO_RXQ_MASK(lro_queue_index), qid));
- airoha_fe_set(eth, REG_CDM_LRO_EN(id), BIT(lro_queue_index));
+union irq_bit {
+ struct {
+ int irq_idx : 8;
+ int reg_idx : 8;
+ int bit_idx : 8;
+ int _pad : 8;
+ };
+ u32 word;
+};
+
+/* RX0/RX1/TX0/TX1 use the same low interrupt-status layout on the
+ * EcoNet QDMA variants supported here. Higher EN7516 rings are
deliberately
+ * left disabled until the Linux datapath grows those rings.
+ */
+
+static const union econet_irq_purpose ECONET_IRQ0_MAP[] = {
+ [0] = IRQ_PURPOSE(DONE, TX, 0),
+ [1] = IRQ_PURPOSE(DONE, RX, 0),
+ [2] = IRQ_PURPOSE(NO_DSCP, TX, 0),
+ [3] = IRQ_PURPOSE(NO_DSCP, RX, 0),
+ [4] = IRQ_PURPOSE(DONE, TX, 1),
+ [5] = IRQ_PURPOSE(DONE, RX, 1),
+ [6] = IRQ_PURPOSE(NO_DSCP, TX, 1),
+ [7] = IRQ_PURPOSE(NO_DSCP, RX, 1),
+ [8] = IRQ_PURPOSE(NO_DSCP, FWD, -1),
+ [9] = IRQ_PURPOSE(NO_DSCP, DONE, 0),
+ [10] = IRQ_PURPOSE(LOW_DSCP, FWD, 0),
+ [11] = IRQ_PURPOSE(OVERFLOW, UNSPEC, -1),
+ [12] = IRQ_PURPOSE(ERR_COHERENT, TX, 0),
+ [13] = IRQ_PURPOSE(ERR_COHERENT, RX, 0),
+ [14] = IRQ_PURPOSE(ERR_COHERENT, TX, 1),
+ [15] = IRQ_PURPOSE(ERR_COHERENT, RX, 1),
+ [16] = IRQ_PURPOSE(GPON_INT, UNSPEC, -1),
+ [17] = IRQ_PURPOSE(EPON_INT, UNSPEC, -1),
+ [18] = IRQ_PURPOSE(XPON_INT, UNSPEC, -1),
+};
+
+static bool econet_valid_irq_bit(union irq_bit bit)
+{
+ return bit.irq_idx == 0 && bit.reg_idx == 0 &&
+ bit.bit_idx < ARRAY_SIZE(ECONET_IRQ0_MAP);
}
-static void airoha_fe_lro_disable(struct airoha_eth *eth, int qdma_id)
+static union econet_irq_purpose econet_irq_purpose(union irq_bit bit)
{
- int i, id = qdma_id + 1;
+ if (WARN_ON_ONCE(!econet_valid_irq_bit(bit)))
+ return (union econet_irq_purpose){0};
- airoha_fe_clear(eth, REG_CDM_LRO_EN(id), LRO_RXQ_EN_MASK);
- airoha_fe_clear(eth, REG_CDM_LRO_LIMIT(id),
- CDM_LRO_AGG_NUM_MASK | CDM_LRO_AGG_SIZE_MASK);
- airoha_fe_clear(eth, REG_CDM_LRO_AGE_TIME(id),
- CDM_LRO_AGE_TIME_MASK | CDM_LRO_AGG_TIME_MASK);
- for (i = 0; i < AIROHA_MAX_NUM_LRO_QUEUES; i++)
- airoha_fe_clear(eth, REG_CDM_LRO_RXQ(id, i), LRO_RXQ_MASK(i));
+ return ECONET_IRQ0_MAP[bit.bit_idx];
}
-static bool airoha_fe_lro_is_enabled(struct airoha_eth *eth, int qdma_id)
+static union irq_bit econet_irq_bit(union econet_irq_purpose purpose)
{
- return airoha_fe_get(eth, REG_CDM_LRO_EN(qdma_id + 1),
- LRO_RXQ_EN_MASK);
+ union irq_bit bit = {0};
+ for (int i = 0; i < ARRAY_SIZE(ECONET_IRQ0_MAP); i++) {
+ if (ECONET_IRQ0_MAP[i].word == purpose.word) {
+ bit.bit_idx = i;
+ return bit;
+ }
+ }
+ return bit;
}
-static void airoha_dev_lro_enable(struct airoha_gdm_dev *dev)
+
+static u32 __iomem *econet_irq_status_reg(struct airoha_qdma_mips *qdma,
+ int bank, int reg)
{
- struct airoha_qdma *qdma = airoha_qdma_deref(dev);
- struct airoha_eth *eth = qdma->eth;
- int qdma_id = qdma - ð->qdma[0];
- int i, lro_queue_index = 0;
+ u8 __iomem *base = qdma->qdma->regs;
- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
- struct airoha_queue *q = &qdma->q_rx[i];
- u32 size;
+ if (bank >= qdma->qdma->eth->soc->irq_banks || reg >=
ECONET_QDMA_IRQ_REGS)
+ return NULL;
- if (!q->ndesc)
- continue;
+ if (reg == 0)
+ return &qdma->regs->int_status;
- if (!airoha_qdma_is_lro_queue(q))
- continue;
+ if (!airoha_is(qdma->qdma->eth, econet_en7528))
+ return NULL;
- size = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size));
- size = min_t(u32, size, CDM_LRO_AGG_SIZE_MASK);
- airoha_fe_lro_rxq_enable(eth, qdma_id, lro_queue_index, i,
- size);
- lro_queue_index++;
- }
+ return (u32 __iomem *)(base +
+ (reg == 1 ? EN7516_QDMA_INT_STATUS1_OFFSET :
+ EN7516_QDMA_INT_STATUS2_OFFSET));
}
-static int airoha_fe_init(struct airoha_eth *eth)
+static u32 __iomem *econet_irq_enable_reg(struct airoha_qdma_mips *qdma,
+ int bank, int reg)
{
- airoha_fe_maccr_init(eth);
+ u8 __iomem *base = qdma->qdma->regs;
- /* PSE IQ reserve */
- airoha_fe_rmw(eth, REG_PSE_IQ_REV1, PSE_IQ_RES1_P2_MASK,
- FIELD_PREP(PSE_IQ_RES1_P2_MASK, 0x10));
- airoha_fe_rmw(eth, REG_PSE_IQ_REV2,
- PSE_IQ_RES2_P5_MASK | PSE_IQ_RES2_P4_MASK,
- FIELD_PREP(PSE_IQ_RES2_P5_MASK, 0x40) |
- FIELD_PREP(PSE_IQ_RES2_P4_MASK, 0x34));
+ if (bank >= qdma->qdma->eth->soc->irq_banks || reg >=
ECONET_QDMA_IRQ_REGS)
+ return NULL;
- /* enable FE copy engine for KA/DPI */
- airoha_fe_wr(eth, REG_FE_PCE_CFG, PCE_DPI_EN_MASK | PCE_KA_EN_MASK);
- /* set vip queue selection to ring 1 */
- airoha_fe_rmw(eth, REG_CDM_FWD_CFG(1), CDM_VIP_QSEL_MASK,
- FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4));
- airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_VIP_QSEL_MASK,
- FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4));
- /* set GDM4 source interface offset to 8 */
- airoha_fe_rmw(eth, REG_GDM_SRC_PORT_SET(4),
- GDM_SPORT_OFF2_MASK |
- GDM_SPORT_OFF1_MASK |
- GDM_SPORT_OFF0_MASK,
- FIELD_PREP(GDM_SPORT_OFF2_MASK, 8) |
- FIELD_PREP(GDM_SPORT_OFF1_MASK, 8) |
- FIELD_PREP(GDM_SPORT_OFF0_MASK, 8));
-
- /* set PSE Page as 128B */
- airoha_fe_rmw(eth, REG_FE_DMA_GLO_CFG,
- FE_DMA_GLO_L2_SPACE_MASK | FE_DMA_GLO_PG_SZ_MASK,
- FIELD_PREP(FE_DMA_GLO_L2_SPACE_MASK, 2) |
- FE_DMA_GLO_PG_SZ_MASK);
- airoha_fe_wr(eth, REG_FE_RST_GLO_CFG,
- FE_RST_CORE_MASK | FE_RST_GDM3_MBI_ARB_MASK |
- FE_RST_GDM4_MBI_ARB_MASK);
- usleep_range(1000, 2000);
+ if (!airoha_is(qdma->qdma->eth, econet_en7528))
+ return bank == 0 && reg == 0 ? &qdma->regs->int_enable : NULL;
- /* connect RxRing1 and RxRing15 to PSE Port0 OQ-1
- * connect other rings to PSE Port0 OQ-0
+ /* EN7516/EN7527 INT1 keeps ENABLE0 at 0x54. The remaining
+ * INT1 registers and all INT2..INT4 enables live in 0x708..0x730.
*/
- airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP0, BIT(4));
- airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP1, BIT(28));
- airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP2, BIT(4));
- airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP3, BIT(28));
+ if (bank == 0 && reg == 0)
+ return &qdma->regs->int_enable;
- airoha_fe_vip_setup(eth);
- airoha_fe_pse_ports_init(eth);
+ return (u32 __iomem *)(base + EN7516_QDMA_INT_ENABLE_BASE +
+ bank * EN7516_QDMA_INT_ENABLE_STRIDE + reg * sizeof(u32));
+}
- airoha_fe_set(eth, REG_GDM_MISC_CFG,
- GDM2_RDM_ACK_WAIT_PREF_MASK |
- GDM2_CHN_VLD_MODE_MASK);
- airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2), CDM_OAM_QSEL_MASK,
- FIELD_PREP(CDM_OAM_QSEL_MASK, 15));
+/* End EcoNet QDMA IRQ layout helpers */
- /* init fragment and assemble Force Port */
- /* NPU Core-3, NPU Bridge Channel-3 */
- airoha_fe_rmw(eth, REG_IP_FRAG_FP,
- IP_FRAGMENT_PORT_MASK | IP_FRAGMENT_NBQ_MASK,
- FIELD_PREP(IP_FRAGMENT_PORT_MASK, 6) |
- FIELD_PREP(IP_FRAGMENT_NBQ_MASK, 3));
- /* QDMA LAN, RX Ring-22 */
- airoha_fe_rmw(eth, REG_IP_FRAG_FP,
- IP_ASSEMBLE_PORT_MASK | IP_ASSEMBLE_NBQ_MASK,
- FIELD_PREP(IP_ASSEMBLE_PORT_MASK, 0) |
- FIELD_PREP(IP_ASSEMBLE_NBQ_MASK, 22));
+static void econet_qdma_set_irqmask(struct airoha_qdma_mips *qdma,
union irq_bit b, bool enable)
+{
+ struct econet_irq *irq;
- airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM3_IDX),
- GDM_PAD_EN_MASK | GDM_STRIP_CRC_MASK);
- airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM4_IDX),
- GDM_PAD_EN_MASK | GDM_STRIP_CRC_MASK);
+ if (WARN_ON_ONCE(b.irq_idx >= qdma->qdma->eth->soc->irq_banks))
+ return;
- /* Enable split for MIB counters for GDM3 and GDM4 */
- airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM3_IDX),
- FE_GDM_TX_MIB_SPLIT_EN_MASK |
- FE_GDM_RX_MIB_SPLIT_EN_MASK);
- airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM4_IDX),
- FE_GDM_TX_MIB_SPLIT_EN_MASK |
- FE_GDM_RX_MIB_SPLIT_EN_MASK);
+ irq = &qdma->irqs[b.irq_idx];
- airoha_fe_crsn_qsel_init(eth);
+ if (WARN_ON_ONCE(b.reg_idx >= ARRAY_SIZE(irq->irqmask)))
+ return;
- airoha_fe_clear(eth, REG_FE_CPORT_CFG, FE_CPORT_QUEUE_XFC_MASK);
- airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PORT_XFC_MASK);
+ if (WARN_ON_ONCE(b.bit_idx >= 32))
+ return;
- /* default aging mode for mbi unlock issue */
- airoha_fe_rmw(eth, REG_GDM_CHN_RLS(2),
- MBI_RX_AGE_SEL_MASK | MBI_TX_AGE_SEL_MASK,
- FIELD_PREP(MBI_RX_AGE_SEL_MASK, 3) |
- FIELD_PREP(MBI_TX_AGE_SEL_MASK, 3));
+ guard(spinlock_irqsave)(&irq->lock_irq);
- /* disable IFC by default */
- airoha_fe_clear(eth, REG_FE_CSR_IFC_CFG, FE_IFC_EN_MASK);
+ if (enable)
+ irq->irqmask[b.reg_idx] |= BIT(b.bit_idx);
+ else
+ irq->irqmask[b.reg_idx] &= ~BIT(b.bit_idx);
- /* enable 1:N vlan action, init vlan table */
- airoha_fe_set(eth, REG_MC_VLAN_EN, MC_VLAN_EN_MASK);
+ if (WARN_ON_ONCE(!irq->mask_reg[b.reg_idx]))
+ return;
- return airoha_fe_mc_vlan_clear(eth);
+ econet_wreg(irq->irqmask[b.reg_idx], irq->mask_reg[b.reg_idx]);
+
+ /* Read irq_enable register in order to guarantee the update above
+ * completes in the spinlock critical section.
+ */
+ econet_rreg(irq->mask_reg[b.reg_idx]);
}
-static int airoha_qdma_fill_rx_queue(struct airoha_queue *q)
+static int econet_qdma_set_xpon_irq_mips(struct airoha_qdma_mips *qdma,
+ enum airoha_xpon_mode mode, bool enable)
{
- struct airoha_qdma *qdma = q->qdma;
- int qid = q - &qdma->q_rx[0];
- int nframes = 0;
+ union econet_irq_purpose purpose;
+ union irq_bit bit;
- while (q->queued < q->ndesc - 1) {
- struct airoha_queue_entry *e = &q->entry[q->head];
- struct airoha_qdma_desc *desc = &q->desc[q->head];
- struct page *page;
- int offset;
- u32 val;
+ if (!qdma || qdma->qdma->id != 1)
+ return -EINVAL;
- page = page_pool_dev_alloc_frag(q->page_pool, &offset,
- q->buf_size);
- if (!page)
- break;
+ switch (mode) {
+ case AIROHA_XPON_MODE_GPON:
+ purpose = IRQ_PURPOSE(GPON_INT, UNSPEC, -1);
+ break;
+ case AIROHA_XPON_MODE_EPON:
+ purpose = IRQ_PURPOSE(EPON_INT, UNSPEC, -1);
+ break;
+ default:
+ return -EINVAL;
+ }
- q->head = (q->head + 1) % q->ndesc;
- q->queued++;
- nframes++;
+ bit = econet_irq_bit(purpose);
+ if (!econet_valid_irq_bit(bit))
+ return -EINVAL;
- offset += AIROHA_RX_HEADROOM;
- e->buf = page_address(page) + offset;
- e->dma_addr = page_pool_get_dma_addr(page) + offset;
- e->dma_len = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size));
+ econet_qdma_set_irqmask(qdma, bit, enable);
+ if (!enable)
+ synchronize_irq(qdma->irqs[bit.irq_idx].irq);
- WRITE_ONCE(desc->tcp_ts_reply, 0);
- val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len);
- WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
- WRITE_ONCE(desc->addr, cpu_to_le32(e->dma_addr));
- val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, q->head);
- WRITE_ONCE(desc->data, cpu_to_le32(val));
- WRITE_ONCE(desc->msg0, 0);
- WRITE_ONCE(desc->msg1, 0);
- WRITE_ONCE(desc->msg2, 0);
- WRITE_ONCE(desc->msg3, 0);
- }
+ return 0;
+}
- if (nframes)
- airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid),
- RX_RING_CPU_IDX_MASK,
- FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
+static int econet_qdma_rx_napi_poll(struct napi_struct *napi, int budget)
+{
+ struct econet_q_rx *q = container_of(napi, struct econet_q_rx, napi);
+ struct airoha_qdma_mips *qdma = q->qdma;
+ int cur, done = 0;
- return nframes;
+ do {
+ cur = econet_qdma_rx_process(q, budget - done);
+ done += cur;
+ } while (cur && done < budget);
+
+ if (done < budget && napi_complete(napi)) {
+ union econet_irq_purpose purpose;
+ union irq_bit b;
+
+ purpose = IRQ_PURPOSE(DONE, RX, q - &qdma->q_rx[0]);
+ b = econet_irq_bit(purpose);
+ econet_qdma_set_irqmask(qdma, b, true);
+ }
+
+ return done;
}
-static struct airoha_gdm_dev *
-airoha_qdma_get_gdm_dev(struct airoha_eth *eth, struct airoha_qdma_desc
*desc)
+static irqreturn_t econet_irq_handler(int irq_num, void *dev_instance)
{
- struct airoha_gdm_port *port;
- u16 p, d;
+ struct econet_irq *irq = dev_instance;
+ struct airoha_qdma_mips *qdma = irq->qdma;
+ unsigned long xpon_pending = 0;
+ int i;
- if (eth->soc->ops.get_dev_from_sport(desc, &p, &d))
- return ERR_PTR(-ENODEV);
+ {
+ guard(spinlock)(&irq->lock_irq);
- if (p >= ARRAY_SIZE(eth->ports))
- return ERR_PTR(-ENODEV);
+ for (i = 0; i < ARRAY_SIZE(irq->irqmask); i++) {
+ unsigned long regval;
+ u32 disable_int = 0;
+ u8 bit;
- port = eth->ports[p];
- if (!port)
- return ERR_PTR(-ENODEV);
+ if (!irq->status_reg[i] || !irq->mask_reg[i])
+ continue;
- if (d >= ARRAY_SIZE(port->devs))
- return ERR_PTR(-ENODEV);
+ regval = econet_rreg(irq->status_reg[i]);
+ regval &= irq->irqmask[i];
- return port->devs[d] ? port->devs[d] : ERR_PTR(-ENODEV);
-}
+ /* You must write the bits back to the status register
+ * or you will keep receiving the same interrupt.
+ */
+ econet_wreg((u32)regval, irq->status_reg[i]);
-static struct sk_buff *airoha_qdma_lro_rx_skb(struct airoha_queue *q,
- struct airoha_qdma_desc *desc,
- struct airoha_queue_entry *e)
-{
- u32 len, th_off, tcp_ack_seq, agg_count, data_off, data_len;
- u32 desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
- u32 msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
- u32 msg2 = le32_to_cpu(READ_ONCE(desc->msg2));
- u32 msg3 = le32_to_cpu(READ_ONCE(desc->msg3));
- struct skb_shared_info *shinfo;
- u16 tcp_win, l2_len;
- struct sk_buff *skb;
- struct tcphdr *th;
- struct page *page;
- bool ipv4, ipv6;
+ for_each_set_bit(bit, ®val, 32) {
+ union econet_irq_purpose p;
- ipv4 = FIELD_GET(QDMA_ETH_RXMSG_IP4_MASK, msg1);
- ipv6 = FIELD_GET(QDMA_ETH_RXMSG_IP6_MASK, msg1);
- if (!ipv4 && !ipv6)
- return NULL;
+ p = econet_irq_purpose((union irq_bit){
+ .irq_idx = irq - &qdma->irqs[0],
+ .reg_idx = i,
+ .bit_idx = bit,
+ });
- l2_len = FIELD_GET(QDMA_ETH_RXMSG_L2_LEN_MASK, msg2);
- len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
+ if (p.type == IPS_DONE && p.source == IPSC_RX) {
+ napi_schedule_irqoff(&qdma->q_rx[p.chain].napi);
+ disable_int |= BIT(bit);
+ continue;
+ }
+ if (p.type == IPS_DONE && p.source == IPSC_TX) {
+ napi_schedule_irqoff(&qdma->q_tx_done[i].napi);
+ disable_int |= BIT(bit);
+ continue;
+ }
+ if (p.type == IPS_GPON_INT) {
+ xpon_pending |= BIT(AIROHA_XPON_MODE_GPON);
+ continue;
+ }
+ if (p.type == IPS_EPON_INT) {
+ xpon_pending |= BIT(AIROHA_XPON_MODE_EPON);
+ continue;
+ }
+ dev_dbg_ratelimited(qdma->qdma->eth->dev,
+ "%s IRQ from %s[%d]\n",
+ econet_irq_purpose_type_str(p.type),
+ econet_irq_purpose_source_str(p.source),
+ p.chain);
+ }
- if (ipv4) {
- struct iphdr *iph;
+ irq->irqmask[i] &= ~disable_int;
+ econet_wreg(irq->irqmask[i], irq->mask_reg[i]);
+ }
+ }
- if (len < l2_len + sizeof(*iph))
- return NULL;
+ /* The EN751221 vendor stack registers GPON/EPON MAC handlers through
+ * QDMA_WAN. Invoke the MAC after acknowledging the QDMA
aggregator and
+ * after dropping irq->lock_irq; the MAC ISR performs its own W1C
and FIFO
+ * drain operations.
+ */
+ if (xpon_pending & BIT(AIROHA_XPON_MODE_GPON))
+ econet_xpon_irq(qdma->qdma->eth, qdma->qdma->id,
+ AIROHA_XPON_MODE_GPON);
+ if (xpon_pending & BIT(AIROHA_XPON_MODE_EPON))
+ econet_xpon_irq(qdma->qdma->eth, qdma->qdma->id,
+ AIROHA_XPON_MODE_EPON);
- iph = (struct iphdr *)(e->buf + l2_len);
- if (iph->protocol != IPPROTO_TCP)
- return NULL;
+ return IRQ_HANDLED;
+}
- if (iph->ihl < 5)
- return NULL;
+static int econet_poll_tx_complete(struct napi_struct *napi, int budget)
+{
+ struct qregs_doneq_state state;
+ struct econet_tx_doneq *done_q;
+ struct airoha_qdma_mips *qdma;
+ int irq_queued;
+ u32 done = 0, head;
+
+ done_q = container_of(napi, struct econet_tx_doneq, napi);
+ qdma = done_q->qdma;
+
+ state = econet_rreg(&done_q->regs->state);
+ head = get_qregs_doneq_state_head_index(&state);
+ head = head % done_q->size;
+ irq_queued = get_qregs_doneq_state_length(&state);
+ /* The done queue entries/descriptors precede the producer state
update. */
+ dma_rmb();
- th_off = l2_len + (iph->ihl << 2);
- if (len < th_off)
- return NULL;
+ while (irq_queued > 0 && done < budget) {
+ u32 index, qid, val = done_q->q[head];
+ struct econet_q_tx_ent *e;
+ struct econet_q_tx *q;
+ struct netdev_queue *txq;
+ struct sk_buff *skb;
+ struct desc desc;
- iph->tot_len = cpu_to_be16(len - l2_len);
- iph->check = 0;
- iph->check = ip_fast_csum((void *)iph, iph->ihl);
- } else {
- struct ipv6hdr *ip6h;
+ if (val == 0xffffffffU)
+ break;
- th_off = l2_len + sizeof(*ip6h);
- if (len < th_off)
- return NULL;
+ done_q->q[head] = 0xffffffffU; /* mark as done */
+ head = (head + 1) % done_q->size;
+ irq_queued--;
+ done++;
- ip6h = (struct ipv6hdr *)(e->buf + l2_len);
- if (ip6h->nexthdr != NEXTHDR_TCP)
- return NULL;
+ qid = FIELD_GET(EN751221_QDMA_IRQ_RING_IDX_MASK, val);
+ if (WARN_ON_ONCE(qid >= ARRAY_SIZE(qdma->q_tx)))
+ continue;
- ip6h->payload_len = cpu_to_be16(len - th_off);
- }
+ q = &qdma->q_tx[qid];
+ if (WARN_ON_ONCE(!q->ndesc))
+ continue;
- if (len < th_off + sizeof(*th))
- return NULL;
+ index = FIELD_GET(EN751221_QDMA_IRQ_DESC_IDX_MASK, val);
+ if (WARN_ON_ONCE(index >= q->ndesc))
+ continue;
- th = (struct tcphdr *)(e->buf + th_off);
- if (th->doff < 5)
- return NULL;
+ guard(spinlock)(&q->lock_bh);
- data_off = th_off + (th->doff << 2);
- if (len < data_off)
- return NULL;
+ desc = q->desc[index];
- tcp_win = FIELD_GET(QDMA_ETH_RXMSG_TCP_WIN_MASK, msg3);
- tcp_ack_seq = le32_to_cpu(READ_ONCE(desc->data));
- th->ack_seq = cpu_to_be32(tcp_ack_seq);
- th->window = cpu_to_be16(tcp_win);
+ if (WARN_ON_ONCE(!is_desc_info_done(&desc.info) &&
+ !is_desc_info_dropped(&desc.info)))
+ continue;
- /* Check tcp timestamp option */
- if (th->doff == (sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4) {
- u32 topt = get_unaligned_be32(th + 1);
+ e = &q->entry[index];
+ skb = e->skb;
- if (topt == ((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
- (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) {
- u8 *ptr = (u8 *)th + sizeof(*th) + 2 * sizeof(__be32);
- __le32 tcp_ts_reply = READ_ONCE(desc->tcp_ts_reply);
+ dma_unmap_single(qdma->qdma->eth->dev, e->dma_addr, e->dma_len,
+ DMA_TO_DEVICE);
+ memset(e, 0, sizeof(*e));
- put_unaligned_be32(le32_to_cpu(tcp_ts_reply), ptr);
+ /* Completion ring can report out of order when hw QoS is
+ * enabled and packets with different priority are queued
+ * to same DMA ring. So we use a linked list to maintain free
+ * entries.
+ */
+ e->freelist_next = 0xffff;
+ q->entry[q->freelist_tail].freelist_next = index;
+ q->freelist_tail = index;
+
+ txq = netdev_get_tx_queue(skb->dev,
+ skb_get_queue_mapping(skb));
+ netdev_tx_completed_queue(txq, 1, skb->len);
+ if (netif_tx_queue_stopped(txq))
+ netif_tx_wake_queue(txq);
+
+ dev_kfree_skb_any(skb);
+ }
+
+ if (done) {
+ int i, len = done >> 7;
+
+ for (i = 0; i < len; i++) {
+ econet_rreg(&qdma->regs->done_queue.pop_back);
+ econet_wreg(0x80U, &qdma->regs->done_queue.pop_back);
}
+ econet_rreg(&qdma->regs->done_queue.pop_back);
+ econet_wreg(done & 0x7f, &qdma->regs->done_queue.pop_back);
}
- if (ipv4) {
- struct iphdr *iph = (struct iphdr *)(e->buf + l2_len);
+ if (done < budget && napi_complete(napi)) {
+ int chain;
- th->check = ~tcp_v4_check(len - th_off, iph->saddr,
- iph->daddr, 0);
- } else {
- struct ipv6hdr *ip6h = (struct ipv6hdr *)(e->buf + l2_len);
+ /*
+ * EN751221 has a single completion list shared by TX0 and TX1.
+ * Either DONE source can therefore schedule this NAPI instance.
+ * Re-arm both sources after draining it; re-arming only TX0 leaves
+ * TX1 permanently masked after its first interrupt.
+ */
+ for (chain = 0; chain < QDMA_NUM_CHAINS; chain++) {
+ union econet_irq_purpose purpose;
+ union irq_bit b;
- th->check = ~tcp_v6_check(len - th_off, &ip6h->saddr,
- &ip6h->daddr, 0);
+ purpose = IRQ_PURPOSE(DONE, TX, chain);
+ b = econet_irq_bit(purpose);
+ econet_qdma_set_irqmask(qdma, b, true);
+ }
}
- skb = napi_alloc_skb(&q->napi, data_off);
- if (!skb)
- return NULL;
+ return done;
+}
- __skb_put(skb, data_off);
- memcpy(skb->data, e->buf, data_off);
+static int econet_qdma_xmit_mips(struct airoha_qdma_mips *qdma, struct
sk_buff *skb,
+ union desc_msg *msg, int qid)
+{
+ struct econet_q_tx *q = &qdma->q_tx[qid];
+ int len = skb_headlen(skb);
+ struct econet_q_tx_ent *e;
+ u16 index, next_index;
+ struct desc *desc;
+ dma_addr_t addr;
+ int ret;
+
+ guard(spinlock_bh)(&q->lock_bh);
+
+ index = q->freelist_head;
+ if (index == 0xffff)
+ return -EBUSY;
- page = virt_to_head_page(e->buf);
- data_len = len - data_off;
- shinfo = skb_shinfo(skb);
- skb_add_rx_frag(skb, shinfo->nr_frags, page,
- e->buf + data_off - page_address(page), data_len,
- q->buf_size);
+ e = &q->entry[index];
+ next_index = e->freelist_next;
+ if (next_index == 0xffff)
+ return -EBUSY;
- shinfo->gso_type = ipv4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
- agg_count = FIELD_GET(QDMA_ETH_RXMSG_AGG_COUNT_MASK, msg2);
- shinfo->gso_size = DIV_ROUND_UP(data_len, agg_count);
- shinfo->gso_segs = agg_count;
+ addr = dma_map_single(qdma->qdma->eth->dev, skb->data, len,
DMA_TO_DEVICE);
+ ret = dma_mapping_error(qdma->qdma->eth->dev, addr);
+ if (unlikely(ret))
+ return ret;
+
+ desc = &q->desc[index];
+ WRITE_ONCE(desc->pkt_addr, addr);
+ WRITE_ONCE(desc->info, ((struct desc_info) {
+ .word = FIELD_PREP(DESC_INFO_PKT_LEN_MASK, len)
+ }));
+ WRITE_ONCE(desc->next_idx, next_index);
+ for (int i = 0; i < ARRAY_SIZE(desc->msg.raw); i++)
+ WRITE_ONCE(desc->msg.raw[i], msg->raw[i]);
+
+ e->skb = skb;
+ e->dma_addr = addr;
+ e->dma_len = len;
+ q->freelist_head = next_index;
- skb->csum_start = skb_headroom(skb) + th_off;
- skb->csum_offset = offsetof(struct tcphdr, check);
- skb->ip_summed = CHECKSUM_PARTIAL;
+ skb_tx_timestamp(skb);
- return skb;
+ /*
+ * Match the vendor QDMA handoff: the descriptor must be globally
visible
+ * before TX_CPUI gives it to hardware. This is required on
non-coherent
+ * MIPS even though the descriptor ring itself is dma_alloc_coherent().
+ */
+ dma_wmb();
+ econet_wreg((u32)next_index, &q->qchain_regs->tx_cpui);
+
+ return q->entry[next_index].freelist_next == 0xffff ?
+ EBUSY : 0;
}
-static struct sk_buff *airoha_qdma_build_rx_skb(struct airoha_queue *q,
- struct airoha_qdma_desc *desc,
- struct airoha_queue_entry *e,
- struct net_device *netdev)
+/* Init functions, no locks, assumed non-concurrent */
+
+static int econet_init_rx_queue(struct econet_q_rx *q,
+ struct airoha_qdma_mips *qdma, int ndesc)
{
- u32 msg2 = le32_to_cpu(READ_ONCE(desc->msg2));
- int qid = q - &q->qdma->q_rx[0];
- struct sk_buff *skb;
+ const struct page_pool_params pp_params = {
+ .order = 0,
+ .pool_size = ndesc,
+ .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+ .dma_dir = DMA_FROM_DEVICE,
+ .max_len = PAGE_SIZE,
+ .nid = NUMA_NO_NODE,
+ .dev = qdma->qdma->eth->dev,
+ .napi = &q->napi,
+ };
+ int threshold;
+ struct qregs_rxring_size rrs;
+ struct qregs_rxring_low rrl;
+ dma_addr_t dma_addr;
- if (FIELD_GET(QDMA_ETH_RXMSG_AGG_COUNT_MASK, msg2) > 1) { /* LRO */
- skb = airoha_qdma_lro_rx_skb(q, desc, e);
- if (!skb)
- return NULL;
- } else {
- u32 desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
- u32 len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
+ q->buf_size = AIROHA_MAX_PACKET_SIZE;
+ q->ndesc = ndesc;
+ q->qdma = qdma;
- skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM, q->buf_size);
- if (!skb)
- return NULL;
+ if (airoha_is(qdma->qdma->eth, econet_en751221))
+ /* QDMA_LAN uses 32 for both rings; QDMA_WAN uses ring_size / 4. */
+ threshold = qdma->qdma->id == 0 ? 32 : ndesc >> 2;
+ else
+ threshold = clamp(ndesc >> 3, 1, 32);
+ q->qchain_regs = (q == &qdma->q_rx[0]) ?
+ &qdma->regs->qchain0 :
+ &qdma->regs->qchain1;
- skb_reserve(skb, AIROHA_RX_HEADROOM);
- __skb_put(skb, len);
- skb->ip_summed = CHECKSUM_UNNECESSARY;
- }
+ q->entry = devm_kzalloc(qdma->qdma->eth->dev, q->ndesc *
sizeof(*q->entry),
+ GFP_KERNEL);
+ if (!q->entry)
+ return -ENOMEM;
- skb_mark_for_recycle(skb);
- skb->dev = netdev;
- skb_record_rx_queue(skb, qid);
- skb->protocol = eth_type_trans(skb, netdev);
+ q->page_pool = page_pool_create(&pp_params);
+ if (IS_ERR(q->page_pool)) {
+ int err = PTR_ERR(q->page_pool);
- return skb;
-}
+ q->page_pool = NULL;
+ return err;
+ }
-static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
-{
- enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
- struct airoha_eth *eth = q->qdma->eth;
- int done = 0;
+ q->desc = dmam_alloc_coherent(qdma->qdma->eth->dev, q->ndesc *
sizeof(*q->desc),
+ &dma_addr, GFP_KERNEL);
+ if (!q->desc)
+ return -ENOMEM;
- while (done < budget) {
- struct airoha_queue_entry *e = &q->entry[q->tail];
- struct airoha_qdma_desc *desc = &q->desc[q->tail];
- u32 hash, reason, msg1, desc_ctrl;
- struct airoha_gdm_dev *dev;
- struct net_device *netdev;
- int data_len, len;
- struct page *page;
+ memset(q->desc, 0, q->ndesc * sizeof(*q->desc));
- desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
- if (!(desc_ctrl & QDMA_DESC_DONE_MASK))
- break;
+ netif_napi_add(qdma->qdma->eth->napi_dev, &q->napi,
econet_qdma_rx_napi_poll);
- dma_rmb();
+ econet_wreg(lower_32_bits(dma_addr), &q->qchain_regs->rxbase);
- q->tail = (q->tail + 1) % q->ndesc;
- q->queued--;
+ /* econet_fill_rx_queue fills everything from current cpu index + 1
up to
+ * but excluding end_i, so to fill every entry we run it with 0 to do
+ * 1,2,3,[...],n, then we run it with 1 to do 0. */
+ econet_fill_rx_queue(q, 0);
+ econet_fill_rx_queue(q, 1);
+ for (int i = 0; i < q->ndesc; i++)
+ if (!q->entry[i].dma_addr)
+ return -ENOMEM;
- dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len,
- dir);
+ /* The RX hardware side considers that hwi == cpui means the queue is
+ * full, not empty. Since cpui starts at 0, we initialize hwi to 1. */
+ econet_wreg(1U, &q->qchain_regs->rx_hwi);
- page = virt_to_head_page(e->buf);
- len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
- data_len = q->skb ? AIROHA_RX_LEN(q->buf_size) : e->dma_len;
- if (!len || data_len < len)
- goto free_frag;
+ rrs = econet_rreg(&qdma->regs->rxring_size);
+ rrl = econet_rreg(&qdma->regs->rxring_low);
- dev = airoha_qdma_get_gdm_dev(eth, desc);
- if (IS_ERR(dev))
- goto free_frag;
+ if (q == &qdma->q_rx[0]) {
+ set_qregs_rxring_size_ring0(&rrs, ndesc);
+ set_qregs_rxring_low_ring0(&rrl, threshold);
+ } else {
+ set_qregs_rxring_size_ring1(&rrs, ndesc);
+ set_qregs_rxring_low_ring1(&rrl, threshold);
+ }
- netdev = netdev_from_priv(dev);
- if (!q->skb) { /* first buffer */
- q->skb = airoha_qdma_build_rx_skb(q, desc, e, netdev);
- if (!q->skb)
- goto free_frag;
- } else { /* scattered frame */
- struct skb_shared_info *shinfo = skb_shinfo(q->skb);
- int nr_frags = shinfo->nr_frags;
+ econet_wreg(rrs, &qdma->regs->rxring_size);
+ econet_wreg(rrl, &qdma->regs->rxring_low);
- if (nr_frags >= ARRAY_SIZE(shinfo->frags))
- goto free_frag;
+ return 0;
+}
- skb_add_rx_frag(q->skb, nr_frags, page,
- e->buf - page_address(page), len,
- q->buf_size);
- }
+static int econet_init_irqs(struct device *dev, struct airoha_qdma_mips
*qdma,
+ int *irqs)
+{
+ int i;
- if (FIELD_GET(QDMA_DESC_MORE_MASK, desc_ctrl))
- continue;
+ if (qdma->qdma->eth->soc->irq_banks > ARRAY_SIZE(qdma->irqs))
+ return -EINVAL;
- if (netdev_uses_dsa(netdev)) {
- struct airoha_gdm_port *port = dev->port;
+ for (i = 0; i < qdma->qdma->eth->soc->irq_banks; i++) {
+ struct econet_irq *irq = &qdma->irqs[i];
+ const char *name;
+ int err;
+ int j;
- /* PPE module requires untagged packets to work
- * properly and it provides DSA port index via the
- * DMA descriptor. Report DSA tag to the DSA stack
- * via skb dst info.
- */
- u32 msg0 = le32_to_cpu(READ_ONCE(desc->msg0));
- u32 sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, msg0);
+ spin_lock_init(&irq->lock_irq);
+ irq->qdma = qdma;
- if (sptag < ARRAY_SIZE(port->dsa_meta) &&
- port->dsa_meta[sptag])
- skb_dst_set_noref(q->skb,
- &port->dsa_meta[sptag]->dst);
+ irq->irq = irqs[i];
+ if (irq->irq < 0)
+ return irq->irq;
+
+ for (j = 0; j < ECONET_QDMA_IRQ_REGS; j++) {
+ irq->status_reg[j] = econet_irq_status_reg(qdma, i, j);
+ irq->mask_reg[j] = econet_irq_enable_reg(qdma, i, j);
}
- msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
- hash = FIELD_GET(AIROHA_RXD4_FOE_ENTRY, msg1);
- if (hash != AIROHA_RXD4_FOE_ENTRY)
- skb_set_hash(q->skb, jhash_1word(hash, 0),
- PKT_HASH_TYPE_L4);
-
- reason = FIELD_GET(AIROHA_RXD4_PPE_CPU_REASON, msg1);
- if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
- airoha_ppe_check_skb(ð->ppe->dev, q->skb, hash,
- false);
+ name = devm_kasprintf(dev, GFP_KERNEL,
+ KBUILD_MODNAME "-%d.%d", qdma->qdma->id, i);
+ if (!name)
+ return -ENOMEM;
- done++;
- napi_gro_receive(&q->napi, q->skb);
- q->skb = NULL;
- continue;
-free_frag:
- if (q->skb) {
- dev_kfree_skb(q->skb);
- q->skb = NULL;
- }
- page_pool_put_full_page(q->page_pool, page, true);
+ err = devm_request_irq(dev, irq->irq,
+ econet_irq_handler, IRQF_SHARED, name,
+ irq);
+ if (err)
+ return err;
}
- airoha_qdma_fill_rx_queue(q);
- return done;
+ return 0;
}
-static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget)
+static int econet_init_tx_doneq(struct econet_tx_doneq *done_q,
+ struct airoha_qdma_mips *qdma, int size,
+ int irq_threshold)
{
- struct airoha_queue *q = container_of(napi, struct airoha_queue, napi);
- int cur, done = 0;
-
- do {
- cur = airoha_qdma_rx_process(q, budget - done);
- done += cur;
- } while (cur && done < budget);
+ dma_addr_t dma_addr;
- if (done < budget && napi_complete(napi)) {
- struct airoha_qdma *qdma = q->qdma;
- int i, qid = q - &qdma->q_rx[0];
- int intr_reg = qid < RX_DONE_HIGH_OFFSET ? QDMA_INT_REG_IDX1
- : QDMA_INT_REG_IDX2;
+ netif_napi_add_tx(qdma->qdma->eth->napi_dev, &done_q->napi,
+ econet_poll_tx_complete);
+ done_q->q = dmam_alloc_coherent(qdma->qdma->eth->dev, size *
sizeof(u32),
+ &dma_addr, GFP_KERNEL);
+ if (!done_q->q)
+ return -ENOMEM;
- for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) {
- if (!(BIT(qid) & RX_IRQ_BANK_PIN_MASK(i)))
- continue;
+ memset(done_q->q, 0xff, size * sizeof(u32));
+ done_q->size = size;
+ done_q->qdma = qdma;
+ done_q->regs = &qdma->regs->done_queue;
- airoha_qdma_irq_enable(&qdma->irq_banks[i], intr_reg,
- BIT(qid % RX_DONE_HIGH_OFFSET));
- }
- }
+ econet_wreg(lower_32_bits(dma_addr), &qdma->regs->done_queue.address);
+ struct qregs_doneq_cfg cfg =
econet_rreg(&qdma->regs->done_queue.config);
+ set_qregs_doneq_cfg_size(&cfg, size);
+ set_qregs_doneq_cfg_int_threshold(&cfg, irq_threshold);
+ econet_wreg(cfg, &qdma->regs->done_queue.config);
- return done;
+ return 0;
}
-static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
- struct airoha_qdma *qdma, int ndesc)
+static int econet_init_tx_queue(struct econet_q_tx *q,
+ struct airoha_qdma_mips *qdma, int size)
{
- struct page_pool_params pp_params = {
- .pool_size = 256,
- .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
- .dma_dir = DMA_FROM_DEVICE,
- .nid = NUMA_NO_NODE,
- .dev = qdma->eth->dev,
- .napi = &q->napi,
- };
- struct airoha_eth *eth = qdma->eth;
- int qid = q - &qdma->q_rx[0], thr;
+ int i, qid = q - &qdma->q_tx[0];
dma_addr_t dma_addr;
- bool lro_q;
+ spin_lock_init(&q->lock_bh);
+ q->ndesc = size;
q->qdma = qdma;
- lro_q = airoha_qdma_is_lro_queue(q);
+ q->qchain_regs = (qid == 0) ?
+ &qdma->regs->qchain0 :
+ &qdma->regs->qchain1;
- q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry),
+ q->entry = devm_kzalloc(qdma->qdma->eth->dev, q->ndesc *
sizeof(*q->entry),
GFP_KERNEL);
if (!q->entry)
return -ENOMEM;
- q->desc = dmam_alloc_coherent(eth->dev, ndesc * sizeof(*q->desc),
+ q->desc = dmam_alloc_coherent(qdma->qdma->eth->dev, q->ndesc *
sizeof(*q->desc),
&dma_addr, GFP_KERNEL);
if (!q->desc)
return -ENOMEM;
- pp_params.order = lro_q ? AIROHA_LRO_PAGE_ORDER : 0;
- pp_params.max_len = PAGE_SIZE << pp_params.order;
-
- q->page_pool = page_pool_create(&pp_params);
- if (IS_ERR(q->page_pool)) {
- int err = PTR_ERR(q->page_pool);
+ memset(q->desc, 0, q->ndesc * sizeof(*q->desc));
- q->page_pool = NULL;
- return err;
+ for (i = 0; i < q->ndesc - 1; i++) {
+ q->entry[i].freelist_next = i + 1;
}
+ q->entry[q->ndesc - 1].freelist_next = 0xffff;
+ q->freelist_tail = q->ndesc - 1;
+ q->freelist_head = 0;
- q->buf_size = lro_q ? pp_params.max_len : pp_params.max_len / 2;
- q->ndesc = ndesc;
- netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll);
-
- airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr);
- airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid),
- RX_RING_SIZE_MASK,
- FIELD_PREP(RX_RING_SIZE_MASK, ndesc));
-
- thr = clamp(ndesc >> 3, 1, 32);
- airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), RX_RING_THR_MASK,
- FIELD_PREP(RX_RING_THR_MASK, thr));
- airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
- FIELD_PREP(RX_RING_DMA_IDX_MASK, q->head));
- if (lro_q)
- airoha_qdma_clear(qdma, REG_RX_SCATTER_CFG(qid),
- RX_RING_SG_EN_MASK);
- else
- airoha_qdma_set(qdma, REG_RX_SCATTER_CFG(qid),
- RX_RING_SG_EN_MASK);
+ econet_wreg(lower_32_bits(dma_addr), &q->qchain_regs->txbase);
- airoha_qdma_fill_rx_queue(q);
+ /* On TX, hardware advances hwi until it is equal to cpui. */
+ econet_wreg(0U, &q->qchain_regs->tx_cpui);
+ econet_wreg(0U, &q->qchain_regs->tx_hwi);
return 0;
}
-static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
+static bool econet_en751221_use_slm(struct airoha_qdma_mips *qdma)
{
- struct airoha_qdma *qdma = q->qdma;
- struct airoha_eth *eth = qdma->eth;
- int qid = q - &qdma->q_rx[0];
+ if (!airoha_is(qdma->qdma->eth, econet_en751221) ||
+ qdma->qdma->id != 1)
+ return false;
- while (q->queued) {
- struct airoha_queue_entry *e = &q->entry[q->tail];
- struct airoha_qdma_desc *desc = &q->desc[q->tail];
- struct page *page = virt_to_head_page(e->buf);
+ /*
+ * The EN7512 SDK enables QDMA_WAN SLM when GET_DRAM_SIZE <= 256.
+ * totalram_pages() is slightly smaller than the fitted DRAM because
+ * kernel reservations are already excluded, which keeps the 256 MiB
+ * boards on the same side of the threshold as the vendor code.
+ */
+ return (totalram_pages() << PAGE_SHIFT) <= EN751221_SLM_MAX_DRAM_SIZE;
+}
- dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len,
- page_pool_get_dma_dir(q->page_pool));
- page_pool_put_full_page(q->page_pool, page, false);
- /* Reset DMA descriptor */
- WRITE_ONCE(desc->tcp_ts_reply, 0);
- WRITE_ONCE(desc->ctrl, 0);
- WRITE_ONCE(desc->addr, 0);
- WRITE_ONCE(desc->data, 0);
- WRITE_ONCE(desc->msg0, 0);
- WRITE_ONCE(desc->msg1, 0);
- WRITE_ONCE(desc->msg2, 0);
- WRITE_ONCE(desc->msg3, 0);
+static int econet_en751221_slm_wait_idle(struct airoha_qdma_mips *qdma)
+{
+ u32 val;
+ int ret;
+
+ ret = read_poll_timeout(readl, val,
+ (val & (EN751221_SLM_EN_ENABLE |
+ EN751221_SLM_EN_ACTIVE)) !=
+ (EN751221_SLM_EN_ENABLE |
+ EN751221_SLM_EN_ACTIVE),
+ USEC_PER_MSEC, 20 * USEC_PER_MSEC, true,
+ &qdma->slm.regs->enable);
+ if (ret)
+ dev_err(qdma->qdma->eth->dev, "SLM is still active after 20 ms\n");
+
+ return ret;
+}
- q->tail = (q->tail + 1) % q->ndesc;
- q->queued--;
- }
+static void econet_en751221_slm_disable(struct airoha_qdma_mips *qdma)
+{
+ if (!qdma->slm.regs || !qdma->slm.enabled)
+ return;
- q->head = q->tail;
- /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is
- * empty.
+ /*
+ * QDMA RX/TX is already stopped by the destroy path. If the SLM busy
+ * indication nevertheless sticks, report it but still disable the
block
+ * before releasing its DMA backing; leaving the aperture live would be
+ * worse than reproducing the vendor's forced-disable behaviour here.
*/
- airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK,
- FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
- airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
- FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail));
+ econet_en751221_slm_wait_idle(qdma);
+ writel(0, &qdma->slm.regs->enable);
+ writel(0, &qdma->slm.regs->virt_size);
+ writel(0, &qdma->slm.regs->phys_size);
+ writel(0, &qdma->slm.regs->virt_base);
+ writel(0, &qdma->slm.regs->phys_base);
+ qdma->slm.enabled = false;
+
+
+ if (qdma->slm.buf) {
+ dmam_free_coherent(qdma->qdma->eth->dev, qdma->slm.buf_size,
+ qdma->slm.buf, qdma->slm.dma_addr);
+ qdma->slm.buf = NULL;
+ qdma->slm.buf_size = 0;
+ qdma->slm.dma_addr = 0;
+ }
}
-static int airoha_qdma_init_rx(struct airoha_qdma *qdma)
+static int econet_en751221_slm_init(struct airoha_qdma_mips *qdma,
+ size_t virtual_size, u32 *hwf_data_addr)
{
- int i;
+ size_t physical_size = min_t(size_t, virtual_size,
+ EN751221_SLM_PHYS_SIZE);
+ dma_addr_t dma_addr;
+ u32 glo_cfg;
+ void *buf;
+ int ret;
- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
- int err;
+ if (!IS_ALIGNED(virtual_size, EN751221_SLM_SECTOR_SIZE) ||
+ !IS_ALIGNED(physical_size, EN751221_SLM_SECTOR_SIZE))
+ return -EINVAL;
- if (!(RX_DONE_INT_MASK & BIT(i))) {
- /* rx-queue not binded to irq */
- continue;
- }
+ qdma->slm.regs = devm_ioremap(qdma->qdma->eth->dev, EN751221_SLM_BASE,
+ EN751221_SLM_REG_SIZE);
+ if (!qdma->slm.regs)
+ return dev_err_probe(qdma->qdma->eth->dev, -ENOMEM,
+ "failed to map EN751221 SLM registers\n");
- err = airoha_qdma_init_rx_queue(&qdma->q_rx[i], qdma,
- RX_DSCP_NUM(i));
- if (err)
- return err;
+ /*
+ * Boot2 sets GLO_CFG[0] when the image was built without SLM support.
+ * Do not clear it from Linux: the remaining GLO_CFG bits are
undocumented
+ * and the board has already completed DRAM/arbiter initialisation.
+ */
+ glo_cfg = readl(&qdma->slm.regs->glo_cfg);
+ if (glo_cfg & EN751221_SLM_GLO_CFG_BYPASS)
+ return dev_err_probe(qdma->qdma->eth->dev, -EOPNOTSUPP,
+ "EN751221 SLM is bypassed by firmware
(GLO_CFG=%08x)\n",
+ glo_cfg);
+
+ ret = econet_en751221_slm_wait_idle(qdma);
+ if (ret)
+ return ret;
+
+ writel(0, &qdma->slm.regs->enable);
+
+ buf = dmam_alloc_coherent(qdma->qdma->eth->dev, physical_size,
&dma_addr,
+ GFP_KERNEL);
+ if (!buf)
+ return dev_err_probe(qdma->qdma->eth->dev, -ENOMEM,
+ "failed to allocate SLM backing memory\n");
+ if (upper_32_bits(dma_addr)) {
+ dmam_free_coherent(qdma->qdma->eth->dev, physical_size, buf,
dma_addr);
+ return dev_err_probe(qdma->qdma->eth->dev, -ERANGE,
+ "SLM backing DMA address must be below 4 GiB\n");
+ }
+
+ qdma->slm.buf = buf;
+ qdma->slm.dma_addr = dma_addr;
+ qdma->slm.buf_size = physical_size;
+
+ writel(virtual_size / EN751221_SLM_SECTOR_SIZE,
+ &qdma->slm.regs->virt_size);
+ writel(physical_size / EN751221_SLM_SECTOR_SIZE,
+ &qdma->slm.regs->phys_size);
+ writel(EN751221_SLM_HWF_VIRT_BASE, &qdma->slm.regs->virt_base);
+ writel(lower_32_bits(dma_addr), &qdma->slm.regs->phys_base);
+ qdma->slm.enabled = true;
+ writel(1, &qdma->slm.regs->enable);
+
+ if (!(readl(&qdma->slm.regs->enable) & BIT(0))) {
+ econet_en751221_slm_disable(qdma);
+ return dev_err_probe(qdma->qdma->eth->dev, -EIO,
+ "failed to enable EN751221 SLM\n");
}
+ *hwf_data_addr = EN751221_SLM_HWF_VIRT_BASE;
+ dev_info(qdma->qdma->eth->dev,
+ "QDMA1 SLM enabled: HWF 0x%08x+%zu KiB -> DMA %pad+%zu KiB\n",
+ EN751221_SLM_HWF_VIRT_BASE, virtual_size / SZ_1K,
+ &qdma->slm.dma_addr, physical_size / SZ_1K);
+
return 0;
}
-static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q)
+static int econet_qdma_init_hfwd(struct airoha_qdma_mips *qdma)
{
- struct airoha_qdma *qdma = q->qdma;
- struct airoha_eth *eth = qdma->eth;
- int i, qid = q - &qdma->q_tx[0];
+ enum qregs_hwf_cfg_pkt_sz buf_size_cfg = QREGS_HWF_CFG_PKT_SZ_2048;
+ int ret, size, index, num_desc = qdma->cfg.num_fwd_descs;
+ struct qregs_hwf_cfg1 cfg1;
+ dma_addr_t dma_addr;
+ u32 buf_size = 2048, hwf_data_addr;
+ struct hwf_cfg cfg;
+ const char *name;
- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
- struct airoha_gdm_port *port = eth->ports[i];
- int d;
+ name = devm_kasprintf(qdma->qdma->eth->dev, GFP_KERNEL,
"qdma%d-buf", qdma->qdma->id);
+ if (!name)
+ return -ENOMEM;
- if (!port)
- continue;
+ while (buf_size < qdma->cfg.fwd_max_packet_size) {
+ buf_size_cfg++;
+ buf_size = 2048 << buf_size_cfg;
+ if (buf_size > 16384) {
+ dev_err(qdma->qdma->eth->dev, "Unsupported hw forwarding
max packet size %d\n",
+ qdma->cfg.fwd_max_packet_size);
+ return -EINVAL;
+ }
+ }
- for (d = 0; d < ARRAY_SIZE(port->devs); d++) {
- struct airoha_gdm_dev *dev = port->devs[d];
- struct net_device *netdev;
- int j;
+ if (econet_en751221_use_slm(qdma)) {
+ size = buf_size * num_desc;
+ ret = econet_en751221_slm_init(qdma, size, &hwf_data_addr);
+ if (ret)
+ return ret;
+ goto hwf_data_ready;
+ }
- if (!dev)
- continue;
+ index = of_property_match_string(qdma->qdma->eth->dev->of_node,
+ "memory-region-names", name);
+ if (index >= 0) {
+ struct reserved_mem *rmem;
+ struct device_node *np;
- if (rcu_access_pointer(dev->qdma) != qdma)
- continue;
+ /* Consume reserved memory for hw forwarding buffers queue if
+ * available in the DTS
+ */
+ np = of_parse_phandle(qdma->qdma->eth->dev->of_node,
"memory-region",
+ index);
+ if (!np)
+ return -ENODEV;
- netdev = netdev_from_priv(dev);
- for (j = 0; j < netdev->num_tx_queues; j++) {
- if (airoha_qdma_get_txq(qdma, j) != qid)
- continue;
+ rmem = of_reserved_mem_lookup(np);
+ of_node_put(np);
+ if (!rmem)
+ return dev_err_probe(qdma->qdma->eth->dev, -EINVAL,
+ "invalid %s memory-region\n", name);
+ if (!rmem->size || upper_32_bits(rmem->base) ||
+ upper_32_bits(rmem->base + rmem->size - 1))
+ return dev_err_probe(qdma->qdma->eth->dev, -ERANGE,
+ "%s memory-region must be below 4 GiB\n",
+ name);
- netif_wake_subqueue(netdev, j);
- }
- }
+ dma_addr = rmem->base;
+ /* Compute the number of hw descriptors according to the
+ * reserved memory size and the payload buffer size
+ */
+ num_desc = div_u64(rmem->size, buf_size);
+ if (!num_desc)
+ return dev_err_probe(qdma->qdma->eth->dev, -EINVAL,
+ "%s memory-region is too small\n", name);
+
+ if (num_desc < qdma->cfg.num_fwd_descs)
+ dev_warn(qdma->qdma->eth->dev,
+ "Reserved memory %pa too small for %d "
+ "hw forwarding descriptors with %d bytes"
+ "payload, reducing to %d descriptors.\n",
+ &rmem->size, qdma->cfg.num_fwd_descs,
+ buf_size, num_desc);
+ } else {
+ /*
+ * QDMA_WAN uses SLM on <= 256 MiB systems. On larger systems
+ * without reserved memory, do not require an 8 MiB contiguous
+ * coherent allocation; keep the exact 4096-entry profile when a
+ * qdma1-buf region exists, otherwise use a conservative fallback.
+ */
+ if (airoha_is(qdma->qdma->eth, econet_en751221) &&
qdma->qdma->id == 1 &&
+ num_desc > 256)
+ num_desc = 256;
+
+ size = buf_size * num_desc;
+ if (!dmam_alloc_coherent(qdma->qdma->eth->dev, size, &dma_addr,
+ GFP_KERNEL))
+ return -ENOMEM;
}
- q->txq_stopped = false;
+
+ hwf_data_addr = lower_32_bits(dma_addr);
+
+hwf_data_ready:
+ econet_wreg(hwf_data_addr, &qdma->regs->hwf_data_addr);
+
+ size = num_desc * sizeof(*qdma->hwf_desc);
+ qdma->hwf_desc = dmam_alloc_coherent(qdma->qdma->eth->dev, size,
&dma_addr, GFP_KERNEL);
+ if (!qdma->hwf_desc)
+ return -ENOMEM;
+
+ econet_wreg(lower_32_bits(dma_addr), &qdma->regs->hwf_desc_addr);
+
+ qdma->num_fwd_descs = num_desc;
+ qdma->fwd_buf_size = buf_size;
+
+ cfg = econet_rreg(&qdma->regs->hwf_cfg);
+ set_qregs_hwf_cfg_pkt_sz(&cfg, buf_size_cfg);
+ set_qregs_hwf_cfg_low_th(&cfg,
+ min_t(int, qdma->cfg.fwd_low_threshold,
+ max_t(int, 1, num_desc >> 3)));
+ econet_wreg(cfg, &qdma->regs->hwf_cfg);
+
+ cfg1 = econet_rreg(&qdma->regs->hwf_cfg1);
+ set_qregs_hwf_cfg1_fwd_desc_n(&cfg1, num_desc);
+ set_qregs_hwf_cfg1_overhead_en(&cfg1, true);
+ set_qregs_hwf_cfg1_overhead(&cfg1, qdma->qdma->id == 0 ? 0x14 : 0x18);
+ set_qregs_hwf_cfg1_start(&cfg1, true);
+ econet_wreg(cfg1, &qdma->regs->hwf_cfg1);
+
+ ret = read_poll_timeout(econet_rreg, cfg1,
+ !is_qregs_hwf_cfg1_start(&cfg1), USEC_PER_MSEC,
+ 30 * USEC_PER_MSEC, true,
+ &qdma->regs->hwf_cfg1);
+ if (ret)
+ dev_err(qdma->qdma->eth->dev,
+ "Error %pe waiting for HW forwarding engine to start",
+ ERR_PTR(ret));
+ return ret;
}
-static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
+static void econet_init_en751221_qdma(struct airoha_qdma_mips *qdma)
{
- struct airoha_tx_irq_queue *irq_q;
- int id, done = 0, irq_queued;
- struct airoha_qdma *qdma;
- struct airoha_eth *eth;
- u32 status, head;
+ struct qregs_tx_congest_cfg cngst_cfg;
+ struct wrr_mode wrr = { 0 };
+ u32 total_min, channel_min, queue_min;
+ u32 total_max, channel_max, queue_max;
+ u32 physical_size, val;
- irq_q = container_of(napi, struct airoha_tx_irq_queue, napi);
- qdma = irq_q->qdma;
- id = irq_q - &qdma->q_tx_irq[0];
- eth = qdma->eth;
+ /*
+ * Match the final PSE buffer policy observed on the production
EN751221
+ * firmware. Userspace retunes the initial SDK values before the
system
+ * reaches its steady state: buffer control and estimation are enabled,
+ * DMA prefetch stays disabled, and LAN/WAN use distinct thresholds.
+ */
+ val = econet_rreg(&qdma->regs->buf_usage_cfg);
+ val &= ~(EN751221_PSE_BUF_CTRL_EN |
+ EN751221_PSE_BUF_PREFETCH_EN |
+ EN751221_PSE_BUF_ESTIMATE_EN |
+ EN751221_PSE_BUF_CH_THR_MASK |
+ EN751221_PSE_BUF_TOTAL_THR_MASK);
+ val |= EN751221_PSE_BUF_CTRL_EN | EN751221_PSE_BUF_ESTIMATE_EN;
+ if (qdma->qdma->id == 0) {
+ val |= FIELD_PREP(EN751221_PSE_BUF_CH_THR_MASK, 16) |
+ FIELD_PREP(EN751221_PSE_BUF_TOTAL_THR_MASK, 128);
+ } else {
+ val |= FIELD_PREP(EN751221_PSE_BUF_CH_THR_MASK, 20) |
+ FIELD_PREP(EN751221_PSE_BUF_TOTAL_THR_MASK, 192);
+ }
+ econet_wreg(val, &qdma->regs->buf_usage_cfg);
- status = airoha_qdma_rr(qdma, REG_IRQ_STATUS(id));
- head = FIELD_GET(IRQ_HEAD_IDX_MASK, status);
- head = head % irq_q->size;
- irq_queued = FIELD_GET(IRQ_ENTRY_LEN_MASK, status);
+ /* Match qdma_dev.c: 16-byte WRR scale, accounting weights by byte. */
+ if (airoha_is(qdma->qdma->eth, econet_en7528))
+ set_qregs_wrr_mode_use_16b(&wrr, false);
+ else
+ set_qregs_wrr_mode_use_16b(&wrr, true);
+ set_qregs_wrr_mode_by_byte(&wrr, true);
+ econet_wreg(wrr, &qdma->regs->wrr_mode);
- while (irq_queued > 0 && done < budget) {
- u32 qid, val = irq_q->q[head];
- struct airoha_qdma_desc *desc;
- struct airoha_queue_entry *e;
- struct airoha_queue *q;
- u32 index, desc_ctrl;
- struct sk_buff *skb;
+ /* The normal dynamic congestion profile uses the TX rate meter. */
+ val = EN751221_TX_RATE_METER_EN |
+ FIELD_PREP(EN751221_TX_RATE_METER_DIV_MASK, 2) |
+ FIELD_PREP(EN751221_TX_RATE_METER_SLICE_MASK, 4000);
+ econet_wreg(val, &qdma->regs->tx_meter_cfg);
- if (val == 0xff)
- break;
+ /*
+ * Start from qdma_set_txq_cngst_auto_config() so reduced fallback
+ * pools remain usable. The production firmware retunes the total
+ * threshold after this initial auto profile, however, and the final
+ * values are materially larger than CONFIG_HWFWD_DSCP_NUM / 5.
+ */
+ if (qdma->num_fwd_descs <= 256) {
+ total_min = 48;
+ channel_min = 2;
+ queue_min = 2;
+ } else {
+ total_min = qdma->num_fwd_descs / 5;
+ channel_min = qdma->num_fwd_descs / 14;
+ queue_min = qdma->num_fwd_descs / 170;
+ }
- irq_q->q[head] = 0xff; /* mark as done */
- head = (head + 1) % irq_q->size;
- irq_queued--;
- done++;
+ physical_size = qdma->num_fwd_descs * qdma->fwd_buf_size;
+ total_max = (physical_size - (physical_size >> 4)) >> 8;
+ channel_max = total_max;
+ queue_max = total_min;
- qid = FIELD_GET(IRQ_RING_IDX_MASK, val);
- if (qid >= ARRAY_SIZE(qdma->q_tx))
- continue;
+ /* Final steady-state values captured from the shipping EN751221
+ * firmware. Keep the auto profile for reduced/no-reserved-memory
+ * fallback pools, where these full-size thresholds would be invalid.
+ */
+ if (qdma->fwd_buf_size == 2048 && qdma->qdma->id == 0 &&
+ qdma->num_fwd_descs == 1024) {
+ total_min = 5120;
+ total_max = 7680;
+ channel_max = 7680;
+ queue_max = 204;
+ } else if (qdma->fwd_buf_size == 2048 && qdma->qdma->id == 1 &&
+ qdma->num_fwd_descs == 4096) {
+ total_min = 2048;
+ total_max = 15360;
+ channel_max = 15360;
+ queue_max = 819;
+ }
- q = &qdma->q_tx[qid];
- if (!q->ndesc)
- continue;
+ airoha_wr((void __iomem *)&qdma->regs->tx_congest_thr, 0,
+ FIELD_PREP(EN751221_TXQ_MAX_THR_MASK, total_max) |
+ FIELD_PREP(EN751221_TXQ_MIN_THR_MASK, total_min));
+ econet_wreg((u32)(FIELD_PREP(EN751221_TXQ_MAX_THR_MASK, channel_max) |
+ FIELD_PREP(EN751221_TXQ_MIN_THR_MASK, channel_min)),
+ &qdma->regs->tx_per_ch_dthr);
+ econet_wreg((u32)(FIELD_PREP(EN751221_TXQ_MAX_THR_MASK, queue_max) |
+ FIELD_PREP(EN751221_TXQ_MIN_THR_MASK, queue_min)),
+ &qdma->regs->tx_per_q_dthr);
- index = FIELD_GET(IRQ_DESC_IDX_MASK, val);
- if (index >= q->ndesc)
- continue;
+ /*
+ * Match the final production dynamic-congestion state: normal and DEI
+ * drops enabled, all three update triggers, 250 us tick and a 1/2 DEI
+ * threshold. The four TX-ring blocking bits stay clear.
+ */
+ cngst_cfg = econet_rreg(&qdma->regs->tx_congest_cfg);
+ set_qregs_tx_congest_cfg_tail_drop_en(&cngst_cfg, true);
+ set_qregs_tx_congest_cfg_dei_drop_en(&cngst_cfg, true);
+ set_qregs_tx_congest_cfg_dyncong_en(&cngst_cfg, true);
+ set_qregs_tx_congest_cfg_max_thr_blk_tx1(&cngst_cfg, false);
+ set_qregs_tx_congest_cfg_min_thr_blk_tx1(&cngst_cfg, false);
+ set_qregs_tx_congest_cfg_max_thr_blk_tx0(&cngst_cfg, false);
+ set_qregs_tx_congest_cfg_min_thr_blk_tx0(&cngst_cfg, false);
+ set_qregs_tx_congest_cfg_dyncong_dei_scale(&cngst_cfg,
+ QREGS_TX_CONGEST_CFG_DYNCONG_DEI_SCALE_HALF);
+ set_qregs_tx_congest_cfg_dyncong_upd_wrr(&cngst_cfg, true);
+ set_qregs_tx_congest_cfg_dyncong_upd_txrx(&cngst_cfg, true);
+ set_qregs_tx_congest_cfg_dyncong_upd_tick(&cngst_cfg, true);
+ cngst_cfg.dyncong_tick = 250;
+ econet_wreg(cngst_cfg, &qdma->regs->tx_congest_cfg);
+}
- spin_lock_bh(&q->lock);
+static int econet_qdma_hw_init(struct airoha_qdma_mips *qdma)
+{
+ struct qregs_qcfg qcfg;
+ int i;
- if (!q->queued)
- goto unlock;
+ for (i = 0; i < qdma->qdma->eth->soc->irq_banks; i++) {
+ int j;
- desc = &q->desc[index];
- desc_ctrl = le32_to_cpu(desc->ctrl);
+ /* Clear only status registers implemented by this QDMA. */
+ for (j = 0; j < ARRAY_SIZE(qdma->irqs[i].status_reg); j++) {
+ if (qdma->irqs[i].status_reg[j])
+ econet_wreg(0xffffffffU, qdma->irqs[i].status_reg[j]);
+ }
- if (!(desc_ctrl & QDMA_DESC_DONE_MASK) &&
- !(desc_ctrl & QDMA_DESC_DROP_MASK))
- goto unlock;
+ /* enable IRQs */
+ for (j = 0; j < ARRAY_SIZE(qdma->irqs[i].irqmask); j++) {
+ u32 mask = 0;
- e = &q->entry[index];
- skb = e->skb;
+ if (!qdma->irqs[i].mask_reg[j])
+ continue;
- dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
- DMA_TO_DEVICE);
- e->dma_addr = 0;
- list_add_tail(&e->list, &q->tx_list);
+ for (int k = 0; k < 32; k++) {
+ union econet_irq_purpose p;
+ union irq_bit bit = {
+ .irq_idx = i,
+ .reg_idx = j,
+ .bit_idx = k,
+ };
+ u32 en = 0;
- WRITE_ONCE(desc->msg0, 0);
- WRITE_ONCE(desc->msg1, 0);
- q->queued--;
+ if (!econet_valid_irq_bit(bit))
+ continue;
- if (skb) {
- struct netdev_queue *txq;
+ p = econet_irq_purpose(bit);
+
+ if (airoha_is(qdma->qdma->eth, econet_en751221)) {
+ /*
+ * Match qdma_dev.c's production mask. HWF
+ * EMPTY/LOW are status/debug conditions, not
+ * runtime interrupts; keeping them enabled while
+ * PPE consumes LMGR descriptors can cause a hard
+ * IRQ storm. Bit 9 is the vendor IRQ_FULL event,
+ * represented as NO_DSCP/DONE in our map.
+ */
+ en |= p.type == IPS_DONE;
+ en |= p.type == IPS_NO_DSCP &&
+ (p.source == IPSC_RX ||
+ p.source == IPSC_DONE);
+ en |= p.type == IPS_ERR_COHERENT;
+ en |= p.type == IPS_OVERFLOW;
+ } else {
+ /* RX and TX done */
+ en |= (p.type == IPS_DONE);
+
+ /* Running out of resources */
+ en |= (p.type == IPS_NO_DSCP &&
+ p.source != IPSC_TX);
+ en |= (p.type == IPS_LOW_DSCP &&
+ p.source != IPSC_TX);
+
+ /* Error conditions */
+ en |= (p.type == IPS_ERR_COHERENT);
+ en |= (p.type == IPS_OVERFLOW);
+
+ /* External hardware */
+ en |= (p.type == IPS_GPON_INT);
+ en |= (p.type == IPS_EPON_INT);
+ en |= (p.type == IPS_XPON_INT);
+ }
+
+ mask |= en << k;
+ }
- txq = skb_get_tx_queue(skb->dev, skb);
- netdev_tx_completed_queue(txq, 1, skb->len);
- dev_kfree_skb_any(skb);
+ qdma->irqs[i].irqmask[j] = mask;
+ econet_wreg(mask, qdma->irqs[i].mask_reg[j]);
}
+ }
- if (q->txq_stopped && q->ndesc - q->queued >= q->free_thr) {
- /* Since multiple net_device TX queues can share the
- * same hw QDMA TX queue, there is no guarantee we have
- * inflight packets queued in hw belonging to a
- * net_device TX queue stopped in the xmit path.
- * In order to avoid any potential net_device TX queue
- * stall, we need to wake all the net_device TX queues
- * feeding the same hw QDMA TX queue.
- */
- airoha_qdma_wake_netdev_txqs(q);
- }
+ qcfg = (struct qregs_qcfg) { 0 };
+ set_qregs_qcfg_msg_word_swap(&qcfg, true);
+ set_qregs_qcfg_dscp_byte_swap(&qcfg, airoha_is(qdma->qdma->eth,
econet_en751221));
+ set_qregs_qcfg_payload_byte_sw(&qcfg, true);
+ if (airoha_is_econet(qdma->qdma->eth))
+ set_qregs_qcfg_dma_pref(&qcfg, QREGS_QCFG_DMA_PREF_ROUND_ROBIN);
+ else
+ set_qregs_qcfg_dma_pref(&qcfg, QREGS_QCFG_DMA_PREF_TX1_FRX_TX0);
+ set_qregs_qcfg_rx_2b_offset(&qcfg, qdma->cfg.rx_2b_offset);
+ set_qregs_qcfg_irq_en(&qcfg, true);
+ set_qregs_qcfg_check_done(&qcfg, !airoha_is_econet(qdma->qdma->eth));
+ set_qregs_qcfg_tx_wb_done(&qcfg, true);
+
+ if (airoha_is(qdma->qdma->eth, econet_en7528) &&
+ qdma->qdma->id == 0)
+ set_qregs_qcfg_tx_immediate_done(&qcfg, true);
+
+ if (airoha_is(qdma->qdma->eth, econet_en751221)) {
+ /*
+ * Stock QDMA_LAN uses TX_IMMEDIATE_DONE (0x9c180075), while
+ * QDMA_WAN uses SLM_RELEASE_EN (0x9c280075). RX_2B_OFFSET is
+ * intentionally kept controlled by cfg because the current
+ * page-pool RX layout cannot safely reproduce the vendor +2 DMA
+ * offset yet. Without that bit the expected values are
+ * 0x1c180075 and 0x1c280075 once DMA is enabled.
+ */
+ if (qdma->qdma->id == 0)
+ set_qregs_qcfg_tx_immediate_done(&qcfg, true);
+ else
+ set_qregs_qcfg_slm_release_en(&qcfg, true);
+ }
+ set_qregs_qcfg_burst_size(&qcfg, QREGS_QCFG_BURST_SIZE_128_BYTES);
+ econet_wreg(qcfg, &qdma->regs->qdma_cfg);
-unlock:
- spin_unlock_bh(&q->lock);
+ econet_wreg(0U, &qdma->regs->rx_int_delay);
+
+ if (airoha_is(qdma->qdma->eth, econet_en751221)) {
+ econet_init_en751221_qdma(qdma);
+ } else {
+ struct qregs_tx_congest_cfg cngst_cfg = { 0 };
+
+ set_qregs_tx_congest_cfg_tail_drop_en(&cngst_cfg, true);
+ set_qregs_tx_congest_cfg_dei_drop_en(&cngst_cfg, true);
+ econet_wreg(cngst_cfg, &qdma->regs->tx_congest_cfg);
}
- if (done) {
- int i, len = done >> 7;
+ /*
+ * The vendor EN751221 LAN driver disables per-channel QDMA TX rate
+ * limiting for channels 0..4 when the external MT7530 is used. The
+ * firmware/bootloader may leave these bits enabled, so explicitly
clear
+ * them instead of inheriting a stale hardware shaper configuration.
+ */
+ {
+ u32 ch_lim_en = econet_rreg(&qdma->regs->ch_lim_en);
- for (i = 0; i < len; i++)
- airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
- IRQ_CLEAR_LEN_MASK, 0x80);
- airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
- IRQ_CLEAR_LEN_MASK, (done & 0x7f));
+ ch_lim_en &= ~GENMASK(4, 0);
+ econet_wreg(ch_lim_en, &qdma->regs->ch_lim_en);
}
- if (done < budget && napi_complete(napi))
- airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0,
- TX_DONE_INT_MASK(id));
+ return 0;
+}
- return done;
+/* End init functions */
+
+static void econet_qdma_destroy_rxq_locked(struct econet_q_rx *q)
+{
+ int i;
+
+ if (!q->page_pool)
+ return;
+
+ for (i = 0; i < q->ndesc; i++) {
+ struct econet_q_rx_ent *e = &q->entry[i];
+ struct page *page;
+
+ if (!e->dma_addr)
+ continue;
+
+ page = virt_to_head_page(e->buf);
+ dma_sync_single_for_cpu(q->qdma->qdma->eth->dev, e->dma_addr, e->dma_len,
+ page_pool_get_dma_dir(q->page_pool));
+ page_pool_put_full_page(q->page_pool, page, false);
+ e->dma_addr = 0;
+ e->buf = NULL;
+ }
+
+ page_pool_destroy(q->page_pool);
+ q->page_pool = NULL;
}
-static int airoha_qdma_init_tx_queue(struct airoha_queue *q,
- struct airoha_qdma *qdma, int size)
+static int econet_qdma_destroy_locked(struct airoha_qdma_mips *qdma)
{
- struct airoha_eth *eth = qdma->eth;
- int i, qid = q - &qdma->q_tx[0];
- dma_addr_t dma_addr;
+ struct qregs_qcfg qcfg;
+ int i;
- spin_lock_init(&q->lock);
- q->qdma = qdma;
- q->free_thr = 1 + MAX_SKB_FRAGS;
- INIT_LIST_HEAD(&q->tx_list);
+ if (WARN_ON_ONCE(qdma->qdma->users))
+ return -EBUSY;
- q->entry = devm_kzalloc(eth->dev, size * sizeof(*q->entry),
- GFP_KERNEL);
- if (!q->entry)
- return -ENOMEM;
+ qcfg = econet_rreg(&qdma->regs->qdma_cfg);
+ set_qregs_qcfg_irq_en(&qcfg, false);
+ set_qregs_qcfg_rx_dma_en(&qcfg, false);
+ set_qregs_qcfg_tx_dma_en(&qcfg, false);
+ econet_wreg(qcfg, &qdma->regs->qdma_cfg);
- q->desc = dmam_alloc_coherent(eth->dev, size * sizeof(*q->desc),
- &dma_addr, GFP_KERNEL);
- if (!q->desc)
- return -ENOMEM;
+ econet_en751221_slm_disable(qdma);
- for (i = 0; i < size; i++) {
- u32 val = FIELD_PREP(QDMA_DESC_DONE_MASK, 1);
+ for (i = 0; i < qdma->qdma->eth->soc->irq_banks; i++) {
+ struct econet_irq *irq = &qdma->irqs[i];
+ int j;
- list_add_tail(&q->entry[i].list, &q->tx_list);
- WRITE_ONCE(q->desc[i].ctrl, cpu_to_le32(val));
+ for (j = 0; j < ARRAY_SIZE(irq->irqmask); j++) {
+ irq->irqmask[j] = 0;
+ if (irq->mask_reg[j])
+ econet_wreg(0U, irq->mask_reg[j]);
+ }
+
+ if (irq->irq > 0)
+ synchronize_irq(irq->irq);
}
- q->ndesc = size;
- /* xmit ring drop default setting */
- airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(qid),
- TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK);
+ for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+ struct econet_q_rx *q = &qdma->q_rx[i];
+
+ econet_qdma_destroy_rxq_locked(q);
+ if (q->napi.dev)
+ netif_napi_del(&q->napi);
+ }
+
+ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_done); i++) {
+ struct econet_tx_doneq *q = &qdma->q_tx_done[i];
+
+ if (q->napi.dev)
+ netif_napi_del(&q->napi);
+ }
- airoha_qdma_wr(qdma, REG_TX_RING_BASE(qid), dma_addr);
- airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK,
- FIELD_PREP(TX_RING_CPU_IDX_MASK, 0));
- airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK,
- FIELD_PREP(TX_RING_DMA_IDX_MASK, 0));
return 0;
}
-static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q,
- struct airoha_qdma *qdma, int size)
+static int econet_qdma_destroy(struct airoha_qdma_mips *qdma)
{
- int id = irq_q - &qdma->q_tx_irq[0];
- struct airoha_eth *eth = qdma->eth;
- dma_addr_t dma_addr;
+ guard(mutex)(&qdma->lock);
- irq_q->q = dmam_alloc_coherent(eth->dev, size * sizeof(u32),
- &dma_addr, GFP_KERNEL);
- if (!irq_q->q)
- return -ENOMEM;
+ return econet_qdma_destroy_locked(qdma);
+}
- memset(irq_q->q, 0xff, size * sizeof(u32));
- irq_q->size = size;
- irq_q->qdma = qdma;
+static void econet_qdma_cleanup_tx(struct airoha_qdma_mips *qdma)
+{
+ int i, j;
- netif_napi_add_tx(eth->napi_dev, &irq_q->napi,
- airoha_qdma_tx_napi_poll);
+ for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
+ struct econet_q_tx *q = &qdma->q_tx[i];
- airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr);
- airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK,
- FIELD_PREP(TX_IRQ_DEPTH_MASK, size));
- airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_THR_MASK,
- FIELD_PREP(TX_IRQ_THR_MASK, 1));
+ guard(spinlock_bh)(&q->lock_bh);
+ for (j = 0; j < q->ndesc; j++) {
+ struct econet_q_tx_ent *e = &q->entry[j];
- return 0;
-}
+ if (!e->dma_addr)
+ continue;
-static int airoha_qdma_init_tx(struct airoha_qdma *qdma)
+ dma_unmap_single(qdma->qdma->eth->dev, e->dma_addr,
+ e->dma_len, DMA_TO_DEVICE);
+ dev_kfree_skb_any(e->skb);
+ memset(e, 0, sizeof(*e));
+
+ e->freelist_next = 0xffff;
+ q->entry[q->freelist_tail].freelist_next = j;
+ q->freelist_tail = j;
+ }
+ }
+}
+
+static int airoha_qdma_mips_xmit(struct airoha_qdma *qdma,
+ struct sk_buff *skb, union desc_msg *msg, int qid)
+{
+ return qdma->econet ? econet_qdma_xmit_mips(qdma->econet, skb, msg,
qid) :
+ -ENODEV;
+}
+
+static int airoha_qdma_mips_set_xpon_irq(struct airoha_qdma *qdma,
+ enum airoha_xpon_mode mode, bool enable)
+{
+ return qdma->econet ? econet_qdma_set_xpon_irq_mips(qdma->econet,
mode, enable) :
+ -ENODEV;
+}
+
+static int airoha_qdma_get_mips_irqs(struct platform_device *pdev,
+ struct airoha_qdma *qdma, int *irqs)
+{
+ struct airoha_eth *eth = qdma->eth;
+ int bank;
+
+ for (bank = 0; bank < eth->soc->irq_banks; bank++) {
+ int irq_idx;
+
+ /* EN751221 exposes only LAN0/WAN0. EN7516/EN7527/EN7528
+ * expose LAN0/WAN0 first and then LAN1..3/WAN1..3.
+ */
+ if (!bank)
+ irq_idx = qdma->id;
+ else
+ irq_idx = qdma->id ? bank + 4 : bank + 1;
+
+ irqs[bank] = platform_get_irq(pdev, irq_idx);
+ if (irqs[bank] < 0)
+ return dev_err_probe(eth->dev, irqs[bank],
+ "failed to get QDMA%u IRQ bank %d\n",
+ qdma->id, bank);
+ }
+
+ return 0;
+}
+
+static int econet_qdma_init_irqs(struct platform_device *pdev,
+ struct airoha_qdma_mips *qdma)
+{
+ int irqs[ECONET_MAX_QDMA_IRQS] = {};
+ int err;
+
+ err = airoha_qdma_get_mips_irqs(pdev, qdma->qdma, irqs);
+ if (err)
+ return err;
+
+ return econet_init_irqs(qdma->qdma->eth->dev, qdma, irqs);
+}
+
+static int econet_qdma_init_rx(struct airoha_qdma_mips *qdma)
{
int i, err;
- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
- err = airoha_qdma_tx_irq_init(&qdma->q_tx_irq[i], qdma,
- IRQ_QUEUE_LEN(i));
+ for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+ err = econet_init_rx_queue(&qdma->q_rx[i], qdma,
+ qdma->cfg.num_rx_descs[i]);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+static int econet_qdma_init_tx(struct airoha_qdma_mips *qdma)
+{
+ int i, err;
+
+ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_done); i++) {
+ err = econet_init_tx_doneq(&qdma->q_tx_done[i], qdma,
+ qdma->cfg.done_list_size[i],
+ qdma->cfg.done_list_irq_threshold[i]);
if (err)
return err;
}
for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
- err = airoha_qdma_init_tx_queue(&qdma->q_tx[i], qdma,
- TX_DSCP_NUM);
+ err = econet_init_tx_queue(&qdma->q_tx[i], qdma,
+ qdma->cfg.num_tx_descs[i]);
if (err)
return err;
}
@@ -1383,433 +1972,4089 @@ static int airoha_qdma_init_tx(struct
airoha_qdma *qdma)
return 0;
}
-static void airoha_qdma_cleanup_tx_queue(struct airoha_queue *q)
+/* Airoha QDMA backend. */
+static void airoha_qdma_set_irqmask(struct airoha_irq_bank *irq_bank,
+ int index, u32 clear, u32 set)
+{
+ struct airoha_qdma *qdma = irq_bank->qdma;
+ int bank = irq_bank - &qdma->irq_banks[0];
+ unsigned long flags;
+
+ if (WARN_ON_ONCE(index >= ARRAY_SIZE(irq_bank->irqmask)))
+ return;
+
+ spin_lock_irqsave(&irq_bank->irq_lock, flags);
+
+ irq_bank->irqmask[index] &= ~clear;
+ irq_bank->irqmask[index] |= set;
+ airoha_qdma_wr(qdma, REG_INT_ENABLE(bank, index),
+ irq_bank->irqmask[index]);
+ /* Read irq_enable register in order to guarantee the update above
+ * completes in the spinlock critical section.
+ */
+ airoha_qdma_rr(qdma, REG_INT_ENABLE(bank, index));
+
+ spin_unlock_irqrestore(&irq_bank->irq_lock, flags);
+}
+
+static void airoha_qdma_irq_enable(struct airoha_irq_bank *irq_bank,
+ int index, u32 mask)
+{
+ airoha_qdma_set_irqmask(irq_bank, index, 0, mask);
+}
+
+static void airoha_qdma_irq_disable(struct airoha_irq_bank *irq_bank,
+ int index, u32 mask)
+{
+ airoha_qdma_set_irqmask(irq_bank, index, mask, 0);
+}
+
+static int airoha_qdma_fill_rx_queue(struct airoha_queue *q)
{
struct airoha_qdma *qdma = q->qdma;
- struct airoha_eth *eth = qdma->eth;
- int i, qid = q - &qdma->q_tx[0];
- u16 index = 0;
+ int qid = q - &qdma->q_rx[0];
+ int nframes = 0;
- spin_lock_bh(&q->lock);
- for (i = 0; i < q->ndesc; i++) {
- struct airoha_queue_entry *e = &q->entry[i];
- struct airoha_qdma_desc *desc = &q->desc[i];
+ while (q->queued < q->ndesc - 1) {
+ struct airoha_queue_entry *e = &q->entry[q->head];
+ struct airoha_qdma_desc *desc = &q->desc[q->head];
+ struct page *page;
+ int offset;
+ u32 val;
- if (!e->dma_addr)
- continue;
+ page = page_pool_dev_alloc_frag(q->page_pool, &offset,
+ q->buf_size);
+ if (!page)
+ break;
- dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
- DMA_TO_DEVICE);
- dev_kfree_skb_any(e->skb);
- e->dma_addr = 0;
- e->skb = NULL;
- list_add_tail(&e->list, &q->tx_list);
+ q->head = (q->head + 1) % q->ndesc;
+ q->queued++;
+ nframes++;
- /* Reset DMA descriptor */
- WRITE_ONCE(desc->ctrl, 0);
- WRITE_ONCE(desc->addr, 0);
- WRITE_ONCE(desc->data, 0);
+ e->buf = page_address(page) + offset;
+ e->dma_addr = page_pool_get_dma_addr(page) + offset;
+ e->dma_len = SKB_WITH_OVERHEAD(q->buf_size);
+
+ WRITE_ONCE(desc->tcp_ts_reply, 0);
+ val = airoha_is(qdma->eth, airoha_en7523, econet_en7580) ?
+ FIELD_PREP(EN7523_QDMA_DESC_LEN_MASK, e->dma_len) :
+ FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len);
+ WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
+ WRITE_ONCE(desc->addr, cpu_to_le32(e->dma_addr));
+ val = FIELD_PREP(QDMA_DESC_NEXT_ID_MASK, q->head);
+ WRITE_ONCE(desc->data, cpu_to_le32(val));
WRITE_ONCE(desc->msg0, 0);
WRITE_ONCE(desc->msg1, 0);
WRITE_ONCE(desc->msg2, 0);
-
- q->queued--;
+ WRITE_ONCE(desc->msg3, 0);
}
- if (!list_empty(&q->tx_list)) {
- struct airoha_queue_entry *e;
-
- e = list_first_entry(&q->tx_list, struct airoha_queue_entry,
- list);
- index = e - q->entry;
+ if (nframes) {
+ /* Publish descriptor contents before handing ownership to DMA. */
+ dma_wmb();
+ airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid),
+ RX_RING_CPU_IDX_MASK,
+ FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
}
- /* Set TX_DMA_IDX to TX_CPU_IDX to notify the hw the QDMA TX ring is
- * empty.
- */
- airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK,
- FIELD_PREP(TX_RING_CPU_IDX_MASK, index));
- airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK,
- FIELD_PREP(TX_RING_DMA_IDX_MASK, index));
- spin_unlock_bh(&q->lock);
+ return nframes;
}
-static int airoha_qdma_init_hfwd_queues(struct airoha_qdma *qdma)
+static struct airoha_gdm_dev *
+airoha_qdma_get_xpon_dev(struct airoha_eth *eth)
{
- int size, index, num_desc = HW_DSCP_NUM;
- struct airoha_eth *eth = qdma->eth;
- int id = qdma - ð->qdma[0];
- u32 status, buf_size;
- dma_addr_t dma_addr;
- const char *name;
-
- name = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d-buf", id);
- if (!name)
- return -ENOMEM;
+ int i, j;
- buf_size = id ? AIROHA_MAX_PACKET_SIZE / 2 : AIROHA_MAX_PACKET_SIZE;
- index = of_property_match_string(eth->dev->of_node,
- "memory-region-names", name);
- if (index >= 0) {
- struct reserved_mem *rmem;
- struct device_node *np;
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
+ struct airoha_gdm_port *port = eth->ports[i];
- /* Consume reserved memory for hw forwarding buffers queue if
- * available in the DTS
- */
- np = of_parse_phandle(eth->dev->of_node, "memory-region",
- index);
- if (!np)
- return -ENODEV;
+ if (!port || port->id != AIROHA_GDM2_IDX)
+ continue;
- rmem = of_reserved_mem_lookup(np);
- of_node_put(np);
- if (!rmem)
- return -ENODEV;
+ for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
+ struct airoha_gdm_dev *dev = port->devs[j];
- dma_addr = rmem->base;
- /* Compute the number of hw descriptors according to the
- * reserved memory size and the payload buffer size
- */
- num_desc = div_u64(rmem->size, buf_size);
- } else {
- size = buf_size * num_desc;
- if (!dmam_alloc_coherent(eth->dev, size, &dma_addr,
- GFP_KERNEL))
- return -ENOMEM;
+ if (dev &&
+ (dev->flags & AIROHA_PRIV_F_XPON_MANAGED))
+ return dev;
+ }
}
- airoha_qdma_wr(qdma, REG_FWD_BUF_BASE, dma_addr);
+ return ERR_PTR(-ENODEV);
+}
- size = num_desc * sizeof(struct airoha_qdma_fwd_desc);
- if (!dmam_alloc_coherent(eth->dev, size, &dma_addr, GFP_KERNEL))
- return -ENOMEM;
+static u32
+airoha_qdma_get_rx_sport(struct airoha_eth *eth,
+ struct airoha_qdma_desc *desc)
+{
+ u32 msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
- airoha_qdma_wr(qdma, REG_FWD_DSCP_BASE, dma_addr);
- /* QDMA0: 2KB. QDMA1: 1KB */
- airoha_qdma_rmw(qdma, REG_HW_FWD_DSCP_CFG,
- HW_FWD_DSCP_PAYLOAD_SIZE_MASK,
- FIELD_PREP(HW_FWD_DSCP_PAYLOAD_SIZE_MASK, !!id));
- airoha_qdma_rmw(qdma, REG_FWD_DSCP_LOW_THR, FWD_DSCP_LOW_THR_MASK,
- FIELD_PREP(FWD_DSCP_LOW_THR_MASK, 128));
- airoha_qdma_rmw(qdma, REG_LMGR_INIT_CFG,
- LMGR_INIT_START | LMGR_SRAM_MODE_MASK |
- HW_FWD_DESC_NUM_MASK,
- FIELD_PREP(HW_FWD_DESC_NUM_MASK, num_desc) |
- LMGR_INIT_START | LMGR_SRAM_MODE_MASK);
+ if (airoha_is(eth, airoha_en7523, econet_en7580))
+ return FIELD_GET(EN7523_QDMA_ETH_RXMSG_SPORT_MASK, msg1);
- return read_poll_timeout(airoha_qdma_rr, status,
- !(status & LMGR_INIT_START), USEC_PER_MSEC,
- 30 * USEC_PER_MSEC, true, qdma,
- REG_LMGR_INIT_CFG);
+ return FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, msg1);
}
-static void airoha_qdma_init_qos(struct airoha_qdma *qdma)
+static struct airoha_gdm_dev *
+airoha_qdma_get_gdm_dev(struct airoha_eth *eth, u32 sport)
{
- airoha_qdma_clear(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_SCALE_MASK);
- airoha_qdma_set(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_BASE_MASK);
-
- airoha_qdma_clear(qdma, REG_PSE_BUF_USAGE_CFG,
- PSE_BUF_ESTIMATE_EN_MASK);
+ struct airoha_gdm_port *port;
+ u16 p, d;
- airoha_qdma_set(qdma, REG_EGRESS_RATE_METER_CFG,
- EGRESS_RATE_METER_EN_MASK |
- EGRESS_RATE_METER_EQ_RATE_EN_MASK);
- /* 2047us x 31 = 63.457ms */
- airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,
- EGRESS_RATE_METER_WINDOW_SZ_MASK,
- FIELD_PREP(EGRESS_RATE_METER_WINDOW_SZ_MASK, 0x1f));
- airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,
- EGRESS_RATE_METER_TIMESLICE_MASK,
- FIELD_PREP(EGRESS_RATE_METER_TIMESLICE_MASK, 0x7ff));
+ if (!eth->soc->ops.get_dev_from_sport ||
+ eth->soc->ops.get_dev_from_sport(eth, sport, &p, &d))
+ return ERR_PTR(-ENODEV);
- /* ratelimit init */
- airoha_qdma_set(qdma, REG_GLB_TRTCM_CFG, GLB_TRTCM_EN_MASK);
- /* fast-tick 25us */
- airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_FAST_TICK_MASK,
- FIELD_PREP(GLB_FAST_TICK_MASK, 25));
- airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_SLOW_TICK_RATIO_MASK,
- FIELD_PREP(GLB_SLOW_TICK_RATIO_MASK, 40));
+ if (p >= eth->soc->max_gdm_ports)
+ return ERR_PTR(-ENODEV);
- airoha_qdma_set(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_TRTCM_EN_MASK);
- airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_FAST_TICK_MASK,
- FIELD_PREP(EGRESS_FAST_TICK_MASK, 25));
- airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG,
- EGRESS_SLOW_TICK_RATIO_MASK,
- FIELD_PREP(EGRESS_SLOW_TICK_RATIO_MASK, 40));
+ port = eth->ports[p];
+ if (!port)
+ return ERR_PTR(-ENODEV);
- airoha_qdma_set(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_TRTCM_EN_MASK);
- airoha_qdma_clear(qdma, REG_INGRESS_TRTCM_CFG,
- INGRESS_TRTCM_MODE_MASK);
- airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_FAST_TICK_MASK,
- FIELD_PREP(INGRESS_FAST_TICK_MASK, 125));
- airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG,
- INGRESS_SLOW_TICK_RATIO_MASK,
- FIELD_PREP(INGRESS_SLOW_TICK_RATIO_MASK, 8));
+ if (d >= ARRAY_SIZE(port->devs))
+ return ERR_PTR(-ENODEV);
- airoha_qdma_set(qdma, REG_SLA_TRTCM_CFG, SLA_TRTCM_EN_MASK);
- airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_FAST_TICK_MASK,
- FIELD_PREP(SLA_FAST_TICK_MASK, 25));
- airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_SLOW_TICK_RATIO_MASK,
- FIELD_PREP(SLA_SLOW_TICK_RATIO_MASK, 40));
+ return port->devs[d] ? port->devs[d] : ERR_PTR(-ENODEV);
+}
+
+static struct sk_buff *airoha_qdma_lro_rx_skb(struct airoha_queue *q,
+ struct airoha_qdma_desc *desc,
+ struct airoha_queue_entry *e)
+{
+ u32 len, th_off, tcp_ack_seq, agg_count, data_off, data_len;
+ u32 desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
+ u32 msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
+ u32 msg2 = le32_to_cpu(READ_ONCE(desc->msg2));
+ u32 msg3 = le32_to_cpu(READ_ONCE(desc->msg3));
+ struct skb_shared_info *shinfo;
+ u16 tcp_win, l2_len;
+ struct sk_buff *skb;
+ struct tcphdr *th;
+ struct page *page;
+ bool ipv4, ipv6;
+
+ switch (q->qdma->eth->soc->version) {
+ case econet_en751221:
+ case econet_en7528:
+ case econet_en7580:
+ return NULL;
+ case airoha_en7523:
+ ipv4 = FIELD_GET(EN7523_QDMA_ETH_RXMSG_IP4_MASK, msg1);
+ ipv6 = FIELD_GET(EN7523_QDMA_ETH_RXMSG_IP6_MASK, msg1);
+ break;
+ case airoha_en7581:
+ case airoha_an7583:
+ ipv4 = FIELD_GET(QDMA_ETH_RXMSG_IP4_MASK, msg1);
+ ipv6 = FIELD_GET(QDMA_ETH_RXMSG_IP6_MASK, msg1);
+ break;
+ }
+ if (!ipv4 && !ipv6)
+ return NULL;
+
+ l2_len = FIELD_GET(QDMA_ETH_RXMSG_L2_LEN_MASK, msg2);
+ len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
+
+ if (ipv4) {
+ struct iphdr *iph;
+
+ if (len < l2_len + sizeof(*iph))
+ return NULL;
+
+ iph = (struct iphdr *)(e->buf + l2_len);
+ if (iph->protocol != IPPROTO_TCP)
+ return NULL;
+
+ if (iph->ihl < 5)
+ return NULL;
+
+ th_off = l2_len + (iph->ihl << 2);
+ if (len < th_off)
+ return NULL;
+
+ iph->tot_len = cpu_to_be16(len - l2_len);
+ iph->check = 0;
+ iph->check = ip_fast_csum((void *)iph, iph->ihl);
+ } else {
+ struct ipv6hdr *ip6h;
+
+ th_off = l2_len + sizeof(*ip6h);
+ if (len < th_off)
+ return NULL;
+
+ ip6h = (struct ipv6hdr *)(e->buf + l2_len);
+ if (ip6h->nexthdr != NEXTHDR_TCP)
+ return NULL;
+
+ ip6h->payload_len = cpu_to_be16(len - th_off);
+ }
+
+ if (len < th_off + sizeof(*th))
+ return NULL;
+
+ th = (struct tcphdr *)(e->buf + th_off);
+ if (th->doff < 5)
+ return NULL;
+
+ data_off = th_off + (th->doff << 2);
+ if (len < data_off)
+ return NULL;
+
+ tcp_win = FIELD_GET(QDMA_ETH_RXMSG_TCP_WIN_MASK, msg3);
+ tcp_ack_seq = le32_to_cpu(READ_ONCE(desc->data));
+ th->ack_seq = cpu_to_be32(tcp_ack_seq);
+ th->window = cpu_to_be16(tcp_win);
+
+ /* Check tcp timestamp option */
+ if (th->doff == (sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4) {
+ u32 topt = get_unaligned_be32(th + 1);
+
+ if (topt == ((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
+ (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) {
+ u8 *ptr = (u8 *)th + sizeof(*th) + 2 * sizeof(__be32);
+ __le32 tcp_ts_reply = READ_ONCE(desc->tcp_ts_reply);
+
+ put_unaligned_be32(le32_to_cpu(tcp_ts_reply), ptr);
+ }
+ }
+
+ if (ipv4) {
+ struct iphdr *iph = (struct iphdr *)(e->buf + l2_len);
+
+ th->check = ~tcp_v4_check(len - th_off, iph->saddr,
+ iph->daddr, 0);
+ } else {
+ struct ipv6hdr *ip6h = (struct ipv6hdr *)(e->buf + l2_len);
+
+ th->check = ~tcp_v6_check(len - th_off, &ip6h->saddr,
+ &ip6h->daddr, 0);
+ }
+
+ skb = napi_alloc_skb(&q->napi, data_off);
+ if (!skb)
+ return NULL;
+
+ __skb_put(skb, data_off);
+ memcpy(skb->data, e->buf, data_off);
+
+ page = virt_to_head_page(e->buf);
+ data_len = len - data_off;
+ shinfo = skb_shinfo(skb);
+ skb_add_rx_frag(skb, shinfo->nr_frags, page,
+ e->buf + data_off - page_address(page), data_len,
+ q->buf_size);
+
+ shinfo->gso_type = ipv4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
+ agg_count = FIELD_GET(QDMA_ETH_RXMSG_AGG_COUNT_MASK, msg2);
+ shinfo->gso_size = DIV_ROUND_UP(data_len, agg_count);
+ shinfo->gso_segs = agg_count;
+
+ skb->csum_start = skb_headroom(skb) + th_off;
+ skb->csum_offset = offsetof(struct tcphdr, check);
+ skb->ip_summed = CHECKSUM_PARTIAL;
+
+ return skb;
+}
+
+static bool airoha_qdma_rx_checksum_ok(struct airoha_eth *eth,
+ struct airoha_qdma_desc *desc)
+{
+ u32 msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
+ u32 ip4, ip6, ip4_fault, l4_valid, l4_fault;
+
+ if (airoha_is(eth, airoha_en7523, econet_en7580)) {
+ ip4 = msg1 & EN7523_QDMA_ETH_RXMSG_IP4_MASK;
+ ip6 = msg1 & EN7523_QDMA_ETH_RXMSG_IP6_MASK;
+ ip4_fault = msg1 & EN7523_QDMA_ETH_RXMSG_IP4F_MASK;
+ l4_valid = msg1 & EN7523_QDMA_ETH_RXMSG_L4_VALID_MASK;
+ l4_fault = msg1 & EN7523_QDMA_ETH_RXMSG_L4F_MASK;
+ } else {
+ ip4 = msg1 & QDMA_ETH_RXMSG_IP4_MASK;
+ ip6 = msg1 & QDMA_ETH_RXMSG_IP6_MASK;
+ ip4_fault = msg1 & QDMA_ETH_RXMSG_IP4F_MASK;
+ l4_valid = msg1 & QDMA_ETH_RXMSG_L4_VALID_MASK;
+ l4_fault = msg1 & QDMA_ETH_RXMSG_L4F_MASK;
+ }
+
+ return (ip4 || ip6) && l4_valid && !ip4_fault && !l4_fault;
+}
+
+static struct sk_buff *airoha_qdma_build_rx_skb(struct airoha_queue *q,
+ struct airoha_qdma_desc *desc,
+ struct airoha_queue_entry *e,
+ struct net_device *netdev,
+ bool raw)
+{
+ u32 msg2 = le32_to_cpu(READ_ONCE(desc->msg2));
+ int qid = q - &q->qdma->q_rx[0];
+ struct sk_buff *skb;
+
+ if (!raw && FIELD_GET(QDMA_ETH_RXMSG_AGG_COUNT_MASK, msg2) > 1) {
/* LRO */
+ skb = airoha_qdma_lro_rx_skb(q, desc, e);
+ if (!skb)
+ return NULL;
+ } else {
+ u32 desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
+ u32 len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
+
+ skb = napi_build_skb(e->buf, q->buf_size);
+ if (!skb)
+ return NULL;
+
+ __skb_put(skb, len);
+ if ((netdev->features & NETIF_F_RXCSUM) &&
+ airoha_qdma_rx_checksum_ok(q->qdma->eth, desc))
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ else
+ skb->ip_summed = CHECKSUM_NONE;
+ }
+
+ skb_mark_for_recycle(skb);
+ skb->dev = netdev;
+ skb_record_rx_queue(skb, qid);
+ if (raw) {
+ skb_reset_mac_header(skb);
+ skb->protocol = 0;
+ } else {
+ skb->protocol = eth_type_trans(skb, netdev);
+ }
+
+ return skb;
+}
+
+
+static bool airoha_qdma_foe_entry_is_valid(struct airoha_eth *eth, u32
hash)
+{
+ return airoha_is(eth, airoha_en7523, econet_en7580) ?
+ hash != EN7523_AIROHA_RXD4_FOE_ENTRY_INVALID :
+ hash != AN7581_AIROHA_RXD4_FOE_ENTRY_INVALID;
+}
+
+static bool airoha_qdma_should_check_ppe_skb(struct airoha_eth *eth,
+ u32 reason)
+{
+ if (reason == AIROHA_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
+ return true;
+
+ if (!airoha_is(eth, airoha_en7523))
+ return false;
+
+ return reason == AIROHA_PPE_CPU_REASON_HIT_UNBIND ||
+ reason == AIROHA_PPE_CPU_REASON_FOE_UNHIT;
+}
+
+static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
+{
+ enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
+ struct airoha_eth *eth = q->qdma->eth;
+ int done = 0;
+
+ while (done < budget) {
+ struct airoha_queue_entry *e = &q->entry[q->tail];
+ struct airoha_qdma_desc *desc = &q->desc[q->tail];
+ u32 hash, reason, msg0, msg1, desc_ctrl;
+ struct airoha_gdm_dev *dev;
+ struct net_device *netdev;
+ bool xpon_oam;
+ int data_len, len;
+ struct page *page;
+
+ desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
+ if (!(desc_ctrl & QDMA_DESC_DONE_MASK))
+ break;
+
+ dma_rmb();
+
+ q->tail = (q->tail + 1) % q->ndesc;
+ q->queued--;
+
+ dma_sync_single_for_cpu(eth->dev, e->dma_addr,
+ SKB_WITH_OVERHEAD(q->buf_size), dir);
+
+ page = virt_to_head_page(e->buf);
+ len = airoha_is(eth, airoha_en7523, econet_en7580) ?
+ FIELD_GET(EN7523_QDMA_DESC_LEN_MASK, desc_ctrl) :
+ FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
+ data_len = q->skb ? q->buf_size
+ : SKB_WITH_OVERHEAD(q->buf_size);
+ if (!len || data_len < len)
+ goto free_frag;
+
+ msg0 = le32_to_cpu(READ_ONCE(desc->msg0));
+ if (airoha_is(eth, airoha_en7523) &&
+ q - &q->qdma->q_rx[0] == 15) {
+ u32 msg2 = le32_to_cpu(READ_ONCE(desc->msg2));
+ u32 msg3 = le32_to_cpu(READ_ONCE(desc->msg3));
+ unsigned int channel, gem_port_id;
+
+ msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
+ channel = FIELD_GET(EN7523_QDMA_ETH_RXMSG_CHAN_MASK, msg0);
+ gem_port_id = FIELD_GET(EN7523_QDMA_ETH_RXMSG_GEM_MASK, msg0);
+ dev_dbg_ratelimited(eth->dev,
+ "QDMA RX15 descriptor: len=%d ctrl=%#010x
msg=%#010x/%#010x/%#010x/%#010x oam=%u channel=%u gem=%u no-mic=%u\n",
+ len, desc_ctrl, msg0, msg1, msg2, msg3,
+ !!(msg0 & EN7523_QDMA_ETH_RXMSG_OAM_MASK),
+ channel, gem_port_id,
+ !!(msg0 & EN7523_QDMA_ETH_RXMSG_NO_MIC_MASK));
+ }
+ xpon_oam = airoha_is(eth, airoha_en7523) &&
+ (msg0 & EN7523_QDMA_ETH_RXMSG_OAM_MASK);
+
+ /* GPON OAM descriptors are not required to carry an Ethernet
+ * source port. Resolve them through the registered xPON GDM2
+ * provider before applying the normal source-port decoder.
+ */
+ if (xpon_oam)
+ dev = airoha_qdma_get_xpon_dev(eth);
+ else
+ dev = airoha_qdma_get_gdm_dev(
+ eth, airoha_qdma_get_rx_sport(eth, desc));
+ if (IS_ERR(dev)) {
+ if (xpon_oam)
+ dev_warn_ratelimited(eth->dev,
+ "xPON OAM RX dropped: no managed GDM2
ring=%td len=%d msg0=%#010x msg1=%#010x\n",
+ q - &q->qdma->q_rx[0], len, msg0,
+ le32_to_cpu(READ_ONCE(desc->msg1)));
+ goto free_frag;
+ }
+
+ netdev = netdev_from_priv(dev);
+ if (!q->skb) { /* first buffer */
+ q->skb = airoha_qdma_build_rx_skb(q, desc, e, netdev,
xpon_oam);
+ if (!q->skb)
+ goto free_frag;
+ } else { /* scattered frame */
+ struct skb_shared_info *shinfo = skb_shinfo(q->skb);
+ int nr_frags = shinfo->nr_frags;
+
+ if (nr_frags >= ARRAY_SIZE(shinfo->frags))
+ goto free_frag;
+
+ skb_add_rx_frag(q->skb, nr_frags, page,
+ e->buf - page_address(page), len,
+ q->buf_size);
+ }
+
+ if (FIELD_GET(QDMA_DESC_MORE_MASK, desc_ctrl))
+ continue;
+
+ if (airoha_is(eth, airoha_en7523)) {
+ struct airoha_xpon_oam_handler *handler;
+
+ if (xpon_oam) {
+ u32 msg2 = le32_to_cpu(READ_ONCE(desc->msg2));
+ u32 msg3 = le32_to_cpu(READ_ONCE(desc->msg3));
+ u16 gem_port_id, channel, sport;
+ u32 flags = 0, skb_len;
+ bool consumed = false;
+
+ gem_port_id = FIELD_GET(EN7523_QDMA_ETH_RXMSG_GEM_MASK,
+ msg0);
+ channel = FIELD_GET(EN7523_QDMA_ETH_RXMSG_CHAN_MASK,
+ msg0);
+ msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
+ sport = FIELD_GET(EN7523_QDMA_ETH_RXMSG_SPORT_MASK,
+ msg1);
+ if (!(msg0 & EN7523_QDMA_ETH_RXMSG_NO_MIC_MASK))
+ flags |= AIROHA_XPON_OAM_RX_F_MIC_PRESENT;
+ if (!(msg0 & EN7523_QDMA_ETH_RXMSG_CRC_ERR_MASK))
+ flags |= AIROHA_XPON_OAM_RX_F_MIC_VALID;
+ else
+ flags |= AIROHA_XPON_OAM_RX_F_CRC_ERROR;
+
+ skb_len = q->skb->len;
+ atomic64_inc(&dev->xpon_oam_rx_packets);
+ atomic64_add(skb_len, &dev->xpon_oam_rx_bytes);
+
+ rcu_read_lock();
+ handler = rcu_dereference(dev->xpon_oam);
+ if (handler && handler->rx)
+ consumed = handler->rx(handler->priv,
+ q->skb, channel,
+ gem_port_id, flags);
+ else
+ atomic64_inc(&dev->xpon_oam_rx_no_handler);
+ rcu_read_unlock();
+
+ if (consumed) {
+ atomic64_inc(&dev->xpon_oam_rx_delivered);
+ } else {
+ atomic64_inc(&dev->xpon_oam_rx_dropped);
+ dev_kfree_skb_any(q->skb);
+ }
+
+ dev_dbg_ratelimited(&netdev->dev,
+ "xPON OAM RX: ring=%td len=%u
msg=%#010x/%#010x/%#010x/%#010x sport=%u channel=%u gem=%u no-mic=%u
crc=%u runt=%u long=%u consumed=%u totals=%lld/%lld/%lld\n",
+ q - &q->qdma->q_rx[0], skb_len,
+ msg0, msg1, msg2, msg3, sport, channel,
+ gem_port_id,
+ !!(msg0 & EN7523_QDMA_ETH_RXMSG_NO_MIC_MASK),
+ !!(msg0 & EN7523_QDMA_ETH_RXMSG_CRC_ERR_MASK),
+ !!(msg0 & EN7523_QDMA_ETH_RXMSG_RUNT_MASK),
+ !!(msg0 & EN7523_QDMA_ETH_RXMSG_LONG_MASK),
+ consumed,
+ (long long)atomic64_read(&dev->xpon_oam_rx_packets),
+ (long long)atomic64_read(&dev->xpon_oam_rx_delivered),
+ (long long)atomic64_read(&dev->xpon_oam_rx_dropped));
+
+ q->skb = NULL;
+ done++;
+ continue;
+ }
+ }
+
+ if (netdev_uses_dsa(netdev)) {
+ struct airoha_gdm_port *port = dev->port;
+ u32 msg0, sptag, sport;
+
+ /* PPE module requires untagged packets to work
+ * properly and it provides DSA port index via the
+ * DMA descriptor. Report DSA tag to the DSA stack
+ * via skb dst info.
+ */
+ msg0 = le32_to_cpu(READ_ONCE(desc->msg0));
+ if (airoha_is(eth, econet_en7580)) {
+ sptag = FIELD_GET(EN7580_QDMA_ETH_RXMSG_SPTAG,
+ msg0);
+ sport = airoha_qdma_get_rx_sport(eth, desc);
+
+ /* MT7530 ingress SPORT 0x10..0x14 = ports 0..4. */
+ if (sport >= 0x10 && sport <= 0x14)
+ sptag = sport - 0x10;
+ } else {
+ sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, msg0);
+ }
+
+ if (sptag < ARRAY_SIZE(port->dsa_meta) &&
+ port->dsa_meta[sptag])
+ skb_dst_set_noref(q->skb,
+ &port->dsa_meta[sptag]->dst);
+ }
+
+ msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
+ hash = airoha_is(eth, airoha_en7523, econet_en7580) ?
+ FIELD_GET(EN7523_QDMA_ETH_RXMSG_PPE_ENTRY_MASK, msg1) :
+ FIELD_GET(AIROHA_RXD4_FOE_ENTRY, msg1);
+
+ reason = airoha_is(eth, airoha_en7523, econet_en7580) ?
+ FIELD_GET(EN7523_QDMA_ETH_RXMSG_CRSN_MASK, msg1) :
+ FIELD_GET(AIROHA_RXD4_PPE_CPU_REASON, msg1);
+
+ if (airoha_qdma_foe_entry_is_valid(eth, hash))
+ skb_set_hash(q->skb, jhash_1word(hash, 0), PKT_HASH_TYPE_L4);
+
+ if (airoha_qdma_foe_entry_is_valid(eth, hash) &&
+ airoha_qdma_should_check_ppe_skb(eth, reason))
+ airoha_ppe_check_skb(ð->ppe->common.dev, q->skb, hash,
+ false);
+
+ done++;
+ napi_gro_receive(&q->napi, q->skb);
+ q->skb = NULL;
+ continue;
+free_frag:
+ if (q->skb) {
+ dev_kfree_skb(q->skb);
+ q->skb = NULL;
+ }
+ page_pool_put_full_page(q->page_pool, page, true);
+ }
+ airoha_qdma_fill_rx_queue(q);
+
+ return done;
+}
+
+static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget)
+{
+ struct airoha_queue *q = container_of(napi, struct airoha_queue, napi);
+ int cur, done = 0;
+
+ do {
+ cur = airoha_qdma_rx_process(q, budget - done);
+ done += cur;
+ } while (cur && done < budget);
+
+ if (done < budget && napi_complete(napi)) {
+ struct airoha_qdma *qdma = q->qdma;
+ int i, qid = q - &qdma->q_rx[0];
+ int intr_reg = qid < RX_DONE_HIGH_OFFSET ? QDMA_INT_REG_IDX1
+ : QDMA_INT_REG_IDX2;
+
+ for (i = 0; i < qdma->eth->soc->irq_banks; i++) {
+ if (!(BIT(qid) & RX_IRQ_BANK_PIN_MASK(i)))
+ continue;
+
+ airoha_qdma_irq_enable(&qdma->irq_banks[i], intr_reg,
+ BIT(qid % RX_DONE_HIGH_OFFSET));
+ }
+ }
+
+ return done;
+}
+
+static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
+ struct airoha_qdma *qdma, int ndesc)
+{
+ struct page_pool_params pp_params = {
+ .pool_size = 256,
+ .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+ .dma_dir = DMA_FROM_DEVICE,
+ .nid = NUMA_NO_NODE,
+ .dev = qdma->eth->dev,
+ .napi = &q->napi,
+ };
+ struct airoha_eth *eth = qdma->eth;
+ int qid = q - &qdma->q_rx[0], thr;
+ dma_addr_t dma_addr;
+ bool lro_q;
+
+ q->qdma = qdma;
+ lro_q = airoha_qdma_is_lro_queue(q);
+
+ q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry),
+ GFP_KERNEL);
+ if (!q->entry)
+ return -ENOMEM;
+
+ q->desc = dmam_alloc_coherent(eth->dev, ndesc * sizeof(*q->desc),
+ &dma_addr, GFP_KERNEL);
+ if (!q->desc)
+ return -ENOMEM;
+
+ pp_params.order = lro_q ?
+ (airoha_is(eth, airoha_en7523) ? EN7523_AIROHA_LRO_PAGE_ORDER :
+ AIROHA_LRO_PAGE_ORDER) : 0;
+ pp_params.max_len = PAGE_SIZE << pp_params.order;
+
+ q->page_pool = page_pool_create(&pp_params);
+ if (IS_ERR(q->page_pool)) {
+ int err = PTR_ERR(q->page_pool);
+
+ q->page_pool = NULL;
+ return err;
+ }
+
+ if (lro_q && airoha_is(eth, airoha_en7523))
+ q->buf_size = SKB_HEAD_ALIGN(EN7523_AIROHA_RXQ_LRO_MAX_AGG_SIZE);
+ else
+ q->buf_size = lro_q ? pp_params.max_len : pp_params.max_len / 2;
+ q->ndesc = ndesc;
+ netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll);
+
+ airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr);
+ airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid),
+ RX_RING_SIZE_MASK,
+ FIELD_PREP(RX_RING_SIZE_MASK, ndesc));
+
+ thr = clamp(ndesc >> 3, 1, 32);
+ airoha_qdma_rmw(qdma, REG_RX_RING_SIZE(qid), RX_RING_THR_MASK,
+ FIELD_PREP(RX_RING_THR_MASK, thr));
+ airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
+ FIELD_PREP(RX_RING_DMA_IDX_MASK, q->head));
+ if (lro_q || (airoha_is(eth, airoha_en7523) && qid == 15))
+ airoha_qdma_clear(qdma, REG_RX_SCATTER_CFG(qid),
+ RX_RING_SG_EN_MASK);
+ else
+ airoha_qdma_set(qdma, REG_RX_SCATTER_CFG(qid),
+ RX_RING_SG_EN_MASK);
+
+ airoha_qdma_fill_rx_queue(q);
+
+ return 0;
+}
+
+static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
+{
+ struct airoha_qdma *qdma = q->qdma;
+ struct airoha_eth *eth = qdma->eth;
+ int qid = q - &qdma->q_rx[0];
+
+ while (q->queued) {
+ struct airoha_queue_entry *e = &q->entry[q->tail];
+ struct airoha_qdma_desc *desc = &q->desc[q->tail];
+ struct page *page = virt_to_head_page(e->buf);
+
+ dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len,
+ page_pool_get_dma_dir(q->page_pool));
+ page_pool_put_full_page(q->page_pool, page, false);
+ /* Reset DMA descriptor */
+ WRITE_ONCE(desc->tcp_ts_reply, 0);
+ WRITE_ONCE(desc->ctrl, 0);
+ WRITE_ONCE(desc->addr, 0);
+ WRITE_ONCE(desc->data, 0);
+ WRITE_ONCE(desc->msg0, 0);
+ WRITE_ONCE(desc->msg1, 0);
+ WRITE_ONCE(desc->msg2, 0);
+ WRITE_ONCE(desc->msg3, 0);
+
+ q->tail = (q->tail + 1) % q->ndesc;
+ q->queued--;
+ }
+
+ q->head = q->tail;
+ /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is
+ * empty.
+ */
+ airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK,
+ FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head));
+ airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
+ FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail));
+}
+
+static int airoha_qdma_modern_init_rx(struct airoha_qdma *qdma)
+{
+ int i;
+
+ for (i = 0; i < qdma->eth->soc->rx_ring; i++) {
+ int err;
+
+ if (!(RX_DONE_INT_MASK & BIT(i))) {
+ /* rx-queue not binded to irq */
+ continue;
+ }
+
+ err = airoha_qdma_init_rx_queue(&qdma->q_rx[i], qdma,
+ RX_DSCP_NUM(i));
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q)
+{
+ struct airoha_qdma *qdma = q->qdma;
+ struct airoha_eth *eth = qdma->eth;
+ int i, qid = q - &qdma->q_tx[0];
+
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
+ struct airoha_gdm_port *port = eth->ports[i];
+ int d;
+
+ if (!port)
+ continue;
+
+ for (d = 0; d < ARRAY_SIZE(port->devs); d++) {
+ struct airoha_gdm_dev *dev = port->devs[d];
+ struct net_device *netdev;
+ int j;
+
+ if (!dev)
+ continue;
+
+ if (rcu_access_pointer(dev->qdma) != qdma)
+ continue;
+
+ netdev = netdev_from_priv(dev);
+ for (j = 0; j < netdev->num_tx_queues; j++) {
+ if (airoha_qdma_get_txq(qdma, j) != qid)
+ continue;
+
+ netif_wake_subqueue(netdev, j);
+ }
+ }
+ }
+ q->txq_stopped = false;
+}
+
+void
+airoha_qdma_unmap_tx_entry(struct airoha_eth *eth,
+ struct airoha_queue_entry *e)
+{
+ if (e->dma_map_page)
+ dma_unmap_page(eth->dev, e->dma_addr, e->dma_len,
+ DMA_TO_DEVICE);
+ else
+ dma_unmap_single(eth->dev, e->dma_addr, e->dma_len,
+ DMA_TO_DEVICE);
+
+ e->dma_addr = 0;
+ e->dma_len = 0;
+ e->dma_map_page = false;
+}
+
+static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget)
+{
+ struct airoha_tx_irq_queue *irq_q;
+ int id, done = 0, irq_queued;
+ struct airoha_qdma *qdma;
+ struct airoha_eth *eth;
+ u32 status, head;
+
+ irq_q = container_of(napi, struct airoha_tx_irq_queue, napi);
+ qdma = irq_q->qdma;
+ id = irq_q - &qdma->q_tx_irq[0];
+ eth = qdma->eth;
+
+ status = airoha_qdma_rr(qdma, REG_IRQ_STATUS(id));
+ head = FIELD_GET(IRQ_HEAD_IDX_MASK, status);
+ head = head % irq_q->size;
+ irq_queued = FIELD_GET(IRQ_ENTRY_LEN_MASK, status);
+
+ while (irq_queued > 0 && done < budget) {
+ struct airoha_qdma_desc *desc;
+ struct airoha_queue_entry *e;
+ struct airoha_queue *q;
+ u32 qid, val, index, desc_ctrl;
+ struct sk_buff *skb;
+ int retry;
+
+ /*
+ * The IRQ status can become visible before the corresponding
+ * completion entry reaches coherent memory. The vendor driver
+ * retries this read for the same reason.
+ */
+ for (retry = 0; retry < 16; retry++) {
+ val = READ_ONCE(irq_q->q[head]);
+ if (val != U32_MAX)
+ break;
+
+ dma_rmb();
+ cpu_relax();
+ }
+ if (val == U32_MAX)
+ break;
+
+ /*
+ * The completion entry is published after the descriptor
writeback.
+ * Order the descriptor read after the queue entry read.
+ */
+ dma_rmb();
+
+ qid = FIELD_GET(IRQ_RING_IDX_MASK, val);
+ if (qid >= eth->soc->tx_ring)
+ goto consume;
+
+ q = &qdma->q_tx[qid];
+ if (!q->ndesc)
+ goto consume;
+
+ index = FIELD_GET(IRQ_DESC_IDX_MASK, val);
+ if (index >= q->ndesc)
+ goto consume;
+
+ spin_lock_bh(&q->lock);
+
+ if (!q->queued)
+ goto unlock_consume;
+
+ e = &q->entry[index];
+ if (!e->dma_addr)
+ goto unlock_consume;
+
+ desc = &q->desc[index];
+ desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
+
+ if (!(desc_ctrl & QDMA_DESC_DONE_MASK) &&
+ !(desc_ctrl & QDMA_DESC_DROP_MASK)) {
+ spin_unlock_bh(&q->lock);
+ break;
+ }
+
+ skb = e->skb;
+ airoha_qdma_unmap_tx_entry(eth, e);
+ e->skb = NULL;
+ list_add_tail(&e->list, &q->tx_list);
+
+ WRITE_ONCE(desc->ctrl, 0);
+ WRITE_ONCE(desc->addr, 0);
+ WRITE_ONCE(desc->data, 0);
+ WRITE_ONCE(desc->msg0, 0);
+ WRITE_ONCE(desc->msg1, 0);
+ WRITE_ONCE(desc->msg2, 0);
+ q->queued--;
+
+ if (skb) {
+ struct netdev_queue *txq;
+
+ txq = skb_get_tx_queue(skb->dev, skb);
+ netdev_tx_completed_queue(txq, 1, skb->len);
+ dev_kfree_skb_any(skb);
+ }
+
+ if (q->txq_stopped && q->ndesc - q->queued >= q->free_thr) {
+ /* Since multiple net_device TX queues can share the
+ * same hw QDMA TX queue, there is no guarantee we have
+ * inflight packets queued in hw belonging to a
+ * net_device TX queue stopped in the xmit path.
+ * In order to avoid any potential net_device TX queue
+ * stall, we need to wake all the net_device TX queues
+ * feeding the same hw QDMA TX queue.
+ */
+ airoha_qdma_wake_netdev_txqs(q);
+ }
+
+unlock_consume:
+ spin_unlock_bh(&q->lock);
+consume:
+ WRITE_ONCE(irq_q->q[head], U32_MAX);
+ head = (head + 1) % irq_q->size;
+ irq_queued--;
+ done++;
+ }
+
+ if (done) {
+ int i, len = done >> 7;
+
+ /* Publish empty markers before returning entries to hardware. */
+ dma_wmb();
+
+ for (i = 0; i < len; i++)
+ airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
+ IRQ_CLEAR_LEN_MASK, 0x80);
+ airoha_qdma_rmw(qdma, REG_IRQ_CLEAR_LEN(id),
+ IRQ_CLEAR_LEN_MASK, (done & 0x7f));
+ }
+
+ if (done < budget && napi_complete(napi))
+ airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0,
+ TX_DONE_INT_MASK(id));
+
+ return done;
+}
+
+static int airoha_qdma_init_tx_queue(struct airoha_queue *q,
+ struct airoha_qdma *qdma, int size)
+{
+ struct airoha_eth *eth = qdma->eth;
+ int i, qid = q - &qdma->q_tx[0];
+ dma_addr_t dma_addr;
+
+ spin_lock_init(&q->lock);
+ q->qdma = qdma;
+ q->free_thr = 1 + MAX_SKB_FRAGS;
+ INIT_LIST_HEAD(&q->tx_list);
+
+ q->entry = devm_kzalloc(eth->dev, size * sizeof(*q->entry),
+ GFP_KERNEL);
+ if (!q->entry)
+ return -ENOMEM;
+
+ q->desc = dmam_alloc_coherent(eth->dev, size * sizeof(*q->desc),
+ &dma_addr, GFP_KERNEL);
+ if (!q->desc)
+ return -ENOMEM;
+
+ for (i = 0; i < size; i++) {
+ u32 val = FIELD_PREP(QDMA_DESC_DONE_MASK, 1);
+
+ list_add_tail(&q->entry[i].list, &q->tx_list);
+ WRITE_ONCE(q->desc[i].ctrl, cpu_to_le32(val));
+ }
+ q->ndesc = size;
+
+ /* xmit ring drop default setting */
+ if (!airoha_is(eth, airoha_en7523))
+ airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(qid),
+ TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK);
+
+ airoha_qdma_wr(qdma, REG_TX_RING_BASE(qid), dma_addr);
+ airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK,
+ FIELD_PREP(TX_RING_CPU_IDX_MASK, 0));
+ airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK,
+ FIELD_PREP(TX_RING_DMA_IDX_MASK, 0));
+
+ return 0;
+}
+
+static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q,
+ struct airoha_qdma *qdma, int size)
+{
+ int id = irq_q - &qdma->q_tx_irq[0];
+ struct airoha_eth *eth = qdma->eth;
+ dma_addr_t dma_addr;
+
+ irq_q->q = dmam_alloc_coherent(eth->dev, size * sizeof(u32),
+ &dma_addr, GFP_KERNEL);
+ if (!irq_q->q)
+ return -ENOMEM;
+
+ memset(irq_q->q, 0xff, size * sizeof(u32));
+ irq_q->size = size;
+ irq_q->qdma = qdma;
+
+ netif_napi_add_tx(eth->napi_dev, &irq_q->napi,
+ airoha_qdma_tx_napi_poll);
+
+ airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr);
+ airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK,
+ FIELD_PREP(TX_IRQ_DEPTH_MASK, size));
+ airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_THR_MASK,
+ FIELD_PREP(TX_IRQ_THR_MASK, 1));
+
+ return 0;
+}
+
+static int airoha_qdma_modern_init_tx(struct airoha_qdma *qdma)
+{
+ int i, err;
+
+ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
+ err = airoha_qdma_tx_irq_init(&qdma->q_tx_irq[i], qdma,
+ IRQ_QUEUE_LEN(i));
+ if (err)
+ return err;
+ }
+
+ for (i = 0; i < qdma->eth->soc->tx_ring; i++) {
+ err = airoha_qdma_init_tx_queue(&qdma->q_tx[i], qdma,
+ TX_DSCP_NUM(i));
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+void airoha_qdma_cleanup_tx_queue(struct airoha_queue *q)
+{
+ struct airoha_qdma *qdma = q->qdma;
+ struct airoha_eth *eth = qdma->eth;
+ int i, qid = q - &qdma->q_tx[0];
+ u16 index = 0;
+
+ spin_lock_bh(&q->lock);
+ for (i = 0; i < q->ndesc; i++) {
+ struct airoha_queue_entry *e = &q->entry[i];
+ struct airoha_qdma_desc *desc = &q->desc[i];
+
+ if (!e->dma_addr)
+ continue;
+
+ airoha_qdma_unmap_tx_entry(eth, e);
+ dev_kfree_skb_any(e->skb);
+ e->skb = NULL;
+ list_add_tail(&e->list, &q->tx_list);
+
+ /* Reset DMA descriptor */
+ WRITE_ONCE(desc->ctrl, 0);
+ WRITE_ONCE(desc->addr, 0);
+ WRITE_ONCE(desc->data, 0);
+ WRITE_ONCE(desc->msg0, 0);
+ WRITE_ONCE(desc->msg1, 0);
+ WRITE_ONCE(desc->msg2, 0);
+
+ q->queued--;
+ }
+
+ if (!list_empty(&q->tx_list)) {
+ struct airoha_queue_entry *e;
+
+ e = list_first_entry(&q->tx_list, struct airoha_queue_entry,
+ list);
+ index = e - q->entry;
+ }
+ /* Set TX_DMA_IDX to TX_CPU_IDX to notify the hw the QDMA TX ring is
+ * empty.
+ */
+ airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK,
+ FIELD_PREP(TX_RING_CPU_IDX_MASK, index));
+ airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK,
+ FIELD_PREP(TX_RING_DMA_IDX_MASK, index));
+
+ spin_unlock_bh(&q->lock);
+}
+
+static int airoha_qdma_modern_init_hfwd(struct airoha_qdma *qdma)
+{
+ int size, index, num_desc = HW_DSCP_NUM;
+ struct airoha_eth *eth = qdma->eth;
+ int id = qdma - ð->qdma[0];
+ u32 status, buf_size;
+ dma_addr_t dma_addr;
+ const char *name;
+
+ name = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d-buf", id);
+ if (!name)
+ return -ENOMEM;
+
+ /* EN7523 reserves 32 MiB for QDMA0/LAN and 16 MiB for QDMA1/WAN.
+ * Keep the vendor 16K HWFWD descriptors on both instances by using
+ * 2 KiB payloads on LAN and 1 KiB payloads on WAN.
+ */
+ buf_size = id ? AIROHA_MAX_PACKET_SIZE / 2 :
+ AIROHA_MAX_PACKET_SIZE;
+ index = of_property_match_string(eth->dev->of_node,
+ "memory-region-names", name);
+ if (index >= 0) {
+ struct reserved_mem *rmem;
+ struct device_node *np;
+
+ /* Consume reserved memory for hw forwarding buffers queue if
+ * available in the DTS
+ */
+ np = of_parse_phandle(eth->dev->of_node, "memory-region",
+ index);
+ if (!np)
+ return -ENODEV;
+
+ rmem = of_reserved_mem_lookup(np);
+ of_node_put(np);
+ if (!rmem)
+ return -ENODEV;
+
+ dma_addr = rmem->base;
+ /* Compute the number of hw descriptors according to the
+ * reserved memory size and the payload buffer size
+ */
+ num_desc = div_u64(rmem->size, buf_size);
+ } else {
+ size = buf_size * num_desc;
+ if (!dmam_alloc_coherent(eth->dev, size, &dma_addr,
+ GFP_KERNEL))
+ return -ENOMEM;
+ }
+
+ airoha_qdma_wr(qdma, REG_FWD_BUF_BASE, dma_addr);
+
+ size = num_desc * sizeof(struct airoha_qdma_fwd_desc);
+ if (!dmam_alloc_coherent(eth->dev, size, &dma_addr, GFP_KERNEL))
+ return -ENOMEM;
+
+ airoha_qdma_wr(qdma, REG_FWD_DSCP_BASE, dma_addr);
+ dev_info(eth->dev,
+ "QDMA%d HWFWD: payload=%u bytes descriptors=%d\n",
+ id, buf_size, num_desc);
+ airoha_qdma_rmw(qdma, REG_HW_FWD_DSCP_CFG,
+ HW_FWD_DSCP_PAYLOAD_SIZE_MASK,
+ FIELD_PREP(HW_FWD_DSCP_PAYLOAD_SIZE_MASK,
+ buf_size != AIROHA_MAX_PACKET_SIZE));
+ airoha_qdma_rmw(qdma, REG_FWD_DSCP_LOW_THR, FWD_DSCP_LOW_THR_MASK,
+ FIELD_PREP(FWD_DSCP_LOW_THR_MASK, 128));
+ airoha_qdma_rmw(qdma, REG_LMGR_INIT_CFG,
+ LMGR_INIT_START | LMGR_SRAM_MODE_MASK |
+ HW_FWD_DESC_NUM_MASK,
+ FIELD_PREP(HW_FWD_DESC_NUM_MASK, num_desc) |
+ LMGR_INIT_START | LMGR_SRAM_MODE_MASK);
+
+ return read_poll_timeout(airoha_qdma_rr, status,
+ !(status & LMGR_INIT_START), USEC_PER_MSEC,
+ 30 * USEC_PER_MSEC, true, qdma,
+ REG_LMGR_INIT_CFG);
+}
+
+static void airoha_qdma_init_qos(struct airoha_qdma *qdma)
+{
+ struct airoha_eth *eth = qdma->eth;
+ u32 meter_cfg, meter_window, meter_timeslice;
+ int id = qdma - ð->qdma[0];
+
+ airoha_qdma_clear(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_SCALE_MASK);
+ airoha_qdma_set(qdma, REG_TXWRR_MODE_CFG, TWRR_WEIGHT_BASE_MASK);
+
+ /* The EN7523 SDK enables PSE buffer estimation only on QDMA WAN.
+ * QDMA1 is the WAN instance in this driver. Later SoCs explicitly
+ * keep this estimator disabled, as does EN7523 QDMA LAN.
+ */
+ if (airoha_is(eth, airoha_en7523) && id == 1)
+ airoha_qdma_set(qdma, REG_PSE_BUF_USAGE_CFG,
+ PSE_BUF_ESTIMATE_EN_MASK);
+ else
+ airoha_qdma_clear(qdma, REG_PSE_BUF_USAGE_CFG,
+ PSE_BUF_ESTIMATE_EN_MASK);
+
+ meter_cfg = EGRESS_RATE_METER_EN_MASK |
+ EGRESS_RATE_METER_EQ_RATE_EN_MASK;
+ meter_window = 0x1f;
+ meter_timeslice = 0x7ff;
+
+ /* The EN7523 SDK uses a shorter sampling interval on QDMA WAN and
+ * leaves equal-rate mode disabled there. QDMA LAN uses the generic
+ * 2047us x 31 = 63.457ms interval.
+ */
+ if (airoha_is(eth, airoha_en7523) && id == 1) {
+ meter_cfg &= ~EGRESS_RATE_METER_EQ_RATE_EN_MASK;
+ meter_window = 20;
+ meter_timeslice = 200; /* 200us x 20 = 4ms */
+ }
+
+ meter_cfg |= FIELD_PREP(EGRESS_RATE_METER_WINDOW_SZ_MASK,
+ meter_window) |
+ FIELD_PREP(EGRESS_RATE_METER_TIMESLICE_MASK,
+ meter_timeslice);
+ airoha_qdma_rmw(qdma, REG_EGRESS_RATE_METER_CFG,
+ EGRESS_RATE_METER_EN_MASK |
+ EGRESS_RATE_METER_EQ_RATE_EN_MASK |
+ EGRESS_RATE_METER_WINDOW_SZ_MASK |
+ EGRESS_RATE_METER_TIMESLICE_MASK,
+ meter_cfg);
+
+ /* ratelimit init */
+ airoha_qdma_set(qdma, REG_GLB_TRTCM_CFG, GLB_TRTCM_EN_MASK);
+ /* fast-tick 25us */
+ airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_FAST_TICK_MASK,
+ FIELD_PREP(GLB_FAST_TICK_MASK, 25));
+ airoha_qdma_rmw(qdma, REG_GLB_TRTCM_CFG, GLB_SLOW_TICK_RATIO_MASK,
+ FIELD_PREP(GLB_SLOW_TICK_RATIO_MASK, 40));
+
+ airoha_qdma_set(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_TRTCM_EN_MASK);
+ airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG, EGRESS_FAST_TICK_MASK,
+ FIELD_PREP(EGRESS_FAST_TICK_MASK, 25));
+ airoha_qdma_rmw(qdma, REG_EGRESS_TRTCM_CFG,
+ EGRESS_SLOW_TICK_RATIO_MASK,
+ FIELD_PREP(EGRESS_SLOW_TICK_RATIO_MASK, 40));
+
+ airoha_qdma_set(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_TRTCM_EN_MASK);
+ airoha_qdma_clear(qdma, REG_INGRESS_TRTCM_CFG,
+ INGRESS_TRTCM_MODE_MASK);
+ airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG, INGRESS_FAST_TICK_MASK,
+ FIELD_PREP(INGRESS_FAST_TICK_MASK, 125));
+ airoha_qdma_rmw(qdma, REG_INGRESS_TRTCM_CFG,
+ INGRESS_SLOW_TICK_RATIO_MASK,
+ FIELD_PREP(INGRESS_SLOW_TICK_RATIO_MASK, 8));
+
+ airoha_qdma_set(qdma, REG_SLA_TRTCM_CFG, SLA_TRTCM_EN_MASK);
+ airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_FAST_TICK_MASK,
+ FIELD_PREP(SLA_FAST_TICK_MASK, 25));
+ airoha_qdma_rmw(qdma, REG_SLA_TRTCM_CFG, SLA_SLOW_TICK_RATIO_MASK,
+ FIELD_PREP(SLA_SLOW_TICK_RATIO_MASK, 40));
}
static void airoha_qdma_init_qos_stats(struct airoha_qdma *qdma)
{
- int i;
+ int i;
+
+ for (i = 0; i < AIROHA_NUM_QOS_CHANNELS; i++) {
+ /* Tx-cpu transferred count */
+ airoha_qdma_wr(qdma, REG_CNTR_VAL(i << 1), 0);
+ airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1),
+ CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
+ CNTR_ALL_DSCP_RING_EN_MASK |
+ FIELD_PREP(CNTR_CHAN_MASK, i));
+ /* Tx-fwd transferred count */
+ airoha_qdma_wr(qdma, REG_CNTR_VAL((i << 1) + 1), 0);
+ airoha_qdma_wr(qdma, REG_CNTR_CFG((i << 1) + 1),
+ CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
+ CNTR_ALL_DSCP_RING_EN_MASK |
+ FIELD_PREP(CNTR_SRC_MASK, 1) |
+ FIELD_PREP(CNTR_CHAN_MASK, i));
+ }
+}
+
+static int airoha_qdma_modern_hw_init(struct airoha_qdma *qdma)
+{
+ int i;
+
+ for (i = 0; i < qdma->eth->soc->irq_banks; i++) {
+ /* clear pending irqs */
+ airoha_qdma_wr(qdma, REG_INT_STATUS(i), 0xffffffff);
+ /* setup rx irqs */
+ airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX0,
+ INT_RX0_MASK(RX_IRQ_BANK_PIN_MASK(i)));
+ airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX1,
+ INT_RX1_MASK(RX_IRQ_BANK_PIN_MASK(i)));
+ airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX2,
+ INT_RX2_MASK(RX_IRQ_BANK_PIN_MASK(i)));
+ airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX3,
+ INT_RX3_MASK(RX_IRQ_BANK_PIN_MASK(i)));
+ }
+ /* setup tx irqs */
+ airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0,
+ TX_COHERENT_LOW_INT_MASK | INT_TX_MASK);
+ airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX4,
+ TX_COHERENT_HIGH_INT_MASK);
+
+ if (airoha_is(qdma->eth, airoha_en7523)) {
+ airoha_qdma_wr(qdma, 0x30, 0x7C000000);
+ airoha_qdma_wr(qdma, 0x34, 0x7C007C00);
+ airoha_qdma_wr(qdma, 0x38, 0x00200000);
+ airoha_qdma_wr(qdma, 0x3C, 0x00200020);
+ airoha_qdma_wr(qdma, 0x40, 0x00000030);
+ airoha_qdma_wr(qdma, 0x6C, 0x00000000);
+ }
+
+ /* setup irq binding */
+ for (i = 0; i < qdma->eth->soc->tx_ring; i++) {
+ if (!qdma->q_tx[i].ndesc)
+ continue;
+
+ if (TX_RING_IRQ_BLOCKING_MAP_MASK & BIT(i))
+ airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i),
+ TX_RING_IRQ_BLOCKING_CFG_MASK);
+ else
+ airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i),
+ TX_RING_IRQ_BLOCKING_CFG_MASK);
+
+ if (airoha_is(qdma->eth, airoha_en7523)) {
+ if (i == 0)
+ airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i),
+ TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK);
+ else
+ airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i),
+ TX_RING_IRQ_BLOCKING_TX_DROP_EN_MASK);
+ }
+ }
+
+ airoha_qdma_wr(qdma, REG_QDMA_GLOBAL_CFG,
+ FIELD_PREP(GLOBAL_CFG_DMA_PREFERENCE_MASK, 3) |
+ GLOBAL_CFG_CPU_TXR_RR_MASK |
+ GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK |
+ GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK |
+ GLOBAL_CFG_MULTICAST_EN_MASK |
+ GLOBAL_CFG_IRQ0_EN_MASK | GLOBAL_CFG_IRQ1_EN_MASK |
+ GLOBAL_CFG_TX_WB_DONE_MASK |
+ FIELD_PREP(GLOBAL_CFG_MAX_ISSUE_NUM_MASK, 2));
+
+ airoha_qdma_init_qos(qdma);
+
+ /* disable qdma rx delay interrupt */
+ for (i = 0; i < qdma->eth->soc->rx_ring; i++) {
+ if (!qdma->q_rx[i].ndesc)
+ continue;
+
+ airoha_qdma_clear(qdma, REG_RX_DELAY_INT_IDX(i),
+ RX_DELAY_INT_MASK);
+ }
+
+ airoha_qdma_set(qdma, REG_TXQ_CNGST_CFG,
+ TXQ_CNGST_DROP_EN | TXQ_CNGST_DEI_DROP_EN);
+ airoha_qdma_init_qos_stats(qdma);
+
+ return 0;
+}
+
+static irqreturn_t airoha_irq_handler(int irq, void *dev_instance)
+{
+ struct airoha_irq_bank *irq_bank = dev_instance;
+ struct airoha_qdma *qdma = irq_bank->qdma;
+ u32 rx_intr_mask = 0, rx_intr1, rx_intr2;
+ u32 intr[ARRAY_SIZE(irq_bank->irqmask)];
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(intr); i++) {
+ intr[i] = airoha_qdma_rr(qdma, REG_INT_STATUS(i));
+ intr[i] &= irq_bank->irqmask[i];
+ airoha_qdma_wr(qdma, REG_INT_STATUS(i), intr[i]);
+ }
+
+ if (!test_bit(DEV_STATE_INITIALIZED, &qdma->eth->state))
+ return IRQ_NONE;
+
+ rx_intr1 = intr[1] & RX_DONE_LOW_INT_MASK;
+ if (rx_intr1) {
+ airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX1, rx_intr1);
+ rx_intr_mask |= rx_intr1;
+ }
+
+ rx_intr2 = intr[2] & RX_DONE_HIGH_INT_MASK;
+ if (rx_intr2) {
+ airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX2, rx_intr2);
+ rx_intr_mask |= (rx_intr2 << 16);
+ }
+
+ for (i = 0; rx_intr_mask && i < qdma->eth->soc->rx_ring; i++) {
+ if (!qdma->q_rx[i].ndesc)
+ continue;
+
+ if (rx_intr_mask & BIT(i))
+ napi_schedule(&qdma->q_rx[i].napi);
+ }
+
+ if (intr[0] & INT_TX_MASK) {
+ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
+ if (!(intr[0] & TX_DONE_INT_MASK(i)))
+ continue;
+
+ airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX0,
+ TX_DONE_INT_MASK(i));
+ napi_schedule(&qdma->q_tx_irq[i].napi);
+ }
+ }
+
+ return IRQ_HANDLED;
+}
+
+static int airoha_qdma_modern_init_irqs(struct platform_device *pdev,
+ struct airoha_qdma *qdma)
+{
+ struct airoha_eth *eth = qdma->eth;
+ int i, id = qdma - ð->qdma[0];
+ qdma->irq_banks = devm_kzalloc(&pdev->dev,
+ sizeof(*qdma->irq_banks) * eth->soc->irq_banks, GFP_KERNEL);
+ if (!qdma->irq_banks)
+ return -ENOMEM;
+
+ for (i = 0; i < eth->soc->irq_banks; i++) {
+ struct airoha_irq_bank *irq_bank = &qdma->irq_banks[i];
+ int err, irq_index = 4 * id + i;
+ const char *name;
+
+ spin_lock_init(&irq_bank->irq_lock);
+ irq_bank->qdma = qdma;
+
+ irq_bank->irq = platform_get_irq(pdev, irq_index);
+ if (irq_bank->irq < 0)
+ return irq_bank->irq;
+
+ name = devm_kasprintf(eth->dev, GFP_KERNEL,
+ KBUILD_MODNAME ".%d", irq_index);
+ if (!name)
+ return -ENOMEM;
+
+ err = devm_request_irq(eth->dev, irq_bank->irq,
+ airoha_irq_handler, IRQF_SHARED, name,
+ irq_bank);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+static int airoha_qdma_prepare(struct platform_device *pdev,
+ struct airoha_eth *eth,
+ struct airoha_qdma *qdma, int id)
+{
+ if (airoha_has_legacy_qdma(eth)) {
+ struct airoha_qdma_mips_cfg cfg;
+ struct airoha_qdma_mips *econet;
+
+ econet_prepare_qdma_cfg(&cfg, eth->soc, id);
+ econet = devm_kzalloc(eth->dev, sizeof(*econet), GFP_KERNEL);
+ if (!econet)
+ return -ENOMEM;
+
+ airoha_qdma_setup(qdma, eth, qdma->regs, id, cfg.num_channels);
+ qdma->econet = econet;
+ econet->qdma = qdma;
+ econet->regs = qdma->regs;
+ econet->cfg = cfg;
+ mutex_init(&econet->lock);
+
+ /* Keep RX/TX stopped while the common init sequence sets up
QDMA. */
+ econet_wreg((struct qregs_qcfg) { 0 }, &econet->regs->qdma_cfg);
+
+ return 0;
+ }
+
+ qdma->q_tx = devm_kcalloc(&pdev->dev, eth->soc->tx_ring,
+ sizeof(*qdma->q_tx), GFP_KERNEL);
+ if (!qdma->q_tx)
+ return -ENOMEM;
+
+ qdma->q_rx = devm_kcalloc(&pdev->dev, eth->soc->rx_ring,
+ sizeof(*qdma->q_rx), GFP_KERNEL);
+ if (!qdma->q_rx)
+ return -ENOMEM;
+
+ airoha_qdma_setup(qdma, eth, qdma->regs, id, AIROHA_NUM_QOS_CHANNELS);
+
+ return 0;
+}
+
+static int airoha_qdma_init_irqs(struct platform_device *pdev,
+ struct airoha_qdma *qdma)
+{
+ if (qdma->econet)
+ return econet_qdma_init_irqs(pdev, qdma->econet);
+
+ return airoha_qdma_modern_init_irqs(pdev, qdma);
+}
+
+static int airoha_qdma_init_rx(struct airoha_qdma *qdma)
+{
+ if (qdma->econet)
+ return econet_qdma_init_rx(qdma->econet);
+
+ return airoha_qdma_modern_init_rx(qdma);
+}
+
+static int airoha_qdma_init_tx(struct airoha_qdma *qdma)
+{
+ if (qdma->econet)
+ return econet_qdma_init_tx(qdma->econet);
+
+ return airoha_qdma_modern_init_tx(qdma);
+}
+
+static int airoha_qdma_init_hfwd(struct airoha_qdma *qdma)
+{
+ if (qdma->econet)
+ return econet_qdma_init_hfwd(qdma->econet);
+
+ return airoha_qdma_modern_init_hfwd(qdma);
+}
+
+static int airoha_qdma_hw_init(struct airoha_qdma *qdma)
+{
+ if (qdma->econet)
+ return econet_qdma_hw_init(qdma->econet);
+
+ return airoha_qdma_modern_hw_init(qdma);
+}
+
+int airoha_qdma_init(struct platform_device *pdev,
+ struct airoha_eth *eth, struct airoha_qdma *qdma)
+{
+ int err, id = qdma - ð->qdma[0];
+ const char *res;
+
+ res = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d", id);
+ if (!res)
+ return -ENOMEM;
+
+ qdma->regs = devm_platform_ioremap_resource_byname(pdev, res);
+ if (IS_ERR(qdma->regs))
+ return dev_err_probe(eth->dev, PTR_ERR(qdma->regs),
+ "failed to iomap qdma%d regs\n", id);
+
+ err = airoha_qdma_prepare(pdev, eth, qdma, id);
+ if (err)
+ return err;
+
+ err = airoha_qdma_init_irqs(pdev, qdma);
+ if (err)
+ return err;
+
+ err = airoha_qdma_init_rx(qdma);
+ if (err)
+ return err;
+
+ err = airoha_qdma_init_tx(qdma);
+ if (err)
+ return err;
+
+ err = airoha_qdma_init_hfwd(qdma);
+ if (err)
+ return err;
+
+ return airoha_qdma_hw_init(qdma);
+}
+
+void airoha_qdma_cleanup(struct airoha_qdma *qdma)
+{
+ int i;
+
+ if (qdma->econet) {
+ econet_qdma_destroy(qdma->econet);
+ qdma->econet = NULL;
+ return;
+ }
+
+ for (i = 0; i < qdma->eth->soc->rx_ring; i++) {
+ if (!qdma->q_rx[i].ndesc)
+ continue;
+
+ netif_napi_del(&qdma->q_rx[i].napi);
+ airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]);
+ if (qdma->q_rx[i].page_pool) {
+ page_pool_destroy(qdma->q_rx[i].page_pool);
+ qdma->q_rx[i].page_pool = NULL;
+ }
+ }
+
+ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
+ if (!qdma->q_tx_irq[i].size)
+ continue;
+
+ netif_napi_del(&qdma->q_tx_irq[i].napi);
+ }
+
+ for (i = 0; i < qdma->eth->soc->tx_ring; i++) {
+ if (!qdma->q_tx[i].ndesc)
+ continue;
+
+ airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]);
+ }
+
+}
+
+/* QDMA runtime lifecycle shared by Airoha and EcoNet layouts. */
+void airoha_qdma_start_napi(struct airoha_qdma *qdma)
+{
+ int i;
+
+ if (qdma->econet) {
+ for (i = 0; i < ARRAY_SIZE(qdma->econet->q_tx_done); i++)
+ napi_enable(&qdma->econet->q_tx_done[i].napi);
+ for (i = 0; i < ARRAY_SIZE(qdma->econet->q_rx); i++)
+ napi_enable(&qdma->econet->q_rx[i].napi);
+ return;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
+ napi_enable(&qdma->q_tx_irq[i].napi);
+
+ for (i = 0; i < qdma->eth->soc->rx_ring; i++) {
+ if (!qdma->q_rx[i].ndesc)
+ continue;
+
+ napi_enable(&qdma->q_rx[i].napi);
+ }
+}
+
+void airoha_qdma_stop_napi(struct airoha_qdma *qdma)
+{
+ int i;
+
+ if (qdma->econet) {
+ for (i = 0; i < ARRAY_SIZE(qdma->econet->q_tx_done); i++)
+ napi_disable(&qdma->econet->q_tx_done[i].napi);
+ for (i = 0; i < ARRAY_SIZE(qdma->econet->q_rx); i++)
+ napi_disable(&qdma->econet->q_rx[i].napi);
+ return;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
+ napi_disable(&qdma->q_tx_irq[i].napi);
+
+ for (i = 0; i < qdma->eth->soc->rx_ring; i++) {
+ if (!qdma->q_rx[i].ndesc)
+ continue;
+
+ napi_disable(&qdma->q_rx[i].napi);
+ }
+}
+
+void airoha_qdma_start(struct airoha_qdma *qdma)
+{
+ if (qdma->econet) {
+ struct qregs_qcfg qcfg;
+
+ guard(mutex)(&qdma->econet->lock);
+ if (qdma->users++ > 0)
+ return;
+
+ qcfg = econet_rreg(&qdma->econet->regs->qdma_cfg);
+ set_qregs_qcfg_rx_dma_en(&qcfg, true);
+ set_qregs_qcfg_tx_dma_en(&qcfg, true);
+ econet_wreg(qcfg, &qdma->econet->regs->qdma_cfg);
+ return;
+ }
+
+ if (qdma->users++ > 0)
+ return;
+
+ airoha_qdma_set(qdma, REG_QDMA_GLOBAL_CFG,
+ GLOBAL_CFG_TX_DMA_EN_MASK |
+ GLOBAL_CFG_RX_DMA_EN_MASK);
+}
+
+void airoha_qdma_stop(struct airoha_qdma *qdma)
+{
+ u32 status;
+
+ if (qdma->econet) {
+ struct qregs_qcfg qcfg;
+
+ guard(mutex)(&qdma->econet->lock);
+ if (WARN_ON_ONCE(qdma->users <= 0))
+ return;
+ if (--qdma->users > 0)
+ return;
+
+ qcfg = econet_rreg(&qdma->econet->regs->qdma_cfg);
+ set_qregs_qcfg_rx_dma_en(&qcfg, false);
+ set_qregs_qcfg_tx_dma_en(&qcfg, false);
+ econet_wreg(qcfg, &qdma->econet->regs->qdma_cfg);
+ econet_qdma_cleanup_tx(qdma->econet);
+ return;
+ }
+
+ if (WARN_ON_ONCE(qdma->users <= 0))
+ return;
+ if (--qdma->users > 0)
+ return;
+
+ airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG,
+ GLOBAL_CFG_TX_DMA_EN_MASK |
+ GLOBAL_CFG_RX_DMA_EN_MASK);
+
+ if (read_poll_timeout(airoha_qdma_rr, status,
+ !(status & (GLOBAL_CFG_TX_DMA_BUSY_MASK |
+ GLOBAL_CFG_RX_DMA_BUSY_MASK)),
+ USEC_PER_MSEC, 50 * USEC_PER_MSEC, true,
+ qdma, REG_QDMA_GLOBAL_CFG))
+ dev_warn(qdma->eth->dev, "QDMA DMA engine busy timeout\n");
+
+ for (int i = 0; i < qdma->eth->soc->tx_ring; i++) {
+ if (!qdma->q_tx[i].ndesc)
+ continue;
+
+ airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]);
+ }
+}
+
+static u16 econet_gdm_oversize_len(struct airoha_gdm_dev *port, int mtu)
+{
+ u16 len = ETH_HLEN + mtu + ETH_FCS_LEN;
+
+ /*
+ * The EN7512/EN7521 SDK programs GDM1_LONG_LEN_VALUE to 1700, not to
+ * the bare 1518-byte Ethernet size. GDM1 sees the in-band MT7530
+ * special tag and can also see customer/service VLAN tags, so
using the
+ * bare MTU wire length causes otherwise valid full-sized frames to be
+ * classified as long packets and dropped before they reach QDMA/PPE.
+ */
+ if (airoha_is(port->eth, econet_en751221) &&
+ port->fport == ETX_FPORT_GDM1)
+ len = max_t(u16, len, EN751221_GDM1_LONG_LEN);
+
+ return len;
+}
+
+static void econet_set_gdm_port_fwd_cfg(struct airoha_gdm_dev *port,
+ enum etx_fport val)
+{
+ struct fwd_cfg fc;
+
+ guard(spinlock)(&port->reg_lock);
+ fc = econet_rreg(&port->econet_regs->fwd_cfg);
+ set_gdm_fwd_cfg_mymac_fport(&fc, val);
+ set_gdm_fwd_cfg_mcast_fport(&fc, val);
+ set_gdm_fwd_cfg_bcast_fport(&fc, val);
+ set_gdm_fwd_cfg_default_fport(&fc, val);
+ /*
+ * Bit 25 is DROP_OVERSIZE on newer Airoha GDMs, but GDM_UNTAG_EN
+ * on EN751221. The vendor EN7512 datapath explicitly leaves UNTAG
+ * disabled when special-tag mode is enabled. Do not preserve the
+ * reset value here: some bootloaders leave bit 25 set.
+ */
+ if (airoha_is_econet(port->eth))
+ fc.word &= ~EN751221_GDM_UNTAG_EN;
+ else
+ set_gdm_fwd_cfg_drop_oversize(&fc, true);
+ econet_wreg(fc, &port->econet_regs->fwd_cfg);
+}
+
+static int econet_validate_xpon_gdm2(struct net_device *netdev,
+ struct airoha_gdm_dev **gdm)
+{
+ struct airoha_gdm_dev *port;
+
+ if (!netdev)
+ return -EINVAL;
+
+ port = netdev_priv(netdev);
+ if (!port->eth || !airoha_is(port->eth, econet_en751221) ||
+ port->fport != ETX_FPORT_GDM2)
+ return -EOPNOTSUPP;
+
+ *gdm = port;
+ return 0;
+}
+
+static int econet_xpon_lookup_service(struct airoha_gdm_dev *port,
+ bool vlan_valid, u16 vlan_id,
+ bool pcp_valid, u8 pcp,
+ struct airoha_xpon_tx_info *info)
+{
+ const struct airoha_xpon_service_cfg *fallback = NULL;
+ int i;
+
+ spin_lock_bh(&port->xpon_service_lock);
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++) {
+ const struct airoha_xpon_service_cfg *service;
+
+ service = &port->xpon_services[i];
+ if (!service->valid)
+ continue;
+ if (service->default_service)
+ fallback = service;
+ if (!service->vlan_valid && !service->pcp_valid)
+ continue;
+ if (service->vlan_valid &&
+ (!vlan_valid || service->vlan_id != vlan_id))
+ continue;
+ if (service->pcp_valid &&
+ (!pcp_valid || service->pcp != pcp))
+ continue;
+ fallback = service;
+ break;
+ }
+
+ if (fallback) {
+ info->gem_port_id = fallback->gem_port_id;
+ info->tcont = fallback->tcont;
+ info->queue = fallback->queue;
+ info->oam = false;
+ }
+ spin_unlock_bh(&port->xpon_service_lock);
+
+ return fallback ? 0 : -ENOENT;
+}
+
+static int econet_xpon_classify(struct airoha_gdm_dev *port,
+ struct sk_buff *skb,
+ struct airoha_xpon_tx_info *info)
+{
+ struct vlan_ethhdr vlan_hdr_buf;
+ const struct vlan_ethhdr *vlan_hdr;
+ u16 vlan_id = 0;
+ u8 pcp = 0;
+ bool vlan_valid;
+
+ vlan_valid = skb_vlan_tag_present(skb);
+ if (vlan_valid) {
+ u16 tci = skb_vlan_tag_get(skb);
+
+ vlan_id = tci & VLAN_VID_MASK;
+ pcp = (tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+ } else {
+ vlan_hdr = skb_header_pointer(skb, 0, sizeof(vlan_hdr_buf),
+ &vlan_hdr_buf);
+ if (vlan_hdr && eth_type_vlan(vlan_hdr->h_vlan_proto)) {
+ u16 tci = ntohs(vlan_hdr->h_vlan_TCI);
+
+ vlan_id = tci & VLAN_VID_MASK;
+ pcp = (tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+ vlan_valid = true;
+ }
+ }
+
+ /*
+ * An untagged frame still carries a priority: it is the default of
+ * zero, and the 802.1p mapper an OLT provisions has an entry for it.
+ * Treating such a frame as having no priority at all leaves it
+ * matching no service on an OLT that expresses its whole service graph
+ * through that mapper, and it is then dropped before ever reaching the
+ * hardware.
+ */
+ return econet_xpon_lookup_service(port, vlan_valid, vlan_id,
+ true, pcp, info);
+}
+
+static netdev_tx_t econet_qdma_xmit(struct sk_buff *skb, struct
net_device *dev)
+{
+ struct airoha_gdm_dev *port = netdev_priv(dev);
+ struct airoha_qdma_skb_meta skb_meta;
+ int qid, ret = 0, len = skb->len;
+ struct netdev_queue *txq;
+ struct airoha_xpon_tx_info xpon_info;
+ union desc_msg msg = {0};
+ bool xpon = false;
+ u8 channel;
+
+ if ((READ_ONCE(port->flags) & AIROHA_PRIV_F_XPON_MANAGED)) {
+ switch (READ_ONCE(port->xpon_mode)) {
+ case AIROHA_XPON_MODE_GPON:
+ if (econet_xpon_classify(port, skb, &xpon_info))
+ goto drop;
+ xpon = true;
+ break;
+ case AIROHA_XPON_MODE_EPON:
+ /* Initial EPON support uses LLID/channel 0. Multi-LLID
+ * classification can be layered on the service API later; the
+ * important part here is to emit a PWAN descriptor instead of
+ * an Ethernet/MT7530 special-tag descriptor.
+ */
+ memset(&xpon_info, 0, sizeof(xpon_info));
+ xpon_info.queue = skb_get_queue_mapping(skb) %
+ ECONET_NUM_QUEUES;
+ xpon = true;
+ break;
+ default:
+ goto drop;
+ }
+ }
+
+ qid = skb_get_queue_mapping(skb);
+ if (xpon) {
+ /* PWAN_FETxMsg_T on EN751221: queue[2:0], channel[10:3],
+ * OAM[11] and GEM[23:12]. The GEM field overlays the Ethernet
+ * MediaTek special-tag field and must therefore be programmed only
+ * for the managed PON datapath.
+ */
+ channel = xpon_info.tcont;
+ set_etx_queue(&msg.etx, xpon_info.queue);
+ set_etx_xpon_gem(&msg.etx, xpon_info.gem_port_id);
+ } else if (airoha_is(port->eth, econet_en751221)) {
+ /*
+ * The EN751221 vendor LAN path does not map Linux flow/hash
+ * queues onto QDMA's eight hardware QoS queues. Unless QoS
+ * explicitly marks txq_is_valid, qdma_transmit_packet() forces
+ * queue 0 and selects the channel from the destination switch
+ * port. Keep that policy here; Linux qid is still used for BQL
+ * accounting and queue stop/wake.
+ */
+ channel = 0;
+ set_etx_queue(&msg.etx, 0);
+ } else {
+ channel = qid / ECONET_NUM_QUEUES;
+ set_etx_queue(&msg.etx, qid % ECONET_NUM_QUEUES);
+ }
+ set_etx_fport(&msg.etx, port->fport);
+
+ txq = netdev_get_tx_queue(dev, qid);
+
+ /* Non-linear skbs are unsupported, we shouldn't get them but
+ * if we do, we'll attempt to linearize. */
+ if (skb_linearize(skb))
+ goto drop;
+
+ /*
+ * Keep the EN751221 PWAN descriptor layout unchanged, but make sure
+ * short GPON Ethernet data frames meet the minimum Ethernet payload
+ * size before they enter QDMA/GDM2. The control/OAM path has its own
+ * framing and must not be padded here.
+ *
+ * skb_put_padto() consumes the skb on allocation failure, so do not
+ * jump to the common drop label in that case.
+ */
+ if (xpon && READ_ONCE(port->xpon_mode) == AIROHA_XPON_MODE_GPON &&
+ !xpon_info.oam && skb->len < ETH_ZLEN) {
+ if (skb_put_padto(skb, ETH_ZLEN)) {
+ dev->stats.tx_dropped++;
+ return NETDEV_TX_OK;
+ }
+
+ len = skb->len;
+ }
+
+ /*
+ * EN751221 keeps the MediaTek DSA special tag in-band, unlike newer
+ * Airoha QDMA which can move it to descriptor metadata. The logical
+ * port-mask/channel classification is nevertheless shared.
+ */
+ if (!xpon && airoha_is(port->eth, econet_en751221)) {
+ airoha_qdma_skb_get_mtk_meta(skb, dev, AIROHA_MTK_TAG_IN_SKB,
+ &skb_meta);
+ if (skb_meta.has_mtk_tag &&
+ skb_meta.channel != AIROHA_MTK_INVALID_CHANNEL)
+ channel = skb_meta.channel;
+ }
+
+ set_etx_channel(&msg.etx, channel);
+
+ /*
+ * EN751221 QDMA can generate IPv4/IPv6 TCP and UDP checksums.
+ * The vendor driver programs all parser checksum-offload bits for a
+ * CHECKSUM_PARTIAL skb and lets the hardware select the applicable
+ * protocol.
+ */
+ if (skb->ip_summed == CHECKSUM_PARTIAL) {
+ set_etx_ico(&msg.etx, true);
+ set_etx_uco(&msg.etx, true);
+ set_etx_tco(&msg.etx, true);
+ }
+
+ netdev_tx_sent_queue(txq, len);
+
+ ret = airoha_qdma_mips_xmit(port->qdma, skb, &msg, 0);
+ if (ret == -EBUSY) {
+ netdev_tx_completed_queue(txq, 1, len);
+ netif_tx_stop_queue(txq);
+ return NETDEV_TX_BUSY;
+ }
+ if (ret < 0) {
+ netdev_tx_completed_queue(txq, 1, len);
+ goto drop;
+ }
+
+ /* Positive EBUSY means this packet was queued and filled the ring. */
+ if (ret == EBUSY)
+ netif_tx_stop_queue(txq);
+
+ return NETDEV_TX_OK;
+
+drop:
+ dev_kfree_skb_any(skb);
+ dev->stats.tx_dropped++;
+ return NETDEV_TX_OK;
+}
+
+static void econet_update_hw_stats(struct airoha_gdm_dev *port)
+{
+ struct gdm_counters __iomem *c = &port->econet_regs->counters;
+ struct clear_counters cc = {0};
+ u32 i = 0;
+
+ guard(spinlock)(&port->port->lock);
+ u64_stats_update_begin(&port->stats.syncp);
+
+ port->stats.tx_ok_pkts += econet_rreg(&c->tx.tx_pkts);
+ port->stats.tx_ok_bytes += econet_rreg(&c->tx.bytes);
+ port->stats.tx_drops += econet_rreg(&c->tx.drops);
+ if (port->g2_stats) {
+ port->stats.tx_broadcast += econet_rreg(&c->tx.g2.tx_bcast);
+ port->stats.tx_multicast += econet_rreg(&c->tx.g2.tx_mcast);
+
+ port->stats.tx_len[i] += econet_rreg(&c->tx.g2.f_less_64);
+ port->stats.tx_len[i++] += econet_rreg(&c->tx.g2.f_64);
+
+ port->stats.tx_len[i++] += econet_rreg(&c->tx.g2.f_65_127);
+ port->stats.tx_len[i++] += econet_rreg(&c->tx.g2.f_128_255);
+ port->stats.tx_len[i++] += econet_rreg(&c->tx.g2.f_256_511);
+ port->stats.tx_len[i++] += econet_rreg(&c->tx.g2.f_512_1023);
+ port->stats.tx_len[i++] += econet_rreg(&c->tx.g2.f_1024_1518);
+ port->stats.tx_len[i++] += econet_rreg(&c->tx.g2.f_more_1518);
+ }
+
+ port->stats.rx_ok_pkts += econet_rreg(&c->rx.pkts);
+ port->stats.rx_ok_bytes += econet_rreg(&c->rx.bytes);
+ port->stats.rx_errors += econet_rreg(&c->rx.drops_err);
+ port->stats.rx_over_errors += econet_rreg(&c->rx.drops_overflow);
+ if (port->g2_stats) {
+ port->stats.rx_drops += econet_rreg(&c->rx.g2.edrops);
+ port->stats.rx_broadcast += econet_rreg(&c->rx.g2.bcast);
+ port->stats.rx_multicast += econet_rreg(&c->rx.g2.mcast);
+ port->stats.rx_crc_error += econet_rreg(&c->rx.g2.ecrc);
+ port->stats.rx_fragment += econet_rreg(&c->rx.g2.efrag);
+ port->stats.rx_jabber += econet_rreg(&c->rx.g2.ejabber);
+
+ i = 0;
+ port->stats.rx_len[i] += econet_rreg(&c->rx.g2.f_less_64);
+ port->stats.rx_len[i++] += econet_rreg(&c->rx.g2.f_64);
+
+ port->stats.rx_len[i++] += econet_rreg(&c->rx.g2.f_65_127);
+ port->stats.rx_len[i++] += econet_rreg(&c->rx.g2.f_128_255);
+ port->stats.rx_len[i++] += econet_rreg(&c->rx.g2.f_256_511);
+ port->stats.rx_len[i++] += econet_rreg(&c->rx.g2.f_512_1023);
+ port->stats.rx_len[i++] += econet_rreg(&c->rx.g2.f_1024_1518);
+ port->stats.rx_len[i++] += econet_rreg(&c->rx.g2.f_more_1518);
+ } else {
+ port->stats.rx_drops += econet_rreg(&c->rx.drops_err);
+ port->stats.rx_drops += econet_rreg(&c->rx.drops_fc);
+ port->stats.rx_drops += econet_rreg(&c->rx.drops_rc);
+ port->stats.rx_drops += econet_rreg(&c->rx.drops_overflow);
+ }
+
+ set_gdm_cl_cnt_rx(&cc, true);
+ set_gdm_cl_cnt_tx(&cc, true);
+ econet_wreg(cc, &port->econet_regs->clear_counters);
+
+ u64_stats_update_end(&port->stats.syncp);
+}
+
+/* Frame-engine platform state is shared by all supported SoCs. */
+
+static struct airoha_gdm_dev *
+airoha_eth_get_gdm_dev(struct airoha_eth *eth, unsigned int id)
+{
+ struct airoha_gdm_port *port;
+
+ if (!id || id > eth->soc->max_gdm_ports)
+ return NULL;
+
+ port = eth->ports[id - 1];
+ return port ? port->devs[0] : NULL;
+}
+
+
+static bool econet_en751221_dsa_sport(u32 sport)
+{
+ return sport >= EN751221_DSA_SPORT_BASE &&
+ sport < EN751221_DSA_SPORT_BASE + EN751221_DSA_NUM_PORTS;
+}
+
+int econet_rx_before_recv(struct airoha_eth *eth, struct sk_buff *skb,
+ u8 sport)
+{
+ struct airoha_gdm_dev *dev;
+ struct net_device *port;
+
+ dev = airoha_qdma_get_gdm_dev(eth, sport);
+ if (IS_ERR(dev))
+ return PTR_ERR(dev);
+
+ port = dev->common.netdev;
+
+ /*
+ * EN7512/EN7521 keeps the MT7530 special tag in-band. The vendor
+ * receive path removes that tag from skb data in software; rxMsgW3 is
+ * only used for descriptor-based tag recovery by the EN7526C special
+ * case. Leave the frame untouched here so the DSA MTK tagger consumes
+ * the wire tag directly.
+ */
+ skb->dev = port;
+ skb->protocol = eth_type_trans(skb, port);
+
+ return 0;
+}
+
+static int airoha_gdm_xpon_start(struct airoha_gdm_dev *dev)
+{
+ int ret = 0;
+
+ mutex_lock(&dev->xpon_lock);
+ if (dev->xpon_ops && !dev->xpon_started) {
+ ret = dev->xpon_ops->start(dev->xpon_priv);
+ if (!ret)
+ dev->xpon_started = true;
+ }
+ mutex_unlock(&dev->xpon_lock);
+
+ return ret;
+}
+
+static void airoha_gdm_xpon_stop(struct airoha_gdm_dev *dev)
+{
+ mutex_lock(&dev->xpon_lock);
+ if (dev->xpon_ops && dev->xpon_started) {
+ dev->xpon_started = false;
+ dev->xpon_ops->stop(dev->xpon_priv);
+ }
+ mutex_unlock(&dev->xpon_lock);
+}
+
+static int econet_set_xpon_mode(struct net_device *netdev,
+ enum airoha_xpon_mode mode)
+{
+ struct airoha_gdm_dev *port;
+ int ret;
+
+ ret = econet_validate_xpon_gdm2(netdev, &port);
+ if (ret)
+ return ret;
+ if (mode != AIROHA_XPON_MODE_GPON && mode != AIROHA_XPON_MODE_EPON)
+ return -EINVAL;
+
+ /* feDevGdm2Cdm2Stop(XPON_ENABLE) starts both protocols from a fully
+ * quiescent GDM2/CDM2 channel map. The EN751221 FE register layout is
+ * the same 0x1500/0x1400 layout described by the shared register file.
+ */
+ airoha_fe_wr(port->eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX), 0);
+ airoha_fe_wr(port->eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX), 0);
+ airoha_fe_wr(port->eth, REG_CDM_HWF_CHN_EN(2), 0);
+ WRITE_ONCE(port->xpon_mode, mode);
+
+ return 0;
+}
+
+static int econet_set_xpon_datapath(struct net_device *netdev,
+ enum airoha_xpon_mode mode, bool enable)
+{
+ struct airoha_gdm_dev *port;
+ int ret;
+
+ ret = econet_validate_xpon_gdm2(netdev, &port);
+ if (ret)
+ return ret;
+
+ switch (mode) {
+ case AIROHA_XPON_MODE_GPON:
+ /* EN7521 feDevGdm2Cdm2Stop(XPON_DISABLE): only downstream
+ * receive channels 0 and 1 are released here. T-CONT TX/HWF
+ * channels are enabled when their Alloc-ID is provisioned.
+ */
+ airoha_fe_rmw(port->eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX),
+ EN751221_GPON_RX_CHN_MASK,
+ enable ? EN751221_GPON_RX_CHN_MASK : 0);
+ break;
+ case AIROHA_XPON_MODE_EPON:
+ /* eponFeChannelEnable(): LLID 0..7 plus TX 16..23 used by
+ * the vendor OAM-favour path.
+ */
+ airoha_fe_rmw(port->eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX),
+ EN751221_EPON_TX_CHN_MASK,
+ enable ? EN751221_EPON_TX_CHN_MASK : 0);
+ airoha_fe_rmw(port->eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX),
+ EN751221_EPON_RX_CHN_MASK,
+ enable ? EN751221_EPON_RX_CHN_MASK : 0);
+ airoha_fe_rmw(port->eth, REG_CDM_HWF_CHN_EN(2),
+ EN751221_EPON_HWF_CHN_MASK,
+ enable ? EN751221_EPON_HWF_CHN_MASK : 0);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int econet_set_xpon_tcont_channel(struct net_device *netdev,
+ unsigned int channel, bool enable)
+{
+ struct airoha_gdm_dev *port;
+ u32 mask;
+ int ret;
+
+ ret = econet_validate_xpon_gdm2(netdev, &port);
+ if (ret)
+ return ret;
+ if (channel >= 32)
+ return -EINVAL;
+
+ mask = BIT(channel);
+ airoha_fe_rmw(port->eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX), mask,
+ enable ? mask : 0);
+ airoha_fe_rmw(port->eth, REG_CDM_HWF_CHN_EN(2), mask,
+ enable ? mask : 0);
+
+ return 0;
+}
+
+static int econet_register_xpon(struct net_device *netdev,
+ enum airoha_xpon_mode mode,
+ const struct airoha_xpon_link_ops *ops,
+ void *priv)
+{
+ struct airoha_gdm_dev *port;
+ unsigned long flags;
+ int ret;
+
+ if (!ops || !ops->start || !ops->stop || !ops->mac_irq)
+ return -EINVAL;
+
+ ret = econet_validate_xpon_gdm2(netdev, &port);
+ if (ret)
+ return ret;
+
+ mutex_lock(&port->xpon_lock);
+ if (port->xpon_ops) {
+ ret = -EBUSY;
+ goto out_unlock;
+ }
+
+ port->xpon_ops = ops;
+ port->xpon_priv = priv;
+ port->xpon_mode = mode;
+ port->flags |= AIROHA_PRIV_F_XPON_MANAGED;
+ spin_lock_irqsave(&port->xpon_state_lock, flags);
+ memset(&port->xpon_link, 0, sizeof(port->xpon_link));
+ port->xpon_link.mode = mode;
+ spin_unlock_irqrestore(&port->xpon_state_lock, flags);
+ netif_carrier_off(netdev);
+
+ /* EN751221 has no standalone xPON platform IRQ. The MAC interrupt is
+ * aggregated into QDMA_WAN bits 16/17 and is enabled only after the
+ * provider callback is fully published.
+ */
+ ret = airoha_qdma_mips_set_xpon_irq(port->qdma, mode, true);
+ if (ret) {
+ port->flags &= ~AIROHA_PRIV_F_XPON_MANAGED;
+ port->xpon_ops = NULL;
+ port->xpon_priv = NULL;
+ goto out_unlock;
+ }
+
+out_unlock:
+ mutex_unlock(&port->xpon_lock);
+ if (ret)
+ return ret;
+
+ if (netif_running(netdev))
+ return airoha_gdm_xpon_start(port);
+
+ return 0;
+}
+
+static void econet_unregister_xpon(struct net_device *netdev,
+ const struct airoha_xpon_link_ops *ops,
+ void *priv)
+{
+ struct airoha_gdm_dev *port;
+ unsigned long flags;
+
+ if (econet_validate_xpon_gdm2(netdev, &port))
+ return;
+ if (READ_ONCE(port->xpon_ops) != ops || READ_ONCE(port->xpon_priv)
!= priv)
+ return;
+
+ /* Mask the QDMA aggregator first and wait out an in-flight hard IRQ
+ * before the provider private pointer can disappear.
+ */
+ airoha_qdma_mips_set_xpon_irq(port->qdma, port->xpon_mode, false);
+ airoha_gdm_xpon_stop(port);
+
+ mutex_lock(&port->xpon_lock);
+ if (port->xpon_ops == ops && port->xpon_priv == priv) {
+ port->xpon_ops = NULL;
+ port->xpon_priv = NULL;
+ port->flags &= ~AIROHA_PRIV_F_XPON_MANAGED;
+ }
+ mutex_unlock(&port->xpon_lock);
+
+ spin_lock_irqsave(&port->xpon_state_lock, flags);
+ memset(&port->xpon_link, 0, sizeof(port->xpon_link));
+ spin_unlock_irqrestore(&port->xpon_state_lock, flags);
+ netif_carrier_off(netdev);
+}
+
+static void econet_xpon_update_link(struct net_device *netdev,
+ const struct airoha_xpon_link_state *state)
+{
+ struct airoha_xpon_link_state new_state;
+ struct airoha_gdm_dev *port;
+ unsigned long flags;
+
+ if (!state || econet_validate_xpon_gdm2(netdev, &port))
+ return;
+ if (!(READ_ONCE(port->flags) & AIROHA_PRIV_F_XPON_MANAGED) ||
+ state->mode != READ_ONCE(port->xpon_mode))
+ return;
+
+ new_state = *state;
+ new_state.valid = true;
+ spin_lock_irqsave(&port->xpon_state_lock, flags);
+ port->xpon_link = new_state;
+ spin_unlock_irqrestore(&port->xpon_state_lock, flags);
+
+ if (new_state.link && netif_running(netdev))
+ netif_carrier_on(netdev);
+ else
+ netif_carrier_off(netdev);
+}
+
+static int econet_xpon_control_start(struct net_device *netdev)
+{
+ struct airoha_gdm_dev *port;
+ int ret;
+
+ ret = econet_validate_xpon_gdm2(netdev, &port);
+ if (ret)
+ return ret;
+
+ mutex_lock(&port->xpon_lock);
+ if (port->xpon_control_started) {
+ ret = 0;
+ goto out;
+ }
+
+ /* OMCI activation is allowed while pon0 is administratively down. Keep
+ * QDMA1 alive independently from ndo_open(), mirroring the vendor WAN
+ * QDMA control-plane ownership.
+ */
+ airoha_qdma_start(port->qdma);
+ port->xpon_control_started = true;
+ ret = 0;
+out:
+ mutex_unlock(&port->xpon_lock);
+ return ret;
+}
+
+static void econet_xpon_control_stop(struct net_device *netdev)
+{
+ struct airoha_gdm_dev *port;
+
+ if (econet_validate_xpon_gdm2(netdev, &port))
+ return;
+
+ mutex_lock(&port->xpon_lock);
+ if (port->xpon_control_started) {
+ port->xpon_control_started = false;
+ airoha_qdma_stop(port->qdma);
+ }
+ mutex_unlock(&port->xpon_lock);
+}
+
+static void econet_xpon_dump_oam_rx_state(struct net_device *netdev)
+{
+ struct airoha_gdm_dev *port;
+
+ if (econet_validate_xpon_gdm2(netdev, &port))
+ return;
+
+ netdev_info(netdev,
+ "xPON OAM RX: packets=%lld bytes=%lld delivered=%lld
dropped=%lld no-handler=%lld txchn=%#010x rxchn=%#010x hwf=%#010x\n",
+ (long long)atomic64_read(&port->xpon_oam_rx_packets),
+ (long long)atomic64_read(&port->xpon_oam_rx_bytes),
+ (long long)atomic64_read(&port->xpon_oam_rx_delivered),
+ (long long)atomic64_read(&port->xpon_oam_rx_dropped),
+ (long long)atomic64_read(&port->xpon_oam_rx_no_handler),
+ airoha_fe_rr(port->eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX)),
+ airoha_fe_rr(port->eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX)),
+ airoha_fe_rr(port->eth, REG_CDM_HWF_CHN_EN(2)));
+}
+
+static int econet_register_xpon_oam(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler)
+{
+ struct airoha_gdm_dev *port;
+ int ret;
+
+ if (!handler || !handler->rx)
+ return -EINVAL;
+ ret = econet_validate_xpon_gdm2(netdev, &port);
+ if (ret)
+ return ret;
+ if (rcu_access_pointer(port->xpon_oam))
+ return -EBUSY;
+
+ rcu_assign_pointer(port->xpon_oam, handler);
+ return 0;
+}
+
+static void econet_unregister_xpon_oam(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler)
+{
+ struct airoha_gdm_dev *port;
+
+ if (econet_validate_xpon_gdm2(netdev, &port))
+ return;
+ if (rcu_access_pointer(port->xpon_oam) != handler)
+ return;
+
+ RCU_INIT_POINTER(port->xpon_oam, NULL);
+ synchronize_rcu();
+}
+
+static int econet_xmit_xpon_oam(struct net_device *netdev, struct
sk_buff *skb,
+ u8 channel, u16 gem_port_id)
+{
+ struct airoha_gdm_dev *port;
+ struct netdev_queue *txq;
+ union desc_msg msg = {};
+ int len, ret;
+
+ ret = econet_validate_xpon_gdm2(netdev, &port);
+ if (ret)
+ return ret;
+ if (!skb || gem_port_id > FIELD_MAX(ETX_XPON_GEM_MASK) ||
+ !READ_ONCE(port->xpon_control_started))
+ return -EINVAL;
+ if (skb_linearize(skb))
+ return -ENOMEM;
+
+ set_etx_queue(&msg.etx, 0);
+ set_etx_channel(&msg.etx, channel);
+ set_etx_oam(&msg.etx, true);
+ set_etx_xpon_gem(&msg.etx, gem_port_id);
+ set_etx_fport(&msg.etx, ETX_FPORT_GDM2);
+
+ skb->dev = netdev;
+ skb_set_queue_mapping(skb, 0);
+ len = skb->len;
+ txq = netdev_get_tx_queue(netdev, 0);
+ netdev_tx_sent_queue(txq, len);
+
+ ret = airoha_qdma_mips_xmit(port->qdma, skb, &msg, 0);
+ if (ret < 0) {
+ netdev_tx_completed_queue(txq, 1, len);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int econet_xpon_get_tx_info(struct net_device *netdev, bool
vlan_valid,
+ u16 vlan_id, bool pcp_valid, u8 pcp,
+ struct airoha_xpon_tx_info *info)
+{
+ struct airoha_gdm_dev *port;
+ int ret;
+
+ if (!info)
+ return -EINVAL;
+ ret = econet_validate_xpon_gdm2(netdev, &port);
+ if (ret)
+ return ret;
+ if (!(READ_ONCE(port->flags) & AIROHA_PRIV_F_XPON_MANAGED) ||
+ READ_ONCE(port->xpon_mode) != AIROHA_XPON_MODE_GPON)
+ return -EOPNOTSUPP;
+
+ return econet_xpon_lookup_service(port, vlan_valid, vlan_id,
+ pcp_valid, pcp, info);
+}
+
+/* Same rule from the classifier's point of view? The cookie is not
part of it. */
+static bool econet_xpon_rule_eq(const struct airoha_xpon_service_cfg *a,
+ const struct airoha_xpon_service_cfg *b)
+{
+ return a->gem_port_id == b->gem_port_id && a->tcont == b->tcont &&
+ a->queue == b->queue &&
+ a->vlan_valid == b->vlan_valid && a->vlan_id == b->vlan_id &&
+ a->pcp_valid == b->pcp_valid && a->pcp == b->pcp &&
+ a->default_service == b->default_service;
+}
+
+/* Slot currently referenced by @cookie, or -1. Caller holds the lock. */
+static int econet_xpon_find_cookie(struct airoha_gdm_dev *port, u32 cookie)
+{
+ int i, j;
+
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++) {
+ if (!port->xpon_services[i].valid)
+ continue;
+ for (j = 0; j < port->xpon_service_ncookies[i]; j++)
+ if (port->xpon_service_cookies[i][j] == cookie)
+ return i;
+ }
+ return -1;
+}
+
+/*
+ * Drop @cookie from whichever rule holds it, freeing the slot once no
cookie
+ * references it any more. Caller holds the lock.
+ */
+static void econet_xpon_forget_cookie(struct airoha_gdm_dev *port, u32
cookie)
+{
+ int i, j;
+
+ i = econet_xpon_find_cookie(port, cookie);
+ if (i < 0)
+ return;
+ for (j = 0; j < port->xpon_service_ncookies[i]; j++) {
+ if (port->xpon_service_cookies[i][j] != cookie)
+ continue;
+ port->xpon_service_ncookies[i]--;
+ port->xpon_service_cookies[i][j] =
+ port->xpon_service_cookies[i][port->xpon_service_ncookies[i]];
+ break;
+ }
+ if (!port->xpon_service_ncookies[i])
+ memset(&port->xpon_services[i], 0,
+ sizeof(port->xpon_services[i]));
+}
+
+static int econet_xpon_add_service(struct net_device *netdev,
+ const struct airoha_xpon_service_cfg *cfg)
+{
+ struct airoha_gdm_dev *port;
+ int empty = -1, i, ret;
+
+ if (!cfg || cfg->gem_port_id > FIELD_MAX(ETX_XPON_GEM_MASK) ||
+ cfg->tcont >= 32 || cfg->queue >= ECONET_NUM_QUEUES)
+ return -EINVAL;
+ ret = econet_validate_xpon_gdm2(netdev, &port);
+ if (ret)
+ return ret;
+
+ spin_lock_bh(&port->xpon_service_lock);
+
+ /* A cookie may come back with a changed rule: detach it first. */
+ econet_xpon_forget_cookie(port, cfg->cookie);
+
+ /*
+ * Share the slot when the very same rule is already installed.
Without
+ * this the table fills with duplicates -- the OMCI agent stages one
+ * service per UNI, so on a 4-LAN ONT every rule lands four or five
+ * times and a real provisioning run exhausts all 256 slots
(measured on
+ * a Movistar AR line: 115 entries on GEM 257 alone, three of them byte
+ * identical apart from the cookie), making the last services fail with
+ * -ENOSPC. Fewer entries also shortens the TX classifier scan.
+ */
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++) {
+ if (!port->xpon_services[i].valid ||
+ !econet_xpon_rule_eq(&port->xpon_services[i], cfg) ||
+ port->xpon_service_ncookies[i] >= AIROHA_XPON_SVC_MAX_COOKIES)
+ continue;
+ port->xpon_service_cookies[i][port->xpon_service_ncookies[i]++] =
+ cfg->cookie;
+ spin_unlock_bh(&port->xpon_service_lock);
+ return 0;
+ }
+
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++) {
+ if (!port->xpon_services[i].valid) {
+ empty = i;
+ break;
+ }
+ }
+ if (empty < 0) {
+ spin_unlock_bh(&port->xpon_service_lock);
+ return -ENOSPC;
+ }
+ if (cfg->default_service)
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++)
+ port->xpon_services[i].default_service = false;
+ port->xpon_services[empty] = *cfg;
+ port->xpon_services[empty].valid = true;
+ port->xpon_service_cookies[empty][0] = cfg->cookie;
+ port->xpon_service_ncookies[empty] = 1;
+ spin_unlock_bh(&port->xpon_service_lock);
+
+ return 0;
+}
+
+static bool econet_xpon_del_service(struct net_device *netdev, u32 cookie,
+ u16 *gem_port_id)
+{
+ struct airoha_gdm_dev *port;
+ bool found = false;
+ int i;
+
+ if (econet_validate_xpon_gdm2(netdev, &port))
+ return false;
+
+ spin_lock_bh(&port->xpon_service_lock);
+ i = econet_xpon_find_cookie(port, cookie);
+ if (i >= 0) {
+ if (gem_port_id)
+ *gem_port_id = port->xpon_services[i].gem_port_id;
+ /* Frees the slot only when this was the last cookie on it. */
+ econet_xpon_forget_cookie(port, cookie);
+ found = true;
+ }
+ spin_unlock_bh(&port->xpon_service_lock);
+
+ return found;
+}
+
+static bool econet_xpon_has_gem_service(struct net_device *netdev,
+ u16 gem_port_id)
+{
+ struct airoha_gdm_dev *port;
+ bool found = false;
+ int i;
+
+ if (econet_validate_xpon_gdm2(netdev, &port))
+ return false;
+
+ spin_lock_bh(&port->xpon_service_lock);
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++)
+ if (port->xpon_services[i].valid &&
+ port->xpon_services[i].gem_port_id == gem_port_id) {
+ found = true;
+ break;
+ }
+ spin_unlock_bh(&port->xpon_service_lock);
+
+ return found;
+}
+
+static void econet_xpon_flush_services(struct net_device *netdev)
+{
+ struct airoha_gdm_dev *port;
+
+ if (econet_validate_xpon_gdm2(netdev, &port))
+ return;
+
+ spin_lock_bh(&port->xpon_service_lock);
+ memset(port->xpon_services, 0, sizeof(port->xpon_services));
+ memset(port->xpon_service_cookies, 0,
sizeof(port->xpon_service_cookies));
+ memset(port->xpon_service_ncookies, 0,
sizeof(port->xpon_service_ncookies));
+ spin_unlock_bh(&port->xpon_service_lock);
+}
+
+bool econet_rx_xpon_oam(struct airoha_eth *eth, u8 qdma_id,
+ struct sk_buff *skb, union desc_msg *msg)
+{
+ struct airoha_xpon_oam_handler *handler;
+ struct airoha_gdm_dev *port;
+ u32 raw0, flags = 0;
+ u16 gem_port_id;
+ u8 channel;
+ u32 skb_len;
+ bool consumed = false;
+
+ if (!airoha_is(eth, econet_en751221) || qdma_id != 1)
+ return false;
+
+ port = airoha_eth_get_gdm_dev(eth, AIROHA_GDM2_IDX);
+ if (!port)
+ return false;
+
+ raw0 = READ_ONCE(msg->raw[0]);
+ if (!(raw0 & ERX_XPON_OAM))
+ return false;
+
+ if (!(READ_ONCE(port->flags) & AIROHA_PRIV_F_XPON_MANAGED))
+ return false;
+
+ gem_port_id = FIELD_GET(ERX_XPON_GEM_MASK, raw0);
+ channel = FIELD_GET(ERX_XPON_CHANNEL_MASK, raw0);
+ if (raw0 & ERX_XPON_CRC_ERROR)
+ flags |= AIROHA_XPON_OAM_RX_F_CRC_ERROR;
+
+ skb->dev = port->common.netdev;
+ skb_len = skb->len;
+ atomic64_inc(&port->xpon_oam_rx_packets);
+ atomic64_add(skb_len, &port->xpon_oam_rx_bytes);
+
+ rcu_read_lock();
+ handler = rcu_dereference(port->xpon_oam);
+ if (handler && handler->rx)
+ consumed = handler->rx(handler->priv, skb, channel,
gem_port_id, flags);
+ else
+ atomic64_inc(&port->xpon_oam_rx_no_handler);
+ rcu_read_unlock();
+
+ if (consumed) {
+ atomic64_inc(&port->xpon_oam_rx_delivered);
+ } else {
+ atomic64_inc(&port->xpon_oam_rx_dropped);
+ dev_kfree_skb_any(skb);
+ }
+
+ dev_dbg_ratelimited(eth->dev,
+ "EN751221 xPON OAM RX: len=%u msg0=%#010x channel=%u
gem=%u crc=%u runt=%u long=%u consumed=%u\n",
+ skb_len, raw0,
+ channel, gem_port_id, !!(raw0 & ERX_XPON_CRC_ERROR),
+ !!(raw0 & ERX_XPON_RUNT), !!(raw0 & ERX_XPON_LONG),
+ consumed);
+
+ return true;
+}
+
+void econet_xpon_irq(struct airoha_eth *eth, u8 qdma_id,
+ enum airoha_xpon_mode mode)
+{
+ const struct airoha_xpon_link_ops *ops;
+ struct airoha_gdm_dev *port;
+ void *xpon_priv;
+
+ if (!airoha_is(eth, econet_en751221) || qdma_id != 1)
+ return;
+
+ port = airoha_eth_get_gdm_dev(eth, AIROHA_GDM2_IDX);
+ if (!port)
+ return;
+ if (!(READ_ONCE(port->flags) & AIROHA_PRIV_F_XPON_MANAGED) ||
READ_ONCE(port->xpon_mode) != mode)
+ return;
+
+ ops = READ_ONCE(port->xpon_ops);
+ xpon_priv = READ_ONCE(port->xpon_priv);
+ if (ops && ops->mac_irq)
+ ops->mac_irq(xpon_priv);
+}
+
+static const struct airoha_eth_xpon_ops econet_xpon_ops = {
+ .set_mode = econet_set_xpon_mode,
+ .set_datapath = econet_set_xpon_datapath,
+ .set_tcont_channel = econet_set_xpon_tcont_channel,
+ .register_link = econet_register_xpon,
+ .unregister_link = econet_unregister_xpon,
+ .update_link = econet_xpon_update_link,
+ .control_start = econet_xpon_control_start,
+ .control_stop = econet_xpon_control_stop,
+ .dump_oam_rx_state = econet_xpon_dump_oam_rx_state,
+ .register_oam = econet_register_xpon_oam,
+ .unregister_oam = econet_unregister_xpon_oam,
+ .xmit_oam = econet_xmit_xpon_oam,
+ .add_service = econet_xpon_add_service,
+ .get_tx_info = econet_xpon_get_tx_info,
+ .del_service = econet_xpon_del_service,
+ .has_gem_service = econet_xpon_has_gem_service,
+ .flush_services = econet_xpon_flush_services,
+};
+
+static int airoha_fe_init(struct airoha_eth *eth);
+
+static void econet_prepare_qdma_cfg(struct airoha_qdma_mips_cfg *cfg,
+ const struct airoha_eth_soc_data *soc,
+ int id)
+{
+ int i;
+
+ memset(cfg, 0, sizeof(*cfg));
+ for (i = 0; i < QDMA_NUM_CHAINS; i++)
+ cfg->num_rx_descs[i] = 128;
+ for (i = 0; i < QDMA_NUM_CHAINS; i++)
+ cfg->num_tx_descs[i] = 128;
+ for (i = 0; i < QDMA_NUM_TX_DONE; i++) {
+ cfg->done_list_size[i] = 256;
+ cfg->done_list_irq_threshold[i] = 1;
+ }
+ cfg->fwd_max_packet_size = AIROHA_MAX_PACKET_SIZE;
+ cfg->fwd_low_threshold = 32;
+ cfg->num_fwd_descs = 256;
+ cfg->num_channels = ECONET_NUM_CHANNELS;
+ /*
+ * Do not enable the legacy RX_2B_OFFSET mode while RX buffers are
backed
+ * by recyclable page-pool fragments. The vendor driver owns a complete
+ * skb for each descriptor and places the DMA address two bytes before
+ * skb->data; that ownership/layout is not equivalent to the current
+ * page-pool zero-copy path on non-coherent MIPS. Keep the stable
layout
+ * until RX_2B_OFFSET is implemented with dedicated EN751221 buffers.
+ */
+ cfg->rx_2b_offset = false;
+ cfg->soc = soc;
+
+ /*
+ * QDMA0 is QDMA_LAN on EN7512/EN7521. Match the descriptor profile
+ * used by the vendor Ethernet/QDMA modules. In particular RX1 and
+ * the hardware-forwarding descriptor pool are deliberately larger
+ * than the generic EcoNet defaults because PPE traffic consumes the
+ * same LMGR resource pool.
+ */
+ if (soc->version == econet_en751221 && id == 0) {
+ cfg->num_rx_descs[0] = 128;
+ cfg->num_rx_descs[1] = 512;
+ cfg->num_tx_descs[0] = 128;
+ cfg->num_tx_descs[1] = 128;
+ cfg->done_list_size[0] = 2048;
+ cfg->done_list_irq_threshold[0] = 16;
+ cfg->num_fwd_descs = 1024;
+ cfg->fwd_low_threshold = 128;
+ cfg->num_channels = 8;
+ } else if (soc->version == econet_en751221 && id == 1) {
+ /* Vendor QDMA_WAN profile. */
+ cfg->num_rx_descs[0] = 512;
+ cfg->num_rx_descs[1] = 256;
+ cfg->num_tx_descs[0] = 1024;
+ cfg->num_tx_descs[1] = 128;
+ cfg->done_list_size[0] = 2048;
+ cfg->done_list_irq_threshold[0] = 16;
+ cfg->num_fwd_descs = 4096;
+ cfg->fwd_low_threshold = 256;
+ cfg->num_channels = 32;
+ }
+}
+
+static int airoha_set_macaddr(struct airoha_gdm_dev *dev, const u8 *addr)
+{
+ u8 ref_addr[ETH_ALEN] __aligned(2);
+ struct airoha_eth *eth = dev->eth;
+ u32 reg, val, lmin, lmax;
+ int i;
+
+ if (eth->soc->mac_addr_mode == AIROHA_MAC_ADDR_GDM_MASK) {
+ struct gdm_mymac_msb msb = {};
+ struct gdm_mymac_lsb lsb = {};
+
+ set_gdm_mymac_msb_lsb_mask(&msb, 0xf8);
+ set_gdm_mymac_msb_a(&msb, addr[0]);
+ set_gdm_mymac_msb_b(&msb, addr[1]);
+ set_gdm_mymac_lsb_c(&lsb, addr[2]);
+ set_gdm_mymac_lsb_d(&lsb, addr[3]);
+ set_gdm_mymac_lsb_e(&lsb, addr[4]);
+ set_gdm_mymac_lsb_f(&lsb, addr[5]);
+
+ scoped_guard(spinlock, &dev->reg_lock) {
+ econet_wreg(lsb, &dev->econet_regs->mymac_lsb);
+ econet_wreg(msb, &dev->econet_regs->mymac_msb);
+ }
+
+ return 0;
+ }
+
+ eth_zero_addr(ref_addr);
+ lmin = (addr[3] << 16) | (addr[4] << 8) | addr[5];
+ lmax = lmin;
+
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
+ struct airoha_gdm_port *port = eth->ports[i];
+ int j;
+
+ if (!port)
+ continue;
+
+ for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
+ struct airoha_gdm_dev *iter_dev;
+ struct net_device *netdev;
+
+ iter_dev = port->devs[j];
+ if (!iter_dev || iter_dev == dev)
+ continue;
+
+ if (airoha_is_lan_gdm_dev(iter_dev) !=
+ airoha_is_lan_gdm_dev(dev))
+ continue;
+
+ netdev = netdev_from_priv(iter_dev);
+ if (netdev->reg_state != NETREG_REGISTERED)
+ continue;
+
+ ether_addr_copy(ref_addr, netdev->dev_addr);
+ val = (netdev->dev_addr[3] << 16) |
+ (netdev->dev_addr[4] << 8) | netdev->dev_addr[5];
+ if (val < lmin)
+ lmin = val;
+ if (val > lmax)
+ lmax = val;
+ }
+ }
+
+ if (!is_zero_ether_addr(ref_addr) && memcmp(ref_addr, addr, 3)) {
+ /* According to the HW design, hw mac address MSBs must be
+ * the same for each net_device with the same LAN/WAN
+ * configuration.
+ */
+ struct net_device *netdev = netdev_from_priv(dev);
+
+ dev_warn(eth->dev,
+ "%s: wrong mac addr, MSBs must be %02x:%02x:%02x\n",
+ netdev->name, ref_addr[0], ref_addr[1],
+ ref_addr[2]);
+ dev_warn(eth->dev, "FE hw forwarding won't work properly\n");
+
+ return -EINVAL;
+ }
+
+ reg = airoha_is_lan_gdm_dev(dev) ? REG_FE_LAN_MAC_H : REG_FE_WAN_MAC_H;
+ val = (addr[0] << 16) | (addr[1] << 8) | addr[2];
+ airoha_fe_wr(eth, reg, val);
+
+ airoha_fe_wr(eth, REG_FE_MAC_LMIN(reg), lmin);
+ airoha_fe_wr(eth, REG_FE_MAC_LMAX(reg), lmax);
+
+ airoha_ppe_init_upd_mem(dev, addr);
+
+ return 0;
+}
+
+static int airoha_validate_xpon_gdm2(struct net_device *netdev,
+ struct airoha_gdm_dev **gdm)
+{
+ struct airoha_gdm_common *common;
+ struct airoha_gdm_dev *dev;
+
+ if (!netdev)
+ return -EINVAL;
+
+ common = airoha_gdm_common_from_netdev(netdev);
+ if (!common || !common->eth ||
+ common->family != AIROHA_ETH_FAMILY_AIROHA)
+ return -ENODEV;
+
+ dev = common->priv;
+ if (!dev || !dev->port || !airoha_is(common->eth, airoha_en7523) ||
+ dev->port->id != AIROHA_GDM2_IDX)
+ return -EOPNOTSUPP;
+
+ *gdm = dev;
+ return 0;
+}
+
+
+static void airoha_eth_update_gpon_pse_buf(struct airoha_gdm_dev *dev,
+ unsigned int active_channels)
+{
+ struct airoha_qdma *qdma = &dev->eth->qdma[1];
+ u32 chan_thr, total_thr, val;
+
+ /* gpon_init_qdma_tx_buff() starts with conservative thresholds while
+ * only the implicit ONU-ID T-CONT exists. xpon_reset_qdma_tx_buf()
+ * expands them after service T-CONTs have been provisioned.
+ */
+ if (active_channels <= 1) {
+ chan_thr = EN7523_GPON_PSE_BUF_INIT_CHAN_THR;
+ total_thr = EN7523_GPON_PSE_BUF_INIT_TOTAL_THR;
+ } else if (active_channels <= EN7523_GPON_PSE_BUF_FEW_CHAN_MAX) {
+ chan_thr = EN7523_GPON_PSE_BUF_FEW_CHAN_THR;
+ total_thr = (EN7523_GPON_PSE_BUF_FEW_CHAN_THR +
+ EN7523_GPON_PSE_BUF_CHAN_OVERHEAD - 1) *
+ active_channels;
+ } else {
+ chan_thr = EN7523_GPON_PSE_BUF_MANY_CHAN_THR;
+ total_thr = (EN7523_GPON_PSE_BUF_MANY_CHAN_THR +
+ EN7523_GPON_PSE_BUF_CHAN_OVERHEAD - 1) *
+ (active_channels - 1) + 1;
+ }
+
+ val = FIELD_PREP(PSE_BUF_CHAN_THR_MASK, chan_thr) |
+ FIELD_PREP(PSE_BUF_TOTAL_THR_MASK, total_thr);
+ airoha_qdma_rmw(qdma, REG_PSE_BUF_USAGE_CFG1,
+ PSE_BUF_CHAN_THR_MASK | PSE_BUF_TOTAL_THR_MASK, val);
+}
+
+static int airoha_xpon_set_mode(struct net_device *netdev,
+ enum airoha_xpon_mode mode)
+{
+ struct airoha_gdm_dev *dev;
+ u32 max_frame_len;
+ int ret;
+
+ ret = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (ret)
+ return ret;
+ if (mode != AIROHA_XPON_MODE_GPON &&
+ mode != AIROHA_XPON_MODE_EPON)
+ return -EINVAL;
+
+ /* FE_API_SET_GDMA_MISC_CONFIG(FE_GDM_SEL_GDMA2, mode).
+ * The vendor SDK sets GDMA2_RLS_MODE only for GPON.
+ */
+ airoha_fe_rmw(dev->eth, REG_GDM_MISC_CFG, GDM2_RLS_MODE_MASK,
+ mode == AIROHA_XPON_MODE_GPON ?
+ GDM2_RLS_MODE_MASK : 0);
+
+ /* gwan_init() in the EN7523 SDK programs GDMA2 to accept GPON
+ * frames from 60 to 2000 bytes. Restore the regular FE maximum when
+ * switching to EPON so the GPON-specific limit does not leak across
+ * modes.
+ */
+ max_frame_len = mode == AIROHA_XPON_MODE_GPON ?
+ EN7523_GPON_MAX_FRAME_LEN : AIROHA_MAX_RX_SIZE;
+ airoha_fe_rmw(dev->eth, REG_GDM_LEN_CFG(AIROHA_GDM2_IDX),
+ GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK,
+ FIELD_PREP(GDM_SHORT_LEN_MASK, ETH_HLEN) |
+ FIELD_PREP(GDM_LONG_LEN_MASK, max_frame_len));
+
+ /* gpon_init_qdma_tx_buff() starts with the implicit ONU-ID T-CONT. */
+ if (mode == AIROHA_XPON_MODE_GPON) {
+ /*
+ * Downstream GPON frames reach GDM2 after the GPON MAC has
+ * already checked them, and a frame that failed decryption is
+ * more useful counted than silently dropped here.
+ */
+ airoha_fe_clear(dev->eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX),
+ GDM_DROP_CRC_ERR_MASK);
+ airoha_eth_update_gpon_pse_buf(dev, 1);
+ }
+
+ /* A mode switch starts from a quiescent GDM2/CDM2 datapath. */
+ airoha_fe_clear(dev->eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX), ~0U);
+ airoha_fe_clear(dev->eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX), ~0U);
+ airoha_fe_clear(dev->eth, REG_CDM_HWF_CHN_EN(2), ~0U);
+
+ return 0;
+}
+
+static int airoha_xpon_set_datapath(struct net_device *netdev,
+ enum airoha_xpon_mode mode, bool enable)
+{
+ struct airoha_gdm_dev *dev;
+ int ret;
+
+ ret = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (ret)
+ return ret;
+
+ switch (mode) {
+ case AIROHA_XPON_MODE_GPON:
+ /* The vendor GPON path enables downstream channels 0 and 1
+ * through the regular GDM2 receive channel register.
+ */
+ airoha_fe_rmw(dev->eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX),
+ EN7523_GDM2_GPON_RX_CHN_MASK,
+ enable ? EN7523_GDM2_GPON_RX_CHN_MASK : 0);
+ break;
+ case AIROHA_XPON_MODE_EPON:
+ /* xpon_1g/eponFeChannelEnable(): LLID channels 0..7,
+ * plus TX channels 16..23 for OAM-favour mode.
+ */
+ airoha_fe_rmw(dev->eth,
+ REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX),
+ EN7523_GDM2_EPON_TX_CHN_MASK,
+ enable ? EN7523_GDM2_EPON_TX_CHN_MASK : 0);
+ airoha_fe_rmw(dev->eth,
+ REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX),
+ EN7523_GDM2_EPON_RX_CHN_MASK,
+ enable ? EN7523_GDM2_EPON_RX_CHN_MASK : 0);
+ airoha_fe_rmw(dev->eth, REG_CDM_HWF_CHN_EN(2),
+ EN7523_CDM2_EPON_HWF_CHN_MASK,
+ enable ? EN7523_CDM2_EPON_HWF_CHN_MASK : 0);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int airoha_qdma_set_gpon_dba_report(struct net_device *netdev,
+ int channel, bool enable);
+
+static int airoha_xpon_set_tcont_channel(struct net_device *netdev,
+ unsigned int channel, bool enable)
+{
+ struct airoha_gdm_dev *dev;
+ u32 mask;
+ int ret;
+
+ ret = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (ret)
+ return ret;
+ if (channel >= 32)
+ return -EINVAL;
+
+ /*
+ * gpon_qos_init() in the EN7523 SDK configures both egress trTCM
+ * buckets for every QDMA WAN channel before enabling the matching
+ * T-CONT. Despite the API name, TX_DBA_REPORT, these parameters
+ * are programmed in the QDMA egress trTCM table as part of the
+ * vendor GPON QoS/DBA setup.
+ */
+ if (dev->xpon_mode == AIROHA_XPON_MODE_GPON && enable) {
+ ret = airoha_qdma_set_gpon_dba_report(netdev, channel, true);
+ if (ret)
+ return ret;
+ }
+
+ mask = BIT(channel);
+ airoha_fe_rmw(dev->eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX),
+ mask, enable ? mask : 0);
+ airoha_fe_rmw(dev->eth, REG_CDM_HWF_CHN_EN(2),
+ mask, enable ? mask : 0);
+
+ if (dev->xpon_mode == AIROHA_XPON_MODE_GPON) {
+ u32 tx_channels;
+
+ tx_channels = airoha_fe_rr(dev->eth,
+ REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX));
+ airoha_eth_update_gpon_pse_buf(dev, hweight32(tx_channels));
+
+ if (!enable) {
+ ret = airoha_qdma_set_gpon_dba_report(netdev, channel, false);
+ if (ret)
+ return ret;
+ }
+
+ netdev_info(netdev,
+ "GPON DBA channel %u %s: CIR/PIR=%#xKbps CBS=%#x
PBS=%#x\n",
+ channel, enable ? "enabled" : "disabled",
+ enable ? EN7523_GPON_DBA_RATE_KBPS : 0,
+ enable ? EN7523_GPON_DBA_CBS_BYTES : 0,
+ enable ? EN7523_GPON_DBA_PBS_BYTES : 0);
+ }
+
+ return 0;
+}
+
+
+static int airoha_xpon_register_link(struct net_device *netdev,
+ enum airoha_xpon_mode mode,
+ const struct airoha_xpon_link_ops *ops,
+ void *priv)
+{
+ struct airoha_gdm_dev *dev;
+ unsigned long flags;
+ int ret;
+
+ if (!ops || !ops->start || !ops->stop)
+ return -EINVAL;
+
+ ret = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (ret)
+ return ret;
+
+ mutex_lock(&dev->xpon_lock);
+ if (dev->xpon_ops) {
+ ret = -EBUSY;
+ goto out;
+ }
+
+ dev->xpon_ops = ops;
+ dev->xpon_priv = priv;
+ dev->xpon_mode = mode;
+ dev->flags |= AIROHA_PRIV_F_XPON_MANAGED;
+ spin_lock_irqsave(&dev->xpon_state_lock, flags);
+ memset(&dev->xpon_link, 0, sizeof(dev->xpon_link));
+ dev->xpon_link.mode = mode;
+ spin_unlock_irqrestore(&dev->xpon_state_lock, flags);
+ netif_carrier_off(netdev);
+out:
+ mutex_unlock(&dev->xpon_lock);
+ if (ret)
+ return ret;
+
+ if (netif_running(netdev))
+ return airoha_gdm_xpon_start(dev);
+
+ return 0;
+}
+
+static void airoha_xpon_unregister_link(struct net_device *netdev,
+ const struct airoha_xpon_link_ops *ops,
+ void *priv)
+{
+ struct airoha_gdm_dev *dev;
+ unsigned long flags;
+
+ if (airoha_validate_xpon_gdm2(netdev, &dev))
+ return;
+
+ airoha_gdm_xpon_stop(dev);
+ mutex_lock(&dev->xpon_lock);
+ if (dev->xpon_ops == ops && dev->xpon_priv == priv) {
+ dev->xpon_ops = NULL;
+ dev->xpon_priv = NULL;
+ }
+ mutex_unlock(&dev->xpon_lock);
+ spin_lock_irqsave(&dev->xpon_state_lock, flags);
+ memset(&dev->xpon_link, 0, sizeof(dev->xpon_link));
+ spin_unlock_irqrestore(&dev->xpon_state_lock, flags);
+ netif_carrier_off(netdev);
+}
+
+static void airoha_xpon_update_link(struct net_device *netdev,
+ const struct airoha_xpon_link_state *state)
+{
+ struct airoha_xpon_link_state new_state;
+ struct airoha_gdm_dev *dev;
+ unsigned long flags;
+
+ if (!state)
+ return;
+ if (airoha_validate_xpon_gdm2(netdev, &dev))
+ return;
+ if (!(dev->flags & AIROHA_PRIV_F_XPON_MANAGED))
+ return;
+ if (state->mode != READ_ONCE(dev->xpon_mode))
+ return;
+
+ new_state = *state;
+ new_state.valid = true;
+
+ spin_lock_irqsave(&dev->xpon_state_lock, flags);
+ dev->xpon_link = new_state;
+ spin_unlock_irqrestore(&dev->xpon_state_lock, flags);
+
+ if (new_state.link && netif_running(netdev))
+ netif_carrier_on(netdev);
+ else
+ netif_carrier_off(netdev);
+}
+
+static void airoha_eth_xpon_dump_rx_desc(struct net_device *netdev,
+ struct airoha_queue *q,
+ unsigned int index, u32 ring_base)
+{
+ struct airoha_qdma_desc *desc;
+ u32 ctrl, msg0, msg1, msg2, msg3;
+ unsigned int channel, desc_len, gem_port_id;
- for (i = 0; i < AIROHA_NUM_QOS_CHANNELS; i++) {
- /* Tx-cpu transferred count */
- airoha_qdma_wr(qdma, REG_CNTR_VAL(i << 1), 0);
- airoha_qdma_wr(qdma, REG_CNTR_CFG(i << 1),
- CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
- CNTR_ALL_DSCP_RING_EN_MASK |
- FIELD_PREP(CNTR_CHAN_MASK, i));
- /* Tx-fwd transferred count */
- airoha_qdma_wr(qdma, REG_CNTR_VAL((i << 1) + 1), 0);
- airoha_qdma_wr(qdma, REG_CNTR_CFG((i << 1) + 1),
- CNTR_EN_MASK | CNTR_ALL_QUEUE_EN_MASK |
- CNTR_ALL_DSCP_RING_EN_MASK |
- FIELD_PREP(CNTR_SRC_MASK, 1) |
- FIELD_PREP(CNTR_CHAN_MASK, i));
+ if (index >= q->ndesc)
+ return;
+
+ desc = &q->desc[index];
+ ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
+ msg0 = le32_to_cpu(READ_ONCE(desc->msg0));
+ msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
+ msg2 = le32_to_cpu(READ_ONCE(desc->msg2));
+ msg3 = le32_to_cpu(READ_ONCE(desc->msg3));
+ desc_len = FIELD_GET(EN7523_QDMA_DESC_LEN_MASK, ctrl);
+ channel = FIELD_GET(EN7523_QDMA_ETH_RXMSG_CHAN_MASK, msg0);
+ gem_port_id = FIELD_GET(EN7523_QDMA_ETH_RXMSG_GEM_MASK, msg0);
+
+ dev_info_ratelimited(&netdev->dev,
+ "GPON OAM RX descriptor: index=%u dma=%#010x
ctrl=%#010x done=%u len=%u msg=%#010x/%#010x/%#010x/%#010x oam=%u
channel=%u gem=%u no-mic=%u\n",
+ index,
+ ring_base + index * sizeof(struct airoha_qdma_desc),
+ ctrl, !!(ctrl & QDMA_DESC_DONE_MASK), desc_len,
+ msg0, msg1, msg2, msg3,
+ !!(msg0 & EN7523_QDMA_ETH_RXMSG_OAM_MASK),
+ channel, gem_port_id,
+ !!(msg0 & EN7523_QDMA_ETH_RXMSG_NO_MIC_MASK));
+}
+
+static void airoha_xpon_dump_oam_rx_state(struct net_device *netdev)
+{
+ struct airoha_gdm_dev *dev;
+ struct airoha_queue *q;
+ struct airoha_qdma *qdma;
+ u32 cpu_raw, dma_raw, global_cfg, int_enable, int_status;
+ u32 ring_base, ring_size, scatter_cfg;
+ u32 cpu_idx, dma_idx;
+ unsigned int head, tail;
+ int ret;
+
+ ret = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (ret)
+ return;
+
+ rcu_read_lock();
+ qdma = rcu_dereference(dev->qdma);
+ if (!qdma || !qdma->q_rx) {
+ rcu_read_unlock();
+ dev_info_ratelimited(&netdev->dev,
+ "GPON OAM RX state: QDMA is not attached\n");
+ return;
+ }
+
+ q = &qdma->q_rx[15];
+ if (!q->desc || !q->ndesc) {
+ rcu_read_unlock();
+ dev_info_ratelimited(&netdev->dev,
+ "GPON OAM RX state: RX ring 15 is not initialized\n");
+ return;
+ }
+
+ head = READ_ONCE(q->head);
+ tail = READ_ONCE(q->tail);
+ global_cfg = airoha_qdma_rr(qdma, REG_QDMA_GLOBAL_CFG);
+ ring_base = airoha_qdma_rr(qdma, REG_RX_RING_BASE(15));
+ ring_size = airoha_qdma_rr(qdma, REG_RX_RING_SIZE(15));
+ scatter_cfg = airoha_qdma_rr(qdma, REG_RX_SCATTER_CFG(15));
+ cpu_raw = airoha_qdma_rr(qdma, REG_RX_CPU_IDX(15));
+ dma_raw = airoha_qdma_rr(qdma, REG_RX_DMA_IDX(15));
+ cpu_idx = FIELD_GET(RX_RING_CPU_IDX_MASK, cpu_raw);
+ dma_idx = FIELD_GET(RX_RING_DMA_IDX_MASK, dma_raw);
+ int_status = airoha_qdma_rr(qdma, REG_INT_STATUS(1));
+ int_enable = airoha_qdma_rr(qdma, REG_INT_ENABLE(0, 1));
+
+ dev_info_ratelimited(&netdev->dev,
+ "GPON OAM RX state: qdma=%td global=%#010x
cdm2-fwd=%#010x ring=15 base=%#010x size=%#010x scatter=%#010x ndesc=%d
queued=%d\n",
+ qdma - &dev->eth->qdma[0], global_cfg,
+ airoha_fe_rr(dev->eth, REG_CDM_FWD_CFG(2)),
+ ring_base, ring_size, scatter_cfg, q->ndesc,
+ READ_ONCE(q->queued));
+ dev_info_ratelimited(&netdev->dev,
+ "GPON OAM RX indices: sw=%u/%u hw=%u/%u
raw=%#010x/%#010x int=%#010x enable=%#010x check-done=%u rx-dma=%u
rx-busy=%u\n",
+ head, tail, cpu_idx, dma_idx, cpu_raw, dma_raw,
+ int_status, int_enable,
+ !!(global_cfg & GLOBAL_CFG_CHECK_DONE_MASK),
+ !!(global_cfg & GLOBAL_CFG_RX_DMA_EN_MASK),
+ !!(global_cfg & GLOBAL_CFG_RX_DMA_BUSY_MASK));
+
+ /* Read descriptor contents after all producer index observations. */
+ dma_rmb();
+ airoha_eth_xpon_dump_rx_desc(netdev, q, q->ndesc - 1, ring_base);
+ airoha_eth_xpon_dump_rx_desc(netdev, q, 0, ring_base);
+ airoha_eth_xpon_dump_rx_desc(netdev, q, 1, ring_base);
+ airoha_eth_xpon_dump_rx_desc(netdev, q, 2, ring_base);
+ rcu_read_unlock();
+}
+
+static int airoha_xpon_register_oam(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler)
+{
+ struct airoha_gdm_dev *dev;
+ int ret;
+
+ if (!handler || !handler->rx)
+ return -EINVAL;
+
+ ret = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (ret)
+ return ret;
+ if (rcu_access_pointer(dev->xpon_oam))
+ return -EBUSY;
+
+ rcu_assign_pointer(dev->xpon_oam, handler);
+ return 0;
+}
+
+static void airoha_xpon_unregister_oam(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler)
+{
+ struct airoha_gdm_dev *dev;
+
+ if (airoha_validate_xpon_gdm2(netdev, &dev))
+ return;
+ if (rcu_access_pointer(dev->xpon_oam) != handler)
+ return;
+
+ RCU_INIT_POINTER(dev->xpon_oam, NULL);
+ synchronize_rcu();
+}
+
+static void airoha_set_gdm_port_fwd_cfg(struct airoha_eth *eth, u32 addr,
+ u32 val)
+{
+ airoha_fe_rmw(eth, addr, GDM_OCFQ_MASK,
+ FIELD_PREP(GDM_OCFQ_MASK, val));
+ airoha_fe_rmw(eth, addr, GDM_MCFQ_MASK,
+ FIELD_PREP(GDM_MCFQ_MASK, val));
+ airoha_fe_rmw(eth, addr, GDM_BCFQ_MASK,
+ FIELD_PREP(GDM_BCFQ_MASK, val));
+ airoha_fe_rmw(eth, addr, GDM_UCFQ_MASK,
+ FIELD_PREP(GDM_UCFQ_MASK, val));
+}
+
+static int airoha_set_vip_for_gdm_port(struct airoha_gdm_dev *dev, bool
enable)
+{
+ struct airoha_gdm_port *port = dev->port;
+ struct airoha_eth *eth = dev->eth;
+ u32 vip_port;
+
+ if (!eth->soc->ops.get_vip_port)
+ return 0;
+
+ vip_port = eth->soc->ops.get_vip_port(port, dev->nbq);
+ if (enable) {
+ airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port);
+ airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port);
+ } else {
+ airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, vip_port);
+ airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, vip_port);
+ }
+
+ return 0;
+}
+
+static void airoha_fe_maccr_init(struct airoha_eth *eth)
+{
+ int p;
+
+ for (p = 1; p <= eth->soc->max_gdm_ports; p++) {
+ airoha_fe_set(eth, REG_GDM_FWD_CFG(p),
+ GDM_TCP_CKSUM_MASK | GDM_UDP_CKSUM_MASK |
+ GDM_IP4_CKSUM_MASK | GDM_DROP_CRC_ERR_MASK);
+ airoha_fe_rmw(eth, REG_GDM_LEN_CFG(p),
+ GDM_SHORT_LEN_MASK | GDM_LONG_LEN_MASK,
+ FIELD_PREP(GDM_SHORT_LEN_MASK, 60) |
+ FIELD_PREP(GDM_LONG_LEN_MASK, AIROHA_MAX_RX_SIZE));
+ }
+
+ airoha_fe_rmw(eth, REG_CDM_VLAN_CTRL(1),
+ CDM_VLAN_MASK | STAG_EN,
+ FIELD_PREP(CDM_VLAN_MASK, 0x8100) |
+ STAG_EN);
+
+ airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PAD);
+}
+
+static void airoha_fe_vip_setup(struct airoha_eth *eth)
+{
+ bool full_l3_vip = airoha_is(eth, econet_en751221,
+ airoha_en7581, airoha_an7583);
+
+ /* EN751221 and EN7581-class FEs keep the L3 protocol VIP entries.
+ * EN7516/EN7527/EN7528 omit them, while EN7523 handles these paths
+ * through its newer classifier setup.
+ */
+ if (full_l3_vip) {
+ /* IPv4 -> ICMP */
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(0), IPPROTO_ICMP);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(0),
+ PATN_FCPU_EN_MASK |
+ FIELD_PREP(PATN_TYPE_MASK, 2) | PATN_EN_MASK);
+
+ /* IPv4 -> IGMP */
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(2), IPPROTO_IGMP);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(2),
+ PATN_FCPU_EN_MASK |
+ FIELD_PREP(PATN_TYPE_MASK, 2) | PATN_EN_MASK);
+
+ /* IPv6 -> ICMPv6 */
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(5), IPPROTO_ICMPV6);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(5),
+ PATN_FCPU_EN_MASK |
+ FIELD_PREP(PATN_TYPE_MASK, 2) | PATN_EN_MASK);
+ }
+
+ /* ARP remains a FE VIP entry on EcoNet and EN7581-class parts.
+ * EN7523 routes ARP through IFC instead.
+ */
+ if (!airoha_is(eth, airoha_en7523)) {
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(1), ETH_P_ARP);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(1),
+ PATN_FCPU_EN_MASK | PATN_EN_MASK);
+ }
+
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(3), ETH_P_PPP_DISC);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(3), PATN_FCPU_EN_MASK | PATN_EN_MASK);
+
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(4), PPP_LCP);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(4),
+ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+ PATN_EN_MASK);
+
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(6), PPP_IPCP);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(6),
+ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+ PATN_EN_MASK);
+
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(7), PPP_CHAP);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(7),
+ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+ PATN_EN_MASK);
+
+ /* The newer EN7528 SDK deliberately leaves BOOTP/DHCPv4 out of
+ * the VIP table. Keep the legacy EN751221 and Airoha behaviour.
+ */
+ if (!airoha_is(eth, econet_en7528)) {
+ /* BOOTP (0x43) */
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(8), 0x43);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(8),
+ PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
+ FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+
+ /* BOOTP (0x44) */
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(9), 0x44);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(9),
+ PATN_FCPU_EN_MASK | PATN_SP_EN_MASK |
+ FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+ }
+
+ /* ISAKMP */
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(10), 0x1f401f4);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(10),
+ PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
+ FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(11), PPP_IPV6CP);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(11),
+ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+ PATN_EN_MASK);
+
+ /* DHCPv6 */
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(12), 0x2220223);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(12),
+ PATN_FCPU_EN_MASK | PATN_DP_EN_MASK | PATN_SP_EN_MASK |
+ FIELD_PREP(PATN_TYPE_MASK, 4) | PATN_EN_MASK);
+
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(19), PPP_PAP);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(19),
+ PATN_FCPU_EN_MASK | FIELD_PREP(PATN_TYPE_MASK, 1) |
+ PATN_EN_MASK);
+
+ /* ETH->ETH_P_1905 (0x893a) */
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(20), 0x893a);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(20),
+ PATN_FCPU_EN_MASK | PATN_EN_MASK);
+
+ airoha_fe_wr(eth, REG_FE_VIP_PATN(21), ETH_P_LLDP);
+ airoha_fe_wr(eth, REG_FE_VIP_EN(21),
+ PATN_FCPU_EN_MASK | PATN_EN_MASK);
+}
+
+static u32 airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth *eth,
+ u32 port, u32 queue)
+{
+ // if (!airoha_is(eth, airoha_en7523))
+ airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
+ PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK,
+ FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
+ FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue));
+
+ return airoha_fe_get(eth, REG_FE_PSE_QUEUE_CFG_VAL,
+ PSE_CFG_OQ_RSV_MASK);
+}
+
+static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth,
+ u32 port, u32 queue, u32 val)
+{
+ airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_VAL, PSE_CFG_OQ_RSV_MASK,
+ FIELD_PREP(PSE_CFG_OQ_RSV_MASK, val));
+ // if (!airoha_is(eth, airoha_en7523))
+ airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
+ PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK |
+ PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK,
+ FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
+ FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue) |
+ PSE_CFG_WR_EN_MASK | PSE_CFG_OQRSV_SEL_MASK);
+}
+
+static u32 airoha_fe_get_pse_all_rsv(struct airoha_eth *eth)
+{
+ return airoha_fe_get(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK);
+}
+
+static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth,
+ u32 port, u32 queue, u32 val)
+{
+ u32 orig_val = airoha_fe_get_pse_queue_rsv_pages(eth, port, queue);
+ u32 tmp, all_rsv, fq_limit;
+
+ airoha_fe_set_pse_queue_rsv_pages(eth, port, queue, val);
+
+ /* modify all rsv */
+ all_rsv = airoha_fe_get_pse_all_rsv(eth);
+ all_rsv += (val - orig_val);
+ airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK,
+ FIELD_PREP(PSE_ALLRSV_MASK, all_rsv));
+
+ if (airoha_is(eth, airoha_en7523)) {
+ airoha_fe_wr(eth, REG_FE_PSE_BUF_SET, 0x2b4);
+ airoha_fe_wr(eth, REG_PSE_SHARE_USED_THD, 0x1e001f4);
+ return 0;
}
+
+ /* modify hthd */
+ fq_limit = airoha_fe_get(eth, eth->soc->pse_fq_cfg, PSE_FQ_LIMIT_MASK);
+ tmp = fq_limit - all_rsv - 0x20;
+ airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
+ PSE_SHARE_USED_HTHD_MASK,
+ FIELD_PREP(PSE_SHARE_USED_HTHD_MASK, tmp));
+
+ tmp = fq_limit - all_rsv - 0x100;
+ airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
+ PSE_SHARE_USED_MTHD_MASK,
+ FIELD_PREP(PSE_SHARE_USED_MTHD_MASK, tmp));
+ tmp = (3 * tmp) >> 2;
+ airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET,
+ PSE_SHARE_USED_LTHD_MASK,
+ FIELD_PREP(PSE_SHARE_USED_LTHD_MASK, tmp));
+
+ return 0;
}
-static int airoha_qdma_hw_init(struct airoha_qdma *qdma)
-{
- int i;
+static void airoha_fe_pse_ports_init(struct airoha_eth *eth)
+{
+ const u32 pse_port_num_queues[] = {
+ [FE_PSE_PORT_CDM1] = 6,
+ [FE_PSE_PORT_GDM1] = 6,
+ [FE_PSE_PORT_GDM2] = 32,
+ [FE_PSE_PORT_GDM3] = 6,
+ [FE_PSE_PORT_PPE1] = 4,
+ [FE_PSE_PORT_CDM2] = 6,
+ [FE_PSE_PORT_CDM3] = 8,
+ [FE_PSE_PORT_CDM4] = 10,
+ [FE_PSE_PORT_PPE2] = 4,
+ [FE_PSE_PORT_GDM4] = 2,
+ [FE_PSE_PORT_CDM5] = 2,
+ };
+ int q;
+
+ if (airoha_ppe_is_enabled(eth, 1)) {
+ u32 all_rsv;
- for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) {
- /* clear pending irqs */
- airoha_qdma_wr(qdma, REG_INT_STATUS(i), 0xffffffff);
- /* setup rx irqs */
- airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX0,
- INT_RX0_MASK(RX_IRQ_BANK_PIN_MASK(i)));
- airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX1,
- INT_RX1_MASK(RX_IRQ_BANK_PIN_MASK(i)));
- airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX2,
- INT_RX2_MASK(RX_IRQ_BANK_PIN_MASK(i)));
- airoha_qdma_irq_enable(&qdma->irq_banks[i], QDMA_INT_REG_IDX3,
- INT_RX3_MASK(RX_IRQ_BANK_PIN_MASK(i)));
+ /* hw misses PPE2 oq rsv */
+ all_rsv = airoha_fe_get_pse_all_rsv(eth);
+ all_rsv += PSE_RSV_PAGES *
+ pse_port_num_queues[FE_PSE_PORT_PPE2];
+ airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK,
+ FIELD_PREP(PSE_ALLRSV_MASK, all_rsv));
}
- /* setup tx irqs */
- airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX0,
- TX_COHERENT_LOW_INT_MASK | INT_TX_MASK);
- airoha_qdma_irq_enable(&qdma->irq_banks[0], QDMA_INT_REG_IDX4,
- TX_COHERENT_HIGH_INT_MASK);
-
- /* setup irq binding */
- for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
- if (!qdma->q_tx[i].ndesc)
- continue;
- if (TX_RING_IRQ_BLOCKING_MAP_MASK & BIT(i))
- airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(i),
- TX_RING_IRQ_BLOCKING_CFG_MASK);
+ /* CDM1 */
+ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM1, q,
+ PSE_QUEUE_RSV_PAGES);
+ /* GDM1 */
+ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM1]; q++)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM1, q,
+ PSE_QUEUE_RSV_PAGES);
+ /* GDM2 */
+ for (q = 6; q < pse_port_num_queues[FE_PSE_PORT_GDM2]; q++)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM2, q, 0);
+ /* GDM3 */
+ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM3]; q++)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM3, q,
+ PSE_QUEUE_RSV_PAGES);
+ /* PPE1 */
+ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE1]; q++) {
+ if (q < pse_port_num_queues[FE_PSE_PORT_PPE1] / 2)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q,
+ PSE_QUEUE_RSV_PAGES);
else
- airoha_qdma_clear(qdma, REG_TX_RING_BLOCKING(i),
- TX_RING_IRQ_BLOCKING_CFG_MASK);
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE1, q, 0);
+ }
+ /* CDM2 */
+ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM2]; q++)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM2, q,
+ PSE_QUEUE_RSV_PAGES);
+ /* CDM3 */
+ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM3] - 1; q++)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM3, q, 0);
+ /* CDM4 */
+ for (q = 4; q < pse_port_num_queues[FE_PSE_PORT_CDM4]; q++)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM4, q,
+ PSE_QUEUE_RSV_PAGES);
+ if (airoha_ppe_is_enabled(eth, 1)) {
+ /* PPE2 */
+ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_PPE2]; q++) {
+ if (q < pse_port_num_queues[FE_PSE_PORT_PPE2] / 2)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2,
+ q,
+ PSE_QUEUE_RSV_PAGES);
+ else
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_PPE2,
+ q, 0);
+ }
}
+ /* GDM4 */
+ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_GDM4]; q++)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_GDM4, q,
+ PSE_QUEUE_RSV_PAGES);
+ /* CDM5 */
+ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM5]; q++)
+ airoha_fe_set_pse_oq_rsv(eth, FE_PSE_PORT_CDM5, q,
+ PSE_QUEUE_RSV_PAGES);
+}
- airoha_qdma_wr(qdma, REG_QDMA_GLOBAL_CFG,
- FIELD_PREP(GLOBAL_CFG_DMA_PREFERENCE_MASK, 3) |
- GLOBAL_CFG_CPU_TXR_RR_MASK |
- GLOBAL_CFG_PAYLOAD_BYTE_SWAP_MASK |
- GLOBAL_CFG_MULTICAST_MODIFY_FP_MASK |
- GLOBAL_CFG_MULTICAST_EN_MASK |
- GLOBAL_CFG_IRQ0_EN_MASK | GLOBAL_CFG_IRQ1_EN_MASK |
- GLOBAL_CFG_TX_WB_DONE_MASK |
- FIELD_PREP(GLOBAL_CFG_MAX_ISSUE_NUM_MASK, 2));
+static int airoha_fe_mc_vlan_clear(struct airoha_eth *eth)
+{
+ int i;
- airoha_qdma_init_qos(qdma);
+ for (i = 0; i < AIROHA_FE_MC_MAX_VLAN_TABLE; i++) {
+ int err, j;
+ u32 val;
- /* disable qdma rx delay interrupt */
- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
- if (!qdma->q_rx[i].ndesc)
- continue;
+ airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
- airoha_qdma_clear(qdma, REG_RX_DELAY_INT_IDX(i),
- RX_DELAY_INT_MASK);
- }
+ val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
+ MC_VLAN_CFG_TABLE_SEL_MASK | MC_VLAN_CFG_RW_MASK;
+ airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
+ err = read_poll_timeout(airoha_fe_rr, val,
+ val & MC_VLAN_CFG_CMD_DONE_MASK,
+ USEC_PER_MSEC, 5 * USEC_PER_MSEC,
+ false, eth, REG_MC_VLAN_CFG);
+ if (err)
+ return err;
- airoha_qdma_set(qdma, REG_TXQ_CNGST_CFG,
- TXQ_CNGST_DROP_EN | TXQ_CNGST_DEI_DROP_EN);
- airoha_qdma_init_qos_stats(qdma);
+ for (j = 0; j < AIROHA_FE_MC_MAX_VLAN_PORT; j++) {
+ airoha_fe_wr(eth, REG_MC_VLAN_DATA, 0x0);
+
+ val = FIELD_PREP(MC_VLAN_CFG_TABLE_ID_MASK, i) |
+ FIELD_PREP(MC_VLAN_CFG_PORT_ID_MASK, j) |
+ MC_VLAN_CFG_RW_MASK;
+ airoha_fe_wr(eth, REG_MC_VLAN_CFG, val);
+ err = read_poll_timeout(airoha_fe_rr, val,
+ val & MC_VLAN_CFG_CMD_DONE_MASK,
+ USEC_PER_MSEC,
+ 5 * USEC_PER_MSEC, false, eth,
+ REG_MC_VLAN_CFG);
+ if (err)
+ return err;
+ }
+ }
return 0;
}
-static irqreturn_t airoha_irq_handler(int irq, void *dev_instance)
+/* EN7516/EN7527/EN7528/EN7580/EN7523 use a 4-bit queue selector
+ * with eight CPU reasons per register. EN7581/AN7583 use a 5-bit
+ * selector with four reasons per register.
+ */
+static bool airoha_fe_crsn_qsel_is_4bit(struct airoha_eth *eth)
{
- struct airoha_irq_bank *irq_bank = dev_instance;
- struct airoha_qdma *qdma = irq_bank->qdma;
- u32 rx_intr_mask = 0, rx_intr1, rx_intr2;
- u32 intr[ARRAY_SIZE(irq_bank->irqmask)];
+ return airoha_is(eth, airoha_en7523, econet_en7528,
+ econet_en7580);
+}
+
+static void airoha_fe_set_crsn_qsel(struct airoha_eth *eth, int cdm,
+ u8 crsn, u8 qsel)
+{
+ bool four_bit = airoha_fe_crsn_qsel_is_4bit(eth);
+ u32 reasons_per_reg = four_bit ? 8 : 4;
+ u32 width = four_bit ? 4 : 5;
+ u32 shift = (crsn % reasons_per_reg) * (four_bit ? 4 : 8);
+ u32 mask = GENMASK(shift + width - 1, shift);
+ u32 reg = crsn / reasons_per_reg;
+
+ airoha_fe_rmw(eth, REG_CDM_CRSN_QSEL(cdm, reg), mask,
+ (u32)qsel << shift);
+}
+
+/* set rx queue for lan->wifi traffic to Q1 */
+static void airoha_fe_crsn_qsel_init(struct airoha_eth *eth)
+{
+ static const struct {
+ u8 crsn;
+ u8 cdm1_qsel;
+ u8 cdm2_qsel;
+ } qsel[] = {
+ { CRSN_22, CDM_CRSN_QSEL_Q1, CDM_CRSN_QSEL_Q1 },
+ { CRSN_08, CDM_CRSN_QSEL_Q1, CDM_CRSN_QSEL_Q1 },
+ { CRSN_21, CDM_CRSN_QSEL_Q1, CDM_CRSN_QSEL_Q1 },
+ { CRSN_24, CDM_CRSN_QSEL_Q6, CDM_CRSN_QSEL_Q6 },
+ { CRSN_25, CDM_CRSN_QSEL_Q1, CDM_CRSN_QSEL_Q1 },
+ };
int i;
- for (i = 0; i < ARRAY_SIZE(intr); i++) {
- intr[i] = airoha_qdma_rr(qdma, REG_INT_STATUS(i));
- intr[i] &= irq_bank->irqmask[i];
- airoha_qdma_wr(qdma, REG_INT_STATUS(i), intr[i]);
+ for (i = 0; i < ARRAY_SIZE(qsel); i++) {
+ airoha_fe_set_crsn_qsel(eth, 1, qsel[i].crsn,
+ qsel[i].cdm1_qsel);
+ airoha_fe_set_crsn_qsel(eth, 2, qsel[i].crsn,
+ qsel[i].cdm2_qsel);
}
+}
- if (!test_bit(DEV_STATE_INITIALIZED, &qdma->eth->state))
- return IRQ_NONE;
+static void airoha_fe_lro_rxq_enable(struct airoha_eth *eth, int qdma_id,
+ int lro_queue_index, int qid,
+ int buf_size)
+{
+ u32 agg_count = airoha_is(eth, airoha_en7523) ?
+ EN7523_AIROHA_RXQ_LRO_MAX_AGG_COUNT :
+ AIROHA_RXQ_LRO_MAX_AGG_COUNT;
+ int id = qdma_id + 1;
- rx_intr1 = intr[1] & RX_DONE_LOW_INT_MASK;
- if (rx_intr1) {
- airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX1, rx_intr1);
- rx_intr_mask |= rx_intr1;
- }
+ airoha_fe_rmw(eth, REG_CDM_LRO_LIMIT(id),
+ CDM_LRO_AGG_NUM_MASK | CDM_LRO_AGG_SIZE_MASK,
+ FIELD_PREP(CDM_LRO_AGG_SIZE_MASK, buf_size) |
+ FIELD_PREP(CDM_LRO_AGG_NUM_MASK, agg_count));
+ airoha_fe_rmw(eth, REG_CDM_LRO_AGE_TIME(id),
+ CDM_LRO_AGE_TIME_MASK | CDM_LRO_AGG_TIME_MASK,
+ FIELD_PREP(CDM_LRO_AGE_TIME_MASK,
+ AIROHA_RXQ_LRO_MAX_AGE_TIME) |
+ FIELD_PREP(CDM_LRO_AGG_TIME_MASK,
+ AIROHA_RXQ_LRO_MAX_AGG_TIME));
- rx_intr2 = intr[2] & RX_DONE_HIGH_INT_MASK;
- if (rx_intr2) {
- airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX2, rx_intr2);
- rx_intr_mask |= (rx_intr2 << 16);
+ switch (eth->soc->version) {
+ case econet_en751221:
+ case econet_en7528:
+ case econet_en7580:
+ break;
+ case airoha_en7523: /* airoha_en7523 stores the map in LRO_EN
itself: enable (24-27) and qid (0-15) */
+ airoha_fe_rmw(eth, REG_CDM_LRO_EN(id),
+ EN7523_LRO_EN_MASK(lro_queue_index) |
+ EN7523_LRO_RXQ_MASK(lro_queue_index),
+ EN7523_LRO_EN_MASK(lro_queue_index) |
+ __field_prep(EN7523_LRO_RXQ_MASK(lro_queue_index),
+ qid));
+ break;
+ case airoha_en7581:
+ case airoha_an7583:
+ airoha_fe_rmw(eth, REG_CDM_LRO_RXQ(id, lro_queue_index),
+ LRO_RXQ_MASK(lro_queue_index),
+ __field_prep(LRO_RXQ_MASK(lro_queue_index), qid));
+ airoha_fe_set(eth, REG_CDM_LRO_EN(id), BIT(lro_queue_index));
+ break;
}
+}
- for (i = 0; rx_intr_mask && i < ARRAY_SIZE(qdma->q_rx); i++) {
- if (!qdma->q_rx[i].ndesc)
- continue;
+static void airoha_fe_lro_disable(struct airoha_eth *eth, int qdma_id)
+{
+ int i, id = qdma_id + 1;
- if (rx_intr_mask & BIT(i))
- napi_schedule(&qdma->q_rx[i].napi);
+ switch (eth->soc->version) {
+ case econet_en751221:
+ case econet_en7528:
+ case econet_en7580:
+ break;
+ case airoha_en7523:
+ airoha_fe_clear(eth, REG_CDM_LRO_EN(id),
+ EN7523_LRO_RXQ_EN_MASK | EN7523_LRO_ALL_RXQ_MASK);
+ break;
+ case airoha_en7581:
+ case airoha_an7583:
+ airoha_fe_clear(eth, REG_CDM_LRO_EN(id), LRO_RXQ_EN_MASK);
+ for (i = 0; i < AIROHA_MAX_NUM_LRO_QUEUES; i++)
+ airoha_fe_clear(eth, REG_CDM_LRO_RXQ(id, i), LRO_RXQ_MASK(i));
+ break;
}
- if (intr[0] & INT_TX_MASK) {
- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
- if (!(intr[0] & TX_DONE_INT_MASK(i)))
- continue;
+ airoha_fe_clear(eth, REG_CDM_LRO_LIMIT(id),
+ CDM_LRO_AGG_NUM_MASK | CDM_LRO_AGG_SIZE_MASK);
+ airoha_fe_clear(eth, REG_CDM_LRO_AGE_TIME(id),
+ CDM_LRO_AGE_TIME_MASK | CDM_LRO_AGG_TIME_MASK);
+}
- airoha_qdma_irq_disable(irq_bank, QDMA_INT_REG_IDX0,
- TX_DONE_INT_MASK(i));
- napi_schedule(&qdma->q_tx_irq[i].napi);
- }
- }
+static bool airoha_fe_lro_is_enabled(struct airoha_eth *eth, int qdma_id)
+{
- return IRQ_HANDLED;
+ switch (eth->soc->version) {
+ case airoha_en7523:
+ return airoha_fe_get(eth, REG_CDM_LRO_EN(qdma_id + 1),
+ EN7523_LRO_RXQ_EN_MASK);
+ case airoha_en7581:
+ case airoha_an7583:
+ return airoha_fe_get(eth, REG_CDM_LRO_EN(qdma_id + 1),
+ LRO_RXQ_EN_MASK);
+ default:
+ return false;
+ }
}
-static int airoha_qdma_init_irq_banks(struct platform_device *pdev,
- struct airoha_qdma *qdma)
+static void airoha_dev_lro_enable(struct airoha_gdm_dev *dev)
{
+ struct airoha_qdma *qdma = airoha_qdma_deref(dev);
struct airoha_eth *eth = qdma->eth;
- int i, id = qdma - ð->qdma[0];
+ int qdma_id = qdma - ð->qdma[0];
+ int i, lro_queue_index = 0;
- for (i = 0; i < ARRAY_SIZE(qdma->irq_banks); i++) {
- struct airoha_irq_bank *irq_bank = &qdma->irq_banks[i];
- int err, irq_index = 4 * id + i;
- const char *name;
+ for (i = 0; i < qdma->eth->soc->rx_ring; i++) {
+ struct airoha_queue *q = &qdma->q_rx[i];
+ u32 size;
- spin_lock_init(&irq_bank->irq_lock);
- irq_bank->qdma = qdma;
+ if (!q->ndesc)
+ continue;
- irq_bank->irq = platform_get_irq(pdev, irq_index);
- if (irq_bank->irq < 0)
- return irq_bank->irq;
+ if (!airoha_qdma_is_lro_queue(q))
+ continue;
- name = devm_kasprintf(eth->dev, GFP_KERNEL,
- KBUILD_MODNAME ".%d", irq_index);
- if (!name)
- return -ENOMEM;
+ size = SKB_WITH_OVERHEAD(q->buf_size);
+ if (airoha_is(eth, airoha_en7523))
+ size = min_t(u32, size,
+ EN7523_AIROHA_RXQ_LRO_MAX_AGG_SIZE);
+ size = min_t(u32, size, CDM_LRO_AGG_SIZE_MASK);
+ airoha_fe_lro_rxq_enable(eth, qdma_id, lro_queue_index, i,
+ size);
+ lro_queue_index++;
+ }
+}
- err = devm_request_irq(eth->dev, irq_bank->irq,
- airoha_irq_handler, IRQF_SHARED, name,
- irq_bank);
- if (err)
- return err;
+static void airoha_fe_pse_iq_init(struct airoha_eth *eth)
+{
+ u32 p2 = 0x10, p5 = 0x40;
+
+ /*
+ * Both EcoNet and newer Airoha SDKs use the same PSE IQ reservation
+ * registers. EN751221 keeps the older P2 reservation, while EN7528
+ * allocates a larger P5 reservation for its GPON multi-TCONT path.
+ */
+ if (airoha_is(eth, econet_en751221))
+ p2 = 0x20;
+ else if (airoha_is(eth, econet_en7528))
+ p5 = 0x80;
+
+ airoha_fe_rmw(eth, REG_PSE_IQ_REV1, PSE_IQ_RES1_P2_MASK,
+ FIELD_PREP(PSE_IQ_RES1_P2_MASK, p2));
+
+ if (airoha_is(eth, econet_en751221)) {
+ airoha_fe_rmw(eth, REG_PSE_IQ_REV2, PSE_IQ_RES2_P5_MASK,
+ FIELD_PREP(PSE_IQ_RES2_P5_MASK, p5));
+ return;
}
- return 0;
+ airoha_fe_rmw(eth, REG_PSE_IQ_REV2,
+ PSE_IQ_RES2_P5_MASK | PSE_IQ_RES2_P4_MASK,
+ FIELD_PREP(PSE_IQ_RES2_P5_MASK, p5) |
+ FIELD_PREP(PSE_IQ_RES2_P4_MASK, 0x34));
}
-static int airoha_qdma_init(struct platform_device *pdev,
- struct airoha_eth *eth,
- struct airoha_qdma *qdma)
+static int airoha_fe_init(struct airoha_eth *eth)
{
- int err, id = qdma - ð->qdma[0];
- const char *res;
+ airoha_fe_pse_iq_init(eth);
- qdma->eth = eth;
- res = devm_kasprintf(eth->dev, GFP_KERNEL, "qdma%d", id);
- if (!res)
- return -ENOMEM;
+ /* VIP classification is shared by the EcoNet and Airoha FEs.
+ * EN7516/EN7527/EN7528 additionally use the same 4-bit CRSN layout
+ * as EN7523. EN751221 keeps its older CRSN programming.
+ */
+ if (airoha_is_econet(eth)) {
+ airoha_fe_vip_setup(eth);
+ if (airoha_is(eth, econet_en7528, econet_en7580))
+ airoha_fe_crsn_qsel_init(eth);
+ return 0;
+ }
- qdma->regs = devm_platform_ioremap_resource_byname(pdev, res);
- if (IS_ERR(qdma->regs))
- return dev_err_probe(eth->dev, PTR_ERR(qdma->regs),
- "failed to iomap qdma%d regs\n", id);
+ airoha_fe_maccr_init(eth);
- err = airoha_qdma_init_irq_banks(pdev, qdma);
- if (err)
- return err;
+ /* enable FE copy engine for KA/DPI */
+ airoha_fe_wr(eth, REG_FE_PCE_CFG, PCE_DPI_EN_MASK | PCE_KA_EN_MASK);
+ /* set vip queue selection to ring 1 */
+ airoha_fe_rmw(eth, REG_CDM_FWD_CFG(1),
+ airoha_is(eth, airoha_en7523) ?
+ EN7523_CDM_VIP_QSEL_MASK : CDM_VIP_QSEL_MASK,
+ airoha_is(eth, airoha_en7523) ?
+ FIELD_PREP(EN7523_CDM_VIP_QSEL_MASK, 0x4) :
+ FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4));
+ airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2),
+ airoha_is(eth, airoha_en7523) ?
+ EN7523_CDM_VIP_QSEL_MASK : CDM_VIP_QSEL_MASK,
+ airoha_is(eth, airoha_en7523) ?
+ FIELD_PREP(EN7523_CDM_VIP_QSEL_MASK, 0x4) :
+ FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4));
+ /* set GDM4 source interface offset to 8 */
+ if (!airoha_is(eth, airoha_en7523))
+ airoha_fe_rmw(eth, REG_GDM_SRC_PORT_SET(4),
+ GDM_SPORT_OFF2_MASK |
+ GDM_SPORT_OFF1_MASK |
+ GDM_SPORT_OFF0_MASK,
+ FIELD_PREP(GDM_SPORT_OFF2_MASK, 8) |
+ FIELD_PREP(GDM_SPORT_OFF1_MASK, 8) |
+ FIELD_PREP(GDM_SPORT_OFF0_MASK, 8));
- err = airoha_qdma_init_rx(qdma);
- if (err)
- return err;
+ /* set PSE Page as 128B */
+ airoha_fe_rmw(eth, REG_FE_DMA_GLO_CFG,
+ FE_DMA_GLO_L2_SPACE_MASK | FE_DMA_GLO_PG_SZ_MASK,
+ FIELD_PREP(FE_DMA_GLO_L2_SPACE_MASK, airoha_is(eth,
airoha_en7523) ? 3 : 2) |
+ FE_DMA_GLO_PG_SZ_MASK);
- err = airoha_qdma_init_tx(qdma);
- if (err)
- return err;
+ /* map GDMP sram to fe */
+ airoha_wr(eth->gdmp_regs, 0x74, 3);
+ /* set PSE buffer to 0x500 = 0x400(pse itself) + 0x100(GDMP buffer) */
+ airoha_fe_wr(eth, eth->soc->pse_fq_cfg, 0x500);
- err = airoha_qdma_init_hfwd_queues(qdma);
- if (err)
- return err;
+ airoha_fe_wr(eth, REG_FE_RST_GLO_CFG,
+ FE_RST_CORE_MASK | FE_RST_GDM3_MBI_ARB_MASK |
+ FE_RST_GDM4_MBI_ARB_MASK);
+ usleep_range(1000, 2000);
- return airoha_qdma_hw_init(qdma);
-}
+ /* connect RxRing1 and RxRing15 to PSE Port0 OQ-1
+ * connect other rings to PSE Port0 OQ-0
+ */
+ airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP0, BIT(4));
+ airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP1, BIT(28));
+ airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP2, BIT(4));
+ airoha_fe_wr(eth, REG_FE_CDM1_OQ_MAP3, BIT(28));
-static void airoha_qdma_cleanup(struct airoha_qdma *qdma)
-{
- int i;
+ airoha_fe_vip_setup(eth);
+ airoha_fe_pse_ports_init(eth);
- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
- if (!qdma->q_rx[i].ndesc)
- continue;
+ /* The vendor SDK enables these GDM2 handshake modes only on the
+ * EN7581-class FE. EN7523 uses the legacy GPON release mode alone;
+ * enabling the EN7581 handshake there can prevent GDM2 from releasing
+ * received frames into the PSE.
+ */
+ if (!airoha_is(eth, airoha_en7523))
+ airoha_fe_set(eth, REG_GDM_MISC_CFG,
+ GDM2_RDM_ACK_WAIT_PREF_MASK |
+ GDM2_CHN_VLD_MODE_MASK);
+ /* EN7523 uses the legacy 4-bit OAM selector at bits 31:28.
+ * EN7581-class FE revisions use the wider high selector.
+ */
+ airoha_fe_rmw(eth, REG_CDM_FWD_CFG(2),
+ airoha_is(eth, airoha_en7523) ?
+ EN7523_CDM_OAM_QSEL_MASK : CDM_OAM_QSEL_MASK,
+ airoha_is(eth, airoha_en7523) ?
+ FIELD_PREP(EN7523_CDM_OAM_QSEL_MASK, 15) :
+ FIELD_PREP(CDM_OAM_QSEL_MASK, 15));
- netif_napi_del(&qdma->q_rx[i].napi);
- airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]);
- if (qdma->q_rx[i].page_pool) {
- page_pool_destroy(qdma->q_rx[i].page_pool);
- qdma->q_rx[i].page_pool = NULL;
- }
+ if (airoha_is(eth, airoha_en7523))
+ airoha_fe_rmw(eth, REG_QDMA_FC_WIFI_SP, WIFI_OFFLOAD_FC_EN_MASK,
+ FIELD_PREP(WIFI_OFFLOAD_FC_EN_MASK, 1));
+
+ /* init fragment and assemble Force Port */
+ if (airoha_is(eth, airoha_en7581, airoha_an7583)) {
+ /* NPU Core-3, NPU Bridge Channel-3 */
+ airoha_fe_rmw(eth, REG_IP_FRAG_FP,
+ IP_FRAGMENT_PORT_MASK | IP_FRAGMENT_NBQ_MASK,
+ FIELD_PREP(IP_FRAGMENT_PORT_MASK, 6) |
+ FIELD_PREP(IP_FRAGMENT_NBQ_MASK, 3));
+ /* QDMA LAN, RX Ring-22 */
+ airoha_fe_rmw(eth, REG_IP_FRAG_FP,
+ IP_ASSEMBLE_PORT_MASK | IP_ASSEMBLE_NBQ_MASK,
+ FIELD_PREP(IP_ASSEMBLE_PORT_MASK, 0) |
+ FIELD_PREP(IP_ASSEMBLE_NBQ_MASK, 22));
}
- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) {
- if (!qdma->q_tx_irq[i].size)
- continue;
+ airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM3_IDX),
+ GDM_PAD_EN_MASK | GDM_STRIP_CRC_MASK);
+ airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM4_IDX),
+ GDM_PAD_EN_MASK | GDM_STRIP_CRC_MASK);
- netif_napi_del(&qdma->q_tx_irq[i].napi);
- }
+ /*
+ * GDM2 is used as the EN7523 xPON datapath. Short Ethernet
+ * packets such as ARP replies and PPPoE discovery frames must
+ * be padded to ETH_ZLEN before being handed to the GPON MAC.
+ * Without this, the upstream transmits runt frames which are
+ * discarded by the OLT or the peer.
+ */
+ if (airoha_is(eth, airoha_en7523))
+ airoha_fe_set(eth, REG_GDM_FWD_CFG(AIROHA_GDM2_IDX),
+ GDM_PAD_EN_MASK);
- for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
- if (!qdma->q_tx[i].ndesc)
- continue;
+ /* Enable split for MIB counters for GDM3 and GDM4 */
+ airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM3_IDX),
+ FE_GDM_TX_MIB_SPLIT_EN_MASK |
+ FE_GDM_RX_MIB_SPLIT_EN_MASK);
+ airoha_fe_set(eth, REG_FE_GDM_MIB_CFG(AIROHA_GDM4_IDX),
+ FE_GDM_TX_MIB_SPLIT_EN_MASK |
+ FE_GDM_RX_MIB_SPLIT_EN_MASK);
- airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]);
+ airoha_fe_crsn_qsel_init(eth);
+
+ /* Match the EN7523 QDMA buffer layout used by the vendor firmware.
+ * QDMA0/LAN has 32 MiB for 16K 2 KiB payloads, while QDMA1/WAN has
+ * 16 MiB for 16K 1 KiB payloads. FAQ thresholds must follow the
+ * payload size selected for the corresponding QDMA.
+ */
+ if (airoha_is(eth, airoha_en7523)) {
+ airoha_fe_wr(eth, REG_CDM_FAQ_CFG(1), 0x07c7);
+ airoha_fe_wr(eth, REG_CDM_FAQ_THR_CFG(1), 0xc40007f0);
+ airoha_fe_wr(eth, REG_CDM_FAQ_CFG(2), 0x07e7);
+ airoha_fe_wr(eth, REG_CDM_FAQ_THR_CFG(2), 0xc40003f0);
}
+
+ airoha_fe_clear(eth, REG_FE_CPORT_CFG, FE_CPORT_QUEUE_XFC_MASK);
+ if (!airoha_is(eth, airoha_en7523))
+ airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PORT_XFC_MASK);
+ else {
+ airoha_fe_set(eth, REG_FE_CPORT_CFG, FE_CPORT_PORT_XFC_MASK |
FE_CPORT_DIS_FE2GSW_CRC);
+ airoha_fe_rmw(eth, REG_FE_CPORT_CFG, FE_CPORT_FE2SW_IPG,
+ FIELD_PREP(FE_CPORT_FE2SW_IPG, 2));
+ }
+
+ /* default aging mode for mbi unlock issue */
+ airoha_fe_rmw(eth, REG_GDM_CHN_RLS(2),
+ MBI_RX_AGE_SEL_MASK | MBI_TX_AGE_SEL_MASK,
+ FIELD_PREP(MBI_RX_AGE_SEL_MASK, 3) |
+ FIELD_PREP(MBI_TX_AGE_SEL_MASK, 3));
+
+ /* disable IFC by default */
+ airoha_fe_clear(eth, REG_FE_CSR_IFC_CFG, FE_IFC_EN_MASK);
+
+ /* enable sp_tag generation */
+ if (airoha_is(eth, airoha_en7523))
+ airoha_fe_set(eth, GDM1_BASE_STAG_EN, CPORT_TX_STAG_EN |
CPORT_RX_STAG_EN | GDM1_RX_LAN_SPORT);
+
+ /* enable 1:N vlan action, init vlan table */
+ airoha_fe_set(eth, REG_MC_VLAN_EN, MC_VLAN_EN_MASK);
+
+ /* enable Frame Engine interrupts */
+ if (airoha_is(eth, airoha_en7523))
+ airoha_fe_set(eth, REG_FE_INT_ENABLE,
+ GDM2_RX_INTR3_MASK | GDM2_RX_INTR2_MASK |
+ GDM2_RX_INTR1_MASK | GDM2_RX_INTR0_MASK |
+ GDM2_TX_INTR2_MASK | GDM2_TX_INTR1_MASK |
+ GDM2_TX_INTR0_MASK | PSE_FQ_EMPTY_MASK);
+
+ return airoha_fe_mc_vlan_clear(eth);
}
+
static int airoha_hw_init(struct platform_device *pdev,
struct airoha_eth *eth)
{
int err, i;
- /* disable xsi */
- err = reset_control_bulk_assert(eth->soc->num_xsi_rsts, eth->xsi_rsts);
- if (err)
- return err;
+ if (eth->soc->num_xsi_rsts) {
+ err = reset_control_bulk_assert(eth->soc->num_xsi_rsts,
+ eth->xsi_rsts);
+ if (err)
+ return err;
+ }
- err = reset_control_bulk_assert(ARRAY_SIZE(eth->rsts), eth->rsts);
+ err = reset_control_bulk_assert(AIROHA_MAX_NUM_RSTS, eth->rsts);
if (err)
return err;
- err = reset_control_bulk_deassert(eth->soc->num_xsi_rsts,
eth->xsi_rsts);
- if (err)
- return err;
+ if (eth->soc->num_xsi_rsts) {
+ err = reset_control_bulk_deassert(eth->soc->num_xsi_rsts,
+ eth->xsi_rsts);
+ if (err)
+ return err;
+ }
msleep(20);
- err = reset_control_bulk_deassert(ARRAY_SIZE(eth->rsts), eth->rsts);
+ err = reset_control_bulk_deassert(AIROHA_MAX_NUM_RSTS, eth->rsts);
if (err)
return err;
@@ -1818,7 +6063,7 @@ static int airoha_hw_init(struct platform_device
*pdev,
if (err)
return err;
- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) {
+ for (i = 0; i < AIROHA_MAX_NUM_QDMA; i++) {
err = airoha_qdma_init(pdev, eth, ð->qdma[i]);
if (err)
goto error;
@@ -1828,12 +6073,29 @@ static int airoha_hw_init(struct platform_device
*pdev,
if (err)
goto error;
- set_bit(DEV_STATE_INITIALIZED, ð->state);
+ /* GDM netdevs on the Airoha path are allocated before PPE so an
+ * nvmem -EPROBE_DEFER cannot leave DMA running. Bind the shared PPE
+ * frontend once hardware initialization has completed.
+ */
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
+ struct airoha_gdm_port *port = eth->ports ? eth->ports[i] : NULL;
+ int j;
+
+ if (!port)
+ continue;
+ for (j = 0; j < ARRAY_SIZE(port->devs); j++)
+ if (port->devs[j])
+ port->devs[j]->common.ppe = eth->ppe_dev;
+ }
+ set_bit(DEV_STATE_INITIALIZED, ð->state);
return 0;
+
error:
- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
- airoha_qdma_cleanup(ð->qdma[i]);
+ for (i = 0; i < AIROHA_MAX_NUM_QDMA; i++) {
+ if (eth->qdma[i].eth)
+ airoha_qdma_cleanup(ð->qdma[i]);
+ }
return err;
}
@@ -1842,40 +6104,12 @@ static void airoha_hw_cleanup(struct airoha_eth
*eth)
{
int i;
+ airoha_ppe_deinit(eth);
for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
airoha_qdma_cleanup(ð->qdma[i]);
- airoha_ppe_deinit(eth);
-}
-
-static void airoha_qdma_start_napi(struct airoha_qdma *qdma)
-{
- int i;
-
- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
- napi_enable(&qdma->q_tx_irq[i].napi);
-
- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
- if (!qdma->q_rx[i].ndesc)
- continue;
-
- napi_enable(&qdma->q_rx[i].napi);
- }
}
-static void airoha_qdma_stop_napi(struct airoha_qdma *qdma)
-{
- int i;
-
- for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++)
- napi_disable(&qdma->q_tx_irq[i].napi);
-
- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
- if (!qdma->q_rx[i].ndesc)
- continue;
- napi_disable(&qdma->q_rx[i].napi);
- }
-}
static void airoha_dev_get_hw_stats(struct airoha_gdm_dev *dev)
{
@@ -2042,12 +6276,25 @@ static void airoha_update_hw_stats(struct
airoha_gdm_dev *dev)
spin_unlock(&port->lock);
}
+static void airoha_gdm_update_hw_stats(struct airoha_gdm_dev *dev)
+{
+ if (dev->common.family == AIROHA_ETH_FAMILY_ECONET)
+ econet_update_hw_stats(dev);
+ else
+ airoha_update_hw_stats(dev);
+}
+
+static bool airoha_gdm_has_extended_stats(struct airoha_gdm_dev *dev)
+{
+ return dev->common.family == AIROHA_ETH_FAMILY_AIROHA || dev->g2_stats;
+}
+
static void airoha_update_netdev_features(struct airoha_gdm_dev *dev)
{
struct airoha_eth *eth = dev->eth;
int i;
- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
struct airoha_gdm_port *port = dev->port;
int j;
@@ -2074,38 +6321,67 @@ static void airoha_update_netdev_features(struct
airoha_gdm_dev *dev)
}
}
-static void airoha_qdma_start(struct airoha_qdma *qdma)
+
+
+static int airoha_xpon_control_start(struct net_device *netdev)
{
- airoha_qdma_set(qdma, REG_QDMA_GLOBAL_CFG,
- GLOBAL_CFG_TX_DMA_EN_MASK |
- GLOBAL_CFG_RX_DMA_EN_MASK);
- qdma->users++;
+ struct airoha_gdm_dev *dev;
+ struct airoha_qdma *qdma;
+ u32 pse_port = FE_PSE_PORT_PPE1;
+ int ret;
+
+ ret = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (ret)
+ return ret;
+
+ mutex_lock(&dev->xpon_lock);
+ if (dev->xpon_control_started)
+ goto out;
+
+ ret = airoha_set_vip_for_gdm_port(dev, true);
+ if (ret)
+ goto out;
+
+ /* GDM2 is a WAN datapath and does not carry a DSA source tag. */
+ airoha_fe_clear(dev->eth, REG_GDM_INGRESS_CFG(dev->port->id),
+ GDM_STAG_EN_MASK);
+
+ qdma = airoha_qdma_deref(dev);
+ airoha_qdma_start(qdma);
+ if (airoha_ppe_is_enabled(dev->eth, 1))
+ pse_port = FE_PSE_PORT_PPE2;
+ airoha_set_gdm_port_fwd_cfg(dev->eth,
+ REG_GDM_FWD_CFG(dev->port->id), pse_port);
+ WRITE_ONCE(dev->xpon_control_started, true);
+out:
+ mutex_unlock(&dev->xpon_lock);
+ return ret;
}
-static void airoha_qdma_stop(struct airoha_qdma *qdma)
+static void airoha_xpon_control_stop(struct net_device *netdev)
{
- u32 status;
+ struct airoha_gdm_dev *dev;
+ struct airoha_qdma *qdma;
- if (--qdma->users)
+ if (airoha_validate_xpon_gdm2(netdev, &dev))
return;
- airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG,
- GLOBAL_CFG_TX_DMA_EN_MASK |
- GLOBAL_CFG_RX_DMA_EN_MASK);
-
- if (read_poll_timeout(airoha_qdma_rr, status,
- !(status & (GLOBAL_CFG_TX_DMA_BUSY_MASK |
- GLOBAL_CFG_RX_DMA_BUSY_MASK)),
- USEC_PER_MSEC, 50 * USEC_PER_MSEC, true,
- qdma, REG_QDMA_GLOBAL_CFG))
- dev_warn(qdma->eth->dev, "QDMA DMA engine busy timeout\n");
-
- for (int i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) {
- if (!qdma->q_tx[i].ndesc)
- continue;
+ mutex_lock(&dev->xpon_lock);
+ if (!dev->xpon_control_started)
+ goto out;
- airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]);
+ WRITE_ONCE(dev->xpon_control_started, false);
+ qdma = airoha_qdma_deref(dev);
+ if (!netif_running(netdev)) {
+ airoha_set_vip_for_gdm_port(dev, false);
+ if (!dev->port->users)
+ airoha_set_gdm_port_fwd_cfg(dev->eth,
+ REG_GDM_FWD_CFG(dev->port->id),
+ FE_PSE_PORT_DROP);
}
+ airoha_qdma_stop(qdma);
+out:
+ mutex_unlock(&dev->xpon_lock);
}
static void airoha_dev_set_mtu(struct net_device *netdev)
@@ -2113,70 +6389,156 @@ static void airoha_dev_set_mtu(struct
net_device *netdev)
struct airoha_gdm_dev *dev = netdev_priv(netdev);
airoha_ppe_set_mtu(dev);
- if (!airoha_is_lan_gdm_dev(dev))
+ if (!airoha_is_lan_gdm_dev(dev)) {
+ /* WAN_MTU0 gates the egress *L2 frame* length for wan-port
+ * forwarding (HIT_BIND_EXCEED_MTU). netdev->mtu is the L3
+ * payload limit, so account for the Ethernet header and any
+ * VLAN/PPPoE tags + FCS, otherwise full-size hw-forwarded
+ * frames are bounced to the CPU and offload never engages.
+ */
+ u32 wan_mtu = netdev->mtu + ETH_HLEN + 2 * VLAN_HLEN +
+ 8 /* PPPoE */ + ETH_FCS_LEN;
+
airoha_fe_rmw(dev->eth, REG_WAN_MTU0,
WAN_MTU0_MASK,
- FIELD_PREP(WAN_MTU0_MASK, netdev->mtu + VLAN_ETH_HLEN));
+ FIELD_PREP(WAN_MTU0_MASK, wan_mtu));
+ }
}
+static int airoha_dev_stop(struct net_device *netdev);
+
static int airoha_dev_open(struct net_device *netdev)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
struct airoha_gdm_port *port = dev->port;
struct airoha_eth *eth = dev->eth;
+ bool econet = dev->common.family == AIROHA_ETH_FAMILY_ECONET;
struct airoha_qdma *qdma;
u32 pse_port = FE_PSE_PORT_PPE1;
- int err, qdma_id;
+ int err;
- /* HW LRO is configured on the QDMA and it is shared between
- * all the devices using it. Refuse to open a second device on
- * the same QDMA if LRO is enabled on any device sharing it.
- */
qdma = airoha_qdma_deref(dev);
- qdma_id = qdma - ð->qdma[0];
+ if (!econet) {
+ int qdma_id = qdma - ð->qdma[0];
- if (qdma->users && airoha_fe_lro_is_enabled(eth, qdma_id)) {
- netdev_warn(netdev, "required to disable HW GRO on QDMA%d\n",
- qdma_id);
- return -EBUSY;
+ /* HW LRO is configured on the QDMA and it is shared between all
+ * the devices using it. Opening a second device on the same QDMA
+ * turns HW GRO off rather than refusing to open.
+ */
+ if (qdma->users && airoha_fe_lro_is_enabled(eth, qdma_id)) {
+ netdev_info(netdev,
+ "disabling HW GRO on shared QDMA%d\n", qdma_id);
+ airoha_fe_lro_disable(eth, qdma_id);
+ }
}
- err = phylink_of_phy_connect(dev->phylink, netdev->dev.of_node, 0);
+ /* Keep fixed-link and real PHY configurations working, but allow an
+ * xPON-managed GDM2 to run without either. In the latter case the xPON
+ * provider owns carrier, speed and duplex.
+ */
+ err = airoha_gdm_phylink_connect(&dev->common,
+ dev->flags & AIROHA_PRIV_F_XPON_MANAGED);
if (err) {
netdev_err(netdev, "could not attach PHY: %d\n", err);
return err;
}
+ if (!dev->common.phylink_started &&
+ (dev->flags & AIROHA_PRIV_F_XPON_MANAGED))
+ netdev_dbg(netdev,
+ "no PHY or fixed-link, using xPON link state\n");
+
+ if (dev->flags & AIROHA_PRIV_F_XPON_MANAGED) {
+ unsigned long flags;
+ bool link;
+
+ /* Adopt a link state already published by the xPON provider. */
+ spin_lock_irqsave(&dev->xpon_state_lock, flags);
+ link = dev->xpon_link.valid && dev->xpon_link.link;
+ spin_unlock_irqrestore(&dev->xpon_state_lock, flags);
+
+ if (link)
+ netif_carrier_on(netdev);
+ else
+ netif_carrier_off(netdev);
+ }
- phylink_start(dev->phylink);
+ if (econet) {
+ bool dsa = netdev_uses_dsa(netdev);
+ struct gdm_len_th rlt;
+ u16 oversize_len;
- netif_tx_start_all_queues(netdev);
- err = airoha_set_vip_for_gdm_port(dev, true);
- if (err)
- return err;
+ /* Keep the MTK special tag on a DSA conduit and disable it on a
+ * direct PHY/WAN GDM. EN751221 additionally needs the legacy
+ * CDM/GDM special-tag bits used by the vendor datapath.
+ */
+ scoped_guard(spinlock, &dev->reg_lock) {
+ if (dev->fport == ETX_FPORT_GDM1) {
+ struct fwd_cfg fc;
+
+ fc = econet_rreg(&dev->econet_regs->fwd_cfg);
+ if (dsa) {
+ fc.word &= ~EN751221_GDM_UNTAG_EN;
+ fc.word |= EN751221_GDM_STAG_EN;
+ } else {
+ fc.word &= ~EN751221_GDM_STAG_EN;
+ }
+ econet_wreg(fc, &dev->econet_regs->fwd_cfg);
+
+ /* CDMA_CSG_CFG is at offset 0 from the GDM1 window. */
+ airoha_rmw(dev->regs, 0, EN751221_CDM_STAG_EN,
+ dsa ? EN751221_CDM_STAG_EN : 0);
+ }
- /* It seems GDM3 and GDM4 needs SPORT enabled to correctly work */
- if (netdev_uses_dsa(netdev) || port->id > 2)
- airoha_fe_set(eth, REG_GDM_INGRESS_CFG(port->id),
- GDM_STAG_EN_MASK);
- else
- airoha_fe_clear(eth, REG_GDM_INGRESS_CFG(port->id),
- GDM_STAG_EN_MASK);
+ econet_wreg((u32)dsa, &dev->econet_regs->stag_en);
+ rlt = econet_rreg(&dev->econet_regs->rx_len_threshold);
+ oversize_len = econet_gdm_oversize_len(dev, netdev->mtu);
+ set_gdm_len_th_runt_len(&rlt, 60);
+ set_gdm_len_th_oversize_len(&rlt, oversize_len);
+ econet_wreg(rlt, &dev->econet_regs->rx_len_threshold);
+ }
- airoha_dev_set_mtu(netdev);
- port->users++;
+ airoha_qdma_start(qdma);
+ netif_tx_start_all_queues(netdev);
+ } else {
+ netif_tx_start_all_queues(netdev);
+ err = airoha_set_vip_for_gdm_port(dev, true);
+ if (err) {
+ netif_tx_disable(netdev);
+ airoha_gdm_phylink_disconnect(&dev->common);
+ return err;
+ }
- airoha_qdma_start(qdma);
+ /* GDM3 and GDM4 need SPORT enabled to correctly work. */
+ if (netdev_uses_dsa(netdev) || port->id > 2)
+ airoha_fe_set(eth, REG_GDM_INGRESS_CFG(port->id),
+ GDM_STAG_EN_MASK);
+ else
+ airoha_fe_clear(eth, REG_GDM_INGRESS_CFG(port->id),
+ GDM_STAG_EN_MASK);
+
+ airoha_dev_set_mtu(netdev);
+ port->users++;
+ airoha_qdma_start(qdma);
+
+ if (!airoha_is_lan_gdm_dev(dev) &&
+ airoha_ppe_is_enabled(eth, 1))
+ pse_port = FE_PSE_PORT_PPE2;
+ airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(port->id),
+ pse_port);
- if (!airoha_is_lan_gdm_dev(dev) &&
- airoha_ppe_is_enabled(eth, 1))
- pse_port = FE_PSE_PORT_PPE2;
- airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(port->id),
- pse_port);
+ /* airoha_qdma_start() above already counted this device in. */
+ if ((netdev->features & NETIF_F_GRO_HW) && qdma->users <= 1)
+ airoha_dev_lro_enable(dev);
- if (netdev->features & NETIF_F_GRO_HW)
- airoha_dev_lro_enable(dev);
+ airoha_update_netdev_features(dev);
+ }
- airoha_update_netdev_features(dev);
+ err = airoha_gdm_xpon_start(dev);
+ if (err) {
+ netdev_err(netdev, "failed to start xPON provider: %d\n", err);
+ airoha_dev_stop(netdev);
+ return err;
+ }
return 0;
}
@@ -2185,31 +6547,54 @@ static int airoha_dev_stop(struct net_device
*netdev)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
struct airoha_gdm_port *port = dev->port;
+ bool econet = dev->common.family == AIROHA_ETH_FAMILY_ECONET;
struct airoha_qdma *qdma;
- int i;
+ airoha_gdm_xpon_stop(dev);
netif_tx_disable(netdev);
- airoha_set_vip_for_gdm_port(dev, false);
- for (i = 0; i < netdev->num_tx_queues; i++)
- netdev_tx_reset_subqueue(netdev, i);
-
qdma = airoha_qdma_deref(dev);
- if (--port->users)
- airoha_ppe_set_mtu(dev);
- else
- airoha_set_gdm_port_fwd_cfg(qdma->eth,
- REG_GDM_FWD_CFG(port->id),
- FE_PSE_PORT_DROP);
- airoha_qdma_stop(qdma);
- airoha_update_netdev_features(dev);
- phylink_stop(dev->phylink);
- phylink_disconnect_phy(dev->phylink);
+ if (econet) {
+ scoped_guard(spinlock, &dev->reg_lock) {
+ econet_wreg(0U, &dev->econet_regs->stag_en);
+ if (dev->fport == ETX_FPORT_GDM1) {
+ struct fwd_cfg fc;
+
+ fc = econet_rreg(&dev->econet_regs->fwd_cfg);
+ fc.word &= ~EN751221_GDM_STAG_EN;
+ econet_wreg(fc, &dev->econet_regs->fwd_cfg);
+ airoha_rmw(dev->regs, 0, EN751221_CDM_STAG_EN, 0);
+ }
+ }
+
+ airoha_qdma_stop(qdma);
+ } else {
+ bool xpon_control;
+ int i;
+
+ netif_carrier_off(netdev);
+ xpon_control = READ_ONCE(dev->xpon_control_started);
+ if (!xpon_control)
+ airoha_set_vip_for_gdm_port(dev, false);
+ for (i = 0; i < netdev->num_tx_queues; i++)
+ netdev_tx_reset_subqueue(netdev, i);
+
+ if (--port->users)
+ airoha_ppe_set_mtu(dev);
+ else if (!xpon_control)
+ airoha_set_gdm_port_fwd_cfg(qdma->eth,
+ REG_GDM_FWD_CFG(port->id),
+ FE_PSE_PORT_DROP);
+ airoha_qdma_stop(qdma);
+ airoha_update_netdev_features(dev);
+ }
+
+ airoha_gdm_phylink_disconnect(&dev->common);
return 0;
}
-static int airoha_dev_set_macaddr(struct net_device *netdev, void *p)
+static int airoha_gdm_dev_set_macaddr(struct net_device *netdev, void *p)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
struct sockaddr *addr = p;
@@ -2248,7 +6633,7 @@ static int airoha_enable_gdm2_loopback(struct
airoha_gdm_dev *dev)
airoha_fe_wr(eth, REG_GDM_TXCHN_EN(AIROHA_GDM2_IDX), 0xffffffff);
airoha_fe_wr(eth, REG_GDM_RXCHN_EN(AIROHA_GDM2_IDX), 0xffff);
- chan = port->id == AIROHA_GDM3_IDX ? airoha_is_7581(eth) ? 4 : 3 : 0;
+ chan = port->id == AIROHA_GDM3_IDX ? airoha_is(eth, airoha_en7581,
airoha_en7523) ? 4 : 3 : 0;
airoha_fe_rmw(eth, REG_GDM_LPBK_CFG(AIROHA_GDM2_IDX),
LPBK_CHAN_MASK | LPBK_MODE_MASK | LPBK_EN_MASK,
FIELD_PREP(LPBK_CHAN_MASK, chan) |
@@ -2276,7 +6661,7 @@ static int airoha_enable_gdm2_loopback(struct
airoha_gdm_dev *dev)
for (i = 0; i < eth->soc->num_ppe; i++)
airoha_ppe_set_cpu_port(dev, i, AIROHA_GDM2_IDX);
- if (port->id == AIROHA_GDM4_IDX && airoha_is_7581(eth)) {
+ if (port->id == AIROHA_GDM4_IDX && airoha_is(eth, airoha_en7581)) {
u32 mask = FC_ID_OF_SRC_PORT_MASK(dev->nbq);
airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, mask,
@@ -2323,7 +6708,7 @@ static int airoha_disable_gdm2_loopback(struct
airoha_gdm_dev *dev)
airoha_fe_set(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX));
airoha_fe_set(eth, REG_FE_IFC_PORT_EN, BIT(AIROHA_GDM2_IDX));
- if (port->id == AIROHA_GDM4_IDX && airoha_is_7581(eth)) {
+ if (port->id == AIROHA_GDM4_IDX && airoha_is(eth, airoha_en7581)) {
u32 mask = FC_ID_OF_SRC_PORT_MASK(dev->nbq);
airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, mask,
@@ -2338,7 +6723,7 @@ airoha_get_wan_gdm_dev(struct airoha_eth *eth)
{
int i;
- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
struct airoha_gdm_port *port = eth->ports[i];
int j;
@@ -2372,6 +6757,20 @@ static void airoha_dev_set_qdma(struct
airoha_gdm_dev *dev)
rcu_assign_pointer(dev->qdma, qdma);
netdev->irq = qdma->irq_banks[0].irq;
+ /* Set GSW P0 as WAN1 */
+ if (airoha_is(dev->qdma->eth, airoha_en7523)) {
+ airoha_fe_rmw(eth, REG_FE_WAN_PORT, WAN1_MASK,
+ FIELD_PREP(WAN1_MASK, 0x10));
+ airoha_fe_rmw(eth, REG_FE_WAN_PORT, WAN1_EN_MASK,
+ FIELD_PREP(WAN1_EN_MASK, 1));
+ airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(dev->port->id),
+ ((dev->port->id == AIROHA_GDM3_IDX ||
+ dev->port->id == AIROHA_GDM4_IDX ||
+ dev->port->id == AIROHA_GDM2_IDX) &&
+ airoha_ppe_is_enabled(eth, 1)) ?
+ FE_PSE_PORT_PPE2 : FE_PSE_PORT_PPE1);
+ }
+
ppe_id = !airoha_is_lan_gdm_dev(dev) && airoha_ppe_is_enabled(eth, 1);
airoha_ppe_set_cpu_port(dev, ppe_id, airoha_get_fe_port(dev));
@@ -2389,6 +6788,43 @@ static int airoha_dev_init(struct net_device *netdev)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
struct airoha_gdm_port *port = dev->port;
+ bool econet = dev->common.family == AIROHA_ETH_FAMILY_ECONET;
+ int err;
+
+ if (econet) {
+ enum etx_fport cpu_fport;
+
+ err = airoha_set_macaddr(dev, netdev->dev_addr);
+ if (err)
+ return err;
+
+ /* Route each GDM port to the CPU side of its matching QDMA. */
+ cpu_fport = dev->fport == ETX_FPORT_GDM2 ?
+ ETX_FPORT_QDMA1_CPU : ETX_FPORT_QDMA0_CPU;
+ econet_set_gdm_port_fwd_cfg(dev, cpu_fport);
+
+ if (dev->fport == ETX_FPORT_GDM1) {
+ struct g1_cport_cfg cport_cfg;
+ struct gdm_vlan vlan;
+
+ /* Match macSetMACCR() in the EN7512 vendor driver. CDMA1
+ * needs the 0x8100 insertion TPID and GDM1 CPORT padding
+ * before LAN traffic is started.
+ */
+ scoped_guard(spinlock, &dev->reg_lock) {
+ vlan = econet_rreg(&dev->econet_regs->vlan);
+ vlan.tpid = ETH_P_8021Q;
+ econet_wreg(vlan, &dev->econet_regs->vlan);
+
+ cport_cfg = econet_rreg(&dev->econet_regs->g1_cport_cfg);
+ set_gdm_g1_cport_cfg_pad(&cport_cfg, true);
+ econet_wreg(cport_cfg,
+ &dev->econet_regs->g1_cport_cfg);
+ }
+ }
+
+ return 0;
+ }
switch (port->id) {
case AIROHA_GDM3_IDX:
@@ -2397,7 +6833,7 @@ static int airoha_dev_init(struct net_device *netdev)
break;
fallthrough;
case AIROHA_GDM2_IDX:
- /* GDM2 is always used as wan */
+ /* GDM2 is always used as wan. */
dev->flags |= AIROHA_PRIV_F_WAN;
break;
default:
@@ -2405,50 +6841,58 @@ static int airoha_dev_init(struct net_device
*netdev)
}
airoha_dev_set_qdma(dev);
- airoha_set_macaddr(dev, netdev->dev_addr);
+ err = airoha_set_macaddr(dev, netdev->dev_addr);
+ if (err)
+ return err;
if (!airoha_is_lan_gdm_dev(dev) &&
- (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX)) {
- int err;
-
- err = airoha_enable_gdm2_loopback(dev);
- if (err)
- return err;
- }
+ (port->id == AIROHA_GDM3_IDX || port->id == AIROHA_GDM4_IDX))
+ return airoha_enable_gdm2_loopback(dev);
return 0;
}
-static void airoha_dev_get_stats64(struct net_device *netdev,
- struct rtnl_link_stats64 *storage)
+static void airoha_gdm_dev_get_stats64(struct net_device *netdev,
+ struct rtnl_link_stats64 *storage)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
unsigned int start;
- airoha_update_hw_stats(dev);
+ airoha_gdm_update_hw_stats(dev);
do {
start = u64_stats_fetch_begin(&dev->stats.syncp);
storage->rx_packets = dev->stats.rx_ok_pkts;
storage->tx_packets = dev->stats.tx_ok_pkts;
storage->rx_bytes = dev->stats.rx_ok_bytes;
storage->tx_bytes = dev->stats.tx_ok_bytes;
- storage->multicast = dev->stats.rx_multicast;
storage->rx_errors = dev->stats.rx_errors;
storage->rx_dropped = dev->stats.rx_drops;
storage->tx_dropped = dev->stats.tx_drops;
- storage->rx_crc_errors = dev->stats.rx_crc_error;
storage->rx_over_errors = dev->stats.rx_over_errors;
+ if (airoha_gdm_has_extended_stats(dev)) {
+ storage->multicast = dev->stats.rx_multicast;
+ storage->rx_crc_errors = dev->stats.rx_crc_error;
+ }
} while (u64_stats_fetch_retry(&dev->stats.syncp, start));
}
-static int airoha_dev_change_mtu(struct net_device *netdev, int mtu)
+static int airoha_gdm_dev_change_mtu(struct net_device *netdev, int mtu)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
- struct airoha_gdm_port *port = dev->port;
WRITE_ONCE(netdev->mtu, mtu);
- if (port->users)
+
+ if (dev->common.family == AIROHA_ETH_FAMILY_ECONET) {
+ struct gdm_len_th rlt;
+
+ guard(spinlock)(&dev->reg_lock);
+ rlt = econet_rreg(&dev->econet_regs->rx_len_threshold);
+ set_gdm_len_th_oversize_len(&rlt,
+ econet_gdm_oversize_len(dev, mtu));
+ econet_wreg(rlt, &dev->econet_regs->rx_len_threshold);
+ } else if (dev->port->users) {
airoha_dev_set_mtu(netdev);
+ }
return 0;
}
@@ -2461,60 +6905,23 @@ static u16 airoha_dev_select_queue(struct
net_device *netdev,
struct airoha_gdm_port *port = dev->port;
int queue, channel;
- /* For dsa device select QoS channel according to the dsa user port
- * index, rely on port id otherwise. Select QoS queue based on the
- * skb priority.
+ /* EcoNet previously had no ndo_select_queue. Preserve the generic
+ * networking-core queue selection now that all GDM netdevs share one
+ * net_device_ops table.
+ */
+ if (dev->common.family == AIROHA_ETH_FAMILY_ECONET)
+ return netdev_pick_tx(netdev, skb, sb_dev);
+
+ /* For DSA devices select the QoS channel according to the DSA user
port
+ * index, rely on port id otherwise. Select QoS queue based on skb
+ * priority.
*/
channel = netdev_uses_dsa(netdev) ? skb_get_queue_mapping(skb) :
port->id;
channel = channel % AIROHA_NUM_QOS_CHANNELS;
- queue = skb->priority % AIROHA_NUM_QOS_QUEUES;
+ queue = (skb->priority - 1) % AIROHA_NUM_QOS_QUEUES;
queue = channel * AIROHA_NUM_QOS_QUEUES + queue;
- return queue < netdev->num_tx_queues ? queue : 0;
-}
-
-static u32 airoha_get_dsa_tag(struct sk_buff *skb, struct net_device *dev)
-{
-#if IS_ENABLED(CONFIG_NET_DSA)
- struct ethhdr *ehdr;
- u8 xmit_tpid;
- u16 tag;
-
- if (!netdev_uses_dsa(dev))
- return 0;
-
- if (dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK)
- return 0;
-
- if (skb_cow_head(skb, 0))
- return 0;
-
- ehdr = (struct ethhdr *)skb->data;
- tag = be16_to_cpu(ehdr->h_proto);
- xmit_tpid = tag >> 8;
-
- switch (xmit_tpid) {
- case MTK_HDR_XMIT_TAGGED_TPID_8100:
- ehdr->h_proto = cpu_to_be16(ETH_P_8021Q);
- tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_8100 << 8);
- break;
- case MTK_HDR_XMIT_TAGGED_TPID_88A8:
- ehdr->h_proto = cpu_to_be16(ETH_P_8021AD);
- tag &= ~(MTK_HDR_XMIT_TAGGED_TPID_88A8 << 8);
- break;
- default:
- /* PPE module requires untagged DSA packets to work properly,
- * so move DSA tag to DMA descriptor.
- */
- memmove(skb->data + MTK_HDR_LEN, skb->data, 2 * ETH_ALEN);
- __skb_pull(skb, MTK_HDR_LEN);
- break;
- }
-
- return tag;
-#else
- return 0;
-#endif
+ return queue % dev->eth->soc->tx_ring;
}
int airoha_get_fe_port(struct airoha_gdm_dev *dev)
@@ -2526,11 +6933,17 @@ int airoha_get_fe_port(struct airoha_gdm_dev *dev)
}
static netdev_features_t airoha_dev_fix_features(struct net_device
*netdev,
- netdev_features_t features)
+ netdev_features_t features)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
struct airoha_qdma *qdma;
+ /* EcoNet had no ndo_fix_features callback. Keep its feature set
+ * unchanged; HW LRO/GRO programming below belongs to the modern FE.
+ */
+ if (dev->common.family == AIROHA_ETH_FAMILY_ECONET)
+ return features;
+
qdma = airoha_qdma_deref(dev);
if (qdma->users > 1)
features &= ~NETIF_F_GRO_HW;
@@ -2543,12 +6956,20 @@ static int airoha_dev_set_features(struct
net_device *netdev,
{
netdev_features_t diff = netdev->features ^ features;
struct airoha_gdm_dev *dev = netdev_priv(netdev);
- struct airoha_qdma *qdma = airoha_qdma_deref(dev);
- int qdma_id = qdma - &dev->eth->qdma[0];
+ struct airoha_qdma *qdma;
+ int qdma_id;
+
+ /* EcoNet had no ndo_set_features callback and does not use the modern
+ * FE LRO registers.
+ */
+ if (dev->common.family == AIROHA_ETH_FAMILY_ECONET)
+ return 0;
if (!(diff & NETIF_F_GRO_HW))
return 0;
+ qdma = airoha_qdma_deref(dev);
+ qdma_id = qdma - &dev->eth->qdma[0];
if (features & NETIF_F_GRO_HW)
airoha_dev_lro_enable(dev);
else
@@ -2557,38 +6978,303 @@ static int airoha_dev_set_features(struct
net_device *netdev,
return 0;
}
-static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
- struct net_device *netdev)
+static int
+airoha_eth_xpon_lookup_service(struct airoha_gdm_dev *dev,
+ bool vlan_valid, u16 vlan_id,
+ bool pcp_valid, u8 pcp,
+ struct airoha_xpon_tx_info *info)
+{
+ const struct airoha_xpon_service_cfg *fallback = NULL;
+ int i;
+
+ spin_lock_bh(&dev->xpon_service_lock);
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++) {
+ const struct airoha_xpon_service_cfg *service;
+
+ service = &dev->xpon_services[i];
+ if (!service->valid)
+ continue;
+ if (service->default_service)
+ fallback = service;
+ if (!service->vlan_valid && !service->pcp_valid)
+ continue;
+ if (service->vlan_valid &&
+ (!vlan_valid || service->vlan_id != vlan_id))
+ continue;
+ if (service->pcp_valid &&
+ (!pcp_valid || service->pcp != pcp))
+ continue;
+ fallback = service;
+ break;
+ }
+
+ if (fallback) {
+ info->gem_port_id = fallback->gem_port_id;
+ info->tcont = fallback->tcont;
+ info->queue = fallback->queue;
+ info->oam = false;
+ }
+ spin_unlock_bh(&dev->xpon_service_lock);
+
+ return fallback ? 0 : -ENOENT;
+}
+
+static int airoha_xpon_get_tx_info(struct net_device *netdev, bool
vlan_valid,
+ u16 vlan_id, bool pcp_valid, u8 pcp,
+ struct airoha_xpon_tx_info *info)
+{
+ struct airoha_gdm_dev *dev;
+ int ret;
+
+ if (!info)
+ return -EINVAL;
+
+ ret = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (ret)
+ return ret;
+ if (!(dev->flags & AIROHA_PRIV_F_XPON_MANAGED) ||
+ dev->xpon_mode != AIROHA_XPON_MODE_GPON)
+ return -EOPNOTSUPP;
+
+ return airoha_eth_xpon_lookup_service(dev, vlan_valid, vlan_id,
+ pcp_valid, pcp, info);
+}
+
+static int
+airoha_eth_xpon_classify(struct airoha_gdm_dev *dev, struct sk_buff *skb,
+ struct airoha_xpon_tx_info *info)
+{
+ u16 vlan_id = 0;
+ u8 pcp = 0;
+ bool vlan_valid;
+
+ vlan_valid = skb_vlan_tag_present(skb);
+ if (vlan_valid) {
+ u16 tci = skb_vlan_tag_get(skb);
+
+ vlan_id = tci & VLAN_VID_MASK;
+ pcp = (tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+ } else {
+ struct vlan_ethhdr vlan_hdr_buf;
+ const struct vlan_ethhdr *vlan_hdr;
+
+ /*
+ * A software 802.1Q device inserts the tag in the Ethernet header
+ * when the lower xPON netdev does not advertise VLAN TX offload.
+ * Classify both that representation and an accelerated skb tag.
+ */
+ vlan_hdr = skb_header_pointer(skb, 0, sizeof(vlan_hdr_buf),
+ &vlan_hdr_buf);
+ if (vlan_hdr && eth_type_vlan(vlan_hdr->h_vlan_proto)) {
+ u16 tci = ntohs(vlan_hdr->h_vlan_TCI);
+
+ vlan_id = tci & VLAN_VID_MASK;
+ pcp = (tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+ vlan_valid = true;
+ }
+ }
+
+ return airoha_eth_xpon_lookup_service(dev, vlan_valid, vlan_id,
+ vlan_valid, pcp, info);
+}
+
+static u8 airoha_skb_ip_version(const struct sk_buff *skb)
+{
+ int offset = skb_network_offset(skb);
+ u8 version, *data;
+
+ if (offset < 0)
+ return 0;
+
+ data = skb_header_pointer(skb, offset, sizeof(version), &version);
+
+ return data ? *data >> 4 : 0;
+}
+
+static u32 airoha_qdma_tx_csum_flags(const struct sk_buff *skb)
+{
+ u32 flags = 0;
+
+ if (skb->ip_summed != CHECKSUM_PARTIAL)
+ return 0;
+
+ if (airoha_skb_ip_version(skb) == 4)
+ flags |= QDMA_ETH_TXMSG_ICO_MASK;
+
+ switch (skb->csum_offset) {
+ case offsetof(struct tcphdr, check):
+ flags |= QDMA_ETH_TXMSG_TCO_MASK;
+ break;
+ case offsetof(struct udphdr, check):
+ flags |= QDMA_ETH_TXMSG_UCO_MASK;
+ break;
+ }
+
+ return flags;
+}
+
+static int airoha_xpon_add_service(struct net_device *netdev,
+ const struct airoha_xpon_service_cfg *cfg)
+{
+ struct airoha_gdm_dev *dev;
+ int empty = -1, i, ret;
+
+ if (!cfg || cfg->gem_port_id > FIELD_MAX(QDMA_ETH_TXMSG_SP_TAG_MASK) ||
+ cfg->tcont >= 32 || cfg->queue >= AIROHA_NUM_QOS_QUEUES)
+ return -EINVAL;
+
+ ret = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (ret)
+ return ret;
+
+ spin_lock_bh(&dev->xpon_service_lock);
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++) {
+ if (!dev->xpon_services[i].valid && empty < 0)
+ empty = i;
+ if (dev->xpon_services[i].valid &&
+ dev->xpon_services[i].cookie == cfg->cookie) {
+ empty = i;
+ break;
+ }
+ }
+ if (empty < 0) {
+ spin_unlock_bh(&dev->xpon_service_lock);
+ return -ENOSPC;
+ }
+ if (cfg->default_service)
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++)
+ dev->xpon_services[i].default_service = false;
+ dev->xpon_services[empty] = *cfg;
+ dev->xpon_services[empty].valid = true;
+ spin_unlock_bh(&dev->xpon_service_lock);
+
+ return 0;
+}
+
+static bool airoha_xpon_del_service(struct net_device *netdev, u32 cookie,
+ u16 *gem_port_id)
+{
+ struct airoha_gdm_dev *dev;
+ bool found = false;
+ int i;
+
+ if (airoha_validate_xpon_gdm2(netdev, &dev))
+ return false;
+
+ spin_lock_bh(&dev->xpon_service_lock);
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++) {
+ if (!dev->xpon_services[i].valid ||
+ dev->xpon_services[i].cookie != cookie)
+ continue;
+ if (gem_port_id)
+ *gem_port_id = dev->xpon_services[i].gem_port_id;
+ memset(&dev->xpon_services[i], 0,
+ sizeof(dev->xpon_services[i]));
+ found = true;
+ break;
+ }
+ spin_unlock_bh(&dev->xpon_service_lock);
+
+ return found;
+}
+
+static bool airoha_xpon_has_gem_service(struct net_device *netdev,
+ u16 gem_port_id)
+{
+ struct airoha_gdm_dev *dev;
+ bool found = false;
+ int i;
+
+ if (airoha_validate_xpon_gdm2(netdev, &dev))
+ return false;
+
+ spin_lock_bh(&dev->xpon_service_lock);
+ for (i = 0; i < AIROHA_XPON_MAX_SERVICES; i++)
+ if (dev->xpon_services[i].valid &&
+ dev->xpon_services[i].gem_port_id == gem_port_id) {
+ found = true;
+ break;
+ }
+ spin_unlock_bh(&dev->xpon_service_lock);
+
+ return found;
+}
+
+static void airoha_xpon_flush_services(struct net_device *netdev)
+{
+ struct airoha_gdm_dev *dev;
+
+ if (airoha_validate_xpon_gdm2(netdev, &dev))
+ return;
+
+ spin_lock_bh(&dev->xpon_service_lock);
+ memset(dev->xpon_services, 0, sizeof(dev->xpon_services));
+ spin_unlock_bh(&dev->xpon_service_lock);
+}
+
+static netdev_tx_t __airoha_dev_xmit(struct sk_buff *skb,
+ struct net_device *netdev,
+ const struct airoha_xpon_tx_info *xpon)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
- u32 nr_frags, tag, msg0, msg1, len;
+ struct airoha_xpon_tx_info classified;
+ struct airoha_qdma_skb_meta skb_meta;
+ bool xpon_oam = xpon && xpon->oam;
+ u32 nr_frags, tag = 0, msg0, msg1, len;
struct airoha_queue_entry *e;
struct airoha_qdma *qdma;
struct netdev_queue *txq;
struct airoha_queue *q;
LIST_HEAD(tx_list);
- int i = 0, qid;
- void *data;
+ int i = 0, qid, chn;
u16 index;
u8 fport;
rcu_read_lock();
qdma = rcu_dereference(dev->qdma);
- qid = airoha_qdma_get_txq(qdma, skb_get_queue_mapping(skb));
- tag = airoha_get_dsa_tag(skb, netdev);
-
- msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
- qid / AIROHA_NUM_QOS_QUEUES) |
- FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK,
- qid % AIROHA_NUM_QOS_QUEUES) |
- FIELD_PREP(QDMA_ETH_TXMSG_SP_TAG_MASK, tag);
- if (skb->ip_summed == CHECKSUM_PARTIAL)
- msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TCO_MASK, 1) |
- FIELD_PREP(QDMA_ETH_TXMSG_UCO_MASK, 1) |
- FIELD_PREP(QDMA_ETH_TXMSG_ICO_MASK, 1);
+ if (!xpon && (dev->flags & AIROHA_PRIV_F_XPON_MANAGED) &&
+ dev->xpon_mode == AIROHA_XPON_MODE_GPON) {
+ if (airoha_eth_xpon_classify(dev, skb, &classified))
+ goto error;
+ xpon = &classified;
+ }
+ qid = airoha_qdma_get_txq(qdma, xpon ? xpon->queue :
+ skb_get_queue_mapping(skb));
+ if (xpon) {
+ if (!airoha_is(qdma->eth, airoha_en7523))
+ goto error;
+
+ msg0 = FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK, xpon->queue) |
+ FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK, xpon->tcont) |
+ FIELD_PREP(QDMA_ETH_TXMSG_SP_TAG_MASK,
+ xpon->gem_port_id);
+ if (xpon->oam)
+ msg0 |= QDMA_ETH_TXMSG_OAM_MASK;
+ } else if (airoha_is(qdma->eth, airoha_en7523, econet_en7580)) {
+ airoha_qdma_skb_get_mtk_meta(skb, netdev,
+ AIROHA_MTK_TAG_TO_DESC,
+ &skb_meta);
+ tag = skb_meta.mtk_tag;
+ chn = skb_meta.channel;
+ msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK, chn) |
+ FIELD_PREP(QDMA_ETH_TXMSG_SP_TAG_MASK, tag | 0x8000);
+ } else {
+ airoha_qdma_skb_get_mtk_meta(skb, netdev,
+ AIROHA_MTK_TAG_TO_DESC,
+ &skb_meta);
+ tag = skb_meta.mtk_tag;
+ chn = skb_meta.channel;
+ msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK,
+ qid / AIROHA_NUM_QOS_QUEUES) |
+ FIELD_PREP(QDMA_ETH_TXMSG_QUEUE_MASK,
+ qid % AIROHA_NUM_QOS_QUEUES) |
+ FIELD_PREP(QDMA_ETH_TXMSG_SP_TAG_MASK, tag);
+ }
+ if (!xpon_oam)
+ msg0 |= airoha_qdma_tx_csum_flags(skb);
/* TSO: fill MSS info in tcp checksum field */
- if (skb_is_gso(skb)) {
+ if (!xpon_oam && skb_is_gso(skb)) {
if (skb_cow_head(skb, 0))
goto error;
@@ -2597,14 +7283,27 @@ static netdev_tx_t airoha_dev_xmit(struct
sk_buff *skb,
__be16 csum = cpu_to_be16(skb_shinfo(skb)->gso_size);
tcp_hdr(skb)->check = (__force __sum16)csum;
- msg0 |= FIELD_PREP(QDMA_ETH_TXMSG_TSO_MASK, 1);
+ msg0 |= QDMA_ETH_TXMSG_TCO_MASK |
+ QDMA_ETH_TXMSG_TSO_MASK;
+ if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
+ msg0 |= QDMA_ETH_TXMSG_ICO_MASK;
}
}
fport = airoha_get_fe_port(dev);
- msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK, dev->nbq) |
- FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) |
- FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);
+ if (airoha_is(qdma->eth, econet_en7580))
+ msg1 = FIELD_PREP(EN7580_QDMA_ETH_TXMSG_NBOQ_MASK,
+ dev->nbq) |
+ FIELD_PREP(EN7580_QDMA_ETH_TXMSG_FPORT_MASK, fport) |
+ FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);
+ else
+ msg1 = FIELD_PREP(QDMA_ETH_TXMSG_NBOQ_MASK,
+ xpon ? xpon->tcont : dev->nbq) |
+ FIELD_PREP(QDMA_ETH_TXMSG_FPORT_MASK, fport) |
+ FIELD_PREP(QDMA_ETH_TXMSG_METER_MASK, 0x7f);
+
+ if (xpon_oam)
+ msg1 |= QDMA_ETH_TXMSG_NO_DROP;
q = &qdma->q_tx[qid];
if (WARN_ON_ONCE(!q->ndesc))
@@ -2625,21 +7324,27 @@ static netdev_tx_t airoha_dev_xmit(struct
sk_buff *skb,
return NETDEV_TX_BUSY;
}
- len = skb_headlen(skb);
- data = skb->data;
-
e = list_first_entry(&q->tx_list, struct airoha_queue_entry,
list);
index = e - q->entry;
- while (true) {
+ while (i < nr_frags) {
struct airoha_qdma_desc *desc = &q->desc[index];
- skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+ bool map_page = i > 0;
dma_addr_t addr;
u32 val;
- addr = dma_map_single(netdev->dev.parent, data, len,
- DMA_TO_DEVICE);
+ if (map_page) {
+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
+
+ len = skb_frag_size(frag);
+ addr = skb_frag_dma_map(netdev->dev.parent, frag, 0,
+ len, DMA_TO_DEVICE);
+ } else {
+ len = skb_headlen(skb);
+ addr = dma_map_single(netdev->dev.parent, skb->data, len,
+ DMA_TO_DEVICE);
+ }
if (unlikely(dma_mapping_error(netdev->dev.parent, addr)))
goto error_unmap;
@@ -2647,12 +7352,15 @@ static netdev_tx_t airoha_dev_xmit(struct
sk_buff *skb,
e->skb = i == nr_frags - 1 ? skb : NULL;
e->dma_addr = addr;
e->dma_len = len;
+ e->dma_map_page = map_page;
e = list_first_entry(&q->tx_list, struct airoha_queue_entry,
list);
index = e - q->entry;
- val = FIELD_PREP(QDMA_DESC_LEN_MASK, len);
+ val = airoha_is(qdma->eth, airoha_en7523, econet_en7580) ?
+ FIELD_PREP(EN7523_QDMA_DESC_LEN_MASK, len) :
+ FIELD_PREP(QDMA_DESC_LEN_MASK, len);
if (i < nr_frags - 1)
val |= FIELD_PREP(QDMA_DESC_MORE_MASK, 1);
WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
@@ -2661,13 +7369,10 @@ static netdev_tx_t airoha_dev_xmit(struct
sk_buff *skb,
WRITE_ONCE(desc->data, cpu_to_le32(val));
WRITE_ONCE(desc->msg0, cpu_to_le32(msg0));
WRITE_ONCE(desc->msg1, cpu_to_le32(msg1));
- WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff));
-
- if (++i == nr_frags)
- break;
+ if (!airoha_is(qdma->eth, airoha_en7523, econet_en7580))
+ WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff));
- data = skb_frag_address(frag);
- len = skb_frag_size(frag);
+ i++;
}
q->queued += i;
@@ -2690,14 +7395,26 @@ static netdev_tx_t airoha_dev_xmit(struct
sk_buff *skb,
error_unmap:
list_for_each_entry(e, &tx_list, list) {
- dma_unmap_single(netdev->dev.parent, e->dma_addr, e->dma_len,
- DMA_TO_DEVICE);
- e->dma_addr = 0;
+ struct airoha_qdma_desc *desc = &q->desc[e - q->entry];
+
+ airoha_qdma_unmap_tx_entry(qdma->eth, e);
+ e->skb = NULL;
+ WRITE_ONCE(desc->ctrl, 0);
+ WRITE_ONCE(desc->addr, 0);
+ WRITE_ONCE(desc->data, 0);
+ WRITE_ONCE(desc->msg0, 0);
+ WRITE_ONCE(desc->msg1, 0);
+ WRITE_ONCE(desc->msg2, 0);
}
list_splice(&tx_list, &q->tx_list);
spin_unlock_bh(&q->lock);
error:
+ if (xpon_oam) {
+ rcu_read_unlock();
+ return NETDEV_TX_BUSY;
+ }
+
dev_kfree_skb_any(skb);
netdev->stats.tx_dropped++;
rcu_read_unlock();
@@ -2705,14 +7422,45 @@ static netdev_tx_t airoha_dev_xmit(struct
sk_buff *skb,
return NETDEV_TX_OK;
}
-static void airoha_ethtool_get_drvinfo(struct net_device *netdev,
- struct ethtool_drvinfo *info)
+static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
+ struct net_device *netdev)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
- struct airoha_eth *eth = dev->eth;
- strscpy(info->driver, eth->dev->driver->name, sizeof(info->driver));
- strscpy(info->bus_info, dev_name(eth->dev), sizeof(info->bus_info));
+ if (airoha_has_legacy_qdma(dev->eth))
+ return econet_qdma_xmit(skb, netdev);
+
+ return __airoha_dev_xmit(skb, netdev, NULL);
+}
+
+static int airoha_xpon_xmit_oam(struct net_device *netdev, struct
sk_buff *skb,
+ u8 channel, u16 gem_port_id)
+{
+ struct airoha_xpon_tx_info info = {
+ .tcont = channel,
+ .queue = 7,
+ .oam = true,
+ };
+ struct airoha_gdm_dev *dev;
+ netdev_tx_t ret;
+ int err;
+
+ err = airoha_validate_xpon_gdm2(netdev, &dev);
+ if (err)
+ return err;
+ if (!skb || gem_port_id > FIELD_MAX(QDMA_ETH_TXMSG_SP_TAG_MASK))
+ return -EINVAL;
+
+ info.gem_port_id = gem_port_id;
+
+ /* OMCI frames are generated internally and are not passed through
+ * dev_queue_xmit(), so associate them with the xPON netdev before
+ * the QDMA completion path performs BQL accounting.
+ */
+ skb->dev = netdev;
+ skb_set_queue_mapping(skb, 7);
+ ret = __airoha_dev_xmit(skb, netdev, &info);
+ return ret == NETDEV_TX_BUSY ? -EBUSY : 0;
}
static void airoha_ethtool_get_mac_stats(struct net_device *netdev,
@@ -2721,16 +7469,18 @@ static void airoha_ethtool_get_mac_stats(struct
net_device *netdev,
struct airoha_gdm_dev *dev = netdev_priv(netdev);
unsigned int start;
- airoha_update_hw_stats(dev);
+ airoha_gdm_update_hw_stats(dev);
do {
start = u64_stats_fetch_begin(&dev->stats.syncp);
stats->FramesTransmittedOK = dev->stats.tx_ok_pkts;
stats->OctetsTransmittedOK = dev->stats.tx_ok_bytes;
- stats->MulticastFramesXmittedOK = dev->stats.tx_multicast;
- stats->BroadcastFramesXmittedOK = dev->stats.tx_broadcast;
stats->FramesReceivedOK = dev->stats.rx_ok_pkts;
stats->OctetsReceivedOK = dev->stats.rx_ok_bytes;
- stats->BroadcastFramesReceivedOK = dev->stats.rx_broadcast;
+ if (airoha_gdm_has_extended_stats(dev)) {
+ stats->MulticastFramesXmittedOK = dev->stats.tx_multicast;
+ stats->BroadcastFramesXmittedOK = dev->stats.tx_broadcast;
+ stats->BroadcastFramesReceivedOK = dev->stats.rx_broadcast;
+ }
} while (u64_stats_fetch_retry(&dev->stats.syncp, start));
}
@@ -2760,7 +7510,10 @@ airoha_ethtool_get_rmon_stats(struct net_device
*netdev,
ARRAY_SIZE(hw_stats->rx_len) + 1);
*ranges = airoha_ethtool_rmon_ranges;
- airoha_update_hw_stats(dev);
+ airoha_gdm_update_hw_stats(dev);
+ if (!airoha_gdm_has_extended_stats(dev))
+ return;
+
do {
int i;
@@ -2780,17 +7533,61 @@ airoha_ethtool_get_link_ksettings(struct
net_device *netdev,
struct ethtool_link_ksettings *cmd)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
+ struct airoha_xpon_link_state state;
+ unsigned long flags;
+
+ if (!(READ_ONCE(dev->flags) & AIROHA_PRIV_F_XPON_MANAGED))
+ return phylink_ethtool_ksettings_get(dev->common.phylink, cmd);
+
+ spin_lock_irqsave(&dev->xpon_state_lock, flags);
+ state = dev->xpon_link;
+ spin_unlock_irqrestore(&dev->xpon_state_lock, flags);
+
+ /*
+ * A legacy fixed-link can still provide settings before the xPON
+ * provider publishes its first state. PHY-less ports are reported
+ * directly from the provider and do not depend on phylink.
+ */
+ if (!state.valid && dev->common.phylink_started)
+ return phylink_ethtool_ksettings_get(dev->common.phylink, cmd);
+
+ ethtool_link_ksettings_zero_link_mode(cmd, supported);
+ ethtool_link_ksettings_zero_link_mode(cmd, advertising);
+ ethtool_link_ksettings_zero_link_mode(cmd, lp_advertising);
+ linkmode_set_bit(ETHTOOL_LINK_MODE_FIBRE_BIT,
+ cmd->link_modes.supported);
+ linkmode_set_bit(ETHTOOL_LINK_MODE_FIBRE_BIT,
+ cmd->link_modes.advertising);
+
+ cmd->base.speed = state.valid ? state.speed : SPEED_UNKNOWN;
+ cmd->base.duplex = state.valid ? state.duplex : DUPLEX_UNKNOWN;
+ cmd->base.autoneg = AUTONEG_DISABLE;
+ cmd->base.port = state.valid ? state.port : PORT_FIBRE;
+ cmd->base.phy_address = 0xff;
+
+ return 0;
+}
+
+static int
+airoha_ethtool_set_link_ksettings(struct net_device *netdev,
+ const struct ethtool_link_ksettings *cmd)
+{
+ struct airoha_gdm_dev *dev = netdev_priv(netdev);
+
+ if (READ_ONCE(dev->flags) & AIROHA_PRIV_F_XPON_MANAGED)
+ return -EOPNOTSUPP;
- return phylink_ethtool_ksettings_get(dev->phylink, cmd);
+ return phylink_ethtool_ksettings_set(dev->common.phylink, cmd);
}
-static int
-airoha_ethtool_set_link_ksettings(struct net_device *netdev,
- const struct ethtool_link_ksettings *cmd)
+static int airoha_ethtool_nway_reset(struct net_device *netdev)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
- return phylink_ethtool_ksettings_set(dev->phylink, cmd);
+ if (READ_ONCE(dev->flags) & AIROHA_PRIV_F_XPON_MANAGED)
+ return -EOPNOTSUPP;
+
+ return phylink_ethtool_nway_reset(dev->common.phylink);
}
static int airoha_qdma_set_chan_tx_sched(struct net_device *netdev,
@@ -2912,7 +7709,8 @@ static int airoha_tc_setup_qdisc_ets(struct
net_device *dev,
if (opt->parent == TC_H_ROOT)
return -EINVAL;
- channel = TC_H_MIN(opt->parent) % AIROHA_NUM_QOS_CHANNELS;
+ channel = TC_H_MAJ(opt->handle) >> 16;
+ channel = channel % AIROHA_NUM_QOS_CHANNELS;
switch (opt->command) {
case TC_ETS_REPLACE:
@@ -3149,6 +7947,50 @@ static int
airoha_qdma_set_trtcm_token_bucket(struct airoha_qdma *qdma,
mode, val);
}
+static int airoha_qdma_set_gpon_dba_report(struct net_device *netdev,
+ int channel, bool enable)
+{
+ struct airoha_gdm_dev *dev = netdev_priv(netdev);
+ struct airoha_qdma *qdma = airoha_qdma_deref(dev);
+ u32 rate = enable ? EN7523_GPON_DBA_RATE_KBPS : 0;
+ int mode, err;
+
+ for (mode = TRTCM_COMMIT_MODE; mode <= TRTCM_PEAK_MODE; mode++) {
+ u32 bucket_size = mode == TRTCM_COMMIT_MODE ?
+ EN7523_GPON_DBA_CBS_BYTES :
+ EN7523_GPON_DBA_PBS_BYTES;
+
+ /* Match QDMA_API_SET_TX_DBA_REPORT(): byte mode, fast tick. */
+ err = airoha_qdma_set_trtcm_config(qdma, channel,
+ REG_EGRESS_TRTCM_CFG, mode,
+ false, TRTCM_PKT_MODE);
+ if (err)
+ return err;
+
+ err = airoha_qdma_set_trtcm_config(qdma, channel,
+ REG_EGRESS_TRTCM_CFG, mode,
+ false, TRTCM_TICK_SEL);
+ if (err)
+ return err;
+
+ err = airoha_qdma_set_trtcm_config(qdma, channel,
+ REG_EGRESS_TRTCM_CFG, mode,
+ enable, TRTCM_METER_MODE);
+ if (err)
+ return err;
+ if (!enable)
+ continue;
+
+ err = airoha_qdma_set_trtcm_token_bucket(qdma, channel,
+ REG_EGRESS_TRTCM_CFG,
+ mode, rate, bucket_size);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
static int airoha_qdma_set_tx_rate_limit(struct net_device *netdev,
int channel, u32 rate,
u32 bucket_size)
@@ -3198,10 +8040,10 @@ static int airoha_tc_htb_modify_queue(struct
net_device *dev,
static int airoha_tc_htb_alloc_leaf_queue(struct net_device *netdev,
struct tc_htb_qopt_offload *opt)
{
- u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS;
- int err, num_tx_queues = AIROHA_NUM_TX_RING + channel + 1;
struct airoha_gdm_dev *dev = netdev_priv(netdev);
struct airoha_qdma *qdma;
+ u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS;
+ int err, num_tx_queues = dev->eth->soc->tx_ring + channel + 1;
/* Here we need to check the requested QDMA channel is not already
* in use by another net_device running on the same QDMA block.
@@ -3229,7 +8071,7 @@ static int airoha_tc_htb_alloc_leaf_queue(struct
net_device *netdev,
}
set_bit(channel, dev->qos_sq_bmap);
- opt->qid = AIROHA_NUM_TX_RING + channel;
+ opt->qid = dev->eth->soc->tx_ring + channel;
return 0;
error:
@@ -3245,7 +8087,7 @@ static int airoha_qdma_set_rx_meter(struct
airoha_gdm_dev *dev,
struct airoha_qdma *qdma = airoha_qdma_deref(dev);
int i;
- for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+ for (i = 0; i < qdma->eth->soc->rx_ring; i++) {
int err;
if (!qdma->q_rx[i].ndesc)
@@ -3364,7 +8206,7 @@ static int airoha_dev_setup_tc_block_cb(enum
tc_setup_type type,
switch (type) {
case TC_SETUP_CLSFLOWER:
- return airoha_ppe_setup_tc_block_cb(ð->ppe->dev, type_data);
+ return airoha_ppe_setup_tc_block_cb(ð->ppe->common.dev,
type_data);
case TC_SETUP_CLSMATCHALL:
return airoha_dev_tc_matchall(netdev, type_data);
default:
@@ -3416,7 +8258,7 @@ static int airoha_dev_setup_tc_block(struct
net_device *dev,
static void airoha_tc_remove_htb_queue(struct net_device *netdev, int
queue)
{
struct airoha_gdm_dev *dev = netdev_priv(netdev);
- int num_tx_queues = AIROHA_NUM_TX_RING;
+ int num_tx_queues = dev->eth->soc->tx_ring;
struct airoha_qdma *qdma;
airoha_qdma_set_tx_rate_limit(netdev, queue, 0, 0);
@@ -3562,8 +8404,7 @@ static int airoha_tc_get_htb_get_leaf_queue(struct
net_device *netdev,
return -EINVAL;
}
- opt->qid = AIROHA_NUM_TX_RING + channel;
-
+ opt->qid = dev->eth->soc->tx_ring + channel;
return 0;
}
@@ -3601,17 +8442,23 @@ static int airoha_tc_setup_qdisc_htb(struct
net_device *netdev,
return 0;
}
-static int airoha_dev_tc_setup(struct net_device *dev,
- enum tc_setup_type type, void *type_data)
+static int airoha_gdm_dev_tc_setup(struct net_device *netdev,
+ enum tc_setup_type type, void *type_data)
{
+ struct airoha_gdm_dev *dev = netdev_priv(netdev);
+
+ if (dev->common.family == AIROHA_ETH_FAMILY_ECONET)
+ return airoha_ppe_dev_setup_tc(dev->common.ppe, netdev, type,
+ type_data);
+
switch (type) {
case TC_SETUP_QDISC_ETS:
- return airoha_tc_setup_qdisc_ets(dev, type_data);
+ return airoha_tc_setup_qdisc_ets(netdev, type_data);
case TC_SETUP_QDISC_HTB:
- return airoha_tc_setup_qdisc_htb(dev, type_data);
+ return airoha_tc_setup_qdisc_htb(netdev, type_data);
case TC_SETUP_BLOCK:
case TC_SETUP_FT:
- return airoha_dev_setup_tc_block(dev, type_data);
+ return airoha_dev_setup_tc_block(netdev, type_data);
default:
return -EOPNOTSUPP;
}
@@ -3621,22 +8468,23 @@ static const struct net_device_ops
airoha_netdev_ops = {
.ndo_init = airoha_dev_init,
.ndo_open = airoha_dev_open,
.ndo_stop = airoha_dev_stop,
- .ndo_change_mtu = airoha_dev_change_mtu,
+ .ndo_change_mtu = airoha_gdm_dev_change_mtu,
.ndo_select_queue = airoha_dev_select_queue,
.ndo_fix_features = airoha_dev_fix_features,
.ndo_set_features = airoha_dev_set_features,
.ndo_start_xmit = airoha_dev_xmit,
- .ndo_get_stats64 = airoha_dev_get_stats64,
- .ndo_set_mac_address = airoha_dev_set_macaddr,
- .ndo_setup_tc = airoha_dev_tc_setup,
+ .ndo_get_stats64 = airoha_gdm_dev_get_stats64,
+ .ndo_set_mac_address = airoha_gdm_dev_set_macaddr,
+ .ndo_setup_tc = airoha_gdm_dev_tc_setup,
};
static const struct ethtool_ops airoha_ethtool_ops = {
- .get_drvinfo = airoha_ethtool_get_drvinfo,
- .get_eth_mac_stats = airoha_ethtool_get_mac_stats,
+ .get_drvinfo = airoha_eth_get_drvinfo,
+ .get_eth_mac_stats = airoha_ethtool_get_mac_stats,
.get_rmon_stats = airoha_ethtool_get_rmon_stats,
.get_link_ksettings = airoha_ethtool_get_link_ksettings,
.set_link_ksettings = airoha_ethtool_set_link_ksettings,
+ .nway_reset = airoha_ethtool_nway_reset,
.get_link = ethtool_op_get_link,
};
@@ -3676,7 +8524,7 @@ bool airoha_is_valid_gdm_dev(struct airoha_eth *eth,
{
int i;
- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
struct airoha_gdm_port *port = eth->ports[i];
int j;
@@ -3692,18 +8540,17 @@ bool airoha_is_valid_gdm_dev(struct airoha_eth *eth,
return false;
}
-/* Nothing to do in MAC, everything is handled in PCS */
-static void airoha_mac_config(struct phylink_config *config, unsigned
int mode,
+/* Nothing to do in MAC, everything is handled in PCS. */
+static void airoha_mac_config(void *priv, unsigned int mode,
const struct phylink_link_state *state)
{
}
-static void airoha_mac_link_up(struct phylink_config *config, struct
phy_device *phy,
+static void airoha_mac_link_up(void *priv, struct phy_device *phy,
unsigned int mode, phy_interface_t interface,
int speed, int duplex, bool tx_pause, bool rx_pause)
{
- struct airoha_gdm_dev *dev = container_of(config, struct
airoha_gdm_dev,
- phylink_config);
+ struct airoha_gdm_dev *dev = priv;
struct airoha_gdm_port *port = dev->port;
struct airoha_eth *eth = dev->eth;
u32 frag_size_tx, frag_size_rx;
@@ -3735,7 +8582,7 @@ static void airoha_mac_link_up(struct
phylink_config *config, struct phy_device
mask = GDM4_SGMII1_TX_FRAG_SIZE_MASK;
val = FIELD_PREP(GDM4_SGMII1_TX_FRAG_SIZE_MASK,
frag_size_tx);
- } else {
+ } else {
mask = GDM4_SGMII0_TX_FRAG_SIZE_MASK;
val = FIELD_PREP(GDM4_SGMII0_TX_FRAG_SIZE_MASK,
frag_size_tx);
@@ -3756,13 +8603,13 @@ static void airoha_mac_link_up(struct
phylink_config *config, struct phy_device
spin_unlock(&port->lock);
}
-/* Nothing to do in MAC, everything is handled in PCS */
-static void airoha_mac_link_down(struct phylink_config *config,
unsigned int mode,
+/* Nothing to do in MAC, everything is handled in PCS. */
+static void airoha_mac_link_down(void *priv, unsigned int mode,
phy_interface_t interface)
{
}
-static const struct phylink_mac_ops airoha_phylink_ops = {
+static const struct airoha_gdm_mac_ops airoha_gdm_mac_ops = {
.mac_config = airoha_mac_config,
.mac_link_up = airoha_mac_link_up,
.mac_link_down = airoha_mac_link_down,
@@ -3784,8 +8631,8 @@ static int airoha_setup_phylink(struct net_device
*netdev)
struct device_node *np = netdev->dev.of_node;
struct airoha_gdm_port *port = dev->port;
struct phylink_config *config;
+ bool econet = airoha_is_econet(dev->eth);
phy_interface_t phy_mode;
- struct phylink *phylink;
int err;
err = of_get_phy_mode(np, &phy_mode);
@@ -3794,27 +8641,44 @@ static int airoha_setup_phylink(struct
net_device *netdev)
return err;
}
- config = &dev->phylink_config;
- config->dev = &netdev->dev;
- config->type = PHYLINK_NETDEV;
+ config = &dev->common.phylink_config;
/*
- * GDM1 only supports internal for Embedded Switch
- * and doesn't require a PCS.
+ * An internal GDM link does not have a PHY or PCS to attach. This is
+ * also how the xPON MAC is represented: the provider owns carrier and
+ * link state while phylink only describes the MAC-facing interface.
+ * EN7523-class GDM1 is internal-only, while EcoNet GDM1/GDM2 may also
+ * be wired to MII/GMII/RGMII interfaces.
*/
- if (port->id == AIROHA_GDM1_IDX) {
+ if (phy_mode == PHY_INTERFACE_MODE_INTERNAL ||
+ (!econet && port->id == AIROHA_GDM1_IDX)) {
config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
MAC_10000FD;
+ if (port->id == AIROHA_GDM2_IDX || econet)
+ config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
+ MAC_10 | MAC_100 | MAC_1000 |
+ MAC_2500FD | MAC_5000FD |
+ MAC_10000FD;
__set_bit(PHY_INTERFACE_MODE_INTERNAL,
config->supported_interfaces);
+ } else if (econet) {
+ config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
+ MAC_10 | MAC_100 | MAC_1000 |
+ MAC_2500FD | MAC_5000FD | MAC_10000FD;
+ __set_bit(PHY_INTERFACE_MODE_MII, config->supported_interfaces);
+ __set_bit(PHY_INTERFACE_MODE_GMII, config->supported_interfaces);
+ __set_bit(PHY_INTERFACE_MODE_RGMII, config->supported_interfaces);
+ __set_bit(PHY_INTERFACE_MODE_RGMII_ID,
config->supported_interfaces);
+ __set_bit(PHY_INTERFACE_MODE_RGMII_RXID,
config->supported_interfaces);
+ __set_bit(PHY_INTERFACE_MODE_RGMII_TXID,
config->supported_interfaces);
} else {
config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
MAC_10 | MAC_100 | MAC_1000 |
MAC_2500FD | MAC_5000FD | MAC_10000FD;
err = fwnode_phylink_pcs_parse(dev_fwnode(config->dev), NULL,
- &dev->phylink_config.num_available_pcs);
+ &dev->common.phylink_config.num_available_pcs);
if (err)
return err;
@@ -3824,25 +8688,27 @@ static int airoha_setup_phylink(struct
net_device *netdev)
config->supported_interfaces);
__set_bit(PHY_INTERFACE_MODE_1000BASEX,
config->supported_interfaces);
- __set_bit(PHY_INTERFACE_MODE_2500BASEX,
- config->supported_interfaces);
- __set_bit(PHY_INTERFACE_MODE_10GBASER,
- config->supported_interfaces);
- __set_bit(PHY_INTERFACE_MODE_USXGMII,
- config->supported_interfaces);
+ /*
+ * Only advertise the higher-rate interfaces when the port is
actually
+ * configured for them. An RTL8221B attached in SGMII mode (1G
copper)
+ * otherwise sees 2500BASEX in host_interfaces and forces its
serdes to
+ * 2500Base-X rate-match, which never links against an SGMII PCS.
+ */
+ if (phy_mode != PHY_INTERFACE_MODE_SGMII &&
+ phy_mode != PHY_INTERFACE_MODE_1000BASEX) {
+ __set_bit(PHY_INTERFACE_MODE_2500BASEX,
+ dev->common.phylink_config.supported_interfaces);
+ __set_bit(PHY_INTERFACE_MODE_10GBASER,
+ dev->common.phylink_config.supported_interfaces);
+ __set_bit(PHY_INTERFACE_MODE_USXGMII,
+ dev->common.phylink_config.supported_interfaces);
+ }
phy_interface_copy(config->pcs_interfaces,
config->supported_interfaces);
}
- phylink = phylink_create(config, of_fwnode_handle(np),
- phy_mode, &airoha_phylink_ops);
- if (IS_ERR(phylink))
- return PTR_ERR(phylink);
-
- dev->phylink = phylink;
-
- return 0;
+ return airoha_gdm_phylink_create(&dev->common, np, phy_mode);
}
static int airoha_alloc_gdm_device(struct airoha_eth *eth,
@@ -3851,12 +8717,27 @@ static int airoha_alloc_gdm_device(struct
airoha_eth *eth,
{
struct net_device *netdev;
struct airoha_gdm_dev *dev;
+ bool econet = airoha_is_econet(eth);
+ bool legacy_qdma = airoha_has_legacy_qdma(eth);
+ unsigned int txqs, rxqs;
u8 index;
int err;
- netdev = devm_alloc_etherdev_mqs(eth->dev, sizeof(*dev),
- AIROHA_NUM_NETDEV_TX_RINGS,
- AIROHA_NUM_RX_RING);
+ if (legacy_qdma) {
+ struct airoha_qdma *qdma;
+
+ if (port->id > AIROHA_MAX_NUM_QDMA)
+ return -EINVAL;
+
+ qdma = ð->qdma[port->id - 1];
+ txqs = qdma->num_channels * ECONET_NUM_QUEUES;
+ rxqs = txqs;
+ } else {
+ txqs = AIROHA_NUM_NETDEV_TX_RINGS(eth->soc);
+ rxqs = eth->soc->rx_ring;
+ }
+
+ netdev = devm_alloc_etherdev_mqs(eth->dev, sizeof(*dev), txqs, rxqs);
if (!netdev) {
dev_err(eth->dev, "alloc_etherdev failed\n");
return -ENOMEM;
@@ -3864,33 +8745,39 @@ static int airoha_alloc_gdm_device(struct
airoha_eth *eth,
netdev->netdev_ops = &airoha_netdev_ops;
netdev->ethtool_ops = &airoha_ethtool_ops;
- netdev->max_mtu = AIROHA_MAX_MTU;
+ if (econet) {
+ netdev->max_mtu = SKB_WITH_OVERHEAD(AIROHA_MAX_PACKET_SIZE) -
+ ETH_HLEN - ETH_FCS_LEN;
+ netdev->hw_features = AIROHA_ETH_HW_FEATURES;
+ if (eth->ppe_dev && eth->ppe_dev->enabled)
+ netdev->hw_features |= NETIF_F_HW_TC;
+ netdev->features |= netdev->hw_features;
+ netdev->vlan_features = netdev->hw_features;
+ } else {
+ netdev->max_mtu = AIROHA_MAX_MTU;
+ netdev->hw_features = AIROHA_HW_FEATURES | NETIF_F_GRO_HW;
+ netdev->features |= AIROHA_HW_FEATURES;
+ netdev->vlan_features = AIROHA_HW_FEATURES;
+ }
+
netdev->watchdog_timeo = 5 * HZ;
- netdev->hw_features = AIROHA_HW_FEATURES | NETIF_F_LRO;
- netdev->features |= AIROHA_HW_FEATURES;
- netdev->vlan_features = AIROHA_HW_FEATURES;
SET_NETDEV_DEV(netdev, eth->dev);
- /* reserve hw queues for HTB offloading */
- err = netif_set_real_num_tx_queues(netdev, AIROHA_NUM_TX_RING);
- if (err)
- return err;
-
- err = of_get_ethdev_address(np, netdev);
- if (err) {
- if (err == -EPROBE_DEFER)
+ if (!legacy_qdma) {
+ /* reserve hw queues for HTB offloading */
+ err = netif_set_real_num_tx_queues(netdev, eth->soc->tx_ring);
+ if (err)
return err;
-
- eth_hw_addr_random(netdev);
- dev_info(eth->dev, "generated random MAC address %pM\n",
- netdev->dev_addr);
}
- /* Allowed nbq for EN7581 on GDM3 port are 4 and 5 for PCIE0
- * and PCIE1 respectively.
- */
+ err = airoha_eth_init_mac_address(eth->dev, np, netdev);
+ if (err)
+ return err;
+
+ /* Allowed nbq for EN7581/EN7523 on GDM3 are 4 and 5. */
index = nbq;
- if (index && airoha_is_7581(eth) && port->id == AIROHA_GDM3_IDX)
+ if (index && airoha_is(eth, airoha_en7581, airoha_en7523) &&
+ port->id == AIROHA_GDM3_IDX)
index -= 4;
if (index >= ARRAY_SIZE(port->devs) || port->devs[index]) {
@@ -3900,10 +8787,42 @@ static int airoha_alloc_gdm_device(struct
airoha_eth *eth,
netdev->dev.of_node = of_node_get(np);
dev = netdev_priv(netdev);
+ airoha_gdm_common_init(&dev->common, eth, netdev,
+ econet ? AIROHA_ETH_FAMILY_ECONET :
+ AIROHA_ETH_FAMILY_AIROHA,
+ port->id,
+ econet ? (port->id == AIROHA_GDM2_IDX ?
+ DPORT_GDMA2 : DPORT_GDMA1) : 0,
+ dev, econet ? NULL : &airoha_gdm_mac_ops);
+ dev->common.ppe = eth->ppe_dev;
u64_stats_init(&dev->stats.syncp);
dev->port = port;
dev->eth = eth;
dev->nbq = nbq;
+ mutex_init(&dev->xpon_lock);
+ spin_lock_init(&dev->xpon_state_lock);
+ spin_lock_init(&dev->xpon_service_lock);
+
+ if (econet) {
+ dev->regs = eth->fe_regs + CDM_BASE(port->id);
+ spin_lock_init(&dev->reg_lock);
+ dev->g2_stats = port->id == AIROHA_GDM2_IDX;
+ dev->fport = port->id == AIROHA_GDM2_IDX ?
+ ETX_FPORT_GDM2 : ETX_FPORT_GDM1;
+ rcu_assign_pointer(dev->qdma, ð->qdma[port->id - 1]);
+ if (port->id == AIROHA_GDM2_IDX)
+ dev->flags |= AIROHA_PRIV_F_WAN;
+ }
+
+ if (of_property_read_bool(np, "airoha,xpon-managed")) {
+ dev->flags |= AIROHA_PRIV_F_XPON_MANAGED;
+ netdev->features |= NETIF_F_GRO_HW;
+
+ /* Keep the validated xPON RX GRO path but do TX segmentation
in SW. */
+ netdev->hw_features &= ~AIROHA_XPON_TX_OFFLOAD_FEATURES;
+ netdev->features &= ~AIROHA_XPON_TX_OFFLOAD_FEATURES;
+ netdev->vlan_features &= ~AIROHA_XPON_TX_OFFLOAD_FEATURES;
+ }
port->devs[index] = dev;
return airoha_setup_phylink(netdev);
@@ -3912,25 +8831,17 @@ static int airoha_alloc_gdm_device(struct
airoha_eth *eth,
static int airoha_alloc_gdm_port(struct airoha_eth *eth,
struct device_node *np)
{
- const __be32 *id_ptr = of_get_property(np, "reg", NULL);
struct airoha_gdm_port *port;
struct device_node *node;
+ bool econet = airoha_is_econet(eth);
int err, nbq, p, d = 0;
- u32 id;
+ u32 id, max_ports = eth->soc->max_gdm_ports;
- if (!id_ptr) {
- dev_err(eth->dev, "missing gdm port id\n");
- return -EINVAL;
- }
+ err = airoha_eth_get_port_id(eth->dev, np, 1, max_ports, &id);
+ if (err)
+ return err;
- id = be32_to_cpup(id_ptr);
p = id - 1;
-
- if (!id || id > ARRAY_SIZE(eth->ports)) {
- dev_err(eth->dev, "invalid gdm port id: %d\n", id);
- return -EINVAL;
- }
-
if (eth->ports[p]) {
dev_err(eth->dev, "duplicate gdm port id: %d\n", id);
return -EINVAL;
@@ -3944,17 +8855,26 @@ static int airoha_alloc_gdm_port(struct
airoha_eth *eth,
spin_lock_init(&port->lock);
eth->ports[p] = port;
- err = airoha_metadata_dst_alloc(port);
- if (err)
- return err;
+ /* Metadata mux destinations belong to the modern descriptor path.
+ * EN751221/EN7528 keep the MT7530 special tag in-band, while EN7580
+ * uses the EcoNet GDM with descriptor-carried switch metadata.
+ */
+ if (!airoha_has_legacy_qdma(eth)) {
+ err = airoha_metadata_dst_alloc(port);
+ if (err)
+ return err;
+ }
+
+ /* Default nbq value to ensure backward compatibility. EcoNet has one
+ * logical device per GDM and therefore always uses nbq 0.
+ */
+ nbq = !econet && id == AIROHA_GDM3_IDX &&
+ airoha_is(eth, airoha_en7581, airoha_en7523) ? 4 : 0;
- /* Default nbq value to ensure backward compatibility */
- nbq = id == AIROHA_GDM3_IDX && airoha_is_7581(eth) ? 4 : 0;
+ if (econet)
+ return airoha_alloc_gdm_device(eth, port, 0, np);
for_each_child_of_node(np, node) {
- /* Multiple external serdes connected to the FE GDM port via an
- * external arbiter.
- */
const __be32 *nbq_ptr;
if (!of_device_is_compatible(node, "airoha,eth-port"))
@@ -3971,7 +8891,6 @@ static int airoha_alloc_gdm_port(struct airoha_eth
*eth,
return -EINVAL;
}
- /* Verify the provided nbq parameter is valid */
nbq = be32_to_cpup(nbq_ptr);
err = eth->soc->ops.get_sport(port, nbq);
if (err < 0) {
@@ -3993,7 +8912,7 @@ static int airoha_register_gdm_devices(struct
airoha_eth *eth)
{
int i;
- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
struct airoha_gdm_port *port = eth->ports[i];
int j;
@@ -4018,167 +8937,303 @@ static int airoha_register_gdm_devices(struct
airoha_eth *eth)
return 0;
}
-static int airoha_probe(struct platform_device *pdev)
+static struct airoha_npu *airoha_eth_ppe_npu_get(struct airoha_eth *eth)
{
- struct reset_control_bulk_data *xsi_rsts;
- struct device_node *np;
- struct airoha_eth *eth;
- int i, err;
+ struct airoha_npu *npu = airoha_npu_get(eth->dev);
- eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL);
- if (!eth)
- return -ENOMEM;
+ return npu ? npu : ERR_PTR(-EOPNOTSUPP);
+}
- eth->soc = of_device_get_match_data(&pdev->dev);
- if (!eth->soc)
- return -EINVAL;
+static void airoha_eth_ppe_npu_put(struct airoha_npu *npu)
+{
+ airoha_npu_put(npu);
+}
- eth->dev = &pdev->dev;
+static int airoha_eth_ppe_npu_init(struct airoha_npu *npu)
+{
+#if IS_ENABLED(CONFIG_NET_AIROHA_NPU)
+ return npu->ops.ppe_init(npu);
+#else
+ return -EOPNOTSUPP;
+#endif
+}
- err = dma_set_mask_and_coherent(eth->dev, DMA_BIT_MASK(32));
- if (err) {
- dev_err(eth->dev, "failed configuring DMA mask\n");
- return err;
- }
+static int airoha_eth_ppe_npu_deinit(struct airoha_npu *npu)
+{
+#if IS_ENABLED(CONFIG_NET_AIROHA_NPU)
+ return npu->ops.ppe_deinit(npu);
+#else
+ return -EOPNOTSUPP;
+#endif
+}
- eth->fe_regs = devm_platform_ioremap_resource_byname(pdev, "fe");
- if (IS_ERR(eth->fe_regs))
- return dev_err_probe(eth->dev, PTR_ERR(eth->fe_regs),
- "failed to iomap fe regs\n");
+static int airoha_eth_ppe_npu_init_stats(struct airoha_npu *npu,
+ dma_addr_t addr, u32 entries)
+{
+#if IS_ENABLED(CONFIG_NET_AIROHA_NPU)
+ return npu->ops.ppe_init_stats(npu, addr, entries);
+#else
+ return -EOPNOTSUPP;
+#endif
+}
- eth->rsts[0].id = "fe";
- eth->rsts[1].id = "pdma";
- eth->rsts[2].id = "qdma";
- err = devm_reset_control_bulk_get_exclusive(eth->dev,
- ARRAY_SIZE(eth->rsts),
- eth->rsts);
- if (err) {
- dev_err(eth->dev, "failed to get bulk reset lines\n");
- return err;
+static void airoha_eth_ppe_npu_stats_read(struct airoha_npu *npu,
+ u32 index,
+ struct airoha_foe_stats *stats)
+{
+#if IS_ENABLED(CONFIG_NET_AIROHA_NPU)
+ memcpy_fromio(stats, &npu->stats[index], sizeof(*stats));
+#endif
+}
+
+static void airoha_eth_ppe_npu_stats_clear(struct airoha_npu *npu, u32
index)
+{
+#if IS_ENABLED(CONFIG_NET_AIROHA_NPU)
+ memset_io(&npu->stats[index], 0, sizeof(*npu->stats));
+#endif
+}
+
+static const struct airoha_ppe_host_ops airoha_ppe_host_ops = {
+ .get_fe_port = airoha_get_fe_port,
+ .is_valid_gdm_dev = airoha_is_valid_gdm_dev,
+ .xpon_get_tx_info = airoha_xpon_get_tx_info,
+ .npu_get = airoha_eth_ppe_npu_get,
+ .npu_put = airoha_eth_ppe_npu_put,
+ .npu_ppe_init = airoha_eth_ppe_npu_init,
+ .npu_ppe_deinit = airoha_eth_ppe_npu_deinit,
+ .npu_ppe_init_stats = airoha_eth_ppe_npu_init_stats,
+ .npu_stats_read = airoha_eth_ppe_npu_stats_read,
+ .npu_stats_clear = airoha_eth_ppe_npu_stats_clear,
+};
+
+static void airoha_eth_cleanup(struct airoha_eth *eth)
+{
+ int i;
+
+ airoha_wed_exit();
+ for (i = 0; i < AIROHA_MAX_NUM_QDMA; i++) {
+ if (eth->qdma[i].eth)
+ airoha_qdma_stop_napi(ð->qdma[i]);
}
- xsi_rsts = devm_kcalloc(eth->dev,
- eth->soc->num_xsi_rsts, sizeof(*xsi_rsts),
- GFP_KERNEL);
- if (!xsi_rsts)
- return -ENOMEM;
+ if (eth->ports) {
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
+ struct airoha_gdm_port *port = eth->ports[i];
+ int j;
- eth->xsi_rsts = xsi_rsts;
- for (i = 0; i < eth->soc->num_xsi_rsts; i++)
- eth->xsi_rsts[i].id = eth->soc->xsi_rsts_names[i];
+ if (!port)
+ continue;
- err = devm_reset_control_bulk_get_exclusive(eth->dev,
- eth->soc->num_xsi_rsts,
- eth->xsi_rsts);
- if (err) {
- dev_err(eth->dev, "failed to get bulk xsi reset lines\n");
- return err;
+ for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
+ struct airoha_gdm_dev *dev = port->devs[j];
+ struct net_device *netdev;
+
+ if (!dev)
+ continue;
+
+ netdev = netdev_from_priv(dev);
+ if (netdev->reg_state == NETREG_REGISTERED)
+ unregister_netdev(netdev);
+ airoha_gdm_phylink_destroy(&dev->common);
+ of_node_put(netdev->dev.of_node);
+ netdev->dev.of_node = NULL;
+ }
+
+ if (!airoha_has_legacy_qdma(eth))
+ airoha_metadata_dst_free(port);
+ }
}
- eth->napi_dev = alloc_netdev_dummy(0);
- if (!eth->napi_dev)
- return -ENOMEM;
+ if (test_bit(DEV_STATE_INITIALIZED, ð->state))
+ airoha_hw_cleanup(eth);
- /* Enable threaded NAPI by default */
- eth->napi_dev->threaded = true;
- strscpy(eth->napi_dev->name, "qdma_eth", sizeof(eth->napi_dev->name));
- platform_set_drvdata(pdev, eth);
+ if (eth->napi_dev) {
+ free_netdev(eth->napi_dev);
+ eth->napi_dev = NULL;
+ }
+}
- err = airoha_hw_init(pdev, eth);
- if (err)
- goto error_netdev_free;
+static const char * const en7523_xsi_rsts_names[] = {
+ "hsi0-mac",
+ "hsi1-mac",
+ "hsi-mac",
+};
- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
- airoha_qdma_start_napi(ð->qdma[i]);
+static int airoha_en751221_get_sport(struct airoha_gdm_port *port, int nbq)
+{
+ (void)port;
+ (void)nbq;
- for_each_child_of_node(pdev->dev.of_node, np) {
- if (!of_device_is_compatible(np, "airoha,eth-mac"))
- continue;
+ return -EINVAL;
+}
- if (!of_device_is_available(np))
- continue;
+static int
+airoha_en751221_get_dev_from_sport(struct airoha_eth *eth, u32 sport,
+ u16 *port, u16 *dev)
+{
+ *dev = 0;
- err = airoha_alloc_gdm_port(eth, np);
- if (err) {
- of_node_put(np);
- goto error_napi_stop;
- }
+ if (sport == ETX_FPORT_GDM2 || sport == ETX_FPORT_QDMA1_CPU) {
+ *port = AIROHA_GDM2_IDX - 1;
+ return 0;
}
- err = airoha_register_gdm_devices(eth);
- if (err)
- goto error_napi_stop;
+ if (sport == ETX_FPORT_GDM1 || sport == ETX_FPORT_QDMA0_CPU ||
+ econet_en751221_dsa_sport(sport)) {
+ *port = AIROHA_GDM1_IDX - 1;
+ return 0;
+ }
+ /*
+ * Keep the generation-1 fallback used before the QDMA lifecycle was
+ * folded into the common core. Once HW flow offload is armed, GDM
+ * ingress is redirected through the PPE and search-miss/CPU-return
+ * descriptors may carry a PPE/otherwise non-GDM SPORT. Dropping an
+ * unknown SPORT here therefore drops ordinary control traffic as well
+ * (ARP/ICMP) and makes enabling HW offload disconnect the interface.
+ *
+ * The legacy EcoNet receive path deliberately returned such frames on
+ * GDM1; preserve that behaviour for EN751221.
+ */
+ dev_info_ratelimited(eth->dev,
+ "EN751221 RX unexpected sport %#x, using GDM1\n",
+ sport);
+ *port = AIROHA_GDM1_IDX - 1;
return 0;
+}
-error_napi_stop:
- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
- airoha_qdma_stop_napi(ð->qdma[i]);
+static int airoha_en7528_get_sport(struct airoha_gdm_port *port, int nbq)
+{
+ (void)port;
+ (void)nbq;
- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
- struct airoha_gdm_port *port = eth->ports[i];
- int j;
+ return -EINVAL;
+}
- if (!port)
- continue;
+static int airoha_en7528_get_dev_from_sport(struct airoha_eth *eth, u32
sport,
+ u16 *port, u16 *dev)
+{
+ *dev = 0;
- for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
- struct airoha_gdm_dev *dev = port->devs[j];
- struct net_device *netdev;
+ switch (sport) {
+ case ETX_FPORT_GDM2:
+ case ETX_FPORT_QDMA1_CPU:
+ *port = AIROHA_GDM2_IDX - 1;
+ return 0;
+ case ETX_FPORT_GDM1:
+ case ETX_FPORT_QDMA0_CPU:
+ case 8 ... 13:
+ case 16 ... 21:
+ *port = AIROHA_GDM1_IDX - 1;
+ return 0;
+ default:
+ dev_info_ratelimited(eth->dev, "RX sport %#x\n", sport);
+ return -EINVAL;
+ }
+}
- if (!dev)
- continue;
+static int airoha_en7580_get_sport(struct airoha_gdm_port *port, int nbq)
+{
+ (void)port;
+ (void)nbq;
- netdev = netdev_from_priv(dev);
- if (netdev->reg_state == NETREG_REGISTERED)
- unregister_netdev(netdev);
- if (dev->phylink)
- phylink_destroy(dev->phylink);
- of_node_put(netdev->dev.of_node);
- }
- airoha_metadata_dst_free(port);
+ return -EINVAL;
+}
+
+static int airoha_en7580_get_dev_from_sport(struct airoha_eth *eth, u32
sport,
+ u16 *port, u16 *dev)
+{
+ *dev = 0;
+
+ switch (sport) {
+ case ETX_FPORT_GDM2:
+ case ETX_FPORT_QDMA1_CPU:
+ *port = AIROHA_GDM2_IDX - 1;
+ return 0;
+ case ETX_FPORT_GDM1:
+ case ETX_FPORT_QDMA0_CPU:
+ case 8 ... 13:
+ case 16 ... 21:
+ *port = AIROHA_GDM1_IDX - 1;
+ return 0;
+ default:
+ dev_info_ratelimited(eth->dev, "RX sport %#x\n", sport);
+ return -EINVAL;
+ }
+}
+
+static int airoha_en7523_get_sport(struct airoha_gdm_port *port, int nbq)
+{
+ switch (port->id) {
+ case AIROHA_GDM3_IDX:
+ if (nbq == 4)
+ return HSGMII_LAN_7523_PCIE0_SRCPORT;
+ if (nbq == 5)
+ return HSGMII_LAN_7523_PCIE1_SRCPORT;
+ if (nbq == 6)
+ return HSGMII_LAN_7523_USB_SRCPORT;
+ if (!nbq)
+ return HSGMII_LAN_7523_ETH_SRCPORT;
+ break;
+ default:
+ break;
+ }
+ return -EINVAL;
+}
+
+static u32 airoha_en7523_get_vip_port(struct airoha_gdm_port *port, int
nbq)
+{
+ switch (port->id) {
+ case AIROHA_GDM2_IDX:
+ /* GDM2 carries the PON datapath and needs its own VIP port. */
+ return BIT(AIROHA_GDM2_IDX);
+ case AIROHA_GDM3_IDX:
+ if (nbq == 4)
+ return XSI_PCIE0_VIP_PORT_MASK;
+ if (nbq == 5)
+ return XSI_PCIE1_VIP_PORT_MASK;
+ if (!nbq)
+ return XSI_ETH_VIP_PORT_MASK;
+ if (nbq == 6)
+ return XSI_USB_VIP_PORT_MASK;
+ break;
+ default:
+ break;
}
- airoha_hw_cleanup(eth);
-error_netdev_free:
- free_netdev(eth->napi_dev);
- platform_set_drvdata(pdev, NULL);
- return err;
+ return 0;
}
-static void airoha_remove(struct platform_device *pdev)
+static int airoha_en7523_get_dev_from_sport(struct airoha_eth *eth, u32
sport,
+ u16 *port, u16 *dev)
{
- struct airoha_eth *eth = platform_get_drvdata(pdev);
- int i;
-
- for (i = 0; i < ARRAY_SIZE(eth->qdma); i++)
- airoha_qdma_stop_napi(ð->qdma[i]);
-
- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
- struct airoha_gdm_port *port = eth->ports[i];
- int j;
-
- if (!port)
- continue;
-
- for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
- struct airoha_gdm_dev *dev = port->devs[j];
- struct net_device *netdev;
-
- if (!dev)
- continue;
+ (void)eth;
- netdev = netdev_from_priv(dev);
- unregister_netdev(netdev);
- phylink_destroy(dev->phylink);
- of_node_put(netdev->dev.of_node);
- }
- airoha_metadata_dst_free(port);
+ *dev = 0;
+ switch (sport) {
+ case 0x10 ... 0x14:
+ *port = 0; /* GDM1 */
+ break;
+ case 0x2 ... 0x4:
+ *port = sport - 1;
+ break;
+ case HSGMII_LAN_7523_PCIE1_SRCPORT:
+ *dev = 1;
+ fallthrough;
+ case HSGMII_LAN_7523_PCIE0_SRCPORT:
+ *port = 2; /* GDM3 */
+ break;
+ case HSGMII_LAN_7523_USB_SRCPORT:
+ *dev = 2;
+ fallthrough;
+ case HSGMII_LAN_7523_ETH_SRCPORT:
+ *port = 2; /* GDM3 */
+ break;
+ default:
+ return -EINVAL;
}
- airoha_hw_cleanup(eth);
- free_netdev(eth->napi_dev);
- platform_set_drvdata(pdev, NULL);
+ return 0;
}
static const char * const en7581_xsi_rsts_names[] = {
@@ -4234,11 +9289,10 @@ static u32 airoha_en7581_get_vip_port(struct
airoha_gdm_port *port, int nbq)
return 0;
}
-static int airoha_en7581_get_dev_from_sport(struct airoha_qdma_desc *desc,
+static int airoha_en7581_get_dev_from_sport(struct airoha_eth *eth, u32
sport,
u16 *port, u16 *dev)
{
- u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK,
- le32_to_cpu(READ_ONCE(desc->msg1)));
+ (void)eth;
*dev = 0;
switch (sport) {
@@ -4315,11 +9369,10 @@ static u32 airoha_an7583_get_vip_port(struct
airoha_gdm_port *port, int nbq)
return 0;
}
-static int airoha_an7583_get_dev_from_sport(struct airoha_qdma_desc *desc,
+static int airoha_an7583_get_dev_from_sport(struct airoha_eth *eth, u32
sport,
u16 *port, u16 *dev)
{
- u32 sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK,
- le32_to_cpu(READ_ONCE(desc->msg1)));
+ (void)eth;
*dev = 0;
switch (sport) {
@@ -4345,11 +9398,100 @@ static int
airoha_an7583_get_dev_from_sport(struct airoha_qdma_desc *desc,
return 0;
}
-static const struct airoha_eth_soc_data en7581_soc_data = {
- .version = 0x7581,
+static const struct airoha_eth_xpon_ops airoha_xpon_ops = {
+ .set_mode = airoha_xpon_set_mode,
+ .set_datapath = airoha_xpon_set_datapath,
+ .set_tcont_channel = airoha_xpon_set_tcont_channel,
+ .register_link = airoha_xpon_register_link,
+ .unregister_link = airoha_xpon_unregister_link,
+ .update_link = airoha_xpon_update_link,
+ .control_start = airoha_xpon_control_start,
+ .control_stop = airoha_xpon_control_stop,
+ .dump_oam_rx_state = airoha_xpon_dump_oam_rx_state,
+ .register_oam = airoha_xpon_register_oam,
+ .unregister_oam = airoha_xpon_unregister_oam,
+ .xmit_oam = airoha_xpon_xmit_oam,
+ .add_service = airoha_xpon_add_service,
+ .get_tx_info = airoha_xpon_get_tx_info,
+ .del_service = airoha_xpon_del_service,
+ .has_gem_service = airoha_xpon_has_gem_service,
+ .flush_services = airoha_xpon_flush_services,
+};
+
+const struct airoha_eth_soc_data econet_en751221_soc_data = {
+ .version = econet_en751221,
+ .mac_addr_mode = AIROHA_MAC_ADDR_GDM_MASK,
+ .xpon_ops = &econet_xpon_ops,
+ .num_ppe = 1,
+ .legacy_qdma = true,
+ .irq_banks = 1,
+ .max_gdm_ports = 2,
+ .pse_fq_cfg = PSE_FQ_CFG,
+ .ppe_dram_entries = 16 * 1024,
+ .foe_format = AIROHA_FOE_FORMAT_V1,
+ .ppe_fport = EN751221_GDM_FPORT_PPE,
+ .ppe_cpu_fport = {
+ EN751221_GDM_FPORT_QDMA0_CPU,
+ EN751221_GDM_FPORT_QDMA1_CPU,
+ },
+ .ops = {
+ .get_sport = airoha_en751221_get_sport,
+ .get_dev_from_sport = airoha_en751221_get_dev_from_sport,
+ },
+};
+
+const struct airoha_eth_soc_data econet_en7528_soc_data = {
+ .version = econet_en7528,
+ .num_ppe = 1,
+ .legacy_qdma = true,
+ .irq_banks = 4,
+ .max_gdm_ports = 2,
+ .pse_fq_cfg = PSE_FQ_CFG,
+ .ppe_dram_entries = 16 * 1024,
+ .foe_format = AIROHA_FOE_FORMAT_V1,
+ .ppe_fport = EN751221_GDM_FPORT_PPE,
+ .ppe_cpu_fport = {
+ EN751221_GDM_FPORT_QDMA0_CPU,
+ EN751221_GDM_FPORT_QDMA1_CPU,
+ },
+ .ops = {
+ .get_sport = airoha_en7528_get_sport,
+ .get_dev_from_sport = airoha_en7528_get_dev_from_sport,
+ },
+};
+
+const struct airoha_eth_soc_data econet_en7580_soc_data = {
+ .version = econet_en7580,
+ .num_ppe = 1,
+ .tx_ring = 8,
+ .rx_ring = 16,
+ .irq_banks = 4,
+ .max_gdm_ports = 2,
+ .ppe_stats_entries = 0,
+ .ppe_sram_entries = 16 * 1024,
+ .ppe_dram_entries = 16 * 1024,
+ .foe_format = AIROHA_FOE_FORMAT_V2,
+ .pse_fq_cfg = PSE_FQ_CFG,
+ .ops = {
+ .get_sport = airoha_en7580_get_sport,
+ .get_dev_from_sport = airoha_en7580_get_dev_from_sport,
+ },
+};
+
+const struct airoha_eth_soc_data airoha_en7581_soc_data = {
+ .version = airoha_en7581,
.xsi_rsts_names = en7581_xsi_rsts_names,
.num_xsi_rsts = ARRAY_SIZE(en7581_xsi_rsts_names),
.num_ppe = 2,
+ .rx_ring = 32,
+ .tx_ring = 32,
+ .irq_banks = 4,
+ .max_gdm_ports = 4,
+ .pse_fq_cfg = PSE_FQ_CFG,
+ .ppe_stats_entries = 4 * 1024,
+ .ppe_sram_entries = 8 * 1024,
+ .ppe_dram_entries = 16 * 1024,
+ .foe_format = AIROHA_FOE_FORMAT_V2_TUNNEL,
.ops = {
.get_sport = airoha_en7581_get_sport,
.get_vip_port = airoha_en7581_get_vip_port,
@@ -4357,11 +9499,20 @@ static const struct airoha_eth_soc_data
en7581_soc_data = {
},
};
-static const struct airoha_eth_soc_data an7583_soc_data = {
- .version = 0x7583,
+const struct airoha_eth_soc_data airoha_an7583_soc_data = {
+ .version = airoha_an7583,
.xsi_rsts_names = an7583_xsi_rsts_names,
.num_xsi_rsts = ARRAY_SIZE(an7583_xsi_rsts_names),
.num_ppe = 1,
+ .rx_ring = 32,
+ .tx_ring = 32,
+ .irq_banks = 4,
+ .max_gdm_ports = 4,
+ .pse_fq_cfg = PSE_FQ_CFG,
+ .ppe_stats_entries = 4 * 1024,
+ .ppe_sram_entries = 8 * 1024,
+ .ppe_dram_entries = 16 * 1024,
+ .foe_format = AIROHA_FOE_FORMAT_V2_TUNNEL,
.ops = {
.get_sport = airoha_an7583_get_sport,
.get_vip_port = airoha_an7583_get_vip_port,
@@ -4369,23 +9520,450 @@ static const struct airoha_eth_soc_data
an7583_soc_data = {
},
};
-static const struct of_device_id of_airoha_match[] = {
- { .compatible = "airoha,en7581-eth", .data = &en7581_soc_data },
- { .compatible = "airoha,an7583-eth", .data = &an7583_soc_data },
+const struct airoha_eth_soc_data airoha_en7523_soc_data = {
+ .version = airoha_en7523,
+ .xpon_ops = &airoha_xpon_ops,
+ .xsi_rsts_names = en7523_xsi_rsts_names,
+ .num_xsi_rsts = ARRAY_SIZE(en7523_xsi_rsts_names),
+ .rx_ring = 16,
+ .tx_ring = 8,
+ .num_ppe = 1,
+ .irq_banks = 2,
+ .max_gdm_ports = 3,
+ .pse_fq_cfg = PSE_FQ_CFG_EN7523,
+ .ppe_stats_entries = 0,
+ /* The vendor PPE runs EN7523 in 80-byte FoE mode with 512 SRAM and
+ * 16K DRAM entries.
+ */
+ .ppe_sram_entries = 512,
+ .ppe_dram_entries = 16 * 1024,
+ .foe_format = AIROHA_FOE_FORMAT_V2,
+ .ops = {
+ .get_sport = airoha_en7523_get_sport,
+ .get_vip_port = airoha_en7523_get_vip_port,
+ .get_dev_from_sport = airoha_en7523_get_dev_from_sport,
+ },
+};
+
+/*
------------------------------------------------------------------------- */
+/* Platform driver and xPON API. */
+
+static int airoha_eth_probe(struct platform_device *pdev)
+{
+ const struct airoha_eth_soc_data *soc;
+ struct reset_control_bulk_data *xsi_rsts;
+ struct device_node *np;
+ struct airoha_eth *eth;
+ bool econet;
+ int i, err;
+
+ soc = of_device_get_match_data(&pdev->dev);
+ if (!soc)
+ return dev_err_probe(&pdev->dev, -EINVAL,
+ "missing Ethernet SoC data\n");
+
+ eth = devm_kzalloc(&pdev->dev, sizeof(*eth), GFP_KERNEL);
+ if (!eth)
+ return -ENOMEM;
+
+ eth->dev = &pdev->dev;
+ eth->soc = soc;
+ eth->ppe_host_ops = &airoha_ppe_host_ops;
+ econet = airoha_is_econet(eth);
+ platform_set_drvdata(pdev, eth);
+
+ err = airoha_eth_set_dma_mask(eth->dev);
+ if (err)
+ goto error;
+
+ eth->fe_regs = devm_platform_ioremap_resource_byname(pdev, "fe");
+ if (IS_ERR(eth->fe_regs)) {
+ err = dev_err_probe(eth->dev, PTR_ERR(eth->fe_regs),
+ "failed to iomap fe regs\n");
+ goto error;
+ }
+
+ eth->gdmp_regs = devm_platform_ioremap_resource_byname(pdev, "gdmp");
+ if (IS_ERR(eth->gdmp_regs)) {
+ err = dev_err_probe(eth->dev, PTR_ERR(eth->gdmp_regs),
+ "failed to iomap gdmp regs\n");
+ goto error;
+ }
+
+ /* These are global FE reset domains on every supported generation.
+ * The DT binding keeps the functional names stable even when a vendor
+ * tree used QDMA-instance-specific labels for the same reset lines.
+ */
+ eth->rsts[0].id = "fe";
+ eth->rsts[1].id = "pdma";
+ eth->rsts[2].id = "qdma";
+ err = devm_reset_control_bulk_get_exclusive(eth->dev,
+ AIROHA_MAX_NUM_RSTS, eth->rsts);
+ if (err) {
+ err = dev_err_probe(eth->dev, err,
+ "failed to get FE reset lines\n");
+ goto error;
+ }
+
+ /* XSI reset storage is not needed on the MIPS FE or on SoCs without
+ * an external-serdes block.
+ */
+ if (eth->soc->num_xsi_rsts) {
+ xsi_rsts = devm_kcalloc(eth->dev, eth->soc->num_xsi_rsts,
+ sizeof(*xsi_rsts), GFP_KERNEL);
+ if (!xsi_rsts) {
+ err = -ENOMEM;
+ goto error;
+ }
+
+ eth->xsi_rsts = xsi_rsts;
+ for (i = 0; i < eth->soc->num_xsi_rsts; i++)
+ eth->xsi_rsts[i].id = eth->soc->xsi_rsts_names[i];
+
+ err = devm_reset_control_bulk_get_exclusive(eth->dev,
+ eth->soc->num_xsi_rsts,
+ eth->xsi_rsts);
+ if (err) {
+ err = dev_err_probe(eth->dev, err,
+ "failed to get XSI reset lines\n");
+ goto error;
+ }
+ }
+
+ eth->ports = devm_kcalloc(eth->dev, eth->soc->max_gdm_ports,
+ sizeof(*eth->ports), GFP_KERNEL);
+ if (!eth->ports) {
+ err = -ENOMEM;
+ goto error;
+ }
+
+ eth->napi_dev = airoha_eth_alloc_napi_dev("qdma_eth");
+ if (!eth->napi_dev) {
+ err = -ENOMEM;
+ goto error;
+ }
+
+ /* Airoha GDMs are allocated before DMA is started so an nvmem backed
+ * MAC address can defer probe safely. EcoNet queue counts are
described
+ * by the MIPS QDMA profile, therefore those two GDMs are allocated
after
+ * airoha_hw_init() has instantiated QDMA0/QDMA1.
+ */
+ if (!econet) {
+ for_each_available_child_of_node(pdev->dev.of_node, np) {
+ if (!of_device_is_compatible(np, "airoha,eth-mac"))
+ continue;
+
+ err = airoha_alloc_gdm_port(eth, np);
+ if (err) {
+ of_node_put(np);
+ goto error;
+ }
+ }
+ }
+
+ err = airoha_hw_init(pdev, eth);
+ if (err)
+ goto error;
+
+ if (econet) {
+ for_each_available_child_of_node(pdev->dev.of_node, np) {
+ if (!of_device_is_compatible(np, "econet,eth-mac"))
+ continue;
+
+ err = airoha_alloc_gdm_port(eth, np);
+ if (err) {
+ of_node_put(np);
+ goto error;
+ }
+ }
+ }
+
+ /* FE error is a distinct interrupt on EN7516/EN7527/EN7528 and the
+ * newer Airoha FE. Do not depend on its positional index in the IRQ
+ * array; EN751221 legitimately has no separate FE-error line.
+ */
+ eth->fe_irq = platform_get_irq_byname_optional(pdev, "fe-error");
+ if (eth->fe_irq < 0 && eth->fe_irq != -ENXIO) {
+ err = dev_err_probe(eth->dev, eth->fe_irq,
+ "failed to get FE error IRQ\n");
+ goto error;
+ }
+
+ for (i = 0; ; i++) {
+ struct device_node *wed_np;
+
+ wed_np = of_parse_phandle(pdev->dev.of_node, "airoha,wed", i);
+ if (!wed_np)
+ break;
+ err = airoha_wed_add_hw(wed_np, i);
+ if (err)
+ goto error;
+ }
+
+ for (i = 0; i < AIROHA_MAX_NUM_QDMA; i++)
+ airoha_qdma_start_napi(ð->qdma[i]);
+
+ err = airoha_register_gdm_devices(eth);
+ if (err)
+ goto error;
+
+ err = airoha_npu_start(eth->dev);
+ if (err && err != -ENODEV && err != -EPROBE_DEFER &&
+ err != -EOPNOTSUPP)
+ dev_warn(eth->dev, "failed to start NPU: %d\n", err);
+
+ return 0;
+
+error:
+ airoha_eth_cleanup(eth);
+ platform_set_drvdata(pdev, NULL);
+ return err;
+}
+
+static void airoha_eth_remove(struct platform_device *pdev)
+{
+ struct airoha_eth *eth = platform_get_drvdata(pdev);
+
+ if (!eth || !eth->soc)
+ return;
+
+ airoha_eth_cleanup(eth);
+ platform_set_drvdata(pdev, NULL);
+}
+
+static const struct airoha_eth_xpon_ops *
+airoha_eth_get_xpon_ops(struct net_device *netdev,
+ struct airoha_gdm_common **gdm_out)
+{
+ struct airoha_gdm_common *gdm;
+ struct airoha_eth *eth;
+
+ if (!netdev)
+ return NULL;
+
+ gdm = airoha_gdm_common_from_netdev(netdev);
+ if (!gdm || gdm->id != 2 || !gdm->eth || !gdm->eth->soc)
+ return NULL;
+
+ eth = gdm->eth;
+ if (!eth->soc->xpon_ops)
+ return NULL;
+
+ if (gdm_out)
+ *gdm_out = gdm;
+
+ return eth->soc->xpon_ops;
+}
+
+struct net_device *airoha_eth_get_xpon_netdev(void)
+{
+ struct net_device *netdev, *found = NULL;
+
+ rtnl_lock();
+ for_each_netdev(&init_net, netdev) {
+ if (!airoha_eth_get_xpon_ops(netdev, NULL))
+ continue;
+
+ dev_hold(netdev);
+ found = netdev;
+ break;
+ }
+ rtnl_unlock();
+
+ return found;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_get_xpon_netdev);
+
+int airoha_eth_set_xpon_mode(struct net_device *netdev,
+ enum airoha_xpon_mode mode)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->set_mode ? ops->set_mode(netdev, mode) :
-EOPNOTSUPP;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_set_xpon_mode);
+
+int airoha_eth_set_xpon_datapath(struct net_device *netdev,
+ enum airoha_xpon_mode mode, bool enable)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->set_datapath ?
+ ops->set_datapath(netdev, mode, enable) : -EOPNOTSUPP;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_set_xpon_datapath);
+
+int airoha_eth_set_xpon_tcont_channel(struct net_device *netdev,
+ unsigned int channel, bool enable)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->set_tcont_channel ?
+ ops->set_tcont_channel(netdev, channel, enable) : -EOPNOTSUPP;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_set_xpon_tcont_channel);
+
+int airoha_eth_register_xpon(struct net_device *netdev,
+ enum airoha_xpon_mode mode,
+ const struct airoha_xpon_link_ops *link_ops,
+ void *priv)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->register_link ?
+ ops->register_link(netdev, mode, link_ops, priv) : -EOPNOTSUPP;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_register_xpon);
+
+void airoha_eth_unregister_xpon(struct net_device *netdev,
+ const struct airoha_xpon_link_ops *link_ops,
+ void *priv)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ if (ops && ops->unregister_link)
+ ops->unregister_link(netdev, link_ops, priv);
+}
+EXPORT_SYMBOL_GPL(airoha_eth_unregister_xpon);
+
+void airoha_eth_xpon_update_link(struct net_device *netdev,
+ const struct airoha_xpon_link_state *state)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ if (ops && ops->update_link)
+ ops->update_link(netdev, state);
+}
+EXPORT_SYMBOL_GPL(airoha_eth_xpon_update_link);
+
+int airoha_eth_xpon_control_start(struct net_device *netdev)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->control_start ? ops->control_start(netdev) :
-EOPNOTSUPP;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_xpon_control_start);
+
+void airoha_eth_xpon_control_stop(struct net_device *netdev)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ if (ops && ops->control_stop)
+ ops->control_stop(netdev);
+}
+EXPORT_SYMBOL_GPL(airoha_eth_xpon_control_stop);
+
+void airoha_eth_xpon_dump_oam_rx_state(struct net_device *netdev)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ if (ops && ops->dump_oam_rx_state)
+ ops->dump_oam_rx_state(netdev);
+}
+EXPORT_SYMBOL_GPL(airoha_eth_xpon_dump_oam_rx_state);
+
+int airoha_eth_register_xpon_oam(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->register_oam ?
+ ops->register_oam(netdev, handler) : -EOPNOTSUPP;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_register_xpon_oam);
+
+void airoha_eth_unregister_xpon_oam(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ if (ops && ops->unregister_oam)
+ ops->unregister_oam(netdev, handler);
+}
+EXPORT_SYMBOL_GPL(airoha_eth_unregister_xpon_oam);
+
+int airoha_eth_xmit_xpon_oam(struct net_device *netdev, struct sk_buff
*skb,
+ u8 channel, u16 gem_port_id)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->xmit_oam ?
+ ops->xmit_oam(netdev, skb, channel, gem_port_id) : -EOPNOTSUPP;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_xmit_xpon_oam);
+
+int airoha_eth_xpon_add_service(struct net_device *netdev,
+ const struct airoha_xpon_service_cfg *cfg)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->add_service ?
+ ops->add_service(netdev, cfg) : -EOPNOTSUPP;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_xpon_add_service);
+
+int airoha_eth_xpon_get_tx_info(struct net_device *netdev, bool vlan_valid,
+ u16 vlan_id, bool pcp_valid, u8 pcp,
+ struct airoha_xpon_tx_info *info)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->get_tx_info ?
+ ops->get_tx_info(netdev, vlan_valid, vlan_id, pcp_valid, pcp,
info) :
+ -EOPNOTSUPP;
+}
+
+bool airoha_eth_xpon_del_service(struct net_device *netdev, u32 cookie,
+ u16 *gem_port_id)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->del_service ?
+ ops->del_service(netdev, cookie, gem_port_id) : false;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_xpon_del_service);
+
+bool airoha_eth_xpon_has_gem_service(struct net_device *netdev, u16
gem_port_id)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ return ops && ops->has_gem_service ?
+ ops->has_gem_service(netdev, gem_port_id) : false;
+}
+EXPORT_SYMBOL_GPL(airoha_eth_xpon_has_gem_service);
+
+void airoha_eth_xpon_flush_services(struct net_device *netdev)
+{
+ const struct airoha_eth_xpon_ops *ops =
airoha_eth_get_xpon_ops(netdev, NULL);
+
+ if (ops && ops->flush_services)
+ ops->flush_services(netdev);
+}
+EXPORT_SYMBOL_GPL(airoha_eth_xpon_flush_services);
+
+static const struct of_device_id airoha_eth_of_match[] = {
+ { .compatible = "econet,en751221-eth", .data =
&econet_en751221_soc_data },
+ { .compatible = "econet,en7528-eth", .data = &econet_en7528_soc_data },
+ { .compatible = "econet,en7580-eth", .data = &econet_en7580_soc_data },
+ { .compatible = "airoha,en7523-eth", .data = &airoha_en7523_soc_data },
+ { .compatible = "airoha,en7581-eth", .data = &airoha_en7581_soc_data },
+ { .compatible = "airoha,an7583-eth", .data = &airoha_an7583_soc_data },
{ /* sentinel */ }
};
-MODULE_DEVICE_TABLE(of, of_airoha_match);
+MODULE_DEVICE_TABLE(of, airoha_eth_of_match);
-static struct platform_driver airoha_driver = {
- .probe = airoha_probe,
- .remove = airoha_remove,
+static struct platform_driver airoha_eth_driver = {
+ .probe = airoha_eth_probe,
+ .remove = airoha_eth_remove,
.driver = {
- .name = KBUILD_MODNAME,
- .of_match_table = of_airoha_match,
+ .name = "airoha-eth",
+ .of_match_table = airoha_eth_of_match,
},
};
-module_platform_driver(airoha_driver);
+module_platform_driver(airoha_eth_driver);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>");
-MODULE_DESCRIPTION("Ethernet driver for Airoha SoC");
+MODULE_AUTHOR("Caleb James DeLisle <cjd@cjdns.fr>");
+MODULE_AUTHOR("Matheus Sampaio Queiroga <srherobrine20@gmail.com>");
+MODULE_DESCRIPTION("Airoha and EcoNet frame-engine Ethernet driver");
diff --git a/drivers/net/ethernet/airoha/airoha_eth.h
b/drivers/net/ethernet/airoha/airoha_eth.h
index 6456e78f8767..b96b59393cbb 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.h
+++ b/drivers/net/ethernet/airoha/airoha_eth.h
@@ -7,19 +7,185 @@
#ifndef AIROHA_ETH_H
#define AIROHA_ETH_H
+#include <linux/atomic.h>
+#include <linux/bitfield.h>
#include <linux/debugfs.h>
#include <linux/etherdevice.h>
#include <linux/iopoll.h>
+#include <linux/io.h>
#include <linux/kernel.h>
+#include <linux/mutex.h>
#include <linux/netdevice.h>
+#include <linux/phylink.h>
+#include <linux/platform_device.h>
#include <linux/reset.h>
+#include <linux/skbuff.h>
+#include <linux/types.h>
#include <linux/soc/airoha/airoha_offload.h>
#include <net/dsa.h>
-#define AIROHA_MAX_NUM_GDM_PORTS 4
-#define AIROHA_MAX_NUM_GDM_DEVS 2
+/* Common Ethernet/GDM/QDMA definitions. */
+struct device;
+struct device_node;
+struct ethtool_drvinfo;
+struct sk_buff;
+struct airoha_ppe_dev;
+struct airoha_eth;
+struct gdm;
+
+#define AIROHA_MTK_INVALID_CHANNEL 7
+#define AIROHA_MTK_HDR_LEN 4
+#define AIROHA_MTK_STAG_PORT_MASK GENMASK(5, 0)
+#define AIROHA_MTK_HDR_XMIT_TAGGED_TPID_8100 1
+#define AIROHA_MTK_HDR_XMIT_TAGGED_TPID_88A8 2
+
+#define AIROHA_XPON_DATA_DUMP_LEN 128
+
+#define EN751221_GPON_RX_CHN_MASK GENMASK(1, 0)
+#define EN751221_EPON_RX_CHN_MASK GENMASK(7, 0)
+#define EN751221_EPON_TX_CHN_MASK (GENMASK(7, 0) | GENMASK(23, 16))
+#define EN751221_EPON_HWF_CHN_MASK GENMASK(7, 0)
+
+#define EN751221_DSA_SPORT_BASE 8
+#define EN751221_DSA_NUM_PORTS 5
+
+#define EN7523_GPON_MAX_FRAME_LEN 2000
+#define EN7523_GPON_PSE_BUF_INIT_CHAN_THR 4
+#define EN7523_GPON_PSE_BUF_INIT_TOTAL_THR 0xe0
+#define EN7523_GPON_PSE_BUF_FEW_CHAN_THR 0x20
+#define EN7523_GPON_PSE_BUF_MANY_CHAN_THR 0x08
+#define EN7523_GPON_PSE_BUF_CHAN_OVERHEAD 0x10
+#define EN7523_GPON_PSE_BUF_FEW_CHAN_MAX 8
+#define EN7523_GPON_DBA_RATE_KBPS 0xa00000
+#define EN7523_GPON_DBA_CBS_BYTES 0x8000
+#define EN7523_GPON_DBA_PBS_BYTES 0xffff
+
+#define AIROHA_XPON_TX_OFFLOAD_FEATURES \
+ (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | \
+ NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6)
+
+enum airoha_mtk_tag_mode {
+ AIROHA_MTK_TAG_IN_SKB,
+ AIROHA_MTK_TAG_TO_DESC,
+};
+
+/**
+ * struct airoha_qdma_skb_meta - common QDMA metadata derived from an skb
+ * @mtk_tag: first 16 bits of the MediaTek DSA special tag
+ * @port_mask: destination-port bitmap carried by @mtk_tag
+ * @channel: first destination port, or AIROHA_MTK_INVALID_CHANNEL
+ * @has_mtk_tag: skb belongs to a MediaTek DSA conduit and carries a tag
+ *
+ * EcoNet and newer Airoha frame engines carry the same logical packet
+ * metadata even though their descriptor bit layouts differ. Keep skb
+ * classification common and leave raw descriptor encoding to each backend.
+ */
+struct airoha_qdma_skb_meta {
+ u16 mtk_tag;
+ u8 port_mask;
+ u8 channel;
+ bool has_mtk_tag;
+};
+
+void airoha_qdma_skb_get_mtk_meta(struct sk_buff *skb,
+ struct net_device *netdev,
+ enum airoha_mtk_tag_mode mode,
+ struct airoha_qdma_skb_meta *meta);
+
+enum airoha_eth_family {
+ AIROHA_ETH_FAMILY_AIROHA,
+ AIROHA_ETH_FAMILY_ECONET,
+};
+
+/**
+ * struct airoha_gdm_mac_ops - SoC-specific MAC callbacks
+ * @mac_config: optional phylink MAC configuration callback
+ * @mac_link_up: optional phylink link-up callback
+ * @mac_link_down: optional phylink link-down callback
+ *
+ * GDM1 and GDM2 are functionally equivalent on the EcoNet and Airoha frame
+ * engines. The common object owns phylink and dispatches only the
hardware
+ * differences to the Ethernet backend.
+ */
+struct airoha_gdm_mac_ops {
+ void (*mac_config)(void *priv, unsigned int mode,
+ const struct phylink_link_state *state);
+ void (*mac_link_up)(void *priv, struct phy_device *phy,
+ unsigned int mode, phy_interface_t interface,
+ int speed, int duplex, bool tx_pause,
+ bool rx_pause);
+ void (*mac_link_down)(void *priv, unsigned int mode,
+ phy_interface_t interface);
+};
+
+#define AIROHA_GDM_COMMON_MAGIC 0x47444d43 /* "GDMC" */
+
+/**
+ * struct airoha_gdm_common - common GDM netdev/phylink state
+ * @magic: identifies a common GDM private object
+ * @family: frame-engine family
+ * @id: one-based GDM identifier
+ * @pse_port: PPE/PSE destination port used by the backend
+ * @eth: common frame-engine host
+ * @netdev: associated Linux network device
+ * @ppe: optional common PPE frontend
+ * @priv: backend GDM object passed to @mac_ops
+ * @mac_ops: optional SoC-specific MAC callbacks
+ * @phylink: phylink instance
+ * @phylink_config: phylink configuration owned by this GDM
+ * @phylink_started: whether phylink_start() has been issued
+ */
+struct airoha_gdm_common {
+ u32 magic;
+ enum airoha_eth_family family;
+ u8 id;
+ u8 pse_port;
+ struct airoha_eth *eth;
+ struct net_device *netdev;
+ struct airoha_ppe_dev *ppe;
+ void *priv;
+ const struct airoha_gdm_mac_ops *mac_ops;
+ struct phylink *phylink;
+ struct phylink_config phylink_config;
+ bool phylink_started;
+};
+
+u32 airoha_rr(void __iomem *base, u32 offset);
+void airoha_wr(void __iomem *base, u32 offset, u32 val);
+u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val);
+
+int airoha_eth_set_dma_mask(struct device *dev);
+int airoha_eth_get_port_id(struct device *dev, struct device_node *np,
+ u32 min, u32 max, u32 *id);
+struct net_device *airoha_eth_alloc_napi_dev(const char *name);
+int airoha_eth_init_mac_address(struct device *dev, struct device_node *np,
+ struct net_device *netdev);
+void airoha_eth_get_drvinfo(struct net_device *netdev,
+ struct ethtool_drvinfo *info);
+
+void airoha_gdm_common_init(struct airoha_gdm_common *gdm,
+ struct airoha_eth *eth, struct net_device *netdev,
+ enum airoha_eth_family family, u8 id,
+ u8 pse_port, void *priv,
+ const struct airoha_gdm_mac_ops *mac_ops);
+int airoha_gdm_phylink_create(struct airoha_gdm_common *gdm,
+ struct device_node *np,
+ phy_interface_t phy_mode);
+int airoha_gdm_phylink_connect(struct airoha_gdm_common *gdm,
+ bool allow_no_phy);
+void airoha_gdm_phylink_disconnect(struct airoha_gdm_common *gdm);
+void airoha_gdm_phylink_destroy(struct airoha_gdm_common *gdm);
+
+static inline struct airoha_gdm_common *
+airoha_gdm_common_from_netdev(struct net_device *netdev)
+{
+ struct airoha_gdm_common *gdm = netdev_priv(netdev);
+
+ return gdm->magic == AIROHA_GDM_COMMON_MAGIC ? gdm : NULL;
+}
+
+#define AIROHA_MAX_NUM_GDM_DEVS 3
#define AIROHA_MAX_NUM_QDMA 2
-#define AIROHA_MAX_NUM_IRQ_BANKS 4
#define AIROHA_MAX_DSA_PORTS 7
#define AIROHA_MAX_NUM_RSTS 3
#define AIROHA_MAX_MTU 9220
@@ -27,35 +193,66 @@
#define AIROHA_MAX_PACKET_SIZE 2048
#define AIROHA_NUM_QOS_CHANNELS 4
#define AIROHA_NUM_QOS_QUEUES 8
-#define AIROHA_NUM_TX_RING 32
-#define AIROHA_NUM_RX_RING 32
-#define AIROHA_NUM_NETDEV_TX_RINGS (AIROHA_NUM_TX_RING + \
+#define AIROHA_NUM_NETDEV_TX_RINGS(_soc) (_soc->tx_ring + \
AIROHA_NUM_QOS_CHANNELS)
#define AIROHA_FE_MC_MAX_VLAN_TABLE 64
#define AIROHA_FE_MC_MAX_VLAN_PORT 16
#define AIROHA_NUM_TX_IRQ 2
-#define AIROHA_RX_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
-#define AIROHA_RX_LEN(_n) ((_n) - AIROHA_RX_HEADROOM)
+/*
+ * Software classification rules rather than a hardware table: one
entry per
+ * UNI and priority combination, all of which can share a single GEM
port and
+ * T-CONT. An OLT mapping eight priorities across several UNIs, plus a
virtual
+ * Ethernet interface point, needs far more than the thirty-two this
started with.
+ */
+#define AIROHA_XPON_MAX_SERVICES 256
+#define AIROHA_XPON_SVC_MAX_COOKIES 8
#define HW_DSCP_NUM 2048
#define IRQ_QUEUE_LEN(_n) ((_n) ? 1024 : 2048)
-#define TX_DSCP_NUM 1024
-#define RX_DSCP_NUM(_n) \
- ((_n) == 2 ? 128 : \
- (_n) == 11 ? 128 : \
- (_n) == 15 ? 128 : \
- (_n) == 0 ? 1024 : 16)
-
-#define AIROHA_LRO_PAGE_ORDER order_base_2(SZ_16K / PAGE_SIZE)
-#define AIROHA_MAX_NUM_LRO_QUEUES 8
-#define AIROHA_RXQ_LRO_EN_MASK GENMASK(7, 0)
-#define AIROHA_RXQ_LRO_MAX_AGG_COUNT 64
-#define AIROHA_RXQ_LRO_MAX_AGG_TIME 100
-#define AIROHA_RXQ_LRO_MAX_AGE_TIME 2000
-
+#define TX_DSCP_NUM(_n) \
+ ((_n) == 0 ? 1536 : \
+ (_n) == 1 ? 128 : \
+ (_n) == 2 ? 128 : \
+ (_n) == 3 ? 128 : \
+ (_n) == 4 ? 128 : \
+ (_n) == 5 ? 128 : \
+ (_n) == 6 ? 1024 : \
+ (_n) == 7 ? 4096 : 1024)
+#define RX_DSCP_NUM(_n) \
+ ((_n) == 0 ? 512 : \
+ (_n) == 1 ? 1024 : \
+ (_n) == 2 ? 128 : \
+ (_n) == 3 ? 16 : \
+ (_n) == 4 ? 16 : \
+ (_n) == 5 ? 8 : \
+ (_n) == 6 ? 8 : \
+ (_n) == 7 ? 16 : \
+ (_n) == 8 ? 16 : \
+ (_n) == 9 ? 16 : \
+ (_n) == 10 ? 16 : \
+ (_n) == 11 ? 128 : \
+ (_n) == 12 ? 16 : \
+ (_n) == 13 ? 16 : \
+ (_n) == 14 ? 16 : \
+ (_n) == 15 ? 128 : 16)
+
+#define AIROHA_LRO_PAGE_ORDER 2
+#define EN7523_AIROHA_LRO_PAGE_ORDER 4
+#define AIROHA_MAX_NUM_LRO_QUEUES 8
+#define EN7523_AIROHA_MAX_NUM_LRO_QUEUES 4
+#define AIROHA_RXQ_LRO_EN_MASK GENMASK(31, 24)
+#define EN7523_AIROHA_RXQ_LRO_EN_MASK GENMASK(14, 11)
+#define AIROHA_RXQ_LRO_MAX_AGG_COUNT 64
+#define EN7523_AIROHA_RXQ_LRO_MAX_AGG_COUNT 30
+#define EN7523_AIROHA_RXQ_LRO_MAX_AGG_SIZE 44000
+#define AIROHA_RXQ_LRO_MAX_AGG_TIME 100
+#define AIROHA_RXQ_LRO_MAX_AGE_TIME 2000 /* 1ms */
+
+#define AIROHA_ETH_HW_FEATURES \
+ (NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_IPV6_CSUM)
+#define AIROHA_QDMA_TSO_HW_FEATURES \
+ (NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6)
#define AIROHA_HW_FEATURES \
- (NETIF_F_IP_CSUM | NETIF_F_RXCSUM | \
- NETIF_F_TSO6 | NETIF_F_IPV6_CSUM | \
- NETIF_F_SG | NETIF_F_TSO | NETIF_F_HW_TC)
+ (AIROHA_ETH_HW_FEATURES | AIROHA_QDMA_TSO_HW_FEATURES | NETIF_F_HW_TC)
#define PSE_RSV_PAGES 128
#define PSE_QUEUE_RSV_PAGES 64
@@ -63,15 +260,9 @@
#define QDMA_METER_IDX(_n) ((_n) & 0xff)
#define QDMA_METER_GROUP(_n) (((_n) >> 8) & 0x3)
-#define PPE_SRAM_NUM_ENTRIES (8 * 1024)
-#define PPE_STATS_NUM_ENTRIES (4 * 1024)
-#define PPE_DRAM_NUM_ENTRIES (16 * 1024)
-#define PPE_ENTRY_SIZE 80
-#define PPE_RAM_NUM_ENTRIES_SHIFT(_n) (__ffs((_n) >> 10))
-
-#define MTK_HDR_LEN 4
-#define MTK_HDR_XMIT_TAGGED_TPID_8100 1
-#define MTK_HDR_XMIT_TAGGED_TPID_88A8 2
+#define AIROHA_FOE_ENTRY_SIZE 80
+#define AIROHA_FOE_ENTRY_WORDS (AIROHA_FOE_ENTRY_SIZE / sizeof(u32))
+#define PPE_RAM_NUM_ENTRIES_SHIFT(_n) ((_n) == 512 ? 7 : __ffs((_n)
>> 10))
enum {
QDMA_INT_REG_IDX0,
@@ -79,6 +270,8 @@ enum {
QDMA_INT_REG_IDX2,
QDMA_INT_REG_IDX3,
QDMA_INT_REG_IDX4,
+ QDMA_INT_REG_IDX5,
+ QDMA_INT_REG_IDX6,
QDMA_INT_REG_MAX
};
@@ -95,6 +288,13 @@ enum {
HSGMII_LAN_7583_USB_SRCPORT,
};
+enum {
+ HSGMII_LAN_7523_PCIE0_SRCPORT = 0x16,
+ HSGMII_LAN_7523_PCIE1_SRCPORT,
+ HSGMII_LAN_7523_USB_SRCPORT,
+ HSGMII_LAN_7523_ETH_SRCPORT = 0xffff,
+};
+
enum {
XSI_PCIE0_VIP_PORT_MASK = BIT(22),
XSI_PCIE1_VIP_PORT_MASK = BIT(23),
@@ -129,6 +329,12 @@ enum airoha_gdm_index {
AIROHA_GDM4_IDX = 4,
};
+enum airoha_xpon_mode {
+ AIROHA_XPON_MODE_GPON,
+ AIROHA_XPON_MODE_EPON,
+ AIROHA_XPON_MODE_XGSPON,
+};
+
enum {
FE_PSE_PORT_CDM1,
FE_PSE_PORT_GDM1,
@@ -193,6 +399,7 @@ struct airoha_queue_entry {
};
dma_addr_t dma_addr;
u16 dma_len;
+ bool dma_map_page;
};
struct airoha_queue {
@@ -258,6 +465,12 @@ enum {
AIROHA_FOE_STATE_FIN
};
+enum airoha_foe_format {
+ AIROHA_FOE_FORMAT_V1,
+ AIROHA_FOE_FORMAT_V2,
+ AIROHA_FOE_FORMAT_V2_TUNNEL,
+};
+
enum {
PPE_PKT_TYPE_IPV4_HNAPT = 0,
PPE_PKT_TYPE_IPV4_ROUTE = 1,
@@ -268,6 +481,47 @@ enum {
PPE_PKT_TYPE_IPV6_6RD = 7,
};
+/* FoE hardware words use the same bit numbering on big and little endian
+ * CPUs. Keep mixed-width fields encoded as u32 words instead of
relying on
+ * C bitfield or u16 member layout.
+ */
+#define AIROHA_FOE_WORD_IB1 0
+#define AIROHA_FOE_PORTS_SPORT GENMASK(31, 16)
+#define AIROHA_FOE_PORTS_DPORT GENMASK(15, 0)
+#define AIROHA_FOE_L2_ETYPE GENMASK(31, 16)
+#define AIROHA_FOE_L2_VLAN1 GENMASK(15, 0)
+#define AIROHA_FOE_L2_DMAC_LO GENMASK(31, 16)
+#define AIROHA_FOE_L2_VLAN2 GENMASK(15, 0)
+#define AIROHA_FOE_L2_SMAC_LO GENMASK(31, 16)
+#define AIROHA_FOE_L2_PPPOE_ID GENMASK(15, 0)
+
+/* FoE v1: EN751221 and EN7528. */
+#define AIROHA_FOE_V1_INVALID_HASH 0xffff
+#define AIROHA_FOE_V1_FP_QDMA_HW 6
+
+#define AIROHA_FOE_V1_IPV4_IB2_WORD 4
+#define AIROHA_FOE_V1_IPV4_DATA_WORD 10
+#define AIROHA_FOE_V1_IPV4_L2_WORD 11
+#define AIROHA_FOE_V1_IPV6_DATA_WORD 13
+#define AIROHA_FOE_V1_IPV6_IB2_WORD 14
+#define AIROHA_FOE_V1_IPV6_L2_WORD 15
+
+#define AIROHA_FOE_V1_IB1_BIND_VLAN_LAYER GENMASK(18, 16)
+#define AIROHA_FOE_V1_IB1_BIND_PPPOE BIT(19)
+#define AIROHA_FOE_V1_IB1_BIND_VLAN_TAG BIT(20)
+#define AIROHA_FOE_V1_IB1_BIND_CACHE BIT(22)
+
+#define AIROHA_FOE_V1_IB2_QID GENMASK(3, 0)
+#define AIROHA_FOE_V1_IB2_PSE_QOS BIT(4)
+#define AIROHA_FOE_V1_IB2_PSE_PORT GENMASK(7, 5)
+#define AIROHA_FOE_V1_IB2_MULTICAST BIT(8)
+#define AIROHA_FOE_V1_IB2_PORT_MG GENMASK(17, 12)
+#define AIROHA_FOE_V1_IB2_PORT_AG GENMASK(23, 18)
+
+#define AIROHA_FOE_V1_ACTDP GENMASK(31, 24)
+#define AIROHA_FOE_V1_SHAPER_ID GENMASK(23, 16)
+#define AIROHA_FOE_V1_CHANNEL GENMASK(15, 8)
+
#define AIROHA_FOE_MAC_SMAC_ID GENMASK(20, 16)
#define AIROHA_FOE_MAC_PPPOE_ID GENMASK(15, 0)
@@ -293,7 +547,6 @@ struct airoha_foe_mac_info {
u16 pppoe_id;
u16 src_mac_lo;
- u32 meter;
};
#define AIROHA_FOE_IB1_UNBIND_PREBIND BIT(24)
@@ -417,8 +670,6 @@ struct airoha_foe_ipv6 {
u32 ib2;
struct airoha_foe_mac_info_common l2;
-
- u32 meter;
};
struct airoha_foe_entry {
@@ -433,10 +684,38 @@ struct airoha_foe_entry {
DECLARE_FLEX_ARRAY(u32, d);
};
};
- u8 data[PPE_ENTRY_SIZE];
+ u32 words[AIROHA_FOE_ENTRY_WORDS];
+ u8 data[AIROHA_FOE_ENTRY_SIZE];
};
};
+static inline bool airoha_foe_v1_is_ipv6_5t(const struct
airoha_foe_entry *entry)
+{
+ return FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, entry->ib1) ==
+ PPE_PKT_TYPE_IPV6_ROUTE_5T;
+}
+
+static inline unsigned int
+airoha_foe_v1_ib2_word(const struct airoha_foe_entry *entry)
+{
+ return airoha_foe_v1_is_ipv6_5t(entry) ? AIROHA_FOE_V1_IPV6_IB2_WORD :
+ AIROHA_FOE_V1_IPV4_IB2_WORD;
+}
+
+static inline unsigned int
+airoha_foe_v1_data_word(const struct airoha_foe_entry *entry)
+{
+ return airoha_foe_v1_is_ipv6_5t(entry) ? AIROHA_FOE_V1_IPV6_DATA_WORD :
+ AIROHA_FOE_V1_IPV4_DATA_WORD;
+}
+
+static inline unsigned int
+airoha_foe_v1_l2_word(const struct airoha_foe_entry *entry)
+{
+ return airoha_foe_v1_is_ipv6_5t(entry) ? AIROHA_FOE_V1_IPV6_L2_WORD :
+ AIROHA_FOE_V1_IPV4_L2_WORD;
+}
+
struct airoha_foe_stats {
u32 bytes;
u32 packets;
@@ -469,6 +748,7 @@ struct airoha_flow_data {
struct {
u16 id;
__be16 proto;
+ u8 prio;
} hdr[2];
u8 num;
} vlan;
@@ -502,6 +782,25 @@ struct airoha_flow_table_entry {
struct rhash_head node;
unsigned long cookie;
+ /* FoE v1 binds a software flow to a hardware-created hash bucket on
+ * RX. The same flow object is kept in the global cookie table; this
+ * list and tuple are only used by the v1 bind-on-RX backend.
+ */
+ struct list_head v1_list;
+ u16 addr_type;
+ u16 src_port;
+ u16 dest_port;
+ union {
+ struct {
+ u32 src_ip;
+ u32 dest_ip;
+ };
+ struct {
+ struct in6_addr src_ip6;
+ struct in6_addr dest_ip6;
+ };
+ };
+
/* Must be last --ends in a flexible-array member. */
struct airoha_foe_entry data;
};
@@ -533,32 +832,187 @@ struct airoha_irq_bank {
int irq;
};
+#define EN751221_SLM_BASE 0x1fa60000
+#define EN751221_SLM_REG_SIZE SZ_256
+#define EN751221_SLM_HWF_VIRT_BASE 0x1b800000
+#define EN751221_SLM_PHYS_SIZE SZ_4M
+#define EN751221_SLM_MAX_DRAM_SIZE SZ_256M
+#define EN751221_SLM_SECTOR_SIZE SZ_256
+#define EN751221_SLM_GLO_CFG_BYPASS BIT(0)
+#define EN751221_SLM_EN_ENABLE BIT(0)
+#define EN751221_SLM_EN_ACTIVE BIT(1)
+
+#define EN7516_QDMA_INT_STATUS1_OFFSET 0x700
+#define EN7516_QDMA_INT_STATUS2_OFFSET 0x704
+#define EN7516_QDMA_INT_ENABLE_BASE 0x704
+#define EN7516_QDMA_INT_ENABLE_STRIDE 0x0c
+
+struct en751221_slm_regs {
+ u32 glo_cfg;
+ u32 enable;
+ u32 reserved_08[2];
+ u32 virt_base;
+ u32 virt_size;
+ u32 phys_base;
+ u32 phys_size;
+ u32 free_min_cnt;
+ u32 free_cur_cnt;
+ u32 free_threshold;
+ u32 reserved_2c;
+ u32 api_cmd;
+ u32 api_base;
+ u32 api_rd_addr;
+ u32 reserved_3c;
+ u32 int_status;
+ u32 int_mask;
+ u32 bus_rd_null_addr;
+ u32 reserved_4c;
+ u32 drop_cmd;
+ u32 drop_addr;
+ u32 drop_cnt;
+};
+
+/**
+ * struct airoha_qdma_slm - EN751221-family SLM state
+ * @regs: SLM control-register window
+ * @buf: CPU address of the backing memory
+ * @dma_addr: DMA address of @buf
+ * @buf_size: size of the backing memory
+ * @enabled: SLM translation is active
+ *
+ * SLM only exists on the MIPS EcoNet QDMA generations. Keep it
attached to
+ * the common QDMA object rather than exposing a second QDMA
implementation.
+ */
+struct airoha_qdma_slm {
+ struct en751221_slm_regs __iomem *regs;
+ void *buf;
+ dma_addr_t dma_addr;
+ size_t buf_size;
+ bool enabled;
+};
+
+struct airoha_qdma_mips;
+
struct airoha_qdma {
struct airoha_eth *eth;
void __iomem *regs;
+ u8 id;
+ u8 num_channels;
int users;
- struct airoha_irq_bank irq_banks[AIROHA_MAX_NUM_IRQ_BANKS];
+ struct airoha_irq_bank *irq_banks;
struct airoha_tx_irq_queue q_tx_irq[AIROHA_NUM_TX_IRQ];
- struct airoha_queue q_tx[AIROHA_NUM_TX_RING];
- struct airoha_queue q_rx[AIROHA_NUM_RX_RING];
+ struct airoha_queue *q_tx;
+ struct airoha_queue *q_rx;
DECLARE_BITMAP(qos_channel_map, AIROHA_NUM_QOS_CHANNELS);
+
+ /* Private queue/ring state for the MIPS QDMA layout. */
+ struct airoha_qdma_mips *econet;
};
enum airoha_priv_flags {
AIROHA_PRIV_F_WAN = BIT(0),
AIROHA_PRIV_F_QOS = BIT(1),
+ AIROHA_PRIV_F_XPON_MANAGED = BIT(2),
+};
+
+#define AIROHA_XPON_OAM_RX_F_MIC_PRESENT BIT(0)
+#define AIROHA_XPON_OAM_RX_F_MIC_VALID BIT(1)
+#define AIROHA_XPON_OAM_RX_F_CRC_ERROR BIT(2)
+
+struct airoha_xpon_oam_handler {
+ bool (*rx)(void *priv, struct sk_buff *skb, u8 channel,
+ u16 gem_port_id, u32 flags);
+ void *priv;
+};
+
+/**
+ * struct airoha_xpon_link_ops - xPON netdev lifecycle notifications
+ * @start: notify the provider that the data netdev has been opened
+ * @stop: notify the provider that the data netdev is being closed
+ *
+ * These callbacks control netdev-facing state only. Protocol control
planes
+ * such as OMCI acquire the shared GDM2/QDMA transport separately.
+ */
+struct airoha_xpon_link_ops {
+ int (*start)(void *priv);
+ void (*stop)(void *priv);
+ /* EcoNet routes the GPON/EPON MAC interrupt through
+ * QDMA1 instead of exposing a dedicated platform IRQ. Providers that
+ * need that path publish the hard-IRQ callback here.
+ */
+ void (*mac_irq)(void *priv);
+};
+
+/**
+ * struct airoha_xpon_link_state - xPON link state exposed by GDM2
+ * @mode: active xPON protocol
+ * @valid: true after the provider has published its first state
+ * @link: true when the protocol is operational
+ * @speed: ethtool-compatible nominal downstream speed
+ * @duplex: ethtool duplex mode
+ * @autoneg: ethtool autonegotiation mode
+ * @port: ethtool port type
+ * @rx_line_rate_bps: exact downstream line rate in bits per second
+ * @tx_line_rate_bps: exact upstream line rate in bits per second
+ *
+ * GPON has asymmetric line rates, while ethtool exposes a single speed.
+ * Keep the exact rates in the provider snapshot and use @speed for the
+ * conventional netdev representation.
+ */
+struct airoha_xpon_link_state {
+ enum airoha_xpon_mode mode;
+ bool valid;
+ bool link;
+ u32 speed;
+ u8 duplex;
+ u8 autoneg;
+ u8 port;
+ u64 rx_line_rate_bps;
+ u64 tx_line_rate_bps;
+};
+
+struct airoha_xpon_service_cfg {
+ u32 cookie;
+ u16 gem_port_id;
+ u16 vlan_id;
+ u8 tcont;
+ u8 queue;
+ u8 pcp;
+ bool vlan_valid;
+ bool pcp_valid;
+ bool default_service;
+ bool valid;
+};
+
+struct airoha_xpon_tx_info {
+ u16 gem_port_id;
+ u8 tcont;
+ u8 queue;
+ bool oam;
};
struct airoha_gdm_dev {
+ /* Must stay first: shared GDM/phylink and PPE port metadata. */
+ struct airoha_gdm_common common;
+
struct airoha_qdma __rcu *qdma;
struct airoha_gdm_port *port;
struct airoha_eth *eth;
+ /* MIPS GDM registers share the same logical GDM device object. */
+ union {
+ void __iomem *regs;
+ struct gdm __iomem *econet_regs;
+ };
+ spinlock_t reg_lock;
+ bool g2_stats;
+ u8 fport;
+
DECLARE_BITMAP(qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS);
/* qos stats counters */
u64 cpu_tx_packets;
@@ -569,8 +1023,32 @@ struct airoha_gdm_dev {
struct airoha_hw_stats stats;
- struct phylink *phylink;
- struct phylink_config phylink_config;
+ struct airoha_xpon_oam_handler __rcu *xpon_oam;
+ atomic64_t xpon_oam_rx_packets;
+ atomic64_t xpon_oam_rx_bytes;
+ atomic64_t xpon_oam_rx_delivered;
+ atomic64_t xpon_oam_rx_dropped;
+ atomic64_t xpon_oam_rx_no_handler;
+
+ /* Serializes xPON registration, netdev notifications and
control-plane IO. */
+ struct mutex xpon_lock;
+ const struct airoha_xpon_link_ops *xpon_ops;
+ void *xpon_priv;
+ enum airoha_xpon_mode xpon_mode;
+ bool xpon_started;
+ bool xpon_control_started;
+ /* Protects the xPON state consumed by netdev and ethtool callbacks. */
+ spinlock_t xpon_state_lock;
+ struct airoha_xpon_link_state xpon_link;
+
+ /* Protects GPON GEM/T-CONT service classification. */
+ spinlock_t xpon_service_lock;
+ struct airoha_xpon_service_cfg
+ xpon_services[AIROHA_XPON_MAX_SERVICES];
+ u32 xpon_service_cookies[AIROHA_XPON_MAX_SERVICES]
+ [AIROHA_XPON_SVC_MAX_COOKIES];
+ u8 xpon_service_ncookies[AIROHA_XPON_MAX_SERVICES];
+
};
struct airoha_gdm_port {
@@ -584,17 +1062,61 @@ struct airoha_gdm_port {
struct metadata_dst *dsa_meta[AIROHA_MAX_DSA_PORTS];
};
-#define AIROHA_RXD4_PPE_CPU_REASON GENMASK(20, 16)
-#define AIROHA_RXD4_FOE_ENTRY GENMASK(15, 0)
+#define AIROHA_RXD4_PPE_CPU_REASON GENMASK(20, 16)
+#define AIROHA_RXD4_FOE_ENTRY GENMASK(15, 0)
+#define AN7581_AIROHA_RXD4_FOE_ENTRY_INVALID 0xffff
+#define EN7523_AIROHA_RXD4_FOE_ENTRY_INVALID 0x7fff
+
+#define AIROHA_PPE_CPU_REASON_FOE_UNHIT 0x0d
+#define AIROHA_PPE_CPU_REASON_HIT_UNBIND 0x0e
+#define AIROHA_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED 0x0f
+
+struct airoha_ppe_host_ops {
+ int (*get_fe_port)(struct airoha_gdm_dev *dev);
+ bool (*is_valid_gdm_dev)(struct airoha_eth *eth,
+ struct airoha_gdm_dev *dev);
+ int (*xpon_get_tx_info)(struct net_device *netdev, bool vlan_valid,
+ u16 vlan_id, bool pcp_valid, u8 pcp,
+ struct airoha_xpon_tx_info *info);
+ struct airoha_npu *(*npu_get)(struct airoha_eth *eth);
+ void (*npu_put)(struct airoha_npu *npu);
+ int (*npu_ppe_init)(struct airoha_npu *npu);
+ int (*npu_ppe_deinit)(struct airoha_npu *npu);
+ int (*npu_ppe_init_stats)(struct airoha_npu *npu, dma_addr_t addr,
+ u32 num_stats_entries);
+ void (*npu_stats_read)(struct airoha_npu *npu, u32 index,
+ struct airoha_foe_stats *stats);
+ void (*npu_stats_clear)(struct airoha_npu *npu, u32 index);
+};
-struct airoha_ppe {
+struct airoha_ppe_common {
struct airoha_ppe_dev dev;
struct airoha_eth *eth;
void *foe;
dma_addr_t foe_dma;
+ struct dentry *debugfs_dir;
+};
+
+struct airoha_ppe_v1 {
+ /* Protects flow ownership in the hardware-created V1 hash buckets. */
+ spinlock_t lock;
+ struct list_head flows;
+ struct airoha_flow_table_entry **foe_owner;
+ bool armed;
+};
+
+struct airoha_ppe {
+ struct airoha_ppe_common common;
+
+ /* TC block ownership is common; only the v1 backend needs to arm the
+ * engine on first bind and disarm it after the last user.
+ */
+ struct list_head block_cb_list;
+
struct rhashtable l2_flows;
+ struct hlist_head pending_flows;
struct hlist_head *foe_flow;
u16 *foe_check_time;
@@ -602,18 +1124,86 @@ struct airoha_ppe {
struct airoha_foe_stats *foe_stats;
dma_addr_t foe_stats_dma;
- struct dentry *debugfs_dir;
+ /* Set once CPU-direct PPE offload has been initialized on systems
+ * with no attached NPU, to avoid re-running setup (and re-requesting
+ * the airoha-npu module) on every flow.
+ */
+ bool offload_setup_done;
+
+ struct airoha_ppe_v1 v1;
+};
+
+enum airoha_ids {
+ econet_en751221 = 0x751221,
+ econet_en7528 = 0x7528,
+ econet_en7580 = 0x7580,
+ airoha_en7523 = 0x7523,
+ airoha_en7581 = 0x7581,
+ airoha_an7583 = 0x7583,
+};
+
+struct platform_device;
+
+struct airoha_eth;
+
+struct airoha_eth_xpon_ops {
+ int (*set_mode)(struct net_device *netdev, enum airoha_xpon_mode mode);
+ int (*set_datapath)(struct net_device *netdev, enum
airoha_xpon_mode mode,
+ bool enable);
+ int (*set_tcont_channel)(struct net_device *netdev, unsigned int
channel,
+ bool enable);
+ int (*register_link)(struct net_device *netdev, enum
airoha_xpon_mode mode,
+ const struct airoha_xpon_link_ops *ops, void *priv);
+ void (*unregister_link)(struct net_device *netdev,
+ const struct airoha_xpon_link_ops *ops, void *priv);
+ void (*update_link)(struct net_device *netdev,
+ const struct airoha_xpon_link_state *state);
+ int (*control_start)(struct net_device *netdev);
+ void (*control_stop)(struct net_device *netdev);
+ void (*dump_oam_rx_state)(struct net_device *netdev);
+ int (*register_oam)(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler);
+ void (*unregister_oam)(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler);
+ int (*xmit_oam)(struct net_device *netdev, struct sk_buff *skb,
+ u8 channel, u16 gem_port_id);
+ int (*add_service)(struct net_device *netdev,
+ const struct airoha_xpon_service_cfg *cfg);
+ int (*get_tx_info)(struct net_device *netdev, bool vlan_valid, u16
vlan_id,
+ bool pcp_valid, u8 pcp,
+ struct airoha_xpon_tx_info *info);
+ bool (*del_service)(struct net_device *netdev, u32 cookie, u16
*gem_port_id);
+ bool (*has_gem_service)(struct net_device *netdev, u16 gem_port_id);
+ void (*flush_services)(struct net_device *netdev);
+};
+
+enum airoha_mac_addr_mode {
+ AIROHA_MAC_ADDR_FE_RANGE,
+ AIROHA_MAC_ADDR_GDM_MASK,
};
struct airoha_eth_soc_data {
- u16 version;
+ enum airoha_ids version;
+ enum airoha_mac_addr_mode mac_addr_mode;
+ const struct airoha_eth_xpon_ops *xpon_ops;
const char * const *xsi_rsts_names;
int num_xsi_rsts;
int num_ppe;
+ int tx_ring, rx_ring;
+ int irq_banks;
+ int max_gdm_ports;
+ bool legacy_qdma;
+ u32 pse_fq_cfg;
+ u32 ppe_stats_entries;
+ u32 ppe_sram_entries;
+ u32 ppe_dram_entries;
+ enum airoha_foe_format foe_format;
+ u8 ppe_fport;
+ u8 ppe_cpu_fport[AIROHA_MAX_NUM_QDMA];
struct {
int (*get_sport)(struct airoha_gdm_port *port, int nbq);
u32 (*get_vip_port)(struct airoha_gdm_port *port, int nbq);
- int (*get_dev_from_sport)(struct airoha_qdma_desc *desc,
+ int (*get_dev_from_sport)(struct airoha_eth *eth, u32 sport,
u16 *port, u16 *dev);
} ops;
};
@@ -625,24 +1215,25 @@ struct airoha_eth {
unsigned long state;
void __iomem *fe_regs;
+ void __iomem *gdmp_regs;
struct airoha_npu __rcu *npu;
struct airoha_ppe *ppe;
+ struct airoha_ppe_dev *ppe_dev;
+ const struct airoha_ppe_host_ops *ppe_host_ops;
struct rhashtable flow_table;
struct reset_control_bulk_data rsts[AIROHA_MAX_NUM_RSTS];
struct reset_control_bulk_data *xsi_rsts;
struct net_device *napi_dev;
+ int fe_irq;
struct airoha_qdma qdma[AIROHA_MAX_NUM_QDMA];
- struct airoha_gdm_port *ports[AIROHA_MAX_NUM_GDM_PORTS];
-};
+ struct airoha_gdm_port **ports;
-u32 airoha_rr(void __iomem *base, u32 offset);
-void airoha_wr(void __iomem *base, u32 offset, u32 val);
-u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val);
+};
#define airoha_fe_rr(eth, offset) \
airoha_rr((eth)->fe_regs, (offset))
@@ -670,9 +1261,22 @@ u32 airoha_rmw(void __iomem *base, u32 offset, u32
mask, u32 val);
#define airoha_qdma_get(qdma, offset, mask) \
FIELD_GET((mask), airoha_qdma_rr((qdma), (offset)))
+void airoha_qdma_setup(struct airoha_qdma *qdma, struct airoha_eth *eth,
+ void __iomem *regs, u8 id, u8 num_channels);
+int airoha_qdma_init(struct platform_device *pdev, struct airoha_eth *eth,
+ struct airoha_qdma *qdma);
+void airoha_qdma_cleanup(struct airoha_qdma *qdma);
+void airoha_qdma_cleanup_tx_queue(struct airoha_queue *q);
+void airoha_qdma_unmap_tx_entry(struct airoha_eth *eth,
+ struct airoha_queue_entry *e);
+void airoha_qdma_start_napi(struct airoha_qdma *qdma);
+void airoha_qdma_stop_napi(struct airoha_qdma *qdma);
+void airoha_qdma_start(struct airoha_qdma *qdma);
+void airoha_qdma_stop(struct airoha_qdma *qdma);
+
static inline u16 airoha_qdma_get_txq(struct airoha_qdma *qdma, u16 qid)
{
- return qid % ARRAY_SIZE(qdma->q_tx);
+ return qid % qdma->eth->soc->tx_ring;
}
static inline bool airoha_is_lan_gdm_dev(struct airoha_gdm_dev *dev)
@@ -680,28 +1284,94 @@ static inline bool airoha_is_lan_gdm_dev(struct
airoha_gdm_dev *dev)
return !(dev->flags & AIROHA_PRIV_F_WAN);
}
-static inline bool airoha_is_7581(struct airoha_eth *eth)
+#define airoha_is(eth, ...) ({ \
+ const enum airoha_ids _ids[] = {__VA_ARGS__, 0}; \
+ bool _found = false; \
+ for (int _i = 0; _ids[_i] != 0; _i++) { \
+ if (_ids[_i] == (eth)->soc->version) { \
+ _found = true; \
+ break; \
+ } \
+ } \
+ _found; \
+})
+
+static inline bool airoha_is_econet(struct airoha_eth *eth)
{
- return eth->soc->version == 0x7581;
+ return airoha_is(eth, econet_en751221, econet_en7528,
+ econet_en7580);
}
-static inline bool airoha_is_7583(struct airoha_eth *eth)
+static inline bool airoha_has_legacy_qdma(struct airoha_eth *eth)
{
- return eth->soc->version == 0x7583;
+ return eth->soc->legacy_qdma;
}
static inline bool airoha_qdma_is_lro_queue(struct airoha_queue *q)
{
struct airoha_qdma *qdma = q->qdma;
int qid = q - &qdma->q_rx[0];
-
- /* EN7581 SoC supports at most 8 LRO rx queues */
- BUILD_BUG_ON(hweight32(AIROHA_RXQ_LRO_EN_MASK) >
- AIROHA_MAX_NUM_LRO_QUEUES);
-
- return !!(AIROHA_RXQ_LRO_EN_MASK & BIT(qid));
+
+ switch (qdma->eth->soc->version) {
+ case airoha_en7523:
+ /* EN7523 11-14 */
+ BUILD_BUG_ON(hweight32(EN7523_AIROHA_RXQ_LRO_EN_MASK) >
+ EN7523_AIROHA_MAX_NUM_LRO_QUEUES);
+
+ return !!(EN7523_AIROHA_RXQ_LRO_EN_MASK & BIT(qid));
+ case airoha_en7581:
+ case airoha_an7583:
+ /* EN7581 SoC supports at most 8 LRO rx queues (24-31) */
+ BUILD_BUG_ON(hweight32(AIROHA_RXQ_LRO_EN_MASK) >
+ AIROHA_MAX_NUM_LRO_QUEUES);
+
+ return !!(AIROHA_RXQ_LRO_EN_MASK & BIT(qid));
+ default:
+ return false;
+ }
}
+extern const struct airoha_eth_soc_data econet_en751221_soc_data;
+extern const struct airoha_eth_soc_data econet_en7528_soc_data;
+extern const struct airoha_eth_soc_data airoha_en7523_soc_data;
+extern const struct airoha_eth_soc_data airoha_en7581_soc_data;
+extern const struct airoha_eth_soc_data airoha_an7583_soc_data;
+
+struct net_device *airoha_eth_get_xpon_netdev(void);
+int airoha_eth_register_xpon(struct net_device *netdev,
+ enum airoha_xpon_mode mode,
+ const struct airoha_xpon_link_ops *ops,
+ void *priv);
+void airoha_eth_unregister_xpon(struct net_device *netdev,
+ const struct airoha_xpon_link_ops *ops,
+ void *priv);
+void airoha_eth_xpon_update_link(struct net_device *netdev,
+ const struct airoha_xpon_link_state *state);
+int airoha_eth_xpon_control_start(struct net_device *netdev);
+void airoha_eth_xpon_control_stop(struct net_device *netdev);
+void airoha_eth_xpon_dump_oam_rx_state(struct net_device *netdev);
+int airoha_eth_register_xpon_oam(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler);
+void airoha_eth_unregister_xpon_oam(struct net_device *netdev,
+ struct airoha_xpon_oam_handler *handler);
+int airoha_eth_xmit_xpon_oam(struct net_device *netdev, struct sk_buff
*skb,
+ u8 channel, u16 gem_port_id);
+int airoha_eth_xpon_add_service(struct net_device *netdev,
+ const struct airoha_xpon_service_cfg *cfg);
+int airoha_eth_xpon_get_tx_info(struct net_device *netdev, bool vlan_valid,
+ u16 vlan_id, bool pcp_valid, u8 pcp,
+ struct airoha_xpon_tx_info *info);
+bool airoha_eth_xpon_del_service(struct net_device *netdev, u32 cookie,
+ u16 *gem_port_id);
+bool airoha_eth_xpon_has_gem_service(struct net_device *netdev,
+ u16 gem_port_id);
+void airoha_eth_xpon_flush_services(struct net_device *netdev);
+int airoha_eth_set_xpon_mode(struct net_device *netdev,
+ enum airoha_xpon_mode mode);
+int airoha_eth_set_xpon_datapath(struct net_device *netdev,
+ enum airoha_xpon_mode mode, bool enable);
+int airoha_eth_set_xpon_tcont_channel(struct net_device *netdev,
+ unsigned int channel, bool enable);
int airoha_get_fe_port(struct airoha_gdm_dev *dev);
bool airoha_is_valid_gdm_dev(struct airoha_eth *eth,
struct airoha_gdm_dev *dev);
@@ -730,14 +1400,3395 @@ struct airoha_foe_entry
*airoha_ppe_foe_get_entry(struct airoha_ppe *ppe,
u32 hash);
void airoha_ppe_foe_entry_get_stats(struct airoha_ppe *ppe, u32 hash,
struct airoha_foe_stats64 *stats);
+void airoha_ppe_v1_read_entry(struct airoha_ppe *ppe, u16 hash,
+ struct airoha_foe_entry *entry);
-#ifdef CONFIG_DEBUG_FS
-int airoha_ppe_debugfs_init(struct airoha_ppe *ppe);
+#if IS_ENABLED(CONFIG_NET_AIROHA_PPE_DEBUGFS)
+int airoha_ppe_debugfs_init(struct airoha_ppe_common *ppe);
#else
-static inline int airoha_ppe_debugfs_init(struct airoha_ppe *ppe)
+static inline int airoha_ppe_debugfs_init(struct airoha_ppe_common *ppe)
{
return 0;
}
#endif
+
+/* EcoNet/EN751x descriptor and register definitions. */
+struct airoha_eth;
+struct airoha_eth_soc_data;
+
+/* QDMA packet descriptors and descriptor messages. */
+#define FIELD_SET(current, mask, val) \
+ (((current) & ~(mask)) | FIELD_PREP((mask), (val)))
+
+/** etx: */
+struct etx {
+ /**
+ * See accessors:
+ * get_etx_unknown0()
+ * set_etx_unknown0()
+ * get_etx_sp_tag()
+ * set_etx_sp_tag()
+ * is_etx_oam()
+ * set_etx_oam()
+ * get_etx_channel()
+ * set_etx_channel()
+ * get_etx_queue()
+ * set_etx_queue()
+ */
+ u32 bitfield_0;
+
+ /**
+ * See accessors:
+ * is_etx_ico()
+ * set_etx_ico()
+ * is_etx_uco()
+ * set_etx_uco()
+ * is_etx_tco()
+ * set_etx_tco()
+ * is_etx_sco()
+ * set_etx_sco()
+ * get_etx_udf_pmap()
+ * set_etx_udf_pmap()
+ * get_etx_fport()
+ * set_etx_fport()
+ * is_etx_vlan_en()
+ * set_etx_vlan_en()
+ * get_etx_vlan_type()
+ * set_etx_vlan_type()
+ * get_etx_vlan_tag()
+ * set_etx_vlan_tag()
+ */
+ u32 bitfield_1;
+
+};
+
+/**
+ * Bitfield accessors for: etx bitfield_0
+ */
+
+#define ETX_UNKNOWN0_MASK GENMASK(31, 28)
+#define ETX_SP_TAG_MASK GENMASK(27, 12)
+/* EN751221 xPON TX message word 0 overlays the Ethernet special-tag
field. */
+#define ETX_XPON_GEM_MASK GENMASK(23, 12)
+#define ETX_XPON_DEI BIT(24)
+#define ETX_XPON_TSE BIT(25)
+#define ETX_XPON_TSID_MASK GENMASK(30, 26)
+#define ETX_OAM BIT(11)
+#define ETX_CHANNEL_MASK GENMASK(10, 3)
+#define ETX_QUEUE_MASK GENMASK(2, 0)
+
+
+/** Unused, called "rev" probably short for reserved */
+static inline u8 get_etx_unknown0(struct etx *x)
+{
+ return FIELD_GET(ETX_UNKNOWN0_MASK, x->bitfield_0);
+}
+static inline void set_etx_unknown0(struct etx *x, u8 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ETX_UNKNOWN0_MASK, v);
+}
+
+/**
+ * MediaTek "Special Tag" format which encapsulates both switch port
number and
+ * possible VLAN
+ */
+static inline u16 get_etx_sp_tag(struct etx *x)
+{
+ return FIELD_GET(ETX_SP_TAG_MASK, x->bitfield_0);
+}
+static inline void set_etx_sp_tag(struct etx *x, u16 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ETX_SP_TAG_MASK, v);
+}
+
+static inline u16 get_etx_xpon_gem(struct etx *x)
+{
+ return FIELD_GET(ETX_XPON_GEM_MASK, x->bitfield_0);
+}
+
+static inline void set_etx_xpon_gem(struct etx *x, u16 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ETX_XPON_GEM_MASK, v);
+}
+
+/** OAM (management) frame, never used with Ethernet transmissions */
+static inline bool is_etx_oam(struct etx *x)
+{
+ return FIELD_GET(ETX_OAM, x->bitfield_0);
+}
+static inline void set_etx_oam(struct etx *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ETX_OAM, v);
+}
+
+/** The channel number for QoS prioritization */
+static inline u8 get_etx_channel(struct etx *x)
+{
+ return FIELD_GET(ETX_CHANNEL_MASK, x->bitfield_0);
+}
+static inline void set_etx_channel(struct etx *x, u8 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ETX_CHANNEL_MASK, v);
+}
+
+/** The queue number for QoS prioritization */
+static inline u8 get_etx_queue(struct etx *x)
+{
+ return FIELD_GET(ETX_QUEUE_MASK, x->bitfield_0);
+}
+static inline void set_etx_queue(struct etx *x, u8 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ETX_QUEUE_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: etx bitfield_1
+ */
+
+enum etx_fport {
+ ETX_FPORT_QDMA0_CPU = 0,
+ ETX_FPORT_GDM1 = 1,
+ ETX_FPORT_GDM2 = 2,
+ ETX_FPORT_QDMA0_HWF = 3,
+ ETX_FPORT_PPE = 4,
+ ETX_FPORT_QDMA1_CPU = 5,
+ ETX_FPORT_QDMA1_HWF = 6,
+ ETX_FPORT_DROP = 7,
+};
+enum etx_vlan_type {
+ ETX_VLAN_TYPE_8100 = 0,
+ ETX_VLAN_TYPE_9100 = 2,
+ ETX_VLAN_TYPE_88A8 = 1,
+ ETX_VLAN_TYPE_UNKNOWN = 3,
+};
+
+#define ETX_ICO BIT(31)
+#define ETX_UCO BIT(30)
+#define ETX_TCO BIT(29)
+#define ETX_SCO BIT(28)
+#define ETX_UDF_PMAP_MASK GENMASK(27, 22)
+#define ETX_FPORT_MASK GENMASK(21, 19)
+#define ETX_VLAN_EN BIT(18)
+#define ETX_VLAN_TYPE_MASK GENMASK(17, 16)
+#define ETX_VLAN_TAG_MASK GENMASK(15, 0)
+
+
+/** Checksum offload, probably IP */
+static inline bool is_etx_ico(struct etx *x)
+{
+ return FIELD_GET(ETX_ICO, x->bitfield_1);
+}
+static inline void set_etx_ico(struct etx *x, bool v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ETX_ICO, v);
+}
+
+/** Checksum offload, probably UDP */
+static inline bool is_etx_uco(struct etx *x)
+{
+ return FIELD_GET(ETX_UCO, x->bitfield_1);
+}
+static inline void set_etx_uco(struct etx *x, bool v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ETX_UCO, v);
+}
+
+/** Checksum offload, probably TCP */
+static inline bool is_etx_tco(struct etx *x)
+{
+ return FIELD_GET(ETX_TCO, x->bitfield_1);
+}
+static inline void set_etx_tco(struct etx *x, bool v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ETX_TCO, v);
+}
+
+/** Unknown, maybe SCTP checksum offload */
+static inline bool is_etx_sco(struct etx *x)
+{
+ return FIELD_GET(ETX_SCO, x->bitfield_1);
+}
+static inline void set_etx_sco(struct etx *x, bool v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ETX_SCO, v);
+}
+
+/** Unknown / unused */
+static inline u8 get_etx_udf_pmap(struct etx *x)
+{
+ return FIELD_GET(ETX_UDF_PMAP_MASK, x->bitfield_1);
+}
+static inline void set_etx_udf_pmap(struct etx *x, u8 v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ETX_UDF_PMAP_MASK, v);
+}
+
+/**
+ * Where in the Frame Engine to send the packet to QDMA0_CPU /
QDMA1_CPU sends
+ * to CPU via the relevant QDMA engine. QDMA0_HWF and QDMA1_HWF goes to
QDMA
+ * hardware forwarding. GDM1 is the LAN, GDM2 is the WAN, and PPE is
the Packet
+ * Processing Engine.
+ */
+static inline enum etx_fport get_etx_fport(struct etx *x)
+{
+ return FIELD_GET(ETX_FPORT_MASK, x->bitfield_1);
+}
+static inline void set_etx_fport(struct etx *x, enum etx_fport v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ETX_FPORT_MASK, v);
+}
+
+/** If 1 then add a vlan header to the packet */
+static inline bool is_etx_vlan_en(struct etx *x)
+{
+ return FIELD_GET(ETX_VLAN_EN, x->bitfield_1);
+}
+static inline void set_etx_vlan_en(struct etx *x, bool v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ETX_VLAN_EN, v);
+}
+
+/** Which type of vlan to add to the packet header */
+static inline enum etx_vlan_type get_etx_vlan_type(struct etx *x)
+{
+ return FIELD_GET(ETX_VLAN_TYPE_MASK, x->bitfield_1);
+}
+static inline void set_etx_vlan_type(struct etx *x, enum etx_vlan_type v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ETX_VLAN_TYPE_MASK, v);
+}
+
+/** The VLAN number, if vlan_en is set */
+static inline u16 get_etx_vlan_tag(struct etx *x)
+{
+ return FIELD_GET(ETX_VLAN_TAG_MASK, x->bitfield_1);
+}
+static inline void set_etx_vlan_tag(struct etx *x, u16 v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ETX_VLAN_TAG_MASK, v);
+}
+
+/**
+ * qdma_desc_erx
+ *
+ * 3 2 1 0
+ * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * 0 | unknown0 |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * 4 |nknwn|I|P|F|T|L|A| sport | crsn | ppe_entry |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * 8 | unknown2 |N|
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * 12 | sp_tag | tci |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * 16
+ *
+ * @unknown0 (32 bit): Unknown / unused field, first word in descriptor
+ * @bitfield_0 (32 bit):
+ * @unknown1 "nknwn" (bits 31..29): Revision number? (unused)
+ * @ip6 "I" (bit 28): IPv6 packet indicator
+ * @ip4 "P" (bit 27): IPv4 packet indicator
+ * @ip4f "F" (bit 26): IPv4 header checksum failure
+ * @tack "T" (bit 25): TCP ACK flag
+ * @l2vld "L" (bit 24): Layer 2 valid flag
+ * @l4f "A" (bit 23): TCP/UDP checksum failure
+ * @sport (bits 22..19): Where the packet came from, mostly unknown /
unused,
+ * needs testing
+ * @crsn (bits 18..14): Most likely a MediaTek PPE CPU_REASON
+ * @ppe_entry (bits 13..0): PPE (Packet Processing Engine) entry index
+ * @bitfield_1 (32 bit):
+ * @unknown2 (bits 31..1): Reserved
+ * @untag "N" (bit 0): VLAN untag flag
+ * @bitfield_3 (32 bit):
+ * @sp_tag (bits 31..16): MediaTek "Special Tag" for switch port/VLAN
encoding
+ * @tci (bits 15..0): The TCI of any vlan tag that was unpopped
beneath the
+ * MTK "Special Tag"
+ */
+struct qdma_desc_erx {
+ u32 unknown0;
+ u32 bitfield_0;
+ u32 bitfield_1;
+ u32 bitfield_3;
+};
+
+/* qdma_desc_erx bitfield_0 */
+
+/* EN751221 xPON RX message word 0. The remaining words overlap the
+ * regular Ethernet RX metadata and are deliberately kept in qdma_desc_erx.
+ */
+#define ERX_XPON_CRC_ERROR BIT(0)
+#define ERX_XPON_RUNT BIT(1)
+#define ERX_XPON_LONG BIT(2)
+#define ERX_XPON_CHANNEL_MASK GENMASK(10, 3)
+#define ERX_XPON_OAM BIT(11)
+#define ERX_XPON_GEM_MASK GENMASK(23, 12)
+
+#define ERX_UNKNOWN1_MASK GENMASK(31, 29)
+#define ERX_IP6 BIT(28)
+#define ERX_IP4 BIT(27)
+#define ERX_IP4F BIT(26)
+#define ERX_TACK BIT(25)
+#define ERX_L2VLD BIT(24)
+#define ERX_L4F BIT(23)
+#define ERX_SPORT_MASK GENMASK(22, 19)
+#define EN7528_ERX_SPORT_MASK GENMASK(23, 19)
+#define ERX_CRSN_MASK GENMASK(18, 14)
+#define ERX_PPE_ENTRY_MASK GENMASK(13, 0)
+
+static inline u8 get_erx_unknown1(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_UNKNOWN1_MASK, x->bitfield_0);
+}
+static inline void set_erx_unknown1(struct qdma_desc_erx *x, u8 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_UNKNOWN1_MASK, v);
+}
+static inline bool is_erx_ip6(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_IP6, x->bitfield_0);
+}
+static inline void set_erx_ip6(struct qdma_desc_erx *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_IP6, v);
+}
+static inline bool is_erx_ip4(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_IP4, x->bitfield_0);
+}
+static inline void set_erx_ip4(struct qdma_desc_erx *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_IP4, v);
+}
+static inline bool is_erx_ip4f(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_IP4F, x->bitfield_0);
+}
+static inline void set_erx_ip4f(struct qdma_desc_erx *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_IP4F, v);
+}
+static inline bool is_erx_tack(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_TACK, x->bitfield_0);
+}
+static inline void set_erx_tack(struct qdma_desc_erx *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_TACK, v);
+}
+static inline bool is_erx_l2vld(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_L2VLD, x->bitfield_0);
+}
+static inline void set_erx_l2vld(struct qdma_desc_erx *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_L2VLD, v);
+}
+static inline bool is_erx_l4f(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_L4F, x->bitfield_0);
+}
+static inline void set_erx_l4f(struct qdma_desc_erx *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_L4F, v);
+}
+static inline u8 get_erx_sport(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_SPORT_MASK, x->bitfield_0);
+}
+static inline void set_erx_sport(struct qdma_desc_erx *x, u8 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_SPORT_MASK, v);
+}
+static inline u8 get_erx_crsn(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_CRSN_MASK, x->bitfield_0);
+}
+static inline void set_erx_crsn(struct qdma_desc_erx *x, u8 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_CRSN_MASK, v);
+}
+static inline u16 get_erx_ppe_entry(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_PPE_ENTRY_MASK, x->bitfield_0);
+}
+static inline void set_erx_ppe_entry(struct qdma_desc_erx *x, u16 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, ERX_PPE_ENTRY_MASK, v);
+}
+
+/* qdma_desc_erx bitfield_1 */
+
+#define ERX_UNKNOWN2_MASK GENMASK(31, 1)
+#define ERX_UNTAG BIT(0)
+
+static inline u32 get_erx_unknown2(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_UNKNOWN2_MASK, x->bitfield_1);
+}
+static inline void set_erx_unknown2(struct qdma_desc_erx *x, u32 v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ERX_UNKNOWN2_MASK, v);
+}
+static inline bool is_erx_untag(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_UNTAG, x->bitfield_1);
+}
+static inline void set_erx_untag(struct qdma_desc_erx *x, bool v)
+{
+ x->bitfield_1 = FIELD_SET(x->bitfield_1, ERX_UNTAG, v);
+}
+
+/* qdma_desc_erx bitfield_3 */
+
+#define ERX_SP_TAG_MASK GENMASK(31, 16)
+#define ERX_TCI_MASK GENMASK(15, 0)
+
+static inline u16 get_erx_sp_tag(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_SP_TAG_MASK, x->bitfield_3);
+}
+static inline void set_erx_sp_tag(struct qdma_desc_erx *x, u16 v)
+{
+ x->bitfield_3 = FIELD_SET(x->bitfield_3, ERX_SP_TAG_MASK, v);
+}
+static inline u16 get_erx_tci(struct qdma_desc_erx *x)
+{
+ return FIELD_GET(ERX_TCI_MASK, x->bitfield_3);
+}
+static inline void set_erx_tci(struct qdma_desc_erx *x, u16 v)
+{
+ x->bitfield_3 = FIELD_SET(x->bitfield_3, ERX_TCI_MASK, v);
+}
+
+/** desc: QDMA Packet Descriptor */
+struct desc {
+ /**
+ * desc_unknown0: Reserved / unused, we still export the symbol so
we can show
+ * it in debugging
+ */
+ u32 unknown0;
+
+ /** desc_info: */
+ struct desc_info {
+ /**
+ * See accessors:
+ * is_desc_info_done()
+ * set_desc_info_done()
+ * is_desc_info_dropped()
+ * set_desc_info_dropped()
+ * is_desc_info_nls()
+ * set_desc_info_nls()
+ * get_desc_info_unknown1()
+ * set_desc_info_unknown1()
+ * get_desc_info_pkt_len()
+ * set_desc_info_pkt_len()
+ */
+ u32 word;
+
+ } info;
+
+ /** desc_pkt_addr: Physical (DMA) address of the packet */
+ u32 pkt_addr;
+
+ /** desc_next_idx: Index of the next descriptor in the ring, max
4096. */
+ u32 next_idx;
+
+ /** desc_msg: This is either Ethernet RX, Ethernet TX, xPON RX, or
xPON TX */
+ union desc_msg {
+ struct qdma_desc_erx erx;
+
+ struct etx etx;
+
+ u32 raw[4];
+
+ } msg;
+
+};
+
+/**
+ * Bitfield accessors for: desc_info word
+ */
+
+#define DESC_INFO_DONE BIT(31)
+#define DESC_INFO_DROPPED BIT(30)
+#define DESC_INFO_NLS BIT(29)
+#define DESC_INFO_UNKNOWN1_MASK GENMASK(28, 16)
+#define DESC_INFO_PKT_LEN_MASK GENMASK(15, 0)
+
+
+/**
+ * Descriptor Done flag, this roughly means that the DSCP "belongs to the
+ * driver", the hardware will set it when it is done receiving or
sending and
+ * will check to make sure it's not touching a DSCP that is not meant
for it.
+ * This is not strictly necessary, you can determine which packets are
yours
+ * only through ring indexes and the TX Done List, and setting and
checking of
+ * this flag can be deactivated.
+ */
+static inline bool is_desc_info_done(struct desc_info *x)
+{
+ return FIELD_GET(DESC_INFO_DONE, x->word);
+}
+static inline void set_desc_info_done(struct desc_info *x, bool v)
+{
+ x->word = FIELD_SET(x->word, DESC_INFO_DONE, v);
+}
+
+/** Packet has been dropped */
+static inline bool is_desc_info_dropped(struct desc_info *x)
+{
+ return FIELD_GET(DESC_INFO_DROPPED, x->word);
+}
+static inline void set_desc_info_dropped(struct desc_info *x, bool v)
+{
+ x->word = FIELD_SET(x->word, DESC_INFO_DROPPED, v);
+}
+
+/** Unknown meaning but used on EN761627 and EN7580 */
+static inline bool is_desc_info_nls(struct desc_info *x)
+{
+ return FIELD_GET(DESC_INFO_NLS, x->word);
+}
+static inline void set_desc_info_nls(struct desc_info *x, bool v)
+{
+ x->word = FIELD_SET(x->word, DESC_INFO_NLS, v);
+}
+
+/**
+ * Reserved / unused, we still export the symbol so we can show it in
debugging
+ */
+static inline u16 get_desc_info_unknown1(struct desc_info *x)
+{
+ return FIELD_GET(DESC_INFO_UNKNOWN1_MASK, x->word);
+}
+static inline void set_desc_info_unknown1(struct desc_info *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, DESC_INFO_UNKNOWN1_MASK, v);
+}
+
+/** Length of the packet in bytes */
+static inline u16 get_desc_info_pkt_len(struct desc_info *x)
+{
+ return FIELD_GET(DESC_INFO_PKT_LEN_MASK, x->word);
+}
+static inline void set_desc_info_pkt_len(struct desc_info *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, DESC_INFO_PKT_LEN_MASK, v);
+}
+
+/** fwdesc: QDMA Hardware Forward Packet Descriptor */
+struct fwdesc {
+ /** fwdesc_pkt_addr: Physical (DMA) address of the packet */
+ u32 pkt_addr;
+
+ /** fwdesc_info: */
+ struct fwdesc_info {
+ /**
+ * See accessors:
+ * is_fwdesc_info_ctx()
+ * set_fwdesc_info_ctx()
+ * is_fwdesc_info_ctx_ring()
+ * set_fwdesc_info_ctx_ring()
+ * get_fwdesc_info_ctx_idx()
+ * set_fwdesc_info_ctx_idx()
+ * get_fwdesc_info_pkt_len()
+ * set_fwdesc_info_pkt_len()
+ */
+ u32 word;
+
+ } info;
+
+ /**
+ * fwdesc_msg: This is either Ethernet RX, Ethernet TX, xPON RX, or
xPON TX
+ */
+ union fwdesc_msg {
+ struct etx etx;
+
+ u32 raw[2];
+
+ } msg;
+
+};
+
+/**
+ * Bitfield accessors for: fwdesc_info word
+ */
+
+#define FWDESC_INFO_CTX BIT(31)
+#define FWDESC_INFO_CTX_RING BIT(28)
+#define FWDESC_INFO_CTX_IDX_MASK GENMASK(27, 16)
+#define FWDESC_INFO_PKT_LEN_MASK GENMASK(15, 0)
+
+
+/** True if there is a context descriptor (i.e. it is send, not a
forward) */
+static inline bool is_fwdesc_info_ctx(struct fwdesc_info *x)
+{
+ return FIELD_GET(FWDESC_INFO_CTX, x->word);
+}
+static inline void set_fwdesc_info_ctx(struct fwdesc_info *x, bool v)
+{
+ x->word = FIELD_SET(x->word, FWDESC_INFO_CTX, v);
+}
+
+/**
+ * If ctx is true then this is the number of the context ring (0 or 1)
because
+ * there are 2 transmit rings.
+ */
+static inline bool is_fwdesc_info_ctx_ring(struct fwdesc_info *x)
+{
+ return FIELD_GET(FWDESC_INFO_CTX_RING, x->word);
+}
+static inline void set_fwdesc_info_ctx_ring(struct fwdesc_info *x, bool v)
+{
+ x->word = FIELD_SET(x->word, FWDESC_INFO_CTX_RING, v);
+}
+
+/**
+ * If ctx is true then this is the index within the TX ring of the context
+ * packet descriptor.
+ */
+static inline u16 get_fwdesc_info_ctx_idx(struct fwdesc_info *x)
+{
+ return FIELD_GET(FWDESC_INFO_CTX_IDX_MASK, x->word);
+}
+static inline void set_fwdesc_info_ctx_idx(struct fwdesc_info *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, FWDESC_INFO_CTX_IDX_MASK, v);
+}
+
+/** Length of the packet in bytes */
+static inline u16 get_fwdesc_info_pkt_len(struct fwdesc_info *x)
+{
+ return FIELD_GET(FWDESC_INFO_PKT_LEN_MASK, x->word);
+}
+static inline void set_fwdesc_info_pkt_len(struct fwdesc_info *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, FWDESC_INFO_PKT_LEN_MASK, v);
+}
+
+/* QDMA register layout and field accessors. */
+#ifndef FIELD_SET
+#define FIELD_SET(current, mask, val) \
+ (((current) & ~(mask)) | FIELD_PREP((mask), (val)))
+#endif
+
+/** qchain_regs: QDMA Chain Registers */
+struct qchain_regs {
+ /** qchain_regs_txbase: TX descriptor array address */
+ u32 txbase;
+
+ /** qchain_regs_rxbase: RX descriptor array address */
+ u32 rxbase;
+
+ /** qchain_regs_tx_cpui: TX ring CPU (driver) index */
+ u32 tx_cpui;
+
+ /** qchain_regs_tx_hwi: TX ring hardware index */
+ u32 tx_hwi;
+
+ /** qchain_regs_rx_cpui: RX ring CPU (driver) index */
+ u32 rx_cpui;
+
+ /** qchain_regs_rx_hwi: TX ring hardware index */
+ u32 rx_hwi;
+
+};
+
+/** qregs: QDMA Registers */
+struct qregs {
+ /** qregs_version: */
+ u32 version;
+
+ /** qregs_qcfg: */
+ struct qregs_qcfg {
+ /**
+ * See accessors:
+ * is_qregs_qcfg_rx_2b_offset()
+ * set_qregs_qcfg_rx_2b_offset()
+ * get_qregs_qcfg_dma_pref()
+ * set_qregs_qcfg_dma_pref()
+ * is_qregs_qcfg_msg_word_swap()
+ * set_qregs_qcfg_msg_word_swap()
+ * is_qregs_qcfg_dscp_byte_swap()
+ * set_qregs_qcfg_dscp_byte_swap()
+ * is_qregs_qcfg_payload_byte_sw()
+ * set_qregs_qcfg_payload_byte_sw()
+ * is_qregs_qcfg_vchnl_map_en()
+ * set_qregs_qcfg_vchnl_map_en()
+ * is_qregs_qcfg_vchnl_map_mode()
+ * set_qregs_qcfg_vchnl_map_mode()
+ * is_qregs_qcfg_qdma_lpbk_rxq_sel()
+ * set_qregs_qcfg_qdma_lpbk_rxq_sel()
+ * is_qregs_qcfg_slm_release_en()
+ * set_qregs_qcfg_slm_release_en()
+ * is_qregs_qcfg_tx_immediate_done()
+ * set_qregs_qcfg_tx_immediate_done()
+ * is_qregs_qcfg_irq_en()
+ * set_qregs_qcfg_irq_en()
+ * is_qregs_qcfg_gdm_loopback()
+ * set_qregs_qcfg_gdm_loopback()
+ * is_qregs_qcfg_qdma_loopback()
+ * set_qregs_qcfg_qdma_loopback()
+ * is_qregs_qcfg_check_done()
+ * set_qregs_qcfg_check_done()
+ * is_qregs_qcfg_tx_wb_done()
+ * set_qregs_qcfg_tx_wb_done()
+ * get_qregs_qcfg_burst_size()
+ * set_qregs_qcfg_burst_size()
+ * is_qregs_qcfg_rx_dma_busy()
+ * set_qregs_qcfg_rx_dma_busy()
+ * is_qregs_qcfg_rx_dma_en()
+ * set_qregs_qcfg_rx_dma_en()
+ * is_qregs_qcfg_tx_dma_busy()
+ * set_qregs_qcfg_tx_dma_busy()
+ * is_qregs_qcfg_tx_dma_en()
+ * set_qregs_qcfg_tx_dma_en()
+ */
+ u32 bitfield_0;
+
+ } qdma_cfg;
+
+ /** qregs_qchain0: */
+ struct qchain_regs qchain0;
+
+ /**
+ * qregs_hwf_desc_addr: Hardware forwarding descriptor table
address. This
+ * memory must be forward-descriptor-size (16 bytes) * fwd_desc_n
in size.
+ */
+ u32 hwf_desc_addr;
+
+ /**
+ * qregs_hwf_data_addr: Hardware forwarding packet content address.
This memory
+ * must be pkt_sz * fwd_desc_n in size.
+ */
+ u32 hwf_data_addr;
+
+ /**
+ * See accessors:
+ * get_qregs_hwf_cfg_pkt_sz()
+ * set_qregs_hwf_cfg_pkt_sz()
+ * get_qregs_hwf_cfg_low_th()
+ * set_qregs_hwf_cfg_low_th()
+ */
+ struct hwf_cfg { u32 word; } hwf_cfg;
+
+ u32 unused_0;
+
+ /**
+ * qregs_hwf_cfg1: Hardware forwarding configuration (also called LMGR)
+ * See accessors:
+ * is_qregs_hwf_cfg1_start()
+ * set_qregs_hwf_cfg1_start()
+ * is_qregs_hwf_cfg1_overhead_en()
+ * set_qregs_hwf_cfg1_overhead_en()
+ * get_qregs_hwf_cfg1_overhead()
+ * set_qregs_hwf_cfg1_overhead()
+ * get_qregs_hwf_cfg1_fwd_desc_n()
+ * set_qregs_hwf_cfg1_fwd_desc_n()
+ */
+ struct qregs_hwf_cfg1 { u32 word; } hwf_cfg1;
+
+ u8 unused_1[12];
+
+ /** qregs_channel_retire: Referred to as QDMA_CSR_LMGR_CHNL_RETIRE */
+ u32 channel_retire;
+
+ u8 unused_2[12];
+
+ /**
+ * qregs_int_status: When an interrupt is triggered, these are the
pending
+ * events
+ */
+ u32 int_status;
+
+ /** qregs_int_enable: Enabled interrupts */
+ u32 int_enable;
+
+ /** qregs_tx_int_delay: Interrupt delay for reducing interrupt load */
+ u32 tx_int_delay;
+
+ /** qregs_rx_int_delay: Interrupt delay for reducing interrupt load */
+ u32 rx_int_delay;
+
+ /** qregs_doneq: */
+ struct qregs_doneq {
+ /** qregs_doneq_addr: */
+ u32 address;
+
+ /**
+ * qregs_doneq_cfg:
+ * See accessors:
+ * get_qregs_doneq_cfg_int_threshold()
+ * set_qregs_doneq_cfg_int_threshold()
+ * get_qregs_doneq_cfg_size()
+ * set_qregs_doneq_cfg_size()
+ */
+ struct qregs_doneq_cfg { u32 word; } config;
+
+ /**
+ * qregs_doneq_pop_back: Pop this number of items from the back
of the the
+ * done queue, max 255
+ */
+ u32 pop_back;
+
+ /**
+ * qregs_doneq_state:
+ * See accessors:
+ * get_qregs_doneq_state_length()
+ * get_qregs_doneq_state_head_index()
+ */
+ struct qregs_doneq_state { u32 word; } state;
+
+ /**
+ * qregs_doneq_wait_time: If there is anything in the queue,
fire an interrupt
+ * after this number of units of time, unit is 20 microseconds.
+ */
+ u32 wait_time;
+
+ } done_queue;
+
+ u8 unused_3[12];
+
+ /**
+ * See accessors:
+ * is_qregs_wrr_mode_use_16b()
+ * set_qregs_wrr_mode_use_16b()
+ * is_qregs_wrr_mode_by_byte()
+ * set_qregs_wrr_mode_by_byte()
+ */
+ struct wrr_mode { u32 word; } wrr_mode;
+
+ u32 unused_4;
+
+ /**
+ * qregs_wrr_weight: Read and write a WRR weight for a
channel+queue pair.
+ * Called QDMA_CSR_TXWRR_WEIGHT_CFG.
+ */
+ struct qregs_wrr_weight {
+ /**
+ * See accessors:
+ * set_qregs_wrr_weight_write()
+ * is_qregs_wrr_weight_done()
+ * set_qregs_wrr_weight_channel()
+ * set_qregs_wrr_weight_queue()
+ */
+ u16 bitfield_0;
+
+ u8 unused_0;
+
+ /**
+ * qregs_wrr_weight_value: The WRR weight to read or write for
the configured
+ * channel+queue.
+ */
+ u8 value;
+
+ } wrr_weight;
+
+ u32 unused_5;
+
+ /**
+ * qregs_buf_usage_cfg: Called QDMA_CSR_PSE_BUF_USAGE_CFG, a
bitfield, TODO
+ * document
+ */
+ u32 buf_usage_cfg;
+
+ /**
+ * qregs_tx_meter_cfg: Called QDMA_CSR_EGRESS_RATEMETER_CFG, a
bitfield, TODO
+ * document
+ */
+ u32 tx_meter_cfg;
+
+ /**
+ * qregs_tx_limit_cfg: Called QDMA_CSR_EGRESS_RATELIMIT_CFG, a
bitfield, TODO
+ * document
+ */
+ u32 tx_limit_cfg;
+
+ /**
+ * qregs_tx_limit_param: Called QDMA_CSR_RATELIMIT_PARAMETER_CFG, a
bitfield,
+ * per-channel tx rate limit, TODO document
+ */
+ u32 tx_limit_param;
+
+ /** qregs_tx_congest_cfg: */
+ struct qregs_tx_congest_cfg {
+ /**
+ * See accessors:
+ * is_qregs_tx_congest_cfg_tail_drop_en()
+ * set_qregs_tx_congest_cfg_tail_drop_en()
+ * is_qregs_tx_congest_cfg_dei_drop_en()
+ * set_qregs_tx_congest_cfg_dei_drop_en()
+ * is_qregs_tx_congest_cfg_dyncong_en()
+ * set_qregs_tx_congest_cfg_dyncong_en()
+ * is_qregs_tx_congest_cfg_max_thr_blk_tx1()
+ * set_qregs_tx_congest_cfg_max_thr_blk_tx1()
+ * is_qregs_tx_congest_cfg_min_thr_blk_tx1()
+ * set_qregs_tx_congest_cfg_min_thr_blk_tx1()
+ * is_qregs_tx_congest_cfg_max_thr_blk_tx0()
+ * set_qregs_tx_congest_cfg_max_thr_blk_tx0()
+ * is_qregs_tx_congest_cfg_min_thr_blk_tx0()
+ * set_qregs_tx_congest_cfg_min_thr_blk_tx0()
+ * get_qregs_tx_congest_cfg_dyncong_margin()
+ * set_qregs_tx_congest_cfg_dyncong_margin()
+ * get_qregs_tx_congest_cfg_dyncong_dei_scale()
+ * set_qregs_tx_congest_cfg_dyncong_dei_scale()
+ * is_qregs_tx_congest_cfg_dyncong_upd_wrr()
+ * set_qregs_tx_congest_cfg_dyncong_upd_wrr()
+ * is_qregs_tx_congest_cfg_dyncong_upd_txrx()
+ * set_qregs_tx_congest_cfg_dyncong_upd_txrx()
+ * is_qregs_tx_congest_cfg_dyncong_upd_tick()
+ * set_qregs_tx_congest_cfg_dyncong_upd_tick()
+ */
+ u16 bitfield_0;
+
+ /**
+ * qregs_tx_congest_cfg_dyncong_tick: Dynamic congestion ticker
rate in
+ * microseconds
+ */
+ u16 dyncong_tick;
+
+ } tx_congest_cfg;
+
+ /** qregs_tx_congest_thr: */
+ struct qregs_tx_congest_thr {
+ /**
+ * qregs_tx_congest_thr_max: When total buffer usage exceeds
this, drop all
+ * packets except VIP
+ */
+ u16 max;
+
+ /**
+ * qregs_tx_congest_thr_min: When total buffer usage exceeds
this, dynamic
+ * congestion control will start
+ */
+ u16 min;
+
+ } tx_congest_thr;
+
+ /**
+ * qregs_tx_per_ch_dthr: Called QDMA_CSR_TXQ_DYN_CHNLTHR_CFG, tx
per-channel
+ * dynamic threshold max/min threshold, TODO document
+ */
+ u32 tx_per_ch_dthr;
+
+ /**
+ * qregs_tx_per_q_dthr: Called QDMA_CSR_TXQ_DYN_QUEUETHR_CFG, tx
per-queue
+ * dynamic threshold max/min threshold, TODO document
+ */
+ u32 tx_per_q_dthr;
+
+ /**
+ * qregs_tx_per_q_sthr: Called QDMA_CSR_STATIC_QUEUE_THR(0..7), tx
per-queue
+ * static thresholds, if dynamic thresholds are disabled, TODO document
+ */
+ u8 unused_6[32];
+
+ u8 unused_7[16];
+
+ /** qregs_debug: */
+ struct qregs_debug {
+ /**
+ * See accessors:
+ * set_qregs_debug_mem_ctl_write()
+ * is_qregs_debug_mem_ctl_done()
+ * set_qregs_debug_mem_ctl_dataset()
+ * set_qregs_debug_mem_ctl_word_of_elem()
+ * set_qregs_debug_mem_ctl_elem()
+ */
+ struct mem_ctl { u32 word; } mem_ctl;
+
+ /**
+ * qregs_debug_mem_lo: Read or write the low bits of a memory
element (use
+ * with mem_ctl) Called QDMA_CSR_DBG_MEM_XS_DATA_LO.
+ */
+ u32 mem_lo;
+
+ /**
+ * qregs_debug_mem_hi: Read or write the high bits of a memory
element (use
+ * with mem_ctl) Called QDMA_CSR_DBG_MEM_XS_DATA_HI. Not used
in practice.
+ */
+ u32 mem_hi;
+
+ u32 unused_0;
+
+ /**
+ * qregs_debug_hwf_desc_free: Called QDMA_CSR_DBG_LMGR_STATUS,
number of free
+ * hardware forwarding descriptors
+ */
+ u32 hwf_desc_free;
+
+ /**
+ * qregs_debug_hwd_buf_used: Number of bytes of buffer used for
hardwre
+ * forwarding
+ */
+ u32 hwd_buf_used;
+
+ /**
+ * qregs_debug_probe_lo: Called QDMA_CSR_DBG_QDMA_PROBE_LO,
unknown usage,
+ * TODO document
+ */
+ u32 probe_lo;
+
+ /**
+ * qregs_debug_probe_hi: Called QDMA_CSR_DBG_QDMA_PROBE_HI,
unknown usage,
+ * TODO document
+ */
+ u32 probe_hi;
+
+ } debug;
+
+ /**
+ * qregs_rxring_size:
+ * See accessors:
+ * get_qregs_rxring_size_ring0()
+ * set_qregs_rxring_size_ring0()
+ * get_qregs_rxring_size_ring1()
+ * set_qregs_rxring_size_ring1()
+ */
+ struct qregs_rxring_size { u32 word; } rxring_size;
+
+ /**
+ * qregs_rxring_low:
+ * See accessors:
+ * get_qregs_rxring_low_ring0()
+ * set_qregs_rxring_low_ring0()
+ * get_qregs_rxring_low_ring1()
+ * set_qregs_rxring_low_ring1()
+ */
+ struct qregs_rxring_low { u32 word; } rxring_low;
+
+ /** qregs_qchain1: */
+ struct qchain_regs qchain1;
+
+ /** qregs_cpu_rx_limit: CPU protection RX limit, TODO document */
+ u32 cpu_rx_limit;
+
+ /** qregs_cpu_rx_limit_val: CPU protection RX limit, TODO document */
+ u32 cpu_rx_limit_val;
+
+ u8 unused_8[20];
+
+ /** qregs_vch_wrr: Virtual channel WRR weighting, TODO document */
+ u32 vch_wrr;
+
+ /** qregs_vch_qmode: Virtual channel QoS mode (WRR / SP), TODO
document */
+ u32 vch_qmode;
+
+ u8 unused_9[28];
+
+ /**
+ * qregs_ch_lim_en: Per-channel rate-limit enable, each bit
corriponds to one
+ * channel
+ */
+ u32 ch_lim_en;
+
+ u8 unused_10[28];
+
+ /**
+ * qregs_ch_qmode: Per-channel queue prioritization mode (WRR /
SP), TODO
+ * document
+ */
+ u8 unused_11[16];
+
+ u8 unused_12[112];
+
+ /**
+ * qregs_ch_tx_rate: Channel data rate, each word corrisponds to 2
channels,
+ * upper 16 bits is the odd channel number, lower 16 is the even.
+ */
+ u8 unused_13[64];
+
+ u8 unused_14[64];
+
+ /**
+ * qregs_ch_drop: Drop counter for normal packets, each byte
corrisponds to one
+ * of the 32 channels.
+ */
+ u8 unused_15[32];
+
+ u8 unused_16[32];
+
+ /**
+ * qregs_ch_dei_drop: Drop counter for Drop-Elligable (DEI)
packets, each byte
+ * corrisponds to one of the 32 channels.
+ */
+ u8 unused_17[32];
+
+ u8 unused_18[32];
+
+ /**
+ * qregs_pkt_ctrs: Every other word is a counter config and a
counter value,
+ * called QDMA_CSR_DBG_CNTR_CFG / QDMA_CSR_DBG_CNTR_VAR. Definitely 40
+ * counters, possible 64. TODO document
+ */
+ u8 unused_19[512];
+
+ u8 unused_20[2816];
+
+};
+
+/**
+ * Bitfield accessors for: qregs_qcfg bitfield_0
+ */
+
+enum qregs_qcfg_dma_pref {
+ QREGS_QCFG_DMA_PREF_ROUND_ROBIN = 0,
+ QREGS_QCFG_DMA_PREF_FRX_TX1_TX0 = 1,
+ QREGS_QCFG_DMA_PREF_TX1_FRX_TX0 = 2,
+ QREGS_QCFG_DMA_PREF_TX1_TX0_FRX = 3,
+};
+enum qregs_qcfg_burst_size {
+ QREGS_QCFG_BURST_SIZE_16_BYTES = 0,
+ QREGS_QCFG_BURST_SIZE_32_BYTES = 1,
+ QREGS_QCFG_BURST_SIZE_64_BYTES = 2,
+ QREGS_QCFG_BURST_SIZE_128_BYTES = 3,
+};
+
+#define QREGS_QCFG_RX_2B_OFFSET BIT(31)
+#define QREGS_QCFG_DMA_PREF_MASK GENMASK(30, 29)
+#define QREGS_QCFG_MSG_WORD_SWAP BIT(28)
+#define QREGS_QCFG_DSCP_BYTE_SWAP BIT(27)
+#define QREGS_QCFG_PAYLOAD_BYTE_SW BIT(26)
+#define QREGS_QCFG_VCHNL_MAP_EN BIT(25)
+#define QREGS_QCFG_VCHNL_MAP_MODE BIT(24)
+#define QREGS_QCFG_QDMA_LPBK_RXQ_SEL BIT(22)
+#define QREGS_QCFG_SLM_RELEASE_EN BIT(21)
+#define QREGS_QCFG_TX_IMMEDIATE_DONE BIT(20)
+#define QREGS_QCFG_IRQ_EN BIT(19)
+#define QREGS_QCFG_GDM_LOOPBACK BIT(17)
+#define QREGS_QCFG_QDMA_LOOPBACK BIT(16)
+#define QREGS_QCFG_CHECK_DONE BIT(7)
+#define QREGS_QCFG_TX_WB_DONE BIT(6)
+#define QREGS_QCFG_BURST_SIZE_MASK GENMASK(5, 4)
+#define QREGS_QCFG_RX_DMA_BUSY BIT(3)
+#define QREGS_QCFG_RX_DMA_EN BIT(2)
+#define QREGS_QCFG_TX_DMA_BUSY BIT(1)
+#define QREGS_QCFG_TX_DMA_EN BIT(0)
+
+
+/** If enabled, use (dscp_pkt_ptr + 2) as starting address for rx
payload */
+static inline bool is_qregs_qcfg_rx_2b_offset(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_RX_2B_OFFSET, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_rx_2b_offset(struct qregs_qcfg *x,
bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_RX_2B_OFFSET, v);
+}
+
+/** DMA channel scheduling preference, FRX means "Forwarding and RX" */
+static inline enum qregs_qcfg_dma_pref get_qregs_qcfg_dma_pref(struct
qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_DMA_PREF_MASK, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_dma_pref(struct qregs_qcfg *x, enum
qregs_qcfg_dma_pref v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_DMA_PREF_MASK, v);
+}
+
+/**
+ * Enable message word swap, don't know what this does but every
implementation
+ * sets it on Big Endian.
+ */
+static inline bool is_qregs_qcfg_msg_word_swap(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_MSG_WORD_SWAP, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_msg_word_swap(struct qregs_qcfg *x,
bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_MSG_WORD_SWAP, v);
+}
+
+/**
+ * Endian-swap packet descriptors (?), drivers always set this on Big
Endian
+ * machines.
+ */
+static inline bool is_qregs_qcfg_dscp_byte_swap(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_DSCP_BYTE_SWAP, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_dscp_byte_swap(struct qregs_qcfg *x,
bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_DSCP_BYTE_SWAP, v);
+}
+
+/**
+ * Endian-swap payload bytes, drivers always set this on Big Endian
machines.
+ */
+static inline bool is_qregs_qcfg_payload_byte_sw(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_PAYLOAD_BYTE_SW, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_payload_byte_sw(struct qregs_qcfg *x,
bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_QCFG_PAYLOAD_BYTE_SW, v);
+}
+
+/** Enable virtual mapping to group queues per physical channel */
+static inline bool is_qregs_qcfg_vchnl_map_en(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_VCHNL_MAP_EN, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_vchnl_map_en(struct qregs_qcfg *x,
bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_VCHNL_MAP_EN, v);
+}
+
+/** Map of 4 virtual channels per physical channel, 0 = map 2 */
+static inline bool is_qregs_qcfg_vchnl_map_mode(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_VCHNL_MAP_MODE, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_vchnl_map_mode(struct qregs_qcfg *x,
bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_VCHNL_MAP_MODE, v);
+}
+
+/**
+ * If enabled, qdma loopback goes to queue 1, otherwise it goes to
queue zero
+ */
+static inline bool is_qregs_qcfg_qdma_lpbk_rxq_sel(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_QDMA_LPBK_RXQ_SEL, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_qdma_lpbk_rxq_sel(struct qregs_qcfg
*x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_QCFG_QDMA_LPBK_RXQ_SEL, v);
+}
+
+/** Enable qdma fwd path release slm_block */
+static inline bool is_qregs_qcfg_slm_release_en(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_SLM_RELEASE_EN, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_slm_release_en(struct qregs_qcfg *x,
bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_SLM_RELEASE_EN, v);
+}
+
+/** QDMA generate pkt_done itself instead of using pse pkt_done */
+static inline bool is_qregs_qcfg_tx_immediate_done(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_TX_IMMEDIATE_DONE, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_tx_immediate_done(struct qregs_qcfg
*x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_QCFG_TX_IMMEDIATE_DONE, v);
+}
+
+/** Enable "interrupt queue" (i.e. Done List) for tx dma done */
+static inline bool is_qregs_qcfg_irq_en(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_IRQ_EN, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_irq_en(struct qregs_qcfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_IRQ_EN, v);
+}
+
+/** Enable gdm loopback tx packet to rx path */
+static inline bool is_qregs_qcfg_gdm_loopback(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_GDM_LOOPBACK, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_gdm_loopback(struct qregs_qcfg *x,
bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_GDM_LOOPBACK, v);
+}
+
+/** Enable hw qdma loopback tx packet to rx path */
+static inline bool is_qregs_qcfg_qdma_loopback(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_QDMA_LOOPBACK, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_qdma_loopback(struct qregs_qcfg *x,
bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_QDMA_LOOPBACK, v);
+}
+
+/**
+ * Check the done bit of descriptor and don't use descriptors which are
marked
+ * done. If disabled, the QDMA engine will determine if a descriptor is
usable
+ * based only on the ring pointers.
+ */
+static inline bool is_qregs_qcfg_check_done(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_CHECK_DONE, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_check_done(struct qregs_qcfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_CHECK_DONE, v);
+}
+
+/**
+ * Set the "done" bit in tx descriptor after sending. If disabled then the
+ * engine will skip setting the done bit and rely on the driver to
check the
+ * Done List (i.e. `irq_en`).
+ */
+static inline bool is_qregs_qcfg_tx_wb_done(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_TX_WB_DONE, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_tx_wb_done(struct qregs_qcfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_TX_WB_DONE, v);
+}
+
+/** Number of bytes per DMA burst */
+static inline enum qregs_qcfg_burst_size
get_qregs_qcfg_burst_size(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_BURST_SIZE_MASK, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_burst_size(struct qregs_qcfg *x, enum
qregs_qcfg_burst_size v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_QCFG_BURST_SIZE_MASK, v);
+}
+
+/** RX DMA engine currently busy */
+static inline bool is_qregs_qcfg_rx_dma_busy(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_RX_DMA_BUSY, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_rx_dma_busy(struct qregs_qcfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_RX_DMA_BUSY, v);
+}
+
+/** Enable RX DMA */
+static inline bool is_qregs_qcfg_rx_dma_en(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_RX_DMA_EN, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_rx_dma_en(struct qregs_qcfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_RX_DMA_EN, v);
+}
+
+/** TX DMA engine currently busy */
+static inline bool is_qregs_qcfg_tx_dma_busy(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_TX_DMA_BUSY, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_tx_dma_busy(struct qregs_qcfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_TX_DMA_BUSY, v);
+}
+
+/** Enable TX DMA */
+static inline bool is_qregs_qcfg_tx_dma_en(struct qregs_qcfg *x)
+{
+ return FIELD_GET(QREGS_QCFG_TX_DMA_EN, x->bitfield_0);
+}
+static inline void set_qregs_qcfg_tx_dma_en(struct qregs_qcfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_QCFG_TX_DMA_EN, v);
+}
+
+/**
+ * Bitfield accessors for: struct hwf_cfg
+ * Hardware forwarding configuration
+ */
+
+enum qregs_hwf_cfg_pkt_sz {
+ QREGS_HWF_CFG_PKT_SZ_2048 = 0,
+ QREGS_HWF_CFG_PKT_SZ_4096 = 1,
+ QREGS_HWF_CFG_PKT_SZ_8192 = 2,
+ QREGS_HWF_CFG_PKT_SZ_16384 = 3,
+};
+
+#define QREGS_HWF_CFG_PKT_SZ_MASK GENMASK(29, 28)
+#define QREGS_HWF_CFG_LOW_TH_MASK GENMASK(12, 0)
+
+
+/**
+ * The size of the packet buffers in hwf_data_addr (and therefore the
maximum
+ * effective MTU).
+ */
+static inline enum qregs_hwf_cfg_pkt_sz get_qregs_hwf_cfg_pkt_sz(struct
hwf_cfg *x)
+{
+ return FIELD_GET(QREGS_HWF_CFG_PKT_SZ_MASK, x->word);
+}
+static inline void set_qregs_hwf_cfg_pkt_sz(struct hwf_cfg *x, enum
qregs_hwf_cfg_pkt_sz v)
+{
+ x->word = FIELD_SET(x->word, QREGS_HWF_CFG_PKT_SZ_MASK, v);
+}
+
+/**
+ * When number of available (not busy) hardware descriptors is below this,
+ * generate an interrupt and pause hardware forwarding.
+ */
+static inline u16 get_qregs_hwf_cfg_low_th(struct hwf_cfg *x)
+{
+ return FIELD_GET(QREGS_HWF_CFG_LOW_TH_MASK, x->word);
+}
+static inline void set_qregs_hwf_cfg_low_th(struct hwf_cfg *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_HWF_CFG_LOW_TH_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: qregs_hwf_cfg1
+ * Register layout (32-bit word):
+ * - Bit 31: START
+ * - Bit 24: OVERHEAD_EN
+ * - Bits 23-16: OVERHEAD (8 bits)
+ * - Bits 15-0: FWD_DESC_N (16 bits)
+ */
+
+#define QREGS_HWF_CFG1_START BIT(31)
+#define QREGS_HWF_CFG1_OVERHEAD_EN BIT(24)
+#define QREGS_HWF_CFG1_OVERHEAD_MASK GENMASK(23, 16)
+#define QREGS_HWF_CFG1_FWD_DESC_N_MASK GENMASK(15, 0)
+
+
+/** Start up the hardware forwarding subsystem */
+static inline bool is_qregs_hwf_cfg1_start(struct qregs_hwf_cfg1 *x)
+{
+ return FIELD_GET(QREGS_HWF_CFG1_START, x->word);
+}
+static inline void set_qregs_hwf_cfg1_start(struct qregs_hwf_cfg1 *x,
bool v)
+{
+ x->word = FIELD_SET(x->word, QREGS_HWF_CFG1_START, v);
+}
+
+/** When set, add overhead to packet size for accounting purposes */
+static inline bool is_qregs_hwf_cfg1_overhead_en(struct qregs_hwf_cfg1 *x)
+{
+ return FIELD_GET(QREGS_HWF_CFG1_OVERHEAD_EN, x->word);
+}
+static inline void set_qregs_hwf_cfg1_overhead_en(struct qregs_hwf_cfg1
*x, bool v)
+{
+ x->word = FIELD_SET(x->word, QREGS_HWF_CFG1_OVERHEAD_EN, v);
+}
+
+/** Amount of overhead to add to packet size for accounting */
+static inline u8 get_qregs_hwf_cfg1_overhead(struct qregs_hwf_cfg1 *x)
+{
+ return FIELD_GET(QREGS_HWF_CFG1_OVERHEAD_MASK, x->word);
+}
+static inline void set_qregs_hwf_cfg1_overhead(struct qregs_hwf_cfg1
*x, u8 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_HWF_CFG1_OVERHEAD_MASK, v);
+}
+
+/** Number of forward descriptors to use */
+static inline u16 get_qregs_hwf_cfg1_fwd_desc_n(struct qregs_hwf_cfg1 *x)
+{
+ return FIELD_GET(QREGS_HWF_CFG1_FWD_DESC_N_MASK, x->word);
+}
+static inline void set_qregs_hwf_cfg1_fwd_desc_n(struct qregs_hwf_cfg1
*x, u16 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_HWF_CFG1_FWD_DESC_N_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: struct wrr_mode
+ * WRR control register, called QDMA_CSR_TXWRR_MODE_CFG.
+ */
+
+#define QREGS_WRR_MODE_USE_16B BIT(31)
+#define QREGS_WRR_MODE_BY_BYTE BIT(3)
+
+
+/** If enabled, weighting is based on 16 byte units, otherwise 64 byte. */
+static inline bool is_qregs_wrr_mode_use_16b(struct wrr_mode *x)
+{
+ return FIELD_GET(QREGS_WRR_MODE_USE_16B, x->word);
+}
+static inline void set_qregs_wrr_mode_use_16b(struct wrr_mode *x, bool v)
+{
+ x->word = FIELD_SET(x->word, QREGS_WRR_MODE_USE_16B, v);
+}
+
+/** If enabled, weighting is by byte, otherwise by packet. */
+static inline bool is_qregs_wrr_mode_by_byte(struct wrr_mode *x)
+{
+ return FIELD_GET(QREGS_WRR_MODE_BY_BYTE, x->word);
+}
+static inline void set_qregs_wrr_mode_by_byte(struct wrr_mode *x, bool v)
+{
+ x->word = FIELD_SET(x->word, QREGS_WRR_MODE_BY_BYTE, v);
+}
+
+/**
+ * Bitfield accessors for: qregs_wrr_weight bitfield_0
+ */
+
+#define QREGS_WRR_WEIGHT_WRITE BIT(15)
+#define QREGS_WRR_WEIGHT_DONE BIT(14)
+#define QREGS_WRR_WEIGHT_CHANNEL_MASK GENMASK(7, 3)
+#define QREGS_WRR_WEIGHT_QUEUE_MASK GENMASK(2, 0)
+
+static inline void set_qregs_wrr_weight_write(struct qregs_wrr_weight
*x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, QREGS_WRR_WEIGHT_WRITE, v);
+}
+
+/**
+ * After performing an operation, this bit asserts from the hardware to
indicate
+ * the operation is done.
+ */
+static inline bool is_qregs_wrr_weight_done(struct qregs_wrr_weight *x)
+{
+ return FIELD_GET(QREGS_WRR_WEIGHT_DONE, x->bitfield_0);
+}
+static inline void set_qregs_wrr_weight_channel(struct qregs_wrr_weight
*x, u8 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_WRR_WEIGHT_CHANNEL_MASK, v);
+}
+static inline void set_qregs_wrr_weight_queue(struct qregs_wrr_weight
*x, u8 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_WRR_WEIGHT_QUEUE_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: qregs_tx_congest_cfg bitfield_0
+ * Configuration for TX congestion dropping
+ */
+
+enum qregs_tx_congest_cfg_dyncong_margin {
+ QREGS_TX_CONGEST_CFG_DYNCONG_MARGIN_0_PCT = 0,
+ QREGS_TX_CONGEST_CFG_DYNCONG_MARGIN_25_PCT = 1,
+ QREGS_TX_CONGEST_CFG_DYNCONG_MARGIN_50_PCT = 2,
+ QREGS_TX_CONGEST_CFG_DYNCONG_MARGIN_100_PCT = 3,
+};
+enum qregs_tx_congest_cfg_dyncong_dei_scale {
+ QREGS_TX_CONGEST_CFG_DYNCONG_DEI_SCALE_HALF = 0,
+ QREGS_TX_CONGEST_CFG_DYNCONG_DEI_SCALE_QUARTER = 1,
+ QREGS_TX_CONGEST_CFG_DYNCONG_DEI_SCALE_EIGHTH = 2,
+ QREGS_TX_CONGEST_CFG_DYNCONG_DEI_SCALE_SIXTEENTH = 3,
+};
+
+#define QREGS_TX_CONGEST_CFG_TAIL_DROP_EN BIT(15)
+#define QREGS_TX_CONGEST_CFG_DEI_DROP_EN BIT(14)
+#define QREGS_TX_CONGEST_CFG_DYNCONG_EN BIT(13)
+#define QREGS_TX_CONGEST_CFG_MAX_THR_BLK_TX1 BIT(11)
+#define QREGS_TX_CONGEST_CFG_MIN_THR_BLK_TX1 BIT(10)
+#define QREGS_TX_CONGEST_CFG_MAX_THR_BLK_TX0 BIT(9)
+#define QREGS_TX_CONGEST_CFG_MIN_THR_BLK_TX0 BIT(8)
+#define QREGS_TX_CONGEST_CFG_DYNCONG_MARGIN_MASK GENMASK(7, 6)
+#define QREGS_TX_CONGEST_CFG_DYNCONG_DEI_SCALE_MASK GENMASK(5, 4)
+#define QREGS_TX_CONGEST_CFG_DYNCONG_UPD_WRR BIT(2)
+#define QREGS_TX_CONGEST_CFG_DYNCONG_UPD_TXRX BIT(1)
+#define QREGS_TX_CONGEST_CFG_DYNCONG_UPD_TICK BIT(0)
+
+static inline bool is_qregs_tx_congest_cfg_tail_drop_en(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_TAIL_DROP_EN, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_tail_drop_en(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_TAIL_DROP_EN, v);
+}
+
+/** Support 802.1ad DEI packet dropping */
+static inline bool is_qregs_tx_congest_cfg_dei_drop_en(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_DEI_DROP_EN, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_dei_drop_en(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_DEI_DROP_EN, v);
+}
+
+/** Enable dynamic congestion algorithm */
+static inline bool is_qregs_tx_congest_cfg_dyncong_en(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_DYNCONG_EN, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_dyncong_en(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_DYNCONG_EN, v);
+}
+
+/**
+ * Block TX Ring1 when TX buffer usage exceeds max threshold (see
+ * tx_congest_thr)
+ */
+static inline bool is_qregs_tx_congest_cfg_max_thr_blk_tx1(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_MAX_THR_BLK_TX1, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_max_thr_blk_tx1(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_MAX_THR_BLK_TX1, v);
+}
+
+/**
+ * Block TX Ring1 when TX buffer usage exceeds min threshold (see
+ * tx_congest_thr)
+ */
+static inline bool is_qregs_tx_congest_cfg_min_thr_blk_tx1(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_MIN_THR_BLK_TX1, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_min_thr_blk_tx1(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_MIN_THR_BLK_TX1, v);
+}
+
+/**
+ * Block TX Ring0 when TX buffer usage exceeds max threshold (see
+ * tx_congest_thr)
+ */
+static inline bool is_qregs_tx_congest_cfg_max_thr_blk_tx0(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_MAX_THR_BLK_TX0, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_max_thr_blk_tx0(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_MAX_THR_BLK_TX0, v);
+}
+
+/**
+ * Block TX Ring0 when TX buffer usage exceeds min threshold (see
+ * tx_congest_thr)
+ */
+static inline bool is_qregs_tx_congest_cfg_min_thr_blk_tx0(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_MIN_THR_BLK_TX0, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_min_thr_blk_tx0(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_MIN_THR_BLK_TX0, v);
+}
+static inline enum qregs_tx_congest_cfg_dyncong_margin
get_qregs_tx_congest_cfg_dyncong_margin(struct qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_DYNCONG_MARGIN_MASK,
x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_dyncong_margin(struct
qregs_tx_congest_cfg *x, enum qregs_tx_congest_cfg_dyncong_margin v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_DYNCONG_MARGIN_MASK, v);
+}
+static inline enum qregs_tx_congest_cfg_dyncong_dei_scale
get_qregs_tx_congest_cfg_dyncong_dei_scale(struct qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_DYNCONG_DEI_SCALE_MASK,
x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_dyncong_dei_scale(struct
qregs_tx_congest_cfg *x, enum qregs_tx_congest_cfg_dyncong_dei_scale v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_DYNCONG_DEI_SCALE_MASK, v);
+}
+
+/** Update dyanmic congestion after each update to WRR weights. */
+static inline bool is_qregs_tx_congest_cfg_dyncong_upd_wrr(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_DYNCONG_UPD_WRR, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_dyncong_upd_wrr(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_DYNCONG_UPD_WRR, v);
+}
+
+/** Update dyanmic congestion after each TX or RX */
+static inline bool is_qregs_tx_congest_cfg_dyncong_upd_txrx(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_DYNCONG_UPD_TXRX, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_dyncong_upd_txrx(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_DYNCONG_UPD_TXRX, v);
+}
+
+/** Update dyanmic congestion after each tick */
+static inline bool is_qregs_tx_congest_cfg_dyncong_upd_tick(struct
qregs_tx_congest_cfg *x)
+{
+ return FIELD_GET(QREGS_TX_CONGEST_CFG_DYNCONG_UPD_TICK, x->bitfield_0);
+}
+static inline void set_qregs_tx_congest_cfg_dyncong_upd_tick(struct
qregs_tx_congest_cfg *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0,
QREGS_TX_CONGEST_CFG_DYNCONG_UPD_TICK, v);
+}
+
+/**
+ * Bitfield accessors for: struct mem_ctl
+ * Called QDMA_CSR_DBG_MEM_XS_CFG, used for debugging access to memory
spaces.
+ */
+
+enum qregs_debug_mem_ctl_dataset {
+ QREGS_DEBUG_MEM_CTL_DATASET_DESC = 0,
+ QREGS_DEBUG_MEM_CTL_DATASET_QUEUE = 1,
+ QREGS_DEBUG_MEM_CTL_DATASET_QOS_WEIGHT_CTR = 2,
+ QREGS_DEBUG_MEM_CTL_DATASET_DMA_IDX = 3,
+ QREGS_DEBUG_MEM_CTL_DATASET_BUF_MON = 4,
+ QREGS_DEBUG_MEM_CTL_DATASET_RL_PARAM = 5,
+ QREGS_DEBUG_MEM_CTL_DATASET_VCH_WEIGHT = 6,
+};
+
+#define QREGS_DEBUG_MEM_CTL_WRITE BIT(31)
+#define QREGS_DEBUG_MEM_CTL_DONE BIT(30)
+#define QREGS_DEBUG_MEM_CTL_DATASET_MASK GENMASK(26, 24)
+#define QREGS_DEBUG_MEM_CTL_WORD_OF_ELEM_MASK GENMASK(20, 16)
+#define QREGS_DEBUG_MEM_CTL_ELEM_MASK GENMASK(15, 0)
+
+static inline void set_qregs_debug_mem_ctl_write(struct mem_ctl *x, bool v)
+{
+ x->word = FIELD_SET(x->word, QREGS_DEBUG_MEM_CTL_WRITE, v);
+}
+
+/** Becomes set when the command is completed */
+static inline bool is_qregs_debug_mem_ctl_done(struct mem_ctl *x)
+{
+ return FIELD_GET(QREGS_DEBUG_MEM_CTL_DONE, x->word);
+}
+static inline void set_qregs_debug_mem_ctl_dataset(struct mem_ctl *x,
enum qregs_debug_mem_ctl_dataset v)
+{
+ x->word = FIELD_SET(x->word, QREGS_DEBUG_MEM_CTL_DATASET_MASK, v);
+}
+static inline void set_qregs_debug_mem_ctl_word_of_elem(struct mem_ctl
*x, u8 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_DEBUG_MEM_CTL_WORD_OF_ELEM_MASK, v);
+}
+static inline void set_qregs_debug_mem_ctl_elem(struct mem_ctl *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_DEBUG_MEM_CTL_ELEM_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: qregs_doneq_cfg
+ * Register layout (32-bit word):
+ * - Bits 27-16: INT_THRESHOLD (when done queue is this full, fire
interrupt)
+ * - Bits 11-0: SIZE (size of done queue buffer in 4-byte units)
+ */
+#define QREGS_DONEQ_CFG_INT_THRESHOLD_MASK GENMASK(27, 16)
+#define QREGS_DONEQ_CFG_SIZE_MASK GENMASK(11, 0)
+
+static inline u16 get_qregs_doneq_cfg_int_threshold(struct
qregs_doneq_cfg *x)
+{
+ return FIELD_GET(QREGS_DONEQ_CFG_INT_THRESHOLD_MASK, x->word);
+}
+static inline void set_qregs_doneq_cfg_int_threshold(struct
qregs_doneq_cfg *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_DONEQ_CFG_INT_THRESHOLD_MASK, v);
+}
+static inline u16 get_qregs_doneq_cfg_size(struct qregs_doneq_cfg *x)
+{
+ return FIELD_GET(QREGS_DONEQ_CFG_SIZE_MASK, x->word);
+}
+static inline void set_qregs_doneq_cfg_size(struct qregs_doneq_cfg *x,
u16 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_DONEQ_CFG_SIZE_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: qregs_doneq_state
+ * Register layout (32-bit word):
+ * - Bits 27-16: LENGTH (number of items waiting in the queue)
+ * - Bits 11-0: HEAD_INDEX (index of the first item in the list)
+ */
+#define QREGS_DONEQ_STATE_LENGTH_MASK GENMASK(27, 16)
+#define QREGS_DONEQ_STATE_HEAD_INDEX_MASK GENMASK(11, 0)
+
+static inline u16 get_qregs_doneq_state_length(struct qregs_doneq_state *x)
+{
+ return FIELD_GET(QREGS_DONEQ_STATE_LENGTH_MASK, x->word);
+}
+static inline u16 get_qregs_doneq_state_head_index(struct
qregs_doneq_state *x)
+{
+ return FIELD_GET(QREGS_DONEQ_STATE_HEAD_INDEX_MASK, x->word);
+}
+
+/**
+ * Bitfield accessors for: qregs_rxring_size
+ * Register layout (32-bit word):
+ * - Bits 11-0: RING0_SIZE (size of RX ring 0, max 4095)
+ * - Bits 27-16: RING1_SIZE (size of RX ring 1, max 4095)
+ */
+#define QREGS_RXRING_SIZE_RING0_MASK GENMASK(11, 0)
+#define QREGS_RXRING_SIZE_RING1_MASK GENMASK(27, 16)
+
+static inline u16 get_qregs_rxring_size_ring0(struct qregs_rxring_size *x)
+{
+ return FIELD_GET(QREGS_RXRING_SIZE_RING0_MASK, x->word);
+}
+static inline void set_qregs_rxring_size_ring0(struct qregs_rxring_size
*x, u16 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_RXRING_SIZE_RING0_MASK, v);
+}
+static inline u16 get_qregs_rxring_size_ring1(struct qregs_rxring_size *x)
+{
+ return FIELD_GET(QREGS_RXRING_SIZE_RING1_MASK, x->word);
+}
+static inline void set_qregs_rxring_size_ring1(struct qregs_rxring_size
*x, u16 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_RXRING_SIZE_RING1_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: qregs_rxring_low
+ * Register layout (32-bit word):
+ * - Bits 11-0: RING0_LOW (interrupt threshold for ring 0, max 4095)
+ * - Bits 27-16: RING1_LOW (interrupt threshold for ring 1, max 4095)
+ */
+#define QREGS_RXRING_LOW_RING0_MASK GENMASK(11, 0)
+#define QREGS_RXRING_LOW_RING1_MASK GENMASK(27, 16)
+
+static inline u16 get_qregs_rxring_low_ring0(struct qregs_rxring_low *x)
+{
+ return FIELD_GET(QREGS_RXRING_LOW_RING0_MASK, x->word);
+}
+static inline void set_qregs_rxring_low_ring0(struct qregs_rxring_low
*x, u16 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_RXRING_LOW_RING0_MASK, v);
+}
+static inline u16 get_qregs_rxring_low_ring1(struct qregs_rxring_low *x)
+{
+ return FIELD_GET(QREGS_RXRING_LOW_RING1_MASK, x->word);
+}
+static inline void set_qregs_rxring_low_ring1(struct qregs_rxring_low
*x, u16 v)
+{
+ x->word = FIELD_SET(x->word, QREGS_RXRING_LOW_RING1_MASK, v);
+}
+
+/* GDM register layout and field accessors. */
+#ifndef FIELD_SET
+#define FIELD_SET(current, mask, val) \
+ (((current) & ~(mask)) | FIELD_PREP((mask), (val)))
+#endif
+
+/** gdm: GDM Registers */
+struct gdm {
+ /** gdm_vlan: VLAN tag control */
+ struct gdm_vlan {
+ /** gdm_vlan_tpid: If inserting a MTK Special Tag, this TPID
will be used */
+ u16 tpid;
+
+ /**
+ * See accessors:
+ * is_gdm_vlan_urx()
+ * set_gdm_vlan_urx()
+ * is_gdm_vlan_ttx()
+ * set_gdm_vlan_ttx()
+ */
+ u16 bitfield_0;
+
+ } vlan;
+
+ /** gdm_pppoe: PPPoE header control */
+ struct gdm_pppoe {
+ /**
+ * See accessors:
+ * is_gdm_pppoe_ttx()
+ * set_gdm_pppoe_ttx()
+ */
+ u16 bitfield_0;
+
+ /** gdm_pppoe_pppid: PPPoE Session ID to insert on transmitted
packets */
+ u16 pppoe_id;
+
+ } pppoe;
+
+ /**
+ * See accessors:
+ * get_gdm_red_cp()
+ * set_gdm_red_cp()
+ * get_gdm_red_lp()
+ * set_gdm_red_lp()
+ * get_gdm_red_hp()
+ * set_gdm_red_hp()
+ * get_gdm_red_dscp_rxm_sz()
+ * set_gdm_red_dscp_rxm_sz()
+ */
+ struct red_drop_criteria { u32 word; } red_drop_criteria;
+
+ /**
+ * gdm_chan_en: Each bit corrisponds to a channel, if one bit is
cleared then
+ * this GDM will refuse to forward traffic from that channel number.
+ */
+ u32 channels_enabled;
+
+ /**
+ * gdm_crsn_prio_lo: Priority of a packet based on the CPU_REASON
for packet
+ * being sent to CPU. Each CPU_REASON corrisponds to 2 bits, 00 =
crit_prio, 01
+ * = low_prio, 10 = high_prio. This word covers CPU_REASONS 0..15
+ */
+ u32 cpu_reason_priority_lo;
+
+ /**
+ * gdm_crsn_prio_hi: Priority of a packet based on the CPU_REASON
for packet
+ * being sent to CPU. Each CPU_REASON corrisponds to 2 bits, 00 =
crit_prio, 01
+ * = low_prio, 10 = high_prio. This word covers CPU_REASONS 16..31
+ */
+ u32 cpu_reason_priority_hi;
+
+ u8 unused_0[232];
+
+ /**
+ * See accessors:
+ * is_gdm_fwd_cfg_vip()
+ * set_gdm_fwd_cfg_vip()
+ * is_gdm_fwd_cfg_l2lu_dcsp_2cpu()
+ * set_gdm_fwd_cfg_l2lu_dcsp_2cpu()
+ * is_gdm_fwd_cfg_l2lu_ctag_2cpu()
+ * set_gdm_fwd_cfg_l2lu_ctag_2cpu()
+ * is_gdm_fwd_cfg_l2lu_stag_2cpu()
+ * set_gdm_fwd_cfg_l2lu_stag_2cpu()
+ * is_gdm_fwd_cfg_g2_underrun_retry()
+ * set_gdm_fwd_cfg_g2_underrun_retry()
+ * is_gdm_fwd_cfg_g2_drop_256b()
+ * set_gdm_fwd_cfg_g2_drop_256b()
+ * is_gdm_fwd_cfg_drop_oversize()
+ * set_gdm_fwd_cfg_drop_oversize()
+ * is_gdm_fwd_cfg_drop_runt()
+ * set_gdm_fwd_cfg_drop_runt()
+ * is_gdm_fwd_cfg_drop_crc()
+ * set_gdm_fwd_cfg_drop_crc()
+ * is_gdm_fwd_cfg_drop_ip4_csum()
+ * set_gdm_fwd_cfg_drop_ip4_csum()
+ * is_gdm_fwd_cfg_drop_tcp_csum()
+ * set_gdm_fwd_cfg_drop_tcp_csum()
+ * is_gdm_fwd_cfg_drop_ucp_csum()
+ * set_gdm_fwd_cfg_drop_ucp_csum()
+ * is_gdm_fwd_cfg_g2_favor_oam()
+ * set_gdm_fwd_cfg_g2_favor_oam()
+ * is_gdm_fwd_cfg_l2lu_brg_cpu()
+ * set_gdm_fwd_cfg_l2lu_brg_cpu()
+ * is_gdm_fwd_cfg_l2lu_unbrg_cpu()
+ * set_gdm_fwd_cfg_l2lu_unbrg_cpu()
+ * is_gdm_fwd_cfg_strip_crc()
+ * set_gdm_fwd_cfg_strip_crc()
+ * get_gdm_fwd_cfg_mymac_fport()
+ * set_gdm_fwd_cfg_mymac_fport()
+ * get_gdm_fwd_cfg_bcast_fport()
+ * set_gdm_fwd_cfg_bcast_fport()
+ * get_gdm_fwd_cfg_mcast_fport()
+ * set_gdm_fwd_cfg_mcast_fport()
+ * get_gdm_fwd_cfg_default_fport()
+ * set_gdm_fwd_cfg_default_fport()
+ */
+ struct fwd_cfg { u32 word; } fwd_cfg;
+
+ /** gdm_tx_shaper: Traffic shaper for TX */
+ u32 tx_shaper;
+
+ /**
+ * gdm_mymac_lsb: Mac address bytes [c,d,e,f] (bytes 2-5 of MAC)
+ * Register layout (32-bit word):
+ * - Bits 31-24: MAC byte c (index 2)
+ * - Bits 23-16: MAC byte d (index 3)
+ * - Bits 15-8: MAC byte e (index 4)
+ * - Bits 7-0: MAC byte f (index 5)
+ * See accessors:
+ * get_gdm_mymac_lsb_c/d/e/f()
+ * set_gdm_mymac_lsb_c/d/e/f()
+ */
+ struct gdm_mymac_lsb { u32 word; } mymac_lsb;
+
+ /**
+ * gdm_mymac_msb: Mac address bytes [a,b] (bytes 0-1 of MAC) and mask
+ * Register layout (32-bit word):
+ * - Bits 31-24: unused
+ * - Bits 23-16: lsb_mask (LSB matching mask)
+ * - Bits 15-8: MAC byte a (index 0)
+ * - Bits 7-0: MAC byte b (index 1)
+ * See accessors:
+ * get_gdm_mymac_msb_lsb_mask/a/b()
+ * set_gdm_mymac_msb_lsb_mask/a/b()
+ */
+ struct gdm_mymac_msb { u32 word; } mymac_msb;
+
+ /** gdm_stag_en: Add Special Tag on RX frames, 0 or 1 */
+ u32 stag_en;
+
+ /**
+ * gdm_len_th:
+ * Register layout (32-bit word):
+ * - Bits 31-16: OVERSIZE_LEN (packet larger than this is oversize,
max 0x3f00)
+ * - Bits 15-0: RUNT_LEN (packet smaller than this is a runt)
+ * See accessors:
+ * get_gdm_len_th_oversize_len()
+ * set_gdm_len_th_oversize_len()
+ * get_gdm_len_th_runt_len()
+ * set_gdm_len_th_runt_len()
+ */
+ struct gdm_len_th { u32 word; } rx_len_threshold;
+
+ /**
+ * See accessors:
+ * get_gdm_pcp_gdm_rx()
+ * set_gdm_pcp_gdm_rx()
+ * get_gdm_pcp_gdm_tx()
+ * set_gdm_pcp_gdm_tx()
+ * get_gdm_pcp_cdm_rx()
+ * set_gdm_pcp_cdm_rx()
+ * get_gdm_pcp_cdm_tx()
+ * set_gdm_pcp_cdm_tx()
+ */
+ struct pcp { u32 word; } pcp;
+
+ /**
+ * See accessors:
+ * get_gdm_lpbk_gap()
+ * set_gdm_lpbk_gap()
+ * get_gdm_lpbk_len()
+ * set_gdm_lpbk_len()
+ * get_gdm_lpbk_chan()
+ * set_gdm_lpbk_chan()
+ * is_gdm_lpbk_gap_mode()
+ * set_gdm_lpbk_gap_mode()
+ * is_gdm_lpbk_len_mode()
+ * set_gdm_lpbk_len_mode()
+ * is_gdm_lpbk_chan_mode()
+ * set_gdm_lpbk_chan_mode()
+ * is_gdm_lpbk_enabled()
+ * set_gdm_lpbk_enabled()
+ */
+ struct loopback { u32 word; } loopback;
+
+ /**
+ * See accessors:
+ * get_gdm_channel_retire_channel()
+ * set_gdm_channel_retire_channel()
+ * is_gdm_channel_retire_done()
+ * set_gdm_channel_retire_done()
+ * is_gdm_channel_retire_release()
+ * set_gdm_channel_retire_release()
+ */
+ struct channel_retire { u32 word; } channel_retire;
+
+ /**
+ * gdm_tx_ch_en: Each bit corripsonds to one of the 32 channels,
set bit to
+ * enable channel
+ */
+ u32 enabled_tx_channels;
+
+ /**
+ * gdm_rx_ch_en: Each bit 0..16 corrisonds to one of the 16 RX
channels, upper
+ * 16 bits are unused
+ */
+ u32 enabled_rx_channels;
+
+ /**
+ * See accessors:
+ * is_gdm_rx7_en()
+ * set_gdm_rx7_en()
+ * get_gdm_rx7_fport()
+ * set_gdm_rx7_fport()
+ */
+ u32 bitfield_0;
+
+ /** gdm_g2_rx_shapers: Incoming traffic shapers, only present in
GDM2 */
+ struct gdm_g2_rx_shapers {
+ /** gdm_g2_rx_shapers_mymac: Shaper for incoming traffic to my
MAC */
+ u32 mymac;
+
+ /** gdm_g2_rx_shapers_bcast: Shaper for incoming broadcast
traffic */
+ u32 bcast;
+
+ /** gdm_g2_rx_shapers_mcast: Shaper for incoming multicast
traffic */
+ u32 mcast;
+
+ /**
+ * gdm_g2_rx_shapers_dflt: Default shaper for incoming traffic
not otherwise
+ * categorized
+ */
+ u32 dflt;
+
+ } g2_rx_shapers;
+
+ /**
+ * See accessors:
+ * is_gdm_g1_cport_cfg_add_crc()
+ * set_gdm_g1_cport_cfg_add_crc()
+ * is_gdm_g1_cport_cfg_pad()
+ * set_gdm_g1_cport_cfg_pad()
+ * is_gdm_g1_cport_cfg_port_xfc()
+ * set_gdm_g1_cport_cfg_port_xfc()
+ * is_gdm_g1_cport_cfg_queue_xfc()
+ * set_gdm_g1_cport_cfg_queue_xfc()
+ * get_gdm_g1_cport_cfg_unknown()
+ * set_gdm_g1_cport_cfg_unknown()
+ */
+ struct g1_cport_cfg { u32 word; } g1_cport_cfg;
+
+ /**
+ * gdm_g1_cport_channel_map: Called FE_CPORT_CHN_MAP, never used,
observed
+ * value is 0x76543210, GDM1 only
+ */
+ u32 g1_cport_channel_map;
+
+ /** gdm_g1_cport_shaper: Traffic shaper for CPORT */
+ u32 g1_cport_shaper;
+
+ /**
+ * gdm_g1_unknown0: Unused, no symbol, observed value 0x000003c0,
GDM1 only.
+ */
+ u32 g1_unknown0;
+
+ /**
+ * gdm_g1_unknown1: Unused, no symbol, observed value 0x00000000,
GDM1 only.
+ */
+ u32 g1_unknown1;
+
+ u8 unused_1[28];
+
+ /**
+ * gdm_tx_chan_active: Each of the 32 bits represents one of the 32
channels, 1
+ * means channel is active used with channel_retire to confirm
channel is
+ * shutdown. Also GDMA1_TX_CHN_VLD
+ */
+ u32 tx_chan_active;
+
+ /**
+ * gdm_rx_chan_active: Unused in practice except on EN7580, each
bit represents
+ * an RX channel, 1 means RX channel is active, used with
channel_retire.
+ * Unsure if 16 or 32 RX channels. Also GDMA1_RX_CHN_VLD
+ */
+ u32 rx_chan_active;
+
+ u8 unused_2[8];
+
+ /** gdm_cdm_counters: Various packet counters for CDM1 / CDM2 */
+ struct gdm_cdm_counters {
+ /**
+ * gdm_cdm_counters_tx: Unused in practice, seems to be frames
successfully
+ * sent by CDM, also CDMA1_TX_OK_CNT
+ */
+ u32 tx;
+
+ u8 unused_0[12];
+
+ /**
+ * gdm_cdm_counters_rxcpu: Unused in practice, seems to be
frames successfully
+ * received to CPU by CDM, also CDMA1_RXCPU_OK_CNT
+ */
+ u32 rxcpu;
+
+ /**
+ * gdm_cdm_counters_rxhwf: Unused in practice, seems to be
frames successfully
+ * received to hardware forwarding chain by CDM, also
CDMA1_RXHWF_OK_CNT
+ */
+ u32 rxhwf;
+
+ /**
+ * gdm_cdm_counters_ka: Unused in practice, seems to be frames
sent to CPU for
+ * NAT keepalive purposes, also CDMA1_RXCPU_KA_CNT
+ */
+ u32 ka;
+
+ u32 unused_1;
+
+ /**
+ * gdm_cdm_counters_rxcpu_drop: Unused in practice, seems to be
frames dropped
+ * for congestion on the RX->CPU chain, also CDMA1_RXCPU_DROP_CNT
+ */
+ u32 rxcpu_drop;
+
+ /**
+ * gdm_cdm_counters_rxhwf_drop: Unused in practice, seems to be
frames dropped
+ * for congestion on the RX->Hardware Forwarding chain, also
+ * CDMA1_RXHWF_DROP_CNT
+ */
+ u32 rxhwf_drop;
+
+ /**
+ * gdm_cdm_counters_unknown0: Unused, no symbol, observed value
matches rxcpu
+ */
+ u32 unknown0;
+
+ /**
+ * gdm_cdm_counters_unknown1: Unused, no symbol, observed value
0x00000000
+ */
+ u32 unknown1;
+
+ /**
+ * gdm_cdm_counters_unknown2: Unused, no symbol, observed value
0x00000000
+ */
+ u32 unknown2;
+
+ /**
+ * gdm_cdm_counters_unknown3: Unused, no symbol, observed value
0x00000000
+ */
+ u32 unknown3;
+
+ /**
+ * gdm_cdm_counters_unknown4: Unused, no symbol, observed value
0x00000000
+ */
+ u32 unknown4;
+
+ /**
+ * gdm_cdm_counters_unknown5: Unused, no symbol, observed value
0x00000000
+ */
+ u32 unknown5;
+
+ } cdm_counters;
+
+ u8 unused_3[48];
+
+ /**
+ * See accessors:
+ * set_gdm_cl_cnt_rx()
+ * set_gdm_cl_cnt_tx()
+ */
+ struct clear_counters { u32 word; } clear_counters;
+
+ u8 unused_4[12];
+
+ /** gdm_counters: Packet and byte counters for GDM port */
+ struct gdm_counters {
+ /** gdm_counters_tx: */
+ struct gdm_counters_tx {
+ /**
+ * gdm_counters_tx_get: Unused in practice, believed to be
number of packets
+ * enregistered for TX, also GDMA1_TX_GET_CNT / GDMA2_TX_GETCNT
+ */
+ u32 get;
+
+ /**
+ * gdm_counters_tx_pkts: Packets successfully transmitted, also
+ * GDMA1_TX_OK_CNT / GDMA2_TX_OKCNT
+ */
+ u32 tx_pkts;
+
+ /**
+ * gdm_counters_tx_drops: Packets dropped on transmission
chain, also
+ * GDMA1_TX_DROP_CNT / GDMA2_TX_DROPCNT
+ */
+ u32 drops;
+
+ /**
+ * gdm_counters_tx_bytes: Bytes successfully transmitted, also
+ * GDMA1_TX_OK_BYTE_CNT / GDMA2_TX_OKBYTE_CNT
+ */
+ u32 bytes;
+
+ /** gdm_counters_tx_g2: GDM2 only extended TX stats */
+ struct gdm_counters_tx_g2 {
+ /**
+ * gdm_counters_tx_g2_epkts: Number of eth frames
transmitted, should match
+ * .pkts, also GDMA2_TX_ETHCNT
+ */
+ u32 tx_epkts;
+
+ /**
+ * gdm_counters_tx_g2_ebytes: Bytes of eth bytes
transmitted, also
+ * GDMA2_TX_ETHLENCNT
+ */
+ u32 tx_ebytes;
+
+ /**
+ * gdm_counters_tx_g2_edrops: Number of eth frames
dropped, see
+ * GDMA2_TX_ETHDROPCNT
+ */
+ u32 edrops;
+
+ /**
+ * gdm_counters_tx_g2_bcast: Number of broadcast frames
transmitted, see
+ * GDMA2_TX_ETHBCDCNT
+ */
+ u32 tx_bcast;
+
+ /**
+ * gdm_counters_tx_g2_mcast: Number of multicast frames
transmitted, see
+ * GDMA2_TX_ETHMULTICASTCNT
+ */
+ u32 tx_mcast;
+
+ /**
+ * gdm_counters_tx_g2_f_less_64: Counter of TX frames
of length less than
+ * 64, see GDMA2_TX_ETH_LESS64_CNT
+ */
+ u32 f_less_64;
+
+ /**
+ * gdm_counters_tx_g2_f_more_1518: Counter of TX frames
of length more than
+ * 1518, see GDMA2_TX_ETH_MORE1518_CNT
+ */
+ u32 f_more_1518;
+
+ /**
+ * gdm_counters_tx_g2_f_64: Counter of TX frames of
length 64, see
+ * GDMA2_TX_ETH_64_CNT
+ */
+ u32 f_64;
+
+ /**
+ * gdm_counters_tx_g2_f_65_127: Counter of TX frames of
length 65-127, see
+ * GDMA2_TX_ETH_65_TO_127_CNT
+ */
+ u32 f_65_127;
+
+ /**
+ * gdm_counters_tx_g2_f_128_255: Counter of TX frames
of length 128-255, see
+ * GDMA2_TX_ETH_128_TO_255_CNT
+ */
+ u32 f_128_255;
+
+ /**
+ * gdm_counters_tx_g2_f_256_511: Counter of TX frames
of length 256-511, see
+ * GDMA2_TX_ETH_256_TO_511_CNT
+ */
+ u32 f_256_511;
+
+ /**
+ * gdm_counters_tx_g2_f_512_1023: Counter of TX frames
of length 512-1023,
+ * see GDMA2_TX_ETH_512_TO_1023_CNT
+ */
+ u32 f_512_1023;
+
+ /**
+ * gdm_counters_tx_g2_f_1024_1518: Counter of TX frames
of length 1024-1518,
+ * see GDMA2_TX_ETH_1024_TO_1518_CNT
+ */
+ u32 f_1024_1518;
+
+ } g2;
+
+ } tx;
+
+ u32 unused_0;
+
+ /** gdm_counters_rx: */
+ struct gdm_counters_rx {
+ /**
+ * gdm_counters_rx_pkts: Number of packets received, see
GDMA1_RX_OK_CNT /
+ * GDMA2_RX_OKCNT
+ */
+ u32 pkts;
+
+ /**
+ * gdm_counters_rx_drops_fc: Packets dropped due to flow
control, thought to
+ * be when port sends PAUSE signal. see GDMA1_RX_FC_DROP_CNT
+ */
+ u32 drops_fc;
+
+ /**
+ * gdm_counters_rx_drops_rc: Packets dropped due to rate
control, thought to
+ * be shapers. See GDMA1_RX_RC_DROP_CNT / GDMA2_RX_RCDROPCNT
+ */
+ u32 drops_rc;
+
+ /**
+ * gdm_counters_rx_drops_overflow: Packets dropped because
we ran out of
+ * queue resources, vendor code logs when this counter
increases so it should
+ * not happen. GDMA2_RX_OVDROPCNT / GDMA1_RX_OVER_DROP_CNT
+ */
+ u32 drops_overflow;
+
+ /**
+ * gdm_counters_rx_drops_err: Packets dropped because of
any error (crc,
+ * IP/TCP/UDP checksum, oversize, runt, etc)
+ */
+ u32 drops_err;
+
+ /**
+ * gdm_counters_rx_bytes: Received bytes, see
GDMA1_RX_BYTECNT /
+ * GDMA2_RX_OKBYTECNT
+ */
+ u32 bytes;
+
+ /** gdm_counters_rx_g2: GDM2 only extended RX stats */
+ struct gdm_counters_rx_g2 {
+ /**
+ * gdm_counters_rx_g2_epkts: Number of eth frames
received, should match
+ * .pkts, also GDMA2_RX_ETHERPCNT
+ */
+ u32 epkts;
+
+ /**
+ * gdm_counters_rx_g2_ebytes: Bytes of eth bytes
received, also
+ * GDMA2_RX_ETHERPLEN
+ */
+ u32 ebytes;
+
+ /**
+ * gdm_counters_rx_g2_edrops: Number of eth frames
dropped, see
+ * GDMA2_RX_ETHDROPCNT
+ */
+ u32 edrops;
+
+ /**
+ * gdm_counters_rx_g2_bcast: Number of broadcast frames
received, see
+ * GDMA2_RX_ETHBCCNT
+ */
+ u32 bcast;
+
+ /**
+ * gdm_counters_rx_g2_mcast: Number of multicast frames
received, see
+ * GDMA2_RX_ETHMCCNT
+ */
+ u32 mcast;
+
+ /**
+ * gdm_counters_rx_g2_ecrc: Number of frames with CRC
errors, see
+ * GDMA2_RX_ETHCRCCNT
+ */
+ u32 ecrc;
+
+ /**
+ * gdm_counters_rx_g2_efrag: Number of frames with CRC
error and which are
+ * less than 64 bytes (probably fragments from
synchronization issue) see
+ * GDMA2_RX_ETHFRACCNT
+ */
+ u32 efrag;
+
+ /**
+ * gdm_counters_rx_g2_ejabber: Number of frames longer
than 1518 with bad
+ * CRC, sign of potentially jabbering hardware, see
GDMA2_RX_ETHJABCNT
+ */
+ u32 ejabber;
+
+ /**
+ * gdm_counters_rx_g2_f_less_64: Counter of TX frames
of length less than
+ * 64, see GDMA2_RX_ETHRUNTCNT
+ */
+ u32 f_less_64;
+
+ /**
+ * gdm_counters_rx_g2_f_more_1518: Counter of TX frames
of length more than
+ * 1518, see GDMA2_RX_ETHLONGCNT
+ */
+ u32 f_more_1518;
+
+ /**
+ * gdm_counters_rx_g2_f_64: Counter of TX frames of
length 64, see
+ * GDMA2_RX_ETH_64_CNT
+ */
+ u32 f_64;
+
+ /**
+ * gdm_counters_rx_g2_f_65_127: Counter of TX frames of
length 65-127, see
+ * GDMA2_RX_ETH_65_TO_127_CNT
+ */
+ u32 f_65_127;
+
+ /**
+ * gdm_counters_rx_g2_f_128_255: Counter of TX frames
of length 128-255, see
+ * GDMA2_RX_ETH_128_TO_255_CNT
+ */
+ u32 f_128_255;
+
+ /**
+ * gdm_counters_rx_g2_f_256_511: Counter of TX frames
of length 256-511, see
+ * GDMA2_RX_ETH_256_TO_511_CNT
+ */
+ u32 f_256_511;
+
+ /**
+ * gdm_counters_rx_g2_f_512_1023: Counter of TX frames
of length 512-1023,
+ * see GDMA2_RX_ETH_512_TO_1023_CNT
+ */
+ u32 f_512_1023;
+
+ /**
+ * gdm_counters_rx_g2_f_1024_1518: Counter of TX frames
of length 1024-1518,
+ * see GDMA2_RX_ETH_1024_TO_1518_CNT
+ */
+ u32 f_1024_1518;
+
+ /** gdm_counters_rx_g2_unknown0: Observed 0x00000000,
possible counter */
+ u32 tx_unknown0;
+
+ /** gdm_counters_rx_g2_unknown1: Observed 0x00000000,
possible counter */
+ u32 unknown1;
+
+ } g2;
+
+ } rx;
+
+ } counters;
+
+};
+
+/**
+ * Bitfield accessors for: gdm_vlan bitfield_0
+ */
+
+#define GDM_VLAN_URX BIT(1)
+#define GDM_VLAN_TTX BIT(0)
+
+
+/** Untag incoming tagged packets (and put the tag in the DESC) */
+static inline bool is_gdm_vlan_urx(struct gdm_vlan *x)
+{
+ return FIELD_GET(GDM_VLAN_URX, x->bitfield_0);
+}
+static inline void set_gdm_vlan_urx(struct gdm_vlan *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, GDM_VLAN_URX, v);
+}
+
+/** Insert MTK Special Tag when sending packets */
+static inline bool is_gdm_vlan_ttx(struct gdm_vlan *x)
+{
+ return FIELD_GET(GDM_VLAN_TTX, x->bitfield_0);
+}
+static inline void set_gdm_vlan_ttx(struct gdm_vlan *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, GDM_VLAN_TTX, v);
+}
+
+/**
+ * Bitfield accessors for: gdm_pppoe bitfield_0
+ */
+
+#define GDM_PPPOE_TTX BIT(0)
+
+
+/** Insert PPPoE header on transmitted packets */
+static inline bool is_gdm_pppoe_ttx(struct gdm_pppoe *x)
+{
+ return FIELD_GET(GDM_PPPOE_TTX, x->bitfield_0);
+}
+static inline void set_gdm_pppoe_ttx(struct gdm_pppoe *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, GDM_PPPOE_TTX, v);
+}
+
+/**
+ * Bitfield accessors for: struct red_drop_criteria
+ */
+
+enum gdm_red_cp {
+ GDM_RED_CP_RL_OR_FULL = 0,
+ GDM_RED_CP_RL_OR_TH = 1,
+ GDM_RED_CP_RL_AND_FULL = 2,
+ GDM_RED_CP_RL_AND_TH = 3,
+};
+enum gdm_red_dscp_rxm_sz {
+ GDM_RED_DSCP_RXM_SZ_4B = 0,
+ GDM_RED_DSCP_RXM_SZ_8B = 1,
+ GDM_RED_DSCP_RXM_SZ_16B = 2,
+ GDM_RED_DSCP_RXM_SZ_32B = 3,
+};
+
+#define GDM_RED_CP_MASK GENMASK(31, 30)
+#define GDM_RED_LP_MASK GENMASK(29, 28)
+#define GDM_RED_HP_MASK GENMASK(27, 26)
+#define GDM_RED_DSCP_RXM_SZ_MASK GENMASK(25, 24)
+
+
+/**
+ * Trigger RED packet dropping on 1. Rate-limit-violation OR buffer is
full 2.
+ * Rate-limit-violation OR buffer has surpassed threshold 3.
+ * Rate-limit-violation WHEN buffer is also full 4.
Rate-limit-violation WHEN
+ * buffer has surpassed threshold For critical priority packets (VIP)
+ */
+static inline enum gdm_red_cp get_gdm_red_cp(struct red_drop_criteria *x)
+{
+ return FIELD_GET(GDM_RED_CP_MASK, x->word);
+}
+static inline void set_gdm_red_cp(struct red_drop_criteria *x, enum
gdm_red_cp v)
+{
+ x->word = FIELD_SET(x->word, GDM_RED_CP_MASK, v);
+}
+
+/** Same as crit_prio, for packets classed as low priority by CPU Reason */
+static inline u8 get_gdm_red_lp(struct red_drop_criteria *x)
+{
+ return FIELD_GET(GDM_RED_LP_MASK, x->word);
+}
+static inline void set_gdm_red_lp(struct red_drop_criteria *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_RED_LP_MASK, v);
+}
+
+/** Same as crit_prio, for packets classed as low priority by CPU Reason */
+static inline u8 get_gdm_red_hp(struct red_drop_criteria *x)
+{
+ return FIELD_GET(GDM_RED_HP_MASK, x->word);
+}
+static inline void set_gdm_red_hp(struct red_drop_criteria *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_RED_HP_MASK, v);
+}
+
+/** Size of the msg field within the packet descriptor */
+static inline enum gdm_red_dscp_rxm_sz get_gdm_red_dscp_rxm_sz(struct
red_drop_criteria *x)
+{
+ return FIELD_GET(GDM_RED_DSCP_RXM_SZ_MASK, x->word);
+}
+static inline void set_gdm_red_dscp_rxm_sz(struct red_drop_criteria *x,
enum gdm_red_dscp_rxm_sz v)
+{
+ x->word = FIELD_SET(x->word, GDM_RED_DSCP_RXM_SZ_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: struct fwd_cfg
+ * Configuration for GDM
+ */
+
+#define GDM_FWD_CFG_VIP BIT(31)
+#define GDM_FWD_CFG_L2LU_DCSP_2CPU BIT(30)
+#define GDM_FWD_CFG_L2LU_CTAG_2CPU BIT(29)
+#define GDM_FWD_CFG_L2LU_STAG_2CPU BIT(28)
+#define GDM_FWD_CFG_G2_UNDERRUN_RETRY BIT(27)
+#define GDM_FWD_CFG_G2_DROP_256B BIT(26)
+#define GDM_FWD_CFG_DROP_OVERSIZE BIT(25)
+#define GDM_FWD_CFG_DROP_RUNT BIT(24)
+#define GDM_FWD_CFG_DROP_CRC BIT(23)
+#define GDM_FWD_CFG_DROP_IP4_CSUM BIT(22)
+#define GDM_FWD_CFG_DROP_TCP_CSUM BIT(21)
+#define GDM_FWD_CFG_DROP_UCP_CSUM BIT(20)
+#define GDM_FWD_CFG_G2_FAVOR_OAM BIT(19)
+#define GDM_FWD_CFG_L2LU_BRG_CPU BIT(18)
+#define GDM_FWD_CFG_L2LU_UNBRG_CPU BIT(17)
+#define GDM_FWD_CFG_STRIP_CRC BIT(16)
+#define GDM_FWD_CFG_MYMAC_FPORT_MASK GENMASK(15, 12)
+#define GDM_FWD_CFG_BCAST_FPORT_MASK GENMASK(11, 8)
+#define GDM_FWD_CFG_MCAST_FPORT_MASK GENMASK(7, 4)
+#define GDM_FWD_CFG_DEFAULT_FPORT_MASK GENMASK(3, 0)
+
+
+/**
+ * GDM2 only, configure whether VIP packets are included in rate control
+ * calculation. They are not actually dropped in any case.
+ */
+static inline bool is_gdm_fwd_cfg_vip(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_VIP, x->word);
+}
+static inline void set_gdm_fwd_cfg_vip(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_VIP, v);
+}
+
+/** On L2LU DSCP lookup miss, send packet to CPU */
+static inline bool is_gdm_fwd_cfg_l2lu_dcsp_2cpu(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_L2LU_DCSP_2CPU, x->word);
+}
+static inline void set_gdm_fwd_cfg_l2lu_dcsp_2cpu(struct fwd_cfg *x,
bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_L2LU_DCSP_2CPU, v);
+}
+
+/** On L2LU CTAG lookup miss, send package to CPU */
+static inline bool is_gdm_fwd_cfg_l2lu_ctag_2cpu(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_L2LU_CTAG_2CPU, x->word);
+}
+static inline void set_gdm_fwd_cfg_l2lu_ctag_2cpu(struct fwd_cfg *x,
bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_L2LU_CTAG_2CPU, v);
+}
+
+/** On L2LU STAG lookup miss, send packet to CPU */
+static inline bool is_gdm_fwd_cfg_l2lu_stag_2cpu(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_L2LU_STAG_2CPU, x->word);
+}
+static inline void set_gdm_fwd_cfg_l2lu_stag_2cpu(struct fwd_cfg *x,
bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_L2LU_STAG_2CPU, v);
+}
+
+/** Unknown/Unused, see GDM2_UNDERRUN_RETRY / GDMA1_FWD_CFG_RETRY_OFFSET */
+static inline bool is_gdm_fwd_cfg_g2_underrun_retry(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_G2_UNDERRUN_RETRY, x->word);
+}
+static inline void set_gdm_fwd_cfg_g2_underrun_retry(struct fwd_cfg *x,
bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_G2_UNDERRUN_RETRY, v);
+}
+
+/** Unknown/unused, referred to as GDM2_DROP_256B */
+static inline bool is_gdm_fwd_cfg_g2_drop_256b(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_G2_DROP_256B, x->word);
+}
+static inline void set_gdm_fwd_cfg_g2_drop_256b(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_G2_DROP_256B, v);
+}
+
+/**
+ * If frame length (with CRC) is greater than
rx_len_threshold.oversize_len,
+ * drop. Referred to as GDM2_DROP_LONG.
+ */
+static inline bool is_gdm_fwd_cfg_drop_oversize(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_DROP_OVERSIZE, x->word);
+}
+static inline void set_gdm_fwd_cfg_drop_oversize(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_DROP_OVERSIZE, v);
+}
+
+/**
+ * If frame length (with CRC) is less than rx_len_threshold.runt_len, drop.
+ * Referred to as GDM2_DROP_RUNT.
+ */
+static inline bool is_gdm_fwd_cfg_drop_runt(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_DROP_RUNT, x->word);
+}
+static inline void set_gdm_fwd_cfg_drop_runt(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_DROP_RUNT, v);
+}
+
+/** If eth CRC invalid, drop. See GDM2_DROP_CRC_ERR */
+static inline bool is_gdm_fwd_cfg_drop_crc(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_DROP_CRC, x->word);
+}
+static inline void set_gdm_fwd_cfg_drop_crc(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_DROP_CRC, v);
+}
+
+/** Drop packet on IPv4 checksum error */
+static inline bool is_gdm_fwd_cfg_drop_ip4_csum(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_DROP_IP4_CSUM, x->word);
+}
+static inline void set_gdm_fwd_cfg_drop_ip4_csum(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_DROP_IP4_CSUM, v);
+}
+
+/** Drop packet on TCP checksum error */
+static inline bool is_gdm_fwd_cfg_drop_tcp_csum(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_DROP_TCP_CSUM, x->word);
+}
+static inline void set_gdm_fwd_cfg_drop_tcp_csum(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_DROP_TCP_CSUM, v);
+}
+
+/** Drop packet on UDP checksum error */
+static inline bool is_gdm_fwd_cfg_drop_ucp_csum(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_DROP_UCP_CSUM, x->word);
+}
+static inline void set_gdm_fwd_cfg_drop_ucp_csum(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_DROP_UCP_CSUM, v);
+}
+
+/**
+ * Favor OAM frames during transmission, this probably means raise their
+ * priority. GDM2 only, EN761221 (EN7521/EN7526) only, see
+ * GDMA2_TX_FAVOR_OAM_OFFSET
+ */
+static inline bool is_gdm_fwd_cfg_g2_favor_oam(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_G2_FAVOR_OAM, x->word);
+}
+static inline void set_gdm_fwd_cfg_g2_favor_oam(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_G2_FAVOR_OAM, v);
+}
+
+/**
+ * Packets which are L2LU misses and which match an L2-Bridge to be
forwarded to
+ * the CPU. Referred to as GDMA1_FWD_CFG_BRG_OFFSET
+ */
+static inline bool is_gdm_fwd_cfg_l2lu_brg_cpu(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_L2LU_BRG_CPU, x->word);
+}
+static inline void set_gdm_fwd_cfg_l2lu_brg_cpu(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_L2LU_BRG_CPU, v);
+}
+
+/**
+ * Packets which are L2LU misses and which are NOT part of a L2-Bridge
to be
+ * forwarded to the CPU. Referred to as GDMA1_FWD_CFG_RUT_OFFSET
+ */
+static inline bool is_gdm_fwd_cfg_l2lu_unbrg_cpu(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_L2LU_UNBRG_CPU, x->word);
+}
+static inline void set_gdm_fwd_cfg_l2lu_unbrg_cpu(struct fwd_cfg *x,
bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_L2LU_UNBRG_CPU, v);
+}
+
+/**
+ * Strip eth crc trailer on RX, this is used with g1_cport_cfg.add_crc
when xPON
+ * WAN and LAN are bridged.
+ */
+static inline bool is_gdm_fwd_cfg_strip_crc(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_STRIP_CRC, x->word);
+}
+static inline void set_gdm_fwd_cfg_strip_crc(struct fwd_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_STRIP_CRC, v);
+}
+
+/** GDM_P_* port for packets with our MAC address */
+static inline u8 get_gdm_fwd_cfg_mymac_fport(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_MYMAC_FPORT_MASK, x->word);
+}
+static inline void set_gdm_fwd_cfg_mymac_fport(struct fwd_cfg *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_MYMAC_FPORT_MASK, v);
+}
+
+/** GDM_P_* port for broadcast packets */
+static inline u8 get_gdm_fwd_cfg_bcast_fport(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_BCAST_FPORT_MASK, x->word);
+}
+static inline void set_gdm_fwd_cfg_bcast_fport(struct fwd_cfg *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_BCAST_FPORT_MASK, v);
+}
+
+/** GDM_P_* port for multicast packets */
+static inline u8 get_gdm_fwd_cfg_mcast_fport(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_MCAST_FPORT_MASK, x->word);
+}
+static inline void set_gdm_fwd_cfg_mcast_fport(struct fwd_cfg *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_MCAST_FPORT_MASK, v);
+}
+
+/** GDM_P_* port for other packets */
+static inline u8 get_gdm_fwd_cfg_default_fport(struct fwd_cfg *x)
+{
+ return FIELD_GET(GDM_FWD_CFG_DEFAULT_FPORT_MASK, x->word);
+}
+static inline void set_gdm_fwd_cfg_default_fport(struct fwd_cfg *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_FWD_CFG_DEFAULT_FPORT_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: struct pcp
+ * These codings are defined in IEEE 802.1ad and define how priority and
+ * drop-eligiblity. should be represented in the 3 bit VLAN PCP header.
8p0d
+ * specifies 8 priorities of which none are drop-eligible. 7p1d specifies 7
+ * priorities + p0d which is the same as p0, but marked drop-eligible. 6p2d
+ * specieies 6 priorities + p0d and p1d, same as p0 and p1, but marked
+ * drop-eligible. 5p3d specifies 5 priorities + p0d, p1d, and p2d, same
as p0,
+ * p1, p2, but marked drop-eligible.
+ */
+
+enum gdm_pcp_gdm_rx {
+ GDM_PCP_GDM_RX_5P3D = 8,
+ GDM_PCP_GDM_RX_6P2D = 4,
+ GDM_PCP_GDM_RX_7P1D = 2,
+ GDM_PCP_GDM_RX_8P0D = 1,
+};
+
+#define GDM_PCP_GDM_RX_MASK GENMASK(15, 12)
+#define GDM_PCP_GDM_TX_MASK GENMASK(11, 8)
+#define GDM_PCP_CDM_RX_MASK GENMASK(7, 4)
+#define GDM_PCP_CDM_TX_MASK GENMASK(3, 0)
+
+static inline enum gdm_pcp_gdm_rx get_gdm_pcp_gdm_rx(struct pcp *x)
+{
+ return FIELD_GET(GDM_PCP_GDM_RX_MASK, x->word);
+}
+static inline void set_gdm_pcp_gdm_rx(struct pcp *x, enum gdm_pcp_gdm_rx v)
+{
+ x->word = FIELD_SET(x->word, GDM_PCP_GDM_RX_MASK, v);
+}
+static inline u8 get_gdm_pcp_gdm_tx(struct pcp *x)
+{
+ return FIELD_GET(GDM_PCP_GDM_TX_MASK, x->word);
+}
+static inline void set_gdm_pcp_gdm_tx(struct pcp *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_PCP_GDM_TX_MASK, v);
+}
+static inline u8 get_gdm_pcp_cdm_rx(struct pcp *x)
+{
+ return FIELD_GET(GDM_PCP_CDM_RX_MASK, x->word);
+}
+static inline void set_gdm_pcp_cdm_rx(struct pcp *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_PCP_CDM_RX_MASK, v);
+}
+static inline u8 get_gdm_pcp_cdm_tx(struct pcp *x)
+{
+ return FIELD_GET(GDM_PCP_CDM_TX_MASK, x->word);
+}
+static inline void set_gdm_pcp_cdm_tx(struct pcp *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_PCP_CDM_TX_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: struct loopback
+ * This is a poorly understood / rarely used register which seems to
control GDM
+ * level loopback. It has gap, len, and channel. Channel seems to
corrispond to
+ * the QDMA channel but the meaning of gap and len are unknown. Names
include
+ * GDMA1_LPBK_CFG, GDMA2_LPBP_CFG, and REG_GDM_LPBK_CFG (Airoha).
+ */
+
+#define GDM_LPBK_GAP_MASK GENMASK(31, 24)
+#define GDM_LPBK_LEN_MASK GENMASK(23, 10)
+#define GDM_LPBK_CHAN_MASK GENMASK(8, 4)
+#define GDM_LPBK_GAP_MODE BIT(3)
+#define GDM_LPBK_LEN_MODE BIT(2)
+#define GDM_LPBK_CHAN_MODE BIT(1)
+#define GDM_LPBK_ENABLED BIT(0)
+
+static inline u8 get_gdm_lpbk_gap(struct loopback *x)
+{
+ return FIELD_GET(GDM_LPBK_GAP_MASK, x->word);
+}
+static inline void set_gdm_lpbk_gap(struct loopback *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_LPBK_GAP_MASK, v);
+}
+static inline u16 get_gdm_lpbk_len(struct loopback *x)
+{
+ return FIELD_GET(GDM_LPBK_LEN_MASK, x->word);
+}
+static inline void set_gdm_lpbk_len(struct loopback *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, GDM_LPBK_LEN_MASK, v);
+}
+static inline u8 get_gdm_lpbk_chan(struct loopback *x)
+{
+ return FIELD_GET(GDM_LPBK_CHAN_MASK, x->word);
+}
+static inline void set_gdm_lpbk_chan(struct loopback *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_LPBK_CHAN_MASK, v);
+}
+static inline bool is_gdm_lpbk_gap_mode(struct loopback *x)
+{
+ return FIELD_GET(GDM_LPBK_GAP_MODE, x->word);
+}
+static inline void set_gdm_lpbk_gap_mode(struct loopback *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_LPBK_GAP_MODE, v);
+}
+static inline bool is_gdm_lpbk_len_mode(struct loopback *x)
+{
+ return FIELD_GET(GDM_LPBK_LEN_MODE, x->word);
+}
+static inline void set_gdm_lpbk_len_mode(struct loopback *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_LPBK_LEN_MODE, v);
+}
+static inline bool is_gdm_lpbk_chan_mode(struct loopback *x)
+{
+ return FIELD_GET(GDM_LPBK_CHAN_MODE, x->word);
+}
+static inline void set_gdm_lpbk_chan_mode(struct loopback *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_LPBK_CHAN_MODE, v);
+}
+static inline bool is_gdm_lpbk_enabled(struct loopback *x)
+{
+ return FIELD_GET(GDM_LPBK_ENABLED, x->word);
+}
+static inline void set_gdm_lpbk_enabled(struct loopback *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_LPBK_ENABLED, v);
+}
+
+/**
+ * Bitfield accessors for: struct channel_retire
+ * Shut down a channel and release its resources
+ */
+
+#define GDM_CHANNEL_RETIRE_CHANNEL_MASK GENMASK(8, 4)
+#define GDM_CHANNEL_RETIRE_DONE BIT(1)
+#define GDM_CHANNEL_RETIRE_RELEASE BIT(0)
+
+
+/** Number of the channel to release */
+static inline u8 get_gdm_channel_retire_channel(struct channel_retire *x)
+{
+ return FIELD_GET(GDM_CHANNEL_RETIRE_CHANNEL_MASK, x->word);
+}
+static inline void set_gdm_channel_retire_channel(struct channel_retire
*x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_CHANNEL_RETIRE_CHANNEL_MASK, v);
+}
+
+/** Read 1 when channel release has completed */
+static inline bool is_gdm_channel_retire_done(struct channel_retire *x)
+{
+ return FIELD_GET(GDM_CHANNEL_RETIRE_DONE, x->word);
+}
+static inline void set_gdm_channel_retire_done(struct channel_retire
*x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_CHANNEL_RETIRE_DONE, v);
+}
+
+/** Write 1 to release the channel */
+static inline bool is_gdm_channel_retire_release(struct channel_retire *x)
+{
+ return FIELD_GET(GDM_CHANNEL_RETIRE_RELEASE, x->word);
+}
+static inline void set_gdm_channel_retire_release(struct channel_retire
*x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_CHANNEL_RETIRE_RELEASE, v);
+}
+
+/**
+ * Bitfield accessors for: gdm bitfield_0
+ * Apparently the lower 8 channels RX channels can be wired to forward
to any
+ * fport on receipt. This is not really ever used and the default value is
+ * disabled + all route to PPE.
+ */
+
+#define GDM_RX7_EN BIT(31)
+#define GDM_RX7_FPORT_MASK GENMASK(30, 28)
+
+static inline bool is_gdm_rx7_en(struct gdm *x)
+{
+ return FIELD_GET(GDM_RX7_EN, x->bitfield_0);
+}
+static inline void set_gdm_rx7_en(struct gdm *x, bool v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, GDM_RX7_EN, v);
+}
+static inline u8 get_gdm_rx7_fport(struct gdm *x)
+{
+ return FIELD_GET(GDM_RX7_FPORT_MASK, x->bitfield_0);
+}
+static inline void set_gdm_rx7_fport(struct gdm *x, u8 v)
+{
+ x->bitfield_0 = FIELD_SET(x->bitfield_0, GDM_RX7_FPORT_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: struct g1_cport_cfg
+ * Not well known what the CPORT config does, a few bits are
understood. Only
+ * used in GDM1, but might still be present in GDM2.
+ */
+
+#define GDM_G1_CPORT_CFG_ADD_CRC BIT(30)
+#define GDM_G1_CPORT_CFG_PAD BIT(26)
+#define GDM_G1_CPORT_CFG_PORT_XFC BIT(25)
+#define GDM_G1_CPORT_CFG_QUEUE_XFC BIT(24)
+#define GDM_G1_CPORT_CFG_UNKNOWN_MASK GENMASK(23, 0)
+
+
+/**
+ * Add the ethernet trailing CRC to incoming frames (from the switch)
this is
+ * used with fwd_cfg.strip_crc when xPON WAN and LAN are bridged
because xPON
+ * PHY always sends us a CRC.
+ */
+static inline bool is_gdm_g1_cport_cfg_add_crc(struct g1_cport_cfg *x)
+{
+ return FIELD_GET(GDM_G1_CPORT_CFG_ADD_CRC, x->word);
+}
+static inline void set_gdm_g1_cport_cfg_add_crc(struct g1_cport_cfg *x,
bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_G1_CPORT_CFG_ADD_CRC, v);
+}
+
+/**
+ * Enable padding (fe_api_set_padding, FE_CPORT_PAD), meaning is
unknown but
+ * this is always set at startup.
+ */
+static inline bool is_gdm_g1_cport_cfg_pad(struct g1_cport_cfg *x)
+{
+ return FIELD_GET(GDM_G1_CPORT_CFG_PAD, x->word);
+}
+static inline void set_gdm_g1_cport_cfg_pad(struct g1_cport_cfg *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_G1_CPORT_CFG_PAD, v);
+}
+
+/**
+ * FE_CPORT_PORT_XFC_MASK, always set on startup. Meaning is thought to
be frame
+ * engine will pause in case of congestion on port (i.e. switch)
+ */
+static inline bool is_gdm_g1_cport_cfg_port_xfc(struct g1_cport_cfg *x)
+{
+ return FIELD_GET(GDM_G1_CPORT_CFG_PORT_XFC, x->word);
+}
+static inline void set_gdm_g1_cport_cfg_port_xfc(struct g1_cport_cfg
*x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_G1_CPORT_CFG_PORT_XFC, v);
+}
+
+/**
+ * FE_CPORT_QUEUE_XFC_MASK, always cleared on startup. Meaning is
throught to be
+ * individual queue will pause in case of congestion on port (i.e. switch)
+ */
+static inline bool is_gdm_g1_cport_cfg_queue_xfc(struct g1_cport_cfg *x)
+{
+ return FIELD_GET(GDM_G1_CPORT_CFG_QUEUE_XFC, x->word);
+}
+static inline void set_gdm_g1_cport_cfg_queue_xfc(struct g1_cport_cfg
*x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_G1_CPORT_CFG_QUEUE_XFC, v);
+}
+
+/** Observed value 0x000a02, never updated, unknown meaning */
+static inline u32 get_gdm_g1_cport_cfg_unknown(struct g1_cport_cfg *x)
+{
+ return FIELD_GET(GDM_G1_CPORT_CFG_UNKNOWN_MASK, x->word);
+}
+static inline void set_gdm_g1_cport_cfg_unknown(struct g1_cport_cfg *x,
u32 v)
+{
+ x->word = FIELD_SET(x->word, GDM_G1_CPORT_CFG_UNKNOWN_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: struct clear_counters
+ */
+
+#define GDM_CL_CNT_RX BIT(1)
+#define GDM_CL_CNT_TX BIT(0)
+
+static inline void set_gdm_cl_cnt_rx(struct clear_counters *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_CL_CNT_RX, v);
+}
+static inline void set_gdm_cl_cnt_tx(struct clear_counters *x, bool v)
+{
+ x->word = FIELD_SET(x->word, GDM_CL_CNT_TX, v);
+}
+
+/**
+ * Bitfield accessors for: gdm_mymac_lsb
+ * MAC address bytes c,d,e,f (indices 2-5)
+ */
+#define GDM_MYMAC_LSB_C_MASK GENMASK(31, 24)
+#define GDM_MYMAC_LSB_D_MASK GENMASK(23, 16)
+#define GDM_MYMAC_LSB_E_MASK GENMASK(15, 8)
+#define GDM_MYMAC_LSB_F_MASK GENMASK(7, 0)
+
+static inline u8 get_gdm_mymac_lsb_c(struct gdm_mymac_lsb *x)
+{
+ return FIELD_GET(GDM_MYMAC_LSB_C_MASK, x->word);
+}
+static inline void set_gdm_mymac_lsb_c(struct gdm_mymac_lsb *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_MYMAC_LSB_C_MASK, v);
+}
+static inline u8 get_gdm_mymac_lsb_d(struct gdm_mymac_lsb *x)
+{
+ return FIELD_GET(GDM_MYMAC_LSB_D_MASK, x->word);
+}
+static inline void set_gdm_mymac_lsb_d(struct gdm_mymac_lsb *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_MYMAC_LSB_D_MASK, v);
+}
+static inline u8 get_gdm_mymac_lsb_e(struct gdm_mymac_lsb *x)
+{
+ return FIELD_GET(GDM_MYMAC_LSB_E_MASK, x->word);
+}
+static inline void set_gdm_mymac_lsb_e(struct gdm_mymac_lsb *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_MYMAC_LSB_E_MASK, v);
+}
+static inline u8 get_gdm_mymac_lsb_f(struct gdm_mymac_lsb *x)
+{
+ return FIELD_GET(GDM_MYMAC_LSB_F_MASK, x->word);
+}
+static inline void set_gdm_mymac_lsb_f(struct gdm_mymac_lsb *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_MYMAC_LSB_F_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: gdm_mymac_msb
+ * MAC address bytes a,b (indices 0-1) and LSB matching mask
+ */
+#define GDM_MYMAC_MSB_LSB_MASK_MASK GENMASK(23, 16)
+#define GDM_MYMAC_MSB_A_MASK GENMASK(15, 8)
+#define GDM_MYMAC_MSB_B_MASK GENMASK(7, 0)
+
+static inline u8 get_gdm_mymac_msb_lsb_mask(struct gdm_mymac_msb *x)
+{
+ return FIELD_GET(GDM_MYMAC_MSB_LSB_MASK_MASK, x->word);
+}
+static inline void set_gdm_mymac_msb_lsb_mask(struct gdm_mymac_msb *x,
u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_MYMAC_MSB_LSB_MASK_MASK, v);
+}
+static inline u8 get_gdm_mymac_msb_a(struct gdm_mymac_msb *x)
+{
+ return FIELD_GET(GDM_MYMAC_MSB_A_MASK, x->word);
+}
+static inline void set_gdm_mymac_msb_a(struct gdm_mymac_msb *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_MYMAC_MSB_A_MASK, v);
+}
+static inline u8 get_gdm_mymac_msb_b(struct gdm_mymac_msb *x)
+{
+ return FIELD_GET(GDM_MYMAC_MSB_B_MASK, x->word);
+}
+static inline void set_gdm_mymac_msb_b(struct gdm_mymac_msb *x, u8 v)
+{
+ x->word = FIELD_SET(x->word, GDM_MYMAC_MSB_B_MASK, v);
+}
+
+/**
+ * Bitfield accessors for: gdm_len_th
+ * Register layout (32-bit word):
+ * - Bits 31-16: OVERSIZE_LEN (packet larger than this is treated as
oversize)
+ * - Bits 15-0: RUNT_LEN (packet smaller than this is treated as a runt)
+ */
+#define GDM_LEN_TH_OVERSIZE_LEN_MASK GENMASK(31, 16)
+#define GDM_LEN_TH_RUNT_LEN_MASK GENMASK(15, 0)
+
+static inline u16 get_gdm_len_th_oversize_len(struct gdm_len_th *x)
+{
+ return FIELD_GET(GDM_LEN_TH_OVERSIZE_LEN_MASK, x->word);
+}
+static inline void set_gdm_len_th_oversize_len(struct gdm_len_th *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, GDM_LEN_TH_OVERSIZE_LEN_MASK, v);
+}
+static inline u16 get_gdm_len_th_runt_len(struct gdm_len_th *x)
+{
+ return FIELD_GET(GDM_LEN_TH_RUNT_LEN_MASK, x->word);
+}
+static inline void set_gdm_len_th_runt_len(struct gdm_len_th *x, u16 v)
+{
+ x->word = FIELD_SET(x->word, GDM_LEN_TH_RUNT_LEN_MASK, v);
+}
+
+/* EcoNet MIPS QDMA layout. The Ethernet block still exposes the same
+ * two QDMA engines and uses AIROHA_MAX_PACKET_SIZE for page-pool sizing.
+ */
+/* Internally these are properties of the QDMA engine, but they are
important
+ * to the GDM port driver because they define how QoS can be done. */
+#define ECONET_NUM_QUEUES 8
+#define ECONET_NUM_CHANNELS 32
+
+/* Number of queues reported to the kernel.
+ * Currently every chan/queue is made available. */
+#define ECONET_NUM_SOFT_QUEUES (ECONET_NUM_CHANNELS * ECONET_NUM_QUEUES)
+
+/* A chain is 1 TX ring + 1 RX ring, each QDMA has 2 chains. */
+#define QDMA_NUM_CHAINS 2
+/* Each QDMA has one queue of TX-complete notifications. */
+#define QDMA_NUM_TX_DONE 1
+/* EN751221 has one IRQ bank and one status/enable register. EN7516/EN7527
+ * expose four IRQ banks, each selecting from three shared status
registers.
+ */
+#define ECONET_MAX_QDMA_IRQS 4
+#define ECONET_QDMA_IRQ_REGS 3
+
+union econet_irq_purpose {
+ struct {
+ enum econet_irq_purpose_type {
+ IPS_INVAL = 0,
+ IPS_DONE,
+ IPS_LOW_DSCP,
+ IPS_NO_DSCP,
+
+ IPS_OVERFLOW,
+ IPS_ERR_COHERENT,
+ IPS_GPON_INT,
+ IPS_EPON_INT,
+ IPS_XPON_INT,
+ } type: 16;
+
+ /* If source is RX, TX, or DONE then chain is the number of the
queue */
+ int chain : 8;
+
+ enum econet_irq_purpose_source {
+ IPSC_RX = 1,
+ IPSC_TX,
+ IPSC_DONE,
+ IPSC_FWD,
+ IPSC_UNSPEC,
+ } source: 8;
+ };
+ u32 word;
+};
+
+static inline char *econet_irq_purpose_type_str(enum
econet_irq_purpose_type t)
+{
+ switch (t) {
+ case IPS_DONE:
+ return "DONE";
+ case IPS_LOW_DSCP:
+ return "LOW_DSCP";
+ case IPS_NO_DSCP:
+ return "NO_DSCP";
+ case IPS_OVERFLOW:
+ return "OVERFLOW";
+ case IPS_ERR_COHERENT:
+ return "ERR_COHERENT";
+ case IPS_GPON_INT:
+ return "GPON_INT";
+ case IPS_EPON_INT:
+ return "EPON_INT";
+ case IPS_XPON_INT:
+ return "XPON_INT";
+ case IPS_INVAL:
+ default:
+ return "INVAL";
+ }
+}
+
+static inline char *econet_irq_purpose_source_str(enum
econet_irq_purpose_source s)
+{
+ switch (s) {
+ case IPSC_RX:
+ return "RX";
+ case IPSC_TX:
+ return "TX";
+ case IPSC_DONE:
+ return "DONE";
+ case IPSC_FWD:
+ return "FWD";
+ case IPSC_UNSPEC:
+ return "UNSPEC";
+ default:
+ return "INVAL";
+ }
+}
+
+
+enum econet_fport {
+ DPORT_CPU = 0,
+ DPORT_GDMA1 = 1,
+ DPORT_GDMA2 = 2,
+ DPORT_UNKNOWN_3 = 3,
+ DPORT_PPE = 4,
+ DPORT_QDMA = 5,
+ DPORT_QDMA_HW = 6,
+ DPORT_DISCARD = 7,
+};
+
+/* Called in softirq context */
+int econet_rx_before_recv(struct airoha_eth *eth, struct sk_buff *skb,
+ u8 sport);
+
+struct airoha_qdma_mips_cfg {
+ int num_rx_descs[QDMA_NUM_CHAINS];
+ int num_tx_descs[QDMA_NUM_CHAINS];
+ int done_list_size[QDMA_NUM_TX_DONE];
+ int done_list_irq_threshold[QDMA_NUM_TX_DONE];
+ int num_fwd_descs;
+ int fwd_max_packet_size;
+ int fwd_low_threshold;
+ int num_channels;
+ bool rx_2b_offset;
+ const struct airoha_eth_soc_data *soc;
+};
+
+bool econet_rx_xpon_oam(struct airoha_eth *eth, u8 qdma_id,
+ struct sk_buff *skb, union desc_msg *msg);
+void econet_xpon_irq(struct airoha_eth *eth, u8 qdma_id,
+ enum airoha_xpon_mode mode);
+
+
+#define econet_rreg(reg) __extension__({ \
+ BUILD_BUG_ON(sizeof(*(reg)) != sizeof(u32)); \
+ union { typeof(*(reg)) v; u32 w; } __r = { \
+ .w = airoha_rr((void __iomem *)(reg), 0), \
+ }; \
+ __r.v; \
+ })
+
+#define econet_wreg(val, reg) do { \
+ BUILD_BUG_ON(sizeof(*(reg)) != sizeof(u32)); \
+ BUILD_BUG_ON(!__same_type(*(reg), (val))); \
+ union { typeof(*(reg)) v; u32 w; } __w = { .v = (val) }; \
+ airoha_wr((void __iomem *)(reg), 0, __w.w); \
+ } while (0)
+
+#define econet_word(val) __extension__({ \
+ union { typeof(val) v; u32 w; } __r = { .v = (val) }; \
+ BUILD_BUG_ON(sizeof(__r) != sizeof(u32)); \
+ __r.w; \
+ })
+
#endif /* AIROHA_ETH_H */
diff --git a/drivers/net/ethernet/airoha/airoha_gpon_omci.c
b/drivers/net/ethernet/airoha/airoha_gpon_omci.c
new file mode 100644
index 000000000000..d47ca3796d0e
--- /dev/null
+++ b/drivers/net/ethernet/airoha/airoha_gpon_omci.c
@@ -0,0 +1,250 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Airoha EN7523 GPON OMCI transport backend
+ */
+
+#include <linux/err.h>
+#include <linux/netdevice.h>
+#include <linux/skbuff.h>
+#include <net/xpon.h>
+#include <net/xpon/omci.h>
+
+#include "airoha_eth.h"
+#include "airoha_gpon_omci.h"
+
+static int airoha_gpon_omci_start_transport(struct omci_device *odev)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+ int ret;
+
+ ret = airoha_eth_xpon_control_start(omci->gdm_dev);
+ if (ret)
+ return ret;
+
+ ret = airoha_gpon_omci_hw_start(omci->hw_priv);
+ if (ret)
+ airoha_eth_xpon_control_stop(omci->gdm_dev);
+
+ return ret;
+}
+
+static void airoha_gpon_omci_stop_transport(struct omci_device *odev)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ airoha_gpon_omci_hw_stop(omci->hw_priv);
+ airoha_eth_xpon_control_stop(omci->gdm_dev);
+}
+
+static int airoha_gpon_omci_xmit(struct omci_device *odev,
+ struct sk_buff *skb, u16 gem_port_id)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ return airoha_eth_xmit_xpon_oam(omci->gdm_dev, skb, 0, gem_port_id);
+}
+
+static int
+airoha_gpon_omci_get_ani_topology(struct omci_device *odev,
+ struct omci_ani_topology *topology)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ return airoha_gpon_omci_hw_get_ani_topology(omci->hw_priv, topology);
+}
+
+static int airoha_gpon_omci_set_tcont(struct omci_device *odev,
+ u16 entity_id, u16 alloc_id,
+ bool valid)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ return airoha_gpon_omci_hw_set_tcont(omci->hw_priv, entity_id,
+ alloc_id, valid);
+}
+
+static int airoha_gpon_omci_set_gem_port(struct omci_device *odev,
+ u16 entity_id,
+ u16 gem_port_id,
+ u16 tcont_entity_id,
+ u8 direction,
+ bool valid,
+ bool encrypted)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ return airoha_gpon_omci_hw_set_gem_port(omci->hw_priv, entity_id,
+ gem_port_id, tcont_entity_id,
+ direction, valid, encrypted);
+}
+
+static int airoha_gpon_omci_set_uni(struct omci_device *odev,
+ u16 entity_id, bool enable)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ return airoha_gpon_omci_hw_set_uni(omci->hw_priv, entity_id, enable);
+}
+
+static int
+airoha_gpon_omci_replace_service(struct omci_device *odev,
+ const struct omci_service_config *service)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ return airoha_gpon_omci_hw_replace_service(omci->hw_priv, service);
+}
+
+static int airoha_gpon_omci_delete_service(struct omci_device *odev,
+ u32 cookie)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ return airoha_gpon_omci_hw_delete_service(omci->hw_priv, cookie);
+}
+
+static int airoha_gpon_omci_get_telemetry(struct omci_device *odev,
+ struct omci_telemetry *telemetry)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ return airoha_gpon_omci_hw_get_telemetry(omci->hw_priv, telemetry);
+}
+
+static int
+airoha_gpon_omci_set_olt_profile(struct omci_device *odev,
+ const struct omci_olt_profile_state *state)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ return airoha_gpon_omci_hw_set_olt_profile(omci->hw_priv, state);
+}
+
+static void airoha_gpon_omci_set_operational(struct omci_device *odev,
+ bool operational)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ airoha_gpon_omci_hw_set_operational(omci->hw_priv, operational);
+}
+
+static void
+airoha_gpon_omci_config_changed(struct omci_device *odev, u16 key,
+ const struct omci_identity *identity)
+{
+ struct airoha_gpon_omci *omci = omci_device_priv(odev);
+
+ airoha_gpon_omci_hw_config_changed(omci->hw_priv, key, identity);
+}
+
+static const struct omci_device_ops airoha_gpon_omci_ops = {
+ .start = airoha_gpon_omci_start_transport,
+ .stop = airoha_gpon_omci_stop_transport,
+ .xmit = airoha_gpon_omci_xmit,
+ .get_ani_topology = airoha_gpon_omci_get_ani_topology,
+ .set_tcont = airoha_gpon_omci_set_tcont,
+ .set_gem_port = airoha_gpon_omci_set_gem_port,
+ .set_uni = airoha_gpon_omci_set_uni,
+ .replace_service = airoha_gpon_omci_replace_service,
+ .delete_service = airoha_gpon_omci_delete_service,
+ .get_telemetry = airoha_gpon_omci_get_telemetry,
+ .set_olt_profile = airoha_gpon_omci_set_olt_profile,
+ .set_operational = airoha_gpon_omci_set_operational,
+ .config_changed = airoha_gpon_omci_config_changed,
+};
+
+int airoha_gpon_omci_register(struct airoha_gpon_omci *omci,
+ struct xpon_device *xpon,
+ struct net_device *gdm_dev,
+ void *hw_priv,
+ const struct omci_identity *identity)
+{
+ int ret;
+
+ omci->gdm_dev = gdm_dev;
+ omci->hw_priv = hw_priv;
+ omci->odev = omci_device_register(xpon,
+ OMCI_CAP_HW_MIC | OMCI_CAP_TELEMETRY,
+ &airoha_gpon_omci_ops, omci);
+ if (IS_ERR(omci->odev))
+ return PTR_ERR(omci->odev);
+
+ omci_device_set_identity_info(omci->odev, identity);
+ ret = omci_device_set_dying_gasp_enabled(omci->odev, true,
+ OMCI_CONFIG_SOURCE_DRIVER);
+ if (ret) {
+ omci_device_unregister(omci->odev);
+ omci->odev = NULL;
+ }
+
+ return ret;
+}
+
+int airoha_gpon_omci_start(struct airoha_gpon_omci *omci)
+{
+ return omci_device_start(omci->odev);
+}
+
+void airoha_gpon_omci_stop(struct airoha_gpon_omci *omci)
+{
+ omci_device_stop(omci->odev);
+}
+
+void airoha_gpon_omci_unregister(struct airoha_gpon_omci *omci)
+{
+ omci_device_unregister(omci->odev);
+ omci->odev = NULL;
+ omci->gdm_dev = NULL;
+ omci->hw_priv = NULL;
+}
+
+void airoha_gpon_omci_set_onu_id(struct airoha_gpon_omci *omci, u16 onu_id)
+{
+ omci_device_set_onu_id(omci->odev, onu_id);
+}
+
+void airoha_gpon_omci_set_channel(struct airoha_gpon_omci *omci,
+ u16 gem_port_id, bool valid)
+{
+ omci_device_set_channel(omci->odev, gem_port_id, valid);
+}
+
+void airoha_gpon_omci_set_state(struct airoha_gpon_omci *omci, u8 state)
+{
+ omci_device_set_state(omci->odev, state);
+}
+
+void airoha_gpon_omci_reconcile_services(struct airoha_gpon_omci *omci)
+{
+ if (omci && omci->odev)
+ omci_device_reconcile_services(omci->odev);
+}
+
+void airoha_gpon_omci_reset_session(struct airoha_gpon_omci *omci)
+{
+ omci_device_reset_session(omci->odev);
+}
+
+int airoha_gpon_omci_send_dying_gasp(struct airoha_gpon_omci *omci)
+{
+ return omci_device_send_dying_gasp(omci->odev);
+}
+
+bool airoha_gpon_omci_receive(void *data, struct sk_buff *skb, u8 channel,
+ u16 gem_port_id, u32 flags)
+{
+ struct airoha_gpon_omci *omci = data;
+ u32 omci_flags = 0;
+
+ (void)channel;
+
+ if (flags & AIROHA_XPON_OAM_RX_F_MIC_PRESENT)
+ omci_flags |= OMCI_F_MIC_PRESENT;
+ if (flags & AIROHA_XPON_OAM_RX_F_MIC_VALID)
+ omci_flags |= OMCI_F_MIC_VALID;
+ if (flags & AIROHA_XPON_OAM_RX_F_CRC_ERROR)
+ omci_flags |= OMCI_F_CRC_ERROR;
+
+ omci_device_receive(omci->odev, skb, gem_port_id, omci_flags);
+ return true;
+}
diff --git a/drivers/net/ethernet/airoha/airoha_gpon_omci.h
b/drivers/net/ethernet/airoha/airoha_gpon_omci.h
new file mode 100644
index 000000000000..a1c76db16519
--- /dev/null
+++ b/drivers/net/ethernet/airoha/airoha_gpon_omci.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _AIROHA_GPON_OMCI_H
+#define _AIROHA_GPON_OMCI_H
+
+#include <linux/types.h>
+
+struct device;
+struct net_device;
+struct omci_device;
+struct omci_identity;
+struct omci_ani_topology;
+struct omci_telemetry;
+struct omci_olt_profile_state;
+struct omci_service_config;
+struct sk_buff;
+struct xpon_device;
+
+struct airoha_gpon_omci {
+ struct omci_device *odev;
+ struct net_device *gdm_dev;
+ void *hw_priv;
+};
+
+int airoha_gpon_omci_register(struct airoha_gpon_omci *omci,
+ struct xpon_device *xpon,
+ struct net_device *gdm_dev,
+ void *hw_priv,
+ const struct omci_identity *identity);
+int airoha_gpon_omci_start(struct airoha_gpon_omci *omci);
+void airoha_gpon_omci_stop(struct airoha_gpon_omci *omci);
+void airoha_gpon_omci_unregister(struct airoha_gpon_omci *omci);
+void airoha_gpon_omci_set_onu_id(struct airoha_gpon_omci *omci, u16
onu_id);
+void airoha_gpon_omci_set_channel(struct airoha_gpon_omci *omci,
+ u16 gem_port_id, bool valid);
+void airoha_gpon_omci_set_state(struct airoha_gpon_omci *omci, u8 state);
+void airoha_gpon_omci_reconcile_services(struct airoha_gpon_omci *omci);
+void airoha_gpon_omci_reset_session(struct airoha_gpon_omci *omci);
+int airoha_gpon_omci_send_dying_gasp(struct airoha_gpon_omci *omci);
+bool airoha_gpon_omci_receive(void *data, struct sk_buff *skb, u8 channel,
+ u16 gem_port_id, u32 flags);
+
+int airoha_gpon_omci_hw_get_ani_topology(void *hw_priv,
+ struct omci_ani_topology *topology);
+int airoha_gpon_omci_hw_start(void *hw_priv);
+void airoha_gpon_omci_hw_stop(void *hw_priv);
+int airoha_gpon_omci_hw_set_tcont(void *hw_priv, u16 entity_id,
+ u16 alloc_id, bool valid);
+int airoha_gpon_omci_hw_set_gem_port(void *hw_priv, u16 entity_id,
+ u16 gem_port_id, u16 tcont_entity_id,
+ u8 direction, bool valid, bool encrypted);
+int airoha_gpon_omci_hw_set_uni(void *hw_priv, u16 entity_id, bool enable);
+int airoha_gpon_omci_hw_replace_service(void *hw_priv,
+ const struct omci_service_config *service);
+int airoha_gpon_omci_hw_delete_service(void *hw_priv, u32 cookie);
+int airoha_gpon_omci_hw_get_telemetry(void *hw_priv,
+ struct omci_telemetry *telemetry);
+int airoha_gpon_omci_hw_set_olt_profile(void *hw_priv,
+ const struct omci_olt_profile_state *state);
+void airoha_gpon_omci_hw_set_operational(void *hw_priv, bool operational);
+void airoha_gpon_omci_hw_config_changed(void *hw_priv, u16 key,
+ const struct omci_identity *identity);
+
+#endif /* _AIROHA_GPON_OMCI_H */
diff --git a/drivers/net/ethernet/airoha/airoha_npu.c
b/drivers/net/ethernet/airoha/airoha_npu.c
index 51eebdf85bec..9da3c1f729b5 100644
--- a/drivers/net/ethernet/airoha/airoha_npu.c
+++ b/drivers/net/ethernet/airoha/airoha_npu.c
@@ -4,16 +4,23 @@
* Author: Lorenzo Bianconi <lorenzo@kernel.org>
*/
+#include <linux/debugfs.h>
#include <linux/devcoredump.h>
#include <linux/firmware.h>
+#include <linux/mfd/syscon.h>
#include <linux/platform_device.h>
#include <linux/of_net.h>
#include <linux/of_platform.h>
#include <linux/of_reserved_mem.h>
#include <linux/regmap.h>
+#include <linux/seq_file.h>
+#include <linux/xarray.h>
+#include <linux/workqueue.h>
#include "airoha_eth.h"
+#define NPU_EN7523_FIRMWARE_DATA "airoha/en7523_npu_data.bin"
+#define NPU_EN7523_FIRMWARE_RV32 "airoha/en7523_npu_rv32.bin"
#define NPU_EN7581_FIRMWARE_DATA "airoha/en7581_npu_data.bin"
#define NPU_EN7581_FIRMWARE_RV32 "airoha/en7581_npu_rv32.bin"
#define NPU_EN7581_7996_FIRMWARE_DATA "airoha/en7581_MT7996_npu_data.bin"
@@ -23,6 +30,7 @@
#define NPU_EN7581_FIRMWARE_RV32_MAX_SIZE 0x200000
#define NPU_EN7581_FIRMWARE_DATA_MAX_SIZE 0x10000
#define NPU_DUMP_SIZE 512
+#define NPU_FIRMWARE_RETRY_MS 1000
/*
* Mailbox DMA payload size. Covers sizeof(struct ppe_mbox_data) (28) and
* WLAN TLV messages (header + payload); largest in-tree WLAN payload
today
@@ -32,14 +40,10 @@
#define REG_NPU_LOCAL_SRAM 0x0
-#define NPU_PC_BASE_ADDR 0x305000
-#define REG_PC_DBG(_n) (0x305000 + ((_n) * 0x100))
-
-#define NPU_CLUSTER_BASE_ADDR 0x306000
-
-#define REG_CR_BOOT_TRIGGER (NPU_CLUSTER_BASE_ADDR + 0x000)
-#define REG_CR_BOOT_CONFIG (NPU_CLUSTER_BASE_ADDR + 0x004)
-#define REG_CR_BOOT_BASE(_n) (NPU_CLUSTER_BASE_ADDR + 0x020 +
((_n) << 2))
+#define REG_PC_DBG(_soc, _n) ((_soc)->pc_base_addr + ((_n) * 0x100))
+#define REG_CR_BOOT_TRIGGER(_soc) ((_soc)->cluster_base_addr + 0x000)
+#define REG_CR_BOOT_CONFIG(_soc) ((_soc)->cluster_base_addr + 0x004)
+#define REG_CR_BOOT_BASE(_soc, _n) ((_soc)->cluster_base_addr + 0x020 +
((_n) << 2))
#define NPU_MBOX_BASE_ADDR 0x30c000
@@ -59,13 +63,13 @@
#define REG_TX_BASE(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x080)
#define REG_TX_DSCP_NUM(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x084)
-#define REG_TX_CPU_IDX(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x088)
-#define REG_TX_DMA_IDX(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x08c)
+#define REG_TX_DMA_IDX(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x088)
+#define REG_TX_CPU_IDX(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x08c)
#define REG_RX_BASE(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x180)
#define REG_RX_DSCP_NUM(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x184)
-#define REG_RX_CPU_IDX(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x188)
-#define REG_RX_DMA_IDX(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x18c)
+#define REG_RX_DMA_IDX(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x188)
+#define REG_RX_CPU_IDX(_n) (NPU_WLAN_BASE_ADDR + ((_n) << 4) +
0x18c)
#define NPU_TIMER_BASE_ADDR 0x310100
#define REG_WDT_TIMER_CTRL(_n) (NPU_TIMER_BASE_ADDR + ((_n) *
0x100))
@@ -119,10 +123,36 @@ struct airoha_npu_fw {
};
struct airoha_npu_soc_data {
+ enum airoha_npu_version version;
+ int max_cores;
+ int cluster_base_addr;
+ int pc_base_addr;
+
struct airoha_npu_fw fw_rv32;
struct airoha_npu_fw fw_data;
+
+ void (*boot_core)(struct airoha_npu *npu, struct reserved_mem *rmem);
+
+ const u32 wlan_func_set[WLAN_FUNC_SET_WAIT_MAX];
+ const u32 wlan_func_get[WLAN_FUNC_GET_WAIT_MAX];
+};
+
+struct airoha_npu_priv {
+ struct airoha_npu npu;
+ struct reserved_mem *rmem;
+ void __iomem *base;
+ void __iomem *fw_addr;
+ /* Serialize the one-time firmware load and core startup. */
+ struct mutex start_lock;
+ struct delayed_work start_work;
+ bool started;
};
+static struct airoha_npu_priv *airoha_npu_to_priv(struct airoha_npu *npu)
+{
+ return container_of(npu, struct airoha_npu_priv, npu);
+}
+
#define MBOX_MSG_FUNC_ID GENMASK(14, 11)
#define MBOX_MSG_STATIC_BUF BIT(5)
#define MBOX_MSG_STATUS GENMASK(4, 2)
@@ -166,6 +196,67 @@ struct wlan_mbox_data {
DECLARE_FLEX_ARRAY(u8, d);
};
+enum airoha_npu_debugfs_counter {
+ AIROHA_NPU_DBG_MBOX_REQUESTS,
+ AIROHA_NPU_DBG_MBOX_SUCCESS,
+ AIROHA_NPU_DBG_MBOX_ERRORS,
+ AIROHA_NPU_DBG_MBOX_TIMEOUTS,
+ AIROHA_NPU_DBG_MBOX_IRQS,
+ AIROHA_NPU_DBG_WDT_IRQS,
+ AIROHA_NPU_DBG_COUNTER_MAX,
+};
+
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+struct airoha_npu_debugfs {
+ struct airoha_npu *npu;
+ struct dentry *dir;
+ atomic64_t counters[AIROHA_NPU_DBG_COUNTER_MAX];
+ u32 retry_times;
+ u32 debug_level;
+ u32 fw_version;
+ bool fw_version_valid;
+};
+
+static DEFINE_XARRAY(airoha_npu_debugfs_ctx);
+
+static void airoha_npu_debugfs_count(struct airoha_npu *npu,
+ enum airoha_npu_debugfs_counter id)
+{
+ struct airoha_npu_debugfs *dbg;
+
+ rcu_read_lock();
+ dbg = xa_load(&airoha_npu_debugfs_ctx, (unsigned long)npu);
+ if (dbg)
+ atomic64_inc(&dbg->counters[id]);
+ rcu_read_unlock();
+}
+
+static u32 airoha_npu_debugfs_level(struct airoha_npu *npu)
+{
+ struct airoha_npu_debugfs *dbg;
+ u32 level = 0;
+
+ rcu_read_lock();
+ dbg = xa_load(&airoha_npu_debugfs_ctx, (unsigned long)npu);
+ if (dbg)
+ level = READ_ONCE(dbg->debug_level);
+ rcu_read_unlock();
+
+ return level;
+}
+#else
+static inline void
+airoha_npu_debugfs_count(struct airoha_npu *npu,
+ enum airoha_npu_debugfs_counter id)
+{
+}
+
+static inline u32 airoha_npu_debugfs_level(struct airoha_npu *npu)
+{
+ return 0;
+}
+#endif
+
static int __airoha_npu_send_msg(struct airoha_npu *npu, int func_id,
const void *data, int len, void *reply,
u16 reply_len)
@@ -180,6 +271,11 @@ static int __airoha_npu_send_msg(struct airoha_npu
*npu, int func_id,
if (reply && reply_len > len)
return -EINVAL;
+ if (npu->soc_data->version == NPU_V1 && func_id > NPU_FUNC_TR471)
+ return -EOPNOTSUPP;
+
+ airoha_npu_debugfs_count(npu, AIROHA_NPU_DBG_MBOX_REQUESTS);
+
/*
* Mailbox payloads are bidirectional (CPU request, NPU response).
* On EN7581+MT7996, streaming DMA_BIDIRECTIONAL against the
@@ -203,8 +299,17 @@ static int __airoha_npu_send_msg(struct airoha_npu
*npu, int func_id,
REG_CR_MBQ0_CTRL(3) + offset,
val, (val & MBOX_MSG_DONE),
100, 100 * MSEC_PER_SEC);
- if (!ret && FIELD_GET(MBOX_MSG_STATUS, val) != NPU_MBOX_SUCCESS)
+ if (ret) {
+ if (ret == -ETIMEDOUT)
+ airoha_npu_debugfs_count(npu,
+ AIROHA_NPU_DBG_MBOX_TIMEOUTS);
+ airoha_npu_debugfs_count(npu, AIROHA_NPU_DBG_MBOX_ERRORS);
+ } else if (FIELD_GET(MBOX_MSG_STATUS, val) != NPU_MBOX_SUCCESS) {
ret = -EINVAL;
+ airoha_npu_debugfs_count(npu, AIROHA_NPU_DBG_MBOX_ERRORS);
+ } else {
+ airoha_npu_debugfs_count(npu, AIROHA_NPU_DBG_MBOX_SUCCESS);
+ }
/* Copy the trailing reply_len bytes of the response. */
if (!ret && reply)
@@ -212,6 +317,10 @@ static int __airoha_npu_send_msg(struct airoha_npu
*npu, int func_id,
spin_unlock_bh(&core->lock);
+ if (airoha_npu_debugfs_level(npu) >= 2)
+ dev_dbg(npu->dev, "mbox func=%d size=%d status=%#x ret=%d\n",
+ func_id, len, val, ret);
+
return ret;
}
@@ -270,28 +379,23 @@ airoha_npu_load_firmware_from_dts(struct device
*dev, void __iomem *addr,
}
static int airoha_npu_run_firmware(struct device *dev, void __iomem *base,
- struct resource *res)
+ void __iomem *fw_addr)
{
const struct airoha_npu_soc_data *soc;
- void __iomem *addr;
int ret;
soc = of_device_get_match_data(dev);
if (!soc)
return -EINVAL;
- addr = devm_ioremap_resource(dev, res);
- if (IS_ERR(addr))
- return PTR_ERR(addr);
-
/* Try to load firmware images using the firmware names provided via
* dts if available.
*/
if (of_find_property(dev->of_node, "firmware-name", NULL))
- return airoha_npu_load_firmware_from_dts(dev, addr, base);
+ return airoha_npu_load_firmware_from_dts(dev, fw_addr, base);
/* Load rv32 npu firmware */
- ret = airoha_npu_load_firmware(dev, addr, soc->fw_rv32.name,
+ ret = airoha_npu_load_firmware(dev, fw_addr, soc->fw_rv32.name,
soc->fw_rv32.max_size);
if (ret)
return ret;
@@ -306,6 +410,8 @@ static irqreturn_t airoha_npu_mbox_handler(int irq,
void *npu_instance)
{
struct airoha_npu *npu = npu_instance;
+ airoha_npu_debugfs_count(npu, AIROHA_NPU_DBG_MBOX_IRQS);
+
/* clear mbox interrupt status */
regmap_write(npu->regmap, REG_CR_MBOX_INT_STATUS,
MBOX_INT_STATUS_MASK);
@@ -333,7 +439,7 @@ static void airoha_npu_wdt_work(struct work_struct
*work)
return;
c = core - &npu->cores[0];
- regmap_bulk_read(npu->regmap, REG_PC_DBG(c), val, ARRAY_SIZE(val));
+ regmap_bulk_read(npu->regmap, REG_PC_DBG(npu->soc_data, c), val,
ARRAY_SIZE(val));
snprintf(dump, NPU_DUMP_SIZE, "PC: %08x SP: %08x LR: %08x\n",
val[0], val[1], val[2]);
@@ -347,6 +453,8 @@ static irqreturn_t airoha_npu_wdt_handler(int irq,
void *core_instance)
int c = core - &npu->cores[0];
u32 val;
+ airoha_npu_debugfs_count(npu, AIROHA_NPU_DBG_WDT_IRQS);
+
regmap_set_bits(npu->regmap, REG_WDT_TIMER_CTRL(c), WDT_INTR_MASK);
if (!regmap_read(npu->regmap, REG_WDT_TIMER_CTRL(c), &val) &&
FIELD_GET(WDT_EN_MASK, val))
@@ -492,6 +600,12 @@ static int airoha_npu_wlan_msg_send(struct
airoha_npu *npu, int ifindex,
struct wlan_mbox_data *wlan_data;
int err, len;
+ if (func_id < 0 || func_id >= WLAN_FUNC_SET_WAIT_MAX)
+ return -EINVAL;
+ if (npu->soc_data->wlan_func_set[func_id] == WLAN_FUNC_SET_WAIT_MAX)
+ return -EOPNOTSUPP;
+ func_id = npu->soc_data->wlan_func_set[func_id];
+
len = sizeof(*wlan_data) + data_len;
wlan_data = kzalloc(len, gfp);
if (!wlan_data)
@@ -515,6 +629,12 @@ static int airoha_npu_wlan_msg_get(struct
airoha_npu *npu, int ifindex,
struct wlan_mbox_data *wlan_data;
int err, len;
+ if (func_id < 0 || func_id >= WLAN_FUNC_GET_WAIT_MAX)
+ return -EINVAL;
+ if (npu->soc_data->wlan_func_get[func_id] == WLAN_FUNC_GET_WAIT_MAX)
+ return -EOPNOTSUPP;
+ func_id = npu->soc_data->wlan_func_get[func_id];
+
len = sizeof(*wlan_data) + data_len;
wlan_data = kzalloc(len, gfp);
if (!wlan_data)
@@ -553,29 +673,32 @@ airoha_npu_wlan_set_reserved_memory(struct
airoha_npu *npu,
static int airoha_npu_wlan_init_memory(struct airoha_npu *npu)
{
- enum airoha_npu_wlan_set_cmd cmd = WLAN_FUNC_SET_WAIT_NPU_BAND0_ONCPU;
+ enum airoha_npu_wlan_set_cmd cmd;
u32 val = 0;
int err;
+ cmd = WLAN_FUNC_SET_WAIT_NPU_BAND0_ONCPU;
err = airoha_npu_wlan_msg_send(npu, 1, cmd, &val, sizeof(val),
GFP_KERNEL);
- if (err)
- return err;
-
- cmd = WLAN_FUNC_SET_WAIT_TX_BUF_CHECK_ADDR;
- err = airoha_npu_wlan_set_reserved_memory(npu, 0, "tx-bufid", cmd);
- if (err)
+ if (err && err != -EOPNOTSUPP) {
+ dev_err(npu->dev, "error on set NPU_BAND0 value: %d\n", err);
return err;
+ }
cmd = WLAN_FUNC_SET_WAIT_PKT_BUF_ADDR;
err = airoha_npu_wlan_set_reserved_memory(npu, 0, "pkt", cmd);
- if (err)
+ if (err) {
+ dev_err(npu->dev, "error on set pkt memory region: %d\n", err);
return err;
+ }
- cmd = WLAN_FUNC_SET_WAIT_TX_PKT_BUF_ADDR;
- err = airoha_npu_wlan_set_reserved_memory(npu, 0, "tx-pkt", cmd);
- if (err)
- return err;
+ if (of_property_match_string(npu->dev->of_node, "memory-region-names",
+ "tx-pkt") >= 0) {
+ cmd = WLAN_FUNC_SET_WAIT_TX_PKT_BUF_ADDR;
+ err = airoha_npu_wlan_set_reserved_memory(npu, 0, "tx-pkt", cmd);
+ if (err)
+ return err;
+ }
if (of_property_match_string(npu->dev->of_node, "memory-region-names",
"ba") >= 0) {
@@ -586,8 +709,12 @@ static int airoha_npu_wlan_init_memory(struct
airoha_npu *npu)
}
cmd = WLAN_FUNC_SET_WAIT_IS_FORCE_TO_CPU;
- return airoha_npu_wlan_msg_send(npu, 0, cmd, &val, sizeof(val),
+ err = airoha_npu_wlan_msg_send(npu, 0, cmd, &val, sizeof(val),
GFP_KERNEL);
+ if (err)
+ dev_err(npu->dev, "error on send force_to_cpu command: %d\n", err);
+
+ return err;
}
static u32 airoha_npu_wlan_queue_addr_get(struct airoha_npu *npu, int qid,
@@ -622,6 +749,645 @@ static void airoha_npu_wlan_irq_disable(struct
airoha_npu *npu, int q)
regmap_clear_bits(npu->regmap, REG_IRQ_RXDONE(q), NPU_IRQ_RX_MASK(q));
}
+
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+#define AIROHA_NPU_MIB_COUNT 32
+#define AIROHA_NPU_DEBUG_LEVEL_MAX 3
+
+static void airoha_npu_debugfs_print_reg(struct seq_file *m,
+ struct airoha_npu *npu,
+ const char *name, u32 reg)
+{
+ u32 val;
+ int err;
+
+ err = regmap_read(npu->regmap, reg, &val);
+ if (err)
+ seq_printf(m, "%-28s [%08x] error=%d\n", name, reg, err);
+ else
+ seq_printf(m, "%-28s [%08x] %08x\n", name, reg, val);
+}
+
+static int airoha_npu_debugfs_status_show(struct seq_file *m, void
*private)
+{
+ struct airoha_npu_debugfs *dbg = m->private;
+ struct airoha_npu *npu = dbg->npu;
+ u32 boot_config = 0, boot_trigger = 0, irq_status = 0;
+ int i;
+
+ regmap_read(npu->regmap, REG_CR_BOOT_CONFIG(npu->soc_data),
+ &boot_config);
+ regmap_read(npu->regmap, REG_CR_BOOT_TRIGGER(npu->soc_data),
+ &boot_trigger);
+ regmap_read(npu->regmap, REG_IRQ_STATUS, &irq_status);
+
+ seq_printf(m, "device: %s\n", dev_name(npu->dev));
+ seq_printf(m, "npu_interface: v%u\n", npu->soc_data->version + 1);
+ seq_printf(m, "cores: %d\n", npu->soc_data->max_cores);
+ if (dbg->fw_version_valid)
+ seq_printf(m, "firmware: v%u.%u (0x%08x)\n",
+ dbg->fw_version >> 16, dbg->fw_version & 0xffff,
+ dbg->fw_version);
+ else
+ seq_puts(m, "firmware: unavailable\n");
+ seq_printf(m, "boot_config: 0x%08x\n", boot_config);
+ seq_printf(m, "boot_trigger: 0x%08x\n", boot_trigger);
+ seq_printf(m, "wlan_irq_status: 0x%08x\n", irq_status);
+ seq_printf(m, "retry_times: %u\n", READ_ONCE(dbg->retry_times));
+ seq_printf(m, "debug_level: %u\n", READ_ONCE(dbg->debug_level));
+
+ for (i = 0; i < (int)ARRAY_SIZE(npu->irqs); i++)
+ seq_printf(m, "wlan_irq%d: %d\n", i, npu->irqs[i]);
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_npu_debugfs_status);
+
+static int airoha_npu_debugfs_rings_show(struct seq_file *m, void *private)
+{
+ struct airoha_npu_debugfs *dbg = m->private;
+ struct airoha_npu *npu = dbg->npu;
+ u32 base, count, dma_idx, cpu_idx;
+ int i;
+
+ seq_puts(m, "ring direction base descriptors dma_idx cpu_idx\n");
+
+ for (i = 0; i < 2; i++) {
+ regmap_read(npu->regmap, REG_RX_BASE(i), &base);
+ regmap_read(npu->regmap, REG_RX_DSCP_NUM(i), &count);
+ regmap_read(npu->regmap, REG_RX_DMA_IDX(i), &dma_idx);
+ regmap_read(npu->regmap, REG_RX_CPU_IDX(i), &cpu_idx);
+ seq_printf(m, "%4d RX 0x%08x %11u %7u %7u\n",
+ i, base, count, dma_idx, cpu_idx);
+ }
+
+ for (i = 0; i < 2; i++) {
+ int ring = i + 2;
+
+ regmap_read(npu->regmap, REG_TX_BASE(ring), &base);
+ regmap_read(npu->regmap, REG_TX_DSCP_NUM(ring), &count);
+ regmap_read(npu->regmap, REG_TX_DMA_IDX(ring), &dma_idx);
+ regmap_read(npu->regmap, REG_TX_CPU_IDX(ring), &cpu_idx);
+ seq_printf(m, "%4d TX 0x%08x %11u %7u %7u\n",
+ ring, base, count, dma_idx, cpu_idx);
+ }
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_npu_debugfs_rings);
+
+static int airoha_npu_debugfs_registers_show(struct seq_file *m,
+ void *private)
+{
+ struct airoha_npu_debugfs *dbg = m->private;
+ struct airoha_npu *npu = dbg->npu;
+ char name[32];
+ int i, ring;
+
+ airoha_npu_debugfs_print_reg(m, npu, "MBOX_INT_STATUS",
+ REG_CR_MBOX_INT_STATUS);
+ for (i = 0; i < 4; i++) {
+ snprintf(name, sizeof(name), "MBOX_INT_MASK%d", i);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_CR_MBOX_INT_MASK(i));
+ snprintf(name, sizeof(name), "MBQ0_CTRL%d", i);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_CR_MBQ0_CTRL(i));
+ snprintf(name, sizeof(name), "MBQ8_CTRL%d", i);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_CR_MBQ8_CTRL(i));
+ }
+
+ airoha_npu_debugfs_print_reg(m, npu, "WLAN_IRQ_STATUS",
+ REG_IRQ_STATUS);
+ for (i = 0; i < 2; i++) {
+ snprintf(name, sizeof(name), "WLAN_IRQ_RXDONE%d", i);
+ airoha_npu_debugfs_print_reg(m, npu, name, REG_IRQ_RXDONE(i));
+
+ snprintf(name, sizeof(name), "RX%d_BASE", i);
+ airoha_npu_debugfs_print_reg(m, npu, name, REG_RX_BASE(i));
+ snprintf(name, sizeof(name), "RX%d_DSCP_NUM", i);
+ airoha_npu_debugfs_print_reg(m, npu, name, REG_RX_DSCP_NUM(i));
+ snprintf(name, sizeof(name), "RX%d_DMA_IDX", i);
+ airoha_npu_debugfs_print_reg(m, npu, name, REG_RX_DMA_IDX(i));
+ snprintf(name, sizeof(name), "RX%d_CPU_IDX", i);
+ airoha_npu_debugfs_print_reg(m, npu, name, REG_RX_CPU_IDX(i));
+
+ ring = i + 2;
+ snprintf(name, sizeof(name), "TX%d_BASE", ring);
+ airoha_npu_debugfs_print_reg(m, npu, name, REG_TX_BASE(ring));
+ snprintf(name, sizeof(name), "TX%d_DSCP_NUM", ring);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_TX_DSCP_NUM(ring));
+ snprintf(name, sizeof(name), "TX%d_DMA_IDX", ring);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_TX_DMA_IDX(ring));
+ snprintf(name, sizeof(name), "TX%d_CPU_IDX", ring);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_TX_CPU_IDX(ring));
+ }
+
+ airoha_npu_debugfs_print_reg(m, npu, "BOOT_TRIGGER",
+ REG_CR_BOOT_TRIGGER(npu->soc_data));
+ airoha_npu_debugfs_print_reg(m, npu, "BOOT_CONFIG",
+ REG_CR_BOOT_CONFIG(npu->soc_data));
+ for (i = 0; i < npu->soc_data->max_cores; i++) {
+ snprintf(name, sizeof(name), "CORE%d_BOOT_BASE", i);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_CR_BOOT_BASE(npu->soc_data, i));
+ snprintf(name, sizeof(name), "CORE%d_PC", i);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_PC_DBG(npu->soc_data, i));
+ snprintf(name, sizeof(name), "CORE%d_SP", i);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_PC_DBG(npu->soc_data, i) + 4);
+ snprintf(name, sizeof(name), "CORE%d_LR", i);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_PC_DBG(npu->soc_data, i) + 8);
+
+ if (npu->soc_data->version >= NPU_V2) {
+ snprintf(name, sizeof(name), "CORE%d_WDT_CTRL", i);
+ airoha_npu_debugfs_print_reg(m, npu, name,
+ REG_WDT_TIMER_CTRL(i));
+ }
+ }
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_npu_debugfs_registers);
+
+static int airoha_npu_debugfs_mib_show(struct seq_file *m, void *private)
+{
+ struct airoha_npu_debugfs *dbg = m->private;
+ struct airoha_npu *npu = dbg->npu;
+ u32 val;
+ int i, err;
+
+ for (i = 0; i < AIROHA_NPU_MIB_COUNT; i++) {
+ err = regmap_read(npu->regmap, REG_CR_NPU_MIB(i), &val);
+ if (err)
+ seq_printf(m, "%02d: error=%d\n", i, err);
+ else
+ seq_printf(m, "%02d: 0x%08x (%u)\n", i, val, val);
+ }
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_npu_debugfs_mib);
+
+static int airoha_npu_debugfs_counters_show(struct seq_file *m,
+ void *private)
+{
+ static const char * const names[AIROHA_NPU_DBG_COUNTER_MAX] = {
+ [AIROHA_NPU_DBG_MBOX_REQUESTS] = "mbox_requests",
+ [AIROHA_NPU_DBG_MBOX_SUCCESS] = "mbox_success",
+ [AIROHA_NPU_DBG_MBOX_ERRORS] = "mbox_errors",
+ [AIROHA_NPU_DBG_MBOX_TIMEOUTS] = "mbox_timeouts",
+ [AIROHA_NPU_DBG_MBOX_IRQS] = "mbox_irqs",
+ [AIROHA_NPU_DBG_WDT_IRQS] = "wdt_irqs",
+ };
+ struct airoha_npu_debugfs *dbg = m->private;
+ int i;
+
+ for (i = 0; i < AIROHA_NPU_DBG_COUNTER_MAX; i++)
+ seq_printf(m, "%-20s %lld\n", names[i],
+ (long long)atomic64_read(&dbg->counters[i]));
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_npu_debugfs_counters);
+
+static void airoha_npu_debugfs_json_string(struct seq_file *m,
+ const char *str)
+{
+ const unsigned char *p = (const unsigned char *)str;
+
+ seq_putc(m, '"');
+ for (; *p; p++) {
+ switch (*p) {
+ case '"':
+ seq_puts(m, "\\\"");
+ break;
+ case '\\':
+ seq_puts(m, "\\\\");
+ break;
+ case '\b':
+ seq_puts(m, "\\b");
+ break;
+ case '\f':
+ seq_puts(m, "\\f");
+ break;
+ case '\n':
+ seq_puts(m, "\\n");
+ break;
+ case '\r':
+ seq_puts(m, "\\r");
+ break;
+ case '\t':
+ seq_puts(m, "\\t");
+ break;
+ default:
+ if (*p < 0x20)
+ seq_printf(m, "\\u%04x", *p);
+ else
+ seq_putc(m, *p);
+ break;
+ }
+ }
+ seq_putc(m, '"');
+}
+
+static void airoha_npu_debugfs_json_nullable_u32(struct seq_file *m,
+ const char *name,
+ u32 value, int err,
+ bool comma)
+{
+ seq_printf(m, "\"%s\": ", name);
+ if (err)
+ seq_puts(m, "null");
+ else
+ seq_printf(m, "%u", value);
+
+ if (comma)
+ seq_puts(m, ", ");
+}
+
+static void airoha_npu_debugfs_json_u32(struct seq_file *m,
+ const char *name, u32 value,
+ int err, bool comma)
+{
+ airoha_npu_debugfs_json_nullable_u32(m, name, value, err, true);
+ seq_printf(m, "\"%s_hex\": ", name);
+ if (err)
+ seq_puts(m, "null");
+ else
+ seq_printf(m, "\"0x%08x\"", value);
+
+ if (comma)
+ seq_puts(m, ", ");
+}
+
+static void airoha_npu_debugfs_json_reg(struct seq_file *m,
+ struct airoha_npu *npu,
+ const char *name, u32 reg,
+ bool *first)
+{
+ u32 value = 0;
+ int err;
+
+ err = regmap_read(npu->regmap, reg, &value);
+ if (!*first)
+ seq_puts(m, ",\n");
+ *first = false;
+
+ seq_puts(m, " ");
+ airoha_npu_debugfs_json_string(m, name);
+ seq_printf(m, ": {\"address\": %u, "
+ "\"address_hex\": \"0x%08x\", ", reg, reg);
+ if (err)
+ seq_printf(m, "\"value\": null, \"value_hex\": null, "
+ "\"error\": %d}", err);
+ else
+ seq_printf(m, "\"value\": %u, "
+ "\"value_hex\": \"0x%08x\", "
+ "\"error\": 0}", value, value);
+}
+
+static void airoha_npu_debugfs_json_ring(struct seq_file *m,
+ struct airoha_npu *npu,
+ bool rx, int ring, bool comma)
+{
+ u32 base = 0, count = 0, dma_idx = 0, cpu_idx = 0;
+ int base_err, count_err, dma_err, cpu_err;
+
+ if (rx) {
+ base_err = regmap_read(npu->regmap, REG_RX_BASE(ring), &base);
+ count_err = regmap_read(npu->regmap, REG_RX_DSCP_NUM(ring),
+ &count);
+ dma_err = regmap_read(npu->regmap, REG_RX_DMA_IDX(ring),
+ &dma_idx);
+ cpu_err = regmap_read(npu->regmap, REG_RX_CPU_IDX(ring),
+ &cpu_idx);
+ } else {
+ base_err = regmap_read(npu->regmap, REG_TX_BASE(ring), &base);
+ count_err = regmap_read(npu->regmap, REG_TX_DSCP_NUM(ring),
+ &count);
+ dma_err = regmap_read(npu->regmap, REG_TX_DMA_IDX(ring),
+ &dma_idx);
+ cpu_err = regmap_read(npu->regmap, REG_TX_CPU_IDX(ring),
+ &cpu_idx);
+ }
+
+ seq_printf(m, " {\"name\": \"%s%d\", "
+ "\"direction\": \"%s\", \"index\": %d, ",
+ rx ? "rx" : "tx", ring, rx ? "rx" : "tx", ring);
+ airoha_npu_debugfs_json_u32(m, "base", base, base_err, true);
+ airoha_npu_debugfs_json_nullable_u32(m, "descriptors", count,
+ count_err, true);
+ airoha_npu_debugfs_json_nullable_u32(m, "dma_idx", dma_idx,
+ dma_err, true);
+ airoha_npu_debugfs_json_nullable_u32(m, "cpu_idx", cpu_idx,
+ cpu_err, true);
+ seq_printf(m, "\"errors\": {\"base\": %d, "
+ "\"descriptors\": %d, \"dma_idx\": %d, "
+ "\"cpu_idx\": %d}}%s\n",
+ base_err, count_err, dma_err, cpu_err, comma ? "," : "");
+}
+
+static int airoha_npu_debugfs_json_show(struct seq_file *m, void *private)
+{
+ static const char * const counter_names[AIROHA_NPU_DBG_COUNTER_MAX] = {
+ [AIROHA_NPU_DBG_MBOX_REQUESTS] = "mbox_requests",
+ [AIROHA_NPU_DBG_MBOX_SUCCESS] = "mbox_success",
+ [AIROHA_NPU_DBG_MBOX_ERRORS] = "mbox_errors",
+ [AIROHA_NPU_DBG_MBOX_TIMEOUTS] = "mbox_timeouts",
+ [AIROHA_NPU_DBG_MBOX_IRQS] = "mbox_irqs",
+ [AIROHA_NPU_DBG_WDT_IRQS] = "wdt_irqs",
+ };
+ struct airoha_npu_debugfs *dbg = m->private;
+ struct airoha_npu *npu = dbg->npu;
+ u32 boot_config = 0, boot_trigger = 0, irq_status = 0;
+ int boot_config_err, boot_trigger_err, irq_status_err;
+ bool first = true;
+ char name[32];
+ int i, ring, err;
+ u32 value;
+
+ (void)private;
+
+ boot_config_err = regmap_read(npu->regmap,
+ REG_CR_BOOT_CONFIG(npu->soc_data),
+ &boot_config);
+ boot_trigger_err = regmap_read(npu->regmap,
+ REG_CR_BOOT_TRIGGER(npu->soc_data),
+ &boot_trigger);
+ irq_status_err = regmap_read(npu->regmap, REG_IRQ_STATUS, &irq_status);
+
+ seq_puts(m, "{\n \"schema_version\": 1,\n \"device\": ");
+ airoha_npu_debugfs_json_string(m, dev_name(npu->dev));
+ seq_printf(m, ",\n \"interface\": {\"version\": %u, "
+ "\"cores\": %d},\n \"firmware\": ",
+ npu->soc_data->version + 1, npu->soc_data->max_cores);
+ if (dbg->fw_version_valid)
+ seq_printf(m, "{\"major\": %u, \"minor\": %u, "
+ "\"raw\": %u, \"raw_hex\": \"0x%08x\"}",
+ dbg->fw_version >> 16, dbg->fw_version & 0xffff,
+ dbg->fw_version, dbg->fw_version);
+ else
+ seq_puts(m, "null");
+
+ seq_printf(m, ",\n \"controls\": {\"retry_times\": %u, "
+ "\"debug_level\": %u},\n \"boot\": {",
+ READ_ONCE(dbg->retry_times), READ_ONCE(dbg->debug_level));
+ airoha_npu_debugfs_json_u32(m, "config", boot_config,
+ boot_config_err, true);
+ airoha_npu_debugfs_json_u32(m, "trigger", boot_trigger,
+ boot_trigger_err, true);
+ seq_printf(m, "\"errors\": {\"config\": %d, "
+ "\"trigger\": %d}},\n \"interrupts\": {",
+ boot_config_err, boot_trigger_err);
+ airoha_npu_debugfs_json_u32(m, "wlan_status", irq_status,
+ irq_status_err, true);
+ seq_printf(m, "\"status_error\": %d, \"wlan_irqs\": [",
+ irq_status_err);
+ for (i = 0; i < (int)ARRAY_SIZE(npu->irqs); i++)
+ seq_printf(m, "%s%d", i ? ", " : "", npu->irqs[i]);
+ seq_puts(m, "]},\n \"rings\": [\n");
+ airoha_npu_debugfs_json_ring(m, npu, true, 0, true);
+ airoha_npu_debugfs_json_ring(m, npu, true, 1, true);
+ airoha_npu_debugfs_json_ring(m, npu, false, 2, true);
+ airoha_npu_debugfs_json_ring(m, npu, false, 3, false);
+
+ seq_puts(m, " ],\n \"counters\": {");
+ for (i = 0; i < AIROHA_NPU_DBG_COUNTER_MAX; i++)
+ seq_printf(m, "%s\"%s\": %lld",
+ i ? ", " : "", counter_names[i],
+ (long long)atomic64_read(&dbg->counters[i]));
+
+ seq_puts(m, "},\n \"mib\": [\n");
+ for (i = 0; i < AIROHA_NPU_MIB_COUNT; i++) {
+ value = 0;
+ err = regmap_read(npu->regmap, REG_CR_NPU_MIB(i), &value);
+ seq_printf(m, " {\"index\": %d, ", i);
+ if (err)
+ seq_printf(m, "\"value\": null, "
+ "\"value_hex\": null, \"error\": %d}",
+ err);
+ else
+ seq_printf(m, "\"value\": %u, "
+ "\"value_hex\": \"0x%08x\", "
+ "\"error\": 0}", value, value);
+ seq_printf(m, "%s\n",
+ i + 1 == AIROHA_NPU_MIB_COUNT ? "" : ",");
+ }
+
+ seq_puts(m, " ],\n \"registers\": {\n");
+ airoha_npu_debugfs_json_reg(m, npu, "MBOX_INT_STATUS",
+ REG_CR_MBOX_INT_STATUS, &first);
+ for (i = 0; i < 4; i++) {
+ snprintf(name, sizeof(name), "MBOX_INT_MASK%d", i);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_CR_MBOX_INT_MASK(i), &first);
+ snprintf(name, sizeof(name), "MBQ0_CTRL%d", i);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_CR_MBQ0_CTRL(i), &first);
+ snprintf(name, sizeof(name), "MBQ8_CTRL%d", i);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_CR_MBQ8_CTRL(i), &first);
+ }
+
+ airoha_npu_debugfs_json_reg(m, npu, "WLAN_IRQ_STATUS",
+ REG_IRQ_STATUS, &first);
+ for (i = 0; i < 2; i++) {
+ snprintf(name, sizeof(name), "WLAN_IRQ_RXDONE%d", i);
+ airoha_npu_debugfs_json_reg(m, npu, name, REG_IRQ_RXDONE(i),
+ &first);
+
+ snprintf(name, sizeof(name), "RX%d_BASE", i);
+ airoha_npu_debugfs_json_reg(m, npu, name, REG_RX_BASE(i),
+ &first);
+ snprintf(name, sizeof(name), "RX%d_DSCP_NUM", i);
+ airoha_npu_debugfs_json_reg(m, npu, name, REG_RX_DSCP_NUM(i),
+ &first);
+ snprintf(name, sizeof(name), "RX%d_DMA_IDX", i);
+ airoha_npu_debugfs_json_reg(m, npu, name, REG_RX_DMA_IDX(i),
+ &first);
+ snprintf(name, sizeof(name), "RX%d_CPU_IDX", i);
+ airoha_npu_debugfs_json_reg(m, npu, name, REG_RX_CPU_IDX(i),
+ &first);
+
+ ring = i + 2;
+ snprintf(name, sizeof(name), "TX%d_BASE", ring);
+ airoha_npu_debugfs_json_reg(m, npu, name, REG_TX_BASE(ring),
+ &first);
+ snprintf(name, sizeof(name), "TX%d_DSCP_NUM", ring);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_TX_DSCP_NUM(ring), &first);
+ snprintf(name, sizeof(name), "TX%d_DMA_IDX", ring);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_TX_DMA_IDX(ring), &first);
+ snprintf(name, sizeof(name), "TX%d_CPU_IDX", ring);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_TX_CPU_IDX(ring), &first);
+ }
+
+ airoha_npu_debugfs_json_reg(m, npu, "BOOT_TRIGGER",
+ REG_CR_BOOT_TRIGGER(npu->soc_data), &first);
+ airoha_npu_debugfs_json_reg(m, npu, "BOOT_CONFIG",
+ REG_CR_BOOT_CONFIG(npu->soc_data), &first);
+ for (i = 0; i < npu->soc_data->max_cores; i++) {
+ snprintf(name, sizeof(name), "CORE%d_BOOT_BASE", i);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_CR_BOOT_BASE(npu->soc_data, i),
+ &first);
+ snprintf(name, sizeof(name), "CORE%d_PC", i);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_PC_DBG(npu->soc_data, i), &first);
+ snprintf(name, sizeof(name), "CORE%d_SP", i);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_PC_DBG(npu->soc_data, i) + 4,
+ &first);
+ snprintf(name, sizeof(name), "CORE%d_LR", i);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_PC_DBG(npu->soc_data, i) + 8,
+ &first);
+
+ if (npu->soc_data->version >= NPU_V2) {
+ snprintf(name, sizeof(name), "CORE%d_WDT_CTRL", i);
+ airoha_npu_debugfs_json_reg(m, npu, name,
+ REG_WDT_TIMER_CTRL(i), &first);
+ }
+ }
+
+ seq_puts(m, "\n }\n}\n");
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_npu_debugfs_json);
+
+static int airoha_npu_debugfs_retry_get(void *data, u64 *val)
+{
+ struct airoha_npu_debugfs *dbg = data;
+
+ *val = READ_ONCE(dbg->retry_times);
+ return 0;
+}
+
+static int airoha_npu_debugfs_retry_set(void *data, u64 val)
+{
+ struct airoha_npu_debugfs *dbg = data;
+ u32 retry_times;
+ int err;
+
+ if (!val || val > U16_MAX)
+ return -ERANGE;
+
+ retry_times = val;
+ err = airoha_npu_wlan_msg_send(dbg->npu, 0,
+ WLAN_FUNC_SET_WAIT_ERROR_RETRY_TIMES,
+ &retry_times, sizeof(retry_times),
+ GFP_KERNEL);
+ if (err)
+ return err;
+
+ WRITE_ONCE(dbg->retry_times, retry_times);
+ return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(airoha_npu_debugfs_retry_fops,
+ airoha_npu_debugfs_retry_get,
+ airoha_npu_debugfs_retry_set, "%llu\n");
+
+static int airoha_npu_debugfs_level_get(void *data, u64 *val)
+{
+ struct airoha_npu_debugfs *dbg = data;
+
+ *val = READ_ONCE(dbg->debug_level);
+ return 0;
+}
+
+static int airoha_npu_debugfs_level_set(void *data, u64 val)
+{
+ struct airoha_npu_debugfs *dbg = data;
+
+ if (val > AIROHA_NPU_DEBUG_LEVEL_MAX)
+ return -ERANGE;
+
+ WRITE_ONCE(dbg->debug_level, val);
+ return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(airoha_npu_debugfs_level_fops,
+ airoha_npu_debugfs_level_get,
+ airoha_npu_debugfs_level_set, "%llu\n");
+
+static int airoha_npu_debugfs_init(struct airoha_npu *npu, u32 fw_version,
+ bool fw_version_valid)
+{
+ struct airoha_npu_debugfs *dbg;
+ int err;
+
+ dbg = devm_kzalloc(npu->dev, sizeof(*dbg), GFP_KERNEL);
+ if (!dbg)
+ return -ENOMEM;
+
+ dbg->npu = npu;
+ dbg->retry_times = 3;
+ dbg->fw_version = fw_version;
+ dbg->fw_version_valid = fw_version_valid;
+
+ dbg->dir = debugfs_create_dir("airoha_npu", NULL);
+ if (IS_ERR_OR_NULL(dbg->dir))
+ return dbg->dir ? PTR_ERR(dbg->dir) : -ENOMEM;
+
+ err = xa_err(xa_store(&airoha_npu_debugfs_ctx,
+ (unsigned long)npu, dbg, GFP_KERNEL));
+ if (err) {
+ debugfs_remove_recursive(dbg->dir);
+ return err;
+ }
+
+ debugfs_create_file("status", 0444, dbg->dir, dbg,
+ &airoha_npu_debugfs_status_fops);
+ debugfs_create_file("rings", 0444, dbg->dir, dbg,
+ &airoha_npu_debugfs_rings_fops);
+ debugfs_create_file("registers", 0444, dbg->dir, dbg,
+ &airoha_npu_debugfs_registers_fops);
+ debugfs_create_file("mib", 0444, dbg->dir, dbg,
+ &airoha_npu_debugfs_mib_fops);
+ debugfs_create_file("counters", 0444, dbg->dir, dbg,
+ &airoha_npu_debugfs_counters_fops);
+ debugfs_create_file("debug.json", 0444, dbg->dir, dbg,
+ &airoha_npu_debugfs_json_fops);
+ debugfs_create_file("retry_times", 0600, dbg->dir, dbg,
+ &airoha_npu_debugfs_retry_fops);
+ debugfs_create_file("debug_level", 0600, dbg->dir, dbg,
+ &airoha_npu_debugfs_level_fops);
+
+ return 0;
+}
+
+static void airoha_npu_debugfs_remove(struct airoha_npu *npu)
+{
+ struct airoha_npu_debugfs *dbg;
+
+ dbg = xa_erase(&airoha_npu_debugfs_ctx, (unsigned long)npu);
+ if (!dbg)
+ return;
+
+ synchronize_rcu();
+ debugfs_remove_recursive(dbg->dir);
+}
+#else
+static inline int airoha_npu_debugfs_init(struct airoha_npu *npu,
+ u32 fw_version,
+ bool fw_version_valid)
+{
+ return 0;
+}
+
+static inline void airoha_npu_debugfs_remove(struct airoha_npu *npu)
+{
+}
+#endif
+
struct airoha_npu *airoha_npu_get(struct device *dev)
{
struct platform_device *pdev;
@@ -649,7 +1415,13 @@ struct airoha_npu *airoha_npu_get(struct device *dev)
npu = platform_get_drvdata(pdev);
if (!npu) {
- npu = ERR_PTR(-ENODEV);
+ npu = ERR_PTR(-EPROBE_DEFER);
+ goto error_module_put;
+ }
+
+ /* Pairs with the release after the NPU cores have been booted. */
+ if (!smp_load_acquire(&airoha_npu_to_priv(npu)->started)) {
+ npu = ERR_PTR(-EPROBE_DEFER);
goto error_module_put;
}
@@ -679,7 +1451,139 @@ void airoha_npu_put(struct airoha_npu *npu)
}
EXPORT_SYMBOL_GPL(airoha_npu_put);
+static void airoha_boot_core_v1(struct airoha_npu *npu,
+ struct reserved_mem *rmem)
+{
+ u32 val = 0x1, boot = 0x1;
+ int core;
+
+ /* setting booting address */
+ for (core = 0; core < npu->soc_data->max_cores; core++) {
+ regmap_write(npu->regmap, REG_CR_BOOT_BASE(npu->soc_data,
core), rmem->base);
+ usleep_range(1000, 2000);
+
+ if (core > 0) {
+ boot = 0x2;
+ regmap_read(npu->regmap, REG_CR_BOOT_CONFIG(npu->soc_data),
&val);
+ val &= 0xff; /* keep old enabled bits */
+ val |= (0x1 << core); /* add new enable bit for CoreX */
+ val |= (0x100 << core); /* add reboot bit for CoreX */
+ }
+
+ regmap_write(npu->regmap, REG_CR_BOOT_CONFIG(npu->soc_data), val);
+ regmap_write(npu->regmap, REG_CR_BOOT_TRIGGER(npu->soc_data),
boot);
+
+ msleep(100);
+ }
+}
+
+static void airoha_boot_core_v2(struct airoha_npu *npu,
+ struct reserved_mem *rmem)
+{
+ int i;
+
+ /* setting booting address */
+ for (i = 0; i < npu->soc_data->max_cores; i++)
+ regmap_write(npu->regmap, REG_CR_BOOT_BASE(npu->soc_data, i),
rmem->base);
+ usleep_range(1000, 2000);
+
+ /* enable NPU cores */
+ regmap_write(npu->regmap, REG_CR_BOOT_CONFIG(npu->soc_data), 0xff);
+ regmap_write(npu->regmap, REG_CR_BOOT_TRIGGER(npu->soc_data), 0x1);
+ msleep(100);
+}
+
+static bool is_fpga(struct regmap *scuclk)
+{
+ u32 val;
+
+ regmap_read(scuclk, 0x9c, &val);
+ return !(val & BIT(0));
+}
+
+static int sram_size_of_l2c(struct regmap *scuclk)
+{
+ u32 val;
+
+ regmap_read(scuclk, 0x280, &val);
+ return val;
+}
+
+static const struct airoha_npu_soc_data en7523_npu_soc_data = {
+ .version = NPU_V1,
+ .max_cores = 4,
+ .cluster_base_addr = 0x308000,
+ .pc_base_addr = 0x308800,
+ .boot_core = airoha_boot_core_v1,
+ .fw_rv32 = {
+ .name = NPU_EN7523_FIRMWARE_RV32,
+ .max_size = NPU_EN7581_FIRMWARE_RV32_MAX_SIZE,
+ },
+ .fw_data = {
+ .name = NPU_EN7523_FIRMWARE_DATA,
+ .max_size = NPU_EN7581_FIRMWARE_DATA_MAX_SIZE,
+ },
+ .wlan_func_get = {
+ [WLAN_FUNC_GET_WAIT_NPU_INFO] = 0,
+ [WLAN_FUNC_GET_WAIT_LAST_RATE] = 1,
+ [WLAN_FUNC_GET_WAIT_COUNTER] = 2,
+ [WLAN_FUNC_GET_WAIT_DBG_COUNTER] = 3,
+ [WLAN_FUNC_GET_WAIT_RXDESC_BASE] = 4,
+ [WLAN_FUNC_GET_WAIT_WCID_DBG_COUNTER] = 5,
+
+ /* Not implemented by the EN7523 V1.001/V1.002 firmware. */
+ [WLAN_FUNC_GET_WAIT_DMA_ADDR] = WLAN_FUNC_GET_WAIT_MAX,
+ [WLAN_FUNC_GET_WAIT_RING_SIZE] = WLAN_FUNC_GET_WAIT_MAX,
+ [WLAN_FUNC_GET_WAIT_NPU_SUPPORT_MAP] = WLAN_FUNC_GET_WAIT_MAX,
+ [WLAN_FUNC_GET_WAIT_MDC_LOCK_ADDRESS] = WLAN_FUNC_GET_WAIT_MAX,
+ [WLAN_FUNC_GET_WAIT_NPU_VERSION] = WLAN_FUNC_GET_WAIT_MAX,
+ },
+ .wlan_func_set = {
+ [WLAN_FUNC_SET_WAIT_PCIE_ADDR] = 0,
+ [WLAN_FUNC_SET_WAIT_DESC] = 1,
+ [WLAN_FUNC_SET_WAIT_NPU_INIT_DONE] = 2,
+ [WLAN_FUNC_SET_WAIT_TRAN_TO_CPU] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_BA_WIN_SIZE] = 4,
+ [WLAN_FUNC_SET_WAIT_DRIVER_MODEL] = 5,
+ [WLAN_FUNC_SET_WAIT_DEL_STA] = 6,
+ [WLAN_FUNC_SET_WAIT_DRAM_BA_NODE_ADDR] = 7,
+ [WLAN_FUNC_SET_WAIT_PKT_BUF_ADDR] = 8,
+ [WLAN_FUNC_SET_WAIT_IS_TEST_NOBA] = 9,
+ [WLAN_FUNC_SET_WAIT_FLUSHONE_TIMEOUT] = 10,
+ [WLAN_FUNC_SET_WAIT_FLUSHALL_TIMEOUT] = 11,
+ [WLAN_FUNC_SET_WAIT_IS_FORCE_TO_CPU] = 12,
+ [WLAN_FUNC_SET_WAIT_PCIE_STATE] = 13,
+ [WLAN_FUNC_SET_WAIT_PCIE_PORT_TYPE] = 14,
+ [WLAN_FUNC_SET_WAIT_ERROR_RETRY_TIMES] = 15,
+ [WLAN_FUNC_SET_WAIT_BAR_INFO] = 16,
+ [WLAN_FUNC_SET_WAIT_FAST_FLAG] = 17,
+ [WLAN_FUNC_SET_WAIT_NPU_BAND0_ONCPU] = 18,
+ [WLAN_FUNC_SET_WAIT_TX_RING_PCIE_ADDR] = 19,
+ [WLAN_FUNC_SET_WAIT_TX_DESC_HW_BASE] = 20,
+ [WLAN_FUNC_SET_WAIT_TX_BUF_SPACE_HW_BASE] = 21,
+
+ /* Not implemented by the EN7523 V1.001/V1.002 firmware. */
+ [WLAN_FUNC_SET_WAIT_RX_RING_FOR_TXDONE_HW_BASE] =
WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_TX_PKT_BUF_ADDR] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_INODE_TXRX_REG_ADDR] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_INODE_DEBUG_FLAG] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_INODE_HW_CFG_INFO] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_INODE_STOP_ACTION] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_INODE_PCIE_SWAP] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_RATELIMIT_CTRL] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_HWNAT_INIT] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_ARHT_CHIP_INFO] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_TX_BUF_CHECK_ADDR] = WLAN_FUNC_SET_WAIT_MAX,
+ [WLAN_FUNC_SET_WAIT_TOKEN_ID_SIZE] = WLAN_FUNC_SET_WAIT_MAX,
+ },
+};
+
static const struct airoha_npu_soc_data en7581_npu_soc_data = {
+ .version = NPU_V2,
+ .max_cores = 8,
+ .cluster_base_addr = 0x306000,
+ .pc_base_addr = 0x305000,
+ .boot_core = airoha_boot_core_v2,
.fw_rv32 = {
.name = NPU_EN7581_FIRMWARE_RV32,
.max_size = NPU_EN7581_FIRMWARE_RV32_MAX_SIZE,
@@ -688,9 +1592,63 @@ static const struct airoha_npu_soc_data
en7581_npu_soc_data = {
.name = NPU_EN7581_FIRMWARE_DATA,
.max_size = NPU_EN7581_FIRMWARE_DATA_MAX_SIZE,
},
+ .wlan_func_get = {
+ [WLAN_FUNC_GET_WAIT_NPU_INFO] = 0,
+ [WLAN_FUNC_GET_WAIT_LAST_RATE] = 1,
+ [WLAN_FUNC_GET_WAIT_COUNTER] = 2,
+ [WLAN_FUNC_GET_WAIT_DBG_COUNTER] = 3,
+ [WLAN_FUNC_GET_WAIT_RXDESC_BASE] = 4,
+ [WLAN_FUNC_GET_WAIT_WCID_DBG_COUNTER] = 5,
+ [WLAN_FUNC_GET_WAIT_DMA_ADDR] = 6,
+ [WLAN_FUNC_GET_WAIT_RING_SIZE] = 7,
+ [WLAN_FUNC_GET_WAIT_NPU_SUPPORT_MAP] = 8,
+ [WLAN_FUNC_GET_WAIT_MDC_LOCK_ADDRESS] = 9,
+ [WLAN_FUNC_GET_WAIT_NPU_VERSION] = 10,
+ },
+ .wlan_func_set = {
+ [WLAN_FUNC_SET_WAIT_PCIE_ADDR] = 0,
+ [WLAN_FUNC_SET_WAIT_DESC] = 1,
+ [WLAN_FUNC_SET_WAIT_NPU_INIT_DONE] = 2,
+ [WLAN_FUNC_SET_WAIT_TRAN_TO_CPU] = 3,
+ [WLAN_FUNC_SET_WAIT_BA_WIN_SIZE] = 4,
+ [WLAN_FUNC_SET_WAIT_DRIVER_MODEL] = 5,
+ [WLAN_FUNC_SET_WAIT_DEL_STA] = 6,
+ [WLAN_FUNC_SET_WAIT_DRAM_BA_NODE_ADDR] = 7,
+ [WLAN_FUNC_SET_WAIT_PKT_BUF_ADDR] = 8,
+ [WLAN_FUNC_SET_WAIT_IS_TEST_NOBA] = 9,
+ [WLAN_FUNC_SET_WAIT_FLUSHONE_TIMEOUT] = 10,
+ [WLAN_FUNC_SET_WAIT_FLUSHALL_TIMEOUT] = 11,
+ [WLAN_FUNC_SET_WAIT_IS_FORCE_TO_CPU] = 12,
+ [WLAN_FUNC_SET_WAIT_PCIE_STATE] = 13,
+ [WLAN_FUNC_SET_WAIT_PCIE_PORT_TYPE] = 14,
+ [WLAN_FUNC_SET_WAIT_ERROR_RETRY_TIMES] = 15,
+ [WLAN_FUNC_SET_WAIT_BAR_INFO] = 16,
+ [WLAN_FUNC_SET_WAIT_FAST_FLAG] = 17,
+ [WLAN_FUNC_SET_WAIT_NPU_BAND0_ONCPU] = 18,
+ [WLAN_FUNC_SET_WAIT_TX_RING_PCIE_ADDR] = 19,
+ [WLAN_FUNC_SET_WAIT_TX_DESC_HW_BASE] = 20,
+ [WLAN_FUNC_SET_WAIT_TX_BUF_SPACE_HW_BASE] = 21,
+ [WLAN_FUNC_SET_WAIT_RX_RING_FOR_TXDONE_HW_BASE] = 22,
+ [WLAN_FUNC_SET_WAIT_TX_PKT_BUF_ADDR] = 23,
+ [WLAN_FUNC_SET_WAIT_INODE_TXRX_REG_ADDR] = 24,
+ [WLAN_FUNC_SET_WAIT_INODE_DEBUG_FLAG] = 25,
+ [WLAN_FUNC_SET_WAIT_INODE_HW_CFG_INFO] = 26,
+ [WLAN_FUNC_SET_WAIT_INODE_STOP_ACTION] = 27,
+ [WLAN_FUNC_SET_WAIT_INODE_PCIE_SWAP] = 28,
+ [WLAN_FUNC_SET_WAIT_RATELIMIT_CTRL] = 29,
+ [WLAN_FUNC_SET_WAIT_HWNAT_INIT] = 30,
+ [WLAN_FUNC_SET_WAIT_ARHT_CHIP_INFO] = 31,
+ [WLAN_FUNC_SET_WAIT_TX_BUF_CHECK_ADDR] = 32,
+ [WLAN_FUNC_SET_WAIT_TOKEN_ID_SIZE] = 33,
+ },
};
static const struct airoha_npu_soc_data an7583_npu_soc_data = {
+ .version = NPU_V2,
+ .max_cores = 8,
+ .cluster_base_addr = 0x306000,
+ .pc_base_addr = 0x305000,
+ .boot_core = airoha_boot_core_v2,
.fw_rv32 = {
.name = NPU_AN7583_FIRMWARE_RV32,
.max_size = NPU_EN7581_FIRMWARE_RV32_MAX_SIZE,
@@ -699,11 +1657,61 @@ static const struct airoha_npu_soc_data
an7583_npu_soc_data = {
.name = NPU_AN7583_FIRMWARE_DATA,
.max_size = NPU_EN7581_FIRMWARE_DATA_MAX_SIZE,
},
+ .wlan_func_get = {
+ [WLAN_FUNC_GET_WAIT_NPU_INFO] = 0,
+ [WLAN_FUNC_GET_WAIT_LAST_RATE] = 1,
+ [WLAN_FUNC_GET_WAIT_COUNTER] = 2,
+ [WLAN_FUNC_GET_WAIT_DBG_COUNTER] = 3,
+ [WLAN_FUNC_GET_WAIT_RXDESC_BASE] = 4,
+ [WLAN_FUNC_GET_WAIT_WCID_DBG_COUNTER] = 5,
+ [WLAN_FUNC_GET_WAIT_DMA_ADDR] = 6,
+ [WLAN_FUNC_GET_WAIT_RING_SIZE] = 7,
+ [WLAN_FUNC_GET_WAIT_NPU_SUPPORT_MAP] = 8,
+ [WLAN_FUNC_GET_WAIT_MDC_LOCK_ADDRESS] = 9,
+ [WLAN_FUNC_GET_WAIT_NPU_VERSION] = 10,
+ },
+ .wlan_func_set = {
+ [WLAN_FUNC_SET_WAIT_PCIE_ADDR] = 0,
+ [WLAN_FUNC_SET_WAIT_DESC] = 1,
+ [WLAN_FUNC_SET_WAIT_NPU_INIT_DONE] = 2,
+ [WLAN_FUNC_SET_WAIT_TRAN_TO_CPU] = 3,
+ [WLAN_FUNC_SET_WAIT_BA_WIN_SIZE] = 4,
+ [WLAN_FUNC_SET_WAIT_DRIVER_MODEL] = 5,
+ [WLAN_FUNC_SET_WAIT_DEL_STA] = 6,
+ [WLAN_FUNC_SET_WAIT_DRAM_BA_NODE_ADDR] = 7,
+ [WLAN_FUNC_SET_WAIT_PKT_BUF_ADDR] = 8,
+ [WLAN_FUNC_SET_WAIT_IS_TEST_NOBA] = 9,
+ [WLAN_FUNC_SET_WAIT_FLUSHONE_TIMEOUT] = 10,
+ [WLAN_FUNC_SET_WAIT_FLUSHALL_TIMEOUT] = 11,
+ [WLAN_FUNC_SET_WAIT_IS_FORCE_TO_CPU] = 12,
+ [WLAN_FUNC_SET_WAIT_PCIE_STATE] = 13,
+ [WLAN_FUNC_SET_WAIT_PCIE_PORT_TYPE] = 14,
+ [WLAN_FUNC_SET_WAIT_ERROR_RETRY_TIMES] = 15,
+ [WLAN_FUNC_SET_WAIT_BAR_INFO] = 16,
+ [WLAN_FUNC_SET_WAIT_FAST_FLAG] = 17,
+ [WLAN_FUNC_SET_WAIT_NPU_BAND0_ONCPU] = 18,
+ [WLAN_FUNC_SET_WAIT_TX_RING_PCIE_ADDR] = 19,
+ [WLAN_FUNC_SET_WAIT_TX_DESC_HW_BASE] = 20,
+ [WLAN_FUNC_SET_WAIT_TX_BUF_SPACE_HW_BASE] = 21,
+ [WLAN_FUNC_SET_WAIT_RX_RING_FOR_TXDONE_HW_BASE] = 22,
+ [WLAN_FUNC_SET_WAIT_TX_PKT_BUF_ADDR] = 23,
+ [WLAN_FUNC_SET_WAIT_INODE_TXRX_REG_ADDR] = 24,
+ [WLAN_FUNC_SET_WAIT_INODE_DEBUG_FLAG] = 25,
+ [WLAN_FUNC_SET_WAIT_INODE_HW_CFG_INFO] = 26,
+ [WLAN_FUNC_SET_WAIT_INODE_STOP_ACTION] = 27,
+ [WLAN_FUNC_SET_WAIT_INODE_PCIE_SWAP] = 28,
+ [WLAN_FUNC_SET_WAIT_RATELIMIT_CTRL] = 29,
+ [WLAN_FUNC_SET_WAIT_HWNAT_INIT] = 30,
+ [WLAN_FUNC_SET_WAIT_ARHT_CHIP_INFO] = 31,
+ [WLAN_FUNC_SET_WAIT_TX_BUF_CHECK_ADDR] = 32,
+ [WLAN_FUNC_SET_WAIT_TOKEN_ID_SIZE] = 33,
+ },
};
static const struct of_device_id of_airoha_npu_match[] = {
- { .compatible = "airoha,en7581-npu", .data = &en7581_npu_soc_data },
- { .compatible = "airoha,an7583-npu", .data = &an7583_npu_soc_data },
+ { .compatible = "airoha,airoha_en7523-npu", .data =
&en7523_npu_soc_data },
+ { .compatible = "airoha,airoha_en7581-npu", .data =
&en7581_npu_soc_data },
+ { .compatible = "airoha,airoha_an7583-npu", .data =
&an7583_npu_soc_data },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, of_airoha_npu_match);
@@ -716,23 +1724,159 @@ static const struct regmap_config regmap_config = {
.disable_locking = true,
};
+static int airoha_npu_probe_cpu_cores(struct airoha_npu *npu)
+{
+ struct airoha_npu_priv *priv = airoha_npu_to_priv(npu);
+ bool npu_version_valid = false;
+ u32 npu_version = 0;
+ int err;
+
+ mutex_lock(&priv->start_lock);
+ if (priv->started) {
+ err = 0;
+ goto out_unlock;
+ }
+
+ err = airoha_npu_run_firmware(npu->dev, priv->base, priv->fw_addr);
+ if (err) {
+ if (err != -EPROBE_DEFER)
+ dev_err(npu->dev, "failed to run npu firmware: %d\n", err);
+ goto out_unlock;
+ }
+
+ /* Set the information consumed by the NPU firmware before booting
it. */
+ regmap_write(npu->regmap, REG_CR_NPU_MIB(10),
+ (u32)priv->rmem->base + npu->soc_data->fw_rv32.max_size);
+ regmap_write(npu->regmap, REG_CR_NPU_MIB(11),
+ sram_size_of_l2c(npu->scu_regmap));
+ regmap_write(npu->regmap, REG_CR_NPU_MIB(12),
+ is_fpga(npu->scu_regmap));
+ regmap_write(npu->regmap, REG_CR_NPU_MIB(21),
+ !of_property_present(npu->dev->of_node,
+ "airoha,enable_npu_tx_uart"));
+ msleep(100);
+
+ /* The Ethernet datapath is fully registered before the cores start. */
+ npu->soc_data->boot_core(npu, priv->rmem);
+ /* Publish the initialized firmware state to airoha_npu_get(). */
+ smp_store_release(&priv->started, true);
+
+ /* Get NPU firmware version when supported. */
+ err = airoha_npu_wlan_msg_get(npu, 0, WLAN_FUNC_GET_WAIT_NPU_VERSION,
+ &npu_version, sizeof(npu_version),
+ GFP_KERNEL);
+ if (!err) {
+ npu_version_valid = true;
+ dev_info(npu->dev, "NPU fw version: %u.%u\n",
+ (npu_version >> 16) & 0xffff, npu_version & 0xffff);
+ } else if (err != -EOPNOTSUPP) {
+ dev_err(npu->dev, "cannot get NPU version: %d\n", err);
+ }
+
+ err = airoha_npu_debugfs_init(npu, npu_version, npu_version_valid);
+ if (err)
+ dev_warn(npu->dev, "failed to initialize debugfs: %d\n", err);
+
+ /* Firmware is running even if optional version/debugfs setup
failed. */
+ err = 0;
+
+out_unlock:
+ mutex_unlock(&priv->start_lock);
+
+ return err;
+}
+
+static void airoha_npu_start_work(struct work_struct *work)
+{
+ struct airoha_npu_priv *priv;
+ int err;
+
+ priv = container_of(to_delayed_work(work), struct airoha_npu_priv,
+ start_work);
+ err = airoha_npu_probe_cpu_cores(&priv->npu);
+ if (err == -EPROBE_DEFER) {
+ mod_delayed_work(system_wq, &priv->start_work,
+ msecs_to_jiffies(NPU_FIRMWARE_RETRY_MS));
+ return;
+ }
+
+ if (err)
+ dev_err(priv->npu.dev, "failed to start NPU: %d\n", err);
+}
+
+int airoha_npu_start(struct device *dev)
+{
+ struct platform_device *pdev;
+ struct device_node *np;
+ struct airoha_npu *npu;
+ int err;
+
+ np = of_parse_phandle(dev->of_node, "airoha,npu", 0);
+ if (!np)
+ return -ENODEV;
+
+ if (!of_device_is_available(np)) {
+ of_node_put(np);
+ return -ENODEV;
+ }
+
+ pdev = of_find_device_by_node(np);
+ of_node_put(np);
+ if (!pdev)
+ return -EPROBE_DEFER;
+
+ device_lock(&pdev->dev);
+ npu = platform_get_drvdata(pdev);
+ if (!npu) {
+ err = -EPROBE_DEFER;
+ goto out_unlock_device;
+ }
+
+ err = airoha_npu_probe_cpu_cores(npu);
+ if (err == -EPROBE_DEFER) {
+ mod_delayed_work(system_wq,
+ &airoha_npu_to_priv(npu)->start_work,
+ msecs_to_jiffies(NPU_FIRMWARE_RETRY_MS));
+ dev_info(npu->dev,
+ "firmware unavailable; NPU startup will be retried\n");
+ err = 0;
+ }
+
+out_unlock_device:
+ device_unlock(&pdev->dev);
+ platform_device_put(pdev);
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(airoha_npu_start);
+
static int airoha_npu_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
+ struct airoha_npu_priv *priv;
+ struct reserved_mem *rmem;
+ struct device_node *np;
struct airoha_npu *npu;
- struct resource res;
void __iomem *base;
int i, irq, err;
- u32 val;
base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(base))
return PTR_ERR(base);
- npu = devm_kzalloc(dev, sizeof(*npu), GFP_KERNEL);
- if (!npu)
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
return -ENOMEM;
+ npu = &priv->npu;
+ priv->base = base;
+ mutex_init(&priv->start_lock);
+ INIT_DELAYED_WORK(&priv->start_work, airoha_npu_start_work);
+
+ npu->soc_data = of_device_get_match_data(dev);
+ if (!npu->soc_data)
+ return -ENODEV;
+
npu->dev = dev;
npu->ops.ppe_init = airoha_npu_ppe_init;
npu->ops.ppe_deinit = airoha_npu_ppe_deinit;
@@ -752,9 +1896,33 @@ static int airoha_npu_probe(struct platform_device
*pdev)
if (IS_ERR(npu->regmap))
return PTR_ERR(npu->regmap);
- err = of_reserved_mem_region_to_resource(dev->of_node, 0, &res);
- if (err)
- return err;
+ np = of_parse_phandle(dev->of_node, "memory-region", 0);
+ if (!np)
+ return -ENODEV;
+
+ rmem = of_reserved_mem_lookup(np);
+ of_node_put(np);
+ if (!rmem)
+ return -ENODEV;
+ priv->rmem = rmem;
+ priv->fw_addr = devm_memremap(dev, rmem->base, rmem->size,
+ MEMREMAP_WC);
+ if (IS_ERR(priv->fw_addr))
+ return PTR_ERR(priv->fw_addr);
+
+ np = of_parse_phandle(dev->of_node, "airoha,scu", 0);
+ if (!np)
+ return dev_err_probe(dev, -EINVAL, "cannot get scuclk\n");
+
+ npu->scu_regmap = syscon_node_to_regmap(np);
+ of_node_put(np);
+ if (IS_ERR(npu->scu_regmap))
+ return PTR_ERR(npu->scu_regmap);
+
+ npu->cores = devm_kcalloc(dev, npu->soc_data->max_cores,
+ sizeof(*npu->cores), GFP_KERNEL);
+ if (!npu->cores)
+ return -ENOMEM;
irq = platform_get_irq(pdev, 0);
if (irq < 0)
@@ -765,27 +1933,9 @@ static int airoha_npu_probe(struct platform_device
*pdev)
if (err)
return err;
- for (i = 0; i < ARRAY_SIZE(npu->cores); i++) {
- struct airoha_npu_core *core = &npu->cores[i];
-
- spin_lock_init(&core->lock);
- core->npu = npu;
-
- irq = platform_get_irq(pdev, i + 1);
- if (irq < 0)
- return irq;
-
- err = devm_request_irq(dev, irq, airoha_npu_wdt_handler,
- IRQF_SHARED, "airoha-npu-wdt", core);
- if (err)
- return err;
-
- INIT_WORK(&core->wdt_work, airoha_npu_wdt_work);
- }
-
- /* wlan IRQ lines */
+ /* WLAN IRQ lines follow the mailbox IRQ on both NPU revisions. */
for (i = 0; i < ARRAY_SIZE(npu->irqs); i++) {
- irq = platform_get_irq(pdev, i + ARRAY_SIZE(npu->cores) + 1);
+ irq = platform_get_irq(pdev, i + 1);
if (irq < 0)
return irq;
@@ -794,44 +1944,40 @@ static int airoha_npu_probe(struct
platform_device *pdev)
err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
if (err)
- return err;
+ return dev_err_probe(dev, err,
+ "No usable coherent DMA configuration\n");
- for (i = 0; i < ARRAY_SIZE(npu->cores); i++) {
+ /*
+ * Prepare the coherent mailbox buffers before the delayed firmware
start.
+ * Only NPU v2 exposes per-core watchdog IRQs.
+ */
+ for (i = 0; i < npu->soc_data->max_cores; i++) {
struct airoha_npu_core *core = &npu->cores[i];
+ spin_lock_init(&core->lock);
+ core->npu = npu;
+ INIT_WORK(&core->wdt_work, airoha_npu_wdt_work);
+
core->buf = dmam_alloc_coherent(dev, AIROHA_NPU_MBOX_SIZE,
&core->addr, GFP_KERNEL);
if (!core->buf)
return -ENOMEM;
- }
- err = airoha_npu_run_firmware(dev, base, &res);
- if (err)
- return err;
-
- regmap_write(npu->regmap, REG_CR_NPU_MIB(10),
- res.start + NPU_EN7581_FIRMWARE_RV32_MAX_SIZE);
- regmap_write(npu->regmap, REG_CR_NPU_MIB(11), 0x40000); /* SRAM 256K */
- regmap_write(npu->regmap, REG_CR_NPU_MIB(12), 0);
- regmap_write(npu->regmap, REG_CR_NPU_MIB(21), 1);
- msleep(100);
-
- /* setting booting address */
- for (i = 0; i < NPU_NUM_CORES; i++)
- regmap_write(npu->regmap, REG_CR_BOOT_BASE(i), res.start);
- usleep_range(1000, 2000);
+ if (npu->soc_data->version < NPU_V2)
+ continue;
- /* enable NPU cores */
- regmap_write(npu->regmap, REG_CR_BOOT_CONFIG, 0xff);
- regmap_write(npu->regmap, REG_CR_BOOT_TRIGGER, 0x1);
- msleep(100);
+ irq = platform_get_irq(pdev, i + npu->soc_data->max_cores + 1);
+ if (irq < 0)
+ return irq;
- if (!airoha_npu_wlan_msg_get(npu, 0, WLAN_FUNC_GET_WAIT_NPU_VERSION,
- &val, sizeof(val), GFP_KERNEL))
- dev_info(dev, "NPU fw version: %0d.%d\n",
- (val >> 16) & 0xffff, val & 0xffff);
+ err = devm_request_irq(dev, irq, airoha_npu_wdt_handler,
+ IRQF_SHARED, "airoha-npu-wdt", core);
+ if (err)
+ return err;
+ }
platform_set_drvdata(pdev, npu);
+ dev_info(dev, "NPU ready; waiting for Ethernet datapath\n");
return 0;
}
@@ -839,9 +1985,13 @@ static int airoha_npu_probe(struct platform_device
*pdev)
static void airoha_npu_remove(struct platform_device *pdev)
{
struct airoha_npu *npu = platform_get_drvdata(pdev);
+ struct airoha_npu_priv *priv = airoha_npu_to_priv(npu);
int i;
- for (i = 0; i < ARRAY_SIZE(npu->cores); i++)
+ cancel_delayed_work_sync(&priv->start_work);
+ airoha_npu_debugfs_remove(npu);
+
+ for (i = 0; i < npu->soc_data->max_cores; i++)
cancel_work_sync(&npu->cores[i].wdt_work);
}
@@ -855,6 +2005,8 @@ static struct platform_driver airoha_npu_driver = {
};
module_platform_driver(airoha_npu_driver);
+MODULE_FIRMWARE(NPU_EN7523_FIRMWARE_DATA);
+MODULE_FIRMWARE(NPU_EN7523_FIRMWARE_RV32);
MODULE_FIRMWARE(NPU_EN7581_FIRMWARE_DATA);
MODULE_FIRMWARE(NPU_EN7581_FIRMWARE_RV32);
MODULE_FIRMWARE(NPU_EN7581_7996_FIRMWARE_DATA);
diff --git a/drivers/net/ethernet/airoha/airoha_ploam.c
b/drivers/net/ethernet/airoha/airoha_ploam.c
new file mode 100644
index 000000000000..f494c06880ef
--- /dev/null
+++ b/drivers/net/ethernet/airoha/airoha_ploam.c
@@ -0,0 +1,762 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * EcoNet GPON PLOAM message processing — hardware-independent layer
+ *
+ * Implements the ITU-T G.984.3 ONU activation state machine (O1–O7) and
+ * all downstream PLOAM message decode / upstream PLOAM message encode
+ * logic. Hardware effects (register writes, carrier changes) are
+ * delegated entirely through struct ploam_ops callbacks.
+ *
+ * Includes the EN7521 PLOAM deduplication filter: every 3rd consecutive
+ * identical message is passed through; intermediate duplicates are
+ * suppressed. For RANGING_TIME the ONU-ID, message type and first five
+ * content bytes are compared, matching the seven-byte vendor comparison.
+ */
+
+#include <linux/slab.h>
+#include <linux/random.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
+#include "airoha_ploam.h"
+
+/* Number of copies used by the EN7523 vendor activation path. */
+#define PLOAM_REPEATED_MSG_REPS 3
+
+/* PHY_TX_EN_BIT_LEN_CONST used for G_PLOu_GUARD_BIT */
+#define PLOAM_GUARD_BIT_LEN 24
+
+struct ploam_priv {
+ const struct ploam_ops *ops;
+ void *hw_priv;
+
+ enum gpon_state state;
+ u8 onu_id;
+ bool emergency_state;
+
+ u8 sn[8];
+ u8 passwd[10];
+
+ /* AES key management */
+ u8 aes_key[16];
+ u8 key_idx;
+ bool key_exchange_pending;
+
+ /* BER REI sequence number (4-bit, wraps at 16) */
+ u8 rei_seq;
+
+ /* Current EqD for O5 incremental adjustment */
+ u32 eqd;
+
+ /* EN7521 downstream PLOAM deduplication filter */
+ struct ploam_msg dedup_prev;
+ bool dedup_prev_valid;
+ u32 dedup_same_cnt;
+};
+
+/* -----------------------------------------------------------------------
+ * Message pack / unpack
+ *
+ * Wire format (big-endian, 12 bytes):
+ * byte 0: ONU-ID (dest_id)
+ * byte 1: message type (msg_id)
+ * bytes 2-11: payload (content[0..9])
+ * -------------------------------------------------------------------- */
+
+static void ploam_unpack(const struct ploam_msg *msg,
+ u8 *onu_id, u8 *type,
+ u8 content[PLOAM_CONTENT_LEN])
+{
+ *onu_id = (msg->value[0] >> 24) & 0xFF;
+ *type = (msg->value[0] >> 16) & 0xFF;
+ content[0] = (msg->value[0] >> 8) & 0xFF;
+ content[1] = msg->value[0] & 0xFF;
+ content[2] = (msg->value[1] >> 24) & 0xFF;
+ content[3] = (msg->value[1] >> 16) & 0xFF;
+ content[4] = (msg->value[1] >> 8) & 0xFF;
+ content[5] = msg->value[1] & 0xFF;
+ content[6] = (msg->value[2] >> 24) & 0xFF;
+ content[7] = (msg->value[2] >> 16) & 0xFF;
+ content[8] = (msg->value[2] >> 8) & 0xFF;
+ content[9] = msg->value[2] & 0xFF;
+}
+
+static void ploam_pack(struct ploam_msg *msg, u8 onu_id, u8 type,
+ const u8 content[PLOAM_CONTENT_LEN])
+{
+ msg->value[0] = ((u32)onu_id << 24) | ((u32)type << 16) |
+ ((u32)content[0] << 8) | content[1];
+ msg->value[1] = ((u32)content[2] << 24) | ((u32)content[3] << 16) |
+ ((u32)content[4] << 8) | content[5];
+ msg->value[2] = ((u32)content[6] << 24) | ((u32)content[7] << 16) |
+ ((u32)content[8] << 8) | content[9];
+}
+
+/* -----------------------------------------------------------------------
+ * EN7521 downstream PLOAM deduplication filter
+ *
+ * Every 3rd consecutive identical message is passed through; duplicates
+ * 1 and 2 are suppressed. For RANGING_TIME the first seven wire bytes
+ * are compared: ONU-ID, message type and content[0..4].
+ * -------------------------------------------------------------------- */
+
+static bool ploam_filter_suppress(struct ploam_priv *pp,
+ const struct ploam_msg *msg, u8 type)
+{
+ bool same;
+
+ if (!pp->dedup_prev_valid) {
+ pp->dedup_prev = *msg;
+ pp->dedup_prev_valid = true;
+ pp->dedup_same_cnt = 1;
+ return false;
+ }
+
+ if (type == PLOAM_DOWN_RANGING_TIME) {
+ /* Match the vendor seven-byte memcmp without depending on
+ * struct layout or host endianness.
+ */
+ same = msg->value[0] == pp->dedup_prev.value[0] &&
+ (msg->value[1] >> 8) ==
+ (pp->dedup_prev.value[1] >> 8);
+ } else {
+ same = (msg->value[0] == pp->dedup_prev.value[0]) &&
+ (msg->value[1] == pp->dedup_prev.value[1]) &&
+ (msg->value[2] == pp->dedup_prev.value[2]);
+ }
+
+ pp->dedup_prev = *msg;
+
+ if (!same) {
+ pp->dedup_same_cnt = 1;
+ return false;
+ }
+
+ pp->dedup_same_cnt++;
+ /* Pass every 3rd (same_cnt==1,3,6,...), suppress 2nd and 3rd of
each triple */
+ return (pp->dedup_same_cnt % 3) != 1;
+}
+
+/* -----------------------------------------------------------------------
+ * Upstream message helpers
+ * -------------------------------------------------------------------- */
+
+static void ploam_send(struct ploam_priv *pp, u8 type,
+ const u8 content[PLOAM_CONTENT_LEN], int times)
+{
+ struct ploam_msg msg;
+
+ ploam_pack(&msg, pp->onu_id, type, content);
+ pp->ops->send_upstream(pp->hw_priv, &msg, times);
+}
+
+static void ploam_send_password(struct ploam_priv *pp)
+{
+ u8 content[PLOAM_CONTENT_LEN] = {};
+
+ memcpy(content, pp->passwd, 10);
+ ploam_send(pp, PLOAM_UP_PASSWORD, content, PLOAM_REPEATED_MSG_REPS);
+}
+
+static void ploam_send_dying_gasp(struct ploam_priv *pp)
+{
+ u8 content[PLOAM_CONTENT_LEN] = {};
+
+ ploam_send(pp, PLOAM_UP_DYING_GASP, content,
+ PLOAM_REPEATED_MSG_REPS);
+}
+
+static void ploam_send_rei(struct ploam_priv *pp, u32 bip_count)
+{
+ u8 content[PLOAM_CONTENT_LEN] = {};
+
+ put_unaligned_be32(bip_count, &content[0]);
+ /* content[4] bits[3:0] = 4-bit sequence number */
+ content[4] = pp->rei_seq & 0x0F;
+ pp->rei_seq = (pp->rei_seq + 1) & 0x0F;
+ ploam_send(pp, PLOAM_UP_REI, content, 1);
+}
+
+/* dm_id = downstream message type being acknowledged;
+ * dm_bytes = first 9 bytes of the downstream message (ONU-ID ..
content[6]) */
+static void ploam_send_ack(struct ploam_priv *pp, u8 dm_id,
+ const u8 dm_bytes[9])
+{
+ u8 content[PLOAM_CONTENT_LEN] = {};
+
+ content[0] = dm_id;
+ memcpy(&content[1], dm_bytes, 9 > (PLOAM_CONTENT_LEN - 1) ?
+ (PLOAM_CONTENT_LEN - 1) : 9);
+ ploam_send(pp, PLOAM_UP_ACK, content, PLOAM_REPEATED_MSG_REPS);
+}
+
+/* Helper: build the 9-byte dm_bytes from a received message's onu_id,
+ * type and content[0..6] for use in Acknowledge */
+static void ploam_make_dm_bytes(u8 dm_bytes[9], u8 onu_id, u8 type,
+ const u8 content[PLOAM_CONTENT_LEN])
+{
+ dm_bytes[0] = onu_id;
+ dm_bytes[1] = type;
+ memcpy(&dm_bytes[2], content, 7);
+}
+
+static void ploam_send_encrypt_key(struct ploam_priv *pp)
+{
+ u8 content[PLOAM_CONTENT_LEN] = {};
+ int i;
+
+ /* Two fragments of 8 key bytes each, as the Nokia/ALCL OLT expects:
+ * octet 3 (content[0]) = key_idx (0 or 1)
+ * octet 4 (content[1]) = frag_idx (0 or 1)
+ * octets 5..12 (content[2..9]) = 8 key bytes */
+ for (i = 0; i < 2; i++) {
+ content[0] = pp->key_idx;
+ content[1] = i;
+ memcpy(&content[2], pp->aes_key + i * 8, 8);
+ ploam_send(pp, PLOAM_UP_ENCRYPT_KEY, content,
+ PLOAM_REPEATED_MSG_REPS);
+ }
+}
+
+/* -----------------------------------------------------------------------
+ * State machine helpers
+ * -------------------------------------------------------------------- */
+
+static void ploam_set_state(struct ploam_priv *pp, enum gpon_state
new_state)
+{
+ if (pp->state == new_state)
+ return;
+
+ pp->state = new_state;
+ pp->ops->state_changed(pp->hw_priv, new_state);
+
+ /*
+ * Do not enqueue Serial_Number_ONU when entering O3. The EN7523 GPON
+ * MAC builds the message from G_VENDOR_ID, G_VS_SN and G_SN_MSG_CFG
+ * and transmits it in the OLT-provided serial-number grant. Sending a
+ * second software-generated message here occurs outside that grant and
+ * also omits the hardware random-delay and power-level fields.
+ */
+}
+
+/* -----------------------------------------------------------------------
+ * Downstream message handlers
+ * Each handler is called after ONU-ID and filter checks pass.
+ *
+ * Field extraction conventions:
+ * content[N] corresponds to raw message byte (N+2) in big-endian order.
+ * Bit-field positions follow the __BIG_ENDIAN layout in
gpon_ploam_raw.h.
+ * -------------------------------------------------------------------- */
+
+/* PLOAM_DOWN_UPSTREAM_OVERHEAD (0x01) — broadcast only */
+static void handle_upstream_overhead(struct ploam_priv *pp,
+ u8 onu_id,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ u8 delim[3];
+ bool delay_mode;
+ u16 delay_time;
+
+ if (onu_id != PLOAM_ONU_BCAST)
+ return;
+ if (pp->state != GPON_O2_STANDBY)
+ return;
+
+ /* c[0]=gbits, c[1]=t1_pbits, c[2]=t2_pbits, c[3]=t3_pbits
+ * c[4..6]=delimiter[0..2]
+ * c[7]: BE layout bits[7:6]=resv, [5]=delay_mode, [4]=sn_mask,
+ * [3:2]=sn_tran_num, [1:0]=tx_power
+ * c[8]=delay_time[0], c[9]=delay_time[1] */
+ delim[0] = c[4];
+ delim[1] = c[5];
+ delim[2] = c[6];
+ delay_mode = (c[7] >> 5) & 1;
+ delay_time = ((u16)c[8] << 8) | c[9];
+
+ pp->ops->set_overhead(pp->hw_priv, c[0], c[1], c[2], c[3],
+ delim, delay_mode, delay_time);
+ ploam_set_state(pp, GPON_O3_SERIAL_NUMBER);
+}
+
+/* PLOAM_DOWN_ASSIGN_ONU_ID (0x03) — broadcast only */
+static void handle_assign_onu_id(struct ploam_priv *pp,
+ u8 onu_id,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ if (onu_id != PLOAM_ONU_BCAST)
+ return;
+ if (pp->state != GPON_O3_SERIAL_NUMBER)
+ return;
+
+ /* c[0]=new ONU-ID, c[1..8]=SN for verification */
+ if (memcmp(&c[1], pp->sn, 8) != 0)
+ return;
+
+ /* Transition to O4 first, then program the ONU-ID register */
+ ploam_set_state(pp, GPON_O4_RANGING);
+ pp->onu_id = c[0];
+ pp->ops->set_onu_id(pp->hw_priv, pp->onu_id);
+}
+
+/* PLOAM_DOWN_RANGING_TIME (0x04) — unicast */
+static void handle_ranging_time(struct ploam_priv *pp,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ u32 new_eqd;
+ u8 eqd_type;
+
+ if (pp->state != GPON_O4_RANGING && pp->state != GPON_O5_OPERATION)
+ return;
+
+ /* c[0] BE: bits[7:1]=resv, bit[0]=eqd_type */
+ eqd_type = c[0] & 1;
+
+ if (eqd_type == 1) {
+ /* Protection path EqD — ignore */
+ return;
+ }
+
+ /* c[1..4] = delay[0..3] big-endian EqD value */
+ new_eqd = get_unaligned_be32(&c[1]);
+
+ if (pp->state == GPON_O4_RANGING) {
+ u32 byte_delay = new_eqd & ~7u;
+ u32 bit_delay = new_eqd & 7u;
+
+ pp->eqd = new_eqd;
+ pp->ops->set_eqd_o4(pp->hw_priv, byte_delay, bit_delay);
+ pp->ops->enable_us_fec(pp->hw_priv);
+ ploam_set_state(pp, GPON_O5_OPERATION);
+ } else {
+ /* O5: incremental EqD adjustment via hardware */
+ pp->ops->adjust_eqd_o5(pp->hw_priv, new_eqd);
+ pp->eqd = new_eqd;
+ }
+}
+
+/* PLOAM_DOWN_DEACTIVATE_ONU_ID (0x05) — broadcast or unicast */
+static void handle_deactivate_onu(struct ploam_priv *pp, u8 onu_id)
+{
+ if (onu_id != pp->onu_id && onu_id != PLOAM_ONU_BCAST)
+ return;
+ if (pp->state != GPON_O4_RANGING &&
+ pp->state != GPON_O5_OPERATION &&
+ pp->state != GPON_O6_POPUP)
+ return;
+
+ pp->ops->deactivate(pp->hw_priv);
+}
+
+/* PLOAM_DOWN_DISABLE_SN (0x06) — broadcast only */
+static void handle_disable_sn(struct ploam_priv *pp,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ /* c[0]=mode, c[1..8]=SN */
+ u8 mode = c[0];
+
+ if (pp->state == GPON_O7_EMERGENCY_STOP) {
+ if (mode == PLOAM_DISABLE_PARTICIPATE_ALL ||
+ (mode == PLOAM_DISABLE_PARTICIPATE &&
+ memcmp(&c[1], pp->sn, 8) == 0)) {
+ pp->emergency_state = false;
+ ploam_set_state(pp, GPON_O2_STANDBY);
+ }
+ } else if (pp->state != GPON_O1_INITIAL) {
+ if (mode == PLOAM_DISABLE_DENIED_ALL ||
+ (mode == PLOAM_DISABLE_DENIED &&
+ memcmp(&c[1], pp->sn, 8) == 0)) {
+ pp->emergency_state = true;
+ ploam_set_state(pp, GPON_O7_EMERGENCY_STOP);
+ }
+ }
+}
+
+/* PLOAM_DOWN_ENCRYPTED_PORT_ID (0x08) — unicast */
+static void handle_encrypted_port_id(struct ploam_priv *pp,
+ u8 onu_id, u8 type,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ u16 port_id;
+ u8 encrypt;
+ u8 dm_bytes[9];
+
+ if (onu_id != pp->onu_id)
+ return;
+
+ /* c[0] BE: bits[7:2]=resv, bits[1:0]=encrypt */
+ encrypt = c[0] & 3;
+ /* c[1]=port_id_m (8 bits), c[2] BE: bits[7:4]=port_id_l,
bits[3:0]=resv */
+ port_id = ((u16)c[1] << 4) | ((c[2] >> 4) & 0xF);
+
+ pp->ops->set_gem_encryption(pp->hw_priv, port_id, encrypt);
+
+ ploam_make_dm_bytes(dm_bytes, onu_id, type, c);
+ ploam_send_ack(pp, type, dm_bytes);
+}
+
+/* PLOAM_DOWN_REQUEST_PASSWORD (0x09) — unicast */
+static void handle_request_password(struct ploam_priv *pp, u8 onu_id)
+{
+ if (onu_id != pp->onu_id)
+ return;
+ if (pp->state == GPON_O5_OPERATION)
+ ploam_send_password(pp);
+}
+
+/* PLOAM_DOWN_ASSIGN_ALLOC_ID (0x0A) — unicast */
+static void handle_assign_alloc_id(struct ploam_priv *pp,
+ u8 onu_id, u8 type,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ u16 alloc_id;
+ bool allocate;
+ u8 dm_bytes[9];
+
+ if (onu_id != pp->onu_id)
+ return;
+ if (pp->state != GPON_O5_OPERATION)
+ goto send_ack;
+
+ /* c[0]=alloc_id_m, c[1] BE: bits[7:4]=alloc_id_l, [3:0]=resv */
+ alloc_id = ((u16)c[0] << 4) | ((c[1] >> 4) & 0xF);
+ /* c[2]=type: 0x01=allocate, 0xFF=deallocate */
+ allocate = (c[2] == 0x01);
+
+ if (alloc_id != pp->onu_id)
+ pp->ops->set_alloc_id(pp->hw_priv, alloc_id, allocate);
+
+send_ack:
+ ploam_make_dm_bytes(dm_bytes, onu_id, type, c);
+ ploam_send_ack(pp, type, dm_bytes);
+}
+
+/* PLOAM_DOWN_POPUP (0x0C) — broadcast or unicast */
+static void handle_popup(struct ploam_priv *pp, u8 onu_id)
+{
+ if (onu_id != pp->onu_id && onu_id != PLOAM_ONU_BCAST)
+ return;
+ if (pp->state != GPON_O6_POPUP)
+ return;
+
+ if (onu_id == PLOAM_ONU_BCAST)
+ ploam_set_state(pp, GPON_O4_RANGING);
+ else
+ ploam_set_state(pp, GPON_O5_OPERATION);
+}
+
+/* PLOAM_DOWN_REQUEST_KEY (0x0D) — unicast */
+static void handle_request_key(struct ploam_priv *pp, u8 onu_id)
+{
+ if (onu_id != pp->onu_id)
+ return;
+ if (pp->state != GPON_O5_OPERATION)
+ return;
+ if (pp->key_exchange_pending)
+ return;
+
+ pp->key_exchange_pending = true;
+ /* Hardware generates a random key and loads shadow regs */
+ pp->ops->request_new_key(pp->hw_priv);
+ /*
+ * Announce the key under the index the OLT is still using and only
+ * then flip: toggling first sends Encryption_Key with the index of a
+ * key the OLT has not been given yet.
+ */
+ ploam_send_encrypt_key(pp);
+ pp->key_idx ^= 1;
+}
+
+/* PLOAM_DOWN_CONFIGURE_PORT_ID (0x0E) — unicast */
+static void handle_configure_port_id(struct ploam_priv *pp,
+ u8 onu_id, u8 type,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ u16 port_id;
+ bool activate;
+ u8 dm_bytes[9];
+
+ if (onu_id != pp->onu_id)
+ return;
+ if (pp->state != GPON_O5_OPERATION)
+ goto send_ack;
+
+ /* c[0] BE: bits[7:1]=resv, bit[0]=activate */
+ activate = c[0] & 1;
+ /* c[1]=port_id_m, c[2] BE: bits[7:4]=port_id_l, [3:0]=resv */
+ port_id = ((u16)c[1] << 4) | ((c[2] >> 4) & 0xF);
+
+ /* Do not acknowledge an OMCC assignment until the hardware path is
+ * ready. The OLT will retry Configure-Port-ID after a transient
+ * command failure.
+ */
+ if (pp->ops->set_omci_gem(pp->hw_priv, port_id, activate))
+ return;
+
+send_ack:
+ ploam_make_dm_bytes(dm_bytes, onu_id, type, c);
+ ploam_send_ack(pp, type, dm_bytes);
+}
+
+/* PLOAM_DOWN_BER_INTERVAL (0x12) — broadcast or unicast */
+static void handle_ber_interval(struct ploam_priv *pp,
+ u8 onu_id, u8 type,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ u32 interval_frames;
+ u32 interval_ms;
+ u8 dm_bytes[9];
+
+ if (onu_id != pp->onu_id && onu_id != PLOAM_ONU_BCAST)
+ return;
+ if (pp->state == GPON_O5_OPERATION) {
+ /* c[0..3] = interval in frames; convert to ms (125 µs/frame) */
+ interval_frames = get_unaligned_be32(&c[0]);
+ /* frames × 125 µs = interval_frames >> 3 ms (≈) */
+ interval_ms = interval_frames >> 3;
+ pp->ops->set_ber_interval(pp->hw_priv, interval_ms);
+ }
+
+ ploam_make_dm_bytes(dm_bytes, onu_id, type, c);
+ ploam_send_ack(pp, type, dm_bytes);
+}
+
+/* PLOAM_DOWN_KEY_SWITCHING_TIME (0x13) — unicast */
+static void handle_key_switching_time(struct ploam_priv *pp,
+ u8 onu_id, u8 type,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ u32 superframe;
+ u8 dm_bytes[9];
+
+ if (onu_id != pp->onu_id)
+ return;
+ if (pp->state == GPON_O5_OPERATION) {
+ superframe = get_unaligned_be32(&c[0]);
+ pp->ops->set_key_switch_time(pp->hw_priv, superframe);
+ }
+
+ ploam_make_dm_bytes(dm_bytes, onu_id, type, c);
+ ploam_send_ack(pp, type, dm_bytes);
+}
+
+/* PLOAM_DOWN_EXTENDED_BURST_LEN (0x14) — broadcast only, O3 only */
+static void handle_extended_burst_length(struct ploam_priv *pp,
+ u8 onu_id,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ if (onu_id != PLOAM_ONU_BCAST)
+ return;
+ if (pp->state != GPON_O3_SERIAL_NUMBER)
+ return;
+
+ /* c[0]=o3_t3_preamble, c[1]=o5_t3_preamble */
+ pp->ops->set_t3_preamble(pp->hw_priv, c[0], c[1]);
+}
+
+/* PLOAM_DOWN_SWIFT_POPUP (0x16) — broadcast only */
+static void handle_swift_popup(struct ploam_priv *pp, u8 onu_id)
+{
+ if (onu_id != PLOAM_ONU_BCAST)
+ return;
+ if (pp->state == GPON_O6_POPUP)
+ ploam_set_state(pp, GPON_O5_OPERATION);
+}
+
+/* PLOAM_DOWN_RANGING_ADJUSTMENT (0x17) — unicast or broadcast */
+static void handle_ranging_adjustment(struct ploam_priv *pp,
+ u8 onu_id, u8 type,
+ const u8 c[PLOAM_CONTENT_LEN])
+{
+ u32 eqd_offset;
+ u8 s_bit;
+ u8 dm_bytes[9];
+
+ if (onu_id != pp->onu_id && onu_id != PLOAM_ONU_BCAST)
+ return;
+ if (pp->state == GPON_O5_OPERATION) {
+ /* c[0] BE: bits[7:2]=resv, bit[1]=s_bit, bit[0]=resv */
+ s_bit = (c[0] >> 1) & 1;
+ eqd_offset = get_unaligned_be32(&c[1]);
+ if (eqd_offset != 0) {
+ u32 new_eqd = s_bit ? pp->eqd - eqd_offset
+ : pp->eqd + eqd_offset;
+ pp->ops->adjust_eqd_o5(pp->hw_priv, new_eqd);
+ pp->eqd = new_eqd;
+ }
+ }
+
+ ploam_make_dm_bytes(dm_bytes, onu_id, type, c);
+ ploam_send_ack(pp, type, dm_bytes);
+}
+
+/* -----------------------------------------------------------------------
+ * Public API
+ * -------------------------------------------------------------------- */
+
+struct ploam_priv *ploam_alloc(const struct ploam_ops *ops, void *hw_priv,
+ const u8 sn[8], const u8 passwd[10])
+{
+ struct ploam_priv *pp;
+
+ pp = kzalloc(sizeof(*pp), GFP_KERNEL);
+ if (!pp)
+ return NULL;
+
+ pp->ops = ops;
+ pp->hw_priv = hw_priv;
+ pp->state = GPON_O1_INITIAL;
+ pp->onu_id = PLOAM_ONU_UNASSIGNED;
+
+ memcpy(pp->sn, sn, sizeof(pp->sn));
+ memcpy(pp->passwd, passwd, sizeof(pp->passwd));
+
+ return pp;
+}
+
+void ploam_free(struct ploam_priv *pp)
+{
+ kfree(pp);
+}
+
+void ploam_set_identity(struct ploam_priv *pp, const u8 sn[8],
+ const u8 passwd[10])
+{
+ memcpy(pp->sn, sn, sizeof(pp->sn));
+ memcpy(pp->passwd, passwd, sizeof(pp->passwd));
+}
+
+void ploam_reset(struct ploam_priv *pp)
+{
+ pp->state = GPON_O1_INITIAL;
+ pp->onu_id = PLOAM_ONU_UNASSIGNED;
+ pp->key_exchange_pending = false;
+ pp->rei_seq = 0;
+ pp->eqd = 0;
+ pp->dedup_prev_valid = false;
+ pp->dedup_same_cnt = 0;
+ /* emergency_state is sticky: only cleared by Disable_SN PARTICIPATE */
+}
+
+void ploam_start(struct ploam_priv *pp)
+{
+ if (pp->state == GPON_O1_INITIAL && !pp->emergency_state)
+ ploam_set_state(pp, GPON_O2_STANDBY);
+}
+
+void ploam_handle_downstream(struct ploam_priv *pp,
+ const struct ploam_msg *msg)
+{
+ u8 onu_id, type;
+ u8 content[PLOAM_CONTENT_LEN];
+
+ ploam_unpack(msg, &onu_id, &type, content);
+
+ /* Drop messages not addressed to us */
+ if (onu_id != PLOAM_ONU_BCAST && onu_id != pp->onu_id)
+ return;
+
+ /* EN7521 deduplication filter */
+ if (ploam_filter_suppress(pp, msg, type))
+ return;
+
+ switch (type) {
+ case PLOAM_DOWN_UPSTREAM_OVERHEAD:
+ handle_upstream_overhead(pp, onu_id, content);
+ break;
+ case PLOAM_DOWN_ASSIGN_ONU_ID:
+ handle_assign_onu_id(pp, onu_id, content);
+ break;
+ case PLOAM_DOWN_RANGING_TIME:
+ handle_ranging_time(pp, content);
+ break;
+ case PLOAM_DOWN_DEACTIVATE_ONU_ID:
+ handle_deactivate_onu(pp, onu_id);
+ break;
+ case PLOAM_DOWN_DISABLE_SN:
+ handle_disable_sn(pp, content);
+ break;
+ case PLOAM_DOWN_ENCRYPTED_PORT_ID:
+ handle_encrypted_port_id(pp, onu_id, type, content);
+ break;
+ case PLOAM_DOWN_REQUEST_PASSWORD:
+ handle_request_password(pp, onu_id);
+ break;
+ case PLOAM_DOWN_ASSIGN_ALLOC_ID:
+ handle_assign_alloc_id(pp, onu_id, type, content);
+ break;
+ case PLOAM_DOWN_POPUP:
+ handle_popup(pp, onu_id);
+ break;
+ case PLOAM_DOWN_REQUEST_KEY:
+ handle_request_key(pp, onu_id);
+ break;
+ case PLOAM_DOWN_CONFIGURE_PORT_ID:
+ handle_configure_port_id(pp, onu_id, type, content);
+ break;
+ case PLOAM_DOWN_BER_INTERVAL:
+ handle_ber_interval(pp, onu_id, type, content);
+ break;
+ case PLOAM_DOWN_KEY_SWITCHING_TIME:
+ handle_key_switching_time(pp, onu_id, type, content);
+ break;
+ case PLOAM_DOWN_EXTENDED_BURST_LEN:
+ handle_extended_burst_length(pp, onu_id, content);
+ break;
+ case PLOAM_DOWN_SWIFT_POPUP:
+ handle_swift_popup(pp, onu_id);
+ break;
+ case PLOAM_DOWN_RANGING_ADJUSTMENT:
+ handle_ranging_adjustment(pp, onu_id, type, content);
+ break;
+ case PLOAM_DOWN_PEE:
+ case PLOAM_DOWN_PST:
+ case PLOAM_DOWN_CHANGE_POWER_LEVEL:
+ case PLOAM_DOWN_PON_ID:
+ case PLOAM_DOWN_SLEEP_ALLOW:
+ /* Stub: message acknowledged by presence in handler table */
+ break;
+ default:
+ break;
+ }
+}
+
+void ploam_notify_dying_gasp(struct ploam_priv *pp)
+{
+ if (pp->state >= GPON_O2_STANDBY)
+ ploam_send_dying_gasp(pp);
+}
+
+void ploam_notify_ber(struct ploam_priv *pp, u32 bip_count)
+{
+ if (pp->state == GPON_O5_OPERATION)
+ ploam_send_rei(pp, bip_count);
+}
+
+void ploam_notify_los(struct ploam_priv *pp)
+{
+ /* LOS while operational: transition to O6 (popup state).
+ * The TO2 timer in the hardware layer will reset to O1 if no
+ * Popup / Swift_Popup is received within 100 ms. */
+ if (pp->state == GPON_O5_OPERATION)
+ ploam_set_state(pp, GPON_O6_POPUP);
+}
+
+enum gpon_state ploam_get_state(const struct ploam_priv *pp)
+{
+ return pp->state;
+}
+
+u8 ploam_get_onu_id(const struct ploam_priv *pp)
+{
+ return pp->onu_id;
+}
+
+u32 ploam_get_eqd(const struct ploam_priv *pp)
+{
+ return pp->eqd;
+}
+
+void ploam_set_aes_key(struct ploam_priv *pp, const u8 key[16])
+{
+ memcpy(pp->aes_key, key, 16);
+ pp->key_exchange_pending = false;
+}
diff --git a/drivers/net/ethernet/airoha/airoha_ploam.h
b/drivers/net/ethernet/airoha/airoha_ploam.h
new file mode 100644
index 000000000000..86a6af2bdc4b
--- /dev/null
+++ b/drivers/net/ethernet/airoha/airoha_ploam.h
@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * EcoNet GPON PLOAM layer — public interface
+ *
+ * This header defines the hardware-independent PLOAM protocol types and
+ * the callback interface through which the PLOAM state machine drives
+ * the hardware layer in gpon.c.
+ */
+
+#ifndef _ECONET_PLOAM_H
+#define _ECONET_PLOAM_H
+
+#include <linux/types.h>
+
+/* PLOAM message: 13 bytes carried in 3 × 32-bit FIFO words.
+ * Layout: value[0][31:24]=ONU-ID, [23:16]=MsgType, [15:0]=content[0-1]
+ * value[1] = content[2-5]
+ * value[2] = content[6-9]
+ * The BIP-8 checksum (byte 12) is computed/verified by hardware.
+ */
+#define PLOAM_WORDS 3
+#define PLOAM_CONTENT_LEN 10
+#define PLOAM_ONU_BCAST 0xFF
+#define PLOAM_ONU_UNASSIGNED 0xFF
+
+/* Downstream PLOAM message type IDs (from ITU-T G.984.3 / vendor ref) */
+#define PLOAM_DOWN_UPSTREAM_OVERHEAD 0x01
+#define PLOAM_DOWN_ASSIGN_ONU_ID 0x03
+#define PLOAM_DOWN_RANGING_TIME 0x04
+#define PLOAM_DOWN_DEACTIVATE_ONU_ID 0x05
+#define PLOAM_DOWN_DISABLE_SN 0x06
+#define PLOAM_DOWN_ENCRYPTED_PORT_ID 0x08
+#define PLOAM_DOWN_REQUEST_PASSWORD 0x09
+#define PLOAM_DOWN_ASSIGN_ALLOC_ID 0x0A
+#define PLOAM_DOWN_POPUP 0x0C
+#define PLOAM_DOWN_REQUEST_KEY 0x0D
+#define PLOAM_DOWN_CONFIGURE_PORT_ID 0x0E
+#define PLOAM_DOWN_PEE 0x0F
+#define PLOAM_DOWN_CHANGE_POWER_LEVEL 0x10
+#define PLOAM_DOWN_PST 0x11
+#define PLOAM_DOWN_BER_INTERVAL 0x12
+#define PLOAM_DOWN_KEY_SWITCHING_TIME 0x13
+#define PLOAM_DOWN_EXTENDED_BURST_LEN 0x14
+#define PLOAM_DOWN_PON_ID 0x15
+#define PLOAM_DOWN_SWIFT_POPUP 0x16
+#define PLOAM_DOWN_RANGING_ADJUSTMENT 0x17
+#define PLOAM_DOWN_SLEEP_ALLOW 0x18
+#define PLOAM_DOWN_MAX_TYPE 0x19
+
+/* Disable_SN mode byte values */
+#define PLOAM_DISABLE_DENIED 0xFF /* unicast SN → go to O7 */
+#define PLOAM_DISABLE_DENIED_ALL 0xF0 /* all ONUs → O7 */
+#define PLOAM_DISABLE_PARTICIPATE 0x00 /* unicast SN → O2 */
+#define PLOAM_DISABLE_PARTICIPATE_ALL 0x0F /* all O7 ONUs → O2
*/
+
+/* Upstream PLOAM message type IDs */
+#define PLOAM_UP_SERIAL_NUMBER_ONU 0x01
+#define PLOAM_UP_PASSWORD 0x02
+#define PLOAM_UP_DYING_GASP 0x03
+#define PLOAM_UP_NO_MESSAGE 0x04
+#define PLOAM_UP_ENCRYPT_KEY 0x05
+#define PLOAM_UP_PEE 0x06
+#define PLOAM_UP_PST 0x07
+#define PLOAM_UP_REI 0x08
+#define PLOAM_UP_ACK 0x09
+#define PLOAM_UP_SLEEP_REQUEST 0x0A
+
+/**
+ * enum gpon_state - ONU activation states per ITU-T G.984.3
+ */
+enum gpon_state {
+ GPON_O1_INITIAL = 1,
+ GPON_O2_STANDBY,
+ GPON_O3_SERIAL_NUMBER,
+ GPON_O4_RANGING,
+ GPON_O5_OPERATION,
+ GPON_O6_POPUP,
+ GPON_O7_EMERGENCY_STOP,
+};
+
+/**
+ * struct ploam_msg - raw PLOAM message as three 32-bit FIFO words
+ */
+struct ploam_msg {
+ u32 value[PLOAM_WORDS];
+};
+
+/**
+ * struct ploam_ops - callbacks from the PLOAM layer to the hardware layer
+ *
+ * @send_upstream: write @times copies of upstream PLOAM to the TX FIFO.
+ * @set_onu_id: program the OLT-assigned ONU-ID into G_ONU_ID.
+ * @set_eqd_o4: program equalization delay in O4; byteDelay = eqd & ~7,
+ * bitDelay = eqd & 7. Hardware writes G_EQD and programs PHY bit delay.
+ * @adjust_eqd_o5: incremental EqD update in O5; hardware reads the current
+ * internal byte delay from DBG_TX_SYNC_OFFSET and adjusts G_EQD.
+ * @enable_us_fec: set GBL_CFG_US_FEC_EN on O4→O5 transition.
+ * @set_overhead: program burst overhead parameters from Upstream_Overhead
+ * PLOAM. guard_bits is the raw message value (PHY_TX_EN_BIT_LEN_CONST=24
+ * should be written to G_PLOu_GUARD_BIT). t1/t2/t3 go to G_PLOu_PRMBL_*,
+ * delim/delay_mode/delay_time go to the PHY and G_PRE_ASSIGNED_DLY.
+ * @set_t3_preamble: program G_PLOu_PRMBL_TYPE3 from Extended_Burst_Length.
+ * @set_key_switch_time: write the AES key-switch superframe counter to
+ * G_AES_CFG so the shadow key becomes active at the OLT-specified frame.
+ * @request_new_key: OLT sent Request_Key; hardware generates a random key,
+ * loads it into shadow registers, returns the key via the aes_key array.
+ * @set_ber_interval: update BER reporting timer to @interval_ms.
+ * @set_omci_gem: configure or clear the OMCI GEM port in G_OMCI_ID;
+ * return zero only when the hardware channel is ready.
+ * @set_gem_encryption: program per-GEM-port encryption mode in GEM
port table.
+ * @set_alloc_id: allocate or deallocate a T-CONT for the given alloc-ID.
+ * @state_changed: ONU activation state has changed; start/stop TO1/TO2
timers,
+ * manage carrier, MBI interface, etc.
+ * @deactivate: OLT sent Deactivate_ONU; trigger a full hardware
disable/reset.
+ */
+struct ploam_ops {
+ void (*send_upstream)(void *priv, const struct ploam_msg *msg, int
times);
+ void (*set_onu_id)(void *priv, u8 onu_id);
+ void (*set_eqd_o4)(void *priv, u32 byte_delay, u32 bit_delay);
+ void (*adjust_eqd_o5)(void *priv, u32 new_eqd);
+ void (*enable_us_fec)(void *priv);
+ void (*set_overhead)(void *priv, u8 guard_bits, u8 t1_pbits, u8
t2_pbits,
+ u8 t3_pbits, const u8 delim[3],
+ bool delay_mode, u16 delay_time);
+ void (*set_t3_preamble)(void *priv, u8 o3_t3, u8 o5_t3);
+ void (*set_key_switch_time)(void *priv, u32 superframe);
+ void (*request_new_key)(void *priv);
+ void (*set_ber_interval)(void *priv, u32 interval_ms);
+ int (*set_omci_gem)(void *priv, u16 gem_port_id, bool valid);
+ void (*set_gem_encryption)(void *priv, u16 port_id, u8 encrypt_mode);
+ void (*set_alloc_id)(void *priv, u16 alloc_id, bool allocate);
+ void (*state_changed)(void *priv, enum gpon_state state);
+ void (*deactivate)(void *priv);
+};
+
+struct ploam_priv;
+
+/* Lifecycle */
+struct ploam_priv *ploam_alloc(const struct ploam_ops *ops, void *hw_priv,
+ const u8 sn[8], const u8 passwd[10]);
+void ploam_free(struct ploam_priv *pp);
+void ploam_set_identity(struct ploam_priv *pp, const u8 sn[8],
+ const u8 passwd[10]);
+void ploam_reset(struct ploam_priv *pp);
+void ploam_start(struct ploam_priv *pp);
+
+/* Downstream processing — call from the HW interrupt handler */
+void ploam_handle_downstream(struct ploam_priv *pp,
+ const struct ploam_msg *msg);
+
+/* Event notifications from hardware to PLOAM layer */
+void ploam_notify_dying_gasp(struct ploam_priv *pp);
+void ploam_notify_ber(struct ploam_priv *pp, u32 bip_count);
+void ploam_notify_los(struct ploam_priv *pp);
+
+/* State queries */
+enum gpon_state ploam_get_state(const struct ploam_priv *pp);
+u8 ploam_get_onu_id(const struct ploam_priv *pp);
+u32 ploam_get_eqd(const struct ploam_priv *pp);
+
+/* Key management — called by hardware after loading key into shadow
regs */
+void ploam_set_aes_key(struct ploam_priv *pp, const u8 key[16]);
+
+#endif /* _ECONET_PLOAM_H */
diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c
b/drivers/net/ethernet/airoha/airoha_ppe.c
index 6ea91ff6a6d2..17d98bb7ff1a 100644
--- a/drivers/net/ethernet/airoha/airoha_ppe.c
+++ b/drivers/net/ethernet/airoha/airoha_ppe.c
@@ -5,14 +5,17 @@
*/
#include <linux/ip.h>
+#include <linux/if_vlan.h>
#include <linux/ipv6.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/rhashtable.h>
#include <net/ipv6.h>
-#include <linux/if_vlan.h>
+#include <net/flow_offload.h>
+#include <net/netlink.h>
#include <net/pkt_cls.h>
#include <net/route.h>
+#include <uapi/linux/ppp_defs.h>
#include "airoha_regs.h"
#include "airoha_eth.h"
@@ -37,15 +40,22 @@ static const struct rhashtable_params
airoha_l2_flow_table_params = {
.automatic_shrinking = true,
};
+static int airoha_ppe_v1_flow_offload_replace(struct net_device *dev,
+ struct flow_cls_offload *f);
+static int airoha_ppe_v1_flow_offload_destroy(struct net_device *dev,
+ struct flow_cls_offload *f);
+static int airoha_ppe_v1_flow_offload_stats(struct flow_cls_offload *f);
+static void airoha_ppe_v1_hw_init(struct airoha_ppe *ppe);
+
static int airoha_ppe_get_num_stats_entries(struct airoha_ppe *ppe)
{
if (!IS_ENABLED(CONFIG_NET_AIROHA_FLOW_STATS))
return -EOPNOTSUPP;
- if (airoha_is_7583(ppe->eth))
+ if (airoha_is(ppe->common.eth, airoha_an7583))
return -EOPNOTSUPP;
- return PPE_STATS_NUM_ENTRIES;
+ return ppe->common.eth->soc->ppe_stats_entries;
}
static int airoha_ppe_get_total_num_stats_entries(struct airoha_ppe *ppe)
@@ -53,7 +63,7 @@ static int
airoha_ppe_get_total_num_stats_entries(struct airoha_ppe *ppe)
int num_stats = airoha_ppe_get_num_stats_entries(ppe);
if (num_stats > 0) {
- struct airoha_eth *eth = ppe->eth;
+ struct airoha_eth *eth = ppe->common.eth;
num_stats = num_stats * eth->soc->num_ppe;
}
@@ -63,20 +73,74 @@ static int
airoha_ppe_get_total_num_stats_entries(struct airoha_ppe *ppe)
static u32 airoha_ppe_get_total_sram_num_entries(struct airoha_ppe *ppe)
{
- struct airoha_eth *eth = ppe->eth;
+ struct airoha_eth *eth = ppe->common.eth;
+
+ return ppe->common.eth->soc->ppe_sram_entries * eth->soc->num_ppe;
+}
+
+static u32 airoha_ppe_get_num_entries_shift(u32 entries)
+{
+ switch (entries) {
+ case 256:
+ return 6;
+ case 512:
+ return 7;
+ default:
+ return __ffs(entries >> 10);
+ }
+}
+
+static int airoha_ppe_hw_to_sw_idx(struct airoha_ppe *ppe, u32 hw_idx,
+ u32 *sw_idx)
+{
+ u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe);
+ u32 dram_num_entries = ppe->common.eth->soc->ppe_dram_entries;
+ u32 ppe_num_entries = sram_num_entries + dram_num_entries;
+
+ if (!airoha_is(ppe->common.eth, airoha_en7523)) {
+ if (hw_idx >= ppe_num_entries)
+ return -ERANGE;
+
+ *sw_idx = hw_idx;
+ return 0;
+ }
+
+ if (hw_idx < sram_num_entries) {
+ *sw_idx = hw_idx;
+ return 0;
+ }
+
+ if (hw_idx >= ppe->common.eth->soc->ppe_dram_entries &&
+ hw_idx < ppe->common.eth->soc->ppe_dram_entries +
dram_num_entries) {
+ *sw_idx = sram_num_entries + hw_idx - dram_num_entries;
+ return 0;
+ }
+
+ return -ERANGE;
+}
+
+static u32 airoha_ppe_sw_to_hw_idx(struct airoha_ppe *ppe, u32 sw_idx)
+{
+ u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe);
- return PPE_SRAM_NUM_ENTRIES * eth->soc->num_ppe;
+ if (airoha_is(ppe->common.eth, airoha_en7523) && sw_idx >=
sram_num_entries)
+ return ppe->common.eth->soc->ppe_dram_entries + sw_idx -
sram_num_entries;
+
+ return sw_idx;
}
u32 airoha_ppe_get_total_num_entries(struct airoha_ppe *ppe)
{
u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe);
- return sram_num_entries + PPE_DRAM_NUM_ENTRIES;
+ return sram_num_entries + ppe->common.eth->soc->ppe_dram_entries;
}
bool airoha_ppe_is_enabled(struct airoha_eth *eth, int index)
{
+ if (eth->soc->foe_format == AIROHA_FOE_FORMAT_V1)
+ return !index && eth->ppe_dev && eth->ppe_dev->enabled;
+
if (index >= eth->soc->num_ppe)
return false;
@@ -85,7 +149,7 @@ bool airoha_ppe_is_enabled(struct airoha_eth *eth,
int index)
static u32 airoha_ppe_get_timestamp(struct airoha_ppe *ppe)
{
- return airoha_fe_get(ppe->eth, REG_FE_FOE_TS,
+ return airoha_fe_get(ppe->common.eth, REG_FE_FOE_TS,
AIROHA_FOE_IB1_BIND_TIMESTAMP);
}
@@ -97,6 +161,8 @@ void airoha_ppe_set_cpu_port(struct airoha_gdm_dev
*dev, u8 ppe_id, u8 fport)
u32 fe_cpu_port;
fe_cpu_port = qdma_id ? FE_PSE_PORT_CDM2 : FE_PSE_PORT_CDM1;
+
+ dev_info(eth->dev, "Setting CPU port for PPE %d, fport %d to
CDM%d\n", ppe_id, fport, qdma_id ? 2 : 1);
airoha_fe_rmw(eth, REG_PPE_DFT_CPORT(ppe_id, fport),
DFT_CPORT_MASK(fport),
__field_prep(DFT_CPORT_MASK(fport), fe_cpu_port));
@@ -128,9 +194,27 @@ void airoha_ppe_set_mtu(struct airoha_gdm_dev *dev)
len = max_t(u32, len, netdev->mtu);
}
- len += VLAN_ETH_HLEN;
+ /* REG_PPE_MTU limits the egress *L2 frame* length, while
netdev->mtu is
+ * only the L3 payload. Without accounting for the Ethernet header (and
+ * any VLAN/PPPoE tags + FCS the egress may carry) a full-size
forwarded
+ * frame is one header larger than the programmed MTU, so the PPE
bounces
+ * every such packet to the CPU with HIT_BIND_EXCEED_MTU and hw offload
+ * never actually forwards. The vendor SDK programs ~2000 here
regardless.
+ */
+ if (len)
+ len += ETH_HLEN + 2 * VLAN_HLEN + 8 /* PPPoE */ + ETH_FCS_LEN;
+
+ /* On EN7523 the hw-forwarded path may egress on a different FP than
+ * this port index, and hw_init already programs a generous MTU for all
+ * FPs; don't undercut it with the (smaller) per-port L2 size here.
+ */
+ if (airoha_is(eth, airoha_en7523))
+ len = max_t(u32, len, 2000);
+
ppe_id = !airoha_is_lan_gdm_dev(dev) && airoha_ppe_is_enabled(eth, 1);
index = port->id == AIROHA_GDM4_IDX ? 7 : port->id;
+
+ dev_info(eth->dev, "Setting PPE %d MTU for index %d to %u\n",
ppe_id, index, len);
airoha_fe_rmw(eth, REG_PPE_MTU(ppe_id, index),
FP_EGRESS_MTU_MASK(index),
__field_prep(FP_EGRESS_MTU_MASK(index), len));
@@ -138,24 +222,37 @@ void airoha_ppe_set_mtu(struct airoha_gdm_dev *dev)
static void airoha_ppe_hw_init(struct airoha_ppe *ppe)
{
- u32 sram_ppe_num_data_entries = PPE_SRAM_NUM_ENTRIES, sram_num_entries;
+ u32 sram_ppe_num_data_entries =
ppe->common.eth->soc->ppe_sram_entries, sram_num_entries;
u32 sram_tb_size, dram_num_entries;
- struct airoha_eth *eth = ppe->eth;
+ struct airoha_eth *eth = ppe->common.eth;
int i, sram_num_stats_entries;
+ switch (eth->soc->foe_format) {
+ case AIROHA_FOE_FORMAT_V1:
+ airoha_ppe_v1_hw_init(ppe);
+ return;
+ case AIROHA_FOE_FORMAT_V2:
+ case AIROHA_FOE_FORMAT_V2_TUNNEL:
+ break;
+ }
+
+ dev_info(eth->dev, "Initializing PPE Hardware\n");
sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe);
- sram_tb_size = sram_num_entries * sizeof(struct airoha_foe_entry);
- dram_num_entries = PPE_RAM_NUM_ENTRIES_SHIFT(PPE_DRAM_NUM_ENTRIES);
+ sram_tb_size = sram_num_entries * AIROHA_FOE_ENTRY_SIZE;
+ dram_num_entries =
airoha_ppe_get_num_entries_shift(eth->soc->ppe_dram_entries);
sram_num_stats_entries = airoha_ppe_get_num_stats_entries(ppe);
if (sram_num_stats_entries > 0)
sram_ppe_num_data_entries -= sram_num_stats_entries;
sram_ppe_num_data_entries =
- PPE_RAM_NUM_ENTRIES_SHIFT(sram_ppe_num_data_entries);
+ airoha_ppe_get_num_entries_shift(sram_ppe_num_data_entries);
for (i = 0; i < eth->soc->num_ppe; i++) {
+ dev_info(eth->dev, "Configuring PPE %d: SRAM entries %u, DRAM
entries %u, stats entries %d\n",
+ i, sram_ppe_num_data_entries, dram_num_entries,
sram_num_stats_entries);
+
airoha_fe_wr(eth, REG_PPE_TB_BASE(i),
- ppe->foe_dma + sram_tb_size);
+ ppe->common.foe_dma + sram_tb_size);
airoha_fe_rmw(eth, REG_PPE_BND_AGE0(i),
PPE_BIND_AGE0_DELTA_NON_L4 |
@@ -168,31 +265,143 @@ static void airoha_ppe_hw_init(struct airoha_ppe
*ppe)
FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP_FIN, 1) |
FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 60));
- airoha_fe_rmw(eth, REG_PPE_TB_HASH_CFG(i),
- PPE_SRAM_TABLE_EN_MASK |
- PPE_SRAM_HASH1_EN_MASK |
- PPE_DRAM_TABLE_EN_MASK |
- PPE_SRAM_HASH0_MODE_MASK |
- PPE_SRAM_HASH1_MODE_MASK |
- PPE_DRAM_HASH0_MODE_MASK |
- PPE_DRAM_HASH1_MODE_MASK,
- FIELD_PREP(PPE_SRAM_TABLE_EN_MASK, 1) |
- FIELD_PREP(PPE_SRAM_HASH1_EN_MASK, 1) |
- FIELD_PREP(PPE_SRAM_HASH1_MODE_MASK, 1) |
- FIELD_PREP(PPE_DRAM_HASH1_MODE_MASK, 3));
-
- airoha_fe_rmw(eth, REG_PPE_TB_CFG(i),
- PPE_TB_CFG_SEARCH_MISS_MASK |
- PPE_SRAM_TB_NUM_ENTRY_MASK |
- PPE_DRAM_TB_NUM_ENTRY_MASK |
- PPE_TB_CFG_KEEPALIVE_MASK |
- PPE_TB_ENTRY_SIZE_MASK,
- FIELD_PREP(PPE_TB_CFG_SEARCH_MISS_MASK, 3) |
- FIELD_PREP(PPE_TB_ENTRY_SIZE_MASK, 0) |
- FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK,
- sram_ppe_num_data_entries) |
- FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK,
- dram_num_entries));
+ if (eth->soc->foe_format == AIROHA_FOE_FORMAT_V2) {
+ /**
+ * the airoha_en7523 support for 64 and 80 bytes, current
use 80 bytes for ppe
+ * 0 = 64 Bytes
+ * 1 = 80 Bytes
+ */
+
+ airoha_fe_rmw(eth, REG_PPE_TB_CFG(i),
+ EN7523_PPE_SRAM_TABLE_EN_MASK |
+ EN7523_PPE_SRAM_HASH1_EN_MASK |
+ EN7523_PPE_DRAM_TABLE_EN_MASK |
+ EN7523_PPE_SRAM_HASH0_MODE_MASK |
+ EN7523_PPE_SRAM_HASH1_MODE_MASK |
+ EN7523_PPE_DRAM_HASH0_MODE_MASK,
+ FIELD_PREP(EN7523_PPE_SRAM_TABLE_EN_MASK, 1) |
+ FIELD_PREP(EN7523_PPE_SRAM_HASH1_EN_MASK, 1) |
+ FIELD_PREP(EN7523_PPE_DRAM_TABLE_EN_MASK, 1) |
+ FIELD_PREP(EN7523_PPE_SRAM_HASH0_MODE_MASK, 3) |
+ FIELD_PREP(EN7523_PPE_SRAM_HASH1_MODE_MASK, 1) |
+ FIELD_PREP(EN7523_PPE_DRAM_HASH0_MODE_MASK, 3));
+
+ airoha_fe_rmw(eth, REG_PPE_TB_CFG(i),
+ PPE_SRAM_TB_NUM_ENTRY_MASK |
+ PPE_DRAM_TB_NUM_ENTRY_MASK |
+ PPE_TB_CFG_SEARCH_MISS_MASK |
+ PPE_TB_CFG_KEEPALIVE_MASK |
+ PPE_TB_CFG_AGE_TCP_FIN_MASK |
+ PPE_TB_CFG_AGE_UDP_MASK |
+ PPE_TB_CFG_AGE_TCP_MASK |
+ PPE_TB_CFG_AGE_UNBIND_MASK |
+ PPE_TB_CFG_AGE_NON_L4_MASK |
+ PPE_TB_CFG_AGE_PREBIND_MASK |
+ PPE_TB_ENTRY_SIZE_MASK,
+ FIELD_PREP(PPE_TB_CFG_SEARCH_MISS_MASK, 3) |
+ FIELD_PREP(PPE_TB_CFG_KEEPALIVE_MASK, 3) |
+ FIELD_PREP(PPE_TB_ENTRY_SIZE_MASK, 1) |
+ PPE_TB_CFG_AGE_TCP_FIN_MASK |
+ PPE_TB_CFG_AGE_UDP_MASK |
+ PPE_TB_CFG_AGE_TCP_MASK |
+ PPE_TB_CFG_AGE_UNBIND_MASK |
+ PPE_TB_CFG_AGE_NON_L4_MASK |
+ PPE_TB_CFG_AGE_PREBIND_MASK |
+ FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK,
sram_ppe_num_data_entries) |
+ FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK,
dram_num_entries));
+
+ if (FIELD_GET(PPE_SRAM_TB_NUM_ENTRY_MASK,
+ airoha_fe_rr(eth, REG_PPE_TB_CFG(i))) !=
sram_ppe_num_data_entries) {
+ u32 tb_cfg = airoha_fe_rr(eth, REG_PPE_TB_CFG(i));
+
+ dev_warn(eth->dev,
+ "EN7523 PPE%d SRAM size mismatch: wrote %u, read
%lu, TB_CFG=%08x\n",
+ i, sram_ppe_num_data_entries,
+ FIELD_GET(PPE_SRAM_TB_NUM_ENTRY_MASK, tb_cfg),
+ tb_cfg);
+ }
+
+ /* Match the working vendor (IOWRT stock) PPE register
+ * setup, dumped via devmem from the EN7523 vendor
+ * firmware (where TB_USED>0, hw-NAT forwards). The generic
+ * mainline EN7581 values leave IPv4_NAPT and IP_PROT
+ * unconfigured so EN7523 never resolves hw-NAT flows.
+ *
+ * The SDK uses 80-byte FoE entries on EN7523. The stock
+ * register dump below has TB_ENTRY_SIZE cleared, so restore
+ * that bit after loading the remaining known-good fields.
+ */
+ airoha_fe_wr(eth, REG_PPE_GLO_CFG(i), 0x00038743);
+ airoha_fe_wr(eth, REG_PPE_PPE_FLOW_CFG(i), 0x06bbf7c0);
+ airoha_fe_wr(eth, REG_PPE_IP_PROTO_CHK(i), 0x000f000f);
+ airoha_fe_wr(eth, REG_PPE_IP_PROTO_CHK(i) + 0x4, 0x04291106);
+ airoha_fe_wr(eth, REG_PPE_IP_PROTO_CHK(i) + 0x8, 0x00003a01);
+ airoha_fe_wr(eth, REG_PPE_TB_CFG(i), 0xef403fb4);
+ airoha_fe_set(eth, REG_PPE_TB_CFG(i), PPE_TB_ENTRY_SIZE_MASK);
+ airoha_fe_wr(eth, REG_PPE_TB_HASH_CFG(i), 0x31003001);
+ /* Remaining vendor PPE config registers (dumped from stock
+ * fw): KA, MIRROR, L2 bridge cfg / ethertype enable.
+ * Offsets relative to GLO_CFG (PPE base + 0x200):
+ * KA=0xE34(+0x34) MIRROR=0xE54(+0x54)
+ * L2B_CFG=0xE88(+0x88) L2B_ETYPE_EN=0xE8C(+0x8c)
+ */
+ airoha_fe_wr(eth, REG_PPE_KEEPALIVE(i),
+ FIELD_PREP(PPE_KEEPALIVE_UDP_MASK, 1) |
+ FIELD_PREP(PPE_KEEPALIVE_TCP_MASK, 1) |
+ FIELD_PREP(PPE_KEEPALIVE_NTU_MASK, 1));
+ airoha_fe_wr(eth, REG_PPE_GLO_CFG(i) + 0x54, 0x00000021);
+ airoha_fe_wr(eth, REG_PPE_GLO_CFG(i) + 0x88, 0x001d077f);
+ airoha_fe_wr(eth, REG_PPE_GLO_CFG(i) + 0x8c, 0x0000001b);
+ /* Give every egress forwarding port a generous MTU so a
+ * full-size offloaded frame is never bounced to the CPU
+ * with HIT_BIND_EXCEED_MTU. The hardware-forwarded path may
+ * egress on an FP other than the wan/lan port index that
+ * airoha_ppe_set_mtu programs, so set all of FP0..FP9 here.
+ * REG_PPE_MTU = PPE base + 0x304; +0x00..+0x10 cover FP0..FP9.
+ */
+ airoha_fe_wr(eth, REG_PPE_MTU_BASE(i) + 0x00, 0x07d407d0);
+ airoha_fe_wr(eth, REG_PPE_MTU_BASE(i) + 0x04, 0x07dc07d8);
+ airoha_fe_wr(eth, REG_PPE_MTU_BASE(i) + 0x08, 0x07e407e0);
+ airoha_fe_wr(eth, REG_PPE_MTU_BASE(i) + 0x0c, 0x07f007e8);
+
+ } else {
+ airoha_fe_rmw(eth, REG_PPE_TB_HASH_CFG(i),
+ EN7581_PPE_SRAM_TABLE_EN_MASK |
+ EN7581_PPE_SRAM_HASH1_EN_MASK |
+ EN7581_PPE_DRAM_TABLE_EN_MASK |
+ EN7581_PPE_SRAM_HASH0_MODE_MASK |
+ EN7581_PPE_SRAM_HASH1_MODE_MASK |
+ EN7581_PPE_DRAM_HASH0_MODE_MASK |
+ EN7581_PPE_DRAM_HASH1_MODE_MASK,
+ FIELD_PREP(EN7581_PPE_SRAM_TABLE_EN_MASK, 1) |
+ FIELD_PREP(EN7581_PPE_SRAM_HASH1_EN_MASK, 1) |
+ FIELD_PREP(EN7581_PPE_DRAM_TABLE_EN_MASK, 1) |
+ FIELD_PREP(EN7581_PPE_SRAM_HASH0_MODE_MASK, 1) |
+ FIELD_PREP(EN7581_PPE_SRAM_HASH1_MODE_MASK, 1) |
+ FIELD_PREP(EN7581_PPE_DRAM_HASH0_MODE_MASK, 1) |
+ FIELD_PREP(EN7581_PPE_DRAM_HASH1_MODE_MASK, 3));
+
+ airoha_fe_rmw(eth, REG_PPE_TB_CFG(i),
+ PPE_TB_CFG_SEARCH_MISS_MASK |
+ PPE_SRAM_TB_NUM_ENTRY_MASK |
+ PPE_DRAM_TB_NUM_ENTRY_MASK |
+ PPE_TB_CFG_KEEPALIVE_MASK |
+ PPE_TB_ENTRY_SIZE_MASK,
+ FIELD_PREP(PPE_TB_CFG_SEARCH_MISS_MASK, 3) |
+ FIELD_PREP(PPE_TB_CFG_KEEPALIVE_MASK, 3) |
+ FIELD_PREP(PPE_TB_ENTRY_SIZE_MASK, 0) |
+ FIELD_PREP(PPE_SRAM_TB_NUM_ENTRY_MASK,
+ sram_ppe_num_data_entries) |
+ FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK,
+ dram_num_entries));
+
+ airoha_fe_set(eth, REG_PPE_PPE_FLOW_CFG(i),
+ PPE_FLOW_CFG_IP4_NAPT_MASK |
+ PPE_FLOW_CFG_IP4_NAT_MASK |
+ PPE_FLOW_CFG_IP6_3T_ROUTE_MASK |
+ PPE_FLOW_CFG_IP6_5T_ROUTE_MASK |
+ PPE_FLOW_CFG_L2_BRIDGE_MASK);
+ }
airoha_fe_rmw(eth, REG_PPE_BIND_RATE(i),
PPE_BIND_RATE_L2B_BIND_MASK |
@@ -200,12 +409,21 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe)
FIELD_PREP(PPE_BIND_RATE_L2B_BIND_MASK, 0x1e) |
FIELD_PREP(PPE_BIND_RATE_BIND_MASK, 0x1e));
+ airoha_fe_rmw(eth, REG_PPE_UNBIND_AGE(i),
+ PPE_UNBIND_AGE_MIN_PACKETS_MASK |
+ PPE_UNBIND_AGE_DELTA_MASK,
+ FIELD_PREP(PPE_UNBIND_AGE_MIN_PACKETS_MASK, 1) |
+ FIELD_PREP(PPE_UNBIND_AGE_DELTA_MASK, 30));
+
airoha_fe_wr(eth, REG_PPE_HASH_SEED(i), PPE_HASH_SEED);
airoha_fe_clear(eth, REG_PPE_PPE_FLOW_CFG(i),
PPE_FLOW_CFG_IP6_6RD_MASK);
+
+ /* Enable PPE */
+ airoha_fe_set(eth, REG_PPE_GLO_CFG(i), PPE_GLO_CFG_EN_MASK);
}
- for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
+ for (i = 0; i < eth->soc->max_gdm_ports; i++) {
struct airoha_gdm_port *port = eth->ports[i];
int j;
@@ -222,7 +440,7 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe)
ppe_id = !airoha_is_lan_gdm_dev(dev) &&
airoha_ppe_is_enabled(eth, 1);
- fport = airoha_get_fe_port(dev);
+ fport = eth->ppe_host_ops->get_fe_port(dev);
airoha_ppe_set_cpu_port(dev, ppe_id, fport);
airoha_ppe_set_mtu(dev);
}
@@ -232,17 +450,38 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe)
static void airoha_ppe_flow_mangle_eth(const struct flow_action_entry
*act, void *eth)
{
void *dest = eth + act->mangle.offset;
- const void *src = &act->mangle.val;
+ u16 val16;
if (act->mangle.offset > 8)
return;
- if (act->mangle.mask == 0xffff) {
- src += 2;
- dest += 2;
+ /*
+ * nf_flow_table builds Ethernet rewrites as one full 32-bit mangle and
+ * two possible 16-bit partial mangles. Do not select the partial
value
+ * by taking a byte pointer into mangle.val: that relies on the CPU
being
+ * little-endian and corrupts both MAC addresses on EN751221 MIPS.
+ *
+ * For mask 0x0000ffff, bits 31:16 contain the replacement and apply to
+ * the upper half of the 32-bit Ethernet word (dest + 2). For mask
+ * 0xffff0000, bits 15:0 contain the replacement and apply to the lower
+ * half at dest. Copying a native u16 obtained by shifting the scalar
+ * preserves the original packet-byte order on both endian variants.
+ */
+ switch (act->mangle.mask) {
+ case 0:
+ memcpy(dest, &act->mangle.val, sizeof(act->mangle.val));
+ break;
+ case 0x0000ffff:
+ val16 = act->mangle.val >> 16;
+ memcpy(dest + 2, &val16, sizeof(val16));
+ break;
+ case 0xffff0000:
+ val16 = act->mangle.val;
+ memcpy(dest, &val16, sizeof(val16));
+ break;
+ default:
+ break;
}
-
- memcpy(dest, src, act->mangle.mask ? 2 : 4);
}
static int airoha_ppe_flow_mangle_ports(const struct flow_action_entry
*act,
@@ -339,15 +578,36 @@ static void airoha_ppe_foe_set_bridge_addrs(struct
airoha_foe_bridge *br,
br->src_mac_lo = get_unaligned_be32(eh->h_source + 2);
}
+static int
+airoha_ppe_xpon_get_tx_info(struct airoha_eth *eth,
+ struct net_device *netdev,
+ const struct airoha_flow_data *data,
+ struct airoha_xpon_tx_info *info)
+{
+ int i;
+
+ for (i = data->vlan.num - 1; i >= 0; i--) {
+ const typeof(data->vlan.hdr[0]) *vlan = &data->vlan.hdr[i];
+
+ if (!eth->ppe_host_ops->xpon_get_tx_info(netdev, true, vlan->id,
+ true, vlan->prio, info))
+ return 0;
+ }
+
+ return eth->ppe_host_ops->xpon_get_tx_info(netdev, false, 0, false,
0, info);
+}
+
static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth,
struct airoha_foe_entry *hwe,
struct net_device *netdev, int type,
struct airoha_flow_data *data,
int l4proto, u8 dsfield)
{
+ struct airoha_xpon_tx_info xpon = {};
u32 qdata = FIELD_PREP(AIROHA_FOE_SHAPER_ID, 0x7f), ports_pad, val;
int wlan_etype = -EINVAL, dsa_port = airoha_get_dsa_port(&netdev);
struct airoha_foe_mac_info_common *l2;
+ bool xpon_flow = false;
u8 smac_id = 0xf;
memset(hwe, 0, sizeof(*hwe));
@@ -361,7 +621,11 @@ static int airoha_ppe_foe_entry_prepare(struct
airoha_eth *eth,
AIROHA_FOE_IB1_BIND_TTL;
hwe->ib1 = val;
- val = FIELD_PREP(AIROHA_FOE_IB2_PORT_AG, 0x1f);
+ /* EN7523 vendor entries use PORT_AG=0 (account group); mainline's 0x1f
+ * yields ib2=0x3e240 vs the vendor's 0x240 for the same flow.
+ */
+ val = FIELD_PREP(AIROHA_FOE_IB2_PORT_AG,
+ airoha_is(eth, airoha_en7523) ? 0x0 : 0x1f);
if (netdev) {
struct airoha_wdma_info info = {};
@@ -369,7 +633,11 @@ static int airoha_ppe_foe_entry_prepare(struct
airoha_eth *eth,
&info)) {
val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, info.idx) |
FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT,
+ airoha_is(eth, airoha_en7523) ?
+ FE_PSE_PORT_GDM3 :
FE_PSE_PORT_CDM4);
+ if (airoha_is(eth, airoha_en7523))
+ val |= AIROHA_FOE_IB2_PSE_QOS;
qdata |= FIELD_PREP(AIROHA_FOE_ACTDP, info.bss);
wlan_etype = FIELD_PREP(AIROHA_FOE_MAC_WDMA_BAND,
info.idx) |
@@ -380,10 +648,27 @@ static int airoha_ppe_foe_entry_prepare(struct
airoha_eth *eth,
struct airoha_gdm_port *port;
u8 pse_port, channel, priority;
- if (!airoha_is_valid_gdm_dev(eth, dev))
+ if (!eth->ppe_host_ops->is_valid_gdm_dev(eth, dev))
return -EINVAL;
port = dev->port;
+ if (!port) {
+ dev_err(eth->dev, "GDM device without parent port
during FOE prepare\n");
+ return -EINVAL;
+ }
+
+ if (airoha_is(eth, airoha_en7523) &&
+ (dev->flags & AIROHA_PRIV_F_XPON_MANAGED) &&
+ dev->xpon_mode == AIROHA_XPON_MODE_GPON) {
+ int ret;
+
+ ret = airoha_ppe_xpon_get_tx_info(eth, netdev, data,
+ &xpon);
+ if (ret)
+ return ret;
+ xpon_flow = true;
+ }
+
if (dsa_port >= 0 || airoha_is_lan_gdm_dev(dev))
pse_port = port->id == 4 ? FE_PSE_PORT_GDM4
: port->id;
@@ -392,14 +677,18 @@ static int airoha_ppe_foe_entry_prepare(struct
airoha_eth *eth,
* loopback
*/
- /* For traffic forwarded to DSA devices select QoS
- * channel according to the DSA user port index, rely
- * on port id otherwise.
+ /* GPON hardware forwarding must carry the same T-CONT,
+ * queue and NBOQ metadata as CPU-originated descriptors.
*/
- channel = dsa_port >= 0 ? dsa_port : port->id;
- channel = channel % AIROHA_NUM_QOS_CHANNELS;
- priority = rt_tos2priority(dsfield);
- priority = priority % AIROHA_NUM_QOS_QUEUES;
+ if (xpon_flow) {
+ channel = xpon.tcont;
+ priority = xpon.queue;
+ } else {
+ channel = dsa_port >= 0 ? dsa_port : port->id;
+ channel %= AIROHA_NUM_QOS_CHANNELS;
+ priority = rt_tos2priority(dsfield);
+ priority %= AIROHA_NUM_QOS_QUEUES;
+ }
qdata |= FIELD_PREP(AIROHA_FOE_CHANNEL, channel) |
FIELD_PREP(AIROHA_FOE_QID, priority);
@@ -409,13 +698,31 @@ static int airoha_ppe_foe_entry_prepare(struct
airoha_eth *eth,
/* For downlink traffic consume SRAM memory for hw
* forwarding descriptors queue.
*/
- if (airoha_is_lan_gdm_dev(dev))
+ /* The vendor SDK defines SUPPORT_FAST_PATH only for
+ * EN7580/EN7581/AN7583. EN7523 must use the normal
+ * PPE-to-QDMA path; setting FAST_PATH makes a bound FOE
+ * entry consume an unsupported descriptor path and drops
+ * the flow as soon as it transitions to BIND.
+ */
+ if (!airoha_is(eth, airoha_en7523) &&
+ airoha_is_lan_gdm_dev(dev))
val |= AIROHA_FOE_IB2_FAST_PATH;
- if (dsa_port >= 0)
+ if (xpon_flow)
+ val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, xpon.tcont);
+ else if (dsa_port >= 0)
val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ,
dsa_port);
+ else if (airoha_is(eth, airoha_en7523) &&
+ port->id == AIROHA_GDM3_IDX &&
+ dev->nbq != 0)
+ val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, dev->nbq);
smac_id = port->id;
+ dev_dbg(eth->dev,
+ "foe_prepare: port_id=%d dev_nbq=%d dsa_port=%d
ib2=%08x nbq=%lu xpon=%u gem=%u channel=%u queue=%u\n",
+ port->id, dev->nbq, dsa_port, val,
+ FIELD_GET(AIROHA_FOE_IB2_NBQ, val), xpon_flow,
+ xpon.gem_port_id, channel, priority);
}
}
@@ -473,6 +780,9 @@ static int airoha_ppe_foe_entry_prepare(struct
airoha_eth *eth,
} else if (dsa_port >= 0) {
l2->etype = BIT(dsa_port);
l2->etype |= !data->vlan.num ? BIT(15) : 0;
+ } else if (xpon_flow) {
+ /* EN7523 reuses the FOE EtherType field as the GPON GEM tag. */
+ l2->etype = xpon.gem_port_id;
} else if (data->pppoe.num) {
l2->etype = ETH_P_PPP_SES;
}
@@ -546,7 +856,7 @@ static u32 airoha_ppe_foe_get_entry_hash(struct
airoha_ppe *ppe,
struct airoha_foe_entry *hwe)
{
int type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1);
- u32 ppe_hash_mask = airoha_ppe_get_total_num_entries(ppe) - 1;
+ u32 ppe_num_entries = airoha_ppe_get_total_num_entries(ppe);
u32 hash, hv1, hv2, hv3;
switch (type) {
@@ -584,20 +894,35 @@ static u32 airoha_ppe_foe_get_entry_hash(struct
airoha_ppe *ppe,
case PPE_PKT_TYPE_IPV6_6RD:
default:
WARN_ON_ONCE(1);
- return ppe_hash_mask;
+ return ppe_num_entries - 1;
}
hash = (hv1 & hv2) | ((~hv1) & hv3);
hash = (hash >> 24) | ((hash & 0xffffff) << 8);
hash ^= hv1 ^ hv2 ^ hv3;
hash ^= hash >> 16;
- hash &= ppe_hash_mask;
+
+ if (airoha_is(ppe->common.eth, airoha_en7523)) {
+ u32 dram_entries = ppe->common.eth->soc->ppe_dram_entries;
+ u32 sram_entries = airoha_ppe_get_total_sram_num_entries(ppe);
+
+ /* For the EN7523, the hardware maps the main BINDs using a
mask on the DRAM.
+ * We masked to the DRAM boundaries and adjusted sw_idx to
point to the area after the SRAM.
+ */
+ hash &= (dram_entries - 1);
+ hash += sram_entries;
+ } else {
+ /* For EN7581/AN7583, ppe_num_entries is a power of 2,
+ * so the bitwise AND works perfectly across the entire table.
+ */
+ hash &= (ppe_num_entries - 1);
+ }
return hash;
}
static int airoha_ppe_foe_get_flow_stats_index(struct airoha_ppe *ppe,
- u32 hash, u32 *index)
+ u32 sw_idx, u32 *index)
{
int ppe_num_stats_entries;
@@ -605,17 +930,55 @@ static int
airoha_ppe_foe_get_flow_stats_index(struct airoha_ppe *ppe,
if (ppe_num_stats_entries < 0)
return ppe_num_stats_entries;
- *index = hash >= ppe_num_stats_entries ? hash - PPE_STATS_NUM_ENTRIES
- : hash;
+ *index = sw_idx >= ppe_num_stats_entries ? sw_idx -
ppe->common.eth->soc->ppe_stats_entries
+ : sw_idx;
return 0;
}
+void airoha_ppe_foe_entry_get_stats(struct airoha_ppe *ppe, u32 sw_idx,
+ struct airoha_foe_stats64 *stats)
+{
+ struct airoha_eth *eth = ppe->common.eth;
+ int ppe_num_stats_entries;
+ struct airoha_npu *npu;
+ u32 index;
+
+ ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe);
+ if (ppe_num_stats_entries <= 0)
+ return;
+
+ if (airoha_ppe_foe_get_flow_stats_index(ppe, sw_idx, &index))
+ return;
+
+ if (index >= ppe_num_stats_entries)
+ return;
+
+ rcu_read_lock();
+
+ npu = rcu_dereference(eth->npu);
+ if (npu && ppe->foe_stats) {
+ u64 packets = ppe->foe_stats[index].packets;
+ u64 bytes = ppe->foe_stats[index].bytes;
+ struct airoha_foe_stats npu_stats;
+
+ if (!eth->ppe_host_ops || !eth->ppe_host_ops->npu_stats_read)
+ goto unlock;
+ eth->ppe_host_ops->npu_stats_read(npu, index, &npu_stats);
+ stats->packets = packets << 32 | npu_stats.packets;
+ stats->bytes = bytes << 32 | npu_stats.bytes;
+ }
+
+unlock:
+ rcu_read_unlock();
+}
+
static void airoha_ppe_foe_flow_stat_entry_reset(struct airoha_ppe *ppe,
struct airoha_npu *npu,
int index)
{
- memset_io(&npu->stats[index], 0, sizeof(*npu->stats));
+ if (ppe->common.eth->ppe_host_ops &&
ppe->common.eth->ppe_host_ops->npu_stats_clear)
+ ppe->common.eth->ppe_host_ops->npu_stats_clear(npu, index);
memset(&ppe->foe_stats[index], 0, sizeof(*ppe->foe_stats));
}
@@ -655,19 +1018,23 @@ static void
airoha_ppe_foe_flow_stats_update(struct airoha_ppe *ppe,
if (type == PPE_PKT_TYPE_BRIDGE) {
data = &hwe->bridge.data;
ib2 = &hwe->bridge.ib2;
- meter = &hwe->bridge.l2.meter;
} else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) {
data = &hwe->ipv6.data;
ib2 = &hwe->ipv6.ib2;
- meter = &hwe->ipv6.meter;
} else {
data = &hwe->ipv4.data;
ib2 = &hwe->ipv4.ib2;
- meter = &hwe->ipv4.l2.meter;
}
+ /* EN7581/AN7583 keep the meter/accounting word at offset 0x40.
+ * EN7523 has no flow-stats table and returns before reaching here.
+ */
+ meter = &hwe->words[0x40 / sizeof(u32)];
+
pse_port = FIELD_GET(AIROHA_FOE_IB2_PSE_PORT, *ib2);
- if (pse_port == FE_PSE_PORT_CDM4)
+ if (pse_port == FE_PSE_PORT_CDM4 ||
+ (airoha_is(ppe->common.eth, airoha_en7523) &&
+ pse_port == FE_PSE_PORT_GDM3))
return;
airoha_ppe_foe_flow_stat_entry_reset(ppe, npu, index);
@@ -688,35 +1055,37 @@ static void
airoha_ppe_foe_flow_stats_update(struct airoha_ppe *ppe,
}
static struct airoha_foe_entry *
-airoha_ppe_foe_get_entry_locked(struct airoha_ppe *ppe, u32 hash)
+airoha_ppe_foe_get_entry_locked(struct airoha_ppe *ppe, u32 sw_idx)
{
u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe);
lockdep_assert_held(&ppe_lock);
- if (hash < sram_num_entries) {
- u32 *hwe = ppe->foe + hash * sizeof(struct airoha_foe_entry);
- bool ppe2 = hash >= PPE_SRAM_NUM_ENTRIES;
- struct airoha_eth *eth = ppe->eth;
+ if (sw_idx < sram_num_entries) {
+ u32 hw_idx = airoha_ppe_sw_to_hw_idx(ppe, sw_idx);
+ u32 *hwe = ppe->common.foe + sw_idx * AIROHA_FOE_ENTRY_SIZE;
+ bool ppe2 = sw_idx >= ppe->common.eth->soc->ppe_sram_entries;
+ struct airoha_eth *eth = ppe->common.eth;
u32 val;
int i;
- airoha_fe_wr(ppe->eth, REG_PPE_RAM_CTRL(ppe2),
- FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hash) |
+ airoha_fe_wr(ppe->common.eth, REG_PPE_RAM_CTRL(ppe2),
+ FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hw_idx) |
PPE_SRAM_CTRL_REQ_MASK);
if (read_poll_timeout_atomic(airoha_fe_rr, val,
val & PPE_SRAM_CTRL_ACK_MASK,
10, 100, false, eth,
- REG_PPE_RAM_CTRL(ppe2)))
+ REG_PPE_RAM_CTRL(ppe2))) {
+ dev_err(eth->dev, "Timeout reading PPE SRAM entry for hash
%u\n", hw_idx);
return NULL;
+ }
- for (i = 0; i < sizeof(struct airoha_foe_entry) / sizeof(*hwe);
- i++)
+ for (i = 0; i < AIROHA_FOE_ENTRY_WORDS; i++)
hwe[i] = airoha_fe_rr(eth,
REG_PPE_RAM_ENTRY(ppe2, i));
}
- return ppe->foe + hash * sizeof(struct airoha_foe_entry);
+ return ppe->common.foe + sw_idx * AIROHA_FOE_ENTRY_SIZE;
}
struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe,
@@ -748,39 +1117,45 @@ static bool airoha_ppe_foe_compare_entry(struct
airoha_flow_table_entry *e,
return !memcmp(&e->data.d, &hwe->d, len - sizeof(hwe->ib1));
}
-static int airoha_ppe_foe_commit_sram_entry(struct airoha_ppe *ppe, u32
hash)
+static int airoha_ppe_foe_commit_sram_entry(struct airoha_ppe *ppe, u32
sw_idx)
{
- struct airoha_foe_entry *hwe = ppe->foe + hash * sizeof(*hwe);
- bool ppe2 = hash >= PPE_SRAM_NUM_ENTRIES;
+ struct airoha_foe_entry *hwe = ppe->common.foe + sw_idx *
AIROHA_FOE_ENTRY_SIZE;
+ bool ppe2 = sw_idx >= ppe->common.eth->soc->ppe_sram_entries;
+ u32 hw_idx = airoha_ppe_sw_to_hw_idx(ppe, sw_idx);
u32 *ptr = (u32 *)hwe, val;
- int i;
+ int i, err;
- for (i = 0; i < sizeof(*hwe) / sizeof(*ptr); i++)
- airoha_fe_wr(ppe->eth, REG_PPE_RAM_ENTRY(ppe2, i), ptr[i]);
+ for (i = 0; i < AIROHA_FOE_ENTRY_WORDS; i++)
+ airoha_fe_wr(ppe->common.eth, REG_PPE_RAM_ENTRY(ppe2, i), ptr[i]);
wmb();
- airoha_fe_wr(ppe->eth, REG_PPE_RAM_CTRL(ppe2),
- FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hash) |
+ airoha_fe_wr(ppe->common.eth, REG_PPE_RAM_CTRL(ppe2),
+ FIELD_PREP(PPE_SRAM_CTRL_ENTRY_MASK, hw_idx) |
PPE_SRAM_CTRL_WR_MASK | PPE_SRAM_CTRL_REQ_MASK);
- return read_poll_timeout_atomic(airoha_fe_rr, val,
+ err = read_poll_timeout_atomic(airoha_fe_rr, val,
val & PPE_SRAM_CTRL_ACK_MASK,
- 10, 100, false, ppe->eth,
+ 10, 100, false, ppe->common.eth,
REG_PPE_RAM_CTRL(ppe2));
+ if (err)
+ dev_err(ppe->common.eth->dev, "Timeout committing SRAM entry
hash %u\n", hw_idx);
+
+ return err;
}
static int airoha_ppe_foe_commit_entry(struct airoha_ppe *ppe,
struct airoha_foe_entry *e,
- u32 hash, bool rx_wlan)
+ u32 sw_idx, bool rx_wlan)
{
u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe);
- struct airoha_foe_entry *hwe = ppe->foe + hash * sizeof(*hwe);
+ struct airoha_foe_entry *hwe = ppe->common.foe + sw_idx *
AIROHA_FOE_ENTRY_SIZE;
u32 ts = airoha_ppe_get_timestamp(ppe);
- struct airoha_eth *eth = ppe->eth;
+ struct airoha_eth *eth = ppe->common.eth;
struct airoha_npu *npu;
int err = 0;
- memcpy(&hwe->d, &e->d, sizeof(*hwe) - sizeof(hwe->ib1));
+ memcpy(&hwe->words[1], &e->words[1],
+ AIROHA_FOE_ENTRY_SIZE - sizeof(hwe->words[0]));
wmb();
e->ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP;
@@ -790,17 +1165,20 @@ static int airoha_ppe_foe_commit_entry(struct
airoha_ppe *ppe,
rcu_read_lock();
npu = rcu_dereference(eth->npu);
- if (!npu) {
- err = -ENODEV;
- goto unlock;
- }
-
- if (!rx_wlan)
- airoha_ppe_foe_flow_stats_update(ppe, npu, hwe, hash);
+ if (!rx_wlan && npu)
+ airoha_ppe_foe_flow_stats_update(ppe, npu, hwe, sw_idx);
+ else if (!npu)
+ dev_dbg(eth->dev, "NPU not attached, skipping FOE stats setup
for entry %u\n",
+ sw_idx);
+
+ /* FOE programming must not depend on the optional NPU/stats path.
+ * EN7523 systems can run with no attached NPU; in that case DRAM
entries
+ * are already written above through the coherent FOE table, while SRAM
+ * entries still need the explicit SRAM_CTRL write below.
+ */
+ if (sw_idx < sram_num_entries)
+ err = airoha_ppe_foe_commit_sram_entry(ppe, sw_idx);
- if (hash < sram_num_entries)
- err = airoha_ppe_foe_commit_sram_entry(ppe, hash);
-unlock:
rcu_read_unlock();
return err;
@@ -855,24 +1233,30 @@ static void
airoha_ppe_foe_flow_remove_entry(struct airoha_ppe *ppe,
static int
airoha_ppe_foe_commit_subflow_entry(struct airoha_ppe *ppe,
struct airoha_flow_table_entry *e,
- u32 hash, bool rx_wlan)
+ u32 sw_idx, bool rx_wlan)
{
u32 mask = AIROHA_FOE_IB1_BIND_PACKET_TYPE | AIROHA_FOE_IB1_BIND_UDP;
struct airoha_foe_entry *hwe_p, hwe;
struct airoha_flow_table_entry *f;
int type;
- hwe_p = airoha_ppe_foe_get_entry_locked(ppe, hash);
- if (!hwe_p)
+ hwe_p = airoha_ppe_foe_get_entry_locked(ppe, sw_idx);
+ if (!hwe_p) {
+ dev_err(ppe->common.eth->dev,
+ "Failed to get locked entry for subflow commit (hash: %u)\n",
+ sw_idx);
return -EINVAL;
+ }
f = kzalloc(sizeof(*f), GFP_ATOMIC);
- if (!f)
+ if (!f) {
+ dev_err(ppe->common.eth->dev, "OOM in subflow commit\n");
return -ENOMEM;
+ }
hlist_add_head(&f->l2_subflow_node, &e->l2_flows);
f->type = FLOW_TYPE_L2_SUBFLOW;
- f->hash = hash;
+ f->hash = sw_idx;
memcpy(&hwe, hwe_p, sizeof(*hwe_p));
hwe.ib1 = (hwe.ib1 & mask) | (e->data.ib1 & ~mask);
@@ -881,9 +1265,6 @@ airoha_ppe_foe_commit_subflow_entry(struct
airoha_ppe *ppe,
if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) {
memcpy(&hwe.ipv6.l2, &e->data.bridge.l2, sizeof(hwe.ipv6.l2));
hwe.ipv6.ib2 = e->data.bridge.ib2;
- /* setting smac_id to 0xf instruct the hw to keep original
- * source mac address
- */
hwe.ipv6.l2.src_mac_hi = FIELD_PREP(AIROHA_FOE_MAC_SMAC_ID,
0xf);
} else {
@@ -896,14 +1277,344 @@ airoha_ppe_foe_commit_subflow_entry(struct
airoha_ppe *ppe,
}
hwe.bridge.data = e->data.bridge.data;
- airoha_ppe_foe_commit_entry(ppe, &hwe, hash, rx_wlan);
+ airoha_ppe_foe_commit_entry(ppe, &hwe, sw_idx, rx_wlan);
return 0;
}
+enum airoha_ppe_tuple_family {
+ AIROHA_PPE_TUPLE_IPV4,
+ AIROHA_PPE_TUPLE_IPV6,
+};
+
+struct airoha_ppe_skb_tuple {
+ enum airoha_ppe_tuple_family family;
+ u8 l4proto;
+ bool has_ports;
+
+ union {
+ struct {
+ __be32 src;
+ __be32 dst;
+ } v4;
+ struct {
+ struct in6_addr src;
+ struct in6_addr dst;
+ } v6;
+ };
+
+ u16 src_port;
+ u16 dst_port;
+};
+
+static bool
+airoha_ppe_skb_read_ports(struct sk_buff *skb, unsigned int off, u8
l4proto,
+ struct airoha_ppe_skb_tuple *tuple)
+{
+ struct udphdr _uh, *uh;
+ struct tcphdr _th, *th;
+
+ tuple->has_ports = false;
+
+ switch (l4proto) {
+ case IPPROTO_TCP:
+ th = skb_header_pointer(skb, off, sizeof(_th), &_th);
+ if (!th)
+ return false;
+
+ tuple->src_port = be16_to_cpu(th->source);
+ tuple->dst_port = be16_to_cpu(th->dest);
+ break;
+ case IPPROTO_UDP:
+ uh = skb_header_pointer(skb, off, sizeof(_uh), &_uh);
+ if (!uh)
+ return false;
+
+ tuple->src_port = be16_to_cpu(uh->source);
+ tuple->dst_port = be16_to_cpu(uh->dest);
+ break;
+ default:
+ return false;
+ }
+
+ tuple->has_ports = true;
+ return true;
+}
+
+static bool
+airoha_ppe_skb_read_ipv4_tuple_at(struct sk_buff *skb, unsigned int off,
+ struct airoha_ppe_skb_tuple *tuple)
+{
+ struct iphdr _iph, *iph;
+ unsigned int l4_off;
+ u16 tot_len;
+
+ if (off + sizeof(_iph) > skb->len)
+ return false;
+
+ iph = skb_header_pointer(skb, off, sizeof(_iph), &_iph);
+ if (!iph || iph->version != 4 || iph->ihl < 5)
+ return false;
+
+ if (iph->protocol != IPPROTO_TCP && iph->protocol != IPPROTO_UDP)
+ return false;
+
+ tot_len = ntohs(iph->tot_len);
+ if (tot_len < iph->ihl * 4 || off + tot_len > skb->len)
+ return false;
+
+ tuple->family = AIROHA_PPE_TUPLE_IPV4;
+ tuple->l4proto = iph->protocol;
+ tuple->v4.src = iph->saddr;
+ tuple->v4.dst = iph->daddr;
+
+ l4_off = off + iph->ihl * 4;
+ return airoha_ppe_skb_read_ports(skb, l4_off, iph->protocol, tuple);
+}
+
+static bool
+airoha_ppe_skb_read_ipv6_tuple_at(struct sk_buff *skb, unsigned int off,
+ struct airoha_ppe_skb_tuple *tuple)
+{
+ struct ipv6_opt_hdr _opth, *opth;
+ struct ip_auth_hdr _authh, *authh;
+ struct frag_hdr _fragh, *fragh;
+ struct ipv6hdr _ip6h, *ip6h;
+ unsigned int l4_off, hdr_len;
+ u8 nexthdr;
+ int i;
+
+ if (off + sizeof(_ip6h) > skb->len)
+ return false;
+
+ ip6h = skb_header_pointer(skb, off, sizeof(_ip6h), &_ip6h);
+ if (!ip6h || ip6h->version != 6)
+ return false;
+
+ /* payload_len can be zero for jumbograms. */
+ if (ip6h->payload_len &&
+ off + sizeof(*ip6h) + ntohs(ip6h->payload_len) > skb->len)
+ return false;
+
+ nexthdr = ip6h->nexthdr;
+ l4_off = off + sizeof(*ip6h);
+
+ /* Traverse a bounded number of IPv6 extension headers. */
+ for (i = 0; i < 8; i++) {
+ switch (nexthdr) {
+ case NEXTHDR_HOP:
+ case NEXTHDR_ROUTING:
+ case NEXTHDR_DEST:
+ opth = skb_header_pointer(skb, l4_off,
+ sizeof(_opth), &_opth);
+ if (!opth)
+ return false;
+
+ hdr_len = ipv6_optlen(opth);
+ if (!hdr_len || l4_off + hdr_len > skb->len)
+ return false;
+
+ nexthdr = opth->nexthdr;
+ l4_off += hdr_len;
+ continue;
+ case NEXTHDR_FRAGMENT:
+ fragh = skb_header_pointer(skb, l4_off,
+ sizeof(_fragh), &_fragh);
+ if (!fragh)
+ return false;
+
+ /* Only the first fragment contains the L4 header. */
+ if (ntohs(fragh->frag_off) & 0xfff8)
+ return false;
+
+ nexthdr = fragh->nexthdr;
+ l4_off += sizeof(*fragh);
+ continue;
+ case NEXTHDR_AUTH:
+ authh = skb_header_pointer(skb, l4_off,
+ sizeof(_authh), &_authh);
+ if (!authh)
+ return false;
+
+ hdr_len = (authh->hdrlen + 2) << 2;
+ if (!hdr_len || l4_off + hdr_len > skb->len)
+ return false;
+
+ nexthdr = authh->nexthdr;
+ l4_off += hdr_len;
+ continue;
+ case NEXTHDR_NONE:
+ case NEXTHDR_ESP:
+ return false;
+ default:
+ goto l4;
+ }
+ }
+
+ return false;
+
+l4:
+ if (nexthdr != IPPROTO_TCP && nexthdr != IPPROTO_UDP)
+ return false;
+
+ tuple->family = AIROHA_PPE_TUPLE_IPV6;
+ tuple->l4proto = nexthdr;
+ tuple->v6.src = ip6h->saddr;
+ tuple->v6.dst = ip6h->daddr;
+
+ return airoha_ppe_skb_read_ports(skb, l4_off, nexthdr, tuple);
+}
+
+static bool
+airoha_ppe_skb_find_tuple(struct sk_buff *skb,
+ struct airoha_ppe_skb_tuple *tuple)
+{
+ static const unsigned int fixed_offsets[] = {
+ 0,
+ ETH_HLEN,
+ ETH_HLEN + VLAN_HLEN,
+ ETH_HLEN + 2 * VLAN_HLEN,
+ };
+ unsigned int i, off, max_scan;
+
+ /* skb->protocol is not reliable at this point on EN7523. */
+ for (i = 0; i < ARRAY_SIZE(fixed_offsets); i++) {
+ off = fixed_offsets[i];
+ if (airoha_ppe_skb_read_ipv4_tuple_at(skb, off, tuple) ||
+ airoha_ppe_skb_read_ipv6_tuple_at(skb, off, tuple))
+ return true;
+ }
+
+ off = skb_network_offset(skb);
+ if (off < skb->len &&
+ (airoha_ppe_skb_read_ipv4_tuple_at(skb, off, tuple) ||
+ airoha_ppe_skb_read_ipv6_tuple_at(skb, off, tuple)))
+ return true;
+
+ /* The packet can still carry a switch/special tag before L3. */
+ max_scan = min_t(unsigned int, skb->len, 96);
+ for (off = 0; off < max_scan; off++) {
+ if (airoha_ppe_skb_read_ipv4_tuple_at(skb, off, tuple) ||
+ airoha_ppe_skb_read_ipv6_tuple_at(skb, off, tuple))
+ return true;
+ }
+
+ return false;
+}
+
+static bool
+airoha_ppe_skb_tuple_is_candidate(const struct airoha_ppe_skb_tuple *tuple)
+{
+ switch (tuple->family) {
+ case AIROHA_PPE_TUPLE_IPV4:
+ return !ipv4_is_multicast(tuple->v4.dst) &&
+ !ipv4_is_lbcast(tuple->v4.dst) &&
+ !ipv4_is_zeronet(tuple->v4.dst);
+ case AIROHA_PPE_TUPLE_IPV6:
+ return !ipv6_addr_any(&tuple->v6.dst) &&
+ !ipv6_addr_is_multicast(&tuple->v6.dst) &&
+ !(ipv6_addr_type(&tuple->v6.dst) & IPV6_ADDR_LINKLOCAL);
+ default:
+ return false;
+ }
+}
+
+static bool
+airoha_ppe_ipv4_tuple_match(const struct airoha_foe_ipv4_tuple *foe,
+ const struct airoha_ppe_skb_tuple *skb,
+ bool with_ports)
+{
+ if (skb->family != AIROHA_PPE_TUPLE_IPV4)
+ return false;
+
+ if (foe->src_ip != be32_to_cpu(skb->v4.src) ||
+ foe->dest_ip != be32_to_cpu(skb->v4.dst))
+ return false;
+
+ if (!with_ports)
+ return true;
+
+ return skb->has_ports && foe->src_port == skb->src_port &&
+ foe->dest_port == skb->dst_port;
+}
+
+static bool
+airoha_ppe_ipv6_tuple_match(const struct airoha_foe_ipv6 *foe,
+ const struct airoha_ppe_skb_tuple *skb,
+ bool with_ports)
+{
+ u32 src[IPV6_ADDR_WORDS], dst[IPV6_ADDR_WORDS];
+
+ if (skb->family != AIROHA_PPE_TUPLE_IPV6)
+ return false;
+
+ ipv6_addr_be32_to_cpu(src, skb->v6.src.s6_addr32);
+ ipv6_addr_be32_to_cpu(dst, skb->v6.dst.s6_addr32);
+
+ if (memcmp(foe->src_ip, src, sizeof(src)) ||
+ memcmp(foe->dest_ip, dst, sizeof(dst)))
+ return false;
+
+ if (!with_ports)
+ return true;
+
+ return skb->has_ports && foe->src_port == skb->src_port &&
+ foe->dest_port == skb->dst_port;
+}
+
+static bool
+airoha_ppe_foe_entry_match_skb_tuple(struct airoha_flow_table_entry *e,
+ const struct airoha_ppe_skb_tuple *skb)
+{
+ u32 type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, e->data.ib1);
+ u8 l4proto = e->data.ib1 & AIROHA_FOE_IB1_BIND_UDP ?
+ IPPROTO_UDP : IPPROTO_TCP;
+
+ if (skb->l4proto != l4proto)
+ return false;
+
+ switch (type) {
+ case PPE_PKT_TYPE_IPV4_ROUTE:
+ return airoha_ppe_ipv4_tuple_match(&e->data.ipv4.orig_tuple,
+ skb, false);
+ case PPE_PKT_TYPE_IPV4_HNAPT:
+ return airoha_ppe_ipv4_tuple_match(&e->data.ipv4.orig_tuple,
+ skb, true) ||
+ airoha_ppe_ipv4_tuple_match(&e->data.ipv4.new_tuple,
+ skb, true);
+ case PPE_PKT_TYPE_IPV6_ROUTE_3T:
+ return airoha_ppe_ipv6_tuple_match(&e->data.ipv6, skb, false);
+ case PPE_PKT_TYPE_IPV6_ROUTE_5T:
+ return airoha_ppe_ipv6_tuple_match(&e->data.ipv6, skb, true);
+ default:
+ return false;
+ }
+}
+
+static struct airoha_flow_table_entry *
+airoha_ppe_find_pending_flow_by_skb(struct airoha_ppe *ppe, struct
sk_buff *skb)
+{
+ struct airoha_ppe_skb_tuple tuple;
+ struct airoha_flow_table_entry *e;
+
+ if (!airoha_ppe_skb_find_tuple(skb, &tuple) ||
+ !airoha_ppe_skb_tuple_is_candidate(&tuple))
+ return NULL;
+
+ hlist_for_each_entry(e, &ppe->pending_flows, list) {
+ if (e->type != FLOW_TYPE_L4 || e->hash != 0xffff)
+ continue;
+
+ if (airoha_ppe_foe_entry_match_skb_tuple(e, &tuple))
+ return e;
+ }
+
+ return NULL;
+}
+
static void airoha_ppe_foe_insert_entry(struct airoha_ppe *ppe,
struct sk_buff *skb,
- u32 hash, bool rx_wlan)
+ u32 sw_idx, bool rx_wlan)
{
struct airoha_flow_table_entry *e;
struct airoha_foe_bridge br = {};
@@ -914,7 +1625,7 @@ static void airoha_ppe_foe_insert_entry(struct
airoha_ppe *ppe,
spin_lock_bh(&ppe_lock);
- hwe = airoha_ppe_foe_get_entry_locked(ppe, hash);
+ hwe = airoha_ppe_foe_get_entry_locked(ppe, sw_idx);
if (!hwe)
goto unlock;
@@ -922,6 +1633,34 @@ static void airoha_ppe_foe_insert_entry(struct
airoha_ppe *ppe,
if (state == AIROHA_FOE_STATE_BIND)
goto unlock;
+ if (airoha_is(ppe->common.eth, airoha_en7523)) {
+ /* On EN7523 the offloaded flows live in ppe->pending_flows (the
+ * SW hash does not match the HW hash), so they are matched by
+ * skb tuple rather than via foe_flow[index]. The HW may present
+ * the slot either as INVALID (FOE_UNHIT) or as an auto-created
+ * UNBIND entry (HIT_UNBIND, when SRAM auto-learn is active);
+ * both must drive the same UNBIND/INVALID -> BIND commit.
+ */
+ e = airoha_ppe_find_pending_flow_by_skb(ppe, skb);
+ if (e) {
+ int err;
+
+ err = airoha_ppe_foe_commit_entry(ppe, &e->data,
+ sw_idx, rx_wlan);
+ if (!err) {
+ hlist_del_init(&e->list);
+ hlist_add_head(&e->list,
+ &ppe->foe_flow[sw_idx]);
+ e->hash = sw_idx;
+ }
+ }
+
+ goto unlock;
+ }
+
+ if (state == AIROHA_FOE_STATE_INVALID)
+ goto unlock;
+
index = airoha_ppe_foe_get_entry_hash(ppe, hwe);
hlist_for_each_entry_safe(e, n, &ppe->foe_flow[index], list) {
if (e->type == FLOW_TYPE_L2_SUBFLOW) {
@@ -936,9 +1675,9 @@ static void airoha_ppe_foe_insert_entry(struct
airoha_ppe *ppe,
if (!airoha_ppe_foe_compare_entry(e, hwe))
continue;
- airoha_ppe_foe_commit_entry(ppe, &e->data, hash, rx_wlan);
+ airoha_ppe_foe_commit_entry(ppe, &e->data, sw_idx, rx_wlan);
commit_done = true;
- e->hash = hash;
+ e->hash = sw_idx;
}
if (commit_done)
@@ -948,7 +1687,7 @@ static void airoha_ppe_foe_insert_entry(struct
airoha_ppe *ppe,
e = rhashtable_lookup_fast(&ppe->l2_flows, &br,
airoha_l2_flow_table_params);
if (e)
- airoha_ppe_foe_commit_subflow_entry(ppe, e, hash, rx_wlan);
+ airoha_ppe_foe_commit_subflow_entry(ppe, e, sw_idx, rx_wlan);
unlock:
spin_unlock_bh(&ppe_lock);
}
@@ -965,8 +1704,12 @@ airoha_ppe_foe_l2_flow_commit_entry(struct
airoha_ppe *ppe,
if (!prev)
return 0;
- if (IS_ERR(prev))
+ if (IS_ERR(prev)) {
+ dev_err(ppe->common.eth->dev,
+ "Error inserting L2 flow to rhashtable: %ld\n",
+ PTR_ERR(prev));
return PTR_ERR(prev);
+ }
return rhashtable_replace_fast(&ppe->l2_flows, &prev->l2_node,
&e->l2_node,
@@ -987,7 +1730,10 @@ static int airoha_ppe_foe_flow_commit_entry(struct
airoha_ppe *ppe,
e->hash = 0xffff;
spin_lock_bh(&ppe_lock);
- hlist_add_head(&e->list, &ppe->foe_flow[hash]);
+ if (airoha_is(ppe->common.eth, airoha_en7523))
+ hlist_add_head(&e->list, &ppe->pending_flows);
+ else
+ hlist_add_head(&e->list, &ppe->foe_flow[hash]);
spin_unlock_bh(&ppe_lock);
return 0;
@@ -1179,6 +1925,7 @@ static int airoha_ppe_flow_offload_replace(struct
airoha_eth *eth,
data.vlan.hdr[data.vlan.num].id = act->vlan.vid;
data.vlan.hdr[data.vlan.num].proto = act->vlan.proto;
+ data.vlan.hdr[data.vlan.num].prio = act->vlan.prio;
data.vlan.num++;
break;
case FLOW_ACTION_VLAN_POP:
@@ -1196,8 +1943,10 @@ static int airoha_ppe_flow_offload_replace(struct
airoha_eth *eth,
}
if (!is_valid_ether_addr(data.eth.h_source) ||
- !is_valid_ether_addr(data.eth.h_dest))
+ !is_valid_ether_addr(data.eth.h_dest)) {
+ dev_err(eth->dev, "Invalid ether addr on flow replace\n");
return -EINVAL;
+ }
err = airoha_ppe_foe_entry_prepare(eth, &hwe, odev, offload_type,
&data, l4proto, dsfield);
@@ -1258,8 +2007,10 @@ static int airoha_ppe_flow_offload_replace(struct
airoha_eth *eth,
return -EOPNOTSUPP;
}
- if (err)
+ if (err) {
+ dev_err(eth->dev, "Failed to mangle flow (err: %d)\n", err);
return err;
+ }
}
if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
@@ -1269,20 +2020,26 @@ static int
airoha_ppe_flow_offload_replace(struct airoha_eth *eth,
}
e = kzalloc(sizeof(*e), GFP_KERNEL);
- if (!e)
+ if (!e) {
+ dev_err(eth->dev, "OOM allocating flow table entry\n");
return -ENOMEM;
+ }
e->cookie = f->cookie;
memcpy(&e->data, &hwe, sizeof(e->data));
err = airoha_ppe_foe_flow_commit_entry(eth->ppe, e);
- if (err)
+ if (err) {
+ dev_err(eth->dev, "Failed to commit PPE flow entry (err:
%d)\n", err);
goto free_entry;
+ }
err = rhashtable_insert_fast(ð->flow_table, &e->node,
airoha_flow_table_params);
- if (err < 0)
+ if (err < 0) {
+ dev_err(eth->dev, "Failed to insert into rhashtable (err:
%d)\n", err);
goto remove_foe_entry;
+ }
return 0;
@@ -1312,41 +2069,6 @@ static int airoha_ppe_flow_offload_destroy(struct
airoha_eth *eth,
return 0;
}
-void airoha_ppe_foe_entry_get_stats(struct airoha_ppe *ppe, u32 hash,
- struct airoha_foe_stats64 *stats)
-{
- struct airoha_eth *eth = ppe->eth;
- int ppe_num_stats_entries;
- struct airoha_npu *npu;
- u32 index;
-
- ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe);
- if (ppe_num_stats_entries < 0)
- return;
-
- if (airoha_ppe_foe_get_flow_stats_index(ppe, hash, &index))
- return;
-
- if (index >= ppe_num_stats_entries)
- return;
-
- rcu_read_lock();
-
- npu = rcu_dereference(eth->npu);
- if (npu) {
- u64 packets = ppe->foe_stats[index].packets;
- u64 bytes = ppe->foe_stats[index].bytes;
- struct airoha_foe_stats npu_stats;
-
- memcpy_fromio(&npu_stats, &npu->stats[index],
- sizeof(*npu->stats));
- stats->packets = packets << 32 | npu_stats.packets;
- stats->bytes = bytes << 32 | npu_stats.bytes;
- }
-
- rcu_read_unlock();
-}
-
static int airoha_ppe_flow_offload_stats(struct airoha_eth *eth,
struct flow_cls_offload *f)
{
@@ -1374,35 +2096,45 @@ static int airoha_ppe_flow_offload_stats(struct
airoha_eth *eth,
}
static int airoha_ppe_flow_offload_cmd(struct airoha_eth *eth,
+ struct net_device *dev,
struct flow_cls_offload *f)
{
+ bool v1 = eth->soc->foe_format == AIROHA_FOE_FORMAT_V1;
+
+ if (v1 && !dev)
+ return -EOPNOTSUPP;
+
switch (f->command) {
case FLOW_CLS_REPLACE:
- return airoha_ppe_flow_offload_replace(eth, f);
+ return v1 ? airoha_ppe_v1_flow_offload_replace(dev, f) :
+ airoha_ppe_flow_offload_replace(eth, f);
case FLOW_CLS_DESTROY:
- return airoha_ppe_flow_offload_destroy(eth, f);
+ return v1 ? airoha_ppe_v1_flow_offload_destroy(dev, f) :
+ airoha_ppe_flow_offload_destroy(eth, f);
case FLOW_CLS_STATS:
- return airoha_ppe_flow_offload_stats(eth, f);
+ return v1 ? airoha_ppe_v1_flow_offload_stats(f) :
+ airoha_ppe_flow_offload_stats(eth, f);
default:
- break;
+ return -EOPNOTSUPP;
}
-
- return -EOPNOTSUPP;
}
static int airoha_ppe_flush_sram_entries(struct airoha_ppe *ppe)
{
u32 sram_num_entries = airoha_ppe_get_total_sram_num_entries(ppe);
- struct airoha_foe_entry *hwe = ppe->foe;
+ struct airoha_foe_entry *hwe = ppe->common.foe;
int i, err = 0;
+ dev_info(ppe->common.eth->dev, "Flushing %u SRAM entries\n",
sram_num_entries);
for (i = 0; i < sram_num_entries; i++) {
- int err;
-
memset(&hwe[i], 0, sizeof(*hwe));
err = airoha_ppe_foe_commit_sram_entry(ppe, i);
- if (err)
+ if (err) {
+ dev_err(ppe->common.eth->dev,
+ "failed to flush SRAM entry %d, err %d\n",
+ i, err);
break;
+ }
}
return err;
@@ -1410,13 +2142,26 @@ static int airoha_ppe_flush_sram_entries(struct
airoha_ppe *ppe)
static struct airoha_npu *airoha_ppe_npu_get(struct airoha_eth *eth)
{
- struct airoha_npu *npu = airoha_npu_get(eth->dev);
+ struct airoha_npu *npu;
- if (IS_ERR(npu)) {
+ if (!eth->ppe_host_ops || !eth->ppe_host_ops->npu_get)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ npu = eth->ppe_host_ops->npu_get(eth);
+ if (!npu)
+ npu = ERR_PTR(-EOPNOTSUPP);
+
+ if (IS_ERR(npu) && PTR_ERR(npu) != -EOPNOTSUPP) {
+ dev_info(eth->dev, "Requesting airoha-npu module\n");
request_module("airoha-npu");
- npu = airoha_npu_get(eth->dev);
+ npu = eth->ppe_host_ops->npu_get(eth);
+ if (!npu)
+ npu = ERR_PTR(-EOPNOTSUPP);
}
+ if (IS_ERR(npu))
+ dev_err(eth->dev, "Failed to get NPU module\n");
+
return npu;
}
@@ -1429,16 +2174,23 @@ static int airoha_ppe_wait_for_npu_init(struct
airoha_eth *eth)
* during the device probe we can just check the configured value
* is not 0 here.
*/
+ dev_info(eth->dev, "Waiting for NPU init on PPE 0\n");
err = read_poll_timeout(airoha_fe_rr, val, val, USEC_PER_MSEC,
100 * USEC_PER_MSEC, false, eth,
REG_PPE_PPE_FLOW_CFG(0));
- if (err)
+ if (err) {
+ dev_err(eth->dev, "Timeout waiting for NPU initialization on
PPE 0\n");
return err;
+ }
- if (airoha_ppe_is_enabled(eth, 1))
+ if (airoha_ppe_is_enabled(eth, 1)) {
+ dev_info(eth->dev, "Waiting for NPU init on PPE 1\n");
err = read_poll_timeout(airoha_fe_rr, val, val, USEC_PER_MSEC,
100 * USEC_PER_MSEC, false, eth,
REG_PPE_PPE_FLOW_CFG(1));
+ if (err)
+ dev_err(eth->dev, "Timeout waiting for NPU initialization
on PPE 1\n");
+ }
return err;
}
@@ -1449,24 +2201,43 @@ static int airoha_ppe_offload_setup(struct
airoha_eth *eth)
struct airoha_ppe *ppe = eth->ppe;
int err, ppe_num_stats_entries;
- if (IS_ERR(npu))
- return PTR_ERR(npu);
-
- err = npu->ops.ppe_init(npu);
- if (err)
- goto error_npu_put;
+ dev_info(eth->dev, "Setting up PPE offload\n");
+ if (IS_ERR(npu)) {
+ /* NPU disabled / unavailable (e.g. EN7523 with no NPU firmware).
+ * Run CPU-direct PPE offload: the FOE commit path already
+ * supports the no-NPU case. Do the HW init once and leave
+ * eth->npu NULL so the commit/stats paths skip the NPU.
+ */
+ if (!ppe->offload_setup_done) {
+ airoha_ppe_hw_init(ppe);
+ ppe->offload_setup_done = true;
+ dev_info(eth->dev,
+ "PPE offload setup completed (no NPU, CPU-direct)\n");
+ }
+ return 0;
+ }
- /* Wait for NPU PPE configuration to complete */
- err = airoha_ppe_wait_for_npu_init(eth);
- if (err)
+ err = eth->ppe_host_ops->npu_ppe_init(npu);
+ if (err && err != -EOPNOTSUPP) {
+ dev_err(eth->dev, "NPU PPE init failed (err: %d)\n", err);
goto error_npu_put;
+ }
- ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe);
- if (ppe_num_stats_entries > 0) {
- err = npu->ops.ppe_init_stats(npu, ppe->foe_stats_dma,
- ppe_num_stats_entries);
+ if (err != -EOPNOTSUPP) {
+ err = airoha_ppe_wait_for_npu_init(eth);
if (err)
goto error_npu_put;
+
+ ppe_num_stats_entries =
airoha_ppe_get_total_num_stats_entries(ppe);
+ if (ppe_num_stats_entries > 0) {
+ err = eth->ppe_host_ops->npu_ppe_init_stats(npu,
+ ppe->foe_stats_dma,
+ ppe_num_stats_entries);
+ if (err) {
+ dev_err(eth->dev, "NPU PPE stats init failed (err:
%d)\n", err);
+ goto error_npu_put;
+ }
+ }
}
airoha_ppe_hw_init(ppe);
@@ -1475,10 +2246,11 @@ static int airoha_ppe_offload_setup(struct
airoha_eth *eth)
rcu_assign_pointer(eth->npu, npu);
synchronize_rcu();
+ dev_info(eth->dev, "PPE offload setup completed successfully\n");
return 0;
error_npu_put:
- airoha_npu_put(npu);
+ eth->ppe_host_ops->npu_put(npu);
return err;
}
@@ -1486,7 +2258,7 @@ static int airoha_ppe_offload_setup(struct
airoha_eth *eth)
int airoha_ppe_setup_tc_block_cb(struct airoha_ppe_dev *dev, void
*type_data)
{
struct airoha_ppe *ppe = dev->priv;
- struct airoha_eth *eth = ppe->eth;
+ struct airoha_eth *eth = ppe->common.eth;
int err = 0;
/* Netfilter flowtable can try to offload flower rules while not all
@@ -1498,10 +2270,12 @@ int airoha_ppe_setup_tc_block_cb(struct
airoha_ppe_dev *dev, void *type_data)
mutex_lock(&flow_offload_mutex);
- if (!eth->npu)
+ if (!eth->npu && !eth->ppe->offload_setup_done) {
+ dev_info(eth->dev, "NPU not attached, setting up offload\n");
err = airoha_ppe_offload_setup(eth);
+ }
if (!err)
- err = airoha_ppe_flow_offload_cmd(eth, type_data);
+ err = airoha_ppe_flow_offload_cmd(eth, NULL, type_data);
mutex_unlock(&flow_offload_mutex);
@@ -1512,19 +2286,19 @@ void airoha_ppe_check_skb(struct airoha_ppe_dev
*dev, struct sk_buff *skb,
u16 hash, bool rx_wlan)
{
struct airoha_ppe *ppe = dev->priv;
- u32 ppe_hash_mask = airoha_ppe_get_total_num_entries(ppe) - 1;
+ u32 sw_idx;
u16 now, diff;
- if (hash > ppe_hash_mask)
+ if (airoha_ppe_hw_to_sw_idx(ppe, hash, &sw_idx))
return;
now = (u16)jiffies;
- diff = now - ppe->foe_check_time[hash];
+ diff = now - ppe->foe_check_time[sw_idx];
if (diff < HZ / 10)
return;
- ppe->foe_check_time[hash] = now;
- airoha_ppe_foe_insert_entry(ppe, skb, hash, rx_wlan);
+ ppe->foe_check_time[sw_idx] = now;
+ airoha_ppe_foe_insert_entry(ppe, skb, sw_idx, rx_wlan);
}
void airoha_ppe_init_upd_mem(struct airoha_gdm_dev *dev, const u8 *addr)
@@ -1533,6 +2307,8 @@ void airoha_ppe_init_upd_mem(struct airoha_gdm_dev
*dev, const u8 *addr)
struct airoha_eth *eth = dev->eth;
u32 val;
+ dev_info(eth->dev, "Initializing UPD mem for port id %d\n", port->id);
+
val = (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) | addr[5];
airoha_fe_wr(eth, REG_UPDMEM_DATA(0), val);
airoha_fe_wr(eth, REG_UPDMEM_CTRL(0),
@@ -1571,8 +2347,10 @@ struct airoha_ppe_dev *airoha_ppe_get_dev(struct
device *dev)
}
eth = platform_get_drvdata(pdev);
- if (!eth)
+ if (!eth) {
+ dev_err(dev, "failed to get platform drvdata for airoha_eth\n");
goto error_module_put;
+ }
if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER)) {
dev_err(&pdev->dev,
@@ -1581,7 +2359,13 @@ struct airoha_ppe_dev *airoha_ppe_get_dev(struct
device *dev)
goto error_module_put;
}
- return ð->ppe->dev;
+ if (!eth->ppe_dev) {
+ dev_err(dev, "Ethernet PPE is not available\n");
+ goto error_module_put;
+ }
+
+ dev_info(dev, "Successfully retrieved PPE device\n");
+ return eth->ppe_dev;
error_module_put:
module_put(THIS_MODULE);
@@ -1594,42 +2378,81 @@ EXPORT_SYMBOL_GPL(airoha_ppe_get_dev);
void airoha_ppe_put_dev(struct airoha_ppe_dev *dev)
{
- struct airoha_ppe *ppe = dev->priv;
- struct airoha_eth *eth = ppe->eth;
-
module_put(THIS_MODULE);
- put_device(eth->dev);
+ if (dev->parent)
+ put_device(dev->parent);
}
EXPORT_SYMBOL_GPL(airoha_ppe_put_dev);
-int airoha_ppe_init(struct airoha_eth *eth)
+static void airoha_ppe_common_init(struct airoha_ppe_common *common,
+ struct airoha_eth *eth, void *priv)
+{
+ common->eth = eth;
+ common->dev.priv = priv;
+ common->dev.parent = eth->dev;
+}
+
+static int airoha_ppe_common_alloc_foe(struct airoha_ppe_common *common,
+ size_t size)
+{
+ struct device *dev = common->eth->dev;
+
+ common->foe = dmam_alloc_coherent(dev, size, &common->foe_dma,
GFP_KERNEL);
+ if (!common->foe)
+ return dev_err_probe(dev, -ENOMEM,
+ "failed to allocate %zu bytes for PPE FoE table\n",
+ size);
+
+ return 0;
+}
+
+static void airoha_ppe_common_enable(struct airoha_ppe_common *common)
+{
+ common->dev.enabled = true;
+ common->eth->ppe_dev = &common->dev;
+}
+
+static void airoha_ppe_common_disable(struct airoha_ppe_common *common)
+{
+ debugfs_remove_recursive(common->debugfs_dir);
+ common->debugfs_dir = NULL;
+ common->dev.enabled = false;
+ if (common->eth->ppe_dev == &common->dev)
+ common->eth->ppe_dev = NULL;
+}
+
+static int airoha_ppe_datapath_init(struct airoha_eth *eth)
{
int foe_size, err, ppe_num_stats_entries;
u32 ppe_num_entries;
struct airoha_ppe *ppe;
+ dev_info(eth->dev, "Starting PPE Initialization\n");
ppe = devm_kzalloc(eth->dev, sizeof(*ppe), GFP_KERNEL);
- if (!ppe)
+ if (!ppe) {
+ dev_err(eth->dev, "OOM allocating struct airoha_ppe\n");
return -ENOMEM;
+ }
- ppe->dev.ops.setup_tc_block_cb = airoha_ppe_setup_tc_block_cb;
- ppe->dev.ops.check_skb = airoha_ppe_check_skb;
- ppe->dev.priv = ppe;
- ppe->eth = eth;
- eth->ppe = ppe;
+ airoha_ppe_common_init(&ppe->common, eth, ppe);
+ ppe->common.dev.ops.setup_tc_block_cb = airoha_ppe_setup_tc_block_cb;
+ ppe->common.dev.ops.check_skb = airoha_ppe_check_skb;
+ INIT_LIST_HEAD(&ppe->block_cb_list);
+ INIT_HLIST_HEAD(&ppe->pending_flows);
ppe_num_entries = airoha_ppe_get_total_num_entries(ppe);
- foe_size = ppe_num_entries * sizeof(struct airoha_foe_entry);
- ppe->foe = dmam_alloc_coherent(eth->dev, foe_size, &ppe->foe_dma,
- GFP_KERNEL);
- if (!ppe->foe)
- return -ENOMEM;
+ foe_size = ppe_num_entries * AIROHA_FOE_ENTRY_SIZE;
+ err = airoha_ppe_common_alloc_foe(&ppe->common, foe_size);
+ if (err)
+ return err;
ppe->foe_flow = devm_kzalloc(eth->dev,
ppe_num_entries * sizeof(*ppe->foe_flow),
GFP_KERNEL);
- if (!ppe->foe_flow)
+ if (!ppe->foe_flow) {
+ dev_err(eth->dev, "OOM allocating FOE flows array\n");
return -ENOMEM;
+ }
ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe);
if (ppe_num_stats_entries > 0) {
@@ -1637,32 +2460,46 @@ int airoha_ppe_init(struct airoha_eth *eth)
ppe->foe_stats = dmam_alloc_coherent(eth->dev, foe_size,
&ppe->foe_stats_dma,
GFP_KERNEL);
- if (!ppe->foe_stats)
+ if (!ppe->foe_stats) {
+ dev_err(eth->dev, "Failed to allocate DMA coherent memory
for FOE stats\n");
return -ENOMEM;
+ }
}
- ppe->foe_check_time = devm_kzalloc(eth->dev,
- ppe_num_entries * sizeof(*ppe->foe_check_time),
- GFP_KERNEL);
- if (!ppe->foe_check_time)
+ ppe->foe_check_time = devm_kcalloc(eth->dev, ppe_num_entries,
+ sizeof(*ppe->foe_check_time), GFP_KERNEL);
+ if (!ppe->foe_check_time) {
+ dev_err(eth->dev, "OOM allocating FOE check time array\n");
return -ENOMEM;
+ }
err = airoha_ppe_flush_sram_entries(ppe);
- if (err)
+ if (err) {
+ dev_err(eth->dev, "Failed to flush SRAM entries during init
(err: %d)\n", err);
return err;
+ }
err = rhashtable_init(ð->flow_table, &airoha_flow_table_params);
- if (err)
+ if (err) {
+ dev_err(eth->dev, "Failed to initialize global flow rhashtable
(err: %d)\n", err);
return err;
+ }
err = rhashtable_init(&ppe->l2_flows, &airoha_l2_flow_table_params);
- if (err)
+ if (err) {
+ dev_err(eth->dev, "Failed to initialize L2 flow rhashtable
(err: %d)\n", err);
goto error_flow_table_destroy;
+ }
- err = airoha_ppe_debugfs_init(ppe);
- if (err)
+ err = airoha_ppe_debugfs_init(&ppe->common);
+ if (err) {
+ dev_err(eth->dev, "Failed to init PPE debugfs (err: %d)\n", err);
goto error_l2_flow_table_destroy;
+ }
+ eth->ppe = ppe;
+ airoha_ppe_common_enable(&ppe->common);
+ dev_info(eth->dev, "PPE Initialization completed successfully\n");
return 0;
error_l2_flow_table_destroy:
@@ -1673,7 +2510,7 @@ int airoha_ppe_init(struct airoha_eth *eth)
return err;
}
-void airoha_ppe_deinit(struct airoha_eth *eth)
+static void airoha_ppe_datapath_deinit(struct airoha_eth *eth)
{
struct airoha_npu *npu;
@@ -1683,13 +2520,1345 @@ void airoha_ppe_deinit(struct airoha_eth *eth)
lockdep_is_held(&flow_offload_mutex));
if (npu) {
synchronize_rcu();
- npu->ops.ppe_deinit(npu);
- airoha_npu_put(npu);
+ if (eth->ppe_host_ops && eth->ppe_host_ops->npu_ppe_deinit)
+ eth->ppe_host_ops->npu_ppe_deinit(npu);
+ if (eth->ppe_host_ops && eth->ppe_host_ops->npu_put)
+ eth->ppe_host_ops->npu_put(npu);
}
mutex_unlock(&flow_offload_mutex);
+ airoha_ppe_common_disable(ð->ppe->common);
rhashtable_destroy(ð->ppe->l2_flows);
rhashtable_destroy(ð->flow_table);
- debugfs_remove(eth->ppe->debugfs_dir);
+ eth->ppe = NULL;
+}
+
+/* FoE v1 backend shared by EN751221 and EN7528. */
+
+static size_t airoha_ppe_v1_foe_size(struct airoha_eth *eth)
+{
+ return (size_t)eth->soc->ppe_dram_entries * AIROHA_FOE_ENTRY_SIZE;
+}
+
+static u32 *airoha_ppe_v1_slot(struct airoha_ppe *ppe, u16 hash)
+{
+ return (u32 *)((u8 *)ppe->common.foe +
+ (size_t)hash * AIROHA_FOE_ENTRY_SIZE);
+}
+
+void airoha_ppe_v1_read_entry(struct airoha_ppe *ppe, u16 hash,
+ struct airoha_foe_entry *entry)
+{
+ u32 *slot = airoha_ppe_v1_slot(ppe, hash);
+ int i;
+
+ dma_rmb();
+ for (i = 0; i < AIROHA_FOE_ENTRY_WORDS; i++)
+ entry->words[i] = READ_ONCE(slot[i]);
+}
+
+static bool airoha_ppe_v1_cache_cmd(struct airoha_ppe *ppe, u32 cmd)
+{
+ void __iomem *reg = ppe->common.eth->fe_regs +
REG_EN751221_PPE_CACHE_CTL;
+ u32 val;
+
+ writel(FIELD_PREP(EN751221_PPE_CACHE_CTL_CMD, cmd) |
+ EN751221_PPE_CACHE_CTL_REQ, reg);
+ return !readl_poll_timeout_atomic(reg, val,
+ !(val & EN751221_PPE_CACHE_CTL_REQ),
+ 1, 100000);
+}
+
+static void airoha_ppe_v1_cache_clean(struct airoha_ppe *ppe)
+{
+ u32 gate = airoha_fe_rr(ppe->common.eth, REG_EN751221_PPE_CAH_GATE);
+
+ if (!(gate & EN751221_PPE_CAH_GATE_EN)) {
+ airoha_fe_wr(ppe->common.eth, REG_EN751221_PPE_CAH_GATE,
+ EN751221_PPE_CAH_GATE_DEFAULT);
+ gate = airoha_fe_rr(ppe->common.eth, REG_EN751221_PPE_CAH_GATE);
+ }
+
+ airoha_fe_wr(ppe->common.eth, REG_EN751221_PPE_CAH_GATE,
+ gate & ~EN751221_PPE_CAH_GATE_EN);
+ if (!airoha_ppe_v1_cache_cmd(ppe, 4))
+ dev_warn(ppe->common.eth->dev,
+ "PPE cache clear-all timed out\n");
+ airoha_fe_wr(ppe->common.eth, REG_EN751221_PPE_CAH_GATE,
+ gate | EN751221_PPE_CAH_GATE_EN);
+}
+
+static u32 *airoha_foe_v1_ib2(struct airoha_foe_entry *entry)
+{
+ return &entry->words[airoha_foe_v1_ib2_word(entry)];
+}
+
+static u32 *airoha_foe_v1_data(struct airoha_foe_entry *entry)
+{
+ return &entry->words[airoha_foe_v1_data_word(entry)];
+}
+
+static u32 *airoha_foe_v1_l2(struct airoha_foe_entry *entry)
+{
+ return &entry->words[airoha_foe_v1_l2_word(entry)];
+}
+
+static void airoha_foe_v1_l2_set(struct airoha_foe_entry *entry,
+ unsigned int word, u32 mask, u32 val)
+{
+ u32 *l2 = airoha_foe_v1_l2(entry);
+
+ l2[word] &= ~mask;
+ l2[word] |= FIELD_PREP(mask, val);
+}
+
+static void airoha_foe_v1_entry_prepare(struct airoha_foe_entry *entry,
+ u8 l4proto, u8 pkt_type,
+ u8 pse_port, const u8 *src_mac,
+ const u8 *dest_mac)
+{
+ u32 *ib2, *l2;
+
+ memset(entry, 0, sizeof(*entry));
+ entry->ib1 = FIELD_PREP(AIROHA_FOE_IB1_BIND_STATE,
+ AIROHA_FOE_STATE_BIND) |
+ FIELD_PREP(AIROHA_FOE_IB1_BIND_PACKET_TYPE, pkt_type) |
+ FIELD_PREP(AIROHA_FOE_IB1_BIND_UDP,
+ l4proto == IPPROTO_UDP) |
+ AIROHA_FOE_V1_IB1_BIND_CACHE | AIROHA_FOE_IB1_BIND_TTL;
+
+ ib2 = airoha_foe_v1_ib2(entry);
+ *ib2 = FIELD_PREP(AIROHA_FOE_V1_IB2_PSE_PORT, pse_port) |
+ FIELD_PREP(AIROHA_FOE_V1_IB2_PORT_MG, 0x3f) |
+ FIELD_PREP(AIROHA_FOE_V1_IB2_PORT_AG, 0x3f);
+ if (is_multicast_ether_addr(dest_mac))
+ *ib2 |= AIROHA_FOE_V1_IB2_MULTICAST;
+
+ l2 = airoha_foe_v1_l2(entry);
+ l2[0] = FIELD_PREP(AIROHA_FOE_L2_ETYPE,
+ pkt_type == PPE_PKT_TYPE_IPV6_ROUTE_5T ?
+ ETH_P_IPV6 : ETH_P_IP);
+ l2[1] = get_unaligned_be32(dest_mac);
+ l2[2] = FIELD_PREP(AIROHA_FOE_L2_DMAC_LO,
+ get_unaligned_be16(dest_mac + 4));
+ l2[3] = get_unaligned_be32(src_mac);
+ l2[4] = FIELD_PREP(AIROHA_FOE_L2_SMAC_LO,
+ get_unaligned_be16(src_mac + 4));
+}
+
+static void airoha_foe_v1_entry_set_ipv4_tuple(struct airoha_foe_entry
*entry,
+ bool egress, __be32 src_addr,
+ __be16 src_port,
+ __be32 dest_addr,
+ __be16 dest_port)
+{
+ unsigned int base = egress ? 5 : 1;
+
+ entry->words[base] = be32_to_cpu(src_addr);
+ entry->words[base + 1] = be32_to_cpu(dest_addr);
+ entry->words[base + 2] =
+ FIELD_PREP(AIROHA_FOE_PORTS_SPORT, be16_to_cpu(src_port)) |
+ FIELD_PREP(AIROHA_FOE_PORTS_DPORT, be16_to_cpu(dest_port));
+}
+
+static void airoha_foe_v1_entry_set_ipv6_tuple(struct airoha_foe_entry
*entry,
+ const struct in6_addr *src_addr,
+ __be16 src_port,
+ const struct in6_addr *dest_addr,
+ __be16 dest_port)
+{
+ int i;
+
+ for (i = 0; i < 4; i++) {
+ entry->words[1 + i] = be32_to_cpu(src_addr->s6_addr32[i]);
+ entry->words[5 + i] = be32_to_cpu(dest_addr->s6_addr32[i]);
+ }
+ entry->words[9] =
+ FIELD_PREP(AIROHA_FOE_PORTS_SPORT, be16_to_cpu(src_port)) |
+ FIELD_PREP(AIROHA_FOE_PORTS_DPORT, be16_to_cpu(dest_port));
+}
+
+static void airoha_foe_v1_entry_set_pse_port(struct airoha_foe_entry
*entry,
+ u8 port)
+{
+ u32 *ib2 = airoha_foe_v1_ib2(entry);
+
+ *ib2 &= ~(AIROHA_FOE_V1_IB2_PSE_PORT | AIROHA_FOE_V1_IB2_PSE_QOS);
+ *ib2 |= FIELD_PREP(AIROHA_FOE_V1_IB2_PSE_PORT, port);
+ if (port == AIROHA_FOE_V1_FP_QDMA_HW)
+ *ib2 |= AIROHA_FOE_V1_IB2_PSE_QOS;
+}
+
+static void airoha_foe_v1_entry_set_queue(struct airoha_foe_entry *entry,
+ u8 queue)
+{
+ u32 *ib2 = airoha_foe_v1_ib2(entry);
+
+ *ib2 &= ~AIROHA_FOE_V1_IB2_QID;
+ *ib2 |= FIELD_PREP(AIROHA_FOE_V1_IB2_QID, queue);
+}
+
+static int airoha_foe_v1_entry_set_xpon(struct airoha_foe_entry *entry,
+ struct net_device *odev,
+ bool vlan_valid, u16 vlan_id,
+ bool pcp_valid, u8 pcp)
+{
+ struct airoha_xpon_tx_info info = {};
+ u32 *data;
+ int err;
+
+ err = airoha_eth_xpon_get_tx_info(odev, vlan_valid, vlan_id,
+ pcp_valid, pcp, &info);
+ if (err)
+ return err;
+ if (info.oam || info.gem_port_id > 0xfff || info.tcont >= 32 ||
+ info.queue >= 8)
+ return -EOPNOTSUPP;
+
+ airoha_foe_v1_entry_set_pse_port(entry, AIROHA_FOE_V1_FP_QDMA_HW);
+ airoha_foe_v1_entry_set_queue(entry, info.queue);
+ data = airoha_foe_v1_data(entry);
+ *data &= ~(AIROHA_FOE_V1_SHAPER_ID | AIROHA_FOE_V1_CHANNEL);
+ *data |= FIELD_PREP(AIROHA_FOE_V1_CHANNEL, info.tcont);
+ airoha_foe_v1_l2_set(entry, 0, AIROHA_FOE_L2_ETYPE,
+ info.gem_port_id);
+
+ return 0;
+}
+
+static int airoha_foe_v1_entry_set_vlan(struct airoha_foe_entry *entry,
+ u16 vid, u8 prio, __be16 proto)
+{
+ if (proto != htons(ETH_P_8021Q) || vid > VLAN_VID_MASK || prio > 7)
+ return -EOPNOTSUPP;
+
+ entry->ib1 &= ~(AIROHA_FOE_V1_IB1_BIND_VLAN_LAYER |
+ AIROHA_FOE_IB1_BIND_VPM);
+ entry->ib1 |= FIELD_PREP(AIROHA_FOE_V1_IB1_BIND_VLAN_LAYER, 1) |
+ FIELD_PREP(AIROHA_FOE_IB1_BIND_VPM, 1);
+ airoha_foe_v1_l2_set(entry, 0, AIROHA_FOE_L2_VLAN1,
+ FIELD_PREP(VLAN_PRIO_MASK, prio) | vid);
+
+ return 0;
+}
+
+static void airoha_foe_v1_entry_set_dsa(struct airoha_foe_entry *entry,
+ int port, bool passthrough)
+{
+ u32 etype = BIT(port) & GENMASK(5, 0);
+
+ if (passthrough)
+ etype |= BIT(7);
+ if (!(entry->ib1 & AIROHA_FOE_V1_IB1_BIND_VLAN_LAYER))
+ entry->ib1 |= FIELD_PREP(AIROHA_FOE_V1_IB1_BIND_VLAN_LAYER, 1);
+ else
+ etype |= BIT(8);
+
+ airoha_foe_v1_l2_set(entry, 0, AIROHA_FOE_L2_ETYPE, etype);
+ airoha_foe_v1_l2_set(entry, 0, AIROHA_FOE_L2_VLAN1, 0);
+ entry->ib1 &= ~AIROHA_FOE_V1_IB1_BIND_VLAN_TAG;
+}
+
+static void airoha_foe_v1_entry_set_pppoe(struct airoha_foe_entry *entry,
+ u16 sid)
+{
+ if (!(entry->ib1 & AIROHA_FOE_V1_IB1_BIND_VLAN_LAYER) ||
+ (entry->ib1 & AIROHA_FOE_V1_IB1_BIND_VLAN_TAG))
+ airoha_foe_v1_l2_set(entry, 0, AIROHA_FOE_L2_ETYPE,
+ ETH_P_PPP_SES);
+ entry->ib1 |= AIROHA_FOE_V1_IB1_BIND_PPPOE;
+ airoha_foe_v1_l2_set(entry, 4, AIROHA_FOE_L2_PPPOE_ID, sid);
+}
+
+static void airoha_foe_v1_entry_set_bind_metadata(struct
airoha_foe_entry *entry)
+{
+ u32 *data = airoha_foe_v1_data(entry);
+ u32 *ib2 = airoha_foe_v1_ib2(entry);
+ u8 port = FIELD_GET(AIROHA_FOE_V1_IB2_PSE_PORT, *ib2);
+
+ entry->ib1 &= ~AIROHA_FOE_V1_IB1_BIND_VLAN_LAYER;
+ entry->ib1 |= FIELD_PREP(AIROHA_FOE_V1_IB1_BIND_VLAN_LAYER, 1);
+
+ *data &= ~AIROHA_FOE_V1_ACTDP;
+ if (port != AIROHA_FOE_V1_FP_QDMA_HW)
+ *data |= FIELD_PREP(AIROHA_FOE_V1_ACTDP, port + 6);
+}
+
+static u32 airoha_foe_v1_packet_type(const struct airoha_foe_entry *entry)
+{
+ return FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, entry->ib1);
+}
+
+static void airoha_ppe_v1_commit_entry(struct airoha_ppe *ppe,
+ struct airoha_foe_entry *entry, u16 hash,
+ const struct airoha_foe_entry *lookup_raw)
+{
+ u32 *slot = airoha_ppe_v1_slot(ppe, hash);
+ u16 timestamp = airoha_fe_rr(ppe->common.eth, REG_FE_FOE_TS) &
+ AIROHA_FOE_IB1_BIND_TIMESTAMP;
+ int i;
+
+ entry->ib1 &= ~AIROHA_FOE_IB1_BIND_TIMESTAMP;
+ entry->ib1 |= FIELD_PREP(AIROHA_FOE_IB1_BIND_TIMESTAMP, timestamp) |
+ AIROHA_FOE_V1_IB1_BIND_CACHE | AIROHA_FOE_IB1_BIND_TTL |
+ AIROHA_FOE_IB1_BIND_KEEPALIVE;
+ airoha_foe_v1_entry_set_bind_metadata(entry);
+
+ if (lookup_raw) {
+ switch (airoha_foe_v1_packet_type(lookup_raw)) {
+ case PPE_PKT_TYPE_IPV4_HNAPT:
+ entry->words[3] = lookup_raw->words[3];
+ fallthrough;
+ case PPE_PKT_TYPE_IPV4_ROUTE:
+ entry->words[1] = lookup_raw->words[1];
+ entry->words[2] = lookup_raw->words[2];
+ break;
+ case PPE_PKT_TYPE_IPV6_ROUTE_5T:
+ for (i = 1; i <= 9; i++)
+ entry->words[i] = lookup_raw->words[i];
+ break;
+ default:
+ break;
+ }
+ }
+
+ for (i = 1; i < AIROHA_FOE_ENTRY_WORDS; i++)
+ WRITE_ONCE(slot[i], entry->words[i]);
+ dma_wmb();
+ WRITE_ONCE(slot[0], entry->words[0]);
+ dma_wmb();
+ airoha_ppe_v1_cache_clean(ppe);
+}
+
+static void airoha_ppe_v1_invalidate_entry(struct airoha_ppe *ppe, u16
hash)
+{
+ if (hash == AIROHA_FOE_V1_INVALID_HASH ||
+ hash >= ppe->common.eth->soc->ppe_dram_entries)
+ return;
+ WRITE_ONCE(airoha_ppe_v1_slot(ppe, hash)[0], 0);
+ /* Publish the invalid state before the caller clears the cache. */
+ dma_wmb();
+}
+
+static u32 airoha_ppe_v1_raw_state(const struct airoha_foe_entry *raw)
+{
+ return FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, raw->ib1);
+}
+
+static void airoha_ppe_v1_clear_owner(struct airoha_ppe *ppe, u16 hash)
+{
+ struct airoha_flow_table_entry *owner;
+
+ if (hash == AIROHA_FOE_V1_INVALID_HASH ||
+ hash >= ppe->common.eth->soc->ppe_dram_entries)
+ return;
+
+ owner = ppe->v1.foe_owner[hash];
+ if (owner && owner->hash == hash)
+ owner->hash = AIROHA_FOE_V1_INVALID_HASH;
+ ppe->v1.foe_owner[hash] = NULL;
+}
+
+static void airoha_ppe_v1_release_flow_slot(struct airoha_ppe *ppe,
+ struct airoha_flow_table_entry *flow,
+ bool invalidate)
+{
+ u16 hash = flow->hash;
+
+ if (hash == AIROHA_FOE_V1_INVALID_HASH ||
+ hash >= ppe->common.eth->soc->ppe_dram_entries ||
+ ppe->v1.foe_owner[hash] != flow) {
+ flow->hash = AIROHA_FOE_V1_INVALID_HASH;
+ return;
+ }
+
+ if (invalidate)
+ airoha_ppe_v1_invalidate_entry(ppe, hash);
+ ppe->v1.foe_owner[hash] = NULL;
+ flow->hash = AIROHA_FOE_V1_INVALID_HASH;
+}
+
+static u16 airoha_ppe_v1_find_bind_way(struct airoha_ppe *ppe,
+ struct airoha_flow_table_entry *flow, u16 hash,
+ struct airoha_foe_entry *lookup_raw,
+ bool *lookup_valid)
+{
+ struct airoha_foe_entry raw[2];
+ u16 way[2] = { hash, hash ^ 1 };
+ u32 state[2];
+ int i;
+
+ *lookup_valid = false;
+
+ /*
+ * FoeHashFun() in the EN7512 SDK returns an even bucket base and then
+ * checks base/base + 1. RX metadata may contain either way, so retain
+ * the reported way as the first preference and inspect its sibling
too.
+ */
+ for (i = 0; i < ARRAY_SIZE(way); i++) {
+ struct airoha_flow_table_entry *owner;
+
+ if (way[i] >= ppe->common.eth->soc->ppe_dram_entries)
+ continue;
+ airoha_ppe_v1_read_entry(ppe, way[i], &raw[i]);
+ state[i] = airoha_ppe_v1_raw_state(&raw[i]);
+
+ owner = ppe->v1.foe_owner[way[i]];
+ if (owner && state[i] != AIROHA_FOE_STATE_BIND) {
+ /* Hardware aging/replacement ended the previous ownership. */
+ airoha_ppe_v1_clear_owner(ppe, way[i]);
+ owner = NULL;
+ }
+ if (owner == flow && state[i] == AIROHA_FOE_STATE_BIND)
+ return way[i];
+ }
+
+ /*
+ * Match the EN7512 FoeHashFun() allocation policy: INVALID and UNBIND
+ * ways are reusable, while BIND and FIN are occupied. An UNBIND way
+ * contains the lookup key generated by the PPE and must be preserved;
+ * an INVALID way has no key, so airoha_ppe_v1_commit_entry()
encodes the
+ * logical tuple from flow->data instead.
+ */
+ for (i = 0; i < ARRAY_SIZE(way); i++) {
+ if (way[i] >= ppe->common.eth->soc->ppe_dram_entries ||
+ ppe->v1.foe_owner[way[i]])
+ continue;
+
+ switch (state[i]) {
+ case AIROHA_FOE_STATE_UNBIND:
+ *lookup_raw = raw[i];
+ *lookup_valid = true;
+ return way[i];
+ case AIROHA_FOE_STATE_INVALID:
+ return way[i];
+ default:
+ break;
+ }
+ }
+
+ return AIROHA_FOE_V1_INVALID_HASH;
}
+
+struct airoha_foe_v1_tuple {
+ u16 addr_type;
+ union {
+ struct {
+ u32 src;
+ u32 dest;
+ } v4;
+ struct {
+ struct in6_addr src;
+ struct in6_addr dest;
+ } v6;
+ };
+ u16 src_port;
+ u16 dest_port;
+};
+
+static bool airoha_foe_v1_parse_tuple(struct sk_buff *skb,
+ struct airoha_foe_v1_tuple *tuple)
+{
+ const u8 *data = skb->data;
+ unsigned int len = skb->len;
+ unsigned int proto_off = 12, off;
+ u16 proto;
+
+ if (len < ETH_HLEN)
+ return false;
+
+ proto = get_unaligned_be16(data + proto_off);
+ if (proto == ETH_P_8021Q || proto == ETH_P_8021AD) {
+ if (len < ETH_HLEN + VLAN_HLEN)
+ return false;
+ proto_off += VLAN_HLEN;
+ proto = get_unaligned_be16(data + proto_off);
+ } else if (proto != ETH_P_IP && proto != ETH_P_IPV6 &&
+ proto != ETH_P_PPP_SES) {
+ /* MTK special tag occupies the ethertype/TCI position. */
+ if (len < ETH_HLEN + 4)
+ return false;
+ proto_off += 4;
+ proto = get_unaligned_be16(data + proto_off);
+ }
+ off = proto_off + sizeof(__be16);
+
+ if (proto == ETH_P_PPP_SES) {
+ u16 ppp_proto;
+
+ if (len < proto_off + 10)
+ return false;
+ ppp_proto = get_unaligned_be16(data + proto_off + 8);
+ if (ppp_proto == PPP_IP)
+ proto = ETH_P_IP;
+ else if (ppp_proto == PPP_IPV6)
+ proto = ETH_P_IPV6;
+ else
+ return false;
+ off = proto_off + 10;
+ }
+
+ memset(tuple, 0, sizeof(*tuple));
+ if (proto == ETH_P_IP) {
+ const u8 *iph, *l4;
+ u8 ihl;
+
+ if (len < off + sizeof(struct iphdr))
+ return false;
+ iph = data + off;
+ if ((iph[0] >> 4) != 4 ||
+ (iph[9] != IPPROTO_TCP && iph[9] != IPPROTO_UDP))
+ return false;
+ ihl = (iph[0] & 0x0f) * 4;
+ if (ihl < sizeof(struct iphdr) || len < off + ihl + 4)
+ return false;
+
+ l4 = iph + ihl;
+ tuple->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
+ tuple->v4.src = get_unaligned_be32(iph + offsetof(struct iphdr,
saddr));
+ tuple->v4.dest = get_unaligned_be32(iph + offsetof(struct
iphdr, daddr));
+ tuple->src_port = get_unaligned_be16(l4);
+ tuple->dest_port = get_unaligned_be16(l4 + 2);
+ return true;
+ }
+
+ if (proto == ETH_P_IPV6) {
+ unsigned short fragoff = 0;
+ unsigned int l4off = off;
+ int flags = 0, nexthdr;
+
+ if (len < off + sizeof(struct ipv6hdr) ||
+ (data[off] >> 4) != 6)
+ return false;
+
+ nexthdr = ipv6_find_hdr(skb, &l4off, -1, &fragoff, &flags);
+ if ((nexthdr != IPPROTO_TCP && nexthdr != IPPROTO_UDP) ||
+ (flags & IP6_FH_F_FRAG) || len < l4off + 4)
+ return false;
+
+ tuple->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
+ memcpy(&tuple->v6.src,
+ data + off + offsetof(struct ipv6hdr, saddr),
+ sizeof(tuple->v6.src));
+ memcpy(&tuple->v6.dest,
+ data + off + offsetof(struct ipv6hdr, daddr),
+ sizeof(tuple->v6.dest));
+ tuple->src_port = get_unaligned_be16(data + l4off);
+ tuple->dest_port = get_unaligned_be16(data + l4off + 2);
+ return true;
+ }
+
+ return false;
+}
+
+static bool
+airoha_foe_v1_flow_matches_tuple(const struct airoha_flow_table_entry
*flow,
+ const struct airoha_foe_v1_tuple *tuple)
+{
+ if (flow->addr_type != tuple->addr_type ||
+ flow->src_port != tuple->src_port ||
+ flow->dest_port != tuple->dest_port)
+ return false;
+
+ if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS)
+ return flow->src_ip == tuple->v4.src &&
+ flow->dest_ip == tuple->v4.dest;
+
+ if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS)
+ return ipv6_addr_equal(&flow->src_ip6, &tuple->v6.src) &&
+ ipv6_addr_equal(&flow->dest_ip6, &tuple->v6.dest);
+
+ return false;
+}
+
+static void airoha_ppe_v1_rx_check(struct airoha_ppe *ppe, struct
sk_buff *skb,
+ u16 hash, u8 reason)
+{
+ struct airoha_foe_v1_tuple tuple;
+ struct airoha_flow_table_entry *flow;
+ struct airoha_foe_entry entry, lookup_raw = {};
+ bool lookup_valid;
+ u16 bind_hash;
+
+ if (!ppe || !READ_ONCE(ppe->v1.armed) ||
+ hash >= ppe->common.eth->soc->ppe_dram_entries)
+ return;
+ if (reason != EN751221_PPE_CPU_REASON_NO_FLOW &&
+ reason != AIROHA_PPE_CPU_REASON_HIT_UNBIND &&
+ reason != AIROHA_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
+ return;
+ if (!airoha_foe_v1_parse_tuple(skb, &tuple))
+ return;
+
+ spin_lock_bh(&ppe->v1.lock);
+ list_for_each_entry(flow, &ppe->v1.flows, v1_list) {
+ if (!airoha_foe_v1_flow_matches_tuple(flow, &tuple))
+ continue;
+
+ bind_hash = airoha_ppe_v1_find_bind_way(ppe, flow, hash,
+ &lookup_raw, &lookup_valid);
+ if (bind_hash == AIROHA_FOE_V1_INVALID_HASH)
+ break;
+ if (ppe->v1.foe_owner[bind_hash] == flow &&
+ flow->hash == bind_hash)
+ break;
+
+ entry = flow->data;
+ if (flow->hash != bind_hash)
+ airoha_ppe_v1_release_flow_slot(ppe, flow, true);
+
+ airoha_ppe_v1_commit_entry(ppe, &entry, bind_hash,
+ lookup_valid ? &lookup_raw : NULL);
+ ppe->v1.foe_owner[bind_hash] = flow;
+ flow->hash = bind_hash;
+ break;
+ }
+ spin_unlock_bh(&ppe->v1.lock);
+}
+
+static void airoha_ppe_check_skb_reason(struct airoha_ppe_dev *ppe_dev,
+ struct sk_buff *skb, u16 hash, u8 reason)
+{
+ struct airoha_ppe *ppe = ppe_dev->priv;
+
+ if (ppe->common.eth->soc->foe_format == AIROHA_FOE_FORMAT_V1)
+ airoha_ppe_v1_rx_check(ppe, skb, hash, reason);
+}
+
+static void airoha_ppe_v1_flush(struct airoha_ppe *ppe)
+{
+ struct airoha_flow_table_entry *flow, *tmp;
+ LIST_HEAD(free_list);
+
+ spin_lock_bh(&ppe->v1.lock);
+ list_for_each_entry(flow, &ppe->v1.flows, v1_list)
+ airoha_ppe_v1_release_flow_slot(ppe, flow, true);
+ list_splice_init(&ppe->v1.flows, &free_list);
+ spin_unlock_bh(&ppe->v1.lock);
+
+ list_for_each_entry_safe(flow, tmp, &free_list, v1_list) {
+ rhashtable_remove_fast(&ppe->common.eth->flow_table, &flow->node,
+ airoha_flow_table_params);
+ list_del(&flow->v1_list);
+ kfree(flow);
+ }
+
+ memset(ppe->common.foe, 0, airoha_ppe_v1_foe_size(ppe->common.eth));
+ /* Publish the cleared table before invalidating the lookup cache. */
+ dma_wmb();
+ airoha_ppe_v1_cache_clean(ppe);
+}
+
+/*
+ * EN751221 vendor hw_nat (PPE type 3) programs the otherwise unnamed
+ * generation-1 GLO_CFG bits 1 and 8 together with byte-swap/hash-offset
+ * and flow-drop-update. Bit 10 is not touched by hw_nat itself, but it
is set
+ * in the production FE state (GLO_CFG = 0x763); the consolidated Linux
driver
+ * owns the FE reset, so restore that steady-state bit explicitly on
enable.
+ */
+#define EN751221_PPE_GLO_CFG_VENDOR_SET (BIT(1) | BIT(8) | \
+ PPE_GLO_CFG_PPE_BSWAP_MASK | \
+ PPE_GLO_CFG_PSE_HASH_OFS_MASK | \
+ PPE_GLO_CFG_FLOW_DROP_UPDATE_MASK)
+#define EN751221_PPE_GLO_CFG_STOCK_BIT10 BIT(10)
+#define EN751221_PPE_GLO_CFG_VENDOR_CLEAR (PPE_GLO_CFG_TTL_DROP_MASK | \
+ PPE_GLO_CFG_IP4_CS_DROP_MASK | \
+ PPE_GLO_CFG_IP4_L4_CS_DROP_MASK)
+
+/*
+ * The type-3 stock flow profile is 0x07e0f740. Bits 21..24 are not named
+ * by the common Airoha register definitions yet, but are part of the
+ * EN751221 L2B/IPv4/IPv6/DS-LITE/6RD profile. IP protocol blacklist mode
+ * (bit 16) is configured separately by the vendor driver.
+ */
+#define EN751221_PPE_FLOW_CFG_TYPE3 (GENMASK(26, 21) | \
+ PPE_FLOW_CFG_L2_BRIDGE_MASK | \
+ PPE_FLOW_CFG_IP4_DSLITE_MASK | \
+ PPE_FLOW_CFG_IP4_NAPT_MASK | \
+ PPE_FLOW_CFG_IP4_NAT_MASK | \
+ PPE_FLOW_CFG_IP6_6RD_MASK | \
+ PPE_FLOW_CFG_IP6_5T_ROUTE_MASK | \
+ PPE_FLOW_CFG_IP6_3T_ROUTE_MASK | \
+ PPE_FLOW_CFG_IP4_TCP_FRAG_MASK)
+
+static void airoha_ppe_v1_set_flow_profile(struct airoha_ppe *ppe)
+{
+ airoha_fe_wr(ppe->common.eth, REG_PPE_PPE_FLOW_CFG(0),
+ EN751221_PPE_FLOW_CFG_TYPE3 |
+ PPE_FLOW_CFG_IP_PROTO_BLACKLIST_MASK);
+}
+
+static void airoha_ppe_v1_hw_init(struct airoha_ppe *ppe)
+{
+ struct airoha_eth *eth = ppe->common.eth;
+ u32 dram_num_entries;
+ u32 tb_cfg;
+
+ dram_num_entries = airoha_ppe_get_num_entries_shift(
+ eth->soc->ppe_dram_entries);
+
+ dev_info(eth->dev, "Initializing generation-1 PPE hardware\n");
+
+ /*
+ * FoE v1 uses CAH_GATE to enable the lookup cache. CAH_CTRL is the
+ * cache command register, not an enable bit. Match the vendor hw_nat
+ * programming and invalidate stale cache contents before publishing
+ * the DMA-backed 80-byte FoE table.
+ */
+ airoha_fe_wr(eth, REG_EN751221_PPE_CAH_GATE,
+ EN751221_PPE_CAH_GATE_DEFAULT);
+ airoha_ppe_v1_cache_clean(ppe);
+
+ memset(ppe->common.foe, 0, airoha_ppe_v1_foe_size(eth));
+ dma_wmb();
+
+ airoha_fe_wr(eth, REG_PPE_TB_BASE(0),
+ lower_32_bits(ppe->common.foe_dma));
+ tb_cfg = PPE_TB_CFG_AGE_TCP_FIN_MASK | PPE_TB_CFG_AGE_UDP_MASK |
+ PPE_TB_CFG_AGE_TCP_MASK | PPE_TB_CFG_AGE_UNBIND_MASK |
+ PPE_TB_CFG_AGE_NON_L4_MASK |
+ FIELD_PREP(PPE_TB_CFG_SEARCH_MISS_MASK, 3) |
+ FIELD_PREP(PPE_TB_CFG_HASH_MODE_MASK, 3) |
+ FIELD_PREP(PPE_TB_CFG_KEEPALIVE_MASK, 3) |
+ FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, dram_num_entries) |
+ PPE_TB_ENTRY_SIZE_MASK;
+ airoha_fe_wr(eth, REG_PPE_TB_CFG(0), tb_cfg);
+ airoha_fe_wr(eth, REG_PPE_IP_PROTO_CHK(0),
+ PPE_IP_PROTO_CHK_IPV4_MASK | PPE_IP_PROTO_CHK_IPV6_MASK);
+
+ /* Keep the same protocol classifier programmed by the vendor stack. */
+ airoha_fe_wr(eth, REG_PPE_IP_PROT(0, 0),
+ FIELD_PREP(GENMASK(7, 0), IPPROTO_TCP) |
+ FIELD_PREP(GENMASK(15, 8), IPPROTO_UDP) |
+ FIELD_PREP(GENMASK(23, 16), IPPROTO_IPV6) |
+ FIELD_PREP(GENMASK(31, 24), IPPROTO_IPIP));
+ airoha_fe_wr(eth, REG_PPE_IP_PROT(0, 1), 0);
+ airoha_fe_wr(eth, REG_PPE_IP_PROT(0, 2), 0);
+ airoha_fe_wr(eth, REG_PPE_IP_PROT(0, 3), 0);
+ airoha_ppe_v1_set_flow_profile(ppe);
+
+ airoha_fe_wr(eth, REG_PPE_UNBIND_AGE(0),
+ FIELD_PREP(PPE_UNBIND_AGE_MIN_PACKETS_MASK, 1000) |
+ FIELD_PREP(PPE_UNBIND_AGE_DELTA_MASK, 3));
+ airoha_fe_wr(eth, REG_PPE_BND_AGE0(0),
+ FIELD_PREP(PPE_BIND_AGE0_DELTA_NON_L4, 15) |
+ FIELD_PREP(PPE_BIND_AGE0_DELTA_UDP, 15));
+ airoha_fe_wr(eth, REG_PPE_BND_AGE1(0),
+ FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP_FIN, 5) |
+ FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 15));
+ airoha_fe_wr(eth, REG_PPE_BIND_LIMIT0(0),
+ FIELD_PREP(PPE_BIND_LIMIT0_HALF_MASK, 800) |
+ FIELD_PREP(PPE_BIND_LIMIT0_QUARTER_MASK, 1600));
+ airoha_fe_wr(eth, REG_PPE_BIND_LIMIT1(0),
+ FIELD_PREP(PPE_BIND_LIMIT1_NON_L4_MASK, 1) |
+ FIELD_PREP(PPE_BIND_LIMIT1_FULL_MASK, 400));
+ airoha_fe_wr(eth, REG_PPE_BIND_RATE(0),
+ FIELD_PREP(PPE_BIND_RATE_BIND_MASK, 30));
+ airoha_fe_wr(eth, REG_PPE_HASH_SEED(0), PPE_HASH_SEED);
+ airoha_fe_wr(eth, REG_PPE_DFT_CPORT_BASE(0), 0);
+ airoha_fe_clear(eth, REG_PPE_GLO_CFG(0), PPE_GLO_CFG_EN_MASK);
+}
+
+static int airoha_ppe_v1_engine_set(struct airoha_ppe *ppe, bool enable)
+{
+ void __iomem *reg = ppe->common.eth->fe_regs + REG_PPE_GLO_CFG(0);
+ u32 val;
+
+ if (enable) {
+ if (readl_poll_timeout_atomic(reg, val,
+ !(val & PPE_GLO_CFG_BUSY_MASK),
+ 10, 10000))
+ return -EBUSY;
+
+ /* Match FUN_00013f50(1) from the stock hw_nat module. */
+ val = readl(reg);
+ val &= ~EN751221_PPE_GLO_CFG_VENDOR_CLEAR;
+ val |= EN751221_PPE_GLO_CFG_VENDOR_SET |
+ EN751221_PPE_GLO_CFG_STOCK_BIT10 | PPE_GLO_CFG_EN_MASK;
+ writel(val, reg);
+ } else {
+ /* Match FUN_00013f50(0), preserving unrelated FE-owned bits. */
+ val = readl(reg);
+ val &= ~(EN751221_PPE_GLO_CFG_VENDOR_SET |
+ PPE_GLO_CFG_TTL_DROP_MASK | PPE_GLO_CFG_EN_MASK);
+ val |= PPE_GLO_CFG_IP4_CS_DROP_MASK |
+ PPE_GLO_CFG_IP4_L4_CS_DROP_MASK;
+ writel(val, reg);
+ }
+ return 0;
+}
+
+static void airoha_ppe_v1_set_gdm_ingress(struct airoha_ppe *ppe, int gdm,
+ u8 fport)
+{
+ u32 mask = GDM_UCFQ_MASK | GDM_BCFQ_MASK |
+ GDM_MCFQ_MASK | GDM_OCFQ_MASK;
+ u32 val = FIELD_PREP(GDM_UCFQ_MASK, fport) |
+ FIELD_PREP(GDM_BCFQ_MASK, fport) |
+ FIELD_PREP(GDM_MCFQ_MASK, fport) |
+ FIELD_PREP(GDM_OCFQ_MASK, fport);
+
+ /*
+ * Stock SetGdmaFwd() sends all four ingress classes to fport 4 while
+ * HWNAT is enabled (the observed low word is 0x4444). Keep this an
+ * RMW so EN751221 special-tag/UNTAG state in the upper bits survives.
+ */
+ airoha_fe_rmw(ppe->common.eth, REG_GDM_FWD_CFG(gdm), mask, val);
+}
+
+static int airoha_ppe_v1_engine_arm(struct airoha_ppe *ppe)
+{
+ int err;
+
+ if (!ppe)
+ return -ENODEV;
+ if (READ_ONCE(ppe->v1.armed))
+ return 0;
+
+ airoha_fe_wr(ppe->common.eth, REG_PPE_TB_CFG(0),
+ FIELD_PREP(PPE_TB_CFG_HASH_MODE_MASK, 3) |
+ FIELD_PREP(PPE_TB_CFG_KEEPALIVE_MASK, 3) |
+ PPE_TB_CFG_AGE_TCP_FIN_MASK | PPE_TB_CFG_AGE_UDP_MASK |
+ PPE_TB_CFG_AGE_TCP_MASK | PPE_TB_CFG_AGE_UNBIND_MASK |
+ PPE_TB_CFG_AGE_NON_L4_MASK |
+ FIELD_PREP(PPE_TB_CFG_SEARCH_MISS_MASK, 3) |
+ PPE_TB_ENTRY_SIZE_MASK |
+ FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, 4));
+ airoha_fe_wr(ppe->common.eth, REG_PPE_KEEPALIVE(0),
+ FIELD_PREP(PPE_KEEPALIVE_UDP_MASK, 1) |
+ FIELD_PREP(PPE_KEEPALIVE_TCP_MASK, 1) |
+ FIELD_PREP(PPE_KEEPALIVE_NTU_MASK, 1));
+
+ /*
+ * Program the complete type-3 datapath before enabling GLO_CFG. The
+ * stock module only calls SetGdmaFwd(1) after cache, FoE/table state,
+ * protocol parsing and the PPE engine itself are ready.
+ */
+ airoha_fe_wr(ppe->common.eth, REG_EN751221_PPE_CAH_GATE,
+ EN751221_PPE_CAH_GATE_DEFAULT);
+ airoha_ppe_v1_set_flow_profile(ppe);
+ airoha_fe_wr(ppe->common.eth, REG_PPE_VPM_TPID(0), ETH_P_8021Q);
+ airoha_fe_wr(ppe->common.eth, REG_CDM_VLAN_CTRL(1),
+ FIELD_PREP(CDM_VLAN_MASK, ETH_P_8021Q) | STAG_EN);
+
+ /* SetGdmaFwd() in the stock module writes PPE_DFT_CPORT = 0x500. */
+ airoha_fe_wr(ppe->common.eth, REG_PPE_DFT_CPORT_BASE(0),
+ FIELD_PREP(DFT_CPORT_MASK(2),
+ ppe->common.eth->soc->ppe_cpu_fport[1]));
+
+ err = airoha_ppe_v1_engine_set(ppe, true);
+ if (err)
+ return err;
+
+ /* GDM redirection is deliberately the last arm step. */
+ airoha_ppe_v1_set_gdm_ingress(ppe, 1, ppe->common.eth->soc->ppe_fport);
+ airoha_ppe_v1_set_gdm_ingress(ppe, 2, ppe->common.eth->soc->ppe_fport);
+ WRITE_ONCE(ppe->v1.armed, true);
+ return 0;
+}
+
+static void airoha_ppe_v1_engine_disarm(struct airoha_ppe *ppe)
+{
+ if (!ppe || !READ_ONCE(ppe->v1.armed))
+ return;
+
+ /* Restore the CPU ingress paths while preserving STAG/UNTAG state. */
+ airoha_ppe_v1_set_gdm_ingress(ppe, 1,
ppe->common.eth->soc->ppe_cpu_fport[0]);
+ airoha_ppe_v1_set_gdm_ingress(ppe, 2,
ppe->common.eth->soc->ppe_cpu_fport[1]);
+ WRITE_ONCE(ppe->v1.armed, false);
+ airoha_ppe_v1_flush(ppe);
+ airoha_ppe_v1_engine_set(ppe, false);
+}
+
+static struct airoha_ppe *airoha_ppe_from_netdev(struct net_device *netdev)
+{
+ struct airoha_gdm_common *gdm;
+
+ gdm = airoha_gdm_common_from_netdev(netdev);
+ if (!gdm || !gdm->ppe)
+ return NULL;
+
+ return gdm->ppe->priv;
+}
+
+static int airoha_ppe_v1_flow_set_output(struct airoha_foe_entry *entry,
+ struct net_device *odev, bool vlan_valid,
+ u16 vlan_id, u8 vlan_prio, bool *xpon)
+{
+ struct airoha_gdm_common *gdm;
+ struct dsa_port *dp;
+ int dsa_port = -1;
+ int err;
+
+ *xpon = false;
+ if (!odev)
+ return -EOPNOTSUPP;
+
+ if (dsa_user_dev_check(odev)) {
+ dp = dsa_port_from_netdev(odev);
+ if (IS_ERR(dp))
+ return PTR_ERR(dp);
+ dsa_port = dp->index;
+ odev = dsa_port_to_conduit(dp);
+ airoha_foe_v1_entry_set_dsa(entry, dsa_port, dp->ds->index != 0);
+ }
+
+ if (!odev)
+ return -EOPNOTSUPP;
+
+ gdm = airoha_gdm_common_from_netdev(odev);
+ if (!gdm || gdm->family != AIROHA_ETH_FAMILY_ECONET)
+ return -EOPNOTSUPP;
+
+ /*
+ * Probe the xPON service API before using direct GDM2 egress. A
managed
+ * GPON port needs QDMA_HW plus GEM/T-CONT metadata; sending a FoE flow
+ * straight to GDM2 bypasses exactly the metadata which the CPU PWAN
+ * descriptor normally supplies.
+ */
+ if (dsa_port < 0) {
+ err = airoha_foe_v1_entry_set_xpon(entry, odev, vlan_valid,
vlan_id,
+ vlan_valid, vlan_prio);
+ if (!err) {
+ *xpon = true;
+ return 0;
+ }
+ if (err != -EOPNOTSUPP)
+ return err;
+ }
+
+ /* VLAN insertion is only implemented for the GPON/QDMA FoE format. */
+ if (vlan_valid)
+ return -EOPNOTSUPP;
+
+ /*
+ * Keep direct GDM egress on the default queue. The EN751221 PPE
+ * flow path does not enable PSE_QOS for GDM1/GDM2 destinations,
+ * and the previous 3 + DSA-port QID mapping is not part of the
+ * vendor DSA special-tag programming.
+ *
+ * The DSA destination is still selected by the in-band special tag;
+ * leaving QID at its reset/default value isolates queue selection from
+ * the hardware forwarding path.
+ */
+ airoha_foe_v1_entry_set_pse_port(entry, gdm->pse_port);
+
+ return 0;
+}
+
+static int airoha_ppe_v1_flow_offload_replace(struct net_device *dev,
+ struct flow_cls_offload *cls)
+{
+ struct airoha_ppe *ppe = airoha_ppe_from_netdev(dev);
+ struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+ struct flow_match_basic basic;
+ struct flow_match_ports ports;
+ struct flow_action_entry *act;
+ struct airoha_flow_table_entry *flow;
+ struct airoha_flow_data data = {};
+ struct airoha_foe_entry entry;
+ struct net_device *odev = NULL;
+ struct in6_addr src_addr6 = {}, dest_addr6 = {};
+ __be32 src_addr = 0, dest_addr = 0;
+ __be16 src_port, dest_port;
+ __be16 vlan_proto = 0;
+ u16 pppoe_sid = 0, vlan_id = 0, addr_type;
+ u8 l4proto, vlan_prio = 0, pkt_type;
+ bool vlan_push = false, xpon = false;
+ int i, err;
+
+ if (!ppe)
+ return -EOPNOTSUPP;
+ if (rhashtable_lookup(&ppe->common.eth->flow_table, &cls->cookie,
+ airoha_flow_table_params))
+ return -EEXIST;
+ if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC) ||
+ !flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
+ NL_SET_ERR_MSG_MOD(cls->common.extack,
+ "FoE v1 requires TCP/UDP 5-tuple flows");
+ return -EOPNOTSUPP;
+ }
+
+ flow_rule_match_basic(rule, &basic);
+ l4proto = basic.key->ip_proto;
+ if (l4proto != IPPROTO_TCP && l4proto != IPPROTO_UDP)
+ return -EOPNOTSUPP;
+ if (basic.mask->n_proto != htons(0xffff) ||
+ basic.mask->ip_proto != 0xff)
+ return -EOPNOTSUPP;
+
+ if (basic.key->n_proto == htons(ETH_P_IP)) {
+ struct flow_match_ipv4_addrs addrs;
+
+ if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS))
+ return -EOPNOTSUPP;
+ flow_rule_match_ipv4_addrs(rule, &addrs);
+ if (addrs.mask->src != htonl(0xffffffff) ||
+ addrs.mask->dst != htonl(0xffffffff)) {
+ NL_SET_ERR_MSG_MOD(cls->common.extack,
+ "FoE v1 requires exact IPv4 addresses");
+ return -EOPNOTSUPP;
+ }
+ src_addr = addrs.key->src;
+ dest_addr = addrs.key->dst;
+ data.v4.src_addr = src_addr;
+ data.v4.dst_addr = dest_addr;
+ addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
+ pkt_type = PPE_PKT_TYPE_IPV4_HNAPT;
+ } else if (basic.key->n_proto == htons(ETH_P_IPV6)) {
+ struct flow_match_ipv6_addrs addrs;
+
+ if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS))
+ return -EOPNOTSUPP;
+ flow_rule_match_ipv6_addrs(rule, &addrs);
+ if (memchr_inv(&addrs.mask->src, 0xff, sizeof(addrs.mask->src)) ||
+ memchr_inv(&addrs.mask->dst, 0xff, sizeof(addrs.mask->dst))) {
+ NL_SET_ERR_MSG_MOD(cls->common.extack,
+ "FoE v1 requires exact IPv6 addresses");
+ return -EOPNOTSUPP;
+ }
+ src_addr6 = addrs.key->src;
+ dest_addr6 = addrs.key->dst;
+ data.v6.src_addr = src_addr6;
+ data.v6.dst_addr = dest_addr6;
+ addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
+ pkt_type = PPE_PKT_TYPE_IPV6_ROUTE_5T;
+ } else {
+ return -EOPNOTSUPP;
+ }
+
+ flow_rule_match_ports(rule, &ports);
+ if (ports.mask->src != htons(0xffff) ||
+ ports.mask->dst != htons(0xffff)) {
+ NL_SET_ERR_MSG_MOD(cls->common.extack,
+ "FoE v1 requires exact TCP/UDP ports");
+ return -EOPNOTSUPP;
+ }
+ if (!flow_action_basic_hw_stats_check(&rule->action,
+ cls->common.extack))
+ return -EOPNOTSUPP;
+
+ src_port = ports.key->src;
+ dest_port = ports.key->dst;
+ data.src_port = src_port;
+ data.dst_port = dest_port;
+
+ flow_action_for_each(i, act, &rule->action) {
+ switch (act->id) {
+ case FLOW_ACTION_MANGLE:
+ switch (act->mangle.htype) {
+ case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
+ airoha_ppe_flow_mangle_eth(act, &data.eth);
+ break;
+ case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
+ if (addr_type != FLOW_DISSECTOR_KEY_IPV4_ADDRS)
+ return -EOPNOTSUPP;
+ err = airoha_ppe_flow_mangle_ipv4(act, &data);
+ if (err)
+ return err;
+ break;
+ case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
+ case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
+ /* IPv6 5T is routing-only on FoE V1; NAT66/port
+ * translation has no rewrite tuple in this format.
+ */
+ if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS)
+ return -EOPNOTSUPP;
+ err = airoha_ppe_flow_mangle_ports(act, &data);
+ if (err)
+ return err;
+ break;
+ case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
+ return -EOPNOTSUPP;
+ default:
+ return -EOPNOTSUPP;
+ }
+ break;
+ case FLOW_ACTION_VLAN_PUSH:
+ if (vlan_push)
+ return -EOPNOTSUPP;
+ vlan_push = true;
+ vlan_id = act->vlan.vid;
+ vlan_prio = act->vlan.prio;
+ vlan_proto = act->vlan.proto;
+ break;
+ case FLOW_ACTION_VLAN_POP:
+ /* No output VLAN is encoded for this direction. */
+ break;
+ case FLOW_ACTION_PPPOE_PUSH:
+ pppoe_sid = act->pppoe.sid;
+ break;
+ case FLOW_ACTION_REDIRECT:
+ odev = act->dev;
+ break;
+ case FLOW_ACTION_CSUM:
+ break;
+ default:
+ NL_SET_ERR_MSG_MOD(cls->common.extack,
+ "unsupported action for FoE v1");
+ return -EOPNOTSUPP;
+ }
+ }
+
+ if (!is_valid_ether_addr(data.eth.h_source) ||
+ !is_valid_ether_addr(data.eth.h_dest)) {
+ NL_SET_ERR_MSG_MOD(cls->common.extack,
+ "missing Ethernet rewrite addresses");
+ return -EOPNOTSUPP;
+ }
+
+ airoha_foe_v1_entry_prepare(&entry, l4proto, pkt_type,
FE_PSE_PORT_GDM1,
+ data.eth.h_source, data.eth.h_dest);
+ if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
+ airoha_foe_v1_entry_set_ipv4_tuple(&entry, false, src_addr,
src_port,
+ dest_addr, dest_port);
+ airoha_foe_v1_entry_set_ipv4_tuple(&entry, true, data.v4.src_addr,
+ data.src_port, data.v4.dst_addr,
+ data.dst_port);
+ } else {
+ airoha_foe_v1_entry_set_ipv6_tuple(&entry, &src_addr6, src_port,
+ &dest_addr6, dest_port);
+ }
+
+ /* Match the vendor ordering: PpeFillInL2Info() installs VLAN/PPPoE
+ * first and PpeSetPortInfo(FOE_MAGIC_GPON) runs last, repurposing
+ * etype for the GEM port ID.
+ */
+ if (vlan_push) {
+ err = airoha_foe_v1_entry_set_vlan(&entry, vlan_id, vlan_prio,
+ vlan_proto);
+ if (err)
+ return err;
+ }
+ if (pppoe_sid)
+ airoha_foe_v1_entry_set_pppoe(&entry, pppoe_sid);
+
+ err = airoha_ppe_v1_flow_set_output(&entry, odev, vlan_push, vlan_id,
+ vlan_prio, &xpon);
+ if (err)
+ return err;
+ if (vlan_push && !xpon)
+ return -EOPNOTSUPP;
+
+ flow = kzalloc(sizeof(*flow), GFP_KERNEL);
+ if (!flow)
+ return -ENOMEM;
+ flow->data = entry;
+ flow->cookie = cls->cookie;
+ flow->addr_type = addr_type;
+ if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
+ flow->src_ip = ntohl(src_addr);
+ flow->dest_ip = ntohl(dest_addr);
+ } else {
+ flow->src_ip6 = src_addr6;
+ flow->dest_ip6 = dest_addr6;
+ }
+ flow->src_port = ntohs(src_port);
+ flow->dest_port = ntohs(dest_port);
+ flow->hash = AIROHA_FOE_V1_INVALID_HASH;
+ INIT_LIST_HEAD(&flow->v1_list);
+
+ err = rhashtable_insert_fast(&ppe->common.eth->flow_table, &flow->node,
+ airoha_flow_table_params);
+ if (err) {
+ kfree(flow);
+ return err;
+ }
+
+ spin_lock_bh(&ppe->v1.lock);
+ list_add_tail(&flow->v1_list, &ppe->v1.flows);
+ spin_unlock_bh(&ppe->v1.lock);
+ return 0;
+}
+
+static int airoha_ppe_v1_flow_offload_destroy(struct net_device *dev,
+ struct flow_cls_offload *cls)
+{
+ struct airoha_ppe *ppe = airoha_ppe_from_netdev(dev);
+ struct airoha_flow_table_entry *flow;
+
+ if (!ppe)
+ return -EOPNOTSUPP;
+
+ flow = rhashtable_lookup(&ppe->common.eth->flow_table, &cls->cookie,
+ airoha_flow_table_params);
+ if (!flow)
+ return -ENOENT;
+
+ spin_lock_bh(&ppe->v1.lock);
+ airoha_ppe_v1_release_flow_slot(ppe, flow, true);
+ airoha_ppe_v1_cache_clean(ppe);
+ list_del_init(&flow->v1_list);
+ spin_unlock_bh(&ppe->v1.lock);
+
+ rhashtable_remove_fast(&ppe->common.eth->flow_table, &flow->node,
+ airoha_flow_table_params);
+ kfree(flow);
+
+ return 0;
+}
+
+static int airoha_ppe_v1_flow_offload_stats(struct flow_cls_offload *cls)
+{
+ flow_stats_update(&cls->stats, 0, 0, 0, jiffies,
+ FLOW_ACTION_HW_STATS_DELAYED);
+ return 0;
+}
+
+static int airoha_ppe_tc_block_cb(enum tc_setup_type type, void *type_data,
+ void *cb_priv)
+{
+ struct net_device *dev = cb_priv;
+ struct airoha_ppe *ppe = airoha_ppe_from_netdev(dev);
+
+ if (!ppe || type != TC_SETUP_CLSFLOWER)
+ return -EOPNOTSUPP;
+
+ return airoha_ppe_flow_offload_cmd(ppe->common.eth, dev, type_data);
+}
+
+static int airoha_ppe_setup_tc_block(struct net_device *dev,
+ struct flow_block_offload *offload)
+{
+ struct airoha_ppe *ppe = airoha_ppe_from_netdev(dev);
+ flow_setup_cb_t *cb = airoha_ppe_tc_block_cb;
+ struct flow_block_cb *block_cb;
+ int err;
+
+ if (!ppe || offload->binder_type !=
FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
+ return -EOPNOTSUPP;
+
+ offload->driver_block_list = &ppe->block_cb_list;
+ switch (offload->command) {
+ case FLOW_BLOCK_BIND:
+ block_cb = flow_block_cb_lookup(offload->block, cb, dev);
+ if (block_cb) {
+ flow_block_cb_incref(block_cb);
+ return 0;
+ }
+ err = airoha_ppe_v1_engine_arm(ppe);
+ if (err)
+ return err;
+ block_cb = flow_block_cb_alloc(cb, dev, dev, NULL);
+ if (IS_ERR(block_cb)) {
+ if (list_empty(&ppe->block_cb_list))
+ airoha_ppe_v1_engine_disarm(ppe);
+ return PTR_ERR(block_cb);
+ }
+ flow_block_cb_incref(block_cb);
+ flow_block_cb_add(block_cb, offload);
+ list_add_tail(&block_cb->driver_list, &ppe->block_cb_list);
+ return 0;
+ case FLOW_BLOCK_UNBIND:
+ block_cb = flow_block_cb_lookup(offload->block, cb, dev);
+ if (!block_cb)
+ return -ENOENT;
+ if (!flow_block_cb_decref(block_cb)) {
+ flow_block_cb_remove(block_cb, offload);
+ list_del(&block_cb->driver_list);
+ }
+ if (list_empty(&ppe->block_cb_list))
+ airoha_ppe_v1_engine_disarm(ppe);
+ return 0;
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static int airoha_ppe_setup_tc(struct airoha_ppe_dev *ppe_dev,
+ struct net_device *dev,
+ enum tc_setup_type type, void *type_data)
+{
+ if (airoha_ppe_from_netdev(dev) != ppe_dev->priv)
+ return -EOPNOTSUPP;
+
+ switch (type) {
+ case TC_SETUP_BLOCK:
+ case TC_SETUP_FT:
+ return airoha_ppe_setup_tc_block(dev, type_data);
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static int airoha_ppe_v1_init(struct airoha_eth *eth)
+{
+ u32 dram_entries = eth->soc->ppe_dram_entries;
+ size_t foe_size = airoha_ppe_v1_foe_size(eth);
+ struct device *dev = eth->dev;
+ struct airoha_ppe *ppe;
+ int err;
+
+ ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL);
+ if (!ppe)
+ return -ENOMEM;
+
+ airoha_ppe_common_init(&ppe->common, eth, ppe);
+ err = airoha_ppe_common_alloc_foe(&ppe->common, foe_size);
+ if (err)
+ return err;
+
+ ppe->v1.foe_owner = devm_kcalloc(dev, dram_entries,
+ sizeof(*ppe->v1.foe_owner), GFP_KERNEL);
+ if (!ppe->v1.foe_owner)
+ return -ENOMEM;
+
+ err = rhashtable_init(ð->flow_table, &airoha_flow_table_params);
+ if (err)
+ return err;
+
+ spin_lock_init(&ppe->v1.lock);
+ INIT_LIST_HEAD(&ppe->v1.flows);
+ INIT_LIST_HEAD(&ppe->block_cb_list);
+
+ airoha_ppe_hw_init(ppe);
+
+ ppe->common.dev.ops.check_skb_reason = airoha_ppe_check_skb_reason;
+ ppe->common.dev.ops.setup_tc = airoha_ppe_setup_tc;
+
+ if (airoha_ppe_debugfs_init(&ppe->common))
+ dev_warn(dev, "failed to initialize generation-1 PPE debugfs\n");
+
+ eth->ppe = ppe;
+ airoha_ppe_common_enable(&ppe->common);
+
+ dev_info(dev, "PPE FoE table at %pad, %u entries\n",
+ &ppe->common.foe_dma, dram_entries);
+ return 0;
+}
+
+static void airoha_ppe_v1_deinit(struct airoha_eth *eth)
+{
+ struct airoha_ppe *ppe = eth->ppe;
+
+ if (!ppe)
+ return;
+
+ if (READ_ONCE(ppe->v1.armed))
+ airoha_ppe_v1_engine_disarm(ppe);
+ else
+ airoha_ppe_v1_flush(ppe);
+ airoha_ppe_common_disable(&ppe->common);
+ rhashtable_destroy(ð->flow_table);
+ eth->ppe = NULL;
+}
+
+int airoha_ppe_init(struct airoha_eth *eth)
+{
+ if (eth->soc->foe_format == AIROHA_FOE_FORMAT_V1)
+ return airoha_ppe_v1_init(eth);
+
+ return airoha_ppe_datapath_init(eth);
+}
+
+void airoha_ppe_deinit(struct airoha_eth *eth)
+{
+ if (eth->soc->foe_format == AIROHA_FOE_FORMAT_V1)
+ airoha_ppe_v1_deinit(eth);
+ else
+ airoha_ppe_datapath_deinit(eth);
+}
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Airoha PPE flow offload");
diff --git a/drivers/net/ethernet/airoha/airoha_ppe_debugfs.c
b/drivers/net/ethernet/airoha/airoha_ppe_debugfs.c
index e46a98514486..844697cb6f36 100644
--- a/drivers/net/ethernet/airoha/airoha_ppe_debugfs.c
+++ b/drivers/net/ethernet/airoha/airoha_ppe_debugfs.c
@@ -5,10 +5,57 @@
*/
#include "airoha_eth.h"
+#include "airoha_regs.h"
+
+static const char *const airoha_ppe_debugfs_type_str[] = {
+ [PPE_PKT_TYPE_IPV4_HNAPT] = "IPv4 5T",
+ [PPE_PKT_TYPE_IPV4_ROUTE] = "IPv4 3T",
+ [PPE_PKT_TYPE_BRIDGE] = "L2B",
+ [PPE_PKT_TYPE_IPV4_DSLITE] = "DS-LITE",
+ [PPE_PKT_TYPE_IPV6_ROUTE_3T] = "IPv6 3T",
+ [PPE_PKT_TYPE_IPV6_ROUTE_5T] = "IPv6 5T",
+ [PPE_PKT_TYPE_IPV6_6RD] = "6RD",
+};
+
+static const char *const airoha_ppe_debugfs_state_short[] = {
+ [AIROHA_FOE_STATE_INVALID] = "INV",
+ [AIROHA_FOE_STATE_UNBIND] = "UNB",
+ [AIROHA_FOE_STATE_BIND] = "BND",
+ [AIROHA_FOE_STATE_FIN] = "FIN",
+};
+
+static const char *const airoha_ppe_debugfs_state_long[] = {
+ [AIROHA_FOE_STATE_INVALID] = "Invalid",
+ [AIROHA_FOE_STATE_UNBIND] = "Unbind",
+ [AIROHA_FOE_STATE_BIND] = "Bind",
+ [AIROHA_FOE_STATE_FIN] = "FIN",
+};
+
+static const char *airoha_ppe_debugfs_type_name(u32 type)
+{
+ if (type < ARRAY_SIZE(airoha_ppe_debugfs_type_str) &&
+ airoha_ppe_debugfs_type_str[type])
+ return airoha_ppe_debugfs_type_str[type];
+
+ return "UNKNOWN";
+}
+
+static const char *airoha_ppe_debugfs_state_name(u32 state, bool
short_name)
+{
+ const char *const *names = short_name ?
airoha_ppe_debugfs_state_short :
+ airoha_ppe_debugfs_state_long;
+
+ if (state < ARRAY_SIZE(airoha_ppe_debugfs_state_short) && names[state])
+ return names[state];
+
+ return "???";
+}
static void airoha_debugfs_ppe_print_tuple(struct seq_file *m,
- void *src_addr, void *dest_addr,
- u16 *src_port, u16 *dest_port,
+ const void *src_addr,
+ const void *dest_addr,
+ const u16 *src_port,
+ const u16 *dest_port,
bool ipv6)
{
__be32 n_addr[IPV6_ADDR_WORDS];
@@ -34,24 +81,158 @@ static void airoha_debugfs_ppe_print_tuple(struct
seq_file *m,
seq_printf(m, ":%d", *dest_port);
}
+static void airoha_debugfs_ppe_print_tuple_json(struct seq_file *m,
+ const char *prefix,
+ const void *src_addr,
+ const void *dest_addr,
+ const u16 *src_port,
+ const u16 *dest_port,
+ bool ipv6)
+{
+ __be32 n_addr[IPV6_ADDR_WORDS];
+
+ if (ipv6) {
+ ipv6_addr_cpu_to_be32(n_addr, src_addr);
+ seq_printf(m, "\"%s_src\":\"%pI6\",", prefix, n_addr);
+ ipv6_addr_cpu_to_be32(n_addr, dest_addr);
+ seq_printf(m, "\"%s_dest\":\"%pI6\"", prefix, n_addr);
+ } else {
+ seq_printf(m, "\"%s_src\":\"%pI4h\",", prefix, src_addr);
+ seq_printf(m, "\"%s_dest\":\"%pI4h\"", prefix, dest_addr);
+ }
+
+ if (src_port)
+ seq_printf(m, ",\"%s_src_port\":%d", prefix, *src_port);
+ if (dest_port)
+ seq_printf(m, ",\"%s_dest_port\":%d", prefix, *dest_port);
+}
+
+static int airoha_ppe_debugfs_foe_json_show(struct seq_file *m, void
*private)
+{
+ struct airoha_ppe *ppe = m->private;
+ u32 ppe_num_entries = airoha_ppe_get_total_num_entries(ppe);
+ bool first_entry = true;
+ int i;
+
+ seq_puts(m, "[\n");
+
+ for (i = 0; i < ppe_num_entries; i++) {
+ const char *state_str, *type_str = "UNKNOWN";
+ void *src_addr = NULL, *dest_addr = NULL;
+ u16 *src_port = NULL, *dest_port = NULL;
+ struct airoha_foe_mac_info_common *l2;
+ unsigned char h_source[ETH_ALEN] = {};
+ struct airoha_foe_stats64 stats = {};
+ unsigned char h_dest[ETH_ALEN];
+ struct airoha_foe_entry *hwe;
+ u32 type, state, ib2, data;
+ bool ipv6 = false;
+
+ hwe = airoha_ppe_foe_get_entry(ppe, i);
+ if (!hwe)
+ continue;
+
+ state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, hwe->ib1);
+ if (!state)
+ continue;
+
+ state_str = airoha_ppe_debugfs_state_name(state, false);
+ type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1);
+ type_str = airoha_ppe_debugfs_type_name(type);
+
+ if (!first_entry)
+ seq_puts(m, ",\n");
+ first_entry = false;
+
+ seq_printf(m, " {\"index\":%d,\"state\":\"%s\",\"type\":\"%s\"",
+ i, state_str, type_str);
+
+ switch (type) {
+ case PPE_PKT_TYPE_IPV4_HNAPT:
+ case PPE_PKT_TYPE_IPV4_DSLITE:
+ src_port = &hwe->ipv4.orig_tuple.src_port;
+ dest_port = &hwe->ipv4.orig_tuple.dest_port;
+ fallthrough;
+ case PPE_PKT_TYPE_IPV4_ROUTE:
+ src_addr = &hwe->ipv4.orig_tuple.src_ip;
+ dest_addr = &hwe->ipv4.orig_tuple.dest_ip;
+ break;
+ case PPE_PKT_TYPE_IPV6_ROUTE_5T:
+ src_port = &hwe->ipv6.src_port;
+ dest_port = &hwe->ipv6.dest_port;
+ fallthrough;
+ case PPE_PKT_TYPE_IPV6_ROUTE_3T:
+ case PPE_PKT_TYPE_IPV6_6RD:
+ src_addr = &hwe->ipv6.src_ip;
+ dest_addr = &hwe->ipv6.dest_ip;
+ ipv6 = true;
+ break;
+ default:
+ break;
+ }
+
+ if (src_addr && dest_addr) {
+ seq_puts(m, ",");
+ airoha_debugfs_ppe_print_tuple_json(m, "orig", src_addr,
dest_addr,
+ src_port, dest_port, ipv6);
+ }
+
+ switch (type) {
+ case PPE_PKT_TYPE_IPV4_HNAPT:
+ case PPE_PKT_TYPE_IPV4_DSLITE:
+ src_port = &hwe->ipv4.new_tuple.src_port;
+ dest_port = &hwe->ipv4.new_tuple.dest_port;
+ fallthrough;
+ case PPE_PKT_TYPE_IPV4_ROUTE:
+ src_addr = &hwe->ipv4.new_tuple.src_ip;
+ dest_addr = &hwe->ipv4.new_tuple.dest_ip;
+ seq_puts(m, ",");
+ airoha_debugfs_ppe_print_tuple_json(m, "new", src_addr,
dest_addr,
+ src_port, dest_port, ipv6);
+ break;
+ default:
+ break;
+ }
+
+ if (type == PPE_PKT_TYPE_BRIDGE) {
+ data = hwe->bridge.data;
+ ib2 = hwe->bridge.ib2;
+ l2 = &hwe->bridge.l2.common;
+ *((__be16 *)&h_source[4]) =
cpu_to_be16(hwe->bridge.l2.src_mac_lo);
+ } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) {
+ data = hwe->ipv6.data;
+ ib2 = hwe->ipv6.ib2;
+ l2 = &hwe->ipv6.l2;
+ *((__be16 *)&h_source[4]) = 0;
+ } else {
+ data = hwe->ipv4.data;
+ ib2 = hwe->ipv4.ib2;
+ l2 = &hwe->ipv4.l2.common;
+ *((__be16 *)&h_source[4]) =
cpu_to_be16(hwe->ipv4.l2.src_mac_lo);
+ }
+
+ airoha_ppe_foe_entry_get_stats(ppe, i, &stats);
+
+ *((__be32 *)h_dest) = cpu_to_be32(l2->dest_mac_hi);
+ *((__be16 *)&h_dest[4]) = cpu_to_be16(l2->dest_mac_lo);
+ *((__be32 *)h_source) = cpu_to_be32(l2->src_mac_hi);
+
+ seq_printf(m,
",\"eth_src\":\"%pM\",\"eth_dest\":\"%pM\",\"etype\":\"0x%04x\",\"data\":\"0x%08x\","
+ "\"vlan1\":%d,\"vlan2\":%d,\"ib1\":\"0x%08x\",\"ib2\":\"0x%08x\","
+ "\"packets\":%llu,\"bytes\":%llu}",
+ h_source, h_dest, l2->etype, data,
+ l2->vlan1, l2->vlan2, hwe->ib1, ib2,
+ stats.packets, stats.bytes);
+ }
+
+ seq_puts(m, "\n]\n");
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_ppe_debugfs_foe_json);
+
static int airoha_ppe_debugfs_foe_show(struct seq_file *m, void *private,
bool bind)
{
- static const char *const ppe_type_str[] = {
- [PPE_PKT_TYPE_IPV4_HNAPT] = "IPv4 5T",
- [PPE_PKT_TYPE_IPV4_ROUTE] = "IPv4 3T",
- [PPE_PKT_TYPE_BRIDGE] = "L2B",
- [PPE_PKT_TYPE_IPV4_DSLITE] = "DS-LITE",
- [PPE_PKT_TYPE_IPV6_ROUTE_3T] = "IPv6 3T",
- [PPE_PKT_TYPE_IPV6_ROUTE_5T] = "IPv6 5T",
- [PPE_PKT_TYPE_IPV6_6RD] = "6RD",
- };
- static const char *const ppe_state_str[] = {
- [AIROHA_FOE_STATE_INVALID] = "INV",
- [AIROHA_FOE_STATE_UNBIND] = "UNB",
- [AIROHA_FOE_STATE_BIND] = "BND",
- [AIROHA_FOE_STATE_FIN] = "FIN",
- };
struct airoha_ppe *ppe = m->private;
u32 ppe_num_entries = airoha_ppe_get_total_num_entries(ppe);
int i;
@@ -79,10 +260,9 @@ static int airoha_ppe_debugfs_foe_show(struct
seq_file *m, void *private,
if (bind && state != AIROHA_FOE_STATE_BIND)
continue;
- state_str = ppe_state_str[state % ARRAY_SIZE(ppe_state_str)];
+ state_str = airoha_ppe_debugfs_state_name(state, true);
type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, hwe->ib1);
- if (type < ARRAY_SIZE(ppe_type_str) && ppe_type_str[type])
- type_str = ppe_type_str[type];
+ type_str = airoha_ppe_debugfs_type_name(type);
seq_printf(m, "%05x %s %7s", i, state_str, type_str);
@@ -132,16 +312,21 @@ static int airoha_ppe_debugfs_foe_show(struct
seq_file *m, void *private,
break;
}
- if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) {
+ if (type == PPE_PKT_TYPE_BRIDGE) {
+ data = hwe->bridge.data;
+ ib2 = hwe->bridge.ib2;
+ l2 = &hwe->bridge.l2.common;
+ *((__be16 *)&h_source[4]) =
cpu_to_be16(hwe->bridge.l2.src_mac_lo);
+ } else if (type >= PPE_PKT_TYPE_IPV6_ROUTE_3T) {
data = hwe->ipv6.data;
ib2 = hwe->ipv6.ib2;
l2 = &hwe->ipv6.l2;
+ *((__be16 *)&h_source[4]) = 0;
} else {
data = hwe->ipv4.data;
ib2 = hwe->ipv4.ib2;
l2 = &hwe->ipv4.l2.common;
- *((__be16 *)&h_source[4]) =
- cpu_to_be16(hwe->ipv4.l2.src_mac_lo);
+ *((__be16 *)&h_source[4]) =
cpu_to_be16(hwe->ipv4.l2.src_mac_lo);
}
airoha_ppe_foe_entry_get_stats(ppe, i, &stats);
@@ -173,13 +358,388 @@ static int
airoha_ppe_debugfs_foe_bind_show(struct seq_file *m, void *private)
}
DEFINE_SHOW_ATTRIBUTE(airoha_ppe_debugfs_foe_bind);
-int airoha_ppe_debugfs_init(struct airoha_ppe *ppe)
+
+static u32 airoha_ppe_v1_debugfs_l2_get(const struct airoha_foe_entry
*entry,
+ unsigned int word, u32 mask)
+{
+ const u32 *l2 = &entry->words[airoha_foe_v1_l2_word(entry)];
+
+ return FIELD_GET(mask, l2[word]);
+}
+
+static void airoha_ppe_v1_debugfs_mac(const struct airoha_foe_entry *entry,
+ u8 *src, u8 *dest)
+{
+ const u32 *l2 = &entry->words[airoha_foe_v1_l2_word(entry)];
+
+ *(__be32 *)&dest[0] = cpu_to_be32(l2[1]);
+ *(__be16 *)&dest[4] = cpu_to_be16(FIELD_GET(AIROHA_FOE_L2_DMAC_LO,
+ l2[2]));
+ *(__be32 *)&src[0] = cpu_to_be32(l2[3]);
+ *(__be16 *)&src[4] = cpu_to_be16(FIELD_GET(AIROHA_FOE_L2_SMAC_LO,
+ l2[4]));
+}
+
+static void airoha_ppe_v1_debugfs_print_ipv4(struct seq_file *m,
+ const struct airoha_foe_entry *entry)
+{
+ u16 src_port = FIELD_GET(AIROHA_FOE_PORTS_SPORT, entry->words[3]);
+ u16 dest_port = FIELD_GET(AIROHA_FOE_PORTS_DPORT, entry->words[3]);
+ u16 new_src_port = FIELD_GET(AIROHA_FOE_PORTS_SPORT, entry->words[7]);
+ u16 new_dest_port = FIELD_GET(AIROHA_FOE_PORTS_DPORT, entry->words[7]);
+ u32 ib2 = entry->words[AIROHA_FOE_V1_IPV4_IB2_WORD];
+ u32 data = entry->words[AIROHA_FOE_V1_IPV4_DATA_WORD];
+ u8 src[ETH_ALEN], dest[ETH_ALEN];
+
+ airoha_ppe_v1_debugfs_mac(entry, src, dest);
+ seq_puts(m, " orig=");
+ airoha_debugfs_ppe_print_tuple(m, &entry->words[1], &entry->words[2],
+ &src_port, &dest_port, false);
+ seq_puts(m, " new=");
+ airoha_debugfs_ppe_print_tuple(m, &entry->words[5], &entry->words[6],
+ &new_src_port, &new_dest_port, false);
+ seq_printf(m,
+ " eth=%pM->%pM etype=%04x vlan=%u,%u pppoe=%u",
+ src, dest,
+ airoha_ppe_v1_debugfs_l2_get(entry, 0, AIROHA_FOE_L2_ETYPE),
+ airoha_ppe_v1_debugfs_l2_get(entry, 0, AIROHA_FOE_L2_VLAN1),
+ airoha_ppe_v1_debugfs_l2_get(entry, 2, AIROHA_FOE_L2_VLAN2),
+ airoha_ppe_v1_debugfs_l2_get(entry, 4, AIROHA_FOE_L2_PPPOE_ID));
+ seq_printf(m, " act_dp=%lu tsid=%lu ch=%lu fp=%lu fqos=%d qid=%lu",
+ FIELD_GET(AIROHA_FOE_V1_ACTDP, data),
+ FIELD_GET(AIROHA_FOE_V1_SHAPER_ID, data),
+ FIELD_GET(AIROHA_FOE_V1_CHANNEL, data),
+ FIELD_GET(AIROHA_FOE_V1_IB2_PSE_PORT, ib2),
+ !!(ib2 & AIROHA_FOE_V1_IB2_PSE_QOS),
+ FIELD_GET(AIROHA_FOE_V1_IB2_QID, ib2));
+ seq_printf(m, " ib1=%08x ib2=%08x", entry->ib1, ib2);
+}
+
+static void airoha_ppe_v1_debugfs_print_ipv6(struct seq_file *m,
+ const struct airoha_foe_entry *entry)
+{
+ u16 src_port = FIELD_GET(AIROHA_FOE_PORTS_SPORT, entry->words[9]);
+ u16 dest_port = FIELD_GET(AIROHA_FOE_PORTS_DPORT, entry->words[9]);
+ u32 ib2 = entry->words[AIROHA_FOE_V1_IPV6_IB2_WORD];
+ u32 data = entry->words[AIROHA_FOE_V1_IPV6_DATA_WORD];
+ u8 src[ETH_ALEN], dest[ETH_ALEN];
+
+ airoha_ppe_v1_debugfs_mac(entry, src, dest);
+ seq_puts(m, " orig=");
+ airoha_debugfs_ppe_print_tuple(m, &entry->words[1], &entry->words[5],
+ &src_port, &dest_port, true);
+ seq_printf(m,
+ " eth=%pM->%pM etype=%04x vlan=%u,%u pppoe=%u",
+ src, dest,
+ airoha_ppe_v1_debugfs_l2_get(entry, 0, AIROHA_FOE_L2_ETYPE),
+ airoha_ppe_v1_debugfs_l2_get(entry, 0, AIROHA_FOE_L2_VLAN1),
+ airoha_ppe_v1_debugfs_l2_get(entry, 2, AIROHA_FOE_L2_VLAN2),
+ airoha_ppe_v1_debugfs_l2_get(entry, 4, AIROHA_FOE_L2_PPPOE_ID));
+ seq_printf(m, " act_dp=%lu tsid=%lu ch=%lu fp=%lu fqos=%d qid=%lu",
+ FIELD_GET(AIROHA_FOE_V1_ACTDP, data),
+ FIELD_GET(AIROHA_FOE_V1_SHAPER_ID, data),
+ FIELD_GET(AIROHA_FOE_V1_CHANNEL, data),
+ FIELD_GET(AIROHA_FOE_V1_IB2_PSE_PORT, ib2),
+ !!(ib2 & AIROHA_FOE_V1_IB2_PSE_QOS),
+ FIELD_GET(AIROHA_FOE_V1_IB2_QID, ib2));
+ seq_printf(m, " ib1=%08x ib2=%08x", entry->ib1, ib2);
+}
+
+static int airoha_ppe_v1_debugfs_foe_show(struct seq_file *m, bool
bind_only)
+{
+ struct airoha_ppe *ppe = m->private;
+ u32 entries = ppe->common.eth->soc->ppe_dram_entries;
+ u32 i;
+
+ for (i = 0; i < entries; i++) {
+ struct airoha_foe_entry entry;
+ u32 state, type;
+
+ spin_lock_bh(&ppe->v1.lock);
+ airoha_ppe_v1_read_entry(ppe, i, &entry);
+ spin_unlock_bh(&ppe->v1.lock);
+ state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, entry.ib1);
+ if (state == AIROHA_FOE_STATE_INVALID)
+ continue;
+ if (bind_only && state != AIROHA_FOE_STATE_BIND)
+ continue;
+
+ type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, entry.ib1);
+ seq_printf(m, "%05x %s %7s", i,
+ airoha_ppe_debugfs_state_name(state, true),
+ airoha_ppe_debugfs_type_name(type));
+
+ if (type == PPE_PKT_TYPE_IPV4_HNAPT ||
+ type == PPE_PKT_TYPE_IPV4_ROUTE)
+ airoha_ppe_v1_debugfs_print_ipv4(m, &entry);
+ else if (type == PPE_PKT_TYPE_IPV6_ROUTE_5T)
+ airoha_ppe_v1_debugfs_print_ipv6(m, &entry);
+ else
+ seq_printf(m, " ib1=%08x", entry.ib1);
+
+ seq_printf(m, " hw_ib1=%08x\n", entry.ib1);
+ }
+
+ return 0;
+}
+
+static int airoha_ppe_v1_debugfs_foe_all_show(struct seq_file *m, void
*private)
+{
+ return airoha_ppe_v1_debugfs_foe_show(m, false);
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_ppe_v1_debugfs_foe_all);
+
+static int airoha_ppe_v1_debugfs_foe_bind_show(struct seq_file *m, void
*private)
+{
+ return airoha_ppe_v1_debugfs_foe_show(m, true);
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_ppe_v1_debugfs_foe_bind);
+
+static int airoha_ppe_v1_debugfs_foe_raw_show(struct seq_file *m, void
*private)
+{
+ struct airoha_ppe *ppe = m->private;
+ u32 entries = ppe->common.eth->soc->ppe_dram_entries;
+ u32 i;
+
+ for (i = 0; i < entries; i++) {
+ struct airoha_foe_entry raw;
+ u32 *words = raw.words;
+ bool nonzero = false;
+ size_t j;
+
+ spin_lock_bh(&ppe->v1.lock);
+ airoha_ppe_v1_read_entry(ppe, i, &raw);
+ spin_unlock_bh(&ppe->v1.lock);
+
+ for (j = 0; j < AIROHA_FOE_ENTRY_WORDS; j++) {
+ if (words[j]) {
+ nonzero = true;
+ break;
+ }
+ }
+ if (!nonzero)
+ continue;
+
+ seq_printf(m, "%05x:", i);
+ for (j = 0; j < AIROHA_FOE_ENTRY_WORDS; j++)
+ seq_printf(m, " %08x", words[j]);
+ seq_putc(m, '\n');
+ }
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_ppe_v1_debugfs_foe_raw);
+
+static int airoha_ppe_v1_debugfs_flows_show(struct seq_file *m, void
*private)
+{
+ struct airoha_ppe *ppe = m->private;
+ struct airoha_flow_table_entry *flow;
+ struct {
+ struct airoha_foe_entry data;
+ unsigned long cookie;
+ u32 hash;
+ } *snapshot;
+ u32 count = 0, n = 0, i;
+
+ spin_lock_bh(&ppe->v1.lock);
+ list_for_each_entry(flow, &ppe->v1.flows, v1_list)
+ count++;
+ spin_unlock_bh(&ppe->v1.lock);
+
+ if (!count)
+ return 0;
+
+ snapshot = kcalloc(count, sizeof(*snapshot), GFP_KERNEL);
+ if (!snapshot)
+ return -ENOMEM;
+
+ spin_lock_bh(&ppe->v1.lock);
+ list_for_each_entry(flow, &ppe->v1.flows, v1_list) {
+ if (n == count)
+ break;
+ snapshot[n].data = flow->data;
+ snapshot[n].cookie = flow->cookie;
+ snapshot[n].hash = flow->hash;
+ n++;
+ }
+ spin_unlock_bh(&ppe->v1.lock);
+
+ for (i = 0; i < n; i++) {
+ u32 type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE,
+ snapshot[i].data.ib1);
+
+ seq_printf(m, "cookie=%lx hash=%04x", snapshot[i].cookie,
+ snapshot[i].hash);
+ if (type == PPE_PKT_TYPE_IPV6_ROUTE_5T)
+ airoha_ppe_v1_debugfs_print_ipv6(m, &snapshot[i].data);
+ else
+ airoha_ppe_v1_debugfs_print_ipv4(m, &snapshot[i].data);
+ seq_putc(m, '\n');
+ }
+
+ kfree(snapshot);
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_ppe_v1_debugfs_flows);
+
+static int airoha_ppe_v1_debugfs_regs_show(struct seq_file *m, void
*private)
+{
+ struct airoha_ppe *ppe = m->private;
+ struct airoha_eth *eth = ppe->common.eth;
+ struct airoha_flow_table_entry *flow;
+ u32 flows = 0;
+
+ spin_lock_bh(&ppe->v1.lock);
+ list_for_each_entry(flow, &ppe->v1.flows, v1_list)
+ flows++;
+ spin_unlock_bh(&ppe->v1.lock);
+
+ seq_printf(m, "armed=%u\n", READ_ONCE(ppe->v1.armed));
+ seq_printf(m, "flows=%u\n", flows);
+ seq_printf(m, "foe_cpu=%px\n", ppe->common.foe);
+ seq_printf(m, "foe_dma=%pad\n", &ppe->common.foe_dma);
+ seq_printf(m, "foe_entries=%u\n", eth->soc->ppe_dram_entries);
+ seq_printf(m, "GLO_CFG=%08x\n", airoha_fe_rr(eth, REG_PPE_GLO_CFG(0)));
+ seq_printf(m, "FLOW_CFG=%08x\n", airoha_fe_rr(eth,
REG_PPE_PPE_FLOW_CFG(0)));
+ seq_printf(m, "IP_PROTO_CHK=%08x\n", airoha_fe_rr(eth,
REG_PPE_IP_PROTO_CHK(0)));
+ seq_printf(m, "TB_CFG=%08x\n", airoha_fe_rr(eth, REG_PPE_TB_CFG(0)));
+ seq_printf(m, "TB_BASE=%08x\n", airoha_fe_rr(eth, REG_PPE_TB_BASE(0)));
+ seq_printf(m, "TB_USED=%08x\n", airoha_fe_rr(eth, REG_PPE_TB_USED(0)));
+ seq_printf(m, "BIND_RATE=%08x\n", airoha_fe_rr(eth,
REG_PPE_BIND_RATE(0)));
+ seq_printf(m, "BIND_LIMIT0=%08x\n", airoha_fe_rr(eth,
REG_PPE_BIND_LIMIT0(0)));
+ seq_printf(m, "BIND_LIMIT1=%08x\n", airoha_fe_rr(eth,
REG_PPE_BIND_LIMIT1(0)));
+ seq_printf(m, "KEEPALIVE=%08x\n", airoha_fe_rr(eth,
REG_PPE_KEEPALIVE(0)));
+ seq_printf(m, "UNBIND_AGE=%08x\n", airoha_fe_rr(eth,
REG_PPE_UNBIND_AGE(0)));
+ seq_printf(m, "BND_AGE0=%08x\n", airoha_fe_rr(eth,
REG_PPE_BND_AGE0(0)));
+ seq_printf(m, "BND_AGE1=%08x\n", airoha_fe_rr(eth,
REG_PPE_BND_AGE1(0)));
+ seq_printf(m, "HASH_SEED=%08x\n", airoha_fe_rr(eth,
REG_PPE_HASH_SEED(0)));
+ seq_printf(m, "DFT_CPORT=%08x\n", airoha_fe_rr(eth,
REG_PPE_DFT_CPORT_BASE(0)));
+ seq_printf(m, "GDM1_FWD_CFG=%08x\n", airoha_fe_rr(eth,
REG_GDM_FWD_CFG(1)));
+ seq_printf(m, "GDM2_FWD_CFG=%08x\n", airoha_fe_rr(eth,
REG_GDM_FWD_CFG(2)));
+ seq_printf(m, "VPM_TPID=%08x\n", airoha_fe_rr(eth,
REG_PPE_VPM_TPID(0)));
+ seq_printf(m, "CACHE_CTL=%08x\n", airoha_fe_rr(eth,
REG_EN751221_PPE_CACHE_CTL));
+ seq_printf(m, "CACHE_GATE=%08x\n", airoha_fe_rr(eth,
REG_EN751221_PPE_CAH_GATE));
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_ppe_v1_debugfs_regs);
+
+static int airoha_ppe_v1_debugfs_foe_json_show(struct seq_file *m, void
*private)
{
- ppe->debugfs_dir = debugfs_create_dir("ppe", NULL);
- debugfs_create_file("entries", 0444, ppe->debugfs_dir, ppe,
- &airoha_ppe_debugfs_foe_all_fops);
- debugfs_create_file("bind", 0444, ppe->debugfs_dir, ppe,
- &airoha_ppe_debugfs_foe_bind_fops);
+ struct airoha_ppe *ppe = m->private;
+ u32 entries = ppe->common.eth->soc->ppe_dram_entries;
+ bool first = true;
+ u32 i;
+
+ seq_puts(m, "[\n");
+ for (i = 0; i < entries; i++) {
+ struct airoha_foe_entry entry;
+ u32 state, type, ib2;
+
+ spin_lock_bh(&ppe->v1.lock);
+ airoha_ppe_v1_read_entry(ppe, i, &entry);
+ spin_unlock_bh(&ppe->v1.lock);
+ state = FIELD_GET(AIROHA_FOE_IB1_BIND_STATE, entry.ib1);
+ if (state == AIROHA_FOE_STATE_INVALID)
+ continue;
+
+ type = FIELD_GET(AIROHA_FOE_IB1_BIND_PACKET_TYPE, entry.ib1);
+ if (!first)
+ seq_puts(m, ",\n");
+ first = false;
+
+ seq_printf(m,
+ "
{\"index\":%u,\"state\":\"%s\",\"type\":\"%s\",\"ib1\":\"0x%08x\"",
+ i, airoha_ppe_debugfs_state_name(state, true),
+ airoha_ppe_debugfs_type_name(type), entry.ib1);
+
+ if (type == PPE_PKT_TYPE_IPV4_HNAPT ||
+ type == PPE_PKT_TYPE_IPV4_ROUTE) {
+ u16 src_port = FIELD_GET(AIROHA_FOE_PORTS_SPORT,
+ entry.words[3]);
+ u16 dest_port = FIELD_GET(AIROHA_FOE_PORTS_DPORT,
+ entry.words[3]);
+ u16 new_src_port = FIELD_GET(AIROHA_FOE_PORTS_SPORT,
+ entry.words[7]);
+ u16 new_dest_port = FIELD_GET(AIROHA_FOE_PORTS_DPORT,
+ entry.words[7]);
+
+ seq_putc(m, ',');
+ airoha_debugfs_ppe_print_tuple_json(
+ m, "orig", &entry.words[1], &entry.words[2],
+ &src_port, &dest_port, false);
+ seq_putc(m, ',');
+ airoha_debugfs_ppe_print_tuple_json(
+ m, "new", &entry.words[5], &entry.words[6],
+ &new_src_port, &new_dest_port, false);
+ ib2 = entry.words[AIROHA_FOE_V1_IPV4_IB2_WORD];
+ seq_printf(m, ",\"ib2\":\"0x%08x\"", ib2);
+ } else if (type == PPE_PKT_TYPE_IPV6_ROUTE_5T) {
+ u16 src_port = FIELD_GET(AIROHA_FOE_PORTS_SPORT,
+ entry.words[9]);
+ u16 dest_port = FIELD_GET(AIROHA_FOE_PORTS_DPORT,
+ entry.words[9]);
+
+ seq_putc(m, ',');
+ airoha_debugfs_ppe_print_tuple_json(
+ m, "orig", &entry.words[1], &entry.words[5],
+ &src_port, &dest_port, true);
+ ib2 = entry.words[AIROHA_FOE_V1_IPV6_IB2_WORD];
+ seq_printf(m, ",\"ib2\":\"0x%08x\"", ib2);
+ }
+
+ seq_putc(m, '}');
+ }
+ seq_puts(m, "\n]\n");
+
+ return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(airoha_ppe_v1_debugfs_foe_json);
+
+static int
+airoha_ppe_debugfs_create_common(struct airoha_ppe_common *common,
+ const struct file_operations *entries_fops,
+ const struct file_operations *json_fops,
+ const struct file_operations *bind_fops)
+{
+ struct dentry *dir;
+ void *priv = common->dev.priv;
+
+ dir = debugfs_create_dir("ppe", NULL);
+ if (IS_ERR(dir))
+ return PTR_ERR(dir);
+
+ common->debugfs_dir = dir;
+ debugfs_create_file("entries", 0444, dir, priv, entries_fops);
+ debugfs_create_file("entries.json", 0444, dir, priv, json_fops);
+ debugfs_create_file("bind", 0444, dir, priv, bind_fops);
+
+ return 0;
+}
+
+int airoha_ppe_debugfs_init(struct airoha_ppe_common *common)
+{
+ int err;
+
+ if (common->eth->soc->foe_format != AIROHA_FOE_FORMAT_V1)
+ return airoha_ppe_debugfs_create_common(common,
+ &airoha_ppe_debugfs_foe_all_fops,
+ &airoha_ppe_debugfs_foe_json_fops,
+ &airoha_ppe_debugfs_foe_bind_fops);
+
+ err = airoha_ppe_debugfs_create_common(common,
+ &airoha_ppe_v1_debugfs_foe_all_fops,
+ &airoha_ppe_v1_debugfs_foe_json_fops,
+ &airoha_ppe_v1_debugfs_foe_bind_fops);
+ if (err)
+ return err;
+
+ debugfs_create_file("raw", 0444, common->debugfs_dir,
+ common->dev.priv, &airoha_ppe_v1_debugfs_foe_raw_fops);
+ debugfs_create_file("flows", 0444, common->debugfs_dir,
+ common->dev.priv, &airoha_ppe_v1_debugfs_flows_fops);
+ debugfs_create_file("regs", 0444, common->debugfs_dir,
+ common->dev.priv, &airoha_ppe_v1_debugfs_regs_fops);
return 0;
}
diff --git a/drivers/net/ethernet/airoha/airoha_regs.h
b/drivers/net/ethernet/airoha/airoha_regs.h
index caa0b95abd75..5980b853afdb 100644
--- a/drivers/net/ethernet/airoha/airoha_regs.h
+++ b/drivers/net/ethernet/airoha/airoha_regs.h
@@ -39,6 +39,54 @@
#define FE_RST_GDM3_MBI_ARB_MASK BIT(2)
#define FE_RST_CORE_MASK BIT(0)
+
+/* EN751221/EN7528 frame-engine layout uses the shared CDM/GDM bases. */
+#define EN751221_CDM_STAG_EN BIT(0)
+#define EN751221_GDM_STAG_EN BIT(24)
+#define EN751221_GDM_UNTAG_EN BIT(25)
+#define EN751221_GDM1_LONG_LEN 1700
+
+/* EN751221 PPE/FoE register and entry layout. */
+#define EN751221_PSE_BUF_CTRL_EN BIT(31)
+#define EN751221_PSE_BUF_PREFETCH_EN BIT(30)
+#define EN751221_PSE_BUF_ESTIMATE_EN BIT(29)
+#define EN751221_PSE_BUF_CH_THR_MASK GENMASK(23, 16)
+#define EN751221_PSE_BUF_TOTAL_THR_MASK GENMASK(7, 0)
+
+#define EN751221_TX_RATE_METER_EN BIT(31)
+#define EN751221_TX_RATE_METER_DIV_MASK GENMASK(17, 16)
+#define EN751221_TX_RATE_METER_SLICE_MASK GENMASK(15, 0)
+#define EN751221_TXQ_MAX_THR_MASK GENMASK(31, 16)
+#define EN751221_TXQ_MIN_THR_MASK GENMASK(15, 0)
+#define EN751221_QDMA_IRQ_RING_IDX_MASK BIT(16)
+#define EN751221_QDMA_IRQ_DESC_IDX_MASK GENMASK(11, 0)
+
+/* EN751221 GDM FWD_CFG destination-port coding. */
+#define EN751221_GDM_FPORT_QDMA0_CPU 0
+#define EN751221_GDM_FPORT_PPE 4
+#define EN751221_GDM_FPORT_QDMA1_CPU 5
+
+/* Legacy PPE cache control retained by the generation-1 engine. */
+#define REG_EN751221_PPE_CACHE_CTL (PPE1_BASE + 0x320)
+#define EN751221_PPE_CACHE_CTL_REQ BIT(8)
+#define EN751221_PPE_CACHE_CTL_CMD GENMASK(14, 12)
+
+#define REG_EN751221_PPE_CAH_GATE (PPE1_BASE + 0x334)
+#define EN751221_PPE_CAH_GATE_EN BIT(0)
+#define EN751221_PPE_CAH_GATE_DEFAULT 0x33
+
+#define EN751221_PPE_CPU_REASON_NO_FLOW 0x07
+
+#define REG_FE_INT_STATUS 0x0008
+#define GDM2_RX_INTR3_MASK BIT(23)
+#define GDM2_RX_INTR2_MASK BIT(22)
+#define GDM2_RX_INTR1_MASK BIT(21)
+#define GDM2_RX_INTR0_MASK BIT(20)
+#define GDM2_TX_INTR2_MASK BIT(18)
+#define GDM2_TX_INTR1_MASK BIT(17)
+#define GDM2_TX_INTR0_MASK BIT(16)
+#define PSE_FQ_EMPTY_MASK BIT(8)
+#define REG_FE_INT_ENABLE 0x000C
#define REG_FE_FOE_TS 0x0010
#define REG_FE_WAN_PORT 0x0024
@@ -71,7 +119,8 @@
#define REG_FE_PSE_QUEUE_CFG_VAL 0x0084
#define PSE_CFG_OQ_RSV_MASK GENMASK(13, 0)
-#define PSE_FQ_CFG 0x008c
+#define PSE_FQ_CFG 0x014c
+#define PSE_FQ_CFG_EN7523 0x008c
#define PSE_FQ_LIMIT_MASK GENMASK(14, 0)
#define REG_FE_PSE_BUF_SET 0x0090
@@ -85,6 +134,10 @@
#define REG_GDM_MISC_CFG 0x0148
#define GDM2_RDM_ACK_WAIT_PREF_MASK BIT(9)
#define GDM2_CHN_VLD_MODE_MASK BIT(5)
+#define GDM2_RLS_MODE_MASK BIT(1)
+
+#define REG_QDMA_FC_WIFI_SP 0x0190
+#define WIFI_OFFLOAD_FC_EN_MASK BIT(6)
#define REG_FE_CSR_IFC_CFG CSR_IFC_BASE
#define FE_IFC_EN_MASK BIT(0)
@@ -113,20 +166,28 @@
#define REG_CDM_VLAN_CTRL(_n) CDM_BASE(_n)
#define CDM_VLAN_MASK GENMASK(31, 16)
+#define STAG_EN BIT(0)
#define REG_CDM_FWD_CFG(_n) (CDM_BASE(_n) + 0x08)
#define CDM_OAM_QSEL_MASK GENMASK(31, 27)
+/* EN7523 follows the 4-bit legacy OAM selector at bits 31:28.
+ * Bits 15:12 are used by EN7528, which is a different FE revision.
+ */
+#define EN7523_CDM_OAM_QSEL_MASK GENMASK(31, 28)
#define CDM_VIP_QSEL_MASK GENMASK(24, 20)
+#define EN7523_CDM_VIP_QSEL_MASK GENMASK(27, 24)
#define REG_CDM_CRSN_QSEL(_n, _m) (CDM_BASE(_n) + 0x10 + ((_m) << 2))
-#define CDM_CRSN_QSEL_REASON_MASK(_n) \
- GENMASK(4 + (((_n) % 4) << 3), (((_n) % 4) << 3))
-
+
#define REG_CDM_LRO_RXQ(_n, _m) (CDM_BASE(_n) + 0x78 + ((_m) &
0x4))
#define LRO_RXQ_MASK(_n) GENMASK(4 + (((_n) & 0x3) << 3), ((_n)
& 0x3) << 3)
+#define EN7523_LRO_RXQ_MASK(_n) GENMASK(((_n) & 0x3) * 4 + 3,
((_n) & 0x3) * 4)
+#define EN7523_LRO_EN_MASK(_n) BIT(24 + ((_n) & 0x3))
#define REG_CDM_LRO_EN(_n) (CDM_BASE(_n) + 0x80)
#define LRO_RXQ_EN_MASK GENMASK(7, 0)
+#define EN7523_LRO_RXQ_EN_MASK GENMASK(27, 24)
+#define EN7523_LRO_ALL_RXQ_MASK GENMASK(15, 0)
#define REG_CDM_LRO_LIMIT(_n) (CDM_BASE(_n) + 0x84)
#define CDM_LRO_AGG_NUM_MASK GENMASK(23, 16)
@@ -136,8 +197,13 @@
#define CDM_LRO_AGE_TIME_MASK GENMASK(31, 16)
#define CDM_LRO_AGG_TIME_MASK GENMASK(15, 0)
+#define REG_CDM_FAQ_CFG(_n) (CDM_BASE(_n) + 0xb0)
+#define REG_CDM_FAQ_THR_CFG(_n) (CDM_BASE(_n) + 0xb4)
+
#define REG_GDM_FWD_CFG(_n) GDM_BASE(_n)
#define GDM_PAD_EN_MASK BIT(28)
+#define GDM_DROP_OVERSIZE_MASK BIT(25)
+#define GDM_DROP_RUNT_MASK BIT(24)
#define GDM_DROP_CRC_ERR_MASK BIT(23)
#define GDM_IP4_CKSUM_MASK BIT(22)
#define GDM_TCP_CKSUM_MASK BIT(21)
@@ -152,6 +218,11 @@
#define GDM_INGRESS_FC_EN_MASK BIT(1)
#define GDM_STAG_EN_MASK BIT(0)
+#define GDM1_BASE_STAG_EN (GDM1_BASE + 0x10)
+#define CPORT_TX_STAG_EN BIT(2)
+#define CPORT_RX_STAG_EN BIT(1)
+#define GDM1_RX_LAN_SPORT BIT(0)
+
#define REG_GDM_LEN_CFG(_n) (GDM_BASE(_n) + 0x14)
#define GDM_SHORT_LEN_MASK GENMASK(13, 0)
#define GDM_LONG_LEN_MASK GENMASK(29, 16)
@@ -172,11 +243,22 @@
#define REG_GDM_TXCHN_EN(_n) (GDM_BASE(_n) + 0x24)
#define REG_GDM_RXCHN_EN(_n) (GDM_BASE(_n) + 0x28)
+#define REG_CDM_HWF_CHN_EN(_n) (CDM_BASE(_n) + 0x0c)
+
+/* GPON downstream traffic uses GDM2 receive channels 0 and 1. */
+#define EN7523_GDM2_GPON_RX_CHN_MASK GENMASK(1, 0)
+#define EN7523_GDM2_EPON_TX_CHN_MASK (GENMASK(7, 0) | GENMASK(23, 16))
+#define EN7523_GDM2_EPON_RX_CHN_MASK GENMASK(7, 0)
+#define EN7523_CDM2_EPON_HWF_CHN_MASK GENMASK(7, 0)
#define REG_FE_CPORT_CFG (GDM1_BASE + 0x40)
+#define FE_CPORT_DIS_FE2GSW_CRC BIT(31)
+#define FE_CPORT_DIS_GSW2FE_CRC BIT(30)
#define FE_CPORT_PAD BIT(26)
#define FE_CPORT_PORT_XFC_MASK BIT(25)
#define FE_CPORT_QUEUE_XFC_MASK BIT(24)
+#define FE_CPORT_FE2SW_IPG GENMASK(15, 8)
+#define FE_CPORT_SW2FE_IPG GENMASK(7, 0)
#define REG_FE_GDM_MIB_CLEAR(_n) (GDM_BASE(_n) + 0xf0)
#define FE_GDM_MIB_RX_CLEAR_MASK BIT(1)
@@ -268,9 +350,12 @@
#define REG_PPE_PPE_FLOW_CFG(_n) (((_n) ? PPE2_BASE :
PPE1_BASE) + 0x204)
#define PPE_FLOW_CFG_IP6_HASH_GRE_KEY_MASK BIT(20)
#define PPE_FLOW_CFG_IP4_HASH_GRE_KEY_MASK BIT(19)
+#define PPE_FLOW_CFG_IP6_MC_HPRI_MASK BIT(25)
+#define PPE_FLOW_CFG_IP_MC_HPIT_MASK BIT(26)
#define PPE_FLOW_CFG_IP4_HASH_FLOW_LABEL_MASK BIT(18)
#define PPE_FLOW_CFG_IP4_NAT_FRAG_MASK BIT(17)
#define PPE_FLOW_CFG_IP_PROTO_BLACKLIST_MASK BIT(16)
+#define PPE_FLOW_CFG_L2_BRIDGE_MASK BIT(15)
#define PPE_FLOW_CFG_IP4_DSLITE_MASK BIT(14)
#define PPE_FLOW_CFG_IP4_NAPT_MASK BIT(13)
#define PPE_FLOW_CFG_IP4_NAT_MASK BIT(12)
@@ -281,12 +366,20 @@
#define PPE_FLOW_CFG_IP4_TCP_FRAG_MASK BIT(6)
#define REG_PPE_IP_PROTO_CHK(_n) (((_n) ? PPE2_BASE :
PPE1_BASE) + 0x208)
+/*
+ * IP protocol list checked by the PPE classifier, four protocol
numbers per
+ * register. setup_ip_chk() in the EN7516 SDK loads TCP, UDP, IPv6 and
IPIP
+ * into PPE_IP_PROT_0 when PPE_FLOW_CFG selects the blacklist mode.
+ */
+#define REG_PPE_IP_PROT(_n, _i) (((_n) ? PPE2_BASE :
PPE1_BASE) + 0x20c + ((_i) << 2))
#define PPE_IP_PROTO_CHK_IPV4_MASK GENMASK(31, 16)
#define PPE_IP_PROTO_CHK_IPV6_MASK GENMASK(15, 0)
#define REG_PPE_TB_CFG(_n) (((_n) ? PPE2_BASE : PPE1_BASE)
+ 0x21c)
#define PPE_SRAM_TB_NUM_ENTRY_MASK GENMASK(26, 24)
#define PPE_TB_CFG_KEEPALIVE_MASK GENMASK(13, 12)
+#define PPE_TB_CFG_HASH_MODE_MASK GENMASK(15, 14)
+#define PPE_TB_CFG_SCAN_MODE_MASK GENMASK(17, 16)
#define PPE_TB_CFG_AGE_TCP_FIN_MASK BIT(11)
#define PPE_TB_CFG_AGE_UDP_MASK BIT(10)
#define PPE_TB_CFG_AGE_TCP_MASK BIT(9)
@@ -298,6 +391,7 @@
#define PPE_DRAM_TB_NUM_ENTRY_MASK GENMASK(2, 0)
#define REG_PPE_TB_BASE(_n) (((_n) ? PPE2_BASE : PPE1_BASE)
+ 0x220)
+#define REG_PPE_TB_USED(_n) (((_n) ? PPE2_BASE : PPE1_BASE)
+ 0x224)
#define REG_PPE_BIND_RATE(_n) (((_n) ? PPE2_BASE :
PPE1_BASE) + 0x228)
#define PPE_BIND_RATE_L2B_BIND_MASK GENMASK(31, 16)
@@ -311,6 +405,11 @@
#define PPE_BIND_LIMIT1_NON_L4_MASK GENMASK(23, 16)
#define PPE_BIND_LIMIT1_FULL_MASK GENMASK(13, 0)
+#define REG_PPE_KEEPALIVE(_n) (((_n) ? PPE2_BASE :
PPE1_BASE) + 0x234)
+#define PPE_KEEPALIVE_UDP_MASK GENMASK(31, 24)
+#define PPE_KEEPALIVE_TCP_MASK GENMASK(23, 16)
+#define PPE_KEEPALIVE_NTU_MASK GENMASK(7, 0)
+
#define REG_PPE_BND_AGE0(_n) (((_n) ? PPE2_BASE :
PPE1_BASE) + 0x23c)
#define PPE_BIND_AGE0_DELTA_NON_L4 GENMASK(30, 16)
#define PPE_BIND_AGE0_DELTA_UDP GENMASK(14, 0)
@@ -331,19 +430,27 @@
#define DFT_CPORT_MASK(_n) GENMASK(3 + (((_n) % 8) << 2),
(((_n) % 8) << 2))
#define REG_PPE_TB_HASH_CFG(_n) (((_n) ? PPE2_BASE :
PPE1_BASE) + 0x250)
-#define PPE_DRAM_HASH1_MODE_MASK GENMASK(31, 28)
-#define PPE_DRAM_HASH1_EN_MASK BIT(24)
-#define PPE_DRAM_HASH0_MODE_MASK GENMASK(23, 20)
-#define PPE_DRAM_TABLE_EN_MASK BIT(16)
-#define PPE_SRAM_HASH1_MODE_MASK GENMASK(15, 12)
-#define PPE_SRAM_HASH1_EN_MASK BIT(8)
-#define PPE_SRAM_HASH0_MODE_MASK GENMASK(7, 4)
-#define PPE_SRAM_TABLE_EN_MASK BIT(0)
+#define EN7581_PPE_DRAM_HASH1_MODE_MASK GENMASK(31, 28)
+#define EN7581_PPE_DRAM_HASH1_EN_MASK BIT(24)
+#define EN7523_PPE_DRAM_HASH0_MODE_MASK GENMASK(15, 14)
+#define EN7581_PPE_DRAM_HASH0_MODE_MASK GENMASK(23, 20)
+#define EN7523_PPE_DRAM_TABLE_EN_MASK BIT(30)
+#define EN7581_PPE_DRAM_TABLE_EN_MASK BIT(16)
+#define EN7523_PPE_SRAM_HASH1_MODE_MASK GENMASK(23, 22)
+#define EN7581_PPE_SRAM_HASH1_MODE_MASK GENMASK(15, 12)
+#define EN7523_PPE_SRAM_HASH1_EN_MASK BIT(29)
+#define EN7581_PPE_SRAM_HASH1_EN_MASK BIT(8)
+#define EN7523_PPE_SRAM_HASH0_MODE_MASK GENMASK(21, 20)
+#define EN7581_PPE_SRAM_HASH0_MODE_MASK GENMASK(7, 4)
+#define EN7523_PPE_SRAM_TABLE_EN_MASK BIT(31)
+#define EN7581_PPE_SRAM_TABLE_EN_MASK BIT(0)
#define REG_PPE_MTU_BASE(_n) (((_n) ? PPE2_BASE :
PPE1_BASE) + 0x304)
#define REG_PPE_MTU(_m, _n) (REG_PPE_MTU_BASE(_m) + (((_n)
/ 2) << 2))
#define FP_EGRESS_MTU_MASK(_n) GENMASK(13 + (((_n) % 2) <<
4), ((_n) % 2) << 4)
+#define REG_PPE_VPM_TPID(_n) (((_n) ? PPE2_BASE : PPE1_BASE)
+ 0x318)
+
#define REG_PPE_RAM_CTRL(_n) (((_n) ? PPE2_BASE :
PPE1_BASE) + 0x31c)
#define PPE_SRAM_CTRL_ACK_MASK BIT(31)
#define PPE_SRAM_CTRL_DUAL_SUCESS_MASK BIT(30)
@@ -450,7 +557,9 @@
((_n) == 1) ? 0x0024 : 0x0020)
#define REG_INT_ENABLE(_b, _n) \
- (((_n) == 4) ? 0x0750 + ((_b) << 5) : \
+ (((_n) == 6) ? 0x0034 + ((_b) << 5) : \
+ ((_n) == 5) ? 0x0030 + ((_b) << 5) : \
+ ((_n) == 4) ? 0x0750 + ((_b) << 5) : \
((_n) == 3) ? 0x0744 + ((_b) << 5) : \
((_n) == 2) ? 0x0740 + ((_b) << 5) : \
((_n) == 1) ? 0x002c + ((_b) << 3) : \
@@ -489,6 +598,13 @@
#define IRQ0_FULL_INT_MASK BIT(1)
#define IRQ0_INT_MASK BIT(0)
+/* EN7523 QDMA_CSR_INT_ENABLE5 30 */
+#define EN7523_RX_COHERENT_LOW_INT_MASK \
+ (RX14_COHERENT_INT_MASK | \
+ RX13_COHERENT_INT_MASK | RX12_COHERENT_INT_MASK | \
+ RX11_COHERENT_INT_MASK | RX10_COHERENT_INT_MASK)
+
+
#define RX_COHERENT_LOW_INT_MASK \
(RX15_COHERENT_INT_MASK | RX14_COHERENT_INT_MASK | \
RX13_COHERENT_INT_MASK | RX12_COHERENT_INT_MASK | \
@@ -551,6 +667,12 @@
#define RX1_DONE_INT_MASK BIT(1)
#define RX0_DONE_INT_MASK BIT(0)
+/* EN7523 QDMA_CSR_INT_ENABLE6 34 */
+#define EN7523_RX_NO_CPU_DSCP_LOW_INT_MASK \
+ (RX14_NO_CPU_DSCP_INT_MASK | \
+ RX13_NO_CPU_DSCP_INT_MASK | RX12_NO_CPU_DSCP_INT_MASK | \
+ RX11_NO_CPU_DSCP_INT_MASK | RX10_NO_CPU_DSCP_INT_MASK)
+
#define RX_NO_CPU_DSCP_LOW_INT_MASK \
(RX15_NO_CPU_DSCP_INT_MASK | RX14_NO_CPU_DSCP_INT_MASK | \
RX13_NO_CPU_DSCP_INT_MASK | RX12_NO_CPU_DSCP_INT_MASK | \
@@ -851,6 +973,10 @@
#define REG_PSE_BUF_USAGE_CFG 0x1028
#define PSE_BUF_ESTIMATE_EN_MASK BIT(29)
+#define REG_PSE_BUF_USAGE_CFG1 0x10d0
+#define PSE_BUF_CHAN_THR_MASK GENMASK(26, 16)
+#define PSE_BUF_TOTAL_THR_MASK GENMASK(10, 0)
+
#define REG_CHAN_QOS_MODE(_n) (0x1040 + ((_n) << 2))
#define CHAN_QOS_MODE_MASK(_n) GENMASK(2 + ((_n) << 2), (_n) << 2)
@@ -877,6 +1003,7 @@
#define QDMA_DESC_DEI_MASK BIT(25)
#define QDMA_DESC_NO_DROP_MASK BIT(24)
#define QDMA_DESC_LEN_MASK GENMASK(15, 0)
+#define EN7523_QDMA_DESC_LEN_MASK GENMASK(15, 0)
/* DATA */
#define QDMA_DESC_NEXT_ID_MASK GENMASK(15, 0)
/* TX MSG0 */
@@ -894,7 +1021,9 @@
#define QDMA_ETH_TXMSG_NO_DROP BIT(31)
#define QDMA_ETH_TXMSG_METER_MASK GENMASK(30, 24) /* 0x7f no
meters */
#define QDMA_ETH_TXMSG_FPORT_MASK GENMASK(23, 20)
+#define EN7580_QDMA_ETH_TXMSG_FPORT_MASK GENMASK(23, 21)
#define QDMA_ETH_TXMSG_NBOQ_MASK GENMASK(19, 15)
+#define EN7580_QDMA_ETH_TXMSG_NBOQ_MASK GENMASK(20, 16)
#define QDMA_ETH_TXMSG_HWF_MASK BIT(14)
#define QDMA_ETH_TXMSG_HOP_MASK BIT(13)
#define QDMA_ETH_TXMSG_PTP_MASK BIT(12)
@@ -902,17 +1031,35 @@
#define QDMA_ETH_TXMSG_ACNT_G0_MASK GENMASK(5, 0) /* 0x3f do not
count */
/* RX MSG0 */
+#define EN7523_QDMA_ETH_RXMSG_CRC_ERR_MASK BIT(11)
+#define EN7523_QDMA_ETH_RXMSG_RUNT_MASK BIT(12)
+#define EN7523_QDMA_ETH_RXMSG_LONG_MASK BIT(13)
+#define EN7523_QDMA_ETH_RXMSG_CHAN_MASK GENMASK(7, 3)
+#define EN7523_QDMA_ETH_RXMSG_OAM_MASK BIT(8)
+#define EN7523_QDMA_ETH_RXMSG_GEM_MASK GENMASK(29, 14)
+#define EN7523_QDMA_ETH_RXMSG_NO_MIC_MASK BIT(30)
#define QDMA_ETH_RXMSG_SPTAG GENMASK(21, 14)
+#define EN7580_QDMA_ETH_RXMSG_SPTAG GENMASK(29, 14)
/* RX MSG1 */
#define QDMA_ETH_RXMSG_DEI_MASK BIT(31)
+#define EN7523_QDMA_ETH_RXMSG_DEI_MASK BIT(30)
#define QDMA_ETH_RXMSG_IP6_MASK BIT(30)
+#define EN7523_QDMA_ETH_RXMSG_IP6_MASK BIT(29)
#define QDMA_ETH_RXMSG_IP4_MASK BIT(29)
+#define EN7523_QDMA_ETH_RXMSG_IP4_MASK BIT(28)
#define QDMA_ETH_RXMSG_IP4F_MASK BIT(28)
+#define EN7523_QDMA_ETH_RXMSG_IP4F_MASK BIT(27)
#define QDMA_ETH_RXMSG_L4_VALID_MASK BIT(27)
+#define EN7523_QDMA_ETH_RXMSG_L4_VALID_MASK BIT(26)
#define QDMA_ETH_RXMSG_L4F_MASK BIT(26)
+#define EN7523_QDMA_ETH_RXMSG_L4F_MASK BIT(25)
#define QDMA_ETH_RXMSG_SPORT_MASK GENMASK(25, 21)
+#define EN7523_QDMA_ETH_RXMSG_SPORT_MASK GENMASK(24, 20)
#define QDMA_ETH_RXMSG_CRSN_MASK GENMASK(20, 16)
+#define EN7523_QDMA_ETH_RXMSG_CRSN_MASK GENMASK(19, 15)
#define QDMA_ETH_RXMSG_PPE_ENTRY_MASK GENMASK(15, 0)
+#define EN7523_QDMA_ETH_RXMSG_PPE_ENTRY_MASK GENMASK(14, 0)
+#define EN7523_QDMA_ETH_RXMSG_UDF_MASK GENMASK(30, 25)
/* RX MSG2 */
#define QDMA_ETH_RXMSG_AGG_COUNT_MASK GENMASK(31, 24)
#define QDMA_ETH_RXMSG_L2_LEN_MASK GENMASK(6, 0)
diff --git a/drivers/net/ethernet/airoha/airoha_wed.c
b/drivers/net/ethernet/airoha/airoha_wed.c
new file mode 100644
index 000000000000..009c80f1f401
--- /dev/null
+++ b/drivers/net/ethernet/airoha/airoha_wed.c
@@ -0,0 +1,1682 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Airoha WED v1 (WiFi Ethernet Datapath) driver
+ *
+ * Implements the mtk_wed_ops interface so that standard mt76 WiFi drivers
+ * can offload TX to WED hardware on Airoha/EcoNet SoCs.
+ *
+ * Based on drivers/net/ethernet/mediatek/mtk_wed.c
+ * Copyright (C) 2021 Felix Fietkau <nbd@nbd.name>
+ *
+ * Airoha WED v1 adaptations:
+ * - No PCIe mirror (these SoCs don't support CR mirror HW)
+ * - No hifsys regmap
+ * - WDMA located via physical address from DT instead of MT7622
hardcoded offsets
+ * - AXI bus interface between WED and WDMA
+ * - WED version 1 (TX offload only, no RX capa)
+ */
+
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/bitfield.h>
+#include <linux/dma-mapping.h>
+#include <linux/skbuff.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_platform.h>
+#include <linux/of_address.h>
+#include <linux/debugfs.h>
+#include <linux/io.h>
+#include <linux/soc/mediatek/mtk_wed.h>
+
+#include "airoha_wed.h"
+
+#define MTK_WED_PKT_SIZE 1900
+#define MTK_WED_BUF_SIZE 2048
+#define MTK_WED_TXD_SIZE 128
+#define AIROHA_WDMA_DESC_CTRL_LAST_SEG1 BIT(14)
+#define MTK_WED_BUF_PER_PAGE (PAGE_SIZE / MTK_WED_BUF_SIZE)
+#define MTK_WED_TX_RING_SIZE 2048
+#define MTK_WED_WDMA_RING_SIZE 1024
+#define MTK_WED_WDMA_DUMMY_RING_SIZE 8
+#define MTK_WED_DLY_INT_CFG 0x210
+#define MTK_WED_DLY_INT_VALUE 0xc014c014
+#define MTK_WED_MT7915_RX_DONE_BAND0 BIT(16)
+#define MTK_WED_MT7915_TX_DONE_MASK (BIT(30) | BIT(31))
+#define AIROHA_WED_EXT_INT_MASK_VENDOR 0x2c008003
+#define AIROHA_WED_PCIE_OFST 0x564
+#define AIROHA_WED_WPDMA_OFST0 0x584
+#define AIROHA_WED_WPDMA_OFST1 0x588
+#define AIROHA_WED_PCIE_OFST_VENDOR 0x04200424
+#define AIROHA_PCIE_INT_MASK 0x420
+#define AIROHA_PCIE_MSI_MASK BIT(23)
+#define ECONET_SCU_LAN_SEL 0x70
+#define ECONET_SCU_LAN_SEL_MASK GENMASK(5, 4)
+#define ECONET_SCU_LAN_SEL_ARB FIELD_PREP(ECONET_SCU_LAN_SEL_MASK, 2)
+
+struct airoha_wed_soc_data {
+ phys_addr_t pcie_base[2];
+ u32 wpdma_base[2];
+ bool set_lan_arb;
+};
+
+static struct mtk_wed_hw *hw_list[2];
+static DEFINE_MUTEX(hw_lock);
+
+#define airoha_wed_info(_dev, _fmt, ...) \
+ dev_dbg((_dev)->hw->dev, "wed%d: " _fmt, \
+ (_dev)->hw->index, ##__VA_ARGS__)
+
+#define airoha_wed_info_ratelimited(_dev, _fmt, ...) \
+ dev_dbg_ratelimited((_dev)->hw->dev, "wed%d: " _fmt, \
+ (_dev)->hw->index, ##__VA_ARGS__)
+
+static const struct mtk_wed_soc_data airoha_wed_v1_data = {
+ .regmap = {
+ .tx_bm_tkid = 0x088,
+ .wpdma_rx_ring = { 0x770, },
+ .reset_idx_tx_mask = GENMASK(3, 0),
+ .reset_idx_rx_mask = GENMASK(17, 16),
+ },
+ .tx_ring_desc_size = sizeof(struct mtk_wdma_desc),
+ .wdma_desc_size = sizeof(struct mtk_wdma_desc),
+};
+
+static const struct airoha_wed_soc_data en751221_wed_data = {
+ .pcie_base = { 0x1fb81000, 0x1fb83000 },
+ .wpdma_base = { 0x20000000, 0x20100000 },
+};
+
+static const struct airoha_wed_soc_data en7528_wed_data = {
+ .pcie_base = { 0x1fb81000, 0x1fb83000 },
+ .wpdma_base = { 0x20000000, 0x20100000 },
+ .set_lan_arb = true,
+};
+
+static const struct airoha_wed_soc_data en7523_wed_data = {
+ .pcie_base = { 0x1fa91000, 0x1fa92000 },
+ .wpdma_base = { 0x20000000, 0x20100000 },
+};
+
+static const struct of_device_id airoha_wed_of_match[] = {
+ { .compatible = "econet,en751221-wed", .data = &en751221_wed_data },
+ { .compatible = "econet,en7528-wed", .data = &en7528_wed_data },
+ { .compatible = "airoha,en7523-wed", .data = &en7523_wed_data },
+ { /* sentinel */ }
+};
+
+static void airoha_wed_rebase_wpdma(struct mtk_wed_device *dev)
+{
+ u32 old_base = dev->wlan.phy_base;
+ u32 new_base = dev->hw->wpdma_base;
+
+ dev->wlan.wpdma_int = new_base + dev->wlan.wpdma_int - old_base;
+ dev->wlan.wpdma_mask = new_base + dev->wlan.wpdma_mask - old_base;
+ dev->wlan.wpdma_phys = new_base + dev->wlan.wpdma_phys - old_base;
+ dev->wlan.wpdma_tx = new_base + dev->wlan.wpdma_tx - old_base;
+ dev->wlan.wpdma_txfree = new_base + dev->wlan.wpdma_txfree - old_base;
+ dev->wlan.wpdma_rx_glo = new_base + dev->wlan.wpdma_rx_glo - old_base;
+ dev->wlan.wpdma_rx[0] = new_base + dev->wlan.wpdma_rx[0] - old_base;
+ dev->wlan.phy_base = new_base;
+}
+
+/*
-------------------------------------------------------------------------
+ * Low-level register helpers
+ *
------------------------------------------------------------------------- */
+
+static void
+wed_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val)
+{
+ regmap_update_bits(dev->hw->regs, reg, mask | val, val);
+}
+
+static void
+wed_set(struct mtk_wed_device *dev, u32 reg, u32 mask)
+{
+ wed_m32(dev, reg, 0, mask);
+}
+
+static void
+wed_clr(struct mtk_wed_device *dev, u32 reg, u32 mask)
+{
+ wed_m32(dev, reg, mask, 0);
+}
+
+static void
+wdma_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val)
+{
+ wdma_w32(dev, reg, (wdma_r32(dev, reg) & ~mask) | val);
+}
+
+static void
+wdma_set(struct mtk_wed_device *dev, u32 reg, u32 mask)
+{
+ wdma_m32(dev, reg, 0, mask);
+}
+
+static void
+wlan_w32(struct mtk_wed_device *dev, u32 phys, u32 val)
+{
+ if (!dev->wlan.base || phys < dev->wlan.phy_base)
+ return;
+
+ writel(val, dev->wlan.base + phys - dev->wlan.phy_base);
+}
+
+static u32
+mtk_wed_wlan_irq_mask(struct mtk_wed_device *dev, u32 mask)
+{
+ return mask | MTK_WED_MT7915_RX_DONE_BAND0;
+}
+
+static u32
+mtk_wed_core_irq_mask(struct mtk_wed_device *dev, u32 mask)
+{
+ if (dev->running)
+ return mtk_wed_wlan_irq_mask(dev, mask);
+
+ return mask & ~MTK_WED_MT7915_RX_DONE_BAND0;
+}
+
+static u32
+mtk_wed_ext_irq_mask(struct mtk_wed_device *dev, u32 mask)
+{
+ return mask ? AIROHA_WED_EXT_INT_MASK_VENDOR : 0;
+}
+
+static u32
+mtk_wed_wpdma_irq_mask(struct mtk_wed_device *dev, u32 mask)
+{
+ return mtk_wed_wlan_irq_mask(dev, mask) |
+ MTK_WED_MT7915_TX_DONE_MASK;
+}
+
+static void
+wdma_clr(struct mtk_wed_device *dev, u32 reg, u32 mask)
+{
+ wdma_m32(dev, reg, mask, 0);
+}
+
+static u32
+mtk_wed_read_reset(struct mtk_wed_device *dev)
+{
+ return wed_r32(dev, MTK_WED_RESET);
+}
+
+static u32
+mtk_wdma_read_reset(struct mtk_wed_device *dev)
+{
+ return wdma_r32(dev, MTK_WDMA_GLO_CFG);
+}
+
+/*
-------------------------------------------------------------------------
+ * WDMA reset helpers
+ *
------------------------------------------------------------------------- */
+
+static int
+mtk_wdma_rx_reset(struct mtk_wed_device *dev)
+{
+ u32 status, mask = MTK_WDMA_GLO_CFG_RX_DMA_BUSY;
+ int i, ret;
+
+ airoha_wed_info(dev, "wdma_rx_reset: glo_cfg=%08x int_status=%08x\n",
+ wdma_r32(dev, MTK_WDMA_GLO_CFG),
+ wdma_r32(dev, MTK_WDMA_INT_STATUS));
+
+ wdma_clr(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_RX_DMA_EN);
+ ret = readx_poll_timeout(mtk_wdma_read_reset, dev, status,
+ !(status & mask), 0, 10000);
+ if (ret)
+ dev_err(dev->hw->dev, "rx reset failed\n");
+
+ wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_RX);
+ wdma_w32(dev, MTK_WDMA_RESET_IDX, 0);
+
+ for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++) {
+ if (dev->rx_wdma[i].desc)
+ continue;
+
+ wdma_w32(dev,
+ MTK_WDMA_RING_RX(i) + MTK_WED_RING_OFS_CPU_IDX, 0);
+ }
+
+ airoha_wed_info(dev, "wdma_rx_reset: done ret=%d glo_cfg=%08x
reset_idx=%08x\n",
+ ret, wdma_r32(dev, MTK_WDMA_GLO_CFG),
+ wdma_r32(dev, MTK_WDMA_RESET_IDX));
+
+ return ret;
+}
+
+/*
-------------------------------------------------------------------------
+ * WED reset
+ *
------------------------------------------------------------------------- */
+
+static u32
+mtk_wed_check_busy(struct mtk_wed_device *dev, u32 reg, u32 mask)
+{
+ return !!(wed_r32(dev, reg) & mask);
+}
+
+static int
+mtk_wed_poll_busy(struct mtk_wed_device *dev, u32 reg, u32 mask)
+{
+ int sleep = 15000;
+ int timeout = 100 * sleep;
+ u32 val;
+
+ return read_poll_timeout(mtk_wed_check_busy, val, !val, sleep,
+ timeout, false, dev, reg, mask);
+}
+
+static void
+mtk_wed_reset(struct mtk_wed_device *dev, u32 mask)
+{
+ u32 status;
+
+ airoha_wed_info(dev, "reset: mask=%08x reset_before=%08x\n",
+ mask, wed_r32(dev, MTK_WED_RESET));
+
+ wed_w32(dev, MTK_WED_RESET, mask);
+ if (readx_poll_timeout(mtk_wed_read_reset, dev, status,
+ !(status & mask), 0, 1000)) {
+ dev_err(dev->hw->dev, "wed%d: reset timeout mask=%08x
status=%08x\n",
+ dev->hw->index, mask, status);
+ WARN_ON_ONCE(1);
+ }
+
+ airoha_wed_info(dev, "reset: mask=%08x reset_after=%08x\n",
+ mask, wed_r32(dev, MTK_WED_RESET));
+}
+
+/*
-------------------------------------------------------------------------
+ * FE reset callbacks (called by the Ethernet driver on reset events)
+ *
------------------------------------------------------------------------- */
+
+void airoha_wed_fe_reset(void)
+{
+ int i;
+
+ mutex_lock(&hw_lock);
+
+ for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
+ struct mtk_wed_hw *hw = hw_list[i];
+ struct mtk_wed_device *dev;
+ int err;
+
+ if (!hw)
+ continue;
+
+ dev = hw->wed_dev;
+ if (!dev || !dev->wlan.reset)
+ continue;
+
+ dev_dbg(hw->dev,
+ "wed%d: fe_reset: invoking WLAN reset callback\n",
+ hw->index);
+ err = dev->wlan.reset(dev);
+ if (err)
+ dev_err(dev->dev, "wlan reset failed: %d\n", err);
+ }
+
+ mutex_unlock(&hw_lock);
+}
+
+void airoha_wed_fe_reset_complete(void)
+{
+ int i;
+
+ mutex_lock(&hw_lock);
+
+ for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
+ struct mtk_wed_hw *hw = hw_list[i];
+ struct mtk_wed_device *dev;
+
+ if (!hw)
+ continue;
+
+ dev = hw->wed_dev;
+ if (!dev || !dev->wlan.reset_complete)
+ continue;
+
+ dev_dbg(hw->dev,
+ "wed%d: fe_reset_complete: invoking WLAN callback\n",
+ hw->index);
+ dev->wlan.reset_complete(dev);
+ }
+
+ mutex_unlock(&hw_lock);
+}
+
+/*
-------------------------------------------------------------------------
+ * WED device assignment
+ *
------------------------------------------------------------------------- */
+
+static struct mtk_wed_hw *
+mtk_wed_assign(struct mtk_wed_device *dev)
+{
+ struct mtk_wed_hw *hw;
+
+ /* Match the WED instance to the PCI root port/domain. */
+ if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) {
+ struct device *device = &dev->wlan.pci_dev->dev;
+ struct pci_dev *pdev = dev->wlan.pci_dev;
+ int index = pci_domain_nr(pdev->bus);
+
+ if (pdev->bus->self)
+ index = PCI_SLOT(pdev->bus->self->devfn);
+
+ dev_dbg(device,
+ "wed assign: bus=pcie index=%d wpdma_phys=%08x
token_start=%u nbuf=%u\n",
+ index, dev->wlan.wpdma_phys, dev->wlan.token_start,
+ dev->wlan.nbuf);
+
+ if (index >= ARRAY_SIZE(hw_list)) {
+ dev_dbg(device,
+ "wed assign: index %d is outside hw_list size %zu\n",
+ index, ARRAY_SIZE(hw_list));
+ return NULL;
+ }
+
+ hw = hw_list[index];
+ if (!hw) {
+ dev_dbg(device,
+ "wed assign: no WED hardware registered for index %d\n",
+ index);
+ return NULL;
+ }
+
+ if (hw->wed_dev) {
+ dev_dbg(device,
+ "wed assign: WED%d is already attached\n",
+ hw->index);
+ return NULL;
+ }
+
+ goto out;
+ }
+
+ pr_debug("airoha_wed: assign: unsupported bus type %u\n",
+ dev->wlan.bus_type);
+
+ return NULL;
+
+out:
+ hw->wed_dev = dev;
+ dev_dbg(hw->dev, "wed%d: assigned to WLAN device %s\n",
+ hw->index, dev_name(&dev->wlan.pci_dev->dev));
+
+ return hw;
+}
+
+/*
-------------------------------------------------------------------------
+ * TX buffer management
+ *
------------------------------------------------------------------------- */
+
+static int
+mtk_wed_tx_buffer_alloc(struct mtk_wed_device *dev)
+{
+ u32 desc_size = dev->hw->soc->tx_ring_desc_size;
+ int i, page_idx = 0, n_pages, ring_size;
+ int token = dev->wlan.token_start;
+ struct mtk_wed_buf *page_list;
+ dma_addr_t desc_phys;
+ void *desc_ptr;
+
+ ring_size = dev->wlan.nbuf & ~(MTK_WED_BUF_PER_PAGE - 1);
+ dev->tx_buf_ring.size = ring_size;
+ n_pages = dev->tx_buf_ring.size / MTK_WED_BUF_PER_PAGE;
+
+ airoha_wed_info(dev, "tx_buffer_alloc: nbuf=%u token_start=%u
ring_size=%d n_pages=%d desc_size=%u\n",
+ dev->wlan.nbuf, dev->wlan.token_start, ring_size,
+ n_pages, desc_size);
+
+ page_list = kcalloc(n_pages, sizeof(*page_list), GFP_KERNEL);
+ if (!page_list)
+ return -ENOMEM;
+
+ dev->tx_buf_ring.pages = page_list;
+
+ desc_ptr = dma_alloc_coherent(dev->hw->dev,
+ dev->tx_buf_ring.size * desc_size,
+ &desc_phys, GFP_KERNEL);
+ if (!desc_ptr)
+ return -ENOMEM;
+
+ dev->tx_buf_ring.desc = desc_ptr;
+ dev->tx_buf_ring.desc_phys = desc_phys;
+
+ airoha_wed_info(dev, "tx_buffer_alloc: desc=%p desc_phys=%pad\n",
+ dev->tx_buf_ring.desc, &dev->tx_buf_ring.desc_phys);
+
+ for (i = 0; i < ring_size; i += MTK_WED_BUF_PER_PAGE) {
+ dma_addr_t page_phys, buf_phys;
+ struct page *page;
+ void *buf;
+ int s;
+
+ page = __dev_alloc_page(GFP_KERNEL | GFP_DMA32);
+ if (!page)
+ return -ENOMEM;
+
+ page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(dev->hw->dev, page_phys)) {
+ __free_page(page);
+ return -ENOMEM;
+ }
+
+ page_list[page_idx].p = page;
+ page_list[page_idx++].phy_addr = page_phys;
+ dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+
+ buf = page_to_virt(page);
+ buf_phys = page_phys;
+
+ for (s = 0; s < MTK_WED_BUF_PER_PAGE; s++) {
+ struct mtk_wdma_desc *desc = desc_ptr;
+ u32 txd_size, ctrl;
+
+ desc->buf0 = cpu_to_le32(buf_phys);
+ txd_size = dev->wlan.init_buf(buf, buf_phys, token++);
+ if (WARN_ON_ONCE(txd_size != MTK_WED_TXD_SIZE))
+ return -EINVAL;
+
+ desc->buf1 = cpu_to_le32(buf_phys + MTK_WED_TXD_SIZE);
+ ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0,
+ MTK_WED_TXD_SIZE) |
+ AIROHA_WDMA_DESC_CTRL_LAST_SEG1 |
+ FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1_V2,
+ MTK_WED_PKT_SIZE);
+ desc->ctrl = cpu_to_le32(ctrl);
+ desc->info = 0;
+
+ desc_ptr += desc_size;
+ buf += MTK_WED_BUF_SIZE;
+ buf_phys += MTK_WED_BUF_SIZE;
+ }
+
+ dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+ }
+
+ airoha_wed_info(dev, "tx_buffer_alloc: completed pages=%d
tokens=%u-%u\n",
+ n_pages, dev->wlan.token_start, token - 1);
+
+ return 0;
+}
+
+static void
+mtk_wed_free_tx_buffer(struct mtk_wed_device *dev)
+{
+ struct mtk_wed_buf *page_list = dev->tx_buf_ring.pages;
+ struct mtk_wed_hw *hw = dev->hw;
+ int i, page_idx = 0;
+
+ if (!page_list)
+ return;
+
+ airoha_wed_info(dev, "free_tx_buffer: size=%d desc=%p
desc_phys=%pad\n",
+ dev->tx_buf_ring.size, dev->tx_buf_ring.desc,
+ &dev->tx_buf_ring.desc_phys);
+
+ if (!dev->tx_buf_ring.desc)
+ goto free_pagelist;
+
+ for (i = 0; i < dev->tx_buf_ring.size; i += MTK_WED_BUF_PER_PAGE) {
+ dma_addr_t page_phy = page_list[page_idx].phy_addr;
+ void *page = page_list[page_idx++].p;
+
+ if (!page)
+ break;
+
+ dma_unmap_page(dev->hw->dev, page_phy, PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+ __free_page(page);
+ }
+
+ dma_free_coherent(dev->hw->dev,
+ dev->tx_buf_ring.size * hw->soc->tx_ring_desc_size,
+ dev->tx_buf_ring.desc,
+ dev->tx_buf_ring.desc_phys);
+
+free_pagelist:
+ kfree(page_list);
+}
+
+/*
-------------------------------------------------------------------------
+ * Ring management
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_free_ring(struct mtk_wed_device *dev, struct mtk_wed_ring *ring)
+{
+ if (!ring->desc)
+ return;
+
+ airoha_wed_info(dev, "free_ring: reg_base=%08x size=%d desc_size=%u
desc=%p desc_phys=%pad\n",
+ ring->reg_base, ring->size, ring->desc_size,
+ ring->desc, &ring->desc_phys);
+
+ dma_free_coherent(dev->hw->dev, ring->size * ring->desc_size,
+ ring->desc, ring->desc_phys);
+}
+
+static void
+mtk_wed_free_tx_rings(struct mtk_wed_device *dev)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++)
+ mtk_wed_free_ring(dev, &dev->tx_ring[i]);
+ for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++)
+ mtk_wed_free_ring(dev, &dev->rx_wdma[i]);
+}
+
+/*
-------------------------------------------------------------------------
+ * Interrupt / external interrupt control
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en, u32 irq_mask)
+{
+ u32 mask = irq_mask ?: wed_r32(dev, MTK_WED_INT_MASK);
+
+ wed_w32(dev, MTK_WED_EXT_INT_MASK,
+ en ? mtk_wed_ext_irq_mask(dev, mask) : 0);
+ wed_r32(dev, MTK_WED_EXT_INT_MASK);
+
+ airoha_wed_info(dev, "ext_int: en=%d mask=%08x effective=%08x
status=%08x\n",
+ en, mask, wed_r32(dev, MTK_WED_EXT_INT_MASK),
+ wed_r32(dev, MTK_WED_EXT_INT_STATUS));
+}
+
+/*
-------------------------------------------------------------------------
+ * DMA enable / disable / stop
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_dma_disable(struct mtk_wed_device *dev)
+{
+ airoha_wed_info(dev, "dma_disable: wed_glo=%08x wpdma_glo=%08x
wdma_glo=%08x\n",
+ wed_r32(dev, MTK_WED_GLO_CFG),
+ wed_r32(dev, MTK_WED_WPDMA_GLO_CFG),
+ wdma_r32(dev, MTK_WDMA_GLO_CFG));
+
+ wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
+ MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
+ MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
+
+ wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
+
+ wed_clr(dev, MTK_WED_GLO_CFG,
+ MTK_WED_GLO_CFG_TX_DMA_EN |
+ MTK_WED_GLO_CFG_RX_DMA_EN);
+
+ wdma_clr(dev, MTK_WDMA_GLO_CFG,
+ MTK_WDMA_GLO_CFG_RX_DMA_EN |
+ MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |
+ MTK_WDMA_GLO_CFG_RX_INFO2_PRERES |
+ MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);
+
+ /* Airoha WED v1 has no PCIe mirror register. */
+
+ airoha_wed_info(dev, "dma_disable: done wed_glo=%08x wpdma_glo=%08x
wdma_glo=%08x\n",
+ wed_r32(dev, MTK_WED_GLO_CFG),
+ wed_r32(dev, MTK_WED_WPDMA_GLO_CFG),
+ wdma_r32(dev, MTK_WDMA_GLO_CFG));
+}
+
+static void
+mtk_wed_stop(struct mtk_wed_device *dev)
+{
+ airoha_wed_info(dev, "stop: running=%d irq_mask=%08x
int_status=%08x\n",
+ dev->running, wed_r32(dev, MTK_WED_INT_MASK),
+ wed_r32(dev, MTK_WED_INT_STATUS));
+
+ mtk_wed_dma_disable(dev);
+ mtk_wed_set_ext_int(dev, false, 0);
+
+ wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, 0);
+ wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, 0);
+ wdma_w32(dev, MTK_WDMA_INT_MASK, 0);
+ wdma_w32(dev, MTK_WDMA_INT_GRP2, 0);
+
+ airoha_wed_info(dev, "stop: completed\n");
+}
+
+static void
+mtk_wed_deinit(struct mtk_wed_device *dev)
+{
+ airoha_wed_info(dev, "deinit: ctrl=%08x\n", wed_r32(dev,
MTK_WED_CTRL));
+
+ mtk_wed_stop(dev);
+
+ wed_clr(dev, MTK_WED_CTRL,
+ MTK_WED_CTRL_WDMA_INT_AGENT_EN |
+ MTK_WED_CTRL_WPDMA_INT_AGENT_EN |
+ MTK_WED_CTRL_WED_TX_BM_EN |
+ MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
+}
+
+/*
-------------------------------------------------------------------------
+ * Detach
+ *
------------------------------------------------------------------------- */
+
+static void
+__mtk_wed_detach(struct mtk_wed_device *dev)
+{
+ struct mtk_wed_hw *hw = dev->hw;
+
+ airoha_wed_info(dev, "detach: wlan=%s running=%d init_done=%d\n",
+ dev->wlan.pci_dev ?
dev_name(&dev->wlan.pci_dev->dev) : "none",
+ dev->running, dev->init_done);
+
+ mtk_wed_deinit(dev);
+
+ mtk_wdma_rx_reset(dev);
+ mtk_wed_reset(dev, MTK_WED_RESET_WED);
+ mtk_wed_free_tx_buffer(dev);
+ mtk_wed_free_tx_rings(dev);
+
+ memset(dev, 0, sizeof(*dev));
+ module_put(THIS_MODULE);
+
+ hw->wed_dev = NULL;
+}
+
+static void
+mtk_wed_detach(struct mtk_wed_device *dev)
+{
+ mutex_lock(&hw_lock);
+ __mtk_wed_detach(dev);
+ mutex_unlock(&hw_lock);
+}
+
+/*
-------------------------------------------------------------------------
+ * Early hardware initialisation
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_set_wpdma(struct mtk_wed_device *dev)
+{
+ wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_phys);
+ wed_w32(dev, AIROHA_WED_WPDMA_OFST0, 0x02040200);
+ wed_w32(dev, AIROHA_WED_WPDMA_OFST1, 0x04000300);
+}
+
+static void
+mtk_wed_hw_init_early(struct mtk_wed_device *dev)
+{
+ u32 mask = MTK_WED_WDMA_GLO_CFG_BT_SIZE |
+ MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE |
+ MTK_WED_WDMA_GLO_CFG_DYNAMIC_SKIP_DMAD_PREP |
+ MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY |
+ MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE |
+ MTK_WED_WDMA_GLO_CFG_AXI_W_AFTER_AW |
+ MTK_WED_WDMA_GLO_CFG_WCOMPLETE_SEL;
+ u32 set = FIELD_PREP(MTK_WED_WDMA_GLO_CFG_BT_SIZE, 2) |
+ MTK_WED_WDMA_GLO_CFG_AXI_W_AFTER_AW |
+ MTK_WED_WDMA_GLO_CFG_WCOMPLETE_SEL;
+
+ mtk_wed_deinit(dev);
+ mtk_wed_reset(dev, MTK_WED_RESET_WED);
+ mtk_wed_set_wpdma(dev);
+ wed_m32(dev, MTK_WED_WDMA_GLO_CFG, mask, set);
+
+ wed_w32(dev, MTK_WED_WDMA_CFG_BASE, 0x1fa00000);
+ wed_w32(dev, MTK_WED_WDMA_OFFSET0,
+ dev->hw->index ? 0x66046620 : 0x62046220);
+ wed_w32(dev, MTK_WED_WDMA_OFFSET1,
+ dev->hw->index ? 0x65006400 : 0x61006000);
+
+ wed_w32(dev, MTK_WED_PCIE_CFG_BASE, dev->hw->pcie_base);
+ wed_w32(dev, AIROHA_WED_PCIE_OFST, AIROHA_WED_PCIE_OFST_VENDOR);
+ wed_w32(dev, MTK_WED_PCIE_INT_CTRL,
+ FIELD_PREP(MTK_WED_PCIE_INT_CTRL_POLL_EN, 1));
+
+ /* WDMA GLO_CFG: mark pre-reserved RX info words (v1 path) */
+ wdma_set(dev, MTK_WDMA_GLO_CFG,
+ MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |
+ MTK_WDMA_GLO_CFG_RX_INFO2_PRERES |
+ MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);
+
+}
+
+/*
-------------------------------------------------------------------------
+ * Main hardware initialisation (called on first start/reset)
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_hw_init(struct mtk_wed_device *dev)
+{
+ if (dev->init_done) {
+ airoha_wed_info(dev, "hw_init: already initialized\n");
+ return;
+ }
+
+ airoha_wed_info(dev, "hw_init: tx_bm_desc=%pad tx_bm_size=%d
token_start=%u nbuf=%u\n",
+ &dev->tx_buf_ring.desc_phys, dev->tx_buf_ring.size,
+ dev->wlan.token_start, dev->wlan.nbuf);
+
+ dev->init_done = true;
+ mtk_wed_set_ext_int(dev, false, 0);
+
+ wed_w32(dev, MTK_WED_TX_BM_BASE, dev->tx_buf_ring.desc_phys);
+ wed_w32(dev, MTK_WED_TX_BM_BUF_LEN, MTK_WED_PKT_SIZE);
+
+ wed_w32(dev, MTK_WED_TX_BM_CTRL,
+ MTK_WED_TX_BM_CTRL_PAUSE |
+ FIELD_PREP(MTK_WED_TX_BM_CTRL_VLD_GRP_NUM,
+ dev->tx_buf_ring.size / 128) |
+ FIELD_PREP(MTK_WED_TX_BM_CTRL_RSV_GRP_NUM,
+ MTK_WED_TX_RING_SIZE / 256));
+ wed_w32(dev, MTK_WED_TX_BM_DYN_THR,
+ FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) |
+ MTK_WED_TX_BM_DYN_THR_HI);
+
+ wed_w32(dev, dev->hw->soc->regmap.tx_bm_tkid,
+ FIELD_PREP(MTK_WED_TX_BM_TKID_START, dev->wlan.token_start) |
+ FIELD_PREP(MTK_WED_TX_BM_TKID_END,
+ dev->wlan.token_start + dev->wlan.nbuf - 1));
+
+ mtk_wed_reset(dev, MTK_WED_RESET_TX_BM);
+
+ wed_set(dev, MTK_WED_CTRL,
+ MTK_WED_CTRL_WED_TX_BM_EN |
+ MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
+
+ wed_clr(dev, MTK_WED_TX_BM_CTRL, MTK_WED_TX_BM_CTRL_PAUSE);
+
+ airoha_wed_info(dev, "hw_init: done ctrl=%08x tx_bm_ctrl=%08x
tx_bm_base=%08x tx_bm_tkid=%08x\n",
+ wed_r32(dev, MTK_WED_CTRL),
+ wed_r32(dev, MTK_WED_TX_BM_CTRL),
+ wed_r32(dev, MTK_WED_TX_BM_BASE),
+ wed_r32(dev, dev->hw->soc->regmap.tx_bm_tkid));
+}
+
+/*
-------------------------------------------------------------------------
+ * Ring helpers
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_ring_reset(struct mtk_wed_ring *ring, int size, bool tx)
+{
+ void *head = (void *)ring->desc;
+ int i;
+
+ for (i = 0; i < size; i++) {
+ struct mtk_wdma_desc *desc =
+ (struct mtk_wdma_desc *)(head + i * ring->desc_size);
+
+ desc->buf0 = 0;
+ desc->ctrl = tx ? cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE)
+ : cpu_to_le32(MTK_WFDMA_DESC_CTRL_TO_HOST);
+ desc->buf1 = 0;
+ desc->info = 0;
+ }
+}
+
+static int
+mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring,
+ int size, u32 desc_size, bool tx)
+{
+ ring->desc = dma_alloc_coherent(dev->hw->dev, size * desc_size,
+ &ring->desc_phys, GFP_KERNEL);
+ if (!ring->desc)
+ return -ENOMEM;
+
+ ring->desc_size = desc_size;
+ ring->size = size;
+ mtk_wed_ring_reset(ring, size, tx);
+
+ airoha_wed_info(dev, "ring_alloc: tx=%d size=%d desc_size=%u
desc=%p desc_phys=%pad\n",
+ tx, size, desc_size, ring->desc, &ring->desc_phys);
+
+ return 0;
+}
+
+/*
-------------------------------------------------------------------------
+ * DMA reset
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_reset_dma(struct mtk_wed_device *dev)
+{
+ bool busy = false;
+ u32 val;
+ int i;
+
+ airoha_wed_info(dev, "reset_dma: start init_done=%d running=%d
wed_glo=%08x wpdma_glo=%08x wdma_glo=%08x\n",
+ dev->init_done, dev->running,
+ wed_r32(dev, MTK_WED_GLO_CFG),
+ wed_r32(dev, MTK_WED_WPDMA_GLO_CFG),
+ wdma_r32(dev, MTK_WDMA_GLO_CFG));
+
+ for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) {
+ if (!dev->tx_ring[i].desc)
+ continue;
+
+ mtk_wed_ring_reset(&dev->tx_ring[i], MTK_WED_TX_RING_SIZE, true);
+ }
+
+ /* 1. Reset WED TX DMA */
+ wed_clr(dev, MTK_WED_GLO_CFG, MTK_WED_GLO_CFG_TX_DMA_EN);
+ busy = mtk_wed_poll_busy(dev, MTK_WED_GLO_CFG,
+ MTK_WED_GLO_CFG_TX_DMA_BUSY);
+ if (busy) {
+ mtk_wed_reset(dev, MTK_WED_RESET_WED_TX_DMA);
+ } else {
+ wed_w32(dev, MTK_WED_RESET_IDX,
+ dev->hw->soc->regmap.reset_idx_tx_mask);
+ wed_w32(dev, MTK_WED_RESET_IDX, 0);
+ }
+
+ /* 2. Reset WDMA RX DMA */
+ busy = !!mtk_wdma_rx_reset(dev);
+ wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
+ if (!busy)
+ busy = mtk_wed_poll_busy(dev, MTK_WED_WDMA_GLO_CFG,
+ MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY);
+ if (busy) {
+ mtk_wed_reset(dev, MTK_WED_RESET_WDMA_INT_AGENT);
+ mtk_wed_reset(dev, MTK_WED_RESET_WDMA_RX_DRV);
+ } else {
+ wed_w32(dev, MTK_WED_WDMA_RESET_IDX,
+ MTK_WED_WDMA_RESET_IDX_RX | MTK_WED_WDMA_RESET_IDX_DRV);
+ wed_w32(dev, MTK_WED_WDMA_RESET_IDX, 0);
+
+ wed_set(dev, MTK_WED_WDMA_GLO_CFG,
+ MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE);
+ wed_clr(dev, MTK_WED_WDMA_GLO_CFG,
+ MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE);
+ }
+
+ /* 3. Reset WED TX-free agent */
+ wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
+
+ for (i = 0; i < 100; i++) {
+ val = FIELD_GET(MTK_WED_TX_BM_INTF_TKFIFO_FDEP,
+ wed_r32(dev, MTK_WED_TX_BM_INTF));
+ if (val == 0x40)
+ break;
+ }
+
+ mtk_wed_reset(dev, MTK_WED_RESET_TX_FREE_AGENT);
+ wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_TX_BM_EN);
+ mtk_wed_reset(dev, MTK_WED_RESET_TX_BM);
+
+ /* 4. Reset WED WPDMA TX/RX drivers */
+ busy = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_GLO_CFG,
+ MTK_WED_WPDMA_GLO_CFG_TX_DRV_BUSY);
+ wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
+ MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
+ MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
+ if (!busy)
+ busy = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_GLO_CFG,
+ MTK_WED_WPDMA_GLO_CFG_RX_DRV_BUSY);
+
+ if (busy) {
+ mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_INT_AGENT);
+ mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_TX_DRV);
+ mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_RX_DRV);
+ } else {
+ wed_w32(dev, MTK_WED_WPDMA_RESET_IDX,
+ MTK_WED_WPDMA_RESET_IDX_TX |
+ MTK_WED_WPDMA_RESET_IDX_RX);
+ wed_w32(dev, MTK_WED_WPDMA_RESET_IDX, 0);
+ }
+
+ dev->init_done = false;
+
+ airoha_wed_info(dev, "reset_dma: done wed_glo=%08x wpdma_glo=%08x
wdma_glo=%08x ctrl=%08x\n",
+ wed_r32(dev, MTK_WED_GLO_CFG),
+ wed_r32(dev, MTK_WED_WPDMA_GLO_CFG),
+ wdma_r32(dev, MTK_WDMA_GLO_CFG),
+ wed_r32(dev, MTK_WED_CTRL));
+}
+
+/*
-------------------------------------------------------------------------
+ * WDMA ring setup
+ *
------------------------------------------------------------------------- */
+
+static int
+mtk_wed_wdma_rx_ring_setup(struct mtk_wed_device *dev, int idx, int size,
+ bool reset)
+{
+ struct mtk_wed_ring *wdma;
+
+ if (idx >= ARRAY_SIZE(dev->rx_wdma))
+ return -EINVAL;
+
+ airoha_wed_info(dev, "wdma_rx_ring_setup: idx=%d size=%d reset=%d\n",
+ idx, size, reset);
+
+ wdma = &dev->rx_wdma[idx];
+ if (!reset && mtk_wed_ring_alloc(dev, wdma, size,
+ dev->hw->soc->wdma_desc_size, true))
+ return -ENOMEM;
+
+ wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE,
+ wdma->desc_phys);
+ wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT, size);
+ wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);
+ if (reset)
+ wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_DMA_IDX,
+ 0);
+
+ wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE,
+ wdma->desc_phys);
+ wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT, size);
+ if (reset) {
+ wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) +
+ MTK_WED_RING_OFS_CPU_IDX, 0);
+ wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) +
+ MTK_WED_RING_OFS_DMA_IDX, 0);
+ }
+
+ airoha_wed_info(dev, "wdma_rx_ring_setup: idx=%d desc=%pad
wdma_base=%08x wed_base=%08x count=%d\n",
+ idx, &wdma->desc_phys,
+ wdma_r32(dev, MTK_WDMA_RING_RX(idx) +
MTK_WED_RING_OFS_BASE),
+ wed_r32(dev, MTK_WED_WDMA_RING_RX(idx) +
MTK_WED_RING_OFS_BASE),
+ size);
+
+ return 0;
+}
+
+/*
-------------------------------------------------------------------------
+ * IRQ configuration
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_configure_irq(struct mtk_wed_device *dev, u32 irq_mask)
+{
+ u32 wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0));
+ u32 core_mask = mtk_wed_core_irq_mask(dev, irq_mask);
+ u32 wlan_mask = mtk_wed_wlan_irq_mask(dev, irq_mask);
+ u32 wpdma_mask = mtk_wed_wpdma_irq_mask(dev, irq_mask);
+
+ dev->hw->irq_mask = irq_mask;
+
+ airoha_wed_info(dev, "configure_irq: irq_mask=%08x wdma_mask=%08x
before int_mask=%08x wpdma_mask=%08x\n",
+ irq_mask, wdma_mask, wed_r32(dev, MTK_WED_INT_MASK),
+ wed_r32(dev, MTK_WED_WPDMA_INT_MASK));
+
+ /* Enable WED control agents */
+ wed_set(dev, MTK_WED_CTRL,
+ MTK_WED_CTRL_WDMA_INT_AGENT_EN |
+ MTK_WED_CTRL_WPDMA_INT_AGENT_EN |
+ MTK_WED_CTRL_WED_TX_BM_EN |
+ MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
+
+ /* v1: use PCIe interrupt trigger for WPDMA */
+ wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER,
MTK_WED_PCIE_INT_TRIGGER_STATUS);
+
+ wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER,
+ MTK_WED_WPDMA_INT_TRIGGER_RX_DONE |
+ MTK_WED_WPDMA_INT_TRIGGER_TX_DONE);
+
+ wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask);
+
+ wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, wdma_mask);
+
+ wdma_w32(dev, MTK_WDMA_INT_MASK, wdma_mask);
+ wdma_w32(dev, MTK_WDMA_INT_GRP2, wdma_mask);
+ wed_w32(dev, MTK_WED_WPDMA_INT_MASK, wpdma_mask);
+ wed_w32(dev, MTK_WED_INT_MASK, core_mask);
+ wed_w32(dev, MTK_WED_EXT_INT_MASK,
+ mtk_wed_ext_irq_mask(dev, irq_mask));
+ wed_w32(dev, MTK_WED_DLY_INT_CFG, MTK_WED_DLY_INT_VALUE);
+ wlan_w32(dev, dev->wlan.wpdma_mask, wlan_mask);
+
+ airoha_wed_info(dev, "configure_irq: done ctrl=%08x pcie_trig=%08x
wpdma_trig=%08x wdma_trig=%08x int_mask=%08x wpdma_mask=%08x
wdma_int_mask=%08x wdma_grp2=%08x\n",
+ wed_r32(dev, MTK_WED_CTRL),
+ wed_r32(dev, MTK_WED_PCIE_INT_TRIGGER),
+ wed_r32(dev, MTK_WED_WPDMA_INT_TRIGGER),
+ wed_r32(dev, MTK_WED_WDMA_INT_TRIGGER),
+ wed_r32(dev, MTK_WED_INT_MASK),
+ wed_r32(dev, MTK_WED_WPDMA_INT_MASK),
+ wdma_r32(dev, MTK_WDMA_INT_MASK),
+ wdma_r32(dev, MTK_WDMA_INT_GRP2));
+}
+
+/*
-------------------------------------------------------------------------
+ * DMA enable
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_dma_enable(struct mtk_wed_device *dev)
+{
+ airoha_wed_info(dev, "dma_enable: start wed_glo=%08x wpdma_glo=%08x
wdma_glo=%08x\n",
+ wed_r32(dev, MTK_WED_GLO_CFG),
+ wed_r32(dev, MTK_WED_WPDMA_GLO_CFG),
+ wdma_r32(dev, MTK_WDMA_GLO_CFG));
+
+ wed_set(dev, MTK_WED_WPDMA_INT_CTRL, MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV);
+ wed_set(dev, MTK_WED_WPDMA_GLO_CFG,
+ MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
+ MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
+ wed_set(dev, MTK_WED_WPDMA_CTRL, MTK_WED_WPDMA_CTRL_SDL1_FIXED);
+ wed_set(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
+ wdma_set(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_RX_DMA_EN);
+ wed_set(dev, MTK_WED_GLO_CFG,
+ MTK_WED_GLO_CFG_TX_DMA_EN |
+ MTK_WED_GLO_CFG_RX_DMA_EN);
+
+ airoha_wed_info(dev, "dma_enable: done wed_glo=%08x wpdma_glo=%08x
wpdma_ctrl=%08x wdma_glo=%08x wdma_int_status=%08x\n",
+ wed_r32(dev, MTK_WED_GLO_CFG),
+ wed_r32(dev, MTK_WED_WPDMA_GLO_CFG),
+ wed_r32(dev, MTK_WED_WPDMA_CTRL),
+ wdma_r32(dev, MTK_WDMA_GLO_CFG),
+ wdma_r32(dev, MTK_WDMA_INT_STATUS));
+}
+
+/*
-------------------------------------------------------------------------
+ * Start / attach / ring setup
+ *
------------------------------------------------------------------------- */
+
+static void
+mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask)
+{
+ u32 pcie_int_mask;
+ int i;
+
+ airoha_wed_info(dev, "start: irq_mask=%08x running=%d init_done=%d\n",
+ irq_mask, dev->running, dev->init_done);
+
+ for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++)
+ if (!dev->rx_wdma[i].desc)
+ mtk_wed_wdma_rx_ring_setup(dev, i,
+ i ? MTK_WED_WDMA_DUMMY_RING_SIZE :
+ MTK_WED_WDMA_RING_SIZE,
+ false);
+
+ pcie_int_mask = readl(dev->hw->pcie + AIROHA_PCIE_INT_MASK);
+ pcie_int_mask |= AIROHA_PCIE_MSI_MASK;
+ writel(pcie_int_mask, dev->hw->pcie + AIROHA_PCIE_INT_MASK);
+
+ mtk_wed_hw_init(dev);
+ mtk_wed_configure_irq(dev, irq_mask);
+ mtk_wed_set_ext_int(dev, true, irq_mask);
+
+ /* Airoha WED v1 has no PCIe mirror register. */
+
+ mtk_wed_dma_enable(dev);
+ dev->running = true;
+ wed_w32(dev, MTK_WED_INT_MASK,
+ mtk_wed_core_irq_mask(dev, irq_mask));
+
+ airoha_wed_info(dev, "start: done running=%d int_mask=%08x
int_status=%08x ext_mask=%08x\n",
+ dev->running, wed_r32(dev, MTK_WED_INT_MASK),
+ wed_r32(dev, MTK_WED_INT_STATUS),
+ wed_r32(dev, MTK_WED_EXT_INT_MASK));
+}
+
+static int
+mtk_wed_attach(struct mtk_wed_device *dev)
+ __releases(RCU)
+{
+ struct mtk_wed_hw *hw;
+ struct device *device;
+ int ret = 0;
+
+ RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
+ "mtk_wed_attach without holding the RCU read lock");
+
+ if (dev->wlan.bus_type == MTK_WED_BUS_PCIE)
+ dev_dbg(&dev->wlan.pci_dev->dev,
+ "wed attach: enter bus_type=%u wpdma_phys=%08x
wpdma_tx=%08x wpdma_txfree=%08x token_start=%u nbuf=%u\n",
+ dev->wlan.bus_type, dev->wlan.wpdma_phys, dev->wlan.wpdma_tx,
+ dev->wlan.wpdma_txfree, dev->wlan.token_start, dev->wlan.nbuf);
+ else
+ pr_debug("airoha_wed: attach: unsupported bus_type=%u\n",
+ dev->wlan.bus_type);
+
+ if (dev->wlan.bus_type != MTK_WED_BUS_PCIE ||
+ !try_module_get(THIS_MODULE))
+ ret = -ENODEV;
+
+ rcu_read_unlock();
+
+ if (ret) {
+ if (dev->wlan.bus_type == MTK_WED_BUS_PCIE)
+ dev_dbg(&dev->wlan.pci_dev->dev,
+ "wed attach: rejected ret=%d\n", ret);
+ return ret;
+ }
+
+ mutex_lock(&hw_lock);
+
+ hw = mtk_wed_assign(dev);
+ if (!hw) {
+ module_put(THIS_MODULE);
+ ret = -ENODEV;
+ goto unlock;
+ }
+
+ device = &dev->wlan.pci_dev->dev;
+ dev_dbg(device, "attaching wed device %d\n", hw->index);
+
+ dev->hw = hw;
+ dev->dev = hw->dev;
+ dev->irq = hw->irq;
+ dev->wdma_idx = hw->index;
+ dev->version = hw->version;
+ airoha_wed_rebase_wpdma(dev);
+
+ ret = dma_set_mask_and_coherent(hw->dev, DMA_BIT_MASK(32));
+ dev_dbg(device,
+ "wed attach: dma_set_mask_and_coherent ret=%d dma_dev=%s\n",
+ ret, dev_name(hw->dev));
+ if (ret)
+ goto out;
+
+ ret = mtk_wed_tx_buffer_alloc(dev);
+ dev_dbg(device, "wed attach: tx_buffer_alloc ret=%d\n", ret);
+ if (ret)
+ goto out;
+
+ mtk_wed_hw_init_early(dev);
+ dev_dbg(device,
+ "wed attach: early init complete irq=%d version=%u wdma_idx=%d\n",
+ dev->irq, dev->version, dev->wdma_idx);
+
+out:
+ if (ret) {
+ dev_err(dev->hw->dev, "failed to attach wed device\n");
+ __mtk_wed_detach(dev);
+ }
+unlock:
+ mutex_unlock(&hw_lock);
+
+ return ret;
+}
+
+static void
+airoha_wed_clear_unused_tx_ring(struct mtk_wed_device *dev)
+{
+ u32 wpdma_phys, offset;
+
+ if (dev->tx_ring[1].desc || !dev->wlan.base ||
+ dev->wlan.wpdma_tx < dev->wlan.phy_base)
+ return;
+
+ wpdma_phys = dev->wlan.wpdma_tx +
+ MTK_WED_RING_TX(1) - MTK_WED_RING_TX(0);
+
+ for (offset = 0; offset <= MTK_WED_RING_OFS_DMA_IDX; offset += 4) {
+ wed_w32(dev, MTK_WED_RING_TX(1) + offset, 0);
+ wlan_w32(dev, wpdma_phys + offset, 0);
+ wed_w32(dev, MTK_WED_WPDMA_RING_TX(1) + offset, 0);
+ }
+}
+
+static int
+mtk_wed_tx_ring_setup(struct mtk_wed_device *dev, int idx,
+ void __iomem *regs, bool reset)
+{
+ struct mtk_wed_ring *ring;
+
+ if (WARN_ON(idx < 0 || idx >= ARRAY_SIZE(dev->tx_ring)))
+ return -EINVAL;
+
+ ring = &dev->tx_ring[idx];
+
+ airoha_wed_info(dev, "tx_ring_setup: idx=%d reset=%d wpdma_regs=%p\n",
+ idx, reset, regs);
+
+ if (!reset && mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE,
+ sizeof(*ring->desc), true))
+ return -ENOMEM;
+
+ if (mtk_wed_wdma_rx_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE,
reset))
+ return -ENOMEM;
+
+ ring->reg_base = MTK_WED_RING_TX(idx);
+ ring->wpdma = regs;
+
+ wed_w32(dev, MTK_WED_RING_TX(idx) + MTK_WED_RING_OFS_BASE,
+ ring->desc_phys);
+ wed_w32(dev, MTK_WED_RING_TX(idx) + MTK_WED_RING_OFS_COUNT,
+ MTK_WED_TX_RING_SIZE);
+ wed_w32(dev, MTK_WED_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);
+ wed_w32(dev, MTK_WED_RING_TX(idx) + MTK_WED_RING_OFS_DMA_IDX, 0);
+
+ /* WED → WPDMA */
+ wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys);
+ wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_COUNT, MTK_WED_TX_RING_SIZE);
+ wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_CPU_IDX, 0);
+ wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_DMA_IDX, 0);
+
+ wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE,
+ ring->desc_phys);
+ wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_COUNT,
+ MTK_WED_TX_RING_SIZE);
+ wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);
+ wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_DMA_IDX, 0);
+
+ if (!idx)
+ airoha_wed_clear_unused_tx_ring(dev);
+
+ airoha_wed_info(dev, "tx_ring_setup: idx=%d desc=%pad
wpdma_base=%08x wed_base=%08x count=%u\n",
+ idx, &ring->desc_phys,
+ wpdma_tx_r32(dev, idx, MTK_WED_RING_OFS_BASE),
+ wed_r32(dev, MTK_WED_WPDMA_RING_TX(idx) +
MTK_WED_RING_OFS_BASE),
+ MTK_WED_TX_RING_SIZE);
+
+ return 0;
+}
+
+static int
+mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs)
+{
+ struct mtk_wed_ring *ring = &dev->txfree_ring;
+ /* For v1: txfree ring is at RX ring index 1 */
+ int index = 1;
+ int i;
+
+ ring->reg_base = MTK_WED_RING_RX(index);
+ ring->wpdma = regs;
+
+ airoha_wed_info(dev, "txfree_ring_setup: index=%d wpdma_regs=%p\n",
+ index, regs);
+
+ for (i = 0; i < 12; i += 4) {
+ u32 val = readl(regs + i);
+
+ wed_w32(dev, MTK_WED_RING_RX(index) + i, val);
+ wed_w32(dev, MTK_WED_WPDMA_RING_RX(index) + i, val);
+ }
+
+ airoha_wed_info(dev, "txfree_ring_setup: index=%d base=%08x
count=%08x cpu_idx=%08x wed_base=%08x wed_count=%08x\n",
+ index, wpdma_txfree_r32(dev, MTK_WED_RING_OFS_BASE),
+ wpdma_txfree_r32(dev, MTK_WED_RING_OFS_COUNT),
+ wpdma_txfree_r32(dev, MTK_WED_RING_OFS_CPU_IDX),
+ wed_r32(dev, MTK_WED_WPDMA_RING_RX(index) +
MTK_WED_RING_OFS_BASE),
+ wed_r32(dev, MTK_WED_WPDMA_RING_RX(index) +
MTK_WED_RING_OFS_COUNT));
+
+ return 0;
+}
+
+/*
-------------------------------------------------------------------------
+ * IRQ get / set mask
+ *
------------------------------------------------------------------------- */
+
+static u32
+mtk_wed_irq_get(struct mtk_wed_device *dev, u32 mask)
+{
+ u32 val, ext_mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK;
+ u32 wpdma_mask = mtk_wed_wpdma_irq_mask(dev, mask);
+
+ val = wed_r32(dev, MTK_WED_EXT_INT_STATUS);
+ wed_w32(dev, MTK_WED_EXT_INT_STATUS, val);
+ airoha_wed_info_ratelimited(dev, "irq_get: raw_ext_status=%08x
ext_mask=%08x\n",
+ val, wed_r32(dev, MTK_WED_EXT_INT_MASK));
+ val &= ext_mask;
+ if (!dev->hw->num_flows)
+ val &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;
+ if (val && net_ratelimit())
+ pr_err("airoha_wed%d: error status=%08x\n", dev->hw->index, val);
+
+ val = wed_r32(dev, MTK_WED_INT_STATUS);
+ airoha_wed_info_ratelimited(dev, "irq_get: raw_int_status=%08x
mask=%08x wed_int_mask=%08x wpdma_int_ctrl=%08x wpdma_mask=%08x\n",
+ val, mask, wed_r32(dev, MTK_WED_INT_MASK),
+ wed_r32(dev, MTK_WED_WPDMA_INT_CTRL),
+ wed_r32(dev, MTK_WED_WPDMA_INT_MASK));
+ val &= wpdma_mask;
+ wed_w32(dev, MTK_WED_INT_STATUS, val); /* ACK */
+
+ airoha_wed_info_ratelimited(dev, "irq_get: ack=%08x\n", val);
+
+ return val;
+}
+
+static void
+mtk_wed_irq_set_mask(struct mtk_wed_device *dev, u32 mask)
+{
+ u32 active_mask = mask | dev->hw->irq_mask;
+ u32 core_mask = mask ? mtk_wed_core_irq_mask(dev, active_mask) : 0;
+
+ airoha_wed_info(dev, "irq_set_mask: mask=%08x old_mask=%08x\n",
+ mask, wed_r32(dev, MTK_WED_INT_MASK));
+
+ wed_w32(dev, MTK_WED_INT_MASK, core_mask);
+ wed_w32(dev, MTK_WED_EXT_INT_MASK,
+ mtk_wed_ext_irq_mask(dev, active_mask));
+
+ airoha_wed_info(dev, "irq_set_mask: new_mask=%08x ext_mask=%08x\n",
+ wed_r32(dev, MTK_WED_INT_MASK),
+ wed_r32(dev, MTK_WED_EXT_INT_MASK));
+}
+
+/*
-------------------------------------------------------------------------
+ * Flow offload callbacks (PPE is not exposed by this WED v1 driver)
+ *
------------------------------------------------------------------------- */
+
+int airoha_wed_flow_add(int index)
+{
+ struct mtk_wed_hw *hw;
+ int ret = 0;
+
+ mutex_lock(&hw_lock);
+
+ if (index < 0 || index >= ARRAY_SIZE(hw_list)) {
+ ret = -ENODEV;
+ goto out;
+ }
+
+ pr_debug("airoha_wed%d: flow_add requested\n", index);
+
+ hw = hw_list[index];
+ if (!hw || !hw->wed_dev) {
+ ret = -ENODEV;
+ goto out;
+ }
+
+ if (!hw->wed_dev->wlan.offload_enable)
+ goto out;
+
+ if (hw->num_flows) {
+ hw->num_flows++;
+ goto out;
+ }
+
+ dev_dbg(hw->dev,
+ "wed%d: flow_add: enabling WLAN offload num_flows=%u\n",
+ hw->index, hw->num_flows);
+
+ ret = hw->wed_dev->wlan.offload_enable(hw->wed_dev);
+ if (!ret) {
+ wed_set(hw->wed_dev, MTK_WED_WDMA_GLO_CFG,
+ MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
+ wdma_set(hw->wed_dev, MTK_WDMA_GLO_CFG,
+ MTK_WDMA_GLO_CFG_RX_DMA_EN);
+ wed_set(hw->wed_dev, MTK_WED_CTRL,
+ MTK_WED_CTRL_WDMA_INT_AGENT_EN);
+ wed_set(hw->wed_dev, MTK_WED_GLO_CFG,
+ MTK_WED_GLO_CFG_TX_DMA_EN |
+ MTK_WED_GLO_CFG_RX_DMA_EN);
+ hw->num_flows++;
+ }
+
+ dev_dbg(hw->dev, "wed%d: flow_add: ret=%d num_flows=%u\n",
+ hw->index, ret, hw->num_flows);
+ mtk_wed_set_ext_int(hw->wed_dev, true, 0);
+
+out:
+ mutex_unlock(&hw_lock);
+ return ret;
+}
+
+void airoha_wed_flow_remove(int index)
+{
+ struct mtk_wed_hw *hw;
+
+ mutex_lock(&hw_lock);
+
+ if (index < 0 || index >= ARRAY_SIZE(hw_list))
+ goto out;
+
+ pr_debug("airoha_wed%d: flow_remove requested\n", index);
+
+ hw = hw_list[index];
+ if (!hw || !hw->wed_dev)
+ goto out;
+
+ if (!hw->wed_dev->wlan.offload_disable)
+ goto out;
+
+ if (!hw->num_flows)
+ goto out;
+
+ if (--hw->num_flows)
+ goto out;
+
+ dev_dbg(hw->dev, "wed%d: flow_remove: disabling WLAN offload\n",
+ hw->index);
+ hw->wed_dev->wlan.offload_disable(hw->wed_dev);
+ wdma_clr(hw->wed_dev, MTK_WDMA_GLO_CFG,
+ MTK_WDMA_GLO_CFG_RX_DMA_EN);
+ mtk_wed_set_ext_int(hw->wed_dev, true, 0);
+
+out:
+ mutex_unlock(&hw_lock);
+}
+
+/* This WED v1 driver has no PPE and no TC offload. */
+static void
+mtk_wed_ppe_check(struct mtk_wed_device *dev, struct sk_buff *skb,
+ u32 reason, u32 hash)
+{
+}
+
+static int
+mtk_wed_setup_tc(struct mtk_wed_device *wed, struct net_device *dev,
+ enum tc_setup_type type, void *type_data)
+{
+ return -EOPNOTSUPP;
+}
+
+static int
+mtk_wed_rx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem
*regs,
+ bool reset)
+{
+ airoha_wed_info(dev, "rx_ring_setup: idx=%d regs=%p reset=%d
unsupported on v1\n",
+ idx, regs, reset);
+
+ return 0;
+}
+
+static int mtk_wed_mcu_msg_update(struct mtk_wed_device *dev, int id,
void *data,
+ int len)
+{
+ airoha_wed_info(dev, "mcu_msg_update: id=%d len=%d unsupported on
v1\n",
+ id, len);
+
+ return 0;
+}
+
+static void
+mtk_wed_start_hw_rro(struct mtk_wed_device *dev, u32 irq_mask, bool reset)
+{
+}
+
+static void
+mtk_wed_rro_rx_ring_setup(struct mtk_wed_device *dev, int idx,
+ void __iomem *regs)
+{
+}
+
+static void
+mtk_wed_msdu_pg_rx_ring_setup(struct mtk_wed_device *dev, int idx, void
__iomem *regs)
+{
+}
+
+static int
+mtk_wed_ind_rx_ring_setup(struct mtk_wed_device *dev, void __iomem *regs)
+{
+ return 0;
+}
+
+static const struct mtk_wed_ops airoha_wed_ops = {
+ .attach = mtk_wed_attach,
+ .tx_ring_setup = mtk_wed_tx_ring_setup,
+ .txfree_ring_setup = mtk_wed_txfree_ring_setup,
+ .start = mtk_wed_start,
+ .stop = mtk_wed_stop,
+ .reset_dma = mtk_wed_reset_dma,
+ .reg_read = wed_r32,
+ .reg_write = wed_w32,
+ .irq_get = mtk_wed_irq_get,
+ .irq_set_mask = mtk_wed_irq_set_mask,
+ .detach = mtk_wed_detach,
+ .ppe_check = mtk_wed_ppe_check,
+ .setup_tc = mtk_wed_setup_tc,
+
+ .rx_ring_setup = mtk_wed_rx_ring_setup,
+ .msg_update = mtk_wed_mcu_msg_update,
+ .start_hw_rro = mtk_wed_start_hw_rro,
+ .rro_rx_ring_setup = mtk_wed_rro_rx_ring_setup,
+ .msdu_pg_rx_ring_setup = mtk_wed_msdu_pg_rx_ring_setup,
+ .ind_rx_ring_setup = mtk_wed_ind_rx_ring_setup,
+};
+
+int airoha_wed_add_hw(struct device_node *np, int index)
+{
+ const struct airoha_wed_soc_data *soc;
+ const struct of_device_id *match;
+ struct platform_device *pdev;
+ struct resource res;
+ struct mtk_wed_hw *hw;
+ struct regmap *scu;
+ struct regmap *regs;
+ void __iomem *pcie;
+ void __iomem *wdma;
+ phys_addr_t wdma_phy;
+ int irq, err;
+
+ if (!np)
+ return -EINVAL;
+
+ if (index < 0 || index >= ARRAY_SIZE(hw_list)) {
+ err = -EINVAL;
+ goto err_node_put;
+ }
+
+ match = of_match_node(airoha_wed_of_match, np);
+ if (!match) {
+ err = -EINVAL;
+ goto err_node_put;
+ }
+ soc = match->data;
+
+ pdev = of_find_device_by_node(np);
+ if (!pdev) {
+ err = -EPROBE_DEFER;
+ goto err_node_put;
+ }
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0) {
+ err = irq;
+ goto err_device_put;
+ }
+
+ regs = syscon_regmap_lookup_by_phandle(np, NULL);
+ if (IS_ERR(regs)) {
+ err = PTR_ERR(regs);
+ goto err_device_put;
+ }
+
+ if (soc->set_lan_arb) {
+ scu = syscon_regmap_lookup_by_phandle(np, "airoha,scu");
+ if (IS_ERR(scu)) {
+ err = PTR_ERR(scu);
+ goto err_device_put;
+ }
+
+ err = regmap_update_bits(scu, ECONET_SCU_LAN_SEL,
+ ECONET_SCU_LAN_SEL_MASK,
+ ECONET_SCU_LAN_SEL_ARB);
+ if (err)
+ goto err_device_put;
+ }
+
+ wdma = of_iomap(np, 1);
+ if (!wdma) {
+ err = -ENOMEM;
+ goto err_device_put;
+ }
+
+ err = of_address_to_resource(np, 1, &res);
+ if (err)
+ goto err_iounmap;
+
+ wdma_phy = res.start;
+ pcie = ioremap(soc->pcie_base[index], SZ_4K);
+ if (!pcie) {
+ err = -ENOMEM;
+ goto err_iounmap;
+ }
+
+ hw = kzalloc(sizeof(*hw), GFP_KERNEL);
+ if (!hw) {
+ err = -ENOMEM;
+ goto err_pcie_iounmap;
+ }
+
+ hw->node = np;
+ hw->regs = regs;
+ hw->dev = &pdev->dev;
+ hw->pcie = pcie;
+ hw->pcie_base = soc->pcie_base[index];
+ hw->wpdma_base = soc->wpdma_base[index];
+ hw->wdma_phy = wdma_phy;
+ hw->wdma = wdma;
+ hw->index = index;
+ hw->irq = irq;
+ hw->version = 1;
+ hw->soc = &airoha_wed_v1_data;
+ snprintf(hw->dirname, sizeof(hw->dirname), "wed%d", index);
+
+ mutex_lock(&hw_lock);
+ if (hw_list[index]) {
+ err = -EEXIST;
+ goto err_unlock;
+ }
+
+ hw_list[index] = hw;
+ airoha_wed_hw_add_debugfs(hw);
+ err = mtk_wed_ops_register(&airoha_wed_ops);
+ if (err) {
+ hw_list[index] = NULL;
+ debugfs_remove(hw->debugfs_dir);
+ goto err_unlock;
+ }
+ mutex_unlock(&hw_lock);
+
+ dev_info(hw->dev, "registered WED%d, irq %d, WDMA %pa\n",
+ hw->index, hw->irq, &hw->wdma_phy);
+
+ return 0;
+
+err_unlock:
+ mutex_unlock(&hw_lock);
+ kfree(hw);
+err_pcie_iounmap:
+ iounmap(pcie);
+err_iounmap:
+ iounmap(wdma);
+err_device_put:
+ put_device(&pdev->dev);
+err_node_put:
+ of_node_put(np);
+
+ return err;
+}
+
+void airoha_wed_exit(void)
+{
+ int i;
+
+ mtk_wed_ops_unregister(&airoha_wed_ops);
+
+ mutex_lock(&hw_lock);
+ for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
+ struct mtk_wed_hw *hw = hw_list[i];
+
+ if (!hw)
+ continue;
+
+ WARN_ON_ONCE(hw->wed_dev);
+ hw_list[i] = NULL;
+ debugfs_remove(hw->debugfs_dir);
+ iounmap(hw->pcie);
+ iounmap(hw->wdma);
+ put_device(hw->dev);
+ of_node_put(hw->node);
+ kfree(hw);
+ }
+ mutex_unlock(&hw_lock);
+}
diff --git a/drivers/net/ethernet/airoha/airoha_wed.h
b/drivers/net/ethernet/airoha/airoha_wed.h
new file mode 100644
index 000000000000..47b6a1f7e626
--- /dev/null
+++ b/drivers/net/ethernet/airoha/airoha_wed.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Airoha WED v1 (Wireless Ethernet Dispatch) private interface.
+ */
+
+#ifndef __AIROHA_WED_PRIV_H
+#define __AIROHA_WED_PRIV_H
+
+#if IS_ENABLED(CONFIG_NET_AIROHA_SOC_WED)
+#include "../mtk_wed_private.h"
+#else
+struct mtk_wed_hw;
+#endif
+
+#if IS_ENABLED(CONFIG_NET_AIROHA_SOC_WED)
+int airoha_wed_add_hw(struct device_node *np, int index);
+void airoha_wed_exit(void);
+int airoha_wed_flow_add(int index);
+void airoha_wed_flow_remove(int index);
+void airoha_wed_fe_reset(void);
+void airoha_wed_fe_reset_complete(void);
+#else
+static inline int airoha_wed_add_hw(struct device_node *np, int index)
+{
+ of_node_put(np);
+ return 0;
+}
+
+static inline void airoha_wed_exit(void)
+{
+}
+
+static inline int airoha_wed_flow_add(int index)
+{
+ return -EINVAL;
+}
+
+static inline void airoha_wed_flow_remove(int index)
+{
+}
+
+static inline void airoha_wed_fe_reset(void)
+{
+}
+
+static inline void airoha_wed_fe_reset_complete(void)
+{
+}
+#endif
+
+#ifdef CONFIG_DEBUG_FS
+void airoha_wed_hw_add_debugfs(struct mtk_wed_hw *hw);
+#else
+static inline void airoha_wed_hw_add_debugfs(struct mtk_wed_hw *hw)
+{
+}
+#endif
+
+#endif
diff --git a/drivers/net/ethernet/airoha/airoha_wed_debugfs.c
b/drivers/net/ethernet/airoha/airoha_wed_debugfs.c
new file mode 100644
index 000000000000..be598635967b
--- /dev/null
+++ b/drivers/net/ethernet/airoha/airoha_wed_debugfs.c
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * EN7523 WED debugfs support (v1 subset — TX info only)
+ *
+ * Based on drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
+ * Copyright (C) 2021 Felix Fietkau <nbd@nbd.name>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/io.h>
+#include <linux/seq_file.h>
+
+#include "airoha_wed.h"
+
+struct reg_dump {
+ const char *name;
+ u16 offset;
+ u8 type;
+};
+
+#define DUMP_WED(_reg) { #_reg, MTK_##_reg }
+#define DUMP_END() { }
+
+static const struct reg_dump wed_regs[] = {
+ DUMP_WED(WED_CTRL),
+ DUMP_WED(WED_RESET),
+ DUMP_WED(WED_GLO_CFG),
+ DUMP_WED(WED_INT_STATUS),
+ DUMP_WED(WED_INT_MASK),
+ DUMP_WED(WED_EXT_INT_STATUS),
+ DUMP_WED(WED_EXT_INT_MASK),
+ DUMP_WED(WED_TX_BM_CTRL),
+ DUMP_WED(WED_TX_BM_BASE),
+ DUMP_WED(WED_TX_BM_BUF_LEN),
+ DUMP_WED(WED_TX_BM_DYN_THR),
+ DUMP_WED(WED_TX_BM_INTF),
+ DUMP_WED(WED_WPDMA_CFG_BASE),
+ DUMP_WED(WED_WPDMA_GLO_CFG),
+ DUMP_WED(WED_WPDMA_INT_TRIGGER),
+ DUMP_WED(WED_WPDMA_INT_CTRL),
+ DUMP_WED(WED_WPDMA_INT_MASK),
+ DUMP_WED(WED_WDMA_CFG_BASE),
+ DUMP_WED(WED_WDMA_OFFSET0),
+ DUMP_WED(WED_WDMA_OFFSET1),
+ DUMP_WED(WED_WDMA_GLO_CFG),
+ DUMP_WED(WED_WDMA_INT_TRIGGER),
+ DUMP_WED(WED_WDMA_INT_CTRL),
+ DUMP_WED(WED_PCIE_CFG_BASE),
+ DUMP_WED(WED_PCIE_INT_TRIGGER),
+ DUMP_WED(WED_PCIE_INT_CTRL),
+ DUMP_END()
+};
+
+static void
+dump_wed_regs(struct seq_file *s, struct mtk_wed_device *dev,
+ const struct reg_dump *regs)
+{
+ const struct reg_dump *cur;
+
+ for (cur = regs; cur->name; cur++) {
+ u32 val;
+
+ regmap_read(dev->hw->regs, cur->offset, &val);
+ seq_printf(s, "%-32s %08x\n", cur->name, val);
+ }
+}
+
+static void
+dump_wed_tx_rings(struct seq_file *s, struct mtk_wed_device *dev)
+{
+ int i;
+
+ seq_puts(s, "\nring path base count cpu_idx dma_idx\n");
+ for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) {
+ static const char * const path[] = { "WED_TX", "WED_WPDMA" };
+ u32 reg[] = { MTK_WED_RING_TX(i), MTK_WED_WPDMA_RING_TX(i) };
+ int j;
+
+ for (j = 0; j < ARRAY_SIZE(reg); j++)
+ seq_printf(s, "%4d %-11s 0x%08x %5u %7u %7u\n", i,
+ path[j],
+ wed_r32(dev, reg[j] + MTK_WED_RING_OFS_BASE),
+ wed_r32(dev, reg[j] + MTK_WED_RING_OFS_COUNT),
+ (u32)(wed_r32(dev, reg[j] + MTK_WED_RING_OFS_CPU_IDX) &
+ GENMASK(15, 0)),
+ (u32)(wed_r32(dev, reg[j] + MTK_WED_RING_OFS_DMA_IDX) &
+ GENMASK(15, 0)));
+
+ if (dev->tx_ring[i].wpdma)
+ seq_printf(s, "%4d %-11s 0x%08x %5u %7u %7u\n", i,
+ "WLAN_WPDMA",
+ readl(dev->tx_ring[i].wpdma + MTK_WED_RING_OFS_BASE),
+ readl(dev->tx_ring[i].wpdma + MTK_WED_RING_OFS_COUNT),
+ (u32)(readl(dev->tx_ring[i].wpdma +
+ MTK_WED_RING_OFS_CPU_IDX) & GENMASK(15, 0)),
+ (u32)(readl(dev->tx_ring[i].wpdma +
+ MTK_WED_RING_OFS_DMA_IDX) & GENMASK(15, 0)));
+ }
+}
+
+static int
+wed_txinfo_show(struct seq_file *s, void *data)
+{
+ struct mtk_wed_hw *hw = s->private;
+ struct mtk_wed_device *dev = hw->wed_dev;
+
+ if (!dev)
+ return 0;
+
+ dump_wed_regs(s, dev, wed_regs);
+
+ dump_wed_tx_rings(s, dev);
+
+ return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(wed_txinfo);
+
+static int
+wed_regval_get(void *data, u64 *val)
+{
+ struct mtk_wed_hw *hw = data;
+
+ regmap_read(hw->regs, hw->debugfs_reg, (u32 *)val);
+
+ return 0;
+}
+
+static int
+wed_regval_set(void *data, u64 val)
+{
+ struct mtk_wed_hw *hw = data;
+
+ regmap_write(hw->regs, hw->debugfs_reg, val);
+
+ return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(wed_regval_fops, wed_regval_get, wed_regval_set,
+ "0x%08llx\n");
+
+void airoha_wed_hw_add_debugfs(struct mtk_wed_hw *hw)
+{
+ struct dentry *dir;
+
+ dir = debugfs_create_dir(hw->dirname, NULL);
+ if (IS_ERR(dir))
+ return;
+
+ hw->debugfs_dir = dir;
+ debugfs_create_u32("regidx", 0600, dir, &hw->debugfs_reg);
+ debugfs_create_file_unsafe("regval", 0600, dir, hw,
+ &wed_regval_fops);
+ debugfs_create_file_unsafe("txinfo", 0400, dir, hw,
+ &wed_txinfo_fops);
+}
diff --git a/drivers/net/ethernet/airoha/airoha_xpon.c
b/drivers/net/ethernet/airoha/airoha_xpon.c
new file mode 100644
index 000000000000..73de8ab5a38d
--- /dev/null
+++ b/drivers/net/ethernet/airoha/airoha_xpon.c
@@ -0,0 +1,4263 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Airoha/EcoNet xPON MAC driver
+ *
+ * Unified GPON/EPON driver for the shared EN751221/EN7523 xPON MAC
complex.
+ * The mode-specific protocol engines remain separate inside this file,
+ * while probe/remove, DT matching and FE/GDM2 ownership are shared.
+ *
+ * The EN7523 register layout and bring-up order follow the vendor xpon_1g
+ * implementation:
+ * - shared xPON region at 0x1fb60000
+ * - GPON registers at +0x4000
+ * - EPON registers at +0x6000
+ * - GDM2 is the WAN datapath for both modes
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitmap.h>
+#include <linux/delay.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/export.h>
+#include <linux/interrupt.h>
+#include <linux/i2c.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/netdevice.h>
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/optical_frontend.h>
+#include <linux/phy/phy.h>
+#include <linux/phy/phy-airoha-xpon.h>
+#include <linux/random.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/sched.h>
+#include <linux/sfp.h>
+#include <linux/timer.h>
+#include <linux/unaligned.h>
+#include <linux/workqueue.h>
+#include <net/xpon/omci.h>
+
+#include "airoha_eth.h"
+#include "airoha_xpon.h"
+#include "airoha_gpon_omci.h"
+#include "airoha_regs.h"
+#include "airoha_ploam.h"
+
+/* Linux 6.18 replaced from_timer() with timer_container_of().
+ * Keep the driver buildable on older kernel trees as well.
+ */
+#ifndef timer_container_of
+#define timer_container_of(var, callback_timer, timer_fieldname) \
+ from_timer(var, callback_timer, timer_fieldname)
+#endif
+
+static const u8 airoha_default_vendor_id[4] = {'M', 'T', 'K', 'G'};
+
+static int airoha_xpon_tx_rearm(struct device *dev,
+ struct optical_frontend *frontend)
+{
+ int ret;
+
+ if (!frontend)
+ return 0;
+
+ ret = optical_frontend_tx_rearm(frontend);
+ if (ret == -EOPNOTSUPP)
+ return 0;
+ if (ret)
+ dev_err(dev, "failed to rearm optical transmitter: %d\n", ret);
+ else
+ dev_info(dev, "optical transmitter safety circuit rearmed\n");
+
+ return ret;
+}
+
+static int airoha_xpon_tx_enable(struct device *dev,
+ struct optical_frontend *frontend,
+ bool enable)
+{
+ int ret;
+
+ if (!frontend)
+ return 0;
+
+ ret = optical_frontend_tx_enable(frontend, enable);
+ if (ret == -EOPNOTSUPP)
+ return 0;
+ if (ret)
+ dev_err(dev, "failed to %s optical transmitter: %d\n",
+ enable ? "enable" : "disable", ret);
+
+ return ret;
+}
+
+static const char *airoha_xpon_mode_name(enum airoha_xpon_mode mode)
+{
+ switch (mode) {
+ case AIROHA_XPON_MODE_GPON:
+ return "GPON";
+ case AIROHA_XPON_MODE_EPON:
+ return "EPON";
+ default:
+ return "unknown";
+ }
+}
+
+static enum xpon_mode airoha_xpon_core_mode(enum airoha_xpon_mode mode)
+{
+ switch (mode) {
+ case AIROHA_XPON_MODE_GPON:
+ return XPON_MODE_GPON;
+ case AIROHA_XPON_MODE_EPON:
+ return XPON_MODE_EPON;
+ default:
+ return XPON_MODE_GPON;
+ }
+}
+
+static int airoha_xpon_get_mode(struct device *dev,
+ const struct airoha_xpon_match_data *data,
+ enum airoha_xpon_mode *mode)
+{
+ const char *name;
+
+ *mode = data->mode;
+ if (!data->mode_from_dt)
+ return 0;
+
+ if (device_property_read_string(dev, "airoha,pon-mode", &name))
+ return 0;
+
+ if (!strcmp(name, "gpon"))
+ *mode = AIROHA_XPON_MODE_GPON;
+ else if (!strcmp(name, "epon"))
+ *mode = AIROHA_XPON_MODE_EPON;
+ else
+ return dev_err_probe(dev, -EINVAL,
+ "invalid airoha,pon-mode '%s'\n", name);
+
+ return 0;
+}
+
+static void airoha_xpon_update_netdev_link(struct xpon_priv *priv, bool
link)
+{
+ struct airoha_xpon_link_state state = {
+ .mode = priv->mode,
+ .link = link,
+ .duplex = DUPLEX_FULL,
+ .autoneg = AUTONEG_DISABLE,
+ .port = PORT_FIBRE,
+ };
+
+ switch (priv->mode) {
+ case AIROHA_XPON_MODE_GPON:
+ state.speed = SPEED_2500;
+ state.rx_line_rate_bps = 2488320000ULL;
+ state.tx_line_rate_bps = 1244160000ULL;
+ break;
+ case AIROHA_XPON_MODE_EPON:
+ state.speed = SPEED_1000;
+ state.rx_line_rate_bps = 1000000000ULL;
+ state.tx_line_rate_bps = 1000000000ULL;
+ break;
+ default:
+ return;
+ }
+
+ if (priv->xpon)
+ xpon_device_report_carrier(priv->xpon, link);
+ airoha_eth_xpon_update_link(priv->gdm_dev, &state);
+}
+
+static int airoha_xpon_set_fe_mode(struct device *dev,
+ struct net_device *netdev,
+ enum airoha_xpon_mode mode)
+{
+ int ret;
+
+ dev_info(dev, "configuring %s FE mode on %s\n",
+ airoha_xpon_mode_name(mode), netdev->name);
+ ret = airoha_eth_set_xpon_mode(netdev, mode);
+ if (ret)
+ dev_err(dev, "failed to configure %s FE mode on %s: %d\n",
+ airoha_xpon_mode_name(mode), netdev->name, ret);
+ else
+ dev_info(dev, "%s FE mode configured on %s\n",
+ airoha_xpon_mode_name(mode), netdev->name);
+
+ return ret;
+}
+
+static int airoha_xpon_set_fe_datapath(struct device *dev,
+ struct net_device *netdev,
+ enum airoha_xpon_mode mode,
+ bool enable)
+{
+ int ret;
+
+ dev_info(dev, "%s %s datapath on %s\n",
+ enable ? "enabling" : "disabling",
+ airoha_xpon_mode_name(mode), netdev->name);
+ ret = airoha_eth_set_xpon_datapath(netdev, mode, enable);
+ if (ret)
+ dev_err(dev, "failed to %s %s datapath on %s: %d\n",
+ enable ? "enable" : "disable",
+ airoha_xpon_mode_name(mode), netdev->name, ret);
+ else
+ dev_info(dev, "%s datapath %s on %s\n",
+ airoha_xpon_mode_name(mode),
+ enable ? "enabled" : "disabled", netdev->name);
+
+ return ret;
+}
+
+static int airoha_xpon_select_wan(struct regmap *scu,
+ const struct airoha_xpon_match_data *data,
+ enum airoha_xpon_mode mode)
+{
+ u32 value;
+
+ value = mode == AIROHA_XPON_MODE_GPON ?
+ XPON_SCU_WAN_MODE_GPON : XPON_SCU_WAN_MODE_EPON;
+
+ return regmap_update_bits(scu, XPON_SCU_WAN_CONF,
+ data->wan_mode_mask, value);
+}
+
+static int airoha_xpon_phy_start(struct device *dev, struct phy *phy,
+ enum airoha_xpon_mode mode,
+ bool *initialized, bool *powered)
+{
+ int submode, ret;
+
+ submode = mode == AIROHA_XPON_MODE_GPON ?
+ AIROHA_XPON_PHY_SUBMODE_GPON :
+ AIROHA_XPON_PHY_SUBMODE_EPON;
+
+ dev_info(dev, "initializing %s digital xPON PHY\n",
+ airoha_xpon_mode_name(mode));
+ ret = phy_init(phy);
+ if (ret)
+ return dev_err_probe(dev, ret, "failed to initialize xPON PHY\n");
+ *initialized = true;
+
+ ret = phy_set_mode_ext(phy, PHY_MODE_ETHERNET, submode);
+ if (ret) {
+ dev_err(dev, "failed to select %s PHY mode: %d\n",
+ airoha_xpon_mode_name(mode), ret);
+ goto err_exit;
+ }
+
+ ret = phy_power_on(phy);
+ if (ret) {
+ dev_err(dev, "failed to start %s digital xPON PHY: %d\n",
+ airoha_xpon_mode_name(mode), ret);
+ goto err_exit;
+ }
+ *powered = true;
+
+ /*
+ * Optical synchronization is asynchronous. The generic PHY remains
+ * powered while LOS is asserted and reports/retries PHY_READY itself.
+ */
+ dev_info(dev, "%s digital xPON PHY started\n",
+ airoha_xpon_mode_name(mode));
+ return 0;
+
+err_exit:
+ phy_exit(phy);
+ *initialized = false;
+ return ret;
+}
+
+static void airoha_xpon_phy_stop(struct device *dev, struct phy *phy,
+ enum airoha_xpon_mode mode,
+ bool *initialized, bool *powered)
+{
+ int ret;
+
+ if (*powered) {
+ ret = phy_power_off(phy);
+ if (ret)
+ dev_warn(dev, "failed to power off %s PHY: %d\n",
+ airoha_xpon_mode_name(mode), ret);
+ *powered = false;
+ }
+
+ if (*initialized) {
+ ret = phy_exit(phy);
+ if (ret)
+ dev_warn(dev, "failed to exit %s PHY: %d\n",
+ airoha_xpon_mode_name(mode), ret);
+ *initialized = false;
+ }
+}
+
+/*
-------------------------------------------------------------------------
+ * GPON implementation
+ *
------------------------------------------------------------------------- */
+/* -----------------------------------------------------------------------
+ * Register offsets from the GPON register window at physical 0x1fb64000.
+ *
+ * The vendor driver maps the encompassing region at 0x1fb60000 and the
+ * generated register structure reserves the first 0x4000 bytes. GPON and
+ * EPON live at fixed offsets in that shared xPON MAC region.
+ * -------------------------------------------------------------------- */
+
+/* -----------------------------------------------------------------------
+ * GPON private data
+ * -------------------------------------------------------------------- */
+
+static const char *gpon_state_name(enum gpon_state state)
+{
+ switch (state) {
+ case GPON_O1_INITIAL:
+ return "O1-initial";
+ case GPON_O2_STANDBY:
+ return "O2-standby";
+ case GPON_O3_SERIAL_NUMBER:
+ return "O3-serial-number";
+ case GPON_O4_RANGING:
+ return "O4-ranging";
+ case GPON_O5_OPERATION:
+ return "O5-operation";
+ case GPON_O6_POPUP:
+ return "O6-popup";
+ case GPON_O7_EMERGENCY_STOP:
+ return "O7-emergency-stop";
+ default:
+ return "unknown";
+ }
+}
+
+static int airoha_xpon_frontend_set_mode(struct xpon_priv *priv)
+{
+ struct optical_frontend_mode mode = {};
+
+ if (!priv->frontend)
+ return 0;
+
+ switch (priv->mode) {
+ case AIROHA_XPON_MODE_GPON:
+ mode.protocol = OPTICAL_FRONTEND_PROTO_GPON;
+ mode.tx_rate = 1244160000ULL;
+ mode.rx_rate = 2488320000ULL;
+ mode.flags = OPTICAL_FRONTEND_MODE_BURST_TX;
+ break;
+ case AIROHA_XPON_MODE_EPON:
+ mode.protocol = OPTICAL_FRONTEND_PROTO_EPON;
+ mode.tx_rate = 1250000000ULL;
+ mode.rx_rate = 1250000000ULL;
+ mode.flags = OPTICAL_FRONTEND_MODE_BURST_TX;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return optical_frontend_set_mode(priv->frontend, &mode);
+}
+
+static int airoha_xpon_reset_mac(struct xpon_priv *priv)
+{
+ int ret;
+
+ if (!priv->mac_reset)
+ return 0;
+
+ dev_info(priv->dev, "resetting %s MAC before session start\n",
+ airoha_xpon_mode_name(priv->mode));
+ ret = reset_control_reset(priv->mac_reset);
+ if (ret) {
+ dev_err(priv->dev, "failed to reset xPON MAC: %d\n", ret);
+ return ret;
+ }
+
+ /*
+ * The reset controller supplies the required reset pulse. Give the MAC
+ * register and indirect table engines time to leave reset before any
+ * GPON/EPON register is accessed.
+ */
+ usleep_range(1000, 2000);
+
+ return 0;
+}
+
+static void gpon_refresh_netdev_link(struct xpon_priv *priv, bool force)
+{
+ bool started, o5, omci, service, link, changed;
+
+ mutex_lock(&priv->link_state_lock);
+ started = READ_ONCE(priv->started);
+ o5 = priv->gpon_o5;
+ omci = priv->omci_operational;
+ service = !bitmap_empty(priv->service_gems, GPON_MAX_GEM_ID);
+ link = started && o5 && omci && service;
+ changed = force || priv->netdev_link != link;
+ priv->netdev_link = link;
+ mutex_unlock(&priv->link_state_lock);
+
+ if (!changed)
+ return;
+
+ dev_info(priv->dev,
+ "GPON netdev link %s: O5=%u OMCI=%u service=%u started=%u\n",
+ link ? "ready" : "not-ready", o5, omci, service, started);
+ airoha_xpon_update_netdev_link(priv, link);
+}
+
+/* -----------------------------------------------------------------------
+ * Register accessors
+ * -------------------------------------------------------------------- */
+
+static inline u32 gpon_read(struct xpon_priv *priv, u32 reg)
+{
+ return readl(priv->gpon_reg + reg);
+}
+
+static inline void gpon_write(struct xpon_priv *priv, u32 reg, u32 val)
+{
+ writel(val, priv->gpon_reg + reg);
+}
+
+static void gpon_dump_activation_regs(struct xpon_priv *priv,
+ const char *reason)
+{
+ u32 phy_tx_frames = 0, phy_tx_bursts = 0;
+ int phy_ret;
+
+ phy_ret = airoha_xpon_phy_get_gpon_tx_counters(priv->phy,
+ &phy_tx_frames,
+ &phy_tx_bursts);
+ dev_info(priv->dev,
+ "GPON activation dump (%s): state=%s onu=%#06x act=%#08x
rsp=%#06x pre_delay=%#010x eqd=%#010x sn_cfg=%#010x guard=%#010x
type12=%#010x type3=%#010x dbg_dly=%#010x tx_sync=%#010x
plou_fifo=%#010x int=%#010x/%#010x pending=%#010x rxq=%u/%u
fast_assign=%u phy_tx=%#010x/%#010x phy_ret=%d\n",
+ reason, gpon_state_name(ploam_get_state(priv->ploam)),
+ gpon_read(priv, GPON_ONU_ID),
+ gpon_read(priv, GPON_ACTIVATION_ST),
+ gpon_read(priv, GPON_RSP_TIME),
+ gpon_read(priv, GPON_PRE_ASSIGNED_DLY),
+ gpon_read(priv, GPON_EQD),
+ gpon_read(priv, GPON_SN_MSG_CFG),
+ gpon_read(priv, GPON_PLOu_GUARD_BIT),
+ gpon_read(priv, GPON_PLOu_PRMBL_TYPE1_2),
+ gpon_read(priv, GPON_PLOu_PRMBL_TYPE3),
+ gpon_read(priv, GPON_DBG_DLY),
+ gpon_read(priv, GPON_DBG_TX_SYNC_OFFSET),
+ gpon_read(priv, GPON_PLOAMu_FIFO_STS),
+ gpon_read(priv, GPON_INT_STATUS),
+ gpon_read(priv, GPON_INT_ENABLE),
+ (u32)atomic_read(&priv->pending_irqs),
+ READ_ONCE(priv->ploam_rx_messages),
+ READ_ONCE(priv->ploam_rx_drops),
+ READ_ONCE(priv->assign_onu_fastpath),
+ phy_tx_frames, phy_tx_bursts, phy_ret);
+}
+
+static inline void gpon_set_bits(struct xpon_priv *priv, u32 reg, u32 bits)
+{
+ gpon_write(priv, reg, gpon_read(priv, reg) | bits);
+}
+
+static inline void gpon_clear_bits(struct xpon_priv *priv, u32 reg, u32
bits)
+{
+ gpon_write(priv, reg, gpon_read(priv, reg) & ~bits);
+}
+
+static inline void gpon_rmw(struct xpon_priv *priv, u32 reg, u32 mask,
u32 val)
+{
+ gpon_write(priv, reg, (gpon_read(priv, reg) & ~mask) | val);
+}
+
+/* -----------------------------------------------------------------------
+ * Low-level hardware helpers
+ * -------------------------------------------------------------------- */
+
+static int gpon_wait_bits(struct xpon_priv *priv, u32 reg, u32 mask,
+ unsigned int timeout_us)
+{
+ u32 val;
+
+ return readl_poll_timeout_atomic(priv->gpon_reg + reg, val, val & mask,
+ 1, timeout_us);
+}
+
+static int gpon_set_fe_mode(struct xpon_priv *priv)
+{
+ return airoha_xpon_set_fe_mode(priv->dev, priv->gdm_dev,
+ AIROHA_XPON_MODE_GPON);
+}
+
+static int gpon_set_fe_datapath(struct xpon_priv *priv, bool enable)
+{
+ return airoha_xpon_set_fe_datapath(priv->dev, priv->gdm_dev,
+ AIROHA_XPON_MODE_GPON, enable);
+}
+
+static int gpon_prepare_hardware(struct xpon_priv *priv)
+{
+ u32 mbi;
+ int ret;
+
+ dev_info(priv->dev,
+ "GPON prepare: mbi=%#08x activation=%#08x int_status=%#08x
int_enable=%#08x\n",
+ gpon_read(priv, GPON_MBI_MPI_STOP),
+ gpon_read(priv, GPON_ACTIVATION_ST),
+ gpon_read(priv, GPON_INT_STATUS),
+ gpon_read(priv, GPON_INT_ENABLE));
+
+ /* The EN7523 FE must release GDM2 in GPON mode before the MAC
starts. */
+ ret = gpon_set_fe_mode(priv);
+ if (ret)
+ return ret;
+
+ /* Select GPON instead of EPON on the shared xPON WAN interface. */
+ dev_info(priv->dev, "selecting GPON on SCU WAN mux\n");
+ ret = airoha_xpon_select_wan(priv->scu, priv->match_data,
+ AIROHA_XPON_MODE_GPON);
+ if (ret)
+ return dev_err_probe(priv->dev, ret,
+ "failed to select GPON WAN mode\n");
+
+ /* Match gponDevMbiStop(XPON_DISABLE). The EN757x vendor sequence
+ * releases the GPON/PSE MBI first, waits 1 ms, releases the two GDM2
+ * downstream channels, then waits another 1 ms. Do not collapse these
+ * steps: the GEM indirect command engine is not ready immediately.
+ */
+ dev_info(priv->dev, "releasing GPON RX/TX MBI\n");
+ gpon_clear_bits(priv, GPON_MBI_MPI_STOP, MBI_RX_STOP | MBI_TX_STOP);
+ mdelay(1);
+ dev_info(priv->dev, "GPON MBI after release: %#08x\n",
+ gpon_read(priv, GPON_MBI_MPI_STOP));
+
+ ret = gpon_set_fe_datapath(priv, true);
+ if (ret)
+ return ret;
+ mdelay(1);
+
+ gpon_write(priv, GPON_DBG_GRP_0, ~0U);
+ gpon_write(priv, GPON_DBG_GRP_1, ~0U);
+
+ /*
+ * EN7523 resets the complete 0x4208 delay register and applies two
+ * additional DBA/BWmap defaults. The older EN7521/EN751221 path does
+ * not execute those writes; only its 0x1c fine internal delay is
common
+ * to the configuration data. Keep the generation-specific writes out
+ * of the common MAC path.
+ */
+ if (priv->match_data->en7523_gpon_defaults)
+ gpon_write(priv, GPON_DBG_DLY, DBG_DLY_RESET_DEFAULT);
+ gpon_rmw(priv, GPON_DBG_DLY, DBG_DLY_FINE_INT_MASK,
+ FIELD_PREP(DBG_DLY_FINE_INT_MASK,
+ priv->match_data->gpon_fine_delay));
+ gpon_rmw(priv, GPON_DBG_IDLE_GEM_THLD, GENMASK(15, 0),
+ GPON_IDLE_GEM_THLD_DEFAULT);
+
+ if (priv->match_data->en7523_gpon_defaults) {
+ gpon_rmw(priv, GPON_GBL_CFG, GBL_CFG_SR_BLK_SIZE_MASK,
+ GPON_DBRU_BLOCK_SIZE_48B);
+ gpon_clear_bits(priv, GPON_DBG_BWM_FILTER_CTRL,
+ BWM_FILTER_LEN_VALID_CHECK_EN);
+ }
+
+ mbi = gpon_read(priv, GPON_MBI_MPI_STOP);
+ if (mbi & (MBI_RX_STOP | MBI_TX_STOP))
+ return dev_err_probe(priv->dev, -EIO,
+ "failed to start GPON MBI: %#08x\n", mbi);
+
+ dev_info(priv->dev,
+ "GPON hardware prepared: mbi=%#08x gbl=%#08x dbg_dly=%#08x
idle_gem=%#08x bwm_filter=%#08x\n",
+ mbi, gpon_read(priv, GPON_GBL_CFG),
+ gpon_read(priv, GPON_DBG_DLY),
+ gpon_read(priv, GPON_DBG_IDLE_GEM_THLD),
+ gpon_read(priv, GPON_DBG_BWM_FILTER_CTRL));
+ return 0;
+}
+
+static void gpon_reset_activation_context(struct xpon_priv *priv)
+{
+ u32 sn_cfg;
+
+ gpon_clear_bits(priv, GPON_GBL_CFG, GBL_CFG_US_FEC_EN);
+ gpon_write(priv, GPON_ONU_ID, PLOAM_ONU_UNASSIGNED);
+ gpon_write(priv, GPON_ACTIVATION_ST, GPON_O1_INITIAL);
+ gpon_write(priv, GPON_PRE_ASSIGNED_DLY, 0);
+ gpon_write(priv, GPON_EQD, 0);
+ gpon_write(priv, GPON_RSP_TIME, GPON_RSP_TIME_RESET);
+
+ /*
+ * Preserve the transmitter power mode while restoring the vendor
+ * serial-request threshold and clearing only the random delay. A zero
+ * threshold makes INT_SN_REQ_CRS continuously retrigger.
+ */
+ sn_cfg = gpon_read(priv, GPON_SN_MSG_CFG);
+ sn_cfg &= ~(SN_MSG_CFG_SN_REQ_THR_MASK |
+ SN_MSG_CFG_RANDOM_DELAY_MASK);
+ sn_cfg |= FIELD_PREP(SN_MSG_CFG_SN_REQ_THR_MASK,
+ GPON_SN_REQ_THRESHOLD);
+ gpon_write(priv, GPON_SN_MSG_CFG, sn_cfg);
+ priv->byte_delay = 0;
+ priv->bit_delay = 0;
+}
+
+static int gpon_dev_init(struct xpon_priv *priv)
+{
+ /*
+ * Reset all activation state that can survive an optical or interface
+ * restart. Do not sweep the indirect GEM/T-CONT tables here; the
EN7523
+ * command engine may not acknowledge those accesses during startup.
+ */
+ gpon_reset_activation_context(priv);
+ mutex_lock(&priv->tcont_lock);
+ memset(priv->tcont_alloc_id, 0xff, sizeof(priv->tcont_alloc_id));
+ memset(priv->tcont_entity_id, 0xff, sizeof(priv->tcont_entity_id));
+ mutex_unlock(&priv->tcont_lock);
+ dev_info(priv->dev,
+ "GPON activation reset: onu=%#x state=%#x rsp=%#x pre=%#x
eqd=%#x\n",
+ gpon_read(priv, GPON_ONU_ID),
+ gpon_read(priv, GPON_ACTIVATION_ST),
+ gpon_read(priv, GPON_RSP_TIME),
+ gpon_read(priv, GPON_PRE_ASSIGNED_DLY),
+ gpon_read(priv, GPON_EQD));
+
+ return 0;
+}
+
+static int gpon_load_credentials(struct xpon_priv *priv)
+{
+ struct omci_identity *identity = &priv->identity;
+ const u8 *mac;
+ int ret;
+
+ ret = omci_identity_load(priv->dev, identity);
+ if (ret)
+ return ret;
+
+ if (!(identity->valid & OMCI_IDENTITY_F_SERIAL_NUMBER) &&
+ device_property_read_bool(priv->dev,
+ "airoha,gpon-serial-from-mac")) {
+ if (!(identity->valid & OMCI_IDENTITY_F_VENDOR_ID))
+ return dev_err_probe(priv->dev, -EINVAL,
+ "missing OMCI vendor ID for MAC-derived GPON
serial\n");
+
+ mac = priv->gdm_dev->dev_addr;
+ if (!is_valid_ether_addr(mac))
+ return dev_err_probe(priv->dev, -EINVAL,
+ "invalid GDM2 MAC for GPON serial derivation\n");
+
+ memcpy(identity->serial_number, identity->vendor_id,
+ sizeof(identity->vendor_id));
+ memcpy(identity->serial_number + sizeof(identity->vendor_id),
+ mac + 2, 4);
+ identity->valid |= OMCI_IDENTITY_F_SERIAL_NUMBER;
+ identity->serial_source = OMCI_CONFIG_SOURCE_DRIVER;
+ dev_info(priv->dev,
+ "derived GPON serial %8phN from GDM2 MAC %pM\n",
+ identity->serial_number, mac);
+ }
+
+ if (!(identity->valid & OMCI_IDENTITY_F_SERIAL_NUMBER)) {
+ gpon_random_serial_number(airoha_default_vendor_id, identity);
+ dev_warn(priv->dev,
+ "GPON serial number missing; generated random development
serial %8phN\n",
+ identity->serial_number);
+ }
+ if (!(identity->valid & OMCI_IDENTITY_F_PASSWORD)) {
+ memset(identity->password, 0, sizeof(identity->password));
+ identity->valid |= OMCI_IDENTITY_F_PASSWORD;
+ identity->password_source = OMCI_CONFIG_SOURCE_DEFAULT;
+ }
+
+ memcpy(priv->hw_sn, identity->serial_number, sizeof(priv->hw_sn));
+ memcpy(priv->hw_passwd, identity->password, sizeof(priv->hw_passwd));
+ dev_info(priv->dev,
+ "loaded normalized GPON identity (serial source %u, password
source %u)\n",
+ identity->serial_source, identity->password_source);
+ return 0;
+}
+
+static void gpon_set_serial_number_regs(struct xpon_priv *priv)
+{
+ u32 random_delay, sn_cfg, sn_req_threshold, tx_power_mode;
+ u32 vendor = get_unaligned_be32(priv->hw_sn);
+ u32 vs_sn = get_unaligned_be32(priv->hw_sn + 4);
+
+ /*
+ * Match vendor gponDevSetSerialNumber(): sn[0] occupies bits 31:24
+ * and sn[7] occupies bits 7:0. These registers feed the automatic
+ * Serial_Number_ONU generator directly.
+ */
+ gpon_write(priv, GPON_VENDOR_ID, vendor);
+ gpon_write(priv, GPON_VS_SN, vs_sn);
+
+ sn_cfg = gpon_read(priv, GPON_SN_MSG_CFG);
+ sn_req_threshold = FIELD_GET(SN_MSG_CFG_SN_REQ_THR_MASK, sn_cfg);
+ tx_power_mode = FIELD_GET(SN_MSG_CFG_TX_POWER_MODE_MASK, sn_cfg);
+ random_delay = FIELD_GET(SN_MSG_CFG_RANDOM_DELAY_MASK, sn_cfg);
+
+ dev_info(priv->dev,
+ "GPON hardware SN response: serial=%8phN vendor=%#08x
vs_sn=%#08x readback=%#08x/%#08x cfg=%#08x threshold=%u tx_power=%u
random_delay=%u\n",
+ priv->hw_sn, vendor, vs_sn,
+ gpon_read(priv, GPON_VENDOR_ID),
+ gpon_read(priv, GPON_VS_SN),
+ sn_cfg, sn_req_threshold, tx_power_mode, random_delay);
+}
+
+static void gpon_load_aes_shadow(struct xpon_priv *priv, const u8 key[16],
+ u32 switch_superframe)
+{
+ int i;
+
+ /*
+ * The shadow key registers are laid out with the least significant
+ * word first, so the key bytes go in reversed word order: word 0 takes
+ * key[12..15]. Loading them straight through decrypts every downstream
+ * frame with a byte-swapped key and the whole GEM port fails CRC.
+ */
+ for (i = 0; i < 4; i++)
+ gpon_write(priv, GPON_AES_SHADOW_KEY0 + i * 4,
+ get_unaligned_be32(key + (3 - i) * 4));
+
+ gpon_write(priv, GPON_AES_CFG,
+ switch_superframe & AES_KEY_SWITCH_CNT_MASK);
+}
+
+static int __gpon_set_tcont_hw(struct xpon_priv *priv, unsigned int index,
+ u16 alloc_id, bool valid)
+{
+ if (index < 16) {
+ unsigned int reg_idx = index / 2;
+ u32 val = gpon_read(priv, GPON_TCONT_ID_0_1 + reg_idx * 4);
+
+ if (index & 1) {
+ val &= ~((TCONT_ID_MASK << TCONT1_ID_SHIFT) |
+ TCONT1_VALID);
+ if (valid)
+ val |= ((alloc_id & TCONT_ID_MASK) <<
+ TCONT1_ID_SHIFT) | TCONT1_VALID;
+ } else {
+ val &= ~(TCONT_ID_MASK | TCONT0_VALID);
+ if (valid)
+ val |= (alloc_id & TCONT_ID_MASK) |
+ TCONT0_VALID;
+ }
+ gpon_write(priv, GPON_TCONT_ID_0_1 + reg_idx * 4, val);
+ return 0;
+ }
+
+ if (index < GPON_MAX_TCONT) {
+ u32 cfg = TCONT16_CMD_EXEC |
+ ((index - 16) << TCONT16_IDX_SHIFT);
+
+ if (valid)
+ cfg |= TCONT16_VALID |
+ (alloc_id & TCONT16_ALLOC_MASK);
+ gpon_write(priv, GPON_TCONT_ID_16_31_CFG, cfg);
+ if (gpon_wait_bits(priv, GPON_TCONT_ID_16_31_STS,
+ TCONT16_CMD_DONE, GPON_CMD_TIMEOUT_US))
+ return -ETIMEDOUT;
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static int gpon_config_tcont_hw(struct xpon_priv *priv, unsigned int index,
+ u16 alloc_id, bool valid)
+{
+ int ret;
+
+ if (!priv->gdm_dev)
+ return -ENODEV;
+
+ if (!valid) {
+ ret = airoha_eth_set_xpon_tcont_channel(priv->gdm_dev, index,
+ false);
+ if (ret)
+ return ret;
+ return __gpon_set_tcont_hw(priv, index, alloc_id, false);
+ }
+
+ ret = __gpon_set_tcont_hw(priv, index, alloc_id, true);
+ if (ret)
+ return ret;
+
+ ret = airoha_eth_set_xpon_tcont_channel(priv->gdm_dev, index, true);
+ if (ret)
+ __gpon_set_tcont_hw(priv, index, alloc_id, false);
+
+ return ret;
+}
+
+static int gpon_set_tcont_hw(struct xpon_priv *priv, unsigned int index,
+ u16 alloc_id, bool valid)
+{
+ int ret;
+
+ if (index >= GPON_MAX_TCONT)
+ return -EINVAL;
+
+ mutex_lock(&priv->tcont_lock);
+ ret = gpon_config_tcont_hw(priv, index, alloc_id, valid);
+ if (!ret) {
+ priv->tcont_alloc_id[index] = valid ? alloc_id :
+ GPON_TCONT_UNASSIGNED;
+ if (!valid)
+ priv->tcont_entity_id[index] =
+ GPON_TCONT_ENTITY_UNASSIGNED;
+ }
+ mutex_unlock(&priv->tcont_lock);
+
+ return ret;
+}
+
+static int gpon_find_tcont_alloc_locked(struct xpon_priv *priv, u16
alloc_id)
+{
+ int index;
+
+ for (index = 1; index < GPON_MAX_TCONT; index++)
+ if (priv->tcont_alloc_id[index] == alloc_id)
+ return index;
+
+ return -ENOENT;
+}
+
+static int gpon_find_tcont_entity_locked(struct xpon_priv *priv, u16
entity_id)
+{
+ int index;
+
+ for (index = 1; index < GPON_MAX_TCONT; index++)
+ if (priv->tcont_entity_id[index] == entity_id)
+ return index;
+
+ return -ENOENT;
+}
+
+static int gpon_find_free_tcont_locked(struct xpon_priv *priv)
+{
+ int index;
+
+ for (index = 1; index < GPON_MAX_TCONT; index++)
+ if (priv->tcont_alloc_id[index] == GPON_TCONT_UNASSIGNED)
+ return index;
+
+ return -ENOSPC;
+}
+
+static int gpon_tcont_entity_to_index(struct xpon_priv *priv, u16
entity_id,
+ unsigned int *index)
+{
+ int found;
+
+ mutex_lock(&priv->tcont_lock);
+ found = gpon_find_tcont_entity_locked(priv, entity_id);
+ if (found < 0 && entity_id >= 0x8000 &&
+ entity_id < 0x8000 + GPON_MAX_TCONT - 1) {
+ unsigned int fallback = (entity_id - 0x8000) + 1;
+
+ if (fallback < GPON_MAX_TCONT &&
+ priv->tcont_alloc_id[fallback] != GPON_TCONT_UNASSIGNED)
+ found = fallback;
+ }
+ if (found >= 0)
+ *index = found;
+ mutex_unlock(&priv->tcont_lock);
+
+ return found < 0 ? found : 0;
+}
+
+static int gpon_read_gem_port_hw(struct xpon_priv *priv, u16 gem_port_id,
+ bool *valid, bool *encrypted)
+{
+ u32 status;
+
+ gpon_write(priv, GPON_GEM_PORT_CFG, gem_port_id);
+ /* Let the command engine clear the completion bit from the previous
+ * transaction before polling it again.
+ */
+ udelay(1);
+ if (gpon_wait_bits(priv, GPON_GEM_PORT_STS, GEM_CMD_DONE,
+ GPON_CMD_TIMEOUT_US))
+ return -ETIMEDOUT;
+
+ status = gpon_read(priv, GPON_GEM_PORT_STS);
+ *valid = !!(status & GEM_STS_VALID);
+ *encrypted = !!(status & GEM_STS_ENCRYPT);
+
+ return 0;
+}
+
+static int gpon_set_gem_port_hw(struct xpon_priv *priv, u16 gem_port_id,
+ bool valid, bool encrypted)
+{
+ bool read_encrypted, read_valid;
+ u32 cfg;
+ int ret, retry;
+
+ if (gem_port_id >= GPON_MAX_GEM_ID)
+ return -EINVAL;
+
+ if (gem_port_id == 0)
+ encrypted = false;
+
+ cfg = GEM_CMD_WRITE | gem_port_id;
+ if (valid)
+ cfg |= GEM_VALID;
+ if (valid && encrypted)
+ cfg |= GEM_ENCRYPT;
+
+ for (retry = 0; retry < 3; retry++) {
+ gpon_write(priv, GPON_GEM_PORT_CFG, cfg);
+ /* G_GEM_PORT_STS may still contain the completion state of the
+ * preceding command on the first bus read.
+ */
+ udelay(1);
+ ret = gpon_wait_bits(priv, GPON_GEM_PORT_STS, GEM_CMD_DONE,
+ GPON_CMD_TIMEOUT_US);
+ if (ret)
+ continue;
+
+ ret = gpon_read_gem_port_hw(priv, gem_port_id,
+ &read_valid, &read_encrypted);
+ if (!ret && read_valid == valid &&
+ read_encrypted == (valid && encrypted))
+ return 0;
+
+ usleep_range(10, 20);
+ }
+
+ dev_err(priv->dev,
+ "GPON GEM port %u configuration failed: valid=%u encrypted=%u
ret=%d\n",
+ gem_port_id, valid, valid && encrypted, ret);
+
+ return ret ?: -EIO;
+}
+
+static int gpon_set_alloc_id_hw(struct xpon_priv *priv, u16 alloc_id,
+ bool allocate)
+{
+ int index, ret = 0;
+
+ if (alloc_id > TCONT_ID_MASK)
+ return -EINVAL;
+
+ mutex_lock(&priv->tcont_lock);
+ index = gpon_find_tcont_alloc_locked(priv, alloc_id);
+ if (index >= 0) {
+ if (!allocate) {
+ ret = gpon_config_tcont_hw(priv, index, alloc_id, false);
+ if (!ret) {
+ priv->tcont_alloc_id[index] = GPON_TCONT_UNASSIGNED;
+ priv->tcont_entity_id[index] =
+ GPON_TCONT_ENTITY_UNASSIGNED;
+ }
+ }
+ goto out;
+ }
+
+ if (!allocate)
+ goto out;
+
+ index = gpon_find_free_tcont_locked(priv);
+ if (index < 0) {
+ ret = index;
+ goto out;
+ }
+
+ ret = gpon_config_tcont_hw(priv, index, alloc_id, true);
+ if (!ret) {
+ priv->tcont_alloc_id[index] = alloc_id;
+ if (priv->tcont_entity_id[index] == GPON_TCONT_ENTITY_UNASSIGNED)
+ priv->tcont_entity_id[index] = 0x8000 + index - 1;
+ }
+out:
+ mutex_unlock(&priv->tcont_lock);
+ return ret;
+}
+
+static int gpon_set_omci_tcont_hw(struct xpon_priv *priv, u16 entity_id,
+ u16 alloc_id, bool valid,
+ unsigned int *channel)
+{
+ int old_index, index, ret = 0;
+
+ if (valid && alloc_id > TCONT_ID_MASK)
+ return -EINVAL;
+
+ mutex_lock(&priv->tcont_lock);
+ old_index = gpon_find_tcont_entity_locked(priv, entity_id);
+ if (!valid) {
+ if (old_index < 0)
+ goto out;
+
+ alloc_id = priv->tcont_alloc_id[old_index];
+ ret = gpon_config_tcont_hw(priv, old_index, alloc_id, false);
+ if (!ret) {
+ priv->tcont_alloc_id[old_index] = GPON_TCONT_UNASSIGNED;
+ priv->tcont_entity_id[old_index] =
+ GPON_TCONT_ENTITY_UNASSIGNED;
+ *channel = old_index;
+ }
+ goto out;
+ }
+
+ index = gpon_find_tcont_alloc_locked(priv, alloc_id);
+ if (index < 0) {
+ index = gpon_find_free_tcont_locked(priv);
+ if (index < 0) {
+ ret = index;
+ goto out;
+ }
+
+ ret = gpon_config_tcont_hw(priv, index, alloc_id, true);
+ if (ret)
+ goto out;
+ priv->tcont_alloc_id[index] = alloc_id;
+ }
+
+ if (old_index >= 0 && old_index != index)
+ priv->tcont_entity_id[old_index] =
+ GPON_TCONT_ENTITY_UNASSIGNED;
+ priv->tcont_entity_id[index] = entity_id;
+ *channel = index;
+out:
+ mutex_unlock(&priv->tcont_lock);
+ return ret;
+}
+
+/* -----------------------------------------------------------------------
+ * Hardware-side PLOAM FIFO access
+ * -------------------------------------------------------------------- */
+
+static inline u32 gpon_ploam_read_word(struct xpon_priv *priv)
+{
+ /*
+ * The GPON FIFO exposes each PLOAM word directly in protocol order:
+ * bits 31:24 contain ONU-ID and bits 23:16 contain the message type.
+ * This is a register value, not a byte array in CPU memory, so
applying
+ * be32_to_cpu() swaps valid messages on little-endian EN7523 systems.
+ */
+ return gpon_read(priv, GPON_PLOAMd_RDATA);
+}
+
+static inline void gpon_ploam_write_word(struct xpon_priv *priv, u32 val)
+{
+ /* Keep the same register/protocol ordering for upstream messages. */
+ gpon_write(priv, GPON_PLOAMu_WDATA, val);
+}
+
+static int gpon_wait_ploam_tx_space(struct xpon_priv *priv, u32 *available)
+{
+ u32 status;
+ int ret;
+
+ ret = readl_poll_timeout_atomic(priv->gpon_reg + GPON_PLOAMu_FIFO_STS,
+ status,
+ (status & PLOAMu_FIFO_AVAIL_MASK) >=
+ PLOAM_WORDS, 1,
+ GPON_PLOAM_TX_TIMEOUT_US);
+ *available = status & PLOAMu_FIFO_AVAIL_MASK;
+
+ return ret;
+}
+
+static void gpon_hw_send_ploam(struct xpon_priv *priv,
+ const struct ploam_msg *msg, int times)
+{
+ u8 onu_id = msg->value[0] >> 24;
+ u8 type = msg->value[0] >> 16;
+ bool ready, los;
+ int ret, t;
+
+ if (type != PLOAM_UP_DYING_GASP) {
+ ret = airoha_xpon_phy_get_link_state(priv->phy, &ready, &los);
+ if (!ret && (!ready || los)) {
+ dev_dbg_ratelimited(priv->dev,
+ "dropping PLOAM type %#04x while PHY ready=%u
LOS=%u\n",
+ type, ready, los);
+ return;
+ }
+ }
+
+ if (type != PLOAM_UP_REI)
+ dev_info(priv->dev,
+ "PLOAM TX: onu=%u type=%#04x copies=%d
words=%08x/%08x/%08x\n",
+ onu_id, type, times, msg->value[0], msg->value[1],
msg->value[2]);
+
+ for (t = 0; t < times; t++) {
+ u32 avail;
+ int ret;
+
+ ret = gpon_wait_ploam_tx_space(priv, &avail);
+ if (ret) {
+ dev_warn_ratelimited(priv->dev,
+ "PLOAM TX FIFO timeout: avail=%u
requested_words=%u copy=%d/%d\n",
+ avail, PLOAM_WORDS, t + 1, times);
+ break;
+ }
+
+ gpon_ploam_write_word(priv, msg->value[0]);
+ gpon_ploam_write_word(priv, msg->value[1]);
+ gpon_ploam_write_word(priv, msg->value[2]);
+ }
+}
+
+static void gpon_ploam_rx_queue_reset(struct xpon_priv *priv)
+{
+ WRITE_ONCE(priv->ploam_rx_head, 0);
+ WRITE_ONCE(priv->ploam_rx_tail, 0);
+}
+
+static bool gpon_ploam_rx_queue_push(struct xpon_priv *priv,
+ const struct ploam_msg *msg)
+{
+ u16 head = READ_ONCE(priv->ploam_rx_head);
+ u16 next = (head + 1) & GPON_PLOAM_RX_QUEUE_MASK;
+
+ /* Pairs with the consumer release when it advances the tail. */
+ if (next == smp_load_acquire(&priv->ploam_rx_tail)) {
+ priv->ploam_rx_drops++;
+ return false;
+ }
+
+ priv->ploam_rx_queue[head] = *msg;
+ /* Publish the message before making the new head visible. */
+ smp_store_release(&priv->ploam_rx_head, next);
+ priv->ploam_rx_messages++;
+
+ return true;
+}
+
+static bool gpon_ploam_rx_queue_pop(struct xpon_priv *priv,
+ struct ploam_msg *msg)
+{
+ u16 tail = READ_ONCE(priv->ploam_rx_tail);
+
+ /* Pairs with the producer release after storing a message. */
+ if (tail == smp_load_acquire(&priv->ploam_rx_head))
+ return false;
+
+ *msg = priv->ploam_rx_queue[tail];
+ /* Finish reading the slot before allowing the producer to reuse it. */
+ smp_store_release(&priv->ploam_rx_tail,
+ (tail + 1) & GPON_PLOAM_RX_QUEUE_MASK);
+
+ return true;
+}
+
+static void gpon_fastpath_assign_onu_id(struct xpon_priv *priv,
+ const struct ploam_msg *msg)
+{
+ u8 msg_sn[8];
+ u8 onu_id;
+
+ if ((msg->value[0] >> 24) != PLOAM_ONU_BCAST ||
+ ((msg->value[0] >> 16) & 0xff) != PLOAM_DOWN_ASSIGN_ONU_ID)
+ return;
+
+ if ((gpon_read(priv, GPON_ACTIVATION_ST) & 0x7) !=
+ GPON_O3_SERIAL_NUMBER)
+ return;
+
+ msg_sn[0] = msg->value[0];
+ msg_sn[1] = msg->value[1] >> 24;
+ msg_sn[2] = msg->value[1] >> 16;
+ msg_sn[3] = msg->value[1] >> 8;
+ msg_sn[4] = msg->value[1];
+ msg_sn[5] = msg->value[2] >> 24;
+ msg_sn[6] = msg->value[2] >> 16;
+ msg_sn[7] = msg->value[2] >> 8;
+ if (memcmp(msg_sn, priv->hw_sn, sizeof(msg_sn)))
+ return;
+
+ onu_id = (msg->value[0] >> 8) & ONU_ID_MASK;
+
+ /*
+ * Match the vendor ISR ordering. The OLT may issue the first ranging
+ * allocation immediately after Assign_ONU-ID, so both registers must
+ * be visible before the deferred protocol state machine runs.
+ */
+ gpon_write(priv, GPON_ACTIVATION_ST, GPON_O4_RANGING);
+ /* Make O4 visible before validating the assigned ONU-ID. */
+ wmb();
+ gpon_write(priv, GPON_ONU_ID, ONU_ID_VLD | onu_id);
+ priv->assign_onu_fastpath++;
+}
+
+static void gpon_drain_ploam_fifo_irq(struct xpon_priv *priv)
+{
+ int budget = GPON_PLOAM_RX_QUEUE_LEN - 1;
+
+ while (budget--) {
+ struct ploam_msg msg;
+ u32 depth;
+
+ depth = gpon_read(priv, GPON_PLOAMd_FIFO_STS) &
+ PLOAMd_FIFO_USED_MASK;
+ if (depth < PLOAM_WORDS)
+ break;
+
+ msg.value[0] = gpon_ploam_read_word(priv);
+ msg.value[1] = gpon_ploam_read_word(priv);
+ msg.value[2] = gpon_ploam_read_word(priv);
+
+ gpon_fastpath_assign_onu_id(priv, &msg);
+ gpon_ploam_rx_queue_push(priv, &msg);
+ }
+}
+
+static void gpon_process_ploam_queue(struct xpon_priv *priv)
+{
+ struct ploam_msg msg;
+
+ while (gpon_ploam_rx_queue_pop(priv, &msg)) {
+ dev_dbg(priv->dev,
+ "PLOAM RX: onu=%u type=%#04x words=%08x/%08x/%08x\n",
+ msg.value[0] >> 24, (msg.value[0] >> 16) & 0xff,
+ msg.value[0], msg.value[1], msg.value[2]);
+ ploam_handle_downstream(priv->ploam, &msg);
+ }
+}
+
+/* -----------------------------------------------------------------------
+ * ploam_ops callbacks
+ * -------------------------------------------------------------------- */
+
+static void gpon_cb_send_upstream(void *hw_priv, const struct ploam_msg
*msg,
+ int times)
+{
+ gpon_hw_send_ploam(hw_priv, msg, times);
+}
+
+static void gpon_cb_set_onu_id(void *hw_priv, u8 onu_id)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ dev_info(priv->dev, "GPON assigned ONU-ID %u\n", onu_id);
+ gpon_write(priv, GPON_ONU_ID, ONU_ID_VLD | (onu_id & ONU_ID_MASK));
+ airoha_gpon_omci_set_onu_id(&priv->omci, onu_id);
+ gpon_dump_activation_regs(priv, "ONU-ID assigned");
+}
+
+/*
+ * EqD is split across the two blocks: the byte-aligned part goes to
the MAC
+ * in G_EQD and the remaining 0-7 bits are a transmitter delay in the PHY.
+ * Program both, or every upstream burst sits up to 7 bit times away
from the
+ * position the OLT ranged.
+ */
+static void gpon_set_bit_delay(struct xpon_priv *priv, u32 bit_delay)
+{
+ int ret;
+
+ ret = airoha_xpon_phy_set_gpon_bit_delay(priv->phy, bit_delay & 7);
+ if (ret)
+ dev_warn(priv->dev,
+ "failed to program GPON PHY bit delay %u: %d\n",
+ bit_delay & 7, ret);
+}
+
+static void gpon_cb_set_eqd_o4(void *hw_priv, u32 byte_delay, u32
bit_delay)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ dev_info(priv->dev, "GPON O4 EqD: byte_delay=%u bit_delay=%u\n",
+ byte_delay, bit_delay);
+ priv->byte_delay = byte_delay;
+ priv->bit_delay = bit_delay;
+ gpon_write(priv, GPON_EQD, byte_delay);
+ gpon_set_bit_delay(priv, bit_delay);
+ gpon_dump_activation_regs(priv, "O4 EqD programmed");
+}
+
+static void gpon_cb_adjust_eqd_o5(void *hw_priv, u32 new_eqd)
+{
+ struct xpon_priv *priv = hw_priv;
+ u32 curr_eqd = ploam_get_eqd(priv->ploam);
+ int delta = (int)new_eqd - (int)curr_eqd;
+ u32 sync_raw;
+ int int_byte_delay;
+ int K, A, B, a;
+
+ if (delta == 0) {
+ dev_info(priv->dev, "GPON O5 EqD unchanged: %u\n", new_eqd);
+ return;
+ }
+
+ dev_info(priv->dev,
+ "GPON O5 EqD adjustment: current=%u new=%u delta=%d\n",
+ curr_eqd, new_eqd, delta);
+
+ /* Read internal byte delay from hardware (EN7521 specific register) */
+ sync_raw = gpon_read(priv, GPON_DBG_TX_SYNC_OFFSET);
+ int_byte_delay = sync_raw & DBG_TX_SYNC_OFFSET_MASK;
+
+ if (delta > 0) {
+ K = delta + (int)priv->bit_delay;
+ A = K >> 3;
+ B = K & 7;
+ a = (((int_byte_delay + (A & 3)) <= 3) ? 0 : 8) +
+ (A - (A & 3)) - (A & 3);
+ priv->byte_delay += (u32)(a << 3);
+ priv->bit_delay = (u32)B;
+ } else {
+ K = -delta - (int)priv->bit_delay;
+ A = (K > 0) ? ((K >> 3) + ((K & 7) ? 1 : 0)) : 0;
+ B = (A << 3) - K;
+ a = ((int_byte_delay >= (A & 3)) ? 0 : 8) +
+ (A - (A & 3)) - (A & 3);
+ priv->byte_delay -= (u32)(a << 3);
+ priv->bit_delay = (u32)B;
+ }
+
+ gpon_write(priv, GPON_EQD, priv->byte_delay);
+ gpon_set_bit_delay(priv, priv->bit_delay);
+ dev_info(priv->dev,
+ "GPON O5 EqD programmed: byte_delay=%u bit_delay=%u sync=%#08x\n",
+ priv->byte_delay, priv->bit_delay, sync_raw);
+}
+
+static void gpon_cb_enable_us_fec(void *hw_priv)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ dev_info(priv->dev, "enabling GPON upstream FEC\n");
+ gpon_set_bits(priv, GPON_GBL_CFG, GBL_CFG_US_FEC_EN);
+}
+
+static void gpon_cb_set_overhead(void *hw_priv,
+ u8 guard_bits, u8 t1_pbits, u8 t2_pbits,
+ u8 t3_pbits, const u8 delim[3],
+ bool delay_mode, u16 delay_time)
+{
+ struct xpon_priv *priv = hw_priv;
+ u32 prmbl, pre_dly;
+ int ret;
+
+ dev_info(priv->dev,
+ "GPON overhead: OLT guard=%u fallback guard=%u t1=%u t2=%u
t3=%u delay_mode=%u delay=%u delim=%02x:%02x:%02x\n",
+ guard_bits, GPON_PHY_GUARD_BIT_NUM, t1_pbits, t2_pbits,
+ t3_pbits, delay_mode, delay_time,
+ delim[0], delim[1], delim[2]);
+
+ /*
+ * Program the guard the OLT asked for. Overriding it with a value
+ * taken from another board's firmware leaves every ranged burst
+ * misaligned with the window the OLT reserved for this ONU, so the
+ * burst is transmitted and never received. Map PLOAM T2 to PHY T1 and
+ * PLOAM T1 to PHY T2, and preserve explicit zero values.
+ */
+ if (!guard_bits)
+ guard_bits = GPON_PHY_GUARD_BIT_NUM;
+
+ ret = airoha_xpon_phy_set_gpon_overhead(priv->phy,
+ guard_bits,
+ t1_pbits, t2_pbits,
+ t3_pbits, delim);
+ if (ret)
+ dev_warn(priv->dev,
+ "failed to program GPON PHY overhead: %d\n", ret);
+
+ gpon_write(priv, GPON_PLOu_GUARD_BIT, guard_bits);
+
+ /* G_PLOu_PRMBL_TYPE1_2: t1 in upper 16 bits, t2 in lower 16 bits */
+ prmbl = ((u32)t1_pbits << 16) | t2_pbits;
+ gpon_write(priv, GPON_PLOu_PRMBL_TYPE1_2, prmbl);
+
+ /* G_PRE_ASSIGNED_DLY */
+ pre_dly = (delay_mode ? PRE_DLY_EN : 0) | (delay_time & PRE_DLY_MASK);
+ gpon_write(priv, GPON_PRE_ASSIGNED_DLY, pre_dly);
+
+ /*
+ * G_PLOu_OVERHEAD and G_PLOu_DELM_BIT are currently undocumented in
+ * this driver. Dump the complete burst-generator window before
+ * assigning layouts to those registers.
+ */
+ dev_info(priv->dev,
+ "GPON MAC burst regs: overhead=%#010x guard=%#010x
type12=%#010x type3=%#010x delimiter=%#010x pre_delay=%#010x\n",
+ gpon_read(priv, GPON_PLOu_OVERHEAD),
+ gpon_read(priv, GPON_PLOu_GUARD_BIT),
+ gpon_read(priv, GPON_PLOu_PRMBL_TYPE1_2),
+ gpon_read(priv, GPON_PLOu_PRMBL_TYPE3),
+ gpon_read(priv, GPON_PLOu_DELM_BIT),
+ gpon_read(priv, GPON_PRE_ASSIGNED_DLY));
+}
+
+static void gpon_cb_set_t3_preamble(void *hw_priv, u8 o3_t3, u8 o5_t3)
+{
+ struct xpon_priv *priv = hw_priv;
+ u32 val;
+ int ret;
+
+ /*
+ * G_PLOu_PRMBL_TYPE3:
+ * bit 24 ebl_en
+ * bits 15:8 O5 extended T3 preamble
+ * bits 7:0 O3/O4 extended T3 preamble
+ *
+ * The stock RTF8225VW SDK reads back 0x01003860 for O3/O4=96
+ * and O5=56. BIT(16) writes a reserved field and leaves EBL off.
+ */
+ val = BIT(24) | ((u32)o5_t3 << 8) | o3_t3;
+ dev_dbg(priv->dev, "GPON T3 preamble: O3=%u O5=%u reg=%#08x\n",
+ o3_t3, o5_t3, val);
+ gpon_write(priv, GPON_PLOu_PRMBL_TYPE3, val);
+
+ ret = airoha_xpon_phy_set_gpon_extended_preamble(priv->phy,
+ o3_t3, o5_t3);
+ if (ret)
+ dev_warn(priv->dev,
+ "failed to program GPON PHY extended preamble: %d\n",
+ ret);
+}
+
+static void gpon_cb_set_key_switch_time(void *hw_priv, u32 superframe)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ dev_info(priv->dev, "GPON AES key switch superframe=%u\n",
+ superframe & AES_KEY_SWITCH_CNT_MASK);
+ gpon_write(priv, GPON_AES_CFG, superframe & AES_KEY_SWITCH_CNT_MASK);
+}
+
+static void gpon_cb_request_new_key(void *hw_priv)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ dev_info(priv->dev, "GPON generating a new AES key\n");
+ get_random_bytes(priv->aes_key, 16);
+ /* Load into shadow registers; switch time set later by
Key_Switching_Time */
+ gpon_load_aes_shadow(priv, priv->aes_key, 0);
+ /* Inform PLOAM layer of the key so it can transmit Encryption_Key */
+ ploam_set_aes_key(priv->ploam, priv->aes_key);
+}
+
+static void gpon_cb_set_ber_interval(void *hw_priv, u32 interval_ms)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ dev_info(priv->dev, "GPON BER reporting interval=%u ms\n",
+ interval_ms);
+ priv->ber_interval_ms = interval_ms;
+ if (interval_ms && READ_ONCE(priv->mac_enabled) &&
+ ploam_get_state(priv->ploam) == GPON_O5_OPERATION)
+ mod_timer(&priv->ber_timer,
+ jiffies + msecs_to_jiffies(interval_ms));
+ else
+ timer_delete(&priv->ber_timer);
+}
+
+static int gpon_cb_set_omci_gem(void *hw_priv, u16 gem_port_id, bool valid)
+{
+ struct xpon_priv *priv = hw_priv;
+ u8 onu_id = ploam_get_onu_id(priv->ploam);
+ u32 reg_val;
+ int ret, tcont_ret;
+
+ if (!valid) {
+ gpon_write(priv, GPON_OMCI_ID, 0);
+ airoha_gpon_omci_set_channel(&priv->omci, gem_port_id, false);
+
+ ret = gpon_set_gem_port_hw(priv, gem_port_id, false, false);
+ if (ret)
+ dev_err(priv->dev,
+ "failed to remove GPON OMCC GEM port %u: %d\n",
+ gem_port_id, ret);
+ tcont_ret = gpon_set_tcont_hw(priv, 0, 0, false);
+ if (!ret)
+ ret = tcont_ret;
+ dev_info(priv->dev, "GPON OMCI GEM disabled: port=%u\n",
+ gem_port_id);
+ return ret;
+ }
+
+ if (onu_id == PLOAM_ONU_UNASSIGNED) {
+ dev_err(priv->dev,
+ "cannot enable GPON OMCC GEM %u without an ONU-ID\n",
+ gem_port_id);
+ return -EINVAL;
+ }
+
+ ret = gpon_set_tcont_hw(priv, 0, onu_id, true);
+ if (ret) {
+ dev_err(priv->dev,
+ "failed to configure GPON OMCC T-CONT: %d\n", ret);
+ return ret;
+ }
+
+ ret = gpon_set_gem_port_hw(priv, gem_port_id, true, false);
+ if (ret) {
+ dev_err(priv->dev,
+ "failed to configure GPON OMCC GEM port %u: %d\n",
+ gem_port_id, ret);
+ gpon_set_tcont_hw(priv, 0, 0, false);
+ return ret;
+ }
+
+ reg_val = OMCI_PORT_VLD | (gem_port_id & OMCI_GPID_MASK);
+ gpon_write(priv, GPON_OMCI_ID, reg_val);
+ if ((gpon_read(priv, GPON_OMCI_ID) &
+ (OMCI_PORT_VLD | OMCI_GPID_MASK)) != reg_val) {
+ dev_err(priv->dev,
+ "failed to enable GPON OMCC register for GEM port %u\n",
+ gem_port_id);
+ gpon_set_gem_port_hw(priv, gem_port_id, false, false);
+ gpon_set_tcont_hw(priv, 0, 0, false);
+ return -EIO;
+ }
+
+ airoha_gpon_omci_set_channel(&priv->omci, gem_port_id, true);
+ dev_info(priv->dev,
+ "GPON OMCC datapath enabled: onu-id=%u tcont=0 gem=%u
reg=%#08x\n",
+ onu_id, gem_port_id, reg_val);
+ dev_info(priv->dev,
+ "GPON OMCC readback: omci=%#010x gem-status=%#010x
tcont0-1=%#010x\n",
+ gpon_read(priv, GPON_OMCI_ID),
+ gpon_read(priv, GPON_GEM_PORT_STS),
+ gpon_read(priv, GPON_TCONT_ID_0_1));
+ airoha_eth_xpon_dump_oam_rx_state(priv->gdm_dev);
+
+ return 0;
+}
+
+static void gpon_cb_set_gem_encryption(void *hw_priv, u16 port_id,
+ u8 encrypt_mode)
+{
+ struct xpon_priv *priv = hw_priv;
+ int ret;
+
+ /*
+ * The OMCC is never encrypted in GPON (G.984.3 12.2), but this
+ * Nokia/ALCL OLT announces encrypt=3 for GEM 0 as well as for the
+ * service ports. Honouring that on GEM 0 decrypts a plaintext OMCC and
+ * every OMCI frame fails, so the ONU never answers and the OLT
+ * deactivates it. Exempt GEM 0 and take the OLT at its word everywhere
+ * else: with the port left in the clear the service ports deliver
+ * high-entropy frames with random MACs and nonsense ethertypes, which
+ * is ciphertext handed up undecrypted.
+ */
+ ret = gpon_set_gem_port_hw(priv, port_id, true,
+ port_id && encrypt_mode == 3);
+ if (ret)
+ dev_err(priv->dev,
+ "failed to update GEM port %u encryption: %d\n",
+ port_id, ret);
+ else
+ dev_info(priv->dev,
+ "GPON GEM encryption update: port=%u mode=%u\n",
+ port_id, encrypt_mode);
+}
+
+static void gpon_cb_set_alloc_id(void *hw_priv, u16 alloc_id, bool
allocate)
+{
+ struct xpon_priv *priv = hw_priv;
+ int ret;
+
+ dev_info(priv->dev, "GPON Alloc-ID %s: %u\n",
+ allocate ? "assign" : "remove", alloc_id);
+ ret = gpon_set_alloc_id_hw(priv, alloc_id, allocate);
+ if (ret)
+ dev_err(priv->dev, "failed to %s Alloc-ID %u: %d\n",
+ allocate ? "assign" : "remove", alloc_id, ret);
+ else
+ airoha_gpon_omci_reconcile_services(&priv->omci);
+}
+
+int airoha_gpon_omci_hw_set_olt_profile(void *hw_priv,
+ const struct omci_olt_profile_state *state)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ if (!state || state->effective == OMCI_OLT_PROFILE_UNSPEC ||
+ state->effective == OMCI_OLT_PROFILE_AUTO)
+ return -EINVAL;
+
+ mutex_lock(&priv->omci_profile_lock);
+ priv->omci_profile = *state;
+ mutex_unlock(&priv->omci_profile_lock);
+
+ dev_info(priv->dev,
+ "OMCI OLT profile applied: configured=%s effective=%s
forced=%s quirks=%#x\n",
+ omci_olt_profile_name(state->configured),
+ omci_olt_profile_name(state->effective),
+ omci_olt_profile_name(state->forced), state->quirks);
+ dev_info(priv->dev, "OMCI OLT identity: vendor=%s equipment=%s\n",
+ state->olt.vendor_id_valid ? state->olt.vendor_id : "unknown",
+ state->olt.equipment_id_valid ?
+ state->olt.equipment_id : "unknown");
+
+ return 0;
+}
+
+void airoha_gpon_omci_hw_set_operational(void *hw_priv, bool operational)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ mutex_lock(&priv->link_state_lock);
+ priv->omci_operational = operational;
+ mutex_unlock(&priv->link_state_lock);
+
+ gpon_refresh_netdev_link(priv, false);
+}
+
+static bool airoha_gpon_config_requires_restart(u16 key)
+{
+ switch (key) {
+ case OMCI_CONFIG_SERIAL_NUMBER:
+ case OMCI_CONFIG_VENDOR_ID:
+ case OMCI_CONFIG_VERSION:
+ case OMCI_CONFIG_EQUIPMENT_ID:
+ case OMCI_CONFIG_PASSWORD:
+ case OMCI_CONFIG_TRAFFIC_MGMT_OPTION:
+ case OMCI_CONFIG_ONU_TYPE:
+ case OMCI_CONFIG_UNI_COUNT:
+ case OMCI_CONFIG_AGENT_FAKE_OMCI:
+ case OMCI_CONFIG_OLT_PROFILE:
+ case OMCI_CONFIG_OLT_PROFILE_FORCE:
+ case OMCI_CONFIG_OMCC_VERSION:
+ case OMCI_CONFIG_HARDWARE_VERSION:
+ case OMCI_CONFIG_SOFTWARE_VERSION_0:
+ case OMCI_CONFIG_SOFTWARE_VERSION_1:
+ return true;
+ default:
+ return false;
+ }
+}
+
+void airoha_gpon_omci_hw_config_changed(void *hw_priv, u16 key,
+ const struct omci_identity *identity)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ if (!identity)
+ return;
+
+ /*
+ * A complete provider/profile apply changes both PLOAM activation data
+ * and the OMCI view exposed after O5. Restart for those
session-defining
+ * values, but keep live agent policy toggles such as permissive
mode and
+ * dying-gasp handling from tearing down an operational service.
+ *
+ * net/omci writes a profile one key at a time. Delay the work
briefly so
+ * serial number, password, profile and managed-entity identity are all
+ * committed before the MAC and digital PHY are reset once.
+ */
+ if (!airoha_gpon_config_requires_restart(key)) {
+ dev_dbg(priv->dev,
+ "OMCI configuration changed (key %u), GPON restart not
required\n",
+ key);
+ return;
+ }
+
+ mutex_lock(&priv->omci_config_lock);
+ priv->pending_identity = *identity;
+ priv->config_restart_key = key;
+ priv->config_restart_pending = true;
+ mutex_unlock(&priv->omci_config_lock);
+
+ dev_info(priv->dev,
+ "GPON session configuration changed (OMCI key %u), scheduling
full restart\n",
+ key);
+ mod_delayed_work(priv->fsm_wq, &priv->restart_work,
+ msecs_to_jiffies(GPON_CONFIG_RESTART_DEBOUNCE_MS));
+}
+
+int airoha_gpon_omci_hw_get_ani_topology(void *hw_priv,
+ struct omci_ani_topology *topology)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ if (!topology)
+ return -EINVAL;
+
+ *topology = (struct omci_ani_topology) {
+ .tcont_base = 0x8000,
+ .scheduler_base = 0x8000,
+ .queue_base = 0x8000,
+ .maximum_queue_size = 0xffff,
+ .allocated_queue_size = 4,
+ .tcont_count = GPON_MAX_TCONT - 1,
+ .queues_per_tcont = AIROHA_NUM_QOS_QUEUES,
+ .queue_config_option = 1,
+ .scheduler_policy = 1,
+ };
+
+ dev_dbg(priv->dev,
+ "OMCI ANI topology: %u T-CONTs, %u queues per T-CONT\n",
+ topology->tcont_count, topology->queues_per_tcont);
+
+ return 0;
+}
+
+int airoha_gpon_omci_hw_set_tcont(void *hw_priv, u16 entity_id,
+ u16 alloc_id, bool valid)
+{
+ struct xpon_priv *priv = hw_priv;
+ unsigned int channel = 0;
+ int ret;
+
+ ret = gpon_set_omci_tcont_hw(priv, entity_id, alloc_id, valid,
+ &channel);
+ if (ret) {
+ dev_err(priv->dev,
+ "failed to configure OMCI T-CONT %#x alloc-id %u: %d\n",
+ entity_id, alloc_id, ret);
+ } else if (valid) {
+ dev_info(priv->dev,
+ "OMCI T-CONT %#x enabled alloc-id %u channel %u\n",
+ entity_id, alloc_id, channel);
+ } else {
+ dev_info(priv->dev, "OMCI T-CONT %#x disabled channel %u\n",
+ entity_id, channel);
+ }
+ return ret;
+}
+
+int airoha_gpon_omci_hw_set_gem_port(void *hw_priv, u16 entity_id,
+ u16 gem_port_id, u16 tcont_entity_id,
+ u8 direction, bool valid, bool encrypted)
+{
+ struct xpon_priv *priv = hw_priv;
+ int ret;
+
+ ret = gpon_set_gem_port_hw(priv, gem_port_id, valid, encrypted);
+ if (ret)
+ return ret;
+
+ gpon_refresh_netdev_link(priv, false);
+ dev_info(priv->dev,
+ "OMCI GEM port %u %s (ME %#x T-CONT %#x direction %u)\n",
+ gem_port_id, valid ? "enabled" : "disabled", entity_id,
+ tcont_entity_id, direction);
+ return 0;
+}
+
+int airoha_gpon_omci_hw_replace_service(void *hw_priv,
+ const struct omci_service_config *service)
+{
+ struct xpon_priv *priv = hw_priv;
+ struct airoha_xpon_service_cfg cfg;
+ unsigned int tcont_index;
+ int ret;
+
+ if (!service || service->multicast || service->vlan_treatment_valid ||
+ service->direction == OMCI_GEM_PORT_DIRECTION_ANI_TO_UNI)
+ return -EOPNOTSUPP;
+
+ cfg = (struct airoha_xpon_service_cfg) {
+ .cookie = service->cookie,
+ .gem_port_id = service->gem_port_id,
+ .vlan_id = service->vlan_id,
+ .queue = service->queue,
+ .pcp = service->pcp,
+ .vlan_valid = service->vlan_valid,
+ .pcp_valid = service->pcp_valid,
+ .default_service = service->default_service,
+ };
+ ret = gpon_tcont_entity_to_index(priv, service->tcont_entity_id,
+ &tcont_index);
+ if (ret && service->alloc_id && service->alloc_id != 0xffff) {
+ unsigned int channel;
+
+ /*
+ * A GPON restart clears the entity->index map in
+ * gpon_dev_init(), but the OLT does not re-run OMCI when the
+ * MIB data sync still matches: it only re-sends Assign_Alloc-ID
+ * over PLOAM, which never reaches the OMCI agent. The agent
+ * then re-applies its retained service graph, every lookup here
+ * returns -ENOENT, and because apply_services is all-or-nothing
+ * the whole set is rolled back -- leaving an ONU in O5 with OMCI
+ * up, service=0 and no datapath at all, permanently.
+ *
+ * The Alloc-ID is enough to rebuild it: program the T-CONT and
+ * re-bind entity->index, which is exactly what a first-time
+ * provisioning does.
+ */
+ ret = gpon_set_omci_tcont_hw(priv, service->tcont_entity_id,
+ service->alloc_id, true, &channel);
+ if (!ret) {
+ tcont_index = channel;
+ dev_info(priv->dev,
+ "rebuilt T-CONT %#x (alloc-id %u) as channel %u after
restart\n",
+ service->tcont_entity_id, service->alloc_id,
+ channel);
+ }
+ }
+ if (ret)
+ return ret;
+ cfg.tcont = tcont_index;
+
+ ret = airoha_eth_xpon_add_service(priv->gdm_dev, &cfg);
+ if (ret)
+ return ret;
+
+ mutex_lock(&priv->link_state_lock);
+ __set_bit(service->gem_port_id, priv->service_gems);
+ mutex_unlock(&priv->link_state_lock);
+ gpon_refresh_netdev_link(priv, false);
+ dev_dbg(priv->dev,
+ "OMCI service %#x: UNI %#x GEM %u T-CONT %#x channel %u queue
%u VLAN %s%u PCP %s%u ANI %s%#x\n",
+ service->cookie, service->uni_entity_id, service->gem_port_id,
+ service->tcont_entity_id, cfg.tcont, service->queue,
+ service->vlan_valid ? "" : "any/", service->vlan_id,
+ service->pcp_valid ? "" : "any/", service->pcp,
+ service->multicast_ani_valid ? "" : "none/",
+ service->multicast_ani_entity_id);
+
+ return 0;
+}
+
+int airoha_gpon_omci_hw_delete_service(void *hw_priv, u32 cookie)
+{
+ struct xpon_priv *priv = hw_priv;
+ u16 gem_port_id;
+
+ if (!airoha_eth_xpon_del_service(priv->gdm_dev, cookie,
+ &gem_port_id))
+ return -ENOENT;
+
+ if (!airoha_eth_xpon_has_gem_service(priv->gdm_dev, gem_port_id)) {
+ mutex_lock(&priv->link_state_lock);
+ __clear_bit(gem_port_id, priv->service_gems);
+ mutex_unlock(&priv->link_state_lock);
+ }
+ gpon_refresh_netdev_link(priv, false);
+
+ return 0;
+}
+
+int airoha_gpon_omci_hw_set_uni(void *hw_priv, u16 entity_id, bool enable)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ dev_dbg(priv->dev, "OMCI UNI port %#x requested %s\n", entity_id,
+ enable ? "enabled" : "disabled");
+
+ /* The EN7523 switch UNI administrative state remains owned by DSA. */
+ return 0;
+}
+
+int airoha_gpon_omci_hw_get_telemetry(void *hw_priv,
+ struct omci_telemetry *telemetry)
+{
+ struct optical_frontend_telemetry optical = {};
+ struct xpon_priv *priv = hw_priv;
+ bool downstream_fec, upstream_fec;
+ int ret;
+
+ if (!telemetry)
+ return -EINVAL;
+
+ memset(telemetry, 0, sizeof(*telemetry));
+ ret = airoha_xpon_phy_get_gpon_fec_status(priv->phy,
+ &downstream_fec,
+ &upstream_fec);
+ if (!ret) {
+ telemetry->downstream_fec = downstream_fec ?
+ OMCI_FEC_STATUS_UP : OMCI_FEC_STATUS_DOWN;
+ telemetry->upstream_fec = upstream_fec ?
+ OMCI_FEC_STATUS_UP : OMCI_FEC_STATUS_DOWN;
+ telemetry->valid |= OMCI_TELEMETRY_F_FEC_DOWNSTREAM |
+ OMCI_TELEMETRY_F_FEC_UPSTREAM;
+ }
+
+ if (!priv->frontend)
+ return telemetry->valid ? 0 : -ENODATA;
+
+ ret = optical_frontend_get_telemetry(priv->frontend, &optical);
+ if (ret)
+ return telemetry->valid ? 0 : ret;
+
+ if (optical.valid & OPTICAL_FRONTEND_TELEMETRY_F_TEMPERATURE) {
+ telemetry->bosa_temperature_mc = optical.temperature_mc;
+ telemetry->valid |= OMCI_TELEMETRY_F_BOSA_TEMPERATURE;
+ }
+ if (optical.valid & OPTICAL_FRONTEND_TELEMETRY_F_VOLTAGE) {
+ telemetry->bosa_voltage_uv = optical.voltage_uv;
+ telemetry->valid |= OMCI_TELEMETRY_F_BOSA_VOLTAGE;
+ }
+ if (optical.valid & OPTICAL_FRONTEND_TELEMETRY_F_BIAS) {
+ telemetry->bosa_bias_ua = optical.bias_ua;
+ telemetry->valid |= OMCI_TELEMETRY_F_BOSA_BIAS;
+ }
+ if (optical.valid & OPTICAL_FRONTEND_TELEMETRY_F_TX_POWER) {
+ telemetry->bosa_tx_power_nw = optical.tx_power_nw;
+ telemetry->valid |= OMCI_TELEMETRY_F_BOSA_TX_POWER;
+ }
+ if (optical.valid & OPTICAL_FRONTEND_TELEMETRY_F_RX_POWER) {
+ telemetry->bosa_rx_power_nw = optical.rx_power_nw;
+ telemetry->valid |= OMCI_TELEMETRY_F_BOSA_RX_POWER;
+ }
+ if (optical.valid & OPTICAL_FRONTEND_TELEMETRY_F_ALARMS) {
+ telemetry->bosa_alarms = optical.alarms;
+ telemetry->valid |= OMCI_TELEMETRY_F_BOSA_ALARMS;
+ }
+
+ return 0;
+}
+
+/* Forward declaration needed by gpon_disable */
+static void gpon_disable(struct xpon_priv *priv);
+
+static void gpon_cb_state_changed(void *hw_priv, enum gpon_state state)
+{
+ struct xpon_priv *priv = hw_priv;
+ enum airoha_xpon_phy_gpon_oper_state phy_state;
+ bool update_phy_state = true;
+ int ret;
+
+ /*
+ * Mirror the vendor PHY state sequencing before updating the MAC
+ * activation state: disabled in O2, ranging formatter in O3/O4 and
+ * operational formatter in O5.
+ */
+ switch (state) {
+ case GPON_O2_STANDBY:
+ phy_state = AIROHA_XPON_PHY_GPON_OPER_DISABLED;
+ break;
+ case GPON_O3_SERIAL_NUMBER:
+ case GPON_O4_RANGING:
+ phy_state = AIROHA_XPON_PHY_GPON_OPER_RANGING;
+ break;
+ case GPON_O5_OPERATION:
+ phy_state = AIROHA_XPON_PHY_GPON_OPER_OPERATION;
+ break;
+ default:
+ update_phy_state = false;
+ break;
+ }
+
+ if (update_phy_state) {
+ ret = airoha_xpon_phy_set_gpon_oper_state(priv->phy, phy_state);
+ if (ret)
+ dev_warn(priv->dev,
+ "failed to update GPON PHY operational state: %d\n",
+ ret);
+ }
+
+ /* Keep the hardware activation state in sync with the PLOAM FSM. */
+ gpon_write(priv, GPON_ACTIVATION_ST, state & 0x7);
+ dev_info(priv->dev, "GPON state -> %s (%u), activation_reg=%#08x\n",
+ gpon_state_name(state), state,
+ gpon_read(priv, GPON_ACTIVATION_ST));
+ airoha_gpon_omci_set_state(&priv->omci, state);
+ if (priv->xpon) {
+ enum xpon_registration_state registration;
+
+ switch (state) {
+ case GPON_O1_INITIAL:
+ case GPON_O7_EMERGENCY_STOP:
+ registration = XPON_REGISTRATION_DOWN;
+ break;
+ case GPON_O2_STANDBY:
+ registration = XPON_REGISTRATION_DISCOVERY;
+ break;
+ case GPON_O3_SERIAL_NUMBER:
+ case GPON_O4_RANGING:
+ case GPON_O6_POPUP:
+ registration = XPON_REGISTRATION_REGISTERING;
+ break;
+ case GPON_O5_OPERATION:
+ registration = XPON_REGISTRATION_OPERATIONAL;
+ break;
+ default:
+ registration = XPON_REGISTRATION_DOWN;
+ break;
+ }
+ xpon_device_report_registration(priv->xpon, registration);
+ }
+
+ if (state == GPON_O4_RANGING || state == GPON_O5_OPERATION)
+ gpon_dump_activation_regs(priv, "state transition");
+
+ mutex_lock(&priv->link_state_lock);
+ priv->gpon_o5 = state == GPON_O5_OPERATION;
+ mutex_unlock(&priv->link_state_lock);
+ gpon_refresh_netdev_link(priv, false);
+
+ if (state != GPON_O5_OPERATION)
+ timer_delete(&priv->ber_timer);
+
+ switch (state) {
+ case GPON_O2_STANDBY:
+ /*
+ * Match the stock SDK for this generation: 0x058b is the
+ * reset/O1 value, and activation starts from O2 with the
+ * generation's own response time.
+ */
+ gpon_write(priv, GPON_RSP_TIME,
+ priv->match_data->gpon_rsp_time_activation);
+ dev_info(priv->dev,
+ "GPON O2 activation response time=%#06x\n",
+ gpon_read(priv, GPON_RSP_TIME));
+ /*
+ * A stale bit delay from the previous session would offset the
+ * serial-number burst of the next ranging cycle.
+ */
+ priv->byte_delay = 0;
+ priv->bit_delay = 0;
+ gpon_set_bit_delay(priv, 0);
+ fallthrough;
+ case GPON_O3_SERIAL_NUMBER:
+ case GPON_O4_RANGING:
+ /* Start / restart TO1 timer */
+ mod_delayed_work(priv->fsm_wq, &priv->to1_work,
+ msecs_to_jiffies(GPON_TO1_MS));
+ break;
+ case GPON_O5_OPERATION:
+ /* Cancel TO1 and start periodic BER reporting only in O5. */
+ cancel_delayed_work(&priv->to1_work);
+ if (priv->ber_interval_ms)
+ mod_timer(&priv->ber_timer,
+ jiffies +
+ msecs_to_jiffies(priv->ber_interval_ms));
+ priv->to1_failures = 0;
+ dev_info(priv->dev, "GPON O5: operational, ONU-ID=%u\n",
+ ploam_get_onu_id(priv->ploam));
+ break;
+ case GPON_O6_POPUP:
+ /* Cancel TO1, start TO2 */
+ cancel_delayed_work(&priv->to1_work);
+ mod_delayed_work(priv->fsm_wq, &priv->to2_work,
+ msecs_to_jiffies(GPON_TO2_MS));
+ break;
+ case GPON_O7_EMERGENCY_STOP:
+ cancel_delayed_work(&priv->to1_work);
+ cancel_delayed_work(&priv->to2_work);
+ break;
+ case GPON_O1_INITIAL:
+ cancel_delayed_work(&priv->to1_work);
+ cancel_delayed_work(&priv->to2_work);
+ break;
+ default:
+ break;
+ }
+}
+
+static void gpon_cb_deactivate(void *hw_priv)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ /*
+ * The callback runs while the ordered IRQ worker is processing the
+ * downstream FIFO. Defer the stop/restart sequence until that worker
+ * has returned; gpon_disable() may cancel work and power down the PHY.
+ */
+ if (READ_ONCE(priv->started) &&
+ READ_ONCE(priv->optical_active))
+ mod_delayed_work(priv->fsm_wq, &priv->restart_work, 0);
+}
+
+static const struct ploam_ops gpon_ploam_ops = {
+ .send_upstream = gpon_cb_send_upstream,
+ .set_onu_id = gpon_cb_set_onu_id,
+ .set_eqd_o4 = gpon_cb_set_eqd_o4,
+ .adjust_eqd_o5 = gpon_cb_adjust_eqd_o5,
+ .enable_us_fec = gpon_cb_enable_us_fec,
+ .set_overhead = gpon_cb_set_overhead,
+ .set_t3_preamble = gpon_cb_set_t3_preamble,
+ .set_key_switch_time = gpon_cb_set_key_switch_time,
+ .request_new_key = gpon_cb_request_new_key,
+ .set_ber_interval = gpon_cb_set_ber_interval,
+ .set_omci_gem = gpon_cb_set_omci_gem,
+ .set_gem_encryption = gpon_cb_set_gem_encryption,
+ .set_alloc_id = gpon_cb_set_alloc_id,
+ .state_changed = gpon_cb_state_changed,
+ .deactivate = gpon_cb_deactivate,
+};
+
+/* -----------------------------------------------------------------------
+ * Enable / disable
+ * -------------------------------------------------------------------- */
+
+static int gpon_enable(struct xpon_priv *priv)
+{
+ u32 fifo_depth, irq_mask, known, pending, unknown;
+ int ret;
+
+ if (READ_ONCE(priv->mac_enabled))
+ return 0;
+
+ if (!READ_ONCE(priv->started) || !READ_ONCE(priv->optical_active))
+ return -ENETDOWN;
+
+ dev_info(priv->dev, "starting GPON MAC from state %s\n",
+ gpon_state_name(ploam_get_state(priv->ploam)));
+
+ /*
+ * EN7523 starts each GPON session from a MAC reset. The EN7521 SDK
+ * initial-enable path instead calls gponDevResetCtrl(XPON_DISABLE),
+ * which releases MBI without asserting RST_CTRL1[31]. A real gen1
+ * recovery reset also stops MBI before asserting that bit, so do not
+ * issue a bare reset here on EN751221.
+ */
+ if (priv->match_data->gpon_reset_on_start) {
+ ret = airoha_xpon_reset_mac(priv);
+ if (ret)
+ goto err_disable_frontend;
+ }
+
+ ret = airoha_xpon_phy_start(priv->dev, priv->phy,
+ AIROHA_XPON_MODE_GPON,
+ &priv->phy_initialized,
+ &priv->phy_powered);
+ if (ret)
+ goto err_disable_frontend;
+
+ ret = gpon_prepare_hardware(priv);
+ if (ret) {
+ dev_err(priv->dev, "failed to prepare GPON hardware: %d\n", ret);
+ goto err_stop_phy;
+ }
+
+ ret = gpon_dev_init(priv);
+ if (ret) {
+ gpon_set_fe_datapath(priv, false);
+ dev_err(priv->dev, "GPON hardware init failed: %d\n", ret);
+ goto err_stop_phy;
+ }
+
+ gpon_set_serial_number_regs(priv);
+ gpon_load_aes_shadow(priv, priv->aes_key, 0);
+
+ /*
+ * Inspect events accumulated while the digital PHY was locking. Known
+ * activation bits help distinguish CDR lock from valid GPON framing.
+ * Error-only interrupts remain masked because the EN7523 can assert
+ * them continuously while GEM delineation is being acquired; counters
+ * remain available through the diagnostic paths.
+ */
+ irq_mask = GPON_INT_DEFAULT_MASK;
+ if (priv->dying_gasp_irq >= 0)
+ irq_mask &= ~INT_DYING_GASP;
+
+ pending = gpon_read(priv, GPON_INT_STATUS);
+ known = pending & (u32)GPON_INT_DEFAULT_MASK;
+ unknown = pending & ~(u32)GPON_INT_DEFAULT_MASK;
+ fifo_depth = gpon_read(priv, GPON_PLOAMd_FIFO_STS) &
+ PLOAMd_FIFO_USED_MASK;
+ if (pending || fifo_depth)
+ dev_info(priv->dev,
+ "GPON events before IRQ enable: raw=%#08x known=%#08x
unknown=%#08x fifo=%u activation=%#08x\n",
+ pending, known, unknown, fifo_depth,
+ gpon_read(priv, GPON_ACTIVATION_ST));
+
+ /*
+ * Enter O2 before unmasking the MAC interrupt source. Any PLOAM queued
+ * in hardware is then processed against a fully initialized FSM.
+ */
+ atomic_set(&priv->pending_irqs, 0);
+ gpon_ploam_rx_queue_reset(priv);
+ WRITE_ONCE(priv->mac_enabled, true);
+ ret = airoha_xpon_tx_enable(priv->dev, priv->frontend, true);
+ if (ret)
+ goto err_disable_mac;
+
+ /* Do not enter O2 until the external transmitter interlock is open. */
+ ploam_start(priv->ploam);
+ WRITE_ONCE(priv->phy_link_known, false);
+ mod_delayed_work(priv->fsm_wq, &priv->phy_link_work, 0);
+
+ /* Vendor gpon_INT_init() clears W1C status before unmasking the
MAC. */
+ gpon_write(priv, GPON_INT_STATUS, ~0U);
+ gpon_write(priv, GPON_INT_ENABLE, irq_mask);
+
+ dev_info(priv->dev,
+ "enabling GPON interrupts on IRQ %d with mask=%#08x
status=%#08x\n",
+ priv->irq, gpon_read(priv, GPON_INT_ENABLE),
+ gpon_read(priv, GPON_INT_STATUS));
+
+ dev_info(priv->dev,
+ "GPON MAC started: state=%s mbi=%#08x int_enable=%#08x\n",
+ gpon_state_name(ploam_get_state(priv->ploam)),
+ gpon_read(priv, GPON_MBI_MPI_STOP),
+ gpon_read(priv, GPON_INT_ENABLE));
+
+ return 0;
+
+err_disable_mac:
+ gpon_disable(priv);
+ return ret;
+err_stop_phy:
+ airoha_xpon_tx_enable(priv->dev, priv->frontend, false);
+ airoha_xpon_phy_stop(priv->dev, priv->phy,
+ AIROHA_XPON_MODE_GPON,
+ &priv->phy_initialized,
+ &priv->phy_powered);
+ return ret;
+err_disable_frontend:
+ airoha_xpon_tx_enable(priv->dev, priv->frontend, false);
+ return ret;
+}
+
+static void gpon_disable(struct xpon_priv *priv)
+{
+ bool mac_enabled = READ_ONCE(priv->mac_enabled);
+ bool phy_active = priv->phy_initialized || priv->phy_powered;
+ bool omci_reset = false;
+ int ret;
+
+ airoha_xpon_tx_enable(priv->dev, priv->frontend, false);
+
+ if (!mac_enabled && !phy_active)
+ goto reset_session;
+
+ dev_info(priv->dev,
+ "stopping GPON MAC: state=%s mbi=%#08x int_status=%#08x\n",
+ gpon_state_name(ploam_get_state(priv->ploam)),
+ gpon_read(priv, GPON_MBI_MPI_STOP),
+ gpon_read(priv, GPON_INT_STATUS));
+
+ /*
+ * Mask the MAC before cancelling queued FSM work. The Linux IRQ line
+ * remains enabled so no activation edge is lost across a restart.
+ */
+ WRITE_ONCE(priv->mac_enabled, false);
+ if (mac_enabled) {
+ gpon_write(priv, GPON_INT_ENABLE, 0);
+ gpon_write(priv, GPON_INT_STATUS, ~0U);
+ synchronize_irq(priv->irq);
+ }
+
+ atomic_set(&priv->pending_irqs, 0);
+ if (current_work() != &priv->irq_work)
+ cancel_work_sync(&priv->irq_work);
+ gpon_ploam_rx_queue_reset(priv);
+
+ if (current_work() == &priv->to1_work.work)
+ cancel_delayed_work(&priv->to1_work);
+ else
+ cancel_delayed_work_sync(&priv->to1_work);
+
+ if (current_work() == &priv->to2_work.work)
+ cancel_delayed_work(&priv->to2_work);
+ else
+ cancel_delayed_work_sync(&priv->to2_work);
+
+ timer_delete_sync(&priv->ber_timer);
+ if (current_work() == &priv->phy_link_work.work)
+ cancel_delayed_work(&priv->phy_link_work);
+ else
+ cancel_delayed_work_sync(&priv->phy_link_work);
+ WRITE_ONCE(priv->phy_link_known, false);
+ WRITE_ONCE(priv->phy_link_up, false);
+
+ if (mac_enabled) {
+ /*
+ * Drain OMCI before disconnecting GDM2. Deactivate_ONU-ID can
+ * share a downstream frame with the last OMCI request, and the
+ * response must reach QDMA before the datapath is disabled.
+ */
+ airoha_gpon_omci_reset_session(&priv->omci);
+ omci_reset = true;
+
+ /*
+ * Match the vendor shutdown path: invalidate the runtime
+ * identities, disconnect GDM2, then stop the GPON/PSE MBI.
+ */
+ gpon_reset_activation_context(priv);
+ gpon_write(priv, GPON_OMCI_ID, 0);
+
+ ret = gpon_set_fe_datapath(priv, false);
+ if (ret)
+ dev_warn(priv->dev,
+ "failed to disable GPON FE datapath: %d\n", ret);
+ airoha_eth_xpon_flush_services(priv->gdm_dev);
+ ret = airoha_eth_set_xpon_mode(priv->gdm_dev,
+ AIROHA_XPON_MODE_GPON);
+ if (ret)
+ dev_warn(priv->dev,
+ "failed to quiesce GPON FE channels: %d\n", ret);
+
+ gpon_set_bits(priv, GPON_MBI_MPI_STOP,
+ MBI_RX_STOP | MBI_TX_STOP);
+ dev_info(priv->dev, "GPON MBI stopped: %#08x\n",
+ gpon_read(priv, GPON_MBI_MPI_STOP));
+ }
+
+ if (phy_active)
+ airoha_xpon_phy_stop(priv->dev, priv->phy,
+ AIROHA_XPON_MODE_GPON,
+ &priv->phy_initialized,
+ &priv->phy_powered);
+
+reset_session:
+ ploam_reset(priv->ploam);
+ if (!omci_reset)
+ airoha_gpon_omci_reset_session(&priv->omci);
+ airoha_gpon_omci_set_state(&priv->omci, GPON_O1_INITIAL);
+ mutex_lock(&priv->link_state_lock);
+ priv->gpon_o5 = false;
+ priv->omci_operational = false;
+ bitmap_zero(priv->service_gems, GPON_MAX_GEM_ID);
+ mutex_unlock(&priv->link_state_lock);
+ gpon_refresh_netdev_link(priv, true);
+ dev_info(priv->dev, "GPON MAC stopped, state reset to %s\n",
+ gpon_state_name(ploam_get_state(priv->ploam)));
+}
+
+/* -----------------------------------------------------------------------
+ * Timers
+ * -------------------------------------------------------------------- */
+
+static void gpon_ber_timer_fn(struct timer_list *t)
+{
+ struct xpon_priv *priv = timer_container_of(priv, t, ber_timer);
+ bool ready, los;
+ int ret;
+
+ if (!READ_ONCE(priv->mac_enabled) || !priv->ber_interval_ms ||
+ ploam_get_state(priv->ploam) != GPON_O5_OPERATION)
+ return;
+
+ ret = airoha_xpon_phy_get_link_state(priv->phy, &ready, &los);
+ if (!ret && (!ready || los)) {
+ mod_delayed_work(priv->fsm_wq, &priv->phy_link_work, 0);
+ return;
+ }
+
+ ploam_notify_ber(priv->ploam, 0);
+ if (READ_ONCE(priv->mac_enabled) && priv->ber_interval_ms &&
+ ploam_get_state(priv->ploam) == GPON_O5_OPERATION)
+ mod_timer(&priv->ber_timer,
+ jiffies +
+ msecs_to_jiffies(priv->ber_interval_ms));
+}
+
+static void airoha_xpon_phy_link_work_fn(struct work_struct *work)
+{
+ struct xpon_priv *priv =
+ container_of(to_delayed_work(work), struct xpon_priv,
+ phy_link_work);
+ bool ready, los, link, changed;
+ int ret;
+
+ if (!READ_ONCE(priv->phy_powered))
+ return;
+
+ ret = airoha_xpon_phy_get_link_state(priv->phy, &ready, &los);
+ if (ret)
+ goto rearm;
+
+ link = ready && !los;
+ changed = !READ_ONCE(priv->phy_link_known) ||
+ READ_ONCE(priv->phy_link_up) != link;
+ WRITE_ONCE(priv->phy_link_known, true);
+ WRITE_ONCE(priv->phy_link_up, link);
+ if (priv->xpon)
+ xpon_device_report_optical(priv->xpon, ready, los);
+
+ if (changed)
+ dev_info(priv->dev,
+ "%s digital PHY link %s: ready=%u LOS=%u\n",
+ airoha_xpon_mode_name(priv->mode),
+ link ? "up" : "down", ready, los);
+
+ if (!link && priv->mode == AIROHA_XPON_MODE_GPON &&
+ ploam_get_state(priv->ploam) == GPON_O5_OPERATION) {
+ dev_warn(priv->dev,
+ "GPON synchronization lost in O5, stopping upstream PLOAM\n");
+ timer_delete(&priv->ber_timer);
+ ploam_notify_los(priv->ploam);
+ }
+
+rearm:
+ if (READ_ONCE(priv->phy_powered))
+ mod_delayed_work(priv->fsm_wq, &priv->phy_link_work,
+ msecs_to_jiffies(XPON_LINK_POLL_MS));
+}
+
+/* TO1: O3/O4 timeout — no Ranging_Time received within 10 s → return
to O2 */
+static void gpon_to1_work_fn(struct work_struct *work)
+{
+ struct xpon_priv *priv =
+ container_of(to_delayed_work(work), struct xpon_priv, to1_work);
+ enum gpon_state st = ploam_get_state(priv->ploam);
+
+ /*
+ * This work runs on the same ordered queue as downstream PLOAM
+ * processing, so Upstream_Overhead cannot re-enter O3 in the middle
+ * of the O3/O4 -> O2 transition.
+ */
+ if (st != GPON_O3_SERIAL_NUMBER && st != GPON_O4_RANGING)
+ return;
+
+ gpon_dump_activation_regs(priv, "TO1 expired");
+ priv->to1_failures++;
+
+ /*
+ * A ranging cycle that does not complete is normal on a busy or
+ * misconfigured OLT, and the recovery for it is cheap: drop back to O2
+ * and answer the next Upstream_Overhead. Restarting the MAC and the
+ * PHY here instead would cost an optical relock on every miss. Only a
+ * long run of failures means the hardware itself is stuck, and that is
+ * what the vendor driver escalates on.
+ */
+ if (priv->to1_failures < GPON_TO1_MAX_RETRIES) {
+ dev_warn(priv->dev,
+ "GPON TO1 expired in O%d (%u/%u), returning to O2\n",
+ (int)st, priv->to1_failures, GPON_TO1_MAX_RETRIES);
+ gpon_write(priv, GPON_ONU_ID, PLOAM_ONU_UNASSIGNED);
+ ploam_reset(priv->ploam);
+ ploam_start(priv->ploam);
+ return;
+ }
+
+ dev_err(priv->dev,
+ "GPON TO1 expired %u times without reaching O5, resetting MAC
and PHY\n",
+ priv->to1_failures);
+ priv->to1_failures = 0;
+ gpon_disable(priv);
+ if (READ_ONCE(priv->started) && READ_ONCE(priv->optical_active))
+ mod_delayed_work(priv->fsm_wq, &priv->restart_work,
+ msecs_to_jiffies(GPON_DEACTIVATE_RESTART_MS));
+}
+
+/* TO2: O6 timeout — no Popup received within 100 ms → full disable */
+static void gpon_to2_work_fn(struct work_struct *work)
+{
+ struct xpon_priv *priv =
+ container_of(to_delayed_work(work), struct xpon_priv, to2_work);
+
+ bool ready, los;
+ int ret;
+
+ if (ploam_get_state(priv->ploam) != GPON_O6_POPUP)
+ return;
+
+ /*
+ * O6 is entered on LOS, so ask the PHY whether the fibre came back
+ * before scheduling anything. While the light is still gone a restart
+ * cannot succeed: it relocks nothing, fails, and re-arms itself, which
+ * on a dirty connector turns one glitch into a restart storm.
Stop the
+ * MAC and wait instead -- gpon_sfp_link_up() restarts the session when
+ * the fibre is back. The vendor TO2 handler makes the same check.
+ */
+ ret = airoha_xpon_phy_get_link_state(priv->phy, &ready, &los);
+ if (!ret && (los || !ready)) {
+ dev_warn(priv->dev,
+ "GPON TO2 expired in O6 with the optical link still down,
waiting for the fibre\n");
+ gpon_disable(priv);
+ return;
+ }
+
+ dev_warn(priv->dev, "GPON TO2 expired in O6, resetting\n");
+ gpon_disable(priv);
+ if (READ_ONCE(priv->started) && READ_ONCE(priv->optical_active))
+ mod_delayed_work(priv->fsm_wq, &priv->restart_work,
+ msecs_to_jiffies(GPON_DEACTIVATE_RESTART_MS));
+}
+
+static bool
+gpon_take_pending_identity(struct xpon_priv *priv,
+ struct omci_identity *identity, u16 *key)
+{
+ bool pending;
+
+ mutex_lock(&priv->omci_config_lock);
+ pending = priv->config_restart_pending;
+ if (pending) {
+ *identity = priv->pending_identity;
+ *key = priv->config_restart_key;
+ priv->config_restart_pending = false;
+ }
+ mutex_unlock(&priv->omci_config_lock);
+
+ return pending;
+}
+
+static void
+gpon_apply_runtime_identity(struct xpon_priv *priv,
+ const struct omci_identity *identity)
+{
+ priv->identity = *identity;
+ memcpy(priv->hw_sn, identity->serial_number, sizeof(priv->hw_sn));
+ memcpy(priv->hw_passwd, identity->password,
+ sizeof(priv->hw_passwd));
+ ploam_set_identity(priv->ploam, priv->hw_sn, priv->hw_passwd);
+
+ dev_info(priv->dev,
+ "applied runtime GPON identity (serial source %u, password
source %u)\n",
+ identity->serial_source, identity->password_source);
+}
+
+static void gpon_restart_work_fn(struct work_struct *work)
+{
+ struct xpon_priv *priv =
+ container_of(to_delayed_work(work), struct xpon_priv,
+ restart_work);
+ struct omci_identity identity;
+ bool config_restart;
+ bool stopped = false;
+ u16 key = OMCI_CONFIG_UNSPEC;
+ int ret;
+
+ config_restart = gpon_take_pending_identity(priv, &identity, &key);
+ if (!READ_ONCE(priv->started) || !READ_ONCE(priv->optical_active)) {
+ if (config_restart)
+ gpon_apply_runtime_identity(priv, &identity);
+ return;
+ }
+
+ if (config_restart) {
+ dev_warn(priv->dev,
+ "restarting GPON after runtime OMCI configuration key %u
changed\n",
+ key);
+ if (READ_ONCE(priv->mac_enabled)) {
+ gpon_disable(priv);
+ stopped = true;
+ }
+ gpon_apply_runtime_identity(priv, &identity);
+ } else if (ploam_get_state(priv->ploam) != GPON_O1_INITIAL) {
+ dev_warn(priv->dev,
+ "restarting GPON after Deactivate_ONU-ID from the OLT\n");
+ gpon_disable(priv);
+ stopped = true;
+ }
+
+ if (stopped) {
+ msleep(GPON_DEACTIVATE_RESTART_MS);
+ if (!READ_ONCE(priv->started) ||
+ !READ_ONCE(priv->optical_active))
+ return;
+ }
+
+ ret = airoha_xpon_tx_rearm(priv->dev, priv->frontend);
+ if (ret) {
+ dev_warn(priv->dev,
+ "retrying optical transmitter rearm in %u ms\n",
+ GPON_REARM_RETRY_MS);
+ if (READ_ONCE(priv->started) &&
+ READ_ONCE(priv->optical_active))
+ mod_delayed_work(priv->fsm_wq, &priv->restart_work,
+ msecs_to_jiffies(GPON_REARM_RETRY_MS));
+ return;
+ }
+
+ ret = gpon_enable(priv);
+ if (ret)
+ dev_err(priv->dev, "failed to restart GPON: %d\n", ret);
+}
+
+int airoha_gpon_omci_hw_start(void *hw_priv)
+{
+ struct xpon_priv *priv = hw_priv;
+ int ret = 0;
+
+ if (READ_ONCE(priv->started))
+ return 0;
+
+ WRITE_ONCE(priv->started, true);
+ gpon_refresh_netdev_link(priv, true);
+ if (priv->sfp_bus) {
+ if (!READ_ONCE(priv->optical_active))
+ sfp_upstream_start(priv->sfp_bus);
+ } else {
+ ret = airoha_xpon_tx_rearm(priv->dev, priv->frontend);
+ if (!ret) {
+ WRITE_ONCE(priv->optical_active, true);
+ ret = gpon_enable(priv);
+ }
+ if (ret)
+ WRITE_ONCE(priv->optical_active, false);
+ }
+
+ if (ret) {
+ WRITE_ONCE(priv->started, false);
+ gpon_refresh_netdev_link(priv, true);
+ }
+
+ return ret;
+}
+
+void airoha_gpon_omci_hw_stop(void *hw_priv)
+{
+ struct xpon_priv *priv = hw_priv;
+
+ if (!READ_ONCE(priv->started))
+ return;
+
+ WRITE_ONCE(priv->started, false);
+ cancel_delayed_work_sync(&priv->restart_work);
+ if (priv->sfp_bus && READ_ONCE(priv->optical_active)) {
+ sfp_upstream_stop(priv->sfp_bus);
+ } else {
+ WRITE_ONCE(priv->optical_active, false);
+ gpon_disable(priv);
+ }
+ gpon_refresh_netdev_link(priv, true);
+}
+
+/* -----------------------------------------------------------------------
+ * Interrupt handler
+ * -------------------------------------------------------------------- */
+
+static void gpon_dump_error_counters(struct xpon_priv *priv, u32 errors)
+{
+ dev_warn_ratelimited(priv->dev,
+ "GPON error: irq=%#x sfifo=%#x bfifo=%#x ctrl=%#x\n",
+ errors, gpon_read(priv, GPON_DBG_BWM_SFIFO_STS),
+ gpon_read(priv, GPON_DBG_BWM_BFIFO_STS),
+ gpon_read(priv, GPON_DBG_ERR_CTRL));
+ dev_warn_ratelimited(priv->dev,
+ "GPON counters: rx=%u/%u/%u tx=%u/%u hec=%u/%u/%u\n",
+ gpon_read(priv, GPON_DBG_RX_GEM_CNT),
+ gpon_read(priv, GPON_DBG_RX_CRC_ERR_CNT),
+ gpon_read(priv, GPON_DBG_RX_GTC_CNT),
+ gpon_read(priv, GPON_DBG_TX_GEM_CNT),
+ gpon_read(priv, GPON_DBG_TX_BST_CNT),
+ gpon_read(priv, GPON_DBG_GEM_HEC_ONE_ERR_CNT),
+ gpon_read(priv, GPON_DBG_GEM_HEC_TWO_ERR_CNT),
+ gpon_read(priv, GPON_DBG_GEM_HEC_UC_ERR_CNT));
+}
+
+static void gpon_irq_work_fn(struct work_struct *work)
+{
+ struct xpon_priv *priv =
+ container_of(work, struct xpon_priv, irq_work);
+ u32 active;
+
+ /*
+ * TO1, TO2 and downstream PLOAM callbacks all execute on fsm_wq.
+ * Loop because the top half may accumulate more bits while the worker
+ * is draining the hardware FIFO.
+ */
+ while ((active = (u32)atomic_xchg(&priv->pending_irqs, 0))) {
+ u32 phy_tx_frames = 0, phy_tx_bursts = 0;
+ u32 sn_cfg;
+ int phy_ret;
+
+ if (!READ_ONCE(priv->mac_enabled))
+ break;
+
+ gpon_process_ploam_queue(priv);
+
+ /*
+ * Keep the serial-number diagnostics compact: one line only when
+ * the MAC receives a serial grant, sends Serial_Number_ONU, or
+ * crosses the serial-request threshold.
+ */
+ if (active & (INT_SN_REQ_RECV | INT_SN_ONU_SEND_O3 |
+ INT_SN_REQ_CRS)) {
+ sn_cfg = gpon_read(priv, GPON_SN_MSG_CFG);
+ phy_ret = airoha_xpon_phy_get_gpon_tx_counters(
+ priv->phy, &phy_tx_frames, &phy_tx_bursts);
+ dev_dbg(priv->dev,
+ "GPON SN event: irq=%#08x cfg=%#010x threshold=%lu
tx_power=%lu random_delay=%lu rsp=%#06x act=%u serial=%#010x/%#010x
guard=%#010x type12=%#010x type3=%#010x pre_delay=%#010x dbg_dly=%#010x
tx_sync=%#010x phy_tx=%#010x/%#010x phy_ret=%d\n",
+ active, sn_cfg,
+ FIELD_GET(SN_MSG_CFG_SN_REQ_THR_MASK, sn_cfg),
+ FIELD_GET(SN_MSG_CFG_TX_POWER_MODE_MASK, sn_cfg),
+ FIELD_GET(SN_MSG_CFG_RANDOM_DELAY_MASK, sn_cfg),
+ gpon_read(priv, GPON_RSP_TIME),
+ gpon_read(priv, GPON_ACTIVATION_ST) & 0x7,
+ gpon_read(priv, GPON_VENDOR_ID),
+ gpon_read(priv, GPON_VS_SN),
+ gpon_read(priv, GPON_PLOu_GUARD_BIT),
+ gpon_read(priv, GPON_PLOu_PRMBL_TYPE1_2),
+ gpon_read(priv, GPON_PLOu_PRMBL_TYPE3),
+ gpon_read(priv, GPON_PRE_ASSIGNED_DLY),
+ gpon_read(priv, GPON_DBG_DLY),
+ gpon_read(priv, GPON_DBG_TX_SYNC_OFFSET),
+ phy_tx_frames, phy_tx_bursts, phy_ret);
+ }
+
+ if (active & INT_DYING_GASP) {
+ int ret;
+
+ dev_warn(priv->dev, "GPON dying-gasp interrupt\n");
+ ploam_notify_dying_gasp(priv->ploam);
+ ret = airoha_gpon_omci_send_dying_gasp(&priv->omci);
+ if (ret && ret != -EOPNOTSUPP && ret != -ENOLINK)
+ dev_warn(priv->dev,
+ "failed to send OMCI dying gasp: %d\n",
+ ret);
+ }
+
+ if (active & INT_LOSS_GEM_DEL)
+ dev_warn_ratelimited(priv->dev,
+ "GPON loss of GEM delineation interrupt\n");
+
+ if (active & GPON_INT_ERROR_MASK)
+ gpon_dump_error_counters(priv, active &
+ (u32)GPON_INT_ERROR_MASK);
+
+ if (active & (INT_RX_ERR | INT_FIFO_ERR))
+ airoha_eth_xpon_dump_oam_rx_state(priv->gdm_dev);
+
+ if (active & INT_TX_LATE_START) {
+ gpon_dump_activation_regs(priv, "TX late start");
+ dev_warn(priv->dev,
+ "GPON upstream burst started late: rsp_time=%#06x
state=%s\n",
+ gpon_read(priv, GPON_RSP_TIME),
+ gpon_state_name(ploam_get_state(priv->ploam)));
+ }
+ }
+
+ if (READ_ONCE(priv->mac_enabled))
+ gpon_process_ploam_queue(priv);
+}
+
+static irqreturn_t gpon_isr(int irq, void *data)
+{
+ struct xpon_priv *priv = data;
+ u32 active, enabled, raw;
+
+ raw = gpon_read(priv, GPON_INT_STATUS);
+ if (!raw)
+ return IRQ_NONE;
+
+ enabled = gpon_read(priv, GPON_INT_ENABLE);
+ active = raw & enabled;
+
+ /* G_INT_STATUS is W1C; acknowledge the complete hardware snapshot. */
+ gpon_write(priv, GPON_INT_STATUS, raw);
+
+ if (active & INT_PLOAMD_RECV)
+ gpon_drain_ploam_fifo_irq(priv);
+
+ if (active) {
+ atomic_or(active, &priv->pending_irqs);
+ queue_work(priv->fsm_wq, &priv->irq_work);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t gpon_dying_gasp_isr(int irq, void *data)
+{
+ struct xpon_priv *priv = data;
+
+ if (!READ_ONCE(priv->mac_enabled))
+ return IRQ_HANDLED;
+
+ atomic_or(INT_DYING_GASP, &priv->pending_irqs);
+ queue_work(priv->fsm_wq, &priv->irq_work);
+
+ return IRQ_HANDLED;
+}
+
+/* -----------------------------------------------------------------------
+ * SFP upstream ops
+ * -------------------------------------------------------------------- */
+
+static void gpon_sfp_attach(void *upstream, struct sfp_bus *bus)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "GPON SFP bus attached\n");
+}
+
+static void gpon_sfp_detach(void *upstream, struct sfp_bus *bus)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "GPON SFP bus detached\n");
+}
+
+static int gpon_sfp_module_insert(void *upstream,
+ const struct sfp_eeprom_id *id)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "GPON SFP module inserted\n");
+ return 0;
+}
+
+static void gpon_sfp_module_remove(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "GPON SFP module removed\n");
+ WRITE_ONCE(priv->optical_active, false);
+ cancel_delayed_work_sync(&priv->restart_work);
+ gpon_disable(priv);
+}
+
+static int gpon_sfp_module_start(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+ enum gpon_state state;
+ int ret = 0;
+
+ state = ploam_get_state(priv->ploam);
+ dev_info(priv->dev, "GPON SFP module start: state=%s\n",
+ gpon_state_name(state));
+ WRITE_ONCE(priv->optical_active, true);
+
+ /* PHY ready: if we were in emergency stop, stay there. */
+ if (state == GPON_O1_INITIAL) {
+ ret = airoha_xpon_tx_rearm(priv->dev, priv->frontend);
+ if (ret)
+ goto err_inactive;
+
+ ret = gpon_enable(priv);
+ }
+
+ if (ret)
+ goto err_inactive;
+
+ return 0;
+
+err_inactive:
+ WRITE_ONCE(priv->optical_active, false);
+ return ret;
+}
+
+static void gpon_sfp_module_stop(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "GPON SFP module stop\n");
+ WRITE_ONCE(priv->optical_active, false);
+ cancel_delayed_work_sync(&priv->restart_work);
+ gpon_disable(priv);
+}
+
+static void gpon_sfp_link_down(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_warn(priv->dev, "GPON optical link down / LOS\n");
+ ploam_notify_los(priv->ploam);
+}
+
+static void gpon_sfp_link_up(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "GPON optical link up\n");
+ if (READ_ONCE(priv->started) &&
+ READ_ONCE(priv->optical_active) &&
+ !READ_ONCE(priv->mac_enabled))
+ mod_delayed_work(priv->fsm_wq, &priv->restart_work, 0);
+}
+
+static const struct sfp_upstream_ops gpon_sfp_ops = {
+ .attach = gpon_sfp_attach,
+ .detach = gpon_sfp_detach,
+ .module_insert = gpon_sfp_module_insert,
+ .module_remove = gpon_sfp_module_remove,
+ .module_start = gpon_sfp_module_start,
+ .module_stop = gpon_sfp_module_stop,
+ .link_up = gpon_sfp_link_up,
+ .link_down = gpon_sfp_link_down,
+};
+
+/* -----------------------------------------------------------------------
+ * GDM2 xPON lifecycle
+ * -------------------------------------------------------------------- */
+
+static int gpon_link_start(void *data)
+{
+ struct xpon_priv *priv = data;
+
+ /* net/omci owns GPON activation; ndo_open only exposes saved
carrier. */
+ gpon_refresh_netdev_link(priv, true);
+ return 0;
+}
+
+static void gpon_link_stop(void *data)
+{
+ struct xpon_priv *priv = data;
+
+ /* net/omci keeps OMCC and activation alive across ndo_stop. */
+ gpon_refresh_netdev_link(priv, true);
+}
+
+static void gpon_mac_irq(void *data)
+{
+ gpon_isr(0, data);
+}
+
+static const struct airoha_xpon_link_ops gpon_link_ops = {
+ .start = gpon_link_start,
+ .stop = gpon_link_stop,
+ .mac_irq = gpon_mac_irq,
+};
+
+/*
-------------------------------------------------------------------------
+ * EPON implementation
+ *
------------------------------------------------------------------------- */
+static const char *epon_llid_state_name(enum airoha_epon_llid_state state)
+{
+ switch (state) {
+ case AIROHA_EPON_LLID_WAIT:
+ return "wait";
+ case AIROHA_EPON_LLID_REGISTERING:
+ return "registering";
+ case AIROHA_EPON_LLID_REGISTERED:
+ return "registered";
+ default:
+ return "unknown";
+ }
+}
+
+
+
+static inline u32 epon_read(struct xpon_priv *priv, u32 reg)
+{
+ return readl(priv->epon_reg + reg);
+}
+
+static inline void epon_write(struct xpon_priv *priv, u32 reg, u32 val)
+{
+ writel(val, priv->epon_reg + reg);
+}
+
+/* --- LLID discovery status helpers --- */
+
+static u32 epon_llid_sts(struct xpon_priv *priv, int idx)
+{
+ return epon_read(priv, EPON_LLID0_DSCVRY_STS + idx * 4);
+}
+
+static void epon_llid_sts_write(struct xpon_priv *priv, int idx, u32 val)
+{
+ epon_write(priv, EPON_LLID0_DSCVRY_STS + idx * 4, val);
+}
+
+/*
+ * Set LLID HW discovery state to REGISTERING.
+ * Pattern from ref eponMpcpDscvFsmWaitHandler /
eponMpcpDiscvGateIntHandler:
+ * clear bits[31:30] (set unregistered) 10 times, then set bits[31:30]=01
+ * while setting all lower bits (0x7FFFFFFF gives bit30=1, rest all-ones).
+ */
+static void epon_llid_set_registering(struct xpon_priv *priv, int idx)
+{
+ u32 tmp;
+ int i;
+
+ dev_info(priv->dev, "EPON LLID%d -> registering\n", idx);
+
+ for (i = 0; i < 10; i++) {
+ tmp = epon_llid_sts(priv, idx) & 0x3FFFFFFF;
+ epon_llid_sts_write(priv, idx, tmp);
+ }
+ tmp = epon_llid_sts(priv, idx) & 0x3FFFFFFF;
+ tmp |= 0x7FFFFFFF; /* bit30=1 → llidDscvrySts=01 (registering) */
+ epon_llid_sts_write(priv, idx, tmp);
+}
+
+/* --- MAC address programming (indirect via e_mac_addr_cfg) --- */
+
+static int epon_wait_mac_cfg(struct xpon_priv *priv)
+{
+ int i;
+
+ /* MAC_ADDR_DONE=1 means busy; wait for it to clear */
+ for (i = 0; i < 100; i++) {
+ if (!(epon_read(priv, EPON_MAC_ADDR_CFG) & MAC_ADDR_DONE))
+ return 0;
+ }
+ return -ETIMEDOUT;
+}
+
+/*
+ * Program per-LLID source MAC address into hardware table.
+ * Two indirect writes: dw_idx=0 for bytes[2-5] (low 32), dw_idx=1 for
+ * bytes[0-1] (high 16). Matches ref eponMacSetMacAddr().
+ */
+static int epon_program_mac_address(struct xpon_priv *priv, int llid_idx,
+ const u8 mac[ETH_ALEN])
+{
+ u32 mac_low = ((u32)mac[2] << 24) | ((u32)mac[3] << 16) |
+ ((u32)mac[4] << 8) | mac[5];
+ u32 mac_high = ((u32)mac[0] << 8) | mac[1];
+ u32 cfg_base = MAC_ADDR_RWCMD | ((llid_idx & 7) <<
MAC_ADDR_LLID_SHIFT);
+
+ if (epon_wait_mac_cfg(priv) < 0) {
+ dev_err(priv->dev, "LLID%d MAC cfg busy\n", llid_idx);
+ return -ETIMEDOUT;
+ }
+ epon_write(priv, EPON_MAC_ADDR_VALUE, mac_low);
+ epon_write(priv, EPON_MAC_ADDR_CFG, cfg_base); /* dw_idx=0 */
+
+ if (epon_wait_mac_cfg(priv) < 0) {
+ dev_err(priv->dev, "LLID%d MAC cfg timeout (low)\n", llid_idx);
+ return -ETIMEDOUT;
+ }
+ dev_info(priv->dev, "programming EPON LLID%d MAC %pM\n",
+ llid_idx, mac);
+ epon_write(priv, EPON_MAC_ADDR_VALUE, mac_high);
+ epon_write(priv, EPON_MAC_ADDR_CFG, cfg_base | MAC_ADDR_DW_IDX);
/* dw_idx=1 */
+
+ if (epon_wait_mac_cfg(priv) < 0) {
+ dev_err(priv->dev, "LLID%d MAC cfg timeout (high)\n", llid_idx);
+ return -ETIMEDOUT;
+ }
+ return 0;
+}
+
+/* --- MPCP discovery control --- */
+
+/*
+ * Submit one MPCP command. In interrupt mode mpcp_cmd_done is hardware
+ * status; REG_REQ_DONE/REG_ACK_DONE drive the software state transition.
+ */
+static void epon_discv_cmd(struct xpon_priv *priv, u32 cmd, int llid_idx)
+{
+ u32 ctrl = cmd | (llid_idx & DSCVRY_TX_MPCP_LLID_MASK);
+
+ dev_info(priv->dev,
+ "EPON discovery command: llid=%d cmd=%#08x ctrl=%#08x\n",
+ llid_idx, cmd, ctrl);
+ epon_write(priv, EPON_LLID_DSCVRY_CTRL, ctrl);
+}
+
+static void epon_report_registration(struct xpon_priv *priv)
+{
+ if (!priv->xpon)
+ return;
+
+ xpon_device_report_registration(priv->xpon,
+ priv->registered_llids ? XPON_REGISTRATION_OPERATIONAL :
+ XPON_REGISTRATION_REGISTERING);
+}
+
+static void epon_llid_drop(struct xpon_priv *priv, int idx)
+{
+ if (priv->llid[idx].valid) {
+ priv->llid[idx].valid = false;
+ if (priv->registered_llids > 0)
+ priv->registered_llids--;
+ }
+ if (priv->oam)
+ xpon_oam_llid_unregistered(priv->oam, idx);
+}
+
+/* --- Security key --- */
+
+static void epon_set_security_key(struct xpon_priv *priv, int llid_idx,
+ int key_idx, u8 key[16])
+{
+ int dw;
+
+ for (dw = 0; dw < 4; dw++) {
+ u32 cfg = SEC_KEY_WRITE_CMD |
+ ((llid_idx & 7) << SEC_KEY_LLID_SHIFT) |
+ ((key_idx & 1) << SEC_KEY_IDX_SHIFT) |
+ ((dw & 3) << SEC_KEY_DW_SHIFT);
+ u32 data = ((u32)key[dw * 4 + 0] << 24) |
+ ((u32)key[dw * 4 + 1] << 16) |
+ ((u32)key[dw * 4 + 2] << 8) |
+ key[dw * 4 + 3];
+
+ epon_write(priv, EPON_SECURITY_KEY_CFG, cfg);
+ epon_write(priv, EPON_SECURITY_KEY_DATA, data);
+ }
+}
+
+/* --- SW reset sequence (ref: eponMacSwReset) --- */
+
+static void epon_sw_reset(struct xpon_priv *priv)
+{
+ u32 raw;
+
+ dev_info(priv->dev, "resetting EPON MAC (external_reset=%u)\n",
+ !!priv->epon_reset_reg);
+ /* Assert external system-level SW reset if mapped. */
+ if (priv->epon_reset_reg) {
+ raw = readl(priv->epon_reset_reg);
+ writel(raw | EPON_EXT_SW_RST_BIT, priv->epon_reset_reg);
+ udelay(EPON_RESET_DELAY_US);
+ writel(raw & ~EPON_EXT_SW_RST_BIT, priv->epon_reset_reg);
+ }
+
+ /* Assert EPON MAC internal SW reset (GLB_CFG bit 4). */
+ raw = epon_read(priv, EPON_GLB_CFG);
+ epon_write(priv, EPON_GLB_CFG, raw | GLB_CFG_EPON_MAC_SW_RST);
+ udelay(EPON_RESET_DELAY_US);
+ raw &= ~GLB_CFG_EPON_MAC_SW_RST;
+ epon_write(priv, EPON_GLB_CFG, raw);
+ udelay(EPON_RESET_DELAY_US);
+
+ /* Enable RPT_TXPRI_CTRL and write post-reset timing parameters. */
+ raw |= GLB_CFG_RPT_TXPRI_CTRL;
+ epon_write(priv, EPON_GLB_CFG, raw);
+
+ epon_write(priv, EPON_GRD_THRSHLD, EPON_TIMEDRIFT_THRSHLD);
+ epon_write(priv, EPON_TRX_ADJUST_TIME1, EPON_TRX_ADJUST_TIME1_DEF);
+ epon_write(priv, EPON_TRX_ADJUST_TIME2, EPON_TRX_ADJUST_TIME2_DEF);
+ epon_write(priv, EPON_TXFETCH_CFG, EPON_TXFETCH_DEFAULT);
+ dev_info(priv->dev,
+ "EPON reset complete: glb_cfg=%#08x txfetch=%#08x guard=%#08x\n",
+ epon_read(priv, EPON_GLB_CFG),
+ epon_read(priv, EPON_TXFETCH_CFG),
+ epon_read(priv, EPON_GRD_THRSHLD));
+}
+
+/* --- Hardware init --- */
+
+/* Vendor xpon_1g order for EPON on EN7523:
+ * select EPON WAN mode, clear GDM2 GPON release mode, stop the EPON MBI,
+ * initialise MAC/LLID state, enable GDM2/CDM2 channels, then release MBI.
+ */
+static int epon_prepare_hardware(struct xpon_priv *priv)
+{
+ int ret;
+
+ dev_info(priv->dev, "preparing EPON FE and WAN mux\n");
+ ret = airoha_xpon_set_fe_mode(priv->dev, priv->gdm_dev,
+ AIROHA_XPON_MODE_EPON);
+ if (ret)
+ return ret;
+
+ dev_info(priv->dev, "selecting EPON on SCU WAN mux\n");
+ ret = airoha_xpon_select_wan(priv->scu, priv->match_data,
+ AIROHA_XPON_MODE_EPON);
+ if (ret)
+ return dev_err_probe(priv->dev, ret,
+ "failed to select EPON WAN mode\n");
+
+ dev_info(priv->dev, "EPON FE and WAN mux prepared\n");
+ return 0;
+}
+
+static void epon_hw_init(struct xpon_priv *priv)
+{
+ u32 cfg;
+ int i;
+ u8 zero_key[16] = {};
+
+ epon_sw_reset(priv);
+
+ cfg = epon_read(priv, EPON_GLB_CFG);
+ cfg |= GLB_CFG_TXMBI_STOP | GLB_CFG_RXMBI_STOP;
+ cfg |= GLB_CFG_MPCP_FWD | GLB_CFG_FCS_ERR_FWD | GLB_CFG_DISCV_BURST_EN;
+ epon_write(priv, EPON_GLB_CFG, cfg);
+
+ epon_write(priv, EPON_PENDING_GNT_NUM, EPON_PENDING_GNT_DEFAULT);
+ epon_write(priv, EPON_MPCP_TIMEOUT_INTVL, EPON_MPCP_TIMEOUT_DEFAULT);
+ epon_write(priv, EPON_RPT_TIMEOUT_INTVL, EPON_RPT_TIMEOUT_DEFAULT);
+ epon_write(priv, EPON_MAX_FUTURE_GNT, EPON_MAX_FUTURE_GNT_DEFAULT);
+ epon_write(priv, EPON_MIN_PROC_TIME, EPON_MIN_PROC_TIME_DEFAULT);
+ epon_write(priv, EPON_LASER_ONOFF_TIME, EPON_LASER_ONOFF_DEFAULT);
+ epon_write(priv, EPON_TX_CAL_CNST, EPON_TX_CAL_CNST_DEFAULT);
+
+ /* Enable hardware dying gasp detection per ref (write magic value) */
+ epon_write(priv, EPON_DYINGGSP_CFG, DYINGGSP_CFG_HW_ENABLE);
+
+ for (i = 0; i < EPON_MAX_LLID; i++)
+ epon_set_security_key(priv, i, 0, zero_key);
+
+ dev_info(priv->dev,
+ "EPON hardware initialized: glb_cfg=%#08x pending_gnt=%#08x
mpcp_timeout=%#08x\n",
+ epon_read(priv, EPON_GLB_CFG),
+ epon_read(priv, EPON_PENDING_GNT_NUM),
+ epon_read(priv, EPON_MPCP_TIMEOUT_INTVL));
+}
+
+/* --- ISR --- */
+
+static irqreturn_t epon_isr(int irq, void *data)
+{
+ struct xpon_priv *priv = data;
+ u32 enabled, raw, status;
+ int idx;
+
+ raw = epon_read(priv, EPON_INT_STATUS);
+ if (!raw)
+ return IRQ_NONE;
+
+ enabled = epon_read(priv, EPON_INT_EN);
+ status = raw & enabled;
+ /* W1C: acknowledge the complete hardware snapshot. */
+ epon_write(priv, EPON_INT_STATUS, raw);
+ if (!status)
+ return IRQ_HANDLED;
+
+ dev_info(priv->dev,
+ "EPON IRQ: status=%#08x enabled=%#08x registered_llids=%d\n",
+ status, enabled, priv->registered_llids);
+
+ if (status & EPON_INT_TIMEDRFT) {
+ u32 drift = epon_read(priv, EPON_TIME_DRFT_STAT) & 0xff;
+
+ dev_info(priv->dev, "EPON: time drift %u\n", drift);
+ epon_write(priv, EPON_TIME_DRFT_STAT, 0);
+ }
+
+ if (status & EPON_INT_MPCP_TIMEOUT) {
+ u32 tmout = epon_read(priv, EPON_RPT_MPCP_TIMEOUT);
+ u8 llidmask = (tmout >> 16) & 0xff;
+
+ for (idx = 0; idx < EPON_MAX_LLID; idx++) {
+ if (!(llidmask & BIT(idx)))
+ continue;
+
+ dev_info(priv->dev, "EPON: MPCP timeout LLID%d\n", idx);
+ epon_llid_drop(priv, idx);
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTERING;
+ epon_llid_set_registering(priv, idx);
+ }
+ epon_write(priv, EPON_RPT_MPCP_TIMEOUT, tmout & 0x0000ff00);
+ if (!priv->registered_llids)
+ airoha_xpon_update_netdev_link(priv, false);
+ epon_report_registration(priv);
+ }
+
+ /*
+ * A discovery GATE starts exactly one REGISTER_REQUEST. Completion is
+ * reported by REG_REQ_DONE; the command-done bit is not
software-owned.
+ */
+ if (status & EPON_INT_DISCV_GATE) {
+ if (priv->xpon && !priv->registered_llids)
+ xpon_device_report_registration(priv->xpon,
+ XPON_REGISTRATION_DISCOVERY);
+ dev_info(priv->dev, "EPON discovery GATE received\n");
+ for (idx = 0; idx < EPON_MAX_LLID; idx++) {
+ if (priv->llid[idx].state != AIROHA_EPON_LLID_REGISTERING)
+ continue;
+ epon_llid_set_registering(priv, idx);
+ epon_discv_cmd(priv, DSCVRY_MPCP_REG_REQ, idx);
+ break;
+ }
+ }
+
+ if (status & EPON_INT_REG_REQ_DONE) {
+ idx = epon_read(priv, EPON_LLID_DSCVRY_CTRL) &
+ DSCVRY_TX_MPCP_LLID_MASK;
+ if (idx < EPON_MAX_LLID &&
+ priv->llid[idx].state == AIROHA_EPON_LLID_REGISTERING) {
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTER_REQUEST;
+ dev_info(priv->dev,
+ "EPON LLID%d REGISTER_REQUEST sent\n", idx);
+ if (priv->xpon && !priv->registered_llids)
+ xpon_device_report_registration(priv->xpon,
+ XPON_REGISTRATION_REGISTERING);
+ }
+ }
+
+ /* REGISTER frames are reported by bits 1..8, one bit per LLID slot. */
+ for (idx = 0; idx < EPON_MAX_LLID; idx++) {
+ u32 sts;
+ int flag;
+ u8 mac[ETH_ALEN];
+ u32 mac_low;
+
+ if (!(status & (EPON_INT_LLID0_RGST << idx)))
+ continue;
+
+ sts = epon_llid_sts(priv, idx);
+ flag = (sts >> LLID_STS_RGST_FLG_SHIFT) & 3;
+ dev_info(priv->dev,
+ "EPON LLID%d registration event: flag=%d status=%#08x
state=%s valid=%u\n",
+ idx, flag, sts,
+ epon_llid_state_name(priv->llid[idx].state),
+ priv->llid[idx].valid);
+
+ switch (flag) {
+ case MPCP_REG_ACK:
+ if (priv->llid[idx].state != AIROHA_EPON_LLID_REGISTER_REQUEST)
+ break;
+ if (!(sts & LLID_STS_VALID)) {
+ dev_err(priv->dev,
+ "EPON: LLID%d ACK without a valid LLID\n", idx);
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTERING;
+ break;
+ }
+
+ priv->llid[idx].value = sts & LLID_STS_VALUE_MASK;
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTER_PENDING;
+
+ ether_addr_copy(mac, priv->gdm_dev->dev_addr);
+ mac_low = ((u32)mac[2] << 24) | ((u32)mac[3] << 16) |
+ ((u32)mac[4] << 8) | mac[5];
+ mac_low += idx;
+ mac[3] = (mac_low >> 16) & 0xff;
+ mac[4] = (mac_low >> 8) & 0xff;
+ mac[5] = mac_low & 0xff;
+ epon_program_mac_address(priv, idx, mac);
+
+ epon_discv_cmd(priv,
+ DSCVRY_MPCP_ACK | DSCVRY_RGSTR_ACK_FLG,
+ idx);
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTER_ACK;
+ break;
+
+ case MPCP_REG_NACK:
+ dev_info(priv->dev, "EPON: LLID%d NACK, retrying\n", idx);
+ epon_llid_drop(priv, idx);
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTERING;
+ epon_llid_set_registering(priv, idx);
+ if (!priv->registered_llids)
+ airoha_xpon_update_netdev_link(priv, false);
+ epon_report_registration(priv);
+ break;
+
+ case MPCP_REG_DE_REGISTER:
+ dev_info(priv->dev, "EPON: LLID%d deregistered\n", idx);
+ epon_llid_drop(priv, idx);
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTERING;
+ epon_llid_set_registering(priv, idx);
+ if (!priv->registered_llids)
+ airoha_xpon_update_netdev_link(priv, false);
+ epon_report_registration(priv);
+ break;
+
+ case MPCP_REG_RE_REGISTER:
+ if (priv->llid[idx].state != AIROHA_EPON_LLID_REGISTERED)
+ break;
+ dev_info(priv->dev, "EPON: LLID%d re-register\n", idx);
+ epon_llid_drop(priv, idx);
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTER_PENDING;
+ epon_discv_cmd(priv,
+ DSCVRY_MPCP_ACK | DSCVRY_RGSTR_ACK_FLG,
+ idx);
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTER_ACK;
+ if (!priv->registered_llids)
+ airoha_xpon_update_netdev_link(priv, false);
+ epon_report_registration(priv);
+ break;
+ }
+ }
+
+ /*
+ * REGISTER_ACK completion is the point at which the LLID becomes
+ * operational. This follows the vendor interrupt-mode MPCP FSM.
+ */
+ if (status & EPON_INT_REG_ACK_DONE) {
+ u32 sts;
+
+ idx = epon_read(priv, EPON_LLID_DSCVRY_CTRL) &
+ DSCVRY_TX_MPCP_LLID_MASK;
+ if (idx < EPON_MAX_LLID &&
+ priv->llid[idx].state == AIROHA_EPON_LLID_REGISTER_ACK) {
+ sts = epon_llid_sts(priv, idx);
+ sts &= 0x3fffffff;
+ sts |= 2U << LLID_STS_DSCVRY_SHIFT;
+ epon_llid_sts_write(priv, idx, sts);
+
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTERED;
+ if (!priv->llid[idx].valid) {
+ priv->llid[idx].valid = true;
+ priv->registered_llids++;
+ }
+ if (priv->oam)
+ xpon_oam_llid_registered(priv->oam, idx,
+ priv->llid[idx].value);
+ dev_info(priv->dev,
+ "EPON LLID%d registered: 0x%04x\n",
+ idx, priv->llid[idx].value);
+ airoha_xpon_update_netdev_link(priv, true);
+ epon_report_registration(priv);
+ }
+ }
+
+ if (status & EPON_INT_RPT_OVRFLW) {
+ u32 tmout = epon_read(priv, EPON_RPT_MPCP_TIMEOUT);
+
+ epon_write(priv, EPON_RPT_MPCP_TIMEOUT, tmout & 0x000000ff);
+ }
+
+ return IRQ_HANDLED;
+}
+
+/* --- Enable / Disable --- */
+
+static int epon_enable(struct xpon_priv *priv)
+{
+ int idx, ret;
+ u32 int_en;
+
+ if (READ_ONCE(priv->mac_enabled))
+ return 0;
+
+ dev_info(priv->dev, "starting EPON MAC\n");
+ ret = airoha_xpon_reset_mac(priv);
+ if (ret)
+ goto err_disable_frontend;
+
+ ret = airoha_xpon_phy_start(priv->dev, priv->phy,
+ AIROHA_XPON_MODE_EPON,
+ &priv->phy_initialized,
+ &priv->phy_powered);
+ if (ret)
+ goto err_disable_frontend;
+
+ ret = epon_prepare_hardware(priv);
+ if (ret)
+ goto err_stop_phy;
+
+ epon_hw_init(priv);
+
+ /* Put all LLIDs into REGISTERING state */
+ for (idx = 0; idx < EPON_MAX_LLID; idx++) {
+ priv->llid[idx].state = AIROHA_EPON_LLID_REGISTERING;
+ priv->llid[idx].valid = false;
+ epon_llid_set_registering(priv, idx);
+ }
+
+ /* Base interrupts */
+ int_en = EPON_INT_DISCV_GATE |
+ EPON_INT_GNT_OVRRUN |
+ EPON_INT_TIMEDRFT |
+ EPON_INT_MPCP_TIMEOUT |
+ EPON_INT_RPT_OVRFLW |
+ EPON_INT_REG_REQ_DONE |
+ EPON_INT_REG_ACK_DONE;
+
+ /* Per-LLID register-frame interrupts */
+ for (idx = 0; idx < EPON_MAX_LLID; idx++)
+ int_en |= (EPON_INT_LLID0_RGST << idx);
+
+ epon_write(priv, EPON_INT_EN, 0);
+ epon_write(priv, EPON_INT_STATUS, ~0U);
+
+ ret = airoha_xpon_set_fe_datapath(priv->dev, priv->gdm_dev,
+ AIROHA_XPON_MODE_EPON, true);
+ if (ret) {
+ epon_write(priv, EPON_INT_EN, 0);
+ goto err_stop_phy;
+ }
+
+ ret = airoha_xpon_tx_enable(priv->dev, priv->frontend, true);
+ if (ret)
+ goto err_disable_datapath;
+
+ /* Release EPON TX/RX MBI only after GDM2/CDM2 channels are ready. */
+ epon_write(priv, EPON_GLB_CFG,
+ epon_read(priv, EPON_GLB_CFG) &
+ ~(GLB_CFG_TXMBI_STOP | GLB_CFG_RXMBI_STOP));
+ usleep_range(50, 100);
+
+ WRITE_ONCE(priv->mac_enabled, true);
+ WRITE_ONCE(priv->phy_link_known, false);
+ mod_delayed_work(priv->fsm_wq, &priv->phy_link_work, 0);
+ epon_write(priv, EPON_INT_STATUS, ~0U);
+ epon_write(priv, EPON_INT_EN, int_en);
+ dev_info(priv->dev, "EPON interrupt mask=%#08x irq=%d\n",
+ int_en, priv->irq);
+ dev_info(priv->dev,
+ "EPON MAC started: glb_cfg=%#08x int_enable=%#08x\n",
+ epon_read(priv, EPON_GLB_CFG), epon_read(priv, EPON_INT_EN));
+
+ return 0;
+
+err_disable_datapath:
+ airoha_xpon_tx_enable(priv->dev, priv->frontend, false);
+ airoha_xpon_set_fe_datapath(priv->dev, priv->gdm_dev,
+ AIROHA_XPON_MODE_EPON, false);
+ goto err_stop_phy_only;
+err_stop_phy:
+ airoha_xpon_tx_enable(priv->dev, priv->frontend, false);
+err_stop_phy_only:
+ airoha_xpon_phy_stop(priv->dev, priv->phy,
+ AIROHA_XPON_MODE_EPON,
+ &priv->phy_initialized,
+ &priv->phy_powered);
+ return ret;
+err_disable_frontend:
+ airoha_xpon_tx_enable(priv->dev, priv->frontend, false);
+ return ret;
+}
+
+static void epon_disable(struct xpon_priv *priv)
+{
+ int idx, ret;
+
+ airoha_xpon_tx_enable(priv->dev, priv->frontend, false);
+
+ if (!READ_ONCE(priv->mac_enabled))
+ return;
+
+ dev_info(priv->dev,
+ "stopping EPON MAC: registered_llids=%d glb_cfg=%#08x\n",
+ priv->registered_llids, epon_read(priv, EPON_GLB_CFG));
+ WRITE_ONCE(priv->mac_enabled, false);
+ cancel_delayed_work_sync(&priv->phy_link_work);
+ WRITE_ONCE(priv->phy_link_known, false);
+ WRITE_ONCE(priv->phy_link_up, false);
+ epon_write(priv, EPON_INT_EN, 0);
+ epon_write(priv, EPON_INT_STATUS, ~0U);
+ synchronize_irq(priv->irq);
+ epon_write(priv, EPON_GLB_CFG,
+ epon_read(priv, EPON_GLB_CFG) |
+ GLB_CFG_TXMBI_STOP | GLB_CFG_RXMBI_STOP);
+ ret = airoha_xpon_set_fe_datapath(priv->dev, priv->gdm_dev,
+ AIROHA_XPON_MODE_EPON, false);
+ if (ret)
+ dev_warn(priv->dev, "failed to disable EPON datapath: %d\n", ret);
+
+ airoha_xpon_phy_stop(priv->dev, priv->phy,
+ AIROHA_XPON_MODE_EPON,
+ &priv->phy_initialized,
+ &priv->phy_powered);
+
+ for (idx = 0; idx < EPON_MAX_LLID; idx++) {
+ priv->llid[idx].state = AIROHA_EPON_LLID_WAIT;
+ priv->llid[idx].valid = false;
+ }
+ priv->registered_llids = 0;
+ airoha_xpon_update_netdev_link(priv, false);
+ dev_info(priv->dev, "EPON MAC stopped: glb_cfg=%#08x\n",
+ epon_read(priv, EPON_GLB_CFG));
+}
+
+/* ---------- SFP upstream ops ---------- */
+
+static void epon_sfp_attach(void *upstream, struct sfp_bus *bus)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "EPON SFP bus attached\n");
+}
+
+static void epon_sfp_detach(void *upstream, struct sfp_bus *bus)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "EPON SFP bus detached\n");
+}
+
+static int epon_sfp_module_insert(void *upstream,
+ const struct sfp_eeprom_id *id)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "EPON SFP module inserted\n");
+ return 0;
+}
+
+static void epon_sfp_module_remove(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "EPON SFP module removed\n");
+ WRITE_ONCE(priv->optical_active, false);
+ epon_disable(priv);
+}
+
+static int epon_sfp_module_start(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+ int ret;
+
+ dev_info(priv->dev, "EPON SFP module start\n");
+
+ ret = airoha_xpon_tx_rearm(priv->dev, priv->frontend);
+ if (ret)
+ return ret;
+
+ ret = epon_enable(priv);
+ if (!ret)
+ WRITE_ONCE(priv->optical_active, true);
+
+ return ret;
+}
+
+static void epon_sfp_module_stop(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "EPON SFP module stop\n");
+ WRITE_ONCE(priv->optical_active, false);
+ epon_disable(priv);
+}
+
+static void epon_sfp_link_down(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_warn(priv->dev, "EPON optical link down / LOS\n");
+ airoha_xpon_update_netdev_link(priv, false);
+}
+
+static void epon_sfp_link_up(void *upstream)
+{
+ struct xpon_priv *priv = upstream;
+
+ dev_info(priv->dev, "EPON optical link up\n");
+}
+
+static const struct sfp_upstream_ops epon_sfp_ops = {
+ .attach = epon_sfp_attach,
+ .detach = epon_sfp_detach,
+ .module_insert = epon_sfp_module_insert,
+ .module_remove = epon_sfp_module_remove,
+ .module_start = epon_sfp_module_start,
+ .module_stop = epon_sfp_module_stop,
+ .link_up = epon_sfp_link_up,
+ .link_down = epon_sfp_link_down,
+};
+
+/* ---------- GDM2 xPON lifecycle ---------- */
+
+static int epon_link_start(void *data)
+{
+ struct xpon_priv *priv = data;
+ int ret = 0;
+
+ if (READ_ONCE(priv->started))
+ return 0;
+ WRITE_ONCE(priv->started, true);
+ airoha_xpon_update_netdev_link(priv, false);
+ if (priv->sfp_bus) {
+ if (!READ_ONCE(priv->optical_active))
+ sfp_upstream_start(priv->sfp_bus);
+ } else {
+ ret = airoha_xpon_tx_rearm(priv->dev, priv->frontend);
+ if (!ret)
+ ret = epon_enable(priv);
+ if (!ret)
+ WRITE_ONCE(priv->optical_active, true);
+ }
+
+ if (ret)
+ WRITE_ONCE(priv->started, false);
+
+ return ret;
+}
+
+static void epon_link_stop(void *data)
+{
+ struct xpon_priv *priv = data;
+
+ if (!READ_ONCE(priv->started))
+ return;
+ WRITE_ONCE(priv->started, false);
+ airoha_xpon_update_netdev_link(priv, false);
+ /* Keep optical activation alive while userspace cycles pon0. */
+}
+
+static void epon_mac_irq(void *data)
+{
+ epon_isr(0, data);
+}
+
+static const struct airoha_xpon_link_ops epon_link_ops = {
+ .start = epon_link_start,
+ .stop = epon_link_stop,
+ .mac_irq = epon_mac_irq,
+};
+
+/*
-------------------------------------------------------------------------
+ * Unified platform driver
+ *
------------------------------------------------------------------------- */
+
+static const struct airoha_xpon_match_data en7523_xpon_data = {
+ .mode = AIROHA_XPON_MODE_GPON,
+ .mode_from_dt = true,
+ .wan_mode_mask = EN7523_SCU_WAN_MODE_MASK,
+ .gpon_fine_delay = DBG_DLY_FINE_INT_DEFAULT,
+ .gpon_rsp_time_activation = GPON_RSP_TIME_ACT_EN7523,
+ .en7523_gpon_defaults = true,
+ .gpon_reset_on_start = true,
+};
+
+static const struct airoha_xpon_match_data en7523_gpon_data = {
+ .mode = AIROHA_XPON_MODE_GPON,
+ .wan_mode_mask = EN7523_SCU_WAN_MODE_MASK,
+ .gpon_fine_delay = DBG_DLY_FINE_INT_DEFAULT,
+ .gpon_rsp_time_activation = GPON_RSP_TIME_ACT_EN7523,
+ .en7523_gpon_defaults = true,
+ .gpon_reset_on_start = true,
+};
+
+static const struct airoha_xpon_match_data en7523_epon_data = {
+ .mode = AIROHA_XPON_MODE_EPON,
+ .wan_mode_mask = EN7523_SCU_WAN_MODE_MASK,
+ .gpon_fine_delay = DBG_DLY_FINE_INT_DEFAULT,
+ .gpon_rsp_time_activation = GPON_RSP_TIME_ACT_EN7523,
+ .en7523_gpon_defaults = true,
+};
+
+static const struct airoha_xpon_match_data en7528_xpon_data = {
+ .mode = AIROHA_XPON_MODE_GPON,
+ .mode_from_dt = true,
+ .wan_mode_mask = EN7528_SCU_WAN_MODE_MASK,
+ .gpon_fine_delay = 0x1c,
+ .gpon_rsp_time_activation = GPON_RSP_TIME_ACT_EN7528,
+ .gpon_reset_on_start = true,
+};
+
+static const struct airoha_xpon_match_data en751221_xpon_data = {
+ .mode = AIROHA_XPON_MODE_GPON,
+ .mode_from_dt = true,
+ .wan_mode_mask = EN751221_SCU_WAN_MODE_MASK,
+ .gpon_fine_delay = 0x1c,
+ .gpon_rsp_time_activation = GPON_RSP_TIME_ACT_EN751221,
+ .mac_irq_via_eth = true,
+ .prepare_before_mmio = true,
+};
+
+static const struct airoha_xpon_match_data en751221_gpon_data = {
+ .mode = AIROHA_XPON_MODE_GPON,
+ .wan_mode_mask = EN751221_SCU_WAN_MODE_MASK,
+ .gpon_fine_delay = 0x1c,
+ .gpon_rsp_time_activation = GPON_RSP_TIME_ACT_EN751221,
+ .mac_irq_via_eth = true,
+ .prepare_before_mmio = true,
+};
+
+static const struct airoha_xpon_match_data en751221_epon_data = {
+ .mode = AIROHA_XPON_MODE_EPON,
+ .wan_mode_mask = EN751221_SCU_WAN_MODE_MASK,
+ .gpon_fine_delay = 0x1c,
+ .gpon_rsp_time_activation = GPON_RSP_TIME_ACT_EN751221,
+ .mac_irq_via_eth = true,
+ .prepare_before_mmio = true,
+};
+
+static bool airoha_xpon_is_gpon(struct xpon_priv *priv)
+{
+ return priv->mode == AIROHA_XPON_MODE_GPON;
+}
+
+static const struct sfp_upstream_ops *
+airoha_xpon_get_sfp_ops(struct xpon_priv *priv)
+{
+ if (airoha_xpon_is_gpon(priv))
+ return &gpon_sfp_ops;
+
+ return &epon_sfp_ops;
+}
+
+static const struct airoha_xpon_link_ops *
+airoha_xpon_get_link_ops(struct xpon_priv *priv)
+{
+ if (airoha_xpon_is_gpon(priv))
+ return &gpon_link_ops;
+
+ return &epon_link_ops;
+}
+
+static int airoha_xpon_request_mac_irq(struct platform_device *pdev,
+ struct xpon_priv *priv,
+ irq_handler_t handler)
+{
+ struct device *dev = &pdev->dev;
+ int ret;
+
+ if (priv->match_data->mac_irq_via_eth) {
+ priv->irq = -1;
+ dev_info(dev,
+ "%s MAC interrupt is routed through the EN751221 QDMA1
aggregator\n",
+ airoha_xpon_mode_name(priv->mode));
+ return 0;
+ }
+
+ priv->irq = platform_get_irq_byname(pdev, "mac");
+ if (priv->irq < 0)
+ priv->irq = platform_get_irq(pdev, 0);
+ if (priv->irq < 0)
+ return dev_err_probe(dev, priv->irq, "needs mac irq\n");
+
+ ret = devm_request_irq(dev, priv->irq, handler, 0, dev_name(dev),
priv);
+ if (ret)
+ return ret;
+
+ dev_info(dev, "%s IRQ %d requested; MAC interrupt mask controls
delivery\n",
+ airoha_xpon_mode_name(priv->mode), priv->irq);
+
+ return 0;
+}
+
+static int
+airoha_xpon_request_dying_gasp_irq(struct platform_device *pdev,
+ struct xpon_priv *priv)
+{
+ struct device *dev = &pdev->dev;
+ int ret;
+
+ priv->dying_gasp_irq =
+ platform_get_irq_byname_optional(pdev, "dying-gasp");
+ if (priv->dying_gasp_irq == -ENXIO) {
+ priv->dying_gasp_irq = -1;
+ dev_info(dev, "no dedicated dying-gasp IRQ, using GPON MAC
event\n");
+ return 0;
+ }
+ if (priv->dying_gasp_irq < 0)
+ return dev_err_probe(dev, priv->dying_gasp_irq,
+ "failed to get dying-gasp IRQ\n");
+
+ ret = devm_request_irq(dev, priv->dying_gasp_irq,
+ gpon_dying_gasp_isr, 0,
+ "airoha-xpon-dying-gasp", priv);
+ if (ret)
+ return dev_err_probe(dev, ret,
+ "failed to request dying-gasp IRQ\n");
+
+ dev_info(dev, "dedicated dying-gasp IRQ %d requested\n",
+ priv->dying_gasp_irq);
+ return 0;
+}
+
+static int airoha_xpon_init_gpon(struct platform_device *pdev,
+ struct xpon_priv *priv)
+{
+ struct device *dev = &pdev->dev;
+ int ret;
+
+ if (!priv->gpon_reg)
+ return dev_err_probe(dev, -EINVAL,
+ "missing GPON register window\n");
+
+ /* The FSM synchronously drains OMCI work queued on system_wq during
+ * session teardown. It is not part of the memory-reclaim path, so do
+ * not mark it WQ_MEM_RECLAIM: a reclaim worker must not flush a
+ * non-reclaim workqueue.
+ */
+ priv->fsm_wq = alloc_ordered_workqueue("%s-gpon-fsm", 0,
+ dev_name(dev));
+ if (!priv->fsm_wq)
+ return -ENOMEM;
+
+ mutex_init(&priv->tcont_lock);
+ mutex_init(&priv->omci_profile_lock);
+ mutex_init(&priv->omci_config_lock);
+ mutex_init(&priv->link_state_lock);
+ bitmap_zero(priv->service_gems, GPON_MAX_GEM_ID);
+ memset(priv->tcont_alloc_id, 0xff, sizeof(priv->tcont_alloc_id));
+ memset(priv->tcont_entity_id, 0xff, sizeof(priv->tcont_entity_id));
+
+ INIT_WORK(&priv->irq_work, gpon_irq_work_fn);
+ INIT_DELAYED_WORK(&priv->to1_work, gpon_to1_work_fn);
+ INIT_DELAYED_WORK(&priv->to2_work, gpon_to2_work_fn);
+ INIT_DELAYED_WORK(&priv->restart_work, gpon_restart_work_fn);
+ atomic_set(&priv->pending_irqs, 0);
+
+ /* Keep the Linux IRQ line enabled and quiesce the MAC at its
source. */
+ gpon_write(priv, GPON_INT_ENABLE, 0);
+ gpon_write(priv, GPON_INT_STATUS, ~0U);
+ ret = airoha_xpon_request_mac_irq(pdev, priv, gpon_isr);
+ if (ret)
+ goto err_destroy_fsm_wq;
+
+ ret = airoha_xpon_request_dying_gasp_irq(pdev, priv);
+ if (ret)
+ goto err_destroy_fsm_wq;
+
+ ret = gpon_load_credentials(priv);
+ if (ret)
+ goto err_destroy_fsm_wq;
+
+ priv->ber_interval_ms = 1000;
+ timer_setup(&priv->ber_timer, gpon_ber_timer_fn, 0);
+
+ priv->ploam = ploam_alloc(&gpon_ploam_ops, priv, priv->hw_sn,
+ priv->hw_passwd);
+ if (!priv->ploam) {
+ ret = -ENOMEM;
+ goto err_destroy_fsm_wq;
+ }
+
+ return 0;
+
+err_destroy_fsm_wq:
+ destroy_workqueue(priv->fsm_wq);
+ priv->fsm_wq = NULL;
+ return ret;
+}
+
+static void airoha_xpon_cleanup_gpon(struct xpon_priv *priv)
+{
+ ploam_free(priv->ploam);
+ priv->ploam = NULL;
+
+ if (priv->fsm_wq) {
+ destroy_workqueue(priv->fsm_wq);
+ priv->fsm_wq = NULL;
+ }
+}
+
+static int airoha_xpon_init_epon(struct platform_device *pdev,
+ struct xpon_priv *priv)
+{
+ int ret;
+
+ if (!priv->epon_reg)
+ return dev_err_probe(&pdev->dev, -EINVAL,
+ "missing EPON register window\n");
+
+ priv->fsm_wq = alloc_ordered_workqueue("airoha-epon", WQ_MEM_RECLAIM);
+ if (!priv->fsm_wq)
+ return -ENOMEM;
+
+ /* Keep the Linux IRQ line enabled and quiesce the MAC at its
source. */
+ epon_write(priv, EPON_INT_EN, 0);
+ epon_write(priv, EPON_INT_STATUS, ~0U);
+ ret = airoha_xpon_request_mac_irq(pdev, priv, epon_isr);
+ if (ret) {
+ destroy_workqueue(priv->fsm_wq);
+ priv->fsm_wq = NULL;
+ }
+
+ return ret;
+}
+
+static void airoha_xpon_cleanup_epon(struct xpon_priv *priv)
+{
+ if (!priv->fsm_wq)
+ return;
+ destroy_workqueue(priv->fsm_wq);
+ priv->fsm_wq = NULL;
+}
+
+static int airoha_xpon_register_gpon_omci(struct xpon_priv *priv)
+{
+ int ret;
+
+ priv->omci_handler.rx = airoha_gpon_omci_receive;
+ priv->omci_handler.priv = &priv->omci;
+ ret = airoha_eth_register_xpon_oam(priv->gdm_dev,
+ &priv->omci_handler);
+ if (ret)
+ return ret;
+
+ ret = airoha_gpon_omci_register(&priv->omci, priv->xpon, priv->gdm_dev,
+ priv, &priv->identity);
+ if (ret)
+ goto err_unregister_oam;
+
+ /*
+ * The OMCI device is zero-initialized. Publish O1 before the first
+ * PLOAM state callback so userspace never observes an invalid O0.
+ */
+ airoha_gpon_omci_set_state(&priv->omci, GPON_O1_INITIAL);
+ ret = airoha_gpon_omci_start(&priv->omci);
+ if (ret)
+ goto err_unregister_omci;
+
+ return 0;
+
+err_unregister_omci:
+ airoha_gpon_omci_stop(&priv->omci);
+err_unregister_oam:
+ airoha_eth_unregister_xpon_oam(priv->gdm_dev, &priv->omci_handler);
+ airoha_gpon_omci_unregister(&priv->omci);
+ return ret;
+}
+
+static void airoha_xpon_unregister_gpon_omci(struct xpon_priv *priv)
+{
+ /* Drain the transport before detaching its RX handler. */
+ airoha_gpon_omci_stop(&priv->omci);
+ airoha_eth_unregister_xpon_oam(priv->gdm_dev, &priv->omci_handler);
+ airoha_gpon_omci_unregister(&priv->omci);
+}
+
+static int airoha_xpon_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ const struct airoha_xpon_match_data *data;
+ const struct airoha_xpon_link_ops *link_ops;
+ const struct sfp_upstream_ops *sfp_ops;
+ struct device_node *eth_node;
+ struct xpon_device_desc xpon_desc = {};
+ struct xpon_priv *priv;
+ struct resource *res;
+ int ret;
+
+ data = device_get_match_data(&pdev->dev);
+ if (!data)
+ return -EINVAL;
+
+ priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+ priv->dev = dev;
+ priv->match_data = data;
+ ret = airoha_xpon_get_mode(dev, data, &priv->mode);
+ if (ret)
+ return ret;
+ INIT_DELAYED_WORK(&priv->phy_link_work,
+ airoha_xpon_phy_link_work_fn);
+
+ dev_info(dev, "xPON initial mode: %s\n",
+ airoha_xpon_mode_name(priv->mode));
+
+ priv->scu = syscon_regmap_lookup_by_phandle(dev->of_node,
+ "airoha,scu");
+ if (IS_ERR(priv->scu)) {
+ ret = dev_err_probe(dev, PTR_ERR(priv->scu),
+ "failed to get SCU regmap\n");
+ goto err_put_gdm;
+ }
+
+ priv->mac_reset = devm_reset_control_get_optional_exclusive(dev,
"mac");
+ if (IS_ERR(priv->mac_reset)) {
+ ret = dev_err_probe(dev, PTR_ERR(priv->mac_reset),
+ "failed to get xPON MAC reset\n");
+ goto err_put_gdm;
+ }
+ if (!priv->mac_reset)
+ dev_warn(dev,
+ "missing xPON MAC reset; session restarts cannot clear all
hardware state\n");
+
+ priv->phy = devm_phy_get(dev, "xpon");
+ if (IS_ERR(priv->phy)) {
+ ret = dev_err_probe(dev, PTR_ERR(priv->phy),
+ "failed to get digital xPON PHY\n");
+ goto err_put_gdm;
+ }
+
+ eth_node = of_parse_phandle(dev->of_node, "ethernet", 0);
+ if (eth_node) {
+ priv->gdm_dev = of_find_net_device_by_node(eth_node);
+ dev_info(dev, "%s datapath phandle: %pOF\n",
+ airoha_xpon_mode_name(priv->mode), eth_node);
+ of_node_put(eth_node);
+ } else {
+ priv->gdm_dev = airoha_eth_get_xpon_netdev();
+ dev_info(dev, "%s datapath discovered automatically\n",
+ airoha_xpon_mode_name(priv->mode));
+ }
+ if (!priv->gdm_dev) {
+ ret = dev_err_probe(dev, -EPROBE_DEFER,
+ "GDM2 netdev is not registered yet\n");
+ goto err_put_gdm;
+ }
+ dev_info(dev, "resolved xPON datapath to %s\n",
+ priv->gdm_dev->name);
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mac");
+ if (!res)
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ ret = dev_err_probe(dev, -EINVAL, "missing mac resource\n");
+ goto err_put_gdm;
+ }
+
+ priv->base = devm_ioremap_resource(dev, res);
+ if (IS_ERR(priv->base)) {
+ ret = dev_err_probe(dev, PTR_ERR(priv->base),
+ "needs xPON mac base\n");
+ goto err_put_gdm;
+ }
+
+ /*
+ * All supported xPON MAC blocks expose GPON and EPON at fixed offsets.
+ * Add a larger resource-size case here when XGSPON support lands.
+ */
+ if (resource_size(res) < V1_XPON_REGION_SIZE) {
+ ret = dev_err_probe(dev, -EINVAL,
+ "unsupported xPON MAC resource size %#llx: %pR\n",
+ (unsigned long long)resource_size(res),
+ res);
+ goto err_put_gdm;
+ }
+
+ /*
+ * The EN751221 xPON_1g SDK selects the shared WAN mux before the
+ * first GPON/EPON MAC register initialization. Keep that ordering:
+ * RST_CTRL1[31] is a runtime MAC reset, not a prerequisite for mapping
+ * or quiescing the interrupt registers during probe.
+ */
+ if (data->prepare_before_mmio) {
+ dev_info(dev, "selecting %s WAN mode before first MAC access\n",
+ airoha_xpon_mode_name(priv->mode));
+ ret = airoha_xpon_select_wan(priv->scu, data, priv->mode);
+ if (ret) {
+ ret = dev_err_probe(dev, ret,
+ "failed to prepare EN751221 xPON WAN mux\n");
+ goto err_put_gdm;
+ }
+
+ dev_info(dev, "%s WAN mode selected before first MAC access\n",
+ airoha_xpon_mode_name(priv->mode));
+ }
+
+ priv->gpon_reg = priv->base + GPON_REG_OFFSET;
+ priv->epon_reg = priv->base + EPON_REG_OFFSET;
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+ "epon-reset");
+ if (res) {
+ priv->epon_reset_reg = devm_ioremap_resource(dev, res);
+ if (IS_ERR(priv->epon_reset_reg)) {
+ ret = dev_err_probe(dev, PTR_ERR(priv->epon_reset_reg),
+ "failed to map EPON reset register\n");
+ goto err_put_gdm;
+ }
+ }
+
+ if (airoha_xpon_is_gpon(priv))
+ ret = airoha_xpon_init_gpon(pdev, priv);
+ else
+ ret = airoha_xpon_init_epon(pdev, priv);
+ if (ret)
+ goto err_put_gdm;
+
+ priv->frontend = devm_optical_frontend_get_optional(dev, "pon");
+ if (IS_ERR(priv->frontend)) {
+ ret = dev_err_probe(dev, PTR_ERR(priv->frontend),
+ "failed to get optical frontend\n");
+ priv->frontend = NULL;
+ goto err_cleanup_mode;
+ }
+
+ ret = airoha_xpon_frontend_set_mode(priv);
+ if (ret) {
+ ret = dev_err_probe(dev, ret,
+ "failed to configure optical frontend mode\n");
+ goto err_cleanup_mode;
+ }
+
+ xpon_desc.netdev = priv->gdm_dev;
+ xpon_desc.optical = priv->frontend ?
+ optical_frontend_get_device(priv->frontend) : NULL;
+ xpon_desc.modes = XPON_MODE_CAP(XPON_MODE_GPON) |
+ XPON_MODE_CAP(XPON_MODE_EPON);
+ xpon_desc.mode = airoha_xpon_core_mode(priv->mode);
+ xpon_desc.priv = priv;
+ priv->xpon = xpon_device_register(dev, &xpon_desc);
+ if (IS_ERR(priv->xpon)) {
+ ret = dev_err_probe(dev, PTR_ERR(priv->xpon),
+ "failed to register generic xPON device\n");
+ priv->xpon = NULL;
+ goto err_cleanup_mode;
+ }
+
+ priv->sfp_bus = sfp_bus_find_fwnode(dev->fwnode);
+ if (IS_ERR(priv->sfp_bus)) {
+ ret = PTR_ERR(priv->sfp_bus);
+ dev_err(dev, "failed to find SFP bus: %d\n", ret);
+ priv->sfp_bus = NULL;
+ goto err_unregister_core;
+ }
+ if (!priv->sfp_bus && !priv->frontend) {
+ ret = -ENODEV;
+ dev_err(dev, "missing SFP or optical frontend reference\n");
+ goto err_unregister_core;
+ }
+
+ if (priv->sfp_bus) {
+ sfp_ops = airoha_xpon_get_sfp_ops(priv);
+ ret = sfp_bus_add_upstream(priv->sfp_bus, priv, sfp_ops);
+ if (ret)
+ goto err_put_sfp;
+ }
+
+ link_ops = airoha_xpon_get_link_ops(priv);
+ ret = airoha_eth_register_xpon(priv->gdm_dev, priv->mode, link_ops,
+ priv);
+ if (ret)
+ goto err_del_upstream;
+
+ airoha_xpon_update_netdev_link(priv, false);
+
+ if (airoha_xpon_is_gpon(priv)) {
+ ret = airoha_xpon_register_gpon_omci(priv);
+ if (ret)
+ goto err_unregister_xpon;
+ }
+
+ platform_set_drvdata(pdev, priv);
+ if (airoha_xpon_is_gpon(priv))
+ dev_info(dev,
+ "GPON probe complete: datapath=%s omci-genl=%u irq=%d
dying-gasp-irq=%d mac-reset=%u default_state=%s\n",
+ priv->gdm_dev->name, omci_device_id(priv->omci.odev),
+ priv->irq, priv->dying_gasp_irq, !!priv->mac_reset,
+ gpon_state_name(ploam_get_state(priv->ploam)));
+ else
+ dev_info(dev, "EPON probe complete: datapath=%s irq=%d\n",
+ priv->gdm_dev->name, priv->irq);
+ return 0;
+
+err_unregister_xpon:
+ airoha_eth_unregister_xpon(priv->gdm_dev, link_ops, priv);
+err_del_upstream:
+ if (priv->sfp_bus)
+ sfp_bus_del_upstream(priv->sfp_bus);
+err_put_sfp:
+ if (priv->sfp_bus)
+ sfp_bus_put(priv->sfp_bus);
+err_unregister_core:
+ xpon_device_unregister(priv->xpon);
+ priv->xpon = NULL;
+err_cleanup_mode:
+ if (airoha_xpon_is_gpon(priv))
+ airoha_xpon_cleanup_gpon(priv);
+ else
+ airoha_xpon_cleanup_epon(priv);
+err_put_gdm:
+ if (priv->gdm_dev) {
+ dev_put(priv->gdm_dev);
+ priv->gdm_dev = NULL;
+ }
+ return ret;
+}
+
+static void airoha_xpon_remove(struct platform_device *pdev)
+{
+ const struct airoha_xpon_link_ops *link_ops;
+ struct xpon_priv *priv;
+
+ priv = platform_get_drvdata(pdev);
+ if (!priv)
+ return;
+
+ dev_info(priv->dev, "removing xPON mode %s\n",
+ airoha_xpon_mode_name(priv->mode));
+
+ if (airoha_xpon_is_gpon(priv))
+ cancel_delayed_work_sync(&priv->restart_work);
+
+ /* net/omci owns and stops the GPON transport before it is detached. */
+ if (airoha_xpon_is_gpon(priv))
+ airoha_xpon_unregister_gpon_omci(priv);
+
+ link_ops = airoha_xpon_get_link_ops(priv);
+ airoha_eth_unregister_xpon(priv->gdm_dev, link_ops, priv);
+
+ if (!airoha_xpon_is_gpon(priv) && priv->sfp_bus &&
+ READ_ONCE(priv->optical_active))
+ sfp_upstream_stop(priv->sfp_bus);
+
+ if (!airoha_xpon_is_gpon(priv)) {
+ epon_disable(priv);
+ } else if (airoha_xpon_is_gpon(priv) &&
+ (READ_ONCE(priv->mac_enabled) || priv->phy_powered)) {
+ /* Defensive fallback for a provider that failed to stop
cleanly. */
+ gpon_disable(priv);
+ } else if (airoha_xpon_is_gpon(priv)) {
+ airoha_xpon_tx_enable(priv->dev, priv->frontend, false);
+ cancel_work_sync(&priv->irq_work);
+ cancel_delayed_work_sync(&priv->to1_work);
+ cancel_delayed_work_sync(&priv->to2_work);
+ }
+
+ xpon_device_unregister(priv->xpon);
+ priv->xpon = NULL;
+
+ if (priv->sfp_bus) {
+ sfp_bus_del_upstream(priv->sfp_bus);
+ sfp_bus_put(priv->sfp_bus);
+ }
+ if (airoha_xpon_is_gpon(priv))
+ airoha_xpon_cleanup_gpon(priv);
+ else
+ airoha_xpon_cleanup_epon(priv);
+ dev_put(priv->gdm_dev);
+ priv->gdm_dev = NULL;
+}
+
+static const struct of_device_id airoha_xpon_of_match[] = {
+ { .compatible = "airoha,en7523-xpon", .data = &en7523_xpon_data },
+ { .compatible = "airoha,en7528-xpon", .data = &en7528_xpon_data },
+ { .compatible = "econet,en751221-xpon", .data = &en751221_xpon_data },
+ {}
+};
+MODULE_DEVICE_TABLE(of, airoha_xpon_of_match);
+
+static struct platform_driver airoha_xpon_driver = {
+ .probe = airoha_xpon_probe,
+ .remove = airoha_xpon_remove,
+ .driver = {
+ .name = "airoha-xpon",
+ .of_match_table = airoha_xpon_of_match,
+ },
+};
+
+module_platform_driver(airoha_xpon_driver);
+
+MODULE_DESCRIPTION("Airoha PON MAC driver");
+MODULE_AUTHOR("Matheus Sampaio Queiroga <srherobrine20@gmail.com>");
+MODULE_AUTHOR("Benjamin Larsson <benjamin.larsson@genexis.eu>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/airoha/airoha_xpon.h
b/drivers/net/ethernet/airoha/airoha_xpon.h
new file mode 100644
index 000000000000..74b3889ef37e
--- /dev/null
+++ b/drivers/net/ethernet/airoha/airoha_xpon.h
@@ -0,0 +1,583 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _AIROHA_XPON_H
+#define _AIROHA_XPON_H
+
+#include <linux/bitmap.h>
+#include <linux/io.h>
+#include <linux/mutex.h>
+#include <linux/netdevice.h>
+#include <linux/optical_frontend.h>
+#include <linux/sfp.h>
+#include <linux/timer.h>
+#include <linux/workqueue.h>
+#include <net/xpon.h>
+#include <net/xpon/omci.h>
+#include <net/xpon/oam.h>
+
+#include "airoha_eth.h"
+#include "airoha_gpon_omci.h"
+#include "airoha_ploam.h"
+
+#define V1_XPON_REGION_SIZE 0x00010000
+#define GPON_REG_OFFSET 0x00004000
+#define EPON_REG_OFFSET 0x00006000
+
+#define XPON_SCU_WAN_CONF 0x070
+#define EN7523_SCU_WAN_MODE_MASK GENMASK(7, 0)
+#define EN7528_SCU_WAN_MODE_MASK GENMASK(2, 0)
+#define EN751221_SCU_WAN_MODE_MASK GENMASK(2, 0)
+#define XPON_SCU_WAN_MODE_GPON 0x00
+#define XPON_SCU_WAN_MODE_EPON 0x01
+
+struct airoha_xpon_match_data {
+ enum airoha_xpon_mode mode;
+ bool mode_from_dt;
+ u32 wan_mode_mask;
+ u8 gpon_fine_delay;
+ u16 gpon_rsp_time_activation;
+ bool en7523_gpon_defaults;
+ bool mac_irq_via_eth;
+ bool prepare_before_mmio;
+ bool gpon_reset_on_start;
+};
+
+
+/* GPON MAC register layout and protocol constants. */
+#define GPON_ONU_ID 0x000
+#define GPON_GBL_CFG 0x004
+#define GPON_INT_STATUS 0x008
+#define GPON_INT_ENABLE 0x00C
+/* T-CONT pair registers (direct access, T-CONTs 0–15) */
+#define GPON_TCONT_ID_0_1 0x020 /* 8 regs, each holds 2 T-CONTs */
+#define GPON_TCONT_ID_14_15 0x03C
+/* T-CONTs 16–31 indirect access */
+#define GPON_TCONT_ID_16_31_CFG 0x180
+#define GPON_TCONT_ID_16_31_STS 0x184
+/* GEM port indirect access */
+#define GPON_GEM_PORT_CFG 0x040
+#define GPON_GEM_PORT_STS 0x044
+/* OMCI GEM port */
+#define GPON_OMCI_ID 0x048
+/* GEM table init control. The EN7523 vendor runtime path does not use
+ * this register during normal activation; it clears GEM entries through
+ * G_GEM_PORT_CFG instead. Keep the offset documented for diagnostics.
+ */
+#define GPON_GEM_TBL_INIT 0x04C
+/* Upstream PLOAM FIFO */
+#define GPON_PLOAMu_FIFO_STS 0x050
+#define GPON_PLOAMu_WDATA 0x054
+/* Downstream PLOAM FIFO */
+#define GPON_PLOAMd_FIFO_STS 0x058
+#define GPON_PLOAMd_RDATA 0x05C
+/* AES shadow key and switch-time config */
+#define GPON_AES_CFG 0x060
+#define GPON_AES_ACTIVE_KEY0 0x064 /* 4 × 32-bit, read-only */
+#define GPON_AES_SHADOW_KEY0 0x074 /* 4 × 32-bit, write to program */
+/* PLOu burst parameters */
+#define GPON_PLOu_OVERHEAD 0x090
+#define GPON_PLOu_GUARD_BIT 0x094
+#define GPON_PLOu_PRMBL_TYPE1_2 0x098
+#define GPON_PLOu_PRMBL_TYPE3 0x09C
+#define GPON_PLOu_DELM_BIT 0x0A0
+#define GPON_PRE_ASSIGNED_DLY 0x0A4
+#define GPON_EQD 0x0A8
+#define GPON_RSP_TIME 0x0AC
+/* Serial number registers */
+#define GPON_VENDOR_ID 0x0B0
+#define GPON_VS_SN 0x0B4
+#define GPON_SN_MSG_CFG 0x0B8
+#define GPON_ACTIVATION_ST 0x0BC
+/* Time of Day */
+#define GPON_TOD_CFG 0x0D0
+#define GPON_NEW_TOD_SEC_L32 0x0D4
+#define GPON_NEW_TOD_NANO_SEC 0x0D8
+#define GPON_CUR_TOD_SEC_L32 0x0DC
+#define GPON_CUR_TOD_NANO_SEC 0x0E0
+/* FCS / MIB tables */
+#define GPON_TX_FCS_TBL_INIT 0x100
+#define GPON_MIB_CTRL_STS 0x120
+#define GPON_MIB_RDATA_L32 0x124
+#define GPON_MIB_RDATA_H32 0x128
+#define GPON_MIB_TBL_INIT 0x134
+/* GPON/PSE memory-bus interface control */
+#define GPON_MBI_MPI_STOP 0x160
+/* EN7523 GPON debug and timing registers */
+#define GPON_DBG_DLY 0x208
+#define GPON_DBG_IDLE_GEM_THLD 0x20C
+#define GPON_DBG_BWM_FILTER_CTRL 0x220
+#define GPON_DBG_BWM_SFIFO_STS 0x224
+#define GPON_DBG_GRP_0 0x228
+#define GPON_DBG_GRP_1 0x22C
+#define GPON_DBG_BWM_BFIFO_STS 0x250
+#define GPON_DBG_ERR_CTRL 0x260
+#define GPON_DBG_RX_GEM_CNT 0x300
+#define GPON_DBG_RX_CRC_ERR_CNT 0x304
+#define GPON_DBG_RX_GTC_CNT 0x308
+#define GPON_DBG_TX_GEM_CNT 0x30C
+#define GPON_DBG_TX_BST_CNT 0x310
+#define GPON_DBG_GEM_HEC_ONE_ERR_CNT 0x330
+#define GPON_DBG_GEM_HEC_TWO_ERR_CNT 0x334
+#define GPON_DBG_GEM_HEC_UC_ERR_CNT 0x338
+/* Debug / TX sync (EN7521 EqD adjustment) */
+#define GPON_DBG_TX_SYNC_OFFSET 0x35C
+/* Power management (always-on domain) */
+#define GPON_SLEEP_GLB_CFG 0x3A4
+#define GPON_SLEEP_CNT 0x3A8
+
+/* -----------------------------------------------------------------------
+ * Register bit definitions
+ * -------------------------------------------------------------------- */
+
+/* G_ONU_ID */
+#define ONU_ID_VLD BIT(15)
+#define ONU_ID_MASK 0xFF
+
+/* G_GBL_CFG */
+#define GBL_CFG_US_FEC_EN BIT(16)
+#define GBL_CFG_SR_BLK_SIZE_MASK GENMASK(7, 0)
+
+/*
+ * The GPON MAC stores the reciprocal of the DBRu block size with the bit
+ * order reversed. The vendor SDK programs a 48-byte block, encoded as
+ * bitrev8(round(2048 / 48)) = bitrev8(43) = 0xd4.
+ */
+#define GPON_DBRU_BLOCK_SIZE_48B 0xd4
+
+/* G_SN_MSG_CFG */
+#define SN_MSG_CFG_SN_REQ_THR_MASK GENMASK(31, 24)
+#define SN_MSG_CFG_TX_POWER_MODE_MASK GENMASK(17, 16)
+#define SN_MSG_CFG_RANDOM_DELAY_MASK GENMASK(11, 0)
+
+/* G_INT_STATUS / G_INT_ENABLE */
+#define INT_PLOAMD_RECV BIT(0)
+#define INT_PLOAMU_SEND BIT(1)
+#define INT_SN_REQ_RECV BIT(2)
+#define INT_SN_ONU_SEND_O3 BIT(3)
+#define INT_RANGING_REQ_RECV BIT(4)
+#define INT_SN_ONU_SEND_O4 BIT(5)
+#define INT_SN_REQ_CRS BIT(6)
+#define INT_LOSS_GEM_DEL BIT(7)
+#define INT_AES_KEY_SWITCH_DONE BIT(8)
+#define INT_TOD_UPDATE_DONE BIT(9)
+#define INT_TOD_1PPS BIT(10)
+#define INT_DYING_GASP BIT(11)
+/*
+ * G_INT_STATUS layout from the vendor EN7521/EN7523 register header.
+ * Bits 13 and 14 are reserved on EN7523. Bit 15 reports completion of
+ * the grant-size calculation used by the EN7523 DBA block.
+ */
+#define INT_CAL_GNT_SIZE_DONE BIT(15)
+#define INT_RX_ERR BIT(16)
+#define INT_FIFO_ERR BIT(17)
+#define INT_BST_SGL_DIFF BIT(18)
+#define INT_TX_LATE_START BIT(19)
+#define INT_RX_EOF_ERR BIT(20)
+#define INT_RX_GEM_INTLV_ERR BIT(21)
+#define INT_BFIFO_FULL BIT(22)
+#define INT_SFIFO_FULL BIT(23)
+#define INT_O5_EQD_ADJ_DONE BIT(24)
+#define INT_OLT_DS_FEC_CHG BIT(25)
+#define INT_ONU_US_FEC_CHG BIT(26)
+#define INT_POP_UP_RECV_O6 BIT(27)
+#define INT_FWI BIT(28)
+#define INT_LWI BIT(29)
+#define INT_BWM_STOP_TIME_ERR BIT(30)
+#define INT_BWM_US_FEC_ERR BIT(31)
+
+#define GPON_INT_ACTIVATION_MASK (INT_PLOAMD_RECV | INT_SN_REQ_RECV | \
+ INT_SN_ONU_SEND_O3 | INT_RANGING_REQ_RECV | \
+ INT_SN_ONU_SEND_O4 | INT_SN_REQ_CRS | \
+ INT_LOSS_GEM_DEL | INT_AES_KEY_SWITCH_DONE | \
+ INT_DYING_GASP | INT_CAL_GNT_SIZE_DONE)
+/*
+ * The EN7523 vendor driver enables only the common receive/burst errors.
+ * The interleave, BWM FIFO and BWM timing interrupts are EN7521-only and
+ * may expose unrelated status bits when enabled on EN7523.
+ */
+#define GPON_INT_ERROR_MASK (INT_RX_ERR | INT_FIFO_ERR | \
+ INT_BST_SGL_DIFF | INT_TX_LATE_START | \
+ INT_RX_EOF_ERR)
+#define GPON_INT_DEFAULT_MASK (GPON_INT_ACTIVATION_MASK | \
+ INT_TX_LATE_START)
+
+/* G_GEM_PORT_CFG */
+#define GEM_CMD_WRITE BIT(31)
+#define GEM_ENCRYPT BIT(17)
+#define GEM_VALID BIT(16)
+
+/* G_GEM_PORT_STS */
+#define GEM_CMD_DONE BIT(31)
+#define GEM_STS_ENCRYPT BIT(1)
+#define GEM_STS_VALID BIT(0)
+
+/* G_GEM_TBL_INIT */
+#define GEM_TBL_INIT_DONE BIT(8)
+#define GEM_TBL_INIT_START BIT(0)
+
+/* G_PLOAMu_FIFO_STS */
+#define PLOAMu_FIFO_AVAIL_MASK 0xFF
+
+/* G_PLOAMd_FIFO_STS */
+#define PLOAMd_FIFO_USED_MASK 0xFF
+
+/* G_TCONT_ID pair register */
+#define TCONT_ID_MASK 0x0FFF
+#define TCONT0_VALID BIT(15)
+#define TCONT1_ID_SHIFT 16
+#define TCONT1_VALID BIT(31)
+#define TCONT_PAIR_INVALID 0x00000000 /* validity bits cleared */
+
+/* G_TCONT_ID_16_31_CFG */
+#define TCONT16_CMD_EXEC BIT(31)
+#define TCONT16_VALID BIT(27)
+#define TCONT16_IDX_SHIFT 16
+#define TCONT16_ALLOC_MASK 0x0FFF
+
+/* G_TCONT_ID_16_31_STS */
+#define TCONT16_CMD_DONE BIT(31)
+
+/* G_MIB_TBL_INIT */
+#define MIB_TBL_INIT_DONE BIT(8)
+#define MIB_TBL_INIT_START BIT(0)
+
+/* G_OMCI_ID */
+#define OMCI_PORT_VLD BIT(16)
+#define OMCI_GPID_MASK 0xFFF
+
+/* G_AES_CFG: bits[29:0] = key-switch superframe counter */
+#define AES_KEY_SWITCH_CNT_MASK 0x3FFFFFFF
+
+/* G_PRE_ASSIGNED_DLY */
+#define PRE_DLY_EN BIT(31)
+#define PRE_DLY_MASK 0xFFFF
+
+/* G_MBI_MPI_STOP */
+#define MBI_RX_STOP BIT(0)
+#define MBI_TX_STOP BIT(8)
+
+/* DBG_DLY */
+#define DBG_DLY_FINE_INT_MASK GENMASK(15, 8)
+#define DBG_DLY_FINE_INT_DEFAULT 0x0D
+#define DBG_DLY_RESET_DEFAULT 0x80800F00
+
+/* DBG_BWM_FILTER_CTRL */
+#define BWM_FILTER_LEN_VALID_CHECK_EN BIT(17)
+
+/* G_DBG_TX_SYNC_OFFSET bits[1:0] = internal byte delay */
+#define DBG_TX_SYNC_OFFSET_MASK GENMASK(1, 0)
+
+/*
+ * G_RSP_TIME carries the ONU response time in units of 32 bits, so one
unit is
+ * 25.72ns at the 1.24416Gbit/s upstream rate and the 0x0551 reset
value is the
+ * 35us of G.984.3. The MAC is held at 0x058b while it is reset or in O1.
+ *
+ * The activation value is generation-specific. The EN7523 vendor driver
+ * switches to 0x0577 before serial-number activation in O2, while the
EN751221
+ * driver programs its configured response time of 0x058b there and
leaves the
+ * FEC-adjusted variant disabled. Responding 0x14 units early on
EN751221 does
+ * not match the timing the OLT ranges against.
+ */
+#define GPON_RSP_TIME_RESET 0x058b
+#define GPON_RSP_TIME_ACT_EN7523 0x0577
+#define GPON_RSP_TIME_ACT_EN7528 0x0577
+#define GPON_RSP_TIME_ACT_EN751221 0x058b
+#define GPON_IDLE_GEM_THLD_DEFAULT 0x001A
+
+/* TO1 timer: 10 seconds in O3/O4 without Ranging_Time → return to O2 */
+#define GPON_TO1_MS 10000
+/*
+ * Consecutive TO1 expiries tolerated before the MAC and the PHY are reset.
+ * Matches the vendor driver: a normal ranging failure only costs a
return to
+ * O2, and only a run of them is treated as wedged hardware.
+ */
+#define GPON_TO1_MAX_RETRIES 20
+/* TO2 timer: 100 ms in O6 without Popup/Swift_Popup → reset to O1 */
+#define GPON_TO2_MS 100
+/* Restart delay after an OLT Deactivate_ONU-ID request. */
+#define GPON_DEACTIVATE_RESTART_MS 500
+/* Coalesce a net/omci apply batch into one complete MAC/PHY restart. */
+#define GPON_CONFIG_RESTART_DEBOUNCE_MS 250
+#define XPON_LINK_POLL_MS 250
+#define GPON_REARM_RETRY_MS 1000
+
+#define GPON_MAX_GEM_ID 4096
+#define GPON_MAX_TCONT 32
+#define GPON_TCONT_UNASSIGNED 0xffff
+#define GPON_TCONT_ENTITY_UNASSIGNED 0xffff
+#define GPON_SN_REQ_THRESHOLD 10
+
+#define GPON_PLOAM_RX_QUEUE_LEN 128
+#define GPON_PLOAM_RX_QUEUE_MASK (GPON_PLOAM_RX_QUEUE_LEN - 1)
+
+#define GPON_CMD_TIMEOUT_US 10000
+#define GPON_TABLE_TIMEOUT_US 100000
+#define GPON_PLOAM_TX_TIMEOUT_US 1000
+
+/*
+ * Fallback TX-enable guard, used only when an OLT announces none. The
guard
+ * time is the OLT's to choose: it sizes the quiet period between upstream
+ * bursts, and every ONU on the PON is told the same value in
+ * Upstream_Overhead.
+ */
+#define GPON_PHY_GUARD_BIT_NUM 20
+
+
+#define EPON_MAX_LLID 8
+
+/* LLID registration states */
+enum airoha_epon_llid_state {
+ AIROHA_EPON_LLID_WAIT = 0,
+ AIROHA_EPON_LLID_REGISTERING,
+ AIROHA_EPON_LLID_REGISTER_REQUEST,
+ AIROHA_EPON_LLID_REGISTER_PENDING,
+ AIROHA_EPON_LLID_REGISTER_ACK,
+ AIROHA_EPON_LLID_REGISTERED,
+};
+
+struct epon_llid {
+ enum airoha_epon_llid_state state;
+ bool valid;
+ u16 value;
+};
+
+struct xpon_priv {
+ void __iomem *base;
+ void __iomem *gpon_reg;
+ void __iomem *epon_reg;
+ void __iomem *epon_reset_reg;
+ struct device *dev;
+ struct regmap *scu;
+ const struct airoha_xpon_match_data *match_data;
+ struct reset_control *mac_reset;
+ struct phy *phy;
+ enum airoha_xpon_mode mode;
+ bool phy_initialized;
+ bool phy_powered;
+ struct net_device *gdm_dev;
+ bool started;
+ bool optical_active;
+ bool mac_enabled;
+ bool phy_link_known;
+ bool phy_link_up;
+ struct xpon_device *xpon;
+ struct xpon_oam *oam;
+ struct airoha_gpon_omci omci;
+ struct airoha_xpon_oam_handler omci_handler;
+ struct sfp_bus *sfp_bus;
+ struct optical_frontend *frontend;
+ int irq;
+ int dying_gasp_irq;
+
+ struct epon_llid llid[EPON_MAX_LLID];
+
+ struct ploam_priv *ploam;
+ int registered_llids;
+
+ u8 hw_sn[8];
+ u8 hw_passwd[10];
+ struct omci_identity identity;
+ u8 aes_key[16];
+
+ /*
+ * Serialize downstream PLOAM protocol handling and activation
+ * timeouts. The IRQ top half only performs the hardware-critical
+ * FIFO drain and ONU-ID fast path.
+ */
+ struct workqueue_struct *fsm_wq;
+ struct work_struct irq_work;
+ struct delayed_work to1_work; /* O3/O4: 10 s → O2 */
+ struct delayed_work to2_work; /* O6: 100 ms → O1 */
+ struct delayed_work restart_work; /* OLT deactivation recovery */
+ struct delayed_work phy_link_work; /* Digital PHY link monitor */
+ atomic_t pending_irqs;
+
+ /*
+ * The IRQ top half drains the hardware FIFO into this single-producer,
+ * single-consumer queue. Keeping the FIFO drain in hard IRQ avoids
+ * losing the first ranging allocation while the ordered workqueue is
+ * processing printk output or earlier PLOAM copies.
+ */
+ struct ploam_msg ploam_rx_queue[GPON_PLOAM_RX_QUEUE_LEN];
+ u16 ploam_rx_head;
+ u16 ploam_rx_tail;
+ u32 ploam_rx_drops;
+ u32 ploam_rx_messages;
+ u32 assign_onu_fastpath;
+
+ /* BER measurement timer */
+ struct timer_list ber_timer;
+ u32 ber_interval_ms;
+
+ /* Protects the hardware T-CONT table and its software allocation
maps. */
+ struct mutex tcont_lock;
+
+ /* Protects the active OMCI OLT interoperability policy. */
+ struct mutex omci_profile_lock;
+ struct omci_olt_profile_state omci_profile;
+
+ /* Protects pending runtime identity updates from OMCI. */
+ struct mutex omci_config_lock;
+ struct omci_identity pending_identity;
+ bool config_restart_pending;
+ u16 config_restart_key;
+
+ /* Protects the GPON carrier readiness inputs. */
+ struct mutex link_state_lock;
+ DECLARE_BITMAP(service_gems, GPON_MAX_GEM_ID);
+ bool gpon_o5;
+ bool omci_operational;
+ bool netdev_link;
+ u16 tcont_alloc_id[GPON_MAX_TCONT];
+ u16 tcont_entity_id[GPON_MAX_TCONT];
+
+ /* EqD state for O5 incremental adjustment */
+ u32 byte_delay;
+ u32 bit_delay;
+
+ /* Consecutive TO1 expiries; cleared on O5. */
+ unsigned int to1_failures;
+
+};
+
+
+/* EPON MAC register layout and MPCP constants. */
+/* Register offsets from EPON MAC base */
+#define EPON_GLB_CFG 0x000
+#define EPON_INT_STATUS 0x004
+#define EPON_INT_EN 0x008
+#define EPON_RPT_MPCP_TIMEOUT 0x00C
+#define EPON_DYINGGSP_CFG 0x010
+#define EPON_PENDING_GNT_NUM 0x014
+#define EPON_LLID0_3_CFG 0x020
+#define EPON_LLID4_7_CFG 0x024
+#define EPON_LLID_DSCVRY_CTRL 0x028
+#define EPON_LLID0_DSCVRY_STS 0x02C
+#define EPON_MAC_ADDR_CFG 0x050
+#define EPON_MAC_ADDR_VALUE 0x054
+#define EPON_SECURITY_KEY_CFG 0x058
+#define EPON_SECURITY_KEY_DATA 0x05C
+#define EPON_RPT_DATA 0x060
+#define EPON_RPT_LEN 0x064
+#define EPON_RPT_CFG 0x068
+#define EPON_LOCAL_TIME 0x080
+#define EPON_TOD_SYNC_X 0x084
+#define EPON_TOD_LTNCY 0x088
+#define EPON_P2P_TX_TAG1 0x08C
+#define EPON_P2P_TX_TAG2 0x090
+#define EPON_TXFETCH_CFG 0x0D0
+#define EPON_SYNC_TIME 0x0D4
+#define EPON_TX_CAL_CNST 0x0D8
+#define EPON_LASER_ONOFF_TIME 0x0DC
+#define EPON_GRD_THRSHLD 0x0E0
+#define EPON_MPCP_TIMEOUT_INTVL 0x0E4
+#define EPON_RPT_TIMEOUT_INTVL 0x0E8
+#define EPON_MAX_FUTURE_GNT 0x0EC
+#define EPON_MIN_PROC_TIME 0x0F0
+#define EPON_TRX_ADJUST_TIME1 0x0F4
+#define EPON_TRX_ADJUST_TIME2 0x0F8
+#define EPON_TIME_DRFT_STAT 0x134
+
+/* e_glb_cfg bits */
+#define GLB_CFG_MODE_SEL BIT(0)
+#define GLB_CFG_RPT_TXPRI_CTRL BIT(1)
+#define GLB_CFG_EPON_MAC_SW_RST BIT(4)
+#define GLB_CFG_TXMBI_STOP BIT(8)
+#define GLB_CFG_RXMBI_STOP BIT(9)
+#define GLB_CFG_FCS_ERR_FWD BIT(17)
+#define GLB_CFG_MPCP_FWD BIT(22)
+#define GLB_CFG_DISCV_BURST_EN BIT(23)
+
+/*
+ * e_int_status / e_int_en bit positions (from epon_reg.h):
+ * bit 0: RCV_DSCVRY_GATE_INT — discovery gate received
+ * bits 1-8: LLID0..7_RCV_RGST_INT — per-LLID REGISTER frame received
+ * bit 9: GNT_BUF_OVRRUN_INT
+ * bit 13: TIMEDRFT_INT — time drift
+ * bit 14: MPCP_TIMEOUT_INT
+ * bit 15: RPT_OVERINTVL_INT
+ * bit 24: REG_REQ_DONE_INT — REGISTER_REQUEST sent by HW
+ * bit 25: REG_ACK_DONE_INT — REGISTER_ACK sent by HW
+ */
+#define EPON_INT_DISCV_GATE BIT(0)
+#define EPON_INT_LLID0_RGST BIT(1) /* BIT(1+n) for LLID n */
+#define EPON_INT_GNT_OVRRUN BIT(9)
+#define EPON_INT_TIMEDRFT BIT(13)
+#define EPON_INT_MPCP_TIMEOUT BIT(14)
+#define EPON_INT_RPT_OVRFLW BIT(15)
+#define EPON_INT_REG_REQ_DONE BIT(24)
+#define EPON_INT_REG_ACK_DONE BIT(25)
+
+/* e_llid_dscvry_ctrl bits (from epon_reg.h REG_e_llid_dscvry_ctrl) */
+#define DSCVRY_MPCP_CMD_MASK (3U << 30)
+#define DSCVRY_MPCP_REG_REQ BIT(30) /* send REGISTER_REQUEST */
+#define DSCVRY_MPCP_NORMAL BIT(31)
+#define DSCVRY_MPCP_ACK (3U << 30) /* send REGISTER_ACK */
+#define DSCVRY_CMD_DONE BIT(16)
+#define DSCVRY_RGSTR_ACK_FLG BIT(12)
+#define DSCVRY_RGSTR_REQ_FLG BIT(8)
+#define DSCVRY_TX_MPCP_LLID_MASK 0x7
+
+/*
+ * e_llid0_dscvry_sts bit layout (little-endian packed struct from
epon_reg.h):
+ * bits [15:0] llidValue
+ * bit 16 llidValid
+ * bits [23:17] reserved
+ * bits [25:24] rgstrFlgSts — see MPCP_REG_* below
+ * bits [29:26] reserved
+ * bits [31:30] llidDscvrySts — 0=unregistered, 1=registering,
2=registered
+ */
+#define LLID_STS_DSCVRY_SHIFT 30
+#define LLID_STS_RGST_FLG_SHIFT 24
+#define LLID_STS_RGST_FLG_MASK (3U << 24)
+#define LLID_STS_VALID BIT(16)
+#define LLID_STS_VALUE_MASK 0xFFFF
+
+/* rgstrFlgSts values */
+#define MPCP_REG_RE_REGISTER 0
+#define MPCP_REG_DE_REGISTER 1
+#define MPCP_REG_ACK 2
+#define MPCP_REG_NACK 3
+
+/*
+ * e_mac_addr_cfg indirect register (from epon_reg.h REG_e_mac_addr_cfg):
+ * bit 0 mac_addr_dw_idx — 0=low 32 bits, 1=high 16 bits
+ * bits [3:1] mac_addr_llid_indx
+ * bit 16 mac_addr_rwcmd_done — 1=busy/in-progress
+ * bit 31 mac_addr_rwcmd — write 1 to trigger write
+ */
+#define MAC_ADDR_RWCMD BIT(31)
+#define MAC_ADDR_DONE BIT(16)
+#define MAC_ADDR_LLID_SHIFT 1
+#define MAC_ADDR_DW_IDX BIT(0)
+
+/* e_security_key_cfg */
+#define SEC_KEY_WRITE_CMD BIT(31)
+#define SEC_KEY_LLID_SHIFT 24
+#define SEC_KEY_IDX_SHIFT 16
+#define SEC_KEY_DW_SHIFT 8
+
+/* Dying gasp init value per ref (hw_dying_gasp_en=1, dygsp_num_of_times=1,
+ * dygsp_code=0, other fields=0x02) */
+#define DYINGGSP_CFG_HW_ENABLE 0x80000102
+
+/* Guard threshold for time drift detection (ref:
EPON_TIMEDRIFT_THRSHLD) */
+#define EPON_TIMEDRIFT_THRSHLD 0x10
+
+/* Default register values */
+#define EPON_PENDING_GNT_DEFAULT 0x40
+#define EPON_MPCP_TIMEOUT_DEFAULT 0x03B9ACA0
+#define EPON_RPT_TIMEOUT_DEFAULT 0x002FAF08
+#define EPON_MAX_FUTURE_GNT_DEFAULT 0x03B9ACA0
+#define EPON_MIN_PROC_TIME_DEFAULT 0x400
+#define EPON_LASER_ONOFF_DEFAULT 0x2020
+#define EPON_TX_CAL_CNST_DEFAULT 0x2612040C
+#define EPON_TXFETCH_DEFAULT 0x202403E8
+#define EPON_TRX_ADJUST_TIME1_DEF 0x004FFFF1
+#define EPON_TRX_ADJUST_TIME2_DEF 0x6
+
+/* External SW reset register bit (REG_E_SW_RST, outside EPON MAC block) */
+#define EPON_EXT_SW_RST_BIT BIT(31)
+#define EPON_RESET_DELAY_US 100
+
+
+#endif /* _AIROHA_XPON_H */
^ permalink raw reply [flat|nested] 9+ messages in thread