Enable dynamic RX flow classification via ethtool set_rxnfc/get_rxnfc. Users can install/remove filter rules to steer packets to specific RX flows based on EtherType patterns via FT3 hardware filters. - Add RXNFC rule tracking structures and lifecycle management - Implement ethtool get_rxnfc/set_rxnfc callbacks - FT3 slot allocation and hardware programming for pattern matching - RX classifier OR table updates based on active rules - Max 15 rules per EMAC, using FT3 slots 0-13,15 (slot 14 reserved) Signed-off-by: Meghana Malladi --- .../net/ethernet/ti/icssg/icssg_classifier.c | 340 ++++++++++++++++++ drivers/net/ethernet/ti/icssg/icssg_ethtool.c | 48 +++ drivers/net/ethernet/ti/icssg/icssg_prueth.c | 27 +- drivers/net/ethernet/ti/icssg/icssg_prueth.h | 32 ++ 4 files changed, 446 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/ti/icssg/icssg_classifier.c b/drivers/net/ethernet/ti/icssg/icssg_classifier.c index 87a8577093dbb..f11539611df75 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_classifier.c +++ b/drivers/net/ethernet/ti/icssg/icssg_classifier.c @@ -105,6 +105,14 @@ enum ft1_cfg_type { ((BIT(slot) << RX_CLASS_FT_FT1_MATCH_SHIFT) & \ RX_CLASS_FT_FT1_MATCH_MASK) +#define RX_CLASS_FT_FT3_MATCH(slot) \ + ((BIT(slot) << RX_CLASS_FT_FT3_MATCH_SHIFT) & \ + RX_CLASS_FT_FT3_MATCH_MASK) + +/* FT3 Configuration registers and modes */ +#define FT3_CFG_REGISTER_EQUAL (1U) +#define FT3_CFG_REGISTER_VLAN_SKIP BIT(2) + /* HSR/PRP protocol frame filtering */ #define HSR_PTP_ETHERTYPE_OFFSET 0x12 #define PRP_PTP_ETHERTYPE_OFFSET 0x0C @@ -532,3 +540,335 @@ void icssg_ft3_hsr_configurations(struct regmap *miig_rt, int slice, } } EXPORT_SYMBOL_GPL(icssg_ft3_hsr_configurations); + +/* RX Flow Classification (ethtool) rule management */ + +/* Allocate free FT3 slot, avoiding reserved slot 14 */ +static int allocate_ft3_slot(struct prueth_emac *emac) +{ + u16 used = emac->rxnfc.ft3_slots_used | ICSSG_FT3_RESERVED_SLOTS; + int slot; + + for (slot = 0; slot < ICSSG_NUM_FT3_SLOTS; slot++) { + if (!(used & BIT(slot))) { + emac->rxnfc.ft3_slots_used |= BIT(slot); + return slot; + } + } + return -ENOSPC; +} + +/* Free allocated FT3 slot */ +static void free_ft3_slot(struct prueth_emac *emac, int slot) +{ + emac->rxnfc.ft3_slots_used &= ~BIT(slot); +} + +/* Program FT3 filter slot with ethertype pattern */ +static void icssg_ft3_program_filter(struct regmap *miig_rt, int slice, + int slot, struct icssg_rxnfc_rule *rule) +{ + const struct ethtool_rx_flow_spec *fs = &rule->flow_spec; + u16 ether_type; + + /* Extract EtherType from flow spec (network byte order) */ + ether_type = fs->h_u.ether_spec.h_proto; + + /* Deactivate filter before programming */ + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_CFG), 0); + + /* EtherType is at byte 12 in standard Ethernet frames */ + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_START), + PRP_PTP_ETHERTYPE_OFFSET); + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_START_AUTO), 0); + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_START_OFFSET), 0); + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_JUMP_OFFSET), 0); + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_LEN), 0); + + /* FT3 compares a 32-bit LE word at FT3_START. EtherType occupies the + * lower 16 bits of that word. FT3_T_MASK=0xFFFF0000 masks the upper + * 16 bits (payload bytes), so only the lower 16 bits are compared. + * + * h_proto is __be16 (network byte order). On LE the raw integer value + * equals the lower 16 bits of the LE 32-bit word at the EtherType + * offset in the frame — e.g. IPv4 0x0800 BE → h_proto = 0x0008 on LE, + * which is exactly what FT3 sees in the lower 16 bits of the word at + * byte 12. No conversion needed; using be16_to_cpu() here would invert + * the byte order and break the match. + * Reference: FT3_PTP_PATTERN = 0xF788 = htons(0x88F7) = raw h_proto value. + */ + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_T), + (__force u32)ether_type); + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_T_MASK), FT3_ETHERTYPE_MASK); + + /* Enable filter: register-equal mode */ + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_CFG), + FT3_CFG_REGISTER_EQUAL); +} + +/* Disable FT3 filter slot */ +static void icssg_ft3_disable_filter(struct regmap *miig_rt, int slice, int slot) +{ + regmap_write(miig_rt, FT3_N_REG(slice, slot, FT3_CFG), 0); +} + +/* Update RX classifiers for all active FT3 rules. + * Classifier N maps to RX flow N (firmware routes classifier-N hits to + * UDMA flow rx_flow_id_base + N). Build a per-classifier OR table so + * that each target queue gets its own classifier entry. + */ +static void icssg_rxnfc_classifier_update(struct prueth_emac *emac) +{ + struct prueth *prueth = emac->prueth; + struct regmap *miig_rt = prueth->miig_rt; + int slice = prueth_emac_slice(emac); + u32 or_table[ICSSG_NUM_CLASSIFIERS] = { }; + struct icssg_rxnfc_rule *rule; + int n; + + if (slice < 0) + return; + + /* Build per-classifier OR table: classifier index = target queue index */ + list_for_each_entry(rule, &emac->rxnfc.rules, list) { + int cls = rule->queue_index; + + if (cls < ICSSG_NUM_CLASSIFIERS) + or_table[cls] |= RX_CLASS_FT_FT3_MATCH(rule->ft3_slot); + } + + /* Write OR tables for user-rule classifiers; skip classifier 0 (default flow). + * Use RX_CLASS_N_REG with RX_CLASS_OR_EN — NOT RX_CLASS_OR_REG which uses + * rx_class_or_base (= rx_class_base + 4), causing an off-by-4 that lands on + * AND_EN of classifier n+1 instead of OR_EN of classifier n. + */ + for (n = 1; n < ICSSG_NUM_CLASSIFIERS; n++) { + regmap_write(miig_rt, + RX_CLASS_N_REG(slice, n, RX_CLASS_OR_EN), + or_table[n]); + } +} + +/* Validate rule parameters */ +static int validate_rule(struct prueth_emac *emac, + struct ethtool_rx_flow_spec *fs) +{ + /* Only ETHER_FLOW supported for Phase 1 */ + if (fs->flow_type != ETHER_FLOW) + return -EOPNOTSUPP; + + /* Check RX queue is valid */ + if (fs->ring_cookie >= emac->rx_flow_num) + return -EINVAL; + + return 0; +} + +/* Check if identical rule already exists */ +static bool rule_exists(struct prueth_emac *emac, + struct ethtool_rx_flow_spec *fs) +{ + struct icssg_rxnfc_rule *rule; + + list_for_each_entry(rule, &emac->rxnfc.rules, list) { + if (rule->flow_spec.flow_type == fs->flow_type && + rule->flow_spec.h_u.ether_spec.h_proto == + fs->h_u.ether_spec.h_proto) { + return true; + } + } + return false; +} + +/* Initialize RX flow classification */ +void icssg_rxnfc_rule_init(struct prueth_emac *emac) +{ + INIT_LIST_HEAD(&emac->rxnfc.rules); + mutex_init(&emac->rxnfc.rxnfc_lock); + emac->rxnfc.rxnfc_rule_count = 0; + emac->rxnfc.ft3_slots_used = ICSSG_FT3_RESERVED_SLOTS; /* Mark slot 14 reserved */ +} +EXPORT_SYMBOL_GPL(icssg_rxnfc_rule_init); + +/* Cleanup RX flow classification */ +void icssg_rxnfc_rule_cleanup(struct prueth_emac *emac) +{ + struct prueth *prueth = emac->prueth; + struct regmap *miig_rt = prueth->miig_rt; + int slice = prueth_emac_slice(emac); + struct icssg_rxnfc_rule *rule, *tmp; + int n; + + if (slice < 0) + return; + + mutex_lock(&emac->rxnfc.rxnfc_lock); + + /* Disable all active FT3 filters */ + list_for_each_entry_safe(rule, tmp, &emac->rxnfc.rules, list) { + icssg_ft3_disable_filter(miig_rt, slice, rule->ft3_slot); + free_ft3_slot(emac, rule->ft3_slot); + list_del(&rule->list); + kfree(rule); + } + + emac->rxnfc.rxnfc_rule_count = 0; + emac->rxnfc.ft3_slots_used = ICSSG_FT3_RESERVED_SLOTS; + + /* Clear all user-rule classifier OR tables (classifier 0 = default, skip) */ + for (n = 1; n < ICSSG_NUM_CLASSIFIERS; n++) + regmap_write(miig_rt, + RX_CLASS_N_REG(slice, n, RX_CLASS_OR_EN), 0); + + mutex_unlock(&emac->rxnfc.rxnfc_lock); +} +EXPORT_SYMBOL_GPL(icssg_rxnfc_rule_cleanup); + +/* Add a new RX flow classification rule */ +int icssg_rxnfc_rule_add(struct prueth_emac *emac, struct ethtool_rxnfc *info) +{ + struct ethtool_rx_flow_spec *fs = &info->fs; + struct prueth *prueth = emac->prueth; + struct regmap *miig_rt = prueth->miig_rt; + int slice = prueth_emac_slice(emac); + struct icssg_rxnfc_rule *rule; + int ft3_slot; + int ret; + + if (slice < 0) + return -EINVAL; + + /* Validate rule parameters */ + ret = validate_rule(emac, fs); + if (ret) + return ret; + + mutex_lock(&emac->rxnfc.rxnfc_lock); + + /* Check max rules */ + if (emac->rxnfc.rxnfc_rule_count >= ICSSG_RXNFC_MAX_RULES) { + ret = -ENOSPC; + goto unlock; + } + + /* Check for duplicate */ + if (rule_exists(emac, fs)) { + ret = -EEXIST; + goto unlock; + } + + /* Allocate FT3 slot */ + ft3_slot = allocate_ft3_slot(emac); + if (ft3_slot < 0) { + ret = -ENOSPC; + goto unlock; + } + + /* Auto-assign location from FT3 slot if RX_CLS_LOC_ANY requested */ + if (fs->location == RX_CLS_LOC_ANY) + fs->location = ft3_slot; + + /* Allocate rule structure */ + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) { + ret = -ENOMEM; + free_ft3_slot(emac, ft3_slot); + goto unlock; + } + + /* Program FT3 hardware */ + rule->flow_spec = *fs; + rule->ft3_slot = ft3_slot; + rule->queue_index = fs->ring_cookie; + rule->location = fs->location; + + icssg_ft3_program_filter(miig_rt, slice, ft3_slot, rule); + + /* Add to rule list */ + list_add(&rule->list, &emac->rxnfc.rules); + emac->rxnfc.rxnfc_rule_count++; + + /* Update classifier */ + icssg_rxnfc_classifier_update(emac); + + ret = 0; + +unlock: + mutex_unlock(&emac->rxnfc.rxnfc_lock); + return ret; +} +EXPORT_SYMBOL_GPL(icssg_rxnfc_rule_add); + +/* Delete a RX flow classification rule */ +int icssg_rxnfc_rule_del(struct prueth_emac *emac, u32 location) +{ + struct prueth *prueth = emac->prueth; + struct regmap *miig_rt = prueth->miig_rt; + int slice = prueth_emac_slice(emac); + struct icssg_rxnfc_rule *rule, *tmp; + int ret = -ENOENT; + + if (slice < 0) + return -EINVAL; + + mutex_lock(&emac->rxnfc.rxnfc_lock); + + list_for_each_entry_safe(rule, tmp, &emac->rxnfc.rules, list) { + if (rule->location == location) { + icssg_ft3_disable_filter(miig_rt, slice, rule->ft3_slot); + free_ft3_slot(emac, rule->ft3_slot); + list_del(&rule->list); + kfree(rule); + emac->rxnfc.rxnfc_rule_count--; + icssg_rxnfc_classifier_update(emac); + ret = 0; + break; + } + } + + mutex_unlock(&emac->rxnfc.rxnfc_lock); + return ret; +} +EXPORT_SYMBOL_GPL(icssg_rxnfc_rule_del); + +/* Get single RX flow classification rule */ +int icssg_rxnfc_rule_get(struct prueth_emac *emac, u32 location, + struct ethtool_rx_flow_spec *fs) +{ + struct icssg_rxnfc_rule *rule; + int ret = -ENOENT; + + mutex_lock(&emac->rxnfc.rxnfc_lock); + + list_for_each_entry(rule, &emac->rxnfc.rules, list) { + if (rule->location == location) { + *fs = rule->flow_spec; + ret = 0; + break; + } + } + + mutex_unlock(&emac->rxnfc.rxnfc_lock); + return ret; +} +EXPORT_SYMBOL_GPL(icssg_rxnfc_rule_get); + +/* Get all RX flow classification rules */ +int icssg_rxnfc_rule_get_all(struct prueth_emac *emac, u32 *rule_locs, + u32 *rule_cnt) +{ + struct icssg_rxnfc_rule *rule; + u32 count = 0; + + mutex_lock(&emac->rxnfc.rxnfc_lock); + + list_for_each_entry(rule, &emac->rxnfc.rules, list) { + rule_locs[count++] = rule->location; + } + + *rule_cnt = count; + + mutex_unlock(&emac->rxnfc.rxnfc_lock); + return 0; +} +EXPORT_SYMBOL_GPL(icssg_rxnfc_rule_get_all); diff --git a/drivers/net/ethernet/ti/icssg/icssg_ethtool.c b/drivers/net/ethernet/ti/icssg/icssg_ethtool.c index 5c2936ecd595c..9f0a1116e530c 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_ethtool.c +++ b/drivers/net/ethernet/ti/icssg/icssg_ethtool.c @@ -299,6 +299,52 @@ static int emac_set_per_queue_coalesce(struct net_device *ndev, u32 queue, return 0; } +static int emac_get_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info, + u32 *rule_locs) +{ + struct prueth_emac *emac = netdev_priv(ndev); + int ret = 0; + + switch (info->cmd) { + case ETHTOOL_GRXRINGS: + info->data = emac->rx_flow_num; + break; + case ETHTOOL_GRXCLSRLCNT: + info->rule_cnt = emac->rxnfc.rxnfc_rule_count; + info->data = ICSSG_RXNFC_MAX_RULES | RX_CLS_LOC_SPECIAL; + break; + case ETHTOOL_GRXCLSRULE: + ret = icssg_rxnfc_rule_get(emac, info->fs.location, &info->fs); + break; + case ETHTOOL_GRXCLSRLALL: { + u32 rule_cnt = 0; + + ret = icssg_rxnfc_rule_get_all(emac, rule_locs, &rule_cnt); + if (!ret) + info->rule_cnt = rule_cnt; + break; + } + default: + return -EOPNOTSUPP; + } + + return ret; +} + +static int emac_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info) +{ + struct prueth_emac *emac = netdev_priv(ndev); + + switch (info->cmd) { + case ETHTOOL_SRXCLSRLINS: + return icssg_rxnfc_rule_add(emac, info); + case ETHTOOL_SRXCLSRLDEL: + return icssg_rxnfc_rule_del(emac, info->fs.location); + default: + return -EOPNOTSUPP; + } +} + const struct ethtool_ops icssg_ethtool_ops = { .get_drvinfo = emac_get_drvinfo, .get_msglevel = emac_get_msglevel, @@ -322,5 +368,7 @@ const struct ethtool_ops icssg_ethtool_ops = { .set_eee = emac_set_eee, .nway_reset = emac_nway_reset, .get_rmon_stats = emac_get_rmon_stats, + .get_rxnfc = emac_get_rxnfc, + .set_rxnfc = emac_set_rxnfc, }; EXPORT_SYMBOL_GPL(icssg_ethtool_ops); diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.c b/drivers/net/ethernet/ti/icssg/icssg_prueth.c index 8338402597b21..f234198d4cc42 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_prueth.c +++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.c @@ -874,10 +874,29 @@ static void prueth_destroy_rxq(struct prueth_emac *emac, int qid) { struct prueth_rx_chn *rx_chn = &emac->rx_chns; struct prueth_rx_flow *flow = &rx_chn->flows[qid]; + dma_addr_t desc_dma; - /* reset napi and hrtimer, and free all pending buffers */ napi_disable(&flow->napi_rx); hrtimer_cancel(&flow->rx_hrtimer); + + /* Drain the UDMA completion ring for this flow before resetting + * the free ring. k3_udma_glue_reset_rx_chn() only drains the + * free ring (descriptors pushed as empty receive buffers). Any + * frames that the hardware already DMA-completed but NAPI hadn't + * popped yet remain in the completion ring with their pages still + * counted as "inflight" in the page pool. Draining them here + * ensures page_pool_destroy() finds zero inflight pages and + * completes immediately instead of stalling for 60+ seconds. + * + * The caller must have stopped new frames from arriving (e.g. via + * icssg_set_port_state(ICSSG_EMAC_PORT_DISABLE)) before calling + * prueth_destroy_rxq() so this loop terminates. + */ + while (!k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, qid, &desc_dma)) { + if (!cppi5_desc_is_tdcm(desc_dma)) + prueth_rx_cleanup(rx_chn, desc_dma); + } + k3_udma_glue_reset_rx_chn(rx_chn->rx_chn, flow->id, rx_chn, prueth_rx_cleanup); @@ -1160,6 +1179,9 @@ static int emac_ndo_stop(struct net_device *ndev) prueth_cleanup_rx_chns(emac); prueth_cleanup_tx_chns(emac); + /* Cleanup RX flow classification rules */ + icssg_rxnfc_rule_cleanup(emac); + prueth->emacs_initialized--; return 0; @@ -1676,6 +1698,9 @@ static int prueth_netdev_init(struct prueth *prueth, emac->ndev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; INIT_WORK(&emac->rx_mode_work, emac_ndo_set_rx_mode_work); + /* Initialize RX flow classification */ + icssg_rxnfc_rule_init(emac); + INIT_DELAYED_WORK(&emac->stats_work, icssg_stats_work_handler); ret = pruss_request_mem_region(prueth->pruss, diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.h b/drivers/net/ethernet/ti/icssg/icssg_prueth.h index 509b9cbda58cf..7a316184aee32 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_prueth.h +++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -192,6 +193,19 @@ struct prueth_swdata { /* Minimum coalesce time in usecs for both Tx and Rx */ #define ICSSG_MIN_COALESCE_USECS 20 +/* RX flow classification (ethtool) */ +#define ICSSG_RXNFC_MAX_RULES 15 /* Max user-installable rules */ +#define ICSSG_FT3_RESERVED_SLOTS BIT(14) /* Slot 14 reserved for HSR/PRP */ + +/* RX flow classification rule structure */ +struct icssg_rxnfc_rule { + struct list_head list; + struct ethtool_rx_flow_spec flow_spec; /* User-provided filter */ + u16 ft3_slot; /* Allocated FT3 slot (0-15, excluding 14) */ + int location; /* Rule location (user-provided) */ + u32 queue_index; /* RX flow to route matches to */ +}; + /* data for each emac port */ struct prueth_emac { bool is_sr1; @@ -255,6 +269,14 @@ struct prueth_emac { struct bpf_prog *xdp_prog; struct xdp_attachment_info xdpi; unsigned long *xsk_queues; + + /* RX flow classification via ethtool */ + struct { + struct list_head rules; /* List of active RXNFC rules */ + struct mutex rxnfc_lock; /* Protect rule list */ + unsigned int rxnfc_rule_count; /* Number of active rules */ + u16 ft3_slots_used; /* Bitmask of allocated FT3 slots */ + } rxnfc; }; /* The buf includes headroom compatible with both skb and xdpf */ @@ -399,6 +421,16 @@ static inline int prueth_emac_slice(struct prueth_emac *emac) } } +/* RX flow classification functions */ +void icssg_rxnfc_rule_init(struct prueth_emac *emac); +void icssg_rxnfc_rule_cleanup(struct prueth_emac *emac); +int icssg_rxnfc_rule_add(struct prueth_emac *emac, struct ethtool_rxnfc *info); +int icssg_rxnfc_rule_del(struct prueth_emac *emac, u32 location); +int icssg_rxnfc_rule_get(struct prueth_emac *emac, u32 location, + struct ethtool_rx_flow_spec *fs); +int icssg_rxnfc_rule_get_all(struct prueth_emac *emac, u32 *rule_locs, + u32 *rule_cnt); + extern const struct ethtool_ops icssg_ethtool_ops; extern const struct dev_pm_ops prueth_dev_pm_ops; -- 2.43.0