Re: [1/4] interconnect: qcom: icc-rpmh: Add QoS config support

[Konrad Dybcio <konrad.dybcio@linaro.org>]

On 22.01.2024 15:30, Odelu Kukatla wrote:
> Introduce support to initialize QoS settings for QNOC platforms.

You should describe why this is useful.

For reference, disabling QoS programming on sm8350 on an android
kernel & userspace yields an inconsistent 1-2% difference in
benchmarks like geekbench or antutu, but perhaps it's useful for
not clogging up the NoCs when there's a lot of multimedia-dram
traffic etc.?

> 
> Change-Id: I068d49cbcfec5d34c01e5adc930eec72d306ed89

This tag has no place upstream

> Signed-off-by: Odelu Kukatla <quic_okukatla@quicinc.com>
> ---
>  drivers/interconnect/qcom/icc-rpmh.c | 158 +++++++++++++++++++++++++++
>  drivers/interconnect/qcom/icc-rpmh.h |  33 ++++++
>  2 files changed, 191 insertions(+)
> 
> diff --git a/drivers/interconnect/qcom/icc-rpmh.c b/drivers/interconnect/qcom/icc-rpmh.c
> index c1aa265c1f4e..49334065ccfa 100644
> --- a/drivers/interconnect/qcom/icc-rpmh.c
> +++ b/drivers/interconnect/qcom/icc-rpmh.c
> @@ -1,8 +1,10 @@
>  // SPDX-License-Identifier: GPL-2.0
>  /*
>   * Copyright (c) 2020, The Linux Foundation. All rights reserved.
> + * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
>   */
>  
> +#include <linux/clk.h>
>  #include <linux/interconnect.h>
>  #include <linux/interconnect-provider.h>
>  #include <linux/module.h>
> @@ -14,6 +16,37 @@
>  #include "icc-common.h"
>  #include "icc-rpmh.h"
>  
> +/* QNOC QoS */
> +#define QOSGEN_MAINCTL_LO(p, qp)	(0x8 + (p->offsets[qp]))
> +#define QOS_SLV_URG_MSG_EN_SHFT		3
> +#define QOS_DFLT_PRIO_MASK		0x7
> +#define QOS_DFLT_PRIO_SHFT		4
> +#define QOS_DISABLE_SHIFT		24

mask + shift -> GENMASK(), then use FIELD_PREP/GET in the callers

These are already defined in icc-rpm.c.. Perhaps they can be factored out
to icc-qnoc.h or something?

[...]

> +
> +static int enable_qos_deps(struct qcom_icc_provider *qp)

Can we perhaps integrate this into .sync_state?

Currently, !synced_state holds all paths (and by extension, all BCMs)
at their max values, so they're definitely enabled, and it conviniently
is also supposed to only fire once.

> +{
> +	struct qcom_icc_bcm *bcm;
> +	bool keepalive;
> +	int ret, i;
> +
> +	for (i = 0; i < qp->num_bcms; i++) {
> +		bcm = qp->bcms[i];
> +		if (bcm_needs_qos_proxy(bcm)) {
> +			keepalive = bcm->keepalive;
> +			bcm->keepalive = true;
> +
> +			qcom_icc_bcm_voter_add(qp->voter, bcm);
> +			ret = qcom_icc_bcm_voter_commit(qp->voter);
> +
> +			bcm->keepalive = keepalive;
> +
> +			if (ret) {
> +				dev_err(qp->dev, "failed to vote BW to %s for QoS\n",
> +					bcm->name);
> +				return ret;
> +			}
> +		}
> +	}
> +
> +	ret = clk_bulk_prepare_enable(qp->num_clks, qp->clks);
> +	if (ret) {
> +		dev_err(qp->dev, "failed to enable clocks for QoS\n");
> +		return ret;
> +	}

if (ret)
	dev_err(qp->dev...

return ret;

> +
> +	return 0;
> +}
> +
> +static void disable_qos_deps(struct qcom_icc_provider *qp)
> +{
> +	struct qcom_icc_bcm *bcm;
> +	int i;
> +
> +	clk_bulk_disable_unprepare(qp->num_clks, qp->clks);
> +
> +	for (i = 0; i < qp->num_bcms; i++) {
> +		bcm = qp->bcms[i];
> +		if (bcm_needs_qos_proxy(bcm)) {
> +			qcom_icc_bcm_voter_add(qp->voter, bcm);
> +			qcom_icc_bcm_voter_commit(qp->voter);
> +		}
> +	}
> +}
> +
> +int qcom_icc_rpmh_configure_qos(struct qcom_icc_provider *qp)
> +{
> +	struct qcom_icc_node *qnode;
> +	size_t i;
> +	int ret;
> +
> +	ret = enable_qos_deps(qp);
> +	if (ret)
> +		return ret;
> +
> +	for (i = 0; i < qp->num_nodes; i++) {
> +		qnode = qp->nodes[i];
> +		if (!qnode)
> +			continue;
> +
> +		if (qnode->qosbox)
> +			qcom_icc_set_qos(qnode);
> +	}
> +
> +	disable_qos_deps(qp);
> +
> +	return ret;
> +}
> +EXPORT_SYMBOL_GPL(qcom_icc_rpmh_configure_qos);

This is simply copypasted from downstream [1].. not necessary at all,
in this patch this func is exclusively called from within this file.

> +
> +static struct regmap *qcom_icc_rpmh_map(struct platform_device *pdev,
> +					const struct qcom_icc_desc *desc)
> +{
> +	void __iomem *base;
> +	struct resource *res;
> +	struct device *dev = &pdev->dev;

Reverse-Christmas-tree throughout the code, please

> +
> +	if (!desc->config)
> +		return NULL;
> +
> +	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
> +	if (!res)
> +		return NULL;
> +
> +	base = devm_ioremap(dev, res->start, resource_size(res));
> +	if (IS_ERR(base))
> +		return ERR_CAST(base);
> +
> +	return devm_regmap_init_mmio(dev, base, desc->config);
> +}


This is devm_platform_get_and_ioremap_resource + devm_regmap_init_mmio

please inline this in the probe func

[...]

>  
> @@ -213,6 +363,8 @@ int qcom_icc_rpmh_probe(struct platform_device *pdev)
>  		if (!qn)
>  			continue;
>  
> +		qn->regmap = dev_get_regmap(qp->dev, NULL);

Why would all nodes need a regmap reference? there's to_qcom_provider()

> +
>  		node = icc_node_create(qn->id);
>  		if (IS_ERR(node)) {
>  			ret = PTR_ERR(node);
> @@ -229,6 +381,10 @@ int qcom_icc_rpmh_probe(struct platform_device *pdev)
>  		data->nodes[i] = node;
>  	}
>  
> +	ret = qcom_icc_rpmh_configure_qos(qp);
> +	if (ret)
> +		goto err_remove_nodes;
> +
>  	ret = icc_provider_register(provider);
>  	if (ret)
>  		goto err_remove_nodes;
> @@ -247,6 +403,7 @@ int qcom_icc_rpmh_probe(struct platform_device *pdev)
>  err_deregister_provider:
>  	icc_provider_deregister(provider);
>  err_remove_nodes:
> +	clk_bulk_put_all(qp->num_clks, qp->clks);

Use devm_clk_bulk_get_all instead

[...]

> + * @nodes: list of interconnect nodes that maps to the provider
> + * @num_nodes: number of @nodes
> + * @regmap: used for QOS registers access

QoS, 'register'

> + * @clks : clks required for register access
> + * @num_clks: number of @clks
>   */
>  struct qcom_icc_provider {
>  	struct icc_provider provider;
> @@ -25,6 +31,11 @@ struct qcom_icc_provider {
>  	struct qcom_icc_bcm * const *bcms;
>  	size_t num_bcms;
>  	struct bcm_voter *voter;
> +	struct qcom_icc_node * const *nodes;
> +	size_t num_nodes;
> +	struct regmap *regmap;
> +	struct clk_bulk_data *clks;
> +	int num_clks;
>  };
>  
>  /**
> @@ -41,6 +52,23 @@ struct bcm_db {
>  	u8 reserved;
>  };
>  
> +/**
> + * struct qcom_icc_qosbox - Qualcomm Technologies, Inc specific QoS config

qosbox -> qos

plus I'm not sure if the full company name adds value to a driver in
drivers/interconnect/qcom..

> + * @prio: priority value assigned to requests on the node
> + * @urg_fwd: if set, master priority is used for requests.

"master priority" meaning "this req goes before anyone else", or "use the
icc provider [master]'s priority value"?

> + * @prio_fwd_disable: if set, master priority is ignored and NOCs default priority is used.

NoC's

This sounds like !(prio || urg_fwd)? Surely it must do something more useful?


> + * @num_ports: number of @ports
> + * @offsets: qos register offsets
> + */
> +
> +struct qcom_icc_qosbox {
> +	u32 prio;
> +	u32 urg_fwd;
> +	bool prio_fwd_disable;
> +	u32 num_ports;
> +	u32 offsets[];

u32 offsets __counted_by(num_ports)

Also, it would probably be more clear if you renamed it to "port_offsets"

> +};
> +
>  #define MAX_LINKS		128
>  #define MAX_BCMS		64
>  #define MAX_BCM_PER_NODE	3
> @@ -58,6 +86,8 @@ struct bcm_db {
>   * @max_peak: current max aggregate value of all peak bw requests
>   * @bcms: list of bcms associated with this logical node
>   * @num_bcms: num of @bcms
> + * @regmap: used for QOS registers access
> + * @qosbox: qos config data associated with node
>   */
>  struct qcom_icc_node {
>  	const char *name;
> @@ -70,6 +100,8 @@ struct qcom_icc_node {
>  	u64 max_peak[QCOM_ICC_NUM_BUCKETS];
>  	struct qcom_icc_bcm *bcms[MAX_BCM_PER_NODE];
>  	size_t num_bcms;
> +	struct regmap *regmap;

Remove

> +	struct qcom_icc_qosbox *qosbox;

Why would this be a pointer and not a const member of the struct?

It seems totally counter-intuitive to reuse QoS settings for more than
one node, given their offsets are unique.

Konrad

[1] https://git.codelinaro.org/clo/la/kernel/msm-5.15/-/blob/kernel.lnx.5.15.r26-rel/drivers/interconnect/qcom/icc-rpmh.c?ref_type=heads#L329-354