[PATCH v10 00/11] drm/etnaviv: Add pci device driver support

[PATCH v10 00/11] drm/etnaviv: Add pci device driver support

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

There is a Vivante GC1000 (v5037) in LS2K1000 and LS7A1000, this GPU is a
PCI device, and it has 2D and 3D cores in the same core. This series is
trying to add PCI device driver support to drm/etnaviv.

v6:
	* Fix build issue on system without CONFIG_PCI enabled
v7:
	* Add a separate patch for the platform driver rearrangement (Bjorn)
	* Switch to runtime check if the GPU is dma coherent or not (Lucas)
	* Add ETNAVIV_PARAM_GPU_COHERENT to allow userspace to query (Lucas)
	* Remove etnaviv_gpu.no_clk member (Lucas)
	* Various Typos and coding style fixed (Bjorn)
v8:
	* Fix typos and remove unnecessary header included (Bjorn).
	* Add a dedicated function to create the virtual master platform
	  device.
v9:
	* Use PCI_VDEVICE() macro (Bjorn)
	* Add trivial stubs for the PCI driver (Bjorn)
	* Remove a redundant dev_err() usage (Bjorn)
	* Clean up etnaviv_pdev_probe() with etnaviv_of_first_available_node()
v10:
	* Add one more cleanup patch
	* Resolve the conflict with a patch from Rob
	* Make the dummy PCI stub inlined
	* Print only if the platform is dma-coherrent

Rob Herring (1):
  drm/etnaviv: Replace of_platform.h with explicit includes

Sui Jingfeng (10):
  drm/etnaviv: Add a dedicated function to register an irq handler
  drm/etnaviv: Add a dedicated function to get various clocks
  drm/etnaviv: Add dedicated functions to create and destroy platform
    device
  drm/etnaviv: Add helpers for private data construction and destruction
  drm/etnaviv: Allow bypass component framework
  drm/etnaviv: Add driver support for the PCI devices
  drm/etnaviv: Add support for the dma coherent device
  drm/etnaviv: Add a dedicated function to create the virtual master
  drm/etnaviv: Clean up etnaviv_pdev_probe() function
  drm/etnaviv: Keep the curly brace aligned

 drivers/gpu/drm/etnaviv/Kconfig             |  10 +
 drivers/gpu/drm/etnaviv/Makefile            |   2 +
 drivers/gpu/drm/etnaviv/etnaviv_drv.c       | 322 ++++++++++++++------
 drivers/gpu/drm/etnaviv/etnaviv_drv.h       |  10 +
 drivers/gpu/drm/etnaviv/etnaviv_gem.c       |  22 +-
 drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c |   7 +-
 drivers/gpu/drm/etnaviv/etnaviv_gpu.c       | 166 ++++++----
 drivers/gpu/drm/etnaviv/etnaviv_gpu.h       |   9 +
 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c   |  75 +++++
 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h   |  18 ++
 include/uapi/drm/etnaviv_drm.h              |   1 +
 11 files changed, 481 insertions(+), 161 deletions(-)
 create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
 create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h

-- 
2.25.1

[PATCH v10 01/11] drm/etnaviv: Add a dedicated function to register an irq handler

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

Because getting IRQ from a device is platform-dependent, PCI devices have
different methods for getting an IRQ. This patch is a preparation to extend
this driver for supporting the PCI devices.

Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 32 +++++++++++++++++++--------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index de8c9894967c..a03e81337d8f 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -1817,6 +1817,27 @@ static const struct of_device_id etnaviv_gpu_match[] = {
 };
 MODULE_DEVICE_TABLE(of, etnaviv_gpu_match);
 
+static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
+{
+	struct device *dev = gpu->dev;
+	int err;
+
+	if (irq < 0)
+		return irq;
+
+	err = devm_request_irq(dev, irq, irq_handler, 0, dev_name(dev), gpu);
+	if (err) {
+		dev_err(dev, "failed to request irq %u: %d\n", irq, err);
+		return err;
+	}
+
+	gpu->irq = irq;
+
+	dev_info(dev, "irq(%d) handler registered\n", irq);
+
+	return 0;
+}
+
 static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -1837,16 +1858,9 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 		return PTR_ERR(gpu->mmio);
 
 	/* Get Interrupt: */
-	gpu->irq = platform_get_irq(pdev, 0);
-	if (gpu->irq < 0)
-		return gpu->irq;
-
-	err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0,
-			       dev_name(gpu->dev), gpu);
-	if (err) {
-		dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err);
+	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
+	if (err)
 		return err;
-	}
 
 	/* Get Clocks: */
 	gpu->clk_reg = devm_clk_get_optional(&pdev->dev, "reg");
-- 
2.25.1

Re: [PATCH v10 01/11] drm/etnaviv: Add a dedicated function to register an irq handler

[Lucas Stach]

Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
> From: Sui Jingfeng <suijingfeng@loongson.cn>
> 
> Because getting IRQ from a device is platform-dependent, PCI devices have
> different methods for getting an IRQ. This patch is a preparation to extend
> this driver for supporting the PCI devices.
> 
> Cc: Lucas Stach <l.stach@pengutronix.de>
> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> Cc: Philipp Zabel <p.zabel@pengutronix.de>
> Cc: Bjorn Helgaas <bhelgaas@google.com>
> Cc: Daniel Vetter <daniel@ffwll.ch>
> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> ---
>  drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 32 +++++++++++++++++++--------
>  1 file changed, 23 insertions(+), 9 deletions(-)
> 
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> index de8c9894967c..a03e81337d8f 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> @@ -1817,6 +1817,27 @@ static const struct of_device_id etnaviv_gpu_match[] = {
>  };
>  MODULE_DEVICE_TABLE(of, etnaviv_gpu_match);
>  
> +static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
> +{
> +	struct device *dev = gpu->dev;
> +	int err;
> +
> +	if (irq < 0)
> +		return irq;
> +
> +	err = devm_request_irq(dev, irq, irq_handler, 0, dev_name(dev), gpu);
> +	if (err) {
> +		dev_err(dev, "failed to request irq %u: %d\n", irq, err);
> +		return err;
> +	}
> +
> +	gpu->irq = irq;
> +
> +	dev_info(dev, "irq(%d) handler registered\n", irq);

There is no reason to put this into the kernel log. It's no different
than other resources to the driver and we don't log each one of those
either.

In fact I don't see any reason for this change in the first place.
Effectively you are moving a single function call into a new function,
which doesn't seem like an improvement.

Regards,
Lucas

> +
> +	return 0;
> +}
> +
>  static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>  {
>  	struct device *dev = &pdev->dev;
> @@ -1837,16 +1858,9 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>  		return PTR_ERR(gpu->mmio);
>  
>  	/* Get Interrupt: */
> -	gpu->irq = platform_get_irq(pdev, 0);
> -	if (gpu->irq < 0)
> -		return gpu->irq;
> -
> -	err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0,
> -			       dev_name(gpu->dev), gpu);
> -	if (err) {
> -		dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err);
> +	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
> +	if (err)
>  		return err;
> -	}
>  
>  	/* Get Clocks: */
>  	gpu->clk_reg = devm_clk_get_optional(&pdev->dev, "reg");

Re: [PATCH v10 01/11] drm/etnaviv: Add a dedicated function to register an irq handler

[Sui Jingfeng]

Hi,

On 2023/6/21 17:07, Lucas Stach wrote:
> Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>
>> Because getting IRQ from a device is platform-dependent, PCI devices have
>> different methods for getting an IRQ. This patch is a preparation to extend
>> this driver for supporting the PCI devices.
>>
>> Cc: Lucas Stach <l.stach@pengutronix.de>
>> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
>> Cc: Philipp Zabel <p.zabel@pengutronix.de>
>> Cc: Bjorn Helgaas <bhelgaas@google.com>
>> Cc: Daniel Vetter <daniel@ffwll.ch>
>> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
>> ---
>>   drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 32 +++++++++++++++++++--------
>>   1 file changed, 23 insertions(+), 9 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> index de8c9894967c..a03e81337d8f 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> @@ -1817,6 +1817,27 @@ static const struct of_device_id etnaviv_gpu_match[] = {
>>   };
>>   MODULE_DEVICE_TABLE(of, etnaviv_gpu_match);
>>   
>> +static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
>> +{
>> +	struct device *dev = gpu->dev;
>> +	int err;
>> +
>> +	if (irq < 0)
>> +		return irq;
>> +
>> +	err = devm_request_irq(dev, irq, irq_handler, 0, dev_name(dev), gpu);
>> +	if (err) {
>> +		dev_err(dev, "failed to request irq %u: %d\n", irq, err);
>> +		return err;
>> +	}
>> +
>> +	gpu->irq = irq;
>> +
>> +	dev_info(dev, "irq(%d) handler registered\n", irq);
> There is no reason to put this into the kernel log.

I want to see the IRQ of the device when debugging,

etnaviv actually print very less.

This serve as a minimal signal  to us the etnaviv_gpu_register_irq() 
function is successful at driver load time.

>   It's no different
> than other resources to the driver and we don't log each one of those
> either.
>
> In fact I don't see any reason for this change in the first place.
> Effectively you are moving a single function call into a new function,
> which doesn't seem like an improvement.

This is to make the patch easy to review, each patch is only introduce a 
small function,

which is paving the way for we introducing the PCI device driver.

Otherwise when we introducing the PCI device driver, the patch is looks 
ugly,

It is difficult to review.

> Regards,
> Lucas
>
>> +
>> +	return 0;
>> +}
>> +
>>   static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>>   {
>>   	struct device *dev = &pdev->dev;
>> @@ -1837,16 +1858,9 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>>   		return PTR_ERR(gpu->mmio);
>>   
>>   	/* Get Interrupt: */
>> -	gpu->irq = platform_get_irq(pdev, 0);
>> -	if (gpu->irq < 0)
>> -		return gpu->irq;
>> -
>> -	err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0,
>> -			       dev_name(gpu->dev), gpu);
>> -	if (err) {
>> -		dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err);
>> +	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
>> +	if (err)
>>   		return err;
>> -	}
>>   
>>   	/* Get Clocks: */
>>   	gpu->clk_reg = devm_clk_get_optional(&pdev->dev, "reg");

-- 
Jingfeng

Re: [PATCH v10 01/11] drm/etnaviv: Add a dedicated function to register an irq handler

[Lucas Stach]

Am Mittwoch, dem 21.06.2023 um 17:20 +0800 schrieb Sui Jingfeng:
> Hi,
> 
> On 2023/6/21 17:07, Lucas Stach wrote:
> > Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
> > > From: Sui Jingfeng <suijingfeng@loongson.cn>
> > > 
> > > Because getting IRQ from a device is platform-dependent, PCI devices have
> > > different methods for getting an IRQ. This patch is a preparation to extend
> > > this driver for supporting the PCI devices.
> > > 
> > > Cc: Lucas Stach <l.stach@pengutronix.de>
> > > Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> > > Cc: Philipp Zabel <p.zabel@pengutronix.de>
> > > Cc: Bjorn Helgaas <bhelgaas@google.com>
> > > Cc: Daniel Vetter <daniel@ffwll.ch>
> > > Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> > > ---
> > >   drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 32 +++++++++++++++++++--------
> > >   1 file changed, 23 insertions(+), 9 deletions(-)
> > > 
> > > diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> > > index de8c9894967c..a03e81337d8f 100644
> > > --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> > > +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> > > @@ -1817,6 +1817,27 @@ static const struct of_device_id etnaviv_gpu_match[] = {
> > >   };
> > >   MODULE_DEVICE_TABLE(of, etnaviv_gpu_match);
> > >   
> > > +static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
> > > +{
> > > +	struct device *dev = gpu->dev;
> > > +	int err;
> > > +
> > > +	if (irq < 0)
> > > +		return irq;
> > > +
> > > +	err = devm_request_irq(dev, irq, irq_handler, 0, dev_name(dev), gpu);
> > > +	if (err) {
> > > +		dev_err(dev, "failed to request irq %u: %d\n", irq, err);
> > > +		return err;
> > > +	}
> > > +
> > > +	gpu->irq = irq;
> > > +
> > > +	dev_info(dev, "irq(%d) handler registered\n", irq);
> > There is no reason to put this into the kernel log.
> 
> I want to see the IRQ of the device when debugging,
> 
> etnaviv actually print very less.
> 
> This serve as a minimal signal  to us the etnaviv_gpu_register_irq() 
> function is successful at driver load time.
> 
And debugging is a very different use-case than normal operation. If
it's needed at all, downgrade this to dev_dbg. This isn't interesting
information for a ordinary user of a system.

> >   It's no different
> > than other resources to the driver and we don't log each one of those
> > either.
> > 
> > In fact I don't see any reason for this change in the first place.
> > Effectively you are moving a single function call into a new function,
> > which doesn't seem like an improvement.
> 
> This is to make the patch easy to review, each patch is only introduce a 
> small function,
> 
What I'm saying is that I don't see the need to introduce this function
at all. All you need to do is move platform_get_irq out into the
platform device code path. The devm_request_irq can stay where it is,
as the only difference between platform and PCI device is how the irq
number is retrieved from the platform.

Regards,
Lucas

> which is paving the way for we introducing the PCI device driver.
> 
> Otherwise when we introducing the PCI device driver, the patch is looks 
> ugly,
> 
> It is difficult to review.
> 
> > Regards,
> > Lucas
> > 
> > > +
> > > +	return 0;
> > > +}
> > > +
> > >   static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
> > >   {
> > >   	struct device *dev = &pdev->dev;
> > > @@ -1837,16 +1858,9 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
> > >   		return PTR_ERR(gpu->mmio);
> > >   
> > >   	/* Get Interrupt: */
> > > -	gpu->irq = platform_get_irq(pdev, 0);
> > > -	if (gpu->irq < 0)
> > > -		return gpu->irq;
> > > -
> > > -	err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0,
> > > -			       dev_name(gpu->dev), gpu);
> > > -	if (err) {
> > > -		dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err);
> > > +	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
> > > +	if (err)
> > >   		return err;
> > > -	}
> > >   
> > >   	/* Get Clocks: */
> > >   	gpu->clk_reg = devm_clk_get_optional(&pdev->dev, "reg");
> 

Re: [PATCH v10 01/11] drm/etnaviv: Add a dedicated function to register an irq handler

[Sui Jingfeng]

Hi,

On 2023/6/21 18:16, Lucas Stach wrote:
> Am Mittwoch, dem 21.06.2023 um 17:20 +0800 schrieb Sui Jingfeng:
>> Hi,
>>
>> On 2023/6/21 17:07, Lucas Stach wrote:
>>> Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
>>>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>>>
>>>> Because getting IRQ from a device is platform-dependent, PCI devices have
>>>> different methods for getting an IRQ. This patch is a preparation to extend
>>>> this driver for supporting the PCI devices.
>>>>
>>>> Cc: Lucas Stach <l.stach@pengutronix.de>
>>>> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
>>>> Cc: Philipp Zabel <p.zabel@pengutronix.de>
>>>> Cc: Bjorn Helgaas <bhelgaas@google.com>
>>>> Cc: Daniel Vetter <daniel@ffwll.ch>
>>>> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
>>>> ---
>>>>    drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 32 +++++++++++++++++++--------
>>>>    1 file changed, 23 insertions(+), 9 deletions(-)
>>>>
>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>>>> index de8c9894967c..a03e81337d8f 100644
>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>>>> @@ -1817,6 +1817,27 @@ static const struct of_device_id etnaviv_gpu_match[] = {
>>>>    };
>>>>    MODULE_DEVICE_TABLE(of, etnaviv_gpu_match);
>>>>    
>>>> +static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
>>>> +{
>>>> +	struct device *dev = gpu->dev;
>>>> +	int err;
>>>> +
>>>> +	if (irq < 0)
>>>> +		return irq;
>>>> +
>>>> +	err = devm_request_irq(dev, irq, irq_handler, 0, dev_name(dev), gpu);
>>>> +	if (err) {
>>>> +		dev_err(dev, "failed to request irq %u: %d\n", irq, err);
>>>> +		return err;
>>>> +	}
>>>> +
>>>> +	gpu->irq = irq;
>>>> +
>>>> +	dev_info(dev, "irq(%d) handler registered\n", irq);
>>> There is no reason to put this into the kernel log.
>> I want to see the IRQ of the device when debugging,
>>
>> etnaviv actually print very less.
>>
>> This serve as a minimal signal  to us the etnaviv_gpu_register_irq()
>> function is successful at driver load time.
>>
> And debugging is a very different use-case than normal operation. If
> it's needed at all, downgrade this to dev_dbg. This isn't interesting
> information for a ordinary user of a system.
>
>>>    It's no different
>>> than other resources to the driver and we don't log each one of those
>>> either.
>>>
>>> In fact I don't see any reason for this change in the first place.
>>> Effectively you are moving a single function call into a new function,
>>> which doesn't seem like an improvement.
>> This is to make the patch easy to review, each patch is only introduce a
>> small function,
>>
> What I'm saying is that I don't see the need to introduce this function
> at all. All you need to do is move platform_get_irq out into the
> platform device code path. The devm_request_irq can stay where it is,
> as the only difference between platform and PCI device is how the irq
> number is retrieved from the platform.

Yes, you are right. I understand what are asking, but my point is:


This patch is paving the way for us to introduce the PCI device driver.

All of the patches before the patch 
v10-0006-drm-etnaviv-Add-driver-support-for-the-PCI-devic.patch

are actually doing the preparation.


Look at the patch 0006, I achieve the goal by 128 insertions and 7 
deletions.

while the only 7 deletions are actually for code shading(convert the 
static function to global function).

There is No large area diff and NO distortion.

The goal is adding a PCI device driver on the top of what we already have.


Before the cleanup, the etnaviv_gpu_platform_probe() function is just 
like is a *glue*.

Originally,  it integrate a lot of irrelevant part together.


1.  Mapping MMIO registers make it platform-dependent;

2.  Calling platform_get_irq(pdev, 0) make it platform-dependent;

3.  Getting Clocks by calling devm_clk_get() make it platform-dependent;

4.  Calling component_add() make it subsytem and framework-dependent;


All of above list item is deny us to introduce the PCI device driver 
wrapper.

It(sub-functional code) is not relevant to each other.

Hence the first five patch is actually do the clean,

for the clarify(and tidy and good looking) of the patch 6.


I will drop the printing, but keep the cleanup function there,

Is this acceptable?


> Regards,
> Lucas
>
>> which is paving the way for we introducing the PCI device driver.
>>
>> Otherwise when we introducing the PCI device driver, the patch is looks
>> ugly,
>>
>> It is difficult to review.
>>
>>> Regards,
>>> Lucas
>>>
>>>> +
>>>> +	return 0;
>>>> +}
>>>> +
>>>>    static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>>>>    {
>>>>    	struct device *dev = &pdev->dev;
>>>> @@ -1837,16 +1858,9 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>>>>    		return PTR_ERR(gpu->mmio);
>>>>    
>>>>    	/* Get Interrupt: */
>>>> -	gpu->irq = platform_get_irq(pdev, 0);
>>>> -	if (gpu->irq < 0)
>>>> -		return gpu->irq;
>>>> -
>>>> -	err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0,
>>>> -			       dev_name(gpu->dev), gpu);
>>>> -	if (err) {
>>>> -		dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err);
>>>> +	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
>>>> +	if (err)
>>>>    		return err;
>>>> -	}
>>>>    
>>>>    	/* Get Clocks: */
>>>>    	gpu->clk_reg = devm_clk_get_optional(&pdev->dev, "reg");

-- 
Jingfeng

Re: [PATCH v10 01/11] drm/etnaviv: Add a dedicated function to register an irq handler

[Lucas Stach]

Am Samstag, dem 24.06.2023 um 23:53 +0800 schrieb Sui Jingfeng:
> Hi,
> 
> On 2023/6/21 18:16, Lucas Stach wrote:
> > Am Mittwoch, dem 21.06.2023 um 17:20 +0800 schrieb Sui Jingfeng:
> > > Hi,
> > > 
> > > On 2023/6/21 17:07, Lucas Stach wrote:
> > > > Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
> > > > > From: Sui Jingfeng <suijingfeng@loongson.cn>
> > > > > 
> > > > > Because getting IRQ from a device is platform-dependent, PCI devices have
> > > > > different methods for getting an IRQ. This patch is a preparation to extend
> > > > > this driver for supporting the PCI devices.
> > > > > 
> > > > > Cc: Lucas Stach <l.stach@pengutronix.de>
> > > > > Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> > > > > Cc: Philipp Zabel <p.zabel@pengutronix.de>
> > > > > Cc: Bjorn Helgaas <bhelgaas@google.com>
> > > > > Cc: Daniel Vetter <daniel@ffwll.ch>
> > > > > Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> > > > > ---
> > > > >    drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 32 +++++++++++++++++++--------
> > > > >    1 file changed, 23 insertions(+), 9 deletions(-)
> > > > > 
> > > > > diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> > > > > index de8c9894967c..a03e81337d8f 100644
> > > > > --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> > > > > +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> > > > > @@ -1817,6 +1817,27 @@ static const struct of_device_id etnaviv_gpu_match[] = {
> > > > >    };
> > > > >    MODULE_DEVICE_TABLE(of, etnaviv_gpu_match);
> > > > >    
> > > > > +static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
> > > > > +{
> > > > > +	struct device *dev = gpu->dev;
> > > > > +	int err;
> > > > > +
> > > > > +	if (irq < 0)
> > > > > +		return irq;
> > > > > +
> > > > > +	err = devm_request_irq(dev, irq, irq_handler, 0, dev_name(dev), gpu);
> > > > > +	if (err) {
> > > > > +		dev_err(dev, "failed to request irq %u: %d\n", irq, err);
> > > > > +		return err;
> > > > > +	}
> > > > > +
> > > > > +	gpu->irq = irq;
> > > > > +
> > > > > +	dev_info(dev, "irq(%d) handler registered\n", irq);
> > > > There is no reason to put this into the kernel log.
> > > I want to see the IRQ of the device when debugging,
> > > 
> > > etnaviv actually print very less.
> > > 
> > > This serve as a minimal signal  to us the etnaviv_gpu_register_irq()
> > > function is successful at driver load time.
> > > 
> > And debugging is a very different use-case than normal operation. If
> > it's needed at all, downgrade this to dev_dbg. This isn't interesting
> > information for a ordinary user of a system.
> > 
> > > >    It's no different
> > > > than other resources to the driver and we don't log each one of those
> > > > either.
> > > > 
> > > > In fact I don't see any reason for this change in the first place.
> > > > Effectively you are moving a single function call into a new function,
> > > > which doesn't seem like an improvement.
> > > This is to make the patch easy to review, each patch is only introduce a
> > > small function,
> > > 
> > What I'm saying is that I don't see the need to introduce this function
> > at all. All you need to do is move platform_get_irq out into the
> > platform device code path. The devm_request_irq can stay where it is,
> > as the only difference between platform and PCI device is how the irq
> > number is retrieved from the platform.
> 
> Yes, you are right. I understand what are asking, but my point is:
> 
> 
> This patch is paving the way for us to introduce the PCI device driver.
> 
> All of the patches before the patch 
> v10-0006-drm-etnaviv-Add-driver-support-for-the-PCI-devic.patch
> 
> are actually doing the preparation.
> 
> 
> Look at the patch 0006, I achieve the goal by 128 insertions and 7 
> deletions.
> 
> while the only 7 deletions are actually for code shading(convert the 
> static function to global function).
> 
> There is No large area diff and NO distortion.
> 
> The goal is adding a PCI device driver on the top of what we already have.
> 
> 
> Before the cleanup, the etnaviv_gpu_platform_probe() function is just 
> like is a *glue*.
> 
> Originally,  it integrate a lot of irrelevant part together.
> 
> 
> 1.  Mapping MMIO registers make it platform-dependent;
> 
> 2.  Calling platform_get_irq(pdev, 0) make it platform-dependent;
> 
I get that and I'm fine with moving this out of this function. But
that's not what you are actually doing in this patch. You are moving
devm_request_irq out of this function, which isn't necessary, since the
request IRQ is not platform dependent. It's the retrieval of the IRQ
number to request (platform_get_irq/pdev->irq) which is platform
dependent.

Regards,
Lucas

> 3.  Getting Clocks by calling devm_clk_get() make it platform-dependent;
> 
> 4.  Calling component_add() make it subsytem and framework-dependent;
> 
> 
> All of above list item is deny us to introduce the PCI device driver 
> wrapper.
> 
> It(sub-functional code) is not relevant to each other.
> 
> Hence the first five patch is actually do the clean,
> 
> for the clarify(and tidy and good looking) of the patch 6.
> 
> 
> I will drop the printing, but keep the cleanup function there,
> 
> Is this acceptable?
> 
> 
> > Regards,
> > Lucas
> > 
> > > which is paving the way for we introducing the PCI device driver.
> > > 
> > > Otherwise when we introducing the PCI device driver, the patch is looks
> > > ugly,
> > > 
> > > It is difficult to review.
> > > 
> > > > Regards,
> > > > Lucas
> > > > 
> > > > > +
> > > > > +	return 0;
> > > > > +}
> > > > > +
> > > > >    static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
> > > > >    {
> > > > >    	struct device *dev = &pdev->dev;
> > > > > @@ -1837,16 +1858,9 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
> > > > >    		return PTR_ERR(gpu->mmio);
> > > > >    
> > > > >    	/* Get Interrupt: */
> > > > > -	gpu->irq = platform_get_irq(pdev, 0);
> > > > > -	if (gpu->irq < 0)
> > > > > -		return gpu->irq;
> > > > > -
> > > > > -	err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0,
> > > > > -			       dev_name(gpu->dev), gpu);
> > > > > -	if (err) {
> > > > > -		dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err);
> > > > > +	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
> > > > > +	if (err)
> > > > >    		return err;
> > > > > -	}
> > > > >    
> > > > >    	/* Get Clocks: */
> > > > >    	gpu->clk_reg = devm_clk_get_optional(&pdev->dev, "reg");
> 

Re: [PATCH v10 01/11] drm/etnaviv: Add a dedicated function to register an irq handler

[Sui Jingfeng]

Hi,

On 2023/6/21 17:07, Lucas Stach wrote:
> Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>
>> Because getting IRQ from a device is platform-dependent, PCI devices have
>> different methods for getting an IRQ. This patch is a preparation to extend
>> this driver for supporting the PCI devices.
>>
>> Cc: Lucas Stach <l.stach@pengutronix.de>
>> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
>> Cc: Philipp Zabel <p.zabel@pengutronix.de>
>> Cc: Bjorn Helgaas <bhelgaas@google.com>
>> Cc: Daniel Vetter <daniel@ffwll.ch>
>> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
>> ---
>>   drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 32 +++++++++++++++++++--------
>>   1 file changed, 23 insertions(+), 9 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> index de8c9894967c..a03e81337d8f 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> @@ -1817,6 +1817,27 @@ static const struct of_device_id etnaviv_gpu_match[] = {
>>   };
>>   MODULE_DEVICE_TABLE(of, etnaviv_gpu_match);
>>   
>> +static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
>> +{
>> +	struct device *dev = gpu->dev;
>> +	int err;
>> +
>> +	if (irq < 0)
>> +		return irq;
>> +
>> +	err = devm_request_irq(dev, irq, irq_handler, 0, dev_name(dev), gpu);
>> +	if (err) {
>> +		dev_err(dev, "failed to request irq %u: %d\n", irq, err);
>> +		return err;
>> +	}
>> +
>> +	gpu->irq = irq;
>> +
>> +	dev_info(dev, "irq(%d) handler registered\n", irq);
> There is no reason to put this into the kernel log. It's no different
> than other resources to the driver and we don't log each one of those
> either.
>
> In fact I don't see any reason for this change in the first place.
> Effectively you are moving a single function call into a new function,
> which doesn't seem like an improvement.

Hi, another reason is that we observed that

It(register irq) has no relationship to rest of the 
etnaviv_gpu_driver_create() funciton,

so it should be stand alone.

After stand alone, it is not platform-dependent any more,

it can be shared by both the PCI device driver and the platform device 
driver.


So, this is what I'm thinking when I create this function.

> Regards,
> Lucas
>
>> +
>> +	return 0;
>> +}
>> +
>>   static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>>   {
>>   	struct device *dev = &pdev->dev;
>> @@ -1837,16 +1858,9 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>>   		return PTR_ERR(gpu->mmio);
>>   
>>   	/* Get Interrupt: */
>> -	gpu->irq = platform_get_irq(pdev, 0);
>> -	if (gpu->irq < 0)
>> -		return gpu->irq;
>> -
>> -	err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0,
>> -			       dev_name(gpu->dev), gpu);
>> -	if (err) {
>> -		dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err);
>> +	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
>> +	if (err)
>>   		return err;
>> -	}
>>   
>>   	/* Get Clocks: */
>>   	gpu->clk_reg = devm_clk_get_optional(&pdev->dev, "reg");

-- 
Jingfeng

[PATCH v10 02/11] drm/etnaviv: Add a dedicated function to get various clocks

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

Because it is also platform-dependent, there are systems where we don't
have DT-based clock drivers supported. For example, discrete PCI GPUs.
Therefire, don't quit if there is no clock subsystem support.
    
For the GPU in LS7A1000 and LS2K1000, the working frequency of the GPU is
rely on the GFX PLL to generate the clock. Typically, the GFX PLL is
configured by the platform firmware.

Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 53 ++++++++++++++++-----------
 1 file changed, 32 insertions(+), 21 deletions(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index a03e81337d8f..5e88fa95dac2 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -1565,6 +1565,35 @@ static irqreturn_t irq_handler(int irq, void *data)
 	return ret;
 }
 
+static int etnaviv_gpu_clk_get(struct etnaviv_gpu *gpu)
+{
+	struct device *dev = gpu->dev;
+
+	gpu->clk_reg = devm_clk_get_optional(dev, "reg");
+	DBG("clk_reg: %p", gpu->clk_reg);
+	if (IS_ERR(gpu->clk_reg))
+		return PTR_ERR(gpu->clk_reg);
+
+	gpu->clk_bus = devm_clk_get_optional(dev, "bus");
+	DBG("clk_bus: %p", gpu->clk_bus);
+	if (IS_ERR(gpu->clk_bus))
+		return PTR_ERR(gpu->clk_bus);
+
+	gpu->clk_core = devm_clk_get(dev, "core");
+	DBG("clk_core: %p", gpu->clk_core);
+	if (IS_ERR(gpu->clk_core))
+		return PTR_ERR(gpu->clk_core);
+	gpu->base_rate_core = clk_get_rate(gpu->clk_core);
+
+	gpu->clk_shader = devm_clk_get_optional(dev, "shader");
+	DBG("clk_shader: %p", gpu->clk_shader);
+	if (IS_ERR(gpu->clk_shader))
+		return PTR_ERR(gpu->clk_shader);
+	gpu->base_rate_shader = clk_get_rate(gpu->clk_shader);
+
+	return 0;
+}
+
 static int etnaviv_gpu_clk_enable(struct etnaviv_gpu *gpu)
 {
 	int ret;
@@ -1863,27 +1892,9 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 		return err;
 
 	/* Get Clocks: */
-	gpu->clk_reg = devm_clk_get_optional(&pdev->dev, "reg");
-	DBG("clk_reg: %p", gpu->clk_reg);
-	if (IS_ERR(gpu->clk_reg))
-		return PTR_ERR(gpu->clk_reg);
-
-	gpu->clk_bus = devm_clk_get_optional(&pdev->dev, "bus");
-	DBG("clk_bus: %p", gpu->clk_bus);
-	if (IS_ERR(gpu->clk_bus))
-		return PTR_ERR(gpu->clk_bus);
-
-	gpu->clk_core = devm_clk_get(&pdev->dev, "core");
-	DBG("clk_core: %p", gpu->clk_core);
-	if (IS_ERR(gpu->clk_core))
-		return PTR_ERR(gpu->clk_core);
-	gpu->base_rate_core = clk_get_rate(gpu->clk_core);
-
-	gpu->clk_shader = devm_clk_get_optional(&pdev->dev, "shader");
-	DBG("clk_shader: %p", gpu->clk_shader);
-	if (IS_ERR(gpu->clk_shader))
-		return PTR_ERR(gpu->clk_shader);
-	gpu->base_rate_shader = clk_get_rate(gpu->clk_shader);
+	err = etnaviv_gpu_clk_get(gpu);
+	if (err)
+		return err;
 
 	/* TODO: figure out max mapped size */
 	dev_set_drvdata(dev, gpu);
-- 
2.25.1

[PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

Also rename the virtual master platform device as etnaviv_platform_device,
for better reflection that it is a platform device, not a DRM device.

Another benefit is that we no longer need to call of_node_put() for three
different cases, Instead, we only need to call it once.

Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_drv.c | 56 +++++++++++++++++++--------
 1 file changed, 39 insertions(+), 17 deletions(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 31a7f59ccb49..cec005035d0e 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -656,12 +656,44 @@ static struct platform_driver etnaviv_platform_driver = {
 	},
 };
 
-static struct platform_device *etnaviv_drm;
+static struct platform_device *etnaviv_platform_device;
 
-static int __init etnaviv_init(void)
+static int etnaviv_create_platform_device(const char *name,
+					  struct platform_device **ppdev)
 {
 	struct platform_device *pdev;
 	int ret;
+
+	pdev = platform_device_alloc(name, PLATFORM_DEVID_NONE);
+	if (!pdev)
+		return -ENOMEM;
+
+	ret = platform_device_add(pdev);
+	if (ret) {
+		platform_device_put(pdev);
+		return ret;
+	}
+
+	*ppdev = pdev;
+
+	return 0;
+}
+
+static void etnaviv_destroy_platform_device(struct platform_device **ppdev)
+{
+	struct platform_device *pdev = *ppdev;
+
+	if (!pdev)
+		return;
+
+	platform_device_unregister(pdev);
+
+	*ppdev = NULL;
+}
+
+static int __init etnaviv_init(void)
+{
+	int ret;
 	struct device_node *np;
 
 	etnaviv_validate_init();
@@ -681,23 +713,13 @@ static int __init etnaviv_init(void)
 	for_each_compatible_node(np, NULL, "vivante,gc") {
 		if (!of_device_is_available(np))
 			continue;
+		of_node_put(np);
 
-		pdev = platform_device_alloc("etnaviv", PLATFORM_DEVID_NONE);
-		if (!pdev) {
-			ret = -ENOMEM;
-			of_node_put(np);
-			goto unregister_platform_driver;
-		}
-
-		ret = platform_device_add(pdev);
-		if (ret) {
-			platform_device_put(pdev);
-			of_node_put(np);
+		ret = etnaviv_create_platform_device("etnaviv",
+						     &etnaviv_platform_device);
+		if (ret)
 			goto unregister_platform_driver;
-		}
 
-		etnaviv_drm = pdev;
-		of_node_put(np);
 		break;
 	}
 
@@ -713,7 +735,7 @@ module_init(etnaviv_init);
 
 static void __exit etnaviv_exit(void)
 {
-	platform_device_unregister(etnaviv_drm);
+	etnaviv_destroy_platform_device(&etnaviv_platform_device);
 	platform_driver_unregister(&etnaviv_platform_driver);
 	platform_driver_unregister(&etnaviv_gpu_driver);
 }
-- 
2.25.1

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Lucas Stach]

Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
> From: Sui Jingfeng <suijingfeng@loongson.cn>
> 
> Also rename the virtual master platform device as etnaviv_platform_device,
> for better reflection that it is a platform device, not a DRM device.
> 
> Another benefit is that we no longer need to call of_node_put() for three
> different cases, Instead, we only need to call it once.
> 
> Cc: Lucas Stach <l.stach@pengutronix.de>
> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> Cc: Philipp Zabel <p.zabel@pengutronix.de>
> Cc: Bjorn Helgaas <bhelgaas@google.com>
> Cc: Daniel Vetter <daniel@ffwll.ch>
> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> ---
>  drivers/gpu/drm/etnaviv/etnaviv_drv.c | 56 +++++++++++++++++++--------
>  1 file changed, 39 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> index 31a7f59ccb49..cec005035d0e 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> @@ -656,12 +656,44 @@ static struct platform_driver etnaviv_platform_driver = {
>  	},
>  };
>  
> -static struct platform_device *etnaviv_drm;
> +static struct platform_device *etnaviv_platform_device;
>  
> -static int __init etnaviv_init(void)
> +static int etnaviv_create_platform_device(const char *name,
> +					  struct platform_device **ppdev)

As the platform device is a global static variable, there is no need to
push it through the parameters of this function. Just use the global
variable directly in this function.

>  {
>  	struct platform_device *pdev;
>  	int ret;
> +
> +	pdev = platform_device_alloc(name, PLATFORM_DEVID_NONE);
> +	if (!pdev)
> +		return -ENOMEM;
> +
> +	ret = platform_device_add(pdev);
> +	if (ret) {
> +		platform_device_put(pdev);
> +		return ret;
> +	}
> +
> +	*ppdev = pdev;
> +
> +	return 0;
> +}
> +
> +static void etnaviv_destroy_platform_device(struct platform_device **ppdev)
> +{
> +	struct platform_device *pdev = *ppdev;

Same here, just use the global variable directly.

Regards,
Lucas

> +
> +	if (!pdev)
> +		return;
> +
> +	platform_device_unregister(pdev);
> +
> +	*ppdev = NULL;
> +}
> +
> +static int __init etnaviv_init(void)
> +{
> +	int ret;
>  	struct device_node *np;
>  
>  	etnaviv_validate_init();
> @@ -681,23 +713,13 @@ static int __init etnaviv_init(void)
>  	for_each_compatible_node(np, NULL, "vivante,gc") {
>  		if (!of_device_is_available(np))
>  			continue;
> +		of_node_put(np);
>  
> -		pdev = platform_device_alloc("etnaviv", PLATFORM_DEVID_NONE);
> -		if (!pdev) {
> -			ret = -ENOMEM;
> -			of_node_put(np);
> -			goto unregister_platform_driver;
> -		}
> -
> -		ret = platform_device_add(pdev);
> -		if (ret) {
> -			platform_device_put(pdev);
> -			of_node_put(np);
> +		ret = etnaviv_create_platform_device("etnaviv",
> +						     &etnaviv_platform_device);
> +		if (ret)
>  			goto unregister_platform_driver;
> -		}
>  
> -		etnaviv_drm = pdev;
> -		of_node_put(np);
>  		break;
>  	}
>  
> @@ -713,7 +735,7 @@ module_init(etnaviv_init);
>  
>  static void __exit etnaviv_exit(void)
>  {
> -	platform_device_unregister(etnaviv_drm);
> +	etnaviv_destroy_platform_device(&etnaviv_platform_device);
>  	platform_driver_unregister(&etnaviv_platform_driver);
>  	platform_driver_unregister(&etnaviv_gpu_driver);
>  }

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Sui Jingfeng]

Hi

On 2023/6/21 17:15, Lucas Stach wrote:
> Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>
>> Also rename the virtual master platform device as etnaviv_platform_device,
>> for better reflection that it is a platform device, not a DRM device.
>>
>> Another benefit is that we no longer need to call of_node_put() for three
>> different cases, Instead, we only need to call it once.
>>
>> Cc: Lucas Stach <l.stach@pengutronix.de>
>> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
>> Cc: Philipp Zabel <p.zabel@pengutronix.de>
>> Cc: Bjorn Helgaas <bhelgaas@google.com>
>> Cc: Daniel Vetter <daniel@ffwll.ch>
>> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
>> ---
>>   drivers/gpu/drm/etnaviv/etnaviv_drv.c | 56 +++++++++++++++++++--------
>>   1 file changed, 39 insertions(+), 17 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> index 31a7f59ccb49..cec005035d0e 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> @@ -656,12 +656,44 @@ static struct platform_driver etnaviv_platform_driver = {
>>   	},
>>   };
>>   
>> -static struct platform_device *etnaviv_drm;
>> +static struct platform_device *etnaviv_platform_device;
>>   
>> -static int __init etnaviv_init(void)
>> +static int etnaviv_create_platform_device(const char *name,
>> +					  struct platform_device **ppdev)
> As the platform device is a global static variable, there is no need to
> push it through the parameters of this function. Just use the global
> variable directly in this function.

A function reference a global static variable is *NOT* a *pure* fucntion,

it degenerate as a procedure,


The function is perfect in the sense that it does not reference any 
global variable.


etnaviv_create_platform_device() is NOT intended to used by one function,

a specific purpose only, but when create this function, I want to create other

platform device with this function.

Say, You want to create a dummy platform device, targeting to bind to the real master

(the single GPU core) . To verify the idea that we choose the first 3D gpu core as master,

other 2D or VG gpu core is not as important as the 3D one.

The should bind to the 3D GPU core (master).


While back to the question you ask, I want etnaviv_create_platform_device() to be generic,

can be used by multiple place for multiple purpose.

I have successfully copy this to a another drm driver by simply renaming.

The body of the function itself does not need to change.

>>   {
>>   	struct platform_device *pdev;
>>   	int ret;
>> +
>> +	pdev = platform_device_alloc(name, PLATFORM_DEVID_NONE);
>> +	if (!pdev)
>> +		return -ENOMEM;
>> +
>> +	ret = platform_device_add(pdev);
>> +	if (ret) {
>> +		platform_device_put(pdev);
>> +		return ret;
>> +	}
>> +
>> +	*ppdev = pdev;
>> +
>> +	return 0;
>> +}
>> +
>> +static void etnaviv_destroy_platform_device(struct platform_device **ppdev)
>> +{
>> +	struct platform_device *pdev = *ppdev;
> Same here, just use the global variable directly.
>
> Regards,
> Lucas
>
>> +
>> +	if (!pdev)
>> +		return;
>> +
>> +	platform_device_unregister(pdev);
>> +
>> +	*ppdev = NULL;
>> +}
>> +
>> +static int __init etnaviv_init(void)
>> +{
>> +	int ret;
>>   	struct device_node *np;
>>   
>>   	etnaviv_validate_init();
>> @@ -681,23 +713,13 @@ static int __init etnaviv_init(void)
>>   	for_each_compatible_node(np, NULL, "vivante,gc") {
>>   		if (!of_device_is_available(np))
>>   			continue;
>> +		of_node_put(np);
>>   
>> -		pdev = platform_device_alloc("etnaviv", PLATFORM_DEVID_NONE);
>> -		if (!pdev) {
>> -			ret = -ENOMEM;
>> -			of_node_put(np);
>> -			goto unregister_platform_driver;
>> -		}
>> -
>> -		ret = platform_device_add(pdev);
>> -		if (ret) {
>> -			platform_device_put(pdev);
>> -			of_node_put(np);
>> +		ret = etnaviv_create_platform_device("etnaviv",
>> +						     &etnaviv_platform_device);
>> +		if (ret)
>>   			goto unregister_platform_driver;
>> -		}
>>   
>> -		etnaviv_drm = pdev;
>> -		of_node_put(np);
>>   		break;
>>   	}
>>   
>> @@ -713,7 +735,7 @@ module_init(etnaviv_init);
>>   
>>   static void __exit etnaviv_exit(void)
>>   {
>> -	platform_device_unregister(etnaviv_drm);
>> +	etnaviv_destroy_platform_device(&etnaviv_platform_device);
>>   	platform_driver_unregister(&etnaviv_platform_driver);
>>   	platform_driver_unregister(&etnaviv_gpu_driver);
>>   }

-- 
Jingfeng

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Lucas Stach]

Am Mittwoch, dem 21.06.2023 um 17:49 +0800 schrieb Sui Jingfeng:
> Hi
> 
> On 2023/6/21 17:15, Lucas Stach wrote:
> > Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
> > > From: Sui Jingfeng <suijingfeng@loongson.cn>
> > > 
> > > Also rename the virtual master platform device as etnaviv_platform_device,
> > > for better reflection that it is a platform device, not a DRM device.
> > > 
> > > Another benefit is that we no longer need to call of_node_put() for three
> > > different cases, Instead, we only need to call it once.
> > > 
> > > Cc: Lucas Stach <l.stach@pengutronix.de>
> > > Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> > > Cc: Philipp Zabel <p.zabel@pengutronix.de>
> > > Cc: Bjorn Helgaas <bhelgaas@google.com>
> > > Cc: Daniel Vetter <daniel@ffwll.ch>
> > > Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> > > ---
> > >   drivers/gpu/drm/etnaviv/etnaviv_drv.c | 56 +++++++++++++++++++--------
> > >   1 file changed, 39 insertions(+), 17 deletions(-)
> > > 
> > > diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> > > index 31a7f59ccb49..cec005035d0e 100644
> > > --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> > > +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> > > @@ -656,12 +656,44 @@ static struct platform_driver etnaviv_platform_driver = {
> > >   	},
> > >   };
> > >   
> > > -static struct platform_device *etnaviv_drm;
> > > +static struct platform_device *etnaviv_platform_device;
> > >   
> > > -static int __init etnaviv_init(void)
> > > +static int etnaviv_create_platform_device(const char *name,
> > > +					  struct platform_device **ppdev)
> > As the platform device is a global static variable, there is no need to
> > push it through the parameters of this function. Just use the global
> > variable directly in this function.
> 
> A function reference a global static variable is *NOT* a *pure* fucntion,
> 
That's right, but all you do with those indirections through the
parameter list is move which of the functions is non-pure, in your case
it's etnaviv_init/etnaviv_exit, with the indirection dropped it's
etnaviv_create_platform_device/etnaviv_destroy_platform_device.

> it degenerate as a procedure,
> 
> 
> The function is perfect in the sense that it does not reference any 
> global variable.
> 
> 
> etnaviv_create_platform_device() is NOT intended to used by one function,
> 
> a specific purpose only, but when create this function, I want to create other
> 
> platform device with this function.
> 
> Say, You want to create a dummy platform device, targeting to bind to the real master
> 
> (the single GPU core) . To verify the idea that we choose the first 3D gpu core as master,
> 
> other 2D or VG gpu core is not as important as the 3D one.
> 
> The should bind to the 3D GPU core (master).
> 
Sorry, I'm not following what you are trying to tell me here. Could you
please rephrase?

> 
> While back to the question you ask, I want etnaviv_create_platform_device() to be generic,
> 
> can be used by multiple place for multiple purpose.
> 
> I have successfully copy this to a another drm driver by simply renaming.
> 
> The body of the function itself does not need to change.

But it isn't shared, in this compilation unit this function is specific
to the etnaviv driver and I don't see why we shouldn't have etnaviv
specifics in there if it makes the code of this driver easier to
follow.

Regards,
Lucas

> 
> > >   {
> > >   	struct platform_device *pdev;
> > >   	int ret;
> > > +
> > > +	pdev = platform_device_alloc(name, PLATFORM_DEVID_NONE);
> > > +	if (!pdev)
> > > +		return -ENOMEM;
> > > +
> > > +	ret = platform_device_add(pdev);
> > > +	if (ret) {
> > > +		platform_device_put(pdev);
> > > +		return ret;
> > > +	}
> > > +
> > > +	*ppdev = pdev;
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static void etnaviv_destroy_platform_device(struct platform_device **ppdev)
> > > +{
> > > +	struct platform_device *pdev = *ppdev;
> > Same here, just use the global variable directly.
> > 
> > Regards,
> > Lucas
> > 
> > > +
> > > +	if (!pdev)
> > > +		return;
> > > +
> > > +	platform_device_unregister(pdev);
> > > +
> > > +	*ppdev = NULL;
> > > +}
> > > +
> > > +static int __init etnaviv_init(void)
> > > +{
> > > +	int ret;
> > >   	struct device_node *np;
> > >   
> > >   	etnaviv_validate_init();
> > > @@ -681,23 +713,13 @@ static int __init etnaviv_init(void)
> > >   	for_each_compatible_node(np, NULL, "vivante,gc") {
> > >   		if (!of_device_is_available(np))
> > >   			continue;
> > > +		of_node_put(np);
> > >   
> > > -		pdev = platform_device_alloc("etnaviv", PLATFORM_DEVID_NONE);
> > > -		if (!pdev) {
> > > -			ret = -ENOMEM;
> > > -			of_node_put(np);
> > > -			goto unregister_platform_driver;
> > > -		}
> > > -
> > > -		ret = platform_device_add(pdev);
> > > -		if (ret) {
> > > -			platform_device_put(pdev);
> > > -			of_node_put(np);
> > > +		ret = etnaviv_create_platform_device("etnaviv",
> > > +						     &etnaviv_platform_device);
> > > +		if (ret)
> > >   			goto unregister_platform_driver;
> > > -		}
> > >   
> > > -		etnaviv_drm = pdev;
> > > -		of_node_put(np);
> > >   		break;
> > >   	}
> > >   
> > > @@ -713,7 +735,7 @@ module_init(etnaviv_init);
> > >   
> > >   static void __exit etnaviv_exit(void)
> > >   {
> > > -	platform_device_unregister(etnaviv_drm);
> > > +	etnaviv_destroy_platform_device(&etnaviv_platform_device);
> > >   	platform_driver_unregister(&etnaviv_platform_driver);
> > >   	platform_driver_unregister(&etnaviv_gpu_driver);
> > >   }
> 

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Sui Jingfeng]

On 2023/6/21 18:23, Lucas Stach wrote:
>> While back to the question you ask, I want etnaviv_create_platform_device() to be generic,
>>
>> can be used by multiple place for multiple purpose.
>>
>> I have successfully copy this to a another drm driver by simply renaming.
>>
>> The body of the function itself does not need to change.
> But it isn't shared,

This can be shared for drm/etnaviv in the future,

currently, we just need an opportunity to use this function.

I want to create a dummy platform device,

let this dummy platform be bound to the single PCI GPU master.


etnaviv_create_platform_device("dummy", &dummy_device);


1) To verify the component code path on PCI case.

2) Possibly for create a device for some other tiny hardware logic
come with the platform

3) Revival component_compare_dev_name() function.

> in this compilation unit this function is specific
> to the etnaviv driver and I don't see why we shouldn't have etnaviv
> specifics in there if it makes the code of this driver easier to
> follow.

-- 
Jingfeng

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Lucas Stach]

Am Mittwoch, dem 21.06.2023 um 21:31 +0800 schrieb Sui Jingfeng:
> On 2023/6/21 18:23, Lucas Stach wrote:
> > > While back to the question you ask, I want etnaviv_create_platform_device() to be generic,
> > > 
> > > can be used by multiple place for multiple purpose.
> > > 
> > > I have successfully copy this to a another drm driver by simply renaming.
> > > 
> > > The body of the function itself does not need to change.
> > But it isn't shared,
> 
> This can be shared for drm/etnaviv in the future,
> 
> currently, we just need an opportunity to use this function.
> 
I'm not convinced, yet.

> I want to create a dummy platform device,
> 
> let this dummy platform be bound to the single PCI GPU master.
> 
> 
> etnaviv_create_platform_device("dummy", &dummy_device);
> 
> 
> 1) To verify the component code path on PCI case.
> 
My favorite option would be to just always use the component path even
when the GPU is on a PCI device to keep both paths mostly aligned. One
could easily image both a 3D and a 2D core being made available though
the same PCI device.

> 2) Possibly for create a device for some other tiny hardware logic
> come with the platform
> 
Do you have something in mind here? Until now I assumed that only the
GPU core is behind the PCI abstraction. Is there something else sharing
the MMIO space?

Regards,
Lucas

> 3) Revival component_compare_dev_name() function.
> 
> > in this compilation unit this function is specific
> > to the etnaviv driver and I don't see why we shouldn't have etnaviv
> > specifics in there if it makes the code of this driver easier to
> > follow.
> 

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Sui Jingfeng]

Hi,

On 2023/6/21 22:00, Lucas Stach wrote:
> Am Mittwoch, dem 21.06.2023 um 21:31 +0800 schrieb Sui Jingfeng:
>> On 2023/6/21 18:23, Lucas Stach wrote:
>>>> While back to the question you ask, I want etnaviv_create_platform_device() to be generic,
>>>>
>>>> can be used by multiple place for multiple purpose.
>>>>
>>>> I have successfully copy this to a another drm driver by simply renaming.
>>>>
>>>> The body of the function itself does not need to change.
>>> But it isn't shared,
>> This can be shared for drm/etnaviv in the future,
>>
>> currently, we just need an opportunity to use this function.
>>
> I'm not convinced, yet.
>
>> I want to create a dummy platform device,
>>
>> let this dummy platform be bound to the single PCI GPU master.
>>
>>
>> etnaviv_create_platform_device("dummy", &dummy_device);
>>
>>
>> 1) To verify the component code path on PCI case.
>>
> My favorite option would be to just always use the component path even
> when the GPU is on a PCI device to keep both paths mostly aligned. One
> could easily image both a 3D and a 2D core being made available though
> the same PCI device.

Component is for something that is possible not available. (or something 
is optional)

Yes it provided flexibly, but don't forget, it rely on the DT.


But for the PCIe device, it always the case that all of the hardware is 
available at the same time

when the device driver(kernel module) is loaded.


>> 2) Possibly for create a device for some other tiny hardware logic
>> come with the platform
>>
> Do you have something in mind here? Until now I assumed that only the
> GPU core is behind the PCI abstraction. Is there something else sharing
> the MMIO space?

A display controller, HDMI phy, vga encoder etc


I have a discrete PCIe GPU card from another vendor,

It integrated display controller and vivante GPU and unknown VPUs.

All of the  hardware block mentioned above sharing the MMIO space.

There are available on the same time when you mount this discrete PCIe 
GPU card on the mother board

>
> Regards,
> Lucas
>
>> 3) Revival component_compare_dev_name() function.
>>
>>> in this compilation unit this function is specific
>>> to the etnaviv driver and I don't see why we shouldn't have etnaviv
>>> specifics in there if it makes the code of this driver easier to
>>> follow.

-- 
Jingfeng

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Sui Jingfeng]

On 2023/6/21 22:35, Sui Jingfeng wrote:
> Yes it provided flexibly, but don't forget, it rely on the DT. 

Yes it provided flexibility, it rely on the DT provide such flexibility.

-- 
Jingfeng

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Lucas Stach]

Am Mittwoch, dem 21.06.2023 um 22:35 +0800 schrieb Sui Jingfeng:
> Hi,
> 
> On 2023/6/21 22:00, Lucas Stach wrote:
> > Am Mittwoch, dem 21.06.2023 um 21:31 +0800 schrieb Sui Jingfeng:
> > > On 2023/6/21 18:23, Lucas Stach wrote:
> > > > > While back to the question you ask, I want etnaviv_create_platform_device() to be generic,
> > > > > 
> > > > > can be used by multiple place for multiple purpose.
> > > > > 
> > > > > I have successfully copy this to a another drm driver by simply renaming.
> > > > > 
> > > > > The body of the function itself does not need to change.
> > > > But it isn't shared,
> > > This can be shared for drm/etnaviv in the future,
> > > 
> > > currently, we just need an opportunity to use this function.
> > > 
> > I'm not convinced, yet.
> > 
> > > I want to create a dummy platform device,
> > > 
> > > let this dummy platform be bound to the single PCI GPU master.
> > > 
> > > 
> > > etnaviv_create_platform_device("dummy", &dummy_device);
> > > 
> > > 
> > > 1) To verify the component code path on PCI case.
> > > 
> > My favorite option would be to just always use the component path even
> > when the GPU is on a PCI device to keep both paths mostly aligned. One
> > could easily image both a 3D and a 2D core being made available though
> > the same PCI device.
> 
> Component is for something that is possible not available. (or something 
> is optional)
> 
> Yes it provided flexibly, but don't forget, it rely on the DT.

The component framework itself doesn't rely on DT in any way. By
providing a appropriate match function you can make it work with any
kind of device. In fact etnaviv supports platform devices instantiated
via board code today. They don't need to come from DT.

If we could make the PCI stuff work the same way, that would be my
preferred option.

> 
> 
> But for the PCIe device, it always the case that all of the hardware is 
> available at the same time
> 
> when the device driver(kernel module) is loaded.
That isn't the issue solved by the component framework. On the existing
SoCs all the hardware is available when the driver is probed. The
component framework just makes sure that we only expose the DRM device
after all GPU cores that should be managed by a single DRM device
instance are probed.

One could easily image a PCI device that containing a 2D and a 3D
Vivante GPU that should be made available through a single DRM device.
In that case you'll also need to use the component framework.

> 
> 
> > > 2) Possibly for create a device for some other tiny hardware logic
> > > come with the platform
> > > 
> > Do you have something in mind here? Until now I assumed that only the
> > GPU core is behind the PCI abstraction. Is there something else sharing
> > the MMIO space?
> 
> A display controller, HDMI phy, vga encoder etc
> 
> 
> I have a discrete PCIe GPU card from another vendor,
> 
> It integrated display controller and vivante GPU and unknown VPUs.
> 
> All of the  hardware block mentioned above sharing the MMIO space.
> 
> There are available on the same time when you mount this discrete PCIe 
> GPU card on the mother board
> 
But they surely should not all be made available through the etnaviv
driver. Etnaviv deals with the Vivante GPUs. If you have a PCI device
with multiple IP cores behind the shared MMIO space you should have a
PCI driver instantiating platform devices so the respective drivers for
those IP cores can bind to the platform device. Etnaviv is not that
driver.

Regards,
Lucas

> > 
> > Regards,
> > Lucas
> > 
> > > 3) Revival component_compare_dev_name() function.
> > > 
> > > > in this compilation unit this function is specific
> > > > to the etnaviv driver and I don't see why we shouldn't have etnaviv
> > > > specifics in there if it makes the code of this driver easier to
> > > > follow.
> 

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Sui Jingfeng]

Hi

On 2023/6/21 23:20, Lucas Stach wrote:
> Am Mittwoch, dem 21.06.2023 um 22:35 +0800 schrieb Sui Jingfeng:
>> Hi,
>>
>> On 2023/6/21 22:00, Lucas Stach wrote:
>>> Am Mittwoch, dem 21.06.2023 um 21:31 +0800 schrieb Sui Jingfeng:
>>>> On 2023/6/21 18:23, Lucas Stach wrote:
>>>>>> While back to the question you ask, I want etnaviv_create_platform_device() to be generic,
>>>>>>
>>>>>> can be used by multiple place for multiple purpose.
>>>>>>
>>>>>> I have successfully copy this to a another drm driver by simply renaming.
>>>>>>
>>>>>> The body of the function itself does not need to change.
>>>>> But it isn't shared,
>>>> This can be shared for drm/etnaviv in the future,
>>>>
>>>> currently, we just need an opportunity to use this function.
>>>>
>>> I'm not convinced, yet.
>>>
>>>> I want to create a dummy platform device,
>>>>
>>>> let this dummy platform be bound to the single PCI GPU master.
>>>>
>>>>
>>>> etnaviv_create_platform_device("dummy", &dummy_device);
>>>>
>>>>
>>>> 1) To verify the component code path on PCI case.
>>>>
>>> My favorite option would be to just always use the component path even
>>> when the GPU is on a PCI device to keep both paths mostly aligned. One
>>> could easily image both a 3D and a 2D core being made available though
>>> the same PCI device.
>> Component is for something that is possible not available. (or something
>> is optional)
>>
>> Yes it provided flexibly, but don't forget, it rely on the DT.
> The component framework itself doesn't rely on DT in any way.

Yes I know that, for example the HDMI audio stuff.

But *your implement* do rely on the DT, this is the point

> By
> providing a appropriate match function you can make it work with any
> kind of device.
Yes, you are right.
>   In fact etnaviv supports platform devices instantiated
> via board code today.
Nice,
>   They don't need to come from DT.
What about the various clock, sir?
> If we could make the PCI stuff work the same way, that would be my
> preferred option.
>
>>
>> But for the PCIe device, it always the case that all of the hardware is
>> available at the same time
>>
>> when the device driver(kernel module) is loaded.
> That isn't the issue solved by the component framework. On the existing
> SoCs all the hardware is available when the driver is probed. The
> component framework just makes sure that we only expose the DRM device
> after all GPU cores that should be managed by a single DRM device
> instance are probed.
>
> One could easily image a PCI device that containing a 2D and a 3D
> Vivante GPU that should be made available through a single DRM device.
> In that case you'll also need to use the component framework.
>
>>
>>>> 2) Possibly for create a device for some other tiny hardware logic
>>>> come with the platform
>>>>
>>> Do you have something in mind here? Until now I assumed that only the
>>> GPU core is behind the PCI abstraction. Is there something else sharing
>>> the MMIO space?
>> A display controller, HDMI phy, vga encoder etc
>>
>>
>> I have a discrete PCIe GPU card from another vendor,
>>
>> It integrated display controller and vivante GPU and unknown VPUs.
>>
>> All of the  hardware block mentioned above sharing the MMIO space.
>>
>> There are available on the same time when you mount this discrete PCIe
>> GPU card on the mother board
>>
> But they surely should not all be made available through the etnaviv
> driver. Etnaviv deals with the Vivante GPUs. If you have a PCI device
> with multiple IP cores behind the shared MMIO space you should have a
> PCI driver instantiating platform devices so the respective drivers for
> those IP cores can bind to the platform device.

I have only one PCI device, let start from the simple case, OK?

I admire your fantastic idea.

let deal with it another patch in the future if such hardware emerged.

Accept the current implement, please ?

>   Etnaviv is not that
> driver.

Yeah, but I notice that there is chipFeatures_DC defined in common.xml.h

I don't know how does this going to used, if a hardware marked it as true.

> Regards,
> Lucas
>
>>> Regards,
>>> Lucas
>>>
>>>> 3) Revival component_compare_dev_name() function.
>>>>
>>>>> in this compilation unit this function is specific
>>>>> to the etnaviv driver and I don't see why we shouldn't have etnaviv
>>>>> specifics in there if it makes the code of this driver easier to
>>>>> follow.

-- 
Jingfeng

Re: [PATCH v10 03/11] drm/etnaviv: Add dedicated functions to create and destroy platform device

[Sui Jingfeng]

Hi,

On 2023/6/21 18:23, Lucas Stach wrote:
> That's right, but all you do with those indirections through the
> parameter list is move which of the functions is non-pure, in your case
> it's etnaviv_init/etnaviv_exit,

But there is a difference,  etnaviv_init() and etnaviv_exit() is 
impossible to be shared

there are only get called once when the module is loaded.

They can never be reused anymore, except here.

And etnaviv_init() and etnaviv_exit() don't have a choice.


But for etnaviv_create_platform_device() function,

there is a possibility to be reused in the future.


> with the indirection dropped it's
> etnaviv_create_platform_device/etnaviv_destroy_platform_device.

-- 
Jingfeng

[PATCH v10 04/11] drm/etnaviv: Add helpers for private data construction and destruction

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

There are numerous members in the struct etnaviv_drm_private, which are
shared by all GPU core. This patch introduces two dedicated functions for
the construction and destruction of the instances of this structure.
The goal is to keep its members from leaking to the outside. The code
needed for error handling can also be simplified.

Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_drv.c | 73 +++++++++++++++++----------
 drivers/gpu/drm/etnaviv/etnaviv_drv.h |  1 +
 2 files changed, 47 insertions(+), 27 deletions(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index cec005035d0e..6a048be02857 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -24,9 +24,47 @@
 #include "etnaviv_perfmon.h"
 
 /*
- * DRM operations:
+ * etnaviv private data construction and destructions:
  */
+static struct etnaviv_drm_private *
+etnaviv_alloc_private(struct device *dev, struct drm_device *drm)
+{
+	struct etnaviv_drm_private *priv;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return ERR_PTR(-ENOMEM);
+
+	priv->drm = drm;
+
+	xa_init_flags(&priv->active_contexts, XA_FLAGS_ALLOC);
+
+	mutex_init(&priv->gem_lock);
+	INIT_LIST_HEAD(&priv->gem_list);
+	priv->num_gpus = 0;
+	priv->shm_gfp_mask = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
 
+	priv->cmdbuf_suballoc = etnaviv_cmdbuf_suballoc_new(dev);
+	if (IS_ERR(priv->cmdbuf_suballoc)) {
+		kfree(priv);
+		dev_err(dev, "Failed to create cmdbuf suballocator\n");
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return priv;
+}
+
+static void etnaviv_free_private(struct etnaviv_drm_private *priv)
+{
+	if (!priv)
+		return;
+
+	etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc);
+
+	xa_destroy(&priv->active_contexts);
+
+	kfree(priv);
+}
 
 static void load_gpu(struct drm_device *dev)
 {
@@ -511,35 +549,21 @@ static int etnaviv_bind(struct device *dev)
 	if (IS_ERR(drm))
 		return PTR_ERR(drm);
 
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv) {
-		dev_err(dev, "failed to allocate private data\n");
-		ret = -ENOMEM;
+	priv = etnaviv_alloc_private(dev, drm);
+	if (IS_ERR(priv)) {
+		ret = PTR_ERR(priv);
 		goto out_put;
 	}
+
 	drm->dev_private = priv;
 
 	dma_set_max_seg_size(dev, SZ_2G);
 
-	xa_init_flags(&priv->active_contexts, XA_FLAGS_ALLOC);
-
-	mutex_init(&priv->gem_lock);
-	INIT_LIST_HEAD(&priv->gem_list);
-	priv->num_gpus = 0;
-	priv->shm_gfp_mask = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
-
-	priv->cmdbuf_suballoc = etnaviv_cmdbuf_suballoc_new(drm->dev);
-	if (IS_ERR(priv->cmdbuf_suballoc)) {
-		dev_err(drm->dev, "Failed to create cmdbuf suballocator\n");
-		ret = PTR_ERR(priv->cmdbuf_suballoc);
-		goto out_free_priv;
-	}
-
 	dev_set_drvdata(dev, drm);
 
 	ret = component_bind_all(dev, drm);
 	if (ret < 0)
-		goto out_destroy_suballoc;
+		goto out_free_priv;
 
 	load_gpu(drm);
 
@@ -551,10 +575,8 @@ static int etnaviv_bind(struct device *dev)
 
 out_unbind:
 	component_unbind_all(dev, drm);
-out_destroy_suballoc:
-	etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc);
 out_free_priv:
-	kfree(priv);
+	etnaviv_free_private(priv);
 out_put:
 	drm_dev_put(drm);
 
@@ -570,12 +592,9 @@ static void etnaviv_unbind(struct device *dev)
 
 	component_unbind_all(dev, drm);
 
-	etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc);
-
-	xa_destroy(&priv->active_contexts);
+	etnaviv_free_private(priv);
 
 	drm->dev_private = NULL;
-	kfree(priv);
 
 	drm_dev_put(drm);
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index b3eb1662e90c..e58f82e698de 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -35,6 +35,7 @@ struct etnaviv_file_private {
 };
 
 struct etnaviv_drm_private {
+	struct drm_device *drm;
 	int num_gpus;
 	struct etnaviv_gpu *gpu[ETNA_MAX_PIPES];
 	gfp_t shm_gfp_mask;
-- 
2.25.1

Re: [PATCH v10 04/11] drm/etnaviv: Add helpers for private data construction and destruction

[Lucas Stach]

Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
> From: Sui Jingfeng <suijingfeng@loongson.cn>
> 
> There are numerous members in the struct etnaviv_drm_private, which are
> shared by all GPU core. This patch introduces two dedicated functions for
> the construction and destruction of the instances of this structure.
> The goal is to keep its members from leaking to the outside. The code
> needed for error handling can also be simplified.
> 
> Cc: Lucas Stach <l.stach@pengutronix.de>
> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> Cc: Philipp Zabel <p.zabel@pengutronix.de>
> Cc: Bjorn Helgaas <bhelgaas@google.com>
> Cc: Daniel Vetter <daniel@ffwll.ch>
> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> ---
>  drivers/gpu/drm/etnaviv/etnaviv_drv.c | 73 +++++++++++++++++----------
>  drivers/gpu/drm/etnaviv/etnaviv_drv.h |  1 +
>  2 files changed, 47 insertions(+), 27 deletions(-)
> 
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> index cec005035d0e..6a048be02857 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> @@ -24,9 +24,47 @@
>  #include "etnaviv_perfmon.h"
>  
>  /*
> - * DRM operations:
> + * etnaviv private data construction and destructions:
>   */
> +static struct etnaviv_drm_private *
> +etnaviv_alloc_private(struct device *dev, struct drm_device *drm)
> +{
> +	struct etnaviv_drm_private *priv;
> +
> +	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
> +	if (!priv)
> +		return ERR_PTR(-ENOMEM);
> +
> +	priv->drm = drm;

That's an unrelated change that you rely on in later patches. If this
is needed at all it needs to be in a separate patch with a explanation
on why it is needed.

Regards,
Lucas

> +
> +	xa_init_flags(&priv->active_contexts, XA_FLAGS_ALLOC);
> +
> +	mutex_init(&priv->gem_lock);
> +	INIT_LIST_HEAD(&priv->gem_list);
> +	priv->num_gpus = 0;
> +	priv->shm_gfp_mask = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
>  
> +	priv->cmdbuf_suballoc = etnaviv_cmdbuf_suballoc_new(dev);
> +	if (IS_ERR(priv->cmdbuf_suballoc)) {
> +		kfree(priv);
> +		dev_err(dev, "Failed to create cmdbuf suballocator\n");
> +		return ERR_PTR(-ENOMEM);
> +	}
> +
> +	return priv;
> +}
> +
> +static void etnaviv_free_private(struct etnaviv_drm_private *priv)
> +{
> +	if (!priv)
> +		return;
> +
> +	etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc);
> +
> +	xa_destroy(&priv->active_contexts);
> +
> +	kfree(priv);
> +}
>  
>  static void load_gpu(struct drm_device *dev)
>  {
> @@ -511,35 +549,21 @@ static int etnaviv_bind(struct device *dev)
>  	if (IS_ERR(drm))
>  		return PTR_ERR(drm);
>  
> -	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
> -	if (!priv) {
> -		dev_err(dev, "failed to allocate private data\n");
> -		ret = -ENOMEM;
> +	priv = etnaviv_alloc_private(dev, drm);
> +	if (IS_ERR(priv)) {
> +		ret = PTR_ERR(priv);
>  		goto out_put;
>  	}
> +
>  	drm->dev_private = priv;
>  
>  	dma_set_max_seg_size(dev, SZ_2G);
>  
> -	xa_init_flags(&priv->active_contexts, XA_FLAGS_ALLOC);
> -
> -	mutex_init(&priv->gem_lock);
> -	INIT_LIST_HEAD(&priv->gem_list);
> -	priv->num_gpus = 0;
> -	priv->shm_gfp_mask = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
> -
> -	priv->cmdbuf_suballoc = etnaviv_cmdbuf_suballoc_new(drm->dev);
> -	if (IS_ERR(priv->cmdbuf_suballoc)) {
> -		dev_err(drm->dev, "Failed to create cmdbuf suballocator\n");
> -		ret = PTR_ERR(priv->cmdbuf_suballoc);
> -		goto out_free_priv;
> -	}
> -
>  	dev_set_drvdata(dev, drm);
>  
>  	ret = component_bind_all(dev, drm);
>  	if (ret < 0)
> -		goto out_destroy_suballoc;
> +		goto out_free_priv;
>  
>  	load_gpu(drm);
>  
> @@ -551,10 +575,8 @@ static int etnaviv_bind(struct device *dev)
>  
>  out_unbind:
>  	component_unbind_all(dev, drm);
> -out_destroy_suballoc:
> -	etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc);
>  out_free_priv:
> -	kfree(priv);
> +	etnaviv_free_private(priv);
>  out_put:
>  	drm_dev_put(drm);
>  
> @@ -570,12 +592,9 @@ static void etnaviv_unbind(struct device *dev)
>  
>  	component_unbind_all(dev, drm);
>  
> -	etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc);
> -
> -	xa_destroy(&priv->active_contexts);
> +	etnaviv_free_private(priv);
>  
>  	drm->dev_private = NULL;
> -	kfree(priv);
>  
>  	drm_dev_put(drm);
>  }
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> index b3eb1662e90c..e58f82e698de 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> @@ -35,6 +35,7 @@ struct etnaviv_file_private {
>  };
>  
>  struct etnaviv_drm_private {
> +	struct drm_device *drm;
>  	int num_gpus;
>  	struct etnaviv_gpu *gpu[ETNA_MAX_PIPES];
>  	gfp_t shm_gfp_mask;

Re: [PATCH v10 04/11] drm/etnaviv: Add helpers for private data construction and destruction

[Sui Jingfeng]

Hi,

On 2023/6/21 17:22, Lucas Stach wrote:
> Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>
>> There are numerous members in the struct etnaviv_drm_private, which are
>> shared by all GPU core. This patch introduces two dedicated functions for
>> the construction and destruction of the instances of this structure.
>> The goal is to keep its members from leaking to the outside. The code
>> needed for error handling can also be simplified.
>>
>> Cc: Lucas Stach <l.stach@pengutronix.de>
>> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
>> Cc: Philipp Zabel <p.zabel@pengutronix.de>
>> Cc: Bjorn Helgaas <bhelgaas@google.com>
>> Cc: Daniel Vetter <daniel@ffwll.ch>
>> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
>> ---
>>   drivers/gpu/drm/etnaviv/etnaviv_drv.c | 73 +++++++++++++++++----------
>>   drivers/gpu/drm/etnaviv/etnaviv_drv.h |  1 +
>>   2 files changed, 47 insertions(+), 27 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> index cec005035d0e..6a048be02857 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> @@ -24,9 +24,47 @@
>>   #include "etnaviv_perfmon.h"
>>   
>>   /*
>> - * DRM operations:
>> + * etnaviv private data construction and destructions:
>>    */
>> +static struct etnaviv_drm_private *
>> +etnaviv_alloc_private(struct device *dev, struct drm_device *drm)
>> +{
>> +	struct etnaviv_drm_private *priv;
>> +
>> +	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
>> +	if (!priv)
>> +		return ERR_PTR(-ENOMEM);
>> +
>> +	priv->drm = drm;
> That's an unrelated change that you rely on in later patches. If this
> is needed at all it needs to be in a separate patch with a explanation
> on why it is needed.

It helps the etnaviv_drm_unbind() function fetch the pointer to the 
struct drm_device,

etnaviv_drm_unbind() became a pure function which don't need to 
reference external variables.


The real rationale is that when we made the single 3D GPU core the be 
the master device,

We no longer create the virtual master. We lost a place to store the 
pointer to

struct drm_device.

We have only one struct device across the whole driver.

There only one dev->driver_data instance in the whole driver,

The etnaviv_gpu_driver_create() function will call dev_set_drvdata(dev, gpu)

function to store the pointer to the instance(struct etnaviv_gpu).

The seat is taken, there is no where to store the pointer to struct 
drm_device.


As the drm is intended to be shared by all GPU core,

so, in the end, even for multiple GPU cores case,

this is still good thing, because the driver could fetch the pointer to 
the struct drm_device

via 'struct etnaviv_drm_private *', they are something in common, that 
is they are both

intended to be shared by the whole program.

> Regards,
> Lucas
>
>> +
>> +	xa_init_flags(&priv->active_contexts, XA_FLAGS_ALLOC);
>> +
>> +	mutex_init(&priv->gem_lock);
>> +	INIT_LIST_HEAD(&priv->gem_list);
>> +	priv->num_gpus = 0;
>> +	priv->shm_gfp_mask = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
>>   
>> +	priv->cmdbuf_suballoc = etnaviv_cmdbuf_suballoc_new(dev);
>> +	if (IS_ERR(priv->cmdbuf_suballoc)) {
>> +		kfree(priv);
>> +		dev_err(dev, "Failed to create cmdbuf suballocator\n");
>> +		return ERR_PTR(-ENOMEM);
>> +	}
>> +
>> +	return priv;
>> +}
>> +
>> +static void etnaviv_free_private(struct etnaviv_drm_private *priv)
>> +{
>> +	if (!priv)
>> +		return;
>> +
>> +	etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc);
>> +
>> +	xa_destroy(&priv->active_contexts);
>> +
>> +	kfree(priv);
>> +}
>>   
>>   static void load_gpu(struct drm_device *dev)
>>   {
>> @@ -511,35 +549,21 @@ static int etnaviv_bind(struct device *dev)
>>   	if (IS_ERR(drm))
>>   		return PTR_ERR(drm);
>>   
>> -	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
>> -	if (!priv) {
>> -		dev_err(dev, "failed to allocate private data\n");
>> -		ret = -ENOMEM;
>> +	priv = etnaviv_alloc_private(dev, drm);
>> +	if (IS_ERR(priv)) {
>> +		ret = PTR_ERR(priv);
>>   		goto out_put;
>>   	}
>> +
>>   	drm->dev_private = priv;
>>   
>>   	dma_set_max_seg_size(dev, SZ_2G);
>>   
>> -	xa_init_flags(&priv->active_contexts, XA_FLAGS_ALLOC);
>> -
>> -	mutex_init(&priv->gem_lock);
>> -	INIT_LIST_HEAD(&priv->gem_list);
>> -	priv->num_gpus = 0;
>> -	priv->shm_gfp_mask = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
>> -
>> -	priv->cmdbuf_suballoc = etnaviv_cmdbuf_suballoc_new(drm->dev);
>> -	if (IS_ERR(priv->cmdbuf_suballoc)) {
>> -		dev_err(drm->dev, "Failed to create cmdbuf suballocator\n");
>> -		ret = PTR_ERR(priv->cmdbuf_suballoc);
>> -		goto out_free_priv;
>> -	}
>> -
>>   	dev_set_drvdata(dev, drm);
>>   
>>   	ret = component_bind_all(dev, drm);
>>   	if (ret < 0)
>> -		goto out_destroy_suballoc;
>> +		goto out_free_priv;
>>   
>>   	load_gpu(drm);
>>   
>> @@ -551,10 +575,8 @@ static int etnaviv_bind(struct device *dev)
>>   
>>   out_unbind:
>>   	component_unbind_all(dev, drm);
>> -out_destroy_suballoc:
>> -	etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc);
>>   out_free_priv:
>> -	kfree(priv);
>> +	etnaviv_free_private(priv);
>>   out_put:
>>   	drm_dev_put(drm);
>>   
>> @@ -570,12 +592,9 @@ static void etnaviv_unbind(struct device *dev)
>>   
>>   	component_unbind_all(dev, drm);
>>   
>> -	etnaviv_cmdbuf_suballoc_destroy(priv->cmdbuf_suballoc);
>> -
>> -	xa_destroy(&priv->active_contexts);
>> +	etnaviv_free_private(priv);
>>   
>>   	drm->dev_private = NULL;
>> -	kfree(priv);
>>   
>>   	drm_dev_put(drm);
>>   }
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> index b3eb1662e90c..e58f82e698de 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> @@ -35,6 +35,7 @@ struct etnaviv_file_private {
>>   };
>>   
>>   struct etnaviv_drm_private {
>> +	struct drm_device *drm;
>>   	int num_gpus;
>>   	struct etnaviv_gpu *gpu[ETNA_MAX_PIPES];
>>   	gfp_t shm_gfp_mask;

-- 
Jingfeng

[PATCH v10 05/11] drm/etnaviv: Allow bypass component framework

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

Originally, component frameworks were used to bind multiple GPU cores to a
virtual master. But there are chips that have only one GPU core integrated.
The component framework can be avoided under some circumstances, Another
reason is that usperspace programs such as X server and Mesa will try to
find the PCI device to use by default. Creating a virtual master device
for PCI GPUs cause unnecessary troubles.

This patch add additional code paths to allow bypassing the component
frameworks, platforms with a single GPU core could probably try the
non-component code path also. This patch is for code shaing between the
PCI driver and the platform driver.

Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_drv.c | 47 ++++++++++-----
 drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 83 +++++++++++++++++----------
 drivers/gpu/drm/etnaviv/etnaviv_gpu.h |  3 +
 3 files changed, 91 insertions(+), 42 deletions(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 6a048be02857..93ca240cd4c0 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -536,10 +536,9 @@ static const struct drm_driver etnaviv_drm_driver = {
 	.minor              = 3,
 };
 
-/*
- * Platform driver:
- */
-static int etnaviv_bind(struct device *dev)
+static struct etnaviv_drm_private *etna_private_ptr;
+
+static int etnaviv_drm_bind(struct device *dev, bool component)
 {
 	struct etnaviv_drm_private *priv;
 	struct drm_device *drm;
@@ -556,12 +555,15 @@ static int etnaviv_bind(struct device *dev)
 	}
 
 	drm->dev_private = priv;
+	etna_private_ptr = priv;
 
 	dma_set_max_seg_size(dev, SZ_2G);
 
-	dev_set_drvdata(dev, drm);
+	if (component)
+		ret = component_bind_all(dev, drm);
+	else
+		ret = etnaviv_gpu_bind(dev, NULL, drm);
 
-	ret = component_bind_all(dev, drm);
 	if (ret < 0)
 		goto out_free_priv;
 
@@ -574,7 +576,10 @@ static int etnaviv_bind(struct device *dev)
 	return 0;
 
 out_unbind:
-	component_unbind_all(dev, drm);
+	if (component)
+		component_unbind_all(dev, drm);
+	else
+		etnaviv_gpu_unbind(dev, NULL, drm);
 out_free_priv:
 	etnaviv_free_private(priv);
 out_put:
@@ -583,14 +588,17 @@ static int etnaviv_bind(struct device *dev)
 	return ret;
 }
 
-static void etnaviv_unbind(struct device *dev)
+static void etnaviv_drm_unbind(struct device *dev, bool component)
 {
-	struct drm_device *drm = dev_get_drvdata(dev);
-	struct etnaviv_drm_private *priv = drm->dev_private;
+	struct etnaviv_drm_private *priv = etna_private_ptr;
+	struct drm_device *drm = priv->drm;
 
 	drm_dev_unregister(drm);
 
-	component_unbind_all(dev, drm);
+	if (component)
+		component_unbind_all(dev, drm);
+	else
+		etnaviv_gpu_unbind(dev, NULL, drm);
 
 	etnaviv_free_private(priv);
 
@@ -599,9 +607,22 @@ static void etnaviv_unbind(struct device *dev)
 	drm_dev_put(drm);
 }
 
+/*
+ * Platform driver:
+ */
+static int etnaviv_master_bind(struct device *dev)
+{
+	return etnaviv_drm_bind(dev, true);
+}
+
+static void etnaviv_master_unbind(struct device *dev)
+{
+	return etnaviv_drm_unbind(dev, true);
+}
+
 static const struct component_master_ops etnaviv_master_ops = {
-	.bind = etnaviv_bind,
-	.unbind = etnaviv_unbind,
+	.bind = etnaviv_master_bind,
+	.unbind = etnaviv_master_unbind,
 };
 
 static int etnaviv_pdev_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index 5e88fa95dac2..059be8c89c5a 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -1737,8 +1737,7 @@ static const struct thermal_cooling_device_ops cooling_ops = {
 	.set_cur_state = etnaviv_gpu_cooling_set_cur_state,
 };
 
-static int etnaviv_gpu_bind(struct device *dev, struct device *master,
-	void *data)
+int etnaviv_gpu_bind(struct device *dev, struct device *master, void *data)
 {
 	struct drm_device *drm = data;
 	struct etnaviv_drm_private *priv = drm->dev_private;
@@ -1769,7 +1768,6 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
 	if (ret < 0)
 		goto out_sched;
 
-
 	gpu->drm = drm;
 	gpu->fence_context = dma_fence_context_alloc(1);
 	xa_init_flags(&gpu->user_fences, XA_FLAGS_ALLOC);
@@ -1798,8 +1796,7 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
 	return ret;
 }
 
-static void etnaviv_gpu_unbind(struct device *dev, struct device *master,
-	void *data)
+void etnaviv_gpu_unbind(struct device *dev, struct device *master, void *data)
 {
 	struct etnaviv_gpu *gpu = dev_get_drvdata(dev);
 
@@ -1867,9 +1864,11 @@ static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
 	return 0;
 }
 
-static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
+/* platform independent */
+
+static int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
+				     int irq, bool component, bool has_clk)
 {
-	struct device *dev = &pdev->dev;
 	struct etnaviv_gpu *gpu;
 	int err;
 
@@ -1877,24 +1876,22 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 	if (!gpu)
 		return -ENOMEM;
 
-	gpu->dev = &pdev->dev;
+	gpu->dev = dev;
+	gpu->mmio = mmio;
 	mutex_init(&gpu->lock);
 	mutex_init(&gpu->sched_lock);
 
-	/* Map registers: */
-	gpu->mmio = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(gpu->mmio))
-		return PTR_ERR(gpu->mmio);
-
 	/* Get Interrupt: */
-	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
+	err = etnaviv_gpu_register_irq(gpu, irq);
 	if (err)
 		return err;
 
 	/* Get Clocks: */
-	err = etnaviv_gpu_clk_get(gpu);
-	if (err)
-		return err;
+	if (has_clk) {
+		err = etnaviv_gpu_clk_get(gpu);
+		if (err)
+			return err;
+	}
 
 	/* TODO: figure out max mapped size */
 	dev_set_drvdata(dev, gpu);
@@ -1904,24 +1901,27 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 	 * autosuspend delay is rather arbitary: no measurements have
 	 * yet been performed to determine an appropriate value.
 	 */
-	pm_runtime_use_autosuspend(gpu->dev);
-	pm_runtime_set_autosuspend_delay(gpu->dev, 200);
-	pm_runtime_enable(gpu->dev);
-
-	err = component_add(&pdev->dev, &gpu_ops);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to register component: %d\n", err);
-		return err;
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_autosuspend_delay(dev, 200);
+	pm_runtime_enable(dev);
+
+	if (component) {
+		err = component_add(dev, &gpu_ops);
+		if (err < 0) {
+			dev_err(dev, "failed to register component: %d\n", err);
+			return err;
+		}
 	}
 
 	return 0;
 }
 
-static int etnaviv_gpu_platform_remove(struct platform_device *pdev)
+static void etnaviv_gpu_driver_destroy(struct device *dev, bool component)
 {
-	component_del(&pdev->dev, &gpu_ops);
-	pm_runtime_disable(&pdev->dev);
-	return 0;
+	if (component)
+		component_del(dev, &gpu_ops);
+
+	pm_runtime_disable(dev);
 }
 
 static int etnaviv_gpu_rpm_suspend(struct device *dev)
@@ -1971,6 +1971,31 @@ static const struct dev_pm_ops etnaviv_gpu_pm_ops = {
 	RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume, NULL)
 };
 
+static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	void __iomem *mmio;
+	int irq;
+
+	/* Map registers: */
+	mmio = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(mmio))
+		return PTR_ERR(mmio);
+
+	irq = platform_get_irq(pdev, 0);
+
+	return etnaviv_gpu_driver_create(dev, mmio, irq, true, true);
+}
+
+static int etnaviv_gpu_platform_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	etnaviv_gpu_driver_destroy(dev, true);
+
+	return 0;
+}
+
 struct platform_driver etnaviv_gpu_driver = {
 	.driver = {
 		.name = "etnaviv-gpu",
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index 98c6f9c320fc..1ec829a649b5 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -206,6 +206,9 @@ void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu);
 int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms);
 void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch);
 
+int etnaviv_gpu_bind(struct device *dev, struct device *master, void *data);
+void etnaviv_gpu_unbind(struct device *dev, struct device *master, void *data);
+
 extern struct platform_driver etnaviv_gpu_driver;
 
 #endif /* __ETNAVIV_GPU_H__ */
-- 
2.25.1

Re: [PATCH v10 05/11] drm/etnaviv: Allow bypass component framework

[Lucas Stach]

Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
> From: Sui Jingfeng <suijingfeng@loongson.cn>
> 
> Originally, component frameworks were used to bind multiple GPU cores to a
> virtual master. But there are chips that have only one GPU core integrated.
> The component framework can be avoided under some circumstances, Another
> reason is that usperspace programs such as X server and Mesa will try to
> find the PCI device to use by default. Creating a virtual master device
> for PCI GPUs cause unnecessary troubles.
> 
> This patch add additional code paths to allow bypassing the component
> frameworks, platforms with a single GPU core could probably try the
> non-component code path also. This patch is for code shaing between the
> PCI driver and the platform driver.
> 
> Cc: Lucas Stach <l.stach@pengutronix.de>
> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> Cc: Philipp Zabel <p.zabel@pengutronix.de>
> Cc: Bjorn Helgaas <bhelgaas@google.com>
> Cc: Daniel Vetter <daniel@ffwll.ch>
> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> ---
>  drivers/gpu/drm/etnaviv/etnaviv_drv.c | 47 ++++++++++-----
>  drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 83 +++++++++++++++++----------
>  drivers/gpu/drm/etnaviv/etnaviv_gpu.h |  3 +
>  3 files changed, 91 insertions(+), 42 deletions(-)
> 
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> index 6a048be02857..93ca240cd4c0 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> @@ -536,10 +536,9 @@ static const struct drm_driver etnaviv_drm_driver = {
>  	.minor              = 3,
>  };
>  
> -/*
> - * Platform driver:
> - */
> -static int etnaviv_bind(struct device *dev)
> +static struct etnaviv_drm_private *etna_private_ptr;

That's not going to fly. You are dropping the virtual master device,
which bundles multiple GPUs together, but you are also only allowing a
single GPU instance via this global private data pointer.

I'm okay with dropping the virtual master and instantiating a DRM
device for each PCI device, but then the driver at least needs to be
able to handle multiple instances.

Also what exactly is the problem with the virtual master device?
Couldn't we just instantiate one of those for each PCI device to
minimize the changes needed to the bind/unbind logic?

Regards,
Lucas

> +
> +static int etnaviv_drm_bind(struct device *dev, bool component)
>  {
>  	struct etnaviv_drm_private *priv;
>  	struct drm_device *drm;
> @@ -556,12 +555,15 @@ static int etnaviv_bind(struct device *dev)
>  	}
>  
>  	drm->dev_private = priv;
> +	etna_private_ptr = priv;
>  
>  	dma_set_max_seg_size(dev, SZ_2G);
>  
> -	dev_set_drvdata(dev, drm);
> +	if (component)
> +		ret = component_bind_all(dev, drm);
> +	else
> +		ret = etnaviv_gpu_bind(dev, NULL, drm);
>  
> -	ret = component_bind_all(dev, drm);
>  	if (ret < 0)
>  		goto out_free_priv;
>  
> @@ -574,7 +576,10 @@ static int etnaviv_bind(struct device *dev)
>  	return 0;
>  
>  out_unbind:
> -	component_unbind_all(dev, drm);
> +	if (component)
> +		component_unbind_all(dev, drm);
> +	else
> +		etnaviv_gpu_unbind(dev, NULL, drm);
>  out_free_priv:
>  	etnaviv_free_private(priv);
>  out_put:
> @@ -583,14 +588,17 @@ static int etnaviv_bind(struct device *dev)
>  	return ret;
>  }
>  
> -static void etnaviv_unbind(struct device *dev)
> +static void etnaviv_drm_unbind(struct device *dev, bool component)
>  {
> -	struct drm_device *drm = dev_get_drvdata(dev);
> -	struct etnaviv_drm_private *priv = drm->dev_private;
> +	struct etnaviv_drm_private *priv = etna_private_ptr;
> +	struct drm_device *drm = priv->drm;
>  
>  	drm_dev_unregister(drm);
>  
> -	component_unbind_all(dev, drm);
> +	if (component)
> +		component_unbind_all(dev, drm);
> +	else
> +		etnaviv_gpu_unbind(dev, NULL, drm);
>  
>  	etnaviv_free_private(priv);
>  
> @@ -599,9 +607,22 @@ static void etnaviv_unbind(struct device *dev)
>  	drm_dev_put(drm);
>  }
>  
> +/*
> + * Platform driver:
> + */
> +static int etnaviv_master_bind(struct device *dev)
> +{
> +	return etnaviv_drm_bind(dev, true);
> +}
> +
> +static void etnaviv_master_unbind(struct device *dev)
> +{
> +	return etnaviv_drm_unbind(dev, true);
> +}
> +
>  static const struct component_master_ops etnaviv_master_ops = {
> -	.bind = etnaviv_bind,
> -	.unbind = etnaviv_unbind,
> +	.bind = etnaviv_master_bind,
> +	.unbind = etnaviv_master_unbind,
>  };
>  
>  static int etnaviv_pdev_probe(struct platform_device *pdev)
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> index 5e88fa95dac2..059be8c89c5a 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> @@ -1737,8 +1737,7 @@ static const struct thermal_cooling_device_ops cooling_ops = {
>  	.set_cur_state = etnaviv_gpu_cooling_set_cur_state,
>  };
>  
> -static int etnaviv_gpu_bind(struct device *dev, struct device *master,
> -	void *data)
> +int etnaviv_gpu_bind(struct device *dev, struct device *master, void *data)
>  {
>  	struct drm_device *drm = data;
>  	struct etnaviv_drm_private *priv = drm->dev_private;
> @@ -1769,7 +1768,6 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
>  	if (ret < 0)
>  		goto out_sched;
>  
> -
>  	gpu->drm = drm;
>  	gpu->fence_context = dma_fence_context_alloc(1);
>  	xa_init_flags(&gpu->user_fences, XA_FLAGS_ALLOC);
> @@ -1798,8 +1796,7 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
>  	return ret;
>  }
>  
> -static void etnaviv_gpu_unbind(struct device *dev, struct device *master,
> -	void *data)
> +void etnaviv_gpu_unbind(struct device *dev, struct device *master, void *data)
>  {
>  	struct etnaviv_gpu *gpu = dev_get_drvdata(dev);
>  
> @@ -1867,9 +1864,11 @@ static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
>  	return 0;
>  }
>  
> -static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
> +/* platform independent */
> +
> +static int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
> +				     int irq, bool component, bool has_clk)
>  {
> -	struct device *dev = &pdev->dev;
>  	struct etnaviv_gpu *gpu;
>  	int err;
>  
> @@ -1877,24 +1876,22 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>  	if (!gpu)
>  		return -ENOMEM;
>  
> -	gpu->dev = &pdev->dev;
> +	gpu->dev = dev;
> +	gpu->mmio = mmio;
>  	mutex_init(&gpu->lock);
>  	mutex_init(&gpu->sched_lock);
>  
> -	/* Map registers: */
> -	gpu->mmio = devm_platform_ioremap_resource(pdev, 0);
> -	if (IS_ERR(gpu->mmio))
> -		return PTR_ERR(gpu->mmio);
> -
>  	/* Get Interrupt: */
> -	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
> +	err = etnaviv_gpu_register_irq(gpu, irq);
>  	if (err)
>  		return err;
>  
>  	/* Get Clocks: */
> -	err = etnaviv_gpu_clk_get(gpu);
> -	if (err)
> -		return err;
> +	if (has_clk) {
> +		err = etnaviv_gpu_clk_get(gpu);
> +		if (err)
> +			return err;
> +	}
>  
>  	/* TODO: figure out max mapped size */
>  	dev_set_drvdata(dev, gpu);
> @@ -1904,24 +1901,27 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>  	 * autosuspend delay is rather arbitary: no measurements have
>  	 * yet been performed to determine an appropriate value.
>  	 */
> -	pm_runtime_use_autosuspend(gpu->dev);
> -	pm_runtime_set_autosuspend_delay(gpu->dev, 200);
> -	pm_runtime_enable(gpu->dev);
> -
> -	err = component_add(&pdev->dev, &gpu_ops);
> -	if (err < 0) {
> -		dev_err(&pdev->dev, "failed to register component: %d\n", err);
> -		return err;
> +	pm_runtime_use_autosuspend(dev);
> +	pm_runtime_set_autosuspend_delay(dev, 200);
> +	pm_runtime_enable(dev);
> +
> +	if (component) {
> +		err = component_add(dev, &gpu_ops);
> +		if (err < 0) {
> +			dev_err(dev, "failed to register component: %d\n", err);
> +			return err;
> +		}
>  	}
>  
>  	return 0;
>  }
>  
> -static int etnaviv_gpu_platform_remove(struct platform_device *pdev)
> +static void etnaviv_gpu_driver_destroy(struct device *dev, bool component)
>  {
> -	component_del(&pdev->dev, &gpu_ops);
> -	pm_runtime_disable(&pdev->dev);
> -	return 0;
> +	if (component)
> +		component_del(dev, &gpu_ops);
> +
> +	pm_runtime_disable(dev);
>  }
>  
>  static int etnaviv_gpu_rpm_suspend(struct device *dev)
> @@ -1971,6 +1971,31 @@ static const struct dev_pm_ops etnaviv_gpu_pm_ops = {
>  	RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume, NULL)
>  };
>  
> +static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
> +{
> +	struct device *dev = &pdev->dev;
> +	void __iomem *mmio;
> +	int irq;
> +
> +	/* Map registers: */
> +	mmio = devm_platform_ioremap_resource(pdev, 0);
> +	if (IS_ERR(mmio))
> +		return PTR_ERR(mmio);
> +
> +	irq = platform_get_irq(pdev, 0);
> +
> +	return etnaviv_gpu_driver_create(dev, mmio, irq, true, true);
> +}
> +
> +static int etnaviv_gpu_platform_remove(struct platform_device *pdev)
> +{
> +	struct device *dev = &pdev->dev;
> +
> +	etnaviv_gpu_driver_destroy(dev, true);
> +
> +	return 0;
> +}
> +
>  struct platform_driver etnaviv_gpu_driver = {
>  	.driver = {
>  		.name = "etnaviv-gpu",
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> index 98c6f9c320fc..1ec829a649b5 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> @@ -206,6 +206,9 @@ void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu);
>  int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms);
>  void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch);
>  
> +int etnaviv_gpu_bind(struct device *dev, struct device *master, void *data);
> +void etnaviv_gpu_unbind(struct device *dev, struct device *master, void *data);
> +
>  extern struct platform_driver etnaviv_gpu_driver;
>  
>  #endif /* __ETNAVIV_GPU_H__ */

Re: [PATCH v10 05/11] drm/etnaviv: Allow bypass component framework

[Sui Jingfeng]

Hi,

On 2023/6/21 17:29, Lucas Stach wrote:
> Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>
>> Originally, component frameworks were used to bind multiple GPU cores to a
>> virtual master. But there are chips that have only one GPU core integrated.
>> The component framework can be avoided under some circumstances, Another
>> reason is that usperspace programs such as X server and Mesa will try to
>> find the PCI device to use by default. Creating a virtual master device
>> for PCI GPUs cause unnecessary troubles.
>>
>> This patch add additional code paths to allow bypassing the component
>> frameworks, platforms with a single GPU core could probably try the
>> non-component code path also. This patch is for code shaing between the
>> PCI driver and the platform driver.
>>
>> Cc: Lucas Stach <l.stach@pengutronix.de>
>> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
>> Cc: Philipp Zabel <p.zabel@pengutronix.de>
>> Cc: Bjorn Helgaas <bhelgaas@google.com>
>> Cc: Daniel Vetter <daniel@ffwll.ch>
>> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
>> ---
>>   drivers/gpu/drm/etnaviv/etnaviv_drv.c | 47 ++++++++++-----
>>   drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 83 +++++++++++++++++----------
>>   drivers/gpu/drm/etnaviv/etnaviv_gpu.h |  3 +
>>   3 files changed, 91 insertions(+), 42 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> index 6a048be02857..93ca240cd4c0 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> @@ -536,10 +536,9 @@ static const struct drm_driver etnaviv_drm_driver = {
>>   	.minor              = 3,
>>   };
>>   
>> -/*
>> - * Platform driver:
>> - */
>> -static int etnaviv_bind(struct device *dev)
>> +static struct etnaviv_drm_private *etna_private_ptr;
> That's not going to fly. You are dropping the virtual master device,
> which bundles multiple GPUs together, but you are also only allowing a
> single GPU instance via this global private data pointer.
>
> I'm okay with dropping the virtual master and instantiating a DRM
> device for each PCI device, but then the driver at least needs to be
> able to handle multiple instances.

Nice catch!


By multiple instances, do you means that multiple vivante GPU core ?


Currently we don't know how to handle multiple PCI GPU instances,

The main reason is that we don't have such a hardware.

I still think,

the PCI device itself still should be the master, other GPU cores can be 
initialized manually.

or bind to this PCI device master with priori knowledge./
/

> Also what exactly is the problem with the virtual master device?

mesa's piple loader will find the PCI device to use by default.

the virtual master device is a platform device.

PCI device go first.

> Couldn't we just instantiate one of those for each PCI device to
> minimize the changes needed to the bind/unbind logic?
If we have such a hardware,

I don't expect each of them could populate to be a stand alone PCI 
device instance.

Often, the mmio registers of the multiple instantiate are offset-ed 
within the same PCI BAR.

Multiple PCI device usually means multiple device driver.


To be honest, this is already the minimal change we can do,

which don't introduce duplication like the patch we made one years ago.


>
> Regards,
> Lucas
>
>> +
>> +static int etnaviv_drm_bind(struct device *dev, bool component)
>>   {
>>   	struct etnaviv_drm_private *priv;
>>   	struct drm_device *drm;
>> @@ -556,12 +555,15 @@ static int etnaviv_bind(struct device *dev)
>>   	}
>>   
>>   	drm->dev_private = priv;
>> +	etna_private_ptr = priv;
>>   
>>   	dma_set_max_seg_size(dev, SZ_2G);
>>   
>> -	dev_set_drvdata(dev, drm);
>> +	if (component)
>> +		ret = component_bind_all(dev, drm);
>> +	else
>> +		ret = etnaviv_gpu_bind(dev, NULL, drm);
>>   
>> -	ret = component_bind_all(dev, drm);
>>   	if (ret < 0)
>>   		goto out_free_priv;
>>   
>> @@ -574,7 +576,10 @@ static int etnaviv_bind(struct device *dev)
>>   	return 0;
>>   
>>   out_unbind:
>> -	component_unbind_all(dev, drm);
>> +	if (component)
>> +		component_unbind_all(dev, drm);
>> +	else
>> +		etnaviv_gpu_unbind(dev, NULL, drm);
>>   out_free_priv:
>>   	etnaviv_free_private(priv);
>>   out_put:
>> @@ -583,14 +588,17 @@ static int etnaviv_bind(struct device *dev)
>>   	return ret;
>>   }
>>   
>> -static void etnaviv_unbind(struct device *dev)
>> +static void etnaviv_drm_unbind(struct device *dev, bool component)
>>   {
>> -	struct drm_device *drm = dev_get_drvdata(dev);
>> -	struct etnaviv_drm_private *priv = drm->dev_private;
>> +	struct etnaviv_drm_private *priv = etna_private_ptr;
>> +	struct drm_device *drm = priv->drm;
>>   
>>   	drm_dev_unregister(drm);
>>   
>> -	component_unbind_all(dev, drm);
>> +	if (component)
>> +		component_unbind_all(dev, drm);
>> +	else
>> +		etnaviv_gpu_unbind(dev, NULL, drm);
>>   
>>   	etnaviv_free_private(priv);
>>   
>> @@ -599,9 +607,22 @@ static void etnaviv_unbind(struct device *dev)
>>   	drm_dev_put(drm);
>>   }
>>   
>> +/*
>> + * Platform driver:
>> + */
>> +static int etnaviv_master_bind(struct device *dev)
>> +{
>> +	return etnaviv_drm_bind(dev, true);
>> +}
>> +
>> +static void etnaviv_master_unbind(struct device *dev)
>> +{
>> +	return etnaviv_drm_unbind(dev, true);
>> +}
>> +
>>   static const struct component_master_ops etnaviv_master_ops = {
>> -	.bind = etnaviv_bind,
>> -	.unbind = etnaviv_unbind,
>> +	.bind = etnaviv_master_bind,
>> +	.unbind = etnaviv_master_unbind,
>>   };
>>   
>>   static int etnaviv_pdev_probe(struct platform_device *pdev)
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> index 5e88fa95dac2..059be8c89c5a 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> @@ -1737,8 +1737,7 @@ static const struct thermal_cooling_device_ops cooling_ops = {
>>   	.set_cur_state = etnaviv_gpu_cooling_set_cur_state,
>>   };
>>   
>> -static int etnaviv_gpu_bind(struct device *dev, struct device *master,
>> -	void *data)
>> +int etnaviv_gpu_bind(struct device *dev, struct device *master, void *data)
>>   {
>>   	struct drm_device *drm = data;
>>   	struct etnaviv_drm_private *priv = drm->dev_private;
>> @@ -1769,7 +1768,6 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
>>   	if (ret < 0)
>>   		goto out_sched;
>>   
>> -
>>   	gpu->drm = drm;
>>   	gpu->fence_context = dma_fence_context_alloc(1);
>>   	xa_init_flags(&gpu->user_fences, XA_FLAGS_ALLOC);
>> @@ -1798,8 +1796,7 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
>>   	return ret;
>>   }
>>   
>> -static void etnaviv_gpu_unbind(struct device *dev, struct device *master,
>> -	void *data)
>> +void etnaviv_gpu_unbind(struct device *dev, struct device *master, void *data)
>>   {
>>   	struct etnaviv_gpu *gpu = dev_get_drvdata(dev);
>>   
>> @@ -1867,9 +1864,11 @@ static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
>>   	return 0;
>>   }
>>   
>> -static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>> +/* platform independent */
>> +
>> +static int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
>> +				     int irq, bool component, bool has_clk)
>>   {
>> -	struct device *dev = &pdev->dev;
>>   	struct etnaviv_gpu *gpu;
>>   	int err;
>>   
>> @@ -1877,24 +1876,22 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>>   	if (!gpu)
>>   		return -ENOMEM;
>>   
>> -	gpu->dev = &pdev->dev;
>> +	gpu->dev = dev;
>> +	gpu->mmio = mmio;
>>   	mutex_init(&gpu->lock);
>>   	mutex_init(&gpu->sched_lock);
>>   
>> -	/* Map registers: */
>> -	gpu->mmio = devm_platform_ioremap_resource(pdev, 0);
>> -	if (IS_ERR(gpu->mmio))
>> -		return PTR_ERR(gpu->mmio);
>> -
>>   	/* Get Interrupt: */
>> -	err = etnaviv_gpu_register_irq(gpu, platform_get_irq(pdev, 0));
>> +	err = etnaviv_gpu_register_irq(gpu, irq);
>>   	if (err)
>>   		return err;
>>   
>>   	/* Get Clocks: */
>> -	err = etnaviv_gpu_clk_get(gpu);
>> -	if (err)
>> -		return err;
>> +	if (has_clk) {
>> +		err = etnaviv_gpu_clk_get(gpu);
>> +		if (err)
>> +			return err;
>> +	}
>>   
>>   	/* TODO: figure out max mapped size */
>>   	dev_set_drvdata(dev, gpu);
>> @@ -1904,24 +1901,27 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>>   	 * autosuspend delay is rather arbitary: no measurements have
>>   	 * yet been performed to determine an appropriate value.
>>   	 */
>> -	pm_runtime_use_autosuspend(gpu->dev);
>> -	pm_runtime_set_autosuspend_delay(gpu->dev, 200);
>> -	pm_runtime_enable(gpu->dev);
>> -
>> -	err = component_add(&pdev->dev, &gpu_ops);
>> -	if (err < 0) {
>> -		dev_err(&pdev->dev, "failed to register component: %d\n", err);
>> -		return err;
>> +	pm_runtime_use_autosuspend(dev);
>> +	pm_runtime_set_autosuspend_delay(dev, 200);
>> +	pm_runtime_enable(dev);
>> +
>> +	if (component) {
>> +		err = component_add(dev, &gpu_ops);
>> +		if (err < 0) {
>> +			dev_err(dev, "failed to register component: %d\n", err);
>> +			return err;
>> +		}
>>   	}
>>   
>>   	return 0;
>>   }
>>   
>> -static int etnaviv_gpu_platform_remove(struct platform_device *pdev)
>> +static void etnaviv_gpu_driver_destroy(struct device *dev, bool component)
>>   {
>> -	component_del(&pdev->dev, &gpu_ops);
>> -	pm_runtime_disable(&pdev->dev);
>> -	return 0;
>> +	if (component)
>> +		component_del(dev, &gpu_ops);
>> +
>> +	pm_runtime_disable(dev);
>>   }
>>   
>>   static int etnaviv_gpu_rpm_suspend(struct device *dev)
>> @@ -1971,6 +1971,31 @@ static const struct dev_pm_ops etnaviv_gpu_pm_ops = {
>>   	RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume, NULL)
>>   };
>>   
>> +static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
>> +{
>> +	struct device *dev = &pdev->dev;
>> +	void __iomem *mmio;
>> +	int irq;
>> +
>> +	/* Map registers: */
>> +	mmio = devm_platform_ioremap_resource(pdev, 0);
>> +	if (IS_ERR(mmio))
>> +		return PTR_ERR(mmio);
>> +
>> +	irq = platform_get_irq(pdev, 0);
>> +
>> +	return etnaviv_gpu_driver_create(dev, mmio, irq, true, true);
>> +}
>> +
>> +static int etnaviv_gpu_platform_remove(struct platform_device *pdev)
>> +{
>> +	struct device *dev = &pdev->dev;
>> +
>> +	etnaviv_gpu_driver_destroy(dev, true);
>> +
>> +	return 0;
>> +}
>> +
>>   struct platform_driver etnaviv_gpu_driver = {
>>   	.driver = {
>>   		.name = "etnaviv-gpu",
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
>> index 98c6f9c320fc..1ec829a649b5 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
>> @@ -206,6 +206,9 @@ void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu);
>>   int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms);
>>   void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch);
>>   
>> +int etnaviv_gpu_bind(struct device *dev, struct device *master, void *data);
>> +void etnaviv_gpu_unbind(struct device *dev, struct device *master, void *data);
>> +
>>   extern struct platform_driver etnaviv_gpu_driver;
>>   
>>   #endif /* __ETNAVIV_GPU_H__ */

-- 
Jingfeng

[PATCH v10 06/11] drm/etnaviv: Add driver support for the PCI devices

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

This patch adds PCI driver support on top of what we already have, take
the GC1000 in LS7A1000/LS2K1000 as the first instance which enjoy the PCI
device driver. There is only one GPU core for the GC1000 in the LS7A1000
and LS2K1000. Therefore, component frameworks can be avoided.

Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/Kconfig           | 10 +++
 drivers/gpu/drm/etnaviv/Makefile          |  2 +
 drivers/gpu/drm/etnaviv/etnaviv_drv.c     | 13 +++-
 drivers/gpu/drm/etnaviv/etnaviv_drv.h     |  3 +
 drivers/gpu/drm/etnaviv/etnaviv_gpu.c     |  8 +--
 drivers/gpu/drm/etnaviv/etnaviv_gpu.h     |  6 ++
 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c | 75 +++++++++++++++++++++++
 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h | 18 ++++++
 8 files changed, 128 insertions(+), 7 deletions(-)
 create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
 create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h

diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig
index faa7fc68b009..1b5b162efb61 100644
--- a/drivers/gpu/drm/etnaviv/Kconfig
+++ b/drivers/gpu/drm/etnaviv/Kconfig
@@ -15,6 +15,16 @@ config DRM_ETNAVIV
 	help
 	  DRM driver for Vivante GPUs.
 
+config DRM_ETNAVIV_PCI_DRIVER
+	bool "enable ETNAVIV PCI driver support"
+	depends on DRM_ETNAVIV
+	depends on PCI
+	default y
+	help
+	  Compile in support for PCI GPUs of Vivante.
+	  For example, the GC1000 in LS7A1000 and LS2K1000.
+	  Say Y if you have such a hardware.
+
 config DRM_ETNAVIV_THERMAL
 	bool "enable ETNAVIV thermal throttling"
 	depends on DRM_ETNAVIV
diff --git a/drivers/gpu/drm/etnaviv/Makefile b/drivers/gpu/drm/etnaviv/Makefile
index 46e5ffad69a6..6829e1ebf2db 100644
--- a/drivers/gpu/drm/etnaviv/Makefile
+++ b/drivers/gpu/drm/etnaviv/Makefile
@@ -16,4 +16,6 @@ etnaviv-y := \
 	etnaviv_perfmon.o \
 	etnaviv_sched.o
 
+etnaviv-$(CONFIG_DRM_ETNAVIV_PCI_DRIVER) += etnaviv_pci_drv.o
+
 obj-$(CONFIG_DRM_ETNAVIV)	+= etnaviv.o
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 93ca240cd4c0..0a365e96d371 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -21,6 +21,7 @@
 #include "etnaviv_gpu.h"
 #include "etnaviv_gem.h"
 #include "etnaviv_mmu.h"
+#include "etnaviv_pci_drv.h"
 #include "etnaviv_perfmon.h"
 
 /*
@@ -538,7 +539,7 @@ static const struct drm_driver etnaviv_drm_driver = {
 
 static struct etnaviv_drm_private *etna_private_ptr;
 
-static int etnaviv_drm_bind(struct device *dev, bool component)
+int etnaviv_drm_bind(struct device *dev, bool component)
 {
 	struct etnaviv_drm_private *priv;
 	struct drm_device *drm;
@@ -588,7 +589,7 @@ static int etnaviv_drm_bind(struct device *dev, bool component)
 	return ret;
 }
 
-static void etnaviv_drm_unbind(struct device *dev, bool component)
+void etnaviv_drm_unbind(struct device *dev, bool component)
 {
 	struct etnaviv_drm_private *priv = etna_private_ptr;
 	struct drm_device *drm = priv->drm;
@@ -746,6 +747,10 @@ static int __init etnaviv_init(void)
 	if (ret != 0)
 		goto unregister_gpu_driver;
 
+	ret = etnaviv_register_pci_driver();
+	if (ret != 0)
+		goto unregister_platform_driver;
+
 	/*
 	 * If the DT contains at least one available GPU device, instantiate
 	 * the DRM platform device.
@@ -763,7 +768,7 @@ static int __init etnaviv_init(void)
 		break;
 	}
 
-	return 0;
+	return ret;
 
 unregister_platform_driver:
 	platform_driver_unregister(&etnaviv_platform_driver);
@@ -778,6 +783,8 @@ static void __exit etnaviv_exit(void)
 	etnaviv_destroy_platform_device(&etnaviv_platform_device);
 	platform_driver_unregister(&etnaviv_platform_driver);
 	platform_driver_unregister(&etnaviv_gpu_driver);
+
+	etnaviv_unregister_pci_driver();
 }
 module_exit(etnaviv_exit);
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index e58f82e698de..9cd72948cfad 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -83,6 +83,9 @@ bool etnaviv_cmd_validate_one(struct etnaviv_gpu *gpu,
 	u32 *stream, unsigned int size,
 	struct drm_etnaviv_gem_submit_reloc *relocs, unsigned int reloc_size);
 
+int etnaviv_drm_bind(struct device *dev, bool component);
+void etnaviv_drm_unbind(struct device *dev, bool component);
+
 #ifdef CONFIG_DEBUG_FS
 void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv,
 	struct seq_file *m);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index 059be8c89c5a..d6a21e97feb1 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -1866,8 +1866,8 @@ static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
 
 /* platform independent */
 
-static int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
-				     int irq, bool component, bool has_clk)
+int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
+			      int irq, bool component, bool has_clk)
 {
 	struct etnaviv_gpu *gpu;
 	int err;
@@ -1916,7 +1916,7 @@ static int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
 	return 0;
 }
 
-static void etnaviv_gpu_driver_destroy(struct device *dev, bool component)
+void etnaviv_gpu_driver_destroy(struct device *dev, bool component)
 {
 	if (component)
 		component_del(dev, &gpu_ops);
@@ -1967,7 +1967,7 @@ static int etnaviv_gpu_rpm_resume(struct device *dev)
 	return 0;
 }
 
-static const struct dev_pm_ops etnaviv_gpu_pm_ops = {
+const struct dev_pm_ops etnaviv_gpu_pm_ops = {
 	RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume, NULL)
 };
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index 1ec829a649b5..8d9833996ed7 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -209,6 +209,12 @@ void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch);
 int etnaviv_gpu_bind(struct device *dev, struct device *master, void *data);
 void etnaviv_gpu_unbind(struct device *dev, struct device *master, void *data);
 
+int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
+			      int irq, bool component, bool has_clk);
+
+void etnaviv_gpu_driver_destroy(struct device *dev, bool component);
+
 extern struct platform_driver etnaviv_gpu_driver;
+extern const struct dev_pm_ops etnaviv_gpu_pm_ops;
 
 #endif /* __ETNAVIV_GPU_H__ */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
new file mode 100644
index 000000000000..78e44a28d30c
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/pci.h>
+
+#include "etnaviv_drv.h"
+#include "etnaviv_gpu.h"
+#include "etnaviv_pci_drv.h"
+
+static int etnaviv_pci_probe(struct pci_dev *pdev,
+			     const struct pci_device_id *ent)
+{
+	struct device *dev = &pdev->dev;
+	void __iomem *mmio;
+	int ret;
+
+	ret = pcim_enable_device(pdev);
+	if (ret) {
+		dev_err(dev, "failed to enable\n");
+		return ret;
+	}
+
+	pci_set_master(pdev);
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret)
+		return ret;
+
+	/* Map registers, assume the PCI bar 0 contain the registers */
+	mmio = pcim_iomap(pdev, 0, 0);
+	if (IS_ERR(mmio))
+		return PTR_ERR(mmio);
+
+	ret = etnaviv_gpu_driver_create(dev, mmio, pdev->irq, false, false);
+	if (ret)
+		return ret;
+
+	return etnaviv_drm_bind(dev, false);
+}
+
+static void etnaviv_pci_remove(struct pci_dev *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	etnaviv_drm_unbind(dev, false);
+
+	etnaviv_gpu_driver_destroy(dev, false);
+
+	pci_clear_master(pdev);
+}
+
+static const struct pci_device_id etnaviv_pci_id_lists[] = {
+	{PCI_VDEVICE(LOONGSON, 0x7a15)},
+	{PCI_VDEVICE(LOONGSON, 0x7a05)},
+	{ }
+};
+
+static struct pci_driver etnaviv_pci_driver = {
+	.name = "etnaviv",
+	.id_table = etnaviv_pci_id_lists,
+	.probe = etnaviv_pci_probe,
+	.remove = etnaviv_pci_remove,
+	.driver.pm = pm_ptr(&etnaviv_gpu_pm_ops),
+};
+
+int etnaviv_register_pci_driver(void)
+{
+	return pci_register_driver(&etnaviv_pci_driver);
+}
+
+void etnaviv_unregister_pci_driver(void)
+{
+	pci_unregister_driver(&etnaviv_pci_driver);
+}
+
+MODULE_DEVICE_TABLE(pci, etnaviv_pci_id_lists);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h
new file mode 100644
index 000000000000..1db559ee5e9b
--- /dev/null
+++ b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __ETNAVIV_PCI_DRV_H__
+#define __ETNAVIV_PCI_DRV_H__
+
+#ifdef CONFIG_DRM_ETNAVIV_PCI_DRIVER
+
+int etnaviv_register_pci_driver(void);
+void etnaviv_unregister_pci_driver(void);
+
+#else
+
+static inline int etnaviv_register_pci_driver(void) { return 0; }
+static inline void etnaviv_unregister_pci_driver(void) { }
+
+#endif
+
+#endif
-- 
2.25.1

Re: [PATCH v10 06/11] drm/etnaviv: Add driver support for the PCI devices

[Lucas Stach]

Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
> From: Sui Jingfeng <suijingfeng@loongson.cn>
> 
> This patch adds PCI driver support on top of what we already have, take
> the GC1000 in LS7A1000/LS2K1000 as the first instance which enjoy the PCI
> device driver. There is only one GPU core for the GC1000 in the LS7A1000
> and LS2K1000. Therefore, component frameworks can be avoided.
> 
> Cc: Lucas Stach <l.stach@pengutronix.de>
> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> Cc: Philipp Zabel <p.zabel@pengutronix.de>
> Cc: Bjorn Helgaas <bhelgaas@google.com>
> Cc: Daniel Vetter <daniel@ffwll.ch>
> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> ---
>  drivers/gpu/drm/etnaviv/Kconfig           | 10 +++
>  drivers/gpu/drm/etnaviv/Makefile          |  2 +
>  drivers/gpu/drm/etnaviv/etnaviv_drv.c     | 13 +++-
>  drivers/gpu/drm/etnaviv/etnaviv_drv.h     |  3 +
>  drivers/gpu/drm/etnaviv/etnaviv_gpu.c     |  8 +--
>  drivers/gpu/drm/etnaviv/etnaviv_gpu.h     |  6 ++
>  drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c | 75 +++++++++++++++++++++++
>  drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h | 18 ++++++
>  8 files changed, 128 insertions(+), 7 deletions(-)
>  create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
>  create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h
> 
> diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig
> index faa7fc68b009..1b5b162efb61 100644
> --- a/drivers/gpu/drm/etnaviv/Kconfig
> +++ b/drivers/gpu/drm/etnaviv/Kconfig
> @@ -15,6 +15,16 @@ config DRM_ETNAVIV
>  	help
>  	  DRM driver for Vivante GPUs.
>  
> +config DRM_ETNAVIV_PCI_DRIVER
> +	bool "enable ETNAVIV PCI driver support"
> +	depends on DRM_ETNAVIV
> +	depends on PCI
> +	default y

Drop the default y. Lots of existing systems with Vivante GPUs on a
platform bus also have a PCI bus, so this will activate the added code
paths in a lot of configurations that don't need it.

> +	help
> +	  Compile in support for PCI GPUs of Vivante.

Compile in support for Vivante GPUs attached via PCI(e).

> +	  For example, the GC1000 in LS7A1000 and LS2K1000.

Not sure if the example is that helpful. Maybe drop this.

> +	  Say Y if you have such a hardware.
> +
>  config DRM_ETNAVIV_THERMAL
>  	bool "enable ETNAVIV thermal throttling"
>  	depends on DRM_ETNAVIV
> diff --git a/drivers/gpu/drm/etnaviv/Makefile b/drivers/gpu/drm/etnaviv/Makefile
> index 46e5ffad69a6..6829e1ebf2db 100644
> --- a/drivers/gpu/drm/etnaviv/Makefile
> +++ b/drivers/gpu/drm/etnaviv/Makefile
> @@ -16,4 +16,6 @@ etnaviv-y := \
>  	etnaviv_perfmon.o \
>  	etnaviv_sched.o
>  
> +etnaviv-$(CONFIG_DRM_ETNAVIV_PCI_DRIVER) += etnaviv_pci_drv.o
> +
>  obj-$(CONFIG_DRM_ETNAVIV)	+= etnaviv.o
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> index 93ca240cd4c0..0a365e96d371 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> @@ -21,6 +21,7 @@
>  #include "etnaviv_gpu.h"
>  #include "etnaviv_gem.h"
>  #include "etnaviv_mmu.h"
> +#include "etnaviv_pci_drv.h"
>  #include "etnaviv_perfmon.h"
>  
>  /*
> @@ -538,7 +539,7 @@ static const struct drm_driver etnaviv_drm_driver = {
>  
>  static struct etnaviv_drm_private *etna_private_ptr;
>  
> -static int etnaviv_drm_bind(struct device *dev, bool component)
> +int etnaviv_drm_bind(struct device *dev, bool component)
>  {
>  	struct etnaviv_drm_private *priv;
>  	struct drm_device *drm;
> @@ -588,7 +589,7 @@ static int etnaviv_drm_bind(struct device *dev, bool component)
>  	return ret;
>  }
>  
> -static void etnaviv_drm_unbind(struct device *dev, bool component)
> +void etnaviv_drm_unbind(struct device *dev, bool component)
>  {
>  	struct etnaviv_drm_private *priv = etna_private_ptr;
>  	struct drm_device *drm = priv->drm;
> @@ -746,6 +747,10 @@ static int __init etnaviv_init(void)
>  	if (ret != 0)
>  		goto unregister_gpu_driver;
>  
> +	ret = etnaviv_register_pci_driver();
> +	if (ret != 0)
> +		goto unregister_platform_driver;
> +
>  	/*
>  	 * If the DT contains at least one available GPU device, instantiate
>  	 * the DRM platform device.
> @@ -763,7 +768,7 @@ static int __init etnaviv_init(void)
>  		break;
>  	}
>  
> -	return 0;
> +	return ret;
>  
>  unregister_platform_driver:
>  	platform_driver_unregister(&etnaviv_platform_driver);
> @@ -778,6 +783,8 @@ static void __exit etnaviv_exit(void)
>  	etnaviv_destroy_platform_device(&etnaviv_platform_device);
>  	platform_driver_unregister(&etnaviv_platform_driver);
>  	platform_driver_unregister(&etnaviv_gpu_driver);
> +
> +	etnaviv_unregister_pci_driver();
>  }
>  module_exit(etnaviv_exit);
>  
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> index e58f82e698de..9cd72948cfad 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> @@ -83,6 +83,9 @@ bool etnaviv_cmd_validate_one(struct etnaviv_gpu *gpu,
>  	u32 *stream, unsigned int size,
>  	struct drm_etnaviv_gem_submit_reloc *relocs, unsigned int reloc_size);
>  
> +int etnaviv_drm_bind(struct device *dev, bool component);
> +void etnaviv_drm_unbind(struct device *dev, bool component);
> +
>  #ifdef CONFIG_DEBUG_FS
>  void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv,
>  	struct seq_file *m);
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> index 059be8c89c5a..d6a21e97feb1 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> @@ -1866,8 +1866,8 @@ static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
>  
>  /* platform independent */
>  
> -static int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
> -				     int irq, bool component, bool has_clk)
> +int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
> +			      int irq, bool component, bool has_clk)
>  {
>  	struct etnaviv_gpu *gpu;
>  	int err;
> @@ -1916,7 +1916,7 @@ static int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
>  	return 0;
>  }
>  
> -static void etnaviv_gpu_driver_destroy(struct device *dev, bool component)
> +void etnaviv_gpu_driver_destroy(struct device *dev, bool component)
>  {
>  	if (component)
>  		component_del(dev, &gpu_ops);
> @@ -1967,7 +1967,7 @@ static int etnaviv_gpu_rpm_resume(struct device *dev)
>  	return 0;
>  }
>  
> -static const struct dev_pm_ops etnaviv_gpu_pm_ops = {
> +const struct dev_pm_ops etnaviv_gpu_pm_ops = {
>  	RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume, NULL)
>  };
>  
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> index 1ec829a649b5..8d9833996ed7 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
> @@ -209,6 +209,12 @@ void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch);
>  int etnaviv_gpu_bind(struct device *dev, struct device *master, void *data);
>  void etnaviv_gpu_unbind(struct device *dev, struct device *master, void *data);
>  
> +int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
> +			      int irq, bool component, bool has_clk);
> +
> +void etnaviv_gpu_driver_destroy(struct device *dev, bool component);
> +
>  extern struct platform_driver etnaviv_gpu_driver;
> +extern const struct dev_pm_ops etnaviv_gpu_pm_ops;
>  
>  #endif /* __ETNAVIV_GPU_H__ */
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
> new file mode 100644
> index 000000000000..78e44a28d30c
> --- /dev/null
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
> @@ -0,0 +1,75 @@
> +// SPDX-License-Identifier: GPL-2.0
> +
> +#include <linux/pci.h>
> +
> +#include "etnaviv_drv.h"
> +#include "etnaviv_gpu.h"
> +#include "etnaviv_pci_drv.h"
> +
> +static int etnaviv_pci_probe(struct pci_dev *pdev,
> +			     const struct pci_device_id *ent)
> +{
> +	struct device *dev = &pdev->dev;
> +	void __iomem *mmio;
> +	int ret;
> +
> +	ret = pcim_enable_device(pdev);
> +	if (ret) {
> +		dev_err(dev, "failed to enable\n");
> +		return ret;
> +	}
> +
> +	pci_set_master(pdev);
> +
> +	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));

Is your device really restricted to 32 bit dma addressing?

> +	if (ret)
> +		return ret;
> +
> +	/* Map registers, assume the PCI bar 0 contain the registers */
> +	mmio = pcim_iomap(pdev, 0, 0);
> +	if (IS_ERR(mmio))
> +		return PTR_ERR(mmio);
> +
> +	ret = etnaviv_gpu_driver_create(dev, mmio, pdev->irq, false, false);
> +	if (ret)
> +		return ret;
> +
> +	return etnaviv_drm_bind(dev, false);
> +}
> +
> +static void etnaviv_pci_remove(struct pci_dev *pdev)
> +{
> +	struct device *dev = &pdev->dev;
> +
> +	etnaviv_drm_unbind(dev, false);
> +
> +	etnaviv_gpu_driver_destroy(dev, false);
> +
> +	pci_clear_master(pdev);
> +}
> +
> +static const struct pci_device_id etnaviv_pci_id_lists[] = {
> +	{PCI_VDEVICE(LOONGSON, 0x7a15)},
> +	{PCI_VDEVICE(LOONGSON, 0x7a05)},
> +	{ }
> +};
> +
> +static struct pci_driver etnaviv_pci_driver = {
> +	.name = "etnaviv",
> +	.id_table = etnaviv_pci_id_lists,
> +	.probe = etnaviv_pci_probe,
> +	.remove = etnaviv_pci_remove,
> +	.driver.pm = pm_ptr(&etnaviv_gpu_pm_ops),
> +};
> +
> +int etnaviv_register_pci_driver(void)
> +{
> +	return pci_register_driver(&etnaviv_pci_driver);
> +}
> +
> +void etnaviv_unregister_pci_driver(void)
> +{
> +	pci_unregister_driver(&etnaviv_pci_driver);
> +}
> +
> +MODULE_DEVICE_TABLE(pci, etnaviv_pci_id_lists);

Hm, I think the amount of code added here is small enough to go
directly into etnaviv_drv.c behind a
#ifdef CONFIG_DRM_ETNAVIV_PCI_DRIVER, which would allow to keep the
referenced functions and pm ops static.

Regards,
Lucas

> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h
> new file mode 100644
> index 000000000000..1db559ee5e9b
> --- /dev/null
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h
> @@ -0,0 +1,18 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +
> +#ifndef __ETNAVIV_PCI_DRV_H__
> +#define __ETNAVIV_PCI_DRV_H__
> +
> +#ifdef CONFIG_DRM_ETNAVIV_PCI_DRIVER
> +
> +int etnaviv_register_pci_driver(void);
> +void etnaviv_unregister_pci_driver(void);
> +
> +#else
> +
> +static inline int etnaviv_register_pci_driver(void) { return 0; }
> +static inline void etnaviv_unregister_pci_driver(void) { }
> +
> +#endif
> +
> +#endif

Re: [PATCH v10 06/11] drm/etnaviv: Add driver support for the PCI devices

[Sui Jingfeng]

Hi,

On 2023/6/21 17:39, Lucas Stach wrote:
> Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>
>> This patch adds PCI driver support on top of what we already have, take
>> the GC1000 in LS7A1000/LS2K1000 as the first instance which enjoy the PCI
>> device driver. There is only one GPU core for the GC1000 in the LS7A1000
>> and LS2K1000. Therefore, component frameworks can be avoided.
>>
>> Cc: Lucas Stach <l.stach@pengutronix.de>
>> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
>> Cc: Philipp Zabel <p.zabel@pengutronix.de>
>> Cc: Bjorn Helgaas <bhelgaas@google.com>
>> Cc: Daniel Vetter <daniel@ffwll.ch>
>> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
>> ---
>>   drivers/gpu/drm/etnaviv/Kconfig           | 10 +++
>>   drivers/gpu/drm/etnaviv/Makefile          |  2 +
>>   drivers/gpu/drm/etnaviv/etnaviv_drv.c     | 13 +++-
>>   drivers/gpu/drm/etnaviv/etnaviv_drv.h     |  3 +
>>   drivers/gpu/drm/etnaviv/etnaviv_gpu.c     |  8 +--
>>   drivers/gpu/drm/etnaviv/etnaviv_gpu.h     |  6 ++
>>   drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c | 75 +++++++++++++++++++++++
>>   drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h | 18 ++++++
>>   8 files changed, 128 insertions(+), 7 deletions(-)
>>   create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
>>   create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h
>>
>> diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig
>> index faa7fc68b009..1b5b162efb61 100644
>> --- a/drivers/gpu/drm/etnaviv/Kconfig
>> +++ b/drivers/gpu/drm/etnaviv/Kconfig
>> @@ -15,6 +15,16 @@ config DRM_ETNAVIV
>>   	help
>>   	  DRM driver for Vivante GPUs.
>>   
>> +config DRM_ETNAVIV_PCI_DRIVER
>> +	bool "enable ETNAVIV PCI driver support"
>> +	depends on DRM_ETNAVIV
>> +	depends on PCI
>> +	default y
> Drop the default y. Lots of existing systems with Vivante GPUs on a
> platform bus also have a PCI bus, so this will activate the added code
> paths in a lot of configurations that don't need it.
>
This is OK,
>> +	help
>> +	  Compile in support for PCI GPUs of Vivante.
> Compile in support for Vivante GPUs attached via PCI(e).
OK, acceptable.
>
>> +	  For example, the GC1000 in LS7A1000 and LS2K1000.
> Not sure if the example is that helpful. Maybe drop this.
OK.
>
>> +	  Say Y if you have such a hardware.
>> +
>>   config DRM_ETNAVIV_THERMAL
>>   	bool "enable ETNAVIV thermal throttling"
>>   	depends on DRM_ETNAVIV
>> diff --git a/drivers/gpu/drm/etnaviv/Makefile b/drivers/gpu/drm/etnaviv/Makefile
>> index 46e5ffad69a6..6829e1ebf2db 100644
>> --- a/drivers/gpu/drm/etnaviv/Makefile
>> +++ b/drivers/gpu/drm/etnaviv/Makefile
>> @@ -16,4 +16,6 @@ etnaviv-y := \
>>   	etnaviv_perfmon.o \
>>   	etnaviv_sched.o
>>   
>> +etnaviv-$(CONFIG_DRM_ETNAVIV_PCI_DRIVER) += etnaviv_pci_drv.o
>> +
>>   obj-$(CONFIG_DRM_ETNAVIV)	+= etnaviv.o
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> index 93ca240cd4c0..0a365e96d371 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> @@ -21,6 +21,7 @@
>>   #include "etnaviv_gpu.h"
>>   #include "etnaviv_gem.h"
>>   #include "etnaviv_mmu.h"
>> +#include "etnaviv_pci_drv.h"
>>   #include "etnaviv_perfmon.h"
>>   
>>   /*
>> @@ -538,7 +539,7 @@ static const struct drm_driver etnaviv_drm_driver = {
>>   
>>   static struct etnaviv_drm_private *etna_private_ptr;
>>   
>> -static int etnaviv_drm_bind(struct device *dev, bool component)
>> +int etnaviv_drm_bind(struct device *dev, bool component)
>>   {
>>   	struct etnaviv_drm_private *priv;
>>   	struct drm_device *drm;
>> @@ -588,7 +589,7 @@ static int etnaviv_drm_bind(struct device *dev, bool component)
>>   	return ret;
>>   }
>>   
>> -static void etnaviv_drm_unbind(struct device *dev, bool component)
>> +void etnaviv_drm_unbind(struct device *dev, bool component)
>>   {
>>   	struct etnaviv_drm_private *priv = etna_private_ptr;
>>   	struct drm_device *drm = priv->drm;
>> @@ -746,6 +747,10 @@ static int __init etnaviv_init(void)
>>   	if (ret != 0)
>>   		goto unregister_gpu_driver;
>>   
>> +	ret = etnaviv_register_pci_driver();
>> +	if (ret != 0)
>> +		goto unregister_platform_driver;
>> +
>>   	/*
>>   	 * If the DT contains at least one available GPU device, instantiate
>>   	 * the DRM platform device.
>> @@ -763,7 +768,7 @@ static int __init etnaviv_init(void)
>>   		break;
>>   	}
>>   
>> -	return 0;
>> +	return ret;
>>   
>>   unregister_platform_driver:
>>   	platform_driver_unregister(&etnaviv_platform_driver);
>> @@ -778,6 +783,8 @@ static void __exit etnaviv_exit(void)
>>   	etnaviv_destroy_platform_device(&etnaviv_platform_device);
>>   	platform_driver_unregister(&etnaviv_platform_driver);
>>   	platform_driver_unregister(&etnaviv_gpu_driver);
>> +
>> +	etnaviv_unregister_pci_driver();
>>   }
>>   module_exit(etnaviv_exit);
>>   
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> index e58f82e698de..9cd72948cfad 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> @@ -83,6 +83,9 @@ bool etnaviv_cmd_validate_one(struct etnaviv_gpu *gpu,
>>   	u32 *stream, unsigned int size,
>>   	struct drm_etnaviv_gem_submit_reloc *relocs, unsigned int reloc_size);
>>   
>> +int etnaviv_drm_bind(struct device *dev, bool component);
>> +void etnaviv_drm_unbind(struct device *dev, bool component);
>> +
>>   #ifdef CONFIG_DEBUG_FS
>>   void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv,
>>   	struct seq_file *m);
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> index 059be8c89c5a..d6a21e97feb1 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> @@ -1866,8 +1866,8 @@ static int etnaviv_gpu_register_irq(struct etnaviv_gpu *gpu, int irq)
>>   
>>   /* platform independent */
>>   
>> -static int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
>> -				     int irq, bool component, bool has_clk)
>> +int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
>> +			      int irq, bool component, bool has_clk)
>>   {
>>   	struct etnaviv_gpu *gpu;
>>   	int err;
>> @@ -1916,7 +1916,7 @@ static int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
>>   	return 0;
>>   }
>>   
>> -static void etnaviv_gpu_driver_destroy(struct device *dev, bool component)
>> +void etnaviv_gpu_driver_destroy(struct device *dev, bool component)
>>   {
>>   	if (component)
>>   		component_del(dev, &gpu_ops);
>> @@ -1967,7 +1967,7 @@ static int etnaviv_gpu_rpm_resume(struct device *dev)
>>   	return 0;
>>   }
>>   
>> -static const struct dev_pm_ops etnaviv_gpu_pm_ops = {
>> +const struct dev_pm_ops etnaviv_gpu_pm_ops = {
>>   	RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume, NULL)
>>   };
>>   
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
>> index 1ec829a649b5..8d9833996ed7 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
>> @@ -209,6 +209,12 @@ void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch);
>>   int etnaviv_gpu_bind(struct device *dev, struct device *master, void *data);
>>   void etnaviv_gpu_unbind(struct device *dev, struct device *master, void *data);
>>   
>> +int etnaviv_gpu_driver_create(struct device *dev, void __iomem *mmio,
>> +			      int irq, bool component, bool has_clk);
>> +
>> +void etnaviv_gpu_driver_destroy(struct device *dev, bool component);
>> +
>>   extern struct platform_driver etnaviv_gpu_driver;
>> +extern const struct dev_pm_ops etnaviv_gpu_pm_ops;
>>   
>>   #endif /* __ETNAVIV_GPU_H__ */
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
>> new file mode 100644
>> index 000000000000..78e44a28d30c
>> --- /dev/null
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
>> @@ -0,0 +1,75 @@
>> +// SPDX-License-Identifier: GPL-2.0
>> +
>> +#include <linux/pci.h>
>> +
>> +#include "etnaviv_drv.h"
>> +#include "etnaviv_gpu.h"
>> +#include "etnaviv_pci_drv.h"
>> +
>> +static int etnaviv_pci_probe(struct pci_dev *pdev,
>> +			     const struct pci_device_id *ent)
>> +{
>> +	struct device *dev = &pdev->dev;
>> +	void __iomem *mmio;
>> +	int ret;
>> +
>> +	ret = pcim_enable_device(pdev);
>> +	if (ret) {
>> +		dev_err(dev, "failed to enable\n");
>> +		return ret;
>> +	}
>> +
>> +	pci_set_master(pdev);
>> +
>> +	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
> Is your device really restricted to 32 bit dma addressing?
Yes, if I don't add this, bugs may happens.
>> +	if (ret)
>> +		return ret;
>> +
>> +	/* Map registers, assume the PCI bar 0 contain the registers */
>> +	mmio = pcim_iomap(pdev, 0, 0);
>> +	if (IS_ERR(mmio))
>> +		return PTR_ERR(mmio);
>> +
>> +	ret = etnaviv_gpu_driver_create(dev, mmio, pdev->irq, false, false);
>> +	if (ret)
>> +		return ret;
>> +
>> +	return etnaviv_drm_bind(dev, false);
>> +}
>> +
>> +static void etnaviv_pci_remove(struct pci_dev *pdev)
>> +{
>> +	struct device *dev = &pdev->dev;
>> +
>> +	etnaviv_drm_unbind(dev, false);
>> +
>> +	etnaviv_gpu_driver_destroy(dev, false);
>> +
>> +	pci_clear_master(pdev);
>> +}
>> +
>> +static const struct pci_device_id etnaviv_pci_id_lists[] = {
>> +	{PCI_VDEVICE(LOONGSON, 0x7a15)},
>> +	{PCI_VDEVICE(LOONGSON, 0x7a05)},
>> +	{ }
>> +};
>> +
>> +static struct pci_driver etnaviv_pci_driver = {
>> +	.name = "etnaviv",
>> +	.id_table = etnaviv_pci_id_lists,
>> +	.probe = etnaviv_pci_probe,
>> +	.remove = etnaviv_pci_remove,
>> +	.driver.pm = pm_ptr(&etnaviv_gpu_pm_ops),
>> +};
>> +
>> +int etnaviv_register_pci_driver(void)
>> +{
>> +	return pci_register_driver(&etnaviv_pci_driver);
>> +}
>> +
>> +void etnaviv_unregister_pci_driver(void)
>> +{
>> +	pci_unregister_driver(&etnaviv_pci_driver);
>> +}
>> +
>> +MODULE_DEVICE_TABLE(pci, etnaviv_pci_id_lists);
> Hm, I think the amount of code added here is small enough to go
> directly into etnaviv_drv.c behind a
> #ifdef CONFIG_DRM_ETNAVIV_PCI_DRIVER, which would allow to keep the
> referenced functions and pm ops static.
OK, this is acceptable.
> Regards,
> Lucas
>
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h
>> new file mode 100644
>> index 000000000000..1db559ee5e9b
>> --- /dev/null
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h
>> @@ -0,0 +1,18 @@
>> +/* SPDX-License-Identifier: GPL-2.0 */
>> +
>> +#ifndef __ETNAVIV_PCI_DRV_H__
>> +#define __ETNAVIV_PCI_DRV_H__
>> +
>> +#ifdef CONFIG_DRM_ETNAVIV_PCI_DRIVER
>> +
>> +int etnaviv_register_pci_driver(void);
>> +void etnaviv_unregister_pci_driver(void);
>> +
>> +#else
>> +
>> +static inline int etnaviv_register_pci_driver(void) { return 0; }
>> +static inline void etnaviv_unregister_pci_driver(void) { }
>> +
>> +#endif
>> +
>> +#endif

-- 
Jingfeng

[PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

Loongson CPUs maintain cache coherency by hardware, which means that the
data in the CPU cache is identical to the data in main system memory. As
for the peripheral device, most of Loongson chips chose to define the
peripherals as DMA coherent by default, device drivers do not need to
maintain the coherency between a processor and an I/O device manually.

There are exceptions, for LS2K1000 SoC, part of peripheral device can be
configured as DMA non-coherent. But there is no released version of such
firmware exist in the market. Peripherals of older LS2K1000 is also DMA
non-coherent, but they are nearly outdated. So, those are trivial cases.

Nevertheless, kernel space still need to do the probe work, because vivante
GPU IP has been integrated into various platform. Hence, this patch add
runtime detection code to probe if a specific GPU is DMA coherent, If the
answer is yes, we are going to utilize such features. On Loongson platform,
When a buffer is accessed by both the GPU and the CPU, the driver should
prefer ETNA_BO_CACHED over ETNA_BO_WC.

This patch also add a new parameter: etnaviv_param_gpu_coherent, which
allow userspace to know if such a feature is available. Because
write-combined BO is still preferred in some case, especially where don't
need CPU read, for example, uploading compiled shader bin.

Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_drv.c       | 35 +++++++++++++++++++++
 drivers/gpu/drm/etnaviv/etnaviv_drv.h       |  6 ++++
 drivers/gpu/drm/etnaviv/etnaviv_gem.c       | 22 ++++++++++---
 drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c |  7 ++++-
 drivers/gpu/drm/etnaviv/etnaviv_gpu.c       |  4 +++
 include/uapi/drm/etnaviv_drm.h              |  1 +
 6 files changed, 70 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 0a365e96d371..d8e788aa16cb 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -5,7 +5,9 @@
 
 #include <linux/component.h>
 #include <linux/dma-mapping.h>
+#include <linux/dma-map-ops.h>
 #include <linux/module.h>
+#include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/uaccess.h>
 
@@ -24,6 +26,34 @@
 #include "etnaviv_pci_drv.h"
 #include "etnaviv_perfmon.h"
 
+static struct device_node *etnaviv_of_first_available_node(void)
+{
+	struct device_node *core_node;
+
+	for_each_compatible_node(core_node, NULL, "vivante,gc") {
+		if (of_device_is_available(core_node))
+			return core_node;
+	}
+
+	return NULL;
+}
+
+static bool etnaviv_is_dma_coherent(struct device *dev)
+{
+	struct device_node *np;
+	bool coherent;
+
+	np = etnaviv_of_first_available_node();
+	if (np) {
+		coherent = of_dma_is_coherent(np);
+		of_node_put(np);
+	} else {
+		coherent = dev_is_dma_coherent(dev);
+	}
+
+	return coherent;
+}
+
 /*
  * etnaviv private data construction and destructions:
  */
@@ -52,6 +82,11 @@ etnaviv_alloc_private(struct device *dev, struct drm_device *drm)
 		return ERR_PTR(-ENOMEM);
 	}
 
+	priv->dma_coherent = etnaviv_is_dma_coherent(dev);
+
+	if (priv->dma_coherent)
+		drm_info(drm, "%s is dma coherent\n", dev_name(dev));
+
 	return priv;
 }
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index 9cd72948cfad..644e5712c050 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -46,6 +46,12 @@ struct etnaviv_drm_private {
 	struct xarray active_contexts;
 	u32 next_context_id;
 
+	/*
+	 * If true, the GPU is capable of snooping cpu cache. Here, it
+	 * also means that cache coherency is enforced by the hardware.
+	 */
+	bool dma_coherent;
+
 	/* list of GEM objects: */
 	struct mutex gem_lock;
 	struct list_head gem_list;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index b5f73502e3dd..39bdc3774f2d 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -343,6 +343,7 @@ void *etnaviv_gem_vmap(struct drm_gem_object *obj)
 static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
 {
 	struct page **pages;
+	pgprot_t prot;
 
 	lockdep_assert_held(&obj->lock);
 
@@ -350,8 +351,19 @@ static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
 	if (IS_ERR(pages))
 		return NULL;
 
-	return vmap(pages, obj->base.size >> PAGE_SHIFT,
-			VM_MAP, pgprot_writecombine(PAGE_KERNEL));
+	switch (obj->flags) {
+	case ETNA_BO_CACHED:
+		prot = PAGE_KERNEL;
+		break;
+	case ETNA_BO_UNCACHED:
+		prot = pgprot_noncached(PAGE_KERNEL);
+		break;
+	case ETNA_BO_WC:
+	default:
+		prot = pgprot_writecombine(PAGE_KERNEL);
+	}
+
+	return vmap(pages, obj->base.size >> PAGE_SHIFT, VM_MAP, prot);
 }
 
 static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
@@ -369,6 +381,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
 {
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
 	struct drm_device *dev = obj->dev;
+	struct etnaviv_drm_private *priv = dev->dev_private;
 	bool write = !!(op & ETNA_PREP_WRITE);
 	int ret;
 
@@ -395,7 +408,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
 			return ret == 0 ? -ETIMEDOUT : ret;
 	}
 
-	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
+	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
 		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
 					 etnaviv_op_to_dma_dir(op));
 		etnaviv_obj->last_cpu_prep_op = op;
@@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
 {
 	struct drm_device *dev = obj->dev;
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	struct etnaviv_drm_private *priv = dev->dev_private;
 
-	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
+	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
 		/* fini without a prep is almost certainly a userspace error */
 		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
 		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
index 3524b5811682..754126992264 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
@@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
 struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
 	struct dma_buf_attachment *attach, struct sg_table *sgt)
 {
+	struct etnaviv_drm_private *priv = dev->dev_private;
 	struct etnaviv_gem_object *etnaviv_obj;
 	size_t size = PAGE_ALIGN(attach->dmabuf->size);
+	u32 cache_flags = ETNA_BO_WC;
 	int ret, npages;
 
-	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
+	if (priv->dma_coherent)
+		cache_flags = ETNA_BO_CACHED;
+
+	ret = etnaviv_gem_new_private(dev, size, cache_flags,
 				      &etnaviv_gem_prime_ops, &etnaviv_obj);
 	if (ret < 0)
 		return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index d6a21e97feb1..d99ac675ce8b 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -164,6 +164,10 @@ int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value)
 		*value = gpu->identity.eco_id;
 		break;
 
+	case ETNAVIV_PARAM_GPU_COHERENT:
+		*value = priv->dma_coherent;
+		break;
+
 	default:
 		DBG("%s: invalid param: %u", dev_name(gpu->dev), param);
 		return -EINVAL;
diff --git a/include/uapi/drm/etnaviv_drm.h b/include/uapi/drm/etnaviv_drm.h
index af024d90453d..76baf45d7158 100644
--- a/include/uapi/drm/etnaviv_drm.h
+++ b/include/uapi/drm/etnaviv_drm.h
@@ -77,6 +77,7 @@ struct drm_etnaviv_timespec {
 #define ETNAVIV_PARAM_GPU_PRODUCT_ID                0x1c
 #define ETNAVIV_PARAM_GPU_CUSTOMER_ID               0x1d
 #define ETNAVIV_PARAM_GPU_ECO_ID                    0x1e
+#define ETNAVIV_PARAM_GPU_COHERENT                  0x1f
 
 #define ETNA_MAX_PIPES 4
 
-- 
2.25.1

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Lucas Stach]

Am Dienstag, dem 20.06.2023 um 17:47 +0800 schrieb Sui Jingfeng:
> From: Sui Jingfeng <suijingfeng@loongson.cn>
> 
> Loongson CPUs maintain cache coherency by hardware, which means that the
> data in the CPU cache is identical to the data in main system memory. As
> for the peripheral device, most of Loongson chips chose to define the
> peripherals as DMA coherent by default, device drivers do not need to
> maintain the coherency between a processor and an I/O device manually.
> 
> There are exceptions, for LS2K1000 SoC, part of peripheral device can be
> configured as DMA non-coherent. But there is no released version of such
> firmware exist in the market. Peripherals of older LS2K1000 is also DMA
> non-coherent, but they are nearly outdated. So, those are trivial cases.
> 
> Nevertheless, kernel space still need to do the probe work, because vivante
> GPU IP has been integrated into various platform. Hence, this patch add
> runtime detection code to probe if a specific GPU is DMA coherent, If the
> answer is yes, we are going to utilize such features. On Loongson platform,
> When a buffer is accessed by both the GPU and the CPU, the driver should
> prefer ETNA_BO_CACHED over ETNA_BO_WC.
> 
> This patch also add a new parameter: etnaviv_param_gpu_coherent, which
> allow userspace to know if such a feature is available. Because
> write-combined BO is still preferred in some case, especially where don't
> need CPU read, for example, uploading compiled shader bin.
> 
> Cc: Lucas Stach <l.stach@pengutronix.de>
> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> Cc: Philipp Zabel <p.zabel@pengutronix.de>
> Cc: Bjorn Helgaas <bhelgaas@google.com>
> Cc: Daniel Vetter <daniel@ffwll.ch>
> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> ---
>  drivers/gpu/drm/etnaviv/etnaviv_drv.c       | 35 +++++++++++++++++++++
>  drivers/gpu/drm/etnaviv/etnaviv_drv.h       |  6 ++++
>  drivers/gpu/drm/etnaviv/etnaviv_gem.c       | 22 ++++++++++---
>  drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c |  7 ++++-
>  drivers/gpu/drm/etnaviv/etnaviv_gpu.c       |  4 +++
>  include/uapi/drm/etnaviv_drm.h              |  1 +
>  6 files changed, 70 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> index 0a365e96d371..d8e788aa16cb 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> @@ -5,7 +5,9 @@
>  
>  #include <linux/component.h>
>  #include <linux/dma-mapping.h>
> +#include <linux/dma-map-ops.h>
>  #include <linux/module.h>
> +#include <linux/of_address.h>
>  #include <linux/of_platform.h>
>  #include <linux/uaccess.h>
>  
> @@ -24,6 +26,34 @@
>  #include "etnaviv_pci_drv.h"
>  #include "etnaviv_perfmon.h"
>  
> +static struct device_node *etnaviv_of_first_available_node(void)
> +{
> +	struct device_node *core_node;
> +
> +	for_each_compatible_node(core_node, NULL, "vivante,gc") {
> +		if (of_device_is_available(core_node))
> +			return core_node;
> +	}
> +
> +	return NULL;
> +}
> +
> +static bool etnaviv_is_dma_coherent(struct device *dev)
> +{
> +	struct device_node *np;
> +	bool coherent;
> +
> +	np = etnaviv_of_first_available_node();
> +	if (np) {
> +		coherent = of_dma_is_coherent(np);
> +		of_node_put(np);
> +	} else {
> +		coherent = dev_is_dma_coherent(dev);
> +	}

This whole dance shouldn't be needed. We transfer the DMA capabilities
from the first node to the virtual master device in the platform device
case, so dev_is_dma_coherent(dev) should always return the right thing.

> +
> +	return coherent;
> +}
> +
>  /*
>   * etnaviv private data construction and destructions:
>   */
> @@ -52,6 +82,11 @@ etnaviv_alloc_private(struct device *dev, struct drm_device *drm)
>  		return ERR_PTR(-ENOMEM);
>  	}
>  
> +	priv->dma_coherent = etnaviv_is_dma_coherent(dev);
> +
> +	if (priv->dma_coherent)
> +		drm_info(drm, "%s is dma coherent\n", dev_name(dev));
> +
>  	return priv;
>  }
>  
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> index 9cd72948cfad..644e5712c050 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> @@ -46,6 +46,12 @@ struct etnaviv_drm_private {
>  	struct xarray active_contexts;
>  	u32 next_context_id;
>  
> +	/*
> +	 * If true, the GPU is capable of snooping cpu cache. Here, it
> +	 * also means that cache coherency is enforced by the hardware.
> +	 */
> +	bool dma_coherent;
> +
No need for this, I think. Just use dev_is_dma_coherent() where you
need to know this.

>  	/* list of GEM objects: */
>  	struct mutex gem_lock;
>  	struct list_head gem_list;
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
> index b5f73502e3dd..39bdc3774f2d 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
> @@ -343,6 +343,7 @@ void *etnaviv_gem_vmap(struct drm_gem_object *obj)
>  static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
>  {
>  	struct page **pages;
> +	pgprot_t prot;
>  
>  	lockdep_assert_held(&obj->lock);
>  
> @@ -350,8 +351,19 @@ static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
>  	if (IS_ERR(pages))
>  		return NULL;
>  
> -	return vmap(pages, obj->base.size >> PAGE_SHIFT,
> -			VM_MAP, pgprot_writecombine(PAGE_KERNEL));
> +	switch (obj->flags) {

switch (obj->flags & ETNA_BO_CACHE_MASK)

> +	case ETNA_BO_CACHED:
> +		prot = PAGE_KERNEL;
> +		break;
> +	case ETNA_BO_UNCACHED:
> +		prot = pgprot_noncached(PAGE_KERNEL);
> +		break;
> +	case ETNA_BO_WC:
> +	default:
> +		prot = pgprot_writecombine(PAGE_KERNEL);
> +	}
> +
> +	return vmap(pages, obj->base.size >> PAGE_SHIFT, VM_MAP, prot);

While that change makes sense it should also be in a separate patch, as
it's a valid change on its own, even if for non-coherent devices.

>  }
>  
>  static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
> @@ -369,6 +381,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
>  {
>  	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>  	struct drm_device *dev = obj->dev;
> +	struct etnaviv_drm_private *priv = dev->dev_private;
>  	bool write = !!(op & ETNA_PREP_WRITE);
>  	int ret;
>  
> @@ -395,7 +408,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
>  			return ret == 0 ? -ETIMEDOUT : ret;
>  	}
>  
> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {

Why do you need this? Isn't dma_sync_sgtable_for_cpu a no-op on your
platform when the device is coherent?

>  		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
>  					 etnaviv_op_to_dma_dir(op));
>  		etnaviv_obj->last_cpu_prep_op = op;
> @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
>  {
>  	struct drm_device *dev = obj->dev;
>  	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
> +	struct etnaviv_drm_private *priv = dev->dev_private;
>  
> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>  		/* fini without a prep is almost certainly a userspace error */
>  		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
>  		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> index 3524b5811682..754126992264 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
>  struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
>  	struct dma_buf_attachment *attach, struct sg_table *sgt)
>  {
> +	struct etnaviv_drm_private *priv = dev->dev_private;
>  	struct etnaviv_gem_object *etnaviv_obj;
>  	size_t size = PAGE_ALIGN(attach->dmabuf->size);
> +	u32 cache_flags = ETNA_BO_WC;
>  	int ret, npages;
>  
> -	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
> +	if (priv->dma_coherent)
> +		cache_flags = ETNA_BO_CACHED;
> +
Drop this change. Instead etnaviv_gem_new_impl() should do the upgrade
from WC to CACHED as necessary by adding something like this:

/*
 * Upgrade WC to CACHED when the device is hardware coherent and the
 * platform doesn't allow mixing cached and writecombined mappings to
 * the same memory area.
 */
if ((flags & ETNA_BO_CACHE_MASK) == ETNA_BO_WC &&
    dev_is_dma_coherent(dev) && !drm_arch_can_wc_memory())
        flags = (flags & ~ETNA_BO_CACHE_MASK) & ETNA_BO_CACHED;

Regards,
Lucas

> +	ret = etnaviv_gem_new_private(dev, size, cache_flags,
>  				      &etnaviv_gem_prime_ops, &etnaviv_obj);
>  	if (ret < 0)
>  		return ERR_PTR(ret);
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> index d6a21e97feb1..d99ac675ce8b 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> @@ -164,6 +164,10 @@ int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value)
>  		*value = gpu->identity.eco_id;
>  		break;
>  
> +	case ETNAVIV_PARAM_GPU_COHERENT:
> +		*value = priv->dma_coherent;
> +		break;
> +
>  	default:
>  		DBG("%s: invalid param: %u", dev_name(gpu->dev), param);
>  		return -EINVAL;
> diff --git a/include/uapi/drm/etnaviv_drm.h b/include/uapi/drm/etnaviv_drm.h
> index af024d90453d..76baf45d7158 100644
> --- a/include/uapi/drm/etnaviv_drm.h
> +++ b/include/uapi/drm/etnaviv_drm.h
> @@ -77,6 +77,7 @@ struct drm_etnaviv_timespec {
>  #define ETNAVIV_PARAM_GPU_PRODUCT_ID                0x1c
>  #define ETNAVIV_PARAM_GPU_CUSTOMER_ID               0x1d
>  #define ETNAVIV_PARAM_GPU_ECO_ID                    0x1e
> +#define ETNAVIV_PARAM_GPU_COHERENT                  0x1f
>  
>  #define ETNA_MAX_PIPES 4
>  

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

On 2023/6/21 18:00, Lucas Stach wrote:
>> +static bool etnaviv_is_dma_coherent(struct device *dev)
>> +{
>> +	struct device_node *np;
>> +	bool coherent;
>> +
>> +	np = etnaviv_of_first_available_node();
>> +	if (np) {
>> +		coherent = of_dma_is_coherent(np);
>> +		of_node_put(np);
>> +	} else {
>> +		coherent = dev_is_dma_coherent(dev);
>> +	}
> This whole dance shouldn't be needed. We transfer the DMA capabilities
> from the first node to the virtual master device in the platform device
> case, so dev_is_dma_coherent(dev) should always return the right thing.
>
OK, I'm fine if this is OK on your platform.
>> +
>> +	return coherent;
>> +}
>> +

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/21 18:00, Lucas Stach wrote:
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> index 9cd72948cfad..644e5712c050 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> @@ -46,6 +46,12 @@ struct etnaviv_drm_private {
>>   	struct xarray active_contexts;
>>   	u32 next_context_id;
>>   
>> +	/*
>> +	 * If true, the GPU is capable of snooping cpu cache. Here, it
>> +	 * also means that cache coherency is enforced by the hardware.
>> +	 */
>> +	bool dma_coherent;
>> +
> No need for this, I think. Just use dev_is_dma_coherent() where you
> need to know this.
>
No, we want this value cached by the driver.

We only need call  dev_is_dma_coherent() once!

We need to reuse this variable on other places.

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Lucas Stach]

Am Mittwoch, dem 21.06.2023 um 22:44 +0800 schrieb Sui Jingfeng:
> Hi,
> 
> On 2023/6/21 18:00, Lucas Stach wrote:
> > > diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> > > index 9cd72948cfad..644e5712c050 100644
> > > --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> > > +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> > > @@ -46,6 +46,12 @@ struct etnaviv_drm_private {
> > >   	struct xarray active_contexts;
> > >   	u32 next_context_id;
> > >   
> > > +	/*
> > > +	 * If true, the GPU is capable of snooping cpu cache. Here, it
> > > +	 * also means that cache coherency is enforced by the hardware.
> > > +	 */
> > > +	bool dma_coherent;
> > > +
> > No need for this, I think. Just use dev_is_dma_coherent() where you
> > need to know this.
> > 
> No, we want this value cached by the driver.
> 
Why? dev_is_dma_coherent() is a header-only function with a single
pointer chasing operation. Your cache is also a single pointer chasing
access, just that we now need storage for this information in both
struct device and struct etnaviv_gpu.

Regards,
Lucas

> We only need call  dev_is_dma_coherent() once!
> 
> We need to reuse this variable on other places.
> 

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

On 2023/6/21 23:23, Lucas Stach wrote:
> Am Mittwoch, dem 21.06.2023 um 22:44 +0800 schrieb Sui Jingfeng:
>> Hi,
>>
>> On 2023/6/21 18:00, Lucas Stach wrote:
>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>>>> index 9cd72948cfad..644e5712c050 100644
>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>>>> @@ -46,6 +46,12 @@ struct etnaviv_drm_private {
>>>>    	struct xarray active_contexts;
>>>>    	u32 next_context_id;
>>>>    
>>>> +	/*
>>>> +	 * If true, the GPU is capable of snooping cpu cache. Here, it
>>>> +	 * also means that cache coherency is enforced by the hardware.
>>>> +	 */
>>>> +	bool dma_coherent;
>>>> +
>>> No need for this, I think. Just use dev_is_dma_coherent() where you
>>> need to know this.
>>>
>> No, we want this value cached by the driver.
>>
> Why? dev_is_dma_coherent() is a header-only function with a single
> pointer chasing operation. Your cache is also a single pointer chasing
> access, just that we now need storage for this information in both
> struct device and struct etnaviv_gpu.


You don't need store it in struct etnaviv_gpu.

As this variable is shared across the device, so it is better to be put 
in the struct etnaviv_drm_private.

I don't think another 4 bytes allocation is something what we can't pay for.


My patch doesn't mentioned that it need to store it inside of struct 
etnaviv_gpu, do I?

> Regards,
> Lucas
>
>> We only need call  dev_is_dma_coherent() once!
>>
>> We need to reuse this variable on other places.
>>
-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Lucas Stach]

Am Mittwoch, dem 21.06.2023 um 23:41 +0800 schrieb Sui Jingfeng:
> On 2023/6/21 23:23, Lucas Stach wrote:
> > Am Mittwoch, dem 21.06.2023 um 22:44 +0800 schrieb Sui Jingfeng:
> > > Hi,
> > > 
> > > On 2023/6/21 18:00, Lucas Stach wrote:
> > > > > diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> > > > > index 9cd72948cfad..644e5712c050 100644
> > > > > --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> > > > > +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> > > > > @@ -46,6 +46,12 @@ struct etnaviv_drm_private {
> > > > >    	struct xarray active_contexts;
> > > > >    	u32 next_context_id;
> > > > >    
> > > > > +	/*
> > > > > +	 * If true, the GPU is capable of snooping cpu cache. Here, it
> > > > > +	 * also means that cache coherency is enforced by the hardware.
> > > > > +	 */
> > > > > +	bool dma_coherent;
> > > > > +
> > > > No need for this, I think. Just use dev_is_dma_coherent() where you
> > > > need to know this.
> > > > 
> > > No, we want this value cached by the driver.
> > > 
> > Why? dev_is_dma_coherent() is a header-only function with a single
> > pointer chasing operation. Your cache is also a single pointer chasing
> > access, just that we now need storage for this information in both
> > struct device and struct etnaviv_gpu.
> 
> 
> You don't need store it in struct etnaviv_gpu.
> 
> As this variable is shared across the device, so it is better to be put 
> in the struct etnaviv_drm_private.
> 
> I don't think another 4 bytes allocation is something what we can't pay for.
> 
> 
> My patch doesn't mentioned that it need to store it inside of struct 
> etnaviv_gpu, do I?

You are right, I was mistaken about the etnaviv struct this is added
to. However there is still the fundamental question: what's the gain of
this cache? The information is already available in struct device and
will be accessed with the same amount of loads if you care that much
about micro-optimization.

Regards,
Lucas

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/22 00:12, Lucas Stach wrote:
> Am Mittwoch, dem 21.06.2023 um 23:41 +0800 schrieb Sui Jingfeng:
>> On 2023/6/21 23:23, Lucas Stach wrote:
>>> Am Mittwoch, dem 21.06.2023 um 22:44 +0800 schrieb Sui Jingfeng:
>>>> Hi,
>>>>
>>>> On 2023/6/21 18:00, Lucas Stach wrote:
>>>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>>>>>> index 9cd72948cfad..644e5712c050 100644
>>>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>>>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>>>>>> @@ -46,6 +46,12 @@ struct etnaviv_drm_private {
>>>>>>     	struct xarray active_contexts;
>>>>>>     	u32 next_context_id;
>>>>>>     
>>>>>> +	/*
>>>>>> +	 * If true, the GPU is capable of snooping cpu cache. Here, it
>>>>>> +	 * also means that cache coherency is enforced by the hardware.
>>>>>> +	 */
>>>>>> +	bool dma_coherent;
>>>>>> +
>>>>> No need for this, I think. Just use dev_is_dma_coherent() where you
>>>>> need to know this.
>>>>>
>>>> No, we want this value cached by the driver.
>>>>
>>> Why? dev_is_dma_coherent() is a header-only function with a single
>>> pointer chasing operation. Your cache is also a single pointer chasing
>>> access, just that we now need storage for this information in both
>>> struct device and struct etnaviv_gpu.
>>
>> You don't need store it in struct etnaviv_gpu.
>>
>> As this variable is shared across the device, so it is better to be put
>> in the struct etnaviv_drm_private.
>>
>> I don't think another 4 bytes allocation is something what we can't pay for.
>>
>>
>> My patch doesn't mentioned that it need to store it inside of struct
>> etnaviv_gpu, do I?
> You are right, I was mistaken about the etnaviv struct this is added
> to. However there is still the fundamental question: what's the gain of
> this cache?

Clearness and short

you approach need to de-reference the pointer struct device *dev every 
time you need to fetch its value.

my name is short, typing it is less time-consuming

>   The information is already available in struct device and
> will be accessed with the same amount of loads if you care that much
> about micro-optimization.

I don't want call it everywhere, its too long.

What if the function you recommend get expanded by some programmer someday?

> Regards,
> Lucas

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/22 00:12, Lucas Stach wrote:
> Am Mittwoch, dem 21.06.2023 um 23:41 +0800 schrieb Sui Jingfeng:
>> On 2023/6/21 23:23, Lucas Stach wrote:
>>> Am Mittwoch, dem 21.06.2023 um 22:44 +0800 schrieb Sui Jingfeng:
>>>> Hi,
>>>>
>>>> On 2023/6/21 18:00, Lucas Stach wrote:
>>>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>>>>>> index 9cd72948cfad..644e5712c050 100644
>>>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>>>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>>>>>> @@ -46,6 +46,12 @@ struct etnaviv_drm_private {
>>>>>>     	struct xarray active_contexts;
>>>>>>     	u32 next_context_id;
>>>>>>     
>>>>>> +	/*
>>>>>> +	 * If true, the GPU is capable of snooping cpu cache. Here, it
>>>>>> +	 * also means that cache coherency is enforced by the hardware.
>>>>>> +	 */
>>>>>> +	bool dma_coherent;
>>>>>> +
>>>>> No need for this, I think. Just use dev_is_dma_coherent() where you
>>>>> need to know this.
>>>>>
>>>> No, we want this value cached by the driver.
>>>>
>>> Why? dev_is_dma_coherent() is a header-only function with a single
>>> pointer chasing operation. Your cache is also a single pointer chasing
>>> access, just that we now need storage for this information in both
>>> struct device and struct etnaviv_gpu.
>>
>> You don't need store it in struct etnaviv_gpu.
>>
>> As this variable is shared across the device, so it is better to be put
>> in the struct etnaviv_drm_private.
>>
>> I don't think another 4 bytes allocation is something what we can't pay for.
>>
>>
>> My patch doesn't mentioned that it need to store it inside of struct
>> etnaviv_gpu, do I?
> You are right, I was mistaken about the etnaviv struct this is added
> to. However there is still the fundamental question: what's the gain of
> this cache? The information is already available in struct device and
> will be accessed with the same amount of loads if you care that much
> about micro-optimization.

Sometime, in some function it is more convenient(easier) to fetch 
'struct etnaviv_drm_private *priv'

than the 'struct device *dev',  I think this is obvious.

> Regards,
> Lucas

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

On 2023/6/21 18:00, Lucas Stach wrote:
>>   	/* list of GEM objects: */
>>   	struct mutex gem_lock;
>>   	struct list_head gem_list;
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
>> index b5f73502e3dd..39bdc3774f2d 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
>> @@ -343,6 +343,7 @@ void *etnaviv_gem_vmap(struct drm_gem_object *obj)
>>   static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
>>   {
>>   	struct page **pages;
>> +	pgprot_t prot;
>>   
>>   	lockdep_assert_held(&obj->lock);
>>   
>> @@ -350,8 +351,19 @@ static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
>>   	if (IS_ERR(pages))
>>   		return NULL;
>>   
>> -	return vmap(pages, obj->base.size >> PAGE_SHIFT,
>> -			VM_MAP, pgprot_writecombine(PAGE_KERNEL));
>> +	switch (obj->flags) {
> switch (obj->flags & ETNA_BO_CACHE_MASK)
>
This is certainly OK, acceptable!

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi

On 2023/6/21 18:00, Lucas Stach wrote:
>> +	case ETNA_BO_CACHED:
>> +		prot = PAGE_KERNEL;
>> +		break;
>> +	case ETNA_BO_UNCACHED:
>> +		prot = pgprot_noncached(PAGE_KERNEL);
>> +		break;
>> +	case ETNA_BO_WC:
>> +	default:
>> +		prot = pgprot_writecombine(PAGE_KERNEL);
>> +	}
>> +
>> +	return vmap(pages, obj->base.size >> PAGE_SHIFT, VM_MAP, prot);
> While that change makes sense it should also be in a separate patch, as
> it's a valid change on its own, even if for non-coherent devices.
>
Accept with pleasure.

I will prepare the patch carefully.

Thanks for the positive comments.


-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

On 2023/6/21 18:00, Lucas Stach wrote:
>>   static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
>> @@ -369,6 +381,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
>>   {
>>   	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>>   	struct drm_device *dev = obj->dev;
>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>   	bool write = !!(op & ETNA_PREP_WRITE);
>>   	int ret;
>>   
>> @@ -395,7 +408,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
>>   			return ret == 0 ? -ETIMEDOUT : ret;
>>   	}
>>   
>> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
>> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
> Why do you need this? Isn't dma_sync_sgtable_for_cpu a no-op on your
> platform when the device is coherent?
>
I need this to show that our hardware is truly dma-coherent!

I have tested that the driver still works like a charm without adding 
this code '!priv->dma_coherent'.


But I'm expressing the idea that a truly dma-coherent just device don't 
need this.

I don't care if it is a no-op.

It is now, it may not in the future.

Even it is, the overhead of function call itself still get involved.

Also, we want to try flush the write buffer with the CPU manually.


Currently, we want the absolute correctness in the concept,

not only the rendering results.

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Lucas Stach]

Am Mittwoch, dem 21.06.2023 um 23:00 +0800 schrieb Sui Jingfeng:
> On 2023/6/21 18:00, Lucas Stach wrote:
> > >   static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
> > > @@ -369,6 +381,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
> > >   {
> > >   	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
> > >   	struct drm_device *dev = obj->dev;
> > > +	struct etnaviv_drm_private *priv = dev->dev_private;
> > >   	bool write = !!(op & ETNA_PREP_WRITE);
> > >   	int ret;
> > >   
> > > @@ -395,7 +408,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
> > >   			return ret == 0 ? -ETIMEDOUT : ret;
> > >   	}
> > >   
> > > -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
> > > +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
> > Why do you need this? Isn't dma_sync_sgtable_for_cpu a no-op on your
> > platform when the device is coherent?
> > 
> I need this to show that our hardware is truly dma-coherent!
> 
> I have tested that the driver still works like a charm without adding 
> this code '!priv->dma_coherent'.
> 
> 
> But I'm expressing the idea that a truly dma-coherent just device don't 
> need this.
> 
> I don't care if it is a no-op.
> 
> It is now, it may not in the future.

And that's exactly the point. If it ever turns into something more than
a no-op on your platform, then that's probably for a good reason and a
driver should not assume that it knows better than the DMA API
implementation what is or is not required on a specific platform to
make DMA work.

> 
> Even it is, the overhead of function call itself still get involved.
> 
cpu_prep/fini aren't total fast paths, you already synchronized with
the GPU here, potentially waiting for jobs to finish, etc. If your
platform no-ops this then the function call will be in the noise.
 
> Also, we want to try flush the write buffer with the CPU manually.
> 
> 
> Currently, we want the absolute correctness in the concept,
> 
> not only the rendering results.

And if you want absolute correctness then calling dma_sync_sgtable_* is
the right thing to do, as it can do much more than just manage caches. 

Right now it also provides SWIOTLB translation if needed.

Regards,
Lucas

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/21 23:33, Lucas Stach wrote:
> Am Mittwoch, dem 21.06.2023 um 23:00 +0800 schrieb Sui Jingfeng:
>> On 2023/6/21 18:00, Lucas Stach wrote:
>>>>    static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
>>>> @@ -369,6 +381,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
>>>>    {
>>>>    	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>>>>    	struct drm_device *dev = obj->dev;
>>>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>>>    	bool write = !!(op & ETNA_PREP_WRITE);
>>>>    	int ret;
>>>>    
>>>> @@ -395,7 +408,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
>>>>    			return ret == 0 ? -ETIMEDOUT : ret;
>>>>    	}
>>>>    
>>>> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
>>>> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>>> Why do you need this? Isn't dma_sync_sgtable_for_cpu a no-op on your
>>> platform when the device is coherent?
>>>
>> I need this to show that our hardware is truly dma-coherent!
>>
>> I have tested that the driver still works like a charm without adding
>> this code '!priv->dma_coherent'.
>>
>>
>> But I'm expressing the idea that a truly dma-coherent just device don't
>> need this.
>>
>> I don't care if it is a no-op.
>>
>> It is now, it may not in the future.
> And that's exactly the point. If it ever turns into something more than
> a no-op on your platform, then that's probably for a good reason and a
> driver should not assume that it knows better than the DMA API
> implementation what is or is not required on a specific platform to
> make DMA work.
>
>> Even it is, the overhead of function call itself still get involved.
>>
> cpu_prep/fini aren't total fast paths, you already synchronized with
> the GPU here, potentially waiting for jobs to finish, etc. If your
> platform no-ops this then the function call will be in the noise.
>   
>> Also, we want to try flush the write buffer with the CPU manually.
>>
>>
>> Currently, we want the absolute correctness in the concept,
>>
>> not only the rendering results.
> And if you want absolute correctness then calling dma_sync_sgtable_* is
> the right thing to do, as it can do much more than just manage caches.

For our hardware, cached mapping don't need calling dma_sync_sgtable_*.

This is the the right thing to do. The hardware already guarantee it for 
use.


We may only want to call it for WC mapping BO,  please don't tangle all 
of this together.

We simply want to do the right thing.

> Right now it also provides SWIOTLB translation if needed.

SWIOTLB introduce the bounce buffer, slower the performance.

We don't need it. It should be avoid.

  I know you know everything. No sugar-coated bullets please.

>
> Regards,
> Lucas

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Lucas Stach]

Am Mittwoch, dem 21.06.2023 um 23:54 +0800 schrieb Sui Jingfeng:
> Hi,
> 
> On 2023/6/21 23:33, Lucas Stach wrote:
> > Am Mittwoch, dem 21.06.2023 um 23:00 +0800 schrieb Sui Jingfeng:
> > > On 2023/6/21 18:00, Lucas Stach wrote:
> > > > >    static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
> > > > > @@ -369,6 +381,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
> > > > >    {
> > > > >    	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
> > > > >    	struct drm_device *dev = obj->dev;
> > > > > +	struct etnaviv_drm_private *priv = dev->dev_private;
> > > > >    	bool write = !!(op & ETNA_PREP_WRITE);
> > > > >    	int ret;
> > > > >    
> > > > > @@ -395,7 +408,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
> > > > >    			return ret == 0 ? -ETIMEDOUT : ret;
> > > > >    	}
> > > > >    
> > > > > -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
> > > > > +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
> > > > Why do you need this? Isn't dma_sync_sgtable_for_cpu a no-op on your
> > > > platform when the device is coherent?
> > > > 
> > > I need this to show that our hardware is truly dma-coherent!
> > > 
> > > I have tested that the driver still works like a charm without adding
> > > this code '!priv->dma_coherent'.
> > > 
> > > 
> > > But I'm expressing the idea that a truly dma-coherent just device don't
> > > need this.
> > > 
> > > I don't care if it is a no-op.
> > > 
> > > It is now, it may not in the future.
> > And that's exactly the point. If it ever turns into something more than
> > a no-op on your platform, then that's probably for a good reason and a
> > driver should not assume that it knows better than the DMA API
> > implementation what is or is not required on a specific platform to
> > make DMA work.
> > 
> > > Even it is, the overhead of function call itself still get involved.
> > > 
> > cpu_prep/fini aren't total fast paths, you already synchronized with
> > the GPU here, potentially waiting for jobs to finish, etc. If your
> > platform no-ops this then the function call will be in the noise.
> >   
> > > Also, we want to try flush the write buffer with the CPU manually.
> > > 
> > > 
> > > Currently, we want the absolute correctness in the concept,
> > > 
> > > not only the rendering results.
> > And if you want absolute correctness then calling dma_sync_sgtable_* is
> > the right thing to do, as it can do much more than just manage caches.
> 
> For our hardware, cached mapping don't need calling dma_sync_sgtable_*.
> 
> This is the the right thing to do. The hardware already guarantee it for 
> use.
> 
And as the HW guarantees it on your platform, your platform
implementation makes this function effectively a no-op. Skipping the
call to this function is breaking the DMA API abstraction, as now the
driver is second guessing the DMA API implementation. I really see no
reason to do this.

> 
> We may only want to call it for WC mapping BO,  please don't tangle all 
> of this together.
> 
> We simply want to do the right thing.
> 
> > Right now it also provides SWIOTLB translation if needed.
> 
> SWIOTLB introduce the bounce buffer, slower the performance.
> 
> We don't need it. It should be avoid.

Sure. If your platform doesn't need it, that's totally fine. But you
can't guarantee that all platforms with coherent Vivante GPUs don't
need this. If it isn't needed the DMA API implementation will skip it
just fine at almost no cost, so the driver really shouldn't try to be
more clever than the platform DMA API implementation.

Regards,
Lucas

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/22 00:07, Lucas Stach wrote:
> And as the HW guarantees it on your platform, your platform
> implementation makes this function effectively a no-op. Skipping the
> call to this function is breaking the DMA API abstraction, as now the
> driver is second guessing the DMA API implementation. I really see no
> reason to do this.

It is the same reason you chose the word 'effectively', not 'difinitely'.

We don't want waste the CPU's time,


  to running the dma_sync_sg_for_cpu funcion() function


```

void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
             int nelems, enum dma_data_direction dir)
{
     const struct dma_map_ops *ops = get_dma_ops(dev);

     BUG_ON(!valid_dma_direction(dir));
     if (dma_map_direct(dev, ops))
         dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
     else if (ops->sync_sg_for_cpu)
         ops->sync_sg_for_cpu(dev, sg, nelems, dir);
     debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
}

```


  to running the this:


```

int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
{
     struct drm_device *dev = obj->dev;
     struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
     struct etnaviv_drm_private *priv = dev->dev_private;

     if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
         /* fini without a prep is almost certainly a userspace error */
         WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
         dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
etnaviv_op_to_dma_dir(etnaviv_obj->last_cpu_prep_op));
         etnaviv_obj->last_cpu_prep_op = 0;
     }

     return 0;
}

```


But, this is acceptable, because we can kill the GEM_CPU_PREP and 
GEM_CPU_FINI ioctl entirely

at userspace for cached buffer, as this is totally not needed for cached 
mapping on our platform.


Well leave this for the WC mapping only,

OK ?

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Lucas Stach]

Am Donnerstag, dem 22.06.2023 um 01:31 +0800 schrieb Sui Jingfeng:
> Hi,
> 
> On 2023/6/22 00:07, Lucas Stach wrote:
> > And as the HW guarantees it on your platform, your platform
> > implementation makes this function effectively a no-op. Skipping the
> > call to this function is breaking the DMA API abstraction, as now the
> > driver is second guessing the DMA API implementation. I really see no
> > reason to do this.
> 
> It is the same reason you chose the word 'effectively', not 'difinitely'.
> 
> We don't want waste the CPU's time,
> 
> 
>   to running the dma_sync_sg_for_cpu funcion() function
> 
> 
> ```
> 
> void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
>              int nelems, enum dma_data_direction dir)
> {
>      const struct dma_map_ops *ops = get_dma_ops(dev);
> 
>      BUG_ON(!valid_dma_direction(dir));
>      if (dma_map_direct(dev, ops))
>          dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
>      else if (ops->sync_sg_for_cpu)
>          ops->sync_sg_for_cpu(dev, sg, nelems, dir);
>      debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
> }
> 
> ```
> 
> 
>   to running the this:
> 
> 
> ```
> 
> int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
> {
>      struct drm_device *dev = obj->dev;
>      struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>      struct etnaviv_drm_private *priv = dev->dev_private;
> 
>      if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>          /* fini without a prep is almost certainly a userspace error */
>          WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
>          dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
> etnaviv_op_to_dma_dir(etnaviv_obj->last_cpu_prep_op));
>          etnaviv_obj->last_cpu_prep_op = 0;
>      }
> 
>      return 0;
> }
> 
> ```
> 
My judgment as the maintainer of this driver is that the small CPU
overhead of calling this function is very well worth it, if the
alternative is breaking the DMA API abstractions.

> 
> But, this is acceptable, because we can kill the GEM_CPU_PREP and 
> GEM_CPU_FINI ioctl entirely
> 
> at userspace for cached buffer, as this is totally not needed for cached 
> mapping on our platform.
> 
And that statement isn't true either. The CPU_PREP/FINI ioctls also
provide fence synchronization between CPU and GPU. There are a few very
specific cases where skipping those ioctls is acceptable (mostly when
the userspace driver explicitly wants unsynchronized access), but in
most cases they are required for correctness.

Regards,
Lucas

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/22 01:53, Lucas Stach wrote:
> Am Donnerstag, dem 22.06.2023 um 01:31 +0800 schrieb Sui Jingfeng:
>> Hi,
>>
>> On 2023/6/22 00:07, Lucas Stach wrote:
>>> And as the HW guarantees it on your platform, your platform
>>> implementation makes this function effectively a no-op. Skipping the
>>> call to this function is breaking the DMA API abstraction, as now the
>>> driver is second guessing the DMA API implementation. I really see no
>>> reason to do this.
>> It is the same reason you chose the word 'effectively', not 'difinitely'.
>>
>> We don't want waste the CPU's time,
>>
>>
>>    to running the dma_sync_sg_for_cpu funcion() function
>>
>>
>> ```
>>
>> void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
>>               int nelems, enum dma_data_direction dir)
>> {
>>       const struct dma_map_ops *ops = get_dma_ops(dev);
>>
>>       BUG_ON(!valid_dma_direction(dir));
>>       if (dma_map_direct(dev, ops))
>>           dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
>>       else if (ops->sync_sg_for_cpu)
>>           ops->sync_sg_for_cpu(dev, sg, nelems, dir);
>>       debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
>> }
>>
>> ```
>>
>>
>>    to running the this:
>>
>>
>> ```
>>
>> int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
>> {
>>       struct drm_device *dev = obj->dev;
>>       struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>>       struct etnaviv_drm_private *priv = dev->dev_private;
>>
>>       if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>>           /* fini without a prep is almost certainly a userspace error */
>>           WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
>>           dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
>> etnaviv_op_to_dma_dir(etnaviv_obj->last_cpu_prep_op));
>>           etnaviv_obj->last_cpu_prep_op = 0;
>>       }
>>
>>       return 0;
>> }
>>
>> ```
>>
> My judgment as the maintainer of this driver is that the small CPU
> overhead of calling this function is very well worth it, if the
> alternative is breaking the DMA API abstractions.
>
>> But, this is acceptable, because we can kill the GEM_CPU_PREP and
>> GEM_CPU_FINI ioctl entirely
>>
>> at userspace for cached buffer, as this is totally not needed for cached
>> mapping on our platform.
>>
> And that statement isn't true either.

Yes, you are right here. I admit.


Because I have suffered such problem in the past when developing 
xf86-video-loongson.

The root cause, I think,  is the CPU don't know when the GPU have 
finished the rendering.

Or there still some data reside in the GPU's cache.


We have to call etna_bo_cpu_prep(etna_bo, DRM_ETNA_PREP_READ) function

to make sure the  data fetch by CPU is the latest.


I realized this knowledge(issue) five month ago in this year, see [1] 
for reference.

I  just forget this thing when doing the debate with you.


[1] 
https://gitlab.freedesktop.org/longxin2019/xf86-video-loongson/-/commit/95f9596eb19223c3109ea1f32c3e086fd1d43bd8

||


>   The CPU_PREP/FINI ioctls also
> provide fence synchronization between CPU and GPU.

You are correct here.

> There are a few very
> specific cases where skipping those ioctls is acceptable (mostly when
> the userspace driver explicitly wants unsynchronized access), but in
> most cases they are required for correctness.

OK, you are extremely correct.

> Regards,
> Lucas

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/21 18:00, Lucas Stach wrote:
>>   		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
>>   					 etnaviv_op_to_dma_dir(op));
>>   		etnaviv_obj->last_cpu_prep_op = op;
>> @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
>>   {
>>   	struct drm_device *dev = obj->dev;
>>   	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>   
>> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
>> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>>   		/* fini without a prep is almost certainly a userspace error */
>>   		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
>>   		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>> index 3524b5811682..754126992264 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>> @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
>>   struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
>>   	struct dma_buf_attachment *attach, struct sg_table *sgt)
>>   {
>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>   	struct etnaviv_gem_object *etnaviv_obj;
>>   	size_t size = PAGE_ALIGN(attach->dmabuf->size);
>> +	u32 cache_flags = ETNA_BO_WC;
>>   	int ret, npages;
>>   
>> -	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
>> +	if (priv->dma_coherent)
>> +		cache_flags = ETNA_BO_CACHED;
>> +
> Drop this change. Instead etnaviv_gem_new_impl() should do the upgrade
> from WC to CACHED as necessary by adding something like this:

I understand you are a profession person in vivante GPU driver domain.

I respect you reviews and instruction.

But, I'm really reluctant to agree with this, is there any space to 
negotiate?

> /*
>   * Upgrade WC to CACHED when the device is hardware coherent and the
>   * platform doesn't allow mixing cached and writecombined mappings to
>   * the same memory area.
>   */
> if ((flags & ETNA_BO_CACHE_MASK) == ETNA_BO_WC &&
>      dev_is_dma_coherent(dev) && !drm_arch_can_wc_memory())
>          flags = (flags & ~ETNA_BO_CACHE_MASK) & ETNA_BO_CACHED;

This is policy, not a mechanism.

Using what cache property is a user-space program's choice.

While you are override the WC with CACHED mapping. This is not correct 
in the concept!

you approach forbidden any possibility to use the WC BO at anywhere.


My approach need only check once, while you approach need at least 3 
check plus

so much bit-wise logic operations,  plus a function call  (&, ==, &&,  
&, ~, &) .

and every time you create a BO. This nasty judgement happens.


Please keep our original implement, it's simple and clear, Please?


-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Lucas Stach]

Am Mittwoch, dem 21.06.2023 um 23:30 +0800 schrieb Sui Jingfeng:
> Hi,
> 
> On 2023/6/21 18:00, Lucas Stach wrote:
> > >   		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
> > >   					 etnaviv_op_to_dma_dir(op));
> > >   		etnaviv_obj->last_cpu_prep_op = op;
> > > @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
> > >   {
> > >   	struct drm_device *dev = obj->dev;
> > >   	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
> > > +	struct etnaviv_drm_private *priv = dev->dev_private;
> > >   
> > > -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
> > > +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
> > >   		/* fini without a prep is almost certainly a userspace error */
> > >   		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
> > >   		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
> > > diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> > > index 3524b5811682..754126992264 100644
> > > --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> > > +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> > > @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
> > >   struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
> > >   	struct dma_buf_attachment *attach, struct sg_table *sgt)
> > >   {
> > > +	struct etnaviv_drm_private *priv = dev->dev_private;
> > >   	struct etnaviv_gem_object *etnaviv_obj;
> > >   	size_t size = PAGE_ALIGN(attach->dmabuf->size);
> > > +	u32 cache_flags = ETNA_BO_WC;
> > >   	int ret, npages;
> > >   
> > > -	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
> > > +	if (priv->dma_coherent)
> > > +		cache_flags = ETNA_BO_CACHED;
> > > +
> > Drop this change. Instead etnaviv_gem_new_impl() should do the upgrade
> > from WC to CACHED as necessary by adding something like this:
> 
> I understand you are a profession person in vivante GPU driver domain.
> 
> I respect you reviews and instruction.
> 
> But, I'm really reluctant to agree with this, is there any space to 
> negotiate?
> 
> > /*
> >   * Upgrade WC to CACHED when the device is hardware coherent and the
> >   * platform doesn't allow mixing cached and writecombined mappings to
> >   * the same memory area.
> >   */
> > if ((flags & ETNA_BO_CACHE_MASK) == ETNA_BO_WC &&
> >      dev_is_dma_coherent(dev) && !drm_arch_can_wc_memory())
> >          flags = (flags & ~ETNA_BO_CACHE_MASK) & ETNA_BO_CACHED;
> 
> This is policy, not a mechanism.
> 
> Using what cache property is a user-space program's choice.
> 
> While you are override the WC with CACHED mapping. This is not correct 
> in the concept!
> 
Please explain why you think that this isn't correct. If using WC
mappings cause a potential loss of coherency on your platform, then we
can not allow the userspace driver to use WC mappings.

As I would like to keep the option of WC mappings, I've asked you if
there are ways to prepare the cache in a way that WC mappings aren't
causing any troubles on your platform. You told me that this might be
possible but needs confirmation from a HW engineer and such
confirmation could take a long time.

With that in mind, our only option right now is to upgrade the mappings
to cached  in order to not lay out traps for the userspace driver.
 
> you approach forbidden any possibility to use the WC BO at anywhere.
> 
> 
> My approach need only check once, while you approach need at least 3 
> check plus
> 
> so much bit-wise logic operations,  plus a function call  (&, ==, &&,  
> &, ~, &) .
> 
> and every time you create a BO. This nasty judgement happens.
> 
BO creation again is not a fast path. You are committing to allocate
new memory, which is a few orders of magnitude more costly than the few
instructions needed for those comparisons.

> 
> Please keep our original implement, it's simple and clear, Please?
> 

It isn't as simple and clear for the userspace interface. It allows
userspace to use WC mappings that would potentially cause loss of
coherency between CPU and GPU, which isn't acceptable.

Regards,
Lucas

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi

On 2023/6/21 23:58, Lucas Stach wrote:
>> you approach forbidden any possibility to use the WC BO at anywhere.
>>
>>
>> My approach need only check once, while you approach need at least 3
>> check plus
>>
>> so much bit-wise logic operations,  plus a function call  (&, ==, &&,
>> &, ~, &) .
>>
>> and every time you create a BO. This nasty judgement happens.
>>
> BO creation again is not a fast path. You are committing to allocate
> new memory, which is a few orders of magnitude more costly than the few
> instructions needed for those comparisons.
>
What's wrong with you point here is that

We are not going make it more worse because it is worse.

We would like same any single bit of the performance.


It's about the beauty, and beauty and correctness is every thing.


My implement is good both in the perspective of beauty and in the 
perspective of the performance.

BO creation is fast or not is irrelevant to what the point we are discuss.

You are always introduce non-critical factor to disturb the discuss,

leading the people go a wrong direction.

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/21 23:58, Lucas Stach wrote:
> Am Mittwoch, dem 21.06.2023 um 23:30 +0800 schrieb Sui Jingfeng:
>> Hi,
>>
>> On 2023/6/21 18:00, Lucas Stach wrote:
>>>>    		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
>>>>    					 etnaviv_op_to_dma_dir(op));
>>>>    		etnaviv_obj->last_cpu_prep_op = op;
>>>> @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
>>>>    {
>>>>    	struct drm_device *dev = obj->dev;
>>>>    	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>>>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>>>    
>>>> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
>>>> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>>>>    		/* fini without a prep is almost certainly a userspace error */
>>>>    		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
>>>>    		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>>>> index 3524b5811682..754126992264 100644
>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>>>> @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
>>>>    struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
>>>>    	struct dma_buf_attachment *attach, struct sg_table *sgt)
>>>>    {
>>>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>>>    	struct etnaviv_gem_object *etnaviv_obj;
>>>>    	size_t size = PAGE_ALIGN(attach->dmabuf->size);
>>>> +	u32 cache_flags = ETNA_BO_WC;
>>>>    	int ret, npages;
>>>>    
>>>> -	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
>>>> +	if (priv->dma_coherent)
>>>> +		cache_flags = ETNA_BO_CACHED;
>>>> +
>>> Drop this change. Instead etnaviv_gem_new_impl() should do the upgrade
>>> from WC to CACHED as necessary by adding something like this:
>> I understand you are a profession person in vivante GPU driver domain.
>>
>> I respect you reviews and instruction.
>>
>> But, I'm really reluctant to agree with this, is there any space to
>> negotiate?
>>
>>> /*
>>>    * Upgrade WC to CACHED when the device is hardware coherent and the
>>>    * platform doesn't allow mixing cached and writecombined mappings to
>>>    * the same memory area.
>>>    */
>>> if ((flags & ETNA_BO_CACHE_MASK) == ETNA_BO_WC &&
>>>       dev_is_dma_coherent(dev) && !drm_arch_can_wc_memory())
>>>           flags = (flags & ~ETNA_BO_CACHE_MASK) & ETNA_BO_CACHED;
>> This is policy, not a mechanism.
>>
>> Using what cache property is a user-space program's choice.
>>
>> While you are override the WC with CACHED mapping. This is not correct
>> in the concept!
>>
> Please explain why you think that this isn't correct.

Again,

this is user-space things!

this is user-space things!

this is user-space things!

I have explained several times.

made the decision for the user-space program is wrong.


> It allows
> userspace to use WC mappings that would potentially cause loss of
> coherency between CPU and GPU, which isn't acceptable.

Before made the WC works correctly,  you need the developing environment.

userspace program can tune the BO cache mapping easily.

Either environment or supply a conf file.


While with your implement, we don't have the opportunity to do debugging 
and the development.

The kernel space is writing hard-code.


> Regards,
> Lucas

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Lucas Stach]

Am Donnerstag, dem 22.06.2023 um 01:21 +0800 schrieb Sui Jingfeng:
> Hi,
> 
> On 2023/6/21 23:58, Lucas Stach wrote:
> > Am Mittwoch, dem 21.06.2023 um 23:30 +0800 schrieb Sui Jingfeng:
> > > Hi,
> > > 
> > > On 2023/6/21 18:00, Lucas Stach wrote:
> > > > >    		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
> > > > >    					 etnaviv_op_to_dma_dir(op));
> > > > >    		etnaviv_obj->last_cpu_prep_op = op;
> > > > > @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
> > > > >    {
> > > > >    	struct drm_device *dev = obj->dev;
> > > > >    	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
> > > > > +	struct etnaviv_drm_private *priv = dev->dev_private;
> > > > >    
> > > > > -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
> > > > > +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
> > > > >    		/* fini without a prep is almost certainly a userspace error */
> > > > >    		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
> > > > >    		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
> > > > > diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> > > > > index 3524b5811682..754126992264 100644
> > > > > --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> > > > > +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> > > > > @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
> > > > >    struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
> > > > >    	struct dma_buf_attachment *attach, struct sg_table *sgt)
> > > > >    {
> > > > > +	struct etnaviv_drm_private *priv = dev->dev_private;
> > > > >    	struct etnaviv_gem_object *etnaviv_obj;
> > > > >    	size_t size = PAGE_ALIGN(attach->dmabuf->size);
> > > > > +	u32 cache_flags = ETNA_BO_WC;
> > > > >    	int ret, npages;
> > > > >    
> > > > > -	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
> > > > > +	if (priv->dma_coherent)
> > > > > +		cache_flags = ETNA_BO_CACHED;
> > > > > +
> > > > Drop this change. Instead etnaviv_gem_new_impl() should do the upgrade
> > > > from WC to CACHED as necessary by adding something like this:
> > > I understand you are a profession person in vivante GPU driver domain.
> > > 
> > > I respect you reviews and instruction.
> > > 
> > > But, I'm really reluctant to agree with this, is there any space to
> > > negotiate?
> > > 
> > > > /*
> > > >    * Upgrade WC to CACHED when the device is hardware coherent and the
> > > >    * platform doesn't allow mixing cached and writecombined mappings to
> > > >    * the same memory area.
> > > >    */
> > > > if ((flags & ETNA_BO_CACHE_MASK) == ETNA_BO_WC &&
> > > >       dev_is_dma_coherent(dev) && !drm_arch_can_wc_memory())
> > > >           flags = (flags & ~ETNA_BO_CACHE_MASK) & ETNA_BO_CACHED;
> > > This is policy, not a mechanism.
> > > 
> > > Using what cache property is a user-space program's choice.
> > > 
> > > While you are override the WC with CACHED mapping. This is not correct
> > > in the concept!
> > > 
> > Please explain why you think that this isn't correct.
> 
> Again,
> 
> this is user-space things!
> 
> this is user-space things!
> 
> this is user-space things!
> 
> I have explained several times.
> 
> made the decision for the user-space program is wrong.
> 
This mode of communication isn't helpful. Please stop it.

As I tried to explain to you multiple times: if userspace can break
coherency by selecting the wrong mapping type then this is something
the kernel should prevent.

> 
> > It allows
> > userspace to use WC mappings that would potentially cause loss of
> > coherency between CPU and GPU, which isn't acceptable.
> 
> Before made the WC works correctly,  you need the developing environment.
> 
> userspace program can tune the BO cache mapping easily.
> 
> Either environment or supply a conf file.
> 
> 
> While with your implement, we don't have the opportunity to do debugging 
> and the development.

You can always add a patch to your local kernel to re-allow WC mappings
while you work on making them behave as expected on your platform. With
the mainline kernel there is no way that the kernel driver will allow
broken coherency.

And as I also mentioned before, there is a clear upgrade path here:
once WC mappings work as expected on your platform we can easily drop
the upgrading from the kernel driver again. The userspace driver can
already be changed to use CACHED BOs where beneficial on your platform
in the meantime.

Regards,
Lucas

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/22 01:45, Lucas Stach wrote:
>> Again,
>>
>> this is user-space things!
>>
>> this is user-space things!
>>
>> this is user-space things!
>>
>> I have explained several times.
>>
>> made the decision for the user-space program is wrong.
>>
> This mode of communication isn't helpful. Please stop it.
>
> As I tried to explain to you multiple times: if userspace can break
> coherency by selecting the wrong mapping type then this is something
> the kernel should prevent.
>
This is still a user-space things.

While my point is that we want to prevent it at userspace.

Please don't forget that sometime a user or a programmer want a wrong thing.
  
Either because of the need of debugging itself or due to the curiosity of

what it would happen if we doing something wrong.

Sometimes, the wrong rendering may more interesting.


The most important thing is that my patch is trying to increase the 
flexibility,

and to increase the possibility. While you idea is to doing the reverse.


-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/22 01:45, Lucas Stach wrote:
> Am Donnerstag, dem 22.06.2023 um 01:21 +0800 schrieb Sui Jingfeng:
>> Hi,
>>
>> On 2023/6/21 23:58, Lucas Stach wrote:
>>> Am Mittwoch, dem 21.06.2023 um 23:30 +0800 schrieb Sui Jingfeng:
>>>> Hi,
>>>>
>>>> On 2023/6/21 18:00, Lucas Stach wrote:
>>>>>>     		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
>>>>>>     					 etnaviv_op_to_dma_dir(op));
>>>>>>     		etnaviv_obj->last_cpu_prep_op = op;
>>>>>> @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
>>>>>>     {
>>>>>>     	struct drm_device *dev = obj->dev;
>>>>>>     	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>>>>>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>>>>>     
>>>>>> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
>>>>>> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>>>>>>     		/* fini without a prep is almost certainly a userspace error */
>>>>>>     		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
>>>>>>     		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
>>>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>>>>>> index 3524b5811682..754126992264 100644
>>>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>>>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>>>>>> @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
>>>>>>     struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
>>>>>>     	struct dma_buf_attachment *attach, struct sg_table *sgt)
>>>>>>     {
>>>>>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>>>>>     	struct etnaviv_gem_object *etnaviv_obj;
>>>>>>     	size_t size = PAGE_ALIGN(attach->dmabuf->size);
>>>>>> +	u32 cache_flags = ETNA_BO_WC;
>>>>>>     	int ret, npages;
>>>>>>     
>>>>>> -	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
>>>>>> +	if (priv->dma_coherent)
>>>>>> +		cache_flags = ETNA_BO_CACHED;
>>>>>> +
>>>>> Drop this change. Instead etnaviv_gem_new_impl() should do the upgrade
>>>>> from WC to CACHED as necessary by adding something like this:
>>>> I understand you are a profession person in vivante GPU driver domain.
>>>>
>>>> I respect you reviews and instruction.
>>>>
>>>> But, I'm really reluctant to agree with this, is there any space to
>>>> negotiate?
>>>>
>>>>> /*
>>>>>     * Upgrade WC to CACHED when the device is hardware coherent and the
>>>>>     * platform doesn't allow mixing cached and writecombined mappings to
>>>>>     * the same memory area.
>>>>>     */
>>>>> if ((flags & ETNA_BO_CACHE_MASK) == ETNA_BO_WC &&
>>>>>        dev_is_dma_coherent(dev) && !drm_arch_can_wc_memory())
>>>>>            flags = (flags & ~ETNA_BO_CACHE_MASK) & ETNA_BO_CACHED;
>>>> This is policy, not a mechanism.
>>>>
>>>> Using what cache property is a user-space program's choice.
>>>>
>>>> While you are override the WC with CACHED mapping. This is not correct
>>>> in the concept!
>>>>
>>> Please explain why you think that this isn't correct.
>> Again,
>>
>> this is user-space things!
>>
>> this is user-space things!
>>
>> this is user-space things!
>>
>> I have explained several times.
>>
>> made the decision for the user-space program is wrong.
>>
> This mode of communication isn't helpful. Please stop it.
>
> As I tried to explain to you multiple times: if userspace can break
> coherency by selecting the wrong mapping type then this is something
> the kernel should prevent.

You are right in overall.


This is the only one benefit which WC mapping is preferred over the 
cached mapping.

As you already told me,  the WC mapping don't *pollute CPU's cache*.


If we can make sure that a BO is *only* going to be used by the GPU,

then we still can choose WC mapping as the cache property of this BO.

As the cache property is CPU side thing.


>>> It allows
>>> userspace to use WC mappings that would potentially cause loss of
>>> coherency between CPU and GPU, which isn't acceptable.
>> Before made the WC works correctly,  you need the developing environment.
>>
>> userspace program can tune the BO cache mapping easily.
>>
>> Either environment or supply a conf file.
>>
>>
>> While with your implement, we don't have the opportunity to do debugging
>> and the development.
> You can always add a patch to your local kernel to re-allow WC mappings
> while you work on making them behave as expected on your platform.


We are doing the things about the *upstream*.


>   With
> the mainline kernel there is no way that the kernel driver will allow
> broken coherency.


A buffer is used by the GPU solely won't break the coherency.


> And as I also mentioned before, there is a clear upgrade path here:
> once WC mappings work as expected on your platform we can easily drop
> the upgrading from the kernel driver again. The userspace driver can
> already be changed to use CACHED BOs where beneficial on your platform
> in the meantime.

For our platform, I think the problem is that the GPU always write to L3 
share cache,

even you use the WC mapping.

As I already said, the WC mapping only influence the CPU side.

How can I control the GPU when a BO is WC mapping ?

Does the GPU know that whether a BO  is WC mapping or not ?

How can I let(or tell) the GPU that

please not write to shared L3 cache,  write to the system RAM directly?

On out platform, the coherency between the CPU and peripheral IO 
device's cache

is maintained by hardware. While at here.  the hardware is the shared L3 
cache.


I guess the current problem is that the GPU don't listen to me,

he still write to CPU cache's even you choose the WC mapping.

Then if you want to read the rendered image by CPU,

the CPU go the un-cached path. Then I think the CPU will get wrong data.

> Regards,
> Lucas

-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Lucas Stach]

Hi,

Am Sonntag, dem 25.06.2023 um 11:51 +0800 schrieb Sui Jingfeng:
> Hi,
> 
> On 2023/6/22 01:45, Lucas Stach wrote:
> > Am Donnerstag, dem 22.06.2023 um 01:21 +0800 schrieb Sui Jingfeng:
> > > Hi,
> > > 
> > > On 2023/6/21 23:58, Lucas Stach wrote:
> > > > Am Mittwoch, dem 21.06.2023 um 23:30 +0800 schrieb Sui Jingfeng:
> > > > > Hi,
> > > > > 
> > > > > On 2023/6/21 18:00, Lucas Stach wrote:
> > > > > > >     		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
> > > > > > >     					 etnaviv_op_to_dma_dir(op));
> > > > > > >     		etnaviv_obj->last_cpu_prep_op = op;
> > > > > > > @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
> > > > > > >     {
> > > > > > >     	struct drm_device *dev = obj->dev;
> > > > > > >     	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
> > > > > > > +	struct etnaviv_drm_private *priv = dev->dev_private;
> > > > > > >     
> > > > > > > -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
> > > > > > > +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
> > > > > > >     		/* fini without a prep is almost certainly a userspace error */
> > > > > > >     		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
> > > > > > >     		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
> > > > > > > diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> > > > > > > index 3524b5811682..754126992264 100644
> > > > > > > --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> > > > > > > +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> > > > > > > @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
> > > > > > >     struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
> > > > > > >     	struct dma_buf_attachment *attach, struct sg_table *sgt)
> > > > > > >     {
> > > > > > > +	struct etnaviv_drm_private *priv = dev->dev_private;
> > > > > > >     	struct etnaviv_gem_object *etnaviv_obj;
> > > > > > >     	size_t size = PAGE_ALIGN(attach->dmabuf->size);
> > > > > > > +	u32 cache_flags = ETNA_BO_WC;
> > > > > > >     	int ret, npages;
> > > > > > >     
> > > > > > > -	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
> > > > > > > +	if (priv->dma_coherent)
> > > > > > > +		cache_flags = ETNA_BO_CACHED;
> > > > > > > +
> > > > > > Drop this change. Instead etnaviv_gem_new_impl() should do the upgrade
> > > > > > from WC to CACHED as necessary by adding something like this:
> > > > > I understand you are a profession person in vivante GPU driver domain.
> > > > > 
> > > > > I respect you reviews and instruction.
> > > > > 
> > > > > But, I'm really reluctant to agree with this, is there any space to
> > > > > negotiate?
> > > > > 
> > > > > > /*
> > > > > >     * Upgrade WC to CACHED when the device is hardware coherent and the
> > > > > >     * platform doesn't allow mixing cached and writecombined mappings to
> > > > > >     * the same memory area.
> > > > > >     */
> > > > > > if ((flags & ETNA_BO_CACHE_MASK) == ETNA_BO_WC &&
> > > > > >        dev_is_dma_coherent(dev) && !drm_arch_can_wc_memory())
> > > > > >            flags = (flags & ~ETNA_BO_CACHE_MASK) & ETNA_BO_CACHED;
> > > > > This is policy, not a mechanism.
> > > > > 
> > > > > Using what cache property is a user-space program's choice.
> > > > > 
> > > > > While you are override the WC with CACHED mapping. This is not correct
> > > > > in the concept!
> > > > > 
> > > > Please explain why you think that this isn't correct.
> > > Again,
> > > 
> > > this is user-space things!
> > > 
> > > this is user-space things!
> > > 
> > > this is user-space things!
> > > 
> > > I have explained several times.
> > > 
> > > made the decision for the user-space program is wrong.
> > > 
> > This mode of communication isn't helpful. Please stop it.
> > 
> > As I tried to explain to you multiple times: if userspace can break
> > coherency by selecting the wrong mapping type then this is something
> > the kernel should prevent.
> 
> You are right in overall.
> 
> 
> This is the only one benefit which WC mapping is preferred over the 
> cached mapping.
> 
> As you already told me,  the WC mapping don't *pollute CPU's cache*.
> 
> 
> If we can make sure that a BO is *only* going to be used by the GPU,
> 
> then we still can choose WC mapping as the cache property of this BO.
> 
> As the cache property is CPU side thing.
> 
But if it only used by the GPU, then WC won't help you either as the
CPU is never touching the memory. The benefit of WC is that the CPU can
write around the cache, so if it uploads data to the GPU this data
won't pollute the cache. If the buffer is only used by the GPU, then
there is no risk of cache pollution through the CPU, right?

> 
> > > > It allows
> > > > userspace to use WC mappings that would potentially cause loss of
> > > > coherency between CPU and GPU, which isn't acceptable.
> > > Before made the WC works correctly,  you need the developing environment.
> > > 
> > > userspace program can tune the BO cache mapping easily.
> > > 
> > > Either environment or supply a conf file.
> > > 
> > > 
> > > While with your implement, we don't have the opportunity to do debugging
> > > and the development.
> > You can always add a patch to your local kernel to re-allow WC mappings
> > while you work on making them behave as expected on your platform.
> 
> 
> We are doing the things about the *upstream*.
> 
Upstream is not about doing experiments. You can always do those
experiments in your development kernel and if they prove to be
beneficial, we can think about how to integrate them with the upstream
kernel without the risk of inadvertently breaking things.

> 
> >   With
> > the mainline kernel there is no way that the kernel driver will allow
> > broken coherency.
> 
> 
> A buffer is used by the GPU solely won't break the coherency.
> 
> 
> > And as I also mentioned before, there is a clear upgrade path here:
> > once WC mappings work as expected on your platform we can easily drop
> > the upgrading from the kernel driver again. The userspace driver can
> > already be changed to use CACHED BOs where beneficial on your platform
> > in the meantime.
> 
> For our platform, I think the problem is that the GPU always write to L3 
> share cache,
> 
> even you use the WC mapping.

As long as that is the case, I think forcing the CPU to go through the
same cache by upgrading the BOs to cached is the right thing to do.
> 
> As I already said, the WC mapping only influence the CPU side.
> 
> How can I control the GPU when a BO is WC mapping ?

Hm, newer GPUs allow to use different AXI attributes (which include the
cachability) controlled via a pagetable setting, but I'm not sure if
the GC1000 on your platform supports this.

Can you ask your hardware guys if there is a way to set the PCI "no-
snoop" transaction flag from the GPU and if that will cause the GPU
memory accesses to bypass the L3 cache?
 
> 
> Does the GPU know that whether a BO  is WC mapping or not ?
> 
> How can I let(or tell) the GPU that
> 
> please not write to shared L3 cache,  write to the system RAM directly?
> 
> On out platform, the coherency between the CPU and peripheral IO 
> device's cache
> 
> is maintained by hardware. While at here.  the hardware is the shared L3 
> cache.
> 
> 
> I guess the current problem is that the GPU don't listen to me,
> 
> he still write to CPU cache's even you choose the WC mapping.
> 
> Then if you want to read the rendered image by CPU,
> 
> the CPU go the un-cached path. Then I think the CPU will get wrong data.
> 
Yes, that sounds plausible and I don't see much of a way around this
other than forcing the CPU to use the same path through the cache.

Regards,
Lucas

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi

On 2023/6/21 23:58, Lucas Stach wrote:
> Am Mittwoch, dem 21.06.2023 um 23:30 +0800 schrieb Sui Jingfeng:
>> Hi,
>>
>> On 2023/6/21 18:00, Lucas Stach wrote:
>>>>    		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
>>>>    					 etnaviv_op_to_dma_dir(op));
>>>>    		etnaviv_obj->last_cpu_prep_op = op;
>>>> @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
>>>>    {
>>>>    	struct drm_device *dev = obj->dev;
>>>>    	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>>>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>>>    
>>>> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
>>>> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>>>>    		/* fini without a prep is almost certainly a userspace error */
>>>>    		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
>>>>    		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
>>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>>>> index 3524b5811682..754126992264 100644
>>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>>>> @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
>>>>    struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
>>>>    	struct dma_buf_attachment *attach, struct sg_table *sgt)
>>>>    {
>>>> +	struct etnaviv_drm_private *priv = dev->dev_private;
>>>>    	struct etnaviv_gem_object *etnaviv_obj;
>>>>    	size_t size = PAGE_ALIGN(attach->dmabuf->size);
>>>> +	u32 cache_flags = ETNA_BO_WC;
>>>>    	int ret, npages;
>>>>    
>>>> -	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
>>>> +	if (priv->dma_coherent)
>>>> +		cache_flags = ETNA_BO_CACHED;
>>>> +
>>> Drop this change. Instead etnaviv_gem_new_impl() should do the upgrade
>>> from WC to CACHED as necessary by adding something like this:
>> I understand you are a profession person in vivante GPU driver domain.
>>
>> I respect you reviews and instruction.
>>
>> But, I'm really reluctant to agree with this, is there any space to
>> negotiate?
>>
>>> /*
>>>    * Upgrade WC to CACHED when the device is hardware coherent and the
>>>    * platform doesn't allow mixing cached and writecombined mappings to
>>>    * the same memory area.
>>>    */
>>> if ((flags & ETNA_BO_CACHE_MASK) == ETNA_BO_WC &&
>>>       dev_is_dma_coherent(dev) && !drm_arch_can_wc_memory())
>>>           flags = (flags & ~ETNA_BO_CACHE_MASK) & ETNA_BO_CACHED;
>> This is policy, not a mechanism.
>>
>> Using what cache property is a user-space program's choice.
>>
>> While you are override the WC with CACHED mapping. This is not correct
>> in the concept!
>>
> Please explain why you think that this isn't correct.

This is NOT always correct because:

when etnaviv_gem_prime_import_sg_table() is called,

drm/etnaviv is importing buffer from other drivers (for example drm/loongson or drm/ingenic).

drm/etnaviv is the importer, and drm/loongson or drm/ingenic is the exporter.


While drm/etnaviv using the WC mapping by default, which may cause *cache aliasing*,


Because the imported buffer originally belong to the KMS driver side.

The KMS driver may choose cached mapping by default.


For drm/ingenic(jz4770), the BO will be cached, but their hardware can't guarantee coherency.

they have to flush the cache manually when do the frame buffer update(damage update or dirty update).

Because cached mapping is more fast than the WC mapping.




For drm/loongson, shared buffer have to pinned into GTT,

By default, we using the cached mapping for the buffers in the system RAM;

But we are lucky, our the hardware bless us,

the hardware maintain the cache coherency for us.


While drm/etnaviv choose the WC mapping for the imported buffer blindly,

when doing the import,

*drm/etnaviv know nothing about the original buffer's caching property.*

This is the problem.


Currently, I guess drm/etnaviv only works for drm/imx,

because drm/imx choose WC mapping as the cache property by default for 
the dumb buffer.

The cache property mapping match, so it works.

But their no communications between the KMS driver and RO driver.


I think, drm/etanviv will also wrong on the ARM64 platforms where cache 
coherency is maintained by the hardware.

If the KMS driver using cached mapping, while drm/etnaviv using WC mapping,

There still will have problems.

I will go to find the hardware, and do the testing for you.


> If using WC
> mappings cause a potential loss of coherency on your platform, then we
> can not allow the userspace driver to use WC mappings.

I have explained with my broken English,

The point is that loss of coherency may because KMS-RO framework have 
defect,

maybe software side bug, please don't make the conclusion too fast.


Xorg DDX driver (EXA base DDX for an example) fallback to the CPU do the 
software rendering.

It is even faster than the hardware accelerated implement,

especially in the case where the CPU is fast and the GPU is weak.

LS7A1000(GC1000) + LS3A5000@2.5Ghz X4 core is such a case.

For Discrete on-board VRAM we will using the WC mapping, it will not be 
shared with other driver,

The CPU use it themself, this is definitionally OK.

In practice, it works like a charm.

For buffers in the system RAM and GTT,

We will using cached mapping for faster CPU software rendering.

hard-coded by the driver.


We are benefited so much from the cached mapping.

While when using write-combine caching mapping for a buffer on LoongArch 
and Loongson Mips CPU,

It is OK for CPU write.

But when you want read from such buffer by the CPU, it is just boil down 
to uncached.

For our platform, uncached read serve as a kind of SYNC.

It will flush the data in the CPU's write buffer.

This cause performance drop.


But we still can use the write-combine caching mapping in the 
circumstances where do not need the CPU read.

So allow write-combine is providing one more choice to the user-space,

It is just that use it at your own risk.


> As I would like to keep the option of WC mappings,

Yeah, I know what's reason.

Because on the platform you love (imx6q, imx8q, for an example),

The hardware don't maintain the cache coherency.

So if fallback to software rendering, you will need to flush the cache frequency.

the performance will drop very much, right?


-- 
Jingfeng

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Robin Murphy]

On 2023-06-20 10:47, Sui Jingfeng wrote:
> From: Sui Jingfeng <suijingfeng@loongson.cn>
> 
> Loongson CPUs maintain cache coherency by hardware, which means that the
> data in the CPU cache is identical to the data in main system memory. As
> for the peripheral device, most of Loongson chips chose to define the
> peripherals as DMA coherent by default, device drivers do not need to
> maintain the coherency between a processor and an I/O device manually.
> 
> There are exceptions, for LS2K1000 SoC, part of peripheral device can be
> configured as DMA non-coherent. But there is no released version of such
> firmware exist in the market. Peripherals of older LS2K1000 is also DMA
> non-coherent, but they are nearly outdated. So, those are trivial cases.
> 
> Nevertheless, kernel space still need to do the probe work, because vivante
> GPU IP has been integrated into various platform. Hence, this patch add
> runtime detection code to probe if a specific GPU is DMA coherent, If the
> answer is yes, we are going to utilize such features. On Loongson platform,
> When a buffer is accessed by both the GPU and the CPU, the driver should
> prefer ETNA_BO_CACHED over ETNA_BO_WC.
> 
> This patch also add a new parameter: etnaviv_param_gpu_coherent, which
> allow userspace to know if such a feature is available. Because
> write-combined BO is still preferred in some case, especially where don't
> need CPU read, for example, uploading compiled shader bin.
> 
> Cc: Lucas Stach <l.stach@pengutronix.de>
> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
> Cc: Philipp Zabel <p.zabel@pengutronix.de>
> Cc: Bjorn Helgaas <bhelgaas@google.com>
> Cc: Daniel Vetter <daniel@ffwll.ch>
> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
> ---
>   drivers/gpu/drm/etnaviv/etnaviv_drv.c       | 35 +++++++++++++++++++++
>   drivers/gpu/drm/etnaviv/etnaviv_drv.h       |  6 ++++
>   drivers/gpu/drm/etnaviv/etnaviv_gem.c       | 22 ++++++++++---
>   drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c |  7 ++++-
>   drivers/gpu/drm/etnaviv/etnaviv_gpu.c       |  4 +++
>   include/uapi/drm/etnaviv_drm.h              |  1 +
>   6 files changed, 70 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> index 0a365e96d371..d8e788aa16cb 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
> @@ -5,7 +5,9 @@
>   
>   #include <linux/component.h>
>   #include <linux/dma-mapping.h>
> +#include <linux/dma-map-ops.h>

/*
  * This header is for implementations of dma_map_ops and related code.
  * It should not be included in drivers just using the DMA API.
  */

>   #include <linux/module.h>
> +#include <linux/of_address.h>
>   #include <linux/of_platform.h>
>   #include <linux/uaccess.h>
>   
> @@ -24,6 +26,34 @@
>   #include "etnaviv_pci_drv.h"
>   #include "etnaviv_perfmon.h"
>   
> +static struct device_node *etnaviv_of_first_available_node(void)
> +{
> +	struct device_node *core_node;
> +
> +	for_each_compatible_node(core_node, NULL, "vivante,gc") {
> +		if (of_device_is_available(core_node))
> +			return core_node;
> +	}
> +
> +	return NULL;
> +}
> +
> +static bool etnaviv_is_dma_coherent(struct device *dev)
> +{
> +	struct device_node *np;
> +	bool coherent;
> +
> +	np = etnaviv_of_first_available_node();
> +	if (np) {
> +		coherent = of_dma_is_coherent(np);
> +		of_node_put(np);
> +	} else {
> +		coherent = dev_is_dma_coherent(dev);
> +	}

Please use device_get_dma_attr() like other well-behaved drivers.

> +
> +	return coherent;
> +}
> +
>   /*
>    * etnaviv private data construction and destructions:
>    */
> @@ -52,6 +82,11 @@ etnaviv_alloc_private(struct device *dev, struct drm_device *drm)
>   		return ERR_PTR(-ENOMEM);
>   	}
>   
> +	priv->dma_coherent = etnaviv_is_dma_coherent(dev);
> +
> +	if (priv->dma_coherent)
> +		drm_info(drm, "%s is dma coherent\n", dev_name(dev));

I'm pretty sure the end-user doesn't care.

> +
>   	return priv;
>   }
>   
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> index 9cd72948cfad..644e5712c050 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
> @@ -46,6 +46,12 @@ struct etnaviv_drm_private {
>   	struct xarray active_contexts;
>   	u32 next_context_id;
>   
> +	/*
> +	 * If true, the GPU is capable of snooping cpu cache. Here, it
> +	 * also means that cache coherency is enforced by the hardware.
> +	 */
> +	bool dma_coherent;
> +
>   	/* list of GEM objects: */
>   	struct mutex gem_lock;
>   	struct list_head gem_list;
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
> index b5f73502e3dd..39bdc3774f2d 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
> @@ -343,6 +343,7 @@ void *etnaviv_gem_vmap(struct drm_gem_object *obj)
>   static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
>   {
>   	struct page **pages;
> +	pgprot_t prot;
>   
>   	lockdep_assert_held(&obj->lock);
>   
> @@ -350,8 +351,19 @@ static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
>   	if (IS_ERR(pages))
>   		return NULL;
>   
> -	return vmap(pages, obj->base.size >> PAGE_SHIFT,
> -			VM_MAP, pgprot_writecombine(PAGE_KERNEL));
> +	switch (obj->flags) {
> +	case ETNA_BO_CACHED:
> +		prot = PAGE_KERNEL;
> +		break;
> +	case ETNA_BO_UNCACHED:
> +		prot = pgprot_noncached(PAGE_KERNEL);
> +		break;
> +	case ETNA_BO_WC:
> +	default:
> +		prot = pgprot_writecombine(PAGE_KERNEL);
> +	}
> +
> +	return vmap(pages, obj->base.size >> PAGE_SHIFT, VM_MAP, prot);
>   }
>   
>   static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
> @@ -369,6 +381,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
>   {
>   	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>   	struct drm_device *dev = obj->dev;
> +	struct etnaviv_drm_private *priv = dev->dev_private;
>   	bool write = !!(op & ETNA_PREP_WRITE);
>   	int ret;
>   
> @@ -395,7 +408,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
>   			return ret == 0 ? -ETIMEDOUT : ret;
>   	}
>   
> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {

TBH the existing condition already looks a bit sketchy - even if 
userspace has a non-cacheable mapping, a DMA sync may need to do things 
other than cache maintenance - but this change certainly isn't making 
things any better. If you can demonstrate a *measurable* and significant 
overhead from calling dma_sync_sgtable_*() on your platform when the 
device is coherent and nothing is bounce-buffered, then the first thing 
we can do is look at making that quicker (for everyone). Otherwise, this 
seems like the famous bad kind of premature optimisation.

Thanks,
Robin.

>   		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
>   					 etnaviv_op_to_dma_dir(op));
>   		etnaviv_obj->last_cpu_prep_op = op;
> @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
>   {
>   	struct drm_device *dev = obj->dev;
>   	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
> +	struct etnaviv_drm_private *priv = dev->dev_private;
>   
> -	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
> +	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>   		/* fini without a prep is almost certainly a userspace error */
>   		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
>   		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> index 3524b5811682..754126992264 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
> @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
>   struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
>   	struct dma_buf_attachment *attach, struct sg_table *sgt)
>   {
> +	struct etnaviv_drm_private *priv = dev->dev_private;
>   	struct etnaviv_gem_object *etnaviv_obj;
>   	size_t size = PAGE_ALIGN(attach->dmabuf->size);
> +	u32 cache_flags = ETNA_BO_WC;
>   	int ret, npages;
>   
> -	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
> +	if (priv->dma_coherent)
> +		cache_flags = ETNA_BO_CACHED;
> +
> +	ret = etnaviv_gem_new_private(dev, size, cache_flags,
>   				      &etnaviv_gem_prime_ops, &etnaviv_obj);
>   	if (ret < 0)
>   		return ERR_PTR(ret);
> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> index d6a21e97feb1..d99ac675ce8b 100644
> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
> @@ -164,6 +164,10 @@ int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value)
>   		*value = gpu->identity.eco_id;
>   		break;
>   
> +	case ETNAVIV_PARAM_GPU_COHERENT:
> +		*value = priv->dma_coherent;
> +		break;
> +
>   	default:
>   		DBG("%s: invalid param: %u", dev_name(gpu->dev), param);
>   		return -EINVAL;
> diff --git a/include/uapi/drm/etnaviv_drm.h b/include/uapi/drm/etnaviv_drm.h
> index af024d90453d..76baf45d7158 100644
> --- a/include/uapi/drm/etnaviv_drm.h
> +++ b/include/uapi/drm/etnaviv_drm.h
> @@ -77,6 +77,7 @@ struct drm_etnaviv_timespec {
>   #define ETNAVIV_PARAM_GPU_PRODUCT_ID                0x1c
>   #define ETNAVIV_PARAM_GPU_CUSTOMER_ID               0x1d
>   #define ETNAVIV_PARAM_GPU_ECO_ID                    0x1e
> +#define ETNAVIV_PARAM_GPU_COHERENT                  0x1f
>   
>   #define ETNA_MAX_PIPES 4
>   

Re: [PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

Hi,

On 2023/6/23 19:52, Robin Murphy wrote:
> On 2023-06-20 10:47, Sui Jingfeng wrote:
>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>
>> Loongson CPUs maintain cache coherency by hardware, which means that the
>> data in the CPU cache is identical to the data in main system memory. As
>> for the peripheral device, most of Loongson chips chose to define the
>> peripherals as DMA coherent by default, device drivers do not need to
>> maintain the coherency between a processor and an I/O device manually.
>>
>> There are exceptions, for LS2K1000 SoC, part of peripheral device can be
>> configured as DMA non-coherent. But there is no released version of such
>> firmware exist in the market. Peripherals of older LS2K1000 is also DMA
>> non-coherent, but they are nearly outdated. So, those are trivial cases.
>>
>> Nevertheless, kernel space still need to do the probe work, because 
>> vivante
>> GPU IP has been integrated into various platform. Hence, this patch add
>> runtime detection code to probe if a specific GPU is DMA coherent, If 
>> the
>> answer is yes, we are going to utilize such features. On Loongson 
>> platform,
>> When a buffer is accessed by both the GPU and the CPU, the driver should
>> prefer ETNA_BO_CACHED over ETNA_BO_WC.
>>
>> This patch also add a new parameter: etnaviv_param_gpu_coherent, which
>> allow userspace to know if such a feature is available. Because
>> write-combined BO is still preferred in some case, especially where 
>> don't
>> need CPU read, for example, uploading compiled shader bin.
>>
>> Cc: Lucas Stach <l.stach@pengutronix.de>
>> Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
>> Cc: Philipp Zabel <p.zabel@pengutronix.de>
>> Cc: Bjorn Helgaas <bhelgaas@google.com>
>> Cc: Daniel Vetter <daniel@ffwll.ch>
>> Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
>> ---
>>   drivers/gpu/drm/etnaviv/etnaviv_drv.c       | 35 +++++++++++++++++++++
>>   drivers/gpu/drm/etnaviv/etnaviv_drv.h       |  6 ++++
>>   drivers/gpu/drm/etnaviv/etnaviv_gem.c       | 22 ++++++++++---
>>   drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c |  7 ++++-
>>   drivers/gpu/drm/etnaviv/etnaviv_gpu.c       |  4 +++
>>   include/uapi/drm/etnaviv_drm.h              |  1 +
>>   6 files changed, 70 insertions(+), 5 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c 
>> b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> index 0a365e96d371..d8e788aa16cb 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
>> @@ -5,7 +5,9 @@
>>     #include <linux/component.h>
>>   #include <linux/dma-mapping.h>
>> +#include <linux/dma-map-ops.h>
>
> /*
>  * This header is for implementations of dma_map_ops and related code.
>  * It should not be included in drivers just using the DMA API.
>  */
>
1)

The driver can not pass the compile, without include the 
linux/dma-map-ops.h


You may be right, but Lucas suggest using dev_is_dma_coherent() function.

because the dev_is_dma_coherent() function is defined in dma-map-ops.h.


for our platform its:


```

static inline bool dev_is_dma_coherent(struct device *dev)
{
     return true;
}

```

2. drm/exynos and drm/msm also include this handler


```

$ cd drivers/gpu/drm

$ find . -name "*.c" -type f | xargs grep "dma-map-ops.h"

./exynos/exynos_drm_dma.c:#include <linux/dma-map-ops.h>
./msm/msm_gem.c:#include <linux/dma-map-ops.h>

```


>>   #include <linux/module.h>
>> +#include <linux/of_address.h>
>>   #include <linux/of_platform.h>
>>   #include <linux/uaccess.h>
>>   @@ -24,6 +26,34 @@
>>   #include "etnaviv_pci_drv.h"
>>   #include "etnaviv_perfmon.h"
>>   +static struct device_node *etnaviv_of_first_available_node(void)
>> +{
>> +    struct device_node *core_node;
>> +
>> +    for_each_compatible_node(core_node, NULL, "vivante,gc") {
>> +        if (of_device_is_available(core_node))
>> +            return core_node;
>> +    }
>> +
>> +    return NULL;
>> +}
>> +
>> +static bool etnaviv_is_dma_coherent(struct device *dev)
>> +{
>> +    struct device_node *np;
>> +    bool coherent;
>> +
>> +    np = etnaviv_of_first_available_node();
>> +    if (np) {
>> +        coherent = of_dma_is_coherent(np);
>> +        of_node_put(np);
>> +    } else {
>> +        coherent = dev_is_dma_coherent(dev);
>> +    }
>
> Please use device_get_dma_attr() like other well-behaved drivers.
>
OK, I agree with you.


```

priv->dma_coherent = device_get_dma_attr(&pdev->dev) == DEV_DMA_COHERENT;

```

Lucas,  Is above code Snippet looks fine to you?


>> +
>> +    return coherent;
>> +}
>> +
>>   /*
>>    * etnaviv private data construction and destructions:
>>    */
>> @@ -52,6 +82,11 @@ etnaviv_alloc_private(struct device *dev, struct 
>> drm_device *drm)
>>           return ERR_PTR(-ENOMEM);
>>       }
>>   +    priv->dma_coherent = etnaviv_is_dma_coherent(dev);
>> +
>> +    if (priv->dma_coherent)
>> +        drm_info(drm, "%s is dma coherent\n", dev_name(dev));
>
> I'm pretty sure the end-user doesn't care.
>
OK, i will delete it at the next version.

>> +
>>       return priv;
>>   }
>>   diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h 
>> b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> index 9cd72948cfad..644e5712c050 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
>> @@ -46,6 +46,12 @@ struct etnaviv_drm_private {
>>       struct xarray active_contexts;
>>       u32 next_context_id;
>>   +    /*
>> +     * If true, the GPU is capable of snooping cpu cache. Here, it
>> +     * also means that cache coherency is enforced by the hardware.
>> +     */
>> +    bool dma_coherent;
>> +
>>       /* list of GEM objects: */
>>       struct mutex gem_lock;
>>       struct list_head gem_list;
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c 
>> b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
>> index b5f73502e3dd..39bdc3774f2d 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
>> @@ -343,6 +343,7 @@ void *etnaviv_gem_vmap(struct drm_gem_object *obj)
>>   static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
>>   {
>>       struct page **pages;
>> +    pgprot_t prot;
>>         lockdep_assert_held(&obj->lock);
>>   @@ -350,8 +351,19 @@ static void *etnaviv_gem_vmap_impl(struct 
>> etnaviv_gem_object *obj)
>>       if (IS_ERR(pages))
>>           return NULL;
>>   -    return vmap(pages, obj->base.size >> PAGE_SHIFT,
>> -            VM_MAP, pgprot_writecombine(PAGE_KERNEL));
>> +    switch (obj->flags) {
>> +    case ETNA_BO_CACHED:
>> +        prot = PAGE_KERNEL;
>> +        break;
>> +    case ETNA_BO_UNCACHED:
>> +        prot = pgprot_noncached(PAGE_KERNEL);
>> +        break;
>> +    case ETNA_BO_WC:
>> +    default:
>> +        prot = pgprot_writecombine(PAGE_KERNEL);
>> +    }
>> +
>> +    return vmap(pages, obj->base.size >> PAGE_SHIFT, VM_MAP, prot);
>>   }
>>     static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
>> @@ -369,6 +381,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object 
>> *obj, u32 op,
>>   {
>>       struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>>       struct drm_device *dev = obj->dev;
>> +    struct etnaviv_drm_private *priv = dev->dev_private;
>>       bool write = !!(op & ETNA_PREP_WRITE);
>>       int ret;
>>   @@ -395,7 +408,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object 
>> *obj, u32 op,
>>               return ret == 0 ? -ETIMEDOUT : ret;
>>       }
>>   -    if (etnaviv_obj->flags & ETNA_BO_CACHED) {
>> +    if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>
> TBH the existing condition already looks a bit sketchy - even if 
> userspace has a non-cacheable mapping, a DMA sync may need to do 
> things other than cache maintenance - but this change certainly isn't 
> making things any better. If you can demonstrate a *measurable* and 
> significant overhead from calling dma_sync_sgtable_*() on your 
> platform when the device is coherent and nothing is bounce-buffered, 
> then the first thing we can do is look at making that quicker (for 
> everyone). Otherwise, this seems like the famous bad kind of premature 
> optimisation.
>
OK, I agree with you, both you and Lucas told to remove this.

I will remove this at next version.


It doesn't hurt much to the performance. On our platform, the 
performance is GPU bound.


> Thanks,
> Robin.
>
>> dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
>>                        etnaviv_op_to_dma_dir(op));
>>           etnaviv_obj->last_cpu_prep_op = op;
>> @@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
>>   {
>>       struct drm_device *dev = obj->dev;
>>       struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
>> +    struct etnaviv_drm_private *priv = dev->dev_private;
>>   -    if (etnaviv_obj->flags & ETNA_BO_CACHED) {
>> +    if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
>>           /* fini without a prep is almost certainly a userspace 
>> error */
>>           WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
>>           dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c 
>> b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>> index 3524b5811682..754126992264 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
>> @@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops 
>> etnaviv_gem_prime_ops = {
>>   struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct 
>> drm_device *dev,
>>       struct dma_buf_attachment *attach, struct sg_table *sgt)
>>   {
>> +    struct etnaviv_drm_private *priv = dev->dev_private;
>>       struct etnaviv_gem_object *etnaviv_obj;
>>       size_t size = PAGE_ALIGN(attach->dmabuf->size);
>> +    u32 cache_flags = ETNA_BO_WC;
>>       int ret, npages;
>>   -    ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
>> +    if (priv->dma_coherent)
>> +        cache_flags = ETNA_BO_CACHED;
>> +
>> +    ret = etnaviv_gem_new_private(dev, size, cache_flags,
>>                         &etnaviv_gem_prime_ops, &etnaviv_obj);
>>       if (ret < 0)
>>           return ERR_PTR(ret);
>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c 
>> b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> index d6a21e97feb1..d99ac675ce8b 100644
>> --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
>> @@ -164,6 +164,10 @@ int etnaviv_gpu_get_param(struct etnaviv_gpu 
>> *gpu, u32 param, u64 *value)
>>           *value = gpu->identity.eco_id;
>>           break;
>>   +    case ETNAVIV_PARAM_GPU_COHERENT:
>> +        *value = priv->dma_coherent;
>> +        break;
>> +
>>       default:
>>           DBG("%s: invalid param: %u", dev_name(gpu->dev), param);
>>           return -EINVAL;
>> diff --git a/include/uapi/drm/etnaviv_drm.h 
>> b/include/uapi/drm/etnaviv_drm.h
>> index af024d90453d..76baf45d7158 100644
>> --- a/include/uapi/drm/etnaviv_drm.h
>> +++ b/include/uapi/drm/etnaviv_drm.h
>> @@ -77,6 +77,7 @@ struct drm_etnaviv_timespec {
>>   #define ETNAVIV_PARAM_GPU_PRODUCT_ID                0x1c
>>   #define ETNAVIV_PARAM_GPU_CUSTOMER_ID               0x1d
>>   #define ETNAVIV_PARAM_GPU_ECO_ID                    0x1e
>> +#define ETNAVIV_PARAM_GPU_COHERENT                  0x1f
>>     #define ETNA_MAX_PIPES 4

-- 
Jingfeng

[PATCH v10 08/11] drm/etnaviv: Add a dedicated function to create the virtual master

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

After introducing the etnaviv_of_first_available_node() helper, the
creation of the virtual master platform device can also be simplified.
So, switch to etnaviv_create_virtual_master() function.

Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_drv.c | 43 ++++++++++++++++-----------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index d8e788aa16cb..47b2cdbb53e2 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -767,10 +767,32 @@ static void etnaviv_destroy_platform_device(struct platform_device **ppdev)
 	*ppdev = NULL;
 }
 
+static int etnaviv_create_virtual_master(void)
+{
+	struct platform_device **master = &etnaviv_platform_device;
+	struct device_node *np;
+
+	/*
+	 * If the DT contains at least one available GPU device, instantiate
+	 * the DRM platform device.
+	 */
+	np = etnaviv_of_first_available_node();
+	if (np) {
+		int ret;
+
+		of_node_put(np);
+
+		ret = etnaviv_create_platform_device("etnaviv", master);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static int __init etnaviv_init(void)
 {
 	int ret;
-	struct device_node *np;
 
 	etnaviv_validate_init();
 
@@ -786,22 +808,9 @@ static int __init etnaviv_init(void)
 	if (ret != 0)
 		goto unregister_platform_driver;
 
-	/*
-	 * If the DT contains at least one available GPU device, instantiate
-	 * the DRM platform device.
-	 */
-	for_each_compatible_node(np, NULL, "vivante,gc") {
-		if (!of_device_is_available(np))
-			continue;
-		of_node_put(np);
-
-		ret = etnaviv_create_platform_device("etnaviv",
-						     &etnaviv_platform_device);
-		if (ret)
-			goto unregister_platform_driver;
-
-		break;
-	}
+	ret = etnaviv_create_virtual_master();
+	if (ret)
+		goto unregister_platform_driver;
 
 	return ret;
 
-- 
2.25.1

[PATCH v10 09/11] drm/etnaviv: Clean up etnaviv_pdev_probe() function

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

Add a dedicate function to do the DMA configuration to the virtual master.
Also replace the &pdev->dev with dev.

Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_drv.c | 65 +++++++++++++++------------
 1 file changed, 36 insertions(+), 29 deletions(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 47b2cdbb53e2..8907cdb8a1f8 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -54,6 +54,40 @@ static bool etnaviv_is_dma_coherent(struct device *dev)
 	return coherent;
 }
 
+static int etnaviv_of_dma_configure(struct device *dev)
+{
+	struct device_node *first_node;
+
+	/*
+	 * PTA and MTLB can have 40 bit base addresses, but
+	 * unfortunately, an entry in the MTLB can only point to a
+	 * 32 bit base address of a STLB. Moreover, to initialize the
+	 * MMU we need a command buffer with a 32 bit address because
+	 * without an MMU there is only an indentity mapping between
+	 * the internal 32 bit addresses and the bus addresses.
+	 *
+	 * To make things easy, we set the dma_coherent_mask to 32
+	 * bit to make sure we are allocating the command buffers and
+	 * TLBs in the lower 4 GiB address space.
+	 */
+	if (dma_set_mask(dev, DMA_BIT_MASK(40)) ||
+	    dma_set_coherent_mask(dev, DMA_BIT_MASK(32))) {
+		dev_err(dev, "No suitable DMA available\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * Apply the same DMA configuration to the virtual etnaviv
+	 * device as the GPU we found. This assumes that all Vivante
+	 * GPUs in the system share the same DMA constraints.
+	 */
+	first_node = etnaviv_of_first_available_node();
+	if (first_node)
+		of_dma_configure(dev, first_node, true);
+
+	return 0;
+}
+
 /*
  * etnaviv private data construction and destructions:
  */
@@ -664,7 +698,6 @@ static const struct component_master_ops etnaviv_master_ops = {
 static int etnaviv_pdev_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct device_node *first_node = NULL;
 	struct component_match *match = NULL;
 
 	if (!dev->platform_data) {
@@ -674,10 +707,7 @@ static int etnaviv_pdev_probe(struct platform_device *pdev)
 			if (!of_device_is_available(core_node))
 				continue;
 
-			if (!first_node)
-				first_node = core_node;
-
-			drm_of_component_match_add(&pdev->dev, &match,
+			drm_of_component_match_add(dev, &match,
 						   component_compare_of, core_node);
 		}
 	} else {
@@ -688,31 +718,8 @@ static int etnaviv_pdev_probe(struct platform_device *pdev)
 			component_match_add(dev, &match, component_compare_dev_name, names[i]);
 	}
 
-	/*
-	 * PTA and MTLB can have 40 bit base addresses, but
-	 * unfortunately, an entry in the MTLB can only point to a
-	 * 32 bit base address of a STLB. Moreover, to initialize the
-	 * MMU we need a command buffer with a 32 bit address because
-	 * without an MMU there is only an indentity mapping between
-	 * the internal 32 bit addresses and the bus addresses.
-	 *
-	 * To make things easy, we set the dma_coherent_mask to 32
-	 * bit to make sure we are allocating the command buffers and
-	 * TLBs in the lower 4 GiB address space.
-	 */
-	if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(40)) ||
-	    dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32))) {
-		dev_dbg(&pdev->dev, "No suitable DMA available\n");
+	if (etnaviv_of_dma_configure(dev))
 		return -ENODEV;
-	}
-
-	/*
-	 * Apply the same DMA configuration to the virtual etnaviv
-	 * device as the GPU we found. This assumes that all Vivante
-	 * GPUs in the system share the same DMA constraints.
-	 */
-	if (first_node)
-		of_dma_configure(&pdev->dev, first_node, true);
 
 	return component_master_add_with_match(dev, &etnaviv_master_ops, match);
 }
-- 
2.25.1

[PATCH v10 10/11] drm/etnaviv: Keep the curly brace aligned

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

No functional change.

Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_drv.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 8907cdb8a1f8..c9f502a89de6 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -185,7 +185,7 @@ static int etnaviv_open(struct drm_device *dev, struct drm_file *file)
 			drm_sched_entity_init(&ctx->sched_entity[i],
 					      DRM_SCHED_PRIORITY_NORMAL, &sched,
 					      1, NULL);
-			}
+		}
 	}
 
 	file->driver_priv = ctx;
-- 
2.25.1

Re: [PATCH v10 00/11] drm/etnaviv: Add pci device driver support

[Christian Gmeiner]

Hi

>
> From: Sui Jingfeng <suijingfeng@loongson.cn>
>
> There is a Vivante GC1000 (v5037) in LS2K1000 and LS7A1000, this GPU is a
> PCI device, and it has 2D and 3D cores in the same core. This series is
> trying to add PCI device driver support to drm/etnaviv.
>

Is it possible to get the lspci output for the GPU? Something like
this: sudo lspci -vvv -s ...


thanks
--
Christian Gmeiner, MSc

https://christian-gmeiner.info/privacypolicy

Re: [PATCH v10 00/11] drm/etnaviv: Add pci device driver support

[Sui Jingfeng]

Hi

On 2023/6/21 15:55, Christian Gmeiner wrote:
> Hi
>
>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>
>> There is a Vivante GC1000 (v5037) in LS2K1000 and LS7A1000, this GPU is a
>> PCI device, and it has 2D and 3D cores in the same core. This series is
>> trying to add PCI device driver support to drm/etnaviv.
>>
> Is it possible to get the lspci output for the GPU? Something like
> this: sudo lspci -vvv -s ...

Yes,


sudo lspci -vvvxxx -s 00:06.0
00:06.0 Multimedia video controller: Loongson Technology LLC Vivante GPU 
(Graphics Processing Unit) (rev 01)
     Control: I/O+ Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop+ ParErr- 
Stepping- SERR- FastB2B- DisINTx-
     Status: Cap- 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- 
<TAbort- <MAbort- >SERR- <PERR- INTx-
     Latency: 0
     Interrupt: pin A routed to IRQ 51
     NUMA node: 0
     Region 0: Memory at e0035200000 (64-bit, non-prefetchable) [size=256K]
     Region 2: Memory at e0030000000 (64-bit, non-prefetchable) [size=64M]
     Region 4: Memory at e0035240000 (64-bit, non-prefetchable) [size=64K]
     Kernel driver in use: etnaviv
     Kernel modules: etnaviv
00: 14 00 15 7a 27 00 00 00 01 00 00 04 00 00 80 00
10: 04 00 20 35 00 00 00 00 04 00 00 30 00 00 00 00
20: 04 00 24 35 00 00 00 00 00 00 00 00 00 00 00 00
30: 00 00 00 00 00 00 00 00 00 00 00 00 5d 01 00 00
40: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
50: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
60: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
70: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
80: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
90: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
a0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
b0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
c0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
d0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
e0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff
f0: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff



> thanks
> --
> Christian Gmeiner, MSc
>
> https://christian-gmeiner.info/privacypolicy

-- 
Jingfeng

Re: [PATCH v10 00/11] drm/etnaviv: Add pci device driver support

[Sui Jingfeng]

Hi,

Below is the gpu info cat from the debugfs,

I guess this is also what you want ?


[root@fedora 0]# cat gpu

0000:00:06.0 Status:
     identity
      model: 0x1000
      revision: 0x5037
      product_id: 0x0
      customer_id: 0x0
      eco_id: 0x0
     features
      major_features: 0xe0286eed
      minor_features0: 0xe9799eff
      minor_features1: 0xbe13b2d9
      minor_features2: 0xca114080
      minor_features3: 0x0e0100a1
      minor_features4: 0x00000000
      minor_features5: 0x00000000
      minor_features6: 0x00000000
      minor_features7: 0x00000000
      minor_features8: 0x00000000
      minor_features9: 0x00000000
      minor_features10: 0x00000000
      minor_features11: 0x00000000
     specs
      stream_count:  4
      register_max: 64
      thread_count: 512
      vertex_cache_size: 8
      shader_core_count: 2
      nn_core_count: 0
      pixel_pipes: 1
      vertex_output_buffer_size: 512
      buffer_size: 0
      instruction_count: 256
      num_constants: 576
      varyings_count: 8
     axi: 0x00000051
     idle: 0x7ffffffe
      FE is not idle
     DMA is running
      address 0: 0x00002ac0
      address 1: 0x00002ac8
      state 0: 0x00000800
      state 1: 0x00000812
      last fetch 64 bit word: 0x380000c8 0x00000701


On 2023/6/21 15:55, Christian Gmeiner wrote:
> Hi
>
>> From: Sui Jingfeng <suijingfeng@loongson.cn>
>>
>> There is a Vivante GC1000 (v5037) in LS2K1000 and LS7A1000, this GPU is a
>> PCI device, and it has 2D and 3D cores in the same core. This series is
>> trying to add PCI device driver support to drm/etnaviv.
>>
> Is it possible to get the lspci output for the GPU? Something like
> this: sudo lspci -vvv -s ...
>
>
> thanks
> --
> Christian Gmeiner, MSc
>
> https://christian-gmeiner.info/privacypolicy

-- 
Jingfeng

[PATCH v10 00/11] drm/etnaviv: Add pci device driver support

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

There is a Vivante GC1000 (v5037) in LS2K1000 and LS7A1000, this GPU is a
PCI device, and it has 2D and 3D cores in the same core. This series is
trying to add PCI device driver support to drm/etnaviv.

v6:
	* Fix build issue on system without CONFIG_PCI enabled
v7:
	* Add a separate patch for the platform driver rearrangement (Bjorn)
	* Switch to runtime check if the GPU is dma coherent or not (Lucas)
	* Add ETNAVIV_PARAM_GPU_COHERENT to allow userspace to query (Lucas)
	* Remove etnaviv_gpu.no_clk member (Lucas)
	* Various Typos and coding style fixed (Bjorn)
v8:
	* Fix typos and remove unnecessary header included (Bjorn).
	* Add a dedicated function to create the virtual master platform
	  device.
v9:
	* Use PCI_VDEVICE() macro (Bjorn)
	* Add trivial stubs for the PCI driver (Bjorn)
	* Remove a redundant dev_err() usage (Bjorn)
	* Clean up etnaviv_pdev_probe() with etnaviv_of_first_available_node()
v10:
	* Add one more cleanup patch
	* Resolve the conflict with a patch from Rob
	* Make the dummy PCI stub inlined
	* Print only if the platform is dma-coherrent

Rob Herring (1):
  drm/etnaviv: Replace of_platform.h with explicit includes

Sui Jingfeng (10):
  drm/etnaviv: Add a dedicated function to register an irq handler
  drm/etnaviv: Add a dedicated function to get various clocks
  drm/etnaviv: Add dedicated functions to create and destroy platform
    device
  drm/etnaviv: Add helpers for private data construction and destruction
  drm/etnaviv: Allow bypass component framework
  drm/etnaviv: Add driver support for the PCI devices
  drm/etnaviv: Add support for the dma coherent device
  drm/etnaviv: Add a dedicated function to create the virtual master
  drm/etnaviv: Clean up etnaviv_pdev_probe() function
  drm/etnaviv: Keep the curly brace aligned

 drivers/gpu/drm/etnaviv/Kconfig             |  10 +
 drivers/gpu/drm/etnaviv/Makefile            |   2 +
 drivers/gpu/drm/etnaviv/etnaviv_drv.c       | 322 ++++++++++++++------
 drivers/gpu/drm/etnaviv/etnaviv_drv.h       |  10 +
 drivers/gpu/drm/etnaviv/etnaviv_gem.c       |  22 +-
 drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c |   7 +-
 drivers/gpu/drm/etnaviv/etnaviv_gpu.c       | 166 ++++++----
 drivers/gpu/drm/etnaviv/etnaviv_gpu.h       |   9 +
 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c   |  75 +++++
 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h   |  18 ++
 include/uapi/drm/etnaviv_drm.h              |   1 +
 11 files changed, 481 insertions(+), 161 deletions(-)
 create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.c
 create mode 100644 drivers/gpu/drm/etnaviv/etnaviv_pci_drv.h

-- 
2.25.1

[PATCH v10 07/11] drm/etnaviv: Add support for the dma coherent device

[Sui Jingfeng]

From: Sui Jingfeng <suijingfeng@loongson.cn>

Loongson CPUs maintain cache coherency by hardware, which means that the
data in the CPU cache is identical to the data in main system memory. As
for the peripheral device, most of Loongson chips chose to define the
peripherals as DMA coherent by default, device drivers do not need to
maintain the coherency between a processor and an I/O device manually.

There are exceptions, for LS2K1000 SoC, part of peripheral device can be
configured as DMA non-coherent. But there is no released version of such
firmware exist in the market. Peripherals of older LS2K1000 is also DMA
non-coherent, but they are nearly outdated. So, those are trivial cases.

Nevertheless, kernel space still need to do the probe work, because vivante
GPU IP has been integrated into various platform. Hence, this patch add
runtime detection code to probe if a specific GPU is DMA coherent, If the
answer is yes, we are going to utilize such features. On Loongson platform,
When a buffer is accessed by both the GPU and the CPU, the driver should
prefer ETNA_BO_CACHED over ETNA_BO_WC.

This patch also add a new parameter: etnaviv_param_gpu_coherent, which
allow userspace to know if such a feature is available. Because
write-combined BO is still preferred in some case, especially where don't
need CPU read, for example, uploading compiled shader bin.

Cc: Lucas Stach <l.stach@pengutronix.de>
Cc: Christian Gmeiner <christian.gmeiner@gmail.com>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Daniel Vetter <daniel@ffwll.ch>
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
---
 drivers/gpu/drm/etnaviv/etnaviv_drv.c       | 35 +++++++++++++++++++++
 drivers/gpu/drm/etnaviv/etnaviv_drv.h       |  6 ++++
 drivers/gpu/drm/etnaviv/etnaviv_gem.c       | 22 ++++++++++---
 drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c |  7 ++++-
 drivers/gpu/drm/etnaviv/etnaviv_gpu.c       |  4 +++
 include/uapi/drm/etnaviv_drm.h              |  1 +
 6 files changed, 70 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 0a365e96d371..d8e788aa16cb 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -5,7 +5,9 @@
 
 #include <linux/component.h>
 #include <linux/dma-mapping.h>
+#include <linux/dma-map-ops.h>
 #include <linux/module.h>
+#include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/uaccess.h>
 
@@ -24,6 +26,34 @@
 #include "etnaviv_pci_drv.h"
 #include "etnaviv_perfmon.h"
 
+static struct device_node *etnaviv_of_first_available_node(void)
+{
+	struct device_node *core_node;
+
+	for_each_compatible_node(core_node, NULL, "vivante,gc") {
+		if (of_device_is_available(core_node))
+			return core_node;
+	}
+
+	return NULL;
+}
+
+static bool etnaviv_is_dma_coherent(struct device *dev)
+{
+	struct device_node *np;
+	bool coherent;
+
+	np = etnaviv_of_first_available_node();
+	if (np) {
+		coherent = of_dma_is_coherent(np);
+		of_node_put(np);
+	} else {
+		coherent = dev_is_dma_coherent(dev);
+	}
+
+	return coherent;
+}
+
 /*
  * etnaviv private data construction and destructions:
  */
@@ -52,6 +82,11 @@ etnaviv_alloc_private(struct device *dev, struct drm_device *drm)
 		return ERR_PTR(-ENOMEM);
 	}
 
+	priv->dma_coherent = etnaviv_is_dma_coherent(dev);
+
+	if (priv->dma_coherent)
+		drm_info(drm, "%s is dma coherent\n", dev_name(dev));
+
 	return priv;
 }
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index 9cd72948cfad..644e5712c050 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -46,6 +46,12 @@ struct etnaviv_drm_private {
 	struct xarray active_contexts;
 	u32 next_context_id;
 
+	/*
+	 * If true, the GPU is capable of snooping cpu cache. Here, it
+	 * also means that cache coherency is enforced by the hardware.
+	 */
+	bool dma_coherent;
+
 	/* list of GEM objects: */
 	struct mutex gem_lock;
 	struct list_head gem_list;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index b5f73502e3dd..39bdc3774f2d 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -343,6 +343,7 @@ void *etnaviv_gem_vmap(struct drm_gem_object *obj)
 static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
 {
 	struct page **pages;
+	pgprot_t prot;
 
 	lockdep_assert_held(&obj->lock);
 
@@ -350,8 +351,19 @@ static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj)
 	if (IS_ERR(pages))
 		return NULL;
 
-	return vmap(pages, obj->base.size >> PAGE_SHIFT,
-			VM_MAP, pgprot_writecombine(PAGE_KERNEL));
+	switch (obj->flags) {
+	case ETNA_BO_CACHED:
+		prot = PAGE_KERNEL;
+		break;
+	case ETNA_BO_UNCACHED:
+		prot = pgprot_noncached(PAGE_KERNEL);
+		break;
+	case ETNA_BO_WC:
+	default:
+		prot = pgprot_writecombine(PAGE_KERNEL);
+	}
+
+	return vmap(pages, obj->base.size >> PAGE_SHIFT, VM_MAP, prot);
 }
 
 static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op)
@@ -369,6 +381,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
 {
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
 	struct drm_device *dev = obj->dev;
+	struct etnaviv_drm_private *priv = dev->dev_private;
 	bool write = !!(op & ETNA_PREP_WRITE);
 	int ret;
 
@@ -395,7 +408,7 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
 			return ret == 0 ? -ETIMEDOUT : ret;
 	}
 
-	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
+	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
 		dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt,
 					 etnaviv_op_to_dma_dir(op));
 		etnaviv_obj->last_cpu_prep_op = op;
@@ -408,8 +421,9 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj)
 {
 	struct drm_device *dev = obj->dev;
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
+	struct etnaviv_drm_private *priv = dev->dev_private;
 
-	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
+	if (!priv->dma_coherent && etnaviv_obj->flags & ETNA_BO_CACHED) {
 		/* fini without a prep is almost certainly a userspace error */
 		WARN_ON(etnaviv_obj->last_cpu_prep_op == 0);
 		dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt,
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
index 3524b5811682..754126992264 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
@@ -112,11 +112,16 @@ static const struct etnaviv_gem_ops etnaviv_gem_prime_ops = {
 struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
 	struct dma_buf_attachment *attach, struct sg_table *sgt)
 {
+	struct etnaviv_drm_private *priv = dev->dev_private;
 	struct etnaviv_gem_object *etnaviv_obj;
 	size_t size = PAGE_ALIGN(attach->dmabuf->size);
+	u32 cache_flags = ETNA_BO_WC;
 	int ret, npages;
 
-	ret = etnaviv_gem_new_private(dev, size, ETNA_BO_WC,
+	if (priv->dma_coherent)
+		cache_flags = ETNA_BO_CACHED;
+
+	ret = etnaviv_gem_new_private(dev, size, cache_flags,
 				      &etnaviv_gem_prime_ops, &etnaviv_obj);
 	if (ret < 0)
 		return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index d6a21e97feb1..d99ac675ce8b 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -164,6 +164,10 @@ int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value)
 		*value = gpu->identity.eco_id;
 		break;
 
+	case ETNAVIV_PARAM_GPU_COHERENT:
+		*value = priv->dma_coherent;
+		break;
+
 	default:
 		DBG("%s: invalid param: %u", dev_name(gpu->dev), param);
 		return -EINVAL;
diff --git a/include/uapi/drm/etnaviv_drm.h b/include/uapi/drm/etnaviv_drm.h
index af024d90453d..76baf45d7158 100644
--- a/include/uapi/drm/etnaviv_drm.h
+++ b/include/uapi/drm/etnaviv_drm.h
@@ -77,6 +77,7 @@ struct drm_etnaviv_timespec {
 #define ETNAVIV_PARAM_GPU_PRODUCT_ID                0x1c
 #define ETNAVIV_PARAM_GPU_CUSTOMER_ID               0x1d
 #define ETNAVIV_PARAM_GPU_ECO_ID                    0x1e
+#define ETNAVIV_PARAM_GPU_COHERENT                  0x1f
 
 #define ETNA_MAX_PIPES 4
 
-- 
2.25.1