* [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to acquire gpu_metrics @ 2023-06-01 1:38 Wenyou Yang 2023-06-11 5:52 ` Yang, WenYou 2023-06-21 4:12 ` Lazar, Lijo 0 siblings, 2 replies; 4+ messages in thread From: Wenyou Yang @ 2023-06-01 1:38 UTC (permalink / raw) To: alexander.deucher, mario.limonciello, christian.koenig, Xinhui.Pan, evan.quan Cc: weiyuan2, richardqi.liang, amd-gfx, dri-devel, linux-kernel, Wenyou Yang To acquire the voltage and current info from gpu_metrics interface, but gpu_metrics_v2_3 doesn't contain them, and to be backward compatible, add new gpu_metrics_v2_4 structure. Acked-by: Evan Quan <evan.quan@amd.com> Signed-off-by: Wenyou Yang <WenYou.Yang@amd.com> --- .../gpu/drm/amd/include/kgd_pp_interface.h | 69 +++++++++++ .../gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c | 109 ++++++++++++++++-- drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c | 3 + 3 files changed, 172 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h index 9f542f6e19ed..0f37dafafcf9 100644 --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h @@ -892,4 +892,73 @@ struct gpu_metrics_v2_3 { uint16_t average_temperature_core[8]; // average CPU core temperature on APUs uint16_t average_temperature_l3[2]; }; + +struct gpu_metrics_v2_4 { + struct metrics_table_header common_header; + + /* Temperature */ + uint16_t temperature_gfx; + uint16_t temperature_soc; + uint16_t temperature_core[8]; + uint16_t temperature_l3[2]; + + /* Utilization */ + uint16_t average_gfx_activity; + uint16_t average_mm_activity; + + /* Driver attached timestamp (in ns) */ + uint64_t system_clock_counter; + + /* Power/Energy */ + uint16_t average_socket_power; + uint16_t average_cpu_power; + uint16_t average_soc_power; + uint16_t average_gfx_power; + uint16_t average_core_power[8]; + + /* Average clocks */ + uint16_t average_gfxclk_frequency; + uint16_t average_socclk_frequency; + uint16_t average_uclk_frequency; + uint16_t average_fclk_frequency; + uint16_t average_vclk_frequency; + uint16_t average_dclk_frequency; + + /* Current clocks */ + uint16_t current_gfxclk; + uint16_t current_socclk; + uint16_t current_uclk; + uint16_t current_fclk; + uint16_t current_vclk; + uint16_t current_dclk; + uint16_t current_coreclk[8]; + uint16_t current_l3clk[2]; + + /* Throttle status (ASIC dependent) */ + uint32_t throttle_status; + + /* Fans */ + uint16_t fan_pwm; + + uint16_t padding[3]; + + /* Throttle status (ASIC independent) */ + uint64_t indep_throttle_status; + + /* Average Temperature */ + uint16_t average_temperature_gfx; + uint16_t average_temperature_soc; + uint16_t average_temperature_core[8]; + uint16_t average_temperature_l3[2]; + + /* Power/Voltage */ + uint16_t average_cpu_voltage; + uint16_t average_soc_voltage; + uint16_t average_gfx_voltage; + + /* Power/Current */ + uint16_t average_cpu_current; + uint16_t average_soc_current; + uint16_t average_gfx_current; +}; #endif diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 067b4e0b026c..185d0b50ee8e 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -1854,6 +1854,86 @@ static ssize_t vangogh_get_gpu_metrics_v2_3(struct smu_context *smu, return sizeof(struct gpu_metrics_v2_3); } +static ssize_t vangogh_get_gpu_metrics_v2_4(struct smu_context *smu, + void **table) +{ + SmuMetrics_t metrics; + struct smu_table_context *smu_table = &smu->smu_table; + struct gpu_metrics_v2_4 *gpu_metrics = + (struct gpu_metrics_v2_4 *)smu_table->gpu_metrics_table; + int ret = 0; + + ret = smu_cmn_get_metrics_table(smu, &metrics, true); + if (ret) + return ret; + + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 4); + + gpu_metrics->temperature_gfx = metrics.Current.GfxTemperature; + gpu_metrics->temperature_soc = metrics.Current.SocTemperature; + memcpy(&gpu_metrics->temperature_core[0], + &metrics.Current.CoreTemperature[0], + sizeof(uint16_t) * 4); + gpu_metrics->temperature_l3[0] = metrics.Current.L3Temperature[0]; + + gpu_metrics->average_temperature_gfx = metrics.Average.GfxTemperature; + gpu_metrics->average_temperature_soc = metrics.Average.SocTemperature; + memcpy(&gpu_metrics->average_temperature_core[0], + &metrics.Average.CoreTemperature[0], + sizeof(uint16_t) * 4); + gpu_metrics->average_temperature_l3[0] = metrics.Average.L3Temperature[0]; + + gpu_metrics->average_gfx_activity = metrics.Current.GfxActivity; + gpu_metrics->average_mm_activity = metrics.Current.UvdActivity; + + gpu_metrics->average_socket_power = metrics.Current.CurrentSocketPower; + gpu_metrics->average_cpu_power = metrics.Current.Power[0]; + gpu_metrics->average_soc_power = metrics.Current.Power[1]; + gpu_metrics->average_gfx_power = metrics.Current.Power[2]; + + gpu_metrics->average_cpu_voltage = metrics.Current.Voltage[0]; + gpu_metrics->average_soc_voltage = metrics.Current.Voltage[1]; + gpu_metrics->average_gfx_voltage = metrics.Current.Voltage[2]; + + gpu_metrics->average_cpu_current = metrics.Current.Current[0]; + gpu_metrics->average_soc_current = metrics.Current.Current[1]; + gpu_metrics->average_gfx_current = metrics.Current.Current[2]; + + memcpy(&gpu_metrics->average_core_power[0], + &metrics.Average.CorePower[0], + sizeof(uint16_t) * 4); + + gpu_metrics->average_gfxclk_frequency = metrics.Average.GfxclkFrequency; + gpu_metrics->average_socclk_frequency = metrics.Average.SocclkFrequency; + gpu_metrics->average_uclk_frequency = metrics.Average.MemclkFrequency; + gpu_metrics->average_fclk_frequency = metrics.Average.MemclkFrequency; + gpu_metrics->average_vclk_frequency = metrics.Average.VclkFrequency; + gpu_metrics->average_dclk_frequency = metrics.Average.DclkFrequency; + + gpu_metrics->current_gfxclk = metrics.Current.GfxclkFrequency; + gpu_metrics->current_socclk = metrics.Current.SocclkFrequency; + gpu_metrics->current_uclk = metrics.Current.MemclkFrequency; + gpu_metrics->current_fclk = metrics.Current.MemclkFrequency; + gpu_metrics->current_vclk = metrics.Current.VclkFrequency; + gpu_metrics->current_dclk = metrics.Current.DclkFrequency; + + memcpy(&gpu_metrics->current_coreclk[0], + &metrics.Current.CoreFrequency[0], + sizeof(uint16_t) * 4); + gpu_metrics->current_l3clk[0] = metrics.Current.L3Frequency[0]; + + gpu_metrics->throttle_status = metrics.Current.ThrottlerStatus; + gpu_metrics->indep_throttle_status = + smu_cmn_get_indep_throttler_status(metrics.Current.ThrottlerStatus, + vangogh_throttler_map); + + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v2_4); +} + static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, void **table) { @@ -1923,23 +2003,34 @@ static ssize_t vangogh_common_get_gpu_metrics(struct smu_context *smu, { uint32_t if_version; uint32_t smu_version; + uint32_t smu_program; + uint32_t fw_version; int ret = 0; ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version); - if (ret) { + if (ret) return ret; - } - if (smu_version >= 0x043F3E00) { - if (if_version < 0x3) - ret = vangogh_get_legacy_gpu_metrics_v2_3(smu, table); + smu_program = (smu_version >> 24) & 0xff; + fw_version = smu_version & 0xffffff; + if (smu_program == 6) { + if (fw_version >= 0x3F0800) + ret = vangogh_get_gpu_metrics_v2_4(smu, table); else ret = vangogh_get_gpu_metrics_v2_3(smu, table); + } else { - if (if_version < 0x3) - ret = vangogh_get_legacy_gpu_metrics(smu, table); - else - ret = vangogh_get_gpu_metrics(smu, table); + if (smu_version >= 0x043F3E00) { + if (if_version < 0x3) + ret = vangogh_get_legacy_gpu_metrics_v2_3(smu, table); + else + ret = vangogh_get_gpu_metrics_v2_3(smu, table); + } else { + if (if_version < 0x3) + ret = vangogh_get_legacy_gpu_metrics(smu, table); + else + ret = vangogh_get_gpu_metrics(smu, table); + } } return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 3ecb900e6ecd..cee5e32b4ff9 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -996,6 +996,9 @@ void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev) case METRICS_VERSION(2, 3): structure_size = sizeof(struct gpu_metrics_v2_3); break; + case METRICS_VERSION(2, 4): + structure_size = sizeof(struct gpu_metrics_v2_4); + break; default: return; } -- 2.39.2 ^ permalink raw reply [flat|nested] 4+ messages in thread
* RE: [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to acquire gpu_metrics 2023-06-01 1:38 [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to acquire gpu_metrics Wenyou Yang @ 2023-06-11 5:52 ` Yang, WenYou 2023-06-20 18:39 ` Limonciello, Mario 2023-06-21 4:12 ` Lazar, Lijo 1 sibling, 1 reply; 4+ messages in thread From: Yang, WenYou @ 2023-06-11 5:52 UTC (permalink / raw) To: Yang, WenYou, Deucher, Alexander, Limonciello, Mario, Koenig, Christian, Pan, Xinhui, Quan, Evan Cc: Yuan, Perry, Liang, Richard qi, amd-gfx, dri-devel, linux-kernel [AMD Official Use Only - General] Any comments? > -----Original Message----- > From: Wenyou Yang <WenYou.Yang@amd.com> > Sent: Thursday, June 1, 2023 9:38 AM > To: Deucher, Alexander <Alexander.Deucher@amd.com>; Limonciello, Mario > <Mario.Limonciello@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; > Pan, Xinhui <Xinhui.Pan@amd.com>; Quan, Evan <Evan.Quan@amd.com> > Cc: Yuan, Perry <Perry.Yuan@amd.com>; Liang, Richard qi > <Richardqi.Liang@amd.com>; amd-gfx@lists.freedesktop.org; dri- > devel@lists.freedesktop.org; linux-kernel@vger.kernel.org; Yang, WenYou > <WenYou.Yang@amd.com> > Subject: [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to acquire > gpu_metrics > > To acquire the voltage and current info from gpu_metrics interface, but > gpu_metrics_v2_3 doesn't contain them, and to be backward compatible, add > new gpu_metrics_v2_4 structure. > > Acked-by: Evan Quan <evan.quan@amd.com> > Signed-off-by: Wenyou Yang <WenYou.Yang@amd.com> > --- > .../gpu/drm/amd/include/kgd_pp_interface.h | 69 +++++++++++ > .../gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c | 109 ++++++++++++++++- > - > drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c | 3 + > 3 files changed, 172 insertions(+), 9 deletions(-) > > diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h > b/drivers/gpu/drm/amd/include/kgd_pp_interface.h > index 9f542f6e19ed..0f37dafafcf9 100644 > --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h > +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h > @@ -892,4 +892,73 @@ struct gpu_metrics_v2_3 { > uint16_t average_temperature_core[8]; // > average CPU core temperature on APUs > uint16_t average_temperature_l3[2]; > }; > + > +struct gpu_metrics_v2_4 { > + struct metrics_table_header common_header; > + > + /* Temperature */ > + uint16_t temperature_gfx; > + uint16_t temperature_soc; > + uint16_t temperature_core[8]; > + uint16_t temperature_l3[2]; > + > + /* Utilization */ > + uint16_t average_gfx_activity; > + uint16_t average_mm_activity; > + > + /* Driver attached timestamp (in ns) */ > + uint64_t system_clock_counter; > + > + /* Power/Energy */ > + uint16_t average_socket_power; > + uint16_t average_cpu_power; > + uint16_t average_soc_power; > + uint16_t average_gfx_power; > + uint16_t average_core_power[8]; > + > + /* Average clocks */ > + uint16_t average_gfxclk_frequency; > + uint16_t average_socclk_frequency; > + uint16_t average_uclk_frequency; > + uint16_t average_fclk_frequency; > + uint16_t average_vclk_frequency; > + uint16_t average_dclk_frequency; > + > + /* Current clocks */ > + uint16_t current_gfxclk; > + uint16_t current_socclk; > + uint16_t current_uclk; > + uint16_t current_fclk; > + uint16_t current_vclk; > + uint16_t current_dclk; > + uint16_t current_coreclk[8]; > + uint16_t current_l3clk[2]; > + > + /* Throttle status (ASIC dependent) */ > + uint32_t throttle_status; > + > + /* Fans */ > + uint16_t fan_pwm; > + > + uint16_t padding[3]; > + > + /* Throttle status (ASIC independent) */ > + uint64_t indep_throttle_status; > + > + /* Average Temperature */ > + uint16_t average_temperature_gfx; > + uint16_t average_temperature_soc; > + uint16_t average_temperature_core[8]; > + uint16_t average_temperature_l3[2]; > + > + /* Power/Voltage */ > + uint16_t average_cpu_voltage; > + uint16_t average_soc_voltage; > + uint16_t average_gfx_voltage; > + > + /* Power/Current */ > + uint16_t average_cpu_current; > + uint16_t average_soc_current; > + uint16_t average_gfx_current; > +}; > #endif > diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c > b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c > index 067b4e0b026c..185d0b50ee8e 100644 > --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c > +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c > @@ -1854,6 +1854,86 @@ static ssize_t vangogh_get_gpu_metrics_v2_3(struct > smu_context *smu, > return sizeof(struct gpu_metrics_v2_3); } > > +static ssize_t vangogh_get_gpu_metrics_v2_4(struct smu_context *smu, > + void **table) > +{ > + SmuMetrics_t metrics; > + struct smu_table_context *smu_table = &smu->smu_table; > + struct gpu_metrics_v2_4 *gpu_metrics = > + (struct gpu_metrics_v2_4 *)smu_table- > >gpu_metrics_table; > + int ret = 0; > + > + ret = smu_cmn_get_metrics_table(smu, &metrics, true); > + if (ret) > + return ret; > + > + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 4); > + > + gpu_metrics->temperature_gfx = metrics.Current.GfxTemperature; > + gpu_metrics->temperature_soc = metrics.Current.SocTemperature; > + memcpy(&gpu_metrics->temperature_core[0], > + &metrics.Current.CoreTemperature[0], > + sizeof(uint16_t) * 4); > + gpu_metrics->temperature_l3[0] = metrics.Current.L3Temperature[0]; > + > + gpu_metrics->average_temperature_gfx = > metrics.Average.GfxTemperature; > + gpu_metrics->average_temperature_soc = > metrics.Average.SocTemperature; > + memcpy(&gpu_metrics->average_temperature_core[0], > + &metrics.Average.CoreTemperature[0], > + sizeof(uint16_t) * 4); > + gpu_metrics->average_temperature_l3[0] = > +metrics.Average.L3Temperature[0]; > + > + gpu_metrics->average_gfx_activity = metrics.Current.GfxActivity; > + gpu_metrics->average_mm_activity = metrics.Current.UvdActivity; > + > + gpu_metrics->average_socket_power = > metrics.Current.CurrentSocketPower; > + gpu_metrics->average_cpu_power = metrics.Current.Power[0]; > + gpu_metrics->average_soc_power = metrics.Current.Power[1]; > + gpu_metrics->average_gfx_power = metrics.Current.Power[2]; > + > + gpu_metrics->average_cpu_voltage = metrics.Current.Voltage[0]; > + gpu_metrics->average_soc_voltage = metrics.Current.Voltage[1]; > + gpu_metrics->average_gfx_voltage = metrics.Current.Voltage[2]; > + > + gpu_metrics->average_cpu_current = metrics.Current.Current[0]; > + gpu_metrics->average_soc_current = metrics.Current.Current[1]; > + gpu_metrics->average_gfx_current = metrics.Current.Current[2]; > + > + memcpy(&gpu_metrics->average_core_power[0], > + &metrics.Average.CorePower[0], > + sizeof(uint16_t) * 4); > + > + gpu_metrics->average_gfxclk_frequency = > metrics.Average.GfxclkFrequency; > + gpu_metrics->average_socclk_frequency = > metrics.Average.SocclkFrequency; > + gpu_metrics->average_uclk_frequency = > metrics.Average.MemclkFrequency; > + gpu_metrics->average_fclk_frequency = > metrics.Average.MemclkFrequency; > + gpu_metrics->average_vclk_frequency = metrics.Average.VclkFrequency; > + gpu_metrics->average_dclk_frequency = > metrics.Average.DclkFrequency; > + > + gpu_metrics->current_gfxclk = metrics.Current.GfxclkFrequency; > + gpu_metrics->current_socclk = metrics.Current.SocclkFrequency; > + gpu_metrics->current_uclk = metrics.Current.MemclkFrequency; > + gpu_metrics->current_fclk = metrics.Current.MemclkFrequency; > + gpu_metrics->current_vclk = metrics.Current.VclkFrequency; > + gpu_metrics->current_dclk = metrics.Current.DclkFrequency; > + > + memcpy(&gpu_metrics->current_coreclk[0], > + &metrics.Current.CoreFrequency[0], > + sizeof(uint16_t) * 4); > + gpu_metrics->current_l3clk[0] = metrics.Current.L3Frequency[0]; > + > + gpu_metrics->throttle_status = metrics.Current.ThrottlerStatus; > + gpu_metrics->indep_throttle_status = > + > smu_cmn_get_indep_throttler_status(metrics.Current.ThrottlerStatus, > + > vangogh_throttler_map); > + > + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); > + > + *table = (void *)gpu_metrics; > + > + return sizeof(struct gpu_metrics_v2_4); } > + > static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, > void **table) > { > @@ -1923,23 +2003,34 @@ static ssize_t > vangogh_common_get_gpu_metrics(struct smu_context *smu, { > uint32_t if_version; > uint32_t smu_version; > + uint32_t smu_program; > + uint32_t fw_version; > int ret = 0; > > ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version); > - if (ret) { > + if (ret) > return ret; > - } > > - if (smu_version >= 0x043F3E00) { > - if (if_version < 0x3) > - ret = vangogh_get_legacy_gpu_metrics_v2_3(smu, > table); > + smu_program = (smu_version >> 24) & 0xff; > + fw_version = smu_version & 0xffffff; > + if (smu_program == 6) { > + if (fw_version >= 0x3F0800) > + ret = vangogh_get_gpu_metrics_v2_4(smu, table); > else > ret = vangogh_get_gpu_metrics_v2_3(smu, table); > + > } else { > - if (if_version < 0x3) > - ret = vangogh_get_legacy_gpu_metrics(smu, table); > - else > - ret = vangogh_get_gpu_metrics(smu, table); > + if (smu_version >= 0x043F3E00) { > + if (if_version < 0x3) > + ret = > vangogh_get_legacy_gpu_metrics_v2_3(smu, table); > + else > + ret = vangogh_get_gpu_metrics_v2_3(smu, > table); > + } else { > + if (if_version < 0x3) > + ret = vangogh_get_legacy_gpu_metrics(smu, > table); > + else > + ret = vangogh_get_gpu_metrics(smu, table); > + } > } > > return ret; > diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c > b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c > index 3ecb900e6ecd..cee5e32b4ff9 100644 > --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c > +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c > @@ -996,6 +996,9 @@ void smu_cmn_init_soft_gpu_metrics(void *table, > uint8_t frev, uint8_t crev) > case METRICS_VERSION(2, 3): > structure_size = sizeof(struct gpu_metrics_v2_3); > break; > + case METRICS_VERSION(2, 4): > + structure_size = sizeof(struct gpu_metrics_v2_4); > + break; > default: > return; > } > -- > 2.39.2 ^ permalink raw reply [flat|nested] 4+ messages in thread
* RE: [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to acquire gpu_metrics 2023-06-11 5:52 ` Yang, WenYou @ 2023-06-20 18:39 ` Limonciello, Mario 0 siblings, 0 replies; 4+ messages in thread From: Limonciello, Mario @ 2023-06-20 18:39 UTC (permalink / raw) To: Yang, WenYou, Deucher, Alexander, Koenig, Christian, Pan, Xinhui, Quan, Evan Cc: Yuan, Perry, Liang, Richard qi, amd-gfx, dri-devel, linux-kernel [Public] You've got an A-b from Evan already on this. It looks fine to me too. Reviewed-by: Mario Limonciello <mario.limonciello@amd.com> > -----Original Message----- > From: Yang, WenYou <WenYou.Yang@amd.com> > Sent: Sunday, June 11, 2023 12:53 AM > To: Yang, WenYou <WenYou.Yang@amd.com>; Deucher, Alexander > <Alexander.Deucher@amd.com>; Limonciello, Mario > <Mario.Limonciello@amd.com>; Koenig, Christian > <Christian.Koenig@amd.com>; Pan, Xinhui <Xinhui.Pan@amd.com>; Quan, > Evan <Evan.Quan@amd.com> > Cc: Yuan, Perry <Perry.Yuan@amd.com>; Liang, Richard qi > <Richardqi.Liang@amd.com>; amd-gfx@lists.freedesktop.org; dri- > devel@lists.freedesktop.org; linux-kernel@vger.kernel.org > Subject: RE: [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to > acquire gpu_metrics > > [AMD Official Use Only - General] > > Any comments? > > > -----Original Message----- > > From: Wenyou Yang <WenYou.Yang@amd.com> > > Sent: Thursday, June 1, 2023 9:38 AM > > To: Deucher, Alexander <Alexander.Deucher@amd.com>; Limonciello, Mario > > <Mario.Limonciello@amd.com>; Koenig, Christian > <Christian.Koenig@amd.com>; > > Pan, Xinhui <Xinhui.Pan@amd.com>; Quan, Evan <Evan.Quan@amd.com> > > Cc: Yuan, Perry <Perry.Yuan@amd.com>; Liang, Richard qi > > <Richardqi.Liang@amd.com>; amd-gfx@lists.freedesktop.org; dri- > > devel@lists.freedesktop.org; linux-kernel@vger.kernel.org; Yang, WenYou > > <WenYou.Yang@amd.com> > > Subject: [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to > acquire > > gpu_metrics > > > > To acquire the voltage and current info from gpu_metrics interface, but > > gpu_metrics_v2_3 doesn't contain them, and to be backward compatible, > add > > new gpu_metrics_v2_4 structure. > > > > Acked-by: Evan Quan <evan.quan@amd.com> > > Signed-off-by: Wenyou Yang <WenYou.Yang@amd.com> > > --- > > .../gpu/drm/amd/include/kgd_pp_interface.h | 69 +++++++++++ > > .../gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c | 109 > ++++++++++++++++- > > - > > drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c | 3 + > > 3 files changed, 172 insertions(+), 9 deletions(-) > > > > diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h > > b/drivers/gpu/drm/amd/include/kgd_pp_interface.h > > index 9f542f6e19ed..0f37dafafcf9 100644 > > --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h > > +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h > > @@ -892,4 +892,73 @@ struct gpu_metrics_v2_3 { > > uint16_t average_temperature_core[8]; // > > average CPU core temperature on APUs > > uint16_t average_temperature_l3[2]; > > }; > > + > > +struct gpu_metrics_v2_4 { > > + struct metrics_table_header common_header; > > + > > + /* Temperature */ > > + uint16_t temperature_gfx; > > + uint16_t temperature_soc; > > + uint16_t temperature_core[8]; > > + uint16_t temperature_l3[2]; > > + > > + /* Utilization */ > > + uint16_t average_gfx_activity; > > + uint16_t average_mm_activity; > > + > > + /* Driver attached timestamp (in ns) */ > > + uint64_t system_clock_counter; > > + > > + /* Power/Energy */ > > + uint16_t average_socket_power; > > + uint16_t average_cpu_power; > > + uint16_t average_soc_power; > > + uint16_t average_gfx_power; > > + uint16_t average_core_power[8]; > > + > > + /* Average clocks */ > > + uint16_t average_gfxclk_frequency; > > + uint16_t average_socclk_frequency; > > + uint16_t average_uclk_frequency; > > + uint16_t average_fclk_frequency; > > + uint16_t average_vclk_frequency; > > + uint16_t average_dclk_frequency; > > + > > + /* Current clocks */ > > + uint16_t current_gfxclk; > > + uint16_t current_socclk; > > + uint16_t current_uclk; > > + uint16_t current_fclk; > > + uint16_t current_vclk; > > + uint16_t current_dclk; > > + uint16_t current_coreclk[8]; > > + uint16_t current_l3clk[2]; > > + > > + /* Throttle status (ASIC dependent) */ > > + uint32_t throttle_status; > > + > > + /* Fans */ > > + uint16_t fan_pwm; > > + > > + uint16_t padding[3]; > > + > > + /* Throttle status (ASIC independent) */ > > + uint64_t indep_throttle_status; > > + > > + /* Average Temperature */ > > + uint16_t average_temperature_gfx; > > + uint16_t average_temperature_soc; > > + uint16_t average_temperature_core[8]; > > + uint16_t average_temperature_l3[2]; > > + > > + /* Power/Voltage */ > > + uint16_t average_cpu_voltage; > > + uint16_t average_soc_voltage; > > + uint16_t average_gfx_voltage; > > + > > + /* Power/Current */ > > + uint16_t average_cpu_current; > > + uint16_t average_soc_current; > > + uint16_t average_gfx_current; > > +}; > > #endif > > diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c > > b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c > > index 067b4e0b026c..185d0b50ee8e 100644 > > --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c > > +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c > > @@ -1854,6 +1854,86 @@ static ssize_t > vangogh_get_gpu_metrics_v2_3(struct > > smu_context *smu, > > return sizeof(struct gpu_metrics_v2_3); } > > > > +static ssize_t vangogh_get_gpu_metrics_v2_4(struct smu_context *smu, > > + void **table) > > +{ > > + SmuMetrics_t metrics; > > + struct smu_table_context *smu_table = &smu->smu_table; > > + struct gpu_metrics_v2_4 *gpu_metrics = > > + (struct gpu_metrics_v2_4 *)smu_table- > > >gpu_metrics_table; > > + int ret = 0; > > + > > + ret = smu_cmn_get_metrics_table(smu, &metrics, true); > > + if (ret) > > + return ret; > > + > > + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 4); > > + > > + gpu_metrics->temperature_gfx = metrics.Current.GfxTemperature; > > + gpu_metrics->temperature_soc = metrics.Current.SocTemperature; > > + memcpy(&gpu_metrics->temperature_core[0], > > + &metrics.Current.CoreTemperature[0], > > + sizeof(uint16_t) * 4); > > + gpu_metrics->temperature_l3[0] = metrics.Current.L3Temperature[0]; > > + > > + gpu_metrics->average_temperature_gfx = > > metrics.Average.GfxTemperature; > > + gpu_metrics->average_temperature_soc = > > metrics.Average.SocTemperature; > > + memcpy(&gpu_metrics->average_temperature_core[0], > > + &metrics.Average.CoreTemperature[0], > > + sizeof(uint16_t) * 4); > > + gpu_metrics->average_temperature_l3[0] = > > +metrics.Average.L3Temperature[0]; > > + > > + gpu_metrics->average_gfx_activity = metrics.Current.GfxActivity; > > + gpu_metrics->average_mm_activity = metrics.Current.UvdActivity; > > + > > + gpu_metrics->average_socket_power = > > metrics.Current.CurrentSocketPower; > > + gpu_metrics->average_cpu_power = metrics.Current.Power[0]; > > + gpu_metrics->average_soc_power = metrics.Current.Power[1]; > > + gpu_metrics->average_gfx_power = metrics.Current.Power[2]; > > + > > + gpu_metrics->average_cpu_voltage = metrics.Current.Voltage[0]; > > + gpu_metrics->average_soc_voltage = metrics.Current.Voltage[1]; > > + gpu_metrics->average_gfx_voltage = metrics.Current.Voltage[2]; > > + > > + gpu_metrics->average_cpu_current = metrics.Current.Current[0]; > > + gpu_metrics->average_soc_current = metrics.Current.Current[1]; > > + gpu_metrics->average_gfx_current = metrics.Current.Current[2]; > > + > > + memcpy(&gpu_metrics->average_core_power[0], > > + &metrics.Average.CorePower[0], > > + sizeof(uint16_t) * 4); > > + > > + gpu_metrics->average_gfxclk_frequency = > > metrics.Average.GfxclkFrequency; > > + gpu_metrics->average_socclk_frequency = > > metrics.Average.SocclkFrequency; > > + gpu_metrics->average_uclk_frequency = > > metrics.Average.MemclkFrequency; > > + gpu_metrics->average_fclk_frequency = > > metrics.Average.MemclkFrequency; > > + gpu_metrics->average_vclk_frequency = metrics.Average.VclkFrequency; > > + gpu_metrics->average_dclk_frequency = > > metrics.Average.DclkFrequency; > > + > > + gpu_metrics->current_gfxclk = metrics.Current.GfxclkFrequency; > > + gpu_metrics->current_socclk = metrics.Current.SocclkFrequency; > > + gpu_metrics->current_uclk = metrics.Current.MemclkFrequency; > > + gpu_metrics->current_fclk = metrics.Current.MemclkFrequency; > > + gpu_metrics->current_vclk = metrics.Current.VclkFrequency; > > + gpu_metrics->current_dclk = metrics.Current.DclkFrequency; > > + > > + memcpy(&gpu_metrics->current_coreclk[0], > > + &metrics.Current.CoreFrequency[0], > > + sizeof(uint16_t) * 4); > > + gpu_metrics->current_l3clk[0] = metrics.Current.L3Frequency[0]; > > + > > + gpu_metrics->throttle_status = metrics.Current.ThrottlerStatus; > > + gpu_metrics->indep_throttle_status = > > + > > smu_cmn_get_indep_throttler_status(metrics.Current.ThrottlerStatus, > > + > > vangogh_throttler_map); > > + > > + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); > > + > > + *table = (void *)gpu_metrics; > > + > > + return sizeof(struct gpu_metrics_v2_4); } > > + > > static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, > > void **table) > > { > > @@ -1923,23 +2003,34 @@ static ssize_t > > vangogh_common_get_gpu_metrics(struct smu_context *smu, { > > uint32_t if_version; > > uint32_t smu_version; > > + uint32_t smu_program; > > + uint32_t fw_version; > > int ret = 0; > > > > ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version); > > - if (ret) { > > + if (ret) > > return ret; > > - } > > > > - if (smu_version >= 0x043F3E00) { > > - if (if_version < 0x3) > > - ret = vangogh_get_legacy_gpu_metrics_v2_3(smu, > > table); > > + smu_program = (smu_version >> 24) & 0xff; > > + fw_version = smu_version & 0xffffff; > > + if (smu_program == 6) { > > + if (fw_version >= 0x3F0800) > > + ret = vangogh_get_gpu_metrics_v2_4(smu, table); > > else > > ret = vangogh_get_gpu_metrics_v2_3(smu, table); > > + > > } else { > > - if (if_version < 0x3) > > - ret = vangogh_get_legacy_gpu_metrics(smu, table); > > - else > > - ret = vangogh_get_gpu_metrics(smu, table); > > + if (smu_version >= 0x043F3E00) { > > + if (if_version < 0x3) > > + ret = > > vangogh_get_legacy_gpu_metrics_v2_3(smu, table); > > + else > > + ret = vangogh_get_gpu_metrics_v2_3(smu, > > table); > > + } else { > > + if (if_version < 0x3) > > + ret = vangogh_get_legacy_gpu_metrics(smu, > > table); > > + else > > + ret = vangogh_get_gpu_metrics(smu, table); > > + } > > } > > > > return ret; > > diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c > > b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c > > index 3ecb900e6ecd..cee5e32b4ff9 100644 > > --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c > > +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c > > @@ -996,6 +996,9 @@ void smu_cmn_init_soft_gpu_metrics(void *table, > > uint8_t frev, uint8_t crev) > > case METRICS_VERSION(2, 3): > > structure_size = sizeof(struct gpu_metrics_v2_3); > > break; > > + case METRICS_VERSION(2, 4): > > + structure_size = sizeof(struct gpu_metrics_v2_4); > > + break; > > default: > > return; > > } > > -- > > 2.39.2 > ^ permalink raw reply [flat|nested] 4+ messages in thread
* RE: [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to acquire gpu_metrics 2023-06-01 1:38 [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to acquire gpu_metrics Wenyou Yang 2023-06-11 5:52 ` Yang, WenYou @ 2023-06-21 4:12 ` Lazar, Lijo 1 sibling, 0 replies; 4+ messages in thread From: Lazar, Lijo @ 2023-06-21 4:12 UTC (permalink / raw) To: Yang, WenYou, Deucher, Alexander, Limonciello, Mario, Koenig, Christian, Pan, Xinhui, Quan, Evan Cc: linux-kernel, dri-devel, Yang, WenYou, amd-gfx, Yuan, Perry, Liang, Richard qi [AMD Official Use Only - General] Could you add the expected units of voltage/current in 2.4 metrics structure? Is it mV/mA or mV/A? Thanks, Lijo -----Original Message----- From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Wenyou Yang Sent: Thursday, June 1, 2023 7:08 AM To: Deucher, Alexander <Alexander.Deucher@amd.com>; Limonciello, Mario <Mario.Limonciello@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Pan, Xinhui <Xinhui.Pan@amd.com>; Quan, Evan <Evan.Quan@amd.com> Cc: linux-kernel@vger.kernel.org; dri-devel@lists.freedesktop.org; Yang, WenYou <WenYou.Yang@amd.com>; amd-gfx@lists.freedesktop.org; Yuan, Perry <Perry.Yuan@amd.com>; Liang, Richard qi <Richardqi.Liang@amd.com> Subject: [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to acquire gpu_metrics To acquire the voltage and current info from gpu_metrics interface, but gpu_metrics_v2_3 doesn't contain them, and to be backward compatible, add new gpu_metrics_v2_4 structure. Acked-by: Evan Quan <evan.quan@amd.com> Signed-off-by: Wenyou Yang <WenYou.Yang@amd.com> --- .../gpu/drm/amd/include/kgd_pp_interface.h | 69 +++++++++++ .../gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c | 109 ++++++++++++++++-- drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c | 3 + 3 files changed, 172 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h index 9f542f6e19ed..0f37dafafcf9 100644 --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h @@ -892,4 +892,73 @@ struct gpu_metrics_v2_3 { uint16_t average_temperature_core[8]; // average CPU core temperature on APUs uint16_t average_temperature_l3[2]; }; + +struct gpu_metrics_v2_4 { + struct metrics_table_header common_header; + + /* Temperature */ + uint16_t temperature_gfx; + uint16_t temperature_soc; + uint16_t temperature_core[8]; + uint16_t temperature_l3[2]; + + /* Utilization */ + uint16_t average_gfx_activity; + uint16_t average_mm_activity; + + /* Driver attached timestamp (in ns) */ + uint64_t system_clock_counter; + + /* Power/Energy */ + uint16_t average_socket_power; + uint16_t average_cpu_power; + uint16_t average_soc_power; + uint16_t average_gfx_power; + uint16_t average_core_power[8]; + + /* Average clocks */ + uint16_t average_gfxclk_frequency; + uint16_t average_socclk_frequency; + uint16_t average_uclk_frequency; + uint16_t average_fclk_frequency; + uint16_t average_vclk_frequency; + uint16_t average_dclk_frequency; + + /* Current clocks */ + uint16_t current_gfxclk; + uint16_t current_socclk; + uint16_t current_uclk; + uint16_t current_fclk; + uint16_t current_vclk; + uint16_t current_dclk; + uint16_t current_coreclk[8]; + uint16_t current_l3clk[2]; + + /* Throttle status (ASIC dependent) */ + uint32_t throttle_status; + + /* Fans */ + uint16_t fan_pwm; + + uint16_t padding[3]; + + /* Throttle status (ASIC independent) */ + uint64_t indep_throttle_status; + + /* Average Temperature */ + uint16_t average_temperature_gfx; + uint16_t average_temperature_soc; + uint16_t average_temperature_core[8]; + uint16_t average_temperature_l3[2]; + + /* Power/Voltage */ + uint16_t average_cpu_voltage; + uint16_t average_soc_voltage; + uint16_t average_gfx_voltage; + + /* Power/Current */ + uint16_t average_cpu_current; + uint16_t average_soc_current; + uint16_t average_gfx_current; +}; #endif diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 067b4e0b026c..185d0b50ee8e 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -1854,6 +1854,86 @@ static ssize_t vangogh_get_gpu_metrics_v2_3(struct smu_context *smu, return sizeof(struct gpu_metrics_v2_3); } +static ssize_t vangogh_get_gpu_metrics_v2_4(struct smu_context *smu, + void **table) +{ + SmuMetrics_t metrics; + struct smu_table_context *smu_table = &smu->smu_table; + struct gpu_metrics_v2_4 *gpu_metrics = + (struct gpu_metrics_v2_4 *)smu_table->gpu_metrics_table; + int ret = 0; + + ret = smu_cmn_get_metrics_table(smu, &metrics, true); + if (ret) + return ret; + + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 4); + + gpu_metrics->temperature_gfx = metrics.Current.GfxTemperature; + gpu_metrics->temperature_soc = metrics.Current.SocTemperature; + memcpy(&gpu_metrics->temperature_core[0], + &metrics.Current.CoreTemperature[0], + sizeof(uint16_t) * 4); + gpu_metrics->temperature_l3[0] = metrics.Current.L3Temperature[0]; + + gpu_metrics->average_temperature_gfx = metrics.Average.GfxTemperature; + gpu_metrics->average_temperature_soc = metrics.Average.SocTemperature; + memcpy(&gpu_metrics->average_temperature_core[0], + &metrics.Average.CoreTemperature[0], + sizeof(uint16_t) * 4); + gpu_metrics->average_temperature_l3[0] = +metrics.Average.L3Temperature[0]; + + gpu_metrics->average_gfx_activity = metrics.Current.GfxActivity; + gpu_metrics->average_mm_activity = metrics.Current.UvdActivity; + + gpu_metrics->average_socket_power = metrics.Current.CurrentSocketPower; + gpu_metrics->average_cpu_power = metrics.Current.Power[0]; + gpu_metrics->average_soc_power = metrics.Current.Power[1]; + gpu_metrics->average_gfx_power = metrics.Current.Power[2]; + + gpu_metrics->average_cpu_voltage = metrics.Current.Voltage[0]; + gpu_metrics->average_soc_voltage = metrics.Current.Voltage[1]; + gpu_metrics->average_gfx_voltage = metrics.Current.Voltage[2]; + + gpu_metrics->average_cpu_current = metrics.Current.Current[0]; + gpu_metrics->average_soc_current = metrics.Current.Current[1]; + gpu_metrics->average_gfx_current = metrics.Current.Current[2]; + + memcpy(&gpu_metrics->average_core_power[0], + &metrics.Average.CorePower[0], + sizeof(uint16_t) * 4); + + gpu_metrics->average_gfxclk_frequency = metrics.Average.GfxclkFrequency; + gpu_metrics->average_socclk_frequency = metrics.Average.SocclkFrequency; + gpu_metrics->average_uclk_frequency = metrics.Average.MemclkFrequency; + gpu_metrics->average_fclk_frequency = metrics.Average.MemclkFrequency; + gpu_metrics->average_vclk_frequency = metrics.Average.VclkFrequency; + gpu_metrics->average_dclk_frequency = metrics.Average.DclkFrequency; + + gpu_metrics->current_gfxclk = metrics.Current.GfxclkFrequency; + gpu_metrics->current_socclk = metrics.Current.SocclkFrequency; + gpu_metrics->current_uclk = metrics.Current.MemclkFrequency; + gpu_metrics->current_fclk = metrics.Current.MemclkFrequency; + gpu_metrics->current_vclk = metrics.Current.VclkFrequency; + gpu_metrics->current_dclk = metrics.Current.DclkFrequency; + + memcpy(&gpu_metrics->current_coreclk[0], + &metrics.Current.CoreFrequency[0], + sizeof(uint16_t) * 4); + gpu_metrics->current_l3clk[0] = metrics.Current.L3Frequency[0]; + + gpu_metrics->throttle_status = metrics.Current.ThrottlerStatus; + gpu_metrics->indep_throttle_status = + smu_cmn_get_indep_throttler_status(metrics.Current.ThrottlerStatus, + vangogh_throttler_map); + + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v2_4); } + static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, void **table) { @@ -1923,23 +2003,34 @@ static ssize_t vangogh_common_get_gpu_metrics(struct smu_context *smu, { uint32_t if_version; uint32_t smu_version; + uint32_t smu_program; + uint32_t fw_version; int ret = 0; ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version); - if (ret) { + if (ret) return ret; - } - if (smu_version >= 0x043F3E00) { - if (if_version < 0x3) - ret = vangogh_get_legacy_gpu_metrics_v2_3(smu, table); + smu_program = (smu_version >> 24) & 0xff; + fw_version = smu_version & 0xffffff; + if (smu_program == 6) { + if (fw_version >= 0x3F0800) + ret = vangogh_get_gpu_metrics_v2_4(smu, table); else ret = vangogh_get_gpu_metrics_v2_3(smu, table); + } else { - if (if_version < 0x3) - ret = vangogh_get_legacy_gpu_metrics(smu, table); - else - ret = vangogh_get_gpu_metrics(smu, table); + if (smu_version >= 0x043F3E00) { + if (if_version < 0x3) + ret = vangogh_get_legacy_gpu_metrics_v2_3(smu, table); + else + ret = vangogh_get_gpu_metrics_v2_3(smu, table); + } else { + if (if_version < 0x3) + ret = vangogh_get_legacy_gpu_metrics(smu, table); + else + ret = vangogh_get_gpu_metrics(smu, table); + } } return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 3ecb900e6ecd..cee5e32b4ff9 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -996,6 +996,9 @@ void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev) case METRICS_VERSION(2, 3): structure_size = sizeof(struct gpu_metrics_v2_3); break; + case METRICS_VERSION(2, 4): + structure_size = sizeof(struct gpu_metrics_v2_4); + break; default: return; } -- 2.39.2 ^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2023-06-21 4:12 UTC | newest] Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2023-06-01 1:38 [PATCH] drm/amd/pm: Vangogh: Add new gpu_metrics_v2_4 to acquire gpu_metrics Wenyou Yang 2023-06-11 5:52 ` Yang, WenYou 2023-06-20 18:39 ` Limonciello, Mario 2023-06-21 4:12 ` Lazar, Lijo
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®