mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Feng Tang <feng.tang@intel.com>
To: Thomas Gleixner <tglx@linutronix.de>,
	Ingo Molnar <mingo@redhat.com>, Borislav Petkov <bp@alien8.de>,
	Dave Hansen <dave.hansen@linux.intel.com>,
	"H . Peter Anvin" <hpa@zytor.com>,
	Peter Zijlstra <peterz@infradead.org>,
	David Woodhouse <dwmw@amazon.co.uk>,
	"Paul E . McKenney" <paulmck@kernel.org>,
	x86@kernel.org, linux-kernel@vger.kernel.org
Cc: rui.zhang@intel.com, tim.c.chen@intel.com,
	Feng Tang <feng.tang@intel.com>
Subject: [Patch v2 2/2] x86/tsc: use logical_packages as a better estimation of socket numbers
Date: Tue, 13 Jun 2023 13:25:23 +0800	[thread overview]
Message-ID: <20230613052523.1106821-2-feng.tang@intel.com> (raw)
In-Reply-To: <20230613052523.1106821-1-feng.tang@intel.com>

Commit b50db7095fe0 ("x86/tsc: Disable clocksource watchdog for TSC
on qualified platorms") was introduced to solve problem that
sometimes TSC clocksource is wrongly judged as unstable by watchdog
like 'jiffies', HPET, etc.

In it, the hardware socket number is a key factor for judging whether
to disable the watchdog for TSC, and 'nr_online_nodes' was chosen as
an estimation due to it is needed in early boot phase before
registering 'tsc-early' clocksource, where all none-boot CPUs are not
brought up yet.

In recent patch review, Dave and Rui pointed out there are many cases
in which 'nr_online_nodes' is not accurate, like:

* numa emulation (numa=fake=4 etc.)
* numa=off
* platforms with CPU-less HBM nodes, CPU-less Optane memory nodes.
* SNC (sub-numa cluster) mode enabled
* 'nr_cpus=' and 'maxcpus=' kernel cmdline parameter setup

Peter Zijlstra suggested 'logical_packages', and it seems to be the
best option we have as it covers all the cases above except the
last one. But it is only usable after smp_init() and all CPUs are
brought up, while 'tsc-early' is initialized before that.

One solution is to skip the watchdog for 'tsc-early' clocksource,
and move the check after smp_init(), while before 'tsc' clocksource
is registered, where 'logical_packages' could be used.

For 'nr_cpus' and 'maxcpus' cmdline case, the global flag
'package_count_unreliable' will be set to true and the watchdog
will be kept as is.

Reported-by: Dave Hansen <dave.hansen@linux.intel.com>
Reported-by: Zhang Rui <rui.zhang@intel.com>
Signed-off-by: Feng Tang <feng.tang@intel.com>
---
Changelog:

  since v1:
   * Handle the 'maxcpus=' and 'nr_cpus=' cmdline parameter cases,
     by keeping watchdog as the package number is unreliable (Dave Hansen)

  since RFC:
   * use 'logical_packages' instead of topology_max_packages(), whose
     implementaion is not accurate, like for heterogeneous systems
     which have combination of Core/Atom CPUs like Alderlake (Dave Hansen)

 arch/x86/include/asm/topology.h |  3 +++
 arch/x86/kernel/smpboot.c       |  2 +-
 arch/x86/kernel/tsc.c           | 43 +++++++++++++--------------------
 3 files changed, 21 insertions(+), 27 deletions(-)

diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index 458c891a8273..d47b7b39e44d 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -124,6 +124,8 @@ extern unsigned int __max_die_per_package;
 extern unsigned int __max_logical_packages;
 #define topology_max_packages()			(__max_logical_packages)
 
+extern unsigned int logical_packages;
+
 static inline int topology_max_die_per_package(void)
 {
 	return __max_die_per_package;
@@ -144,6 +146,7 @@ bool topology_is_primary_thread(unsigned int cpu);
 bool topology_smt_supported(void);
 #else
 #define topology_max_packages()			(1)
+#define logical_packages	1
 static inline int
 topology_update_package_map(unsigned int apicid, unsigned int cpu) { return 0; }
 static inline int
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index b78770b0c43d..c5ac9eb17cd7 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -104,7 +104,7 @@ EXPORT_PER_CPU_SYMBOL(cpu_info);
 /* Logical package management. We might want to allocate that dynamically */
 unsigned int __max_logical_packages __read_mostly;
 EXPORT_SYMBOL(__max_logical_packages);
-static unsigned int logical_packages __read_mostly;
+unsigned int logical_packages __read_mostly;
 static unsigned int logical_die __read_mostly;
 
 /* Maximum number of SMT threads on any online core */
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 344698852146..fe762f7268a0 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -1146,8 +1146,7 @@ static struct clocksource clocksource_tsc_early = {
 	.uncertainty_margin	= 32 * NSEC_PER_MSEC,
 	.read			= read_tsc,
 	.mask			= CLOCKSOURCE_MASK(64),
-	.flags			= CLOCK_SOURCE_IS_CONTINUOUS |
-				  CLOCK_SOURCE_MUST_VERIFY,
+	.flags			= CLOCK_SOURCE_IS_CONTINUOUS,
 	.vdso_clock_mode	= VDSO_CLOCKMODE_TSC,
 	.enable			= tsc_cs_enable,
 	.resume			= tsc_resume,
@@ -1195,12 +1194,6 @@ void mark_tsc_unstable(char *reason)
 
 EXPORT_SYMBOL_GPL(mark_tsc_unstable);
 
-static void __init tsc_disable_clocksource_watchdog(void)
-{
-	clocksource_tsc_early.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
-	clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
-}
-
 bool tsc_clocksource_watchdog_disabled(void)
 {
 	return !(clocksource_tsc.flags & CLOCK_SOURCE_MUST_VERIFY) &&
@@ -1223,23 +1216,6 @@ static void __init check_system_tsc_reliable(void)
 #endif
 	if (boot_cpu_has(X86_FEATURE_TSC_RELIABLE))
 		tsc_clocksource_reliable = 1;
-
-	/*
-	 * Disable the clocksource watchdog when the system has:
-	 *  - TSC running at constant frequency
-	 *  - TSC which does not stop in C-States
-	 *  - the TSC_ADJUST register which allows to detect even minimal
-	 *    modifications
-	 *  - not more than two sockets. As the number of sockets cannot be
-	 *    evaluated at the early boot stage where this has to be
-	 *    invoked, check the number of online memory nodes as a
-	 *    fallback solution which is an reasonable estimate.
-	 */
-	if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC) &&
-	    boot_cpu_has(X86_FEATURE_NONSTOP_TSC) &&
-	    boot_cpu_has(X86_FEATURE_TSC_ADJUST) &&
-	    nr_online_nodes <= 2)
-		tsc_disable_clocksource_watchdog();
 }
 
 /*
@@ -1455,6 +1431,21 @@ static int __init init_tsc_clocksource(void)
 	if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3))
 		clocksource_tsc.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
 
+	/*
+	 * Disable the clocksource watchdog when the system has:
+	 *  - TSC running at constant frequency
+	 *  - TSC which does not stop in C-States
+	 *  - the TSC_ADJUST register which allows to detect even minimal
+	 *    modifications
+	 *  - not more than two sockets.
+	 */
+	if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC) &&
+	    boot_cpu_has(X86_FEATURE_NONSTOP_TSC) &&
+	    boot_cpu_has(X86_FEATURE_TSC_ADJUST) &&
+	    is_package_count_reliable() &&
+	    logical_packages <= 2)
+		clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
+
 	/*
 	 * When TSC frequency is known (retrieved via MSR or CPUID), we skip
 	 * the refined calibration and directly register it as a clocksource.
@@ -1590,7 +1581,7 @@ void __init tsc_init(void)
 	}
 
 	if (tsc_clocksource_reliable || no_tsc_watchdog)
-		tsc_disable_clocksource_watchdog();
+		clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
 
 	clocksource_register_khz(&clocksource_tsc_early, tsc_khz);
 	detect_art();
-- 
2.34.1


  reply	other threads:[~2023-06-13  5:36 UTC|newest]

Thread overview: 23+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2023-06-13  5:25 [Patch v2 1/2] smp: Add helper function to mark possible bad package number Feng Tang
2023-06-13  5:25 ` Feng Tang [this message]
2023-06-15  9:20   ` [Patch v2 2/2] x86/tsc: use logical_packages as a better estimation of socket numbers Peter Zijlstra
2023-06-16  6:53     ` Zhang, Rui
2023-06-16  8:02       ` Peter Zijlstra
2023-06-16  8:10         ` Peter Zijlstra
2023-06-16  9:19           ` Zhang, Rui
2023-06-16  9:42             ` Peter Zijlstra
2023-06-16 11:23               ` Zhang, Rui
2023-06-16 11:47                 ` Feng Tang
2023-06-16  8:22         ` Peter Zijlstra
2023-06-19 10:42         ` Feng Tang
2023-06-16  7:18     ` Feng Tang
2023-06-22 14:27       ` Thomas Gleixner
2023-06-22 23:07         ` Thomas Gleixner
2023-06-23 15:49           ` Zhang, Rui
     [not found]           ` <ZJW0gi5oQQbxf8Df@feng-clx>
2023-06-25 14:51             ` Feng Tang
2023-06-27 11:14               ` Thomas Gleixner
2023-06-29 13:27                 ` Feng Tang
2023-07-17 13:38                   ` Feng Tang
2023-07-26 19:37                     ` Thomas Gleixner
2023-07-27  1:24                       ` Feng Tang
2023-06-23 15:36         ` Zhang, Rui

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20230613052523.1106821-2-feng.tang@intel.com \
    --to=feng.tang@intel.com \
    --cc=bp@alien8.de \
    --cc=dave.hansen@linux.intel.com \
    --cc=dwmw@amazon.co.uk \
    --cc=hpa@zytor.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=mingo@redhat.com \
    --cc=paulmck@kernel.org \
    --cc=peterz@infradead.org \
    --cc=rui.zhang@intel.com \
    --cc=tglx@linutronix.de \
    --cc=tim.c.chen@intel.com \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®