[PATCH] riscv: Support inlined clocksource reads for the time counter

[PATCH] riscv: Support inlined clocksource reads for the time counter

[Chen Pei]

The commit 2e27beeb66e4 ("timekeeping: Allow inlining clocksource::read()")
lets an architecture inline the clocksource read behind a static branch, so
that a read which is a single instruction stops paying for the indirect
call in front of it. The branch is only enabled for a clocksource
advertising CLOCK_SOURCE_CAN_INLINE_READ. The commit b27801189f7f ("x86:
Inline TSC reads in timekeeping") wired x86 up to it.

The overhead matters for consumers that need accurate timestamps and take
them very often: 1PPS edge timestamps, PTP stamping, and polling mode PPS
drivers all end up in the timekeeping fast path.

RISC-V has the same shape. In S-mode the counter is the time CSR, so on
64-bit a read is a single csr_read(CSR_TIME), and
riscv_clocksource.read() is a wrapper whose whole body is that one
instruction. Every fast path timestamp therefore pays an indirect call
and a return to reach a single CSR read. Letting timekeeping inline the
read issues that instruction at the call site and drops the call, which
is where the gain comes from.

Signed-off-by: Chen Pei <cp0613@linux.alibaba.com>
---
A few notes:

- The gain is in instructions executed per read. Without inlining, a
  read is a function pointer load, an indirect call, and a frame around
  the rdtime - 11 instructions in all; inlined it is a jump-label branch
  and the rdtime - 2. The read is no longer shared, so each fast path
  function in timekeeping.o grows by 11 instructions (438 bytes).

- RV32 is left out on purpose. There the read is a retry sequence
  (rdtimeh, rdtime, rdtimeh) rather than one instruction, so the saving
  is much smaller while the sequence is duplicated at every read site,
  which is the wrong trade on the small in-order cores with tight
  I-caches that run 32-bit Linux. rv32 depends on NONPORTABLE and
  nobody has asked for it; if hardware data says otherwise, enabling it
  is a one-line Kconfig change.

- M-mode is left out because RISCV_TIMER is not built there and the
  CLINT clocksource reads MMIO, so there is no cheap counter read to
  inline.

 arch/riscv/Kconfig                     |  1 +
 arch/riscv/include/asm/clock_inlined.h | 14 ++++++++++++++
 drivers/clocksource/timer-riscv.c      |  3 ++-
 3 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/include/asm/clock_inlined.h

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index d6c2dbf8455c..f5d314ef37b8 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -90,6 +90,7 @@ config RISCV
 	select ARCH_WANT_LD_ORPHAN_WARN
 	select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
 	select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
+	select ARCH_WANTS_CLOCKSOURCE_READ_INLINE if !RISCV_M_MODE && 64BIT
 	select ARCH_WANTS_NO_INSTR
 	select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE
 	select ARCH_WEAK_RELEASE_ACQUIRE if ARCH_USE_QUEUED_SPINLOCKS
diff --git a/arch/riscv/include/asm/clock_inlined.h b/arch/riscv/include/asm/clock_inlined.h
new file mode 100644
index 000000000000..450add2436e4
--- /dev/null
+++ b/arch/riscv/include/asm/clock_inlined.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_RISCV_CLOCK_INLINED_H
+#define _ASM_RISCV_CLOCK_INLINED_H
+
+#include <asm/timex.h>
+
+struct clocksource;
+
+static __always_inline u64 arch_inlined_clocksource_read(struct clocksource *cs)
+{
+	return get_cycles64();
+}
+
+#endif
diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c
index b310e6721fac..768e0d0eaeb0 100644
--- a/drivers/clocksource/timer-riscv.c
+++ b/drivers/clocksource/timer-riscv.c
@@ -96,7 +96,8 @@ static struct clocksource riscv_clocksource = {
 	.name		= "riscv_clocksource",
 	.rating		= 400,
 	.mask		= CLOCKSOURCE_MASK(64),
-	.flags		= CLOCK_SOURCE_IS_CONTINUOUS,
+	.flags		= CLOCK_SOURCE_IS_CONTINUOUS |
+			  CLOCK_SOURCE_CAN_INLINE_READ,
 	.read		= riscv_clocksource_rdtime,
 	.vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER,
 };
-- 
2.50.1