[patch 0/7] timekeeping: Decouple xtime_lock from xtime seqcount

[patch 0/7] timekeeping: Decouple xtime_lock from xtime seqcount

[Thomas Gleixner]

xtime_lock which serializes the update of time relevant data
structures is held over a large code pathes which results in extended
seqlock contention times on the reader side. This series decouples the
lock from the seqcount and reduces the seqcount protected sections to
those which actually update the reader visible data.

Thanks,

	tglx
---
 jiffies.c       |    4 
 ntp.c           |   16 +--
 tick-common.c   |    8 -
 tick-internal.h |    3 
 tick-sched.c    |   12 +-
 timekeeping.c   |  237 +++++++++++++++++++++++++++++++-------------------------
 6 files changed, 157 insertions(+), 123 deletions(-)

[patch 2/7] ntp: Shorten xtime seqcount sections

[Thomas Gleixner]

[-- Attachment #1: xtime-ntp-shorten-seqcount-sections.patch --]
[-- Type: text/plain, Size: 2938 bytes --]

We only need to bump the seqcount for the actual leap seconds
injection.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 kernel/time/ntp.c         |    8 --------
 kernel/time/timekeeping.c |    2 ++
 2 files changed, 2 insertions(+), 8 deletions(-)

Index: linux-2.6/kernel/time/ntp.c
===================================================================
--- linux-2.6.orig/kernel/time/ntp.c
+++ linux-2.6/kernel/time/ntp.c
@@ -359,7 +359,6 @@ static enum hrtimer_restart ntp_leap_sec
 	enum hrtimer_restart res = HRTIMER_NORESTART;
 
 	raw_spin_lock(&xtime_lock);
-	write_seqcount_begin(&xtime_seq);
 
 	switch (time_state) {
 	case TIME_OK:
@@ -389,7 +388,6 @@ static enum hrtimer_restart ntp_leap_sec
 		break;
 	}
 
-	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock(&xtime_lock);
 
 	return res;
@@ -666,7 +664,6 @@ int do_adjtimex(struct timex *txc)
 	getnstimeofday(&ts);
 
 	raw_spin_lock_irq(&xtime_lock);
-	write_seqcount_begin(&xtime_seq);
 
 	if (txc->modes & ADJ_ADJTIME) {
 		long save_adjust = time_adjust;
@@ -708,7 +705,6 @@ int do_adjtimex(struct timex *txc)
 	/* fill PPS status fields */
 	pps_fill_timex(txc);
 
-	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock_irq(&xtime_lock);
 
 	txc->time.tv_sec = ts.tv_sec;
@@ -908,7 +904,6 @@ void hardpps(const struct timespec *phas
 	pts_norm = pps_normalize_ts(*phase_ts);
 
 	raw_spin_lock_irqsave(&xtime_lock, flags);
-	write_seqcount_begin(&xtime_seq);
 
 	/* clear the error bits, they will be set again if needed */
 	time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
@@ -921,7 +916,6 @@ void hardpps(const struct timespec *phas
 	 * just start the frequency interval */
 	if (unlikely(pps_fbase.tv_sec == 0)) {
 		pps_fbase = *raw_ts;
-		write_seqcount_end(&xtime_seq);
 		raw_spin_unlock_irqrestore(&xtime_lock, flags);
 		return;
 	}
@@ -937,7 +931,6 @@ void hardpps(const struct timespec *phas
 		time_status |= STA_PPSJITTER;
 		/* restart the frequency calibration interval */
 		pps_fbase = *raw_ts;
-		write_seqcount_end(&xtime_seq);
 		raw_spin_unlock_irqrestore(&xtime_lock, flags);
 		pr_err("hardpps: PPSJITTER: bad pulse\n");
 		return;
@@ -955,7 +948,6 @@ void hardpps(const struct timespec *phas
 
 	hardpps_update_phase(pts_norm.nsec);
 
-	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 }
 EXPORT_SYMBOL(hardpps);
Index: linux-2.6/kernel/time/timekeeping.c
===================================================================
--- linux-2.6.orig/kernel/time/timekeeping.c
+++ linux-2.6/kernel/time/timekeeping.c
@@ -173,10 +173,12 @@ int __read_mostly timekeeping_suspended;
 /* must hold xtime_lock */
 void timekeeping_leap_insert(int leapsecond)
 {
+	write_seqcount_begin(&xtime_seq);
 	xtime.tv_sec += leapsecond;
 	wall_to_monotonic.tv_sec -= leapsecond;
 	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
 			timekeeper.mult);
+	write_seqcount_end(&xtime_seq);
 }
 
 /**

[patch 1/7] timekeeping: Split xtime lock into lock and seqcount

[Thomas Gleixner]

[-- Attachment #1: xtime-split-lock-and-seqcount.patch --]
[-- Type: text/plain, Size: 16675 bytes --]

This allows us to shorten the sequence sections to the small sections
which actually update the timekeeping data structures, so the reader
blocked section becomes minimal.

Scripted conversion. No functional change.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 kernel/time/jiffies.c       |    4 -
 kernel/time/ntp.c           |   24 +++++++----
 kernel/time/tick-common.c   |   10 ++--
 kernel/time/tick-internal.h |    3 -
 kernel/time/tick-sched.c    |   16 ++++---
 kernel/time/timekeeping.c   |   95 +++++++++++++++++++++++++-------------------
 6 files changed, 91 insertions(+), 61 deletions(-)

Index: linux-2.6/kernel/time/jiffies.c
===================================================================
--- linux-2.6.orig/kernel/time/jiffies.c
+++ linux-2.6/kernel/time/jiffies.c
@@ -74,9 +74,9 @@ u64 get_jiffies_64(void)
 	u64 ret;
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 		ret = jiffies_64;
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 	return ret;
 }
 EXPORT_SYMBOL(get_jiffies_64);
Index: linux-2.6/kernel/time/ntp.c
===================================================================
--- linux-2.6.orig/kernel/time/ntp.c
+++ linux-2.6/kernel/time/ntp.c
@@ -358,7 +358,8 @@ static enum hrtimer_restart ntp_leap_sec
 {
 	enum hrtimer_restart res = HRTIMER_NORESTART;
 
-	write_seqlock(&xtime_lock);
+	raw_spin_lock(&xtime_lock);
+	write_seqcount_begin(&xtime_seq);
 
 	switch (time_state) {
 	case TIME_OK:
@@ -388,7 +389,8 @@ static enum hrtimer_restart ntp_leap_sec
 		break;
 	}
 
-	write_sequnlock(&xtime_lock);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock(&xtime_lock);
 
 	return res;
 }
@@ -663,7 +665,8 @@ int do_adjtimex(struct timex *txc)
 
 	getnstimeofday(&ts);
 
-	write_seqlock_irq(&xtime_lock);
+	raw_spin_lock_irq(&xtime_lock);
+	write_seqcount_begin(&xtime_seq);
 
 	if (txc->modes & ADJ_ADJTIME) {
 		long save_adjust = time_adjust;
@@ -705,7 +708,8 @@ int do_adjtimex(struct timex *txc)
 	/* fill PPS status fields */
 	pps_fill_timex(txc);
 
-	write_sequnlock_irq(&xtime_lock);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock_irq(&xtime_lock);
 
 	txc->time.tv_sec = ts.tv_sec;
 	txc->time.tv_usec = ts.tv_nsec;
@@ -903,7 +907,8 @@ void hardpps(const struct timespec *phas
 
 	pts_norm = pps_normalize_ts(*phase_ts);
 
-	write_seqlock_irqsave(&xtime_lock, flags);
+	raw_spin_lock_irqsave(&xtime_lock, flags);
+	write_seqcount_begin(&xtime_seq);
 
 	/* clear the error bits, they will be set again if needed */
 	time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
@@ -916,7 +921,8 @@ void hardpps(const struct timespec *phas
 	 * just start the frequency interval */
 	if (unlikely(pps_fbase.tv_sec == 0)) {
 		pps_fbase = *raw_ts;
-		write_sequnlock_irqrestore(&xtime_lock, flags);
+		write_seqcount_end(&xtime_seq);
+		raw_spin_unlock_irqrestore(&xtime_lock, flags);
 		return;
 	}
 
@@ -931,7 +937,8 @@ void hardpps(const struct timespec *phas
 		time_status |= STA_PPSJITTER;
 		/* restart the frequency calibration interval */
 		pps_fbase = *raw_ts;
-		write_sequnlock_irqrestore(&xtime_lock, flags);
+		write_seqcount_end(&xtime_seq);
+		raw_spin_unlock_irqrestore(&xtime_lock, flags);
 		pr_err("hardpps: PPSJITTER: bad pulse\n");
 		return;
 	}
@@ -948,7 +955,8 @@ void hardpps(const struct timespec *phas
 
 	hardpps_update_phase(pts_norm.nsec);
 
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 }
 EXPORT_SYMBOL(hardpps);
 
Index: linux-2.6/kernel/time/tick-common.c
===================================================================
--- linux-2.6.orig/kernel/time/tick-common.c
+++ linux-2.6/kernel/time/tick-common.c
@@ -63,13 +63,15 @@ int tick_is_oneshot_available(void)
 static void tick_periodic(int cpu)
 {
 	if (tick_do_timer_cpu == cpu) {
-		write_seqlock(&xtime_lock);
+		raw_spin_lock(&xtime_lock);
+		write_seqcount_begin(&xtime_seq);
 
 		/* Keep track of the next tick event */
 		tick_next_period = ktime_add(tick_next_period, tick_period);
 
 		do_timer(1);
-		write_sequnlock(&xtime_lock);
+		write_seqcount_end(&xtime_seq);
+		raw_spin_unlock(&xtime_lock);
 	}
 
 	update_process_times(user_mode(get_irq_regs()));
@@ -130,9 +132,9 @@ void tick_setup_periodic(struct clock_ev
 		ktime_t next;
 
 		do {
-			seq = read_seqbegin(&xtime_lock);
+			seq = read_seqcount_begin(&xtime_seq);
 			next = tick_next_period;
-		} while (read_seqretry(&xtime_lock, seq));
+		} while (read_seqcount_retry(&xtime_seq, seq));
 
 		clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
 
Index: linux-2.6/kernel/time/tick-internal.h
===================================================================
--- linux-2.6.orig/kernel/time/tick-internal.h
+++ linux-2.6/kernel/time/tick-internal.h
@@ -141,4 +141,5 @@ static inline int tick_device_is_functio
 #endif
 
 extern void do_timer(unsigned long ticks);
-extern seqlock_t xtime_lock;
+extern raw_spinlock_t xtime_lock;
+extern seqcount_t xtime_seq;
Index: linux-2.6/kernel/time/tick-sched.c
===================================================================
--- linux-2.6.orig/kernel/time/tick-sched.c
+++ linux-2.6/kernel/time/tick-sched.c
@@ -56,7 +56,8 @@ static void tick_do_update_jiffies64(kti
 		return;
 
 	/* Reevalute with xtime_lock held */
-	write_seqlock(&xtime_lock);
+	raw_spin_lock(&xtime_lock);
+	write_seqcount_begin(&xtime_seq);
 
 	delta = ktime_sub(now, last_jiffies_update);
 	if (delta.tv64 >= tick_period.tv64) {
@@ -79,7 +80,8 @@ static void tick_do_update_jiffies64(kti
 		/* Keep the tick_next_period variable up to date */
 		tick_next_period = ktime_add(last_jiffies_update, tick_period);
 	}
-	write_sequnlock(&xtime_lock);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock(&xtime_lock);
 }
 
 /*
@@ -89,12 +91,14 @@ static ktime_t tick_init_jiffy_update(vo
 {
 	ktime_t period;
 
-	write_seqlock(&xtime_lock);
+	raw_spin_lock(&xtime_lock);
+	write_seqcount_begin(&xtime_seq);
 	/* Did we start the jiffies update yet ? */
 	if (last_jiffies_update.tv64 == 0)
 		last_jiffies_update = tick_next_period;
 	period = last_jiffies_update;
-	write_sequnlock(&xtime_lock);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock(&xtime_lock);
 	return period;
 }
 
@@ -345,11 +349,11 @@ void tick_nohz_stop_sched_tick(int inidl
 	ts->idle_calls++;
 	/* Read jiffies and the time when jiffies were updated last */
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 		last_update = last_jiffies_update;
 		last_jiffies = jiffies;
 		time_delta = timekeeping_max_deferment();
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 
 	if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu) ||
 	    arch_needs_cpu(cpu)) {
Index: linux-2.6/kernel/time/timekeeping.c
===================================================================
--- linux-2.6.orig/kernel/time/timekeeping.c
+++ linux-2.6/kernel/time/timekeeping.c
@@ -136,10 +136,11 @@ static inline s64 timekeeping_get_ns_raw
 }
 
 /*
- * This read-write spinlock protects us from races in SMP while
- * playing with xtime.
+ * xtime_seq  allows lockless readers to observe updates
+ * xtime_lock protects the time keeping code
  */
-__cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
+__cacheline_aligned_in_smp seqcount_t xtime_seq;
+DEFINE_RAW_SPINLOCK(xtime_lock);
 
 
 /*
@@ -222,7 +223,7 @@ void getnstimeofday(struct timespec *ts)
 	WARN_ON(timekeeping_suspended);
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 
 		*ts = xtime;
 		nsecs = timekeeping_get_ns();
@@ -230,7 +231,7 @@ void getnstimeofday(struct timespec *ts)
 		/* If arch requires, add in gettimeoffset() */
 		nsecs += arch_gettimeoffset();
 
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 
 	timespec_add_ns(ts, nsecs);
 }
@@ -245,12 +246,12 @@ ktime_t ktime_get(void)
 	WARN_ON(timekeeping_suspended);
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 		secs = xtime.tv_sec + wall_to_monotonic.tv_sec;
 		nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec;
 		nsecs += timekeeping_get_ns();
 
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 	/*
 	 * Use ktime_set/ktime_add_ns to create a proper ktime on
 	 * 32-bit architectures without CONFIG_KTIME_SCALAR.
@@ -276,12 +277,12 @@ void ktime_get_ts(struct timespec *ts)
 	WARN_ON(timekeeping_suspended);
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 		*ts = xtime;
 		tomono = wall_to_monotonic;
 		nsecs = timekeeping_get_ns();
 
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 
 	set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
 				ts->tv_nsec + tomono.tv_nsec + nsecs);
@@ -309,7 +310,7 @@ void getnstime_raw_and_real(struct times
 	do {
 		u32 arch_offset;
 
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 
 		*ts_raw = raw_time;
 		*ts_real = xtime;
@@ -322,7 +323,7 @@ void getnstime_raw_and_real(struct times
 		nsecs_raw += arch_offset;
 		nsecs_real += arch_offset;
 
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 
 	timespec_add_ns(ts_raw, nsecs_raw);
 	timespec_add_ns(ts_real, nsecs_real);
@@ -361,7 +362,8 @@ int do_settimeofday(const struct timespe
 	if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
 		return -EINVAL;
 
-	write_seqlock_irqsave(&xtime_lock, flags);
+	raw_spin_lock_irqsave(&xtime_lock, flags);
+	write_seqcount_begin(&xtime_seq);
 
 	timekeeping_forward_now();
 
@@ -377,7 +379,8 @@ int do_settimeofday(const struct timespe
 	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
 				timekeeper.mult);
 
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 
 	/* signal hrtimers about time change */
 	clock_was_set();
@@ -401,7 +404,8 @@ int timekeeping_inject_offset(struct tim
 	if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
 		return -EINVAL;
 
-	write_seqlock_irqsave(&xtime_lock, flags);
+	raw_spin_lock_irqsave(&xtime_lock, flags);
+	write_seqcount_begin(&xtime_seq);
 
 	timekeeping_forward_now();
 
@@ -414,7 +418,8 @@ int timekeeping_inject_offset(struct tim
 	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
 				timekeeper.mult);
 
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 
 	/* signal hrtimers about time change */
 	clock_was_set();
@@ -486,11 +491,11 @@ void getrawmonotonic(struct timespec *ts
 	s64 nsecs;
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 		nsecs = timekeeping_get_ns_raw();
 		*ts = raw_time;
 
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 
 	timespec_add_ns(ts, nsecs);
 }
@@ -506,11 +511,11 @@ int timekeeping_valid_for_hres(void)
 	int ret;
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 
 		ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
 
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 
 	return ret;
 }
@@ -518,7 +523,7 @@ int timekeeping_valid_for_hres(void)
 /**
  * timekeeping_max_deferment - Returns max time the clocksource can be deferred
  *
- * Caller must observe xtime_lock via read_seqbegin/read_seqretry to
+ * Caller must observe xtime_seq via read_seqcount_begin/req_seqcount_retry to
  * ensure that the clocksource does not change!
  */
 u64 timekeeping_max_deferment(void)
@@ -568,7 +573,8 @@ void __init timekeeping_init(void)
 	read_persistent_clock(&now);
 	read_boot_clock(&boot);
 
-	write_seqlock_irqsave(&xtime_lock, flags);
+	raw_spin_lock_irqsave(&xtime_lock, flags);
+	write_seqcount_begin(&xtime_seq);
 
 	ntp_init();
 
@@ -589,7 +595,8 @@ void __init timekeeping_init(void)
 				-boot.tv_sec, -boot.tv_nsec);
 	total_sleep_time.tv_sec = 0;
 	total_sleep_time.tv_nsec = 0;
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 }
 
 /* time in seconds when suspend began */
@@ -636,7 +643,8 @@ void timekeeping_inject_sleeptime(struct
 	if (!(ts.tv_sec == 0 && ts.tv_nsec == 0))
 		return;
 
-	write_seqlock_irqsave(&xtime_lock, flags);
+	raw_spin_lock_irqsave(&xtime_lock, flags);
+	write_seqcount_begin(&xtime_seq);
 	timekeeping_forward_now();
 
 	__timekeeping_inject_sleeptime(delta);
@@ -646,7 +654,8 @@ void timekeeping_inject_sleeptime(struct
 	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
 				timekeeper.mult);
 
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 
 	/* signal hrtimers about time change */
 	clock_was_set();
@@ -669,7 +678,8 @@ static void timekeeping_resume(void)
 
 	clocksource_resume();
 
-	write_seqlock_irqsave(&xtime_lock, flags);
+	raw_spin_lock_irqsave(&xtime_lock, flags);
+	write_seqcount_begin(&xtime_seq);
 
 	if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
 		ts = timespec_sub(ts, timekeeping_suspend_time);
@@ -679,7 +689,8 @@ static void timekeeping_resume(void)
 	timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
 	timekeeper.ntp_error = 0;
 	timekeeping_suspended = 0;
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 
 	touch_softlockup_watchdog();
 
@@ -697,7 +708,8 @@ static int timekeeping_suspend(void)
 
 	read_persistent_clock(&timekeeping_suspend_time);
 
-	write_seqlock_irqsave(&xtime_lock, flags);
+	raw_spin_lock_irqsave(&xtime_lock, flags);
+	write_seqcount_begin(&xtime_seq);
 	timekeeping_forward_now();
 	timekeeping_suspended = 1;
 
@@ -720,7 +732,8 @@ static int timekeeping_suspend(void)
 		timekeeping_suspend_time =
 			timespec_add(timekeeping_suspend_time, delta_delta);
 	}
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 
 	clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
 	clocksource_suspend();
@@ -1011,13 +1024,13 @@ void get_monotonic_boottime(struct times
 	WARN_ON(timekeeping_suspended);
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 		*ts = xtime;
 		tomono = wall_to_monotonic;
 		sleep = total_sleep_time;
 		nsecs = timekeeping_get_ns();
 
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 
 	set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec,
 			ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec + nsecs);
@@ -1068,10 +1081,10 @@ struct timespec current_kernel_time(void
 	unsigned long seq;
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 
 		now = xtime;
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 
 	return now;
 }
@@ -1083,11 +1096,11 @@ struct timespec get_monotonic_coarse(voi
 	unsigned long seq;
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 
 		now = xtime;
 		mono = wall_to_monotonic;
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 
 	set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
 				now.tv_nsec + mono.tv_nsec);
@@ -1119,11 +1132,11 @@ void get_xtime_and_monotonic_and_sleep_o
 	unsigned long seq;
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 		*xtim = xtime;
 		*wtom = wall_to_monotonic;
 		*sleep = total_sleep_time;
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 }
 
 /**
@@ -1135,9 +1148,9 @@ ktime_t ktime_get_monotonic_offset(void)
 	struct timespec wtom;
 
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqcount_begin(&xtime_seq);
 		wtom = wall_to_monotonic;
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqcount_retry(&xtime_seq, seq));
 	return timespec_to_ktime(wtom);
 }
 
@@ -1149,7 +1162,9 @@ ktime_t ktime_get_monotonic_offset(void)
  */
 void xtime_update(unsigned long ticks)
 {
-	write_seqlock(&xtime_lock);
+	raw_spin_lock(&xtime_lock);
+	write_seqcount_begin(&xtime_seq);
 	do_timer(ticks);
-	write_sequnlock(&xtime_lock);
+	write_seqcount_end(&xtime_seq);
+	raw_spin_unlock(&xtime_lock);
 }

[patch 3/7] timekeeping: Make seqcount sections smaller

[Thomas Gleixner]

[-- Attachment #1: xtime-timekeeping-seqcount-sections.patch --]
[-- Type: text/plain, Size: 2706 bytes --]

Move the seqcount bump inside of do_timer() and remove it from
sections which only need the xtime_lock serialization.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 kernel/time/tick-common.c |    2 --
 kernel/time/tick-sched.c  |    4 ----
 kernel/time/timekeeping.c |    4 ++--
 3 files changed, 2 insertions(+), 8 deletions(-)

Index: linux-2.6/kernel/time/tick-common.c
===================================================================
--- linux-2.6.orig/kernel/time/tick-common.c
+++ linux-2.6/kernel/time/tick-common.c
@@ -64,13 +64,11 @@ static void tick_periodic(int cpu)
 {
 	if (tick_do_timer_cpu == cpu) {
 		raw_spin_lock(&xtime_lock);
-		write_seqcount_begin(&xtime_seq);
 
 		/* Keep track of the next tick event */
 		tick_next_period = ktime_add(tick_next_period, tick_period);
 
 		do_timer(1);
-		write_seqcount_end(&xtime_seq);
 		raw_spin_unlock(&xtime_lock);
 	}
 
Index: linux-2.6/kernel/time/tick-sched.c
===================================================================
--- linux-2.6.orig/kernel/time/tick-sched.c
+++ linux-2.6/kernel/time/tick-sched.c
@@ -57,7 +57,6 @@ static void tick_do_update_jiffies64(kti
 
 	/* Reevalute with xtime_lock held */
 	raw_spin_lock(&xtime_lock);
-	write_seqcount_begin(&xtime_seq);
 
 	delta = ktime_sub(now, last_jiffies_update);
 	if (delta.tv64 >= tick_period.tv64) {
@@ -80,7 +79,6 @@ static void tick_do_update_jiffies64(kti
 		/* Keep the tick_next_period variable up to date */
 		tick_next_period = ktime_add(last_jiffies_update, tick_period);
 	}
-	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock(&xtime_lock);
 }
 
@@ -92,12 +90,10 @@ static ktime_t tick_init_jiffy_update(vo
 	ktime_t period;
 
 	raw_spin_lock(&xtime_lock);
-	write_seqcount_begin(&xtime_seq);
 	/* Did we start the jiffies update yet ? */
 	if (last_jiffies_update.tv64 == 0)
 		last_jiffies_update = tick_next_period;
 	period = last_jiffies_update;
-	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock(&xtime_lock);
 	return period;
 }
Index: linux-2.6/kernel/time/timekeeping.c
===================================================================
--- linux-2.6.orig/kernel/time/timekeeping.c
+++ linux-2.6/kernel/time/timekeeping.c
@@ -1116,8 +1116,10 @@ struct timespec get_monotonic_coarse(voi
  */
 void do_timer(unsigned long ticks)
 {
+	write_seqcount_begin(&xtime_seq);
 	jiffies_64 += ticks;
 	update_wall_time();
+	write_seqcount_end(&xtime_seq);
 	calc_global_load(ticks);
 }
 
@@ -1165,8 +1167,6 @@ ktime_t ktime_get_monotonic_offset(void)
 void xtime_update(unsigned long ticks)
 {
 	raw_spin_lock(&xtime_lock);
-	write_seqcount_begin(&xtime_seq);
 	do_timer(ticks);
-	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock(&xtime_lock);
 }

[patch 4/7] timekeeper: Reorder so the hot data is together

[Thomas Gleixner]

[-- Attachment #1: timekeeper-reorder.patch --]
[-- Type: text/plain, Size: 922 bytes --]

Keep all the interesting data in a single cache line.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 kernel/time/timekeeping.c |    4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

Index: linux-2.6-tip/kernel/time/timekeeping.c
===================================================================
--- linux-2.6-tip.orig/kernel/time/timekeeping.c
+++ linux-2.6-tip/kernel/time/timekeeping.c
@@ -27,6 +27,8 @@ struct timekeeper {
 	struct clocksource *clock;
 	/* The shift value of the current clocksource. */
 	int	shift;
+	/* NTP adjusted clock multiplier */
+	u32	mult;
 
 	/* Number of clock cycles in one NTP interval. */
 	cycle_t cycle_interval;
@@ -45,8 +47,6 @@ struct timekeeper {
 	/* Shift conversion between clock shifted nano seconds and
 	 * ntp shifted nano seconds. */
 	int	ntp_error_shift;
-	/* NTP adjusted clock multiplier */
-	u32	mult;
 };
 
 static struct timekeeper timekeeper;

[patch 6/7] timekeeping: Add extra timekeeper/xtime structs for calculation

[Thomas Gleixner]

[-- Attachment #1: timekeeping-add-calc-struct.patch --]
[-- Type: text/plain, Size: 8596 bytes --]

The timekeeping/xtime is directly changed by the update_wall_time()
code which requires that the seqcount protection has to spawn the full
region. Create shadow structs which are used for calculation and then
update the real ones after the calculation finished.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 kernel/time/timekeeping.c |   95 ++++++++++++++++++++++++++--------------------
 1 file changed, 55 insertions(+), 40 deletions(-)

Index: linux-2.6-tip/kernel/time/timekeeping.c
===================================================================
--- linux-2.6-tip.orig/kernel/time/timekeeping.c
+++ linux-2.6-tip/kernel/time/timekeeping.c
@@ -51,6 +51,11 @@ struct timekeeper {
 
 static struct timekeeper timekeeper;
 
+/* Used to decouple calculations from timekeeper/xtime */
+static struct timekeeper tk_calc;
+static struct timespec xtime_calc;
+static void timekeeping_update_calc(void);
+
 /**
  * timekeeper_setup_internals - Set up internals to use clocksource clock.
  *
@@ -99,6 +104,7 @@ static void timekeeper_setup_internals(s
 	 * to counteract clock drifting.
 	 */
 	timekeeper.mult = clock->mult;
+	timekeeping_update_calc();
 }
 
 /* Timekeeper helper functions. */
@@ -170,6 +176,12 @@ static struct timespec raw_time;
 /* flag for if timekeeping is suspended */
 int __read_mostly timekeeping_suspended;
 
+static void timekeeping_update_calc(void)
+{
+	xtime_calc = xtime;
+	tk_calc = timekeeper;
+}
+
 static void timekeeping_update(bool clearntp)
 {
 	if (clearntp) {
@@ -178,6 +190,7 @@ static void timekeeping_update(bool clea
 	}
 	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
 			timekeeper.mult);
+	timekeeping_update_calc();
 }
 
 /* must hold xtime_lock */
@@ -596,6 +609,7 @@ void __init timekeeping_init(void)
 				-boot.tv_sec, -boot.tv_nsec);
 	total_sleep_time.tv_sec = 0;
 	total_sleep_time.tv_nsec = 0;
+	timekeeping_update_calc();
 	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 }
@@ -772,7 +786,7 @@ static __always_inline int timekeeping_b
 	 * here.  This is tuned so that an error of about 1 msec is adjusted
 	 * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
 	 */
-	error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
+	error2 = tk_calc.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
 	error2 = abs(error2);
 	for (look_ahead = 0; error2 > 0; look_ahead++)
 		error2 >>= 2;
@@ -781,8 +795,8 @@ static __always_inline int timekeeping_b
 	 * Now calculate the error in (1 << look_ahead) ticks, but first
 	 * remove the single look ahead already included in the error.
 	 */
-	tick_error = tick_length >> (timekeeper.ntp_error_shift + 1);
-	tick_error -= timekeeper.xtime_interval >> 1;
+	tick_error = tick_length >> (tk_calc.ntp_error_shift + 1);
+	tick_error -= tk_calc.xtime_interval >> 1;
 	error = ((error - tick_error) >> look_ahead) + tick_error;
 
 	/* Finally calculate the adjustment shift value.  */
@@ -809,10 +823,10 @@ static __always_inline int timekeeping_b
  */
 static void timekeeping_adjust(s64 offset)
 {
-	s64 error, interval = timekeeper.cycle_interval;
+	s64 error, interval = tk_calc.cycle_interval;
 	int adj;
 
-	error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1);
+	error = tk_calc.ntp_error >> (tk_calc.ntp_error_shift - 1);
 	if (error > interval) {
 		error >>= 2;
 		if (likely(error <= interval))
@@ -830,11 +844,10 @@ static void timekeeping_adjust(s64 offse
 	} else
 		return;
 
-	timekeeper.mult += adj;
-	timekeeper.xtime_interval += interval;
-	timekeeper.xtime_nsec -= offset;
-	timekeeper.ntp_error -= (interval - offset) <<
-				timekeeper.ntp_error_shift;
+	tk_calc.mult += adj;
+	tk_calc.xtime_interval += interval;
+	tk_calc.xtime_nsec -= offset;
+	tk_calc.ntp_error -= (interval - offset) << tk_calc.ntp_error_shift;
 }
 
 
@@ -849,26 +862,26 @@ static void timekeeping_adjust(s64 offse
  */
 static cycle_t logarithmic_accumulation(cycle_t offset, int shift)
 {
-	u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift;
+	u64 nsecps = (u64)NSEC_PER_SEC << tk_calc.shift;
 	u64 raw_nsecs;
 
 	/* If the offset is smaller then a shifted interval, do nothing */
-	if (offset < timekeeper.cycle_interval<<shift)
+	if (offset < tk_calc.cycle_interval<<shift)
 		return offset;
 
 	/* Accumulate one shifted interval */
-	offset -= timekeeper.cycle_interval << shift;
-	timekeeper.clock->cycle_last += timekeeper.cycle_interval << shift;
+	offset -= tk_calc.cycle_interval << shift;
+	tk_calc.clock->cycle_last += tk_calc.cycle_interval << shift;
 
-	timekeeper.xtime_nsec += timekeeper.xtime_interval << shift;
-	while (timekeeper.xtime_nsec >= nsecps) {
-		timekeeper.xtime_nsec -= nsecps;
-		xtime.tv_sec++;
+	tk_calc.xtime_nsec += tk_calc.xtime_interval << shift;
+	while (tk_calc.xtime_nsec >= nsecps) {
+		tk_calc.xtime_nsec -= nsecps;
+		xtime_calc.tv_sec++;
 		second_overflow();
 	}
 
 	/* Accumulate raw time */
-	raw_nsecs = timekeeper.raw_interval << shift;
+	raw_nsecs = tk_calc.raw_interval << shift;
 	raw_nsecs += raw_time.tv_nsec;
 	if (raw_nsecs >= NSEC_PER_SEC) {
 		u64 raw_secs = raw_nsecs;
@@ -878,10 +891,10 @@ static cycle_t logarithmic_accumulation(
 	raw_time.tv_nsec = raw_nsecs;
 
 	/* Accumulate error between NTP and clock interval */
-	timekeeper.ntp_error += tick_length << shift;
-	timekeeper.ntp_error -=
-	    (timekeeper.xtime_interval + timekeeper.xtime_remainder) <<
-				(timekeeper.ntp_error_shift + shift);
+	tk_calc.ntp_error += tick_length << shift;
+	tk_calc.ntp_error -=
+	    (tk_calc.xtime_interval + tk_calc.xtime_remainder) <<
+				(tk_calc.ntp_error_shift + shift);
 
 	return offset;
 }
@@ -902,14 +915,14 @@ static void update_wall_time(void)
 	if (unlikely(timekeeping_suspended))
 		return;
 
-	clock = timekeeper.clock;
+	clock = tk_calc.clock;
 
 #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
-	offset = timekeeper.cycle_interval;
+	offset = tk_calc.cycle_interval;
 #else
 	offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
 #endif
-	timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift;
+	tk_calc.xtime_nsec = (s64)xtime.tv_nsec << tk_calc.shift;
 
 	/*
 	 * With NO_HZ we may have to accumulate many cycle_intervals
@@ -919,14 +932,14 @@ static void update_wall_time(void)
 	 * chunk in one go, and then try to consume the next smaller
 	 * doubled multiple.
 	 */
-	shift = ilog2(offset) - ilog2(timekeeper.cycle_interval);
+	shift = ilog2(offset) - ilog2(tk_calc.cycle_interval);
 	shift = max(0, shift);
 	/* Bound shift to one less then what overflows tick_length */
 	maxshift = (8*sizeof(tick_length) - (ilog2(tick_length)+1)) - 1;
 	shift = min(shift, maxshift);
-	while (offset >= timekeeper.cycle_interval) {
+	while (offset >= tk_calc.cycle_interval) {
 		offset = logarithmic_accumulation(offset, shift);
-		if(offset < timekeeper.cycle_interval<<shift)
+		if(offset < tk_calc.cycle_interval<<shift)
 			shift--;
 	}
 
@@ -949,10 +962,10 @@ static void update_wall_time(void)
 	 * We'll correct this error next time through this function, when
 	 * xtime_nsec is not as small.
 	 */
-	if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
-		s64 neg = -(s64)timekeeper.xtime_nsec;
-		timekeeper.xtime_nsec = 0;
-		timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
+	if (unlikely((s64)tk_calc.xtime_nsec < 0)) {
+		s64 neg = -(s64)tk_calc.xtime_nsec;
+		tk_calc.xtime_nsec = 0;
+		tk_calc.ntp_error += neg << tk_calc.ntp_error_shift;
 	}
 
 
@@ -960,21 +973,23 @@ static void update_wall_time(void)
 	 * Store full nanoseconds into xtime after rounding it up and
 	 * add the remainder to the error difference.
 	 */
-	xtime.tv_nsec =	((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1;
-	timekeeper.xtime_nsec -= (s64) xtime.tv_nsec << timekeeper.shift;
-	timekeeper.ntp_error +=	timekeeper.xtime_nsec <<
-				timekeeper.ntp_error_shift;
+	xtime_calc.tv_nsec = ((s64) tk_calc.xtime_nsec >> tk_calc.shift) + 1;
+	tk_calc.xtime_nsec -= (s64) xtime_calc.tv_nsec << tk_calc.shift;
+	tk_calc.ntp_error += tk_calc.xtime_nsec << tk_calc.ntp_error_shift;
 
 	/*
 	 * Finally, make sure that after the rounding
 	 * xtime.tv_nsec isn't larger then NSEC_PER_SEC
 	 */
-	if (unlikely(xtime.tv_nsec >= NSEC_PER_SEC)) {
-		xtime.tv_nsec -= NSEC_PER_SEC;
-		xtime.tv_sec++;
+	if (unlikely(xtime_calc.tv_nsec >= NSEC_PER_SEC)) {
+		xtime_calc.tv_nsec -= NSEC_PER_SEC;
+		xtime_calc.tv_sec++;
 		second_overflow();
 	}
 
+	timekeeper = tk_calc;
+	xtime = xtime_calc;
+
 	/* check to see if there is a new clocksource to use */
 	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
 				timekeeper.mult);

[patch 5/7] timekeeping: Move common updates to a function

[Thomas Gleixner]

[-- Attachment #1: timekeeping-collapse-copied-code.patch --]
[-- Type: text/plain, Size: 2166 bytes --]

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 kernel/time/timekeeping.c |   33 ++++++++++++++-------------------
 1 file changed, 14 insertions(+), 19 deletions(-)

Index: linux-2.6-tip/kernel/time/timekeeping.c
===================================================================
--- linux-2.6-tip.orig/kernel/time/timekeeping.c
+++ linux-2.6-tip/kernel/time/timekeeping.c
@@ -170,14 +170,23 @@ static struct timespec raw_time;
 /* flag for if timekeeping is suspended */
 int __read_mostly timekeeping_suspended;
 
+static void timekeeping_update(bool clearntp)
+{
+	if (clearntp) {
+		timekeeper.ntp_error = 0;
+		ntp_clear();
+	}
+	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
+			timekeeper.mult);
+}
+
 /* must hold xtime_lock */
 void timekeeping_leap_insert(int leapsecond)
 {
 	write_seqcount_begin(&xtime_seq);
 	xtime.tv_sec += leapsecond;
 	wall_to_monotonic.tv_sec -= leapsecond;
-	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
-			timekeeper.mult);
+	timekeeping_update(false);
 	write_seqcount_end(&xtime_seq);
 }
 
@@ -375,12 +384,7 @@ int do_settimeofday(const struct timespe
 
 	xtime = *tv;
 
-	timekeeper.ntp_error = 0;
-	ntp_clear();
-
-	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
-				timekeeper.mult);
-
+	timekeeping_update(true);
 	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 
@@ -414,12 +418,7 @@ int timekeeping_inject_offset(struct tim
 	xtime = timespec_add(xtime, *ts);
 	wall_to_monotonic = timespec_sub(wall_to_monotonic, *ts);
 
-	timekeeper.ntp_error = 0;
-	ntp_clear();
-
-	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
-				timekeeper.mult);
-
+	timekeeping_update(true);
 	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 
@@ -651,11 +650,7 @@ void timekeeping_inject_sleeptime(struct
 
 	__timekeeping_inject_sleeptime(delta);
 
-	timekeeper.ntp_error = 0;
-	ntp_clear();
-	update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
-				timekeeper.mult);
-
+	timekeeping_update(true);
 	write_seqcount_end(&xtime_seq);
 	raw_spin_unlock_irqrestore(&xtime_lock, flags);
 

[patch 7/7] timekeeping: Reduce seqcount section to update timekeeper/xtime

[Thomas Gleixner]

[-- Attachment #1: timekeeping-reduce-seqcnt-section-more.patch --]
[-- Type: text/plain, Size: 1058 bytes --]

The calculation of the timekeeping values is now decoupled from
timekeeper/xtime which are used by the readers. So we can confine the
seqcount section to the mere update of the reader side data.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 kernel/time/timekeeping.c |    8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

Index: linux-2.6-tip/kernel/time/timekeeping.c
===================================================================
--- linux-2.6-tip.orig/kernel/time/timekeeping.c
+++ linux-2.6-tip/kernel/time/timekeeping.c
@@ -986,7 +986,10 @@ static void update_wall_time(void)
 		xtime_calc.tv_sec++;
 		second_overflow();
 	}
+}
 
+static void update_timekeeper(void)
+{
 	timekeeper = tk_calc;
 	xtime = xtime_calc;
 
@@ -1126,10 +1129,13 @@ struct timespec get_monotonic_coarse(voi
  */
 void do_timer(unsigned long ticks)
 {
+	update_wall_time();
+
 	write_seqcount_begin(&xtime_seq);
 	jiffies_64 += ticks;
-	update_wall_time();
+	update_timekeeper();
 	write_seqcount_end(&xtime_seq);
+
 	calc_global_load(ticks);
 }
 

Re: [patch 0/7] timekeeping: Decouple xtime_lock from xtime seqcount

[Eric Dumazet]

Le dimanche 13 novembre 2011 à 23:19 +0000, Thomas Gleixner a écrit :
> xtime_lock which serializes the update of time relevant data
> structures is held over a large code pathes which results in extended
> seqlock contention times on the reader side. This series decouples the
> lock from the seqcount and reduces the seqcount protected sections to
> those which actually update the reader visible data.
> 
> Thanks,

Thanks a lot Thomas, I'll take a look on this serie ASAP.