[PATCH 0/3] timers: Disable memory pre-allocation of timer debug objects

[PATCH 0/3] timers: Disable memory pre-allocation of timer debug objects

[Waiman Long]

A circular locking dependency was reported by lockdep involving printk()
called from within the memory allocator, console driver with timeout
capability and timer code calling debug_object_activate() doing object
pool refill by allocating memory.

One simple way to break this circular locking dependency is to disable
memory allocation when timer debug objects are being handled which is
what this series is about. The majority of the changes is in the
debugobjects code with a one-line change in the timer code.

I have also considered updating the console driver to not hold the lock
when setting up timeout, but many console drivers are using timeout in
some ways. So this problem may not specific to just a single one.

Waiman Long (3):
  debugobjects: Add ODEBUG_FLAG_NO_ALLOC to disable memory allocation
  debugobjects: Show the state of debug_objects_enabled
  timers: Disable memory pre-allocation of timer debug objects

 include/linux/debugobjects.h |  6 ++++++
 kernel/time/timer.c          |  1 +
 lib/debugobjects.c           | 29 +++++++++++++++++------------
 3 files changed, 24 insertions(+), 12 deletions(-)

-- 
2.49.0

[PATCH 1/3] debugobjects: Add ODEBUG_FLAG_NO_ALLOC to disable memory allocation

[Waiman Long]

Some of the objects associated with debug_obj may be handled
in a sensitive context that allowing debug_obj pre-allocation
in debug_objects_fill_pool() may casue deadlock. Add a new flags
parameter to the debug_obj_descr structure as well as adding the new
ODEBUG_FLAG_NO_ALLOC flag to enable us to disallow memory allocation
for those types of objects.

Signed-off-by: Waiman Long <longman@redhat.com>
---
 include/linux/debugobjects.h |  6 ++++++
 lib/debugobjects.c           | 10 +++++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/include/linux/debugobjects.h b/include/linux/debugobjects.h
index 8b95545e7924..a058c7dba898 100644
--- a/include/linux/debugobjects.h
+++ b/include/linux/debugobjects.h
@@ -41,6 +41,7 @@ struct debug_obj {
  * struct debug_obj_descr - object type specific debug description structure
  *
  * @name:		name of the object typee
+ * @flags:		debug object flags
  * @debug_hint:		function returning address, which have associated
  *			kernel symbol, to allow identify the object
  * @is_static_object:	return true if the obj is static, otherwise return false
@@ -58,6 +59,7 @@ struct debug_obj {
  */
 struct debug_obj_descr {
 	const char		*name;
+	unsigned long		 flags;
 	void *(*debug_hint)(void *addr);
 	bool (*is_static_object)(void *addr);
 	bool (*fixup_init)(void *addr, enum debug_obj_state state);
@@ -67,6 +69,10 @@ struct debug_obj_descr {
 	bool (*fixup_assert_init)(void *addr, enum debug_obj_state state);
 };
 
+enum debug_obj_flags {
+	ODEBUG_FLAG_NO_ALLOC = 0x1,	/* Disallow debug object pre-allocation */
+};
+
 #ifdef CONFIG_DEBUG_OBJECTS
 extern void debug_object_init      (void *addr, const struct debug_obj_descr *descr);
 extern void
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 7f50c4480a4e..52bc77b41f48 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -694,7 +694,7 @@ static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket
 	return NULL;
 }
 
-static void debug_objects_fill_pool(void)
+static void debug_objects_fill_pool(bool no_alloc)
 {
 	if (!static_branch_likely(&obj_cache_enabled))
 		return;
@@ -705,7 +705,7 @@ static void debug_objects_fill_pool(void)
 	/* Try reusing objects from obj_to_free_list */
 	fill_pool_from_freelist();
 
-	if (likely(!pool_should_refill(&pool_global)))
+	if (likely(!pool_should_refill(&pool_global) || no_alloc))
 		return;
 
 	/*
@@ -734,7 +734,7 @@ __debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack
 	struct debug_bucket *db;
 	unsigned long flags;
 
-	debug_objects_fill_pool();
+	debug_objects_fill_pool(descr->flags & ODEBUG_FLAG_NO_ALLOC);
 
 	db = get_bucket((unsigned long) addr);
 
@@ -811,7 +811,7 @@ int debug_object_activate(void *addr, const struct debug_obj_descr *descr)
 	if (!debug_objects_enabled)
 		return 0;
 
-	debug_objects_fill_pool();
+	debug_objects_fill_pool(descr->flags & ODEBUG_FLAG_NO_ALLOC);
 
 	db = get_bucket((unsigned long) addr);
 
@@ -1000,7 +1000,7 @@ void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr)
 	if (!debug_objects_enabled)
 		return;
 
-	debug_objects_fill_pool();
+	debug_objects_fill_pool(descr->flags & ODEBUG_FLAG_NO_ALLOC);
 
 	db = get_bucket((unsigned long) addr);
 
-- 
2.49.0

[PATCH 2/3] debugobjects: Show the state of debug_objects_enabled

[Waiman Long]

In the rare case that debug_objects got disabled because we are running
out of free debug objects, it is not easy to figure this out. Fix that
by showing the state of "debug_objects_enabled" in the stats debugfs
file as well as always printing a message in the console log.

Signed-off-by: Waiman Long <longman@redhat.com>
---
 lib/debugobjects.c | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 52bc77b41f48..0e9f44db9043 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -125,6 +125,12 @@ static int __init disable_object_debug(char *str)
 }
 early_param("no_debug_objects", disable_object_debug);
 
+static void debug_objects_disable(const char *msg)
+{
+	debug_objects_enabled = false;
+	pr_warn("debug_objects disabled: %s\n", msg);
+}
+
 static const char *obj_states[ODEBUG_STATE_MAX] = {
 	[ODEBUG_STATE_NONE]		= "none",
 	[ODEBUG_STATE_INIT]		= "initialized",
@@ -690,7 +696,7 @@ static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket
 	}
 
 	/* Out of memory. Do the cleanup outside of the locked region */
-	debug_objects_enabled = false;
+	debug_objects_disable("out of memory");
 	return NULL;
 }
 
@@ -1161,6 +1167,8 @@ static int debug_stats_show(struct seq_file *m, void *v)
 	seq_printf(m, "on_free_list  : %u\n", pool_count(&pool_to_free));
 	seq_printf(m, "objs_allocated: %d\n", debug_objects_allocated);
 	seq_printf(m, "objs_freed    : %d\n", debug_objects_freed);
+	seq_printf(m, "debug_objects : %s\n", debug_objects_enabled ? "enabled"
+								    : "disabled");
 	return 0;
 }
 DEFINE_SHOW_ATTRIBUTE(debug_stats);
@@ -1314,7 +1322,7 @@ check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
 out:
 	raw_spin_unlock_irqrestore(&db->lock, flags);
 	if (res)
-		debug_objects_enabled = false;
+		debug_object_disable("selftest");
 	return res;
 }
 
@@ -1486,11 +1494,8 @@ void __init debug_objects_mem_init(void)
 	cache = kmem_cache_create("debug_objects_cache", sizeof (struct debug_obj), 0,
 				  SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE, NULL);
 
-	if (!cache || !debug_objects_replace_static_objects(cache)) {
-		debug_objects_enabled = false;
-		pr_warn("Out of memory.\n");
-		return;
-	}
+	if (!cache || !debug_objects_replace_static_objects(cache))
+		debug_objects_disable("out of memory");
 
 	/*
 	 * Adjust the thresholds for allocating and freeing objects
-- 
2.49.0

[PATCH 3/3] timers: Disable memory pre-allocation of timer debug objects

[Waiman Long]

A circular locking dependency lockdep splat was hit recently with a
debug kernel. The dependency chain (in reverse order) is:

  -> #3 (&zone->lock){-.-.}-{2:2}:
  -> #2 (&base->lock){-.-.}-{2:2}:
  -> #1 (&console_sch_key){-.-.}-{2:2}:
  -> #0 (console_owner){..-.}-{0:0}:

The last one is from calling printk() within the rmqueue_bulk() call in
mm/page_alloc.c. The "base->lock" is from lock_timer_base() and first
one is due to calling add_timer_on() leading to debug_object_activate()
doing actual memory allocation acquiring the zone lock.

The console_sch_key comes from a s390 console driver in driver/s390/cio.
The console_sch_key -> timer dependency happens because the console
driver is setting a timeout value while holding its lock. Apparently it
is pretty common for a console driver to use timer for timeout or other
timing purposes. So this may happen to other console drivers as well.

One way to break this circular locking dependency is to disallow any
memory allocation when a timer debug object is being handled. Do this by
setting the ODEBUG_FLAG_NO_ALLOC flag in the timer_debug_descr structure.

The figures below show the number of times the debug_objects_fill_pool()
function has reached the statement right before and after the no_alloc
check in initial bootup and after running a parallel kernel build on
a 2-socket 96-threads x86-64 system.

			 Before      After     non-timer %
		 	 ------      -----     -----------
  Initial bootup	  150,730     148,198     98.3%
  Parallel kernel build 5,974,464   5,893,116     98.6%

So from object pre-allocation perspective, timer debug objects represent
just a small slice of the total number of debug objects to be processed.

The allocation of debug_object to the global pool happens when its object
count falls below the (256 + 16 * num_possible_cpus) threshold. Even
then, there may still be free objects available in the percpu pool. So
the chance that debug_objects gets disabled because it is running out
of free debug_object should be minimal.

Signed-off-by: Waiman Long <longman@redhat.com>
---
 kernel/time/timer.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 553fa469d7cc..e0be64591e43 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -775,6 +775,7 @@ static bool timer_fixup_assert_init(void *addr, enum debug_obj_state state)
 
 static const struct debug_obj_descr timer_debug_descr = {
 	.name			= "timer_list",
+	.flags			= ODEBUG_FLAG_NO_ALLOC,
 	.debug_hint		= timer_debug_hint,
 	.is_static_object	= timer_is_static_object,
 	.fixup_init		= timer_fixup_init,
-- 
2.49.0

Re: [PATCH 2/3] debugobjects: Show the state of debug_objects_enabled

[kernel test robot]

Hi Waiman,

kernel test robot noticed the following build errors:

[auto build test ERROR on tip/timers/core]
[also build test ERROR on akpm-mm/mm-everything tip/core/debugobjects linus/master v6.15 next-20250605]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Waiman-Long/debugobjects-Add-ODEBUG_FLAG_NO_ALLOC-to-disable-memory-allocation/20250605-061211
base:   tip/timers/core
patch link:    https://lore.kernel.org/r/20250604220926.870760-3-longman%40redhat.com
patch subject: [PATCH 2/3] debugobjects: Show the state of debug_objects_enabled
config: arm-randconfig-002-20250606 (https://download.01.org/0day-ci/archive/20250606/202506060634.mQtyT7cN-lkp@intel.com/config)
compiler: arm-linux-gnueabi-gcc (GCC) 11.5.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250606/202506060634.mQtyT7cN-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202506060634.mQtyT7cN-lkp@intel.com/

All errors (new ones prefixed by >>):

   lib/debugobjects.c: In function 'check_results':
>> lib/debugobjects.c:1325:17: error: implicit declaration of function 'debug_object_disable'; did you mean 'debug_objects_disable'? [-Werror=implicit-function-declaration]
    1325 |                 debug_object_disable("selftest");
         |                 ^~~~~~~~~~~~~~~~~~~~
         |                 debug_objects_disable
   cc1: some warnings being treated as errors


vim +1325 lib/debugobjects.c

  1288	
  1289	static int __init
  1290	check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
  1291	{
  1292		struct debug_bucket *db;
  1293		struct debug_obj *obj;
  1294		unsigned long flags;
  1295		int res = -EINVAL;
  1296	
  1297		db = get_bucket((unsigned long) addr);
  1298	
  1299		raw_spin_lock_irqsave(&db->lock, flags);
  1300	
  1301		obj = lookup_object(addr, db);
  1302		if (!obj && state != ODEBUG_STATE_NONE) {
  1303			WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
  1304			goto out;
  1305		}
  1306		if (obj && obj->state != state) {
  1307			WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
  1308			       obj->state, state);
  1309			goto out;
  1310		}
  1311		if (fixups != debug_objects_fixups) {
  1312			WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
  1313			       fixups, debug_objects_fixups);
  1314			goto out;
  1315		}
  1316		if (warnings != debug_objects_warnings) {
  1317			WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
  1318			       warnings, debug_objects_warnings);
  1319			goto out;
  1320		}
  1321		res = 0;
  1322	out:
  1323		raw_spin_unlock_irqrestore(&db->lock, flags);
  1324		if (res)
> 1325			debug_object_disable("selftest");
  1326		return res;
  1327	}
  1328	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki