[PATCH wq/for-7.4 v2 3/3] workqueue: add a percpu_concurrency_managed workqueue attribute

[Breno Leitao <leitao@debian.org>]

Introduce percpu_concurrency_managed, similar to the discussion in [1],
and use it when deciding to do concurrency management, instead of
relying on WQ_PERCPU (the only WQ type that does CM as of now).

Do not set it for WQ_BH. BH uses static per-cpu pools too, but it forces
max_active to INT_MAX and does not impose the normal max-active
throttle.

The attr is workqueue-only. wqattrs_clear_for_pool() clears it before
attrs are stored in a worker_pool. Pool selection still uses WQ_PERCPU;
attrs do not switch a workqueue onto static per-cpu pools yet.

Link: https://lore.kernel.org/all/8a1437751a6da2c166284b1ca562fb0a@kernel.org/ [1]
Signed-off-by: Breno Leitao <leitao@debian.org>
---
 include/linux/workqueue.h |  9 +++++++++
 kernel/workqueue.c        | 40 ++++++++++++++++++++++++++--------------
 2 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index a283766a192aaf..32747f797c4449 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -204,6 +204,15 @@ struct workqueue_attrs {
 	 */
 	enum wq_affn_scope affn_scope;
 
+	/**
+	 * @percpu_concurrency_managed: use per-cpu concurrency accounting
+	 *
+	 * Workqueues with this set meter active work per CPU against
+	 * percpu_max_active. Workqueues with this clear either use unbound
+	 * max_active accounting or do not impose a max_active limit.
+	 */
+	bool percpu_concurrency_managed;
+
 	/**
 	 * @ordered: work items must be executed one by one in queueing order
 	 */
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 545332d6118159..7059fc1bd9b090 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -424,9 +424,9 @@ struct workqueue_struct {
 
 static int wq_user_max_active(struct workqueue_struct *wq)
 {
-	if (wq->flags & WQ_UNBOUND)
-		return READ_ONCE(wq->saved_max_active);
-	return READ_ONCE(wq->saved_percpu_max_active);
+	if (READ_ONCE(wq->attrs->percpu_concurrency_managed))
+		return READ_ONCE(wq->saved_percpu_max_active);
+	return READ_ONCE(wq->saved_max_active);
 }
 
 /*
@@ -1861,11 +1861,13 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
 
 	lockdep_assert_held(&pool->lock);
 
-	/*
-	 * A concurrency-managed per-cpu pool accounts nr_active per pwq, so
-	 * pwq->nr_active against wq->percpu_max_active is sufficient.
-	 */
-	if (is_percpu_pool(pool)) {
+	/* BH workqueues use per-cpu pools but do not throttle max_active. */
+	if (pool->flags & POOL_BH) {
+		obtained = true;
+		goto out;
+	}
+
+	if (READ_ONCE(wq->attrs->percpu_concurrency_managed)) {
 		obtained = pwq->nr_active < READ_ONCE(wq->percpu_max_active);
 		goto out;
 	}
@@ -2091,6 +2093,7 @@ static void node_activate_pending_pwq(struct wq_node_nr_active *nna,
  */
 static void pwq_dec_nr_active(struct pool_workqueue *pwq)
 {
+	struct workqueue_struct *wq = pwq->wq;
 	struct worker_pool *pool = pwq->pool;
 	struct wq_node_nr_active *nna;
 
@@ -2102,16 +2105,15 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq)
 	 */
 	pwq->nr_active--;
 
-	/*
-	 * A concurrency-managed per-cpu pool only needs to kick the first
-	 * inactive work item on @pwq itself.
-	 */
-	if (is_percpu_pool(pool)) {
+	if (pool->flags & POOL_BH)
+		return;
+
+	if (READ_ONCE(wq->attrs->percpu_concurrency_managed)) {
 		pwq_activate_first_inactive(pwq, false);
 		return;
 	}
 
-	nna = wq_node_nr_active(pwq->wq, pool->node);
+	nna = wq_node_nr_active(wq, pool->node);
 
 	/*
 	 * If @pwq is for an unbound workqueue, it's more complicated because
@@ -5034,6 +5036,7 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to,
 	 * get_unbound_pool() explicitly clears the fields.
 	 */
 	to->affn_scope = from->affn_scope;
+	to->percpu_concurrency_managed = from->percpu_concurrency_managed;
 	to->ordered = from->ordered;
 }
 
@@ -5044,6 +5047,7 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to,
 static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs)
 {
 	attrs->affn_scope = WQ_AFFN_NR_TYPES;
+	attrs->percpu_concurrency_managed = false;
 	attrs->ordered = false;
 	if (attrs->affn_strict)
 		cpumask_copy(attrs->cpumask, cpu_possible_mask);
@@ -5851,6 +5855,10 @@ int apply_workqueue_attrs(struct workqueue_struct *wq,
 	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
 		return -EINVAL;
 
+	/* Attrs cannot switch a workqueue to per-cpu pools yet. */
+	if (WARN_ON(attrs->percpu_concurrency_managed))
+		return -EINVAL;
+
 	mutex_lock(&wq_pool_mutex);
 	ret = apply_workqueue_attrs_locked(wq, attrs);
 	mutex_unlock(&wq_pool_mutex);
@@ -5942,6 +5950,10 @@ static struct workqueue_attrs *alloc_wq_std_attrs(struct workqueue_struct *wq)
 	if (wq->flags & __WQ_ORDERED)
 		attrs->ordered = true;
 
+	/* BH uses per-cpu pools but not max_active concurrency management. */
+	if ((wq->flags & WQ_PERCPU) && !(wq->flags & WQ_BH))
+		attrs->percpu_concurrency_managed = true;
+
 	return attrs;
 }
 

-- 
2.53.0-Meta