[RFCv2 PATCH 13/23] sched: Introduce weighted/unweighted switch in load related functions

[Morten Rasmussen <morten.rasmussen@arm.com>]

From: Dietmar Eggemann <dietmar.eggemann@arm.com>

Add weighted/unweighted switch as an additional argument to cpu_load(),
source_load(), target_load() and task_h_load() to enable the user to
either ask for weighted or unweighted load signal. Use 0 (weighted) for
all existing occurrences of these functions.

Signed-off-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
---
 kernel/sched/fair.c |   65 +++++++++++++++++++++++++++------------------------
 1 file changed, 35 insertions(+), 30 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 784fdab..37e9ea1 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -665,7 +665,7 @@ static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se)
 }
 
 #ifdef CONFIG_SMP
-static unsigned long task_h_load(struct task_struct *p);
+static unsigned long task_h_load(struct task_struct *p, int uw);
 
 static inline void __update_task_entity_contrib(struct sched_entity *se);
 
@@ -1014,9 +1014,9 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page,
 	return group_faults(p, dst_nid) < (group_faults(p, src_nid) * 3 / 4);
 }
 
-static unsigned long cpu_load(const int cpu);
-static unsigned long source_load(int cpu, int type);
-static unsigned long target_load(int cpu, int type);
+static unsigned long cpu_load(const int cpu, int uw);
+static unsigned long source_load(int cpu, int type, int uw);
+static unsigned long target_load(int cpu, int type, int uw);
 static unsigned long capacity_of(int cpu);
 static long effective_load(struct task_group *tg, int cpu, long wl, long wg);
 
@@ -1045,7 +1045,7 @@ static void update_numa_stats(struct numa_stats *ns, int nid)
 		struct rq *rq = cpu_rq(cpu);
 
 		ns->nr_running += rq->nr_running;
-		ns->load += cpu_load(cpu);
+		ns->load += cpu_load(cpu, 0);
 		ns->compute_capacity += capacity_of(cpu);
 
 		cpus++;
@@ -1215,12 +1215,12 @@ balance:
 	orig_src_load = env->src_stats.load;
 
 	/* XXX missing capacity terms */
-	load = task_h_load(env->p);
+	load = task_h_load(env->p, 0);
 	dst_load = orig_dst_load + load;
 	src_load = orig_src_load - load;
 
 	if (cur) {
-		load = task_h_load(cur);
+		load = task_h_load(cur, 0);
 		dst_load -= load;
 		src_load += load;
 	}
@@ -4036,9 +4036,10 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 
 #ifdef CONFIG_SMP
 /* Used instead of source_load when we know the type == 0 */
-static unsigned long cpu_load(const int cpu)
+static unsigned long cpu_load(const int cpu, int uw)
 {
-	return cpu_rq(cpu)->cfs.runnable_load_avg;
+	return uw ? cpu_rq(cpu)->cfs.uw_runnable_load_avg :
+			cpu_rq(cpu)->cfs.runnable_load_avg;
 }
 
 /*
@@ -4048,30 +4049,32 @@ static unsigned long cpu_load(const int cpu)
  * We want to under-estimate the load of migration sources, to
  * balance conservatively.
  */
-static unsigned long source_load(int cpu, int type)
+static unsigned long source_load(int cpu, int type, int uw)
 {
 	struct rq *rq = cpu_rq(cpu);
-	unsigned long total = cpu_load(cpu);
+	unsigned long total = cpu_load(cpu, uw);
 
 	if (type == 0 || !sched_feat(LB_BIAS))
 		return total;
 
-	return min(rq->cpu_load[type-1], total);
+	return uw ? min(rq->uw_cpu_load[type-1], total) :
+			min(rq->cpu_load[type-1], total);
 }
 
 /*
  * Return a high guess at the load of a migration-target cpu weighted
  * according to the scheduling class and "nice" value.
  */
-static unsigned long target_load(int cpu, int type)
+static unsigned long target_load(int cpu, int type, int uw)
 {
 	struct rq *rq = cpu_rq(cpu);
-	unsigned long total = cpu_load(cpu);
+	unsigned long total = cpu_load(cpu, uw);
 
 	if (type == 0 || !sched_feat(LB_BIAS))
 		return total;
 
-	return max(rq->cpu_load[type-1], total);
+	return uw ? max(rq->uw_cpu_load[type-1], total) :
+			max(rq->cpu_load[type-1], total);
 }
 
 static unsigned long capacity_of(int cpu)
@@ -4292,8 +4295,8 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
 	idx	  = sd->wake_idx;
 	this_cpu  = smp_processor_id();
 	prev_cpu  = task_cpu(p);
-	load	  = source_load(prev_cpu, idx);
-	this_load = target_load(this_cpu, idx);
+	load	  = source_load(prev_cpu, idx, 0);
+	this_load = target_load(this_cpu, idx, 0);
 
 	/*
 	 * If sync wakeup then subtract the (maximum possible)
@@ -4349,7 +4352,7 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
 
 	if (balanced ||
 	    (this_load <= load &&
-	     this_load + target_load(prev_cpu, idx) <= tl_per_task)) {
+	     this_load + target_load(prev_cpu, idx, 0) <= tl_per_task)) {
 		/*
 		 * This domain has SD_WAKE_AFFINE and
 		 * p is cache cold in this domain, and
@@ -4398,9 +4401,9 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p,
 		for_each_cpu(i, sched_group_cpus(group)) {
 			/* Bias balancing toward cpus of our domain */
 			if (local_group)
-				load = source_load(i, load_idx);
+				load = source_load(i, load_idx, 0);
 			else
-				load = target_load(i, load_idx);
+				load = target_load(i, load_idx, 0);
 
 			avg_load += load;
 		}
@@ -4433,7 +4436,7 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
 
 	/* Traverse only the allowed CPUs */
 	for_each_cpu_and(i, sched_group_cpus(group), tsk_cpus_allowed(p)) {
-		load = cpu_load(i);
+		load = cpu_load(i, 0);
 
 		if (load < min_load || (load == min_load && i == this_cpu)) {
 			min_load = load;
@@ -5402,7 +5405,7 @@ static int move_tasks(struct lb_env *env)
 		if (!can_migrate_task(p, env))
 			goto next;
 
-		load = task_h_load(p);
+		load = task_h_load(p, 0);
 
 		if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed)
 			goto next;
@@ -5542,12 +5545,14 @@ static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
 	}
 }
 
-static unsigned long task_h_load(struct task_struct *p)
+static unsigned long task_h_load(struct task_struct *p, int uw)
 {
 	struct cfs_rq *cfs_rq = task_cfs_rq(p);
+	unsigned long task_load = uw ? p->se.avg.uw_load_avg_contrib
+				     : p->se.avg.load_avg_contrib;
 
 	update_cfs_rq_h_load(cfs_rq);
-	return div64_ul(p->se.avg.load_avg_contrib * cfs_rq->h_load,
+	return div64_ul(task_load * cfs_rq->h_load,
 			cfs_rq->runnable_load_avg + 1);
 }
 #else
@@ -5555,9 +5560,9 @@ static inline void update_blocked_averages(int cpu)
 {
 }
 
-static unsigned long task_h_load(struct task_struct *p)
+static unsigned long task_h_load(struct task_struct *p, int uw)
 {
-	return p->se.avg.load_avg_contrib;
+	return uw ? p->se.avg.uw_load_avg_contrib : p->se.avg.load_avg_contrib;
 }
 #endif
 
@@ -5916,9 +5921,9 @@ static inline void update_sg_lb_stats(struct lb_env *env,
 
 		/* Bias balancing toward cpus of our domain */
 		if (local_group)
-			load = target_load(i, load_idx);
+			load = target_load(i, load_idx, 0);
 		else
-			load = source_load(i, load_idx);
+			load = source_load(i, load_idx, 0);
 
 		sgs->group_load += load;
 		sgs->sum_nr_running += rq->nr_running;
@@ -5926,7 +5931,7 @@ static inline void update_sg_lb_stats(struct lb_env *env,
 		sgs->nr_numa_running += rq->nr_numa_running;
 		sgs->nr_preferred_running += rq->nr_preferred_running;
 #endif
-		sgs->sum_weighted_load += cpu_load(i);
+		sgs->sum_weighted_load += cpu_load(i, 0);
 		if (idle_cpu(i))
 			sgs->idle_cpus++;
 	}
@@ -6421,7 +6426,7 @@ static struct rq *find_busiest_queue(struct lb_env *env,
 		if (!capacity_factor)
 			capacity_factor = fix_small_capacity(env->sd, group);
 
-		load = cpu_load(i);
+		load = cpu_load(i, 0);
 
 		/*
 		 * When comparing with imbalance, use cpu_load()
-- 
1.7.9.5