[PATCH v3 23/26] mm: Split out NR_FREE_PAGES_BLOCKS_[UN]MAPPED

[Brendan Jackman <jackmanb@google.com>]

In order to infer whether compaction is likely to enable an allocation
that maps/unmaps a pageblock, we need to know how many fully-free
pageblocks of each type there are.

Signed-off-by: Brendan Jackman <jackmanb@google.com>
---
 include/linux/mmzone.h |  5 ++++-
 include/linux/vmstat.h | 10 ++++++++++
 mm/compaction.c        |  5 ++---
 mm/page_alloc.c        | 13 ++++++++++---
 mm/vmscan.c            | 49 ++++++++++++++++++++++++++++---------------------
 mm/vmstat.c            |  3 ++-
 6 files changed, 56 insertions(+), 29 deletions(-)

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 2533a228f71fd..e063a59bbc883 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -181,7 +181,10 @@ enum numa_stat_item {
 
 enum zone_stat_item {
 	NR_FREE_PAGES,
-	NR_FREE_PAGES_BLOCKS,
+	/* Number of free pages in entirely free pageblocks, with direct map */
+	NR_FREE_PAGES_BLOCKS_MAPPED,
+	/* Ditto, without direct map */
+	NR_FREE_PAGES_BLOCKS_UNMAPPED,
 	NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry */
 	NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE,
 	NR_ZONE_ACTIVE_ANON,
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index 5b31d8e7ae405..debd593756149 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -224,6 +224,16 @@ static inline unsigned long zone_page_state(struct zone *zone,
 	return x;
 }
 
+/*
+ * Approx number of pages in entirely free pageblocks. Due to races this could
+ * actually return a value more than the number of pages in the zone.
+ */
+static inline unsigned long zone_free_pages_blocks(struct zone *zone)
+{
+	return  zone_page_state(zone, NR_FREE_PAGES_BLOCKS_MAPPED) +
+		zone_page_state(zone, NR_FREE_PAGES_BLOCKS_UNMAPPED);
+}
+
 /*
  * More accurate version that also considers the currently pending
  * deltas. For that we need to loop over all cpus to find the current
diff --git a/mm/compaction.c b/mm/compaction.c
index c9eb3947ffc79..ed12d2fc6fad3 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -2353,8 +2353,7 @@ static enum compact_result __compact_finished(struct compact_control *cc)
 		if (__zone_watermark_ok(cc->zone, cc->order,
 					high_wmark_pages(cc->zone),
 					cc->highest_zoneidx, cc->alloc_flags,
-					zone_page_state(cc->zone,
-							NR_FREE_PAGES_BLOCKS)))
+					zone_free_pages_blocks(cc->zone)))
 			return COMPACT_SUCCESS;
 
 		return COMPACT_CONTINUE;
@@ -2538,7 +2537,7 @@ compaction_suit_allocation_order(struct zone *zone, unsigned int order,
 	unsigned long watermark;
 
 	if (kcompactd && defrag_mode)
-		free_pages = zone_page_state(zone, NR_FREE_PAGES_BLOCKS);
+		free_pages = zone_free_pages_blocks(zone);
 	else
 		free_pages = zone_page_state(zone, NR_FREE_PAGES);
 
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index ac2f6190117ae..d12ce84662ab7 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -866,6 +866,13 @@ static inline void account_freepages(struct zone *zone, int nr_pages,
 			   zone->nr_free_highatomic + nr_pages);
 }
 
+static inline enum zone_stat_item free_pages_blocks_stat(freetype_t ft)
+{
+	if (freetype_flags(ft) & FREETYPE_UNMAPPED)
+		return NR_FREE_PAGES_BLOCKS_UNMAPPED;
+	return NR_FREE_PAGES_BLOCKS_MAPPED;
+}
+
 /* Used for pages not on another list */
 static inline void __add_to_free_list(struct page *page, struct zone *zone,
 				      unsigned int order, freetype_t freetype,
@@ -890,7 +897,7 @@ static inline void __add_to_free_list(struct page *page, struct zone *zone,
 	area->nr_free++;
 
 	if (order >= pageblock_order && !is_migrate_isolate(free_to_migratetype(freetype)))
-		__mod_zone_page_state(zone, NR_FREE_PAGES_BLOCKS, nr_pages);
+		__mod_zone_page_state(zone, free_pages_blocks_stat(freetype), nr_pages);
 }
 
 /*
@@ -926,7 +933,7 @@ static inline void move_to_free_list(struct page *page, struct zone *zone,
 	    is_migrate_isolate(old_mt) != is_migrate_isolate(new_mt)) {
 		if (!is_migrate_isolate(old_mt))
 			nr_pages = -nr_pages;
-		__mod_zone_page_state(zone, NR_FREE_PAGES_BLOCKS, nr_pages);
+		__mod_zone_page_state(zone, free_pages_blocks_stat(new_ft), nr_pages);
 	}
 }
 
@@ -954,7 +961,7 @@ static inline void __del_page_from_free_list(struct page *page, struct zone *zon
 	zone->free_area[order].nr_free--;
 
 	if (order >= pageblock_order && !is_migrate_isolate(free_to_migratetype(freetype)))
-		__mod_zone_page_state(zone, NR_FREE_PAGES_BLOCKS, -nr_pages);
+		__mod_zone_page_state(zone, free_pages_blocks_stat(freetype), -nr_pages);
 }
 
 static inline void del_page_from_free_list(struct page *page, struct zone *zone,
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 566c4e837c7d5..5789c39a0a729 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -6935,6 +6935,28 @@ static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)
 	return false;
 }
 
+/*
+ * Helper to get *FREE_PAGES* zone stats with accuracy heuristics.
+ *
+ * When there is a high number of CPUs in the system, the cumulative error from
+ * the vmstat per-cpu cache can blur the line between the watermarks. In that
+ * case, be safe and get an accurate snapshot.
+ *
+ * TODO: NR_FREE_PAGES_BLOCKS_* move in steps of pageblock_nr_pages, while the
+ * vmstat pcp threshold is limited to 125. On many configurations that counter
+ * won't actually be per-cpu cached. But keep things simple for now; revisit
+ * when somebody cares.
+ */
+static inline unsigned long get_free_pages_stat(struct zone *zone,
+						enum zone_stat_item item)
+{
+	unsigned long free_pages = zone_page_state(zone, item);
+
+	if (zone->percpu_drift_mark && free_pages < zone->percpu_drift_mark)
+		return zone_page_state_snapshot(zone, item);
+	return free_pages;
+}
+
 /*
  * Returns true if there is an eligible zone balanced for the request order
  * and highest_zoneidx
@@ -6950,7 +6972,6 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
 	 * meet watermarks.
 	 */
 	for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) {
-		enum zone_stat_item item;
 		unsigned long free_pages;
 
 		if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)
@@ -6968,26 +6989,12 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
 		 * has dropped order, simply ensure there are enough
 		 * base pages for compaction, wake kcompactd & sleep.
 		 */
-		if (defrag_mode && order)
-			item = NR_FREE_PAGES_BLOCKS;
-		else
-			item = NR_FREE_PAGES;
-
-		/*
-		 * When there is a high number of CPUs in the system,
-		 * the cumulative error from the vmstat per-cpu cache
-		 * can blur the line between the watermarks. In that
-		 * case, be safe and get an accurate snapshot.
-		 *
-		 * TODO: NR_FREE_PAGES_BLOCKS moves in steps of
-		 * pageblock_nr_pages, while the vmstat pcp threshold
-		 * is limited to 125. On many configurations that
-		 * counter won't actually be per-cpu cached. But keep
-		 * things simple for now; revisit when somebody cares.
-		 */
-		free_pages = zone_page_state(zone, item);
-		if (zone->percpu_drift_mark && free_pages < zone->percpu_drift_mark)
-			free_pages = zone_page_state_snapshot(zone, item);
+		if (defrag_mode && order) {
+			free_pages = get_free_pages_stat(zone, NR_FREE_PAGES_BLOCKS_UNMAPPED) +
+				get_free_pages_stat(zone, NR_FREE_PAGES_BLOCKS_MAPPED);
+		} else {
+			free_pages = get_free_pages_stat(zone, NR_FREE_PAGES);
+		}
 
 		if (__zone_watermark_ok(zone, order, mark, highest_zoneidx,
 					0, free_pages))
diff --git a/mm/vmstat.c b/mm/vmstat.c
index cb57714539fb5..f5ab6ab641c6d 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1200,7 +1200,8 @@ const char * const vmstat_text[] = {
 	/* enum zone_stat_item counters */
 #define I(x) (x)
 	[I(NR_FREE_PAGES)]			= "nr_free_pages",
-	[I(NR_FREE_PAGES_BLOCKS)]		= "nr_free_pages_blocks",
+	[I(NR_FREE_PAGES_BLOCKS_MAPPED)]	= "nr_free_pages_blocks_mapped",
+	[I(NR_FREE_PAGES_BLOCKS_UNMAPPED)]	= "nr_free_pages_blocks_unmapped",
 	[I(NR_ZONE_INACTIVE_ANON)]		= "nr_zone_inactive_anon",
 	[I(NR_ZONE_ACTIVE_ANON)]		= "nr_zone_active_anon",
 	[I(NR_ZONE_INACTIVE_FILE)]		= "nr_zone_inactive_file",

-- 
2.54.0