[PATCH v3 16/26] mm: add definitions for allocating unmapped pages

[Brendan Jackman <jackmanb@google.com>]

Create ALLOC_UNMAPPED, which requests pages that are not present in the
direct map. Since this feature has a cost (e.g. more freelists), it's
behind a kconfig. Unlike other conditionally-defined alloc flags, it
doesn't fall back to being 0. This prevents building code that uses
ALLOC_UNMAPPED but doesn't depend on the necessary kconfig, since that
would lead to invisible security issues.

Create a freetype flag to record that pages on the freelists with this
flag are unmapped. This is currently only needed for MIGRATE_UNMOVABLE
pages, so the freetype encoding remains trivial.

Also create the corresponding pageblock flag to record the same thing.

To keep patches from being too overwhelming, the actual implementation
is added separately, this is just types, Kconfig boilerplate, etc.

Acked-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Signed-off-by: Brendan Jackman <jackmanb@google.com>
---
 include/linux/freetype.h | 70 ++++++++++++++++++++++++++++++++++++++++--------
 mm/Kconfig               |  3 +++
 mm/page_alloc.h          | 18 +++++++++++++
 3 files changed, 80 insertions(+), 11 deletions(-)

diff --git a/include/linux/freetype.h b/include/linux/freetype.h
index 3b0d44023b6a1..37e88dcccdecc 100644
--- a/include/linux/freetype.h
+++ b/include/linux/freetype.h
@@ -2,6 +2,7 @@
 #ifndef _LINUX_FREETYPE_H
 #define _LINUX_FREETYPE_H
 
+#include <linux/log2.h>
 #include <linux/types.h>
 #include <linux/mmdebug.h>
 
@@ -64,20 +65,47 @@ static inline bool migratetype_is_mergeable(int mt)
 	return mt < MIGRATE_PCPTYPES;
 }
 
+enum {
+	/* Defined unconditionally as a hack to avoid a zero-width bitfield. */
+	FREETYPE_UNMAPPED_BIT,
+	NUM_FREETYPE_FLAGS,
+};
+
 /*
  * A freetype is the identifier for a page freelist. This consists of a
  * migratetype, and other bits which encode orthogonal properties of memory.
  */
 typedef struct {
-	int migratetype;
+	unsigned int migratetype : order_base_2(MIGRATE_TYPES);
+	unsigned int flags : NUM_FREETYPE_FLAGS;
 } freetype_t;
 
+#ifdef CONFIG_PAGE_ALLOC_UNMAPPED
+#define FREETYPE_UNMAPPED			BIT(FREETYPE_UNMAPPED_BIT)
+#define NUM_UNMAPPED_FREETYPES			1
+#else
+#define FREETYPE_UNMAPPED			0
+#define NUM_UNMAPPED_FREETYPES			0
+#endif
+
+#define FREETYPE_FLAGS_MASK FREETYPE_UNMAPPED
+
 /*
  * Return a dense linear index for freetypes that have lists in the free area.
  * Return -1 for other freetypes.
  */
 static inline int freetype_idx(freetype_t freetype)
 {
+	/* For FREETYPE_UNMAPPED, only MIGRATE_UNMOVABLE has an index. */
+	if (freetype.flags & FREETYPE_UNMAPPED) {
+		VM_WARN_ON_ONCE(freetype.flags & ~FREETYPE_UNMAPPED);
+		if (freetype.migratetype != MIGRATE_UNMOVABLE)
+			return -1;
+		return MIGRATE_TYPES;
+	}
+	/* No other flags are supported. */
+	VM_WARN_ON_ONCE(freetype.flags);
+
 	return freetype.migratetype;
 }
 
@@ -85,33 +113,53 @@ static inline freetype_t freetype_from_idx(unsigned int idx)
 {
 	freetype_t freetype;
 
-	freetype.migratetype = idx;
+	if (idx == MIGRATE_TYPES) {
+		freetype.flags = FREETYPE_UNMAPPED;
+		freetype.migratetype = MIGRATE_UNMOVABLE;
+	} else {
+		VM_WARN_ON_ONCE(idx < 0 || idx > MIGRATE_TYPES);
+		freetype.flags = 0;
+		freetype.migratetype = idx;
+	}
 	return freetype;
 }
 
-/* No freetype flags actually exist yet. */
-#define NR_FREETYPE_IDXS MIGRATE_TYPES
+/* One for each migratetype, plus one for MIGRATE_UNMOVABLE-FREETYPE_UNMAPPED */
+#define NR_FREETYPE_IDXS (MIGRATE_TYPES + NUM_UNMAPPED_FREETYPES)
 
 static inline unsigned int freetype_flags(freetype_t freetype)
 {
-	/* No flags supported yet. */
-	return 0;
+	return freetype.flags;
 }
 
+#ifdef CONFIG_PAGE_ALLOC_UNMAPPED
+static inline bool freetype_unmapped(freetype_t freetype)
+{
+	return !!(freetype.flags & FREETYPE_UNMAPPED);
+}
+#else
+static inline bool freetype_unmapped(freetype_t freetype)
+{
+	return false;
+}
+#endif
+
 static inline bool freetypes_equal(freetype_t a, freetype_t b)
 {
-	return a.migratetype == b.migratetype;
+	return a.migratetype == b.migratetype && a.flags == b.flags;
 }
 
 static inline freetype_t migrate_to_freetype(enum migratetype mt,
 					     unsigned int flags)
 {
-	freetype_t freetype;
+	freetype_t freetype = {
+		.migratetype = mt,
+		.flags = flags,
+	};
 
-	/* No flags supported yet. */
-	VM_WARN_ON_ONCE(flags);
+	VM_WARN_ON_ONCE(flags & ~FREETYPE_FLAGS_MASK);
+	VM_WARN_ON_ONCE(mt >= MIGRATE_TYPES);
 
-	freetype.migratetype = mt;
 	return freetype;
 }
 
diff --git a/mm/Kconfig b/mm/Kconfig
index adf71788971a2..aa5f041ce1a67 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -1530,3 +1530,6 @@ config MERMAP_KUNIT_TEST
 source "mm/damon/Kconfig"
 
 endmenu
+
+config PAGE_ALLOC_UNMAPPED
+	bool
diff --git a/mm/page_alloc.h b/mm/page_alloc.h
index 9928aa9012588..fac8e5304bb03 100644
--- a/mm/page_alloc.h
+++ b/mm/page_alloc.h
@@ -56,6 +56,24 @@
  * alloc_tag_sub_check().
  */
 #define ALLOC_NO_CODETAG       0x1000
+#ifdef CONFIG_PAGE_ALLOC_UNMAPPED
+/*
+ * Allocate pages that aren't present in the direct map. If the caller changes
+ * direct map presence, it must be restored to the previous state before freeing
+ * the page. (This is true regardless of ALLOC_UNMAPPED).
+ *
+ * This uses the mermap (when __GFP_ZERO), so it's only valid to allocate with
+ * this flag where that's valid, namely from process context after the mermap
+ * has been initialised for that process. This also means that the allocator
+ * leaves behind stale TLB entries in the mermap region. The caller is
+ * responsible for ensuring they are flushed as needed.
+ *
+ * This is currently incompatible with __GFP_MOVABLE and __GFP_RECLAIMABLE, but
+ * only because of allocator implementation details, if a usecase arises this
+ * restriction could be dropped.
+ */
+#define ALLOC_UNMAPPED	       0x2000
+#endif
 
 /* Flags that allow allocations below the min watermark. */
 #define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)

-- 
2.54.0