[PATCH 08/13] mm/page_alloc: route unaccepted isolated pages

[Kevin Loughlin <kevinloughlin@google.com>]

When free page reporting unaccepts isolated pages on confidential
computing guests, __putback_isolated_page() must not return those
unaccepted pages directly to zone->free_area; doing so would violate the
allocator invariant that all pages on zone->free_area are accepted.

Update __putback_isolated_page(), __free_pages_ok(),
__free_frozen_pages(), and free_unref_folios() to route dynamically
unaccepted pages to zone->unaccepted_pages via __free_unaccepted_locked()
(falling back to immediate acceptance for isolated or CMA pageblocks).

In addition:
- Extract free_pages_prepare_init() so post-acceptance poisoning,
  zeroing, KASAN tagging, arch_free_page(), and debug_pagealloc
  unmapping run after accept_memory() without touching unaccepted pages
  in __free_pages_prepare() or re-running metadata teardown in
  __accept_page().
- Pass FPI_SKIP_REPORT_NOTIFY when __accept_page() frees a freshly
  accepted block into the buddy allocator so it is not immediately
  re-queued for free page reporting before allocation.

Signed-off-by: Kevin Loughlin <kevinloughlin@google.com>
---
 mm/page_alloc.c | 120 ++++++++++++++++++++++++++++++++++--------------
 1 file changed, 85 insertions(+), 35 deletions(-)

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 46ae8c6f3814..c10b77304633 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1298,12 +1298,53 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)
 
 #endif /* CONFIG_MEM_ALLOC_PROFILING */
 
+static inline bool should_free_unaccepted(struct page *page,
+					  unsigned int order,
+					  fpi_t fpi_flags)
+{
+	return !(fpi_flags & FPI_NOLOCK) &&
+	       page_contains_unaccepted(page, order);
+}
+
+static void free_pages_prepare_init(struct page *page, unsigned int order)
+{
+	bool skip_kasan_poison = should_skip_kasan_poison(page);
+	bool init = want_init_on_free();
+
+	kernel_poison_pages(page, 1 << order);
+
+	/*
+	 * As memory initialization might be integrated into KASAN,
+	 * KASAN poisoning and memory initialization code must be
+	 * kept together to avoid discrepancies in behavior.
+	 *
+	 * With hardware tag-based KASAN, memory tags must be set before the
+	 * page becomes unavailable via debug_pagealloc or arch_free_page.
+	 */
+	if (!skip_kasan_poison) {
+		kasan_poison_pages(page, order, init);
+
+		/* Memory is already initialized if KASAN did it internally. */
+		if (kasan_has_integrated_init())
+			init = false;
+	}
+	if (init)
+		clear_highpages_kasan_tagged(page, 1 << order);
+
+	/*
+	 * arch_free_page() can make the page's contents inaccessible.  s390
+	 * does this.  So nothing which can access the page's contents should
+	 * happen after this.
+	 */
+	arch_free_page(page, order);
+
+	debug_pagealloc_unmap_pages(page, 1 << order);
+}
+
 static __always_inline bool __free_pages_prepare(struct page *page,
 		unsigned int order, fpi_t fpi_flags)
 {
 	int bad = 0;
-	bool skip_kasan_poison = should_skip_kasan_poison(page);
-	bool init = want_init_on_free();
 	bool compound = PageCompound(page);
 	struct folio *folio = page_folio(page);
 
@@ -1411,34 +1452,10 @@ static __always_inline bool __free_pages_prepare(struct page *page,
 					   PAGE_SIZE << order);
 	}
 
-	kernel_poison_pages(page, 1 << order);
-
-	/*
-	 * As memory initialization might be integrated into KASAN,
-	 * KASAN poisoning and memory initialization code must be
-	 * kept together to avoid discrepancies in behavior.
-	 *
-	 * With hardware tag-based KASAN, memory tags must be set before the
-	 * page becomes unavailable via debug_pagealloc or arch_free_page.
-	 */
-	if (!skip_kasan_poison) {
-		kasan_poison_pages(page, order, init);
-
-		/* Memory is already initialized if KASAN did it internally. */
-		if (kasan_has_integrated_init())
-			init = false;
-	}
-	if (init)
-		clear_highpages_kasan_tagged(page, 1 << order);
-
-	/*
-	 * arch_free_page() can make the page's contents inaccessible.  s390
-	 * does this.  So nothing which can access the page's contents should
-	 * happen after this.
-	 */
-	arch_free_page(page, order);
+	if (should_free_unaccepted(page, order, fpi_flags))
+		return true;
 
-	debug_pagealloc_unmap_pages(page, 1 << order);
+	free_pages_prepare_init(page, order);
 
 	return true;
 }
@@ -1572,14 +1589,31 @@ static void free_one_page(struct zone *zone, struct page *page,
 	__count_vm_events(PGFREE, 1 << order);
 }
 
+static bool handle_unaccepted_free(struct page *page, unsigned int order,
+				   fpi_t fpi_flags)
+{
+	if (!should_free_unaccepted(page, order, fpi_flags))
+		return false;
+
+	if (__free_unaccepted(page, order))
+		return true;
+
+	accept_memory(page_to_phys(page), PAGE_SIZE << order);
+	free_pages_prepare_init(page, order);
+	return false;
+}
+
 static void __free_pages_ok(struct page *page, unsigned int order,
 			    fpi_t fpi_flags)
 {
 	unsigned long pfn = page_to_pfn(page);
 	struct zone *zone = page_zone(page);
 
-	if (__free_pages_prepare(page, order, fpi_flags))
+	if (__free_pages_prepare(page, order, fpi_flags)) {
+		if (handle_unaccepted_free(page, order, fpi_flags))
+			return;
 		free_one_page(zone, page, pfn, order, fpi_flags);
+	}
 }
 
 void __meminit __free_pages_core(struct page *page, unsigned int order,
@@ -2955,7 +2989,8 @@ static void __free_frozen_pages(struct page *page, unsigned int order,
 	unsigned long pfn = page_to_pfn(page);
 	int migratetype;
 
-	if (!pcp_allowed_order(order)) {
+	if (!pcp_allowed_order(order) ||
+	    should_free_unaccepted(page, order, fpi_flags)) {
 		__free_pages_ok(page, order, fpi_flags);
 		return;
 	}
@@ -3024,9 +3059,13 @@ void free_unref_folios(struct folio_batch *folios)
 			continue;
 		/*
 		 * Free orders not handled on the PCP directly to the
-		 * allocator.
+		 * allocator, as well as any pages containing unaccepted memory.
 		 */
-		if (!pcp_allowed_order(order)) {
+		if (!pcp_allowed_order(order) ||
+		    should_free_unaccepted(&folio->page, order, FPI_NONE)) {
+			if (handle_unaccepted_free(&folio->page, order,
+						   FPI_NONE))
+				continue;
 			free_one_page(folio_zone(folio), &folio->page,
 				      pfn, order, FPI_NONE);
 			continue;
@@ -3187,6 +3226,16 @@ void __putback_isolated_page(struct page *page, unsigned int order, int mt)
 	/* zone lock should be held when this function is called */
 	lockdep_assert_held(&zone->lock);
 
+	if (page_contains_unaccepted(page, order)) {
+		if (__free_unaccepted_locked(page, zone, order))
+			return;
+		accept_memory(page_to_phys(page), PAGE_SIZE << order);
+		free_pages_prepare_init(page, order);
+		split_large_buddy(zone, page, page_to_pfn(page), order,
+				  FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
+		return;
+	}
+
 	/* Return isolated page to tail of freelist. */
 	__free_one_page(page, page_to_pfn(page), zone, order, mt,
 			FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
@@ -7933,8 +7982,9 @@ static void __accept_page(struct zone *zone, unsigned long *flags,
 	spin_unlock_irqrestore(&zone->lock, *flags);
 
 	accept_memory(page_to_phys(page), PAGE_SIZE << order);
-
-	__free_pages_ok(page, order, FPI_TO_TAIL);
+	free_pages_prepare_init(page, order);
+	free_one_page(zone, page, page_to_pfn(page), order,
+		      FPI_TO_TAIL | FPI_SKIP_REPORT_NOTIFY);
 }
 
 void accept_page(struct page *page)
-- 
2.56.0.307.g5fe7bd680c-goog