[PATCH 06/13] efi/unaccepted: add mark_memory_unaccepted()

[Kevin Loughlin <kevinloughlin@google.com>]

Introduce mark_memory_unaccepted() to mark a physical memory range as
unaccepted in the EFI unaccepted memory bitmap. Use
unaccepted_memory_order() to query the page order matching the EFI
unaccepted memory unit_size granularity.

Synchronize mark_memory_unaccepted() with concurrent accept_memory()
callers via accepting_list and only invoke arch_unaccept_memory() on
sub-ranges that are currently marked accepted in the bitmap.

On platforms with dynamic unacceptance (arch_has_unaccept_memory()),
load_unaligned_zeropad() across page boundaries is recovered via
EX_TYPE_ZEROPAD exception fixup. Thus, skip extending accept_memory()
and range_contains_unaccepted_memory() by an extra unit_size guard
block to avoid corrupting the acceptance state of adjacent 2MB blocks.

Signed-off-by: Kevin Loughlin <kevinloughlin@google.com>
---
 drivers/firmware/efi/unaccepted_memory.c | 107 ++++++++++++++++++++++-
 include/linux/mm.h                       |  12 +++
 2 files changed, 115 insertions(+), 4 deletions(-)

diff --git a/drivers/firmware/efi/unaccepted_memory.c b/drivers/firmware/efi/unaccepted_memory.c
index 4a8ec8d6a571..128bc1b113cd 100644
--- a/drivers/firmware/efi/unaccepted_memory.c
+++ b/drivers/firmware/efi/unaccepted_memory.c
@@ -5,6 +5,8 @@
 #include <linux/spinlock.h>
 #include <linux/crash_dump.h>
 #include <linux/nmi.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
 #include <asm/unaccepted_memory.h>
 
 /* Protects unaccepted memory bitmap and accepting_list */
@@ -86,7 +88,16 @@ void accept_memory(phys_addr_t start, unsigned long size)
 	 *    if 'size+end' is aligned on a unit_size boundary. (immediately
 	 *    following this comment)
 	 */
-	if (!(end % unit_size))
+	/*
+	 * On platforms without implemented dynamic unacceptance support,
+	 * load_unaligned_zeropad() across a page boundary into unaccepted
+	 * memory causes a fatal VMM exit rather than a recoverable exception.
+	 * When dynamic unacceptance is supported (arch_has_unaccept_memory()),
+	 * load_unaligned_zeropad() is recovered via EX_TYPE_ZEROPAD exception
+	 * fixup, whereas extending the range by unit_size would accept
+	 * adjacent dynamically-unaccepted blocks.
+	 */
+	if (!arch_has_unaccept_memory() && !(end % unit_size))
 		end += unit_size;
 
 	/* Make sure not to overrun the bitmap */
@@ -191,10 +202,11 @@ bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size)
 	end -= unaccepted->phys_base;
 
 	/*
-	 * Also consider the unaccepted state of the *next* page. See fix #1 in
-	 * the comment on load_unaligned_zeropad() in accept_memory().
+	 * Also consider the unaccepted state of the *next* page on platforms
+	 * without implemented dynamic unacceptance support. See fix #1 in the
+	 * comment on load_unaligned_zeropad() in accept_memory().
 	 */
-	if (!(end % unit_size))
+	if (!arch_has_unaccept_memory() && !(end % unit_size))
 		end += unit_size;
 
 	/* Make sure not to overrun the bitmap */
@@ -215,6 +227,93 @@ bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size)
 	return ret;
 }
 
+unsigned int unaccepted_memory_order(void)
+{
+	struct efi_unaccepted_memory *unaccepted = efi_get_unaccepted_table();
+
+	if (unaccepted && unaccepted->unit_size >= PAGE_SIZE)
+		return ilog2(unaccepted->unit_size >> PAGE_SHIFT);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(unaccepted_memory_order);
+
+int mark_memory_unaccepted(phys_addr_t start, unsigned long size)
+{
+	struct efi_unaccepted_memory *unaccepted;
+	unsigned long range_start, range_end;
+	struct accept_range range, *entry;
+	phys_addr_t end = start + size;
+	unsigned long flags;
+	u64 unit_size;
+
+	if (!arch_has_unaccept_memory())
+		return -EOPNOTSUPP;
+
+	unaccepted = efi_get_unaccepted_table();
+	if (!unaccepted)
+		return -EOPNOTSUPP;
+
+	unit_size = unaccepted->unit_size;
+	if (WARN_ON_ONCE(!IS_ALIGNED(start, unit_size) ||
+			 !IS_ALIGNED(size, unit_size)))
+		return -EINVAL;
+
+	if (WARN_ON_ONCE(start < unaccepted->phys_base || end <= start))
+		return -ERANGE;
+
+	start -= unaccepted->phys_base;
+	end -= unaccepted->phys_base;
+
+	/* Make sure not to overrun the bitmap */
+	if (WARN_ON_ONCE(end > unaccepted->size * unit_size * BITS_PER_BYTE))
+		return -ERANGE;
+
+	range.start = start / unit_size;
+	range.end = end / unit_size;
+retry:
+	spin_lock_irqsave(&unaccepted_memory_lock, flags);
+
+	list_for_each_entry(entry, &accepting_list, list) {
+		if (entry->end <= range.start)
+			continue;
+		if (entry->start >= range.end)
+			continue;
+
+		spin_unlock_irqrestore(&unaccepted_memory_lock, flags);
+		goto retry;
+	}
+
+	list_add(&range.list, &accepting_list);
+
+	range_start = range.start;
+	for_each_clear_bitrange_from(range_start, range_end,
+				     unaccepted->bitmap, range.end) {
+		unsigned long phys_start, phys_end;
+		unsigned long len = range_end - range_start;
+
+		phys_start = range_start * unit_size + unaccepted->phys_base;
+		phys_end = range_end * unit_size + unaccepted->phys_base;
+
+		bitmap_set(unaccepted->bitmap, range_start, len);
+		spin_unlock(&unaccepted_memory_lock);
+
+		debug_pagealloc_map_pages(pfn_to_page(PHYS_PFN(phys_start)),
+					  (phys_end - phys_start) >> PAGE_SHIFT);
+		arch_unaccept_memory(phys_start, phys_end);
+
+		spin_lock(&unaccepted_memory_lock);
+	}
+
+	list_del(&range.list);
+
+	touch_softlockup_watchdog();
+
+	spin_unlock_irqrestore(&unaccepted_memory_lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mark_memory_unaccepted);
+
 #ifdef CONFIG_PROC_VMCORE
 static bool unaccepted_memory_vmcore_pfn_is_ram(struct vmcore_cb *cb,
 						unsigned long pfn)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 6882ad400935..27bb40651c85 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -5362,6 +5362,8 @@ int set_anon_vma_name(unsigned long addr, unsigned long size,
 
 bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size);
 void accept_memory(phys_addr_t start, unsigned long size);
+unsigned int unaccepted_memory_order(void);
+int mark_memory_unaccepted(phys_addr_t start, unsigned long size);
 
 #else
 
@@ -5375,6 +5377,16 @@ static inline void accept_memory(phys_addr_t start, unsigned long size)
 {
 }
 
+static inline unsigned int unaccepted_memory_order(void)
+{
+	return 0;
+}
+
+static inline int mark_memory_unaccepted(phys_addr_t start, unsigned long size)
+{
+	return -EOPNOTSUPP;
+}
+
 #endif
 
 static inline bool pfn_is_unaccepted_memory(unsigned long pfn)
-- 
2.56.0.307.g5fe7bd680c-goog