[PATCH v2 02/13] crash-core: Flush caches on CRASH_WIPE_SECRETS

Replace memzero_expclit with our custom wrapper that additionally
flushes the target address ranges from CPU caches.

On ARM64, while memzero_explicit is already reasonably reliable for
wiping secrets from memory during kdump, it can theoretically leave
residue in DRAM when the last memzero_explicit writes are still in
flight in caches when shutting down the caches in machine_kexec.

We need to swap every memzero_explicit call because ARM64 do not have a
wholesale "flush all caches" primitive and only support flushes targeted
to a particular address range.

Architectures other than ARM64, notably x86-64, are not affected by this
cache flush issue.

Signed-off-by: Jan Sebastian Götte <[email protected]>
---
 arch/arm64/kernel/machine_kexec.c | 20 ++++++++++++++++++++
 include/linux/crash_core.h        | 21 +++++++++++++++++++++
 kernel/crash_core.c               | 23 +++++++++++++++++++++++
 3 files changed, 64 insertions(+)

diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
index c5693a32e49b..12bf3c90f519 100644
--- a/arch/arm64/kernel/machine_kexec.c
+++ b/arch/arm64/kernel/machine_kexec.c
@@ -6,6 +6,7 @@
  * Copyright (C) Huawei Futurewei Technologies.
  */
 
+#include <linux/crash_core.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/kernel.h>
@@ -15,6 +16,7 @@
 #include <linux/set_memory.h>
 #include <linux/smp.h>
 
+#include <asm/barrier.h>
 #include <asm/cacheflush.h>
 #include <asm/cpu_ops.h>
 #include <asm/daifflags.h>
@@ -221,6 +223,24 @@ void machine_crash_shutdown(struct pt_regs *regs)
 	pr_info("Starting crashdump kernel...\n");
 }
 
+#ifdef CONFIG_CRASH_WIPE_SECRETS
+/*
+ * Queue each wiped range for cleaning to the point of coherency.  Deferring
+ * the barrier until the notifier chain is complete avoids one DSB per range.
+ */
+void arch_crash_wipe_range(void *addr, size_t size)
+{
+	unsigned long start = (unsigned long)addr;
+
+	dcache_clean_poc_nosync(start, start + size);
+}
+
+void arch_crash_wipe_flush(void)
+{
+	dsb(sy);
+}
+#endif
+
 #if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_HIBERNATION)
 /*
  * To preserve the crash dump kernel image, the relevant memory segments
diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h
index 4230463f3faa..d3e4192b7e6d 100644
--- a/include/linux/crash_core.h
+++ b/include/linux/crash_core.h
@@ -6,6 +6,7 @@
 #include <linux/elfcore.h>
 #include <linux/elf.h>
 #include <linux/notifier.h>
+#include <linux/string.h>
 
 struct kimage;
 
@@ -15,6 +16,26 @@ struct crash_mem {
 	struct range ranges[] __counted_by(max_nr_ranges);
 };
 
+#ifdef CONFIG_CRASH_WIPE_SECRETS
+/*
+ * Record a range that has already been wiped, or wipe and record it in one
+ * operation.  Architectures may use the ranges to push the wipes out to
+ * memory before kexec disables the caches.
+ */
+void crash_wipe_cache_range(void *addr, size_t size);
+void crash_wipe_memzero(void *addr, size_t size);
+
+void arch_crash_wipe_range(void *addr, size_t size);
+void arch_crash_wipe_flush(void);
+#else
+static inline void crash_wipe_cache_range(void *addr, size_t size) { }
+
+static inline void crash_wipe_memzero(void *addr, size_t size)
+{
+	memzero_explicit(addr, size);
+}
+#endif
+
 #ifdef CONFIG_CRASH_DUMP
 
 int crash_shrink_memory(unsigned long new_size);
diff --git a/kernel/crash_core.c b/kernel/crash_core.c
index 95f5c0415e60..42faf8d0a4c4 100644
--- a/kernel/crash_core.c
+++ b/kernel/crash_core.c
@@ -51,10 +51,33 @@ int crash_wipe_secrets_unregister(struct notifier_block *nb)
 }
 EXPORT_SYMBOL_GPL(crash_wipe_secrets_unregister);
 
+/*
+ * Some kexec paths disable the data cache without first cleaning it.  Give
+ * architectures valid virtual ranges for the wiped data, then let them defer
+ * any completion barrier until all crash-wipe callbacks have run.
+ */
+void __weak arch_crash_wipe_range(void *addr, size_t size) { }
+void __weak arch_crash_wipe_flush(void) { }
+
+void crash_wipe_cache_range(void *addr, size_t size)
+{
+	if (size)
+		arch_crash_wipe_range(addr, size);
+}
+EXPORT_SYMBOL_GPL(crash_wipe_cache_range);
+
+void crash_wipe_memzero(void *addr, size_t size)
+{
+	memzero_explicit(addr, size);
+	crash_wipe_cache_range(addr, size);
+}
+EXPORT_SYMBOL_GPL(crash_wipe_memzero);
+
 static void crash_wipe_secrets(void)
 {
 	pr_info("Wiping sensitive secrets...\n");
 	atomic_notifier_call_chain(&crash_wipe_secrets_notifier_list, 0, NULL);
+	arch_crash_wipe_flush();
 	pr_info("Done wiping secrets.\n");
 }
 #else

-- 
2.53.0