Pablo Greco cb7c71
From 5b7a08e0ea5bb51f886f2f5e2373de2bf5981dd2 Mon Sep 17 00:00:00 2001
Pablo Greco cb7c71
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Pablo Greco cb7c71
Date: Thu, 28 Nov 2019 20:58:05 +0100
Pablo Greco cb7c71
Subject: [PATCH 1/2] arm64: KVM: Invoke compute_layout() before alternatives
Pablo Greco cb7c71
 are applied
Pablo Greco cb7c71
Pablo Greco cb7c71
compute_layout() is invoked as part of an alternative fixup under
Pablo Greco cb7c71
stop_machine(). This function invokes get_random_long() which acquires a
Pablo Greco cb7c71
sleeping lock on -RT which can not be acquired in this context.
Pablo Greco cb7c71
Pablo Greco cb7c71
Rename compute_layout() to kvm_compute_layout() and invoke it before
Pablo Greco cb7c71
stop_machine() applies the alternatives. Add a __init prefix to
Pablo Greco cb7c71
kvm_compute_layout() because the caller has it, too (and so the code can be
Pablo Greco cb7c71
discarded after boot).
Pablo Greco cb7c71
Pablo Greco cb7c71
Reviewed-by: James Morse <james.morse@arm.com>
Pablo Greco cb7c71
Acked-by: Marc Zyngier <maz@kernel.org>
Pablo Greco cb7c71
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Pablo Greco cb7c71
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Pablo Greco cb7c71
---
Pablo Greco cb7c71
 arch/arm64/include/asm/kvm_mmu.h | 1 +
Pablo Greco cb7c71
 arch/arm64/kernel/smp.c          | 4 ++++
Pablo Greco cb7c71
 arch/arm64/kvm/va_layout.c       | 8 +-------
Pablo Greco cb7c71
 3 files changed, 6 insertions(+), 7 deletions(-)
Pablo Greco cb7c71
Pablo Greco cb7c71
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
Pablo Greco cb7c71
index befe37d..53d846f 100644
Pablo Greco cb7c71
--- a/arch/arm64/include/asm/kvm_mmu.h
Pablo Greco cb7c71
+++ b/arch/arm64/include/asm/kvm_mmu.h
Pablo Greco cb7c71
@@ -91,6 +91,7 @@
Pablo Greco cb7c71
 
Pablo Greco cb7c71
 void kvm_update_va_mask(struct alt_instr *alt,
Pablo Greco cb7c71
 			__le32 *origptr, __le32 *updptr, int nr_inst);
Pablo Greco cb7c71
+void kvm_compute_layout(void);
Pablo Greco cb7c71
 
Pablo Greco cb7c71
 static inline unsigned long __kern_hyp_va(unsigned long v)
Pablo Greco cb7c71
 {
Pablo Greco cb7c71
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
Pablo Greco cb7c71
index dc9fe87..02d41ea 100644
Pablo Greco cb7c71
--- a/arch/arm64/kernel/smp.c
Pablo Greco cb7c71
+++ b/arch/arm64/kernel/smp.c
Pablo Greco cb7c71
@@ -31,6 +31,7 @@
Pablo Greco cb7c71
 #include <linux of.h="">
Pablo Greco cb7c71
 #include <linux irq_work.h="">
Pablo Greco cb7c71
 #include <linux kexec.h="">
Pablo Greco cb7c71
+#include <linux kvm_host.h="">
Pablo Greco cb7c71
 
Pablo Greco cb7c71
 #include <asm alternative.h="">
Pablo Greco cb7c71
 #include <asm atomic.h="">
Pablo Greco cb7c71
@@ -39,6 +40,7 @@
Pablo Greco cb7c71
 #include <asm cputype.h="">
Pablo Greco cb7c71
 #include <asm cpu_ops.h="">
Pablo Greco cb7c71
 #include <asm daifflags.h="">
Pablo Greco cb7c71
+#include <asm kvm_mmu.h="">
Pablo Greco cb7c71
 #include <asm mmu_context.h="">
Pablo Greco cb7c71
 #include <asm numa.h="">
Pablo Greco cb7c71
 #include <asm pgtable.h="">
Pablo Greco cb7c71
@@ -408,6 +410,8 @@ static void __init hyp_mode_check(void)
Pablo Greco cb7c71
 			   "CPU: CPUs started in inconsistent modes");
Pablo Greco cb7c71
 	else
Pablo Greco cb7c71
 		pr_info("CPU: All CPU(s) started at EL1\n");
Pablo Greco cb7c71
+	if (IS_ENABLED(CONFIG_KVM_ARM_HOST))
Pablo Greco cb7c71
+		kvm_compute_layout();
Pablo Greco cb7c71
 }
Pablo Greco cb7c71
 
Pablo Greco cb7c71
 void __init smp_cpus_done(unsigned int max_cpus)
Pablo Greco cb7c71
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
Pablo Greco cb7c71
index 2cf7d4b..dab1fea 100644
Pablo Greco cb7c71
--- a/arch/arm64/kvm/va_layout.c
Pablo Greco cb7c71
+++ b/arch/arm64/kvm/va_layout.c
Pablo Greco cb7c71
@@ -22,7 +22,7 @@
Pablo Greco cb7c71
 static u64 tag_val;
Pablo Greco cb7c71
 static u64 va_mask;
Pablo Greco cb7c71
 
Pablo Greco cb7c71
-static void compute_layout(void)
Pablo Greco cb7c71
+__init void kvm_compute_layout(void)
Pablo Greco cb7c71
 {
Pablo Greco cb7c71
 	phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
Pablo Greco cb7c71
 	u64 hyp_va_msb;
Pablo Greco cb7c71
@@ -110,9 +110,6 @@ void __init kvm_update_va_mask(struct alt_instr *alt,
Pablo Greco cb7c71
 
Pablo Greco cb7c71
 	BUG_ON(nr_inst != 5);
Pablo Greco cb7c71
 
Pablo Greco cb7c71
-	if (!has_vhe() && !va_mask)
Pablo Greco cb7c71
-		compute_layout();
Pablo Greco cb7c71
-
Pablo Greco cb7c71
 	for (i = 0; i < nr_inst; i++) {
Pablo Greco cb7c71
 		u32 rd, rn, insn, oinsn;
Pablo Greco cb7c71
 
Pablo Greco cb7c71
@@ -156,9 +153,6 @@ void kvm_patch_vector_branch(struct alt_instr *alt,
Pablo Greco cb7c71
 		return;
Pablo Greco cb7c71
 	}
Pablo Greco cb7c71
 
Pablo Greco cb7c71
-	if (!va_mask)
Pablo Greco cb7c71
-		compute_layout();
Pablo Greco cb7c71
-
Pablo Greco cb7c71
 	/*
Pablo Greco cb7c71
 	 * Compute HYP VA by using the same computation as kern_hyp_va()
Pablo Greco cb7c71
 	 */
Pablo Greco cb7c71
-- 
Pablo Greco cb7c71
1.8.3.1
Pablo Greco cb7c71
Pablo Greco cb7c71
Pablo Greco cb7c71
From d2d093ca412b7fd66acd745d00e3ebd4764fe127 Mon Sep 17 00:00:00 2001
Pablo Greco cb7c71
From: Russell King <rmk+kernel@armlinux.org.uk>
Pablo Greco cb7c71
Date: Sat, 28 Dec 2019 11:57:14 +0000
Pablo Greco cb7c71
Subject: [PATCH 2/2] arm64: kvm: Fix IDMAP overlap with HYP VA
Pablo Greco cb7c71
Pablo Greco cb7c71
Booting 5.4 on LX2160A reveals that KVM is non-functional:
Pablo Greco cb7c71
Pablo Greco cb7c71
kvm: Limiting the IPA size due to kernel Virtual Address limit
Pablo Greco cb7c71
kvm [1]: IPA Size Limit: 43bits
Pablo Greco cb7c71
kvm [1]: IDMAP intersecting with HYP VA, unable to continue
Pablo Greco cb7c71
kvm [1]: error initializing Hyp mode: -22
Pablo Greco cb7c71
Pablo Greco cb7c71
Debugging shows:
Pablo Greco cb7c71
Pablo Greco cb7c71
kvm [1]: IDMAP page: 81a26000
Pablo Greco cb7c71
kvm [1]: HYP VA range: 0:22ffffffff
Pablo Greco cb7c71
Pablo Greco cb7c71
as RAM is located at:
Pablo Greco cb7c71
Pablo Greco cb7c71
80000000-fbdfffff : System RAM
Pablo Greco cb7c71
2080000000-237fffffff : System RAM
Pablo Greco cb7c71
Pablo Greco cb7c71
Comparing this with the same kernel on Armada 8040 shows:
Pablo Greco cb7c71
Pablo Greco cb7c71
kvm: Limiting the IPA size due to kernel Virtual Address limit
Pablo Greco cb7c71
kvm [1]: IPA Size Limit: 43bits
Pablo Greco cb7c71
kvm [1]: IDMAP page: 2a26000
Pablo Greco cb7c71
kvm [1]: HYP VA range: 4800000000:493fffffff
Pablo Greco cb7c71
...
Pablo Greco cb7c71
kvm [1]: Hyp mode initialized successfully
Pablo Greco cb7c71
Pablo Greco cb7c71
which indicates that hyp_va_msb is set, and is always set to the
Pablo Greco cb7c71
opposite value of the idmap page to avoid the overlap. This does not
Pablo Greco cb7c71
happen with the LX2160A.
Pablo Greco cb7c71
Pablo Greco cb7c71
Further debugging shows vabits_actual = 39, kva_msb = 38 on LX2160A and
Pablo Greco cb7c71
kva_msb = 33 on Armada 8040. Looking at the bit layout of the HYP VA,
Pablo Greco cb7c71
there is still one bit available for hyp_va_msb. Set this bit
Pablo Greco cb7c71
appropriately. This allows KVM to be functional on the LX2160A, but
Pablo Greco cb7c71
without any HYP VA randomisation:
Pablo Greco cb7c71
Pablo Greco cb7c71
kvm: Limiting the IPA size due to kernel Virtual Address limit
Pablo Greco cb7c71
kvm [1]: IPA Size Limit: 43bits
Pablo Greco cb7c71
kvm [1]: IDMAP page: 81a24000
Pablo Greco cb7c71
kvm [1]: HYP VA range: 4000000000:62ffffffff
Pablo Greco cb7c71
...
Pablo Greco cb7c71
kvm [1]: Hyp mode initialized successfully
Pablo Greco cb7c71
Pablo Greco cb7c71
Fixes: ed57cac83e05 ("arm64: KVM: Introduce EL2 VA randomisation")
Pablo Greco cb7c71
Signed-off-by: Russell King <rmk+kernel@armlinux.org.uk>
Pablo Greco cb7c71
[maz: small additional cleanups, preserved case where the tag
Pablo Greco cb7c71
 is legitimately 0 and we can just use the mask, Fixes tag]
Pablo Greco cb7c71
Signed-off-by: Marc Zyngier <maz@kernel.org>
Pablo Greco cb7c71
Link: https://lore.kernel.org/r/E1ilAiY-0000MA-RG@rmk-PC.armlinux.org.uk
Pablo Greco cb7c71
---
Pablo Greco cb7c71
 arch/arm64/kvm/va_layout.c | 56 +++++++++++++++++++++-------------------------
Pablo Greco cb7c71
 1 file changed, 25 insertions(+), 31 deletions(-)
Pablo Greco cb7c71
Pablo Greco cb7c71
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
Pablo Greco cb7c71
index dab1fea..a4f48c1 100644
Pablo Greco cb7c71
--- a/arch/arm64/kvm/va_layout.c
Pablo Greco cb7c71
+++ b/arch/arm64/kvm/va_layout.c
Pablo Greco cb7c71
@@ -13,52 +13,46 @@
Pablo Greco cb7c71
 #include <asm kvm_mmu.h="">
Pablo Greco cb7c71
 
Pablo Greco cb7c71
 /*
Pablo Greco cb7c71
- * The LSB of the random hyp VA tag or 0 if no randomization is used.
Pablo Greco cb7c71
+ * The LSB of the HYP VA tag
Pablo Greco cb7c71
  */
Pablo Greco cb7c71
 static u8 tag_lsb;
Pablo Greco cb7c71
 /*
Pablo Greco cb7c71
- * The random hyp VA tag value with the region bit if hyp randomization is used
Pablo Greco cb7c71
+ * The HYP VA tag value with the region bit
Pablo Greco cb7c71
  */
Pablo Greco cb7c71
 static u64 tag_val;
Pablo Greco cb7c71
 static u64 va_mask;
Pablo Greco cb7c71
 
Pablo Greco cb7c71
+/*
Pablo Greco cb7c71
+ * We want to generate a hyp VA with the following format (with V ==
Pablo Greco cb7c71
+ * vabits_actual):
Pablo Greco cb7c71
+ *
Pablo Greco cb7c71
+ *  63 ... V |     V-1    | V-2 .. tag_lsb | tag_lsb - 1 .. 0
Pablo Greco cb7c71
+ *  ---------------------------------------------------------
Pablo Greco cb7c71
+ * | 0000000 | hyp_va_msb |   random tag   |  kern linear VA |
Pablo Greco cb7c71
+ *           |--------- tag_val -----------|----- va_mask ---|
Pablo Greco cb7c71
+ *
Pablo Greco cb7c71
+ * which does not conflict with the idmap regions.
Pablo Greco cb7c71
+ */
Pablo Greco cb7c71
 __init void kvm_compute_layout(void)
Pablo Greco cb7c71
 {
Pablo Greco cb7c71
 	phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
Pablo Greco cb7c71
 	u64 hyp_va_msb;
Pablo Greco cb7c71
-	int kva_msb;
Pablo Greco cb7c71
 
Pablo Greco cb7c71
 	/* Where is my RAM region? */
Pablo Greco cb7c71
 	hyp_va_msb  = idmap_addr & BIT(vabits_actual - 1);
Pablo Greco cb7c71
 	hyp_va_msb ^= BIT(vabits_actual - 1);
Pablo Greco cb7c71
 
Pablo Greco cb7c71
-	kva_msb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
Pablo Greco cb7c71
+	tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
Pablo Greco cb7c71
 			(u64)(high_memory - 1));
Pablo Greco cb7c71
 
Pablo Greco cb7c71
-	if (kva_msb == (vabits_actual - 1)) {
Pablo Greco cb7c71
-		/*
Pablo Greco cb7c71
-		 * No space in the address, let's compute the mask so
Pablo Greco cb7c71
-		 * that it covers (vabits_actual - 1) bits, and the region
Pablo Greco cb7c71
-		 * bit. The tag stays set to zero.
Pablo Greco cb7c71
-		 */
Pablo Greco cb7c71
-		va_mask  = BIT(vabits_actual - 1) - 1;
Pablo Greco cb7c71
-		va_mask |= hyp_va_msb;
Pablo Greco cb7c71
-	} else {
Pablo Greco cb7c71
-		/*
Pablo Greco cb7c71
-		 * We do have some free bits to insert a random tag.
Pablo Greco cb7c71
-		 * Hyp VAs are now created from kernel linear map VAs
Pablo Greco cb7c71
-		 * using the following formula (with V == vabits_actual):
Pablo Greco cb7c71
-		 *
Pablo Greco cb7c71
-		 *  63 ... V |     V-1    | V-2 .. tag_lsb | tag_lsb - 1 .. 0
Pablo Greco cb7c71
-		 *  ---------------------------------------------------------
Pablo Greco cb7c71
-		 * | 0000000 | hyp_va_msb |    random tag  |  kern linear VA |
Pablo Greco cb7c71
-		 */
Pablo Greco cb7c71
-		tag_lsb = kva_msb;
Pablo Greco cb7c71
-		va_mask = GENMASK_ULL(tag_lsb - 1, 0);
Pablo Greco cb7c71
-		tag_val = get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb);
Pablo Greco cb7c71
-		tag_val |= hyp_va_msb;
Pablo Greco cb7c71
-		tag_val >>= tag_lsb;
Pablo Greco cb7c71
+	va_mask = GENMASK_ULL(tag_lsb - 1, 0);
Pablo Greco cb7c71
+	tag_val = hyp_va_msb;
Pablo Greco cb7c71
+
Pablo Greco cb7c71
+	if (tag_lsb != (vabits_actual - 1)) {
Pablo Greco cb7c71
+		/* We have some free bits to insert a random tag. */
Pablo Greco cb7c71
+		tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb);
Pablo Greco cb7c71
 	}
Pablo Greco cb7c71
+	tag_val >>= tag_lsb;
Pablo Greco cb7c71
 }
Pablo Greco cb7c71
 
Pablo Greco cb7c71
 static u32 compute_instruction(int n, u32 rd, u32 rn)
Pablo Greco cb7c71
@@ -117,11 +111,11 @@ void __init kvm_update_va_mask(struct alt_instr *alt,
Pablo Greco cb7c71
 		 * VHE doesn't need any address translation, let's NOP
Pablo Greco cb7c71
 		 * everything.
Pablo Greco cb7c71
 		 *
Pablo Greco cb7c71
-		 * Alternatively, if we don't have any spare bits in
Pablo Greco cb7c71
-		 * the address, NOP everything after masking that
Pablo Greco cb7c71
-		 * kernel VA.
Pablo Greco cb7c71
+		 * Alternatively, if the tag is zero (because the layout
Pablo Greco cb7c71
+		 * dictates it and we don't have any spare bits in the
Pablo Greco cb7c71
+		 * address), NOP everything after masking the kernel VA.
Pablo Greco cb7c71
 		 */
Pablo Greco cb7c71
-		if (has_vhe() || (!tag_lsb && i > 0)) {
Pablo Greco cb7c71
+		if (has_vhe() || (!tag_val && i > 0)) {
Pablo Greco cb7c71
 			updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
Pablo Greco cb7c71
 			continue;
Pablo Greco cb7c71
 		}
Pablo Greco cb7c71
-- 
Pablo Greco cb7c71
1.8.3.1
Pablo Greco cb7c71