Commit 904cabf4 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

Merge branch kvm-arm64/hyp-stack-guard into kvmarm-master/next 



* kvm-arm64/hyp-stack-guard:
  : .
  : Harden the EL2 stack by providing stack guards, courtesy of
  : Kalesh Singh.
  : .
  KVM: arm64: Symbolize the nVHE HYP addresses
  KVM: arm64: Detect and handle hypervisor stack overflows
  KVM: arm64: Add guard pages for pKVM (protected nVHE) hypervisor stack
  KVM: arm64: Add guard pages for KVM nVHE hypervisor stack
  KVM: arm64: Introduce pkvm_alloc_private_va_range()
  KVM: arm64: Introduce hyp_alloc_private_va_range()

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parents b2c4caf3 6ccf9cb5

arch/arm64/include/asm/kvm_asm.h

+1 −0
Original line number Diff line number Diff line
@@ -169,6 +169,7 @@ struct kvm_nvhe_init_params { 
	unsigned long tcr_el2;

	unsigned long tpidr_el2;

	unsigned long stack_hyp_va;

	unsigned long stack_pa;

	phys_addr_t pgd_pa;

	unsigned long hcr_el2;

	unsigned long vttbr;

arch/arm64/include/asm/kvm_mmu.h

+3 −0
Original line number Diff line number Diff line
@@ -154,6 +154,9 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) 
int kvm_share_hyp(void *from, void *to);

void kvm_unshare_hyp(void *from, void *to);

int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot);

int __create_hyp_mappings(unsigned long start, unsigned long size,

			  unsigned long phys, enum kvm_pgtable_prot prot);

int hyp_alloc_private_va_range(size_t size, unsigned long *haddr);

int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,

			   void __iomem **kaddr,

			   void __iomem **haddr);

arch/arm64/kvm/arm.c

+34 −3
Original line number Diff line number Diff line
@@ -1479,7 +1479,6 @@ static void cpu_prepare_hyp_mode(int cpu) 
	tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;

	params->tcr_el2 = tcr;



	params->stack_hyp_va = kern_hyp_va(per_cpu(kvm_arm_hyp_stack_page, cpu) + PAGE_SIZE);

	params->pgd_pa = kvm_mmu_get_httbr();

	if (is_protected_kvm_enabled())

		params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
@@ -1929,14 +1928,46 @@ static int init_hyp_mode(void) 
	 * Map the Hyp stack pages

	 */

	for_each_possible_cpu(cpu) {

		struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);

		char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);

		err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE,

					  PAGE_HYP);

		unsigned long hyp_addr;



		/*

		 * Allocate a contiguous HYP private VA range for the stack

		 * and guard page. The allocation is also aligned based on

		 * the order of its size.

		 */

		err = hyp_alloc_private_va_range(PAGE_SIZE * 2, &hyp_addr);

		if (err) {

			kvm_err("Cannot allocate hyp stack guard page\n");

			goto out_err;

		}



		/*

		 * Since the stack grows downwards, map the stack to the page

		 * at the higher address and leave the lower guard page

		 * unbacked.

		 *

		 * Any valid stack address now has the PAGE_SHIFT bit as 1

		 * and addresses corresponding to the guard page have the

		 * PAGE_SHIFT bit as 0 - this is used for overflow detection.

		 */

		err = __create_hyp_mappings(hyp_addr + PAGE_SIZE, PAGE_SIZE,

					    __pa(stack_page), PAGE_HYP);

		if (err) {

			kvm_err("Cannot map hyp stack\n");

			goto out_err;

		}



		/*

		 * Save the stack PA in nvhe_init_params. This will be needed

		 * to recreate the stack mapping in protected nVHE mode.

		 * __hyp_pa() won't do the right thing there, since the stack

		 * has been mapped in the flexible private VA space.

		 */

		params->stack_pa = __pa(stack_page);



		params->stack_hyp_va = hyp_addr + (2 * PAGE_SIZE);

	}



	for_each_possible_cpu(cpu) {

arch/arm64/kvm/handle_exit.c

+5 −8
Original line number Diff line number Diff line
@@ -322,13 +322,8 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, 
	u64 elr_in_kimg = __phys_to_kimg(elr_phys);

	u64 hyp_offset = elr_in_kimg - kaslr_offset() - elr_virt;

	u64 mode = spsr & PSR_MODE_MASK;

	u64 panic_addr = elr_virt + hyp_offset;



	/*

	 * The nVHE hyp symbols are not included by kallsyms to avoid issues

	 * with aliasing. That means that the symbols cannot be printed with the

	 * "%pS" format specifier, so fall back to the vmlinux address if

	 * there's no better option.

	 */

	if (mode != PSR_MODE_EL2t && mode != PSR_MODE_EL2h) {

		kvm_err("Invalid host exception to nVHE hyp!\n");

	} else if (ESR_ELx_EC(esr) == ESR_ELx_EC_BRK64 &&
@@ -348,9 +343,11 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, 
		if (file)

			kvm_err("nVHE hyp BUG at: %s:%u!\n", file, line);

		else

			kvm_err("nVHE hyp BUG at: %016llx!\n", elr_virt + hyp_offset);

			kvm_err("nVHE hyp BUG at: [<%016llx>] %pB!\n", panic_addr,

					(void *)panic_addr);

	} else {

		kvm_err("nVHE hyp panic at: %016llx!\n", elr_virt + hyp_offset);

		kvm_err("nVHE hyp panic at: [<%016llx>] %pB!\n", panic_addr,

				(void *)panic_addr);

	}



	/*

arch/arm64/kvm/hyp/include/nvhe/mm.h

+4 −2
Original line number Diff line number Diff line
@@ -19,8 +19,10 @@ int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back); 
int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot);

int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot);

int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot);

unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size,

					    enum kvm_pgtable_prot prot);

int __pkvm_create_private_mapping(phys_addr_t phys, size_t size,

				  enum kvm_pgtable_prot prot,

				  unsigned long *haddr);

int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr);



static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size,

				     unsigned long *start, unsigned long *end)