Commit 0ba0e5d5 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman
Browse files

KVM: PPC: Book3S HV: Split P8 from P9 path guest vCPU TLB flushing 



This creates separate functions for old and new paths for vCPU TLB
flushing, which will reduce complexity of the next change.

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20211123095231.1036501-43-npiggin@gmail.com
parent a089a686

arch/powerpc/include/asm/kvm_ppc.h

+1 −2
Original line number Diff line number Diff line
@@ -552,8 +552,7 @@ extern void kvm_hv_vm_activated(void); 
extern void kvm_hv_vm_deactivated(void);

extern bool kvm_hv_mode_active(void);



extern void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu,

					struct kvm_nested_guest *nested);

extern void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu);



#else

static inline void __init kvm_cma_reserve(void)

arch/powerpc/kvm/book3s_hv_builtin.c

+8 −45
Original line number Diff line number Diff line
@@ -682,22 +682,6 @@ static void flush_guest_tlb(struct kvm *kvm) 
	unsigned long rb, set;



	rb = PPC_BIT(52);	/* IS = 2 */

	if (kvm_is_radix(kvm)) {

		/* R=1 PRS=1 RIC=2 */

		asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)

			     : : "r" (rb), "i" (1), "i" (1), "i" (2),

			       "r" (0) : "memory");

		for (set = 1; set < kvm->arch.tlb_sets; ++set) {

			rb += PPC_BIT(51);	/* increment set number */

			/* R=1 PRS=1 RIC=0 */

			asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)

				     : : "r" (rb), "i" (1), "i" (1), "i" (0),

				       "r" (0) : "memory");

		}

		asm volatile("ptesync": : :"memory");

		// POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.

		asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory");

	} else {

	for (set = 0; set < kvm->arch.tlb_sets; ++set) {

		/* R=0 PRS=0 RIC=0 */

		asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
@@ -706,36 +690,15 @@ static void flush_guest_tlb(struct kvm *kvm) 
		rb += PPC_BIT(51);	/* increment set number */

	}

	asm volatile("ptesync": : :"memory");

		// POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.

		if (cpu_has_feature(CPU_FTR_ARCH_300))

			asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");

	}

}



void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu,

				 struct kvm_nested_guest *nested)

void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu)

{

	cpumask_t *need_tlb_flush;



	/*

	 * On POWER9, individual threads can come in here, but the

	 * TLB is shared between the 4 threads in a core, hence

	 * invalidating on one thread invalidates for all.

	 * Thus we make all 4 threads use the same bit.

	 */

	if (cpu_has_feature(CPU_FTR_ARCH_300))

		pcpu = cpu_first_tlb_thread_sibling(pcpu);



	if (nested)

		need_tlb_flush = &nested->need_tlb_flush;

	else

		need_tlb_flush = &kvm->arch.need_tlb_flush;



	if (cpumask_test_cpu(pcpu, need_tlb_flush)) {

	if (cpumask_test_cpu(pcpu, &kvm->arch.need_tlb_flush)) {

		flush_guest_tlb(kvm);



		/* Clear the bit after the TLB flush */

		cpumask_clear_cpu(pcpu, need_tlb_flush);

		cpumask_clear_cpu(pcpu, &kvm->arch.need_tlb_flush);

	}

}

EXPORT_SYMBOL_GPL(kvmppc_check_need_tlb_flush);

arch/powerpc/kvm/book3s_hv_p9_entry.c

+61 −1
Original line number Diff line number Diff line
@@ -632,6 +632,66 @@ static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu) 
	}

}



static void flush_guest_tlb(struct kvm *kvm)

{

	unsigned long rb, set;



	rb = PPC_BIT(52);	/* IS = 2 */

	if (kvm_is_radix(kvm)) {

		/* R=1 PRS=1 RIC=2 */

		asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)

			     : : "r" (rb), "i" (1), "i" (1), "i" (2),

			       "r" (0) : "memory");

		for (set = 1; set < kvm->arch.tlb_sets; ++set) {

			rb += PPC_BIT(51);	/* increment set number */

			/* R=1 PRS=1 RIC=0 */

			asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)

				     : : "r" (rb), "i" (1), "i" (1), "i" (0),

				       "r" (0) : "memory");

		}

		asm volatile("ptesync": : :"memory");

		// POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.

		asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory");

	} else {

		for (set = 0; set < kvm->arch.tlb_sets; ++set) {

			/* R=0 PRS=0 RIC=0 */

			asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)

				     : : "r" (rb), "i" (0), "i" (0), "i" (0),

				       "r" (0) : "memory");

			rb += PPC_BIT(51);	/* increment set number */

		}

		asm volatile("ptesync": : :"memory");

		// POWER9 congruence-class TLBIEL leaves ERAT. Flush it now.

		asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");

	}

}



static void check_need_tlb_flush(struct kvm *kvm, int pcpu,

				 struct kvm_nested_guest *nested)

{

	cpumask_t *need_tlb_flush;



	/*

	 * On POWER9, individual threads can come in here, but the

	 * TLB is shared between the 4 threads in a core, hence

	 * invalidating on one thread invalidates for all.

	 * Thus we make all 4 threads use the same bit.

	 */

	pcpu = cpu_first_tlb_thread_sibling(pcpu);



	if (nested)

		need_tlb_flush = &nested->need_tlb_flush;

	else

		need_tlb_flush = &kvm->arch.need_tlb_flush;



	if (cpumask_test_cpu(pcpu, need_tlb_flush)) {

		flush_guest_tlb(kvm);



		/* Clear the bit after the TLB flush */

		cpumask_clear_cpu(pcpu, need_tlb_flush);

	}

}



int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr, u64 *tb)

{

	struct p9_host_os_sprs host_os_sprs;
@@ -819,7 +879,7 @@ int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpc 
	}



	/* TLBIEL uses LPID=LPIDR, so run this after setting guest LPID */

	kvmppc_check_need_tlb_flush(kvm, vc->pcpu, nested);

	check_need_tlb_flush(kvm, vc->pcpu, nested);



	/*

	 * P9 suppresses the HDEC exception when LPCR[HDICE] = 0,