Commit 4645d11f authored by Marc Zyngier's avatar Marc Zyngier
Browse files

KVM: arm64: vgic-v3: Implement MMIO-based LPI invalidation 



Since GICv4.1, it has become legal for an implementation to advertise
GICR_{INVLPIR,INVALLR,SYNCR} while having an ITS, allowing for a more
efficient invalidation scheme (no guest command queue contention when
multiple CPUs are generating invalidations).

Provide the invalidation registers as a primitive to their ITS
counterpart. Note that we don't advertise them to the guest yet
(the architecture allows an implementation to do this).

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
Reviewed-by: default avatarOliver Upton <oupton@google.com>
Link: https://lore.kernel.org/r/20220405182327.205520-4-maz@kernel.org
parent 94828468

arch/arm64/kvm/vgic/vgic-its.c

+41 −21
Original line number Diff line number Diff line
@@ -1272,6 +1272,11 @@ static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its, 
	return 0;

}



int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq)

{

	return update_lpi_config(kvm, irq, NULL, true);

}



/*

 * The INV command syncs the configuration bits from the memory table.

 * Must be called with the its_lock mutex held.
@@ -1288,7 +1293,41 @@ static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its, 
	if (!ite)

		return E_ITS_INV_UNMAPPED_INTERRUPT;



	return update_lpi_config(kvm, ite->irq, NULL, true);

	return vgic_its_inv_lpi(kvm, ite->irq);

}



/**

 * vgic_its_invall - invalidate all LPIs targetting a given vcpu

 * @vcpu: the vcpu for which the RD is targetted by an invalidation

 *

 * Contrary to the INVALL command, this targets a RD instead of a

 * collection, and we don't need to hold the its_lock, since no ITS is

 * involved here.

 */

int vgic_its_invall(struct kvm_vcpu *vcpu)

{

	struct kvm *kvm = vcpu->kvm;

	int irq_count, i = 0;

	u32 *intids;



	irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids);

	if (irq_count < 0)

		return irq_count;



	for (i = 0; i < irq_count; i++) {

		struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intids[i]);

		if (!irq)

			continue;

		update_lpi_config(kvm, irq, vcpu, false);

		vgic_put_irq(kvm, irq);

	}



	kfree(intids);



	if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)

		its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);



	return 0;

}



/*
@@ -1305,32 +1344,13 @@ static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its, 
	u32 coll_id = its_cmd_get_collection(its_cmd);

	struct its_collection *collection;

	struct kvm_vcpu *vcpu;

	struct vgic_irq *irq;

	u32 *intids;

	int irq_count, i;



	collection = find_collection(its, coll_id);

	if (!its_is_collection_mapped(collection))

		return E_ITS_INVALL_UNMAPPED_COLLECTION;



	vcpu = kvm_get_vcpu(kvm, collection->target_addr);



	irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids);

	if (irq_count < 0)

		return irq_count;



	for (i = 0; i < irq_count; i++) {

		irq = vgic_get_irq(kvm, NULL, intids[i]);

		if (!irq)

			continue;

		update_lpi_config(kvm, irq, vcpu, false);

		vgic_put_irq(kvm, irq);

	}



	kfree(intids);



	if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)

		its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);

	vgic_its_invall(vcpu);



	return 0;

}

arch/arm64/kvm/vgic/vgic-mmio-v3.c

+66 −0
Original line number Diff line number Diff line
@@ -543,6 +543,63 @@ static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu, 
			   pendbaser) != old_pendbaser);

}



static unsigned long vgic_mmio_read_sync(struct kvm_vcpu *vcpu,

					 gpa_t addr, unsigned int len)

{

	return !!atomic_read(&vcpu->arch.vgic_cpu.syncr_busy);

}



static void vgic_set_rdist_busy(struct kvm_vcpu *vcpu, bool busy)

{

	if (busy) {

		atomic_inc(&vcpu->arch.vgic_cpu.syncr_busy);

		smp_mb__after_atomic();

	} else {

		smp_mb__before_atomic();

		atomic_dec(&vcpu->arch.vgic_cpu.syncr_busy);

	}

}



static void vgic_mmio_write_invlpi(struct kvm_vcpu *vcpu,

				   gpa_t addr, unsigned int len,

				   unsigned long val)

{

	struct vgic_irq *irq;



	/*

	 * If the guest wrote only to the upper 32bit part of the

	 * register, drop the write on the floor, as it is only for

	 * vPEs (which we don't support for obvious reasons).

	 *

	 * Also discard the access if LPIs are not enabled.

	 */

	if ((addr & 4) || !vgic_lpis_enabled(vcpu))

		return;



	vgic_set_rdist_busy(vcpu, true);



	irq = vgic_get_irq(vcpu->kvm, NULL, lower_32_bits(val));

	if (irq) {

		vgic_its_inv_lpi(vcpu->kvm, irq);

		vgic_put_irq(vcpu->kvm, irq);

	}



	vgic_set_rdist_busy(vcpu, false);

}



static void vgic_mmio_write_invall(struct kvm_vcpu *vcpu,

				   gpa_t addr, unsigned int len,

				   unsigned long val)

{

	/* See vgic_mmio_write_invlpi() for the early return rationale */

	if ((addr & 4) || !vgic_lpis_enabled(vcpu))

		return;



	vgic_set_rdist_busy(vcpu, true);

	vgic_its_invall(vcpu);

	vgic_set_rdist_busy(vcpu, false);

}



/*

 * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the

 * redistributors, while SPIs are covered by registers in the distributor
@@ -648,6 +705,15 @@ static const struct vgic_register_region vgic_v3_rd_registers[] = { 
	REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER,

		vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8,

		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),

	REGISTER_DESC_WITH_LENGTH(GICR_INVLPIR,

		vgic_mmio_read_raz, vgic_mmio_write_invlpi, 8,

		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),

	REGISTER_DESC_WITH_LENGTH(GICR_INVALLR,

		vgic_mmio_read_raz, vgic_mmio_write_invall, 8,

		VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),

	REGISTER_DESC_WITH_LENGTH(GICR_SYNCR,

		vgic_mmio_read_sync, vgic_mmio_write_wi, 4,

		VGIC_ACCESS_32bit),

	REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,

		vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,

		VGIC_ACCESS_32bit),

arch/arm64/kvm/vgic/vgic.h

+4 −0
Original line number Diff line number Diff line
@@ -318,6 +318,10 @@ void vgic_lpi_translation_cache_init(struct kvm *kvm); 
void vgic_lpi_translation_cache_destroy(struct kvm *kvm);

void vgic_its_invalidate_cache(struct kvm *kvm);



/* GICv4.1 MMIO interface */

int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq);

int vgic_its_invall(struct kvm_vcpu *vcpu);



bool vgic_supports_direct_msis(struct kvm *kvm);

int vgic_v4_init(struct kvm *kvm);

void vgic_v4_teardown(struct kvm *kvm);

include/kvm/arm_vgic.h

+1 −0
Original line number Diff line number Diff line
@@ -344,6 +344,7 @@ struct vgic_cpu { 
	struct vgic_io_device	rd_iodev;

	struct vgic_redist_region *rdreg;

	u32 rdreg_index;

	atomic_t syncr_busy;



	/* Contains the attributes and gpa of the LPI pending tables. */

	u64 pendbaser;