Commit 38ddf7da authored by Peter Collingbourne's avatar Peter Collingbourne Committed by Will Deacon
Browse files

arm64: mte: avoid clearing PSTATE.TCO on entry unless necessary 



On some microarchitectures, clearing PSTATE.TCO is expensive. Clearing
TCO is only necessary if in-kernel MTE is enabled, or if MTE is
enabled in the userspace process in synchronous (or, soon, asymmetric)
mode, because we do not report uaccess faults to userspace in none
or asynchronous modes. Therefore, adjust the kernel entry code to
clear TCO only if necessary.

Because it is now possible to switch to a task in which TCO needs to
be clear from a task in which TCO is set, we also need to do the same
thing on task switch.

Signed-off-by: default avatarPeter Collingbourne <pcc@google.com>
Link: https://linux-review.googlesource.com/id/I52d82a580bd0500d420be501af2c35fa8c90729e


Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20220219012945.894950-2-pcc@google.com


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent f9b5e46f

arch/arm64/include/asm/mte.h

+22 −0
Original line number Original line Diff line number Diff line
@@ -11,7 +11,9 @@ 
#ifndef __ASSEMBLY__

#ifndef __ASSEMBLY__





#include <linux/bitfield.h>

#include <linux/bitfield.h>

#include <linux/kasan-enabled.h>

#include <linux/page-flags.h>

#include <linux/page-flags.h>

#include <linux/sched.h>

#include <linux/types.h>

#include <linux/types.h>





#include <asm/pgtable-types.h>

#include <asm/pgtable-types.h>
@@ -86,6 +88,26 @@ static inline int mte_ptrace_copy_tags(struct task_struct *child, 




#endif /* CONFIG_ARM64_MTE */

#endif /* CONFIG_ARM64_MTE */





static inline void mte_disable_tco_entry(struct task_struct *task)

{

	if (!system_supports_mte())

		return;



	/*

	 * Re-enable tag checking (TCO set on exception entry). This is only

	 * necessary if MTE is enabled in either the kernel or the userspace

	 * task in synchronous or asymmetric mode (SCTLR_EL1.TCF0 bit 0 is set

	 * for both). With MTE disabled in the kernel and disabled or

	 * asynchronous in userspace, tag check faults (including in uaccesses)

	 * are not reported, therefore there is no need to re-enable checking.

	 * This is beneficial on microarchitectures where re-enabling TCO is

	 * expensive.

	 */

	if (kasan_hw_tags_enabled() ||

	    (task->thread.sctlr_user & (1UL << SCTLR_EL1_TCF0_SHIFT)))

		asm volatile(SET_PSTATE_TCO(0));

}



#ifdef CONFIG_KASAN_HW_TAGS

#ifdef CONFIG_KASAN_HW_TAGS

/* Whether the MTE asynchronous mode is enabled. */

/* Whether the MTE asynchronous mode is enabled. */

DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);

DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);

arch/arm64/kernel/entry-common.c

+3 −0
Original line number Original line Diff line number Diff line
@@ -6,6 +6,7 @@ 
 */

 */





#include <linux/context_tracking.h>

#include <linux/context_tracking.h>

#include <linux/kasan.h>

#include <linux/linkage.h>

#include <linux/linkage.h>

#include <linux/lockdep.h>

#include <linux/lockdep.h>

#include <linux/ptrace.h>

#include <linux/ptrace.h>
@@ -56,6 +57,7 @@ static void noinstr enter_from_kernel_mode(struct pt_regs *regs) 
{

{

	__enter_from_kernel_mode(regs);

	__enter_from_kernel_mode(regs);

	mte_check_tfsr_entry();

	mte_check_tfsr_entry();

	mte_disable_tco_entry(current);

}

}





/*

/*
@@ -103,6 +105,7 @@ static __always_inline void __enter_from_user_mode(void) 
	CT_WARN_ON(ct_state() != CONTEXT_USER);

	CT_WARN_ON(ct_state() != CONTEXT_USER);

	user_exit_irqoff();

	user_exit_irqoff();

	trace_hardirqs_off_finish();

	trace_hardirqs_off_finish();

	mte_disable_tco_entry(current);

}

}





static __always_inline void enter_from_user_mode(struct pt_regs *regs)

static __always_inline void enter_from_user_mode(struct pt_regs *regs)

arch/arm64/kernel/entry.S

+0 −7
Original line number Original line Diff line number Diff line
@@ -308,13 +308,6 @@ alternative_if ARM64_HAS_IRQ_PRIO_MASKING 
	msr_s	SYS_ICC_PMR_EL1, x20

	msr_s	SYS_ICC_PMR_EL1, x20

alternative_else_nop_endif

alternative_else_nop_endif





	/* Re-enable tag checking (TCO set on exception entry) */

#ifdef CONFIG_ARM64_MTE

alternative_if ARM64_MTE

	SET_PSTATE_TCO(0)

alternative_else_nop_endif

#endif



	/*

	/*

	 * Registers that may be useful after this macro is invoked:

	 * Registers that may be useful after this macro is invoked:

	 *

	 *

arch/arm64/kernel/mte.c

+3 −0
Original line number Original line Diff line number Diff line
@@ -253,6 +253,9 @@ void mte_thread_switch(struct task_struct *next) 
	mte_update_sctlr_user(next);

	mte_update_sctlr_user(next);

	mte_update_gcr_excl(next);

	mte_update_gcr_excl(next);





	/* TCO may not have been disabled on exception entry for the current task. */

	mte_disable_tco_entry(next);



	/*

	/*

	 * Check if an async tag exception occurred at EL1.

	 * Check if an async tag exception occurred at EL1.

	 *

	 *