Commit 3b619e22 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Will Deacon
Browse files

arm64: implement dynamic shadow call stack for Clang 



Implement dynamic shadow call stack support on Clang, by parsing the
unwind tables at init time to locate all occurrences of PACIASP/AUTIASP
instructions, and replacing them with the shadow call stack push and pop
instructions, respectively.

This is useful because the overhead of the shadow call stack is
difficult to justify on hardware that implements pointer authentication
(PAC), and given that the PAC instructions are executed as NOPs on
hardware that doesn't, we can just replace them without breaking
anything. As PACIASP/AUTIASP are guaranteed to be paired with respect to
manipulations of the return address, replacing them 1:1 with shadow call
stack pushes and pops is guaranteed to result in the desired behavior.

Signed-off-by: default avatarArd Biesheuvel <ardb@kernel.org>
Reviewed-by: default avatarSami Tolvanen <samitolvanen@google.com>
Tested-by: default avatarSami Tolvanen <samitolvanen@google.com>
Link: https://lore.kernel.org/r/20221027155908.1940624-4-ardb@kernel.org


Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 9beccca0

arch/arm64/Kconfig

+9 −0
Original line number Original line Diff line number Diff line
@@ -2160,6 +2160,15 @@ config ARCH_NR_GPIO 




          If unsure, leave the default value.

          If unsure, leave the default value.





config UNWIND_PATCH_PAC_INTO_SCS

	bool "Enable shadow call stack dynamically using code patching"

	# needs Clang with https://reviews.llvm.org/D111780 incorporated

	depends on CC_IS_CLANG && CLANG_VERSION >= 150000

	depends on ARM64_PTR_AUTH_KERNEL && CC_HAS_BRANCH_PROT_PAC_RET

	depends on SHADOW_CALL_STACK

	select UNWIND_TABLES

	select DYNAMIC_SCS



endmenu # "Kernel Features"

endmenu # "Kernel Features"





menu "Boot options"

menu "Boot options"

arch/arm64/Makefile

+8 −2
Original line number Original line Diff line number Diff line
@@ -77,10 +77,16 @@ branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address= 
# We enable additional protection for leaf functions as there is some

# We enable additional protection for leaf functions as there is some

# narrow potential for ROP protection benefits and no substantial

# narrow potential for ROP protection benefits and no substantial

# performance impact has been observed.

# performance impact has been observed.

PACRET-y := pac-ret+leaf



# Using a shadow call stack in leaf functions is too costly, so avoid PAC there

# as well when we may be patching PAC into SCS

PACRET-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) := pac-ret



ifeq ($(CONFIG_ARM64_BTI_KERNEL),y)

ifeq ($(CONFIG_ARM64_BTI_KERNEL),y)

branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=pac-ret+leaf+bti

branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI) := -mbranch-protection=$(PACRET-y)+bti

else

else

branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=pac-ret+leaf

branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=$(PACRET-y)

endif

endif

# -march=armv8.3-a enables the non-nops instructions for PAC, to avoid the

# -march=armv8.3-a enables the non-nops instructions for PAC, to avoid the

# compiler to generate them and consequently to break the single image contract

# compiler to generate them and consequently to break the single image contract

arch/arm64/include/asm/scs.h

+49 −0
Original line number Original line Diff line number Diff line
@@ -5,6 +5,7 @@ 
#ifdef __ASSEMBLY__

#ifdef __ASSEMBLY__





#include <asm/asm-offsets.h>

#include <asm/asm-offsets.h>

#include <asm/sysreg.h>





#ifdef CONFIG_SHADOW_CALL_STACK

#ifdef CONFIG_SHADOW_CALL_STACK

	scs_sp	.req	x18

	scs_sp	.req	x18
@@ -24,6 +25,54 @@ 
	.endm

	.endm

#endif /* CONFIG_SHADOW_CALL_STACK */

#endif /* CONFIG_SHADOW_CALL_STACK */







#else



#include <linux/scs.h>

#include <asm/cpufeature.h>



#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS

static inline bool should_patch_pac_into_scs(void)

{

	u64 reg;



	/*

	 * We only enable the shadow call stack dynamically if we are running

	 * on a system that does not implement PAC or BTI. PAC and SCS provide

	 * roughly the same level of protection, and BTI relies on the PACIASP

	 * instructions serving as landing pads, preventing us from patching

	 * those instructions into something else.

	 */

	reg = read_sysreg_s(SYS_ID_AA64ISAR1_EL1);

	if (SYS_FIELD_GET(ID_AA64ISAR1_EL1, APA, reg) |

	    SYS_FIELD_GET(ID_AA64ISAR1_EL1, API, reg))

		return false;



	reg = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);

	if (SYS_FIELD_GET(ID_AA64ISAR2_EL1, APA3, reg))

		return false;



	if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) {

		reg = read_sysreg_s(SYS_ID_AA64PFR1_EL1);

		if (reg & (0xf << ID_AA64PFR1_EL1_BT_SHIFT))

			return false;

	}

	return true;

}



static inline void dynamic_scs_init(void)

{

	if (should_patch_pac_into_scs()) {

		pr_info("Enabling dynamic shadow call stack\n");

		static_branch_enable(&dynamic_scs_enabled);

	}

}

#else

static inline void dynamic_scs_init(void) {}

#endif



int scs_patch(const u8 eh_frame[], int size);



#endif /* __ASSEMBLY __ */

#endif /* __ASSEMBLY __ */





#endif /* _ASM_SCS_H */
 
#endif /* _ASM_SCS_H */

arch/arm64/kernel/Makefile

+2 −0
Original line number Original line Diff line number Diff line
@@ -80,6 +80,8 @@ obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o 
obj-$(CONFIG_ARM64_MTE)			+= mte.o

obj-$(CONFIG_ARM64_MTE)			+= mte.o

obj-y					+= vdso-wrap.o

obj-y					+= vdso-wrap.o

obj-$(CONFIG_COMPAT_VDSO)		+= vdso32-wrap.o

obj-$(CONFIG_COMPAT_VDSO)		+= vdso32-wrap.o

obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS)	+= patch-scs.o

CFLAGS_patch-scs.o			+= -mbranch-protection=none





# Force dependency (vdso*-wrap.S includes vdso.so through incbin)

# Force dependency (vdso*-wrap.S includes vdso.so through incbin)

$(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so

$(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so

arch/arm64/kernel/head.S

+3 −0
Original line number Original line Diff line number Diff line
@@ -462,6 +462,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) 
	bl	early_fdt_map			// Try mapping the FDT early

	bl	early_fdt_map			// Try mapping the FDT early

	mov	x0, x20				// pass the full boot status

	mov	x0, x20				// pass the full boot status

	bl	init_feature_override		// Parse cpu feature overrides

	bl	init_feature_override		// Parse cpu feature overrides

#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS

	bl	scs_patch_vmlinux

#endif

	mov	x0, x20

	mov	x0, x20

	bl	finalise_el2			// Prefer VHE if possible

	bl	finalise_el2			// Prefer VHE if possible

	ldp	x29, x30, [sp], #16

	ldp	x29, x30, [sp], #16