Merge tag 'x86_paravirt_for_v5.12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip (1255f440) · Commits · jan.koester / Linux

arch/x86/entry/entry_64.S

+12 −14

Original line number	Diff line number	Diff line
		@@ -46,14 +46,6 @@
		.code64
		.section .entry.text, "ax"

		#ifdef CONFIG_PARAVIRT_XXL
		SYM_CODE_START(native_usergs_sysret64)
		UNWIND_HINT_EMPTY
		swapgs
		sysretq
		SYM_CODE_END(native_usergs_sysret64)
		#endif /* CONFIG_PARAVIRT_XXL */

		/*
		* 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
		*
		@@ -123,7 +115,12 @@ SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL)
		* Try to use SYSRET instead of IRET if we're returning to
		* a completely clean 64-bit userspace context. If we're not,
		* go to the slow exit path.
		* In the Xen PV case we must use iret anyway.
		*/

		ALTERNATIVE "", "jmp swapgs_restore_regs_and_return_to_usermode", \
		X86_FEATURE_XENPV

		movq RCX(%rsp), %rcx
		movq RIP(%rsp), %r11

		@@ -215,7 +212,8 @@ syscall_return_via_sysret:

		popq %rdi
		popq %rsp
		USERGS_SYSRET64
		swapgs
		sysretq
		SYM_CODE_END(entry_SYSCALL_64)

		/*
		@@ -669,7 +667,7 @@ native_irq_return_ldt:
		*/

		pushq %rdi /* Stash user RDI */
		SWAPGS /* to kernel GS */
		swapgs /* to kernel GS */
		SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */

		movq PER_CPU_VAR(espfix_waddr), %rdi
		@@ -699,7 +697,7 @@ native_irq_return_ldt:
		orq PER_CPU_VAR(espfix_stack), %rax

		SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
		SWAPGS /* to user GS */
		swapgs /* to user GS */
		popq %rdi /* Restore user RDI */

		movq %rax, %rsp
		@@ -943,7 +941,7 @@ SYM_CODE_START_LOCAL(paranoid_entry)
		ret

		.Lparanoid_entry_swapgs:
		SWAPGS
		swapgs

		/*
		* The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
		@@ -1001,7 +999,7 @@ SYM_CODE_START_LOCAL(paranoid_exit)
		jnz restore_regs_and_return_to_kernel

		/* We are returning to a context with user GSBASE */
		SWAPGS_UNSAFE_STACK
		swapgs
		jmp restore_regs_and_return_to_kernel
		SYM_CODE_END(paranoid_exit)

		@@ -1426,7 +1424,7 @@ nmi_no_fsgsbase:
		jnz nmi_restore

		nmi_swapgs:
		SWAPGS_UNSAFE_STACK
		swapgs

		nmi_restore:
		POP_REGS

arch/x86/include/asm/idtentry.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -585,6 +585,9 @@ DECLARE_IDTENTRY_MCE(X86_TRAP_MC, exc_machine_check);
		#else
		DECLARE_IDTENTRY_RAW(X86_TRAP_MC, exc_machine_check);
		#endif
		#ifdef CONFIG_XEN_PV
		DECLARE_IDTENTRY_RAW(X86_TRAP_MC, xenpv_exc_machine_check);
		#endif
		#endif

		/* NMI */
		@@ -605,6 +608,9 @@ DECLARE_IDTENTRY_RAW(X86_TRAP_DB, xenpv_exc_debug);

		/* #DF */
		DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault);
		#ifdef CONFIG_XEN_PV
		DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault);
		#endif

		/* #VC */
		#ifdef CONFIG_AMD_MEM_ENCRYPT

arch/x86/include/asm/irqflags.h

+14 −32

Original line number	Diff line number	Diff line
		@@ -35,15 +35,6 @@ extern __always_inline unsigned long native_save_fl(void)
		return flags;
		}

		extern inline void native_restore_fl(unsigned long flags);
		extern inline void native_restore_fl(unsigned long flags)
		{
		asm volatile("push %0 ; popf"
		: /* no output */
		:"g" (flags)
		:"memory", "cc");
		}

		static __always_inline void native_irq_disable(void)
		{
		asm volatile("cli": : :"memory");
		@@ -79,11 +70,6 @@ static __always_inline unsigned long arch_local_save_flags(void)
		return native_save_fl();
		}

		static __always_inline void arch_local_irq_restore(unsigned long flags)
		{
		native_restore_fl(flags);
		}

		static __always_inline void arch_local_irq_disable(void)
		{
		native_irq_disable();
		@@ -131,25 +117,7 @@ static __always_inline unsigned long arch_local_irq_save(void)
		#define SAVE_FLAGS(x) pushfq; popq %rax
		#endif

		#define SWAPGS swapgs
		/*
		* Currently paravirt can't handle swapgs nicely when we
		* don't have a stack we can rely on (such as a user space
		* stack). So we either find a way around these or just fault
		* and emulate if a guest tries to call swapgs directly.
		*
		* Either way, this is a good way to document that we don't
		* have a reliable stack. x86_64 only.
		*/
		#define SWAPGS_UNSAFE_STACK swapgs

		#define INTERRUPT_RETURN jmp native_iret
		#define USERGS_SYSRET64 \
		swapgs; \
		sysretq;
		#define USERGS_SYSRET32 \
		swapgs; \
		sysretl

		#else
		#define INTERRUPT_RETURN iret
		@@ -170,6 +138,20 @@ static __always_inline int arch_irqs_disabled(void)

		return arch_irqs_disabled_flags(flags);
		}

		static __always_inline void arch_local_irq_restore(unsigned long flags)
		{
		if (!arch_irqs_disabled_flags(flags))
		arch_local_irq_enable();
		}
		#else
		#ifdef CONFIG_X86_64
		#ifdef CONFIG_XEN_PV
		#define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV
		#else
		#define SWAPGS swapgs
		#endif
		#endif
		#endif /* !__ASSEMBLY__ */

		#endif

arch/x86/include/asm/paravirt.h

+0 −30

Original line number	Diff line number	Diff line
		@@ -648,11 +648,6 @@ static inline notrace unsigned long arch_local_save_flags(void)
		return PVOP_CALLEE0(unsigned long, irq.save_fl);
		}

		static inline notrace void arch_local_irq_restore(unsigned long f)
		{
		PVOP_VCALLEE1(irq.restore_fl, f);
		}

		static inline notrace void arch_local_irq_disable(void)
		{
		PVOP_VCALLEE0(irq.irq_disable);
		@@ -776,31 +771,6 @@ extern void default_banner(void);

		#ifdef CONFIG_X86_64
		#ifdef CONFIG_PARAVIRT_XXL
		/*
		* If swapgs is used while the userspace stack is still current,
		* there's no way to call a pvop. The PV replacement must be
		* inlined, or the swapgs instruction must be trapped and emulated.
		*/
		#define SWAPGS_UNSAFE_STACK \
		PARA_SITE(PARA_PATCH(PV_CPU_swapgs), swapgs)

		/*
		* Note: swapgs is very special, and in practise is either going to be
		* implemented with a single "swapgs" instruction or something very
		* special. Either way, we don't need to save any registers for
		* it.
		*/
		#define SWAPGS \
		PARA_SITE(PARA_PATCH(PV_CPU_swapgs), \
		ANNOTATE_RETPOLINE_SAFE; \
		call PARA_INDIRECT(pv_ops+PV_CPU_swapgs); \
		)

		#define USERGS_SYSRET64 \
		PARA_SITE(PARA_PATCH(PV_CPU_usergs_sysret64), \
		ANNOTATE_RETPOLINE_SAFE; \
		jmp PARA_INDIRECT(pv_ops+PV_CPU_usergs_sysret64);)

		#ifdef CONFIG_DEBUG_ENTRY
		#define SAVE_FLAGS(clobbers) \
		PARA_SITE(PARA_PATCH(PV_IRQ_save_fl), \

arch/x86/include/asm/paravirt_types.h

+2 −15

Original line number	Diff line number	Diff line
		@@ -156,20 +156,10 @@ struct pv_cpu_ops {

		u64 (*read_pmc)(int counter);

		/*
		* Switch to usermode gs and return to 64-bit usermode using
		* sysret. Only used in 64-bit kernels to return to 64-bit
		* processes. Usermode register state, including %rsp, must
		* already be restored.
		*/
		void (*usergs_sysret64)(void);

		/* Normal iret. Jump to this with the standard iret stack
		frame set up. */
		void (*iret)(void);

		void (*swapgs)(void);

		void (start_context_switch)(struct task_struct prev);
		void (end_context_switch)(struct task_struct next);
		#endif
		@@ -178,16 +168,13 @@ struct pv_cpu_ops {
		struct pv_irq_ops {
		#ifdef CONFIG_PARAVIRT_XXL
		/*
		* Get/set interrupt state. save_fl and restore_fl are only
		* expected to use X86_EFLAGS_IF; all other bits
		* returned from save_fl are undefined, and may be ignored by
		* restore_fl.
		* Get/set interrupt state. save_fl is expected to use X86_EFLAGS_IF;
		* all other bits returned from save_fl are undefined.
		*
		* NOTE: These functions callers expect the callee to preserve
		* more registers than the standard C calling convention.
		*/
		struct paravirt_callee_save save_fl;
		struct paravirt_callee_save restore_fl;
		struct paravirt_callee_save irq_disable;
		struct paravirt_callee_save irq_enable;