Commit 314ef685 authored by David S. Miller's avatar David S. Miller
Browse files

[SPARC64]: Refine register window trap handling. 



When saving and restoing trap state, do the window spill/fill
handling inline so that we never trap deeper than 2 trap levels.
This is important for chips like Niagara.

The window fixup code is massively simplified, and many more
improvements are now possible.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent ffe483d5

arch/sparc64/kernel/etrap.S

+25 −79
Original line number Diff line number Diff line
@@ -55,7 +55,31 @@ etrap_irq: 
		rd	%y, %g3

		stx	%g1, [%g2 + STACKFRAME_SZ + PT_V9_TNPC]

		st	%g3, [%g2 + STACKFRAME_SZ + PT_V9_Y]

		save	%g2, -STACK_BIAS, %sp	! Ordering here is critical



		rdpr	%cansave, %g1

		brnz,pt %g1, etrap_save

		 nop



		rdpr	%cwp, %g1

		add	%g1, 2, %g1

		wrpr	%g1, %cwp

		be,pt	%xcc, etrap_user_spill

		 mov	ASI_AIUP, %g3



		rdpr	%otherwin, %g3

		brz	%g3, etrap_kernel_spill

		 mov	ASI_AIUS, %g3



etrap_user_spill:



		wr	%g3, 0x0, %asi

		ldx	[%g6 + TI_FLAGS], %g3

		and	%g3, _TIF_32BIT, %g3

		brnz,pt	%g3, etrap_user_spill_32bit

		 nop

		ba,a,pt	%xcc, etrap_user_spill_64bit



etrap_save:	save	%g2, -STACK_BIAS, %sp

		mov	%g6, %l6



		bne,pn	%xcc, 3f
@@ -176,83 +200,5 @@ etraptl1: /* Save tstate/tpc/tnpc of TL 1-->4 and the tl register itself. 
		ba,pt	%xcc, 1b

		 andcc	%g1, TSTATE_PRIV, %g0



		.align	64

		.globl	scetrap

scetrap:

		TRAP_LOAD_THREAD_REG(%g6, %g1)

		rdpr	%pil, %g2

		rdpr	%tstate, %g1

		sllx	%g2, 20, %g3

		andcc	%g1, TSTATE_PRIV, %g0

		or	%g1, %g3, %g1

		bne,pn	%xcc, 1f

		 sub	%sp, (STACKFRAME_SZ+TRACEREG_SZ-STACK_BIAS), %g2

		wrpr	%g0, 7, %cleanwin



		sllx	%g1, 51, %g3

		sethi	%hi(TASK_REGOFF), %g2

		or	%g2, %lo(TASK_REGOFF), %g2

		brlz,pn	%g3, 1f

		 add	%g6, %g2, %g2

		wr	%g0, 0, %fprs

1:		rdpr	%tpc, %g3

		stx	%g1, [%g2 + STACKFRAME_SZ + PT_V9_TSTATE]



		rdpr	%tnpc, %g1

		stx	%g3, [%g2 + STACKFRAME_SZ + PT_V9_TPC]

		stx	%g1, [%g2 + STACKFRAME_SZ + PT_V9_TNPC]

		save	%g2, -STACK_BIAS, %sp	! Ordering here is critical

		mov	%g6, %l6

		bne,pn	%xcc, 2f

		 mov	ASI_P, %l7

		rdpr	%canrestore, %g3



		rdpr	%wstate, %g2

		wrpr	%g0, 0, %canrestore

		sll	%g2, 3, %g2

		mov	PRIMARY_CONTEXT, %l4

		wrpr	%g3, 0, %otherwin

		wrpr	%g2, 0, %wstate

		sethi	%hi(sparc64_kern_pri_context), %g2

		ldx	[%g2 + %lo(sparc64_kern_pri_context)], %g3

		stxa	%g3, [%l4] ASI_DMMU

		sethi	%hi(KERNBASE), %l4

		flush	%l4



		mov	ASI_AIUS, %l7

2:		mov	%g4, %l4

		mov	%g5, %l5

		add	%g7, 0x4, %l2

		wrpr	%g0, ETRAP_PSTATE1, %pstate

		stx	%g1, [%sp + PTREGS_OFF + PT_V9_G1]

		stx	%g2, [%sp + PTREGS_OFF + PT_V9_G2]

		sllx	%l7, 24, %l7



		stx	%g3, [%sp + PTREGS_OFF + PT_V9_G3]

		rdpr	%cwp, %l0

		stx	%g4, [%sp + PTREGS_OFF + PT_V9_G4]

		stx	%g5, [%sp + PTREGS_OFF + PT_V9_G5]

		stx	%g6, [%sp + PTREGS_OFF + PT_V9_G6]

		stx	%g7, [%sp + PTREGS_OFF + PT_V9_G7]

		or	%l7, %l0, %l7

		sethi	%hi(TSTATE_RMO | TSTATE_PEF), %l0



		or	%l7, %l0, %l7

		wrpr	%l2, %tnpc

		wrpr	%l7, (TSTATE_PRIV | TSTATE_IE), %tstate

		stx	%i0, [%sp + PTREGS_OFF + PT_V9_I0]

		stx	%i1, [%sp + PTREGS_OFF + PT_V9_I1]

		stx	%i2, [%sp + PTREGS_OFF + PT_V9_I2]

		stx	%i3, [%sp + PTREGS_OFF + PT_V9_I3]

		stx	%i4, [%sp + PTREGS_OFF + PT_V9_I4]



		stx	%i5, [%sp + PTREGS_OFF + PT_V9_I5]

		stx	%i6, [%sp + PTREGS_OFF + PT_V9_I6]

		mov	%l6, %g6

		stx	%i7, [%sp + PTREGS_OFF + PT_V9_I7]

		LOAD_PER_CPU_BASE(%g5, %g6, %g4, %g3, %l1)

		ldx	[%g6 + TI_TASK], %g4

		done



#undef TASK_REGOFF

#undef ETRAP_PSTATE1

arch/sparc64/kernel/process.c

+18 −2
Original line number Diff line number Diff line
@@ -541,6 +541,18 @@ void synchronize_user_stack(void) 
	}

}



static void stack_unaligned(unsigned long sp)

{

	siginfo_t info;



	info.si_signo = SIGBUS;

	info.si_errno = 0;

	info.si_code = BUS_ADRALN;

	info.si_addr = (void __user *) sp;

	info.si_trapno = 0;

	force_sig_info(SIGBUS, &info, current);

}



void fault_in_user_windows(void)

{

	struct thread_info *t = current_thread_info();
@@ -556,13 +568,17 @@ void fault_in_user_windows(void) 
	flush_user_windows();

	window = get_thread_wsaved();



	if (window != 0) {

	if (likely(window != 0)) {

		window -= 1;

		do {

			unsigned long sp = (t->rwbuf_stkptrs[window] + bias);

			struct reg_window *rwin = &t->reg_window[window];



			if (copy_to_user((char __user *)sp, rwin, winsize))

			if (unlikely(sp & 0x7UL))

				stack_unaligned(sp);



			if (unlikely(copy_to_user((char __user *)sp,

						  rwin, winsize)))

				goto barf;

		} while (window--);

	}

arch/sparc64/kernel/rtrap.S

+56 −2
Original line number Diff line number Diff line
@@ -267,15 +267,69 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 


		wrpr			%l2, %g0, %canrestore

		wrpr			%l1, %g0, %wstate

		brnz,pt			%l2, user_rtt_restore

		 wrpr			%g0, %g0, %otherwin



		ldx			[%g6 + TI_FLAGS], %g3

		wr			%g0, ASI_AIUP, %asi

		rdpr			%cwp, %g1

		andcc			%g3, _TIF_32BIT, %g0

		sub			%g1, 1, %g1

		bne,pt			%xcc, user_rtt_fill_32bit

		 wrpr			%g1, %cwp

		ba,a,pt			%xcc, user_rtt_fill_64bit



user_rtt_fill_fixup:

		rdpr	%cwp, %g1

		add	%g1, 1, %g1

		wrpr	%g1, 0x0, %cwp



		rdpr	%wstate, %g2

		sll	%g2, 3, %g2

		wrpr	%g2, 0x0, %wstate



		/* We know %canrestore and %otherwin are both zero.  */



		sethi	%hi(sparc64_kern_pri_context), %g2

		ldx	[%g2 + %lo(sparc64_kern_pri_context)], %g2

		mov	PRIMARY_CONTEXT, %g1

		stxa	%g2, [%g1] ASI_DMMU

		sethi	%hi(KERNBASE), %g1

		flush	%g1



		or	%g4, FAULT_CODE_WINFIXUP, %g4

		stb	%g4, [%g6 + TI_FAULT_CODE]

		stx	%g5, [%g6 + TI_FAULT_ADDR]



		mov	%g6, %l1

		wrpr	%g0, 0x0, %tl

		wrpr	%g0, RTRAP_PSTATE, %pstate

		mov	%l1, %g6

		ldx	[%g6 + TI_TASK], %g4

		LOAD_PER_CPU_BASE(%g5, %g6, %g1, %g2, %g3)

		call	do_sparc64_fault

		 add	%sp, PTREGS_OFF, %o0

		ba,pt	%xcc, rtrap

		 nop



user_rtt_pre_restore:

		add			%g1, 1, %g1

		wrpr			%g1, 0x0, %cwp



user_rtt_restore:

		restore

		rdpr			%canrestore, %g1

		wrpr			%g1, 0x0, %cleanwin

		retry

		nop



kern_rtt:	restore

kern_rtt:	rdpr			%canrestore, %g1

		brz,pn			%g1, kern_rtt_fill

		 nop

kern_rtt_restore:

		restore

		retry



to_kernel:

#ifdef CONFIG_PREEMPT

		ldsw			[%g6 + TI_PRE_COUNT], %l5

arch/sparc64/kernel/tsb.S

+0 −1
Original line number Diff line number Diff line
@@ -115,7 +115,6 @@ sparc64_realfault_common: 
	ba,pt	%xcc, rtrap_clr_l6		! Restore cpu state

	 nop					! Delay slot (fill me)



	.globl	winfix_trampoline

winfix_trampoline:

	rdpr	%tpc, %g3			! Prepare winfixup TNPC

	or	%g3, 0x7c, %g3			! Compute branch offset

arch/sparc64/kernel/ttable.S

+8 −8
Original line number Diff line number Diff line
@@ -92,11 +92,11 @@ tl0_resv07c: BTRAP(0x7c) BTRAP(0x7d) BTRAP(0x7e) BTRAP(0x7f) 
tl0_s0n:	SPILL_0_NORMAL

tl0_s1n:	SPILL_1_NORMAL

tl0_s2n:	SPILL_2_NORMAL

tl0_s3n:	SPILL_3_NORMAL

tl0_s4n:	SPILL_4_NORMAL

tl0_s5n:	SPILL_5_NORMAL

tl0_s6n:	SPILL_6_NORMAL

tl0_s7n:	SPILL_7_NORMAL

tl0_s3n:	SPILL_0_NORMAL_ETRAP

tl0_s4n:	SPILL_1_GENERIC_ETRAP

tl0_s5n:	SPILL_1_GENERIC_ETRAP_FIXUP

tl0_s6n:	SPILL_2_GENERIC_ETRAP

tl0_s7n:	SPILL_2_GENERIC_ETRAP_FIXUP

tl0_s0o:	SPILL_0_OTHER

tl0_s1o:	SPILL_1_OTHER

tl0_s2o:	SPILL_2_OTHER
@@ -110,9 +110,9 @@ tl0_f1n: FILL_1_NORMAL 
tl0_f2n:	FILL_2_NORMAL

tl0_f3n:	FILL_3_NORMAL

tl0_f4n:	FILL_4_NORMAL

tl0_f5n:	FILL_5_NORMAL

tl0_f6n:	FILL_6_NORMAL

tl0_f7n:	FILL_7_NORMAL

tl0_f5n:	FILL_0_NORMAL_RTRAP

tl0_f6n:	FILL_1_GENERIC_RTRAP

tl0_f7n:	FILL_2_GENERIC_RTRAP

tl0_f0o:	FILL_0_OTHER

tl0_f1o:	FILL_1_OTHER

tl0_f2o:	FILL_2_OTHER