Commit 739ad2e4 authored by Janis Schoetterl-Glausch's avatar Janis Schoetterl-Glausch Committed by Heiko Carstens
Browse files

s390/uaccess: limit number of retries for cmpxchg_user_key() 



cmpxchg_user_key() for byte and short values is implemented via a one word
cmpxchg loop. Give up trying to perform the cmpxchg if it fails too often
because of contention on the cache line. This ensures that the thread
cannot become stuck in the kernel.

Signed-off-by: default avatarJanis Schoetterl-Glausch <scgl@linux.ibm.com>
Link: https://lore.kernel.org/r/20221117100745.3253896-1-scgl@linux.ibm.com


Signed-off-by: default avatarHeiko Carstens <hca@linux.ibm.com>
parent 51098f0e

arch/s390/include/asm/uaccess.h

+25 −8
Original line number Diff line number Diff line
@@ -392,6 +392,8 @@ do { \ 


void __cmpxchg_user_key_called_with_bad_pointer(void);



#define CMPXCHG_USER_KEY_MAX_LOOPS 128



static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,

					      __uint128_t old, __uint128_t new,

					      unsigned long key, int size)
@@ -401,6 +403,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 
	switch (size) {

	case 1: {

		unsigned int prev, shift, mask, _old, _new;

		unsigned long count;



		shift = (3 ^ (address & 3)) << 3;

		address ^= address & 3;
@@ -410,6 +413,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 
		asm volatile(

			"	spka	0(%[key])\n"

			"	sacf	256\n"

			"	llill	%[count],%[max_loops]\n"

			"0:	l	%[prev],%[address]\n"

			"1:	nr	%[prev],%[mask]\n"

			"	xilf	%[mask],0xffffffff\n"
@@ -421,7 +425,8 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 
			"	xr	%[tmp],%[prev]\n"

			"	xr	%[new],%[tmp]\n"

			"	nr	%[tmp],%[mask]\n"

			"	jz	2b\n"

			"	jnz	5f\n"

			"	brct	%[count],2b\n"

			"5:	sacf	768\n"

			"	spka	%[default_key]\n"

			EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
@@ -433,15 +438,20 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 
			  [address] "+Q" (*(int *)address),

			  [tmp] "+&d" (_old),

			  [new] "+&d" (_new),

			  [mask] "+&d" (mask)

			: [key] "a" (key << 4),

			  [default_key] "J" (PAGE_DEFAULT_KEY)

			  [mask] "+&d" (mask),

			  [count] "=a" (count)

			: [key] "%[count]" (key << 4),

			  [default_key] "J" (PAGE_DEFAULT_KEY),

			  [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)

			: "memory", "cc");

		*(unsigned char *)uval = prev >> shift;

		if (!count)

			rc = -EAGAIN;

		return rc;

	}

	case 2: {

		unsigned int prev, shift, mask, _old, _new;

		unsigned long count;



		shift = (2 ^ (address & 2)) << 3;

		address ^= address & 2;
@@ -451,6 +461,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 
		asm volatile(

			"	spka	0(%[key])\n"

			"	sacf	256\n"

			"	llill	%[count],%[max_loops]\n"

			"0:	l	%[prev],%[address]\n"

			"1:	nr	%[prev],%[mask]\n"

			"	xilf	%[mask],0xffffffff\n"
@@ -462,7 +473,8 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 
			"	xr	%[tmp],%[prev]\n"

			"	xr	%[new],%[tmp]\n"

			"	nr	%[tmp],%[mask]\n"

			"	jz	2b\n"

			"	jnz	5f\n"

			"	brct	%[count],2b\n"

			"5:	sacf	768\n"

			"	spka	%[default_key]\n"

			EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
@@ -474,11 +486,15 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 
			  [address] "+Q" (*(int *)address),

			  [tmp] "+&d" (_old),

			  [new] "+&d" (_new),

			  [mask] "+&d" (mask)

			: [key] "a" (key << 4),

			  [default_key] "J" (PAGE_DEFAULT_KEY)

			  [mask] "+&d" (mask),

			  [count] "=a" (count)

			: [key] "%[count]" (key << 4),

			  [default_key] "J" (PAGE_DEFAULT_KEY),

			  [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)

			: "memory", "cc");

		*(unsigned short *)uval = prev >> shift;

		if (!count)

			rc = -EAGAIN;

		return rc;

	}

	case 4:	{
@@ -568,6 +584,7 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 
 *

 * Return:     0: cmpxchg executed

 *	       -EFAULT: an exception happened when trying to access *@ptr

 *	       -EAGAIN: maxed out number of retries (byte and short only)

 */

#define cmpxchg_user_key(ptr, uval, old, new, key)			\

({									\