Commit 030c6ff6 authored by Uros Bizjak's avatar Uros Bizjak Committed by Andrew Morton
Browse files

lib/genalloc: use try_cmpxchg in {set,clear}_bits_ll 

Use try_cmpxchg instead of cmpxchg (*ptr, old, new) == old in
{set,clear}_bits_ll.  x86 CMPXCHG instruction returns success in ZF flag,
so this change saves a compare after cmpxchg (and related move instruction
in front of cmpxchg).

Also, try_cmpxchg implicitly assigns old *ptr value to "old" when cmpxchg
fails.

Note that the value from *ptr should be read using READ_ONCE to prevent
the compiler from merging, refetching or reordering the read.

The patch also declares these two functions inline, to ensure inlining.

No functional change intended.

Link: https://lkml.kernel.org/r/20230118150703.4024-1-ubizjak@gmail.com


Signed-off-by: default avatarUros Bizjak <ubizjak@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent f3296f80

lib/genalloc.c

+8 −10
Original line number Original line Diff line number Diff line
@@ -40,32 +40,30 @@ static inline size_t chunk_size(const struct gen_pool_chunk *chunk) 
	return chunk->end_addr - chunk->start_addr + 1;

	return chunk->end_addr - chunk->start_addr + 1;

}

}





static int set_bits_ll(unsigned long *addr, unsigned long mask_to_set)

static inline int

set_bits_ll(unsigned long *addr, unsigned long mask_to_set)

{

{

	unsigned long val, nval;

	unsigned long val = READ_ONCE(*addr);





	nval = *addr;

	do {

	do {

		val = nval;

		if (val & mask_to_set)

		if (val & mask_to_set)

			return -EBUSY;

			return -EBUSY;

		cpu_relax();

		cpu_relax();

	} while ((nval = cmpxchg(addr, val, val | mask_to_set)) != val);

	} while (!try_cmpxchg(addr, &val, val | mask_to_set));





	return 0;

	return 0;

}

}





static int clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear)

static inline int

clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear)

{

{

	unsigned long val, nval;

	unsigned long val = READ_ONCE(*addr);





	nval = *addr;

	do {

	do {

		val = nval;

		if ((val & mask_to_clear) != mask_to_clear)

		if ((val & mask_to_clear) != mask_to_clear)

			return -EBUSY;

			return -EBUSY;

		cpu_relax();

		cpu_relax();

	} while ((nval = cmpxchg(addr, val, val & ~mask_to_clear)) != val);

	} while (!try_cmpxchg(addr, &val, val & ~mask_to_clear));





	return 0;

	return 0;

}

}