Commit e07af262 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'arc-5.15-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc 

Pull ARC updates from Vineet Gupta:
 "Finally a big pile of changes for ARC (atomics/mm). These are from our
  internal arc64 tree, preparing mainline for eventual arc64 support.
  I'm spreading them out to avoid tsunami of patches in one release.

   - MM rework:
       - Implement up to 4 paging levels
       - Enable STRICT_MM_TYPECHECK
       - switch pgtable_t back to 'struct page *'

   - Atomics rework / implement relaxed accessors

   - Retire legacy MMUv1,v2; ARC750 cores

   - A few other build errors, typos"

* tag 'arc-5.15-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc: (33 commits)
  ARC: mm: vmalloc sync from kernel to user table to update PMD ...
  ARC: mm: support 4 levels of page tables
  ARC: mm: support 3 levels of page tables
  ARC: mm: switch to asm-generic/pgalloc.h
  ARC: mm: switch pgtable_t back to struct page *
  ARC: mm: hack to allow 2 level build with 4 level code
  ARC: mm: disintegrate pgtable.h into levels and flags
  ARC: mm: disintegrate mmu.h (arcv2 bits out)
  ARC: mm: move MMU specific bits out of entry code ...
  ARC: mm: move MMU specific bits out of ASID allocator
  ARC: mm: non-functional code movement/cleanup
  ARC: mm: pmd_populate* to use the canonical set_pmd (and drop pmd_set)
  ARC: ioremap: use more commonly used PAGE_KERNEL based uncached flag
  ARC: mm: Enable STRICT_MM_TYPECHECKS
  ARC: mm: Fixes to allow STRICT_MM_TYPECHECKS
  ARC: mm: move mmu/cache externs out to setup.h
  ARC: mm: remove tlb paranoid code
  ARC: mm: use SCRATCH_DATA0 register for caching pgdir in ARCv2 only
  ARC: retire MMUv1 and MMUv2 support
  ARC: retire ARC750 support
  ...
parents 063df71a 56809a28

arch/arc/Kconfig

+8 −33
Original line number Original line Diff line number Diff line
@@ -116,16 +116,9 @@ choice 
	default ARC_CPU_770 if ISA_ARCOMPACT

	default ARC_CPU_770 if ISA_ARCOMPACT

	default ARC_CPU_HS if ISA_ARCV2

	default ARC_CPU_HS if ISA_ARCV2





if ISA_ARCOMPACT



config ARC_CPU_750D

	bool "ARC750D"

	select ARC_CANT_LLSC

	help

	  Support for ARC750 core



config ARC_CPU_770

config ARC_CPU_770

	bool "ARC770"

	bool "ARC770"

	depends on ISA_ARCOMPACT

	select ARC_HAS_SWAPE

	select ARC_HAS_SWAPE

	help

	help

	  Support for ARC770 core introduced with Rel 4.10 (Summer 2011)

	  Support for ARC770 core introduced with Rel 4.10 (Summer 2011)
@@ -135,8 +128,6 @@ config ARC_CPU_770 
	  -Caches: New Prog Model, Region Flush

	  -Caches: New Prog Model, Region Flush

	  -Insns: endian swap, load-locked/store-conditional, time-stamp-ctr

	  -Insns: endian swap, load-locked/store-conditional, time-stamp-ctr





endif #ISA_ARCOMPACT



config ARC_CPU_HS

config ARC_CPU_HS

	bool "ARC-HS"

	bool "ARC-HS"

	depends on ISA_ARCV2

	depends on ISA_ARCV2
@@ -274,33 +265,17 @@ config ARC_DCCM_BASE 




choice

choice

	prompt "MMU Version"

	prompt "MMU Version"

	default ARC_MMU_V3 if ARC_CPU_770

	default ARC_MMU_V3 if ISA_ARCOMPACT

	default ARC_MMU_V2 if ARC_CPU_750D

	default ARC_MMU_V4 if ISA_ARCV2

	default ARC_MMU_V4 if ARC_CPU_HS



if ISA_ARCOMPACT



config ARC_MMU_V1

	bool "MMU v1"

	help

	  Orig ARC700 MMU



config ARC_MMU_V2

	bool "MMU v2"

	help

	  Fixed the deficiency of v1 - possible thrashing in memcpy scenario

	  when 2 D-TLB and 1 I-TLB entries index into same 2way set.





config ARC_MMU_V3

config ARC_MMU_V3

	bool "MMU v3"

	bool "MMU v3"

	depends on ARC_CPU_770

	depends on ISA_ARCOMPACT

	help

	help

	  Introduced with ARC700 4.10: New Features

	  Introduced with ARC700 4.10: New Features

	  Variable Page size (1k-16k), var JTLB size 128 x (2 or 4)

	  Variable Page size (1k-16k), var JTLB size 128 x (2 or 4)

	  Shared Address Spaces (SASID)

	  Shared Address Spaces (SASID)





endif



config ARC_MMU_V4

config ARC_MMU_V4

	bool "MMU v4"

	bool "MMU v4"

	depends on ISA_ARCV2

	depends on ISA_ARCV2
@@ -319,7 +294,6 @@ config ARC_PAGE_SIZE_8K 




config ARC_PAGE_SIZE_16K

config ARC_PAGE_SIZE_16K

	bool "16KB"

	bool "16KB"

	depends on ARC_MMU_V3 || ARC_MMU_V4





config ARC_PAGE_SIZE_4K

config ARC_PAGE_SIZE_4K

	bool "4KB"

	bool "4KB"
@@ -340,6 +314,10 @@ config ARC_HUGEPAGE_16M 




endchoice

endchoice





config PGTABLE_LEVELS

	int "Number of Page table levels"

	default 2



config ARC_COMPACT_IRQ_LEVELS

config ARC_COMPACT_IRQ_LEVELS

	depends on ISA_ARCOMPACT

	depends on ISA_ARCOMPACT

	bool "Setup Timer IRQ as high Priority"

	bool "Setup Timer IRQ as high Priority"
@@ -563,9 +541,6 @@ config ARC_DW2_UNWIND 
	  If you don't debug the kernel, you can say N, but we may not be able

	  If you don't debug the kernel, you can say N, but we may not be able

	  to solve problems without frame unwind information

	  to solve problems without frame unwind information





config ARC_DBG_TLB_PARANOIA

	bool "Paranoia Checks in Low Level TLB Handlers"



config ARC_DBG_JUMP_LABEL

config ARC_DBG_JUMP_LABEL

	bool "Paranoid checks in Static Keys (jump labels) code"

	bool "Paranoid checks in Static Keys (jump labels) code"

	depends on JUMP_LABEL

	depends on JUMP_LABEL

arch/arc/include/asm/atomic-llsc.h

0 → 100644
+97 −0
Original line number Original line Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */



#ifndef _ASM_ARC_ATOMIC_LLSC_H

#define _ASM_ARC_ATOMIC_LLSC_H



#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))



#define ATOMIC_OP(op, c_op, asm_op)					\

static inline void arch_atomic_##op(int i, atomic_t *v)			\

{									\

	unsigned int val;						\

									\

	__asm__ __volatile__(						\

	"1:	llock   %[val], [%[ctr]]		\n"		\

	"	" #asm_op " %[val], %[val], %[i]	\n"		\

	"	scond   %[val], [%[ctr]]		\n"		\

	"	bnz     1b				\n"		\

	: [val]	"=&r"	(val) /* Early clobber to prevent reg reuse */	\

	: [ctr]	"r"	(&v->counter), /* Not "m": llock only supports reg direct addr mode */	\

	  [i]	"ir"	(i)						\

	: "cc");							\

}									\



#define ATOMIC_OP_RETURN(op, c_op, asm_op)				\

static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v)	\

{									\

	unsigned int val;						\

									\

	__asm__ __volatile__(						\

	"1:	llock   %[val], [%[ctr]]		\n"		\

	"	" #asm_op " %[val], %[val], %[i]	\n"		\

	"	scond   %[val], [%[ctr]]		\n"		\

	"	bnz     1b				\n"		\

	: [val]	"=&r"	(val)						\

	: [ctr]	"r"	(&v->counter),					\

	  [i]	"ir"	(i)						\

	: "cc");							\

									\

	return val;							\

}



#define arch_atomic_add_return_relaxed		arch_atomic_add_return_relaxed

#define arch_atomic_sub_return_relaxed		arch_atomic_sub_return_relaxed



#define ATOMIC_FETCH_OP(op, c_op, asm_op)				\

static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\

{									\

	unsigned int val, orig;						\

									\

	__asm__ __volatile__(						\

	"1:	llock   %[orig], [%[ctr]]		\n"		\

	"	" #asm_op " %[val], %[orig], %[i]	\n"		\

	"	scond   %[val], [%[ctr]]		\n"		\

	"	bnz     1b				\n"		\

	: [val]	"=&r"	(val),						\

	  [orig] "=&r" (orig)						\

	: [ctr]	"r"	(&v->counter),					\

	  [i]	"ir"	(i)						\

	: "cc");							\

									\

	return orig;							\

}



#define arch_atomic_fetch_add_relaxed		arch_atomic_fetch_add_relaxed

#define arch_atomic_fetch_sub_relaxed		arch_atomic_fetch_sub_relaxed



#define arch_atomic_fetch_and_relaxed		arch_atomic_fetch_and_relaxed

#define arch_atomic_fetch_andnot_relaxed	arch_atomic_fetch_andnot_relaxed

#define arch_atomic_fetch_or_relaxed		arch_atomic_fetch_or_relaxed

#define arch_atomic_fetch_xor_relaxed		arch_atomic_fetch_xor_relaxed



#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_OP_RETURN(op, c_op, asm_op)				\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(add, +=, add)

ATOMIC_OPS(sub, -=, sub)



#undef ATOMIC_OPS

#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(and, &=, and)

ATOMIC_OPS(andnot, &= ~, bic)

ATOMIC_OPS(or, |=, or)

ATOMIC_OPS(xor, ^=, xor)



#define arch_atomic_andnot		arch_atomic_andnot



#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



#endif

arch/arc/include/asm/atomic-spinlock.h

0 → 100644
+102 −0
Original line number Original line Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */



#ifndef _ASM_ARC_ATOMIC_SPLOCK_H

#define _ASM_ARC_ATOMIC_SPLOCK_H



/*

 * Non hardware assisted Atomic-R-M-W

 * Locking would change to irq-disabling only (UP) and spinlocks (SMP)

 */



static inline void arch_atomic_set(atomic_t *v, int i)

{

	/*

	 * Independent of hardware support, all of the atomic_xxx() APIs need

	 * to follow the same locking rules to make sure that a "hardware"

	 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn

	 * sequence

	 *

	 * Thus atomic_set() despite being 1 insn (and seemingly atomic)

	 * requires the locking.

	 */

	unsigned long flags;



	atomic_ops_lock(flags);

	WRITE_ONCE(v->counter, i);

	atomic_ops_unlock(flags);

}



#define arch_atomic_set_release(v, i)	arch_atomic_set((v), (i))



#define ATOMIC_OP(op, c_op, asm_op)					\

static inline void arch_atomic_##op(int i, atomic_t *v)			\

{									\

	unsigned long flags;						\

									\

	atomic_ops_lock(flags);						\

	v->counter c_op i;						\

	atomic_ops_unlock(flags);					\

}



#define ATOMIC_OP_RETURN(op, c_op, asm_op)				\

static inline int arch_atomic_##op##_return(int i, atomic_t *v)		\

{									\

	unsigned long flags;						\

	unsigned int temp;						\

									\

	/*								\

	 * spin lock/unlock provides the needed smp_mb() before/after	\

	 */								\

	atomic_ops_lock(flags);						\

	temp = v->counter;						\

	temp c_op i;							\

	v->counter = temp;						\

	atomic_ops_unlock(flags);					\

									\

	return temp;							\

}



#define ATOMIC_FETCH_OP(op, c_op, asm_op)				\

static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\

{									\

	unsigned long flags;						\

	unsigned int orig;						\

									\

	/*								\

	 * spin lock/unlock provides the needed smp_mb() before/after	\

	 */								\

	atomic_ops_lock(flags);						\

	orig = v->counter;						\

	v->counter c_op i;						\

	atomic_ops_unlock(flags);					\

									\

	return orig;							\

}



#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_OP_RETURN(op, c_op, asm_op)				\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(add, +=, add)

ATOMIC_OPS(sub, -=, sub)



#undef ATOMIC_OPS

#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(and, &=, and)

ATOMIC_OPS(andnot, &= ~, bic)

ATOMIC_OPS(or, |=, or)

ATOMIC_OPS(xor, ^=, xor)



#define arch_atomic_andnot		arch_atomic_andnot

#define arch_atomic_fetch_andnot	arch_atomic_fetch_andnot



#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



#endif

arch/arc/include/asm/atomic.h

+26 −418
Original line number Original line Diff line number Diff line
@@ -17,435 +17,43 @@ 
#define arch_atomic_read(v)  READ_ONCE((v)->counter)

#define arch_atomic_read(v)  READ_ONCE((v)->counter)





#ifdef CONFIG_ARC_HAS_LLSC

#ifdef CONFIG_ARC_HAS_LLSC



#include <asm/atomic-llsc.h>

#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))



#define ATOMIC_OP(op, c_op, asm_op)					\

static inline void arch_atomic_##op(int i, atomic_t *v)			\

{									\

	unsigned int val;						\

									\

	__asm__ __volatile__(						\

	"1:	llock   %[val], [%[ctr]]		\n"		\

	"	" #asm_op " %[val], %[val], %[i]	\n"		\

	"	scond   %[val], [%[ctr]]		\n"		\

	"	bnz     1b				\n"		\

	: [val]	"=&r"	(val) /* Early clobber to prevent reg reuse */	\

	: [ctr]	"r"	(&v->counter), /* Not "m": llock only supports reg direct addr mode */	\

	  [i]	"ir"	(i)						\

	: "cc");							\

}									\



#define ATOMIC_OP_RETURN(op, c_op, asm_op)				\

static inline int arch_atomic_##op##_return(int i, atomic_t *v)		\

{									\

	unsigned int val;						\

									\

	/*								\

	 * Explicit full memory barrier needed before/after as		\

	 * LLOCK/SCOND themselves don't provide any such semantics	\

	 */								\

	smp_mb();							\

									\

	__asm__ __volatile__(						\

	"1:	llock   %[val], [%[ctr]]		\n"		\

	"	" #asm_op " %[val], %[val], %[i]	\n"		\

	"	scond   %[val], [%[ctr]]		\n"		\

	"	bnz     1b				\n"		\

	: [val]	"=&r"	(val)						\

	: [ctr]	"r"	(&v->counter),					\

	  [i]	"ir"	(i)						\

	: "cc");							\

									\

	smp_mb();							\

									\

	return val;							\

}



#define ATOMIC_FETCH_OP(op, c_op, asm_op)				\

static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\

{									\

	unsigned int val, orig;						\

									\

	/*								\

	 * Explicit full memory barrier needed before/after as		\

	 * LLOCK/SCOND themselves don't provide any such semantics	\

	 */								\

	smp_mb();							\

									\

	__asm__ __volatile__(						\

	"1:	llock   %[orig], [%[ctr]]		\n"		\

	"	" #asm_op " %[val], %[orig], %[i]	\n"		\

	"	scond   %[val], [%[ctr]]		\n"		\

	"	bnz     1b				\n"		\

	: [val]	"=&r"	(val),						\

	  [orig] "=&r" (orig)						\

	: [ctr]	"r"	(&v->counter),					\

	  [i]	"ir"	(i)						\

	: "cc");							\

									\

	smp_mb();							\

									\

	return orig;							\

}



#else	/* !CONFIG_ARC_HAS_LLSC */



#ifndef CONFIG_SMP



 /* violating atomic_xxx API locking protocol in UP for optimization sake */

#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))



#else

#else

#include <asm/atomic-spinlock.h>

#endif





static inline void arch_atomic_set(atomic_t *v, int i)

#define arch_atomic_cmpxchg(v, o, n)					\

{

({									\

	/*

	arch_cmpxchg(&((v)->counter), (o), (n));			\

	 * Independent of hardware support, all of the atomic_xxx() APIs need

})

	 * to follow the same locking rules to make sure that a "hardware"

	 * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn

	 * sequence

	 *

	 * Thus atomic_set() despite being 1 insn (and seemingly atomic)

	 * requires the locking.

	 */

	unsigned long flags;





	atomic_ops_lock(flags);

#ifdef arch_cmpxchg_relaxed

	WRITE_ONCE(v->counter, i);

#define arch_atomic_cmpxchg_relaxed(v, o, n)				\

	atomic_ops_unlock(flags);

({									\

}

	arch_cmpxchg_relaxed(&((v)->counter), (o), (n));		\

})

#endif





#define arch_atomic_set_release(v, i)	arch_atomic_set((v), (i))

#define arch_atomic_xchg(v, n)						\

({									\

	arch_xchg(&((v)->counter), (n));				\

})





#ifdef arch_xchg_relaxed

#define arch_atomic_xchg_relaxed(v, n)					\

({									\

	arch_xchg_relaxed(&((v)->counter), (n));			\

})

#endif

#endif





/*

/*

 * Non hardware assisted Atomic-R-M-W

 * 64-bit atomics

 * Locking would change to irq-disabling only (UP) and spinlocks (SMP)

 */

 */



#define ATOMIC_OP(op, c_op, asm_op)					\

static inline void arch_atomic_##op(int i, atomic_t *v)			\

{									\

	unsigned long flags;						\

									\

	atomic_ops_lock(flags);						\

	v->counter c_op i;						\

	atomic_ops_unlock(flags);					\

}



#define ATOMIC_OP_RETURN(op, c_op, asm_op)				\

static inline int arch_atomic_##op##_return(int i, atomic_t *v)		\

{									\

	unsigned long flags;						\

	unsigned long temp;						\

									\

	/*								\

	 * spin lock/unlock provides the needed smp_mb() before/after	\

	 */								\

	atomic_ops_lock(flags);						\

	temp = v->counter;						\

	temp c_op i;							\

	v->counter = temp;						\

	atomic_ops_unlock(flags);					\

									\

	return temp;							\

}



#define ATOMIC_FETCH_OP(op, c_op, asm_op)				\

static inline int arch_atomic_fetch_##op(int i, atomic_t *v)		\

{									\

	unsigned long flags;						\

	unsigned long orig;						\

									\

	/*								\

	 * spin lock/unlock provides the needed smp_mb() before/after	\

	 */								\

	atomic_ops_lock(flags);						\

	orig = v->counter;						\

	v->counter c_op i;						\

	atomic_ops_unlock(flags);					\

									\

	return orig;							\

}



#endif /* !CONFIG_ARC_HAS_LLSC */



#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_OP_RETURN(op, c_op, asm_op)				\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(add, +=, add)

ATOMIC_OPS(sub, -=, sub)



#undef ATOMIC_OPS

#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(and, &=, and)

ATOMIC_OPS(andnot, &= ~, bic)

ATOMIC_OPS(or, |=, or)

ATOMIC_OPS(xor, ^=, xor)



#define arch_atomic_andnot		arch_atomic_andnot

#define arch_atomic_fetch_andnot	arch_atomic_fetch_andnot



#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP



#ifdef CONFIG_GENERIC_ATOMIC64

#ifdef CONFIG_GENERIC_ATOMIC64



#include <asm-generic/atomic64.h>

#include <asm-generic/atomic64.h>



#else

#else	/* Kconfig ensures this is only enabled with needed h/w assist */

#include <asm/atomic64-arcv2.h>



#endif

/*

 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)

 *  - The address HAS to be 64-bit aligned

 *  - There are 2 semantics involved here:

 *    = exclusive implies no interim update between load/store to same addr

 *    = both words are observed/updated together: this is guaranteed even

 *      for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()

 *      is NOT required to use LLOCKD+SCONDD, STD suffices

 */



typedef struct {

	s64 __aligned(8) counter;

} atomic64_t;



#define ATOMIC64_INIT(a) { (a) }



static inline s64 arch_atomic64_read(const atomic64_t *v)

{

	s64 val;



	__asm__ __volatile__(

	"	ldd   %0, [%1]	\n"

	: "=r"(val)

	: "r"(&v->counter));



	return val;

}



static inline void arch_atomic64_set(atomic64_t *v, s64 a)

{

	/*

	 * This could have been a simple assignment in "C" but would need

	 * explicit volatile. Otherwise gcc optimizers could elide the store

	 * which borked atomic64 self-test

	 * In the inline asm version, memory clobber needed for exact same

	 * reason, to tell gcc about the store.

	 *

	 * This however is not needed for sibling atomic64_add() etc since both

	 * load/store are explicitly done in inline asm. As long as API is used

	 * for each access, gcc has no way to optimize away any load/store

	 */

	__asm__ __volatile__(

	"	std   %0, [%1]	\n"

	:

	: "r"(a), "r"(&v->counter)

	: "memory");

}



#define ATOMIC64_OP(op, op1, op2)					\

static inline void arch_atomic64_##op(s64 a, atomic64_t *v)		\

{									\

	s64 val;							\

									\

	__asm__ __volatile__(						\

	"1:				\n"				\

	"	llockd  %0, [%1]	\n"				\

	"	" #op1 " %L0, %L0, %L2	\n"				\

	"	" #op2 " %H0, %H0, %H2	\n"				\

	"	scondd   %0, [%1]	\n"				\

	"	bnz     1b		\n"				\

	: "=&r"(val)							\

	: "r"(&v->counter), "ir"(a)					\

	: "cc");							\

}									\



#define ATOMIC64_OP_RETURN(op, op1, op2)		        	\

static inline s64 arch_atomic64_##op##_return(s64 a, atomic64_t *v)	\

{									\

	s64 val;							\

									\

	smp_mb();							\

									\

	__asm__ __volatile__(						\

	"1:				\n"				\

	"	llockd   %0, [%1]	\n"				\

	"	" #op1 " %L0, %L0, %L2	\n"				\

	"	" #op2 " %H0, %H0, %H2	\n"				\

	"	scondd   %0, [%1]	\n"				\

	"	bnz     1b		\n"				\

	: [val] "=&r"(val)						\

	: "r"(&v->counter), "ir"(a)					\

	: "cc");	/* memory clobber comes from smp_mb() */	\

									\

	smp_mb();							\

									\

	return val;							\

}



#define ATOMIC64_FETCH_OP(op, op1, op2)		        		\

static inline s64 arch_atomic64_fetch_##op(s64 a, atomic64_t *v)	\

{									\

	s64 val, orig;							\

									\

	smp_mb();							\

									\

	__asm__ __volatile__(						\

	"1:				\n"				\

	"	llockd   %0, [%2]	\n"				\

	"	" #op1 " %L1, %L0, %L3	\n"				\

	"	" #op2 " %H1, %H0, %H3	\n"				\

	"	scondd   %1, [%2]	\n"				\

	"	bnz     1b		\n"				\

	: "=&r"(orig), "=&r"(val)					\

	: "r"(&v->counter), "ir"(a)					\

	: "cc");	/* memory clobber comes from smp_mb() */	\

									\

	smp_mb();							\

									\

	return orig;							\

}



#define ATOMIC64_OPS(op, op1, op2)					\

	ATOMIC64_OP(op, op1, op2)					\

	ATOMIC64_OP_RETURN(op, op1, op2)				\

	ATOMIC64_FETCH_OP(op, op1, op2)



ATOMIC64_OPS(add, add.f, adc)

ATOMIC64_OPS(sub, sub.f, sbc)

ATOMIC64_OPS(and, and, and)

ATOMIC64_OPS(andnot, bic, bic)

ATOMIC64_OPS(or, or, or)

ATOMIC64_OPS(xor, xor, xor)



#define arch_atomic64_andnot		arch_atomic64_andnot

#define arch_atomic64_fetch_andnot	arch_atomic64_fetch_andnot



#undef ATOMIC64_OPS

#undef ATOMIC64_FETCH_OP

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP



static inline s64

arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new)

{

	s64 prev;



	smp_mb();



	__asm__ __volatile__(

	"1:	llockd  %0, [%1]	\n"

	"	brne    %L0, %L2, 2f	\n"

	"	brne    %H0, %H2, 2f	\n"

	"	scondd  %3, [%1]	\n"

	"	bnz     1b		\n"

	"2:				\n"

	: "=&r"(prev)

	: "r"(ptr), "ir"(expected), "r"(new)

	: "cc");	/* memory clobber comes from smp_mb() */



	smp_mb();



	return prev;

}



static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)

{

	s64 prev;



	smp_mb();



	__asm__ __volatile__(

	"1:	llockd  %0, [%1]	\n"

	"	scondd  %2, [%1]	\n"

	"	bnz     1b		\n"

	"2:				\n"

	: "=&r"(prev)

	: "r"(ptr), "r"(new)

	: "cc");	/* memory clobber comes from smp_mb() */



	smp_mb();



	return prev;

}



/**

 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive

 * @v: pointer of type atomic64_t

 *

 * The function returns the old value of *v minus 1, even if

 * the atomic variable, v, was not decremented.

 */



static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)

{

	s64 val;



	smp_mb();



	__asm__ __volatile__(

	"1:	llockd  %0, [%1]	\n"

	"	sub.f   %L0, %L0, 1	# w0 - 1, set C on borrow\n"

	"	sub.c   %H0, %H0, 1	# if C set, w1 - 1\n"

	"	brlt    %H0, 0, 2f	\n"

	"	scondd  %0, [%1]	\n"

	"	bnz     1b		\n"

	"2:				\n"

	: "=&r"(val)

	: "r"(&v->counter)

	: "cc");	/* memory clobber comes from smp_mb() */



	smp_mb();



	return val;

}

#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive



/**

 * arch_atomic64_fetch_add_unless - add unless the number is a given value

 * @v: pointer of type atomic64_t

 * @a: the amount to add to v...

 * @u: ...unless v is equal to u.

 *

 * Atomically adds @a to @v, if it was not @u.

 * Returns the old value of @v

 */

static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)

{

	s64 old, temp;



	smp_mb();



	__asm__ __volatile__(

	"1:	llockd  %0, [%2]	\n"

	"	brne	%L0, %L4, 2f	# continue to add since v != u \n"

	"	breq.d	%H0, %H4, 3f	# return since v == u \n"

	"2:				\n"

	"	add.f   %L1, %L0, %L3	\n"

	"	adc     %H1, %H0, %H3	\n"

	"	scondd  %1, [%2]	\n"

	"	bnz     1b		\n"

	"3:				\n"

	: "=&r"(old), "=&r" (temp)

	: "r"(&v->counter), "r"(a), "r"(u)

	: "cc");	/* memory clobber comes from smp_mb() */



	smp_mb();



	return old;

}

#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless



#endif	/* !CONFIG_GENERIC_ATOMIC64 */





#endif	/* !__ASSEMBLY__ */

#endif	/* !__ASSEMBLY__ */

arch/arc/include/asm/atomic64-arcv2.h

0 → 100644
+250 −0

File added.

Preview size limit exceeded, changes collapsed.