path: root/sys/dev/pci/drm/include/linux/atomic.h

/* $OpenBSD: atomic.h,v 1.17 2022/01/14 06:53:14 jsg Exp $ */
/**
 * \file drm_atomic.h
 * Atomic operations used in the DRM which may or may not be provided by the OS.
 * 
 * \author Eric Anholt <anholt@FreeBSD.org>
 */

/*-
 * Copyright 2004 Eric Anholt
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef _DRM_LINUX_ATOMIC_H_
#define _DRM_LINUX_ATOMIC_H_

#include <sys/types.h>
#include <sys/mutex.h>
#include <machine/intr.h>
#include <linux/types.h>
#include <linux/compiler.h>	/* via x86/include/asm/atomic.h */

#define ATOMIC_INIT(x)		(x)

#define atomic_set(p, v)	WRITE_ONCE(*(p), (v))
#define atomic_read(p)		READ_ONCE(*(p))
#define atomic_inc(p)		__sync_fetch_and_add(p, 1)
#define atomic_dec(p)		__sync_fetch_and_sub(p, 1)
#define atomic_add(n, p)	__sync_fetch_and_add(p, n)
#define atomic_sub(n, p)	__sync_fetch_and_sub(p, n)
#define atomic_and(n, p)	__sync_fetch_and_and(p, n)
#define atomic_or(n, p)		atomic_setbits_int(p, n)
#define atomic_add_return(n, p) __sync_add_and_fetch(p, n)
#define atomic_sub_return(n, p) __sync_sub_and_fetch(p, n)
#define atomic_sub_and_test(n, p)	(atomic_sub_return(n, p) == 0)
#define atomic_inc_return(v)	atomic_add_return(1, (v))
#define atomic_dec_return(v)	atomic_sub_return(1, (v))
#define atomic_dec_and_test(v)	(atomic_dec_return(v) == 0)
#define atomic_inc_and_test(v)	(atomic_inc_return(v) == 0)
#define atomic_cmpxchg(p, o, n)	__sync_val_compare_and_swap(p, o, n)
#define cmpxchg(p, o, n)	__sync_val_compare_and_swap(p, o, n)
#define cmpxchg64(p, o, n)	__sync_val_compare_and_swap(p, o, n)
#define atomic_set_release(p, v)	atomic_set((p), (v))
#define atomic_andnot(bits, p)		atomic_clearbits_int(p,bits)
#define atomic_fetch_inc(p)		__sync_fetch_and_add(p, 1)
#define atomic_fetch_xor(n, p)		__sync_fetch_and_xor(p, n)

#define try_cmpxchg(p, op, n)						\
({									\
	__typeof(p) __op = (__typeof((p)))(op);				\
	__typeof(*(p)) __o = *__op;					\
	__typeof(*(p)) __p = __sync_val_compare_and_swap((p), (__o), (n)); \
	if (__p != __o)							\
		*__op = __p;						\
	(__p == __o);							\
})

static inline bool
atomic_try_cmpxchg(volatile int *p, int *op, int n)
{
	return try_cmpxchg(p, op, n);
}

static inline int
atomic_xchg(volatile int *v, int n)
{
	__sync_synchronize();
	return __sync_lock_test_and_set(v, n);
}

#define xchg(v, n)	__sync_lock_test_and_set(v, n)

static inline int
atomic_add_unless(volatile int *v, int n, int u)
{
	int o;

	do {
		o = *v;
		if (o == u)
			return 0;
	} while (__sync_val_compare_and_swap(v, o, o +n) != o);

	return 1;
}

#define atomic_inc_not_zero(v)	atomic_add_unless((v), 1, 0)

static inline int
atomic_dec_if_positive(volatile int *v)
{
	int r, o;

	do {
		o = *v;
		r = o - 1;
		if (r < 0)
			break;
	} while (__sync_val_compare_and_swap(v, o, r) != o);

	return r;
}

#define atomic_long_read(p)	READ_ONCE(*(p))

/* 32 bit powerpc lacks 64 bit atomics */
#if !defined(__powerpc__) || defined(__powerpc64__)

typedef int64_t atomic64_t;

#define ATOMIC64_INIT(x)	(x)

#define atomic64_set(p, v)	WRITE_ONCE(*(p), (v))
#define atomic64_read(p)	READ_ONCE(*(p))

static inline int64_t
atomic64_xchg(volatile int64_t *v, int64_t n)
{
	__sync_synchronize();
	return __sync_lock_test_and_set(v, n);
}

#define atomic64_add(n, p)	__sync_fetch_and_add_8(p, n)
#define atomic64_sub(n, p)	__sync_fetch_and_sub_8(p, n)
#define atomic64_inc(p)		__sync_fetch_and_add_8(p, 1)
#define atomic64_add_return(n, p) __sync_add_and_fetch_8(p, n)
#define atomic64_inc_return(p)	__sync_add_and_fetch_8(p, 1)

#else

extern struct mutex atomic64_mtx;

typedef struct {
	volatile int64_t val;
} atomic64_t;

#define ATOMIC64_INIT(x)	{ (x) }

static inline void
atomic64_set(atomic64_t *v, int64_t i)
{
	mtx_enter(&atomic64_mtx);
	v->val = i;
	mtx_leave(&atomic64_mtx);
}

static inline int64_t
atomic64_read(atomic64_t *v)
{
	int64_t val;

	mtx_enter(&atomic64_mtx);
	val = v->val;
	mtx_leave(&atomic64_mtx);

	return val;
}

static inline int64_t
atomic64_xchg(atomic64_t *v, int64_t n)
{
	int64_t val;

	mtx_enter(&atomic64_mtx);
	val = v->val;
	v->val = n;
	mtx_leave(&atomic64_mtx);

	return val;
}

static inline void
atomic64_add(int i, atomic64_t *v)
{
	mtx_enter(&atomic64_mtx);
	v->val += i;
	mtx_leave(&atomic64_mtx);
}

#define atomic64_inc(p)		atomic64_add(p, 1)

static inline int64_t
atomic64_add_return(int i, atomic64_t *v)
{
	int64_t val;

	mtx_enter(&atomic64_mtx);
	val = v->val + i;
	v->val = val;
	mtx_leave(&atomic64_mtx);

	return val;
}

#define atomic64_inc_return(p)		atomic64_add_return(p, 1)

static inline void
atomic64_sub(int i, atomic64_t *v)
{
	mtx_enter(&atomic64_mtx);
	v->val -= i;
	mtx_leave(&atomic64_mtx);
}
#endif

static inline int64_t
atomic64_cmpxchg(atomic64_t *p, int64_t o, int64_t n)
{
	return atomic_cmpxchg(p, o, n);
}

#ifdef __LP64__
typedef int64_t atomic_long_t;
#define atomic_long_set(p, v)		atomic64_set(p, v)
#define atomic_long_xchg(v, n)		atomic64_xchg(v, n)
#define atomic_long_cmpxchg(p, o, n)	atomic_cmpxchg(p, o, n)
#define atomic_long_add(i, v)		atomic64_add(i, v)
#define atomic_long_sub(i, v)		atomic64_sub(i, v)
#else
typedef int32_t atomic_long_t;
#define atomic_long_set(p, v)		atomic_set(p, v)
#define atomic_long_xchg(v, n)		atomic_xchg(v, n)
#define atomic_long_cmpxchg(p, o, n)	atomic_cmpxchg(p, o, n)
#define atomic_long_add(i, v)		atomic_add(i, v)
#define atomic_long_sub(i, v)		atomic_sub(i, v)
#endif

static inline atomic_t
test_and_set_bit(u_int b, volatile void *p)
{
	unsigned int m = 1 << (b & 0x1f);
	unsigned int prev = __sync_fetch_and_or((volatile u_int *)p + (b >> 5), m);
	return (prev & m) != 0;
}

static inline void
clear_bit(u_int b, volatile void *p)
{
	atomic_clearbits_int(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
}

static inline void
clear_bit_unlock(u_int b, volatile void *p)
{
	membar_enter();
	clear_bit(b, p);
}

static inline void
set_bit(u_int b, volatile void *p)
{
	atomic_setbits_int(((volatile u_int *)p) + (b >> 5), 1 << (b & 0x1f));
}

static inline void
__clear_bit(u_int b, volatile void *p)
{
	volatile u_int *ptr = (volatile u_int *)p;
	ptr[b >> 5] &= ~(1 << (b & 0x1f));
}

static inline void
__set_bit(u_int b, volatile void *p)
{
	volatile u_int *ptr = (volatile u_int *)p;
	ptr[b >> 5] |= (1 << (b & 0x1f));
}

static inline int
test_bit(u_int b, const volatile void *p)
{
	return !!(((volatile u_int *)p)[b >> 5] & (1 << (b & 0x1f)));
}

static inline int
__test_and_set_bit(u_int b, volatile void *p)
{
	unsigned int m = 1 << (b & 0x1f);
	volatile u_int *ptr = (volatile u_int *)p;
	unsigned int prev = ptr[b >> 5];
	ptr[b >> 5] |= m;
	
	return (prev & m) != 0;
}

static inline int
test_and_clear_bit(u_int b, volatile void *p)
{
	unsigned int m = 1 << (b & 0x1f);
	unsigned int prev = __sync_fetch_and_and((volatile u_int *)p + (b >> 5), ~m);
	return (prev & m) != 0;
}

static inline int
__test_and_clear_bit(u_int b, volatile void *p)
{
	volatile u_int *ptr = (volatile u_int *)p;
	int rv = !!(ptr[b >> 5] & (1 << (b & 0x1f)));
	ptr[b >> 5] &= ~(1 << (b & 0x1f));
	return rv;
}

static inline int
find_first_zero_bit(volatile void *p, int max)
{
	int b;
	volatile u_int *ptr = (volatile u_int *)p;

	for (b = 0; b < max; b += 32) {
		if (ptr[b >> 5] != ~0) {
			for (;;) {
				if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
					return b;
				b++;
			}
		}
	}
	return max;
}

static inline int
find_next_zero_bit(volatile void *p, int max, int b)
{
	volatile u_int *ptr = (volatile u_int *)p;

	for (; b < max; b += 32) {
		if (ptr[b >> 5] != ~0) {
			for (;;) {
				if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0)
					return b;
				b++;
			}
		}
	}
	return max;
}

static inline int
find_first_bit(volatile void *p, int max)
{
	int b;
	volatile u_int *ptr = (volatile u_int *)p;

	for (b = 0; b < max; b += 32) {
		if (ptr[b >> 5] != 0) {
			for (;;) {
				if (ptr[b >> 5] & (1 << (b & 0x1f)))
					return b;
				b++;
			}
		}
	}
	return max;
}

static inline int
find_next_bit(volatile void *p, int max, int b)
{
	volatile u_int *ptr = (volatile u_int *)p;

	for (; b < max; b+= 32) {
		if (ptr[b >> 5] != 0) {
			for (;;) {
				if (ptr[b >> 5] & (1 << (b & 0x1f)))
					return b;
				b++;
			}
		}
	}
	return max;
}

#define for_each_set_bit(b, p, max) \
	for ((b) = find_first_bit((p), (max));			\
	     (b) < (max);					\
	     (b) = find_next_bit((p), (max), (b) + 1))

#define for_each_clear_bit(b, p, max) \
	for ((b) = find_first_zero_bit((p), (max));		\
	     (b) < (max);					\
	     (b) = find_next_zero_bit((p), (max), (b) + 1))

#if defined(__i386__)
#define rmb()	__asm __volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
#define wmb()	__asm __volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
#define mb()	__asm __volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
#define smp_mb()	__asm __volatile("lock; addl $0,-4(%%esp)" : : : "memory", "cc")
#define smp_rmb()	__membar("")
#define smp_wmb()	__membar("")
#define __smp_store_mb(var, value)	do { (void)xchg(&var, value); } while (0)
#define smp_mb__after_atomic()	do { } while (0)
#define smp_mb__before_atomic()	do { } while (0)
#elif defined(__amd64__)
#define rmb()	__membar("lfence")
#define wmb()	__membar("sfence")
#define mb()	__membar("mfence")
#define smp_mb()	__asm __volatile("lock; addl $0,-4(%%rsp)" : : : "memory", "cc")
#define smp_rmb()	__membar("")
#define smp_wmb()	__membar("")
#define __smp_store_mb(var, value)	do { (void)xchg(&var, value); } while (0)
#define smp_mb__after_atomic()	do { } while (0)
#define smp_mb__before_atomic()	do { } while (0)
#elif defined(__aarch64__)
#define rmb()	__membar("dsb ld")
#define wmb()	__membar("dsb st")
#define mb()	__membar("dsb sy")
#elif defined(__arm__)
#define rmb()	__membar("dsb sy")
#define wmb()	__membar("dsb sy")
#define mb()	__membar("dsb sy")
#elif defined(__mips64__)
#define rmb()	mips_sync() 
#define wmb()	mips_sync()
#define mb()	mips_sync()
#elif defined(__powerpc64__)
#define rmb()	__membar("sync")
#define wmb()	__membar("sync")
#define mb()	__membar("sync")
#define smp_rmb()	__membar("lwsync")
#define smp_wmb()	__membar("lwsync")
#elif defined(__powerpc__)
#define rmb()	__membar("sync")
#define wmb()	__membar("sync")
#define mb()	__membar("sync")
#define smp_wmb()	__membar("eieio")
#elif defined(__riscv)
#define rmb()	__membar("fence ir,ir")
#define wmb()	__membar("fence ow,ow")
#define mb()	__membar("fence iorw,iorw")
#define smp_rmb()	__membar("fence r,r")
#define smp_wmb()	__membar("fence w,w")
#define smp_mb()	__membar("fence rw,rw")
#elif defined(__sparc64__)
#define rmb()	membar_sync()
#define wmb()	membar_sync()
#define mb()	membar_sync()
#endif

#ifndef smp_rmb
#define smp_rmb()	rmb()
#endif

#ifndef smp_wmb
#define smp_wmb()	wmb()
#endif

#ifndef mmiowb
#define mmiowb()	wmb()
#endif

#ifndef smp_mb__before_atomic
#define smp_mb__before_atomic()	mb()
#endif

#ifndef smp_mb__after_atomic
#define smp_mb__after_atomic()	mb()
#endif

#ifndef smp_store_mb
#define smp_store_mb(x, v)	do { x = v; mb(); } while (0)
#endif

#endif