SYS/SYS_AtomicImpl.h

Go to the documentation of this file.

/*
 * PROPRIETARY INFORMATION.  This software is proprietary to
 * Side Effects Software Inc., and is not to be reproduced,
 * transmitted, or disclosed in any way without written permission.
 *
 * Produced by:
 *      Side Effects Software Inc
 *      123 Front Street West, Suite 1401
 *      Toronto, Ontario
 *      Canada   M5J 2M2
 *      416-504-9876
 *
 * NAME:        SYS_AtomicImpl.h (SYS Library, C++)
 *
 * COMMENTS:    Platform-specific atomic operations implementation.
 */

#ifndef __SYS_ATOMICIMPL_H_INCLUDED__
#define __SYS_ATOMICIMPL_H_INCLUDED__

#include "SYS_Types.h"
#include "SYS_MemoryOrder.h"
#ifdef WIN32
    #if defined(_MSC_VER) && _MSC_VER >= 1400
        #include <intrin.h>
    #else
        #error "This file will not build with MSVC2003 and older"
    #endif
    #define SYS_ATOMIC_INLINE   SYS_FORCE_INLINE
#else
    #define SYS_ATOMIC_INLINE   inline
#endif
#ifdef SOLARIS
#include <bits/atomicity.h> // for __test_and_set
#endif
#ifdef MBSD     
#include <libkern/OSAtomic.h>
#endif


namespace SYS_AtomicImpl
{

template <typename T> T test_and_set(T *addr, T val) {}
template <typename T> T test_and_add(T *addr, T val) {}
template <typename T> T compare_and_swap(volatile T *addr, T oldval, T newval) {}

// Group these together because the load/store implementation is the same
#if defined(LINUX) || defined(MBSD_INTEL)

#if defined(LINUX)

// GCC 4.0 doesn't support __sync_lock_test_and_set,
// but GCC 4.1 and onwards do
#if !SYS_CHECK_GCC(4, 1)
#error "Unsupported gcc version"
#endif

template <>
SYS_ATOMIC_INLINE int32
test_and_set<int32>(int32 *addr, int32 val)
{
    return __sync_lock_test_and_set(addr, val);
}

template <>
SYS_ATOMIC_INLINE int32
test_and_add<int32>(int32 *addr, int32 val)
{
    return __sync_fetch_and_add(addr, val);
}

// if (*addr = oldval) *addr = newval
template <>
SYS_ATOMIC_INLINE int32
compare_and_swap<int32>(volatile int32 *addr, int32 oldval, int32 newval)
{
    return __sync_val_compare_and_swap(const_cast<int32 *>(addr),oldval,newval);
}

#if defined(AMD64)

template <>
SYS_ATOMIC_INLINE int64
test_and_set<int64>(int64 *addr, int64 val)
{
    return __sync_lock_test_and_set(addr, val);
}

template <>
SYS_ATOMIC_INLINE int64
test_and_add<int64>(int64 *addr, int64 val)
{
    return __sync_fetch_and_add(addr, val);
}

template <>
SYS_ATOMIC_INLINE int64
compare_and_swap<int64>(volatile int64 *addr, int64 oldval, int64 newval)
{
    return __sync_val_compare_and_swap(const_cast<int64 *>(addr),oldval,newval);
}

#endif // AMD64

#else // LINUX

//
// Code for MBSD_INTEL
//

template <>
SYS_ATOMIC_INLINE static int32
test_and_set<int32>(int32 *addr, int32 val)
{
    int32       oldval;
    __asm__ __volatile__("lock xchgl %0, %1"
            : "=r"(oldval), "=m"(*(addr))
            : "0"(val), "m"(*(addr)));
    return oldval;
}

template <>
SYS_ATOMIC_INLINE static int32
test_and_add<int32>(int32 *addr, int32 val)
{
    return __sync_fetch_and_add(addr, val);
}

template <>
SYS_ATOMIC_INLINE static int32
compare_and_swap<int32>(volatile int32 *addr, int32 oldval, int32 newval)
{
    return __sync_val_compare_and_swap(const_cast<int32 *>(addr),oldval,newval);
}

// NOTE: MBSD_INTEL implies AMD64
template <>
SYS_ATOMIC_INLINE static int64
test_and_set<int64>(int64 *addr, int64 val)
{
    return __sync_lock_test_and_set(addr, val);
}

// NOTE: MBSD_INTEL implies AMD64
template <>
SYS_ATOMIC_INLINE static int64
test_and_add<int64>(int64 *addr, int64 val)
{
    return __sync_fetch_and_add(addr, val);
}

// NOTE: MBSD_INTEL implies AMD64
template <>
SYS_ATOMIC_INLINE static int64
compare_and_swap<int64>(volatile int64 *addr, int64 oldval, int64 newval)
{
    return __sync_val_compare_and_swap(const_cast<int64 *>(addr),oldval,newval);
}

#endif // defined(LINUX) || defined(MBSD_INTEL)

template <typename T>
SYS_ATOMIC_INLINE static void
store(T *addr, T val, SYS_MemoryOrder order)
{
    if (order == SYS_MEMORY_ORDER_RELAXED)
    {
        // Use volatile to force the compiler to issue a store instruction, but
        // we don't care what the CPU does.
        *static_cast<volatile T *>(addr) = val;
    }
    else if (order == SYS_MEMORY_ORDER_RELEASE)
    {
        // We use a full memory barrier here, as a release barrier (via a
        // __sync_lock_release(&dummy) call) provides no guarantee about
        // the subsequent store crossing it.
        __sync_synchronize();   // full memory barrier
        *static_cast<volatile T *>(addr) = val;
    }
    else if (order == SYS_MEMORY_ORDER_SEQ_CST)
    {
        T       dummy = 1;

        // __sync_lock_release() is a release barrier, ensuring all previous
        // memory stores are globally visible, and all previous memory loads
        // have been satisfied.
        __sync_lock_release(&dummy);                    // release barrier

        // __sync_lock_test_and_set is an acquire barrier, preventing any
        // subsequent memory references from moving before this operation.
        // Consequently, this store will be globally visible before any
        // subsequent stores or loads are processed.
        (void)__sync_lock_test_and_set(addr, val);      // acquire barrier
    }
    else
    {
        // Invalid order!
    }
}

template <typename T>
SYS_ATOMIC_INLINE static T
load(const T *addr, SYS_MemoryOrder order)
{
    if (order == SYS_MEMORY_ORDER_RELAXED)
    {
        return *static_cast<const volatile T *>(addr);
    }
    else if (order == SYS_MEMORY_ORDER_CONSUME ||
             order == SYS_MEMORY_ORDER_ACQUIRE)
    {
        T       val = *static_cast<const volatile T *>(addr);

        // We use a full memory barrier here, as an acquire barrier (via a
        // __sync_lock_test_and_set(&dummy, 1) call) provides no guarantee
        // about the preceding load crossing it.
        __sync_synchronize();   // full memory barrier

        return val;
    }
    else if (order == SYS_MEMORY_ORDER_SEQ_CST)
    {
        T       tmp = 0;

        return __sync_val_compare_and_swap(const_cast<T *>(addr), tmp, tmp);
    }
    else
    {
        // Invalid order!
        return *addr;
    }
}

#elif defined(WIN32)

#pragma intrinsic (_InterlockedExchange)
#pragma intrinsic (_InterlockedExchangeAdd)
#pragma intrinsic (_InterlockedCompareExchange)

template <>
SYS_FORCE_INLINE static int32
test_and_set<int32>(int32 *addr, int32 val)
{
    return (int32)_InterlockedExchange((long *)addr, (long)val);
}

template <>
SYS_FORCE_INLINE static int32
test_and_add<int32>(int32 *addr, int32 val)
{
    return (int32)_InterlockedExchangeAdd((long *)addr, (long)val);
}

template <>
SYS_FORCE_INLINE static int32
compare_and_swap<int32>(volatile int32 *addr, int32 oldval, int32 newval)
{
    return _InterlockedCompareExchange((volatile long *)addr, newval, oldval);
}

#if defined(AMD64)

#pragma intrinsic (_InterlockedExchange64)
#pragma intrinsic (_InterlockedExchangeAdd64)
#pragma intrinsic (_InterlockedCompareExchange64)

template <>
SYS_FORCE_INLINE static int64
test_and_set<int64>(int64 *addr, int64 val)
{
    return _InterlockedExchange64(addr, val);
}

template <>
SYS_FORCE_INLINE static int64
test_and_add<int64>(int64 *addr, int64 val)
{
    return _InterlockedExchangeAdd64(addr, val);
}

template <>
SYS_FORCE_INLINE static int64
compare_and_swap<int64>(volatile int64 *addr, int64 oldval, int64 newval)
{
    return _InterlockedCompareExchange64(addr, newval, oldval);
}

#endif // AMD64

// The following implementations of store() and load() are valid only for
// MS Visual C++ 2005 and higher.
#if defined(_MSC_VER) && _MSC_VER >= 1400
#pragma intrinsic (_ReadBarrier)
#pragma intrinsic (_WriteBarrier)
#pragma intrinsic (_InterlockedCompareExchange)

template <typename T>
SYS_FORCE_INLINE static void
store(T *addr, T val, SYS_MemoryOrder order)
{
    // In Visual C++ 2005 and up, reads from volatile variables are defined to
    // have read-acquire semantics, and writes to volatile variables are
    // defined to have write-release semantics.  The compiler will not move
    // any reads and writes past them, and on Windows will ensure that the CPU
    // does not do so either.  Thus there is no need for explicit calls to
    // _ReadWriteBarrier().
    //
    // NOTE:
    //   Visual Studio 2005 had a bug (subsequently fixed) on IA64 where the
    //   compiler did not respect the acquire/release semantics for volatile
    //   floats.
    //   http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=288218
    if (order == SYS_MEMORY_ORDER_RELAXED)
    {
        // We want to force the compiler to respect this write.  One way would
        // be to treat addr as volatile, but, as already explained, that would
        // enforce release semantics on both the compiler and CPU. It should
        // theoretically be more efficient to simply prevent the compiler from
        // allowing optimization using any following writes.
        *addr = val;
        _WriteBarrier();
    }
    else if (order == SYS_MEMORY_ORDER_RELEASE)
    {
        *static_cast<volatile T *>(addr) = val;
    }
    else if (order == SYS_MEMORY_ORDER_SEQ_CST)
    {
        (void)test_and_set(addr, val);
    }
    else
    {
        // Invalid order!
    }
}

template <typename T>
SYS_FORCE_INLINE static T
load(const T *addr, SYS_MemoryOrder order)
{
    // In Visual C++ 2005 and up, reads from volatile variables are defined to
    // have read-acquire semantics, and writes to volatile variables are
    // defined to have write-release semantics.  The compiler will not move
    // any reads and writes past them, and on Windows will ensure that the CPU
    // does not do so either.  Thus there is no need for explicit calls to
    // _ReadWriteBarrier().
    //
    // NOTE:
    //   Visual Studio 2005 had a bug (subsequently fixed) on IA64 where the
    //   compiler did not respect the acquire/release semantics for volatile
    //   floats.
    //   http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=288218
    if (order == SYS_MEMORY_ORDER_RELAXED)
    {
        // We want to force the compiler to respect this read.  One way would
        // be to treat addr as volatile, but, as already explained, that would
        // enforce acquire semantics on both the compiler and CPU. It should
        // theoretically be more efficient to simply prevent the compiler from
        // equating earlier reads with this one.
        _ReadBarrier();
        return *addr;
    }
    else if (order == SYS_MEMORY_ORDER_CONSUME ||
             order == SYS_MEMORY_ORDER_ACQUIRE)
    {
        return *static_cast<const volatile T *>(addr);
    }
    else if (order == SYS_MEMORY_ORDER_SEQ_CST)
    {
        T       tmp = 0;
        return compare_and_swap(const_cast<T *>(addr), tmp, tmp);
    }
    else
    {
        // Invalid order!
        return *addr;
    }
}
#else
#error store() only implemented under WIN32/WIN64 for MSVC 2005 and up
#error load() only implemented under WIN32/WIN64 for MSVC 2005 and up
#endif // _MSC_VER

#else

#error "Unsupported platform"
#endif

}

#endif // __SYS_ATOMICIMPL_H_INCLUDED__

---