00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018 #ifndef __SYS_ATOMICIMPL_H_INCLUDED__
00019 #define __SYS_ATOMICIMPL_H_INCLUDED__
00020
00021 #include "SYS_Types.h"
00022 #include "SYS_MemoryOrder.h"
00023 #ifdef WIN32
00024 #if defined(_MSC_VER) && _MSC_VER >= 1400
00025 #include <intrin.h>
00026 #else
00027 #error "This file will not build with MSVC2003 and older"
00028 #endif
00029 #define SYS_ATOMIC_INLINE SYS_FORCE_INLINE
00030 #else
00031 #define SYS_ATOMIC_INLINE inline
00032 #endif
00033 #ifdef SOLARIS
00034 #include <bits/atomicity.h>
00035 #endif
00036 #ifdef MBSD
00037 #include <libkern/OSAtomic.h>
00038 #endif
00039
00040
00041 namespace SYS_AtomicImpl
00042 {
00043
00044 template <typename T> T test_and_set(T *addr, T val) {}
00045 template <typename T> T test_and_add(T *addr, T val) {}
00046 template <typename T> T compare_and_swap(volatile T *addr, T oldval, T newval) {}
00047
00048
00049 #if defined(LINUX) || defined(MBSD_INTEL)
00050
00051 #if defined(LINUX)
00052
00053
00054
00055 #if !SYS_CHECK_GCC(4, 1)
00056 #error "Unsupported gcc version"
00057 #endif
00058
00059 template <>
00060 SYS_ATOMIC_INLINE int32
00061 test_and_set<int32>(int32 *addr, int32 val)
00062 {
00063 return __sync_lock_test_and_set(addr, val);
00064 }
00065
00066 template <>
00067 SYS_ATOMIC_INLINE int32
00068 test_and_add<int32>(int32 *addr, int32 val)
00069 {
00070 return __sync_fetch_and_add(addr, val);
00071 }
00072
00073
00074 template <>
00075 SYS_ATOMIC_INLINE int32
00076 compare_and_swap<int32>(volatile int32 *addr, int32 oldval, int32 newval)
00077 {
00078 return __sync_val_compare_and_swap(const_cast<int32 *>(addr),oldval,newval);
00079 }
00080
00081 #if defined(AMD64)
00082
00083 template <>
00084 SYS_ATOMIC_INLINE int64
00085 test_and_set<int64>(int64 *addr, int64 val)
00086 {
00087 return __sync_lock_test_and_set(addr, val);
00088 }
00089
00090 template <>
00091 SYS_ATOMIC_INLINE int64
00092 test_and_add<int64>(int64 *addr, int64 val)
00093 {
00094 return __sync_fetch_and_add(addr, val);
00095 }
00096
00097 template <>
00098 SYS_ATOMIC_INLINE int64
00099 compare_and_swap<int64>(volatile int64 *addr, int64 oldval, int64 newval)
00100 {
00101 return __sync_val_compare_and_swap(const_cast<int64 *>(addr),oldval,newval);
00102 }
00103
00104 #endif // AMD64
00105
00106 #else // LINUX
00107
00108
00109
00110
00111
00112 template <>
00113 SYS_ATOMIC_INLINE static int32
00114 test_and_set<int32>(int32 *addr, int32 val)
00115 {
00116 int32 oldval;
00117 __asm__ __volatile__("lock xchgl %0, %1"
00118 : "=r"(oldval), "=m"(*(addr))
00119 : "0"(val), "m"(*(addr)));
00120 return oldval;
00121 }
00122
00123 template <>
00124 SYS_ATOMIC_INLINE static int32
00125 test_and_add<int32>(int32 *addr, int32 val)
00126 {
00127 return __sync_fetch_and_add(addr, val);
00128 }
00129
00130 template <>
00131 SYS_ATOMIC_INLINE static int32
00132 compare_and_swap<int32>(volatile int32 *addr, int32 oldval, int32 newval)
00133 {
00134 return __sync_val_compare_and_swap(const_cast<int32 *>(addr),oldval,newval);
00135 }
00136
00137
00138 template <>
00139 SYS_ATOMIC_INLINE static int64
00140 test_and_set<int64>(int64 *addr, int64 val)
00141 {
00142 return __sync_lock_test_and_set(addr, val);
00143 }
00144
00145
00146 template <>
00147 SYS_ATOMIC_INLINE static int64
00148 test_and_add<int64>(int64 *addr, int64 val)
00149 {
00150 return __sync_fetch_and_add(addr, val);
00151 }
00152
00153
00154 template <>
00155 SYS_ATOMIC_INLINE static int64
00156 compare_and_swap<int64>(volatile int64 *addr, int64 oldval, int64 newval)
00157 {
00158 return __sync_val_compare_and_swap(const_cast<int64 *>(addr),oldval,newval);
00159 }
00160
00161 #endif // defined(LINUX) || defined(MBSD_INTEL)
00162
00163 template <typename T>
00164 SYS_ATOMIC_INLINE static void
00165 store(T *addr, T val, SYS_MemoryOrder order)
00166 {
00167 if (order == SYS_MEMORY_ORDER_RELAXED)
00168 {
00169
00170
00171 *static_cast<volatile T *>(addr) = val;
00172 }
00173 else if (order == SYS_MEMORY_ORDER_RELEASE)
00174 {
00175
00176
00177
00178 __sync_synchronize();
00179 *static_cast<volatile T *>(addr) = val;
00180 }
00181 else if (order == SYS_MEMORY_ORDER_SEQ_CST)
00182 {
00183 T dummy = 1;
00184
00185
00186
00187
00188 __sync_lock_release(&dummy);
00189
00190
00191
00192
00193
00194 (void)__sync_lock_test_and_set(addr, val);
00195 }
00196 else
00197 {
00198
00199 }
00200 }
00201
00202 template <typename T>
00203 SYS_ATOMIC_INLINE static T
00204 load(const T *addr, SYS_MemoryOrder order)
00205 {
00206 if (order == SYS_MEMORY_ORDER_RELAXED)
00207 {
00208 return *static_cast<const volatile T *>(addr);
00209 }
00210 else if (order == SYS_MEMORY_ORDER_CONSUME ||
00211 order == SYS_MEMORY_ORDER_ACQUIRE)
00212 {
00213 T val = *static_cast<const volatile T *>(addr);
00214
00215
00216
00217
00218 __sync_synchronize();
00219
00220 return val;
00221 }
00222 else if (order == SYS_MEMORY_ORDER_SEQ_CST)
00223 {
00224 T tmp = 0;
00225
00226 return __sync_val_compare_and_swap(const_cast<T *>(addr), tmp, tmp);
00227 }
00228 else
00229 {
00230
00231 return *addr;
00232 }
00233 }
00234
00235 #elif defined(WIN32)
00236
00237 #pragma intrinsic (_InterlockedExchange)
00238 #pragma intrinsic (_InterlockedExchangeAdd)
00239 #pragma intrinsic (_InterlockedCompareExchange)
00240
00241 template <>
00242 SYS_FORCE_INLINE static int32
00243 test_and_set<int32>(int32 *addr, int32 val)
00244 {
00245 return (int32)_InterlockedExchange((long *)addr, (long)val);
00246 }
00247
00248 template <>
00249 SYS_FORCE_INLINE static int32
00250 test_and_add<int32>(int32 *addr, int32 val)
00251 {
00252 return (int32)_InterlockedExchangeAdd((long *)addr, (long)val);
00253 }
00254
00255 template <>
00256 SYS_FORCE_INLINE static int32
00257 compare_and_swap<int32>(volatile int32 *addr, int32 oldval, int32 newval)
00258 {
00259 return _InterlockedCompareExchange((volatile long *)addr, newval, oldval);
00260 }
00261
00262 #if defined(AMD64)
00263
00264 #pragma intrinsic (_InterlockedExchange64)
00265 #pragma intrinsic (_InterlockedExchangeAdd64)
00266 #pragma intrinsic (_InterlockedCompareExchange64)
00267
00268 template <>
00269 SYS_FORCE_INLINE static int64
00270 test_and_set<int64>(int64 *addr, int64 val)
00271 {
00272 return _InterlockedExchange64(addr, val);
00273 }
00274
00275 template <>
00276 SYS_FORCE_INLINE static int64
00277 test_and_add<int64>(int64 *addr, int64 val)
00278 {
00279 return _InterlockedExchangeAdd64(addr, val);
00280 }
00281
00282 template <>
00283 SYS_FORCE_INLINE static int64
00284 compare_and_swap<int64>(volatile int64 *addr, int64 oldval, int64 newval)
00285 {
00286 return _InterlockedCompareExchange64(addr, newval, oldval);
00287 }
00288
00289 #endif // AMD64
00290
00291
00292
00293 #if defined(_MSC_VER) && _MSC_VER >= 1400
00294 #pragma intrinsic (_ReadBarrier)
00295 #pragma intrinsic (_WriteBarrier)
00296 #pragma intrinsic (_InterlockedCompareExchange)
00297
00298 template <typename T>
00299 SYS_FORCE_INLINE static void
00300 store(T *addr, T val, SYS_MemoryOrder order)
00301 {
00302
00303
00304
00305
00306
00307
00308
00309
00310
00311
00312
00313
00314 if (order == SYS_MEMORY_ORDER_RELAXED)
00315 {
00316
00317
00318
00319
00320
00321 *addr = val;
00322 _WriteBarrier();
00323 }
00324 else if (order == SYS_MEMORY_ORDER_RELEASE)
00325 {
00326 *static_cast<volatile T *>(addr) = val;
00327 }
00328 else if (order == SYS_MEMORY_ORDER_SEQ_CST)
00329 {
00330 (void)test_and_set(addr, val);
00331 }
00332 else
00333 {
00334
00335 }
00336 }
00337
00338 template <typename T>
00339 SYS_FORCE_INLINE static T
00340 load(const T *addr, SYS_MemoryOrder order)
00341 {
00342
00343
00344
00345
00346
00347
00348
00349
00350
00351
00352
00353
00354 if (order == SYS_MEMORY_ORDER_RELAXED)
00355 {
00356
00357
00358
00359
00360
00361 _ReadBarrier();
00362 return *addr;
00363 }
00364 else if (order == SYS_MEMORY_ORDER_CONSUME ||
00365 order == SYS_MEMORY_ORDER_ACQUIRE)
00366 {
00367 return *static_cast<const volatile T *>(addr);
00368 }
00369 else if (order == SYS_MEMORY_ORDER_SEQ_CST)
00370 {
00371 T tmp = 0;
00372 return compare_and_swap(const_cast<T *>(addr), tmp, tmp);
00373 }
00374 else
00375 {
00376
00377 return *addr;
00378 }
00379 }
00380 #else
00381 #error store() only implemented under WIN32/WIN64 for MSVC 2005 and up
00382 #error load() only implemented under WIN32/WIN64 for MSVC 2005 and up
00383 #endif // _MSC_VER
00384
00385 #else
00386
00387 #error "Unsupported platform"
00388 #endif
00389
00390 }
00391
00392 #endif // __SYS_ATOMICIMPL_H_INCLUDED__