Atomic.h

Go to the documentation of this file.

//============================================================================
//  Copyright (c) Kitware, Inc.
//  All rights reserved.
//  See LICENSE.txt for details.
//
//  This software is distributed WITHOUT ANY WARRANTY; without even
//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
//  PURPOSE.  See the above copyright notice for more information.
//============================================================================
#ifndef vtk_m_Atomic_h
#define vtk_m_Atomic_h
 
#include <vtkm/List.h>
 
#include <vtkm/internal/Windows.h>
 
#include <atomic>
 
namespace vtkm
{
 
enum class MemoryOrder
{
  Relaxed,
 
  Acquire,
 
  Release,
 
  AcquireAndRelease,
 
  SequentiallyConsistent
};
 
namespace internal
{
 
VTKM_EXEC_CONT inline std::memory_order StdAtomicMemOrder(vtkm::MemoryOrder order)
{
  switch (order)
  {
    case vtkm::MemoryOrder::Relaxed:
      return std::memory_order_relaxed;
    case vtkm::MemoryOrder::Acquire:
      return std::memory_order_acquire;
    case vtkm::MemoryOrder::Release:
      return std::memory_order_release;
    case vtkm::MemoryOrder::AcquireAndRelease:
      return std::memory_order_acq_rel;
    case vtkm::MemoryOrder::SequentiallyConsistent:
      return std::memory_order_seq_cst;
  }
 
  // Should never reach here, but avoid compiler warnings
  return std::memory_order_seq_cst;
}
 
} // namespace internal
 
} // namespace vtkm
 
 
#if defined(VTKM_CUDA_DEVICE_PASS)
 
namespace vtkm
{
namespace detail
{
 
// Fence to ensure that previous non-atomic stores are visible to other threads.
VTKM_EXEC_CONT inline void AtomicStoreFence(vtkm::MemoryOrder order)
{
  if ((order == vtkm::MemoryOrder::Release) || (order == vtkm::MemoryOrder::AcquireAndRelease) ||
      (order == vtkm::MemoryOrder::SequentiallyConsistent))
  {
    __threadfence();
  }
}
 
// Fence to ensure that previous non-atomic stores are visible to other threads.
VTKM_EXEC_CONT inline void AtomicLoadFence(vtkm::MemoryOrder order)
{
  if ((order == vtkm::MemoryOrder::Acquire) || (order == vtkm::MemoryOrder::AcquireAndRelease) ||
      (order == vtkm::MemoryOrder::SequentiallyConsistent))
  {
    __threadfence();
  }
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicLoadImpl(T* const addr, vtkm::MemoryOrder order)
{
  volatile T* const vaddr = addr; /* volatile to bypass cache*/
  if (order == vtkm::MemoryOrder::SequentiallyConsistent)
  {
    __threadfence();
  }
  const T value = *vaddr;
  /* fence to ensure that dependent reads are correctly ordered */
  AtomicLoadFence(order);
  return value;
}
 
template <typename T>
VTKM_EXEC_CONT inline void AtomicStoreImpl(T* addr, T value, vtkm::MemoryOrder order)
{
  volatile T* vaddr = addr; /* volatile to bypass cache */
  /* fence to ensure that previous non-atomic stores are visible to other threads */
  AtomicStoreFence(order);
  *vaddr = value;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicAddImpl(T* addr, T arg, vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  auto result = atomicAdd(addr, arg);
  AtomicLoadFence(order);
  return result;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicAndImpl(T* addr, T mask, vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  auto result = atomicAnd(addr, mask);
  AtomicLoadFence(order);
  return result;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicOrImpl(T* addr, T mask, vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  auto result = atomicOr(addr, mask);
  AtomicLoadFence(order);
  return result;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicXorImpl(T* addr, T mask, vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  auto result = atomicXor(addr, mask);
  AtomicLoadFence(order);
  return result;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicNotImpl(T* addr, vtkm::MemoryOrder order)
{
  return AtomicXorImpl(addr, static_cast<T>(~T{ 0u }), order);
}
 
template <typename T>
VTKM_EXEC_CONT inline bool AtomicCompareExchangeImpl(T* addr,
                                                     T* expected,
                                                     T desired,
                                                     vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  auto result = atomicCAS(addr, *expected, desired);
  AtomicLoadFence(order);
  if (result == *expected)
  {
    return true;
  }
  else
  {
    *expected = result;
    return false;
  }
}
#if __CUDA_ARCH__ < 200
VTKM_EXEC_CONT inline vtkm::Float32 AtomicAddImpl(vtkm::Float32* address,
                                                  vtkm::Float32 value,
                                                  vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  vtkm::UInt32 assumed;
  vtkm::UInt32 old = __float_as_int(*address);
  do
  {
    assumed = old;
    old = atomicCAS(reinterpret_cast<vtkm::UInt32*>(address),
                    assumed,
                    __float_as_int(__int_as_float(assumed) + value));
  } while (assumed != old);
  AtomicLoadFence(order);
  return __int_as_float(old);
}
#endif
#if __CUDA_ARCH__ < 600
VTKM_EXEC_CONT inline vtkm::Float64 AtomicAddImpl(vtkm::Float64* address,
                                                  vtkm::Float64 value,
                                                  vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  vtkm::UInt64 assumed;
  vtkm::UInt64 old = __double_as_longlong(*address);
  do
  {
    assumed = old;
    old = atomicCAS(reinterpret_cast<vtkm::UInt64*>(address),
                    assumed,
                    __double_as_longlong(__longlong_as_double(assumed) + value));
  } while (assumed != old);
  AtomicLoadFence(order);
  return __longlong_as_double(old);
}
#endif
}
} // namespace vtkm::detail
 
#elif defined(VTKM_ENABLE_KOKKOS)
 
VTKM_THIRDPARTY_PRE_INCLUDE
// Superhack! Kokkos_Macros.hpp defines macros to include modifiers like __device__.
// However, we don't want to actually use those if compiling this with a standard
// C++ compiler (because this particular code does not run on a device). Thus,
// we want to disable that behavior when not using the device compiler. To do that,
// we are going to have to load the KokkosCore_config.h file (which you are not
// supposed to do), then undefine the device enables if necessary, then load
// Kokkos_Macros.hpp to finish the state.
#ifndef KOKKOS_MACROS_HPP
#define KOKKOS_MACROS_HPP
#include <KokkosCore_config.h>
#undef KOKKOS_MACROS_HPP
#define KOKKOS_DONT_INCLUDE_CORE_CONFIG_H
 
#if defined(KOKKOS_ENABLE_CUDA) && !defined(VTKM_CUDA)
#undef KOKKOS_ENABLE_CUDA
 
// In later versions we need to directly deactivate Kokkos_Setup_Cuda.hpp
#if KOKKOS_VERSION >= 30401
#define KOKKOS_CUDA_SETUP_HPP_
#endif
#endif
 
#if defined(KOKKOS_ENABLE_HIP) && !defined(VTKM_HIP)
#undef KOKKOS_ENABLE_HIP
#endif
 
#endif //KOKKOS_MACROS_HPP not loaded
 
#include <Kokkos_Atomic.hpp>
VTKM_THIRDPARTY_POST_INCLUDE
 
namespace vtkm
{
namespace detail
{
 
// Fence to ensure that previous non-atomic stores are visible to other threads.
VTKM_EXEC_CONT inline void AtomicStoreFence(vtkm::MemoryOrder order)
{
  if ((order == vtkm::MemoryOrder::Release) || (order == vtkm::MemoryOrder::AcquireAndRelease) ||
      (order == vtkm::MemoryOrder::SequentiallyConsistent))
  {
    Kokkos::memory_fence();
  }
}
 
// Fence to ensure that previous non-atomic stores are visible to other threads.
VTKM_EXEC_CONT inline void AtomicLoadFence(vtkm::MemoryOrder order)
{
  if ((order == vtkm::MemoryOrder::Acquire) || (order == vtkm::MemoryOrder::AcquireAndRelease) ||
      (order == vtkm::MemoryOrder::SequentiallyConsistent))
  {
    Kokkos::memory_fence();
  }
}
#ifdef KOKKOS_INTERNAL_NOT_PARALLEL
#define VTKM_DESUL_MEM_SCOPE desul::MemoryScopeCaller()
#else
#define VTKM_DESUL_MEM_SCOPE desul::MemoryScopeDevice()
#endif
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicLoadImpl(T* const addr, vtkm::MemoryOrder order)
{
  switch (order)
  {
    case vtkm::MemoryOrder::Relaxed:
      return desul::atomic_load(addr, desul::MemoryOrderRelaxed(), VTKM_DESUL_MEM_SCOPE);
    case vtkm::MemoryOrder::Acquire:
    case vtkm::MemoryOrder::Release:           // Release doesn't make sense. Use Acquire.
    case vtkm::MemoryOrder::AcquireAndRelease: // Release doesn't make sense. Use Acquire.
      return desul::atomic_load(addr, desul::MemoryOrderAcquire(), VTKM_DESUL_MEM_SCOPE);
    case vtkm::MemoryOrder::SequentiallyConsistent:
      return desul::atomic_load(addr, desul::MemoryOrderSeqCst(), VTKM_DESUL_MEM_SCOPE);
  }
 
  // Should never reach here, but avoid compiler warnings
  return desul::atomic_load(addr, desul::MemoryOrderSeqCst(), VTKM_DESUL_MEM_SCOPE);
}
 
template <typename T>
VTKM_EXEC_CONT inline void AtomicStoreImpl(T* addr, T value, vtkm::MemoryOrder order)
{
  switch (order)
  {
    case vtkm::MemoryOrder::Relaxed:
      desul::atomic_store(addr, value, desul::MemoryOrderRelaxed(), VTKM_DESUL_MEM_SCOPE);
      break;
    case vtkm::MemoryOrder::Acquire: // Acquire doesn't make sense. Use Release.
    case vtkm::MemoryOrder::Release:
    case vtkm::MemoryOrder::AcquireAndRelease: // Acquire doesn't make sense. Use Release.
      desul::atomic_store(addr, value, desul::MemoryOrderRelease(), VTKM_DESUL_MEM_SCOPE);
      break;
    case vtkm::MemoryOrder::SequentiallyConsistent:
      desul::atomic_store(addr, value, desul::MemoryOrderSeqCst(), VTKM_DESUL_MEM_SCOPE);
      break;
  }
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicAddImpl(T* addr, T arg, vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  T result = Kokkos::atomic_fetch_add(addr, arg);
  AtomicLoadFence(order);
  return result;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicAndImpl(T* addr, T mask, vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  T result = Kokkos::atomic_fetch_and(addr, mask);
  AtomicLoadFence(order);
  return result;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicOrImpl(T* addr, T mask, vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  T result = Kokkos::atomic_fetch_or(addr, mask);
  AtomicLoadFence(order);
  return result;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicXorImpl(T* addr, T mask, vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  T result = Kokkos::atomic_fetch_xor(addr, mask);
  AtomicLoadFence(order);
  return result;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicNotImpl(T* addr, vtkm::MemoryOrder order)
{
  return AtomicXorImpl(addr, static_cast<T>(~T{ 0u }), order);
}
 
template <typename T>
VTKM_EXEC_CONT inline bool AtomicCompareExchangeImpl(T* addr,
                                                     T* expected,
                                                     T desired,
                                                     vtkm::MemoryOrder order)
{
  AtomicStoreFence(order);
  T oldValue = Kokkos::atomic_compare_exchange(addr, *expected, desired);
  AtomicLoadFence(order);
  if (oldValue == *expected)
  {
    return true;
  }
  else
  {
    *expected = oldValue;
    return false;
  }
}
}
} // namespace vtkm::detail
 
#elif defined(VTKM_MSVC)
 
// Supports vtkm::UInt8, vtkm::UInt16, vtkm::UInt32, vtkm::UInt64
 
#include <cstdint>
#include <cstring>
#include <intrin.h> // For MSVC atomics
 
namespace vtkm
{
namespace detail
{
 
template <typename To, typename From>
VTKM_EXEC_CONT inline To BitCast(const From& src)
{
  // The memcpy should be removed by the compiler when possible, but this
  // works around a host of issues with bitcasting using reinterpret_cast.
  VTKM_STATIC_ASSERT(sizeof(From) == sizeof(To));
  To dst;
  std::memcpy(&dst, &src, sizeof(From));
  return dst;
}
 
template <typename T>
VTKM_EXEC_CONT inline T BitCast(T&& src)
{
  return std::forward<T>(src);
}
 
// Note about Load and Store implementations:
//
// "Simple reads and writes to properly-aligned 32-bit variables are atomic
//  operations"
//
// "Simple reads and writes to properly aligned 64-bit variables are atomic on
// 64-bit Windows. Reads and writes to 64-bit values are not guaranteed to be
// atomic on 32-bit Windows."
//
// "Reads and writes to variables of other sizes [than 32 or 64 bits] are not
// guaranteed to be atomic on any platform."
//
// https://docs.microsoft.com/en-us/windows/desktop/sync/interlocked-variable-access
 
VTKM_EXEC_CONT inline vtkm::UInt8 AtomicLoadImpl(vtkm::UInt8* const addr, vtkm::MemoryOrder order)
{
  // This assumes that the memory interface is smart enough to load a 32-bit
  // word atomically and a properly aligned 8-bit word from it.
  // We could build address masks and do shifts to perform this manually if
  // this assumption is incorrect.
  auto result = *static_cast<volatile vtkm::UInt8* const>(addr);
  std::atomic_thread_fence(internal::StdAtomicMemOrder(order));
  return result;
}
VTKM_EXEC_CONT inline vtkm::UInt16 AtomicLoadImpl(vtkm::UInt16* const addr, vtkm::MemoryOrder order)
{
  // This assumes that the memory interface is smart enough to load a 32-bit
  // word atomically and a properly aligned 16-bit word from it.
  // We could build address masks and do shifts to perform this manually if
  // this assumption is incorrect.
  auto result = *static_cast<volatile vtkm::UInt16* const>(addr);
  std::atomic_thread_fence(internal::StdAtomicMemOrder(order));
  return result;
}
VTKM_EXEC_CONT inline vtkm::UInt32 AtomicLoadImpl(vtkm::UInt32* const addr, vtkm::MemoryOrder order)
{
  auto result = *static_cast<volatile vtkm::UInt32* const>(addr);
  std::atomic_thread_fence(internal::StdAtomicMemOrder(order));
  return result;
}
VTKM_EXEC_CONT inline vtkm::UInt64 AtomicLoadImpl(vtkm::UInt64* const addr, vtkm::MemoryOrder order)
{
  auto result = *static_cast<volatile vtkm::UInt64* const>(addr);
  std::atomic_thread_fence(internal::StdAtomicMemOrder(order));
  return result;
}
 
VTKM_EXEC_CONT inline void AtomicStoreImpl(vtkm::UInt8* addr,
                                           vtkm::UInt8 val,
                                           vtkm::MemoryOrder vtkmNotUsed(order))
{
  // There doesn't seem to be an atomic store instruction in the windows
  // API, so just exchange and discard the result.
  _InterlockedExchange8(reinterpret_cast<volatile CHAR*>(addr), BitCast<CHAR>(val));
}
VTKM_EXEC_CONT inline void AtomicStoreImpl(vtkm::UInt16* addr,
                                           vtkm::UInt16 val,
                                           vtkm::MemoryOrder vtkmNotUsed(order))
{
  // There doesn't seem to be an atomic store instruction in the windows
  // API, so just exchange and discard the result.
  _InterlockedExchange16(reinterpret_cast<volatile SHORT*>(addr), BitCast<SHORT>(val));
}
VTKM_EXEC_CONT inline void AtomicStoreImpl(vtkm::UInt32* addr,
                                           vtkm::UInt32 val,
                                           vtkm::MemoryOrder order)
{
  std::atomic_thread_fence(internal::StdAtomicMemOrder(order));
  *addr = val;
}
VTKM_EXEC_CONT inline void AtomicStoreImpl(vtkm::UInt64* addr,
                                           vtkm::UInt64 val,
                                           vtkm::MemoryOrder order)
{
  std::atomic_thread_fence(internal::StdAtomicMemOrder(order));
  *addr = val;
}
 
#define VTKM_ATOMIC_OP(vtkmName, winName, vtkmType, winType, suffix)                             \
  VTKM_EXEC_CONT inline vtkmType vtkmName(vtkmType* addr, vtkmType arg, vtkm::MemoryOrder order) \
  {                                                                                              \
    return BitCast<vtkmType>(                                                                    \
      winName##suffix(reinterpret_cast<volatile winType*>(addr), BitCast<winType>(arg)));        \
  }
 
#define VTKM_ATOMIC_OPS_FOR_TYPE(vtkmType, winType, suffix)                                     \
  VTKM_ATOMIC_OP(AtomicAddImpl, _InterlockedExchangeAdd, vtkmType, winType, suffix)             \
  VTKM_ATOMIC_OP(AtomicAndImpl, _InterlockedAnd, vtkmType, winType, suffix)                     \
  VTKM_ATOMIC_OP(AtomicOrImpl, _InterlockedOr, vtkmType, winType, suffix)                       \
  VTKM_ATOMIC_OP(AtomicXorImpl, _InterlockedXor, vtkmType, winType, suffix)                     \
  VTKM_EXEC_CONT inline vtkmType AtomicNotImpl(vtkmType* addr, vtkm::MemoryOrder order)         \
  {                                                                                             \
    return AtomicXorImpl(addr, static_cast<vtkmType>(~vtkmType{ 0u }), order);                  \
  }                                                                                             \
  VTKM_EXEC_CONT inline bool AtomicCompareExchangeImpl(                                         \
    vtkmType* addr, vtkmType* expected, vtkmType desired, vtkm::MemoryOrder vtkmNotUsed(order)) \
  {                                                                                             \
    vtkmType result = BitCast<vtkmType>(                                                        \
      _InterlockedCompareExchange##suffix(reinterpret_cast<volatile winType*>(addr),            \
                                          BitCast<winType>(desired),                            \
                                          BitCast<winType>(*expected)));                        \
    if (result == *expected)                                                                    \
    {                                                                                           \
      return true;                                                                              \
    }                                                                                           \
    else                                                                                        \
    {                                                                                           \
      *expected = result;                                                                       \
      return false;                                                                             \
    }                                                                                           \
  }
 
VTKM_ATOMIC_OPS_FOR_TYPE(vtkm::UInt8, CHAR, 8)
VTKM_ATOMIC_OPS_FOR_TYPE(vtkm::UInt16, SHORT, 16)
VTKM_ATOMIC_OPS_FOR_TYPE(vtkm::UInt32, LONG, )
VTKM_ATOMIC_OPS_FOR_TYPE(vtkm::UInt64, LONG64, 64)
 
#undef VTKM_ATOMIC_OPS_FOR_TYPE
 
VTKM_EXEC_CONT inline vtkm::Float32 AtomicAddImpl(vtkm::Float32* address,
                                                  vtkm::Float32 value,
                                                  vtkm::MemoryOrder vtkmNotUsed(order))
{
  LONG assumed;
  LONG old = BitCast<LONG>(*address);
  do
  {
    assumed = old;
    old = _InterlockedCompareExchange(reinterpret_cast<volatile LONG*>(address),
                                      BitCast<LONG>(BitCast<vtkm::Float32>(assumed) + value),
                                      assumed);
  } while (assumed != old);
  return BitCast<vtkm::Float32>(old);
}
 
VTKM_EXEC_CONT inline vtkm::Float64 AtomicAddImpl(vtkm::Float64* address,
                                                  vtkm::Float64 value,
                                                  vtkm::MemoryOrder vtkmNotUsed(order))
{
  LONG64 assumed;
  LONG64 old = BitCast<LONG64>(*address);
  do
  {
    assumed = old;
    old = _InterlockedCompareExchange64(reinterpret_cast<volatile LONG64*>(address),
                                        BitCast<LONG64>(BitCast<vtkm::Float64>(assumed) + value),
                                        assumed);
  } while (assumed != old);
  return BitCast<vtkm::Float64>(old);
}
 
}
} // namespace vtkm::detail
 
#else // gcc/clang for CPU
 
// Supports vtkm::UInt8, vtkm::UInt16, vtkm::UInt32, vtkm::UInt64
 
#include <cstdint>
#include <cstring>
 
namespace vtkm
{
namespace detail
{
 
VTKM_EXEC_CONT inline int GccAtomicMemOrder(vtkm::MemoryOrder order)
{
  switch (order)
  {
    case vtkm::MemoryOrder::Relaxed:
      return __ATOMIC_RELAXED;
    case vtkm::MemoryOrder::Acquire:
      return __ATOMIC_ACQUIRE;
    case vtkm::MemoryOrder::Release:
      return __ATOMIC_RELEASE;
    case vtkm::MemoryOrder::AcquireAndRelease:
      return __ATOMIC_ACQ_REL;
    case vtkm::MemoryOrder::SequentiallyConsistent:
      return __ATOMIC_SEQ_CST;
  }
 
  // Should never reach here, but avoid compiler warnings
  return __ATOMIC_SEQ_CST;
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicLoadImpl(T* const addr, vtkm::MemoryOrder order)
{
  return __atomic_load_n(addr, GccAtomicMemOrder(order));
}
 
template <typename T>
VTKM_EXEC_CONT inline void AtomicStoreImpl(T* addr, T value, vtkm::MemoryOrder order)
{
  return __atomic_store_n(addr, value, GccAtomicMemOrder(order));
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicAddImpl(T* addr, T arg, vtkm::MemoryOrder order)
{
  return __atomic_fetch_add(addr, arg, GccAtomicMemOrder(order));
}
 
#include <vtkmstd/bit_cast.h>
 
// TODO: Use enable_if to write one version for both Float32 and Float64.
VTKM_EXEC_CONT inline vtkm::Float32 AtomicAddImpl(vtkm::Float32* addr,
                                                  vtkm::Float32 arg,
                                                  vtkm::MemoryOrder order)
{
  vtkm::UInt32 expected = vtkmstd::bit_cast<vtkm::UInt32>(*addr);
  vtkm::UInt32 desired;
 
  do
  {
    desired = vtkmstd::bit_cast<vtkm::UInt32>(vtkmstd::bit_cast<vtkm::Float32>(expected) + arg);
  } while (
    !__atomic_compare_exchange_n(reinterpret_cast<vtkm::UInt32*>(addr),
                                 &expected, // reloads expected with *addr prior to the operation
                                 desired,
                                 false,
                                 GccAtomicMemOrder(order),
                                 GccAtomicMemOrder(order)));
  // return the "old" value that was in the memory.
  return vtkmstd::bit_cast<vtkm::Float32>(expected);
}
 
// TODO: Use enable_if to write one version for both Float32 and Float64.
VTKM_EXEC_CONT inline vtkm::Float64 AtomicAddImpl(vtkm::Float64* addr,
                                                  vtkm::Float64 arg,
                                                  vtkm::MemoryOrder order)
{
  vtkm::UInt64 expected = vtkmstd::bit_cast<vtkm::UInt64>(*addr);
  vtkm::UInt64 desired;
 
  do
  {
    desired = vtkmstd::bit_cast<vtkm::UInt64>(vtkmstd::bit_cast<vtkm::Float64>(expected) + arg);
  } while (
    !__atomic_compare_exchange_n(reinterpret_cast<vtkm::UInt64*>(addr),
                                 &expected, // reloads expected with *addr prior to the operation
                                 desired,
                                 false,
                                 GccAtomicMemOrder(order),
                                 GccAtomicMemOrder(order)));
  // return the "old" value that was in the memory.
  return vtkmstd::bit_cast<vtkm::Float64>(expected);
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicAndImpl(T* addr, T mask, vtkm::MemoryOrder order)
{
  return __atomic_fetch_and(addr, mask, GccAtomicMemOrder(order));
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicOrImpl(T* addr, T mask, vtkm::MemoryOrder order)
{
  return __atomic_fetch_or(addr, mask, GccAtomicMemOrder(order));
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicXorImpl(T* addr, T mask, vtkm::MemoryOrder order)
{
  return __atomic_fetch_xor(addr, mask, GccAtomicMemOrder(order));
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicNotImpl(T* addr, vtkm::MemoryOrder order)
{
  return AtomicXorImpl(addr, static_cast<T>(~T{ 0u }), order);
}
 
template <typename T>
VTKM_EXEC_CONT inline bool AtomicCompareExchangeImpl(T* addr,
                                                     T* expected,
                                                     T desired,
                                                     vtkm::MemoryOrder order)
{
  return __atomic_compare_exchange_n(
    addr, expected, desired, false, GccAtomicMemOrder(order), GccAtomicMemOrder(order));
}
}
} // namespace vtkm::detail
 
#endif // gcc/clang
 
namespace vtkm
{
 
namespace detail
{
 
template <typename T>
using OppositeSign = typename std::conditional<std::is_signed<T>::value,
                                               typename std::make_unsigned<T>::type,
                                               typename std::make_signed<T>::type>::type;
 
} // namespace detail
 
using AtomicTypePreferred = vtkm::UInt32;
 
using AtomicTypesSupported = vtkm::List<vtkm::UInt32, vtkm::UInt64>;
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicLoad(T* const pointer,
                                   vtkm::MemoryOrder order = vtkm::MemoryOrder::Acquire)
{
  return detail::AtomicLoadImpl(pointer, order);
}
 
template <typename T>
VTKM_EXEC_CONT inline void AtomicStore(T* pointer,
                                       T value,
                                       vtkm::MemoryOrder order = vtkm::MemoryOrder::Release)
{
  detail::AtomicStoreImpl(pointer, value, order);
}
template <typename T>
VTKM_EXEC_CONT inline void AtomicStore(T* pointer,
                                       detail::OppositeSign<T> value,
                                       vtkm::MemoryOrder order = vtkm::MemoryOrder::Release)
{
  detail::AtomicStoreImpl(pointer, static_cast<T>(value), order);
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicAdd(
  T* pointer,
  T operand,
  vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicAddImpl(pointer, operand, order);
}
template <typename T, typename std::enable_if<std::is_integral<T>::value>::type* = nullptr>
VTKM_EXEC_CONT inline T AtomicAdd(
  T* pointer,
  detail::OppositeSign<T> operand,
  vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicAddImpl(pointer, static_cast<T>(operand), order);
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicAnd(
  T* pointer,
  T operand,
  vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicAndImpl(pointer, operand, order);
}
template <typename T>
VTKM_EXEC_CONT inline T AtomicAnd(
  T* pointer,
  detail::OppositeSign<T> operand,
  vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicAndImpl(pointer, static_cast<T>(operand), order);
}
 
template <typename T>
VTKM_EXEC_CONT inline T
AtomicOr(T* pointer, T operand, vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicOrImpl(pointer, operand, order);
}
template <typename T>
VTKM_EXEC_CONT inline T AtomicOr(
  T* pointer,
  detail::OppositeSign<T> operand,
  vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicOrImpl(pointer, static_cast<T>(operand), order);
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicXor(
  T* pointer,
  T operand,
  vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicXorImpl(pointer, operand, order);
}
template <typename T>
VTKM_EXEC_CONT inline T AtomicXor(
  T* pointer,
  detail::OppositeSign<T> operand,
  vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicXorImpl(pointer, static_cast<T>(operand), order);
}
 
template <typename T>
VTKM_EXEC_CONT inline T AtomicNot(
  T* pointer,
  vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicNotImpl(pointer, order);
}
 
template <typename T>
VTKM_EXEC_CONT inline bool AtomicCompareExchange(
  T* shared,
  T* expected,
  T desired,
  vtkm::MemoryOrder order = vtkm::MemoryOrder::SequentiallyConsistent)
{
  return detail::AtomicCompareExchangeImpl(shared, expected, desired, order);
}
 
} // namespace vtkm
 
#endif //vtk_m_Atomic_h