libs/backends/wavesaudio/wavesapi/akupara/threading/atomic_ops.hpp

   1 /*
   2 *  Akupara/threading/atomic_ops.hpp
   3 *
   4 *
   5 *  Created by Udi Barzilai on 06/06.
   6 *  Copyright 2006 __MyCompanyName__. All rights reserved.
   7 *
   8 */
   9 #if !defined(_AKUPARA_THREADING_ATOMIC_OPS_HPP__INCLUDED_)
  10 #define _AKUPARA_THREADING_ATOMIC_OPS_HPP__INCLUDED_
  11
  12 #include "Akupara/basics.hpp"  // for EXPECT macro
  13 #include "Akupara/compiletime_functions.hpp" // for TR1 stuff, signed/unsigned stuff
  14
  15 namespace Akupara
  16 {
  17     namespace threading
  18     {
  19         namespace atomic
  20         {
  21             namespace machine
  22             {
  23                 // Machine capabilities
  24                 // The following templates are specialized by the machine-specific headers to indicate
  25                 // the capabilities of the machine being compiled for. A true 'value' member for a given
  26                 // byte count means that there is an implementation of the corresponding atomic operation.
  27                 //-------------------------------------
  28                 template<unsigned int _byte_count> struct implements_load          : public false_type {};  // simple assignment from memory (assumes naturally aligned address)
  29                 template<unsigned int _byte_count> struct implements_store         : public false_type {};  // simple assignment to memory (assumes naturally aligned address)
  30                 template<unsigned int _byte_count> struct implements_CAS           : public false_type {};  // compare_and_store()
  31                 template<unsigned int _byte_count> struct implements_LL_SC         : public false_type {};  // load_linked(), store_conditional()
  32                 template<unsigned int _byte_count> struct implements_add           : public false_type {};  // add(), subtract()
  33                 template<unsigned int _byte_count> struct implements_fetch_and_add : public false_type {};  // fetch_and_add(), fetch_and_subtract()
  34                 template<unsigned int _byte_count> struct implements_add_and_fetch : public false_type {};  // add_and_fetch(), subtract_and_fetch()
  35                 //-------------------------------------
  36
  37
  38                 //-------------------------------------
  39                 // functions in this namespace may or may not be implemented, for any integer types, as specified by the machine capabilities templates above
  40                 template<typename _integer_type> bool compare_and_store(volatile _integer_type * operand_address, const _integer_type & expected_value, const _integer_type & value_to_store);
  41
  42                 template<typename _integer_type> _integer_type load_linked(volatile _integer_type * operand_address);
  43                 template<typename _integer_type> bool store_conditional(volatile _integer_type * operand_address, const _integer_type & value_to_store);
  44
  45                 template<typename _integer_type> void add(volatile _integer_type * operand_address, const _integer_type & addend);
  46                 template<typename _integer_type> void subtract(volatile _integer_type * operand_address, const _integer_type & subtrahend);
  47
  48                 template<typename _integer_type> _integer_type fetch_and_add(volatile _integer_type * operand_address, const _integer_type & addend);
  49                 template<typename _integer_type> _integer_type fetch_and_subtract(volatile _integer_type * operand_address, const _integer_type & subtrahend);
  50
  51                 template<typename _integer_type> _integer_type add_and_fetch(volatile _integer_type * operand_address, const _integer_type & addend);
  52                 template<typename _integer_type> _integer_type subtract_and_fetch(volatile _integer_type * operand_address, const _integer_type & subtrahend);
  53
  54                 void memory_barrier_read();
  55                 void memory_barrier_write();
  56                 void memory_barrier_readwrite();
  57                 //-------------------------------------
  58
  59             } // namespace machine
  60         } // namespace atomic
  61     } // namespace threading
  62 } // namespace Akupara
  63
  64 // Include the machine-specific implementations; these only implement the templates above for some of the _signed_ integer types
  65 #if defined(__GNUC__) && defined(__POWERPC__)
  66 #include "atomic_ops_gcc_ppc.hpp"
  67 #endif // defined(__GNUC__) && defined(__POWERPC__)
  68
  69 #if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
  70 #include "atomic_ops_gcc_x86.hpp"
  71 #endif // defined(__GNUC__) && defined(__i386__)
  72
  73 #if defined(_MSC_VER) && defined(_M_IX86)
  74 #include "atomic_ops_msvc_x86.hpp"
  75 #endif // defined(_MSC_VER) && defined(_M_IX86)
  76
  77 #if defined(_MSC_VER) && defined(_M_X64)
  78 #include "atomic_ops_msvc_x86_64.hpp"
  79 #endif // defined(_MSC_VER) && defined(_M_X64)
  80
  81 namespace Akupara
  82 {
  83     namespace threading
  84     {
  85         namespace atomic
  86         {
  87
  88
  89             // Select the most convenient atomic integer type based on the machine's ability to load/store atomically
  90             // The definition below selects that largest atomically accessible integer up to the size of int
  91             //----------------------------------------------------------------------------------------
  92             namespace detail
  93             {
  94                 template<unsigned int _byte_count>
  95                 struct largest_atomic_byte_count_upto
  96                 {
  97                     static const unsigned int value =
  98                         machine::implements_load<_byte_count>::value && machine::implements_store<_byte_count>::value ?
  99 _byte_count :
 100                     largest_atomic_byte_count_upto<_byte_count/2>::value;
 101                 };
 102
 103                 template<>
 104                 struct largest_atomic_byte_count_upto<0> { static const unsigned int value = 0; };
 105
 106                 const unsigned int k_byte_count_best_atomic = largest_atomic_byte_count_upto<sizeof(int)>::value;
 107             }
 108             typedef   signed_integer_with_byte_count< detail::k_byte_count_best_atomic >::type signed_integer_type;
 109             typedef unsigned_integer_with_byte_count< detail::k_byte_count_best_atomic >::type unsigned_integer_type;
 110             typedef signed_integer_type integer_type;
 111             //----------------------------------------------------------------------------------------
 112
 113             //----------------------------------------------------------------------------------------
 114             // These need to be implemented by all machines
 115             using machine::memory_barrier_read;
 116             using machine::memory_barrier_write;
 117             using machine::memory_barrier_readwrite;
 118             //----------------------------------------------------------------------------------------
 119
 120             //----------------------------------------------------------------------------------------
 121             // These may or may not be implemented, but if they aren't, we can't help much
 122             using machine::load_linked;
 123             using machine::store_conditional;
 124             //----------------------------------------------------------------------------------------
 125
 126
 127             //----------------------------------------------------------------------------------------
 128             // CAS implementation
 129             namespace detail
 130             {
 131                 template<
 132                     typename _integer_type,
 133                     bool _implements_CAS   = machine::implements_CAS  <sizeof(_integer_type)>::value,
 134                     bool _implements_LL_SC = machine::implements_LL_SC<sizeof(_integer_type)>::value>
 135                 struct implementation_CAS
 136                 {
 137                     static const bool s_exists = false;
 138                 };
 139                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 140                 // specialization for native CAS support
 141                 template<typename _integer_type, bool _implements_LL_SC>
 142                 struct implementation_CAS<_integer_type, true, _implements_LL_SC>
 143                 {
 144                     static const bool s_exists = true;
 145                     static inline bool compare_and_store(volatile _integer_type * operand_address, const _integer_type & expected_value, const _integer_type & value_to_store)
 146                     {
 147                         return machine::compare_and_store(operand_address, expected_value, value_to_store);
 148                     }
 149                 };
 150                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 151                 // specialization for cases with no CAS but with LL/SC
 152                 template<typename _integer_type>
 153                 struct implementation_CAS<_integer_type, false, true>
 154                 {
 155                     static const bool s_exists = true;
 156                     static inline bool compare_and_store(volatile _integer_type * operand_address, const _integer_type & expected_value, const _integer_type & value_to_store)
 157                     {
 158                         while (machine::load_linked(operand_address) == expected_value)
 159                             if (AKUPARA_EXPECT_TRUE(machine::store_conditional(operand_address, value_to_store)))
 160                                 return true;
 161                         return false;
 162                     }
 163                 };
 164                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 165             } // namespace detail
 166             // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 167             template<typename _integer_type>
 168             inline bool compare_and_store(volatile _integer_type * operand_address, const _integer_type & expected_value, const _integer_type & value_to_store)
 169             {
 170                 // if your compiler can't find the function to call here then there is no implementation available for your machine
 171                 return detail::implementation_CAS<_integer_type>::compare_and_store(operand_address, expected_value, value_to_store);
 172             }
 173             //----------------------------------------------------------------------------------------
 174
 175
 176
 177
 178
 179             //----------------------------------------------------------------------------------------
 180             // fetch_and_add
 181             namespace detail
 182             {
 183                 template<
 184                     typename _integer_type,
 185                     bool _0 = machine::implements_fetch_and_add<sizeof(_integer_type)>::value,
 186                     bool _1 = machine::implements_add_and_fetch<sizeof(_integer_type)>::value,
 187                     bool _2 = machine::implements_LL_SC        <sizeof(_integer_type)>::value,
 188                     bool _3 = machine::implements_CAS          <sizeof(_integer_type)>::value>
 189                 struct implementation_FAA
 190                 {
 191                     static const bool s_exists = false;
 192                 };
 193                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 194                 // specialization for native support
 195                 template<typename _integer_type, bool _1, bool _2, bool _3>
 196                 struct implementation_FAA<_integer_type, true, _1, _2, _3>
 197                 {
 198                     static const bool s_exists = true;
 199                     static inline _integer_type fetch_and_add(volatile _integer_type * operand_address, const _integer_type & addend)
 200                     {
 201                         return machine::fetch_and_add(operand_address, addend);
 202                     }
 203                 };
 204                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 205                 // specialization using add_and_fetch
 206                 template<typename _integer_type, bool _2, bool _3>
 207                 struct implementation_FAA<_integer_type, false, true, _2, _3>
 208                 {
 209                     static const bool s_exists = true;
 210                     static inline _integer_type fetch_and_add(volatile _integer_type * operand_address, const _integer_type & addend)
 211                     {
 212                         return machine::add_and_fetch(operand_address, addend) - addend;
 213                     }
 214                 };
 215                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 216                 // specialization using LL/SC
 217                 template<typename _integer_type, bool _3>
 218                 struct implementation_FAA<_integer_type, false, false, true, _3>
 219                 {
 220                     static const bool s_exists = true;
 221                     static inline _integer_type fetch_and_add(volatile _integer_type * operand_address, const _integer_type & addend)
 222                     {
 223                         _integer_type old_value;
 224                         do
 225                         old_value  = machine::load_linked(operand_address);
 226                         while (!machine::store_conditional(operand_address, old_value+addend));
 227                         return old_value;
 228                     }
 229                 };
 230                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 231                 // specialization using CAS
 232                 template<typename _integer_type>
 233                 struct implementation_FAA<_integer_type, false, false, false, true>
 234                 {
 235                     static const bool s_exists = true;
 236                     static inline _integer_type fetch_and_add(volatile _integer_type * operand_address, const _integer_type & addend)
 237                     {
 238                         _integer_type old_value;
 239                         do
 240                         old_value  = *operand_address;
 241                         while (!machine::compare_and_store(operand_address, old_value, old_value+addend));
 242                         return old_value;
 243                     }
 244                 };
 245                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 246             } // namespace detail
 247             template<typename _integer_type>
 248             inline _integer_type fetch_and_add(volatile _integer_type * operand_address, const _integer_type & addend)
 249             {
 250                 // if your compiler can't find the function to call here then there is no implementation available for your machine
 251                 return detail::implementation_FAA<_integer_type>::fetch_and_add(operand_address, addend);
 252             }
 253             //----------------------------------------------------------------------------------------
 254
 255
 256
 257
 258             //----------------------------------------------------------------------------------------
 259             // add_and_fetch
 260             namespace detail
 261             {
 262                 template<
 263                     typename _integer_type,
 264                     bool _0 = machine::implements_add_and_fetch<sizeof(_integer_type)>::value,
 265                     bool _1 = machine::implements_fetch_and_add<sizeof(_integer_type)>::value,
 266                     bool _2 = machine::implements_LL_SC        <sizeof(_integer_type)>::value,
 267                     bool _3 = machine::implements_CAS          <sizeof(_integer_type)>::value>
 268                 struct implementation_AAF
 269                 {
 270                     static const bool s_exists = false;
 271                 };
 272                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 273                 // specialization for native support
 274                 template<typename _integer_type, bool _1, bool _2, bool _3>
 275                 struct implementation_AAF<_integer_type, true, _1, _2, _3>
 276                 {
 277                     static const bool s_exists = true;
 278                     static inline _integer_type add_and_fetch(volatile _integer_type * operand_address, const _integer_type & addend)
 279                     {
 280                         return machine::add_and_fetch(operand_address, addend);
 281                     }
 282                 };
 283                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 284                 // specialization using add_and_fetch
 285                 template<typename _integer_type, bool _2, bool _3>
 286                 struct implementation_AAF<_integer_type, false, true, _2, _3>
 287                 {
 288                     static const bool s_exists = true;
 289                     static inline _integer_type add_and_fetch(volatile _integer_type * operand_address, const _integer_type & addend)
 290                     {
 291                         return machine::fetch_and_add(operand_address, addend) + addend;
 292                     }
 293                 };
 294                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 295                 // specialization using LL/SC
 296                 template<typename _integer_type, bool _3>
 297                 struct implementation_AAF<_integer_type, false, false, true, _3>
 298                 {
 299                     static const bool s_exists = true;
 300                     static inline _integer_type add_and_fetch(volatile _integer_type * operand_address, const _integer_type & addend)
 301                     {
 302                         _integer_type new_value;
 303                         do
 304                         new_value  = machine::load_linked(operand_address)+addend;
 305                         while (!machine::store_conditional(operand_address, new_value));
 306                         return new_value;
 307                     }
 308                 };
 309                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 310                 // specialization using CAS
 311                 template<typename _integer_type>
 312                 struct implementation_AAF<_integer_type, false, false, false, true>
 313                 {
 314                     static const bool s_exists = true;
 315                     static inline _integer_type add_and_fetch(volatile _integer_type * operand_address, const _integer_type & addend)
 316                     {
 317                         _integer_type old_value, new_value;
 318                         do
 319                         old_value = *operand_address, new_value  = old_value + addend;
 320                         while (!machine::compare_and_store(operand_address, old_value, new_value));
 321                         return new_value;
 322                     }
 323                 };
 324                 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 325             } // namespace detail
 326             template<typename _integer_type>
 327             inline _integer_type add_and_fetch(volatile _integer_type * operand_address, const _integer_type & addend)
 328             {
 329                 // if your compiler can't find the function to call here then there is no implementation available for your machine
 330                 return detail::implementation_AAF<_integer_type>::add_and_fetch(operand_address, addend);
 331             }
 332             //----------------------------------------------------------------------------------------
 333
 334
 335
 336             //----------------------------------------------------------------------------------------
 337             // add
 338             template<typename _integer_type>
 339             inline void add(volatile _integer_type * operand_address, const _integer_type & addend)
 340             {
 341                 if (machine::implements_add<sizeof(_integer_type)>::value)
 342                     machine::add(operand_address, addend);
 343                 else if (machine::implements_fetch_and_add<sizeof(_integer_type)>::value)
 344                     machine::fetch_and_add(operand_address, addend);
 345                 else if (machine::implements_add_and_fetch<sizeof(_integer_type)>::value)
 346                     machine::add_and_fetch(operand_address, addend);
 347                 else
 348                     fetch_and_add(operand_address, addend); // this will simulate using CAS or LL/SC (or it will fail the compilation if neither is available)
 349             }
 350             //----------------------------------------------------------------------------------------
 351
 352
 353
 354             //----------------------------------------------------------------------------------------
 355             // TODO: this is where we add implementations for:
 356             // - functions not implemented by the machine
 357             // - functions that take unsigned types (routed to call the signed versions with appropriate conversions)
 358             // For now we add nothing, so developers will need to stick to what their machine can do, and use signed
 359             // integers only.
 360             using machine::subtract;
 361             using machine::subtract_and_fetch;
 362             using machine::fetch_and_subtract;
 363             //----------------------------------------------------------------------------------------
 364
 365
 366
 367             //---------------------------------------------------------------------
 368             template<class _base_type, unsigned int _bytes_per_cache_line=machine::k_bytes_per_cache_line>
 369             struct pad_to_cache_line : public _base_type
 370             {
 371             private:
 372                 typedef pad_to_cache_line this_type;
 373                 typedef _base_type base_type;
 374             public:
 375                 static const unsigned int s_bytes_per_cache_line = _bytes_per_cache_line;
 376             private:
 377                 int m_padding[(s_bytes_per_cache_line - sizeof(base_type))/sizeof(int)];
 378             public:
 379                 pad_to_cache_line() {}
 380                 template<typename _arg_type> pad_to_cache_line(_arg_type arg) : base_type(arg) {}
 381             };
 382             //---------------------------------------------------------------------
 383
 384         } // namespace atomic
 385     } // namespace threading
 386 } // namespace Akupara
 387
 388 #endif // _AKUPARA_THREADING_ATOMIC_OPS_HPP__INCLUDED_