/* Copyright (c) 2005-2017 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #ifndef __TBB_enumerable_thread_specific_H #define __TBB_enumerable_thread_specific_H #include "atomic.h" #include "concurrent_vector.h" #include "tbb_thread.h" #include "tbb_allocator.h" #include "cache_aligned_allocator.h" #include "aligned_space.h" #include "internal/_template_helpers.h" #include "internal/_tbb_hash_compare_impl.h" #include "tbb_profiling.h" #include // for memcpy #if _WIN32||_WIN64 #include "machine/windows_api.h" #else #include #endif #define __TBB_ETS_USE_CPP11 \ (__TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ && __TBB_CPP11_DECLTYPE_PRESENT && __TBB_CPP11_LAMBDAS_PRESENT) namespace tbb { //! enum for selecting between single key and key-per-instance versions enum ets_key_usage_type { ets_key_per_instance, ets_no_key }; namespace interface6 { // Forward declaration to use in internal classes template class enumerable_thread_specific; //! @cond namespace internal { using namespace tbb::internal; template class ets_base: tbb::internal::no_copy { protected: typedef tbb_thread::id key_type; #if __TBB_PROTECTED_NESTED_CLASS_BROKEN public: #endif struct slot; struct array { array* next; size_t lg_size; slot& at( size_t k ) { return ((slot*)(void*)(this+1))[k]; } size_t size() const {return size_t(1)<>(8*sizeof(size_t)-lg_size); } }; struct slot { key_type key; void* ptr; bool empty() const {return key == key_type();} bool match( key_type k ) const {return key == k;} bool claim( key_type k ) { // TODO: maybe claim ptr, because key_type is not guaranteed to fit into word size return atomic_compare_and_swap(key, k, key_type()) == key_type(); } }; #if __TBB_PROTECTED_NESTED_CLASS_BROKEN protected: #endif //! Root of linked list of arrays of decreasing size. /** NULL if and only if my_count==0. Each array in the list is half the size of its predecessor. */ atomic my_root; atomic my_count; virtual void* create_local() = 0; virtual void* create_array(size_t _size) = 0; // _size in bytes virtual void free_array(void* ptr, size_t _size) = 0; // _size in bytes array* allocate( size_t lg_size ) { size_t n = size_t(1)<(create_array( sizeof(array)+n*sizeof(slot) )); a->lg_size = lg_size; std::memset( a+1, 0, n*sizeof(slot) ); return a; } void free(array* a) { size_t n = size_t(1)<<(a->lg_size); free_array( (void *)a, size_t(sizeof(array)+n*sizeof(slot)) ); } ets_base() {my_root=NULL; my_count=0;} virtual ~ets_base(); // g++ complains if this is not virtual void* table_lookup( bool& exists ); void table_clear(); // The following functions are not used in concurrent context, // so we don't need synchronization and ITT annotations there. void table_elementwise_copy( const ets_base& other, void*(*add_element)(ets_base&, void*) ) { __TBB_ASSERT(!my_root,NULL); __TBB_ASSERT(!my_count,NULL); if( !other.my_root ) return; array* root = my_root = allocate(other.my_root->lg_size); root->next = NULL; my_count = other.my_count; size_t mask = root->mask(); for( array* r=other.my_root; r; r=r->next ) { for( size_t i=0; isize(); ++i ) { slot& s1 = r->at(i); if( !s1.empty() ) { for( size_t j = root->start(tbb::tbb_hash()(s1.key)); ; j=(j+1)&mask ) { slot& s2 = root->at(j); if( s2.empty() ) { s2.ptr = add_element(*this, s1.ptr); s2.key = s1.key; break; } else if( s2.match(s1.key) ) break; } } } } } void table_swap( ets_base& other ) { __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); tbb::internal::swap(my_root, other.my_root); tbb::internal::swap(my_count, other.my_count); } }; template ets_base::~ets_base() { __TBB_ASSERT(!my_root, NULL); } template void ets_base::table_clear() { while( array* r = my_root ) { my_root = r->next; free(r); } my_count = 0; } template void* ets_base::table_lookup( bool& exists ) { const key_type k = tbb::this_tbb_thread::get_id(); __TBB_ASSERT(k != key_type(),NULL); void* found; size_t h = tbb::tbb_hash()(k); for( array* r=my_root; r; r=r->next ) { call_itt_notify(acquired,r); size_t mask=r->mask(); for(size_t i = r->start(h); ;i=(i+1)&mask) { slot& s = r->at(i); if( s.empty() ) break; if( s.match(k) ) { if( r==my_root ) { // Success at top level exists = true; return s.ptr; } else { // Success at some other level. Need to insert at top level. exists = true; found = s.ptr; goto insert; } } } } // Key does not yet exist. The density of slots in the table does not exceed 0.5, // for if this will occur a new table is allocated with double the current table // size, which is swapped in as the new root table. So an empty slot is guaranteed. exists = false; found = create_local(); { size_t c = ++my_count; array* r = my_root; call_itt_notify(acquired,r); if( !r || c>r->size()/2 ) { size_t s = r ? r->lg_size : 2; while( c>size_t(1)<<(s-1) ) ++s; array* a = allocate(s); for(;;) { a->next = r; call_itt_notify(releasing,a); array* new_r = my_root.compare_and_swap(a,r); if( new_r==r ) break; call_itt_notify(acquired, new_r); if( new_r->lg_size>=s ) { // Another thread inserted an equal or bigger array, so our array is superfluous. free(a); break; } r = new_r; } } } insert: // Whether a slot has been found in an older table, or if it has been inserted at this level, // it has already been accounted for in the total. Guaranteed to be room for it, and it is // not present, so search for empty slot and use it. array* ir = my_root; call_itt_notify(acquired, ir); size_t mask = ir->mask(); for(size_t i = ir->start(h);;i=(i+1)&mask) { slot& s = ir->at(i); if( s.empty() ) { if( s.claim(k) ) { s.ptr = found; return found; } } } } //! Specialization that exploits native TLS template <> class ets_base: protected ets_base { typedef ets_base super; #if _WIN32||_WIN64 #if __TBB_WIN8UI_SUPPORT typedef DWORD tls_key_t; void create_key() { my_key = FlsAlloc(NULL); } void destroy_key() { FlsFree(my_key); } void set_tls(void * value) { FlsSetValue(my_key, (LPVOID)value); } void* get_tls() { return (void *)FlsGetValue(my_key); } #else typedef DWORD tls_key_t; void create_key() { my_key = TlsAlloc(); } void destroy_key() { TlsFree(my_key); } void set_tls(void * value) { TlsSetValue(my_key, (LPVOID)value); } void* get_tls() { return (void *)TlsGetValue(my_key); } #endif #else typedef pthread_key_t tls_key_t; void create_key() { pthread_key_create(&my_key, NULL); } void destroy_key() { pthread_key_delete(my_key); } void set_tls( void * value ) const { pthread_setspecific(my_key, value); } void* get_tls() const { return pthread_getspecific(my_key); } #endif tls_key_t my_key; virtual void* create_local() __TBB_override = 0; virtual void* create_array(size_t _size) __TBB_override = 0; // _size in bytes virtual void free_array(void* ptr, size_t _size) __TBB_override = 0; // size in bytes protected: ets_base() {create_key();} ~ets_base() {destroy_key();} void* table_lookup( bool& exists ) { void* found = get_tls(); if( found ) { exists=true; } else { found = super::table_lookup(exists); set_tls(found); } return found; } void table_clear() { destroy_key(); create_key(); super::table_clear(); } void table_swap( ets_base& other ) { using std::swap; __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); swap(my_key, other.my_key); super::table_swap(other); } }; //! Random access iterator for traversing the thread local copies. template< typename Container, typename Value > class enumerable_thread_specific_iterator #if defined(_WIN64) && defined(_MSC_VER) // Ensure that Microsoft's internal template function _Val_type works correctly. : public std::iterator #endif /* defined(_WIN64) && defined(_MSC_VER) */ { //! current position in the concurrent_vector Container *my_container; typename Container::size_type my_index; mutable Value *my_value; template friend enumerable_thread_specific_iterator operator+( ptrdiff_t offset, const enumerable_thread_specific_iterator& v ); template friend bool operator==( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ); template friend bool operator<( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ); template friend ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ); template friend class enumerable_thread_specific_iterator; public: enumerable_thread_specific_iterator( const Container &container, typename Container::size_type index ) : my_container(&const_cast(container)), my_index(index), my_value(NULL) {} //! Default constructor enumerable_thread_specific_iterator() : my_container(NULL), my_index(0), my_value(NULL) {} template enumerable_thread_specific_iterator( const enumerable_thread_specific_iterator& other ) : my_container( other.my_container ), my_index( other.my_index), my_value( const_cast(other.my_value) ) {} enumerable_thread_specific_iterator operator+( ptrdiff_t offset ) const { return enumerable_thread_specific_iterator(*my_container, my_index + offset); } enumerable_thread_specific_iterator &operator+=( ptrdiff_t offset ) { my_index += offset; my_value = NULL; return *this; } enumerable_thread_specific_iterator operator-( ptrdiff_t offset ) const { return enumerable_thread_specific_iterator( *my_container, my_index-offset ); } enumerable_thread_specific_iterator &operator-=( ptrdiff_t offset ) { my_index -= offset; my_value = NULL; return *this; } Value& operator*() const { Value* value = my_value; if( !value ) { value = my_value = (*my_container)[my_index].value(); } __TBB_ASSERT( value==(*my_container)[my_index].value(), "corrupt cache" ); return *value; } Value& operator[]( ptrdiff_t k ) const { return (*my_container)[my_index + k].value; } Value* operator->() const {return &operator*();} enumerable_thread_specific_iterator& operator++() { ++my_index; my_value = NULL; return *this; } enumerable_thread_specific_iterator& operator--() { --my_index; my_value = NULL; return *this; } //! Post increment enumerable_thread_specific_iterator operator++(int) { enumerable_thread_specific_iterator result = *this; ++my_index; my_value = NULL; return result; } //! Post decrement enumerable_thread_specific_iterator operator--(int) { enumerable_thread_specific_iterator result = *this; --my_index; my_value = NULL; return result; } // STL support typedef ptrdiff_t difference_type; typedef Value value_type; typedef Value* pointer; typedef Value& reference; typedef std::random_access_iterator_tag iterator_category; }; template enumerable_thread_specific_iterator operator+( ptrdiff_t offset, const enumerable_thread_specific_iterator& v ) { return enumerable_thread_specific_iterator( v.my_container, v.my_index + offset ); } template bool operator==( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ) { return i.my_index==j.my_index && i.my_container == j.my_container; } template bool operator!=( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ) { return !(i==j); } template bool operator<( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ) { return i.my_index bool operator>( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ) { return j bool operator>=( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ) { return !(i bool operator<=( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ) { return !(j ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, const enumerable_thread_specific_iterator& j ) { return i.my_index-j.my_index; } template class segmented_iterator #if defined(_WIN64) && defined(_MSC_VER) : public std::iterator #endif { template friend bool operator==(const segmented_iterator& i, const segmented_iterator& j); template friend bool operator!=(const segmented_iterator& i, const segmented_iterator& j); template friend class segmented_iterator; public: segmented_iterator() {my_segcont = NULL;} segmented_iterator( const SegmentedContainer& _segmented_container ) : my_segcont(const_cast(&_segmented_container)), outer_iter(my_segcont->end()) { } ~segmented_iterator() {} typedef typename SegmentedContainer::iterator outer_iterator; typedef typename SegmentedContainer::value_type InnerContainer; typedef typename InnerContainer::iterator inner_iterator; // STL support typedef ptrdiff_t difference_type; typedef Value value_type; typedef typename SegmentedContainer::size_type size_type; typedef Value* pointer; typedef Value& reference; typedef std::input_iterator_tag iterator_category; // Copy Constructor template segmented_iterator(const segmented_iterator& other) : my_segcont(other.my_segcont), outer_iter(other.outer_iter), // can we assign a default-constructed iterator to inner if we're at the end? inner_iter(other.inner_iter) {} // assignment template segmented_iterator& operator=( const segmented_iterator& other) { if(this != &other) { my_segcont = other.my_segcont; outer_iter = other.outer_iter; if(outer_iter != my_segcont->end()) inner_iter = other.inner_iter; } return *this; } // allow assignment of outer iterator to segmented iterator. Once it is // assigned, move forward until a non-empty inner container is found or // the end of the outer container is reached. segmented_iterator& operator=(const outer_iterator& new_outer_iter) { __TBB_ASSERT(my_segcont != NULL, NULL); // check that this iterator points to something inside the segmented container for(outer_iter = new_outer_iter ;outer_iter!=my_segcont->end(); ++outer_iter) { if( !outer_iter->empty() ) { inner_iter = outer_iter->begin(); break; } } return *this; } // pre-increment segmented_iterator& operator++() { advance_me(); return *this; } // post-increment segmented_iterator operator++(int) { segmented_iterator tmp = *this; operator++(); return tmp; } bool operator==(const outer_iterator& other_outer) const { __TBB_ASSERT(my_segcont != NULL, NULL); return (outer_iter == other_outer && (outer_iter == my_segcont->end() || inner_iter == outer_iter->begin())); } bool operator!=(const outer_iterator& other_outer) const { return !operator==(other_outer); } // (i)* RHS reference operator*() const { __TBB_ASSERT(my_segcont != NULL, NULL); __TBB_ASSERT(outer_iter != my_segcont->end(), "Dereferencing a pointer at end of container"); __TBB_ASSERT(inner_iter != outer_iter->end(), NULL); // should never happen return *inner_iter; } // i-> pointer operator->() const { return &operator*();} private: SegmentedContainer* my_segcont; outer_iterator outer_iter; inner_iterator inner_iter; void advance_me() { __TBB_ASSERT(my_segcont != NULL, NULL); __TBB_ASSERT(outer_iter != my_segcont->end(), NULL); // not true if there are no inner containers __TBB_ASSERT(inner_iter != outer_iter->end(), NULL); // not true if the inner containers are all empty. ++inner_iter; while(inner_iter == outer_iter->end() && ++outer_iter != my_segcont->end()) { inner_iter = outer_iter->begin(); } } }; // segmented_iterator template bool operator==( const segmented_iterator& i, const segmented_iterator& j ) { if(i.my_segcont != j.my_segcont) return false; if(i.my_segcont == NULL) return true; if(i.outer_iter != j.outer_iter) return false; if(i.outer_iter == i.my_segcont->end()) return true; return i.inner_iter == j.inner_iter; } // != template bool operator!=( const segmented_iterator& i, const segmented_iterator& j ) { return !(i==j); } template struct construct_by_default: tbb::internal::no_assign { void construct(void*where) {new(where) T();} // C++ note: the () in T() ensure zero initialization. construct_by_default( int ) {} }; template struct construct_by_exemplar: tbb::internal::no_assign { const T exemplar; void construct(void*where) {new(where) T(exemplar);} construct_by_exemplar( const T& t ) : exemplar(t) {} #if __TBB_ETS_USE_CPP11 construct_by_exemplar( T&& t ) : exemplar(std::move(t)) {} #endif }; template struct construct_by_finit: tbb::internal::no_assign { Finit f; void construct(void* where) {new(where) T(f());} construct_by_finit( const Finit& f_ ) : f(f_) {} #if __TBB_ETS_USE_CPP11 construct_by_finit( Finit&& f_ ) : f(std::move(f_)) {} #endif }; #if __TBB_ETS_USE_CPP11 template struct construct_by_args: tbb::internal::no_assign { internal::stored_pack pack; void construct(void* where) { internal::call( [where](const typename strip

::type&... args ){ new(where) T(args...); }, pack ); } construct_by_args( P&& ... args ) : pack(std::forward

(args)...) {} }; #endif // storage for initialization function pointer // TODO: consider removing the template parameter T here and in callback_leaf template class callback_base { public: // Clone *this virtual callback_base* clone() const = 0; // Destruct and free *this virtual void destroy() = 0; // Need virtual destructor to satisfy GCC compiler warning virtual ~callback_base() { } // Construct T at where virtual void construct(void* where) = 0; }; template class callback_leaf: public callback_base, Constructor { #if __TBB_ETS_USE_CPP11 template callback_leaf( P&& ... params ) : Constructor(std::forward

(params)...) {} #else template callback_leaf( const X& x ) : Constructor(x) {} #endif // TODO: make the construction/destruction consistent (use allocator.construct/destroy) typedef typename tbb::tbb_allocator my_allocator_type; callback_base* clone() const __TBB_override { return make(*this); } void destroy() __TBB_override { my_allocator_type().destroy(this); my_allocator_type().deallocate(this,1); } void construct(void* where) __TBB_override { Constructor::construct(where); } public: #if __TBB_ETS_USE_CPP11 template static callback_base* make( P&& ... params ) { void* where = my_allocator_type().allocate(1); return new(where) callback_leaf( std::forward

(params)... ); } #else template static callback_base* make( const X& x ) { void* where = my_allocator_type().allocate(1); return new(where) callback_leaf(x); } #endif }; //! Template for recording construction of objects in table /** All maintenance of the space will be done explicitly on push_back, and all thread local copies must be destroyed before the concurrent vector is deleted. The flag is_built is initialized to false. When the local is successfully-constructed, set the flag to true or call value_committed(). If the constructor throws, the flag will be false. */ template struct ets_element { tbb::aligned_space my_space; bool is_built; ets_element() { is_built = false; } // not currently-built U* value() { return my_space.begin(); } U* value_committed() { is_built = true; return my_space.begin(); } ~ets_element() { if(is_built) { my_space.begin()->~U(); is_built = false; } } }; // A predicate that can be used for a compile-time compatibility check of ETS instances // Ideally, it should have been declared inside the ETS class, but unfortunately // in that case VS2013 does not enable the variadic constructor. template struct is_compatible_ets { static const bool value = false; }; template struct is_compatible_ets< T, enumerable_thread_specific > { static const bool value = internal::is_same_type::value; }; #if __TBB_ETS_USE_CPP11 // A predicate that checks whether, for a variable 'foo' of type T, foo() is a valid expression template class is_callable_no_args { private: typedef char yes[1]; typedef char no [2]; template static yes& decide( decltype(declval()())* ); template static no& decide(...); public: static const bool value = (sizeof(decide(NULL)) == sizeof(yes)); }; #endif } // namespace internal //! @endcond //! The enumerable_thread_specific container /** enumerable_thread_specific has the following properties: - thread-local copies are lazily created, with default, exemplar or function initialization. - thread-local copies do not move (during lifetime, and excepting clear()) so the address of a copy is invariant. - the contained objects need not have operator=() defined if combine is not used. - enumerable_thread_specific containers may be copy-constructed or assigned. - thread-local copies can be managed by hash-table, or can be accessed via TLS storage for speed. - outside of parallel contexts, the contents of all thread-local copies are accessible by iterator or using combine or combine_each methods @par Segmented iterator When the thread-local objects are containers with input_iterators defined, a segmented iterator may be used to iterate over all the elements of all thread-local copies. @par combine and combine_each - Both methods are defined for enumerable_thread_specific. - combine() requires the type T have operator=() defined. - neither method modifies the contents of the object (though there is no guarantee that the applied methods do not modify the object.) - Both are evaluated in serial context (the methods are assumed to be non-benign.) @ingroup containers */ template , ets_key_usage_type ETS_key_type=ets_no_key > class enumerable_thread_specific: internal::ets_base { template friend class enumerable_thread_specific; typedef internal::padded< internal::ets_element > padded_element; //! A generic range, used to create range objects from the iterators template class generic_range_type: public blocked_range { public: typedef T value_type; typedef T& reference; typedef const T& const_reference; typedef I iterator; typedef ptrdiff_t difference_type; generic_range_type( I begin_, I end_, size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {} template generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {} generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} }; typedef typename Allocator::template rebind< padded_element >::other padded_allocator_type; typedef tbb::concurrent_vector< padded_element, padded_allocator_type > internal_collection_type; internal::callback_base *my_construct_callback; internal_collection_type my_locals; // TODO: consider unifying the callback mechanism for all create_local* methods below // (likely non-compatible and requires interface version increase) void* create_local() __TBB_override { padded_element& lref = *my_locals.grow_by(1); my_construct_callback->construct(lref.value()); return lref.value_committed(); } static void* create_local_by_copy( internal::ets_base& base, void* p ) { enumerable_thread_specific& ets = static_cast(base); padded_element& lref = *ets.my_locals.grow_by(1); new(lref.value()) T(*static_cast(p)); return lref.value_committed(); } #if __TBB_ETS_USE_CPP11 static void* create_local_by_move( internal::ets_base& base, void* p ) { enumerable_thread_specific& ets = static_cast(base); padded_element& lref = *ets.my_locals.grow_by(1); new(lref.value()) T(std::move(*static_cast(p))); return lref.value_committed(); } #endif typedef typename Allocator::template rebind< uintptr_t >::other array_allocator_type; // _size is in bytes void* create_array(size_t _size) __TBB_override { size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); return array_allocator_type().allocate(nelements); } void free_array( void* _ptr, size_t _size) __TBB_override { size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); array_allocator_type().deallocate( reinterpret_cast(_ptr),nelements); } public: //! Basic types typedef Allocator allocator_type; typedef T value_type; typedef T& reference; typedef const T& const_reference; typedef T* pointer; typedef const T* const_pointer; typedef typename internal_collection_type::size_type size_type; typedef typename internal_collection_type::difference_type difference_type; // Iterator types typedef typename internal::enumerable_thread_specific_iterator< internal_collection_type, value_type > iterator; typedef typename internal::enumerable_thread_specific_iterator< internal_collection_type, const value_type > const_iterator; // Parallel range types typedef generic_range_type< iterator > range_type; typedef generic_range_type< const_iterator > const_range_type; //! Default constructor. Each local instance of T is default constructed. enumerable_thread_specific() : my_construct_callback( internal::callback_leaf >::make(/*dummy argument*/0) ){} //! Constructor with initializer functor. Each local instance of T is constructed by T(finit()). template ::type>::value>::type #endif > explicit enumerable_thread_specific( Finit finit ) : my_construct_callback( internal::callback_leaf >::make( tbb::internal::move(finit) ) ){} //! Constructor with exemplar. Each local instance of T is copy-constructed from the exemplar. explicit enumerable_thread_specific( const T& exemplar ) : my_construct_callback( internal::callback_leaf >::make( exemplar ) ){} #if __TBB_ETS_USE_CPP11 explicit enumerable_thread_specific( T&& exemplar ) : my_construct_callback( internal::callback_leaf >::make( std::move(exemplar) ) ){} //! Variadic constructor with initializer arguments. Each local instance of T is constructed by T(args...) template ::type>::value && !internal::is_compatible_ets::type>::value && !internal::is_same_type::type>::value >::type> enumerable_thread_specific( P1&& arg1, P&& ... args ) : my_construct_callback( internal::callback_leaf >::make( std::forward(arg1), std::forward

(args)... ) ){} #endif //! Destructor ~enumerable_thread_specific() { if(my_construct_callback) my_construct_callback->destroy(); // Deallocate the hash table before overridden free_array() becomes inaccessible this->internal::ets_base::table_clear(); } //! returns reference to local, discarding exists reference local() { bool exists; return local(exists); } //! Returns reference to calling thread's local copy, creating one if necessary reference local(bool& exists) { void* ptr = this->table_lookup(exists); return *(T*)ptr; } //! Get the number of local copies size_type size() const { return my_locals.size(); } //! true if there have been no local copies created bool empty() const { return my_locals.empty(); } //! begin iterator iterator begin() { return iterator( my_locals, 0 ); } //! end iterator iterator end() { return iterator(my_locals, my_locals.size() ); } //! begin const iterator const_iterator begin() const { return const_iterator(my_locals, 0); } //! end const iterator const_iterator end() const { return const_iterator(my_locals, my_locals.size()); } //! Get range for parallel algorithms range_type range( size_t grainsize=1 ) { return range_type( begin(), end(), grainsize ); } //! Get const range for parallel algorithms const_range_type range( size_t grainsize=1 ) const { return const_range_type( begin(), end(), grainsize ); } //! Destroys local copies void clear() { my_locals.clear(); this->table_clear(); // callback is not destroyed } private: template void internal_copy(const enumerable_thread_specific& other) { #if __TBB_ETS_USE_CPP11 && TBB_USE_ASSERT // this tests is_compatible_ets __TBB_STATIC_ASSERT( (internal::is_compatible_ets::type>::value), "is_compatible_ets fails" ); #endif // Initialize my_construct_callback first, so that it is valid even if rest of this routine throws an exception. my_construct_callback = other.my_construct_callback->clone(); __TBB_ASSERT(my_locals.size()==0,NULL); my_locals.reserve(other.size()); this->table_elementwise_copy( other, create_local_by_copy ); } void internal_swap(enumerable_thread_specific& other) { using std::swap; __TBB_ASSERT( this!=&other, NULL ); swap(my_construct_callback, other.my_construct_callback); // concurrent_vector::swap() preserves storage space, // so addresses to the vector kept in ETS hash table remain valid. swap(my_locals, other.my_locals); this->internal::ets_base::table_swap(other); } #if __TBB_ETS_USE_CPP11 template void internal_move(enumerable_thread_specific&& other) { #if TBB_USE_ASSERT // this tests is_compatible_ets __TBB_STATIC_ASSERT( (internal::is_compatible_ets::type>::value), "is_compatible_ets fails" ); #endif my_construct_callback = other.my_construct_callback; other.my_construct_callback = NULL; __TBB_ASSERT(my_locals.size()==0,NULL); my_locals.reserve(other.size()); this->table_elementwise_copy( other, create_local_by_move ); } #endif public: enumerable_thread_specific( const enumerable_thread_specific& other ) : internal::ets_base() /* prevents GCC warnings with -Wextra */ { internal_copy(other); } template enumerable_thread_specific( const enumerable_thread_specific& other ) { internal_copy(other); } #if __TBB_ETS_USE_CPP11 enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() { internal_swap(other); } template enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() { internal_move(std::move(other)); } #endif enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) { if( this != &other ) { this->clear(); my_construct_callback->destroy(); internal_copy( other ); } return *this; } template enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) { __TBB_ASSERT( static_cast(this)!=static_cast(&other), NULL ); // Objects of different types this->clear(); my_construct_callback->destroy(); internal_copy(other); return *this; } #if __TBB_ETS_USE_CPP11 enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) { if( this != &other ) internal_swap(other); return *this; } template enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) { __TBB_ASSERT( static_cast(this)!=static_cast(&other), NULL ); // Objects of different types this->clear(); my_construct_callback->destroy(); internal_move(std::move(other)); return *this; } #endif // combine_func_t has signature T(T,T) or T(const T&, const T&) template T combine(combine_func_t f_combine) { if(begin() == end()) { internal::ets_element location; my_construct_callback->construct(location.value()); return *location.value_committed(); } const_iterator ci = begin(); T my_result = *ci; while(++ci != end()) my_result = f_combine( my_result, *ci ); return my_result; } // combine_func_t takes T by value or by [const] reference, and returns nothing template void combine_each(combine_func_t f_combine) { for(iterator ci = begin(); ci != end(); ++ci) { f_combine( *ci ); } } }; // enumerable_thread_specific template< typename Container > class flattened2d { // This intermediate typedef is to address issues with VC7.1 compilers typedef typename Container::value_type conval_type; public: //! Basic types typedef typename conval_type::size_type size_type; typedef typename conval_type::difference_type difference_type; typedef typename conval_type::allocator_type allocator_type; typedef typename conval_type::value_type value_type; typedef typename conval_type::reference reference; typedef typename conval_type::const_reference const_reference; typedef typename conval_type::pointer pointer; typedef typename conval_type::const_pointer const_pointer; typedef typename internal::segmented_iterator iterator; typedef typename internal::segmented_iterator const_iterator; flattened2d( const Container &c, typename Container::const_iterator b, typename Container::const_iterator e ) : my_container(const_cast(&c)), my_begin(b), my_end(e) { } explicit flattened2d( const Container &c ) : my_container(const_cast(&c)), my_begin(c.begin()), my_end(c.end()) { } iterator begin() { return iterator(*my_container) = my_begin; } iterator end() { return iterator(*my_container) = my_end; } const_iterator begin() const { return const_iterator(*my_container) = my_begin; } const_iterator end() const { return const_iterator(*my_container) = my_end; } size_type size() const { size_type tot_size = 0; for(typename Container::const_iterator i = my_begin; i != my_end; ++i) { tot_size += i->size(); } return tot_size; } private: Container *my_container; typename Container::const_iterator my_begin; typename Container::const_iterator my_end; }; template flattened2d flatten2d(const Container &c, const typename Container::const_iterator b, const typename Container::const_iterator e) { return flattened2d(c, b, e); } template flattened2d flatten2d(const Container &c) { return flattened2d(c); } } // interface6 namespace internal { using interface6::internal::segmented_iterator; } using interface6::enumerable_thread_specific; using interface6::flattened2d; using interface6::flatten2d; } // namespace tbb #endif