From aa7491901457d2994e991a7346373366fcff268f Mon Sep 17 00:00:00 2001 From: Ilya Date: Sun, 27 Jan 2019 18:25:32 +0300 Subject: [PATCH 1/3] infrastructure for lazy skip list create unit tests create stress tests note: algorythm is lock-free (see SkipListSet) --- cds/container/details/lazy_skip_list_base.h | 235 ++++ .../details/make_lazy_skip_list_set.h | 89 ++ cds/container/impl/lazy_skip_list_set.h | 296 ++++ cds/container/lazy_skip_list_set_dhp.h | 9 + cds/container/lazy_skip_list_set_hp.h | 9 + cds/intrusive/details/lazy_skip_list_base.h | 523 +++++++ cds/intrusive/impl/lazy_skip_list.h | 1220 +++++++++++++++++ cds/intrusive/lazy_skip_list_dhp.h | 7 + cds/intrusive/lazy_skip_list_hp.h | 12 + test/include/cds_test/stat_lazyskiplist_out.h | 61 + test/stress/set/del3/CMakeLists.txt | 1 + test/stress/set/del3/set_del3_lazy_skip.cpp | 8 + test/stress/set/delodd/CMakeLists.txt | 1 + .../set/delodd/set_delodd_lazy_skip.cpp | 8 + test/stress/set/insdel_find/CMakeLists.txt | 1 + .../set_insdelfind_lazy_skip_hp.cpp | 8 + test/stress/set/insdel_func/CMakeLists.txt | 1 + .../insdel_func/set_insdel_func_lazy_skip.cpp | 8 + test/stress/set/insdel_string/CMakeLists.txt | 1 + .../set_insdel_string_lazy_skip.cpp | 8 + test/stress/set/set_type_lazy_skip_list.h | 267 ++++ test/unit/intrusive-set/CMakeLists.txt | 12 + .../intrusive_lazyskiplist_dhp.cpp | 334 +++++ .../intrusive_lazyskiplist_hp.cpp | 335 +++++ test/unit/set/CMakeLists.txt | 12 + test/unit/set/lazy_skiplist_dhp.cpp | 32 + test/unit/set/lazy_skiplist_hp.cpp | 33 + test/unit/set/lazy_skiplist_hp_inl.h | 177 +++ 28 files changed, 3708 insertions(+) create mode 100644 cds/container/details/lazy_skip_list_base.h create mode 100644 cds/container/details/make_lazy_skip_list_set.h create mode 100644 cds/container/impl/lazy_skip_list_set.h create mode 100644 cds/container/lazy_skip_list_set_dhp.h create mode 100644 cds/container/lazy_skip_list_set_hp.h create mode 100644 cds/intrusive/details/lazy_skip_list_base.h create mode 100644 cds/intrusive/impl/lazy_skip_list.h create mode 100644 cds/intrusive/lazy_skip_list_dhp.h create mode 100644 cds/intrusive/lazy_skip_list_hp.h create mode 100644 test/include/cds_test/stat_lazyskiplist_out.h create mode 100644 test/stress/set/del3/set_del3_lazy_skip.cpp create mode 100644 test/stress/set/delodd/set_delodd_lazy_skip.cpp create mode 100644 test/stress/set/insdel_find/set_insdelfind_lazy_skip_hp.cpp create mode 100644 test/stress/set/insdel_func/set_insdel_func_lazy_skip.cpp create mode 100644 test/stress/set/insdel_string/set_insdel_string_lazy_skip.cpp create mode 100644 test/stress/set/set_type_lazy_skip_list.h create mode 100644 test/unit/intrusive-set/intrusive_lazyskiplist_dhp.cpp create mode 100644 test/unit/intrusive-set/intrusive_lazyskiplist_hp.cpp create mode 100644 test/unit/set/lazy_skiplist_dhp.cpp create mode 100644 test/unit/set/lazy_skiplist_hp.cpp create mode 100644 test/unit/set/lazy_skiplist_hp_inl.h diff --git a/cds/container/details/lazy_skip_list_base.h b/cds/container/details/lazy_skip_list_base.h new file mode 100644 index 000000000..a6947a89b --- /dev/null +++ b/cds/container/details/lazy_skip_list_base.h @@ -0,0 +1,235 @@ +#ifndef CDSLIB_CONTAINER_DETAILS_LAZY_SKIP_LIST_BASE_H +#define CDSLIB_CONTAINER_DETAILS_LAZY_SKIP_LIST_BASE_H + +#include +#include +#include + +namespace cds { namespace container { + + namespace lazy_skip_list { + template + using random_level_generator = cds::intrusive::lazy_skip_list::random_level_generator; + + template + using xor_shift = cds::intrusive::lazy_skip_list::xor_shift; + + typedef cds::intrusive::lazy_skip_list::xorshift32 xorshift32; + + typedef cds::intrusive::lazy_skip_list::xorshift24 xorshift24; + + typedef cds::intrusive::lazy_skip_list::xorshift16 xorshift16; + + using cds::intrusive::lazy_skip_list::xorshift; + + template + using turbo = cds::intrusive::lazy_skip_list::turbo; + + typedef cds::intrusive::lazy_skip_list::turbo32 turbo32; + + typedef cds::intrusive::lazy_skip_list::turbo24 turbo24; + + typedef cds::intrusive::lazy_skip_list::turbo16 turbo16; + + using cds::intrusive::lazy_skip_list::turbo_pascal; + + template + using stat = cds::intrusive::lazy_skip_list::stat < EventCounter >; + + typedef cds::intrusive::lazy_skip_list::empty_stat empty_stat; + + struct traits + { + typedef opt::none compare; + typedef opt::none less; + typedef atomicity::empty_item_counter item_counter; + typedef opt::v::relaxed_ordering memory_model; + typedef turbo32 random_level_generator; + typedef CDS_DEFAULT_ALLOCATOR allocator; + typedef cds::backoff::Default back_off; + typedef empty_stat stat; + typedef opt::none key_accessor; + }; + + template + struct make_traits { +# ifdef CDS_DOXYGEN_INVOKED + typedef implementation_defined type ; +# else + typedef typename cds::opt::make_options< + typename cds::opt::find_type_traits< traits, Options... >::type + ,Options... + >::type type; +# endif + }; + + namespace details { + + template + class node_allocator + { + protected: + typedef Node node_type; + typedef Traits traits; + + typedef typename node_type::tower_item_type node_tower_item; + + typedef typename std::allocator_traits::template rebind_alloc tower_allocator_type; + typedef typename std::allocator_traits::template rebind_alloc node_allocator_type; + + static size_t const c_nTowerItemSize = sizeof(node_tower_item); + static size_t const c_nNodePadding = sizeof(node_type) % c_nTowerItemSize; + static size_t const c_nNodeSize = sizeof(node_type) + (c_nNodePadding ? (c_nTowerItemSize - c_nNodePadding) : 0); + + static constexpr size_t node_size( unsigned int nHeight ) noexcept + { + return c_nNodeSize + (nHeight - 1) * c_nTowerItemSize; + } + + static unsigned char * alloc_space( unsigned int nHeight ) + { + unsigned char * pMem; + size_t const sz = node_size( nHeight ); + + if ( nHeight > 1 ) { + pMem = tower_allocator_type().allocate( sz ); + + assert( (((uintptr_t) pMem) & (alignof(node_type) - 1)) == 0 ); + assert( (((uintptr_t) (pMem + c_nNodeSize)) & (alignof(node_tower_item) - 1)) == 0 ); + return pMem; + } + else + pMem = reinterpret_cast( node_allocator_type().allocate( 1 )); + + return pMem; + } + + static void free_space( unsigned char * p, unsigned int nHeight ) + { + assert( p != nullptr ); + + if ( nHeight == 1 ) + node_allocator_type().deallocate( reinterpret_cast(p), 1 ); + else + tower_allocator_type().deallocate( p, node_size(nHeight)); + } + + public: + template + node_type * New( unsigned int nHeight, Q const& v ) + { + unsigned char * pMem = alloc_space( nHeight ); + node_type * p = new( pMem ) + node_type( nHeight, nHeight > 1 ? reinterpret_cast(pMem + c_nNodeSize) : nullptr, v ); + return p; + } + + template + node_type * New( unsigned int nHeight, Args&&... args ) + { + unsigned char * pMem = alloc_space( nHeight ); + node_type * p = new( pMem ) + node_type( nHeight, nHeight > 1 ? reinterpret_cast(pMem + c_nNodeSize) : nullptr, + std::forward(args)... ); + return p; + } + + void Delete( node_type * p ) + { + assert( p != nullptr ); + + unsigned int nHeight = p->height(); + node_allocator_type a; + std::allocator_traits::destroy( a, p ); + free_space( reinterpret_cast(p), nHeight ); + } + }; + + template + struct dummy_node_builder { + typedef IntrusiveNode intrusive_node_type; + + template + static intrusive_node_type * make_tower( intrusive_node_type * pNode, RandomGen& /*gen*/ ) { return pNode ; } + static intrusive_node_type * make_tower( intrusive_node_type * pNode, unsigned int /*nHeight*/ ) { return pNode ; } + static void dispose_tower( intrusive_node_type * pNode ) + { + pNode->release_tower(); + } + + struct node_disposer { + void operator()( intrusive_node_type * /*pNode*/ ) const {} + }; + }; + + template + class iterator + { + typedef ForwardIterator intrusive_iterator; + typedef typename intrusive_iterator::value_type node_type; + typedef typename node_type::stored_value_type value_type; + static bool const c_isConst = intrusive_iterator::c_isConst; + + typedef typename std::conditional< c_isConst, value_type const&, value_type&>::type value_ref; + template friend class iterator; + + intrusive_iterator m_It; + + public: // for internal use only!!! + iterator( intrusive_iterator const& it ) + : m_It( it ) + {} + + public: + iterator() + : m_It() + {} + + iterator( iterator const& s) + : m_It( s.m_It ) + {} + + value_type * operator ->() const + { + return &( m_It.operator->()->m_Value ); + } + + value_ref operator *() const + { + return m_It.operator*().m_Value; + } + + iterator& operator ++() + { + ++m_It; + return *this; + } + + iterator& operator = (iterator const& src) + { + m_It = src.m_It; + return *this; + } + + template + bool operator ==(iterator const& i ) const + { + return m_It == i.m_It; + } + template + bool operator !=(iterator const& i ) const + { + return !( *this == i ); + } + }; + + } // namespace details + + } // namespace lazy_skip_list + + template + class LazySkipListSet; + +}} // namespace cds::container + +#endif // #ifndef CDSLIB_CONTAINER_DETAILS_LAZY_SKIP_LIST_BASE_H diff --git a/cds/container/details/make_lazy_skip_list_set.h b/cds/container/details/make_lazy_skip_list_set.h new file mode 100644 index 000000000..20b973257 --- /dev/null +++ b/cds/container/details/make_lazy_skip_list_set.h @@ -0,0 +1,89 @@ +#ifndef CDSLIB_CONTAINER_DETAILS_MAKE_LAZY_SKIP_LIST_SET_H +#define CDSLIB_CONTAINER_DETAILS_MAKE_LAZY_SKIP_LIST_SET_H + +#include +#include + +namespace cds { namespace container { namespace details { + + template + struct make_lazy_skip_list_set + { + typedef GC gc; + typedef T value_type; + typedef Traits traits; + + typedef cds::intrusive::lazy_skip_list::node< gc > intrusive_node_type; + + struct node_type: public intrusive_node_type + { + typedef intrusive_node_type base_class; + typedef typename base_class::atomic_marked_ptr atomic_marked_ptr; + typedef value_type stored_value_type; + + value_type m_Value; + + template + node_type( unsigned int nHeight, atomic_marked_ptr * pTower, Q&& v ) + : m_Value( std::forward( v )) + { + init_tower( nHeight, pTower ); + } + + template + node_type( unsigned int nHeight, atomic_marked_ptr * pTower, Q&& q, Args&&... args ) + : m_Value( std::forward(q), std::forward(args)... ) + { + init_tower( nHeight, pTower ); + } + + node_type() = delete; + + private: + void init_tower( unsigned nHeight, atomic_marked_ptr* pTower ) + { + if ( nHeight > 1 ) { + new ( pTower ) atomic_marked_ptr[nHeight - 1]; + base_class::make_tower( nHeight, pTower ); + } + } + }; + + typedef lazy_skip_list::details::node_allocator< node_type, traits> node_allocator; + + struct node_deallocator { + void operator ()( node_type * pNode ) + { + node_allocator().Delete( pNode ); + } + }; + + typedef lazy_skip_list::details::dummy_node_builder dummy_node_builder; + + struct value_accessor + { + value_type const& operator()( node_type const& node ) const + { + return node.m_Value; + } + }; + + typedef typename opt::details::make_comparator< value_type, traits >::type key_comparator; + + template + using less_wrapper = cds::details::compare_wrapper< node_type, cds::opt::details::make_comparator_from_less, value_accessor >; + + class intrusive_traits: public cds::intrusive::lazy_skip_list::make_traits< + cds::opt::type_traits< traits > + ,cds::intrusive::opt::hook< intrusive::lazy_skip_list::base_hook< cds::opt::gc< gc > > > + ,cds::intrusive::opt::disposer< node_deallocator > + ,cds::intrusive::lazy_skip_list::internal_node_builder< dummy_node_builder > + ,cds::opt::compare< cds::details::compare_wrapper< node_type, key_comparator, value_accessor > > + >::type + {}; + + typedef cds::intrusive::LazySkipListSet< gc, node_type, intrusive_traits> type; + }; +}}} // namespace cds::container::details + +#endif //#ifndef CDSLIB_CONTAINER_DETAILS_MAKE_LAZY_SKIP_LIST_SET_H diff --git a/cds/container/impl/lazy_skip_list_set.h b/cds/container/impl/lazy_skip_list_set.h new file mode 100644 index 000000000..cacb81e08 --- /dev/null +++ b/cds/container/impl/lazy_skip_list_set.h @@ -0,0 +1,296 @@ +#ifndef CDSLIB_CONTAINER_IMPL_LAZY_SKIP_LIST_SET_H +#define CDSLIB_CONTAINER_IMPL_LAZY_SKIP_LIST_SET_H + +#include +#include + +namespace cds { namespace container { + + template < + typename GC, + typename T, +#ifdef CDS_DOXYGEN_INVOKED + typename Traits = lazy_skip_list::traits +#else + typename Traits +#endif + > + class LazySkipListSet: +#ifdef CDS_DOXYGEN_INVOKED + protected intrusive::LazySkipListSet< GC, T, Traits > +#else + protected details::make_lazy_skip_list_set< GC, T, Traits >::type +#endif + { + typedef details::make_lazy_skip_list_set< GC, T, Traits > maker; + typedef typename maker::type base_class; + + public: + typedef GC gc; + typedef T value_type; + typedef Traits traits; + + typedef typename base_class::back_off back_off; + typedef typename traits::allocator allocator_type; + typedef typename base_class::item_counter item_counter; + typedef typename maker::key_comparator key_comparator; + typedef typename base_class::memory_model memory_model; + typedef typename traits::random_level_generator random_level_generator; + typedef typename traits::stat stat; + + static size_t const c_nHazardPtrCount = base_class::c_nHazardPtrCount; + + protected: + typedef typename maker::node_type node_type; + typedef typename maker::node_allocator node_allocator; + + typedef std::unique_ptr< node_type, typename maker::node_deallocator > scoped_node_ptr; + + public: + typedef typename gc::template guarded_ptr< node_type, value_type, details::guarded_ptr_cast_set > guarded_ptr; + + protected: + unsigned int random_level() + { + return base_class::random_level(); + } + + public: + LazySkipListSet() + : base_class() + {} + + ~LazySkipListSet() + {} + + public: + typedef lazy_skip_list::details::iterator< typename base_class::iterator > iterator; + typedef lazy_skip_list::details::iterator< typename base_class::const_iterator > const_iterator; + + iterator begin() + { + return iterator( base_class::begin()); + } + + const_iterator begin() const + { + return const_iterator( base_class::begin()); + } + + const_iterator cbegin() const + { + return const_iterator( base_class::cbegin()); + } + + iterator end() + { + return iterator( base_class::end()); + } + + const_iterator end() const + { + return const_iterator( base_class::end()); + } + + const_iterator cend() const + { + return const_iterator( base_class::cend()); + } + + public: + template + bool insert( Q const& val ) + { + scoped_node_ptr sp( node_allocator().New( random_level(), val )); + if ( base_class::insert( *sp.get())) { + sp.release(); + return true; + } + return false; + } + + template + bool insert( Q const& val, Func f ) + { + scoped_node_ptr sp( node_allocator().New( random_level(), val )); + if ( base_class::insert( *sp.get(), [&f]( node_type& v ) { f( v.m_Value ); } )) { + sp.release(); + return true; + } + return false; + } + + template + std::pair update( const Q& val, Func func, bool bInsert = true ) + { + scoped_node_ptr sp( node_allocator().New( random_level(), val )); + std::pair bRes = base_class::update( *sp, + [&func, &val](bool bNew, node_type& node, node_type&){ func( bNew, node.m_Value, val ); }, + bInsert ); + if ( bRes.first && bRes.second ) + sp.release(); + return bRes; + } + + template + CDS_DEPRECATED("ensure() is deprecated, use update()") + std::pair ensure( const Q& val, Func func ) + { + return update( val, func, true ); + } + + template + bool emplace( Args&&... args ) + { + scoped_node_ptr sp( node_allocator().New( random_level(), std::forward(args)... )); + if ( base_class::insert( *sp.get())) { + sp.release(); + return true; + } + return false; + } + + template + bool erase( Q const& key ) + { + return base_class::erase( key ); + } + + template + bool erase_with( Q const& key, Less pred ) + { + CDS_UNUSED( pred ); + return base_class::erase_with( key, cds::details::predicate_wrapper< node_type, Less, typename maker::value_accessor >()); + } + + template + bool erase( Q const& key, Func f ) + { + return base_class::erase( key, [&f]( node_type const& node) { f( node.m_Value ); } ); + } + + template + bool erase_with( Q const& key, Less pred, Func f ) + { + CDS_UNUSED( pred ); + return base_class::erase_with( key, cds::details::predicate_wrapper< node_type, Less, typename maker::value_accessor >(), + [&f]( node_type const& node) { f( node.m_Value ); } ); + } + + template + guarded_ptr extract( Q const& key ) + { + return base_class::extract_( key, typename base_class::key_comparator()); + } + + template + guarded_ptr extract_with( Q const& key, Less pred ) + { + CDS_UNUSED( pred ); + typedef cds::details::predicate_wrapper< node_type, Less, typename maker::value_accessor > wrapped_less; + return base_class::extract_( key, cds::opt::details::make_comparator_from_less()); + } + + guarded_ptr extract_min() + { + return base_class::extract_min_(); + } + + guarded_ptr extract_max() + { + return base_class::extract_max_(); + } + + template + bool find( Q& key, Func f ) + { + return base_class::find( key, [&f]( node_type& node, Q& v ) { f( node.m_Value, v ); }); + } + + template + bool find( Q const& key, Func f ) + { + return base_class::find( key, [&f]( node_type& node, Q& v ) { f( node.m_Value, v ); } ); + } + + template + bool find_with( Q& key, Less pred, Func f ) + { + CDS_UNUSED( pred ); + return base_class::find_with( key, cds::details::predicate_wrapper< node_type, Less, typename maker::value_accessor >(), + [&f]( node_type& node, Q& v ) { f( node.m_Value, v ); } ); + } + + template + bool find_with( Q const& key, Less pred, Func f ) + { + CDS_UNUSED( pred ); + return base_class::find_with( key, cds::details::predicate_wrapper< node_type, Less, typename maker::value_accessor >(), + [&f]( node_type& node, Q const& v ) { f( node.m_Value, v ); } ); + } + + template + bool contains( Q const& key ) + { + return base_class::contains( key ); + } + + template + CDS_DEPRECATED("deprecated, use contains()") + bool find( Q const& key ) + { + return contains( key ); + } + + template + bool contains( Q const& key, Less pred ) + { + CDS_UNUSED( pred ); + return base_class::contains( key, cds::details::predicate_wrapper< node_type, Less, typename maker::value_accessor >()); + } + + template + CDS_DEPRECATED("deprecated, use contains()") + bool find_with( Q const& key, Less pred ) + { + return contains( key, pred ); + } + + template + guarded_ptr get( Q const& key ) + { + return base_class::get_with_( key, typename base_class::key_comparator()); + } + + template + guarded_ptr get_with( Q const& key, Less pred ) + { + CDS_UNUSED( pred ); + typedef cds::details::predicate_wrapper< node_type, Less, typename maker::value_accessor > wrapped_less; + return base_class::get_with_( key, cds::opt::details::make_comparator_from_less< wrapped_less >()); + } + + void clear() + { + base_class::clear(); + } + + bool empty() const + { + return base_class::empty(); + } + + size_t size() const + { + return base_class::size(); + } + + stat const& statistics() const + { + return base_class::statistics(); + } + }; + +}} // namespace cds::container + +#endif // #ifndef CDSLIB_CONTAINER_IMPL_LAZY_SKIP_LIST_SET_H diff --git a/cds/container/lazy_skip_list_set_dhp.h b/cds/container/lazy_skip_list_set_dhp.h new file mode 100644 index 000000000..50f652d18 --- /dev/null +++ b/cds/container/lazy_skip_list_set_dhp.h @@ -0,0 +1,9 @@ +#ifndef CDSLIB_CONTAINER_LAZY_SKIP_LIST_SET_DHP_H +#define CDSLIB_CONTAINER_LAZY_SKIP_LIST_SET_DHP_H + +#include +#include +#include +#include + +#endif // #ifndef CDSLIB_CONTAINER_LAZY_SKIP_LIST_SET_DHP_H diff --git a/cds/container/lazy_skip_list_set_hp.h b/cds/container/lazy_skip_list_set_hp.h new file mode 100644 index 000000000..95e9430ce --- /dev/null +++ b/cds/container/lazy_skip_list_set_hp.h @@ -0,0 +1,9 @@ +#ifndef CDSLIB_CONTAINER_LAZY_SKIP_LIST_SET_HP_H +#define CDSLIB_CONTAINER_LAZY_SKIP_LIST_SET_HP_H + +#include +#include +#include +#include + +#endif // #ifndef CDSLIB_CONTAINER_LAZY_SKIP_LIST_SET_HP_H diff --git a/cds/intrusive/details/lazy_skip_list_base.h b/cds/intrusive/details/lazy_skip_list_base.h new file mode 100644 index 000000000..9f2390fbf --- /dev/null +++ b/cds/intrusive/details/lazy_skip_list_base.h @@ -0,0 +1,523 @@ +#ifndef CDSLIB_INTRUSIVE_DETAILS_LAZY_SKIP_LIST_BASE_H +#define CDSLIB_INTRUSIVE_DETAILS_LAZY_SKIP_LIST_BASE_H + +#include +#include +#include +#include +#include +#include + +namespace cds { namespace intrusive { + namespace lazy_skip_list { + static unsigned int const c_nHeightLimit = 32; + + template + class node + { + public: + typedef GC gc; + typedef Lock lock_type; + typedef Tag tag; + + typedef cds::details::marked_ptr marked_ptr; + typedef typename gc::template atomic_marked_ptr< marked_ptr> atomic_marked_ptr; + typedef atomic_marked_ptr tower_item_type; + + + protected: + atomic_marked_ptr m_pNext; + unsigned int m_nHeight; + atomic_marked_ptr* m_arrNext; + atomics::atomic m_nUnlink; + mutable lock_type m_lock; + + public: + node() + : m_pNext( nullptr ) + , m_nHeight( 1 ) + , m_arrNext( nullptr ) + { + m_nUnlink.store( 1, atomics::memory_order_release ); + } + + + void make_tower( unsigned int nHeight, atomic_marked_ptr * nextTower ) + { + assert( nHeight > 0 ); + assert( (nHeight == 1 && nextTower == nullptr) // bottom-list node + || (nHeight > 1 && nextTower != nullptr) // node at level of more than 0 + ); + + m_arrNext = nextTower; + m_nHeight = nHeight; + m_nUnlink.store( nHeight, atomics::memory_order_release ); + } + + atomic_marked_ptr * release_tower() + { + atomic_marked_ptr * pTower = m_arrNext; + m_arrNext = nullptr; + m_nHeight = 1; + return pTower; + } + + atomic_marked_ptr * get_tower() const + { + return m_arrNext; + } + + bool has_tower() const + { + return m_nHeight > 1; + } + + atomic_marked_ptr& next( unsigned int nLevel ) + { + assert( nLevel < height()); + assert( nLevel == 0 || (nLevel > 0 && m_arrNext != nullptr)); + + if ( nLevel ) { + // TSan: data race between m_arrNext[ nLevel - 1 ] and make_tower() + // In fact, m_arrNext is a const array that is never changed + CDS_TSAN_ANNOTATE_HAPPENS_BEFORE( &m_arrNext[ nLevel - 1 ] ); + return m_arrNext[nLevel - 1]; + } + return m_pNext; + } + + atomic_marked_ptr const& next( unsigned int nLevel ) const + { + assert( nLevel < height()); + assert( nLevel == 0 || nLevel > 0 && m_arrNext != nullptr ); + + if ( nLevel ) { + CDS_TSAN_ANNOTATE_HAPPENS_BEFORE( &m_arrNext[nLevel - 1] ); + return m_arrNext[nLevel - 1]; + } + return m_pNext; + } + + atomic_marked_ptr& operator[]( unsigned int nLevel ) + { + return next( nLevel ); + } + + atomic_marked_ptr const& operator[]( unsigned int nLevel ) const + { + return next( nLevel ); + } + + unsigned int height() const + { + return m_nHeight; + } + + void clear() + { + assert( m_arrNext == nullptr ); + m_pNext.store( marked_ptr(), atomics::memory_order_release ); + } + + bool is_cleared() const + { + return m_pNext == atomic_marked_ptr() + && m_arrNext == nullptr + && m_nHeight <= 1; + } + + bool level_unlinked( unsigned nCount = 1 ) + { + return m_nUnlink.fetch_sub( nCount, atomics::memory_order_relaxed ) == 1; + } + + bool is_upper_level( unsigned nLevel ) const + { + return m_nUnlink.load( atomics::memory_order_relaxed ) == nLevel + 1; + } + + void lock() const + { + m_lock.lock(); + } + + void unlock() const + { + m_lock.unlock(); + } + }; + + struct undefined_gc; + struct default_hook { + typedef undefined_gc gc; + typedef opt::none tag; + typedef std::mutex lock_type; + }; + + template < typename HookType, typename... Options> + struct hook + { + typedef typename opt::make_options< default_hook, Options...>::type options; + typedef typename options::gc gc; + typedef typename options::lock_type lock_type; + typedef typename options::tag tag; + typedef node node_type; + typedef HookType hook_type; + }; + + template < typename... Options > + struct base_hook: public hook< opt::base_hook_tag, Options... > + {}; + + template < size_t MemberOffset, typename... Options > + struct member_hook: public hook< opt::member_hook_tag, Options... > + { + static const size_t c_nMemberOffset = MemberOffset; + }; + + template + struct traits_hook: public hook< opt::traits_hook_tag, Options... > + { + typedef NodeTraits node_traits; + }; + + template + struct random_level_generator { + template + struct pack: public Base + { + typedef Type random_level_generator; + }; + }; + + template + class xor_shift { + atomics::atomic m_nSeed; + + static_assert( MaxHeight > 1, "MaxHeight" ); + static_assert( MaxHeight <= c_nHeightLimit, "MaxHeight is too large" ); + static unsigned int const c_nBitMask = (1u << ( MaxHeight - 1 )) - 1; + + public: + static unsigned int const c_nUpperBound = MaxHeight; + + xor_shift() + { + m_nSeed.store( (unsigned int) cds::OS::Timer::random_seed(), atomics::memory_order_relaxed ); + } + + unsigned int operator()() + { + unsigned int x = m_nSeed.load( atomics::memory_order_relaxed ); + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + m_nSeed.store( x, atomics::memory_order_relaxed ); + unsigned int nLevel = ((x & 0x00000001) != 0) ? 0 : cds::bitop::LSB( (~(x >> 1)) & c_nBitMask ); + + assert( nLevel < c_nUpperBound ); + return nLevel; + } + }; + + typedef xor_shift xorshift32; + + // For backward compatibility + typedef xorshift32 xorshift; + + typedef xor_shift< 24 > xorshift24; + + typedef xor_shift< 16 > xorshift16; + + template + class turbo + { + atomics::atomic m_nSeed; + + static_assert( MaxHeight > 1, "MaxHeight" ); + static_assert( MaxHeight <= c_nHeightLimit, "MaxHeight is too large" ); + static unsigned int const c_nBitMask = (1u << ( MaxHeight - 1 )) - 1; + public: + static unsigned int const c_nUpperBound = MaxHeight; + + turbo() + { + m_nSeed.store( (unsigned int) cds::OS::Timer::random_seed(), atomics::memory_order_relaxed ); + } + + unsigned int operator()() + { + /* + The low bits are apparently not very random (the original used only + upper 16 bits) so we traverse from highest bit down (i.e., test + sign), thus hardly ever use lower bits. + */ + unsigned int x = m_nSeed.load( atomics::memory_order_relaxed ) * 134775813 + 1; + m_nSeed.store( x, atomics::memory_order_relaxed ); + unsigned int nLevel = ( x & 0x80000000 ) ? ( c_nUpperBound - 1 - cds::bitop::MSBnz( (x & c_nBitMask ) | 1 )) : 0; + + assert( nLevel < c_nUpperBound ); + return nLevel; + } + }; + + typedef turbo turbo32; + + // For backward compatibility + typedef turbo32 turbo_pascal; + + typedef turbo< 24 > turbo24; + + typedef turbo< 16 > turbo16; + + template + struct stat { + typedef EventCounter event_counter ; + + event_counter m_nNodeHeightAdd[c_nHeightLimit] ; + event_counter m_nNodeHeightDel[c_nHeightLimit] ; + event_counter m_nInsertSuccess ; + event_counter m_nInsertFailed ; + event_counter m_nInsertRetries ; + event_counter m_nUpdateExist ; + event_counter m_nUpdateNew ; + event_counter m_nUnlinkSuccess ; + event_counter m_nUnlinkFailed ; + event_counter m_nEraseSuccess ; + event_counter m_nEraseFailed ; + event_counter m_nEraseRetry ; + event_counter m_nFindFastSuccess ; + event_counter m_nFindFastFailed ; + event_counter m_nFindSlowSuccess ; + event_counter m_nFindSlowFailed ; + event_counter m_nRenewInsertPosition ; + event_counter m_nLogicDeleteWhileInsert; + event_counter m_nRemoveWhileInsert ; + event_counter m_nFastErase ; + event_counter m_nFastExtract ; + event_counter m_nSlowErase ; + event_counter m_nSlowExtract ; + event_counter m_nExtractSuccess ; + event_counter m_nExtractFailed ; + event_counter m_nExtractRetries ; + event_counter m_nExtractMinSuccess ; + event_counter m_nExtractMinFailed ; + event_counter m_nExtractMinRetries ; + event_counter m_nExtractMaxSuccess ; + event_counter m_nExtractMaxFailed ; + event_counter m_nExtractMaxRetries ; + event_counter m_nEraseWhileFind ; + event_counter m_nExtractWhileFind ; + event_counter m_nMarkFailed ; + event_counter m_nEraseContention ; + + void onAddNode( unsigned int nHeight ) + { + assert( nHeight > 0 && nHeight <= sizeof(m_nNodeHeightAdd) / sizeof(m_nNodeHeightAdd[0])); + ++m_nNodeHeightAdd[nHeight - 1]; + } + void onRemoveNode( unsigned int nHeight ) + { + assert( nHeight > 0 && nHeight <= sizeof(m_nNodeHeightDel) / sizeof(m_nNodeHeightDel[0])); + ++m_nNodeHeightDel[nHeight - 1]; + } + + void onInsertSuccess() { ++m_nInsertSuccess ; } + void onInsertFailed() { ++m_nInsertFailed ; } + void onInsertRetry() { ++m_nInsertRetries ; } + void onUpdateExist() { ++m_nUpdateExist ; } + void onUpdateNew() { ++m_nUpdateNew ; } + void onUnlinkSuccess() { ++m_nUnlinkSuccess ; } + void onUnlinkFailed() { ++m_nUnlinkFailed ; } + void onEraseSuccess() { ++m_nEraseSuccess ; } + void onEraseFailed() { ++m_nEraseFailed ; } + void onEraseRetry() { ++m_nEraseRetry; } + void onFindFastSuccess() { ++m_nFindFastSuccess ; } + void onFindFastFailed() { ++m_nFindFastFailed ; } + void onFindSlowSuccess() { ++m_nFindSlowSuccess ; } + void onFindSlowFailed() { ++m_nFindSlowFailed ; } + void onEraseWhileFind() { ++m_nEraseWhileFind ; } + void onExtractWhileFind() { ++m_nExtractWhileFind ; } + void onRenewInsertPosition() { ++m_nRenewInsertPosition; } + void onLogicDeleteWhileInsert() { ++m_nLogicDeleteWhileInsert; } + void onRemoveWhileInsert() { ++m_nRemoveWhileInsert; } + void onFastErase() { ++m_nFastErase; } + void onFastExtract() { ++m_nFastExtract; } + void onSlowErase() { ++m_nSlowErase; } + void onSlowExtract() { ++m_nSlowExtract; } + void onExtractSuccess() { ++m_nExtractSuccess; } + void onExtractFailed() { ++m_nExtractFailed; } + void onExtractRetry() { ++m_nExtractRetries; } + void onExtractMinSuccess() { ++m_nExtractMinSuccess; } + void onExtractMinFailed() { ++m_nExtractMinFailed; } + void onExtractMinRetry() { ++m_nExtractMinRetries; } + void onExtractMaxSuccess() { ++m_nExtractMaxSuccess; } + void onExtractMaxFailed() { ++m_nExtractMaxFailed; } + void onExtractMaxRetry() { ++m_nExtractMaxRetries; } + void onMarkFailed() { ++m_nMarkFailed; } + void onEraseContention() { ++m_nEraseContention; } + }; + + struct empty_stat { + void onAddNode( unsigned int /*nHeight*/ ) const {} + void onRemoveNode( unsigned int /*nHeight*/ ) const {} + void onInsertSuccess() const {} + void onInsertFailed() const {} + void onInsertRetry() const {} + void onUpdateExist() const {} + void onUpdateNew() const {} + void onUnlinkSuccess() const {} + void onUnlinkFailed() const {} + void onEraseSuccess() const {} + void onEraseFailed() const {} + void onEraseRetry() const {} + void onFindFastSuccess() const {} + void onFindFastFailed() const {} + void onFindSlowSuccess() const {} + void onFindSlowFailed() const {} + void onEraseWhileFind() const {} + void onExtractWhileFind() const {} + void onRenewInsertPosition() const {} + void onLogicDeleteWhileInsert() const {} + void onRemoveWhileInsert() const {} + void onFastErase() const {} + void onFastExtract() const {} + void onSlowErase() const {} + void onSlowExtract() const {} + void onExtractSuccess() const {} + void onExtractFailed() const {} + void onExtractRetry() const {} + void onExtractMinSuccess() const {} + void onExtractMinFailed() const {} + void onExtractMinRetry() const {} + void onExtractMaxSuccess() const {} + void onExtractMaxFailed() const {} + void onExtractMaxRetry() const {} + void onMarkFailed() const {} + void onEraseContention() const {} + }; + + // For internal use only!!! + template + struct internal_node_builder { + template + struct pack: public Base + { + typedef Type internal_node_builder; + }; + }; + + struct traits + { + typedef base_hook<> hook; + + typedef opt::none compare; + + typedef opt::none less; + + typedef opt::v::empty_disposer disposer; + + typedef atomicity::empty_item_counter item_counter; + + typedef opt::v::relaxed_ordering memory_model; + + typedef turbo32 random_level_generator; + + typedef CDS_DEFAULT_ALLOCATOR allocator; + + typedef cds::backoff::Default back_off; + + typedef empty_stat stat; + + typedef opt::v::rcu_throw_deadlock rcu_check_deadlock; + + // For internal use only!!! + typedef opt::none internal_node_builder; + }; + + template + struct make_traits { +# ifdef CDS_DOXYGEN_INVOKED + typedef implementation_defined type ; +# else + typedef typename cds::opt::make_options< + typename cds::opt::find_type_traits< traits, Options... >::type + ,Options... + >::type type; +# endif + }; + + namespace details { + template + class head_node: public Node + { + typedef Node node_type; + typename node_type::atomic_marked_ptr m_Tower[lazy_skip_list::c_nHeightLimit]; + + public: + head_node( unsigned int nHeight ) + { + for ( size_t i = 0; i < sizeof(m_Tower) / sizeof(m_Tower[0]); ++i ) + m_Tower[i].store( typename node_type::marked_ptr(), atomics::memory_order_relaxed ); + + node_type::make_tower( nHeight, m_Tower ); + } + + node_type * head() const + { + return const_cast( static_cast(this)); + } + }; + + template + struct intrusive_node_builder + { + typedef NodeType node_type; + typedef AtomicNodePtr atomic_node_ptr; + typedef Alloc allocator_type; + + typedef cds::details::Allocator< atomic_node_ptr, allocator_type > tower_allocator; + + template + static node_type * make_tower( node_type * pNode, RandomGen& gen ) + { + return make_tower( pNode, gen() + 1 ); + } + + static node_type * make_tower( node_type * pNode, unsigned int nHeight ) + { + if ( nHeight > 1 ) + pNode->make_tower( nHeight, tower_allocator().NewArray( nHeight - 1, nullptr )); + return pNode; + } + + static void dispose_tower( node_type * pNode ) + { + unsigned int nHeight = pNode->height(); + if ( nHeight > 1 ) + tower_allocator().Delete( pNode->release_tower(), nHeight ); + } + + struct node_disposer { + void operator()( node_type * pNode ) + { + dispose_tower( pNode ); + } + }; + }; + + // Forward declaration + template + class iterator; + + } // namespace details + + } // namespace lazy_skip_list + + // Forward declaration + template + class LazySkipListSet; + +}} // namespace cds::intrusive + +#endif // #ifndef CDSLIB_INTRUSIVE_DETAILS_LAZY_SKIP_LIST_BASE_H diff --git a/cds/intrusive/impl/lazy_skip_list.h b/cds/intrusive/impl/lazy_skip_list.h new file mode 100644 index 000000000..c841d4ae1 --- /dev/null +++ b/cds/intrusive/impl/lazy_skip_list.h @@ -0,0 +1,1220 @@ +#ifndef CDSLIB_INTRUSIVE_IMPL_LAZY_SKIP_LIST_H +#define CDSLIB_INTRUSIVE_IMPL_LAZY_SKIP_LIST_H + +#include +#include +#include // ref +#include +#include +#include + +namespace cds { namespace intrusive { + + namespace lazy_skip_list { namespace details { + + template + class iterator { + public: + typedef GC gc; + typedef NodeTraits node_traits; + typedef BackOff back_off; + typedef typename node_traits::node_type node_type; + typedef typename node_traits::value_type value_type; + static constexpr bool const c_isConst = IsConst; + + typedef typename std::conditional< c_isConst, value_type const&, value_type&>::type value_ref; + + protected: + typedef typename node_type::marked_ptr marked_ptr; + typedef typename node_type::atomic_marked_ptr atomic_marked_ptr; + + typename gc::Guard m_guard; + node_type * m_pNode; + + protected: + static value_type * gc_protect( marked_ptr p ) + { + return node_traits::to_value_ptr( p.ptr()); + } + + void next() + { + typename gc::Guard g; + g.copy( m_guard ); + back_off bkoff; + + for (;;) { + if ( m_pNode->next( m_pNode->height() - 1 ).load( atomics::memory_order_acquire ).bits()) { + // Current node is marked as deleted. So, its next pointer can point to anything + // In this case we interrupt our iteration and returns end() iterator. + *this = iterator(); + return; + } + + marked_ptr p = m_guard.protect( (*m_pNode)[0], gc_protect ); + node_type * pp = p.ptr(); + if ( p.bits()) { + // p is marked as deleted. Spin waiting for physical removal + bkoff(); + continue; + } + else if ( pp && pp->next( pp->height() - 1 ).load( atomics::memory_order_relaxed ).bits()) { + // p is marked as deleted. Spin waiting for physical removal + bkoff(); + continue; + } + + m_pNode = pp; + break; + } + } + + public: // for internal use only!!! + iterator( node_type& refHead ) + : m_pNode( nullptr ) + { + back_off bkoff; + + for (;;) { + marked_ptr p = m_guard.protect( refHead[0], gc_protect ); + if ( !p.ptr()) { + // empty skip-list + m_guard.clear(); + break; + } + + node_type * pp = p.ptr(); + // Logically deleted node is marked from highest level + if ( !pp->next( pp->height() - 1 ).load( atomics::memory_order_acquire ).bits()) { + m_pNode = pp; + break; + } + + bkoff(); + } + } + + public: + iterator() + : m_pNode( nullptr ) + {} + + iterator( iterator const& s) + : m_pNode( s.m_pNode ) + { + m_guard.assign( node_traits::to_value_ptr(m_pNode)); + } + + value_type * operator ->() const + { + assert( m_pNode != nullptr ); + assert( node_traits::to_value_ptr( m_pNode ) != nullptr ); + + return node_traits::to_value_ptr( m_pNode ); + } + + value_ref operator *() const + { + assert( m_pNode != nullptr ); + assert( node_traits::to_value_ptr( m_pNode ) != nullptr ); + + return *node_traits::to_value_ptr( m_pNode ); + } + + iterator& operator ++() + { + next(); + return *this; + } + + iterator& operator =(const iterator& src) + { + m_pNode = src.m_pNode; + m_guard.copy( src.m_guard ); + return *this; + } + + template + bool operator ==(iterator const& i ) const + { + return m_pNode == i.m_pNode; + } + template + bool operator !=(iterator const& i ) const + { + return !( *this == i ); + } + }; + }} // namespace lazy_skip_list::details + + template < + class GC + ,typename T +#ifdef CDS_DOXYGEN_INVOKED + ,typename Traits = lazy_skip_list::traits +#else + ,typename Traits +#endif + > + class LazySkipListSet + { + public: + typedef GC gc; + typedef T value_type; + typedef Traits traits; + + typedef typename traits::hook hook; + typedef typename hook::node_type node_type; + +# ifdef CDS_DOXYGEN_INVOKED + typedef implementation_defined key_comparator; +# else + typedef typename opt::details::make_comparator< value_type, traits >::type key_comparator; +# endif + + typedef typename traits::disposer disposer; + typedef typename get_node_traits< value_type, node_type, hook>::type node_traits; + + typedef typename traits::item_counter item_counter; + typedef typename traits::memory_model memory_model; + typedef typename traits::random_level_generator random_level_generator; + typedef typename traits::allocator allocator_type; + typedef typename traits::back_off back_off; + typedef typename traits::stat stat; + + public: + typedef typename gc::template guarded_ptr< value_type > guarded_ptr; + + static unsigned int const c_nMaxHeight = std::conditional< + (random_level_generator::c_nUpperBound <= lazy_skip_list::c_nHeightLimit), + std::integral_constant< unsigned int, random_level_generator::c_nUpperBound >, + std::integral_constant< unsigned int, lazy_skip_list::c_nHeightLimit > + >::type::value; + + static unsigned int const c_nMinHeight = 5; + + // c_nMaxHeight * 2 - pPred/pSucc guards + // + 1 - for erase, unlink + // + 1 - for clear + // + 1 - for help_remove() + static size_t const c_nHazardPtrCount = c_nMaxHeight * 2 + 3; + + protected: + typedef typename node_type::atomic_marked_ptr atomic_node_ptr; + typedef typename node_type::marked_ptr marked_node_ptr; + + protected: + typedef lazy_skip_list::details::intrusive_node_builder< node_type, atomic_node_ptr, allocator_type > intrusive_node_builder; + + typedef typename std::conditional< + std::is_same< typename traits::internal_node_builder, cds::opt::none >::value + ,intrusive_node_builder + ,typename traits::internal_node_builder + >::type node_builder; + + typedef std::unique_ptr< node_type, typename node_builder::node_disposer > scoped_node_ptr; + + struct position { + node_type * pPrev[ c_nMaxHeight ]; + node_type * pSucc[ c_nMaxHeight ]; + + typename gc::template GuardArray< c_nMaxHeight * 2 > guards; + node_type * pCur; // guarded by one of guards + }; + + public: + LazySkipListSet() + : m_Head( c_nMaxHeight ) + , m_nHeight( c_nMinHeight ) + { + static_assert( (std::is_same< gc, typename node_type::gc >::value), "GC and node_type::gc must be the same type" ); + + gc::check_available_guards( c_nHazardPtrCount ); + + // Barrier for head node + atomics::atomic_thread_fence( memory_model::memory_order_release ); + } + + ~LazySkipListSet() + { + destroy(); + } + + public: + + typedef lazy_skip_list::details::iterator< gc, node_traits, back_off, false > iterator; + + typedef lazy_skip_list::details::iterator< gc, node_traits, back_off, true > const_iterator; + + iterator begin() + { + return iterator( *m_Head.head()); + } + + const_iterator begin() const + { + return const_iterator( *m_Head.head()); + } + const_iterator cbegin() const + { + return const_iterator( *m_Head.head()); + } + + iterator end() + { + return iterator(); + } + + const_iterator end() const + { + return const_iterator(); + } + const_iterator cend() const + { + return const_iterator(); + } + + public: + + bool insert( value_type& val ) + { + return insert( val, []( value_type& ) {} ); + } + + template + bool insert( value_type& val, Func f ) + { + typename gc::Guard gNew; + gNew.assign( &val ); + + node_type * pNode = node_traits::to_node_ptr( val ); + scoped_node_ptr scp( pNode ); + unsigned int nHeight = pNode->height(); + bool bTowerOk = pNode->has_tower(); // nHeight > 1 && pNode->get_tower() != nullptr; + bool bTowerMade = false; + + position pos; + while ( true ) + { + if ( find_position( val, pos, key_comparator(), true )) { + // scoped_node_ptr deletes the node tower if we create it + if ( !bTowerMade ) + scp.release(); + + m_Stat.onInsertFailed(); + return false; + } + + if ( !bTowerOk ) { + build_node( pNode ); + nHeight = pNode->height(); + bTowerMade = pNode->has_tower(); + bTowerOk = true; + } + + if ( !insert_at_position( val, pNode, pos, f )) { + m_Stat.onInsertRetry(); + continue; + } + + increase_height( nHeight ); + ++m_ItemCounter; + m_Stat.onAddNode( nHeight ); + m_Stat.onInsertSuccess(); + scp.release(); + return true; + } + } + + template + std::pair update( value_type& val, Func func, bool bInsert = true ) + { + typename gc::Guard gNew; + gNew.assign( &val ); + + node_type * pNode = node_traits::to_node_ptr( val ); + scoped_node_ptr scp( pNode ); + unsigned int nHeight = pNode->height(); + bool bTowerOk = pNode->has_tower(); + bool bTowerMade = false; + + position pos; + while ( true ) + { + bool bFound = find_position( val, pos, key_comparator(), true ); + if ( bFound ) { + // scoped_node_ptr deletes the node tower if we create it before + if ( !bTowerMade ) + scp.release(); + + func( false, *node_traits::to_value_ptr(pos.pCur), val ); + m_Stat.onUpdateExist(); + return std::make_pair( true, false ); + } + + if ( !bInsert ) { + scp.release(); + return std::make_pair( false, false ); + } + + if ( !bTowerOk ) { + build_node( pNode ); + nHeight = pNode->height(); + bTowerMade = pNode->has_tower(); + bTowerOk = true; + } + + if ( !insert_at_position( val, pNode, pos, [&func]( value_type& item ) { func( true, item, item ); })) { + m_Stat.onInsertRetry(); + continue; + } + + increase_height( nHeight ); + ++m_ItemCounter; + scp.release(); + m_Stat.onAddNode( nHeight ); + m_Stat.onUpdateNew(); + return std::make_pair( true, true ); + } + } + + template + CDS_DEPRECATED("ensure() is deprecated, use update()") + std::pair ensure( value_type& val, Func func ) + { + return update( val, func, true ); + } + + bool unlink( value_type& val ) + { + position pos; + + if ( !find_position( val, pos, key_comparator(), false )) { + m_Stat.onUnlinkFailed(); + return false; + } + + node_type * pDel = pos.pCur; + assert( key_comparator()( *node_traits::to_value_ptr( pDel ), val ) == 0 ); + + unsigned int nHeight = pDel->height(); + typename gc::Guard gDel; + gDel.assign( node_traits::to_value_ptr(pDel)); + + if ( node_traits::to_value_ptr( pDel ) == &val && try_remove_at( pDel, pos, [](value_type const&) {} )) { + --m_ItemCounter; + m_Stat.onRemoveNode( nHeight ); + m_Stat.onUnlinkSuccess(); + return true; + } + + m_Stat.onUnlinkFailed(); + return false; + } + + template + guarded_ptr extract( Q const& key ) + { + return extract_( key, key_comparator()); + } + + template + guarded_ptr extract_with( Q const& key, Less pred ) + { + CDS_UNUSED( pred ); + return extract_( key, cds::opt::details::make_comparator_from_less()); + } + + guarded_ptr extract_min() + { + return extract_min_(); + } + + guarded_ptr extract_max() + { + return extract_max_(); + } + + template + bool erase( Q const& key ) + { + return erase_( key, key_comparator(), [](value_type const&) {} ); + } + + template + bool erase_with( Q const& key, Less pred ) + { + CDS_UNUSED( pred ); + return erase_( key, cds::opt::details::make_comparator_from_less(), [](value_type const&) {} ); + } + + template + bool erase( Q const& key, Func f ) + { + return erase_( key, key_comparator(), f ); + } + + template + bool erase_with( Q const& key, Less pred, Func f ) + { + CDS_UNUSED( pred ); + return erase_( key, cds::opt::details::make_comparator_from_less(), f ); + } + + template + bool find( Q& key, Func f ) + { + return find_with_( key, key_comparator(), f ); + } + + template + bool find( Q const& key, Func f ) + { + return find_with_( key, key_comparator(), f ); + } + + template + bool find_with( Q& key, Less pred, Func f ) + { + CDS_UNUSED( pred ); + return find_with_( key, cds::opt::details::make_comparator_from_less(), f ); + } + + template + bool find_with( Q const& key, Less pred, Func f ) + { + CDS_UNUSED( pred ); + return find_with_( key, cds::opt::details::make_comparator_from_less(), f ); + } + + template + bool contains( Q const& key ) + { + return find_with_( key, key_comparator(), [](value_type& , Q const& ) {} ); + } + + template + CDS_DEPRECATED("deprecated, use contains()") + bool find( Q const& key ) + { + return contains( key ); + } + + template + bool contains( Q const& key, Less pred ) + { + CDS_UNUSED( pred ); + return find_with_( key, cds::opt::details::make_comparator_from_less(), [](value_type& , Q const& ) {} ); + } + + template + CDS_DEPRECATED("deprecated, use contains()") + bool find_with( Q const& key, Less pred ) + { + return contains( key, pred ); + } + + template + guarded_ptr get( Q const& key ) + { + return get_with_( key, key_comparator()); + } + + template + guarded_ptr get_with( Q const& key, Less pred ) + { + CDS_UNUSED( pred ); + return get_with_( key, cds::opt::details::make_comparator_from_less()); + } + + size_t size() const + { + return m_ItemCounter; + } + + bool empty() const + { + return m_Head.head()->next( 0 ).load( memory_model::memory_order_relaxed ) == nullptr; + } + + void clear() + { + while ( extract_min_()); + } + + static constexpr unsigned int max_height() noexcept + { + return c_nMaxHeight; + } + + stat const& statistics() const + { + return m_Stat; + } + + protected: + + unsigned int random_level() + { + // Random generator produces a number from range [0..31] + // We need a number from range [1..32] + return m_RandomLevelGen() + 1; + } + + template + node_type * build_node( Q v ) + { + return node_builder::make_tower( v, m_RandomLevelGen ); + } + + static value_type * gc_protect( marked_node_ptr p ) + { + return node_traits::to_value_ptr( p.ptr()); + } + + static void dispose_node( void* p ) + { + assert( p != nullptr ); + value_type* pVal = reinterpret_cast( p ); + typename node_builder::node_disposer()( node_traits::to_node_ptr( pVal )); + disposer()( pVal ); + } + + void help_remove( int nLevel, node_type* pPred, marked_node_ptr pCur ) + { + if ( pCur->is_upper_level( nLevel )) { + marked_node_ptr p( pCur.ptr()); + typename gc::Guard hp; + marked_node_ptr pSucc = hp.protect( pCur->next( nLevel ), gc_protect ); + + if ( pSucc.bits() && + pPred->next( nLevel ).compare_exchange_strong( p, marked_node_ptr( pSucc.ptr()), + memory_model::memory_order_acquire, atomics::memory_order_relaxed )) + { + if ( pCur->level_unlinked()) { + gc::retire( node_traits::to_value_ptr( pCur.ptr()), dispose_node ); + m_Stat.onEraseWhileFind(); + } + } + } + } + + template + bool find_position( Q const& val, position& pos, Compare cmp, bool bStopIfFound ) + { + node_type * pPred; + marked_node_ptr pSucc; + marked_node_ptr pCur; + + // Hazard pointer array: + // pPred: [nLevel * 2] + // pSucc: [nLevel * 2 + 1] + + retry: + pPred = m_Head.head(); + int nCmp = 1; + + for ( int nLevel = static_cast( c_nMaxHeight - 1 ); nLevel >= 0; --nLevel ) { + pos.guards.assign( nLevel * 2, node_traits::to_value_ptr( pPred )); + while ( true ) { + pCur = pos.guards.protect( nLevel * 2 + 1, pPred->next( nLevel ), gc_protect ); + if ( pCur.bits()) { + // pCur.bits() means that pPred is logically deleted + goto retry; + } + + if ( pCur.ptr() == nullptr ) { + // end of list at level nLevel - goto next level + break; + } + + // pSucc contains deletion mark for pCur + pSucc = pCur->next( nLevel ).load( memory_model::memory_order_acquire ); + + if ( pPred->next( nLevel ).load( memory_model::memory_order_acquire ).all() != pCur.ptr()) + goto retry; + + if ( pSucc.bits()) { + // pCur is marked, i.e. logically deleted + // try to help deleting pCur + help_remove( nLevel, pPred, pCur ); + goto retry; + } + else { + nCmp = cmp( *node_traits::to_value_ptr( pCur.ptr()), val ); + if ( nCmp < 0 ) { + pPred = pCur.ptr(); + pos.guards.copy( nLevel * 2, nLevel * 2 + 1 ); // pPrev guard := cur guard + } + else if ( nCmp == 0 && bStopIfFound ) + goto found; + else + break; + } + } + + // Next level + pos.pPrev[nLevel] = pPred; + pos.pSucc[nLevel] = pCur.ptr(); + } + + if ( nCmp != 0 ) + return false; + + found: + pos.pCur = pCur.ptr(); + return pCur.ptr() && nCmp == 0; + } + + bool find_min_position( position& pos ) + { + node_type * pPred; + marked_node_ptr pSucc; + marked_node_ptr pCur; + + // Hazard pointer array: + // pPred: [nLevel * 2] + // pSucc: [nLevel * 2 + 1] + + retry: + pPred = m_Head.head(); + + for ( int nLevel = static_cast( c_nMaxHeight - 1 ); nLevel >= 0; --nLevel ) { + pos.guards.assign( nLevel * 2, node_traits::to_value_ptr( pPred )); + pCur = pos.guards.protect( nLevel * 2 + 1, pPred->next( nLevel ), gc_protect ); + + // pCur.bits() means that pPred is logically deleted + // head cannot be deleted + assert( pCur.bits() == 0 ); + + if ( pCur.ptr()) { + + // pSucc contains deletion mark for pCur + pSucc = pCur->next( nLevel ).load( memory_model::memory_order_acquire ); + + if ( pPred->next( nLevel ).load( memory_model::memory_order_acquire ).all() != pCur.ptr()) + goto retry; + + if ( pSucc.bits()) { + // pCur is marked, i.e. logically deleted. + // try to help deleting pCur + help_remove( nLevel, pPred, pCur ); + goto retry; + } + } + + // Next level + pos.pPrev[nLevel] = pPred; + pos.pSucc[nLevel] = pCur.ptr(); + } + + return ( pos.pCur = pCur.ptr()) != nullptr; + } + + bool find_max_position( position& pos ) + { + node_type * pPred; + marked_node_ptr pSucc; + marked_node_ptr pCur; + + // Hazard pointer array: + // pPred: [nLevel * 2] + // pSucc: [nLevel * 2 + 1] + + retry: + pPred = m_Head.head(); + + for ( int nLevel = static_cast( c_nMaxHeight - 1 ); nLevel >= 0; --nLevel ) { + pos.guards.assign( nLevel * 2, node_traits::to_value_ptr( pPred )); + while ( true ) { + pCur = pos.guards.protect( nLevel * 2 + 1, pPred->next( nLevel ), gc_protect ); + if ( pCur.bits()) { + // pCur.bits() means that pPred is logically deleted + goto retry; + } + + if ( pCur.ptr() == nullptr ) { + // end of the list at level nLevel - goto next level + break; + } + + // pSucc contains deletion mark for pCur + pSucc = pCur->next( nLevel ).load( memory_model::memory_order_acquire ); + + if ( pPred->next( nLevel ).load( memory_model::memory_order_acquire ).all() != pCur.ptr()) + goto retry; + + if ( pSucc.bits()) { + // pCur is marked, i.e. logically deleted. + // try to help deleting pCur + help_remove( nLevel, pPred, pCur ); + goto retry; + } + else { + if ( !pSucc.ptr()) + break; + + pPred = pCur.ptr(); + pos.guards.copy( nLevel * 2, nLevel * 2 + 1 ); + } + } + + // Next level + pos.pPrev[nLevel] = pPred; + pos.pSucc[nLevel] = pCur.ptr(); + } + + return ( pos.pCur = pCur.ptr()) != nullptr; + } + + bool renew_insert_position( value_type& val, node_type * pNode, position& pos ) + { + node_type * pPred; + marked_node_ptr pSucc; + marked_node_ptr pCur; + key_comparator cmp; + + // Hazard pointer array: + // pPred: [nLevel * 2] + // pSucc: [nLevel * 2 + 1] + + retry: + pPred = m_Head.head(); + int nCmp = 1; + + for ( int nLevel = static_cast( c_nMaxHeight - 1 ); nLevel >= 0; --nLevel ) { + pos.guards.assign( nLevel * 2, node_traits::to_value_ptr( pPred )); + while ( true ) { + pCur = pos.guards.protect( nLevel * 2 + 1, pPred->next( nLevel ), gc_protect ); + if ( pCur.bits()) { + // pCur.bits() means that pPred is logically deleted + goto retry; + } + + if ( pCur.ptr() == nullptr ) { + // end of list at level nLevel - goto next level + break; + } + + // pSucc contains deletion mark for pCur + pSucc = pCur->next( nLevel ).load( memory_model::memory_order_acquire ); + + if ( pPred->next( nLevel ).load( memory_model::memory_order_acquire ).all() != pCur.ptr()) + goto retry; + + if ( pSucc.bits()) { + // pCur is marked, i.e. logically deleted + if ( pCur.ptr() == pNode ) { + // Node is removing while we are inserting it + return false; + } + // try to help deleting pCur + help_remove( nLevel, pPred, pCur ); + goto retry; + } + else { + nCmp = cmp( *node_traits::to_value_ptr( pCur.ptr()), val ); + if ( nCmp < 0 ) { + pPred = pCur.ptr(); + pos.guards.copy( nLevel * 2, nLevel * 2 + 1 ); // pPrev guard := cur guard + } + else + break; + } + } + + // Next level + pos.pPrev[nLevel] = pPred; + pos.pSucc[nLevel] = pCur.ptr(); + } + + return nCmp == 0; + } + + template + bool insert_at_position( value_type& val, node_type * pNode, position& pos, Func f ) + { + unsigned int const nHeight = pNode->height(); + + for ( unsigned int nLevel = 1; nLevel < nHeight; ++nLevel ) + pNode->next( nLevel ).store( marked_node_ptr(), memory_model::memory_order_relaxed ); + + // Insert at level 0 + { + marked_node_ptr p( pos.pSucc[0] ); + pNode->next( 0 ).store( p, memory_model::memory_order_release ); + if ( !pos.pPrev[0]->next( 0 ).compare_exchange_strong( p, marked_node_ptr( pNode ), memory_model::memory_order_release, atomics::memory_order_relaxed )) + return false; + + f( val ); + } + + // Insert at level 1..max + for ( unsigned int nLevel = 1; nLevel < nHeight; ++nLevel ) { + marked_node_ptr p; + while ( true ) { + marked_node_ptr pSucc( pos.pSucc[nLevel] ); + + // Set pNode->next + // pNode->next can have "logical deleted" flag if another thread is removing pNode right now + if ( !pNode->next( nLevel ).compare_exchange_strong( p, pSucc, + memory_model::memory_order_release, atomics::memory_order_acquire )) + { + // pNode has been marked as removed while we are inserting it + // Stop inserting + assert( p.bits() != 0 ); + + // Here pNode is linked at least level 0 so level_unlinked() cannot returns true + CDS_VERIFY_FALSE( pNode->level_unlinked( nHeight - nLevel )); + + // pNode is linked up to nLevel - 1 + // Remove it via find_position() + find_position( val, pos, key_comparator(), false ); + + m_Stat.onLogicDeleteWhileInsert(); + return true; + } + p = pSucc; + + // Link pNode into the list at nLevel + if ( pos.pPrev[nLevel]->next( nLevel ).compare_exchange_strong( pSucc, marked_node_ptr( pNode ), + memory_model::memory_order_release, atomics::memory_order_relaxed )) + { + // go to next level + break; + } + + // Renew insert position + m_Stat.onRenewInsertPosition(); + + if ( !renew_insert_position( val, pNode, pos )) { + // The node has been deleted while we are inserting it + // Update current height for concurent removing + CDS_VERIFY_FALSE( pNode->level_unlinked( nHeight - nLevel )); + + m_Stat.onRemoveWhileInsert(); + + // help to removing val + find_position( val, pos, key_comparator(), false ); + return true; + } + } + } + return true; + } + + template + bool try_remove_at( node_type * pDel, position& pos, Func f ) + { + assert( pDel != nullptr ); + + marked_node_ptr pSucc; + back_off bkoff; + + // logical deletion (marking) + for ( unsigned int nLevel = pDel->height() - 1; nLevel > 0; --nLevel ) { + pSucc = pDel->next( nLevel ).load( memory_model::memory_order_relaxed ); + if ( pSucc.bits() == 0 ) { + bkoff.reset(); + while ( !( pDel->next( nLevel ).compare_exchange_weak( pSucc, pSucc | 1, + memory_model::memory_order_release, atomics::memory_order_acquire ) + || pSucc.bits() != 0 )) + { + bkoff(); + m_Stat.onMarkFailed(); + } + } + } + + marked_node_ptr p( pDel->next( 0 ).load( memory_model::memory_order_relaxed ).ptr()); + while ( true ) { + if ( pDel->next( 0 ).compare_exchange_strong( p, p | 1, memory_model::memory_order_release, atomics::memory_order_acquire )) + { + f( *node_traits::to_value_ptr( pDel )); + + // Physical deletion + // try fast erase + p = pDel; + + for ( int nLevel = static_cast( pDel->height() - 1 ); nLevel >= 0; --nLevel ) { + + pSucc = pDel->next( nLevel ).load( memory_model::memory_order_acquire ); + if ( pos.pPrev[nLevel]->next( nLevel ).compare_exchange_strong( p, marked_node_ptr( pSucc.ptr()), + memory_model::memory_order_acq_rel, atomics::memory_order_relaxed )) + { + pDel->level_unlinked(); + } + else { + // Make slow erase +# ifdef CDS_DEBUG + if ( find_position( *node_traits::to_value_ptr( pDel ), pos, key_comparator(), false )) + assert( pDel != pos.pCur ); +# else + find_position( *node_traits::to_value_ptr( pDel ), pos, key_comparator(), false ); +# endif + m_Stat.onSlowErase(); + return true; + } + } + + // Fast erasing success + gc::retire( node_traits::to_value_ptr( pDel ), dispose_node ); + m_Stat.onFastErase(); + return true; + } + else if ( p.bits()) { + // Another thread is deleting pDel right now + m_Stat.onEraseContention(); + return false; + } + m_Stat.onEraseRetry(); + bkoff(); + } + } + + enum finsd_fastpath_result { + find_fastpath_found, + find_fastpath_not_found, + find_fastpath_abort + }; + template + finsd_fastpath_result find_fastpath( Q& val, Compare cmp, Func f ) + { + node_type * pPred; + marked_node_ptr pCur; + marked_node_ptr pNull; + + // guard array: + // 0 - pPred on level N + // 1 - pCur on level N + typename gc::template GuardArray<2> guards; + back_off bkoff; + unsigned attempt = 0; + + try_again: + pPred = m_Head.head(); + for ( int nLevel = static_cast( m_nHeight.load( memory_model::memory_order_relaxed ) - 1 ); nLevel >= 0; --nLevel ) { + pCur = guards.protect( 1, pPred->next( nLevel ), gc_protect ); + + while ( pCur != pNull ) { + if ( pCur.bits()) { + // pPred is being removed + if ( ++attempt < 4 ) { + bkoff(); + goto try_again; + } + + return find_fastpath_abort; + } + + if ( pCur.ptr()) { + int nCmp = cmp( *node_traits::to_value_ptr( pCur.ptr()), val ); + if ( nCmp < 0 ) { + guards.copy( 0, 1 ); + pPred = pCur.ptr(); + pCur = guards.protect( 1, pCur->next( nLevel ), gc_protect ); + } + else if ( nCmp == 0 ) { + // found + f( *node_traits::to_value_ptr( pCur.ptr()), val ); + return find_fastpath_found; + } + else { + // pCur > val - go down + break; + } + } + } + } + + return find_fastpath_not_found; + } + + template + bool find_slowpath( Q& val, Compare cmp, Func f ) + { + position pos; + if ( find_position( val, pos, cmp, true )) { + assert( cmp( *node_traits::to_value_ptr( pos.pCur ), val ) == 0 ); + + f( *node_traits::to_value_ptr( pos.pCur ), val ); + return true; + } + else + return false; + } + + template + bool find_with_( Q& val, Compare cmp, Func f ) + { + switch ( find_fastpath( val, cmp, f )) { + case find_fastpath_found: + m_Stat.onFindFastSuccess(); + return true; + case find_fastpath_not_found: + m_Stat.onFindFastFailed(); + return false; + default: + break; + } + + if ( find_slowpath( val, cmp, f )) { + m_Stat.onFindSlowSuccess(); + return true; + } + + m_Stat.onFindSlowFailed(); + return false; + } + + template + guarded_ptr get_with_( Q const& val, Compare cmp ) + { + guarded_ptr gp; + if ( find_with_( val, cmp, [&gp]( value_type& found, Q const& ) { gp.reset( &found ); } )) + return gp; + return guarded_ptr(); + } + + template + bool erase_( Q const& val, Compare cmp, Func f ) + { + position pos; + + if ( !find_position( val, pos, cmp, false )) { + m_Stat.onEraseFailed(); + return false; + } + + node_type * pDel = pos.pCur; + typename gc::Guard gDel; + gDel.assign( node_traits::to_value_ptr( pDel )); + assert( cmp( *node_traits::to_value_ptr( pDel ), val ) == 0 ); + + unsigned int nHeight = pDel->height(); + if ( try_remove_at( pDel, pos, f )) { + --m_ItemCounter; + m_Stat.onRemoveNode( nHeight ); + m_Stat.onEraseSuccess(); + return true; + } + + m_Stat.onEraseFailed(); + return false; + } + + template + guarded_ptr extract_( Q const& val, Compare cmp ) + { + position pos; + + guarded_ptr gp; + for (;;) { + if ( !find_position( val, pos, cmp, false )) { + m_Stat.onExtractFailed(); + return guarded_ptr(); + } + + node_type * pDel = pos.pCur; + gp.reset( node_traits::to_value_ptr( pDel )); + assert( cmp( *node_traits::to_value_ptr( pDel ), val ) == 0 ); + + unsigned int nHeight = pDel->height(); + if ( try_remove_at( pDel, pos, []( value_type const& ) {} )) { + --m_ItemCounter; + m_Stat.onRemoveNode( nHeight ); + m_Stat.onExtractSuccess(); + return gp; + } + m_Stat.onExtractRetry(); + } + } + + guarded_ptr extract_min_() + { + position pos; + + guarded_ptr gp; + for ( ;;) { + if ( !find_min_position( pos )) { + // The list is empty + m_Stat.onExtractMinFailed(); + return guarded_ptr(); + } + + node_type * pDel = pos.pCur; + + unsigned int nHeight = pDel->height(); + gp.reset( node_traits::to_value_ptr( pDel )); + + if ( try_remove_at( pDel, pos, []( value_type const& ) {} )) { + --m_ItemCounter; + m_Stat.onRemoveNode( nHeight ); + m_Stat.onExtractMinSuccess(); + return gp; + } + + m_Stat.onExtractMinRetry(); + } + } + + guarded_ptr extract_max_() + { + position pos; + + guarded_ptr gp; + for ( ;;) { + if ( !find_max_position( pos )) { + // The list is empty + m_Stat.onExtractMaxFailed(); + return guarded_ptr(); + } + + node_type * pDel = pos.pCur; + + unsigned int nHeight = pDel->height(); + gp.reset( node_traits::to_value_ptr( pDel )); + + if ( try_remove_at( pDel, pos, []( value_type const& ) {} )) { + --m_ItemCounter; + m_Stat.onRemoveNode( nHeight ); + m_Stat.onExtractMaxSuccess(); + return gp; + } + + m_Stat.onExtractMaxRetry(); + } + } + + void increase_height( unsigned int nHeight ) + { + unsigned int nCur = m_nHeight.load( memory_model::memory_order_relaxed ); + if ( nCur < nHeight ) + m_nHeight.compare_exchange_strong( nCur, nHeight, memory_model::memory_order_relaxed, atomics::memory_order_relaxed ); + } + + void destroy() + { + node_type* p = m_Head.head()->next( 0 ).load( atomics::memory_order_relaxed ).ptr(); + while ( p ) { + node_type* pNext = p->next( 0 ).load( atomics::memory_order_relaxed ).ptr(); + dispose_node( node_traits::to_value_ptr( p )); + p = pNext; + } + } + + + private: + + lazy_skip_list::details::head_node< node_type > m_Head; + + random_level_generator m_RandomLevelGen; + atomics::atomic m_nHeight; + item_counter m_ItemCounter; + mutable stat m_Stat; + }; + +}} // namespace cds::intrusive + + +#endif // #ifndef CDSLIB_INTRUSIVE_IMPL_LAZY_SKIP_LIST_H diff --git a/cds/intrusive/lazy_skip_list_dhp.h b/cds/intrusive/lazy_skip_list_dhp.h new file mode 100644 index 000000000..6fbe567c6 --- /dev/null +++ b/cds/intrusive/lazy_skip_list_dhp.h @@ -0,0 +1,7 @@ +#ifndef CDSLIB_INTRUSIVE_LAZY_SKIP_LIST_DHP_H +#define CDSLIB_INTRUSIVE_LAZY_SKIP_LIST_DHP_H + +#include +#include + +#endif diff --git a/cds/intrusive/lazy_skip_list_hp.h b/cds/intrusive/lazy_skip_list_hp.h new file mode 100644 index 000000000..68593d7ed --- /dev/null +++ b/cds/intrusive/lazy_skip_list_hp.h @@ -0,0 +1,12 @@ +// Copyright (c) 2006-2018 Maxim Khizhinsky +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt) + +#ifndef CDSLIB_INTRUSIVE_LAZY_SKIP_LIST_HP_H +#define CDSLIB_INTRUSIVE_LAZY_SKIP_LIST_HP_H + +#include +#include + +#endif diff --git a/test/include/cds_test/stat_lazyskiplist_out.h b/test/include/cds_test/stat_lazyskiplist_out.h new file mode 100644 index 000000000..7b92b1ba6 --- /dev/null +++ b/test/include/cds_test/stat_lazyskiplist_out.h @@ -0,0 +1,61 @@ +#ifndef CDSTEST_STAT_LAZYSKIPLIST_OUT_H +#define CDSTEST_STAT_LAZYSKIPLIST_OUT_H + +#include + +namespace cds_test { + + static inline property_stream& operator <<( property_stream& o, cds::intrusive::lazy_skip_list::empty_stat const& /*s*/ ) + { + return o; + } + + static inline property_stream& operator <<( property_stream& o, cds::intrusive::lazy_skip_list::stat<> const& s ) + { + { + std::stringstream stm; + for ( unsigned int i = 0; i < sizeof( s.m_nNodeHeightAdd ) / sizeof( s.m_nNodeHeightAdd[0] ); ++i ) + stm << " +" << s.m_nNodeHeightAdd[i].get() << "/-" << s.m_nNodeHeightDel[i].get(); + o << CDSSTRESS_STAT_OUT_( "stat.level_ins_del", stm.str().substr( 1 ).c_str()); + } + + return o + << CDSSTRESS_STAT_OUT( s, m_nInsertSuccess ) + << CDSSTRESS_STAT_OUT( s, m_nInsertFailed ) + << CDSSTRESS_STAT_OUT( s, m_nInsertRetries ) + << CDSSTRESS_STAT_OUT( s, m_nUpdateExist ) + << CDSSTRESS_STAT_OUT( s, m_nUpdateNew ) + << CDSSTRESS_STAT_OUT( s, m_nUnlinkSuccess ) + << CDSSTRESS_STAT_OUT( s, m_nUnlinkFailed ) + << CDSSTRESS_STAT_OUT( s, m_nExtractSuccess ) + << CDSSTRESS_STAT_OUT( s, m_nExtractFailed ) + << CDSSTRESS_STAT_OUT( s, m_nExtractRetries ) + << CDSSTRESS_STAT_OUT( s, m_nExtractMinSuccess ) + << CDSSTRESS_STAT_OUT( s, m_nExtractMinFailed ) + << CDSSTRESS_STAT_OUT( s, m_nExtractMinRetries ) + << CDSSTRESS_STAT_OUT( s, m_nExtractMaxSuccess ) + << CDSSTRESS_STAT_OUT( s, m_nExtractMaxFailed ) + << CDSSTRESS_STAT_OUT( s, m_nExtractMaxRetries ) + << CDSSTRESS_STAT_OUT( s, m_nEraseSuccess ) + << CDSSTRESS_STAT_OUT( s, m_nEraseFailed ) + << CDSSTRESS_STAT_OUT( s, m_nEraseRetry ) + << CDSSTRESS_STAT_OUT( s, m_nFindFastSuccess ) + << CDSSTRESS_STAT_OUT( s, m_nFindFastFailed ) + << CDSSTRESS_STAT_OUT( s, m_nFindSlowSuccess ) + << CDSSTRESS_STAT_OUT( s, m_nFindSlowFailed ) + << CDSSTRESS_STAT_OUT( s, m_nRenewInsertPosition ) + << CDSSTRESS_STAT_OUT( s, m_nLogicDeleteWhileInsert ) + << CDSSTRESS_STAT_OUT( s, m_nRemoveWhileInsert ) + << CDSSTRESS_STAT_OUT( s, m_nFastErase ) + << CDSSTRESS_STAT_OUT( s, m_nSlowErase ) + << CDSSTRESS_STAT_OUT( s, m_nFastExtract ) + << CDSSTRESS_STAT_OUT( s, m_nSlowExtract ) + << CDSSTRESS_STAT_OUT( s, m_nEraseWhileFind ) + << CDSSTRESS_STAT_OUT( s, m_nExtractWhileFind ) + << CDSSTRESS_STAT_OUT( s, m_nMarkFailed ) + << CDSSTRESS_STAT_OUT( s, m_nEraseContention ); + } + +} // namespace cds_test + +#endif // #ifndef CDSTEST_STAT_LAZYSKIPLIST_OUT_H diff --git a/test/stress/set/del3/CMakeLists.txt b/test/stress/set/del3/CMakeLists.txt index 40aec3c0d..746f1046e 100644 --- a/test/stress/set/del3/CMakeLists.txt +++ b/test/stress/set/del3/CMakeLists.txt @@ -6,6 +6,7 @@ set(CDSSTRESS_SET_DEL3_SOURCES set_del3_cuckoo.cpp set_del3_ellentree.cpp set_del3_feldman_hashset.cpp + set_del3_lazy_skip.cpp set_del3_michael.cpp set_del3_skip.cpp set_del3_split.cpp diff --git a/test/stress/set/del3/set_del3_lazy_skip.cpp b/test/stress/set/del3/set_del3_lazy_skip.cpp new file mode 100644 index 000000000..9f7686fb3 --- /dev/null +++ b/test/stress/set/del3/set_del3_lazy_skip.cpp @@ -0,0 +1,8 @@ +#include "set_del3.h" +#include "set_type_lazy_skip_list.h" + +namespace set { + + CDSSTRESS_LazySkipListSet( Set_Del3, run_test_extract, key_thread, size_t ) + +} // namespace set diff --git a/test/stress/set/delodd/CMakeLists.txt b/test/stress/set/delodd/CMakeLists.txt index d16782a23..c5f27fd1d 100644 --- a/test/stress/set/delodd/CMakeLists.txt +++ b/test/stress/set/delodd/CMakeLists.txt @@ -6,6 +6,7 @@ set(CDSSTRESS_SET_DELODD_SOURCES set_delodd_cuckoo.cpp set_delodd_ellentree.cpp set_delodd_feldman_hashset.cpp + set_delodd_lazy_skip.cpp set_delodd_michael.cpp set_delodd_skip.cpp set_delodd_split.cpp diff --git a/test/stress/set/delodd/set_delodd_lazy_skip.cpp b/test/stress/set/delodd/set_delodd_lazy_skip.cpp new file mode 100644 index 000000000..4ab0f2425 --- /dev/null +++ b/test/stress/set/delodd/set_delodd_lazy_skip.cpp @@ -0,0 +1,8 @@ +#include "set_delodd.h" +#include "set_type_lazy_skip_list.h" + +namespace set { + + CDSSTRESS_LazySkipListSet( Set_DelOdd, run_test_extract, key_thread, size_t ) + +} // namespace set diff --git a/test/stress/set/insdel_find/CMakeLists.txt b/test/stress/set/insdel_find/CMakeLists.txt index 2d0ceb77f..85833ce72 100644 --- a/test/stress/set/insdel_find/CMakeLists.txt +++ b/test/stress/set/insdel_find/CMakeLists.txt @@ -7,6 +7,7 @@ set(CDSSTRESS_SET_INSDELFIND_HP_SOURCES set_insdelfind.cpp set_insdelfind_ellentree_hp.cpp set_insdelfind_feldman_hashset_hp.cpp + set_insdelfind_lazy_skip_hp.cpp set_insdelfind_michael_hp.cpp set_insdelfind_skip_hp.cpp set_insdelfind_split_hp.cpp diff --git a/test/stress/set/insdel_find/set_insdelfind_lazy_skip_hp.cpp b/test/stress/set/insdel_find/set_insdelfind_lazy_skip_hp.cpp new file mode 100644 index 000000000..093dd6eac --- /dev/null +++ b/test/stress/set/insdel_find/set_insdelfind_lazy_skip_hp.cpp @@ -0,0 +1,8 @@ +#include "set_insdelfind.h" +#include "set_type_lazy_skip_list.h" + +namespace set { + + CDSSTRESS_LazySkipListSet_HP( Set_InsDelFind, run_test, size_t, size_t ) + +} // namespace set diff --git a/test/stress/set/insdel_func/CMakeLists.txt b/test/stress/set/insdel_func/CMakeLists.txt index 43f4d3a02..3f04a6869 100644 --- a/test/stress/set/insdel_func/CMakeLists.txt +++ b/test/stress/set/insdel_func/CMakeLists.txt @@ -6,6 +6,7 @@ set(CDSSTRESS_SET_INSDEL_FUNC_SOURCES set_insdel_func_cuckoo.cpp set_insdel_func_ellentree.cpp set_insdel_func_feldman_hashset.cpp + set_insdel_func_lazy_skip.cpp set_insdel_func_michael.cpp set_insdel_func_skip.cpp set_insdel_func_split.cpp diff --git a/test/stress/set/insdel_func/set_insdel_func_lazy_skip.cpp b/test/stress/set/insdel_func/set_insdel_func_lazy_skip.cpp new file mode 100644 index 000000000..66d234674 --- /dev/null +++ b/test/stress/set/insdel_func/set_insdel_func_lazy_skip.cpp @@ -0,0 +1,8 @@ +#include "set_insdel_func.h" +#include "set_type_lazy_skip_list.h" + +namespace set { + + CDSSTRESS_LazySkipListSet( Set_InsDel_func, run_test, size_t, value ) + +} // namespace set diff --git a/test/stress/set/insdel_string/CMakeLists.txt b/test/stress/set/insdel_string/CMakeLists.txt index 57b0d56a2..18ddfe639 100644 --- a/test/stress/set/insdel_string/CMakeLists.txt +++ b/test/stress/set/insdel_string/CMakeLists.txt @@ -6,6 +6,7 @@ set(CDSSTRESS_SET_INSDEL_STRING_SOURCES set_insdel_string_cuckoo.cpp set_insdel_string_ellentree.cpp set_insdel_string_feldman_hashset.cpp + set_insdel_string_lazy_skip.cpp set_insdel_string_michael.cpp set_insdel_string_skip.cpp set_insdel_string_split.cpp diff --git a/test/stress/set/insdel_string/set_insdel_string_lazy_skip.cpp b/test/stress/set/insdel_string/set_insdel_string_lazy_skip.cpp new file mode 100644 index 000000000..f87e1742e --- /dev/null +++ b/test/stress/set/insdel_string/set_insdel_string_lazy_skip.cpp @@ -0,0 +1,8 @@ +#include "set_insdel_string.h" +#include "set_type_lazy_skip_list.h" + +namespace set { + + CDSSTRESS_LazySkipListSet( Set_InsDel_string, run_test_extract, std::string, size_t ) + +} // namespace set diff --git a/test/stress/set/set_type_lazy_skip_list.h b/test/stress/set/set_type_lazy_skip_list.h new file mode 100644 index 000000000..eb97eb2a2 --- /dev/null +++ b/test/stress/set/set_type_lazy_skip_list.h @@ -0,0 +1,267 @@ +#ifndef CDSUNIT_SET_TYPE_LAZY_SKIP_LIST_H +#define CDSUNIT_SET_TYPE_LAZY_SKIP_LIST_H + +#include "set_type.h" + +#include +#include + +#include + +namespace set { + + template + class LazySkipListSet : public cc::LazySkipListSet + { + typedef cc::LazySkipListSet base_class; + public: + template + LazySkipListSet( Config const& /*cfg*/ ) + {} + + // for testing + static constexpr bool const c_bExtractSupported = true; + static constexpr bool const c_bLoadFactorDepended = false; + static constexpr bool const c_bEraseExactKey = false; + }; + + struct tag_LazySkipListSet; + + template + struct set_type< tag_LazySkipListSet, Key, Val >: public set_type_base< Key, Val > + { + typedef set_type_base< Key, Val > base_class; + typedef typename base_class::key_val key_val; + typedef typename base_class::compare compare; + typedef typename base_class::less less; + typedef typename base_class::hash hash; + + class traits_LazySkipListSet_less_turbo32: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::turbo32 > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_turbo32 > LazySkipListSet_hp_less_turbo32; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_turbo32 > LazySkipListSet_dhp_less_turbo32; + + class traits_LazySkipListSet_less_turbo24: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::turbo24 > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_turbo24 > LazySkipListSet_hp_less_turbo24; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_turbo24 > LazySkipListSet_dhp_less_turbo24; + + class traits_LazySkipListSet_less_turbo16: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::turbo16 > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_turbo16 > LazySkipListSet_hp_less_turbo16; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_turbo16 > LazySkipListSet_dhp_less_turbo16; + + class traits_LazySkipListSet_less_turbo32_seqcst: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::turbo32 > + ,co::memory_model< co::v::sequential_consistent > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_turbo32_seqcst > LazySkipListSet_hp_less_turbo32_seqcst; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_turbo32_seqcst > LazySkipListSet_dhp_less_turbo32_seqcst; + + class traits_LazySkipListSet_less_turbo32_stat: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::turbo32 > + ,co::stat< cc::lazy_skip_list::stat<> > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_turbo32_stat > LazySkipListSet_hp_less_turbo32_stat; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_turbo32_stat > LazySkipListSet_dhp_less_turbo32_stat; + + class traits_LazySkipListSet_less_turbo24_stat: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::turbo24 > + ,co::stat< cc::lazy_skip_list::stat<> > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_turbo24_stat > LazySkipListSet_hp_less_turbo24_stat; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_turbo24_stat > LazySkipListSet_dhp_less_turbo24_stat; + + class traits_LazySkipListSet_less_turbo16_stat: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::turbo16 > + ,co::stat< cc::lazy_skip_list::stat<> > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_turbo16_stat > LazySkipListSet_hp_less_turbo16_stat; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_turbo16_stat > LazySkipListSet_dhp_less_turbo16_stat; + + class traits_LazySkipListSet_cmp_turbo32: public cc::lazy_skip_list::make_traits < + co::compare< compare > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::turbo32 > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_cmp_turbo32 > LazySkipListSet_hp_cmp_turbo32; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_cmp_turbo32 > LazySkipListSet_dhp_cmp_turbo32; + + class traits_LazySkipListSet_cmp_turbo32_stat: public cc::lazy_skip_list::make_traits < + co::compare< compare > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::turbo32 > + ,co::stat< cc::lazy_skip_list::stat<> > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_cmp_turbo32_stat > LazySkipListSet_hp_cmp_turbo32_stat; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_cmp_turbo32_stat > LazySkipListSet_dhp_cmp_turbo32_stat; + + class traits_LazySkipListSet_less_xorshift32: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::xorshift32 > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_xorshift32 > LazySkipListSet_hp_less_xorshift32; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_xorshift32 > LazySkipListSet_dhp_less_xorshift32; + + class traits_LazySkipListSet_less_xorshift24: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::xorshift24 > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_xorshift24 > LazySkipListSet_hp_less_xorshift24; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_xorshift24 > LazySkipListSet_dhp_less_xorshift24; + + class traits_LazySkipListSet_less_xorshift16: public cc::lazy_skip_list::make_traits < + co::less< less > + , cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::xorshift16 > + , co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_xorshift16 > LazySkipListSet_hp_less_xorshift16; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_xorshift16 > LazySkipListSet_dhp_less_xorshift16; + + class traits_LazySkipListSet_less_xorshift32_stat: public cc::lazy_skip_list::make_traits < + co::less< less > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::xorshift32 > + ,co::stat< cc::lazy_skip_list::stat<> > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_xorshift32_stat > LazySkipListSet_hp_less_xorshift32_stat; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_xorshift32_stat > LazySkipListSet_dhp_less_xorshift32_stat; + + class traits_LazySkipListSet_less_xorshift24_stat: public cc::lazy_skip_list::make_traits < + co::less< less > + , cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::xorshift24 > + , co::stat< cc::lazy_skip_list::stat<> > + , co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_xorshift24_stat > LazySkipListSet_hp_less_xorshift24_stat; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_xorshift24_stat > LazySkipListSet_dhp_less_xorshift24_stat; + + class traits_LazySkipListSet_less_xorshift16_stat: public cc::lazy_skip_list::make_traits < + co::less< less > + , cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::xorshift16 > + , co::stat< cc::lazy_skip_list::stat<> > + , co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_less_xorshift16_stat > LazySkipListSet_hp_less_xorshift16_stat; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_less_xorshift16_stat > LazySkipListSet_dhp_less_xorshift16_stat; + + class traits_LazySkipListSet_cmp_xorshift32: public cc::lazy_skip_list::make_traits < + co::compare< compare > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::xorshift32 > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_cmp_xorshift32 > LazySkipListSet_hp_cmp_xorshift32; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_cmp_xorshift32 > LazySkipListSet_dhp_cmp_xorshift32; + + class traits_LazySkipListSet_cmp_xorshift32_stat: public cc::lazy_skip_list::make_traits < + co::compare< compare > + ,cc::lazy_skip_list::random_level_generator< cc::lazy_skip_list::xorshift32 > + ,co::stat< cc::lazy_skip_list::stat<> > + ,co::item_counter< cds::atomicity::cache_friendly_item_counter > + >::type + {}; + typedef LazySkipListSet< cds::gc::HP, key_val, traits_LazySkipListSet_cmp_xorshift32_stat > LazySkipListSet_hp_cmp_xorshift32_stat; + typedef LazySkipListSet< cds::gc::DHP, key_val, traits_LazySkipListSet_cmp_xorshift32_stat > LazySkipListSet_dhp_cmp_xorshift32_stat; + }; + + template + static inline void print_stat( cds_test::property_stream& o, LazySkipListSet const& s ) + { + o << s.statistics(); + } + +} // namespace set + +#define CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipList_set_type, key_type, value_type ) \ + TEST_F( fixture, LazySkipList_set_type ) \ + { \ + typedef set::set_type< tag_LazySkipListSet, key_type, value_type >::LazySkipList_set_type set_type; \ + test_case(); \ + } + + +#if defined(CDS_STRESS_TEST_LEVEL) && CDS_STRESS_TEST_LEVEL > 1 +# define CDSSTRESS_LazySkipListSet_HP_2( fixture, test_case, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_less_turbo32_seqcst, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_less_turbo32_seqcst, key_type, value_type ) \ + +#else +# define CDSSTRESS_LazySkipListSet_HP_2( fixture, test_case, key_type, value_type ) +#endif + +#if defined(CDS_STRESS_TEST_LEVEL) && CDS_STRESS_TEST_LEVEL == 1 +# define CDSSTRESS_LazySkipListSet_HP_1( fixture, test_case, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_less_turbo32, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_less_turbo32_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_cmp_turbo32, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_cmp_turbo32_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_less_xorshift32, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_less_xorshift32_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_cmp_xorshift32, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_cmp_xorshift32_stat, key_type, value_type ) \ + +#else +# define CDSSTRESS_LazySkipListSet_HP_1( fixture, test_case, key_type, value_type ) +#endif + + +#define CDSSTRESS_LazySkipListSet_HP( fixture, test_case, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_less_turbo32, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_less_turbo24, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_less_turbo16, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_less_turbo32_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_less_turbo24_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_less_turbo16_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_cmp_turbo32, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_cmp_turbo32_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_less_xorshift32, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_less_xorshift24, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_less_xorshift16, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_less_xorshift32_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_less_xorshift24_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_less_xorshift16_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_dhp_cmp_xorshift32, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_case( fixture, test_case, LazySkipListSet_hp_cmp_xorshift32_stat, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_HP_1( fixture, test_case, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_HP_2( fixture, test_case, key_type, value_type ) \ + + +#define CDSSTRESS_LazySkipListSet( fixture, test_case, key_type, value_type ) \ + CDSSTRESS_LazySkipListSet_HP( fixture, test_case, key_type, value_type ) \ + +#endif // #ifndef CDSUNIT_SET_TYPE_LAZY_SKIP_LIST_H diff --git a/test/unit/intrusive-set/CMakeLists.txt b/test/unit/intrusive-set/CMakeLists.txt index fa62df2be..c3bfeb867 100644 --- a/test/unit/intrusive-set/CMakeLists.txt +++ b/test/unit/intrusive-set/CMakeLists.txt @@ -79,6 +79,17 @@ add_executable(${UNIT_ISET_SKIP} ${UNIT_ISET_SKIP_SOURCES}) target_link_libraries(${UNIT_ISET_SKIP} ${CDS_TEST_LIBRARIES}) add_test(NAME ${UNIT_ISET_SKIP} COMMAND ${UNIT_ISET_SKIP} WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) +# intrusive::LazySkipListSet +set(UNIT_ISET_LAZYSKIP unit-iset-lazy-skip) +set(UNIT_ISET_LAZYSKIP_SOURCES + ../main.cpp + intrusive_lazyskiplist_hp.cpp + intrusive_lazyskiplist_dhp.cpp +) +add_executable(${UNIT_ISET_LAZYSKIP} ${UNIT_ISET_LAZYSKIP_SOURCES}) +target_link_libraries(${UNIT_ISET_LAZYSKIP} ${CDS_TEST_LIBRARIES}) +add_test(NAME ${UNIT_ISET_LAZYSKIP} COMMAND ${UNIT_ISET_LAZYSKIP} WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) + # intrusive::SplitListSet set(UNIT_ISET_SPLIT_MICHAEL unit-iset-split-michael) set(UNIT_ISET_SPLIT_MICHAEL_SOURCES @@ -131,6 +142,7 @@ add_custom_target( unit-iset ${UNIT_ISET_MICHAEL_ITERABLE} ${UNIT_ISET_MICHAEL_LAZY} ${UNIT_ISET_SKIP} + ${UNIT_ISET_LAZYSKIP} ${UNIT_ISET_SPLIT_MICHAEL} ${UNIT_ISET_SPLIT_ITERABLE} ${UNIT_ISET_SPLIT_LAZY} diff --git a/test/unit/intrusive-set/intrusive_lazyskiplist_dhp.cpp b/test/unit/intrusive-set/intrusive_lazyskiplist_dhp.cpp new file mode 100644 index 000000000..009503f0b --- /dev/null +++ b/test/unit/intrusive-set/intrusive_lazyskiplist_dhp.cpp @@ -0,0 +1,334 @@ +// Copyright (c) 2006-2018 Maxim Khizhinsky +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include "test_intrusive_set_hp.h" + +#include + +namespace { + namespace ci = cds::intrusive; + typedef cds::gc::DHP gc_type; + + class IntrusiveLazySkipListSet_DHP : public cds_test::intrusive_set_hp + { + protected: + typedef cds_test::intrusive_set_hp base_class; + + protected: + typedef typename base_class::base_int_item< ci::lazy_skip_list::node< gc_type>> base_item_type; + typedef typename base_class::member_int_item< ci::lazy_skip_list::node< gc_type>> member_item_type; + + void SetUp() + { + typedef ci::LazySkipListSet< gc_type, base_item_type, + typename ci::lazy_skip_list::make_traits< + ci::opt::hook>> + ,ci::opt::disposer + ,ci::opt::compare + >::type + > set_type; + + cds::gc::dhp::smr::construct( set_type::c_nHazardPtrCount ); + cds::threading::Manager::attachThread(); + } + + void TearDown() + { + cds::threading::Manager::detachThread(); + cds::gc::dhp::smr::destruct(); + } + }; + + + TEST_F( IntrusiveLazySkipListSet_DHP, base_cmp ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, base_less ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, base_cmpmix ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef base_class::less less; + typedef ci::lazy_skip_list::stat<> stat; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, base_xorshift32 ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift32 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, base_xorshift24 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift24 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, base_xorshift16 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift16 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, base_turbo32 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo32 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, base_turbo24 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo24 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, base_turbo16 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo16 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, member_cmp ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof(member_item_type, hMember), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, member_less ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef ci::opt::v::sequential_consistent memory_model; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, member_cmpmix ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef base_class::less less; + typedef ci::lazy_skip_list::stat<> stat; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, member_xorshift32 ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift32 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, member_xorshift24 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift24 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, member_xorshift16 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift16 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, member_turbo32 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo32 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, member_turbo24 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo24 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_DHP, member_turbo16 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo16 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + +} // namespace diff --git a/test/unit/intrusive-set/intrusive_lazyskiplist_hp.cpp b/test/unit/intrusive-set/intrusive_lazyskiplist_hp.cpp new file mode 100644 index 000000000..735fd2a88 --- /dev/null +++ b/test/unit/intrusive-set/intrusive_lazyskiplist_hp.cpp @@ -0,0 +1,335 @@ +// Copyright (c) 2006-2018 Maxim Khizhinsky +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include "test_intrusive_set_hp.h" + +#include + +namespace { + namespace ci = cds::intrusive; + typedef cds::gc::HP gc_type; + + class IntrusiveLazySkipListSet_HP : public cds_test::intrusive_set_hp + { + protected: + typedef cds_test::intrusive_set_hp base_class; + + protected: + typedef typename base_class::base_int_item< ci::lazy_skip_list::node< gc_type>> base_item_type; + typedef typename base_class::member_int_item< ci::lazy_skip_list::node< gc_type>> member_item_type; + + void SetUp() + { + typedef ci::LazySkipListSet< gc_type, base_item_type, + typename ci::lazy_skip_list::make_traits< + ci::opt::hook>> + ,ci::opt::disposer + ,ci::opt::compare + >::type + > set_type; + + // +1 - for guarded_ptr + cds::gc::hp::GarbageCollector::Construct( set_type::c_nHazardPtrCount + 1, 1, 16 ); + cds::threading::Manager::attachThread(); + } + + void TearDown() + { + cds::threading::Manager::detachThread(); + cds::gc::hp::GarbageCollector::Destruct( true ); + } + }; + + + TEST_F( IntrusiveLazySkipListSet_HP, base_cmp ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, base_less ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, base_cmpmix ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef base_class::less less; + typedef ci::lazy_skip_list::stat<> stat; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, base_xorshift32 ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift32 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, base_xorshift24 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift24 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, base_xorshift16 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift16 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, base_turbo32 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo32 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, base_turbo24 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo24 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, base_turbo16 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::base_hook< ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo16 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, base_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, member_cmp ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof(member_item_type, hMember), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, member_less ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef ci::opt::v::sequential_consistent memory_model; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, member_cmpmix ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef base_class::less less; + typedef ci::lazy_skip_list::stat<> stat; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, member_xorshift32 ) + { + struct traits : public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift32 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, member_xorshift24 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift24 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, member_xorshift16 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::xorshift16 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, member_turbo32 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo32 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, member_turbo24 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo24 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + + TEST_F( IntrusiveLazySkipListSet_HP, member_turbo16 ) + { + struct traits: public ci::lazy_skip_list::traits + { + typedef ci::lazy_skip_list::member_hook< offsetof( member_item_type, hMember ), ci::opt::gc< gc_type >> hook; + typedef mock_disposer disposer; + typedef cmp compare; + typedef ci::lazy_skip_list::turbo16 random_level_generator; + }; + + typedef ci::LazySkipListSet< gc_type, member_item_type, traits > set_type; + + set_type s; + test( s ); + } + +} // namespace diff --git a/test/unit/set/CMakeLists.txt b/test/unit/set/CMakeLists.txt index 0487d6617..22d59b7e8 100644 --- a/test/unit/set/CMakeLists.txt +++ b/test/unit/set/CMakeLists.txt @@ -77,6 +77,17 @@ add_executable(${UNIT_SET_SKIP} ${UNIT_SET_SKIP_SOURCES}) target_link_libraries(${UNIT_SET_SKIP} ${CDS_TEST_LIBRARIES}) add_test(NAME ${UNIT_SET_SKIP} COMMAND ${UNIT_SET_SKIP} WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) +# LazySkipListSet +set(UNIT_SET_LAZY_SKIP unit-set-lazy-skip) +set(UNIT_SET_LAZY_SKIP_SOURCES + ../main.cpp + lazy_skiplist_hp.cpp + lazy_skiplist_dhp.cpp +) +add_executable(${UNIT_SET_LAZY_SKIP} ${UNIT_SET_LAZY_SKIP_SOURCES}) +target_link_libraries(${UNIT_SET_LAZY_SKIP} ${CDS_TEST_LIBRARIES}) +add_test(NAME ${UNIT_SET_LAZY_SKIP} COMMAND ${UNIT_SET_LAZY_SKIP} WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) + # SplitListSet set(UNIT_SET_SPLIT_MICHAEL unit-set-split-michael) set(UNIT_SET_SPLIT_MICHAEL_SOURCES @@ -128,6 +139,7 @@ add_custom_target( unit-set ${UNIT_SET_MICHAEL_ITERABLE} ${UNIT_SET_MICHAEL_LAZY} ${UNIT_SET_SKIP} + ${UNIT_SET_LAZY_SKIP} ${UNIT_SET_SPLIT_MICHAEL} ${UNIT_SET_SPLIT_ITERABLE} ${UNIT_SET_SPLIT_LAZY} diff --git a/test/unit/set/lazy_skiplist_dhp.cpp b/test/unit/set/lazy_skiplist_dhp.cpp new file mode 100644 index 000000000..5e4d24bff --- /dev/null +++ b/test/unit/set/lazy_skiplist_dhp.cpp @@ -0,0 +1,32 @@ +#include "test_ordered_set_hp.h" + +#include + +namespace { + namespace cc = cds::container; + typedef cds::gc::DHP gc_type; + + class LazySkipListSet_DHP : public cds_test::container_ordered_set_hp + { + protected: + typedef cds_test::container_ordered_set_hp base_class; + + void SetUp() + { + typedef cc::LazySkipListSet< gc_type, int_item > set_type; + + cds::gc::dhp::smr::construct( set_type::c_nHazardPtrCount ); + cds::threading::Manager::attachThread(); + } + + void TearDown() + { + cds::threading::Manager::detachThread(); + cds::gc::dhp::smr::destruct(); + } + }; + +# define CDSTEST_FIXTURE_NAME LazySkipListSet_DHP +# include "lazy_skiplist_hp_inl.h" + +} // namespace diff --git a/test/unit/set/lazy_skiplist_hp.cpp b/test/unit/set/lazy_skiplist_hp.cpp new file mode 100644 index 000000000..c10cab44f --- /dev/null +++ b/test/unit/set/lazy_skiplist_hp.cpp @@ -0,0 +1,33 @@ +#include "test_ordered_set_hp.h" + +#include + +namespace { + namespace cc = cds::container; + typedef cds::gc::HP gc_type; + + class LazySkipListSet_HP: public cds_test::container_ordered_set_hp + { + protected: + typedef cds_test::container_ordered_set_hp base_class; + + void SetUp() + { + typedef cc::LazySkipListSet< gc_type, int_item > set_type; + + // +1 - for guarded_ptr + cds::gc::hp::GarbageCollector::Construct( set_type::c_nHazardPtrCount + 1, 1, 16 ); + cds::threading::Manager::attachThread(); + } + + void TearDown() + { + cds::threading::Manager::detachThread(); + cds::gc::hp::GarbageCollector::Destruct( true ); + } + }; + +# define CDSTEST_FIXTURE_NAME LazySkipListSet_HP +# include "lazy_skiplist_hp_inl.h" + +} diff --git a/test/unit/set/lazy_skiplist_hp_inl.h b/test/unit/set/lazy_skiplist_hp_inl.h new file mode 100644 index 000000000..bfb44d3e6 --- /dev/null +++ b/test/unit/set/lazy_skiplist_hp_inl.h @@ -0,0 +1,177 @@ +TEST_F( CDSTEST_FIXTURE_NAME, compare ) +{ + typedef cc::LazySkipListSet< gc_type, int_item, + typename cc::lazy_skip_list::make_traits< + cds::opt::compare< cmp > + >::type + > set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, less ) +{ + typedef cc::LazySkipListSet< gc_type, int_item, + typename cc::lazy_skip_list::make_traits< + cds::opt::less< base_class::less > + >::type + > set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, cmpmix ) +{ + typedef cc::LazySkipListSet< gc_type, int_item, + typename cc::lazy_skip_list::make_traits< + cds::opt::less< base_class::less > + ,cds::opt::compare< cmp > + >::type + > set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, item_counting ) +{ + struct set_traits: public cc::lazy_skip_list::traits + { + typedef cmp compare; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + }; + typedef cc::LazySkipListSet< gc_type, int_item, set_traits >set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, backoff ) +{ + struct set_traits: public cc::lazy_skip_list::traits + { + typedef cmp compare; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef cds::backoff::yield back_off; + }; + typedef cc::LazySkipListSet< gc_type, int_item, set_traits >set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, stat ) +{ + struct set_traits: public cc::lazy_skip_list::traits + { + typedef cmp compare; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef cds::backoff::yield back_off; + typedef cc::lazy_skip_list::stat<> stat; + }; + typedef cc::LazySkipListSet< gc_type, int_item, set_traits >set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, xorshift32 ) +{ + struct set_traits: public cc::lazy_skip_list::traits + { + typedef cmp compare; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef cc::lazy_skip_list::stat<> stat; + typedef cc::lazy_skip_list::xorshift32 random_level_generator; + }; + typedef cc::LazySkipListSet< gc_type, int_item, set_traits >set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, xorshift24 ) +{ + struct set_traits: public cc::lazy_skip_list::traits + { + typedef cmp compare; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef cc::lazy_skip_list::stat<> stat; + typedef cc::lazy_skip_list::xorshift24 random_level_generator; + }; + typedef cc::LazySkipListSet< gc_type, int_item, set_traits >set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, xorshift16 ) +{ + struct set_traits: public cc::lazy_skip_list::traits + { + typedef cmp compare; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef cc::lazy_skip_list::stat<> stat; + typedef cc::lazy_skip_list::xorshift16 random_level_generator; + }; + typedef cc::LazySkipListSet< gc_type, int_item, set_traits >set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, turbo32 ) +{ + struct set_traits: public cc::lazy_skip_list::traits + { + typedef cmp compare; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef cc::lazy_skip_list::stat<> stat; + typedef cc::lazy_skip_list::turbo32 random_level_generator; + }; + typedef cc::LazySkipListSet< gc_type, int_item, set_traits >set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, turbo24 ) +{ + struct set_traits: public cc::lazy_skip_list::traits + { + typedef cmp compare; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef cc::lazy_skip_list::stat<> stat; + typedef cc::lazy_skip_list::turbo24 random_level_generator; + }; + typedef cc::LazySkipListSet< gc_type, int_item, set_traits >set_type; + + set_type s; + test( s ); +} + +TEST_F( CDSTEST_FIXTURE_NAME, turbo16 ) +{ + struct set_traits: public cc::lazy_skip_list::traits + { + typedef cmp compare; + typedef base_class::less less; + typedef cds::atomicity::item_counter item_counter; + typedef cc::lazy_skip_list::stat<> stat; + typedef cc::lazy_skip_list::turbo16 random_level_generator; + }; + typedef cc::LazySkipListSet< gc_type, int_item, set_traits >set_type; + + set_type s; + test( s ); +} From 4848e8b7f70fb90dddecc3f25386504900c711a0 Mon Sep 17 00:00:00 2001 From: Ilya Date: Wed, 30 Jan 2019 00:34:46 +0300 Subject: [PATCH 2/3] lock-free changade to fine-grained locks --- cds/intrusive/impl/lazy_skip_list.h | 896 +++++++++++----------------- 1 file changed, 344 insertions(+), 552 deletions(-) diff --git a/cds/intrusive/impl/lazy_skip_list.h b/cds/intrusive/impl/lazy_skip_list.h index c841d4ae1..c2a86dc58 100644 --- a/cds/intrusive/impl/lazy_skip_list.h +++ b/cds/intrusive/impl/lazy_skip_list.h @@ -32,9 +32,9 @@ namespace cds { namespace intrusive { node_type * m_pNode; protected: - static value_type * gc_protect( marked_ptr p ) + static value_type* gc_protect(marked_ptr p) { - return node_traits::to_value_ptr( p.ptr()); + return node_traits::to_value_ptr(p.ptr()); } void next() @@ -169,11 +169,11 @@ namespace cds { namespace intrusive { # ifdef CDS_DOXYGEN_INVOKED typedef implementation_defined key_comparator; # else - typedef typename opt::details::make_comparator< value_type, traits >::type key_comparator; + typedef typename opt::details::make_comparator::type key_comparator; # endif typedef typename traits::disposer disposer; - typedef typename get_node_traits< value_type, node_type, hook>::type node_traits; + typedef typename get_node_traits::type node_traits; typedef typename traits::item_counter item_counter; typedef typename traits::memory_model memory_model; @@ -183,7 +183,7 @@ namespace cds { namespace intrusive { typedef typename traits::stat stat; public: - typedef typename gc::template guarded_ptr< value_type > guarded_ptr; + typedef typename gc::template guarded_ptr guarded_ptr; static unsigned int const c_nMaxHeight = std::conditional< (random_level_generator::c_nUpperBound <= lazy_skip_list::c_nHeightLimit), @@ -191,7 +191,11 @@ namespace cds { namespace intrusive { std::integral_constant< unsigned int, lazy_skip_list::c_nHeightLimit > >::type::value; - static unsigned int const c_nMinHeight = 5; + static unsigned int const c_nMinHeight = std::conditional< + (5 <= c_nMaxHeight), + std::integral_constant, + std::integral_constant + >::type::value; // c_nMaxHeight * 2 - pPred/pSucc guards // + 1 - for erase, unlink @@ -215,24 +219,73 @@ namespace cds { namespace intrusive { typedef std::unique_ptr< node_type, typename node_builder::node_disposer > scoped_node_ptr; struct position { - node_type * pPrev[ c_nMaxHeight ]; - node_type * pSucc[ c_nMaxHeight ]; + public: + position() + : m_LockedCounter(0) + { + for (auto& pCurr : m_LockedArray) { + pCurr = nullptr; + } + } + + ~position() + { + unlockAll(); + } + + void addLockPtr(node_type* pNode) + { + assert(pNode != nullptr); + assert(m_LockedCounter >= 2 * c_nMaxHeight + 1); + + if (contains(pNode)) + return; + + m_LockedArray[m_LockedCounter++] = pNode; + } + + bool contains(node_type* pNode) + { + for (auto pCurr : m_LockedArray) { + if (pNode == pCurr) + return true; + } + return false; + } + + void unlockAll() + { + for (auto& pNode : m_LockedArray) { + if (!pNode) + return; + + pNode->unlock(); + pNode = nullptr; + } + + m_LockedCounter = 0; + } + + public: + node_type* pPrev[c_nMaxHeight]; + node_type* pCur; - typename gc::template GuardArray< c_nMaxHeight * 2 > guards; - node_type * pCur; // guarded by one of guards + private: + node_type* m_LockedArray[2 * c_nMaxHeight + 1]; + unsigned int m_LockedCounter; }; public: LazySkipListSet() - : m_Head( c_nMaxHeight ) - , m_nHeight( c_nMinHeight ) + : m_Head(c_nMaxHeight) + , m_nHeight(c_nMinHeight) { static_assert( (std::is_same< gc, typename node_type::gc >::value), "GC and node_type::gc must be the same type" ); - gc::check_available_guards( c_nHazardPtrCount ); + gc::check_available_guards(c_nHazardPtrCount); // Barrier for head node - atomics::atomic_thread_fence( memory_model::memory_order_release ); + atomics::atomic_thread_fence(memory_model::memory_order_release); } ~LazySkipListSet() @@ -276,106 +329,82 @@ namespace cds { namespace intrusive { public: - bool insert( value_type& val ) + bool insert(value_type& val) { return insert( val, []( value_type& ) {} ); } template - bool insert( value_type& val, Func f ) + bool insert(value_type& val, Func f) { typename gc::Guard gNew; - gNew.assign( &val ); + gNew.assign(&val); - node_type * pNode = node_traits::to_node_ptr( val ); - scoped_node_ptr scp( pNode ); + node_type* pNode = node_traits::to_node_ptr(val); + scoped_node_ptr scp(pNode); unsigned int nHeight = pNode->height(); - bool bTowerOk = pNode->has_tower(); // nHeight > 1 && pNode->get_tower() != nullptr; - bool bTowerMade = false; position pos; - while ( true ) - { - if ( find_position( val, pos, key_comparator(), true )) { - // scoped_node_ptr deletes the node tower if we create it - if ( !bTowerMade ) - scp.release(); - m_Stat.onInsertFailed(); - return false; - } + if (find_position(val, pos, key_comparator())) { + scp.release(); + m_Stat.onInsertFailed(); + return false; + } - if ( !bTowerOk ) { - build_node( pNode ); - nHeight = pNode->height(); - bTowerMade = pNode->has_tower(); - bTowerOk = true; - } + if (!pNode->has_tower()) { + build_node(pNode); + nHeight = pNode->height(); + } - if ( !insert_at_position( val, pNode, pos, f )) { - m_Stat.onInsertRetry(); - continue; - } + insert_at_position(val, pNode, pos, f); - increase_height( nHeight ); - ++m_ItemCounter; - m_Stat.onAddNode( nHeight ); - m_Stat.onInsertSuccess(); - scp.release(); - return true; - } + increase_height(nHeight); + ++m_ItemCounter; + m_Stat.onAddNode(nHeight); + m_Stat.onInsertSuccess(); + scp.release(); + pos.unlockAll(); + return true; } template std::pair update( value_type& val, Func func, bool bInsert = true ) { typename gc::Guard gNew; - gNew.assign( &val ); + gNew.assign(&val); - node_type * pNode = node_traits::to_node_ptr( val ); - scoped_node_ptr scp( pNode ); + node_type* pNode = node_traits::to_node_ptr(val); + scoped_node_ptr scp(pNode); unsigned int nHeight = pNode->height(); - bool bTowerOk = pNode->has_tower(); - bool bTowerMade = false; position pos; - while ( true ) - { - bool bFound = find_position( val, pos, key_comparator(), true ); - if ( bFound ) { - // scoped_node_ptr deletes the node tower if we create it before - if ( !bTowerMade ) - scp.release(); - - func( false, *node_traits::to_value_ptr(pos.pCur), val ); - m_Stat.onUpdateExist(); - return std::make_pair( true, false ); - } - - if ( !bInsert ) { - scp.release(); - return std::make_pair( false, false ); - } - - if ( !bTowerOk ) { - build_node( pNode ); - nHeight = pNode->height(); - bTowerMade = pNode->has_tower(); - bTowerOk = true; - } - if ( !insert_at_position( val, pNode, pos, [&func]( value_type& item ) { func( true, item, item ); })) { - m_Stat.onInsertRetry(); - continue; - } + if (find_position(val, pos, key_comparator())) { + scp.release(); + func(false, *node_traits::to_value_ptr(pos.pCur), val); + m_Stat.onUpdateExist(); + return std::make_pair(true, false); + } - increase_height( nHeight ); - ++m_ItemCounter; + if (!bInsert) { scp.release(); - m_Stat.onAddNode( nHeight ); - m_Stat.onUpdateNew(); - return std::make_pair( true, true ); + return std::make_pair(false, false); } + + if (!pNode->has_tower()) { + build_node(pNode); + nHeight = pNode->height(); + } + + insert_at_position(val, pNode, pos, [&func](value_type& item) { func(true, item, item); }); + + increase_height(nHeight); + ++m_ItemCounter; + scp.release(); + m_Stat.onAddNode(nHeight); + m_Stat.onUpdateNew(); + return std::make_pair(true, true); } template @@ -385,25 +414,23 @@ namespace cds { namespace intrusive { return update( val, func, true ); } - bool unlink( value_type& val ) + bool unlink(value_type& val) { position pos; - if ( !find_position( val, pos, key_comparator(), false )) { + if (!find_position(val, pos, key_comparator())) { m_Stat.onUnlinkFailed(); return false; } - node_type * pDel = pos.pCur; - assert( key_comparator()( *node_traits::to_value_ptr( pDel ), val ) == 0 ); + node_type* pDel = pos.pCur; unsigned int nHeight = pDel->height(); - typename gc::Guard gDel; - gDel.assign( node_traits::to_value_ptr(pDel)); - if ( node_traits::to_value_ptr( pDel ) == &val && try_remove_at( pDel, pos, [](value_type const&) {} )) { + if (node_traits::to_value_ptr(pDel) == &val) { + remove_at(pDel, pos, [](value_type const&) {}); --m_ItemCounter; - m_Stat.onRemoveNode( nHeight ); + m_Stat.onRemoveNode(nHeight); m_Stat.onUnlinkSuccess(); return true; } @@ -413,16 +440,16 @@ namespace cds { namespace intrusive { } template - guarded_ptr extract( Q const& key ) + guarded_ptr extract(const Q& key) { - return extract_( key, key_comparator()); + return extract_(key, key_comparator()); } template - guarded_ptr extract_with( Q const& key, Less pred ) + guarded_ptr extract_with(const Q& key, Less pred) { - CDS_UNUSED( pred ); - return extract_( key, cds::opt::details::make_comparator_from_less()); + CDS_UNUSED(pred); + return extract_(key, cds::opt::details::make_comparator_from_less()); } guarded_ptr extract_min() @@ -436,16 +463,16 @@ namespace cds { namespace intrusive { } template - bool erase( Q const& key ) + bool erase(const Q& key) { - return erase_( key, key_comparator(), [](value_type const&) {} ); + return erase_(key, key_comparator(), [](const value_type&) {} ); } template - bool erase_with( Q const& key, Less pred ) + bool erase_with(const Q& key, Less pred) { - CDS_UNUSED( pred ); - return erase_( key, cds::opt::details::make_comparator_from_less(), [](value_type const&) {} ); + CDS_UNUSED(pred); + return erase_(key, cds::opt::details::make_comparator_from_less(), [](const value_type&) {} ); } template @@ -455,76 +482,76 @@ namespace cds { namespace intrusive { } template - bool erase_with( Q const& key, Less pred, Func f ) + bool erase_with(const Q& key, Less pred, Func f) { - CDS_UNUSED( pred ); - return erase_( key, cds::opt::details::make_comparator_from_less(), f ); + CDS_UNUSED(pred); + return erase_(key, cds::opt::details::make_comparator_from_less(), f); } template - bool find( Q& key, Func f ) + bool find(Q& key, Func f) { - return find_with_( key, key_comparator(), f ); + return find_with_(key, key_comparator(), f); } template - bool find( Q const& key, Func f ) + bool find(const Q& key, Func f) { - return find_with_( key, key_comparator(), f ); + return find_with_(key, key_comparator(), f); } template - bool find_with( Q& key, Less pred, Func f ) + bool find_with(Q& key, Less pred, Func f) { - CDS_UNUSED( pred ); - return find_with_( key, cds::opt::details::make_comparator_from_less(), f ); + CDS_UNUSED(pred); + return find_with_(key, cds::opt::details::make_comparator_from_less(), f); } template - bool find_with( Q const& key, Less pred, Func f ) + bool find_with(const Q& key, Less pred, Func f) { - CDS_UNUSED( pred ); - return find_with_( key, cds::opt::details::make_comparator_from_less(), f ); + CDS_UNUSED(pred); + return find_with_(key, cds::opt::details::make_comparator_from_less(), f); } template - bool contains( Q const& key ) + bool contains(const Q& key ) { - return find_with_( key, key_comparator(), [](value_type& , Q const& ) {} ); + return find_with_(key, key_comparator(), [](value_type& , const Q&) {} ); } template CDS_DEPRECATED("deprecated, use contains()") - bool find( Q const& key ) + bool find(const Q& key) { - return contains( key ); + return contains(key); } template - bool contains( Q const& key, Less pred ) + bool contains(const Q& key, Less pred) { - CDS_UNUSED( pred ); - return find_with_( key, cds::opt::details::make_comparator_from_less(), [](value_type& , Q const& ) {} ); + CDS_UNUSED(pred); + return find_with_(key, cds::opt::details::make_comparator_from_less(), [](value_type& , const Q&) {} ); } template CDS_DEPRECATED("deprecated, use contains()") - bool find_with( Q const& key, Less pred ) + bool find_with(const Q& key, Less pred) { - return contains( key, pred ); + return contains(key, pred); } template - guarded_ptr get( Q const& key ) + guarded_ptr get(const Q& key) { - return get_with_( key, key_comparator()); + return get_with_(key, key_comparator()); } template - guarded_ptr get_with( Q const& key, Less pred ) + guarded_ptr get_with(const Q& key, Less pred) { - CDS_UNUSED( pred ); - return get_with_( key, cds::opt::details::make_comparator_from_less()); + CDS_UNUSED(pred); + return get_with_(key, cds::opt::details::make_comparator_from_less()); } size_t size() const @@ -534,12 +561,12 @@ namespace cds { namespace intrusive { bool empty() const { - return m_Head.head()->next( 0 ).load( memory_model::memory_order_relaxed ) == nullptr; + return m_Head.head()->next(0).load(memory_model::memory_order_relaxed) == nullptr; } void clear() { - while ( extract_min_()); + while (extract_min_()); } static constexpr unsigned int max_height() noexcept @@ -547,7 +574,7 @@ namespace cds { namespace intrusive { return c_nMaxHeight; } - stat const& statistics() const + const stat& statistics() const { return m_Stat; } @@ -562,414 +589,201 @@ namespace cds { namespace intrusive { } template - node_type * build_node( Q v ) + node_type* build_node(Q v) { - return node_builder::make_tower( v, m_RandomLevelGen ); + return node_builder::make_tower(v, m_RandomLevelGen); } - static value_type * gc_protect( marked_node_ptr p ) + static value_type* gc_protect(marked_node_ptr p) { - return node_traits::to_value_ptr( p.ptr()); + return node_traits::to_value_ptr(p.ptr()); } - static void dispose_node( void* p ) + static void dispose_node(void* p) { - assert( p != nullptr ); - value_type* pVal = reinterpret_cast( p ); - typename node_builder::node_disposer()( node_traits::to_node_ptr( pVal )); - disposer()( pVal ); + assert(p != nullptr); + value_type* pVal = reinterpret_cast(p); + typename node_builder::node_disposer()(node_traits::to_node_ptr(pVal)); + disposer()(pVal); } - void help_remove( int nLevel, node_type* pPred, marked_node_ptr pCur ) + template + bool find_position(const Q& val, position& pos, Compare cmp) { - if ( pCur->is_upper_level( nLevel )) { - marked_node_ptr p( pCur.ptr()); - typename gc::Guard hp; - marked_node_ptr pSucc = hp.protect( pCur->next( nLevel ), gc_protect ); - - if ( pSucc.bits() && - pPred->next( nLevel ).compare_exchange_strong( p, marked_node_ptr( pSucc.ptr()), - memory_model::memory_order_acquire, atomics::memory_order_relaxed )) - { - if ( pCur->level_unlinked()) { - gc::retire( node_traits::to_value_ptr( pCur.ptr()), dispose_node ); - m_Stat.onEraseWhileFind(); - } - } - } - } + typename gc::template GuardArray<2> guards; - template - bool find_position( Q const& val, position& pos, Compare cmp, bool bStopIfFound ) - { - node_type * pPred; - marked_node_ptr pSucc; - marked_node_ptr pCur; + node_type* pPrev; + marked_node_ptr pCurr; + node_type* pPrevAtThisLevel = m_Head.head(); - // Hazard pointer array: - // pPred: [nLevel * 2] - // pSucc: [nLevel * 2 + 1] + int cmpResult = 1; - retry: - pPred = m_Head.head(); - int nCmp = 1; + for (int nLevel = static_cast(c_nMaxHeight - 1); nLevel >= 0; --nLevel) { + retryAtLevel: + pPrev = pPrevAtThisLevel; + guards.assign(0, pPrev); - for ( int nLevel = static_cast( c_nMaxHeight - 1 ); nLevel >= 0; --nLevel ) { - pos.guards.assign( nLevel * 2, node_traits::to_value_ptr( pPred )); - while ( true ) { - pCur = pos.guards.protect( nLevel * 2 + 1, pPred->next( nLevel ), gc_protect ); - if ( pCur.bits()) { - // pCur.bits() means that pPred is logically deleted - goto retry; - } + while (true) { + pCurr = guards.protect(1, pPrev->next(nLevel), gc_protect); - if ( pCur.ptr() == nullptr ) { + if (nullptr == pCurr.ptr()) { // end of list at level nLevel - goto next level break; } - // pSucc contains deletion mark for pCur - pSucc = pCur->next( nLevel ).load( memory_model::memory_order_acquire ); + cmpResult = cmp(*node_traits::to_value_ptr(pCurr.ptr()), val); + if (cmpResult < 0) { + pPrev = pCurr.ptr(); + guards.copy(0, 1); // pPrev guard := pCurr guard + } else { + if (!pos.contains(pPrev)) { + pPrev->lock(); + } - if ( pPred->next( nLevel ).load( memory_model::memory_order_acquire ).all() != pCur.ptr()) - goto retry; + if (pPrev->next(nLevel).load(memory_model::memory_order_relaxed) != pCurr.ptr()) { + if (!pos.contains(pPrev)) { + pPrev->unlock(); + } - if ( pSucc.bits()) { - // pCur is marked, i.e. logically deleted - // try to help deleting pCur - help_remove( nLevel, pPred, pCur ); - goto retry; - } - else { - nCmp = cmp( *node_traits::to_value_ptr( pCur.ptr()), val ); - if ( nCmp < 0 ) { - pPred = pCur.ptr(); - pos.guards.copy( nLevel * 2, nLevel * 2 + 1 ); // pPrev guard := cur guard + goto retryAtLevel; } - else if ( nCmp == 0 && bStopIfFound ) - goto found; - else - break; + break; } } - // Next level - pos.pPrev[nLevel] = pPred; - pos.pSucc[nLevel] = pCur.ptr(); + pos.addLockPtr(pPrev); + pos.pPrev[nLevel] = pPrev; } - if ( nCmp != 0 ) - return false; + if (cmpResult == 0) { + pos.pCur = pCurr.ptr(); + pos.pCur->lock(); + pos.addLockPtr(pos.pCur); + + } - found: - pos.pCur = pCur.ptr(); - return pCur.ptr() && nCmp == 0; + return cmpResult == 0; } - bool find_min_position( position& pos ) + bool find_min_position(position& pos) { - node_type * pPred; - marked_node_ptr pSucc; - marked_node_ptr pCur; - - // Hazard pointer array: - // pPred: [nLevel * 2] - // pSucc: [nLevel * 2 + 1] - - retry: - pPred = m_Head.head(); - - for ( int nLevel = static_cast( c_nMaxHeight - 1 ); nLevel >= 0; --nLevel ) { - pos.guards.assign( nLevel * 2, node_traits::to_value_ptr( pPred )); - pCur = pos.guards.protect( nLevel * 2 + 1, pPred->next( nLevel ), gc_protect ); - - // pCur.bits() means that pPred is logically deleted - // head cannot be deleted - assert( pCur.bits() == 0 ); - - if ( pCur.ptr()) { + typename gc::template GuardArray<2> guards; - // pSucc contains deletion mark for pCur - pSucc = pCur->next( nLevel ).load( memory_model::memory_order_acquire ); + node_type* pPrev = m_Head.head(); + marked_node_ptr pCurr; - if ( pPred->next( nLevel ).load( memory_model::memory_order_acquire ).all() != pCur.ptr()) - goto retry; + pPrev->lock(); + pos.addLockPtr(pPrev); - if ( pSucc.bits()) { - // pCur is marked, i.e. logically deleted. - // try to help deleting pCur - help_remove( nLevel, pPred, pCur ); - goto retry; - } - } + for (int nLevel = static_cast(c_nMaxHeight - 1); nLevel >= 0; --nLevel) { + pCurr = pPrev->next(nLevel); + pos.pPrev[nLevel] = pPrev; + } - // Next level - pos.pPrev[nLevel] = pPred; - pos.pSucc[nLevel] = pCur.ptr(); + if (nullptr != (pos.pCur = pCurr.ptr())) { + pos.pCur->lock(); + pos.addLockPtr(pos.pCur); } - return ( pos.pCur = pCur.ptr()) != nullptr; + return pos.pCur != nullptr; } - bool find_max_position( position& pos ) + bool find_max_position(position& pos) { - node_type * pPred; - marked_node_ptr pSucc; - marked_node_ptr pCur; + typename gc::template GuardArray<2> guards; - // Hazard pointer array: - // pPred: [nLevel * 2] - // pSucc: [nLevel * 2 + 1] + node_type* pPrev; + marked_node_ptr pCurr; + node_type* pPrevAtThisLevel = m_Head.head(); - retry: - pPred = m_Head.head(); + for (int nLevel = static_cast(c_nMaxHeight - 1); nLevel >= 0; --nLevel) { + retryAtLevel: + pPrev = pPrevAtThisLevel; + guards.assign(0, pPrev); - for ( int nLevel = static_cast( c_nMaxHeight - 1 ); nLevel >= 0; --nLevel ) { - pos.guards.assign( nLevel * 2, node_traits::to_value_ptr( pPred )); - while ( true ) { - pCur = pos.guards.protect( nLevel * 2 + 1, pPred->next( nLevel ), gc_protect ); - if ( pCur.bits()) { + while (true) { + pCurr = guards.protect(1, pPrev->next(nLevel), gc_protect); + if (pCurr.bits()) { // pCur.bits() means that pPred is logically deleted - goto retry; - } - - if ( pCur.ptr() == nullptr ) { - // end of the list at level nLevel - goto next level - break; + goto retryAtLevel; } - // pSucc contains deletion mark for pCur - pSucc = pCur->next( nLevel ).load( memory_model::memory_order_acquire ); + if (nullptr == pCurr || nullptr == pCurr->next(nLevel).load(memory_model::memory_order_relaxed)) { + bool lockThisNode = !pos.contains(pPrev); + if (lockThisNode) { + pPrev->lock(); + } - if ( pPred->next( nLevel ).load( memory_model::memory_order_acquire ).all() != pCur.ptr()) - goto retry; + if (pPrev->next(nLevel).load(memory_model::memory_order_relaxed) != pCurr.ptr()) { + if (lockThisNode) { + pPrev->unlock(); + } + goto retryAtLevel; + } - if ( pSucc.bits()) { - // pCur is marked, i.e. logically deleted. - // try to help deleting pCur - help_remove( nLevel, pPred, pCur ); - goto retry; + break; } - else { - if ( !pSucc.ptr()) - break; - pPred = pCur.ptr(); - pos.guards.copy( nLevel * 2, nLevel * 2 + 1 ); - } + pPrev = pCurr.ptr(); + guards.copy(0, 1); // pPrev guard := pCurr guard } - // Next level - pos.pPrev[nLevel] = pPred; - pos.pSucc[nLevel] = pCur.ptr(); + pos.addLockPtr(pPrev); + pos.pPrev[nLevel] = pPrev; } - return ( pos.pCur = pCur.ptr()) != nullptr; - } - - bool renew_insert_position( value_type& val, node_type * pNode, position& pos ) - { - node_type * pPred; - marked_node_ptr pSucc; - marked_node_ptr pCur; - key_comparator cmp; - - // Hazard pointer array: - // pPred: [nLevel * 2] - // pSucc: [nLevel * 2 + 1] - - retry: - pPred = m_Head.head(); - int nCmp = 1; - - for ( int nLevel = static_cast( c_nMaxHeight - 1 ); nLevel >= 0; --nLevel ) { - pos.guards.assign( nLevel * 2, node_traits::to_value_ptr( pPred )); - while ( true ) { - pCur = pos.guards.protect( nLevel * 2 + 1, pPred->next( nLevel ), gc_protect ); - if ( pCur.bits()) { - // pCur.bits() means that pPred is logically deleted - goto retry; - } - - if ( pCur.ptr() == nullptr ) { - // end of list at level nLevel - goto next level - break; - } - - // pSucc contains deletion mark for pCur - pSucc = pCur->next( nLevel ).load( memory_model::memory_order_acquire ); - - if ( pPred->next( nLevel ).load( memory_model::memory_order_acquire ).all() != pCur.ptr()) - goto retry; - - if ( pSucc.bits()) { - // pCur is marked, i.e. logically deleted - if ( pCur.ptr() == pNode ) { - // Node is removing while we are inserting it - return false; - } - // try to help deleting pCur - help_remove( nLevel, pPred, pCur ); - goto retry; - } - else { - nCmp = cmp( *node_traits::to_value_ptr( pCur.ptr()), val ); - if ( nCmp < 0 ) { - pPred = pCur.ptr(); - pos.guards.copy( nLevel * 2, nLevel * 2 + 1 ); // pPrev guard := cur guard - } - else - break; - } - } - - // Next level - pos.pPrev[nLevel] = pPred; - pos.pSucc[nLevel] = pCur.ptr(); + if (nullptr != (pos.pCur = pCurr.ptr())) { + pos.pCur->lock(); + pos.addLockPtr(pos.pCur); } - return nCmp == 0; + return pos.pCur != nullptr; } template - bool insert_at_position( value_type& val, node_type * pNode, position& pos, Func f ) + void insert_at_position(value_type& val, node_type* pNode, position& pos, Func f) { unsigned int const nHeight = pNode->height(); - for ( unsigned int nLevel = 1; nLevel < nHeight; ++nLevel ) - pNode->next( nLevel ).store( marked_node_ptr(), memory_model::memory_order_relaxed ); + //for each level item set next to nullptr + for (unsigned int nLevel = 1; nLevel < nHeight; ++nLevel) + pNode->next( nLevel ).store(marked_node_ptr(), memory_model::memory_order_relaxed ); // Insert at level 0 - { - marked_node_ptr p( pos.pSucc[0] ); - pNode->next( 0 ).store( p, memory_model::memory_order_release ); - if ( !pos.pPrev[0]->next( 0 ).compare_exchange_strong( p, marked_node_ptr( pNode ), memory_model::memory_order_release, atomics::memory_order_relaxed )) - return false; - - f( val ); - } + marked_node_ptr pNodePtr(pNode); + pNode->lock(); + pNode->next(0).store(marked_node_ptr(pos.pPrev[0]->next(0)), memory_model::memory_order_relaxed); + pos.pPrev[0]->next(0).store(pNodePtr, memory_model::memory_order_relaxed); + f(val); // Insert at level 1..max - for ( unsigned int nLevel = 1; nLevel < nHeight; ++nLevel ) { - marked_node_ptr p; - while ( true ) { - marked_node_ptr pSucc( pos.pSucc[nLevel] ); - - // Set pNode->next - // pNode->next can have "logical deleted" flag if another thread is removing pNode right now - if ( !pNode->next( nLevel ).compare_exchange_strong( p, pSucc, - memory_model::memory_order_release, atomics::memory_order_acquire )) - { - // pNode has been marked as removed while we are inserting it - // Stop inserting - assert( p.bits() != 0 ); - - // Here pNode is linked at least level 0 so level_unlinked() cannot returns true - CDS_VERIFY_FALSE( pNode->level_unlinked( nHeight - nLevel )); - - // pNode is linked up to nLevel - 1 - // Remove it via find_position() - find_position( val, pos, key_comparator(), false ); - - m_Stat.onLogicDeleteWhileInsert(); - return true; - } - p = pSucc; - - // Link pNode into the list at nLevel - if ( pos.pPrev[nLevel]->next( nLevel ).compare_exchange_strong( pSucc, marked_node_ptr( pNode ), - memory_model::memory_order_release, atomics::memory_order_relaxed )) - { - // go to next level - break; - } - - // Renew insert position - m_Stat.onRenewInsertPosition(); - - if ( !renew_insert_position( val, pNode, pos )) { - // The node has been deleted while we are inserting it - // Update current height for concurent removing - CDS_VERIFY_FALSE( pNode->level_unlinked( nHeight - nLevel )); - - m_Stat.onRemoveWhileInsert(); - - // help to removing val - find_position( val, pos, key_comparator(), false ); - return true; - } - } + for (unsigned int nLevel = 1; nLevel < nHeight; ++nLevel) { + pNode->next(nLevel).store(marked_node_ptr(pos.pPrev[nLevel]->next(nLevel)), memory_model::memory_order_relaxed); + pos.pPrev[nLevel]->next(nLevel).store(pNodePtr, memory_model::memory_order_relaxed); } - return true; + + pNode->unlock(); } template - bool try_remove_at( node_type * pDel, position& pos, Func f ) + void remove_at(node_type* pDel, position& pos, Func f) { - assert( pDel != nullptr ); + assert(pDel != nullptr); marked_node_ptr pSucc; back_off bkoff; - // logical deletion (marking) - for ( unsigned int nLevel = pDel->height() - 1; nLevel > 0; --nLevel ) { - pSucc = pDel->next( nLevel ).load( memory_model::memory_order_relaxed ); - if ( pSucc.bits() == 0 ) { - bkoff.reset(); - while ( !( pDel->next( nLevel ).compare_exchange_weak( pSucc, pSucc | 1, - memory_model::memory_order_release, atomics::memory_order_acquire ) - || pSucc.bits() != 0 )) - { - bkoff(); - m_Stat.onMarkFailed(); - } - } - } - - marked_node_ptr p( pDel->next( 0 ).load( memory_model::memory_order_relaxed ).ptr()); - while ( true ) { - if ( pDel->next( 0 ).compare_exchange_strong( p, p | 1, memory_model::memory_order_release, atomics::memory_order_acquire )) - { - f( *node_traits::to_value_ptr( pDel )); - - // Physical deletion - // try fast erase - p = pDel; - - for ( int nLevel = static_cast( pDel->height() - 1 ); nLevel >= 0; --nLevel ) { + f(*node_traits::to_value_ptr(pDel)); - pSucc = pDel->next( nLevel ).load( memory_model::memory_order_acquire ); - if ( pos.pPrev[nLevel]->next( nLevel ).compare_exchange_strong( p, marked_node_ptr( pSucc.ptr()), - memory_model::memory_order_acq_rel, atomics::memory_order_relaxed )) - { - pDel->level_unlinked(); - } - else { - // Make slow erase -# ifdef CDS_DEBUG - if ( find_position( *node_traits::to_value_ptr( pDel ), pos, key_comparator(), false )) - assert( pDel != pos.pCur ); -# else - find_position( *node_traits::to_value_ptr( pDel ), pos, key_comparator(), false ); -# endif - m_Stat.onSlowErase(); - return true; - } - } + marked_node_ptr pNext(pDel->next(0).load(memory_model::memory_order_relaxed).ptr()); - // Fast erasing success - gc::retire( node_traits::to_value_ptr( pDel ), dispose_node ); - m_Stat.onFastErase(); - return true; - } - else if ( p.bits()) { - // Another thread is deleting pDel right now - m_Stat.onEraseContention(); - return false; - } - m_Stat.onEraseRetry(); - bkoff(); + for (int nLevel = static_cast(pDel->height() - 1); nLevel >= 0; --nLevel) { + pSucc = pDel->next(nLevel).load(memory_model::memory_order_relaxed); + pos.pPrev[nLevel]->next(nLevel).store(pSucc); } + + gc::retire(node_traits::to_value_ptr(pDel), dispose_node); + m_Stat.onFastErase(); } enum finsd_fastpath_result { @@ -977,8 +791,9 @@ namespace cds { namespace intrusive { find_fastpath_not_found, find_fastpath_abort }; + template - finsd_fastpath_result find_fastpath( Q& val, Compare cmp, Func f ) + finsd_fastpath_result find_fastpath(Q& val, Compare cmp, Func f) { node_type * pPred; marked_node_ptr pCur; @@ -1034,10 +849,8 @@ namespace cds { namespace intrusive { bool find_slowpath( Q& val, Compare cmp, Func f ) { position pos; - if ( find_position( val, pos, cmp, true )) { - assert( cmp( *node_traits::to_value_ptr( pos.pCur ), val ) == 0 ); - - f( *node_traits::to_value_ptr( pos.pCur ), val ); + if (find_position(val, pos, cmp)) { + f(*node_traits::to_value_ptr(pos.pCur), val); return true; } else @@ -1045,9 +858,9 @@ namespace cds { namespace intrusive { } template - bool find_with_( Q& val, Compare cmp, Func f ) + bool find_with_(Q& val, Compare cmp, Func f) { - switch ( find_fastpath( val, cmp, f )) { + switch (find_fastpath(val, cmp, f)) { case find_fastpath_found: m_Stat.onFindFastSuccess(); return true; @@ -1058,7 +871,7 @@ namespace cds { namespace intrusive { break; } - if ( find_slowpath( val, cmp, f )) { + if (find_slowpath(val, cmp, f)) { m_Stat.onFindSlowSuccess(); return true; } @@ -1071,63 +884,55 @@ namespace cds { namespace intrusive { guarded_ptr get_with_( Q const& val, Compare cmp ) { guarded_ptr gp; - if ( find_with_( val, cmp, [&gp]( value_type& found, Q const& ) { gp.reset( &found ); } )) + if (find_with_(val, cmp, [&gp](value_type& found, const Q&) { gp.reset(&found); } )) return gp; return guarded_ptr(); } template - bool erase_( Q const& val, Compare cmp, Func f ) + bool erase_(const Q& val, Compare cmp, Func f) { position pos; - if ( !find_position( val, pos, cmp, false )) { + if (!find_position(val, pos, cmp)) { m_Stat.onEraseFailed(); return false; } - node_type * pDel = pos.pCur; + node_type* pDel = pos.pCur; typename gc::Guard gDel; - gDel.assign( node_traits::to_value_ptr( pDel )); - assert( cmp( *node_traits::to_value_ptr( pDel ), val ) == 0 ); + gDel.assign(node_traits::to_value_ptr(pDel)); + assert(cmp(*node_traits::to_value_ptr(pDel), val) == 0); unsigned int nHeight = pDel->height(); - if ( try_remove_at( pDel, pos, f )) { - --m_ItemCounter; - m_Stat.onRemoveNode( nHeight ); - m_Stat.onEraseSuccess(); - return true; - } - - m_Stat.onEraseFailed(); - return false; + remove_at(pDel, pos, f); + --m_ItemCounter; + m_Stat.onRemoveNode(nHeight); + m_Stat.onEraseSuccess(); + return true; } template - guarded_ptr extract_( Q const& val, Compare cmp ) + guarded_ptr extract_(const Q& val, Compare cmp) { position pos; - guarded_ptr gp; - for (;;) { - if ( !find_position( val, pos, cmp, false )) { - m_Stat.onExtractFailed(); - return guarded_ptr(); - } + if (!find_position(val, pos, cmp)) { + m_Stat.onExtractFailed(); + return guarded_ptr(); + } - node_type * pDel = pos.pCur; - gp.reset( node_traits::to_value_ptr( pDel )); - assert( cmp( *node_traits::to_value_ptr( pDel ), val ) == 0 ); + node_type* pDel = pos.pCur; + guarded_ptr gp; + gp.reset(node_traits::to_value_ptr(pDel)); + assert(cmp(*node_traits::to_value_ptr(pDel), val) == 0); - unsigned int nHeight = pDel->height(); - if ( try_remove_at( pDel, pos, []( value_type const& ) {} )) { - --m_ItemCounter; - m_Stat.onRemoveNode( nHeight ); - m_Stat.onExtractSuccess(); - return gp; - } - m_Stat.onExtractRetry(); - } + unsigned int nHeight = pDel->height(); + remove_at(pDel, pos, [](const value_type&) {}); + --m_ItemCounter; + m_Stat.onRemoveNode(nHeight); + m_Stat.onExtractSuccess(); + return gp; } guarded_ptr extract_min_() @@ -1135,27 +940,22 @@ namespace cds { namespace intrusive { position pos; guarded_ptr gp; - for ( ;;) { - if ( !find_min_position( pos )) { - // The list is empty - m_Stat.onExtractMinFailed(); - return guarded_ptr(); - } - - node_type * pDel = pos.pCur; + if (!find_min_position(pos)) { + // The list is empty + m_Stat.onExtractMinFailed(); + return guarded_ptr(); + } - unsigned int nHeight = pDel->height(); - gp.reset( node_traits::to_value_ptr( pDel )); + node_type* pDel = pos.pCur; - if ( try_remove_at( pDel, pos, []( value_type const& ) {} )) { - --m_ItemCounter; - m_Stat.onRemoveNode( nHeight ); - m_Stat.onExtractMinSuccess(); - return gp; - } + unsigned int nHeight = pDel->height(); + gp.reset(node_traits::to_value_ptr(pDel)); - m_Stat.onExtractMinRetry(); - } + remove_at(pDel, pos, [](const value_type&) {} ); + --m_ItemCounter; + m_Stat.onRemoveNode(nHeight); + m_Stat.onExtractMinSuccess(); + return gp; } guarded_ptr extract_max_() @@ -1163,49 +963,41 @@ namespace cds { namespace intrusive { position pos; guarded_ptr gp; - for ( ;;) { - if ( !find_max_position( pos )) { - // The list is empty - m_Stat.onExtractMaxFailed(); - return guarded_ptr(); - } - - node_type * pDel = pos.pCur; + if (!find_max_position(pos)) { + // The list is empty + m_Stat.onExtractMaxFailed(); + return guarded_ptr(); + } - unsigned int nHeight = pDel->height(); - gp.reset( node_traits::to_value_ptr( pDel )); + node_type* pDel = pos.pCur; - if ( try_remove_at( pDel, pos, []( value_type const& ) {} )) { - --m_ItemCounter; - m_Stat.onRemoveNode( nHeight ); - m_Stat.onExtractMaxSuccess(); - return gp; - } + unsigned int nHeight = pDel->height(); + gp.reset(node_traits::to_value_ptr(pDel)); - m_Stat.onExtractMaxRetry(); - } + remove_at(pDel, pos, [](const value_type&) {} ); + --m_ItemCounter; + m_Stat.onRemoveNode(nHeight); + m_Stat.onExtractMaxSuccess(); + return gp; } - void increase_height( unsigned int nHeight ) + void increase_height(unsigned int nHeight) { - unsigned int nCur = m_nHeight.load( memory_model::memory_order_relaxed ); - if ( nCur < nHeight ) - m_nHeight.compare_exchange_strong( nCur, nHeight, memory_model::memory_order_relaxed, atomics::memory_order_relaxed ); + unsigned int nCurr = m_nHeight.load(memory_model::memory_order_relaxed); + if (nCurr < nHeight) + m_nHeight.compare_exchange_strong(nCurr, nHeight, memory_model::memory_order_relaxed, atomics::memory_order_relaxed); } void destroy() { - node_type* p = m_Head.head()->next( 0 ).load( atomics::memory_order_relaxed ).ptr(); - while ( p ) { - node_type* pNext = p->next( 0 ).load( atomics::memory_order_relaxed ).ptr(); - dispose_node( node_traits::to_value_ptr( p )); - p = pNext; + node_type* pNext = nullptr; + for (node_type* p = m_Head.head()->next(0).load(atomics::memory_order_relaxed).ptr(); p != nullptr; p = pNext) { + node_type* pNext = p->next(0).load(atomics::memory_order_relaxed).ptr(); + dispose_node(node_traits::to_value_ptr(p)); } } - private: - lazy_skip_list::details::head_node< node_type > m_Head; random_level_generator m_RandomLevelGen; From c16698aa84ff1a83bb64933a03915b5e2c902085 Mon Sep 17 00:00:00 2001 From: Ilya Date: Mon, 4 Feb 2019 09:57:35 +0300 Subject: [PATCH 3/3] fix dead lock and segfault --- cds/intrusive/impl/lazy_skip_list.h | 277 +++++++++++++++++----------- 1 file changed, 168 insertions(+), 109 deletions(-) diff --git a/cds/intrusive/impl/lazy_skip_list.h b/cds/intrusive/impl/lazy_skip_list.h index c2a86dc58..88258de3f 100644 --- a/cds/intrusive/impl/lazy_skip_list.h +++ b/cds/intrusive/impl/lazy_skip_list.h @@ -197,11 +197,8 @@ namespace cds { namespace intrusive { std::integral_constant >::type::value; - // c_nMaxHeight * 2 - pPred/pSucc guards - // + 1 - for erase, unlink - // + 1 - for clear - // + 1 - for help_remove() - static size_t const c_nHazardPtrCount = c_nMaxHeight * 2 + 3; + // pPrev / pCurr + static size_t const c_nHazardPtrCount = 3; protected: typedef typename node_type::atomic_marked_ptr atomic_node_ptr; @@ -236,11 +233,12 @@ namespace cds { namespace intrusive { void addLockPtr(node_type* pNode) { assert(pNode != nullptr); - assert(m_LockedCounter >= 2 * c_nMaxHeight + 1); if (contains(pNode)) return; + assert(m_LockedCounter <= c_nMaxHeight); + m_LockedArray[m_LockedCounter++] = pNode; } @@ -257,7 +255,7 @@ namespace cds { namespace intrusive { { for (auto& pNode : m_LockedArray) { if (!pNode) - return; + break; pNode->unlock(); pNode = nullptr; @@ -271,7 +269,7 @@ namespace cds { namespace intrusive { node_type* pCur; private: - node_type* m_LockedArray[2 * c_nMaxHeight + 1]; + node_type* m_LockedArray[c_nMaxHeight + 1]; unsigned int m_LockedCounter; }; @@ -280,7 +278,7 @@ namespace cds { namespace intrusive { : m_Head(c_nMaxHeight) , m_nHeight(c_nMinHeight) { - static_assert( (std::is_same< gc, typename node_type::gc >::value), "GC and node_type::gc must be the same type" ); + static_assert(std::is_same::value, "GC and node_type::gc must be the same type"); gc::check_available_guards(c_nHazardPtrCount); @@ -301,16 +299,16 @@ namespace cds { namespace intrusive { iterator begin() { - return iterator( *m_Head.head()); + return iterator(*m_Head.head()); } const_iterator begin() const { - return const_iterator( *m_Head.head()); + return const_iterator(*m_Head.head()); } const_iterator cbegin() const { - return const_iterator( *m_Head.head()); + return const_iterator(*m_Head.head()); } iterator end() @@ -331,7 +329,7 @@ namespace cds { namespace intrusive { bool insert(value_type& val) { - return insert( val, []( value_type& ) {} ); + return insert(val, [](value_type&) {}); } template @@ -346,7 +344,7 @@ namespace cds { namespace intrusive { position pos; - if (find_position(val, pos, key_comparator())) { + if (find_position(val, pos, key_comparator(), true)) { scp.release(); m_Stat.onInsertFailed(); return false; @@ -364,7 +362,6 @@ namespace cds { namespace intrusive { m_Stat.onAddNode(nHeight); m_Stat.onInsertSuccess(); scp.release(); - pos.unlockAll(); return true; } @@ -380,7 +377,7 @@ namespace cds { namespace intrusive { position pos; - if (find_position(val, pos, key_comparator())) { + if (find_position(val, pos, key_comparator(), true)) { scp.release(); func(false, *node_traits::to_value_ptr(pos.pCur), val); m_Stat.onUpdateExist(); @@ -418,7 +415,7 @@ namespace cds { namespace intrusive { { position pos; - if (!find_position(val, pos, key_comparator())) { + if (!find_position(val, pos, key_comparator(), false)) { m_Stat.onUnlinkFailed(); return false; } @@ -476,7 +473,7 @@ namespace cds { namespace intrusive { } template - bool erase( Q const& key, Func f ) + bool erase(const Q& key, Func f) { return erase_( key, key_comparator(), f ); } @@ -585,7 +582,9 @@ namespace cds { namespace intrusive { { // Random generator produces a number from range [0..31] // We need a number from range [1..32] - return m_RandomLevelGen() + 1; + unsigned int level = m_RandomLevelGen() + 1; + assert(level <= c_nMaxHeight); + return level; } template @@ -608,136 +607,201 @@ namespace cds { namespace intrusive { } template - bool find_position(const Q& val, position& pos, Compare cmp) + bool find_position(const Q& val, position& pos, Compare cmp, bool bStopIfFound) { typename gc::template GuardArray<2> guards; - node_type* pPrev; - marked_node_ptr pCurr; - node_type* pPrevAtThisLevel = m_Head.head(); - - int cmpResult = 1; + node_type * pPred; + marked_node_ptr pSucc; + marked_node_ptr pCur; - for (int nLevel = static_cast(c_nMaxHeight - 1); nLevel >= 0; --nLevel) { - retryAtLevel: - pPrev = pPrevAtThisLevel; - guards.assign(0, pPrev); + retry: + pPred = m_Head.head(); + int nCmp = 1; + for ( int nLevel = static_cast(c_nMaxHeight - 1); nLevel >= 0; --nLevel) { + guards.assign(0, node_traits::to_value_ptr(pPred)); while (true) { - pCurr = guards.protect(1, pPrev->next(nLevel), gc_protect); + pCur = guards.protect(1, pPred->next(nLevel), gc_protect); + if ( pCur.bits()) { + // pCur.bits() means that pPred is logically deleted + pos.unlockAll(); + goto retry; + } - if (nullptr == pCurr.ptr()) { + if ( pCur.ptr() == nullptr ) { // end of list at level nLevel - goto next level break; } - cmpResult = cmp(*node_traits::to_value_ptr(pCurr.ptr()), val); - if (cmpResult < 0) { - pPrev = pCurr.ptr(); - guards.copy(0, 1); // pPrev guard := pCurr guard - } else { - if (!pos.contains(pPrev)) { - pPrev->lock(); - } + // pSucc contains deletion mark for pCur + pSucc = pCur->next(nLevel).load(memory_model::memory_order_acquire); - if (pPrev->next(nLevel).load(memory_model::memory_order_relaxed) != pCurr.ptr()) { - if (!pos.contains(pPrev)) { - pPrev->unlock(); - } + if (pPred->next(nLevel).load(memory_model::memory_order_acquire).all() != pCur.ptr()) { + pos.unlockAll(); + goto retry; + } - goto retryAtLevel; + if (pSucc.bits()) { + // pCur is marked, i.e. logically deleted + // try to help deleting pCur +// help_remove(nLevel, pPred, pCur); + goto retry; + } + else { + nCmp = cmp(*node_traits::to_value_ptr(pCur.ptr()), val); + if ( nCmp < 0 ) { + pPred = pCur.ptr(); + guards.copy(0, 1); } - break; + else if (nCmp == 0 && bStopIfFound) + goto found; + else + break; } } - pos.addLockPtr(pPrev); - pos.pPrev[nLevel] = pPrev; - } + // Next level + if (!pos.contains(pPred)) { + pPred->lock(); + pos.addLockPtr(pPred); + } - if (cmpResult == 0) { - pos.pCur = pCurr.ptr(); - pos.pCur->lock(); - pos.addLockPtr(pos.pCur); + if (pPred->next(nLevel).load(memory_model::memory_order_acquire).all() != pCur.ptr()) { + pos.unlockAll(); + goto retry; + } + pos.pPrev[nLevel] = pPred; } - return cmpResult == 0; + if (nCmp != 0) + return false; + + found: + pos.pCur = pCur.ptr(); + return pCur.ptr() && nCmp == 0; } bool find_min_position(position& pos) { typename gc::template GuardArray<2> guards; - node_type* pPrev = m_Head.head(); - marked_node_ptr pCurr; + node_type * pPred; + marked_node_ptr pSucc; + marked_node_ptr pCur; - pPrev->lock(); - pos.addLockPtr(pPrev); + retry: + pPred = m_Head.head(); for (int nLevel = static_cast(c_nMaxHeight - 1); nLevel >= 0; --nLevel) { - pCurr = pPrev->next(nLevel); - pos.pPrev[nLevel] = pPrev; - } + guards.assign(0, node_traits::to_value_ptr( pPred )); + pCur = guards.protect(1, pPred->next(nLevel), gc_protect); + + // pCur.bits() means that pPred is logically deleted + // head cannot be deleted + assert(pCur.bits() == 0); - if (nullptr != (pos.pCur = pCurr.ptr())) { - pos.pCur->lock(); - pos.addLockPtr(pos.pCur); + if (pCur.ptr()) { + // pSucc contains deletion mark for pCur + pSucc = pCur->next(nLevel).load(memory_model::memory_order_acquire); + + if (pPred->next(nLevel).load(memory_model::memory_order_acquire).all() != pCur.ptr()) { + pos.unlockAll(); + goto retry; + } + + if (pSucc.bits()) { + // pCur is marked, i.e. logically deleted. + // try to help deleting pCur +// help_remove(nLevel, pPred, pCur); + pos.unlockAll(); + goto retry; + } + } + + if (!pos.contains(pPred)) { + pPred->lock(); + pos.addLockPtr(pPred); + } + + if (pPred->next(nLevel).load(memory_model::memory_order_acquire).all() != pCur.ptr()) { + pos.unlockAll(); + goto retry; + } + + // Next level + pos.pPrev[nLevel] = pPred; } - return pos.pCur != nullptr; + return (pos.pCur = pCur.ptr()) != nullptr; } bool find_max_position(position& pos) { typename gc::template GuardArray<2> guards; - node_type* pPrev; - marked_node_ptr pCurr; - node_type* pPrevAtThisLevel = m_Head.head(); + node_type * pPred; + marked_node_ptr pSucc; + marked_node_ptr pCur; - for (int nLevel = static_cast(c_nMaxHeight - 1); nLevel >= 0; --nLevel) { - retryAtLevel: - pPrev = pPrevAtThisLevel; - guards.assign(0, pPrev); + retry: + pPred = m_Head.head(); - while (true) { - pCurr = guards.protect(1, pPrev->next(nLevel), gc_protect); - if (pCurr.bits()) { + for (int nLevel = static_cast(c_nMaxHeight - 1); nLevel >= 0; --nLevel) { + guards.assign(0, node_traits::to_value_ptr(pPred)); + while ( true ) { + pCur = guards.protect(1, pPred->next(nLevel), gc_protect); + if (pCur.bits()) { // pCur.bits() means that pPred is logically deleted - goto retryAtLevel; + pos.unlockAll(); + goto retry; } - if (nullptr == pCurr || nullptr == pCurr->next(nLevel).load(memory_model::memory_order_relaxed)) { - bool lockThisNode = !pos.contains(pPrev); - if (lockThisNode) { - pPrev->lock(); - } + if (pCur.ptr() == nullptr) { + // end of the list at level nLevel - goto next level + break; + } - if (pPrev->next(nLevel).load(memory_model::memory_order_relaxed) != pCurr.ptr()) { - if (lockThisNode) { - pPrev->unlock(); - } - goto retryAtLevel; - } + // pSucc contains deletion mark for pCur + pSucc = pCur->next(nLevel).load(memory_model::memory_order_acquire); - break; + if (pPred->next(nLevel).load(memory_model::memory_order_acquire).all() != pCur.ptr()) { + pos.unlockAll(); + goto retry; } - pPrev = pCurr.ptr(); - guards.copy(0, 1); // pPrev guard := pCurr guard + if ( pSucc.bits()) { + // pCur is marked, i.e. logically deleted. + // try to help deleting pCur +// help_remove(nLevel, pPred, pCur); + pos.unlockAll(); + goto retry; + } + else { + if (!pSucc.ptr()) + break; + + pPred = pCur.ptr(); + guards.copy(0, 1); + } } - pos.addLockPtr(pPrev); - pos.pPrev[nLevel] = pPrev; - } + if (!pos.contains(pPred)) { + pPred->lock(); + pos.addLockPtr(pPred); + } + + if (pPred->next(nLevel).load(memory_model::memory_order_acquire).all() != pCur.ptr()) { + pos.unlockAll(); + goto retry; + } - if (nullptr != (pos.pCur = pCurr.ptr())) { - pos.pCur->lock(); - pos.addLockPtr(pos.pCur); + // Next level + pos.pPrev[nLevel] = pPred; } - return pos.pCur != nullptr; + return (pos.pCur = pCur.ptr()) != nullptr; } template @@ -747,22 +811,20 @@ namespace cds { namespace intrusive { //for each level item set next to nullptr for (unsigned int nLevel = 1; nLevel < nHeight; ++nLevel) - pNode->next( nLevel ).store(marked_node_ptr(), memory_model::memory_order_relaxed ); + pNode->next( nLevel ).store(marked_node_ptr(), memory_model::memory_order_relaxed); // Insert at level 0 - marked_node_ptr pNodePtr(pNode); pNode->lock(); + pos.addLockPtr(pNode); pNode->next(0).store(marked_node_ptr(pos.pPrev[0]->next(0)), memory_model::memory_order_relaxed); - pos.pPrev[0]->next(0).store(pNodePtr, memory_model::memory_order_relaxed); + pos.pPrev[0]->next(0).store(marked_node_ptr(pNode), memory_model::memory_order_relaxed); f(val); // Insert at level 1..max for (unsigned int nLevel = 1; nLevel < nHeight; ++nLevel) { pNode->next(nLevel).store(marked_node_ptr(pos.pPrev[nLevel]->next(nLevel)), memory_model::memory_order_relaxed); - pos.pPrev[nLevel]->next(nLevel).store(pNodePtr, memory_model::memory_order_relaxed); + pos.pPrev[nLevel]->next(nLevel).store(marked_node_ptr(pNode), memory_model::memory_order_relaxed); } - - pNode->unlock(); } template @@ -771,12 +833,9 @@ namespace cds { namespace intrusive { assert(pDel != nullptr); marked_node_ptr pSucc; - back_off bkoff; f(*node_traits::to_value_ptr(pDel)); - marked_node_ptr pNext(pDel->next(0).load(memory_model::memory_order_relaxed).ptr()); - for (int nLevel = static_cast(pDel->height() - 1); nLevel >= 0; --nLevel) { pSucc = pDel->next(nLevel).load(memory_model::memory_order_relaxed); pos.pPrev[nLevel]->next(nLevel).store(pSucc); @@ -809,7 +868,7 @@ namespace cds { namespace intrusive { try_again: pPred = m_Head.head(); for ( int nLevel = static_cast( m_nHeight.load( memory_model::memory_order_relaxed ) - 1 ); nLevel >= 0; --nLevel ) { - pCur = guards.protect( 1, pPred->next( nLevel ), gc_protect ); + pCur = guards.protect(1, pPred->next( nLevel ), gc_protect); while ( pCur != pNull ) { if ( pCur.bits()) { @@ -849,7 +908,7 @@ namespace cds { namespace intrusive { bool find_slowpath( Q& val, Compare cmp, Func f ) { position pos; - if (find_position(val, pos, cmp)) { + if (find_position(val, pos, cmp, true)) { f(*node_traits::to_value_ptr(pos.pCur), val); return true; } @@ -894,7 +953,7 @@ namespace cds { namespace intrusive { { position pos; - if (!find_position(val, pos, cmp)) { + if (!find_position(val, pos, cmp, false)) { m_Stat.onEraseFailed(); return false; } @@ -917,7 +976,7 @@ namespace cds { namespace intrusive { { position pos; - if (!find_position(val, pos, cmp)) { + if (!find_position(val, pos, cmp, false)) { m_Stat.onExtractFailed(); return guarded_ptr(); }