From 6da7f0f999473cbb3c5af5594041ee885423c5b7 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Tue, 30 Nov 2021 16:27:30 -0600 Subject: [PATCH] New/improved support for initialization and finalization routines in ThreadPool (#19) * clang-tidy fixes * Improvements to ThreadPool constructors * Support for finalization functions in ThreadPool * Bumped version * PTL_INSTALL_HEADERS + PTL_INSTALL_CONFIG --- .clang-tidy | 2 ++ CMakeLists.txt | 8 ++++- VERSION | 2 +- source/CMakeLists.txt | 24 ++++++++------ source/PTL/TaskGroup.hh | 5 +-- source/PTL/TaskGroup.icc | 1 + source/PTL/ThreadPool.hh | 71 +++++++++++++++++++++++++++++----------- source/PTL/Types.hh | 36 -------------------- source/PTL/Utility.hh | 39 +++++++++++++++++++++- source/ThreadPool.cc | 22 ++++++++++++- source/UserTaskQueue.cc | 1 + 11 files changed, 139 insertions(+), 72 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 695e31b..d713dd4 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -18,6 +18,7 @@ modernize-*,\ -modernize-use-using,\ -modernize-use-auto,\ -modernize-concat-nested-namespaces,\ +-modernize-use-nodiscard,\ performance-*,\ readability-*,\ -readability-function-size,\ @@ -38,6 +39,7 @@ readability-*,\ -readability-static-accessed-through-instance,\ -readability-const-return-type,\ -readability-function-cognitive-complexity,\ +-readability-redundant-access-specifiers,\ " HeaderFilterRegex: 'source/[^/]*\.(hh|cc)$' CheckOptions: diff --git a/CMakeLists.txt b/CMakeLists.txt index 42e5a9c..f3aae59 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,13 @@ include(PTLBuildSettings) # User options ptl_add_option(PTL_USE_TBB "Enable TBB" ON) ptl_add_option(PTL_USE_LOCKS "Enable mutex locking in task subqueues for extra safety" OFF) +ptl_add_option(PTL_INSTALL_HEADERS "Install the headers" ON) +ptl_add_option(PTL_INSTALL_CONFIG "Install the cmake configuration" ON) + +if(DEFINED PTL_DEVELOPER_INSTALL) + set(PTL_INSTALL_HEADERS ${PTL_DEVELOPER_INSTALL} CACHE BOOL "Set via PTL_DEVELOPER_INSTALL" FORCE) + set(PTL_INSTALL_CONFIG ${PTL_DEVELOPER_INSTALL} CACHE BOOL "Set via PTL_DEVELOPER_INSTALL" FORCE) +endif() ################################################################################ # Build Dependencies @@ -92,4 +99,3 @@ endif() if(PTL_MASTER_PROJECT) ptl_print_features() endif() - diff --git a/VERSION b/VERSION index 7ec1d6d..ccbccc3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.0 +2.2.0 diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index adf8552..f7a5a49 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -91,13 +91,17 @@ endif() # # ################################################################################ -# install export -install(EXPORT ${PROJECT_NAME}Targets - NAMESPACE PTL:: - DESTINATION ${PTL_INSTALL_CMAKEDIR} - COMPONENT Development) - -# headers -install(FILES ${ptl_headers} - DESTINATION ${PTL_INSTALL_INCLUDEDIR}/PTL - COMPONENT Development) +if(PTL_INSTALL_CONFIG) + # install export + install(EXPORT ${PROJECT_NAME}Targets + NAMESPACE PTL:: + DESTINATION ${PTL_INSTALL_CMAKEDIR} + COMPONENT Development) +endif() + +if(PTL_INSTALL_HEADERS) + # headers + install(FILES ${ptl_headers} + DESTINATION ${PTL_INSTALL_INCLUDEDIR}/PTL + COMPONENT Development) +endif() diff --git a/source/PTL/TaskGroup.hh b/source/PTL/TaskGroup.hh index e3a4afd..621b6dd 100644 --- a/source/PTL/TaskGroup.hh +++ b/source/PTL/TaskGroup.hh @@ -35,6 +35,7 @@ #include "PTL/Task.hh" #include "PTL/ThreadData.hh" #include "PTL/ThreadPool.hh" +#include "PTL/Utility.hh" #include #include @@ -113,9 +114,9 @@ public: // define move-construct TaskGroup(this_type&& rhs) = default; // delete copy-assign - this_type& operator=(const this_type& rhs) = delete; + TaskGroup& operator=(const this_type& rhs) = delete; // define move-assign - this_type& operator=(this_type&& rhs) = default; + TaskGroup& operator=(this_type&& rhs) = default; public: template diff --git a/source/PTL/TaskGroup.icc b/source/PTL/TaskGroup.icc index 38324f9..b8af898 100644 --- a/source/PTL/TaskGroup.icc +++ b/source/PTL/TaskGroup.icc @@ -30,6 +30,7 @@ // --------------------------------------------------------------- #include "PTL/Types.hh" +#include "PTL/Utility.hh" namespace PTL { diff --git a/source/PTL/ThreadPool.hh b/source/PTL/ThreadPool.hh index 37c5096..a7f0868 100644 --- a/source/PTL/ThreadPool.hh +++ b/source/PTL/ThreadPool.hh @@ -84,25 +84,50 @@ public: using task_pointer = std::shared_ptr; using task_queue_t = VUserTaskQueue; // containers - typedef std::deque thread_list_t; - typedef std::vector bool_list_t; - typedef std::map thread_id_map_t; - typedef std::map thread_index_map_t; - using thread_vec_t = std::vector; - using thread_data_t = std::vector>; + using thread_list_t = std::deque; + using bool_list_t = std::vector; + using thread_id_map_t = std::map; + using thread_index_map_t = std::map; + using thread_vec_t = std::vector; + using thread_data_t = std::vector>; // functions - typedef std::function initialize_func_t; - typedef std::function affinity_func_t; + using initialize_func_t = std::function; + using finalize_func_t = std::function; + using affinity_func_t = std::function; + + static affinity_func_t& affinity_functor() + { + static affinity_func_t _v = [](intmax_t) { + static std::atomic assigned; + intmax_t _assign = assigned++; + return _assign % Thread::hardware_concurrency(); + }; + return _v; + } + + static initialize_func_t& initialization_functor() + { + static initialize_func_t _v = []() {}; + return _v; + } + + static finalize_func_t& finalization_functor() + { + static finalize_func_t _v = []() {}; + return _v; + } public: // Constructor and Destructors ThreadPool(const size_type& pool_size, VUserTaskQueue* task_queue = nullptr, bool _use_affinity = GetEnv("PTL_CPU_AFFINITY", false), - affinity_func_t = [](intmax_t) { - static std::atomic assigned; - intmax_t _assign = assigned++; - return _assign % Thread::hardware_concurrency(); - }); + affinity_func_t = affinity_functor(), + initialize_func_t = initialization_functor(), + finalize_func_t = finalization_functor()); + ThreadPool(const size_type& pool_size, initialize_func_t, finalize_func_t, + bool _use_affinity = GetEnv("PTL_CPU_AFFINITY", false), + affinity_func_t = affinity_functor(), + VUserTaskQueue* task_queue = nullptr); // Virtual destructors are required by abstract classes // so add it by default, just in case virtual ~ThreadPool(); @@ -149,11 +174,16 @@ public: // only relevant when compiled with PTL_USE_TBB static tbb_global_control_t*& tbb_global_control(); - void set_initialization(initialize_func_t f) { m_init_func = f; } + void set_initialization(initialize_func_t f) { m_init_func = std::move(f); } + void set_finalization(finalize_func_t f) { m_fini_func = std::move(f); } + void reset_initialization() { - auto f = []() {}; - m_init_func = f; + m_init_func = []() {}; + } + void reset_finalization() + { + m_fini_func = []() {}; } public: @@ -175,7 +205,7 @@ public: return (m_thread_awake) ? m_thread_awake->load() : 0; } - void set_affinity(affinity_func_t f) { m_affinity_func = f; } + void set_affinity(affinity_func_t f) { m_affinity_func = std::move(f); } void set_affinity(intmax_t i, Thread&); void set_verbose(int n) { m_verbose = n; } @@ -257,8 +287,9 @@ private: tbb_task_group_t* m_tbb_task_group = nullptr; // functions - initialize_func_t m_init_func = []() {}; - affinity_func_t m_affinity_func; + initialize_func_t m_init_func = initialization_functor(); + finalize_func_t m_fini_func = finalization_functor(); + affinity_func_t m_affinity_func = affinity_functor(); private: // Private static variables @@ -354,7 +385,7 @@ ThreadPool::run_on_this(task_pointer&& _task) if(m_tbb_tp && m_tbb_task_group) { - auto _arena = get_task_arena(); + auto* _arena = get_task_arena(); _arena->execute([this, _func]() { this->m_tbb_task_group->run(_func); }); } else diff --git a/source/PTL/Types.hh b/source/PTL/Types.hh index 42d5dde..bab3aff 100644 --- a/source/PTL/Types.hh +++ b/source/PTL/Types.hh @@ -123,42 +123,6 @@ GetSharedPointerPairMasterInstance() //======================================================================================// -struct ScopeDestructor -{ - template - ScopeDestructor(FuncT&& _func) - : m_functor(std::forward(_func)) - {} - - // delete copy operations - ScopeDestructor(const ScopeDestructor&) = delete; - ScopeDestructor& operator=(const ScopeDestructor&) = delete; - - // allow move operations - ScopeDestructor(ScopeDestructor&& rhs) noexcept - : m_functor(std::move(rhs.m_functor)) - { - rhs.m_functor = []() {}; - } - - ScopeDestructor& operator=(ScopeDestructor&& rhs) noexcept - { - if(this != &rhs) - { - m_functor = std::move(rhs.m_functor); - rhs.m_functor = []() {}; - } - return *this; - } - - ~ScopeDestructor() { m_functor(); } - -private: - std::function m_functor = []() {}; -}; - -//======================================================================================// - } // namespace PTL // Forward declation of void type argument for usage in direct object diff --git a/source/PTL/Utility.hh b/source/PTL/Utility.hh index 2966824..8f03b7a 100644 --- a/source/PTL/Utility.hh +++ b/source/PTL/Utility.hh @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -312,7 +313,7 @@ GetEnv(const std::string& env_id, const EnvChoiceList& _choices, Tp _default template Tp -GetChoice(const EnvChoiceList& _choices, const std::string str_var) +GetChoice(const EnvChoiceList& _choices, const std::string& str_var) { auto asupper = [](std::string var) { for(auto& itr : var) @@ -363,4 +364,40 @@ PrintEnv(std::ostream& os = std::cout) //--------------------------------------------------------------------------------------// +struct ScopeDestructor +{ + template + ScopeDestructor(FuncT&& _func) + : m_functor(std::forward(_func)) + {} + + // delete copy operations + ScopeDestructor(const ScopeDestructor&) = delete; + ScopeDestructor& operator=(const ScopeDestructor&) = delete; + + // allow move operations + ScopeDestructor(ScopeDestructor&& rhs) noexcept + : m_functor(std::move(rhs.m_functor)) + { + rhs.m_functor = []() {}; + } + + ScopeDestructor& operator=(ScopeDestructor&& rhs) noexcept + { + if(this != &rhs) + { + m_functor = std::move(rhs.m_functor); + rhs.m_functor = []() {}; + } + return *this; + } + + ~ScopeDestructor() { m_functor(); } + +private: + std::function m_functor = []() {}; +}; + +//--------------------------------------------------------------------------------------// + } // namespace PTL diff --git a/source/ThreadPool.cc b/source/ThreadPool.cc index 01dfc1f..cfcfd69 100644 --- a/source/ThreadPool.cc +++ b/source/ThreadPool.cc @@ -32,6 +32,7 @@ #include "PTL/Globals.hh" #include "PTL/ThreadData.hh" #include "PTL/UserTaskQueue.hh" +#include "PTL/Utility.hh" #include "PTL/VUserTaskQueue.hh" #include @@ -138,11 +139,14 @@ ThreadPool::get_this_thread_id() //======================================================================================// ThreadPool::ThreadPool(const size_type& pool_size, VUserTaskQueue* task_queue, - bool _use_affinity, affinity_func_t _affinity_func) + bool _use_affinity, affinity_func_t _affinity_func, + initialize_func_t _init_func, finalize_func_t _fini_func) : m_use_affinity(_use_affinity) , m_pool_state(std::make_shared(thread_pool::state::NONINIT)) , m_task_queue(task_queue) , m_affinity_func(std::move(_affinity_func)) +, m_init_func(std::move(_init_func)) +, m_fini_func(std::move(_fini_func)) { auto master_id = get_this_thread_id(); if(master_id != 0 && m_verbose > 1) @@ -157,6 +161,17 @@ ThreadPool::ThreadPool(const size_type& pool_size, VUserTaskQueue* task_queue, m_task_queue = new UserTaskQueue(m_pool_size); } +ThreadPool::ThreadPool(const size_type& pool_size, initialize_func_t _init_func, + finalize_func_t _fini_func, bool _use_affinity, + affinity_func_t _affinity_func, VUserTaskQueue* task_queue) +: ThreadPool{ pool_size, + task_queue, + _use_affinity, + std::move(_affinity_func), + std::move(_init_func), + std::move(_fini_func) } +{} + //======================================================================================// ThreadPool::~ThreadPool() @@ -251,6 +266,8 @@ ThreadPool::initialize_threadpool(size_type proposed_size) // create task group (used for async) if(!m_tbb_task_group) m_tbb_task_group = new tbb_task_group_t(); + + execute_on_all_threads([this]() { m_init_func(); }); return m_pool_size; } #endif @@ -375,6 +392,7 @@ ThreadPool::destroy_threadpool() #if defined(PTL_USE_TBB) if(m_tbb_task_group) { + execute_on_all_threads([this]() { m_fini_func(); }); auto _func = [&]() { m_tbb_task_group->wait(); }; if(m_tbb_task_arena) m_tbb_task_arena->execute(_func); @@ -552,6 +570,8 @@ ThreadPool::execute_thread(VUserTaskQueue* _task_queue) // initialization function m_init_func(); + // finalization function (executed when scope is destroyed) + ScopeDestructor _fini{ [this]() { m_fini_func(); } }; ThreadId tid = ThisThread::get_id(); ThreadData* data = thread_data(); diff --git a/source/UserTaskQueue.cc b/source/UserTaskQueue.cc index 1443b1d..60d05e9 100644 --- a/source/UserTaskQueue.cc +++ b/source/UserTaskQueue.cc @@ -27,6 +27,7 @@ #include "PTL/Task.hh" #include "PTL/TaskGroup.hh" #include "PTL/ThreadPool.hh" +#include "PTL/Utility.hh" #include