Update Intel TBB library to v2021.1.1

CHANGELOG.md

@@ -1,3 +1,12 @@
+CasparCG 2.3.2 Beta
+==========================================
+
+### Producers
+##### Fixes
+* Packages: Update TBB library to v2021.1.1 - fixes CPU and memory growth when deleting threads
+* FFmpeg: Fix possible deadlock leading to producer not being cleaned up correctly
+
+
 CasparCG 2.3.1 Stable
 ==========================================
 

src/CMakeLists.txt

@@ -4,7 +4,7 @@ CMAKE_MINIMUM_REQUIRED (VERSION 3.0)
 PROJECT ("CasparCG Server")
 SET (CONFIG_VERSION_MAJOR 2)
 SET (CONFIG_VERSION_MINOR 3)
-SET (CONFIG_VERSION_BUG 1)
+SET (CONFIG_VERSION_BUG 2)
 SET (CONFIG_VERSION_TAG "Dev")
 
 option(ENABLE_HTML "Enable HTML module, require CEF" ON)

src/CMakeModules/Bootstrap_Windows.cmake

@@ -99,10 +99,10 @@ casparcg_add_runtime_dependency("${FFMPEG_BIN_PATH}/swscale-5.dll")
 
 # TBB
 set(TBB_INCLUDE_PATH "${PACKAGES_FOLDER}/tbb/include")
-set(TBB_BIN_PATH "${PACKAGES_FOLDER}/tbb/bin/win32")
-link_directories("${PACKAGES_FOLDER}/tbb/lib/win32")
-casparcg_add_runtime_dependency("${TBB_BIN_PATH}/tbb.dll")
-casparcg_add_runtime_dependency("${TBB_BIN_PATH}/tbb_debug.dll")
+set(TBB_BIN_PATH "${PACKAGES_FOLDER}/tbb/bin/intel64")
+link_directories("${PACKAGES_FOLDER}/tbb/lib/intel64")
+casparcg_add_runtime_dependency("${TBB_BIN_PATH}/tbb12.dll")
+casparcg_add_runtime_dependency("${TBB_BIN_PATH}/tbb12_debug.dll")
 casparcg_add_runtime_dependency("${TBB_BIN_PATH}/tbbmalloc.dll")
 casparcg_add_runtime_dependency("${TBB_BIN_PATH}/tbbmalloc_debug.dll")
 casparcg_add_runtime_dependency("${TBB_BIN_PATH}/tbbmalloc_proxy.dll")

src/accelerator/ogl/util/buffer.cpp

@@ -22,16 +22,14 @@
 
 #include <common/gl/gl_check.h>
 
-#include <tbb/atomic.h>
-
 #include <GL/glew.h>
 
 namespace caspar { namespace accelerator { namespace ogl {
 
-static tbb::atomic<int>         g_w_total_count;
-static tbb::atomic<std::size_t> g_w_total_size;
-static tbb::atomic<int>         g_r_total_count;
-static tbb::atomic<std::size_t> g_r_total_size;
+static std::atomic<int>         g_w_total_count;
+static std::atomic<std::size_t> g_w_total_size;
+static std::atomic<int>         g_r_total_count;
+static std::atomic<std::size_t> g_r_total_size;
 
 struct buffer::impl
 {

src/common/compiler/vs/disable_silly_warnings.h

@@ -23,6 +23,7 @@
 #pragma warning(disable : 4100) // unreferenced formal parameter
 #pragma warning(disable : 4127) // conditional expression is constant
 #pragma warning(disable : 4180) // qualifier applied to function type has no meaning; ignored
+#pragma warning(disable : 4324) // padding a structure for alignment
 #pragma warning(disable : 4355) // 'this' : used in base member initializer list
 #pragma warning(disable : 4482) // nonstandard extension used: enum 'enum' used in qualified name
 #pragma warning(disable : 4503) // decorated name length exceeded, name was truncated
@@ -31,6 +32,7 @@
 #pragma warning(disable : 4714) // marked as __forceinline not inlined
 #pragma warning(disable : 4505) // unreferenced local function has been
 #pragma warning(disable : 4481) // nonstandard extension used: override specifier 'override'
+#pragma warning(disable : 4834) // discarding return value of function with 'nodiscard' attribute
 #pragma warning(disable : 4996) // function call with parameters that may be unsafe
 #pragma warning(                                                                                                       \
     disable : 4334) // '<<': result of 32 - bit shift implicitly converted to 64 bits(was 64 - bit shift intended ?)

src/modules/ffmpeg/producer/av_input.cpp

@@ -101,6 +101,10 @@ void Input::abort()
 {
     abort_request_ = true;
     ic_cond_.notify_all();
+
+    std::shared_ptr<AVPacket> packet;
+    while (buffer_.try_pop(packet))
+        ;
 }
 
 void Input::reset()

src/packages/tbb/bin/intel64/tbb.def

@@ -0,0 +1,138 @@
+; Copyright (c) 2005-2020 Intel Corporation
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+;     http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+; This file is organized with a section for each .cpp file.
+
+EXPORTS
+
+; Assertions (assert.cpp)
+?assertion_failure@r1@detail@tbb@@YAXPEBDH00@Z
+
+; ITT (tbb_profiling.cpp)
+?call_itt_notify@r1@detail@tbb@@YAXHPEAX@Z
+?create_itt_sync@r1@detail@tbb@@YAXPEAXPEB_W1@Z
+?itt_make_task_group@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K12W4string_resource_index@d0@23@@Z
+?itt_task_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K12W4string_resource_index@d0@23@@Z
+?itt_task_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@@Z
+?itt_set_sync_name@r1@detail@tbb@@YAXPEAXPEB_W@Z
+?itt_metadata_str_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_KW4string_resource_index@d0@23@PEBD@Z
+?itt_relation_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_KW4itt_relation@d0@23@12@Z
+?itt_region_begin@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K12W4string_resource_index@d0@23@@Z
+?itt_region_end@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_K@Z
+?itt_metadata_ptr_add@r1@detail@tbb@@YAXW4itt_domain_enum@d1@23@PEAX_KW4string_resource_index@d0@23@1@Z
+
+; Allocators (tbb_allocator.cpp)
+?cache_aligned_allocate@r1@detail@tbb@@YAPEAX_K@Z
+?cache_aligned_deallocate@r1@detail@tbb@@YAXPEAX@Z
+?cache_line_size@r1@detail@tbb@@YA_KXZ
+?allocate_memory@r1@detail@tbb@@YAPEAX_K@Z
+?deallocate_memory@r1@detail@tbb@@YAXPEAX@Z
+?is_tbbmalloc_used@r1@detail@tbb@@YA_NXZ
+
+; Small object pool (small_object_pool.cpp)
+?allocate@r1@detail@tbb@@YAPEAXAEAPEAVsmall_object_pool@d1@23@_KAEBUexecution_data@523@@Z
+?allocate@r1@detail@tbb@@YAPEAXAEAPEAVsmall_object_pool@d1@23@_K@Z
+?deallocate@r1@detail@tbb@@YAXAEAVsmall_object_pool@d1@23@PEAX_KAEBUexecution_data@523@@Z
+?deallocate@r1@detail@tbb@@YAXAEAVsmall_object_pool@d1@23@PEAX_K@Z
+
+; Error handling (exception.cpp)
+?throw_exception@r1@detail@tbb@@YAXW4exception_id@d0@23@@Z
+?what@bad_last_alloc@r1@detail@tbb@@UEBAPEBDXZ
+?what@user_abort@r1@detail@tbb@@UEBAPEBDXZ
+?what@missing_wait@r1@detail@tbb@@UEBAPEBDXZ
+
+; RTM Mutex (rtm_mutex.cpp)
+?try_acquire@r1@detail@tbb@@YA_NAEAVrtm_mutex@d1@23@AEAVscoped_lock@4523@@Z
+?acquire@r1@detail@tbb@@YAXAEAVrtm_mutex@d1@23@AEAVscoped_lock@4523@_N@Z
+?release@r1@detail@tbb@@YAXAEAVscoped_lock@rtm_mutex@d1@23@@Z
+
+; RTM RW Mutex (rtm_rw_mutex.cpp)
+?acquire_writer@r1@detail@tbb@@YAXAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z
+?acquire_reader@r1@detail@tbb@@YAXAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z
+?upgrade@r1@detail@tbb@@YA_NAEAVscoped_lock@rtm_rw_mutex@d1@23@@Z
+?downgrade@r1@detail@tbb@@YA_NAEAVscoped_lock@rtm_rw_mutex@d1@23@@Z
+?try_acquire_writer@r1@detail@tbb@@YA_NAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@@Z
+?try_acquire_reader@r1@detail@tbb@@YA_NAEAVrtm_rw_mutex@d1@23@AEAVscoped_lock@4523@@Z
+?release@r1@detail@tbb@@YAXAEAVscoped_lock@rtm_rw_mutex@d1@23@@Z
+
+; Tasks and partitioners (task.cpp)
+?suspend@r1@detail@tbb@@YAXP6AXPEAXPEAUsuspend_point_type@123@@Z0@Z
+?resume@r1@detail@tbb@@YAXPEAUsuspend_point_type@123@@Z
+?current_suspend_point@r1@detail@tbb@@YAPEAUsuspend_point_type@123@XZ
+?notify_waiters@r1@detail@tbb@@YAX_K@Z
+
+; Task dispatcher (task_dispatcher.cpp)
+?spawn@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@@Z
+?spawn@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@G@Z
+?execute_and_wait@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@AEAVwait_context@523@1@Z
+?execution_slot@r1@detail@tbb@@YAGPEBUexecution_data@d1@23@@Z
+?wait@r1@detail@tbb@@YAXAEAVwait_context@d1@23@AEAVtask_group_context@523@@Z
+?submit@r1@detail@tbb@@YAXAEAVtask@d1@23@AEAVtask_group_context@523@PEAVarena@123@_K@Z
+?current_context@r1@detail@tbb@@YAPEAVtask_group_context@d1@23@XZ
+
+; Task group context (task_group_context.cpp)
+?initialize@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z
+?destroy@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z
+?is_group_execution_cancelled@r1@detail@tbb@@YA_NAEAVtask_group_context@d1@23@@Z
+?reset@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z
+?cancel_group_execution@r1@detail@tbb@@YA_NAEAVtask_group_context@d1@23@@Z
+?capture_fp_settings@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@@Z
+
+; Task arena (arena.cpp)
+?max_concurrency@r1@detail@tbb@@YAHPEBVtask_arena_base@d1@23@@Z
+?initialize@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@@Z
+?terminate@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@@Z
+?execute@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@AEAVdelegate_base@523@@Z
+?wait@r1@detail@tbb@@YAXAEAVtask_arena_base@d1@23@@Z
+
+; NUMA support (governor.cpp)
+?numa_node_count@r1@detail@tbb@@YAIXZ
+?fill_numa_indices@r1@detail@tbb@@YAXPEAH@Z
+?numa_default_concurrency@r1@detail@tbb@@YAHH@Z
+?attach@r1@detail@tbb@@YA_NAEAVtask_arena_base@d1@23@@Z
+?isolate_within_arena@r1@detail@tbb@@YAXAEAVdelegate_base@d1@23@_J@Z
+?enqueue@r1@detail@tbb@@YAXAEAVtask@d1@23@PEAVtask_arena_base@523@@Z
+
+; Observer (observer_proxy.cpp)
+?observe@r1@detail@tbb@@YAXAEAVtask_scheduler_observer@d1@23@_N@Z
+
+; Queuing RW Mutex (queuing_rw_mutex.cpp)
+?construct@r1@detail@tbb@@YAXAEAVqueuing_rw_mutex@d1@23@@Z
+?try_acquire@r1@detail@tbb@@YA_NAEAVqueuing_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z
+?acquire@r1@detail@tbb@@YAXAEAVqueuing_rw_mutex@d1@23@AEAVscoped_lock@4523@_N@Z
+?release@r1@detail@tbb@@YAXAEAVscoped_lock@queuing_rw_mutex@d1@23@@Z
+?upgrade_to_writer@r1@detail@tbb@@YA_NAEAVscoped_lock@queuing_rw_mutex@d1@23@@Z
+?downgrade_to_reader@r1@detail@tbb@@YA_NAEAVscoped_lock@queuing_rw_mutex@d1@23@@Z
+
+; Global control (global_control.cpp)
+?global_control_active_value@r1@detail@tbb@@YA_KH@Z
+?create@r1@detail@tbb@@YAXAEAVglobal_control@d1@23@@Z
+?destroy@r1@detail@tbb@@YAXAEAVglobal_control@d1@23@@Z
+?get@r1@detail@tbb@@YAXAEAVtask_scheduler_handle@d1@23@@Z
+?finalize@r1@detail@tbb@@YA_NAEAVtask_scheduler_handle@d1@23@_J@Z
+
+; Parallel pipeline (parallel_pipeline.cpp)
+?set_end_of_input@r1@detail@tbb@@YAXAEAVbase_filter@d1@23@@Z
+?parallel_pipeline@r1@detail@tbb@@YAXAEAVtask_group_context@d1@23@_KAEBVfilter_node@523@@Z
+
+; Concurrent bounded queue (concurrent_bounded_queue.cpp)
+?allocate_bounded_queue_rep@r1@detail@tbb@@YAPEAE_K@Z
+?deallocate_bounded_queue_rep@r1@detail@tbb@@YAXPEAE_K@Z
+?wait_bounded_queue_monitor@r1@detail@tbb@@YAXPEAVconcurrent_monitor@123@_K_JAEAVdelegate_base@d1@23@@Z
+?abort_bounded_queue_monitors@r1@detail@tbb@@YAXPEAVconcurrent_monitor@123@@Z
+?notify_bounded_queue_monitor@r1@detail@tbb@@YAXPEAVconcurrent_monitor@123@_K1@Z
+
+;; Versioning (version.cpp)
+TBB_runtime_interface_version
+TBB_runtime_version

src/packages/tbb/bin/intel64/tbbmalloc.def

@@ -0,0 +1,47 @@
+; Copyright (c) 2005-2020 Intel Corporation
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+;     http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+
+EXPORTS
+
+; frontend.cpp
+scalable_calloc
+scalable_free
+scalable_malloc
+scalable_realloc
+scalable_posix_memalign
+scalable_aligned_malloc
+scalable_aligned_realloc
+scalable_aligned_free
+scalable_msize
+scalable_allocation_mode
+scalable_allocation_command
+__TBB_malloc_safer_free
+__TBB_malloc_safer_realloc
+__TBB_malloc_safer_msize
+__TBB_malloc_safer_aligned_msize
+__TBB_malloc_safer_aligned_realloc
+
+; memory pool stuff
+?pool_create@rml@@YAPEAVMemoryPool@1@_JPEBUMemPoolPolicy@1@@Z
+?pool_create_v1@rml@@YA?AW4MemPoolError@1@_JPEBUMemPoolPolicy@1@PEAPEAVMemoryPool@1@@Z
+?pool_destroy@rml@@YA_NPEAVMemoryPool@1@@Z
+?pool_malloc@rml@@YAPEAXPEAVMemoryPool@1@_K@Z
+?pool_free@rml@@YA_NPEAVMemoryPool@1@PEAX@Z
+?pool_reset@rml@@YA_NPEAVMemoryPool@1@@Z
+?pool_realloc@rml@@YAPEAXPEAVMemoryPool@1@PEAX_K@Z
+?pool_aligned_realloc@rml@@YAPEAXPEAVMemoryPool@1@PEAX_K2@Z
+?pool_aligned_malloc@rml@@YAPEAXPEAVMemoryPool@1@_K1@Z
+?pool_identify@rml@@YAPEAVMemoryPool@1@PEAX@Z
+?pool_msize@rml@@YA_KPEAVMemoryPool@1@PEAX@Z
+

src/packages/tbb/include/oneapi/tbb.h

@@ -0,0 +1,73 @@
+/*
+    Copyright (c) 2005-2020 Intel Corporation
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+#ifndef __TBB_tbb_H
+#define __TBB_tbb_H
+
+/**
+    This header bulk-includes declarations or definitions of all the functionality
+    provided by TBB (save for tbbmalloc and 3rd party dependent headers).
+
+    If you use only a few TBB constructs, consider including specific headers only.
+    Any header listed below can be included independently of others.
+**/
+
+#include "oneapi/tbb/blocked_range.h"
+#include "oneapi/tbb/blocked_range2d.h"
+#include "oneapi/tbb/blocked_range3d.h"
+#if TBB_PREVIEW_BLOCKED_RANGE_ND
+#include "tbb/blocked_rangeNd.h"
+#endif
+#include "oneapi/tbb/cache_aligned_allocator.h"
+#include "oneapi/tbb/combinable.h"
+#include "oneapi/tbb/concurrent_hash_map.h"
+#if TBB_PREVIEW_CONCURRENT_LRU_CACHE
+#include "tbb/concurrent_lru_cache.h"
+#endif
+#include "oneapi/tbb/concurrent_priority_queue.h"
+#include "oneapi/tbb/concurrent_queue.h"
+#include "oneapi/tbb/concurrent_unordered_map.h"
+#include "oneapi/tbb/concurrent_unordered_set.h"
+#include "oneapi/tbb/concurrent_map.h"
+#include "oneapi/tbb/concurrent_set.h"
+#include "oneapi/tbb/concurrent_vector.h"
+#include "oneapi/tbb/enumerable_thread_specific.h"
+#include "oneapi/tbb/flow_graph.h"
+#include "oneapi/tbb/global_control.h"
+#include "oneapi/tbb/info.h"
+#include "oneapi/tbb/null_mutex.h"
+#include "oneapi/tbb/null_rw_mutex.h"
+#include "oneapi/tbb/parallel_for.h"
+#include "oneapi/tbb/parallel_for_each.h"
+#include "oneapi/tbb/parallel_invoke.h"
+#include "oneapi/tbb/parallel_pipeline.h"
+#include "oneapi/tbb/parallel_reduce.h"
+#include "oneapi/tbb/parallel_scan.h"
+#include "oneapi/tbb/parallel_sort.h"
+#include "oneapi/tbb/partitioner.h"
+#include "oneapi/tbb/queuing_mutex.h"
+#include "oneapi/tbb/queuing_rw_mutex.h"
+#include "oneapi/tbb/spin_mutex.h"
+#include "oneapi/tbb/spin_rw_mutex.h"
+#include "oneapi/tbb/task.h"
+#include "oneapi/tbb/task_arena.h"
+#include "oneapi/tbb/task_group.h"
+#include "oneapi/tbb/task_scheduler_observer.h"
+#include "oneapi/tbb/tbb_allocator.h"
+#include "oneapi/tbb/tick_count.h"
+#include "oneapi/tbb/version.h"
+
+#endif /* __TBB_tbb_H */