We’ve been tracking the instability issues with our cameras and one of the issues we found is the following:
#include <chrono>
#include <iostream>
#include <iterator>
#include <mutex>
#include <ranges>
#include <syncstream>
#include <vector>
#include <thread>
#include <utility>
#include <sl/CameraOne.hpp>
int main(int argc, char* argv[]) {
// std::mutex mutex { };
auto func = [&](auto stopToken, auto& device) {
sl::InitParametersOne initParams { };
initParams.camera_resolution = sl::RESOLUTION::HD1080;
initParams.camera_fps = 15;
initParams.sdk_verbose = 1;
initParams.input.setFromSerialNumber(device.serial_number);
sl::CameraOne camera { };
{
// std::lock_guard lock { mutex };
if (auto err = camera.open(initParams); err != sl::ERROR_CODE::SUCCESS) {
std::cout << "open failed: " << sl::toString(err) << std::endl;
return;
}
}
std::size_t counter { 0 };
while (!stopToken.stop_requested()) {
auto t1 = std::chrono::steady_clock::now();
if (auto err = camera.grab(); err != sl::ERROR_CODE::SUCCESS) {
std::osyncstream(std::cout) << counter << " " << std::chrono::steady_clock::now() - t1 << " error" << std::endl;
} else {
std::osyncstream(std::cout) << counter << " " << std::chrono::steady_clock::now() - t1 << " ok" << std::endl;
}
++counter;
std::this_thread::sleep_for(std::chrono::milliseconds { 10 });
}
std::osyncstream(std::cout) << "thread closing" << std::endl;
camera.close();
std::osyncstream(std::cout) << "thread done" << std::endl;
};
std::vector<std::jthread> threads { };
std::ranges::copy(
sl::CameraOne::getDeviceList() |
std::ranges::views::filter([](auto&& d) {
return d.camera_model == sl::MODEL::ZED_XONE_GS ||
d.camera_model == sl::MODEL::ZED_XONE_UHD ||
d.camera_model == sl::MODEL::ZED_XONE_HDR;
}) |
std::ranges::views::take(3) |
std::ranges::views::transform([&](auto&& d) {
return std::jthread { [d, &func](auto stopToken) {
return func(stopToken, d);
} };
}),
std::back_inserter(threads)
);
std::osyncstream(std::cout) << "sleeping for 15s" << std::endl;
std::this_thread::sleep_for(std::chrono::seconds { 15 });
std::ranges::for_each(
threads,
[](auto&& t) {
t.request_stop();
t.join();
}
);
std::cout << "done" << std::endl;
return 0;
}
cmake_minimum_required(VERSION 3.10)
project (zedone_tsan VERSION 0.1)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
SET(CMAKE_BUILD_TYPE "RelWithDebInfo")
# ASAN
#add_compile_options(-fsanitize=address -fno-omit-frame-pointer)
#add_link_options(-fsanitize=address)
# TSAN
add_compile_options(-fsanitize=thread)
add_link_options(-fsanitize=thread)
# UBSAN
#add_compile_options(-fsanitize=undefined)
#add_link_options(-fsanitize=undefined)
find_package(ZED REQUIRED)
find_package(CUDA ${ZED_CUDA_VERSION} REQUIRED)
add_executable(${PROJECT_NAME} zedone_tsan.cpp)
target_include_directories(${PROJECT_NAME} PRIVATE
${ZED_INCLUDE_DIRS}
${CUDA_INCLUDE_DIRS}
)
link_directories(${ZED_LIBRARY_DIR} ${CUDA_LIBRARY_DIRS})
target_link_libraries(${PROJECT_NAME} PRIVATE
${ZED_LIBRARIES}
${CUDA_CUDA_LIBRARY}
)
Three Zed X One cameras, Zed Link Capture Card Duo, ZedSDK 5.4.1, CUDA version: V13.2.86, on Jetson Orin NX with Jetpack 7.2.1, the program is compiled with gcc 13.3.0 with thread sanitizer enabled. ZED_Diagnostics is green. The program above gives the first report on `open` and the second one on `grab`:
WARNING: ThreadSanitizer: data race (pid=38839)
Write of size 8 at 0xffffa0fe4510 by thread T12 (mutexes: write M0):
#0 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:115 (libtsan.so.2+0x930fc) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:107 (libtsan.so.2+0x930fc)
#2 <null> <null> (libnvargus_socketclient.so+0x196f2c) (BuildId: 66cda73d5955063b65e82a1c027dded2c727992b)
#3 operator()<std::stop_token> <stripped>/zedone_tsan.cpp:56 (zedone_tsan+0x66a4) (BuildId: 82e9a6a46629fa6af7ccc2450f1549285a759ff0)
#4 __invoke_impl<void, main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:61 (zedone_tsan+0x66a4)
#5 __invoke<main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:96 (zedone_tsan+0x66a4)
#6 _M_invoke<0, 1> /usr/include/c++/13/bits/std_thread.h:292 (zedone_tsan+0x66a4)
#7 operator() /usr/include/c++/13/bits/std_thread.h:299 (zedone_tsan+0x66a4)
#8 _M_run /usr/include/c++/13/bits/std_thread.h:244 (zedone_tsan+0x66a4)
#9 <null> <null> (libstdc++.so.6+0xe1adc) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
Previous write of size 8 at 0xffffa0fe4510 by thread T13 (mutexes: write M1):
#0 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:115 (libtsan.so.2+0x930fc) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:107 (libtsan.so.2+0x930fc)
#2 <null> <null> (libnvargus_socketclient.so+0x196f2c) (BuildId: 66cda73d5955063b65e82a1c027dded2c727992b)
#3 operator()<std::stop_token> <stripped>/zedone_tsan.cpp:56 (zedone_tsan+0x66a4) (BuildId: 82e9a6a46629fa6af7ccc2450f1549285a759ff0)
#4 __invoke_impl<void, main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:61 (zedone_tsan+0x66a4)
#5 __invoke<main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:96 (zedone_tsan+0x66a4)
#6 _M_invoke<0, 1> /usr/include/c++/13/bits/std_thread.h:292 (zedone_tsan+0x66a4)
#7 operator() /usr/include/c++/13/bits/std_thread.h:299 (zedone_tsan+0x66a4)
#8 _M_run /usr/include/c++/13/bits/std_thread.h:244 (zedone_tsan+0x66a4)
#9 <null> <null> (libstdc++.so.6+0xe1adc) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
Location is global '<null>' at 0x000000000000 (libnvargus_socketclient.so+0x294510)
Mutex M0 (0xffffac4e5f30) created at:
#0 pthread_mutex_lock ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1341 (libtsan.so.2+0x5cc14) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 sl::ArgusProvider::getInstance() <null> (libsl_zed.so+0x7c4518) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
#2 __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58 (libc.so.6+0x284c0) (BuildId: d5ef86dde36cbd3289566cf5098226035d76f2e1)
Mutex M1 (0xffffac4e42e8) created at:
#0 pthread_mutex_lock ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1341 (libtsan.so.2+0x5cc14) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 sl::ArgusCamera::getGMSLDevices(bool, bool) <null> (libsl_zed.so+0x7abfe4) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
#2 __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58 (libc.so.6+0x284c0) (BuildId: d5ef86dde36cbd3289566cf5098226035d76f2e1)
Thread T12 (tid=38935, running) created by main thread at:
#0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1022 (libtsan.so.2+0x5dd98) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xe1c64) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
#2 __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58 (libc.so.6+0x284c0) (BuildId: d5ef86dde36cbd3289566cf5098226035d76f2e1)
Thread T13 (tid=38936, running) created by main thread at:
#0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1022 (libtsan.so.2+0x5dd98) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xe1c64) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
#2 __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58 (libc.so.6+0x284c0) (BuildId: d5ef86dde36cbd3289566cf5098226035d76f2e1)
SUMMARY: ThreadSanitizer: data race ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:115 in memcpy
WARNING: ThreadSanitizer: data race (pid=38839)
Write of size 8 at 0xffff92a03c78 by thread T18:
#0 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:115 (libtsan.so.2+0x930fc) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:107 (libtsan.so.2+0x930fc)
#2 sl::ArgusCamera::readSingle(unsigned char*, unsigned long long&) <null> (libsl_zed.so+0x7922ac) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
Previous read of size 8 at 0xffff92a03c78 by thread T11 (mutexes: write M0):
#0 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:115 (libtsan.so.2+0x930fc) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:107 (libtsan.so.2+0x930fc)
#2 <null> <null> (libsl_zed.so+0x202030c) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
#3 operator()<std::stop_token> <stripped>/zedone_tsan.cpp:56 (zedone_tsan+0x66a4) (BuildId: 82e9a6a46629fa6af7ccc2450f1549285a759ff0)
#4 __invoke_impl<void, main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:61 (zedone_tsan+0x66a4)
#5 __invoke<main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:96 (zedone_tsan+0x66a4)
#6 _M_invoke<0, 1> /usr/include/c++/13/bits/std_thread.h:292 (zedone_tsan+0x66a4)
#7 operator() /usr/include/c++/13/bits/std_thread.h:299 (zedone_tsan+0x66a4)
#8 _M_run /usr/include/c++/13/bits/std_thread.h:244 (zedone_tsan+0x66a4)
#9 <null> <null> (libstdc++.so.6+0xe1adc) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
Location is heap block of size 2792 at 0xffff92a03c00 allocated by thread T11:
#0 operator new(unsigned long) ../../../../src/libsanitizer/tsan/tsan_new_delete.cpp:64 (libtsan.so.2+0xad458) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 sl::GMSLInput::setUp(int&, int, int, int, bool, sl::IMU_FREQ, bool, bool, int) <null> (libsl_zed.so+0x7e6a78) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
#2 operator()<std::stop_token> <stripped>/zedone_tsan.cpp:56 (zedone_tsan+0x66a4) (BuildId: 82e9a6a46629fa6af7ccc2450f1549285a759ff0)
#3 __invoke_impl<void, main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:61 (zedone_tsan+0x66a4)
#4 __invoke<main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:96 (zedone_tsan+0x66a4)
#5 _M_invoke<0, 1> /usr/include/c++/13/bits/std_thread.h:292 (zedone_tsan+0x66a4)
#6 operator() /usr/include/c++/13/bits/std_thread.h:299 (zedone_tsan+0x66a4)
#7 _M_run /usr/include/c++/13/bits/std_thread.h:244 (zedone_tsan+0x66a4)
#8 <null> <null> (libstdc++.so.6+0xe1adc) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
Mutex M0 (0xffff926056f8) created at:
#0 pthread_mutex_trylock ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1354 (libtsan.so.2+0x5b0a4) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 <null> <null> (libsl_zed.so+0x2020758) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
#2 operator()<std::stop_token> <stripped>/zedone_tsan.cpp:56 (zedone_tsan+0x66a4) (BuildId: 82e9a6a46629fa6af7ccc2450f1549285a759ff0)
#3 __invoke_impl<void, main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:61 (zedone_tsan+0x66a4)
#4 __invoke<main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:96 (zedone_tsan+0x66a4)
#5 _M_invoke<0, 1> /usr/include/c++/13/bits/std_thread.h:292 (zedone_tsan+0x66a4)
#6 operator() /usr/include/c++/13/bits/std_thread.h:299 (zedone_tsan+0x66a4)
#7 _M_run /usr/include/c++/13/bits/std_thread.h:244 (zedone_tsan+0x66a4)
#8 <null> <null> (libstdc++.so.6+0xe1adc) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
Thread T18 (tid=39284, running) created by thread T11 at:
#0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1022 (libtsan.so.2+0x5dd98) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xe1c64) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
#2 operator()<std::stop_token> <stripped>/zedone_tsan.cpp:56 (zedone_tsan+0x66a4) (BuildId: 82e9a6a46629fa6af7ccc2450f1549285a759ff0)
#3 __invoke_impl<void, main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:61 (zedone_tsan+0x66a4)
#4 __invoke<main(int, char**)::<lambda(auto:56&&)>::<lambda(auto:57)>, std::stop_token> /usr/include/c++/13/bits/invoke.h:96 (zedone_tsan+0x66a4)
#5 _M_invoke<0, 1> /usr/include/c++/13/bits/std_thread.h:292 (zedone_tsan+0x66a4)
#6 operator() /usr/include/c++/13/bits/std_thread.h:299 (zedone_tsan+0x66a4)
#7 _M_run /usr/include/c++/13/bits/std_thread.h:244 (zedone_tsan+0x66a4)
#8 <null> <null> (libstdc++.so.6+0xe1adc) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
Thread T11 (tid=38934, running) created by main thread at:
#0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1022 (libtsan.so.2+0x5dd98) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xe1c64) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
#2 __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58 (libc.so.6+0x284c0) (BuildId: d5ef86dde36cbd3289566cf5098226035d76f2e1)
SUMMARY: ThreadSanitizer: data race ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:115 in memcpy
Fwiw, while undesirable synchronising around `open` eliminates the first report. Doing the same for `grab` isn’t really possible.