We’ve been tracking the instability issues with our cameras and one of the issues we found is the following:
#include <iostream>
#include <sl/Camera.hpp>
int main(int argc, char* argv[]) {
sl::InitParameters initParams { };
initParams.camera_resolution = sl::RESOLUTION::HD1080;
initParams.camera_fps = 15;
sl::Camera camera;
if (auto err = camera.open(initParams); err != sl::ERROR_CODE::SUCCESS) {
std::cout << "open failed: " << sl::toString(err) << std::endl;
return EXIT_FAILURE;
}
std::cout << "closing" << std::endl;
camera.close();
std::cout << "done" << std::endl;
return 0;
}
cmake_minimum_required(VERSION 3.10)
project (zedx_tsan VERSION 0.1)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
SET(CMAKE_BUILD_TYPE "RelWithDebInfo")
# ASAN
#add_compile_options(-fsanitize=address -fno-omit-frame-pointer)
#add_link_options(-fsanitize=address)
# TSAN
add_compile_options(-fsanitize=thread)
add_link_options(-fsanitize=thread)
# UBSAN
#add_compile_options(-fsanitize=undefined)
#add_link_options(-fsanitize=undefined)
find_package(ZED REQUIRED)
find_package(CUDA ${ZED_CUDA_VERSION} REQUIRED)
add_executable(${PROJECT_NAME} zedx_tsan.cpp)
target_include_directories(${PROJECT_NAME} PRIVATE
${ZED_INCLUDE_DIRS}
${CUDA_INCLUDE_DIRS}
)
link_directories(${ZED_LIBRARY_DIR} ${CUDA_LIBRARY_DIRS})
target_link_libraries(${PROJECT_NAME} PRIVATE
${ZED_LIBRARIES}
${CUDA_CUDA_LIBRARY}
)
Zed X camera, Zed Link Capture Card Duo, ZedSDK 5.4.1, CUDA version: V13.2.86, on Jetson Orin NX with Jetpack 7.2.1, the program is compiled with gcc 13.3.0 with thread sanitizer enabled. ZED_Diagnostics is green. The program above gives the following report when `open` is called:
==================
WARNING: ThreadSanitizer: data race (pid=138119)
Write of size 8 at 0xffff81a00078 by thread T15 (mutexes: write M0):
#0 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:115 (libtsan.so.2+0x930fc) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:107 (libtsan.so.2+0x930fc)
#2 sl::ArgusCamera::readSBSFromDual(unsigned char*, unsigned long long&) <null> (libsl_zed.so+0x791a7c) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
Previous read of size 8 at 0xffff81a00078 by thread T24:
#0 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:115 (libtsan.so.2+0x930fc) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:107 (libtsan.so.2+0x930fc)
#2 <null> <null> (libsl_zed.so+0x1fc0dd0) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
#3 std::__future_base::_Async_state_impl<std::thread::_Invoker<std::tuple<sl::ERROR_CODE (sl::CameraMemberHandler::*)(), sl::CameraMemberHandler*> >, sl::ERROR_CODE>::_M_run() <null> (libsl_zed.so+0x1f68b68) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
As if synchronized via sleep:
#0 nanosleep ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:397 (libtsan.so.2+0x5bba8) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 sl::ArgusCamera::consume() <null> (libsl_zed.so+0x7940c4) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
Location is heap block of size 2792 at 0xffff81a00000 allocated by main thread:
#0 operator new(unsigned long) ../../../../src/libsanitizer/tsan/tsan_new_delete.cpp:64 (libtsan.so.2+0xad458) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 sl::GMSLInput::setUp(int&, int, int, int, bool, sl::IMU_FREQ, bool, bool, int) <null> (libsl_zed.so+0x7e6a78) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
#2 __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58 (libc.so.6+0x284c0) (BuildId: d5ef86dde36cbd3289566cf5098226035d76f2e1)
Mutex M0 (0xffff81a00570) created at:
#0 pthread_mutex_lock ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1341 (libtsan.so.2+0x5cc14) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 sl::ArgusCamera::readSBSFromDual(unsigned char*, unsigned long long&) <null> (libsl_zed.so+0x79185c) (BuildId: 3584267cb36b2d701f8a7ce36ee700a9a1ea4514)
Thread T15 (tid=138259, running) created by main thread at:
#0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1022 (libtsan.so.2+0x5dd98) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xe1c64) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
#2 __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58 (libc.so.6+0x284c0) (BuildId: d5ef86dde36cbd3289566cf5098226035d76f2e1)
Thread T24 (tid=138288, running) created by main thread at:
#0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1022 (libtsan.so.2+0x5dd98) (BuildId: dc0d657f42587fea53114ce058dbdfe5834e0205)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xe1c64) (BuildId: 7c239d1743727054caa3df1ab866bfa72cd4dd93)
#2 __libc_start_call_main ../sysdeps/nptl/libc_start_call_main.h:58 (libc.so.6+0x284c0) (BuildId: d5ef86dde36cbd3289566cf5098226035d76f2e1)
SUMMARY: ThreadSanitizer: data race ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc:115 in memcpy
==================
This may be one of the causes for the issues we face when Argus daemon gets rebooted in the background. Fwiw this happens regardless whether we have any other cameras connected or not.