I am experiencing an issue with cameras/streaming crashing with my camera system. Here is a (hopefully) detailed description of the system I have currently:
I have 3 cameras (2x ZED X Mini, 1x ZED X) connected to an Nvidia Jetson Orin AGX via StereoLabs’ capture card with GMSL2 fakra cables.
After streaming for some amount of time, one of the cameras will crash. The only configuration for which we’ve found to have no issues with is 1 camera, SVGA resolution at 60 fps. Here is the behavior for several cases:
- 1x SVGA @ 60, any bitrate: completely stable
- 1x 1080 @ 60, 32k bitrate: ~48fps, (do we crash?)
- 2x SVGA @ 60, 12k bitrate: 60fps, ~50fps (respectively)
- 3x SVGA @ 60, 12k bitrate: all cameras start below 60 fps, but over a long period of time (>20 minutes) creep to 60 fps each. After a few minutes, one or two of the cameras dips far below 60 fps (<45 fps) and then crashes
- 3x SVGA @ 60, 32k bitrate: one camera sits decently comfortably at 60fps, while the two others sit around 30-40, and after some time the program crashes.
- 1x 1080 @ 60, 2x SVGA @ 60, 32k bitrate: no camera hits 60fps, crashes fairly fast.
After doing some profiling, I’ve deduced that the issue is not CPU or GPU limitations. Reducing bitrate helps make the system last a lot longer but still eventually crashes.
I’ve attached our code below:
#!/usr/bin/env python3
import pyzed.sl as sl
import sys
import time
import argparse
import threading
import signal
# Global variable to handle exit
exit_app = False
def signal_handler(signal, frame):
"""Handle Ctrl+C to properly exit the program"""
global exit_app
exit_app = True
print("\nCtrl+C pressed. Exiting...")
def parse_args():
parser = argparse.ArgumentParser(description='ZED Multi-Camera Streaming Sender')
parser.add_argument('--head_resolution', type=str, default='HD1080',
choices=['HD1200', 'HD1080', 'SVGA', 'None'],
help='Camera resolution (default: HD1080)')
parser.add_argument('--wrist_resolution', type=str, default='SVGA',
choices=['HD1200', 'HD1080', 'SVGA', 'None'],
help='Camera resolution (default: SVGA)')
parser.add_argument('--fps', type=int, default=60,
help='Frames per second (default: 60)')
parser.add_argument('--base-port', type=int, default=30000,
help='Base streaming port - each camera will use base_port + (2 * camera_index) (default: 30000)')
parser.add_argument('--bitrate', type=int, default=32768,
help='Streaming bitrate in Kbits/s (default: 32768 for high-bandwidth networks)')
parser.add_argument('--codec', type=str, default='H264',
choices=['H264', 'H265'],
help='Streaming codec (default: H264 - potentially slightly faster)')
parser.add_argument('--serials', nargs='*', type=int,
help='Specific camera serial numbers to use (if not provided, uses all available cameras)')
parser.add_argument('--status-interval', type=int, default=500,
help='Frames between status reports per camera (default: 1000 for less overhead)')
parser.add_argument('--chunk-size', type=int, default=8192,
help='Network chunk size in bytes (default: 8192 for low latency, range: 1024-32768)')
return parser.parse_args()
def get_resolution(res_str):
"""Convert resolution string to ZED SDK resolution enum"""
res_map = {
'HD1200': sl.RESOLUTION.HD1200, # ZED X native resolution: 1920x1200
'HD1080': sl.RESOLUTION.HD1080, # 1920x1080
'SVGA': sl.RESOLUTION.SVGA, # 960x600
'None': None, # Omit camera
}
return res_map.get(res_str, None)
def get_codec(codec_str):
"""Convert codec string to ZED SDK codec enum"""
codec_map = {
'H264': sl.STREAMING_CODEC.H264,
'H265': sl.STREAMING_CODEC.H265
}
return codec_map.get(codec_str, sl.STREAMING_CODEC.H264)
def print_device_info(devs, device_type="Camera"):
"""Print information about detected devices"""
if devs:
print(f"\n{device_type} devices found:")
for dev in devs:
print(f" ID: {dev.id}, Model: {dev.camera_model}, S/N: {dev.serial_number}, State: {dev.camera_state}")
else:
print(f"No {device_type} devices found")
def acquisition(zed, camera_info, args):
"""Acquisition thread function to continuously grab frames"""
# Pre-allocate all variables to avoid runtime allocations
camera_model = camera_info['model']
camera_serial = camera_info['serial']
camera_port = camera_info['port']
camera_id = f"{camera_model}_SN{camera_serial}"
frame_count = 0
start_time = time.time()
# Runtime parameters
runtime_params = sl.RuntimeParameters()
runtime_params.enable_depth = False
print(f"[{camera_id}] Starting acquisition thread on port {camera_port}")
while not exit_app:
if zed.grab(runtime_params) == sl.ERROR_CODE.SUCCESS:
frame_count += 1
# Print status every N frames
if frame_count % args.status_interval == 0:
elapsed_time = time.time() - start_time
fps = frame_count / elapsed_time
print(f"[{camera_id}] Frames: {frame_count}, FPS: {fps:.1f}")
# Reset counters for fresh FPS measurement
frame_count = 0
start_time = time.time()
print(f"[{camera_id}] QUIT - disabling streaming and closing camera")
# Cleanup
zed.disable_streaming()
zed.close()
def open_camera(zed, serial_number, camera_model, port, args):
"""Open a camera with given serial number and enable streaming with given port"""
# Determine camera type and create appropriate init parameters
if isinstance(zed, sl.Camera):
init_params = sl.InitParameters()
init_params.depth_mode = sl.DEPTH_MODE.NONE # Disable depth on sender to save computation
elif isinstance(zed, sl.CameraOne):
init_params = sl.CameraOne.InitParametersOne()
else:
print(f"Unsupported camera type: {type(zed)}")
return False, None
# Configure camera parameters - use appropriate resolution based on camera type
if str(camera_model) == "ZED X":
resolution_param = args.head_resolution
elif str(camera_model) == "ZED X Mini":
resolution_param = args.wrist_resolution
else:
print(f"Unsupported camera model: {camera_model}")
return False, None
resolution = get_resolution(resolution_param)
if resolution is None:
print(f"Skipping camera SN{serial_number}")
return False, None
if str(camera_model) == "ZED X":
init_params.camera_resolution = resolution
elif str(camera_model) == "ZED X Mini":
init_params.camera_resolution = resolution
else:
print(f"Unsupported camera model: {camera_model}")
return False, None
init_params.camera_fps = args.fps
init_params.sdk_verbose = 0 # Always disable for performance
# Ultra-performance optimizations to maximize frames sent without computing extra layers
init_params.camera_disable_self_calib = True # Always disable for streaming
init_params.enable_image_enhancement = False
init_params.camera_image_flip = sl.FLIP_MODE.OFF
init_params.enable_image_validity_check = 0
init_params.async_grab_camera_recovery = True
init_params.depth_stabilization = 0
# Set camera parameters
init_params.set_from_serial_number(serial_number)
# Open the camera
open_err = zed.open(init_params)
if open_err != sl.ERROR_CODE.SUCCESS:
print(f"ZED SN{serial_number} Error: {open_err}")
zed.close()
return False, None
# Get camera information
cam_info = zed.get_camera_information()
print(f"Opened: {cam_info.camera_model}_SN{serial_number}")
# Configure streaming parameters
stream_params = sl.StreamingParameters()
stream_params.codec = get_codec(args.codec)
stream_params.port = port
stream_params.bitrate = args.bitrate
stream_params.gop_size = 2 # keyframe interval
stream_params.adaptative_bitrate = False # Fixed bitrate for consistency
stream_params.chunk_size = args.chunk_size
stream_params.target_framerate = args.fps
# Enable streaming
stream_err = zed.enable_streaming(stream_params)
if stream_err != sl.ERROR_CODE.SUCCESS:
print(f"ZED SN{serial_number} Streaming initialization error: {stream_err}")
zed.close()
return False, None
print(f" Streaming enabled on port {port}")
# Return camera info for the acquisition thread
camera_info = {
'model': cam_info.camera_model,
'serial': serial_number,
'port': port
}
return True, camera_info
def main():
global exit_app
args = parse_args()
print(f"Starting ZED Multi-Camera Streaming Sender")
print(f"Head Resolution: {args.head_resolution}, {args.wrist_resolution}")
print(f"FPS: {args.fps}")
print(f"Base Port: {args.base_port}")
print(f"Bitrate: {args.bitrate} Kbits/s")
print(f"Codec: {args.codec}")
print(f"Chunk Size: {args.chunk_size} bytes")
# Get the list of available ZED cameras
dev_stereo_list = sl.Camera.get_device_list()
print_device_info(dev_stereo_list, "Stereo Camera")
if sys.platform != "win32":
dev_mono_list = sl.CameraOne.get_device_list()
print_device_info(dev_mono_list, "Mono Camera")
else:
dev_mono_list = []
# Filter cameras by serial numbers if specified
if args.serials:
print(f"\nFiltering cameras by serial numbers: {args.serials}")
dev_stereo_list = [dev for dev in dev_stereo_list if dev.serial_number in args.serials]
dev_mono_list = [dev for dev in dev_mono_list if dev.serial_number in args.serials]
print(f"Filtered to {len(dev_stereo_list)} stereo and {len(dev_mono_list)} mono cameras")
nb_cameras = len(dev_stereo_list) + len(dev_mono_list)
if nb_cameras == 0:
print("No ZED cameras detected or available, exiting")
return 1
print(f"\nAttempting to open {nb_cameras} cameras...")
# Create camera objects and open them
zeds = []
camera_infos = []
threads = []
cameras_opened = 0
# Open stereo cameras
for i, dev in enumerate(dev_stereo_list):
port = args.base_port + (2 * i)
zed = sl.Camera()
success, cam_info = open_camera(zed, dev.serial_number, dev.camera_model, port, args)
if success:
zeds.append(zed)
camera_infos.append(cam_info)
cameras_opened += 1
else:
print(f"Failed to open stereo camera SN{dev.serial_number}")
# Open mono cameras
for i, dev in enumerate(dev_mono_list):
port = args.base_port + (2 * (len(dev_stereo_list) + i))
zed = sl.CameraOne()
success, cam_info = open_camera(zed, dev.serial_number, dev.camera_model, port, args)
if success:
zeds.append(zed)
camera_infos.append(cam_info)
cameras_opened += 1
else:
print(f"Failed to open mono camera SN{dev.serial_number}")
if cameras_opened == 0:
print("No cameras could be opened, exiting")
return 1
print(f"\nSuccessfully opened {cameras_opened} cameras")
# Create and start acquisition threads
print("Starting acquisition threads...")
for i, (zed, cam_info) in enumerate(zip(zeds, camera_infos)):
thread = threading.Thread(target=acquisition, args=(zed, cam_info, args))
thread.daemon = True # Allow main thread to exit even if acquisition threads are running
threads.append(thread)
thread.start()
# Set up signal handler for Ctrl+C
signal.signal(signal.SIGINT, signal_handler)
print("All cameras streaming! Press Ctrl+C to stop")
# Main loop - minimize work here
try:
while not exit_app:
time.sleep(0.1) # Small sleep to prevent busy waiting
except KeyboardInterrupt:
exit_app = True
# Cleanup
print("\nStopping all streams...")
exit_app = True
time.sleep(0.2) # Give threads time to see the exit flag
# Wait for all threads to finish
print("Waiting for acquisition threads to finish...")
for thread in threads:
thread.join(timeout=2.0) # Wait up to 2 seconds for each thread
print("All cameras closed successfully")
return 0
if __name__ == "__main__":
sys.exit(main())
If anyone has any suggestions or fixes, I’d greatly appreciate it!
