How to save the same frame of images, depth and point cloud simultaneously?

       if frame_count == 30:
            #cam.retrieve_image(image, sl.VIEW.LEFT)  
            cam.retrieve_image(image, sl.VIEW.LEFT_UNRECTIFIED) 
            img_left = "left_" + str(svo_position) + ".png"
            img_left = os.path.join(left_img_path, img_left)
            img = image.write(img_left)
            if img == sl.ERROR_CODE.SUCCESS:
                print("Saved image : ",img_left)
            else:
                print("Something wrong happened in image saving... ")

            # depth
            cam.retrieve_measure(depth, sl.MEASURE.DEPTH)  
            depth_data = depth.get_data()
            depth_left = "left_" + str(svo_position) + '.npy'
            depth_left = os.path.join(left_depth_path, depth_left)
            np.save(depth_left, depth_data)

            #depth_img
            cam.retrieve_image(image, sl.VIEW.NORMALS)  
            img_left_depth = "left_depth_" + str(svo_position) + ".png"
            img_left_depth = os.path.join(left_img_depth_path, img_left_depth)
            img = image.write(img_left_depth)
            if img == sl.ERROR_CODE.SUCCESS:
                print("Saved image : ",img_left_depth)
            else:
                print("Something wrong happened in image saving... ")

            #point_cloud
             cam.retrieve_measure(point_cloud, sl.MEASURE.XYZRGBA)
             point_left = 'left_' + str(svo_position) + '.ply'
             point_left = os.path.join(left_point_path, point_left)
             err = point_cloud.write(point_left)
             if (err == sl.ERROR_CODE.SUCCESS):
                 print("Current .ply file saving succeed")
             else:
                 print("Current .ply file failed")

Hello, I want to capture the image, depth and point cloud simultaneously every 30 frames through the recorded svo2. However, the point cloud captured in this way is incorrect. How can I solve this problem?

Hi @pigcat88
Welcome to the StereoLabs community

Please share the full code with the initial camera setup

import sys
import pyzed.sl as sl
import cv2
import argparse
import os
import numpy as np


def progress_bar(percent_done, bar_length=50):
    # Display progress bar
    done_length = int(bar_length * percent_done / 100)
    bar = '=' * done_length + '-' * (bar_length - done_length)
    sys.stdout.write('[%s] %i%s\r' % (bar, percent_done, '%'))
    sys.stdout.flush()


def main():
    filepath = opt.input_svo_file  # Path to the .svo file to be playbacked
    input_type = sl.InputType()
    input_type.set_from_svo_file(filepath)  # Set init parameter to run from the .svo
    init = sl.InitParameters(input_t=input_type, svo_real_time_mode=False)
    init.camera_disable_self_calib = True
    init.optional_opencv_calibration_file = "zed_calibration_1080.yml"  # Read the calibration parameters of opencv
    init.depth_mode = sl.DEPTH_MODE.LAST
    init.coordinate_units = sl.UNIT.METER
    init.depth_minimum_distance = 0.2
    init.depth_maximum_distance = 20
    #init.enable_right_side_measure = True  #Allow the calculation of the depth on the right side
    cam = sl.Camera()
    status = cam.open(init)
    if status != sl.ERROR_CODE.SUCCESS:  # Ensure the camera opened succesfully
        print("Camera Open", status, "Exit program.")
        exit(1)

    runtime = sl.RuntimeParameters()
    # Depth Map Filling Mode
    runtime.enable_fill_mode = True
    #Adjust the confidence level
    #runtime.confidence_threshold = 90

    image = sl.Mat()
    depth = sl.Mat()
    point_cloud = sl.Mat()



    nb_frames = cam.get_svo_number_of_frames()
    print("[Info] SVO contains ", nb_frames, " frames")

    frame_count = 0  #Frame counter
    prefix_path = 'da-6'
    left_img_path = os.path.join(prefix_path,'left/img')
    left_depth_path = os.path.join(prefix_path,'left/depth')
    left_img_depth_path = os.path.join(prefix_path, 'left/depth_img')
    left_point_path = os.path.join(prefix_path, 'left/point')

    right_img_path = os.path.join(prefix_path, 'right/img')
    right_depth_path = os.path.join(prefix_path, 'right/depth')
    right_img_depth_path = os.path.join(prefix_path, 'right/depth_img')
    right_point_path = os.path.join(prefix_path, 'right/point')

    while True:
        err = cam.grab(runtime)
        if err == sl.ERROR_CODE.SUCCESS:
            # The position of the current frame
            svo_position = cam.get_svo_position()
            # Save the left image and the depth map every 15 frames
            if frame_count == 30:
                cam.retrieve_image(image, sl.VIEW.LEFT_UNRECTIFIED)
                img_left = "left_" + str(svo_position) + ".png"
                img_left = os.path.join(left_img_path, img_left)
                img = image.write(img_left)
                if img == sl.ERROR_CODE.SUCCESS:
                    print("Saved image : ",img_left)
                else:
                    print("Something wrong happened in image saving... ")

                # Save the left depth
                cam.retrieve_measure(depth, sl.MEASURE.DEPTH)
                depth_data = depth.get_data()
                depth_left = "left_" + str(svo_position) + '.npy'
                depth_left = os.path.join(left_depth_path, depth_left)
                np.save(depth_left, depth_data)

                #Save the left depth map
                cam.retrieve_image(image, sl.VIEW.NORMALS)
                img_left_depth = "left_depth_" + str(svo_position) + ".png"
                img_left_depth = os.path.join(left_img_depth_path, img_left_depth)
                img = image.write(img_left_depth)
                if img == sl.ERROR_CODE.SUCCESS:
                    print("Saved image : ",img_left_depth)
                else:
                    print("Something wrong happened in image saving... ")

                #Save the point cloud on the left
                cam.retrieve_measure(point_cloud, sl.MEASURE.XYZRGBA)
                point_left = 'left_' + str(svo_position) + '.ply'
                point_left = os.path.join(left_point_path, point_left)
                err = point_cloud.write(point_left)
                if (err == sl.ERROR_CODE.SUCCESS):
                    print("Current .ply file saving succeed")
                else:
                    print("Current .ply file failed")


                # Save the right picture
                # cam.retrieve_image(image, sl.VIEW.RIGHT)
                # img_right = "right_" + str(svo_position) + ".png"
                # img_right = os.path.join(right_img_path, img_right)
                # img = image.write(img_right)
                # if img == sl.ERROR_CODE.SUCCESS:
                #     print("Saved image : ",img_right)
                # else:
                #     print("Something wrong happened in image saving... ")
                #
                # Save the right depth
                # cam.retrieve_measure(depth, sl.MEASURE.DEPTH_RIGHT)
                # depth_data = depth.get_data()
                # depth_right = str(svo_position)+'_right_depth.npy'
                # depth_right = os.path.join(right_depth_path, depth_right)
                # np.save(depth_right, depth_data)
                #
                # Save the depth map on the right
                # cam.retrieve_image(depth, sl.VIEW.NORMALS_RIGHT)
                # img_right_depth = "right_depth_" + str(svo_position) + ".png"
                # img_right_depth = os.path.join(right_img_depth_path, img_right_depth)
                # img = depth.write(img_right_depth)
                # if img == sl.ERROR_CODE.SUCCESS:
                #     print("Saved image : ",img_right_depth)
                # else:
                #     print("Something wrong happened in image saving... ")

                # Save the point cloud on the right
                # cam.retrieve_measure(point_cloud, sl.MEASURE.XYZRGBA_RIGHT)
                # point_right = 'right_' + str(svo_position) + '.ply'
                # point_right = os.path.join(right_point_path, point_right)
                # err = point_cloud.write(point_right)
                # if (err == sl.ERROR_CODE.SUCCESS):
                #     print("Current .ply file saving succeed")
                # else:
                #     print("Current .ply file failed")

                frame_count = 1  # Reset the frame counter
            else:
                frame_count += 1  # Increase the frame counter

        elif err == sl.ERROR_CODE.END_OF_SVOFILE_REACHED:  # Check if the .svo has ended
            progress_bar(100, 30)
            # print("SVO end has been reached. Looping back to 0")
            # cam.set_svo_position(0)
            print("SVO end has been reached. Exiting program.")
            break
        else:
            print("Grab ZED : ", err)
            break
    cv2.destroyAllWindows()
    cam.close()


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--input_svo_file', type=str, help='Path to the SVO file', required=True)
    opt = parser.parse_args()
    if not opt.input_svo_file.endswith(".svo") and not opt.input_svo_file.endswith(".svo2"):
        print("--input_svo_file parameter should be a .svo file but is not : ", opt.input_svo_file, "Exit program.")
        exit()
    if not os.path.isfile(opt.input_svo_file):
        print("--input_svo_file parameter should be an existing file but is not : ", opt.input_svo_file,
              "Exit program.")
        exit()
    main()

Thank you very much for your reply. I have uploaded the complete code and related comments.

Hello @pigcat,

As I look to your code, I see that you init your camera with a depth_mode parameter as sl.DEPTH_MODE.LAST. Why are you using this enum which is not in the doc ? We provide several depth modes. Please use one of these: DEPTH_MODE Class Reference | API Reference | Stereolabs

Let us know if it solves your issue !

Best regards,

1 Like

This doesn’t seem to be a matter of deep mode selection. I have tested each deep mode.
In this example “zed-sdk/depth sensing/depth sensing/python/depth_sensing.py at master · stereolabs/zed-sdk · GitHub”, The point cloud played and the point cloud saved by pressing “S” are normal. However, this code cannot meet my requirements. I need to save the image, depth, and point cloud of the same frame simultaneously, and repeat this operation at certain intervals.

I used ZED2I underwater. After calibration with opencv, the depth information obtained by sl.DEPTH_MODE.LAST seems to be the most accurate. Although I don’t know exactly which depth mode sl.DEPTH_MODE.LAST is because the documentation doesn’t specify it. However, the inaccurate saving of point clouds has nothing to do with the depth at which I use this mode, because the point clouds saved in other depth modes are also incorrect.

This is not a depth mode. The SDK will replace this value with a valid mode.

When you work underwater you must recalibrate the camera because the factory calibration is performed in the air. The water is like a lens, so the default calibration is not good, resulting in bad depth information.

Read more here

and here

Thank you for your answer. Regarding underwater usage and calibration, I have reviewed and completed the underwater recalibration work.
The key issue now is I can effectively see through this code “zed-sdk/depth sensing/depth sensing/python/depth_sensing.py at master · stereolabs/zed-sdk · GitHub” Point cloud, but I can’t save the point cloud as I thought. What should I do next to save the point cloud corresponding to a certain frame of image at the same time and save it automatically at certain time intervals?

This is a Python exercise, not a ZED SDK issue.
I recommend you use a timer to measure the elapsed time and call grab and the required retrieve_measure and retrieve_image functions at the required frequency.

This link could help you: https://stackoverflow.com/questions/3393612/run-certain-code-every-n-seconds