Hey,
i want to send obj detection data to another computer over OSC/UDP.
attached is a snippet of the processing loop.
right now an osc message is sent every 200 ms for 80% of the time … sometimes around 250ms, outliers around 3000 and 0. i would like to get it down to a rate of 15 frames.
iḿ running a zed 2i on a ZED BOX Orin NX 8GB. in the end the project needs 2 cameras sending object detectino data to the other computer.
Are the specs of the Orin NX 8 GB good enough? ( also in regard to my code quality ^^)
attached is also a pastebin with the whole code ( OSC bit is line 330 to 370 )
#if ENABLE_GUI
gl_viewer_available &&
#endif
!quit && zed.grab(runtime_parameters) == ERROR_CODE::SUCCESS)
{
// update confidence threshold based on TrackBar
if (detection_parameters_rt.object_class_filter.empty())
detection_parameters_rt.detection_confidence_threshold = detection_confidence;
else // if using class filter, set confidence for each class
for (auto& it : detection_parameters_rt.object_class_filter)
detection_parameters_rt.object_class_detection_confidence_threshold[it] = detection_confidence;
returned_state = zed.retrieveObjects(objects, detection_parameters_rt);
if (returned_state == ERROR_CODE::SUCCESS)
{
//what rate here?
// Iterate through detected objects and send data via OSC
for (int i = 0; i < objects.object_list.size(); i++)
{
sl::ObjectData& obj = objects.object_list[i];
float objVol = calculateVolume(obj.dimensions);
float velocity = calcTotalVelocity(obj.velocity);
std::string label = objectClassToString( obj.label );
std::string state = trackingStateToString(obj.tracking_state);
//cout << "OSC sending " << endl;
// Send OSC message (you'll need to replace this with your actual OSC send function)
sendOSCMessage( "/front",
obj.id,
obj.position.x,
obj.position.y,
obj.position.z,
label,
state,
objVol, // size
velocity, // summed velocity
obj.confidence);
// Right after sending an OSC message:
auto currentTimePoint = std::chrono::high_resolution_clock::now();
// Calculating the interval (difference) in milliseconds
auto interval = std::chrono::duration_cast<std::chrono::milliseconds>(currentTimePoint - previousTimePoint).count();
// Update previousTimePoint for the next iteration
previousTimePoint = currentTimePoint;
// Output the interval
std::cout << "OSC message sent, interval since last message: " << interval << " ms" << std::endl;
}
#if ENABLE_BATCHING_REID
// store the id of detetected objects
for(auto &it: objects.object_list)
id_counter[it.id] = 1;
// check if bacthed trajectories are available
std::vector<sl::ObjectsBatch> objectsBatch;
if(zed.getObjectsBatch(objectsBatch)==sl::ERROR_CODE::SUCCESS)
{
if(objectsBatch.size()){
std::cout<<"During last batch processing: "<<id_counter.size()<<" Object were detected: ";
for(auto it :id_counter) std::cout<<it.first<<" ";
std::cout<<"\nWhile "<<objectsBatch.size()<<" different only after reID: ";
for(auto it :objectsBatch) std::cout<<it.id<<" ";
std::cout<<std::endl;
id_counter.clear();
}
}
#endif
#if ENABLE_GUI
zed.retrieveMeasure(point_cloud, MEASURE::XYZRGBA, MEM::GPU, pc_resolution);
zed.getPosition(cam_w_pose, REFERENCE_FRAME::WORLD);
zed.retrieveImage(image_left, VIEW::LEFT, MEM::CPU, display_resolution);
image_render_left.copyTo(image_left_ocv);
track_view_generator.generate_view(objects, image_left_ocv,img_scale,cam_w_pose, image_track_ocv, objects.is_tracked);
// Tiefenbild
// viewer.updateData(point_cloud, objects.object_list, cam_w_pose.pose_data);
// gl_viewer_available = viewer.isAvailable();
// CONVERT BETWEEN CV_8UC4 AND CV_8UC3
cv::Mat img_bgra; // Your CV_8UC4 image
cv::Mat global_image_c3; // C3 image
cv::cvtColor(global_image, global_image_c3, cv::COLOR_BGRA2BGR);
cout << "frame: " << frameCnt++ << endl;
// Write the frame into the file
video.write(global_image_c3);
// as image_left_ocv and image_track_ocv are both ref of global_image, no need to update it
cv::imshow(window_name, global_image_c3);
#if ENABLE_REC
#endif
key = cv::waitKey(10);
if (key == 'i') {
track_view_generator.zoomIn();
} else if (key == 'o') {
track_view_generator.zoomOut();
} else if (key == 'q') {
quit = true;
} else if (key == 'p') {
zed.setSVOPosition(2900);
//detection_parameters_rt.object_class_filter.clear();
//detection_parameters_rt.object_class_filter.push_back(OBJECT_CLASS::PERSON);
//detection_parameters_rt.object_class_detection_confidence_threshold[OBJECT_CLASS::PERSON] = detection_confidence;
cout << "Person only" << endl;
} else if (key == 'v') {
detection_parameters_rt.object_class_filter.clear();
detection_parameters_rt.object_class_filter.push_back(OBJECT_CLASS::VEHICLE);
detection_parameters_rt.object_class_detection_confidence_threshold[OBJECT_CLASS::VEHICLE] = detection_confidence;
cout << "Vehicle only" << endl;
} else if (key == 'c') {
detection_parameters_rt.object_class_filter.clear();
detection_parameters_rt.object_class_detection_confidence_threshold.clear();
cout << "Clear Filters" << endl;
}
//cout << "Current Position: " << zed.getSVOPosition() << ", Total Frames: " << zed.getSVONumberOfFrames() << ", Is Playback: " << is_playback << endl;
//if (is_playback && zed.getSVOPosition() == zed.getSVONumberOfFrames() ) // framerate playblack problem
if (is_playback && zed.getSVOPosition() == 200 ) // framerate playblack problem
{
cout << "setSVOPos 0" << endl;
//zed.setSVOPosition(0); // LOOP file
quit = true;
}
#endif
}
} // ENABLE GUI / while loop - END