From ffa78bf33a5d7b4d3535dbcf4134ac4107746a22 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 10 Oct 2022 13:24:27 +0800 Subject: [PATCH 01/40] fix engine api --- vino_core_lib/include/vino_core_lib/engines/engine.h | 9 +++++---- .../include/vino_core_lib/engines/engine_manager.h | 2 +- vino_core_lib/src/engines/engine.cpp | 2 +- vino_core_lib/src/engines/engine_manager.cpp | 12 ++++++------ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/engines/engine.h b/vino_core_lib/include/vino_core_lib/engines/engine.h index 5b862ac0..8302ba02 100644 --- a/vino_core_lib/include/vino_core_lib/engines/engine.h +++ b/vino_core_lib/include/vino_core_lib/engines/engine.h @@ -25,6 +25,7 @@ #include "vino_core_lib/models/base_model.h" #include "inference_engine.hpp" +#include "openvino/openvino.hpp" /** * @class NetworkEngine @@ -48,12 +49,12 @@ class Engine /** * @brief Using an Inference Request to initialize the inference Engine. */ - Engine(InferenceEngine::InferRequest::Ptr&); + Engine(ov::InferRequest &); /** * @brief Get the inference request this instance holds. * @return The inference request this instance holds. */ - inline InferenceEngine::InferRequest::Ptr& getRequest() + inline ov::InferRequest & getRequest() { return request_; } @@ -65,11 +66,11 @@ class Engine template void setCompletionCallback(const T& callbackToSet) { - request_->SetCompletionCallback(callbackToSet); + request_.set_callback(callbackToSet); } private: - InferenceEngine::InferRequest::Ptr request_ = nullptr; + ov::InferRequest request_; }; } // namespace Engines diff --git a/vino_core_lib/include/vino_core_lib/engines/engine_manager.h b/vino_core_lib/include/vino_core_lib/engines/engine_manager.h index ac908394..8dc1f09d 100644 --- a/vino_core_lib/include/vino_core_lib/engines/engine_manager.h +++ b/vino_core_lib/include/vino_core_lib/engines/engine_manager.h @@ -52,7 +52,7 @@ class EngineManager std::shared_ptr createEngine_beforeV2019R2(const std::string&, const std::shared_ptr&); #endif - std::shared_ptr createEngine_V2019R2_plus(const std::string&, const std::shared_ptr&); + std::shared_ptr createEngine_V2022(const std::string&, const std::shared_ptr&); }; } // namespace Engines diff --git a/vino_core_lib/src/engines/engine.cpp b/vino_core_lib/src/engines/engine.cpp index 38bd9b97..9066e838 100644 --- a/vino_core_lib/src/engines/engine.cpp +++ b/vino_core_lib/src/engines/engine.cpp @@ -27,7 +27,7 @@ Engines::Engine::Engine(InferenceEngine::InferencePlugin plg, const Models::Base } #endif -Engines::Engine::Engine(InferenceEngine::InferRequest::Ptr& request) +Engines::Engine::Engine(ov::InferRequest & request) { request_ = request; } diff --git a/vino_core_lib/src/engines/engine_manager.cpp b/vino_core_lib/src/engines/engine_manager.cpp index 15fc9371..b9d1335b 100644 --- a/vino_core_lib/src/engines/engine_manager.cpp +++ b/vino_core_lib/src/engines/engine_manager.cpp @@ -33,18 +33,18 @@ std::shared_ptr Engines::EngineManager::createEngine(const std: #if (defined(USE_OLD_E_PLUGIN_API)) return createEngine_beforeV2019R2(device, model); #else - return createEngine_V2019R2_plus(device, model); + return createEngine_V2022(device, model); #endif } -std::shared_ptr Engines::EngineManager::createEngine_V2019R2_plus( +std::shared_ptr Engines::EngineManager::createEngine_V2022( const std::string& device, const std::shared_ptr& model) { - InferenceEngine::Core core; - auto executable_network = core.LoadNetwork(model->getNetReader(), device); - auto request = executable_network.CreateInferRequestPtr(); + ov::Core core; + ov::CompiledModel executable_network = core.compile_model(model->getNetReader(), device); + ov::InferRequest infer_request = executable_network.create_infer_request(); - return std::make_shared(request); + return std::make_shared(infer_request); } #if (defined(USE_OLD_E_PLUGIN_API)) From ac2e596f0ccec5be92201a1855759a3f72cb3ba8 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Tue, 11 Oct 2022 16:24:09 +0800 Subject: [PATCH 02/40] fix pipeline api --- vino_core_lib/src/pipeline.cpp | 25 +++++---- vino_core_lib/src/pipeline_manager.cpp | 78 +++++++++++++------------- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/vino_core_lib/src/pipeline.cpp b/vino_core_lib/src/pipeline.cpp index 758fbcfa..8f6e0f32 100644 --- a/vino_core_lib/src/pipeline.cpp +++ b/vino_core_lib/src/pipeline.cpp @@ -245,18 +245,19 @@ void Pipeline::printPipeline() void Pipeline::setCallback() { - for (auto& pair : name_to_detection_map_) - { + for (auto & pair : name_to_detection_map_) { std::string detection_name = pair.first; - - std::function callb; - callb = [detection_name, this]() { - this->callback(detection_name); - return; - }; - pair.second->getEngine()->getRequest()->SetCompletionCallback(callb); - slog::debug << "Set Callback for Detection: " << detection_name << slog::endl; - } + std::function callb; + callb = [detection_name, self = this](std::exception_ptr ex) + { + if (ex) + throw ex; + + self->callback(detection_name); + return; + }; + pair.second->getEngine()->getRequest().set_callback(callb); + } } void Pipeline::callback(const std::string& detection_name) @@ -294,7 +295,7 @@ void Pipeline::callback(const std::string& detection_name) increaseInferenceCounter(); next_detection_ptr->submitRequest(); auto request = next_detection_ptr->getEngine()->getRequest(); - request->Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY); + request.wait(); } } } diff --git a/vino_core_lib/src/pipeline_manager.cpp b/vino_core_lib/src/pipeline_manager.cpp index 8d5b4d91..902073b2 100644 --- a/vino_core_lib/src/pipeline_manager.cpp +++ b/vino_core_lib/src/pipeline_manager.cpp @@ -27,11 +27,11 @@ #include "vino_core_lib/inferences/emotions_detection.h" #include "vino_core_lib/inferences/face_detection.h" #include "vino_core_lib/inferences/head_pose_detection.h" -#include "vino_core_lib/inferences/face_reidentification.h" +// #include "vino_core_lib/inferences/face_reidentification.h" #include "vino_core_lib/inferences/person_attribs_detection.h" #include "vino_core_lib/inferences/vehicle_attribs_detection.h" #include "vino_core_lib/inferences/license_plate_detection.h" -#include "vino_core_lib/inferences/landmarks_detection.h" +// #include "vino_core_lib/inferences/landmarks_detection.h" #include "vino_core_lib/inputs/image_input.h" #include "vino_core_lib/inputs/realsense_camera.h" #include "vino_core_lib/inputs/realsense_camera_topic.h" @@ -44,11 +44,11 @@ #include "vino_core_lib/models/head_pose_detection_model.h" #include "vino_core_lib/models/object_detection_ssd_model.h" // #include "vino_core_lib/models/object_detection_yolov2voc_model.h" -#include "vino_core_lib/models/face_reidentification_model.h" +// #include "vino_core_lib/models/face_reidentification_model.h" #include "vino_core_lib/models/person_attribs_detection_model.h" #include "vino_core_lib/models/vehicle_attribs_detection_model.h" #include "vino_core_lib/models/license_plate_detection_model.h" -#include "vino_core_lib/models/landmarks_detection_model.h" +// #include "vino_core_lib/models/landmarks_detection_model.h" #include "vino_core_lib/outputs/image_window_output.h" #include "vino_core_lib/outputs/ros_topic_output.h" #include "vino_core_lib/outputs/rviz_output.h" @@ -251,18 +251,18 @@ PipelineManager::parseInference(const Params::ParamManager::PipelineRawData& par { object = createPersonReidentification(infer); } - else if (infer.name == kInferTpye_FaceReidentification) - { - object = createFaceReidentification(infer); - } + // else if (infer.name == kInferTpye_FaceReidentification) + // { + // object = createFaceReidentification(infer); + // } else if (infer.name == kInferTpye_PersonAttribsDetection) { object = createPersonAttribsDetection(infer); } - else if (infer.name == kInferTpye_LandmarksDetection) - { - object = createLandmarksDetection(infer); - } + // else if (infer.name == kInferTpye_LandmarksDetection) + // { + // object = createLandmarksDetection(infer); + // } else if (infer.name == kInferTpye_VehicleAttribsDetection) { object = createVehicleAttribsDetection(infer); @@ -432,33 +432,33 @@ PipelineManager::createPersonAttribsDetection(const Params::ParamManager::Infere return attribs_inference_ptr; } -std::shared_ptr -PipelineManager::createFaceReidentification(const Params::ParamManager::InferenceRawData& infer) -{ - auto model = std::make_shared(infer.label, infer.model, infer.batch); - slog::debug << "for test in createFaceReidentification()" << slog::endl; - model->modelInit(); - auto engine = engine_manager_.createEngine(infer.engine, model); - auto attribs_inference_ptr = std::make_shared(infer.confidence_threshold); - attribs_inference_ptr->loadNetwork(model); - attribs_inference_ptr->loadEngine(engine); - - return attribs_inference_ptr; -} - -std::shared_ptr -PipelineManager::createLandmarksDetection( - const Params::ParamManager::InferenceRawData & infer) -{ - auto model = std::make_shared(infer.label, infer.model, infer.batch); - model->modelInit(); - auto engine = engine_manager_.createEngine(infer.engine, model); - auto landmarks_inference_ptr = std::make_shared(); - landmarks_inference_ptr->loadNetwork(model); - landmarks_inference_ptr->loadEngine(engine); - - return landmarks_inference_ptr; -} +// std::shared_ptr +// PipelineManager::createFaceReidentification(const Params::ParamManager::InferenceRawData& infer) +// { +// auto model = std::make_shared(infer.label, infer.model, infer.batch); +// slog::debug << "for test in createFaceReidentification()" << slog::endl; +// model->modelInit(); +// auto engine = engine_manager_.createEngine(infer.engine, model); +// auto attribs_inference_ptr = std::make_shared(infer.confidence_threshold); +// attribs_inference_ptr->loadNetwork(model); +// attribs_inference_ptr->loadEngine(engine); + +// return attribs_inference_ptr; +// } + +// std::shared_ptr +// PipelineManager::createLandmarksDetection( +// const Params::ParamManager::InferenceRawData & infer) +// { +// auto model = std::make_shared(infer.label, infer.model, infer.batch); +// model->modelInit(); +// auto engine = engine_manager_.createEngine(infer.engine, model); +// auto landmarks_inference_ptr = std::make_shared(); +// landmarks_inference_ptr->loadNetwork(model); +// landmarks_inference_ptr->loadEngine(engine); + +// return landmarks_inference_ptr; +// } void PipelineManager::threadPipeline(const char* name) { From 9dfd32f4e787e9998f106e5bcc517684fa933089 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Tue, 11 Oct 2022 16:26:49 +0800 Subject: [PATCH 03/40] fix base_inference api --- .../include/vino_core_lib/inferences/base_inference.h | 11 ++++++----- vino_core_lib/src/inferences/base_inference.cpp | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/inferences/base_inference.h b/vino_core_lib/include/vino_core_lib/inferences/base_inference.h index 8ded62ba..688a839d 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/base_inference.h +++ b/vino_core_lib/include/vino_core_lib/inferences/base_inference.h @@ -29,6 +29,7 @@ #include "vino_core_lib/models/base_model.h" #include "vino_core_lib/slog.h" #include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" namespace Outputs @@ -43,14 +44,14 @@ class BaseOutput; * @param[in] batch_index Indicates the batch index for the frame. */ template -void matU8ToBlob(const cv::Mat& orig_image, InferenceEngine::Blob::Ptr& blob, float scale_factor = 1.0, +void matU8ToBlob(const cv::Mat& orig_image, ov::Tensor & input_tensor, float scale_factor = 1.0, int batch_index = 0) { - InferenceEngine::SizeVector blob_size = blob->getTensorDesc().getDims(); + ov::Shape blob_size = input_tensor.get_shape(); const size_t width = blob_size[3]; const size_t height = blob_size[2]; const size_t channels = blob_size[1]; - T* blob_data = blob->buffer().as(); + T * blob_data = input_tensor.data(); cv::Mat resized_image(orig_image); if (width != (size_t)orig_image.size().width || height != (size_t)orig_image.size().height) @@ -198,8 +199,8 @@ class BaseInference << ") processed by inference" << slog::endl; return false; } - InferenceEngine::Blob::Ptr input_blob = engine_->getRequest()->GetBlob(input_name); - matU8ToBlob(frame, input_blob, scale_factor, batch_index); + ov::Tensor input_tensor = engine_->getRequest().get_tensor(input_name); + matU8ToBlob(frame, input_tensor, scale_factor, batch_index); enqueued_frames_ += 1; return true; } diff --git a/vino_core_lib/src/inferences/base_inference.cpp b/vino_core_lib/src/inferences/base_inference.cpp index 9112d669..5e122fcb 100644 --- a/vino_core_lib/src/inferences/base_inference.cpp +++ b/vino_core_lib/src/inferences/base_inference.cpp @@ -41,7 +41,7 @@ void vino_core_lib::BaseInference::loadEngine(const std::shared_ptrgetRequest() == nullptr) + if (!engine_->getRequest()) { return false; } @@ -51,13 +51,13 @@ bool vino_core_lib::BaseInference::submitRequest() } enqueued_frames_ = 0; results_fetched_ = false; - engine_->getRequest()->StartAsync(); + engine_->getRequest().start_async(); return true; } bool vino_core_lib::BaseInference::SynchronousRequest() { - if (engine_->getRequest() == nullptr) + if (!engine_->getRequest()) { return false; } @@ -67,7 +67,7 @@ bool vino_core_lib::BaseInference::SynchronousRequest() } enqueued_frames_ = 0; results_fetched_ = false; - engine_->getRequest()->Infer(); + engine_->getRequest().infer(); return true; } From aed1c24d8f0d0d35441dc7d952d135a113af4f6e Mon Sep 17 00:00:00 2001 From: wujiawei Date: Tue, 11 Oct 2022 16:31:05 +0800 Subject: [PATCH 04/40] fix version info --- .../vino_core_lib/utils/version_info.hpp | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/utils/version_info.hpp b/vino_core_lib/include/vino_core_lib/utils/version_info.hpp index e26e46f3..65367668 100644 --- a/vino_core_lib/include/vino_core_lib/utils/version_info.hpp +++ b/vino_core_lib/include/vino_core_lib/utils/version_info.hpp @@ -50,27 +50,15 @@ inline std::string& trim(std::string& s) return s; } -static std::ostream& operator<<(std::ostream& os, const InferenceEngine::Version* version) +static std::ostream & operator<<(std::ostream & os, const ov::Version& version) { os << "\n\tAPI version ............ "; - if (nullptr == version) - { - os << "UNKNOWN"; - } - else - { - os << version->apiVersion.major << "." << version->apiVersion.minor; - if (nullptr != version->buildNumber) - { - os << "\n\t" - << "Build .................. " << version->buildNumber; - } - if (nullptr != version->description) - { - os << "\n\t" - << "Description ............ " << version->description; - } - } + os << OPENVINO_VERSION_MAJOR << "." << OPENVINO_VERSION_MINOR << "." << OPENVINO_VERSION_PATCH; + os << "\n\t" << + "Build .................. " << version.buildNumber; + os << "\n\t" << + "Description ............ " << version.description; + return os; } From 04d7a0d684c2923e958e39cb63532c06bba2e5c4 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Tue, 11 Oct 2022 16:33:02 +0800 Subject: [PATCH 05/40] fix base model --- vino_core_lib/include/vino_core_lib/models/base_model.h | 9 +++++---- vino_core_lib/src/models/base_model.cpp | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/base_model.h b/vino_core_lib/include/vino_core_lib/models/base_model.h index 4ccb9714..91ed67bd 100644 --- a/vino_core_lib/include/vino_core_lib/models/base_model.h +++ b/vino_core_lib/include/vino_core_lib/models/base_model.h @@ -31,6 +31,7 @@ #include #include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "vino_core_lib/models/attributes/base_attribute.h" namespace Engines @@ -100,7 +101,7 @@ class BaseModel : public ModelAttribute return attr_; } - inline InferenceEngine::CNNNetwork getNetReader() const + inline std::shared_ptr getNetReader() const { return net_reader_; } @@ -111,10 +112,10 @@ class BaseModel : public ModelAttribute * @brief Set the layer property (layer layout, layer precision, etc.). * @param[in] network_reader The reader of the network to be set. */ - virtual bool updateLayerProperty(InferenceEngine::CNNNetwork& network_reader) = 0; + virtual bool updateLayerProperty(std::shared_ptr& network_reader) = 0; - InferenceEngine::Core engine; - InferenceEngine::CNNNetwork net_reader_; + ov::Core engine; + std::shared_ptr net_reader_; void setFrameSize(const int& w, const int& h) { frame_size_.width = w; diff --git a/vino_core_lib/src/models/base_model.cpp b/vino_core_lib/src/models/base_model.cpp index 223eccdc..903c0524 100644 --- a/vino_core_lib/src/models/base_model.cpp +++ b/vino_core_lib/src/models/base_model.cpp @@ -46,7 +46,7 @@ void Models::BaseModel::modelInit() // Read network model ///net_reader_->ReadNetwork(model_loc_); - net_reader_ = engine.ReadNetwork(model_loc_); + net_reader_ = engine.read_model(model_loc_); // Extract model name and load it's weights // remove extension size_t last_index = model_loc_.find_last_of("."); @@ -56,11 +56,11 @@ void Models::BaseModel::modelInit() // Read labels (if any) std::string label_file_name = label_loc_.substr(0, last_index); //std::string label_file_name = raw_name + ".labels"; - loadLabelsFromFile(label_file_name); + loadLabelsFromFile(label_loc_); // Set batch size to given max_batch_size_ slog::info << "Batch size is set to " << max_batch_size_ << slog::endl; - net_reader_.setBatchSize(max_batch_size_); + // net_reader_.setBatchSize(max_batch_size_); updateLayerProperty(net_reader_); } From 4921a24d7a8668041ab954be7142dc2b6f31a816 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Wed, 12 Oct 2022 12:44:42 +0800 Subject: [PATCH 06/40] fix age gender detection pipeline --- .../models/age_gender_detection_model.h | 7 +- .../src/inferences/age_gender_detection.cpp | 8 +-- .../src/models/age_gender_detection_model.cpp | 66 +++++++++++-------- 3 files changed, 47 insertions(+), 34 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/age_gender_detection_model.h b/vino_core_lib/include/vino_core_lib/models/age_gender_detection_model.h index aae2de2d..d28a0d6c 100644 --- a/vino_core_lib/include/vino_core_lib/models/age_gender_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/age_gender_detection_model.h @@ -63,7 +63,12 @@ class AgeGenderDetectionModel : public BaseModel const std::string getModelCategory() const override; protected: - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; + std::string input_tensor_name_; + std::string output_tensor_name_; + + std::vector> inputs_info_; + std::vector> outputs_info_; }; } // namespace Models diff --git a/vino_core_lib/src/inferences/age_gender_detection.cpp b/vino_core_lib/src/inferences/age_gender_detection.cpp index 59064f81..7b2c4e3e 100644 --- a/vino_core_lib/src/inferences/age_gender_detection.cpp +++ b/vino_core_lib/src/inferences/age_gender_detection.cpp @@ -70,13 +70,13 @@ bool vino_core_lib::AgeGenderDetection::fetchResults() if (!can_fetch) return false; auto request = getEngine()->getRequest(); - InferenceEngine::Blob::Ptr genderBlob = request->GetBlob(valid_model_->getOutputGenderName()); - InferenceEngine::Blob::Ptr ageBlob = request->GetBlob(valid_model_->getOutputAgeName()); + ov::Tensor gender_tensor = request.get_tensor(valid_model_->getOutputGenderName()); + ov::Tensor age_tensor = request.get_tensor(valid_model_->getOutputAgeName()); for (size_t i = 0; i < results_.size(); ++i) { - results_[i].age_ = ageBlob->buffer().as()[i] * 100; - results_[i].male_prob_ = genderBlob->buffer().as()[i * 2 + 1]; + results_[i].age_ = age_tensor.data()[i] * 100; + results_[i].male_prob_ = gender_tensor.data()[i * 2 + 1]; } return true; } diff --git a/vino_core_lib/src/models/age_gender_detection_model.cpp b/vino_core_lib/src/models/age_gender_detection_model.cpp index 6ef493b9..64ad2972 100644 --- a/vino_core_lib/src/models/age_gender_detection_model.cpp +++ b/vino_core_lib/src/models/age_gender_detection_model.cpp @@ -30,34 +30,37 @@ Models::AgeGenderDetectionModel::AgeGenderDetectionModel(const std::string& labe : BaseModel(label_loc, model_loc, max_batch_size) { } -bool Models::AgeGenderDetectionModel::updateLayerProperty(InferenceEngine::CNNNetwork& net_reader) +bool Models::AgeGenderDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); - if (input_info_map.size() != 1) - { - slog::warn << "This model seems not Age-Gender-like, which should have only one input," - << " but we got " << std::to_string(input_info_map.size()) << "inputs" << slog::endl; + inputs_info_ = net_reader->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + + ov::Shape input_tensor_shape = net_reader->input().get_shape(); + if (inputs_info_.size() != 1) { + slog::warn << "This model seems not Age-Gender-like, which should have only one input, but we got" + << std::to_string(input_tensor_shape.size()) << "inputs" + << slog::endl; return false; } - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::FP32); - input_info->setLayout(InferenceEngine::Layout::NCHW); - addInputInfo("input", input_info_map.begin()->first); + + addInputInfo("input", input_tensor_name_); + const ov::Layout tensor_layout{"NCHW"}; + input_info.tensor(). + set_element_type(ov::element::f32). + set_layout(tensor_layout); + // set output property - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); - if (output_info_map.size() != 2) - { - // throw std::logic_error("Age/Gender Recognition network should have two output layers"); - slog::warn << "This model seems not Age-gender like, which should have and only have 2" - " outputs, but we got " - << std::to_string(output_info_map.size()) << "outputs" << slog::endl; + outputs_info_ = net_reader->outputs(); + if (outputs_info_.size() != 2) { + slog::warn << "This model seems not Age-Gender-like, which should have and only have 2 outputs, but we got" + << std::to_string(outputs_info_.size()) << "outputs" + << slog::endl; return false; } - auto it = output_info_map.begin(); - InferenceEngine::DataPtr age_output_ptr = (it++)->second; - InferenceEngine::DataPtr gender_output_ptr = (it++)->second; #if(0) /// // Check More Configuration: @@ -81,17 +84,22 @@ bool Models::AgeGenderDetectionModel::updateLayerProperty(InferenceEngine::CNNNe slog::info << "Gender layer: " << gender_output_ptr->getCreatorLayer().lock()->name << slog::endl; #endif + auto age_output_info = outputs_info_[1]; + ppp.output(age_output_info.get_any_name()). + tensor(). + set_element_type(ov::element::f32). + set_layout(tensor_layout); + auto gender_output_info = outputs_info_[0]; + ppp.output(gender_output_info.get_any_name()). + tensor(). + set_element_type(ov::element::f32). + set_layout(tensor_layout); - age_output_ptr->setPrecision(InferenceEngine::Precision::FP32); - age_output_ptr->setLayout(InferenceEngine::Layout::NCHW); - gender_output_ptr->setPrecision(InferenceEngine::Precision::FP32); - gender_output_ptr->setLayout(InferenceEngine::Layout::NCHW); - - // output_age_ = age_output_ptr->name; - addOutputInfo("age", age_output_ptr->getName()); - // output_gender_ = gender_output_ptr->name; - addOutputInfo("gender", gender_output_ptr->getName()); + net_reader = ppp.build(); + ov::set_batch(net_reader_, getMaxBatchSize()); + addOutputInfo("age", age_output_info.get_any_name()); + addOutputInfo("gender", gender_output_info.get_any_name()); printAttribute(); return true; } From 0b224e26edf4d9de8d8b657ea3a7e208ca4ffbe2 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Wed, 12 Oct 2022 12:48:56 +0800 Subject: [PATCH 07/40] fix emotion detection pipeline --- .../inferences/emotions_detection.h | 1 + .../models/emotion_detection_model.h | 9 ++- .../src/inferences/emotions_detection.cpp | 7 ++- .../src/models/emotion_detection_model.cpp | 62 +++++++++---------- 4 files changed, 41 insertions(+), 38 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/inferences/emotions_detection.h b/vino_core_lib/include/vino_core_lib/inferences/emotions_detection.h index b7e2adda..56ce404c 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/emotions_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/emotions_detection.h @@ -29,6 +29,7 @@ #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/models/emotion_detection_model.h" #include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" namespace Outputs diff --git a/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h b/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h index 100d1348..b784d23e 100644 --- a/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h @@ -41,10 +41,15 @@ class EmotionDetectionModel : public BaseModel * @return Name of the model. */ const std::string getModelCategory() const override; - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; private: - bool verifyOutputLayer(const InferenceEngine::DataPtr& ptr); + // bool verifyOutputLayer(const InferenceEngine::DataPtr& ptr); + std::string input_tensor_name_; + std::string output_tensor_name_; + + std::vector> inputs_info_; + std::vector> outputs_info_; }; } // namespace Models diff --git a/vino_core_lib/src/inferences/emotions_detection.cpp b/vino_core_lib/src/inferences/emotions_detection.cpp index ca6d1c9f..c3cb600b 100644 --- a/vino_core_lib/src/inferences/emotions_detection.cpp +++ b/vino_core_lib/src/inferences/emotions_detection.cpp @@ -80,11 +80,12 @@ bool vino_core_lib::EmotionsDetection::fetchResults() /// std::cout << "label_length = " << label_length << std::endl; std::string output_name = valid_model_->getOutputName(); /// std::cout << "output_name = " << output_name << std::endl; - InferenceEngine::Blob::Ptr emotions_blob = getEngine()->getRequest()->GetBlob(output_name); + ov::Tensor emotions_tensor = getEngine()->getRequest().get_tensor(output_name); /** emotions vector must have the same size as number of channels in model output. Default output format is NCHW so we check index 1 */ - int64 num_of_channels = emotions_blob->getTensorDesc().getDims().at(1); + ov::Shape shape = emotions_tensor.get_shape(); + int64 num_of_channels = shape[1]; /// std::cout << "num_of_channels " << num_of_channels << std::endl; if (num_of_channels != label_length) { @@ -98,7 +99,7 @@ bool vino_core_lib::EmotionsDetection::fetchResults() /** we identify an index of the most probable emotion in output array for idx image to return appropriate emotion name */ - auto emotions_values = emotions_blob->buffer().as(); + auto emotions_values = emotions_tensor.data(); for (unsigned int idx = 0; idx < results_.size(); ++idx) { auto output_idx_pos = emotions_values + idx; diff --git a/vino_core_lib/src/models/emotion_detection_model.cpp b/vino_core_lib/src/models/emotion_detection_model.cpp index 2a556681..41fd0f08 100644 --- a/vino_core_lib/src/models/emotion_detection_model.cpp +++ b/vino_core_lib/src/models/emotion_detection_model.cpp @@ -29,52 +29,48 @@ Models::EmotionDetectionModel::EmotionDetectionModel(const std::string& label_lo { } -bool Models::EmotionDetectionModel::updateLayerProperty(InferenceEngine::CNNNetwork& net_reader) +bool Models::EmotionDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); - if (input_info_map.size() != 1) - { - slog::warn << "This model seems not Age-Gender-like, which should have only one input," - << " but we got " << std::to_string(input_info_map.size()) << "inputs" << slog::endl; + inputs_info_ = net_reader->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + + ov::Shape input_tensor_shape = net_reader->input().get_shape(); + if (inputs_info_.size() != 1) { + slog::warn << "This model seems not Emotion-detection-model-like, which should have only one input, but we got" + << std::to_string(input_tensor_shape.size()) << "inputs" + << slog::endl; return false; } - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::FP32); - input_info->setLayout(InferenceEngine::Layout::NCHW); - addInputInfo("input", input_info_map.begin()->first); + + addInputInfo("input", input_tensor_name_); + const ov::Layout tensor_layout{"NCHW"}; + input_info.tensor(). + set_element_type(ov::element::f32). + set_layout(tensor_layout); // set output property - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); - if (output_info_map.size() != 1) - { - // throw std::logic_error("Age/Gender Recognition network should have two output layers"); - slog::warn << "This model should have and only have 1 output, but we got " << std::to_string(output_info_map.size()) - << "outputs" << slog::endl; + outputs_info_ = net_reader->outputs(); + output_tensor_name_ = net_reader->output().get_any_name(); + ov::preprocess::OutputInfo& output_info = ppp.output(output_tensor_name_); + if (outputs_info_.size() != 1) { + slog::warn << "This model should have and only have 1 output, but we got " + << std::to_string(outputs_info_.size()) << "outputs" + << slog::endl; return false; } - /// InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; - /// slog::info << "Emotions layer: " << output_data_ptr->getCreatorLayer().lock()->name << slog::endl; - /// output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); - /// output_data_ptr->setLayout(InferenceEngine::Layout::NCHW); - addOutputInfo("output", output_info_map.begin()->first); + + net_reader = ppp.build(); + ov::set_batch(net_reader_, getMaxBatchSize()); + addOutputInfo("output", output_tensor_name_); + printAttribute(); return true; ///verifyOutputLayer(output_data_ptr); } -bool Models::EmotionDetectionModel::verifyOutputLayer(const InferenceEngine::DataPtr& ptr) -{ -/// if (ptr->getCreatorLayer().lock()->type != "SoftMax") -/// { -/// slog::err << "In Emotion network, gender layer (" << ptr->getCreatorLayer().lock()->name -/// << ") should be a SoftMax, but was: " << ptr->getCreatorLayer().lock()->type << slog::endl; -/// return false; -/// } - - return true; -} - const std::string Models::EmotionDetectionModel::getModelCategory() const { return "Emotions Detection"; From 451dfcdde6a7c11b844d81e493a306af33547cb1 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Wed, 12 Oct 2022 12:50:58 +0800 Subject: [PATCH 08/40] fix head pose detection pipeline --- .../inferences/head_pose_detection.h | 1 + .../models/head_pose_detection_model.h | 7 ++- .../src/inferences/head_pose_detection.cpp | 12 ++--- .../src/models/head_pose_detection_model.cpp | 44 +++++++++++-------- 4 files changed, 39 insertions(+), 25 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/inferences/head_pose_detection.h b/vino_core_lib/include/vino_core_lib/inferences/head_pose_detection.h index a2b61f8f..7209eec0 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/head_pose_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/head_pose_detection.h @@ -29,6 +29,7 @@ #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/models/head_pose_detection_model.h" #include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" namespace vino_core_lib diff --git a/vino_core_lib/include/vino_core_lib/models/head_pose_detection_model.h b/vino_core_lib/include/vino_core_lib/models/head_pose_detection_model.h index aceedb21..340fcfb0 100644 --- a/vino_core_lib/include/vino_core_lib/models/head_pose_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/head_pose_detection_model.h @@ -65,12 +65,17 @@ class HeadPoseDetectionModel : public BaseModel * @return Name of the model. */ const std::string getModelCategory() const override; - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; private: std::string output_angle_r_ = "angle_r_fc"; std::string output_angle_p_ = "angle_p_fc"; std::string output_angle_y_ = "angle_y_fc"; + std::string input_tensor_name_; + std::string output_tensor_name_; + + std::vector> inputs_info_; + std::vector> outputs_info_; }; } // namespace Models diff --git a/vino_core_lib/src/inferences/head_pose_detection.cpp b/vino_core_lib/src/inferences/head_pose_detection.cpp index 6005739d..09cc8444 100644 --- a/vino_core_lib/src/inferences/head_pose_detection.cpp +++ b/vino_core_lib/src/inferences/head_pose_detection.cpp @@ -69,15 +69,15 @@ bool vino_core_lib::HeadPoseDetection::fetchResults() if (!can_fetch) return false; auto request = getEngine()->getRequest(); - InferenceEngine::Blob::Ptr angle_r = request->GetBlob(valid_model_->getOutputOutputAngleR()); - InferenceEngine::Blob::Ptr angle_p = request->GetBlob(valid_model_->getOutputOutputAngleP()); - InferenceEngine::Blob::Ptr angle_y = request->GetBlob(valid_model_->getOutputOutputAngleY()); + ov::Tensor angle_r = request.get_tensor(valid_model_->getOutputOutputAngleR()); + ov::Tensor angle_p = request.get_tensor(valid_model_->getOutputOutputAngleP()); + ov::Tensor angle_y = request.get_tensor(valid_model_->getOutputOutputAngleY()); for (int i = 0; i < getResultsLength(); ++i) { - results_[i].angle_r_ = angle_r->buffer().as()[i]; - results_[i].angle_p_ = angle_p->buffer().as()[i]; - results_[i].angle_y_ = angle_y->buffer().as()[i]; + results_[i].angle_r_ = angle_r.data()[i]; + results_[i].angle_p_ = angle_p.data()[i]; + results_[i].angle_y_ = angle_y.data()[i]; } return true; } diff --git a/vino_core_lib/src/models/head_pose_detection_model.cpp b/vino_core_lib/src/models/head_pose_detection_model.cpp index aee4f956..020d293d 100644 --- a/vino_core_lib/src/models/head_pose_detection_model.cpp +++ b/vino_core_lib/src/models/head_pose_detection_model.cpp @@ -32,38 +32,46 @@ Models::HeadPoseDetectionModel::HeadPoseDetectionModel(const std::string& label_ { } -bool Models::HeadPoseDetectionModel::updateLayerProperty(InferenceEngine::CNNNetwork& net_reader) +bool Models::HeadPoseDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); + auto input_info_map = net_reader->inputs(); if (input_info_map.size() != 1) { slog::warn << "This model should have only one input, but we got" << std::to_string(input_info_map.size()) << "inputs" << slog::endl; return false; } - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); - addInputInfo("input", input_info_map.begin()->first); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + const ov::Layout input_tensor_layout{"NCHW"}; + input_info.tensor(). + set_element_type(ov::element::u8). + set_layout(input_tensor_layout); + addInputInfo("input", input_tensor_name_); // set output property - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); - for (auto& output : output_info_map) - { - output.second->setPrecision(InferenceEngine::Precision::FP32); - output.second->setLayout(InferenceEngine::Layout::NC); + auto output_info_map = net_reader->outputs(); + std::vector outputs_name; + for (auto & output_item : output_info_map) { + std::string output_tensor_name_ = output_item.get_any_name(); + const ov::Layout output_tensor_layout{"NC"}; + ppp.output(output_tensor_name_). + tensor(). + set_element_type(ov::element::f32). + set_layout(output_tensor_layout); + outputs_name.push_back(output_tensor_name_); } - for (const std::string& outName : { output_angle_r_, output_angle_p_, output_angle_y_ }) - { - if (output_info_map.find(outName) == output_info_map.end()) - { + net_reader = ppp.build(); + ov::set_batch(net_reader_, getMaxBatchSize()); + + for (const std::string& outName : {output_angle_r_, output_angle_p_, output_angle_y_}) { + if (find(outputs_name.begin(), outputs_name.end(), outName) == outputs_name.end()) { throw std::logic_error("There is no " + outName + " output in Head Pose Estimation network"); - } - else - { + } else { addOutputInfo(outName, outName); } } From f005a2f38e2b1d7502f32f1c992d50ec30c15d8a Mon Sep 17 00:00:00 2001 From: wujiawei Date: Wed, 12 Oct 2022 15:37:48 +0800 Subject: [PATCH 09/40] fix person reidentification pipeline --- .../models/person_reidentification_model.h | 2 +- .../inferences/person_reidentification.cpp | 4 +-- .../models/person_reidentification_model.cpp | 27 ++++++++++--------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/person_reidentification_model.h b/vino_core_lib/include/vino_core_lib/models/person_reidentification_model.h index 6476208f..972ccf7c 100644 --- a/vino_core_lib/include/vino_core_lib/models/person_reidentification_model.h +++ b/vino_core_lib/include/vino_core_lib/models/person_reidentification_model.h @@ -45,7 +45,7 @@ class PersonReidentificationModel : public BaseModel const std::string getModelCategory() const override; protected: - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; // void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; std::string input_; diff --git a/vino_core_lib/src/inferences/person_reidentification.cpp b/vino_core_lib/src/inferences/person_reidentification.cpp index c18623d5..bbf18006 100644 --- a/vino_core_lib/src/inferences/person_reidentification.cpp +++ b/vino_core_lib/src/inferences/person_reidentification.cpp @@ -73,9 +73,9 @@ bool vino_core_lib::PersonReidentification::fetchResults() return false; } bool found_result = false; - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + ov::InferRequest request = getEngine()->getRequest(); std::string output = valid_model_->getOutputName(); - const float* output_values = request->GetBlob(output)->buffer().as(); + const float* output_values = request.get_tensor(output).data(); for (int i = 0; i < getResultsLength(); i++) { std::vector new_person = std::vector(output_values + 256 * i, output_values + 256 * i + 256); diff --git a/vino_core_lib/src/models/person_reidentification_model.cpp b/vino_core_lib/src/models/person_reidentification_model.cpp index d2c28d3f..c0c0e566 100644 --- a/vino_core_lib/src/models/person_reidentification_model.cpp +++ b/vino_core_lib/src/models/person_reidentification_model.cpp @@ -25,22 +25,25 @@ Models::PersonReidentificationModel::PersonReidentificationModel(const std::stri { } -bool Models::PersonReidentificationModel::updateLayerProperty(InferenceEngine::CNNNetwork& netreader) +bool Models::PersonReidentificationModel::updateLayerProperty(std::shared_ptr& net_reader) { slog::info << "Checking Inputs for Model" << getModelName() << slog::endl; - auto network = netreader; - - InferenceEngine::InputsDataMap input_info_map(network.getInputsInfo()); - - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); + auto network = net_reader; + auto input_info_map = net_reader->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + input_ = input_info_map[0].get_any_name(); + const ov::Layout input_tensor_layout{"NCHW"}; + ppp.input(input_). + tensor(). + set_element_type(ov::element::u8). + set_layout(input_tensor_layout); // set output property - InferenceEngine::OutputsDataMap output_info_map(network.getOutputsInfo()); - // set input and output layer name - input_ = input_info_map.begin()->first; - output_ = output_info_map.begin()->first; + auto output_info_map = net_reader->outputs(); + output_ = output_info_map[0].get_any_name(); + + net_reader = ppp.build(); + ov::set_batch(net_reader_, getMaxBatchSize()); return true; } From 4273a4010f45e1e44883fcade77898c078a9e410 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Wed, 12 Oct 2022 20:47:18 +0800 Subject: [PATCH 10/40] fix object segmentation pipeline --- .../models/object_segmentation_model.h | 7 +- .../src/inferences/object_segmentation.cpp | 26 ++--- .../src/models/object_segmentation_model.cpp | 103 +++++++++--------- 3 files changed, 71 insertions(+), 65 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/object_segmentation_model.h b/vino_core_lib/include/vino_core_lib/models/object_segmentation_model.h index faf0fc88..887749c3 100644 --- a/vino_core_lib/include/vino_core_lib/models/object_segmentation_model.h +++ b/vino_core_lib/include/vino_core_lib/models/object_segmentation_model.h @@ -47,13 +47,16 @@ class ObjectSegmentationModel : public BaseModel * @return Name of the model. */ const std::string getModelCategory() const override; - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; private: int max_proposal_count_; int object_size_; + std::string input_tensor_name_; + std::string output_tensor_name_; - InferenceEngine::InputsDataMap input_info_; + std::vector> inputs_info_; + ov::Shape input_shape_; }; } // namespace Models #endif // VINO_CORE_LIB__MODELS__OBJECT_SEGMENTATION_MODEL_H diff --git a/vino_core_lib/src/inferences/object_segmentation.cpp b/vino_core_lib/src/inferences/object_segmentation.cpp index ec6d6e84..4c81a0c1 100644 --- a/vino_core_lib/src/inferences/object_segmentation.cpp +++ b/vino_core_lib/src/inferences/object_segmentation.cpp @@ -112,27 +112,27 @@ bool vino_core_lib::ObjectSegmentation::fetchResults() } bool found_result = false; results_.clear(); - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + ov::InferRequest infer_request = getEngine()->getRequest(); slog::debug << "Analyzing Detection results..." << slog::endl; std::string detection_output = valid_model_->getOutputName("detection"); std::string mask_output = valid_model_->getOutputName("masks"); - const InferenceEngine::Blob::Ptr do_blob = request->GetBlob(detection_output.c_str()); - const auto do_data = do_blob->buffer().as(); - const auto masks_blob = request->GetBlob(mask_output.c_str()); - const auto masks_data = masks_blob->buffer().as(); - const InferenceEngine::SizeVector& outSizeVector = masks_blob->getTensorDesc().getDims(); + ov::Tensor output_tensor = infer_request.get_tensor(detection_output); + const auto out_data = output_tensor.data(); + ov::Shape out_shape = output_tensor.get_shape(); + ov::Tensor masks_tensor = infer_request.get_tensor(detection_output.c_str()); + const auto masks_data = masks_tensor.data(); int output_des, output_h, output_w; - switch(outSizeVector.size()) { + switch(out_shape.size()) { case 3: output_des = 0; - output_h = outSizeVector[1]; - output_w = outSizeVector[2]; + output_h = out_shape[1]; + output_w = out_shape[2]; break; case 4: - output_des = outSizeVector[1]; - output_h = outSizeVector[2]; - output_w = outSizeVector[3]; + output_des = out_shape[1]; + output_h = out_shape[2]; + output_w = out_shape[3]; break; default: throw std::runtime_error("Unexpected output blob shape. Only 4D and 3D output blobs are" @@ -142,7 +142,7 @@ bool vino_core_lib::ObjectSegmentation::fetchResults() slog::debug << "output h " << output_h << slog::endl; slog::debug << "output description " << output_des << slog::endl; - const float* detections = request->GetBlob(detection_output)->buffer().as(); + const auto detections = output_tensor.data(); std::vector& labels = valid_model_->getLabels(); slog::debug << "label size " << labels.size() << slog::endl; diff --git a/vino_core_lib/src/models/object_segmentation_model.cpp b/vino_core_lib/src/models/object_segmentation_model.cpp index 22b4d785..f2fe2d87 100644 --- a/vino_core_lib/src/models/object_segmentation_model.cpp +++ b/vino_core_lib/src/models/object_segmentation_model.cpp @@ -35,21 +35,22 @@ bool Models::ObjectSegmentationModel::enqueue(const std::shared_ptrgetTensorDesc().getDims().size() << slog::endl; - if (inputInfoItem.second->getTensorDesc().getDims().size() == 4) + // slog::debug << "first tensor" << inputInfoItem.second->getTensorDesc().getDims().size() << slog::endl; + auto dims = inputInfoItem.get_shape(); + if (dims.size()==4) { matToBlob(frame, input_frame_loc, 1.0, 0, engine); } // Fill second input tensor with image info - if (inputInfoItem.second->getTensorDesc().getDims().size() == 2) + if (dims.size() == 2) { - InferenceEngine::Blob::Ptr input = engine->getRequest()->GetBlob(inputInfoItem.first); - auto data = input->buffer().as::value_type*>(); - data[0] = static_cast(frame.rows); // height + ov::Tensor in_tensor = engine->getRequest().get_tensor(inputInfoItem); + auto data = in_tensor.data(); + data[0] = static_cast(frame.rows); // height data[1] = static_cast(frame.cols); // width data[2] = 1; } @@ -84,11 +85,8 @@ bool Models::ObjectSegmentationModel::matToBlob(const cv::Mat& orig_image, const return false; } - InferenceEngine::TensorDesc tDesc(InferenceEngine::Precision::U8, { 1, channels, height, width }, - InferenceEngine::Layout::NHWC); - - auto shared_blob = InferenceEngine::make_shared_blob(tDesc, orig_image.data); - engine->getRequest()->SetBlob(getInputName(), shared_blob); + ov::Tensor input_tensor{ov::element::u8, input_shape_, orig_image.data}; + engine->getRequest().set_tensor(input_tensor_name_, input_tensor); return true; } @@ -98,57 +96,61 @@ const std::string Models::ObjectSegmentationModel::getModelCategory() const return "Object Segmentation"; } -bool Models::ObjectSegmentationModel::updateLayerProperty(InferenceEngine::CNNNetwork& net_reader) +bool Models::ObjectSegmentationModel::updateLayerProperty(std::shared_ptr& net_reader) { slog::info << "Checking INPUTS for Model" << getModelName() << slog::endl; - auto network = net_reader; - input_info_ = InferenceEngine::InputsDataMap(network.getInputsInfo()); - - InferenceEngine::ICNNNetwork::InputShapes inputShapes = network.getInputShapes(); - slog::debug << "input size" << inputShapes.size() << slog::endl; - if (inputShapes.size() != 1) - { + inputs_info_ = net_reader->inputs(); + slog::debug<<"input size"<second; - slog::debug << "channel size" << in_size_vector[1] << "dimensional" << in_size_vector.size() << slog::endl; - if (in_size_vector.size() != 4 || in_size_vector[1] != 3) - { - // throw std::runtime_error("3-channel 4-dimensional model's input is expected"); - slog::warn << "3-channel 4-dimensional model's input is expected, but we got " << std::to_string(in_size_vector[1]) - << " channels and " << std::to_string(in_size_vector.size()) << " dimensions." << slog::endl; - return false; - } - in_size_vector[0] = 1; - network.reshape(inputShapes); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); - InferenceEngine::InputInfo& inputInfo = *network.getInputsInfo().begin()->second; - inputInfo.getPreProcess().setResizeAlgorithm(InferenceEngine::ResizeAlgorithm::RESIZE_BILINEAR); - inputInfo.setLayout(InferenceEngine::Layout::NHWC); - inputInfo.setPrecision(InferenceEngine::Precision::U8); + const ov::Layout tensor_layout{"NCHW"}; + input_info.tensor(). + set_element_type(ov::element::u8). + set_layout(tensor_layout); + input_info.preprocess().resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR); + addInputInfo("input", input_tensor_name_); - // InferenceEngine::InputInfo::Ptr input_info = input_info_.begin()->second; - addInputInfo("input", input_info_.begin()->first.c_str()); - // addInputInfo("input", inputShapes.begin()->first); - InferenceEngine::OutputsDataMap outputsDataMap = network.getOutputsInfo(); - if (outputsDataMap.size() != 1) + auto outputs_info = net_reader->outputs(); + if (outputs_info.size() != 1) { // throw std::runtime_error("Demo supports topologies only with 1 output"); slog::warn << "This inference sample should have only one output, but we got" - << std::to_string(outputsDataMap.size()) << "outputs" << slog::endl; + << std::to_string(outputs_info.size()) << "outputs" << slog::endl; + return false; + } + output_tensor_name_ = net_reader->output().get_any_name(); + auto data = net_reader->output(); + + ov::preprocess::OutputInfo& output_info = ppp.output(output_tensor_name_); + output_info.tensor().set_element_type(ov::element::f32); + net_reader = ppp.build(); + // ov::set_batch(net_reader_, getMaxBatchSize()); + + input_shape_ = net_reader->input().get_shape(); + + std::vector &in_size_vector = input_shape_; + slog::debug<<"channel size"<second; - data.setPrecision(InferenceEngine::Precision::FP32); - - const InferenceEngine::SizeVector& outSizeVector = data.getTensorDesc().getDims(); + auto& outSizeVector = data.get_shape(); int outChannels, outHeight, outWidth; slog::debug << "output size vector " << outSizeVector.size() << slog::endl; switch (outSizeVector.size()) @@ -176,10 +178,11 @@ bool Models::ObjectSegmentationModel::updateLayerProperty(InferenceEngine::CNNNe } slog::debug << "output width " << outWidth << slog::endl; - slog::debug << "output hEIGHT " << outHeight << slog::endl; + slog::debug << "output HEIGHT " << outHeight << slog::endl; slog::debug << "output CHANNALS " << outChannels << slog::endl; - addOutputInfo("masks", (outputsDataMap.begin()++)->first); - addOutputInfo("detection", outputsDataMap.begin()->first); + slog::debug << "output name " << output_tensor_name_<< slog::endl; + addOutputInfo("masks", output_tensor_name_); + addOutputInfo("detection", output_tensor_name_); // const InferenceEngine::CNNLayerPtr output_layer = // network.getLayerByName(outputsDataMap.begin()->first.c_str()); From 6d5b7a0a8352536ae88079da9357f923ee72c14c Mon Sep 17 00:00:00 2001 From: wujiawei Date: Wed, 12 Oct 2022 20:49:37 +0800 Subject: [PATCH 11/40] fix object detection(ssd) pipeline --- .../models/object_detection_ssd_model.h | 2 +- .../src/models/attributes/ssd_model_attr.cpp | 33 +++++----- .../src/models/object_detection_ssd_model.cpp | 63 +++++++++++-------- 3 files changed, 56 insertions(+), 42 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/object_detection_ssd_model.h b/vino_core_lib/include/vino_core_lib/models/object_detection_ssd_model.h index d539e9f7..dea7e052 100644 --- a/vino_core_lib/include/vino_core_lib/models/object_detection_ssd_model.h +++ b/vino_core_lib/include/vino_core_lib/models/object_detection_ssd_model.h @@ -52,7 +52,7 @@ class ObjectDetectionSSDModel : public ObjectDetectionModel */ const std::string getModelCategory() const override; - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; }; } // namespace Models #endif // VINO_CORE_LIB__MODELS__OBJECT_DETECTION_SSD_MODEL_H diff --git a/vino_core_lib/src/models/attributes/ssd_model_attr.cpp b/vino_core_lib/src/models/attributes/ssd_model_attr.cpp index 21365bc9..ead9ffe9 100644 --- a/vino_core_lib/src/models/attributes/ssd_model_attr.cpp +++ b/vino_core_lib/src/models/attributes/ssd_model_attr.cpp @@ -30,38 +30,41 @@ Models::SSDModelAttr::SSDModelAttr( } bool Models::SSDModelAttr::updateLayerProperty( - const InferenceEngine::CNNNetwork & net_reader) + const std::shared_ptr & net_reader) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); + auto input_info_map = net_reader->inputs(); if (input_info_map.size() != 1) { slog::warn << "This model seems not SSDNet-like, SSDnet has only one input, but we got " << std::to_string(input_info_map.size()) << "inputs" << slog::endl; return false; } - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - addInputInfo("input", input_info_map.begin()->first); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + input_info.tensor().set_element_type(ov::element::u8); + addInputInfo("input", input_tensor_name_); - const InferenceEngine::SizeVector input_dims = input_info->getTensorDesc().getDims(); + ov::Shape input_dims = input_info_map[0].get_shape(); setInputHeight(input_dims[2]); setInputWidth(input_dims[3]); slog::info << "Checking OUTPUTs for model " << getModelName() << slog::endl; - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); - if (output_info_map.size() != 1) { + auto outputs_info = net_reader->outputs(); + if (outputs_info.size() != 1) { slog::warn << "This model seems not SSDNet-like! We got " - << std::to_string(output_info_map.size()) << "outputs, but SSDnet has only one." + << std::to_string(outputs_info.size()) << "outputs, but SSDnet has only one." << slog::endl; return false; } - InferenceEngine::DataPtr & output_data_ptr = output_info_map.begin()->second; - addOutputInfo("output", output_info_map.begin()->first); - slog::info << "Checking Object Detection output ... Name=" << output_info_map.begin()->first + ov::preprocess::OutputInfo& output_info = ppp.output(); + addOutputInfo("output", net_reader->output().get_any_name()); + slog::info << "Checking Object Detection output ... Name=" << net_reader->output().get_any_name() << slog::endl; - output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); + + output_info.tensor().set_element_type(ov::element::f32); ///TODO: double check this part: BEGIN #if(0) /// @@ -91,7 +94,9 @@ bool Models::SSDModelAttr::updateLayerProperty( ///TODO: double check this part: END // last dimension of output layer should be 7 - const InferenceEngine::SizeVector output_dims = output_data_ptr->getTensorDesc().getDims(); + auto outputsDataMap = net_reader->outputs(); + auto & data = outputsDataMap[0]; + ov::Shape output_dims = data.get_shape(); setMaxProposalCount(static_cast(output_dims[2])); slog::info << "max proposal count is: " << getMaxProposalCount() << slog::endl; diff --git a/vino_core_lib/src/models/object_detection_ssd_model.cpp b/vino_core_lib/src/models/object_detection_ssd_model.cpp index 582882d9..5049d7db 100644 --- a/vino_core_lib/src/models/object_detection_ssd_model.cpp +++ b/vino_core_lib/src/models/object_detection_ssd_model.cpp @@ -56,13 +56,15 @@ bool Models::ObjectDetectionSSDModel::matToBlob(const cv::Mat& orig_image, const std::string input_name = getInputName(); slog::debug << "add input image to blob: " << input_name << slog::endl; - InferenceEngine::Blob::Ptr input_blob = engine->getRequest()->GetBlob(input_name); + ov::Tensor in_tensor = engine->getRequest().get_tensor(input_name); + + ov::Shape in_shape = in_tensor.get_shape(); + const int width = in_shape[3]; + const int height = in_shape[2]; + const int channels = in_shape[1]; + + u_int8_t * blob_data = (u_int8_t *)in_tensor.data(); - InferenceEngine::SizeVector blob_size = input_blob->getTensorDesc().getDims(); - const int width = blob_size[3]; - const int height = blob_size[2]; - const int channels = blob_size[1]; - u_int8_t* blob_data = input_blob->buffer().as(); cv::Mat resized_image(orig_image); if (width != orig_image.size().width || height != orig_image.size().height) @@ -99,9 +101,9 @@ bool Models::ObjectDetectionSSDModel::fetchResults(const std::shared_ptrgetRequest(); + ov::InferRequest request = engine->getRequest(); std::string output = getOutputName(); - const float* detections = request->GetBlob(output)->buffer().as(); + const float * detections = (float * )request.get_tensor(output).data(); slog::debug << "Analyzing Detection results..." << slog::endl; auto max_proposal_count = getMaxProposalCount(); @@ -147,38 +149,43 @@ bool Models::ObjectDetectionSSDModel::fetchResults(const std::shared_ptr& net_reader) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); - if (input_info_map.size() != 1) - { + auto input_info_map = net_reader->inputs(); + if (input_info_map.size() != 1) { slog::warn << "This model seems not SSDNet-like, SSDnet has only one input, but we got " - << std::to_string(input_info_map.size()) << "inputs" << slog::endl; + << std::to_string(input_info_map.size()) << "inputs" << slog::endl; return false; } + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + + input_info.tensor().set_element_type(ov::element::u8); + addInputInfo("input", input_tensor_name_); - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - addInputInfo("input", input_info_map.begin()->first); + ov::Shape input_dims = input_info_map[0].get_shape(); - const InferenceEngine::SizeVector input_dims = input_info->getTensorDesc().getDims(); setInputHeight(input_dims[2]); setInputWidth(input_dims[3]); slog::info << "Checking OUTPUTs for model " << getModelName() << slog::endl; - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); - if (output_info_map.size() != 1) - { - slog::warn << "This model seems not SSDNet-like! We got " << std::to_string(output_info_map.size()) - << "outputs, but SSDnet has only one." << slog::endl; + auto outputs_info = net_reader->outputs(); + if (outputs_info.size() != 1) { + slog::warn << "This model seems not SSDNet-like! We got " + << std::to_string(outputs_info.size()) << "outputs, but SSDnet has only one." + << slog::endl; return false; } - InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; - addOutputInfo("output", output_info_map.begin()->first); - slog::info << "Checking Object Detection output ... Name=" << output_info_map.begin()->first << slog::endl; - output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); + ov::preprocess::OutputInfo& output_info = ppp.output(); + addOutputInfo("output", net_reader->output().get_any_name()); + slog::info << "Checking Object Detection output ... Name=" << net_reader->output().get_any_name() + << slog::endl; + + output_info.tensor().set_element_type(ov::element::f32); + net_reader = ppp.build(); /// TODO: double check this part: BEGIN #if(0) @@ -209,7 +216,9 @@ bool Models::ObjectDetectionSSDModel::updateLayerProperty(InferenceEngine::CNNNe /// TODO: double check this part: END // last dimension of output layer should be 7 - const InferenceEngine::SizeVector output_dims = output_data_ptr->getTensorDesc().getDims(); + auto outputsDataMap = net_reader->outputs(); + auto & data = outputsDataMap[0]; + ov::Shape output_dims = data.get_shape(); setMaxProposalCount(static_cast(output_dims[2])); slog::info << "max proposal count is: " << getMaxProposalCount() << slog::endl; From 853f773b8efef9ac89d33145671f62fb5c454474 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Thu, 13 Oct 2022 11:56:01 +0800 Subject: [PATCH 12/40] fix license plate detection pipeline --- .../models/license_plate_detection_model.h | 2 +- .../inferences/license_plate_detection.cpp | 15 ++++++----- .../models/license_plate_detection_model.cpp | 27 +++++++++++-------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/license_plate_detection_model.h b/vino_core_lib/include/vino_core_lib/models/license_plate_detection_model.h index 17af5ed3..c4b02fcb 100644 --- a/vino_core_lib/include/vino_core_lib/models/license_plate_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/license_plate_detection_model.h @@ -53,7 +53,7 @@ class LicensePlateDetectionModel : public BaseModel const std::string getModelCategory() const override; protected: - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; // up to 88 items per license plate, ended with "-1" const int max_sequence_size_ = 88; std::string input_; diff --git a/vino_core_lib/src/inferences/license_plate_detection.cpp b/vino_core_lib/src/inferences/license_plate_detection.cpp index 87b4a97f..8dd08695 100644 --- a/vino_core_lib/src/inferences/license_plate_detection.cpp +++ b/vino_core_lib/src/inferences/license_plate_detection.cpp @@ -44,13 +44,14 @@ void vino_core_lib::LicensePlateDetection::loadNetwork( void vino_core_lib::LicensePlateDetection::fillSeqBlob() { - InferenceEngine::Blob::Ptr seq_blob = getEngine()->getRequest()->GetBlob(valid_model_->getSeqInputName()); - int max_sequence_size = seq_blob->getTensorDesc().getDims()[0]; + ov::Tensor seq_tensor = getEngine()->getRequest().get_tensor( + valid_model_->getSeqInputName()); + int max_sequence_size = seq_tensor.get_shape()[0]; // second input is sequence, which is some relic from the training // it should have the leading 0.0f and rest 1.0f - float* blob_data = seq_blob->buffer().as(); - blob_data[0] = 0.0f; - std::fill(blob_data + 1, blob_data + max_sequence_size, 1.0f); + float * tensor_data = seq_tensor.data(); + tensor_data[0] = 0.0f; + std::fill(tensor_data + 1, tensor_data + max_sequence_size, 1.0f); } bool vino_core_lib::LicensePlateDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) @@ -82,9 +83,9 @@ bool vino_core_lib::LicensePlateDetection::fetchResults() return false; } bool found_result = false; - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + ov::InferRequest request = getEngine()->getRequest(); std::string output = valid_model_->getOutputName(); - const float* output_values = request->GetBlob(output)->buffer().as(); + const float* output_values = request.get_tensor(output).data(); for (int i = 0; i < getResultsLength(); i++) { std::string license = ""; diff --git a/vino_core_lib/src/models/license_plate_detection_model.cpp b/vino_core_lib/src/models/license_plate_detection_model.cpp index 7a7f2e74..b59726b6 100644 --- a/vino_core_lib/src/models/license_plate_detection_model.cpp +++ b/vino_core_lib/src/models/license_plate_detection_model.cpp @@ -25,33 +25,38 @@ Models::LicensePlateDetectionModel::LicensePlateDetectionModel(const std::string { } -bool Models::LicensePlateDetectionModel::updateLayerProperty(InferenceEngine::CNNNetwork& net_reader) +bool Models::LicensePlateDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); + auto input_info_map = net_reader->inputs(); if (input_info_map.size() != 2) { throw std::logic_error("Vehicle Attribs topology should have only two inputs"); } - auto sequence_input = (++input_info_map.begin()); - if (sequence_input->second->getTensorDesc().getDims()[0] != getMaxSequenceSize()) + auto sequence_input = input_info_map[1]; + if (sequence_input.get_shape()[0] != getMaxSequenceSize()) { throw std::logic_error("License plate detection max sequence size dismatch"); } - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); + auto output_info_map = net_reader->outputs(); if (output_info_map.size() != 1) { throw std::logic_error("Vehicle Attribs Network expects networks having one output"); } - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + std::string input_tensor_name_ = input_info_map[0].get_any_name(); + const ov::Layout tensor_layout{"NCHW"}; + ppp.input(input_tensor_name_). + tensor(). + set_element_type(ov::element::u8). + set_layout(tensor_layout); + net_reader = ppp.build(); // set input and output layer name - input_ = input_info_map.begin()->first; - seq_input_ = (++input_info_map.begin())->first; - output_ = output_info_map.begin()->first; + input_ = input_tensor_name_; + seq_input_ = sequence_input.get_any_name(); + output_ = net_reader->output().get_any_name(); return true; } From daae598606e6a3d001cb9435787af1a01df37c91 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Thu, 13 Oct 2022 11:58:54 +0800 Subject: [PATCH 13/40] fix vehicle attribs detection pipeline --- .../models/vehicle_attribs_detection_model.h | 2 +- .../inferences/vehicle_attribs_detection.cpp | 6 ++-- .../vehicle_attribs_detection_model.cpp | 32 ++++++++----------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h b/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h index ad8e5acd..300a6fb2 100644 --- a/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h @@ -51,7 +51,7 @@ class VehicleAttribsDetectionModel : public BaseModel protected: // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; // void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; std::string input_; std::string color_output_; std::string type_output_; diff --git a/vino_core_lib/src/inferences/vehicle_attribs_detection.cpp b/vino_core_lib/src/inferences/vehicle_attribs_detection.cpp index 13e6d918..ff9ecd65 100644 --- a/vino_core_lib/src/inferences/vehicle_attribs_detection.cpp +++ b/vino_core_lib/src/inferences/vehicle_attribs_detection.cpp @@ -71,13 +71,13 @@ bool vino_core_lib::VehicleAttribsDetection::fetchResults() return false; } bool found_result = false; - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + ov::InferRequest request = getEngine()->getRequest(); // std::string color_name = valid_model_->getColorOutputName(); // std::string type_name = valid_model_->getTypeOutputName(); std::string color_name = valid_model_->getOutputName("color_output_"); std::string type_name = valid_model_->getOutputName("type_output_"); - const float* color_values = request->GetBlob(color_name)->buffer().as(); - const float* type_values = request->GetBlob(type_name)->buffer().as(); + const float * color_values = request.get_tensor(color_name).data(); + const float * type_values = request.get_tensor(type_name).data(); for (int i = 0; i < getResultsLength(); i++) { auto color_id = std::max_element(color_values, color_values + 7) - color_values; diff --git a/vino_core_lib/src/models/vehicle_attribs_detection_model.cpp b/vino_core_lib/src/models/vehicle_attribs_detection_model.cpp index a1c5c1b5..03854adc 100644 --- a/vino_core_lib/src/models/vehicle_attribs_detection_model.cpp +++ b/vino_core_lib/src/models/vehicle_attribs_detection_model.cpp @@ -25,37 +25,33 @@ Models::VehicleAttribsDetectionModel::VehicleAttribsDetectionModel(const std::st { } -bool Models::VehicleAttribsDetectionModel::updateLayerProperty(InferenceEngine::CNNNetwork& net_reader) +bool Models::VehicleAttribsDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); + auto input_info_map = net_reader->inputs(); if (input_info_map.size() != 1) { throw std::logic_error("Vehicle Attribs topology should have only one input"); } - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); + auto output_info_map = net_reader->outputs(); if (output_info_map.size() != 2) { throw std::logic_error("Vehicle Attribs Network expects networks having two outputs"); } - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); - - // set input and output layer name - input_ = input_info_map.begin()->first; - auto output_iter = output_info_map.begin(); - // color_output_ = (output_iter++)->second->name; - // type_output_ = (output_iter++)->second->name; - InferenceEngine::DataPtr color_output_ptr = (output_iter++)->second; - InferenceEngine::DataPtr type_output_ptr = (output_iter++)->second; - - addOutputInfo("color_output_", color_output_ptr->getName()); - // output_gender_ = gender_output_ptr->name; - addOutputInfo("type_output_", type_output_ptr->getName()); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + const ov::Layout tensor_layout{"NCHW"}; + input_info.tensor(). + set_element_type(ov::element::u8). + set_layout(tensor_layout); + net_reader = ppp.build(); + addInputInfo("input", input_tensor_name_); + addOutputInfo("color_output_", output_info_map[1].get_any_name()); + addOutputInfo("type_output_", output_info_map[0].get_any_name()); printAttribute(); return true; } From de67e5960486055287e5705c2f8142d31f53e606 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Thu, 13 Oct 2022 15:32:20 +0800 Subject: [PATCH 14/40] fix person attribs detection pipeline --- .../models/person_attribs_detection_model.h | 7 +++- .../inferences/person_attribs_detection.cpp | 14 ++++---- .../models/person_attribs_detection_model.cpp | 35 +++++++++---------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h b/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h index 01e9c01b..e9ba4c24 100644 --- a/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h @@ -41,9 +41,14 @@ class PersonAttribsDetectionModel : public BaseModel protected: // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; // void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; std::string input_; std::string output_; + std::string input_tensor_name_; + std::string output_tensor_name_; + + std::vector> inputs_info_; + std::vector> outputs_info_; }; } // namespace Models #endif // VINO_CORE_LIB__MODELS__PERSON_ATTRIBS_DETECTION_MODEL_H diff --git a/vino_core_lib/src/inferences/person_attribs_detection.cpp b/vino_core_lib/src/inferences/person_attribs_detection.cpp index 46728e29..f3b8574c 100644 --- a/vino_core_lib/src/inferences/person_attribs_detection.cpp +++ b/vino_core_lib/src/inferences/person_attribs_detection.cpp @@ -95,7 +95,7 @@ bool vino_core_lib::PersonAttribsDetection::fetchResults() return false; } bool found_result = false; - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + ov::InferRequest request = getEngine()->getRequest(); slog::debug << "Analyzing Attributes Detection results..." << slog::endl; std::string attribute_output = valid_model_->getOutputName("attributes_output_"); std::string top_output = valid_model_->getOutputName("top_output_"); @@ -104,13 +104,13 @@ bool vino_core_lib::PersonAttribsDetection::fetchResults() /*auto attri_values = request->GetBlob(attribute_output)->buffer().as(); auto top_values = request->GetBlob(top_output)->buffer().as(); auto bottom_values = request->GetBlob(bottom_output)->buffer().as();*/ - InferenceEngine::Blob::Ptr attribBlob = request->GetBlob(attribute_output); - InferenceEngine::Blob::Ptr topBlob = request->GetBlob(top_output); - InferenceEngine::Blob::Ptr bottomBlob = request->GetBlob(bottom_output); + ov::Tensor attrib_tensor = request.get_tensor(attribute_output); + ov::Tensor top_tensor = request.get_tensor(top_output); + ov::Tensor bottom_tensor = request.get_tensor(bottom_output); - auto attri_values = attribBlob->buffer().as(); - auto top_values = topBlob->buffer().as(); - auto bottom_values = bottomBlob->buffer().as(); + auto attri_values = attrib_tensor.data(); + auto top_values = top_tensor.data(); + auto bottom_values = bottom_tensor.data(); int net_attrib_length = net_attributes_.size(); for (int i = 0; i < getResultsLength(); i++) diff --git a/vino_core_lib/src/models/person_attribs_detection_model.cpp b/vino_core_lib/src/models/person_attribs_detection_model.cpp index f4e8e618..d0f584bd 100644 --- a/vino_core_lib/src/models/person_attribs_detection_model.cpp +++ b/vino_core_lib/src/models/person_attribs_detection_model.cpp @@ -25,37 +25,34 @@ Models::PersonAttribsDetectionModel::PersonAttribsDetectionModel(const std::stri { } -bool Models::PersonAttribsDetectionModel::updateLayerProperty(InferenceEngine::CNNNetwork& net_reader) +bool Models::PersonAttribsDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); + auto input_info_map = net_reader->inputs(); if (input_info_map.size() != 1) { throw std::logic_error("Person Attribs topology should have only one input"); } - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); - addInputInfo("input", input_info_map.begin()->first); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + const ov::Layout tensor_layout{"NCHW"}; + input_info.tensor(). + set_element_type(ov::element::u8). + set_layout(tensor_layout); slog::info << "Checking OUTPUTs for model " << getModelName() << slog::endl; - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); + auto output_info_map = net_reader->outputs(); if (output_info_map.size() != 3) { throw std::logic_error("Person Attribs Network expects networks having 3 output"); } - input_ = input_info_map.begin()->first; - output_ = output_info_map.begin()->first; - - auto output_iter = output_info_map.begin(); - InferenceEngine::DataPtr attribute_output_ptr = (output_iter++)->second; - InferenceEngine::DataPtr top_output_ptr = (output_iter++)->second; - InferenceEngine::DataPtr bottom_output_ptr = (output_iter++)->second; - - addOutputInfo("attributes_output_", attribute_output_ptr->getName()); - // output_gender_ = gender_output_ptr->name; - addOutputInfo("top_output_", top_output_ptr->getName()); - addOutputInfo("bottom_output_", bottom_output_ptr->getName()); + net_reader = ppp.build(); + addInputInfo("input", input_tensor_name_); + addOutputInfo("attributes_output_",output_info_map[2].get_any_name()); + //output_gender_ = gender_output_ptr->name; + addOutputInfo("top_output_", output_info_map[1].get_any_name()); + addOutputInfo("bottom_output_", output_info_map[0].get_any_name()); printAttribute(); return true; } From ef7c0306de882c8c99b1249214429f4ee52a2779 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Thu, 13 Oct 2022 15:33:47 +0800 Subject: [PATCH 15/40] fix object detection(yolo) pipeline --- .../models/object_detection_yolov2voc_model.h | 4 +- .../object_detection_yolov2voc_model.cpp | 74 +++++++++++-------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/object_detection_yolov2voc_model.h b/vino_core_lib/include/vino_core_lib/models/object_detection_yolov2voc_model.h index e619fa56..12f18099 100644 --- a/vino_core_lib/include/vino_core_lib/models/object_detection_yolov2voc_model.h +++ b/vino_core_lib/include/vino_core_lib/models/object_detection_yolov2voc_model.h @@ -49,11 +49,11 @@ class ObjectDetectionYolov2Model : public ObjectDetectionModel * @return Name of the model. */ const std::string getModelCategory() const override; - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; protected: int getEntryIndex(int side, int lcoords, int lclasses, int location, int entry); - InferenceEngine::InputInfo::Ptr input_info_ = nullptr; + // InferenceEngine::InputInfo::Ptr input_info_ = nullptr; }; } // namespace Models #endif // VINO_CORE_LIB__MODELS__OBJECT_DETECTION_YOLOV2VOC_MODEL_H diff --git a/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp b/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp index 5b712d13..ba9b83e4 100644 --- a/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp +++ b/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp @@ -26,11 +26,11 @@ Models::ObjectDetectionYolov2Model::ObjectDetectionYolov2Model(const std::string { } -bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const InferenceEngine::CNNNetwork& net_reader) +bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const std::shared_ptr& net_reader) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); + auto input_info_map = net_reader->inputs(); if (input_info_map.size() != 1) { slog::warn << "This model seems not Yolo-like, which has only one input, but we got " @@ -38,24 +38,30 @@ bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const InferenceEngi return false; } - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::FP32); - input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); - input_info_ = input_info; - addInputInfo("input", input_info_map.begin()->first); + // set input property + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + const ov::Layout input_tensor_layout{"NCHW"}; + input_info.tensor(). + set_element_type(ov::element::f32). + set_layout(input_tensor_layout); + addInputInfo("input", input_tensor_name_); // set output property - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); - if (output_info_map.size() != 1) - { - slog::warn << "This model seems not Yolo-like! We got " << std::to_string(output_info_map.size()) - << "outputs, but SSDnet has only one." << slog::endl; + auto output_info_map = net_reader -> outputs(); + if (output_info_map.size() != 1) { + slog::warn << "This model seems not Yolo-like! We got " + << std::to_string(output_info_map.size()) << "outputs, but SSDnet has only one." + << slog::endl; return false; } - InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; - output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); - addOutputInfo("output", output_info_map.begin()->first); - slog::info << "Checking Object Detection output ... Name=" << output_info_map.begin()->first << slog::endl; + ov::preprocess::OutputInfo& output_info = ppp.output(); + addOutputInfo("output", net_reader->output().get_any_name()); + output_info.tensor().set_element_type(ov::element::f32); + slog::info << "Checking Object Detection output ... Name=" << net_reader->output().get_any_name() + << slog::endl; + net_reader = ppp.build(); #if(0) const InferenceEngine::CNNLayerPtr output_layer = @@ -85,7 +91,9 @@ bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const InferenceEngi } #endif // last dimension of output layer should be 7 - const InferenceEngine::SizeVector output_dims = output_data_ptr->getTensorDesc().getDims(); + auto outputsDataMap = net_reader->outputs(); + auto & data = outputsDataMap[0]; + ov::Shape output_dims = data.get_shape(); setMaxProposalCount(static_cast(output_dims[2])); slog::info << "max proposal count is: " << getMaxProposalCount() << slog::endl; @@ -137,13 +145,14 @@ bool Models::ObjectDetectionYolov2Model::matToBlob(const cv::Mat& orig_image, co } std::string input_name = getInputName(); - InferenceEngine::Blob::Ptr input_blob = engine->getRequest()->GetBlob(input_name); + ov::Tensor input_tensor = + engine->getRequest().get_tensor(input_name); - InferenceEngine::SizeVector blob_size = input_blob->getTensorDesc().getDims(); + ov::Shape blob_size = input_tensor.get_shape(); const int width = blob_size[3]; const int height = blob_size[2]; const int channels = blob_size[1]; - float* blob_data = input_blob->buffer().as(); + float * blob_data = input_tensor.data(); int dx = 0; int dy = 0; @@ -223,16 +232,17 @@ bool Models::ObjectDetectionYolov2Model::fetchResults(const std::shared_ptrgetRequest(); + ov::InferRequest request = engine->getRequest(); std::string output = getOutputName(); - std::vector& labels = getLabels(); - const float* detections = request->GetBlob(output) - ->buffer() - .as::value_type*>(); - InferenceEngine::CNNLayerPtr layer = getNetReader()->getNetwork().getLayerByName(output.c_str()); - int input_height = input_info_->getTensorDesc().getDims()[2]; - int input_width = input_info_->getTensorDesc().getDims()[3]; + std::vector & labels = getLabels(); + const float * detections = (float * )request.get_tensor(output).data(); + + std::string input = getInputName(); + auto input_tensor = request.get_tensor(input); + ov::Shape input_shape = input_tensor.get_shape(); + int input_height = input_shape[2]; + int input_width = input_shape[3]; // --------------------------- Validating output parameters -------------------------------- if (layer != nullptr && layer->type != "RegionYolo") @@ -243,13 +253,13 @@ bool Models::ObjectDetectionYolov2Model::fetchResults(const std::shared_ptrGetParamAsInt("num"); const int coords = layer->GetParamAsInt("coords"); const int classes = layer->GetParamAsInt("classes"); - auto blob = request->GetBlob(output); - const int out_blob_h = static_cast(blob->getTensorDesc().getDims()[2]); - ; + auto output_tensor = request.get_tensor(output); + ov::Shape output_shape = output_tensor.get_shape(); + const int out_tensor_h = static_cast(output_shape[2]);; std::vector anchors = { 0.572730, 0.677385, 1.874460, 2.062530, 3.338430, 5.474340, 7.882820, 3.527780, 9.770520, 9.168280 }; - auto side = out_blob_h; + auto side = out_tensor_h; auto side_square = side * side; // --------------------------- Parsing YOLO Region output ------------------------------------- From 0d3d74a627ca027e809af64d829ac841603e2944 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Sun, 16 Oct 2022 20:44:13 +0800 Subject: [PATCH 16/40] fix face reidentification pipeline --- .../inferences/face_reidentification.h | 1 + .../models/face_reidentification_model.h | 2 +- .../src/inferences/face_reidentification.cpp | 11 +++--- .../models/face_reidentification_model.cpp | 35 ++++++++++--------- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/inferences/face_reidentification.h b/vino_core_lib/include/vino_core_lib/inferences/face_reidentification.h index 7f8291ce..562661cc 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/face_reidentification.h +++ b/vino_core_lib/include/vino_core_lib/inferences/face_reidentification.h @@ -26,6 +26,7 @@ #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/inferences/base_reidentification.h" #include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace namespace vino_core_lib diff --git a/vino_core_lib/include/vino_core_lib/models/face_reidentification_model.h b/vino_core_lib/include/vino_core_lib/models/face_reidentification_model.h index a02844e7..5a43b762 100644 --- a/vino_core_lib/include/vino_core_lib/models/face_reidentification_model.h +++ b/vino_core_lib/include/vino_core_lib/models/face_reidentification_model.h @@ -42,7 +42,7 @@ class FaceReidentificationModel : public BaseModel * @brief Get the name of this detection model. * @return Name of the model. */ - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; const std::string getModelCategory() const override; protected: diff --git a/vino_core_lib/src/inferences/face_reidentification.cpp b/vino_core_lib/src/inferences/face_reidentification.cpp index 61277bbb..ae6f51c5 100644 --- a/vino_core_lib/src/inferences/face_reidentification.cpp +++ b/vino_core_lib/src/inferences/face_reidentification.cpp @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - + /** * @brief a header file with declaration of FaceReidentification class and * FaceReidentificationResult class @@ -71,10 +71,13 @@ bool vino_core_lib::FaceReidentification::fetchResults() return false; } bool found_result = false; - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + ov::InferRequest request = getEngine()->getRequest(); std::string output = valid_model_->getOutputName(); - const float* output_values = request->GetBlob(output)->buffer().as(); - int result_length = request->GetBlob(output)->getTensorDesc().getDims()[1]; + ov::Tensor output_tensor = request.get_tensor(output); + const float* output_values = output_tensor.data(); + ov::Shape output_shape = output_tensor.get_shape(); + int result_length = output_shape[1]; + for (int i = 0; i < getResultsLength(); i++) { std::vector new_face = diff --git a/vino_core_lib/src/models/face_reidentification_model.cpp b/vino_core_lib/src/models/face_reidentification_model.cpp index bcf2c399..6d32c375 100644 --- a/vino_core_lib/src/models/face_reidentification_model.cpp +++ b/vino_core_lib/src/models/face_reidentification_model.cpp @@ -25,28 +25,31 @@ Models::FaceReidentificationModel::FaceReidentificationModel(const std::string& { } -bool Models::FaceReidentificationModel::updateLayerProperty(InferenceEngine::CNNNetwork& net_reader) +bool Models::FaceReidentificationModel::updateLayerProperty(std::shared_ptr& net_reader) { // set input property - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); - InferenceEngine::InputInfo::Ptr input_info = input_info_map.begin()->second; - input_info->setPrecision(InferenceEngine::Precision::U8); - input_info->getInputData()->setLayout(InferenceEngine::Layout::NCHW); + auto inputs_info_map = net_reader->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + + const ov::Layout tensor_layout{"NCHW"}; + input_info.tensor(). + set_element_type(ov::element::u8). + set_layout(tensor_layout); + addInputInfo("input", input_tensor_name_); + // set output property - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); - InferenceEngine::DataPtr& output_data_ptr = output_info_map.begin()->second; - output_data_ptr->setPrecision(InferenceEngine::Precision::FP32); - output_data_ptr->setLayout(InferenceEngine::Layout::NCHW); - // set input and output layer name - input_ = input_info_map.begin()->first; - output_ = output_info_map.begin()->first; + auto outputs_info = net_reader->outputs(); + std::string output_tensor_name = net_reader->output().get_any_name(); + ov::preprocess::OutputInfo& output_info = ppp.output(output_tensor_name); + output_info.tensor().set_element_type(ov::element::f32); + addOutputInfo("output", output_tensor_name); + net_reader = ppp.build(); + return true; } -// void Models::FaceReidentificationModel::checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr& net_reader) -// { -// } - const std::string Models::FaceReidentificationModel::getModelCategory() const { return "Face Reidentification"; From 0060965dadc87962a7657f44f1c21e8fdf7389d5 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Sun, 16 Oct 2022 20:46:17 +0800 Subject: [PATCH 17/40] fix landmarks detection pipeline --- .../inferences/landmarks_detection.h | 1 + .../models/landmarks_detection_model.h | 2 +- .../src/inferences/landmarks_detection.cpp | 9 ++-- .../src/models/landmarks_detection_model.cpp | 49 ++++++++++++------- 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/inferences/landmarks_detection.h b/vino_core_lib/include/vino_core_lib/inferences/landmarks_detection.h index eaf3a070..6df6f32c 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/landmarks_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/landmarks_detection.h @@ -25,6 +25,7 @@ #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" #include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace namespace vino_core_lib diff --git a/vino_core_lib/include/vino_core_lib/models/landmarks_detection_model.h b/vino_core_lib/include/vino_core_lib/models/landmarks_detection_model.h index 69afc695..43b8eaf1 100644 --- a/vino_core_lib/include/vino_core_lib/models/landmarks_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/landmarks_detection_model.h @@ -42,7 +42,7 @@ class LandmarksDetectionModel : public BaseModel * @brief Get the name of this detection model. * @return Name of the model. */ - bool updateLayerProperty(InferenceEngine::CNNNetwork&) override; + bool updateLayerProperty(std::shared_ptr&) override; const std::string getModelCategory() const override; protected: diff --git a/vino_core_lib/src/inferences/landmarks_detection.cpp b/vino_core_lib/src/inferences/landmarks_detection.cpp index cea90fde..8726da80 100644 --- a/vino_core_lib/src/inferences/landmarks_detection.cpp +++ b/vino_core_lib/src/inferences/landmarks_detection.cpp @@ -68,10 +68,13 @@ bool vino_core_lib::LandmarksDetection::fetchResults() return false; } bool found_result = false; - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); + ov::InferRequest request = getEngine()->getRequest(); std::string output = valid_model_->getOutputName(); - const float* output_values = request->GetBlob(output)->buffer().as(); - int result_length = request->GetBlob(output)->getTensorDesc().getDims()[1]; + ov::Tensor output_tensor = request.get_tensor(output); + const float* output_values = output_tensor.data(); + ov::Shape output_shape = output_tensor.get_shape(); + int result_length = output_shape[1]; + for (int i = 0; i < getResultsLength(); i++) { std::vector coordinates = diff --git a/vino_core_lib/src/models/landmarks_detection_model.cpp b/vino_core_lib/src/models/landmarks_detection_model.cpp index d0c2687b..d92cade6 100644 --- a/vino_core_lib/src/models/landmarks_detection_model.cpp +++ b/vino_core_lib/src/models/landmarks_detection_model.cpp @@ -24,24 +24,34 @@ Models::LandmarksDetectionModel::LandmarksDetectionModel(const std::string& labe : BaseModel(label_loc, model_loc, max_batch_size) { } - -bool Models::LandmarksDetectionModel::updateLayerProperty(InferenceEngine::CNNNetwork& net_reader) + +bool Models::LandmarksDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) { //INPUT - InferenceEngine::InputsDataMap input_info_map(net_reader.getInputsInfo()); - auto input_layerName = input_info_map.begin()->first; - auto input_layerData = input_info_map.begin()->second; - auto input_layerDims = input_layerData->getTensorDesc().getDims(); + auto inputs_info_map = net_reader->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); + + auto input_layerName = inputs_info_map[0]; + auto input_layerData = inputs_info_map[1]; + addInputInfo("input_layerName", input_layerName.get_any_name()); + addInputInfo("input_layerData", input_layerData.get_any_name()); + ov::Shape input_layerDims = input_layerData.get_shape(); if (input_layerDims.size() == 4) { - input_layerData->setLayout(InferenceEngine::Layout::NCHW); - input_layerData->setPrecision(InferenceEngine::Precision::U8); + const ov::Layout tensor_layout{"NCHW"}; + ppp.input(input_layerData.get_any_name()).tensor(). + set_element_type(ov::element::u8). + set_layout(tensor_layout); } else if (input_layerDims.size() == 2) { - input_layerData->setLayout(InferenceEngine::Layout::NC); - input_layerData->setPrecision(InferenceEngine::Precision::FP32); + const ov::Layout tensor_layout{"NC"}; + ppp.input(input_layerData.get_any_name()).tensor(). + set_element_type(ov::element::f32). + set_layout(tensor_layout); } else { @@ -49,16 +59,17 @@ bool Models::LandmarksDetectionModel::updateLayerProperty(InferenceEngine::CNNNe } // OUTPUT - InferenceEngine::OutputsDataMap output_info_map(net_reader.getOutputsInfo()); - auto output_layerName = output_info_map.begin()->first; - auto output_layerData = output_info_map.begin()->second; - auto output_layerDims = output_layerData->getTensorDesc().getDims(); + std::string output_tensor_name = net_reader->output().get_any_name(); + ov::preprocess::OutputInfo& output_info = ppp.output(output_tensor_name); + auto outputs_info_map = net_reader->outputs(); + auto output_layerName = outputs_info_map[0]; + auto output_layerData = outputs_info_map[1]; + ov::Shape output_layerDims = output_layerData.get_shape(); + ppp.output(output_layerData.get_any_name()).tensor().set_element_type(ov::element::f32); - output_layerData->setPrecision(InferenceEngine::Precision::FP32); - // output_layerData->setLayout(InferenceEngine::Layout::NCHW); - // set input and output layer name - input_ = input_layerName; - output_ = output_layerName; + net_reader = ppp.build(); + addOutputInfo("output_layerName", output_layerName.get_any_name()); + addOutputInfo("output_layerData", output_layerData.get_any_name()); return true; } From b120aa02182739bea0849ab63b9e7826bb1d8657 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Sun, 16 Oct 2022 20:47:12 +0800 Subject: [PATCH 18/40] fix pipeline manager --- vino_core_lib/src/pipeline_manager.cpp | 70 +++++++++++++------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/vino_core_lib/src/pipeline_manager.cpp b/vino_core_lib/src/pipeline_manager.cpp index 902073b2..a96fdb14 100644 --- a/vino_core_lib/src/pipeline_manager.cpp +++ b/vino_core_lib/src/pipeline_manager.cpp @@ -251,18 +251,18 @@ PipelineManager::parseInference(const Params::ParamManager::PipelineRawData& par { object = createPersonReidentification(infer); } - // else if (infer.name == kInferTpye_FaceReidentification) - // { - // object = createFaceReidentification(infer); - // } + else if (infer.name == kInferTpye_FaceReidentification) + { + object = createFaceReidentification(infer); + } else if (infer.name == kInferTpye_PersonAttribsDetection) { object = createPersonAttribsDetection(infer); } - // else if (infer.name == kInferTpye_LandmarksDetection) - // { - // object = createLandmarksDetection(infer); - // } + else if (infer.name == kInferTpye_LandmarksDetection) + { + object = createLandmarksDetection(infer); + } else if (infer.name == kInferTpye_VehicleAttribsDetection) { object = createVehicleAttribsDetection(infer); @@ -432,33 +432,33 @@ PipelineManager::createPersonAttribsDetection(const Params::ParamManager::Infere return attribs_inference_ptr; } -// std::shared_ptr -// PipelineManager::createFaceReidentification(const Params::ParamManager::InferenceRawData& infer) -// { -// auto model = std::make_shared(infer.label, infer.model, infer.batch); -// slog::debug << "for test in createFaceReidentification()" << slog::endl; -// model->modelInit(); -// auto engine = engine_manager_.createEngine(infer.engine, model); -// auto attribs_inference_ptr = std::make_shared(infer.confidence_threshold); -// attribs_inference_ptr->loadNetwork(model); -// attribs_inference_ptr->loadEngine(engine); - -// return attribs_inference_ptr; -// } - -// std::shared_ptr -// PipelineManager::createLandmarksDetection( -// const Params::ParamManager::InferenceRawData & infer) -// { -// auto model = std::make_shared(infer.label, infer.model, infer.batch); -// model->modelInit(); -// auto engine = engine_manager_.createEngine(infer.engine, model); -// auto landmarks_inference_ptr = std::make_shared(); -// landmarks_inference_ptr->loadNetwork(model); -// landmarks_inference_ptr->loadEngine(engine); - -// return landmarks_inference_ptr; -// } +std::shared_ptr +PipelineManager::createFaceReidentification(const Params::ParamManager::InferenceRawData& infer) +{ + auto model = std::make_shared(infer.label, infer.model, infer.batch); + slog::debug << "for test in createFaceReidentification()" << slog::endl; + model->modelInit(); + auto engine = engine_manager_.createEngine(infer.engine, model); + auto attribs_inference_ptr = std::make_shared(infer.confidence_threshold); + attribs_inference_ptr->loadNetwork(model); + attribs_inference_ptr->loadEngine(engine); + + return attribs_inference_ptr; +} + +std::shared_ptr +PipelineManager::createLandmarksDetection( + const Params::ParamManager::InferenceRawData & infer) +{ + auto model = std::make_shared(infer.label, infer.model, infer.batch); + model->modelInit(); + auto engine = engine_manager_.createEngine(infer.engine, model); + auto landmarks_inference_ptr = std::make_shared(); + landmarks_inference_ptr->loadNetwork(model); + landmarks_inference_ptr->loadEngine(engine); + + return landmarks_inference_ptr; +} void PipelineManager::threadPipeline(const char* name) { From 4eb28696ab66131d25fb5d9a9c6c9ba95f515cce Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 17 Oct 2022 17:03:24 +0800 Subject: [PATCH 19/40] fix pipeline params API --- vino_sample/src/pipeline_with_params.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vino_sample/src/pipeline_with_params.cpp b/vino_sample/src/pipeline_with_params.cpp index 09ed868c..887d0b00 100644 --- a/vino_sample/src/pipeline_with_params.cpp +++ b/vino_sample/src/pipeline_with_params.cpp @@ -54,7 +54,7 @@ #include "vino_core_lib/pipeline.h" #include "vino_core_lib/pipeline_manager.h" #include "vino_core_lib/slog.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" #include "vino_sample/utility.hpp" @@ -91,7 +91,7 @@ int main(int argc, char** argv) try { - std::cout << "InferenceEngine: " << InferenceEngine::GetInferenceEngineVersion() << std::endl; + std::cout << "OpenVINO: " << ov::get_openvino_version << std::endl; // ----- Parsing and validation of input args----------------------- if (!parseAndCheckCommandLine(argc, argv)) From 439c569c2bcbec2fdb15e667568ed543c6436ac9 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 17 Oct 2022 17:04:35 +0800 Subject: [PATCH 20/40] fix CMakeLists --- vino_core_lib/CMakeLists.txt | 25 ++++++------------ vino_sample/CMakeLists.txt | 49 +++++++++++------------------------- 2 files changed, 21 insertions(+), 53 deletions(-) diff --git a/vino_core_lib/CMakeLists.txt b/vino_core_lib/CMakeLists.txt index 1659874b..e1c5da63 100644 --- a/vino_core_lib/CMakeLists.txt +++ b/vino_core_lib/CMakeLists.txt @@ -17,7 +17,7 @@ cmake_minimum_required (VERSION 3.0.2) project(vino_core_lib) #################################### -## to use new InferenceEngine API (InferenceEngine::Core), +## to use new OpenVINO API (ov::core), ## then, uncomment below line #add_definitions(-DUSE_IE_CORE) #################################### @@ -27,9 +27,8 @@ project(vino_core_lib) ## add_definitions(-DLOG_LEVEL_DEBUG) #################################### -message(STATUS "Looking for inference engine configuration file at: ${CMAKE_PREFIX_PATH}") -find_package(InferenceEngine REQUIRED) -if (NOT InferenceEngine_FOUND) +find_package(OpenVINO REQUIRED) +if (NOT OpenVINO_FOUND) message(FATAL_ERROR "") endif() @@ -44,7 +43,7 @@ find_package( vino_pipeline_srv_msgs image_transport cv_bridge - InferenceEngine + OpenVINO ) # Find OpenCV libray if exists @@ -65,27 +64,17 @@ catkin_package( image_transport object_msgs ) - +set(OpenVINO_LIBRARIES openvino::runtime) include_directories ( include ${catkin_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/include - ${InferenceEngine_INCLUDE_DIRS} - ${InferenceEngine_INCLUDE_DIRS}/../samples - ${InferenceEngine_DIR}/../src + ${INTEL_OPENVINO_DIR} ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/gflags/include ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/extension/include ${CMAKE_CURRENT_SOURCE_DIR}/../vino_param_lib/include ) -#if (NOT DEFINED ENV{CPU_EXTENSION_LIB}) -# message(FATAL_ERROR "Please set ENV CPU_EXTENSION_LIB with 'export CPU_EXTENSION_LIB='") -#endif() -#set (CpuExtension_lib $ENV{CPU_EXTENSION_LIB}) -#add_library(cpu_extension SHARED IMPORTED) -#set_target_properties(cpu_extension PROPERTIES -# IMPORTED_LOCATION $ENV{CPU_EXTENSION_LIB}) - # Flags if(UNIX OR APPLE) # Linker flags. @@ -142,7 +131,7 @@ if (SUPPORT_SSE41 EQUAL 0) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.1") endif() -set(DEPENDENCIES realsense2 ${OpenCV_LIBS}) +set(DEPENDENCIES realsense2 ${OpenCV_LIBS} openvino::runtime) add_library(${PROJECT_NAME} SHARED src/services/frame_processing_server.cpp diff --git a/vino_sample/CMakeLists.txt b/vino_sample/CMakeLists.txt index f3464497..50f499b3 100644 --- a/vino_sample/CMakeLists.txt +++ b/vino_sample/CMakeLists.txt @@ -16,9 +16,8 @@ cmake_minimum_required(VERSION 3.0.2) project(vino_sample) -message(STATUS "Looking for inference engine configuration file at: ${CMAKE_PREFIX_PATH}") -find_package(InferenceEngine) -if (NOT InferenceEngine_FOUND) +find_package(OpenVINO) +if (NOT OpenVINO_FOUND) message(FATAL_ERROR "") endif() @@ -62,12 +61,7 @@ include_directories ( include ${catkin_INCLUDE_DIRS} ${OpenCV_INCLUDE_DIRS} - ${InferenceEngine_INCLUDE_DIRS} - ${InferenceEngine_INCLUDE_DIRS}/../samples/build/thirdparty/gflags/include - ${InferenceEngine_INCLUDE_DIRS}/../build/samples/thirdparty/gflags/include - ${InferenceEngine_INCLUDE_DIRS}/../samples - ${InferenceEngine_DIR}/../src - #${vino_core_lib_INCLUDE_DIRS} + ${OpenVINO_DIR} ${vino_param_lib_INCLUDE_DIRS} ) @@ -80,22 +74,6 @@ else() return() endif() -#if (NOT DEFINED ENV{CPU_EXTENSION_LIB}) -# message(FATAL_ERROR "Please set ENV CPU_EXTENSION_LIB with 'export CPU_EXTENSION_LIB='") -#endif() -#set (CpuExtension_lib $ENV{CPU_EXTENSION_LIB}) -#add_library(cpu_extension SHARED IMPORTED) -#set_target_properties(cpu_extension PROPERTIES -# IMPORTED_LOCATION $ENV{CPU_EXTENSION_LIB}) - -#if (NOT DEFINED ENV{GFLAGS_LIB}) -# message(FATAL_ERROR "Please set ENV GFLAGS_LIB with 'export GFLAGS_LIB='") -#endif() -#set (Gflags_lib $ENV{GFLAGS_LIB}) -#add_library(gflags STATIC IMPORTED) -#set_target_properties(gflags PROPERTIES -# IMPORTED_LOCATION $ENV{GFLAGS_LIB}) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") add_executable(pipeline_with_params @@ -108,11 +86,12 @@ add_dependencies(pipeline_with_params ${vino_core_lib_TARGETS} ) +set(OpenCV_LIBRARIES openvino::runtime) target_link_libraries(pipeline_with_params ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) @@ -130,7 +109,7 @@ target_link_libraries(image_people_server ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) @@ -149,7 +128,7 @@ target_link_libraries(image_people_client ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) @@ -167,7 +146,7 @@ target_link_libraries(image_object_server ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) @@ -186,7 +165,7 @@ target_link_libraries(image_object_client ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) @@ -205,7 +184,7 @@ target_link_libraries(image_segmentation_client ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) @@ -223,7 +202,7 @@ target_link_libraries(image_segmentation_server ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) @@ -242,7 +221,7 @@ target_link_libraries(image_reidentification_client ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) @@ -260,7 +239,7 @@ target_link_libraries(image_reidentification_server ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) @@ -278,7 +257,7 @@ target_link_libraries(pipeline_service_client ${catkin_LIBRARIES} gflags ${vino_param_lib_LIBRARIES} - ${InferenceEngine_LIBRARIES} + ${OpenVINO_LIBRARIES} ${OpenCV_LIBRARIES} ) From 68a5aa4ea9993d02e4bc27a6b19ad82794b02696 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 17 Oct 2022 17:18:29 +0800 Subject: [PATCH 21/40] clean code and fix head files --- .../include/vino_core_lib/engines/engine.h | 1 - .../vino_core_lib/engines/engine_manager.h | 2 +- .../inferences/age_gender_detection.h | 2 +- .../vino_core_lib/inferences/base_inference.h | 1 - .../inferences/emotions_detection.h | 1 - .../vino_core_lib/inferences/face_detection.h | 2 +- .../inferences/face_reidentification.h | 1 - .../inferences/head_pose_detection.h | 1 - .../inferences/landmarks_detection.h | 1 - .../inferences/license_plate_detection.h | 2 +- .../inferences/object_detection.h | 2 +- .../inferences/object_segmentation.h | 2 +- .../inferences/person_attribs_detection.h | 2 +- .../inferences/person_reidentification.h | 2 +- .../inferences/vehicle_attribs_detection.h | 2 +- .../models/attributes/base_attribute.h | 4 +-- .../include/vino_core_lib/models/base_model.h | 2 -- .../models/emotion_detection_model.h | 1 - .../models/face_detection_model.h | 1 - .../models/object_detection_yolov2voc_model.h | 1 - .../models/person_attribs_detection_model.h | 4 --- .../models/person_reidentification_model.h | 2 -- .../models/vehicle_attribs_detection_model.h | 2 -- vino_core_lib/src/engines/engine_manager.cpp | 2 +- .../src/inferences/emotions_detection.cpp | 9 +----- .../inferences/license_plate_detection.cpp | 2 -- .../src/inferences/object_detection.cpp | 4 --- .../src/inferences/object_segmentation.cpp | 2 -- .../inferences/person_attribs_detection.cpp | 28 +------------------ .../inferences/vehicle_attribs_detection.cpp | 2 -- vino_core_lib/src/inputs/image_input.cpp | 1 - vino_core_lib/src/inputs/image_topic.cpp | 6 ---- vino_core_lib/src/inputs/ip_camera.cpp | 2 +- vino_core_lib/src/inputs/realsense_camera.cpp | 2 -- vino_core_lib/src/inputs/standard_camera.cpp | 1 - vino_core_lib/src/inputs/video_input.cpp | 1 - vino_core_lib/src/models/base_model.cpp | 6 ---- .../src/models/emotion_detection_model.cpp | 2 +- .../src/models/object_detection_ssd_model.cpp | 6 ++-- .../src/models/object_segmentation_model.cpp | 20 +------------ .../models/person_attribs_detection_model.cpp | 1 - vino_core_lib/src/outputs/base_output.cpp | 1 - .../src/outputs/image_window_output.cpp | 1 - .../src/outputs/ros_service_output.cpp | 1 - .../src/outputs/ros_topic_output.cpp | 9 ------ vino_core_lib/src/outputs/rviz_output.cpp | 1 - vino_core_lib/src/pipeline.cpp | 7 ----- vino_core_lib/src/pipeline_manager.cpp | 4 --- vino_sample/src/image_object_server.cpp | 2 +- vino_sample/src/image_people_server.cpp | 2 +- .../src/image_reidentification_server.cpp | 2 +- vino_sample/src/image_segmentation_server.cpp | 2 +- 52 files changed, 23 insertions(+), 147 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/engines/engine.h b/vino_core_lib/include/vino_core_lib/engines/engine.h index 8302ba02..45d315f1 100644 --- a/vino_core_lib/include/vino_core_lib/engines/engine.h +++ b/vino_core_lib/include/vino_core_lib/engines/engine.h @@ -24,7 +24,6 @@ #pragma once #include "vino_core_lib/models/base_model.h" -#include "inference_engine.hpp" #include "openvino/openvino.hpp" /** diff --git a/vino_core_lib/include/vino_core_lib/engines/engine_manager.h b/vino_core_lib/include/vino_core_lib/engines/engine_manager.h index 8dc1f09d..f3314e2c 100644 --- a/vino_core_lib/include/vino_core_lib/engines/engine_manager.h +++ b/vino_core_lib/include/vino_core_lib/engines/engine_manager.h @@ -23,7 +23,7 @@ #include "vino_core_lib/models/base_model.h" #include "vino_core_lib/engines/engine.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" namespace Engines { diff --git a/vino_core_lib/include/vino_core_lib/inferences/age_gender_detection.h b/vino_core_lib/include/vino_core_lib/inferences/age_gender_detection.h index ed6f9373..0bd6977b 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/age_gender_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/age_gender_detection.h @@ -28,7 +28,7 @@ #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/models/age_gender_detection_model.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" namespace Outputs diff --git a/vino_core_lib/include/vino_core_lib/inferences/base_inference.h b/vino_core_lib/include/vino_core_lib/inferences/base_inference.h index 688a839d..7087b141 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/base_inference.h +++ b/vino_core_lib/include/vino_core_lib/inferences/base_inference.h @@ -28,7 +28,6 @@ #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/models/base_model.h" #include "vino_core_lib/slog.h" -#include "inference_engine.hpp" #include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" diff --git a/vino_core_lib/include/vino_core_lib/inferences/emotions_detection.h b/vino_core_lib/include/vino_core_lib/inferences/emotions_detection.h index 56ce404c..1e951a3e 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/emotions_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/emotions_detection.h @@ -28,7 +28,6 @@ #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/models/emotion_detection_model.h" -#include "inference_engine.hpp" #include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" diff --git a/vino_core_lib/include/vino_core_lib/inferences/face_detection.h b/vino_core_lib/include/vino_core_lib/inferences/face_detection.h index d50b8ec4..9c9cfb3e 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/face_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/face_detection.h @@ -34,7 +34,7 @@ #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/inferences/object_detection.h" #include "vino_core_lib/models/face_detection_model.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace diff --git a/vino_core_lib/include/vino_core_lib/inferences/face_reidentification.h b/vino_core_lib/include/vino_core_lib/inferences/face_reidentification.h index 562661cc..5af636c5 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/face_reidentification.h +++ b/vino_core_lib/include/vino_core_lib/inferences/face_reidentification.h @@ -25,7 +25,6 @@ #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/inferences/base_reidentification.h" -#include "inference_engine.hpp" #include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace diff --git a/vino_core_lib/include/vino_core_lib/inferences/head_pose_detection.h b/vino_core_lib/include/vino_core_lib/inferences/head_pose_detection.h index 7209eec0..a2ba6be3 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/head_pose_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/head_pose_detection.h @@ -28,7 +28,6 @@ #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/models/head_pose_detection_model.h" -#include "inference_engine.hpp" #include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" diff --git a/vino_core_lib/include/vino_core_lib/inferences/landmarks_detection.h b/vino_core_lib/include/vino_core_lib/inferences/landmarks_detection.h index 6df6f32c..0460eb98 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/landmarks_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/landmarks_detection.h @@ -24,7 +24,6 @@ #include "vino_core_lib/models/landmarks_detection_model.h" #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" -#include "inference_engine.hpp" #include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace diff --git a/vino_core_lib/include/vino_core_lib/inferences/license_plate_detection.h b/vino_core_lib/include/vino_core_lib/inferences/license_plate_detection.h index 5aad30fe..c7e4c43c 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/license_plate_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/license_plate_detection.h @@ -24,7 +24,7 @@ #include "vino_core_lib/models/license_plate_detection_model.h" #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace namespace vino_core_lib diff --git a/vino_core_lib/include/vino_core_lib/inferences/object_detection.h b/vino_core_lib/include/vino_core_lib/inferences/object_detection.h index 89baa8ef..6bb957c0 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/object_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/object_detection.h @@ -29,7 +29,7 @@ #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/inferences/base_filter.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace namespace vino_core_lib diff --git a/vino_core_lib/include/vino_core_lib/inferences/object_segmentation.h b/vino_core_lib/include/vino_core_lib/inferences/object_segmentation.h index ddacd48e..e9b877f2 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/object_segmentation.h +++ b/vino_core_lib/include/vino_core_lib/inferences/object_segmentation.h @@ -27,7 +27,7 @@ #include "vino_core_lib/models/object_segmentation_model.h" #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace namespace vino_core_lib diff --git a/vino_core_lib/include/vino_core_lib/inferences/person_attribs_detection.h b/vino_core_lib/include/vino_core_lib/inferences/person_attribs_detection.h index 726bafa3..3404f5e6 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/person_attribs_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/person_attribs_detection.h @@ -24,7 +24,7 @@ #include "vino_core_lib/models/person_attribs_detection_model.h" #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace namespace vino_core_lib diff --git a/vino_core_lib/include/vino_core_lib/inferences/person_reidentification.h b/vino_core_lib/include/vino_core_lib/inferences/person_reidentification.h index da7a4419..3a9beff5 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/person_reidentification.h +++ b/vino_core_lib/include/vino_core_lib/inferences/person_reidentification.h @@ -25,7 +25,7 @@ #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" #include "vino_core_lib/inferences/base_reidentification.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace namespace vino_core_lib diff --git a/vino_core_lib/include/vino_core_lib/inferences/vehicle_attribs_detection.h b/vino_core_lib/include/vino_core_lib/inferences/vehicle_attribs_detection.h index c95eb942..4b38011e 100644 --- a/vino_core_lib/include/vino_core_lib/inferences/vehicle_attribs_detection.h +++ b/vino_core_lib/include/vino_core_lib/inferences/vehicle_attribs_detection.h @@ -24,7 +24,7 @@ #include "vino_core_lib/models/vehicle_attribs_detection_model.h" #include "vino_core_lib/engines/engine.h" #include "vino_core_lib/inferences/base_inference.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" // namespace namespace vino_core_lib diff --git a/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h b/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h index 99e801d0..208a8293 100644 --- a/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h +++ b/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h @@ -26,7 +26,7 @@ #include #include -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "vino_core_lib/slog.h" namespace Models @@ -90,7 +90,7 @@ class ModelAttribute slog::info << "--------------------------------" << slog::endl; } - virtual bool updateLayerProperty(const InferenceEngine::CNNNetwork&) + virtual bool updateLayerProperty(std::shared_ptr&) { return false; } diff --git a/vino_core_lib/include/vino_core_lib/models/base_model.h b/vino_core_lib/include/vino_core_lib/models/base_model.h index 91ed67bd..a4fa0afe 100644 --- a/vino_core_lib/include/vino_core_lib/models/base_model.h +++ b/vino_core_lib/include/vino_core_lib/models/base_model.h @@ -29,8 +29,6 @@ #include #include #include - -#include "inference_engine.hpp" #include "openvino/openvino.hpp" #include "vino_core_lib/models/attributes/base_attribute.h" diff --git a/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h b/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h index b784d23e..a9a7de55 100644 --- a/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h @@ -44,7 +44,6 @@ class EmotionDetectionModel : public BaseModel bool updateLayerProperty(std::shared_ptr&) override; private: - // bool verifyOutputLayer(const InferenceEngine::DataPtr& ptr); std::string input_tensor_name_; std::string output_tensor_name_; diff --git a/vino_core_lib/include/vino_core_lib/models/face_detection_model.h b/vino_core_lib/include/vino_core_lib/models/face_detection_model.h index 82a7d190..8ae714d8 100644 --- a/vino_core_lib/include/vino_core_lib/models/face_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/face_detection_model.h @@ -35,7 +35,6 @@ class FaceDetectionModel : public ObjectDetectionModel { public: FaceDetectionModel(const std::string& label_loc, const std::string& model_loc, int batch_size = 1); - // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; /** * @brief Get the name of this detection model. * @return Name of the model. diff --git a/vino_core_lib/include/vino_core_lib/models/object_detection_yolov2voc_model.h b/vino_core_lib/include/vino_core_lib/models/object_detection_yolov2voc_model.h index 12f18099..1ccd1bde 100644 --- a/vino_core_lib/include/vino_core_lib/models/object_detection_yolov2voc_model.h +++ b/vino_core_lib/include/vino_core_lib/models/object_detection_yolov2voc_model.h @@ -53,7 +53,6 @@ class ObjectDetectionYolov2Model : public ObjectDetectionModel protected: int getEntryIndex(int side, int lcoords, int lclasses, int location, int entry); - // InferenceEngine::InputInfo::Ptr input_info_ = nullptr; }; } // namespace Models #endif // VINO_CORE_LIB__MODELS__OBJECT_DETECTION_YOLOV2VOC_MODEL_H diff --git a/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h b/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h index e9ba4c24..c2fd4411 100644 --- a/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h @@ -30,8 +30,6 @@ class PersonAttribsDetectionModel : public BaseModel { public: PersonAttribsDetectionModel(const std::string& label_loc, const std::string& model_loc, int batch_size = 1); - // inline const std::string getInputName() {return input_;} - // inline const std::string getOutputName() {return output_;} /** * @brief Get the name of this detection model. * @return Name of the model. @@ -39,8 +37,6 @@ class PersonAttribsDetectionModel : public BaseModel const std::string getModelCategory() const override; protected: - // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; - // void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; bool updateLayerProperty(std::shared_ptr&) override; std::string input_; std::string output_; diff --git a/vino_core_lib/include/vino_core_lib/models/person_reidentification_model.h b/vino_core_lib/include/vino_core_lib/models/person_reidentification_model.h index 972ccf7c..0b471176 100644 --- a/vino_core_lib/include/vino_core_lib/models/person_reidentification_model.h +++ b/vino_core_lib/include/vino_core_lib/models/person_reidentification_model.h @@ -46,8 +46,6 @@ class PersonReidentificationModel : public BaseModel protected: bool updateLayerProperty(std::shared_ptr&) override; - // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; - // void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; std::string input_; std::string output_; }; diff --git a/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h b/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h index 300a6fb2..33051fe0 100644 --- a/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h @@ -49,8 +49,6 @@ class VehicleAttribsDetectionModel : public BaseModel const std::string getModelCategory() const override; protected: - // void checkLayerProperty(const InferenceEngine::CNNNetReader::Ptr &) override; - // void setLayerProperty(InferenceEngine::CNNNetReader::Ptr) override; bool updateLayerProperty(std::shared_ptr&) override; std::string input_; std::string color_output_; diff --git a/vino_core_lib/src/engines/engine_manager.cpp b/vino_core_lib/src/engines/engine_manager.cpp index b9d1335b..dfd5933f 100644 --- a/vino_core_lib/src/engines/engine_manager.cpp +++ b/vino_core_lib/src/engines/engine_manager.cpp @@ -22,7 +22,7 @@ #include "vino_core_lib/models/base_model.h" #include "vino_core_lib/utils/version_info.hpp" #include -#include +#include #if (defined(USE_OLD_E_PLUGIN_API)) #include #endif diff --git a/vino_core_lib/src/inferences/emotions_detection.cpp b/vino_core_lib/src/inferences/emotions_detection.cpp index c3cb600b..c0a4d90f 100644 --- a/vino_core_lib/src/inferences/emotions_detection.cpp +++ b/vino_core_lib/src/inferences/emotions_detection.cpp @@ -77,16 +77,11 @@ bool vino_core_lib::EmotionsDetection::fetchResults() if (!can_fetch) return false; int label_length = static_cast(valid_model_->getLabels().size()); - /// std::cout << "label_length = " << label_length << std::endl; std::string output_name = valid_model_->getOutputName(); - /// std::cout << "output_name = " << output_name << std::endl; ov::Tensor emotions_tensor = getEngine()->getRequest().get_tensor(output_name); - /** emotions vector must have the same size as number of channels - in model output. Default output format is NCHW so we check index 1 */ - ov::Shape shape = emotions_tensor.get_shape(); int64 num_of_channels = shape[1]; - /// std::cout << "num_of_channels " << num_of_channels << std::endl; + if (num_of_channels != label_length) { slog::err << "Output size (" << num_of_channels << ") of the Emotions Recognition network is not equal " @@ -97,8 +92,6 @@ bool vino_core_lib::EmotionsDetection::fetchResults() std::to_string(label_length) + ")"); } - /** we identify an index of the most probable emotion in output array - for idx image to return appropriate emotion name */ auto emotions_values = emotions_tensor.data(); for (unsigned int idx = 0; idx < results_.size(); ++idx) { diff --git a/vino_core_lib/src/inferences/license_plate_detection.cpp b/vino_core_lib/src/inferences/license_plate_detection.cpp index 8dd08695..16076182 100644 --- a/vino_core_lib/src/inferences/license_plate_detection.cpp +++ b/vino_core_lib/src/inferences/license_plate_detection.cpp @@ -47,8 +47,6 @@ void vino_core_lib::LicensePlateDetection::fillSeqBlob() ov::Tensor seq_tensor = getEngine()->getRequest().get_tensor( valid_model_->getSeqInputName()); int max_sequence_size = seq_tensor.get_shape()[0]; - // second input is sequence, which is some relic from the training - // it should have the leading 0.0f and rest 1.0f float * tensor_data = seq_tensor.data(); tensor_data[0] = 0.0f; std::fill(tensor_data + 1, tensor_data + max_sequence_size, 1.0f); diff --git a/vino_core_lib/src/inferences/object_detection.cpp b/vino_core_lib/src/inferences/object_detection.cpp index 5947f675..9aef1017 100644 --- a/vino_core_lib/src/inferences/object_detection.cpp +++ b/vino_core_lib/src/inferences/object_detection.cpp @@ -68,10 +68,6 @@ bool vino_core_lib::ObjectDetection::enqueue(const cv::Mat& frame, const cv::Rec return false; } - // nonsense!! - // Result r(input_frame_loc); - // results_.clear(); - // results_.emplace_back(r); enqueued_frames_ += 1; return true; } diff --git a/vino_core_lib/src/inferences/object_segmentation.cpp b/vino_core_lib/src/inferences/object_segmentation.cpp index 4c81a0c1..29589aca 100644 --- a/vino_core_lib/src/inferences/object_segmentation.cpp +++ b/vino_core_lib/src/inferences/object_segmentation.cpp @@ -163,14 +163,12 @@ bool vino_core_lib::ObjectSegmentation::fetchResults() { colored_mask.at(rowId, colId)[ch] = colors_[classId][ch]; } - // classId = static_cast(predictions[rowId * output_w + colId]); } else { for (int chId = 0; chId < output_des; ++chId) { float prob = detections[chId * output_h * output_w + rowId * output_w + colId]; - // float prob = predictions[chId * output_h * output_w + rowId * output_w+ colId]; if (prob > maxProb) { classId = chId; diff --git a/vino_core_lib/src/inferences/person_attribs_detection.cpp b/vino_core_lib/src/inferences/person_attribs_detection.cpp index f3b8574c..fda7b304 100644 --- a/vino_core_lib/src/inferences/person_attribs_detection.cpp +++ b/vino_core_lib/src/inferences/person_attribs_detection.cpp @@ -63,30 +63,7 @@ bool vino_core_lib::PersonAttribsDetection::submitRequest() { return vino_core_lib::BaseInference::submitRequest(); } -/* -bool vino_core_lib::PersonAttribsDetection::fetchResults() -{ - bool can_fetch = vino_core_lib::BaseInference::fetchResults(); - if (!can_fetch) {return false;} - bool found_result = false; - InferenceEngine::InferRequest::Ptr request = getEngine()->getRequest(); - std::string output = valid_model_->getOutputName(); - const float * output_values = request->GetBlob(output)->buffer().as(); - int net_attrib_length = net_attributes_.size(); - for (int i = 0; i < getResultsLength(); i++) { - results_[i].male_probability_ = output_values[i * net_attrib_length]; - std::string attrib = ""; - for (int j = 1; j < net_attrib_length; j++) { - attrib += (output_values[i * net_attrib_length + j] > attribs_confidence_) ? - net_attributes_[j] + ", " : ""; - } - results_[i].attributes_ = attrib; - found_result = true; - } - if (!found_result) {results_.clear();} - return true; -} -*/ + bool vino_core_lib::PersonAttribsDetection::fetchResults() { bool can_fetch = vino_core_lib::BaseInference::fetchResults(); @@ -101,9 +78,6 @@ bool vino_core_lib::PersonAttribsDetection::fetchResults() std::string top_output = valid_model_->getOutputName("top_output_"); std::string bottom_output = valid_model_->getOutputName("bottom_output_"); - /*auto attri_values = request->GetBlob(attribute_output)->buffer().as(); - auto top_values = request->GetBlob(top_output)->buffer().as(); - auto bottom_values = request->GetBlob(bottom_output)->buffer().as();*/ ov::Tensor attrib_tensor = request.get_tensor(attribute_output); ov::Tensor top_tensor = request.get_tensor(top_output); ov::Tensor bottom_tensor = request.get_tensor(bottom_output); diff --git a/vino_core_lib/src/inferences/vehicle_attribs_detection.cpp b/vino_core_lib/src/inferences/vehicle_attribs_detection.cpp index ff9ecd65..700a1f9c 100644 --- a/vino_core_lib/src/inferences/vehicle_attribs_detection.cpp +++ b/vino_core_lib/src/inferences/vehicle_attribs_detection.cpp @@ -72,8 +72,6 @@ bool vino_core_lib::VehicleAttribsDetection::fetchResults() } bool found_result = false; ov::InferRequest request = getEngine()->getRequest(); - // std::string color_name = valid_model_->getColorOutputName(); - // std::string type_name = valid_model_->getTypeOutputName(); std::string color_name = valid_model_->getOutputName("color_output_"); std::string type_name = valid_model_->getOutputName("type_output_"); const float * color_values = request.get_tensor(color_name).data(); diff --git a/vino_core_lib/src/inputs/image_input.cpp b/vino_core_lib/src/inputs/image_input.cpp index c802ce86..df043458 100644 --- a/vino_core_lib/src/inputs/image_input.cpp +++ b/vino_core_lib/src/inputs/image_input.cpp @@ -52,7 +52,6 @@ bool Input::Image::read(cv::Mat* frame) return false; } *frame = image_; - // setHeader("image_frame"); return true; } diff --git a/vino_core_lib/src/inputs/image_topic.cpp b/vino_core_lib/src/inputs/image_topic.cpp index d5aedef9..f06909da 100644 --- a/vino_core_lib/src/inputs/image_topic.cpp +++ b/vino_core_lib/src/inputs/image_topic.cpp @@ -45,13 +45,8 @@ bool Input::ImageTopic::initialize(size_t width, size_t height) void Input::ImageTopic::cb(const sensor_msgs::Image::ConstPtr& image_msg) { slog::debug << "Receiving a new image from Camera topic." << slog::endl; - // setHeader(image_msg->header); image_ = cv_bridge::toCvCopy(image_msg, "bgr8")->image; - // Suppose Image Topic is sent within BGR order, so the below line would work. - // image_ = cv::Mat(image_msg->height, image_msg->width, CV_8UC3, - // const_cast(&image_msg->data[0]), image_msg->step); - image_count_.increaseCounter(); } @@ -64,7 +59,6 @@ bool Input::ImageTopic::read(cv::Mat* frame) } *frame = image_; - // lockHeader(); image_count_.decreaseCounter(); return true; } diff --git a/vino_core_lib/src/inputs/ip_camera.cpp b/vino_core_lib/src/inputs/ip_camera.cpp index d4977f5f..b6fefe4c 100644 --- a/vino_core_lib/src/inputs/ip_camera.cpp +++ b/vino_core_lib/src/inputs/ip_camera.cpp @@ -43,7 +43,7 @@ bool Input::IpCamera::read(cv::Mat * frame) return false; } cap.grab(); - // setHeader("ip_camera_frame"); + bool retrieved = cap.retrieve(*frame); if (retrieved) { diff --git a/vino_core_lib/src/inputs/realsense_camera.cpp b/vino_core_lib/src/inputs/realsense_camera.cpp index 9bd7c0d1..fac4064a 100644 --- a/vino_core_lib/src/inputs/realsense_camera.cpp +++ b/vino_core_lib/src/inputs/realsense_camera.cpp @@ -46,7 +46,6 @@ bool Input::RealSenseCamera::initialize(size_t width, size_t height) setWidth(width); setHeight(height); - // bypass RealSense's bug: several captured frames after HW is inited are with wrong data. bypassFewFramesOnceInited(); return isInit(); @@ -74,7 +73,6 @@ bool Input::RealSenseCamera::read(cv::Mat* frame) return false; } - // setHeader("realsense_camera_frame"); return true; } diff --git a/vino_core_lib/src/inputs/standard_camera.cpp b/vino_core_lib/src/inputs/standard_camera.cpp index 426badf4..98feac10 100644 --- a/vino_core_lib/src/inputs/standard_camera.cpp +++ b/vino_core_lib/src/inputs/standard_camera.cpp @@ -45,7 +45,6 @@ bool Input::StandardCamera::read(cv::Mat* frame) return false; } cap.grab(); - // setHeader("standard_camera_frame"); return cap.retrieve(*frame); } diff --git a/vino_core_lib/src/inputs/video_input.cpp b/vino_core_lib/src/inputs/video_input.cpp index a4f88ee7..40b72d72 100644 --- a/vino_core_lib/src/inputs/video_input.cpp +++ b/vino_core_lib/src/inputs/video_input.cpp @@ -58,6 +58,5 @@ bool Input::Video::read(cv::Mat* frame) return false; } cap.grab(); - // setHeader("video_frame"); return cap.retrieve(*frame); } diff --git a/vino_core_lib/src/models/base_model.cpp b/vino_core_lib/src/models/base_model.cpp index 903c0524..fc9bb586 100644 --- a/vino_core_lib/src/models/base_model.cpp +++ b/vino_core_lib/src/models/base_model.cpp @@ -37,7 +37,6 @@ Models::BaseModel::BaseModel(const std::string& label_loc, const std::string& mo throw std::logic_error("model file name is empty!"); } - ///net_reader_ = std::make_shared(); } void Models::BaseModel::modelInit() @@ -45,22 +44,17 @@ void Models::BaseModel::modelInit() slog::info << "Loading network files: " << model_loc_ << slog::endl; // Read network model - ///net_reader_->ReadNetwork(model_loc_); net_reader_ = engine.read_model(model_loc_); // Extract model name and load it's weights // remove extension size_t last_index = model_loc_.find_last_of("."); std::string raw_name = model_loc_.substr(0, last_index); - ///std::string bin_file_name = raw_name + ".bin"; - ///net_reader_->ReadWeights(bin_file_name); // Read labels (if any) std::string label_file_name = label_loc_.substr(0, last_index); - //std::string label_file_name = raw_name + ".labels"; loadLabelsFromFile(label_loc_); // Set batch size to given max_batch_size_ slog::info << "Batch size is set to " << max_batch_size_ << slog::endl; - // net_reader_.setBatchSize(max_batch_size_); updateLayerProperty(net_reader_); } diff --git a/vino_core_lib/src/models/emotion_detection_model.cpp b/vino_core_lib/src/models/emotion_detection_model.cpp index 41fd0f08..c4e1410d 100644 --- a/vino_core_lib/src/models/emotion_detection_model.cpp +++ b/vino_core_lib/src/models/emotion_detection_model.cpp @@ -68,7 +68,7 @@ bool Models::EmotionDetectionModel::updateLayerProperty(std::shared_ptr()); } const std::string Models::ObjectDetectionSSDModel::getModelCategory() const @@ -114,7 +113,6 @@ bool Models::ObjectDetectionSSDModel::fetchResults(const std::shared_ptrinputs(); slog::debug<<"input size"<outputs(); if (outputs_info.size() != 1) { - // throw std::runtime_error("Demo supports topologies only with 1 output"); slog::warn << "This inference sample should have only one output, but we got" << std::to_string(outputs_info.size()) << "outputs" << slog::endl; return false; @@ -136,14 +134,12 @@ bool Models::ObjectSegmentationModel::updateLayerProperty(std::shared_ptrinput().get_shape(); std::vector &in_size_vector = input_shape_; slog::debug<<"channel size"<first.c_str()); - ///const InferenceEngine::CNNLayerPtr output_layer = network.getLayerByName(getOutputName("detection").c_str()); -// const int num_classes = output_layer->GetParamAsInt("num_classes"); -// slog::info << "Checking Object Segmentation output ... num_classes=" << num_classes << slog::endl; - #if 0 if (getLabels().size() != num_classes) { @@ -203,14 +192,7 @@ bool Models::ObjectSegmentationModel::updateLayerProperty(std::shared_ptr(output_dims[2])); - slog::info << "max proposal count is: " << getMaxProposalCount() << slog::endl; - auto object_size = static_cast(output_dims[3]); - setObjectSize(object_size); - - slog::debug << "model size" << output_dims.size() << slog::endl;*/ + printAttribute(); slog::info << "This model is SSDNet-like, Layer Property updated!" << slog::endl; return true; diff --git a/vino_core_lib/src/models/person_attribs_detection_model.cpp b/vino_core_lib/src/models/person_attribs_detection_model.cpp index d0f584bd..48d5d5dc 100644 --- a/vino_core_lib/src/models/person_attribs_detection_model.cpp +++ b/vino_core_lib/src/models/person_attribs_detection_model.cpp @@ -50,7 +50,6 @@ bool Models::PersonAttribsDetectionModel::updateLayerProperty(std::shared_ptrname; addOutputInfo("top_output_", output_info_map[1].get_any_name()); addOutputInfo("bottom_output_", output_info_map[0].get_any_name()); printAttribute(); diff --git a/vino_core_lib/src/outputs/base_output.cpp b/vino_core_lib/src/outputs/base_output.cpp index f40572a5..a7db68f9 100644 --- a/vino_core_lib/src/outputs/base_output.cpp +++ b/vino_core_lib/src/outputs/base_output.cpp @@ -19,7 +19,6 @@ void Outputs::BaseOutput::setPipeline(Pipeline* const pipeline) { - // pipeline->printPipeline(); pipeline_ = pipeline; } diff --git a/vino_core_lib/src/outputs/image_window_output.cpp b/vino_core_lib/src/outputs/image_window_output.cpp index 7b5a6aee..0d44267c 100644 --- a/vino_core_lib/src/outputs/image_window_output.cpp +++ b/vino_core_lib/src/outputs/image_window_output.cpp @@ -37,7 +37,6 @@ Outputs::ImageWindowOutput::ImageWindowOutput(const std::string& output_name, in void Outputs::ImageWindowOutput::feedFrame(const cv::Mat& frame) { - // frame_ = frame; frame_ = frame.clone(); if (camera_matrix_.empty()) { diff --git a/vino_core_lib/src/outputs/ros_service_output.cpp b/vino_core_lib/src/outputs/ros_service_output.cpp index 3798f433..5531e6f8 100644 --- a/vino_core_lib/src/outputs/ros_service_output.cpp +++ b/vino_core_lib/src/outputs/ros_service_output.cpp @@ -130,6 +130,5 @@ void Outputs::RosServiceOutput::clearData() age_gender_topic_ = nullptr; emotions_topic_ = nullptr; headpose_topic_ = nullptr; - // segmented_object_topic_ = nullptr; person_reid_topic_ = nullptr; } diff --git a/vino_core_lib/src/outputs/ros_topic_output.cpp b/vino_core_lib/src/outputs/ros_topic_output.cpp index c2a881ea..fa6fa147 100644 --- a/vino_core_lib/src/outputs/ros_topic_output.cpp +++ b/vino_core_lib/src/outputs/ros_topic_output.cpp @@ -74,7 +74,6 @@ void Outputs::RosTopicOutput::accept(const std::vector"; auto loc = r.getLocation(); attribs.roi.x_offset = loc.x; attribs.roi.y_offset = loc.y; @@ -92,7 +91,6 @@ void Outputs::RosTopicOutput::accept(const std::vector"; auto loc = r.getLocation(); plate.roi.x_offset = loc.x; plate.roi.y_offset = loc.y; @@ -109,7 +107,6 @@ void Outputs::RosTopicOutput::accept(const std::vector"; auto loc = r.getLocation(); person_attrib.roi.x_offset = loc.x; person_attrib.roi.y_offset = loc.y; @@ -126,7 +123,6 @@ void Outputs::RosTopicOutput::accept(const std::vector"; auto loc = r.getLocation(); face.roi.x_offset = loc.x; face.roi.y_offset = loc.y; @@ -143,7 +139,6 @@ void Outputs::RosTopicOutput::accept(const std::vector"; auto loc = r.getLocation(); person.roi.x_offset = loc.x; person.roi.y_offset = loc.y; @@ -160,7 +155,6 @@ void Outputs::RosTopicOutput::accept(const std::vector"; auto loc = r.getLocation(); object.roi.x_offset = loc.x; object.roi.y_offset = loc.y; @@ -286,7 +280,6 @@ void Outputs::RosTopicOutput::accept(const std::vector"; auto loc = r.getLocation(); landmark.roi.x_offset = loc.x; landmark.roi.y_offset = loc.y; @@ -307,7 +300,6 @@ void Outputs::RosTopicOutput::accept(const std::vectorgetInputDevice()->getLockedHeader(); if (vehicle_attribs_topic_ != nullptr) { vino_people_msgs::VehicleAttribsStamped vehicle_attribs_msg; @@ -342,7 +334,6 @@ void Outputs::RosTopicOutput::handleOutput() } if (segmented_objects_topic_ != nullptr) { - // slog::info << "publishing faces outputs." << slog::endl; vino_people_msgs::ObjectsInMasks segmented_msg; segmented_msg.header = header; segmented_msg.objects_vector.swap(segmented_objects_topic_->objects_vector); diff --git a/vino_core_lib/src/outputs/rviz_output.cpp b/vino_core_lib/src/outputs/rviz_output.cpp index 6ac58b5a..e750d6ac 100644 --- a/vino_core_lib/src/outputs/rviz_output.cpp +++ b/vino_core_lib/src/outputs/rviz_output.cpp @@ -96,7 +96,6 @@ void Outputs::RvizOutput::handleOutput() image_window_output_->decorateFrame(); cv::Mat frame = image_window_output_->getFrame(); std_msgs::Header header = getHeader(); - // std_msgs::Header header = getPipeline()->getInputDevice()->getLockedHeader(); std::shared_ptr cv_ptr = std::make_shared(header, "bgr8", frame); sensor_msgs::Image image_msg; image_topic_ = cv_ptr->toImageMsg(); diff --git a/vino_core_lib/src/pipeline.cpp b/vino_core_lib/src/pipeline.cpp index 8f6e0f32..be6897a0 100644 --- a/vino_core_lib/src/pipeline.cpp +++ b/vino_core_lib/src/pipeline.cpp @@ -196,15 +196,12 @@ void Pipeline::runOnce() if (!input_device_->read(&frame_)) { - // throw std::logic_error("Failed to get frame from cv::VideoCapture"); - // slog::warn << "Failed to get frame from input_device." << slog::endl; return; // do nothing if now frame read out } width_ = frame_.cols; height_ = frame_.rows; slog::debug << "DEBUG: in Pipeline run process..." << slog::endl; - // auto t0 = std::chrono::high_resolution_clock::now(); for (auto pos = next_.equal_range(input_device_name_); pos.first != pos.second; ++pos.first) { std::string detection_name = pos.first->second; @@ -224,13 +221,9 @@ void Pipeline::runOnce() std::unique_lock lock(counter_mutex_); cv_.wait(lock, [self = this]() { return self->counter_ == 0; }); - // auto t1 = std::chrono::high_resolution_clock::now(); - // typedef std::chrono::duration> ms; - slog::debug << "DEBUG: in Pipeline run process...handleOutput" << slog::endl; for (auto& pair : name_to_output_map_) { - // slog::info << "Handling Output ..." << pair.first << slog::endl; pair.second->handleOutput(); } } diff --git a/vino_core_lib/src/pipeline_manager.cpp b/vino_core_lib/src/pipeline_manager.cpp index a96fdb14..3d5022d3 100644 --- a/vino_core_lib/src/pipeline_manager.cpp +++ b/vino_core_lib/src/pipeline_manager.cpp @@ -341,10 +341,6 @@ PipelineManager::createObjectDetection(const Params::ParamManager::InferenceRawD { object_detection_model = std::make_shared(infer.label, infer.model, infer.batch); } - // if (infer.model_type == kInferTpye_ObjectDetectionTypeYolov2voc) - // { - // object_detection_model = std::make_shared(infer.model, infer.batch); - // } slog::debug << "for test in createObjectDetection(), Created SSDModel" << slog::endl; object_inference_ptr = std::make_shared( diff --git a/vino_sample/src/image_object_server.cpp b/vino_sample/src/image_object_server.cpp index ee1294e7..93a8a93a 100644 --- a/vino_sample/src/image_object_server.cpp +++ b/vino_sample/src/image_object_server.cpp @@ -54,7 +54,7 @@ #include "vino_core_lib/pipeline.h" #include "vino_core_lib/pipeline_manager.h" #include "vino_core_lib/slog.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" #include "vino_sample/utility.hpp" diff --git a/vino_sample/src/image_people_server.cpp b/vino_sample/src/image_people_server.cpp index 2b183da8..480f75da 100644 --- a/vino_sample/src/image_people_server.cpp +++ b/vino_sample/src/image_people_server.cpp @@ -54,7 +54,7 @@ #include "vino_core_lib/pipeline.h" #include "vino_core_lib/pipeline_manager.h" #include "vino_core_lib/slog.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" #include "vino_sample/utility.hpp" diff --git a/vino_sample/src/image_reidentification_server.cpp b/vino_sample/src/image_reidentification_server.cpp index 39e0aa42..3ff93d48 100644 --- a/vino_sample/src/image_reidentification_server.cpp +++ b/vino_sample/src/image_reidentification_server.cpp @@ -54,7 +54,7 @@ #include "vino_core_lib/pipeline.h" #include "vino_core_lib/pipeline_manager.h" #include "vino_core_lib/slog.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" #include "vino_sample/utility.hpp" #include diff --git a/vino_sample/src/image_segmentation_server.cpp b/vino_sample/src/image_segmentation_server.cpp index f4c3c8de..97368624 100644 --- a/vino_sample/src/image_segmentation_server.cpp +++ b/vino_sample/src/image_segmentation_server.cpp @@ -54,7 +54,7 @@ #include "vino_core_lib/pipeline.h" #include "vino_core_lib/pipeline_manager.h" #include "vino_core_lib/slog.h" -#include "inference_engine.hpp" +#include "openvino/openvino.hpp" #include "opencv2/opencv.hpp" #include "vino_sample/utility.hpp" From 374b648d165fa6a435de7d98a576dc8437c35e83 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Tue, 18 Oct 2022 09:24:34 +0800 Subject: [PATCH 22/40] Add NCHW and NHWC layout compatibility for object_segmentation and object_detection_ssd model --- .../src/inferences/object_segmentation.cpp | 29 +++++++++---------- .../src/models/object_detection_ssd_model.cpp | 15 ++++++++++ .../src/models/object_segmentation_model.cpp | 29 ++++++++++++------- vino_core_lib/src/pipeline_manager.cpp | 8 ++--- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/vino_core_lib/src/inferences/object_segmentation.cpp b/vino_core_lib/src/inferences/object_segmentation.cpp index 29589aca..f4c5e0e3 100644 --- a/vino_core_lib/src/inferences/object_segmentation.cpp +++ b/vino_core_lib/src/inferences/object_segmentation.cpp @@ -122,21 +122,20 @@ bool vino_core_lib::ObjectSegmentation::fetchResults() ov::Shape out_shape = output_tensor.get_shape(); ov::Tensor masks_tensor = infer_request.get_tensor(detection_output.c_str()); const auto masks_data = masks_tensor.data(); - int output_des, output_h, output_w; - switch(out_shape.size()) { - case 3: - output_des = 0; - output_h = out_shape[1]; - output_w = out_shape[2]; - break; - case 4: - output_des = out_shape[1]; - output_h = out_shape[2]; - output_w = out_shape[3]; - break; - default: - throw std::runtime_error("Unexpected output blob shape. Only 4D and 3D output blobs are" - "supported."); + + size_t output_w, output_h, output_des, output_extra = 0; + if (out_shape.size() == 3) { + output_w = out_shape[2]; + output_h = out_shape[1]; + output_des = out_shape[0]; + } else if (out_shape.size() == 4) { + output_w = out_shape[3]; + output_h = out_shape[2]; + output_des = out_shape[1]; + output_extra = out_shape[0]; + } else { + slog::warn << "unexpected output shape: " <outputs(); diff --git a/vino_core_lib/src/models/object_segmentation_model.cpp b/vino_core_lib/src/models/object_segmentation_model.cpp index 96256dc1..e375b557 100644 --- a/vino_core_lib/src/models/object_segmentation_model.cpp +++ b/vino_core_lib/src/models/object_segmentation_model.cpp @@ -85,7 +85,7 @@ bool Models::ObjectSegmentationModel::matToBlob(const cv::Mat& orig_image, const return false; } - ov::Tensor input_tensor{ov::element::u8, input_shape_, orig_image.data}; + ov::Tensor input_tensor = ov::Tensor(ov::element::u8, {1, height, width, channels}, orig_image.data); engine->getRequest().set_tensor(input_tensor_name_, input_tensor); return true; @@ -113,11 +113,23 @@ bool Models::ObjectSegmentationModel::updateLayerProperty(std::shared_ptrinput().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); - const ov::Layout tensor_layout{"NCHW"}; + ov::Layout tensor_layout = ov::Layout("NHWC"); + ov::Layout expect_layout = ov::Layout("NCHW"); + ov::Shape input_shape = net_reader->input().get_shape(); + if (input_shape[1] == 3) + expect_layout = ov::Layout("NCHW"); + else if (input_shape[3] == 3) + expect_layout = ov::Layout("NHWC"); + else + slog::warn << "unexpect input shape " << input_shape << slog::endl; + input_info.tensor(). set_element_type(ov::element::u8). - set_layout(tensor_layout); - input_info.preprocess().resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR); + set_layout(tensor_layout). + set_spatial_dynamic_shape(); + input_info.preprocess(). + convert_layout(expect_layout). + resize(ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR); addInputInfo("input", input_tensor_name_); @@ -135,13 +147,10 @@ bool Models::ObjectSegmentationModel::updateLayerProperty(std::shared_ptrinput().get_shape(); - - std::vector &in_size_vector = input_shape_; - slog::debug<<"channel size"< &in_size_vector = input_shape; + slog::debug<<"dimensional"< Date: Tue, 18 Oct 2022 14:05:15 +0800 Subject: [PATCH 23/40] clean code --- vino_core_lib/CMakeLists.txt | 2 +- vino_core_lib/include/vino_core_lib/engines/engine.h | 4 ++-- vino_core_lib/include/vino_core_lib/engines/engine_manager.h | 4 ++-- vino_core_lib/include/vino_core_lib/utils/version_info.hpp | 2 +- vino_sample/CMakeLists.txt | 2 +- vino_sample/src/pipeline_with_params.cpp | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/vino_core_lib/CMakeLists.txt b/vino_core_lib/CMakeLists.txt index e1c5da63..726951ec 100644 --- a/vino_core_lib/CMakeLists.txt +++ b/vino_core_lib/CMakeLists.txt @@ -69,7 +69,7 @@ include_directories ( include ${catkin_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/include - ${INTEL_OPENVINO_DIR} + ${OPENVINO_DIR} ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/gflags/include ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/extension/include ${CMAKE_CURRENT_SOURCE_DIR}/../vino_param_lib/include diff --git a/vino_core_lib/include/vino_core_lib/engines/engine.h b/vino_core_lib/include/vino_core_lib/engines/engine.h index 45d315f1..cfc45ec2 100644 --- a/vino_core_lib/include/vino_core_lib/engines/engine.h +++ b/vino_core_lib/include/vino_core_lib/engines/engine.h @@ -38,7 +38,7 @@ class Engine public: #if (defined(USE_OLD_E_PLUGIN_API)) /** - * DEPRECATED! instead of using Engine(InferenceEngine::InferRequest::Ptr &) + * DEPRECATED! instead of using Engine(ov::InferRequest &) * @brief Create an NetworkEngine instance * from a inference plugin and an inference network. */ @@ -46,7 +46,7 @@ class Engine #endif /** - * @brief Using an Inference Request to initialize the inference Engine. + * @brief Using an Inference Request to initialize the OpenVINO Engine. */ Engine(ov::InferRequest &); /** diff --git a/vino_core_lib/include/vino_core_lib/engines/engine_manager.h b/vino_core_lib/include/vino_core_lib/engines/engine_manager.h index f3314e2c..dceca2e5 100644 --- a/vino_core_lib/include/vino_core_lib/engines/engine_manager.h +++ b/vino_core_lib/include/vino_core_lib/engines/engine_manager.h @@ -29,13 +29,13 @@ namespace Engines { /** * @class EngineManager - * @brief This class is used to create and manage Inference engines. + * @brief This class is used to create and manage OpenVINO engines. */ class EngineManager { public: /** - * @brief Create InferenceEngine instance by given Engine Name and Network. + * @brief Create OpenVINO instance by given Engine Name and Network. * @return The shared pointer of created Engine instance. */ std::shared_ptr createEngine(const std::string&, const std::shared_ptr&); diff --git a/vino_core_lib/include/vino_core_lib/utils/version_info.hpp b/vino_core_lib/include/vino_core_lib/utils/version_info.hpp index 65367668..7c30354f 100644 --- a/vino_core_lib/include/vino_core_lib/utils/version_info.hpp +++ b/vino_core_lib/include/vino_core_lib/utils/version_info.hpp @@ -13,7 +13,7 @@ // limitations under the License. // -// @brief a header file with version information about Inference Engine. +// @brief a header file with version information about OpenVINO. // @file version_info.hpp // diff --git a/vino_sample/CMakeLists.txt b/vino_sample/CMakeLists.txt index 50f499b3..a8532f92 100644 --- a/vino_sample/CMakeLists.txt +++ b/vino_sample/CMakeLists.txt @@ -86,7 +86,7 @@ add_dependencies(pipeline_with_params ${vino_core_lib_TARGETS} ) -set(OpenCV_LIBRARIES openvino::runtime) +set(OpenVINO_LIBRARIES openvino::runtime) target_link_libraries(pipeline_with_params ${catkin_LIBRARIES} gflags diff --git a/vino_sample/src/pipeline_with_params.cpp b/vino_sample/src/pipeline_with_params.cpp index 887d0b00..6ef94ef4 100644 --- a/vino_sample/src/pipeline_with_params.cpp +++ b/vino_sample/src/pipeline_with_params.cpp @@ -91,7 +91,7 @@ int main(int argc, char** argv) try { - std::cout << "OpenVINO: " << ov::get_openvino_version << std::endl; + std::cout << "OpenVINO: " << ov::get_openvino_version() << std::endl; // ----- Parsing and validation of input args----------------------- if (!parseAndCheckCommandLine(argc, argv)) From dc8e99698a3949360bcdb1b81ef59838b27156e0 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Wed, 19 Oct 2022 13:51:57 +0800 Subject: [PATCH 24/40] update docs for ov2.0 --- ..._started_with_Melodic_Ubuntu18.04_ov2.0.md | 196 ++++++++++++++++++ ...g_started_with_Noetic_Ubuntu20.04_ov2.0.md | 194 +++++++++++++++++ 2 files changed, 390 insertions(+) create mode 100644 doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md create mode 100644 doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md diff --git a/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md b/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md new file mode 100644 index 00000000..1e3b3de1 --- /dev/null +++ b/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md @@ -0,0 +1,196 @@ +# ROS_MELODIC_OpenVINO_Toolkit + +**NOTE:** +Below steps have been tested on **Ubuntu 20.04**. + +## 1. Environment Setup +* Install ROS Melodic ([guide](http://wiki.ros.org/melodic/Installation/Ubuntu)) + +* Install Intel® OpenVINO™ Toolkit Version: 2022.1 ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)). +* * Install from an achive file([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)). +Tips: Both runtime and development tool are needed, `pip` is recommended for installing the development tool ([guide](https://docs.openvino.ai/latest/openvino_docs_install_guides_install_dev_tools.html)). +* * Install from source code([guide](https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux)). + +* Install Intel® RealSense ™ SDK ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)). + +* Install Dependency + * Install gflags + ``` + sudo apt-get install -y libgflags-dev + ``` + +## 2. Build and Installation +* Install ROS_OpenVINO_Toolkit packages +``` +mkdir -p ~/catkin_ws/src +cd ~/catkin_ws/src +git clone https://github.com/intel/ros_openvino_toolkit -b melodic +git clone https://github.com/intel/object_msgs +git clone https://github.com/ros-perception/vision_opencv.git -b melodic +git clone https://github.com/IntelRealSense/realsense-ros.git +cd realsense-ros +git checkout `git tag | sort -V | grep -P "^2.\d+\.\d+" | tail -1` +``` +* Install dependencies +``` +sudo apt-get install ros-melodic-ddynamic-reconfigure +``` +* Build package +``` +source /opt/ros/melodic/setup.bash +source /setupvars.sh +cd ~/catkin_ws +catkin_make_isolated +source ./devel_isolated/setup.bash +``` + +## 3. Running the Demo +### Install OpenVINO 2022.1 by source code +* See all available models +``` +cd /openvino/thirdparty/open_model_zoo/tools/model_tools +sudo python3 downloader.py --print_all +``` + +* Download the optimized Intermediate Representation (IR) of model (execute once): +``` +cd /openvino/thirdparty/open_model_zoo/tools/model_tools +sudo python3 downloader.py --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output +sudo python3 downloader.py --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output +sudo python3 downloader.py --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output +sudo python3 downloader.py --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output +sudo python3 downloader.py --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output +sudo python3 downloader.py --name person-reidentification-retail-0277 --output_dir /opt/openvino_toolkit/models/person-reidentification/output +sudo python3 downloader.py --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output +sudo python3 downloader.py --name face-reidentification-retail-0095 --output_dir /opt/openvino_toolkit/models/face-reidentification-retail/output +sudo python3 downloader.py --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recognition/output +sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output +``` + +* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi)need to be converted to intermediate representation (For example the model for object detection) + * mobilenet-ssd (replaced ssd_mobilenet_v2_coco to mobilenet-ssd for object detection in ov2022) + ``` + cd /openvino/thirdparty/open_model_zoo/tools/model_tools + sudo python3 downloader.py --name mobilenet-ssd --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output + sudo python3 converter.py --name mobilenet-ssd -d /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/ -o /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert + ``` + * vehicle-license-plate-detection-barrier-0123 + ``` + cd /openvino/thirdparty/open_model_zoo/tools/model_tools + sudo python3 downloader.py --name vehicle-license-plate-detection-barrier-0123 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output + sudo python3 converter.py --name vehicle-license-plate-detection-barrier-0123 -d /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output -o /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output/convert + ``` + * deeplabv3 + ``` + cd /openvino/thirdparty/open_model_zoo/tools/model_tools + sudo python3 downloader.py --name deeplabv3 --output_dir /opt/openvino_toolkit/models/deeplabv3/output + sudo python3 converter.py --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert + ``` + +* Copy label files (execute once) +* Before launch, copy label files to the same model path, make sure the model path and label path match the ros_openvino_toolkit/vino_launch/param/xxxx.yaml. +``` + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 + sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels +``` +### Install OpenVINO 2022.1 by PIP +* OMZ tools are provided for downloading and converting OMZ models in ov2022([guide](https://pypi.org/project/openvino-dev/)). + +* See all available models +``` +omz_downloader --print_all +``` + +* Download the optimized Intermediate Representation (IR) of model (execute once), for example: +``` +omz_downloader --name face-detection-adas-0001 -output_dir /opt/openvino_toolkit/models/face_detection/output +omz_downloader --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output +omz_downloader --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output +omz_downloader --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output +omz_downloader --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output +omz_downloader --name person-reidentification-retail-0277 --output_dir /opt/openvino_toolkit/models/person-reidentification/output +omz_downloader --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output +omz_downloader --name semantic-segmentation-adas-0001 --output_dir /opt/openvino_toolkit/models/semantic-segmentation/output +omz_downloader --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recognition/output +omz_downloader --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output +``` +* Copy label files (execute once) +``` + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 + sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels +``` + +* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to intermediate representation (such as the model for object detection): + * mobilenet-ssd (replaced ssd_mobilenet_v2_coco to mobilenet-ssd for object detection in ov2022) + ``` + omz_downloader --name mobilenet-ssd --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output + omz_converter --name mobilenet-ssd -d /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output -o /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert + ``` + * vehicle-license-plate-detection-barrier-0123 + ``` + omz_downloader --name vehicle-license-plate-detection-barrier-0123 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output + omz_converter --name vehicle-license-plate-detection-barrier-0123 -d /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output -o /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output/convert + ``` + + * deeplabv3 + ``` + omz_downloader --name deeplabv3 --output_dir /opt/openvino_toolkit/models/deeplabv3/output + omz_converter --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert + ``` + +* Please check the parameter configuration in ros2_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. + * run face detection sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_people.launch + ``` + * run person reidentification sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_face_reidentification.launch + ``` + * run person reidentification sample code input from StandardCamera + ``` + roslaunch vino_launch pipeline_reidentification.launch + ``` + * run face detection sample code input from Image. + ``` + roslaunch vino_launch pipeline_image.launch + ``` + * run object sample + ``` + roslaunch vino_launch pipeline_object.launch + ``` + * run object topic sample + ``` + roslaunch vino_launch pipeline_object_topic.launch + ``` + * run object segmentation sample code input from RealSenseCamera. + ``` + roslaunch vino_launch pipeline_segmentation.launch + ``` + * run vehicle detection sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_vehicle_detection.launch + ``` + * run video sample + ``` + roslaunch vino_launch pipeline_video.launch + ``` + +# More Information + +###### *Any security issue should be reported using process at https://01.org/security* + + + diff --git a/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md b/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md new file mode 100644 index 00000000..048eda2d --- /dev/null +++ b/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md @@ -0,0 +1,194 @@ +# ROS_NOETIC_OpenVINO_Toolkit + +**NOTE:** +Below steps have been tested on **Ubuntu 20.04**. + +## 1. Environment Setup +* Install ROS Noetic ([guide](http://wiki.ros.org/noetic/Installation/Ubuntu)) + +* Install Intel® OpenVINO™ Toolkit Version: 2022.1 ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)). +* * Install from an achive file ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)). +Tips: Both runtime and development tool are needed, `pip` is recommended for installing the development tool ([guide](https://docs.openvino.ai/latest/openvino_docs_install_guides_install_dev_tools.html)). +* * Install from source code ([guide](https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux)). + +* Install Intel® RealSense ™ SDK ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)). + +* Install Dependency + * Install gflags + ``` + sudo apt-get install -y libgflags-dev + ``` + +## 2. Build and Installation +* Install ROS_OpenVINO_Toolkit packages +``` +mkdir -p ~/catkin_ws/src +cd ~/catkin_ws/src +git clone https://github.com/intel/ros_openvino_toolkit -b noetic +git clone https://github.com/intel/object_msgs +git clone https://github.com/IntelRealSense/realsense-ros.git +cd realsense-ros +git checkout `git tag | sort -V | grep -P "^2.\d+\.\d+" | tail -1` +``` +* Install dependencies +``` +sudo apt-get install ros-noetic-ddynamic-reconfigure +``` +* Build package +``` +source /opt/ros/noetic/setup.bash +source /setupvars.sh +cd ~/catkin_ws +catkin_make_isolated +source ./devel_isolated/setup.bash +``` + +## 3. Running the Demo +### Install OpenVINO 2022.1 by source code +* See all available models +``` +cd /openvino/thirdparty/open_model_zoo/tools/model_tools +sudo python3 downloader.py --print_all +``` + +* Download the optimized Intermediate Representation (IR) of model (execute once): +``` +cd /openvino/thirdparty/open_model_zoo/tools/model_tools +sudo python3 downloader.py --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output +sudo python3 downloader.py --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output +sudo python3 downloader.py --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output +sudo python3 downloader.py --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output +sudo python3 downloader.py --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output +sudo python3 downloader.py --name person-reidentification-retail-0277 --output_dir /opt/openvino_toolkit/models/person-reidentification/output +sudo python3 downloader.py --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output +sudo python3 downloader.py --name face-reidentification-retail-0095 --output_dir /opt/openvino_toolkit/models/face-reidentification-retail/output +sudo python3 downloader.py --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recognition/output +sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output +``` + +* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi)need to be converted to intermediate representation (For example the model for object detection) + * mobilenet-ssd (replaced ssd_mobilenet_v2_coco to mobilenet-ssd for object detection in ov2022) + ``` + cd /openvino/thirdparty/open_model_zoo/tools/model_tools + sudo python3 downloader.py --name mobilenet-ssd --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output + sudo python3 converter.py --name mobilenet-ssd -d /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/ -o /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert + ``` + * vehicle-license-plate-detection-barrier-0123 + ``` + cd /openvino/thirdparty/open_model_zoo/tools/model_tools + sudo python3 downloader.py --name vehicle-license-plate-detection-barrier-0123 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output + sudo python3 converter.py --name vehicle-license-plate-detection-barrier-0123 -d /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output -o /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output/convert + ``` + * deeplabv3 + ``` + cd /openvino/thirdparty/open_model_zoo/tools/model_tools + sudo python3 downloader.py --name deeplabv3 --output_dir /opt/openvino_toolkit/models/deeplabv3/output + sudo python3 converter.py --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert + ``` + +* Copy label files (execute once) +* Before launch, copy label files to the same model path, make sure the model path and label path match the ros_openvino_toolkit/vino_launch/param/xxxx.yaml. +``` + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 + sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels +``` +### Install OpenVINO 2022.1 by PIP +* OMZ tools are provided for downloading and converting OMZ models in ov2022 ([guide](https://pypi.org/project/openvino-dev/)). + +* See all available models +``` +omz_downloader --print_all +``` + +* Download the optimized Intermediate Representation (IR) of model (execute once), for example: +``` +omz_downloader --name face-detection-adas-0001 -output_dir /opt/openvino_toolkit/models/face_detection/output +omz_downloader --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output +omz_downloader --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output +omz_downloader --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output +omz_downloader --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output +omz_downloader --name person-reidentification-retail-0277 --output_dir /opt/openvino_toolkit/models/person-reidentification/output +omz_downloader --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output +omz_downloader --name semantic-segmentation-adas-0001 --output_dir /opt/openvino_toolkit/models/semantic-segmentation/output +omz_downloader --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recognition/output +omz_downloader --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output +``` +* Copy label files (execute once) +``` + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 + sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels +``` + +* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to intermediate representation (such as the model for object detection): + * mobilenet-ssd (replaced ssd_mobilenet_v2_coco to mobilenet-ssd for object detection in ov2022) + ``` + omz_downloader --name mobilenet-ssd --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output + omz_converter --name mobilenet-ssd -d /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output -o /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert + ``` + * vehicle-license-plate-detection-barrier-0123 + ``` + omz_downloader --name vehicle-license-plate-detection-barrier-0123 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output + omz_converter --name vehicle-license-plate-detection-barrier-0123 -d /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output -o /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output/convert + ``` + + * deeplabv3 + ``` + omz_downloader --name deeplabv3 --output_dir /opt/openvino_toolkit/models/deeplabv3/output + omz_converter --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert + ``` + +* Please check the parameter configuration in ros2_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. + * run face detection sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_people.launch + ``` + * run person reidentification sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_face_reidentification.launch + ``` + * run person reidentification sample code input from StandardCamera + ``` + roslaunch vino_launch pipeline_reidentification.launch + ``` + * run face detection sample code input from Image. + ``` + roslaunch vino_launch pipeline_image.launch + ``` + * run object sample + ``` + roslaunch vino_launch pipeline_object.launch + ``` + * run object topic sample + ``` + roslaunch vino_launch pipeline_object_topic.launch + ``` + * run object segmentation sample code input from RealSenseCamera. + ``` + roslaunch vino_launch pipeline_segmentation.launch + ``` + * run vehicle detection sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_vehicle_detection.launch + ``` + * run video sample + ``` + roslaunch vino_launch pipeline_video.launch + ``` + +# More Information + +###### *Any security issue should be reported using process at https://01.org/security* + + From b79a632821180fd25f6d4504e18fdd6be9435e76 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Thu, 20 Oct 2022 14:02:26 +0800 Subject: [PATCH 25/40] fix docs --- ..._started_with_Melodic_Ubuntu18.04_ov2.0.md | 34 +++++++++---------- ...g_started_with_Noetic_Ubuntu20.04_ov2.0.md | 30 ++++++++-------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md b/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md index 1e3b3de1..9e5af285 100644 --- a/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md +++ b/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md @@ -1,15 +1,15 @@ # ROS_MELODIC_OpenVINO_Toolkit **NOTE:** -Below steps have been tested on **Ubuntu 20.04**. +Below steps have been tested on **Ubuntu 18.04**. ## 1. Environment Setup -* Install ROS Melodic ([guide](http://wiki.ros.org/melodic/Installation/Ubuntu)) +* Install ROS Melodic ([guide](http://wiki.ros.org/melodic/Installation/Ubuntu)). * Install Intel® OpenVINO™ Toolkit Version: 2022.1 ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)). -* * Install from an achive file([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)). + * Install from an achive file ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)). Tips: Both runtime and development tool are needed, `pip` is recommended for installing the development tool ([guide](https://docs.openvino.ai/latest/openvino_docs_install_guides_install_dev_tools.html)). -* * Install from source code([guide](https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux)). + * Install from source code ([guide](https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux)). * Install Intel® RealSense ™ SDK ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)). @@ -90,17 +90,17 @@ sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --outpu * Copy label files (execute once) * Before launch, copy label files to the same model path, make sure the model path and label path match the ros_openvino_toolkit/vino_launch/param/xxxx.yaml. ``` - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels ``` ### Install OpenVINO 2022.1 by PIP -* OMZ tools are provided for downloading and converting OMZ models in ov2022([guide](https://pypi.org/project/openvino-dev/)). +* OMZ tools are provided for downloading and converting OMZ models in ov2022 ([guide](https://pypi.org/project/openvino-dev/)). * See all available models ``` @@ -109,7 +109,7 @@ omz_downloader --print_all * Download the optimized Intermediate Representation (IR) of model (execute once), for example: ``` -omz_downloader --name face-detection-adas-0001 -output_dir /opt/openvino_toolkit/models/face_detection/output +omz_downloader --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output omz_downloader --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output omz_downloader --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output omz_downloader --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output @@ -122,11 +122,11 @@ omz_downloader --name license-plate-recognition-barrier-0001 --output_dir /opt/o ``` * Copy label files (execute once) ``` - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels @@ -150,7 +150,7 @@ omz_downloader --name license-plate-recognition-barrier-0001 --output_dir /opt/o omz_converter --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert ``` -* Please check the parameter configuration in ros2_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. +* Please check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. * run face detection sample code input from StandardCamera. ``` roslaunch vino_launch pipeline_people.launch diff --git a/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md b/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md index 048eda2d..becfd227 100644 --- a/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md +++ b/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md @@ -4,12 +4,12 @@ Below steps have been tested on **Ubuntu 20.04**. ## 1. Environment Setup -* Install ROS Noetic ([guide](http://wiki.ros.org/noetic/Installation/Ubuntu)) +* Install ROS Noetic ([guide](http://wiki.ros.org/noetic/Installation/Ubuntu)). * Install Intel® OpenVINO™ Toolkit Version: 2022.1 ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)). -* * Install from an achive file ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)). + * Install from an achive file ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)). Tips: Both runtime and development tool are needed, `pip` is recommended for installing the development tool ([guide](https://docs.openvino.ai/latest/openvino_docs_install_guides_install_dev_tools.html)). -* * Install from source code ([guide](https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux)). + * Install from source code ([guide](https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux)). * Install Intel® RealSense ™ SDK ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)). @@ -89,11 +89,11 @@ sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --outpu * Copy label files (execute once) * Before launch, copy label files to the same model path, make sure the model path and label path match the ros_openvino_toolkit/vino_launch/param/xxxx.yaml. ``` - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels @@ -108,7 +108,7 @@ omz_downloader --print_all * Download the optimized Intermediate Representation (IR) of model (execute once), for example: ``` -omz_downloader --name face-detection-adas-0001 -output_dir /opt/openvino_toolkit/models/face_detection/output +omz_downloader --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output omz_downloader --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output omz_downloader --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output omz_downloader --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output @@ -121,11 +121,11 @@ omz_downloader --name license-plate-recognition-barrier-0001 --output_dir /opt/o ``` * Copy label files (execute once) ``` - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros2_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels @@ -149,7 +149,7 @@ omz_downloader --name license-plate-recognition-barrier-0001 --output_dir /opt/o omz_converter --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert ``` -* Please check the parameter configuration in ros2_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. +* Please check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. * run face detection sample code input from StandardCamera. ``` roslaunch vino_launch pipeline_people.launch From c6f4fa31a434e95ea9d64a0aec415f5fab5bc1fe Mon Sep 17 00:00:00 2001 From: wujiawei Date: Fri, 11 Nov 2022 11:01:51 +0800 Subject: [PATCH 26/40] fix param name --- vino_core_lib/include/vino_core_lib/common.h | 2 +- .../include/vino_core_lib/models/base_model.h | 6 +++--- vino_core_lib/src/engines/engine.cpp | 2 +- vino_core_lib/src/engines/engine_manager.cpp | 4 ++-- .../src/models/age_gender_detection_model.cpp | 16 +++++++-------- .../src/models/attributes/ssd_model_attr.cpp | 18 ++++++++--------- vino_core_lib/src/models/base_model.cpp | 8 ++++---- .../src/models/emotion_detection_model.cpp | 18 ++++++++--------- .../src/models/face_detection_model.cpp | 8 ++++---- .../models/face_reidentification_model.cpp | 14 ++++++------- .../src/models/head_pose_detection_model.cpp | 14 ++++++------- .../src/models/landmarks_detection_model.cpp | 14 ++++++------- .../models/license_plate_detection_model.cpp | 12 +++++------ .../src/models/object_detection_ssd_model.cpp | 20 +++++++++---------- .../object_detection_yolov2voc_model.cpp | 20 +++++++++---------- .../src/models/object_segmentation_model.cpp | 18 ++++++++--------- .../models/person_attribs_detection_model.cpp | 12 +++++------ .../models/person_reidentification_model.cpp | 14 ++++++------- .../vehicle_attribs_detection_model.cpp | 12 +++++------ 19 files changed, 116 insertions(+), 116 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/common.h b/vino_core_lib/include/vino_core_lib/common.h index 97d76f7f..141c97d0 100644 --- a/vino_core_lib/include/vino_core_lib/common.h +++ b/vino_core_lib/include/vino_core_lib/common.h @@ -38,7 +38,7 @@ #include #include -///#include +///#include ///#include /// #include ///#include diff --git a/vino_core_lib/include/vino_core_lib/models/base_model.h b/vino_core_lib/include/vino_core_lib/models/base_model.h index a4fa0afe..95529186 100644 --- a/vino_core_lib/include/vino_core_lib/models/base_model.h +++ b/vino_core_lib/include/vino_core_lib/models/base_model.h @@ -99,9 +99,9 @@ class BaseModel : public ModelAttribute return attr_; } - inline std::shared_ptr getNetReader() const + inline std::shared_ptr getModel() const { - return net_reader_; + return model_; } protected: @@ -113,7 +113,7 @@ class BaseModel : public ModelAttribute virtual bool updateLayerProperty(std::shared_ptr& network_reader) = 0; ov::Core engine; - std::shared_ptr net_reader_; + std::shared_ptr model_; void setFrameSize(const int& w, const int& h) { frame_size_.width = w; diff --git a/vino_core_lib/src/engines/engine.cpp b/vino_core_lib/src/engines/engine.cpp index 9066e838..13c8294b 100644 --- a/vino_core_lib/src/engines/engine.cpp +++ b/vino_core_lib/src/engines/engine.cpp @@ -23,7 +23,7 @@ #if (defined(USE_OLD_E_PLUGIN_API)) Engines::Engine::Engine(InferenceEngine::InferencePlugin plg, const Models::BaseModel::Ptr base_model) { - request_ = (plg.LoadNetwork(base_model->getNetReader()->getNetwork(), {})).CreateInferRequestPtr(); + request_ = (plg.LoadNetwork(base_model->getModel()->getNetwork(), {})).CreateInferRequestPtr(); } #endif diff --git a/vino_core_lib/src/engines/engine_manager.cpp b/vino_core_lib/src/engines/engine_manager.cpp index dfd5933f..2aa9cd50 100644 --- a/vino_core_lib/src/engines/engine_manager.cpp +++ b/vino_core_lib/src/engines/engine_manager.cpp @@ -41,7 +41,7 @@ std::shared_ptr Engines::EngineManager::createEngine_V2022( const std::string& device, const std::shared_ptr& model) { ov::Core core; - ov::CompiledModel executable_network = core.compile_model(model->getNetReader(), device); + ov::CompiledModel executable_network = core.compile_model(model->getModel(), device); ov::InferRequest infer_request = executable_network.create_infer_request(); return std::make_shared(infer_request); @@ -59,7 +59,7 @@ std::shared_ptr Engines::EngineManager::createEngine_beforeV201 slog::info << "Created plugin for " << device << slog::endl; } - auto executeable_network = plugins_for_devices_[device].LoadNetwork(model->getNetReader()->getNetwork(), {}); + auto executeable_network = plugins_for_devices_[device].LoadNetwork(model->getModel()->getNetwork(), {}); auto request = executeable_network.CreateInferRequestPtr(); return std::make_shared(request); diff --git a/vino_core_lib/src/models/age_gender_detection_model.cpp b/vino_core_lib/src/models/age_gender_detection_model.cpp index 64ad2972..8c7e1301 100644 --- a/vino_core_lib/src/models/age_gender_detection_model.cpp +++ b/vino_core_lib/src/models/age_gender_detection_model.cpp @@ -30,16 +30,16 @@ Models::AgeGenderDetectionModel::AgeGenderDetectionModel(const std::string& labe : BaseModel(label_loc, model_loc, max_batch_size) { } -bool Models::AgeGenderDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) +bool Models::AgeGenderDetectionModel::updateLayerProperty(std::shared_ptr& model) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - inputs_info_ = net_reader->inputs(); - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - input_tensor_name_ = net_reader->input().get_any_name(); + inputs_info_ = model->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); - ov::Shape input_tensor_shape = net_reader->input().get_shape(); + ov::Shape input_tensor_shape = model->input().get_shape(); if (inputs_info_.size() != 1) { slog::warn << "This model seems not Age-Gender-like, which should have only one input, but we got" << std::to_string(input_tensor_shape.size()) << "inputs" @@ -54,7 +54,7 @@ bool Models::AgeGenderDetectionModel::updateLayerProperty(std::shared_ptroutputs(); + outputs_info_ = model->outputs(); if (outputs_info_.size() != 2) { slog::warn << "This model seems not Age-Gender-like, which should have and only have 2 outputs, but we got" << std::to_string(outputs_info_.size()) << "outputs" @@ -95,8 +95,8 @@ bool Models::AgeGenderDetectionModel::updateLayerProperty(std::shared_ptr & net_reader) + const std::shared_ptr & model) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - auto input_info_map = net_reader->inputs(); + auto input_info_map = model->inputs(); if (input_info_map.size() != 1) { slog::warn << "This model seems not SSDNet-like, SSDnet has only one input, but we got " << std::to_string(input_info_map.size()) << "inputs" << slog::endl; return false; } - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + std::string input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); input_info.tensor().set_element_type(ov::element::u8); addInputInfo("input", input_tensor_name_); @@ -52,7 +52,7 @@ bool Models::SSDModelAttr::updateLayerProperty( setInputWidth(input_dims[3]); slog::info << "Checking OUTPUTs for model " << getModelName() << slog::endl; - auto outputs_info = net_reader->outputs(); + auto outputs_info = model->outputs(); if (outputs_info.size() != 1) { slog::warn << "This model seems not SSDNet-like! We got " << std::to_string(outputs_info.size()) << "outputs, but SSDnet has only one." @@ -60,8 +60,8 @@ bool Models::SSDModelAttr::updateLayerProperty( return false; } ov::preprocess::OutputInfo& output_info = ppp.output(); - addOutputInfo("output", net_reader->output().get_any_name()); - slog::info << "Checking Object Detection output ... Name=" << net_reader->output().get_any_name() + addOutputInfo("output", model->output().get_any_name()); + slog::info << "Checking Object Detection output ... Name=" << model->output().get_any_name() << slog::endl; output_info.tensor().set_element_type(ov::element::f32); @@ -69,7 +69,7 @@ bool Models::SSDModelAttr::updateLayerProperty( ///TODO: double check this part: BEGIN #if(0) /// const InferenceEngine::CNNLayerPtr output_layer = - net_reader->getNetwork().getLayerByName(output_info_map.begin()->first.c_str()); + model->getNetwork().getLayerByName(output_info_map.begin()->first.c_str()); // output layer should have attribute called num_classes slog::info << "Checking Object Detection num_classes" << slog::endl; if (output_layer->params.find("num_classes") == output_layer->params.end()) { @@ -94,7 +94,7 @@ bool Models::SSDModelAttr::updateLayerProperty( ///TODO: double check this part: END // last dimension of output layer should be 7 - auto outputsDataMap = net_reader->outputs(); + auto outputsDataMap = model->outputs(); auto & data = outputsDataMap[0]; ov::Shape output_dims = data.get_shape(); setMaxProposalCount(static_cast(output_dims[2])); diff --git a/vino_core_lib/src/models/base_model.cpp b/vino_core_lib/src/models/base_model.cpp index fc9bb586..1319dd36 100644 --- a/vino_core_lib/src/models/base_model.cpp +++ b/vino_core_lib/src/models/base_model.cpp @@ -44,7 +44,7 @@ void Models::BaseModel::modelInit() slog::info << "Loading network files: " << model_loc_ << slog::endl; // Read network model - net_reader_ = engine.read_model(model_loc_); + model_ = engine.read_model(model_loc_); // Extract model name and load it's weights // remove extension size_t last_index = model_loc_.find_last_of("."); @@ -56,15 +56,15 @@ void Models::BaseModel::modelInit() // Set batch size to given max_batch_size_ slog::info << "Batch size is set to " << max_batch_size_ << slog::endl; - updateLayerProperty(net_reader_); + updateLayerProperty(model_); } #if 0 bool Models::BaseModel::updateLayerProperty( - InferenceEngine::CNNNetReader::Ptr net_reader) + InferenceEngine::CNNNetReader::Ptr model) { #if 0 - if (!updateLayerProperty(net_reader)){ + if (!updateLayerProperty(model)){ slog::warn << "The model(name: " << getModelName() << ") failed to update Layer Property!" << slog::endl; return false; diff --git a/vino_core_lib/src/models/emotion_detection_model.cpp b/vino_core_lib/src/models/emotion_detection_model.cpp index c4e1410d..d41f86ad 100644 --- a/vino_core_lib/src/models/emotion_detection_model.cpp +++ b/vino_core_lib/src/models/emotion_detection_model.cpp @@ -29,16 +29,16 @@ Models::EmotionDetectionModel::EmotionDetectionModel(const std::string& label_lo { } -bool Models::EmotionDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) +bool Models::EmotionDetectionModel::updateLayerProperty(std::shared_ptr& model) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - inputs_info_ = net_reader->inputs(); - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - input_tensor_name_ = net_reader->input().get_any_name(); + inputs_info_ = model->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); - ov::Shape input_tensor_shape = net_reader->input().get_shape(); + ov::Shape input_tensor_shape = model->input().get_shape(); if (inputs_info_.size() != 1) { slog::warn << "This model seems not Emotion-detection-model-like, which should have only one input, but we got" << std::to_string(input_tensor_shape.size()) << "inputs" @@ -53,8 +53,8 @@ bool Models::EmotionDetectionModel::updateLayerProperty(std::shared_ptroutputs(); - output_tensor_name_ = net_reader->output().get_any_name(); + outputs_info_ = model->outputs(); + output_tensor_name_ = model->output().get_any_name(); ov::preprocess::OutputInfo& output_info = ppp.output(output_tensor_name_); if (outputs_info_.size() != 1) { slog::warn << "This model should have and only have 1 output, but we got " @@ -63,8 +63,8 @@ bool Models::EmotionDetectionModel::updateLayerProperty(std::shared_ptrparams.find("num_classes") == output_layer->params.end()) { diff --git a/vino_core_lib/src/models/face_reidentification_model.cpp b/vino_core_lib/src/models/face_reidentification_model.cpp index 6d32c375..a2c91fa0 100644 --- a/vino_core_lib/src/models/face_reidentification_model.cpp +++ b/vino_core_lib/src/models/face_reidentification_model.cpp @@ -25,12 +25,12 @@ Models::FaceReidentificationModel::FaceReidentificationModel(const std::string& { } -bool Models::FaceReidentificationModel::updateLayerProperty(std::shared_ptr& net_reader) +bool Models::FaceReidentificationModel::updateLayerProperty(std::shared_ptr& model) { // set input property - auto inputs_info_map = net_reader->inputs(); - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - std::string input_tensor_name_ = net_reader->input().get_any_name(); + auto inputs_info_map = model->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + std::string input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); const ov::Layout tensor_layout{"NCHW"}; @@ -40,12 +40,12 @@ bool Models::FaceReidentificationModel::updateLayerProperty(std::shared_ptroutputs(); - std::string output_tensor_name = net_reader->output().get_any_name(); + auto outputs_info = model->outputs(); + std::string output_tensor_name = model->output().get_any_name(); ov::preprocess::OutputInfo& output_info = ppp.output(output_tensor_name); output_info.tensor().set_element_type(ov::element::f32); addOutputInfo("output", output_tensor_name); - net_reader = ppp.build(); + model = ppp.build(); return true; } diff --git a/vino_core_lib/src/models/head_pose_detection_model.cpp b/vino_core_lib/src/models/head_pose_detection_model.cpp index 020d293d..e1fd9d67 100644 --- a/vino_core_lib/src/models/head_pose_detection_model.cpp +++ b/vino_core_lib/src/models/head_pose_detection_model.cpp @@ -32,19 +32,19 @@ Models::HeadPoseDetectionModel::HeadPoseDetectionModel(const std::string& label_ { } -bool Models::HeadPoseDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) +bool Models::HeadPoseDetectionModel::updateLayerProperty(std::shared_ptr& model) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - auto input_info_map = net_reader->inputs(); + auto input_info_map = model->inputs(); if (input_info_map.size() != 1) { slog::warn << "This model should have only one input, but we got" << std::to_string(input_info_map.size()) << "inputs" << slog::endl; return false; } - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + std::string input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); const ov::Layout input_tensor_layout{"NCHW"}; input_info.tensor(). @@ -53,7 +53,7 @@ bool Models::HeadPoseDetectionModel::updateLayerProperty(std::shared_ptroutputs(); + auto output_info_map = model->outputs(); std::vector outputs_name; for (auto & output_item : output_info_map) { std::string output_tensor_name_ = output_item.get_any_name(); @@ -65,8 +65,8 @@ bool Models::HeadPoseDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) +bool Models::LandmarksDetectionModel::updateLayerProperty(std::shared_ptr& model) { //INPUT - auto inputs_info_map = net_reader->inputs(); - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - std::string input_tensor_name_ = net_reader->input().get_any_name(); + auto inputs_info_map = model->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + std::string input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); auto input_layerName = inputs_info_map[0]; @@ -59,15 +59,15 @@ bool Models::LandmarksDetectionModel::updateLayerProperty(std::shared_ptroutput().get_any_name(); + std::string output_tensor_name = model->output().get_any_name(); ov::preprocess::OutputInfo& output_info = ppp.output(output_tensor_name); - auto outputs_info_map = net_reader->outputs(); + auto outputs_info_map = model->outputs(); auto output_layerName = outputs_info_map[0]; auto output_layerData = outputs_info_map[1]; ov::Shape output_layerDims = output_layerData.get_shape(); ppp.output(output_layerData.get_any_name()).tensor().set_element_type(ov::element::f32); - net_reader = ppp.build(); + model = ppp.build(); addOutputInfo("output_layerName", output_layerName.get_any_name()); addOutputInfo("output_layerData", output_layerData.get_any_name()); return true; diff --git a/vino_core_lib/src/models/license_plate_detection_model.cpp b/vino_core_lib/src/models/license_plate_detection_model.cpp index b59726b6..2280f699 100644 --- a/vino_core_lib/src/models/license_plate_detection_model.cpp +++ b/vino_core_lib/src/models/license_plate_detection_model.cpp @@ -25,10 +25,10 @@ Models::LicensePlateDetectionModel::LicensePlateDetectionModel(const std::string { } -bool Models::LicensePlateDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) +bool Models::LicensePlateDetectionModel::updateLayerProperty(std::shared_ptr& model) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - auto input_info_map = net_reader->inputs(); + auto input_info_map = model->inputs(); if (input_info_map.size() != 2) { throw std::logic_error("Vehicle Attribs topology should have only two inputs"); @@ -38,25 +38,25 @@ bool Models::LicensePlateDetectionModel::updateLayerProperty(std::shared_ptroutputs(); + auto output_info_map = model->outputs(); if (output_info_map.size() != 1) { throw std::logic_error("Vehicle Attribs Network expects networks having one output"); } - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); std::string input_tensor_name_ = input_info_map[0].get_any_name(); const ov::Layout tensor_layout{"NCHW"}; ppp.input(input_tensor_name_). tensor(). set_element_type(ov::element::u8). set_layout(tensor_layout); - net_reader = ppp.build(); + model = ppp.build(); // set input and output layer name input_ = input_tensor_name_; seq_input_ = sequence_input.get_any_name(); - output_ = net_reader->output().get_any_name(); + output_ = model->output().get_any_name(); return true; } diff --git a/vino_core_lib/src/models/object_detection_ssd_model.cpp b/vino_core_lib/src/models/object_detection_ssd_model.cpp index a157ebbf..e49f3ec8 100644 --- a/vino_core_lib/src/models/object_detection_ssd_model.cpp +++ b/vino_core_lib/src/models/object_detection_ssd_model.cpp @@ -147,18 +147,18 @@ bool Models::ObjectDetectionSSDModel::fetchResults(const std::shared_ptr& net_reader) +bool Models::ObjectDetectionSSDModel::updateLayerProperty(std::shared_ptr& model) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - auto input_info_map = net_reader->inputs(); + auto input_info_map = model->inputs(); if (input_info_map.size() != 1) { slog::warn << "This model seems not SSDNet-like, SSDnet has only one input, but we got " << std::to_string(input_info_map.size()) << "inputs" << slog::endl; return false; } - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + std::string input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); input_info.tensor().set_element_type(ov::element::u8); @@ -185,7 +185,7 @@ bool Models::ObjectDetectionSSDModel::updateLayerProperty(std::shared_ptroutputs(); + auto outputs_info = model->outputs(); if (outputs_info.size() != 1) { slog::warn << "This model seems not SSDNet-like! We got " << std::to_string(outputs_info.size()) << "outputs, but SSDnet has only one." @@ -193,17 +193,17 @@ bool Models::ObjectDetectionSSDModel::updateLayerProperty(std::shared_ptroutput().get_any_name()); - slog::info << "Checking Object Detection output ... Name=" << net_reader->output().get_any_name() + addOutputInfo("output", model->output().get_any_name()); + slog::info << "Checking Object Detection output ... Name=" << model->output().get_any_name() << slog::endl; output_info.tensor().set_element_type(ov::element::f32); - net_reader = ppp.build(); + model = ppp.build(); /// TODO: double check this part: BEGIN #if(0) const InferenceEngine::CNNLayerPtr output_layer = - net_reader->getNetwork().getLayerByName(output_info_map.begin()->first.c_str()); + model->getNetwork().getLayerByName(output_info_map.begin()->first.c_str()); // output layer should have attribute called num_classes slog::info << "Checking Object Detection num_classes" << slog::endl; if (output_layer->params.find("num_classes") == output_layer->params.end()) @@ -229,7 +229,7 @@ bool Models::ObjectDetectionSSDModel::updateLayerProperty(std::shared_ptroutputs(); + auto outputsDataMap = model->outputs(); auto & data = outputsDataMap[0]; ov::Shape output_dims = data.get_shape(); setMaxProposalCount(static_cast(output_dims[2])); diff --git a/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp b/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp index ba9b83e4..4233b5fd 100644 --- a/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp +++ b/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp @@ -26,11 +26,11 @@ Models::ObjectDetectionYolov2Model::ObjectDetectionYolov2Model(const std::string { } -bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const std::shared_ptr& net_reader) +bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const std::shared_ptr& model) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - auto input_info_map = net_reader->inputs(); + auto input_info_map = model->inputs(); if (input_info_map.size() != 1) { slog::warn << "This model seems not Yolo-like, which has only one input, but we got " @@ -39,8 +39,8 @@ bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const std::shared_p } // set input property - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + std::string input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); const ov::Layout input_tensor_layout{"NCHW"}; input_info.tensor(). @@ -49,7 +49,7 @@ bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const std::shared_p addInputInfo("input", input_tensor_name_); // set output property - auto output_info_map = net_reader -> outputs(); + auto output_info_map = model -> outputs(); if (output_info_map.size() != 1) { slog::warn << "This model seems not Yolo-like! We got " << std::to_string(output_info_map.size()) << "outputs, but SSDnet has only one." @@ -57,15 +57,15 @@ bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const std::shared_p return false; } ov::preprocess::OutputInfo& output_info = ppp.output(); - addOutputInfo("output", net_reader->output().get_any_name()); + addOutputInfo("output", model->output().get_any_name()); output_info.tensor().set_element_type(ov::element::f32); - slog::info << "Checking Object Detection output ... Name=" << net_reader->output().get_any_name() + slog::info << "Checking Object Detection output ... Name=" << model->output().get_any_name() << slog::endl; - net_reader = ppp.build(); + model = ppp.build(); #if(0) const InferenceEngine::CNNLayerPtr output_layer = - net_reader->getNetwork().getLayerByName(output_info_map.begin()->first.c_str()); + model->getNetwork().getLayerByName(output_info_map.begin()->first.c_str()); // output layer should have attribute called num_classes slog::info << "Checking Object Detection num_classes" << slog::endl; if (output_layer == nullptr || output_layer->params.find("classes") == output_layer->params.end()) @@ -91,7 +91,7 @@ bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const std::shared_p } #endif // last dimension of output layer should be 7 - auto outputsDataMap = net_reader->outputs(); + auto outputsDataMap = model->outputs(); auto & data = outputsDataMap[0]; ov::Shape output_dims = data.get_shape(); setMaxProposalCount(static_cast(output_dims[2])); diff --git a/vino_core_lib/src/models/object_segmentation_model.cpp b/vino_core_lib/src/models/object_segmentation_model.cpp index e375b557..b0b601de 100644 --- a/vino_core_lib/src/models/object_segmentation_model.cpp +++ b/vino_core_lib/src/models/object_segmentation_model.cpp @@ -96,11 +96,11 @@ const std::string Models::ObjectSegmentationModel::getModelCategory() const return "Object Segmentation"; } -bool Models::ObjectSegmentationModel::updateLayerProperty(std::shared_ptr& net_reader) +bool Models::ObjectSegmentationModel::updateLayerProperty(std::shared_ptr& model) { slog::info << "Checking INPUTS for Model" << getModelName() << slog::endl; - inputs_info_ = net_reader->inputs(); + inputs_info_ = model->inputs(); slog::debug<<"input size"<input().get_any_name(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); ov::Layout tensor_layout = ov::Layout("NHWC"); ov::Layout expect_layout = ov::Layout("NCHW"); - ov::Shape input_shape = net_reader->input().get_shape(); + ov::Shape input_shape = model->input().get_shape(); if (input_shape[1] == 3) expect_layout = ov::Layout("NCHW"); else if (input_shape[3] == 3) @@ -133,19 +133,19 @@ bool Models::ObjectSegmentationModel::updateLayerProperty(std::shared_ptroutputs(); + auto outputs_info = model->outputs(); if (outputs_info.size() != 1) { slog::warn << "This inference sample should have only one output, but we got" << std::to_string(outputs_info.size()) << "outputs" << slog::endl; return false; } - output_tensor_name_ = net_reader->output().get_any_name(); - auto data = net_reader->output(); + output_tensor_name_ = model->output().get_any_name(); + auto data = model->output(); ov::preprocess::OutputInfo& output_info = ppp.output(output_tensor_name_); output_info.tensor().set_element_type(ov::element::f32); - net_reader = ppp.build(); + model = ppp.build(); std::vector &in_size_vector = input_shape; slog::debug<<"dimensional"<& net_reader) +bool Models::PersonAttribsDetectionModel::updateLayerProperty(std::shared_ptr& model) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; - auto input_info_map = net_reader->inputs(); + auto input_info_map = model->inputs(); if (input_info_map.size() != 1) { throw std::logic_error("Person Attribs topology should have only one input"); } - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + std::string input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); const ov::Layout tensor_layout{"NCHW"}; input_info.tensor(). @@ -42,12 +42,12 @@ bool Models::PersonAttribsDetectionModel::updateLayerProperty(std::shared_ptroutputs(); + auto output_info_map = model->outputs(); if (output_info_map.size() != 3) { throw std::logic_error("Person Attribs Network expects networks having 3 output"); } - net_reader = ppp.build(); + model = ppp.build(); addInputInfo("input", input_tensor_name_); addOutputInfo("attributes_output_",output_info_map[2].get_any_name()); addOutputInfo("top_output_", output_info_map[1].get_any_name()); diff --git a/vino_core_lib/src/models/person_reidentification_model.cpp b/vino_core_lib/src/models/person_reidentification_model.cpp index c0c0e566..d1241647 100644 --- a/vino_core_lib/src/models/person_reidentification_model.cpp +++ b/vino_core_lib/src/models/person_reidentification_model.cpp @@ -25,13 +25,13 @@ Models::PersonReidentificationModel::PersonReidentificationModel(const std::stri { } -bool Models::PersonReidentificationModel::updateLayerProperty(std::shared_ptr& net_reader) +bool Models::PersonReidentificationModel::updateLayerProperty(std::shared_ptr& model) { slog::info << "Checking Inputs for Model" << getModelName() << slog::endl; - auto network = net_reader; - auto input_info_map = net_reader->inputs(); - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); + auto network = model; + auto input_info_map = model->inputs(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); input_ = input_info_map[0].get_any_name(); const ov::Layout input_tensor_layout{"NCHW"}; ppp.input(input_). @@ -39,11 +39,11 @@ bool Models::PersonReidentificationModel::updateLayerProperty(std::shared_ptroutputs(); + auto output_info_map = model->outputs(); output_ = output_info_map[0].get_any_name(); - net_reader = ppp.build(); - ov::set_batch(net_reader_, getMaxBatchSize()); + model = ppp.build(); + ov::set_batch(model_, getMaxBatchSize()); return true; } diff --git a/vino_core_lib/src/models/vehicle_attribs_detection_model.cpp b/vino_core_lib/src/models/vehicle_attribs_detection_model.cpp index 03854adc..938abe34 100644 --- a/vino_core_lib/src/models/vehicle_attribs_detection_model.cpp +++ b/vino_core_lib/src/models/vehicle_attribs_detection_model.cpp @@ -25,29 +25,29 @@ Models::VehicleAttribsDetectionModel::VehicleAttribsDetectionModel(const std::st { } -bool Models::VehicleAttribsDetectionModel::updateLayerProperty(std::shared_ptr& net_reader) +bool Models::VehicleAttribsDetectionModel::updateLayerProperty(std::shared_ptr& model) { slog::info << "Checking INPUTs for model " << getModelName() << slog::endl; // set input property - auto input_info_map = net_reader->inputs(); + auto input_info_map = model->inputs(); if (input_info_map.size() != 1) { throw std::logic_error("Vehicle Attribs topology should have only one input"); } - auto output_info_map = net_reader->outputs(); + auto output_info_map = model->outputs(); if (output_info_map.size() != 2) { throw std::logic_error("Vehicle Attribs Network expects networks having two outputs"); } - ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(net_reader); - std::string input_tensor_name_ = net_reader->input().get_any_name(); + ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); + std::string input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); const ov::Layout tensor_layout{"NCHW"}; input_info.tensor(). set_element_type(ov::element::u8). set_layout(tensor_layout); - net_reader = ppp.build(); + model = ppp.build(); addInputInfo("input", input_tensor_name_); addOutputInfo("color_output_", output_info_map[1].get_any_name()); From 7adb00c5339841d2d3ae2dc18887bf660f425b0a Mon Sep 17 00:00:00 2001 From: wujiawei Date: Fri, 11 Nov 2022 11:21:09 +0800 Subject: [PATCH 27/40] code refine: move input&output info definition to base attribute --- .../vino_core_lib/models/age_gender_detection_model.h | 5 ----- .../vino_core_lib/models/attributes/base_attribute.h | 4 ++++ .../include/vino_core_lib/models/emotion_detection_model.h | 7 ------- .../vino_core_lib/models/head_pose_detection_model.h | 5 ----- .../vino_core_lib/models/object_segmentation_model.h | 4 ---- .../vino_core_lib/models/person_attribs_detection_model.h | 5 ----- vino_core_lib/src/models/age_gender_detection_model.cpp | 2 +- vino_core_lib/src/models/attributes/ssd_model_attr.cpp | 2 +- vino_core_lib/src/models/emotion_detection_model.cpp | 2 +- vino_core_lib/src/models/head_pose_detection_model.cpp | 4 ++-- vino_core_lib/src/models/license_plate_detection_model.cpp | 2 +- vino_core_lib/src/models/object_detection_ssd_model.cpp | 2 +- .../src/models/object_detection_yolov2voc_model.cpp | 2 +- vino_core_lib/src/models/object_segmentation_model.cpp | 4 ++-- .../src/models/person_attribs_detection_model.cpp | 2 +- vino_core_lib/src/models/person_reidentification_model.cpp | 2 +- .../src/models/vehicle_attribs_detection_model.cpp | 2 +- 17 files changed, 17 insertions(+), 39 deletions(-) diff --git a/vino_core_lib/include/vino_core_lib/models/age_gender_detection_model.h b/vino_core_lib/include/vino_core_lib/models/age_gender_detection_model.h index d28a0d6c..c9177ec2 100644 --- a/vino_core_lib/include/vino_core_lib/models/age_gender_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/age_gender_detection_model.h @@ -64,11 +64,6 @@ class AgeGenderDetectionModel : public BaseModel protected: bool updateLayerProperty(std::shared_ptr&) override; - std::string input_tensor_name_; - std::string output_tensor_name_; - - std::vector> inputs_info_; - std::vector> outputs_info_; }; } // namespace Models diff --git a/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h b/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h index 208a8293..d18aa7ed 100644 --- a/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h +++ b/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h @@ -187,6 +187,10 @@ class ModelAttribute protected: ModelAttr attr_; + std::string input_tensor_name_; + std::string output_tensor_name_; + std::vector> inputs_info_; + std::vector> outputs_info_; }; #if 0 // not used diff --git a/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h b/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h index a9a7de55..313ea022 100644 --- a/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/emotion_detection_model.h @@ -42,13 +42,6 @@ class EmotionDetectionModel : public BaseModel */ const std::string getModelCategory() const override; bool updateLayerProperty(std::shared_ptr&) override; - -private: - std::string input_tensor_name_; - std::string output_tensor_name_; - - std::vector> inputs_info_; - std::vector> outputs_info_; }; } // namespace Models diff --git a/vino_core_lib/include/vino_core_lib/models/head_pose_detection_model.h b/vino_core_lib/include/vino_core_lib/models/head_pose_detection_model.h index 340fcfb0..0f8074a4 100644 --- a/vino_core_lib/include/vino_core_lib/models/head_pose_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/head_pose_detection_model.h @@ -71,11 +71,6 @@ class HeadPoseDetectionModel : public BaseModel std::string output_angle_r_ = "angle_r_fc"; std::string output_angle_p_ = "angle_p_fc"; std::string output_angle_y_ = "angle_y_fc"; - std::string input_tensor_name_; - std::string output_tensor_name_; - - std::vector> inputs_info_; - std::vector> outputs_info_; }; } // namespace Models diff --git a/vino_core_lib/include/vino_core_lib/models/object_segmentation_model.h b/vino_core_lib/include/vino_core_lib/models/object_segmentation_model.h index 887749c3..adc26bb1 100644 --- a/vino_core_lib/include/vino_core_lib/models/object_segmentation_model.h +++ b/vino_core_lib/include/vino_core_lib/models/object_segmentation_model.h @@ -52,10 +52,6 @@ class ObjectSegmentationModel : public BaseModel private: int max_proposal_count_; int object_size_; - std::string input_tensor_name_; - std::string output_tensor_name_; - - std::vector> inputs_info_; ov::Shape input_shape_; }; } // namespace Models diff --git a/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h b/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h index c2fd4411..afa02629 100644 --- a/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/person_attribs_detection_model.h @@ -40,11 +40,6 @@ class PersonAttribsDetectionModel : public BaseModel bool updateLayerProperty(std::shared_ptr&) override; std::string input_; std::string output_; - std::string input_tensor_name_; - std::string output_tensor_name_; - - std::vector> inputs_info_; - std::vector> outputs_info_; }; } // namespace Models #endif // VINO_CORE_LIB__MODELS__PERSON_ATTRIBS_DETECTION_MODEL_H diff --git a/vino_core_lib/src/models/age_gender_detection_model.cpp b/vino_core_lib/src/models/age_gender_detection_model.cpp index 8c7e1301..b7790368 100644 --- a/vino_core_lib/src/models/age_gender_detection_model.cpp +++ b/vino_core_lib/src/models/age_gender_detection_model.cpp @@ -96,7 +96,7 @@ bool Models::AgeGenderDetectionModel::updateLayerProperty(std::shared_ptrinput().get_any_name(); + input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); input_info.tensor().set_element_type(ov::element::u8); addInputInfo("input", input_tensor_name_); diff --git a/vino_core_lib/src/models/emotion_detection_model.cpp b/vino_core_lib/src/models/emotion_detection_model.cpp index d41f86ad..d5e10839 100644 --- a/vino_core_lib/src/models/emotion_detection_model.cpp +++ b/vino_core_lib/src/models/emotion_detection_model.cpp @@ -64,7 +64,7 @@ bool Models::EmotionDetectionModel::updateLayerProperty(std::shared_ptrinput().get_any_name(); + input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); const ov::Layout input_tensor_layout{"NCHW"}; input_info.tensor(). @@ -66,7 +66,7 @@ bool Models::HeadPoseDetectionModel::updateLayerProperty(std::shared_ptrinput().get_any_name(); + input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); input_info.tensor().set_element_type(ov::element::u8); diff --git a/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp b/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp index 4233b5fd..b07b7e9b 100644 --- a/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp +++ b/vino_core_lib/src/models/object_detection_yolov2voc_model.cpp @@ -40,7 +40,7 @@ bool Models::ObjectDetectionYolov2Model::updateLayerProperty(const std::shared_p // set input property ov::preprocess::PrePostProcessor ppp = ov::preprocess::PrePostProcessor(model); - std::string input_tensor_name_ = model->input().get_any_name(); + input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); const ov::Layout input_tensor_layout{"NCHW"}; input_info.tensor(). diff --git a/vino_core_lib/src/models/object_segmentation_model.cpp b/vino_core_lib/src/models/object_segmentation_model.cpp index b0b601de..9db8206c 100644 --- a/vino_core_lib/src/models/object_segmentation_model.cpp +++ b/vino_core_lib/src/models/object_segmentation_model.cpp @@ -181,9 +181,9 @@ bool Models::ObjectSegmentationModel::updateLayerProperty(std::shared_ptrinput().get_any_name(); + input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); const ov::Layout tensor_layout{"NCHW"}; input_info.tensor(). diff --git a/vino_core_lib/src/models/person_reidentification_model.cpp b/vino_core_lib/src/models/person_reidentification_model.cpp index d1241647..4d1a956f 100644 --- a/vino_core_lib/src/models/person_reidentification_model.cpp +++ b/vino_core_lib/src/models/person_reidentification_model.cpp @@ -43,7 +43,7 @@ bool Models::PersonReidentificationModel::updateLayerProperty(std::shared_ptrinput().get_any_name(); + input_tensor_name_ = model->input().get_any_name(); ov::preprocess::InputInfo& input_info = ppp.input(input_tensor_name_); const ov::Layout tensor_layout{"NCHW"}; input_info.tensor(). From 5ad3792dd11be27ffc1a1c0393524b67b09c5aef Mon Sep 17 00:00:00 2001 From: wujiawei Date: Tue, 22 Nov 2022 01:56:29 +0800 Subject: [PATCH 28/40] update doc and model list for ros --- data/model_list/convert_model.lst | 5 + data/model_list/download_model.lst | 16 ++ ..._started_with_Melodic_Ubuntu18.04_ov2.0.md | 196 ------------------ .../getting_started_with_ros_ov2.0.md | 152 ++++++++++++++ 4 files changed, 173 insertions(+), 196 deletions(-) create mode 100644 data/model_list/convert_model.lst create mode 100644 data/model_list/download_model.lst delete mode 100644 doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md create mode 100644 doc/quick_start/getting_started_with_ros_ov2.0.md diff --git a/data/model_list/convert_model.lst b/data/model_list/convert_model.lst new file mode 100644 index 00000000..2304f169 --- /dev/null +++ b/data/model_list/convert_model.lst @@ -0,0 +1,5 @@ +# This file can be used with the --list option of the model converter. +mobilenet-ssd +deeplabv3 +mask_rcnn_inception_resnet_v2_atrous_coco +vehicle-license-plate-detection-barrier-0123 diff --git a/data/model_list/download_model.lst b/data/model_list/download_model.lst new file mode 100644 index 00000000..f485052e --- /dev/null +++ b/data/model_list/download_model.lst @@ -0,0 +1,16 @@ +# This file can be used with the --list option of the model downloader. +face-detection-adas-0001 +age-gender-recognition-retail-0013 +emotions-recognition-retail-0003 +landmarks-regression-retail-0009 +license-plate-recognition-barrier-0001 +person-detection-retail-0013 +person-attributes-recognition-crossroad-0230 +person-reidentification-retail-0277 +vehicle-attributes-recognition-barrier-0039 +vehicle-license-plate-detection-barrier-0106 +head-pose-estimation-adas-0001 +human-pose-estimation-0001 +semantic-segmentation-adas-0001 +mobilenet-ssd +deeplabv3 diff --git a/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md b/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md deleted file mode 100644 index 9e5af285..00000000 --- a/doc/quick_start/getting_started_with_Melodic_Ubuntu18.04_ov2.0.md +++ /dev/null @@ -1,196 +0,0 @@ -# ROS_MELODIC_OpenVINO_Toolkit - -**NOTE:** -Below steps have been tested on **Ubuntu 18.04**. - -## 1. Environment Setup -* Install ROS Melodic ([guide](http://wiki.ros.org/melodic/Installation/Ubuntu)). - -* Install Intel® OpenVINO™ Toolkit Version: 2022.1 ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)). - * Install from an achive file ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)). -Tips: Both runtime and development tool are needed, `pip` is recommended for installing the development tool ([guide](https://docs.openvino.ai/latest/openvino_docs_install_guides_install_dev_tools.html)). - * Install from source code ([guide](https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux)). - -* Install Intel® RealSense ™ SDK ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)). - -* Install Dependency - * Install gflags - ``` - sudo apt-get install -y libgflags-dev - ``` - -## 2. Build and Installation -* Install ROS_OpenVINO_Toolkit packages -``` -mkdir -p ~/catkin_ws/src -cd ~/catkin_ws/src -git clone https://github.com/intel/ros_openvino_toolkit -b melodic -git clone https://github.com/intel/object_msgs -git clone https://github.com/ros-perception/vision_opencv.git -b melodic -git clone https://github.com/IntelRealSense/realsense-ros.git -cd realsense-ros -git checkout `git tag | sort -V | grep -P "^2.\d+\.\d+" | tail -1` -``` -* Install dependencies -``` -sudo apt-get install ros-melodic-ddynamic-reconfigure -``` -* Build package -``` -source /opt/ros/melodic/setup.bash -source /setupvars.sh -cd ~/catkin_ws -catkin_make_isolated -source ./devel_isolated/setup.bash -``` - -## 3. Running the Demo -### Install OpenVINO 2022.1 by source code -* See all available models -``` -cd /openvino/thirdparty/open_model_zoo/tools/model_tools -sudo python3 downloader.py --print_all -``` - -* Download the optimized Intermediate Representation (IR) of model (execute once): -``` -cd /openvino/thirdparty/open_model_zoo/tools/model_tools -sudo python3 downloader.py --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output -sudo python3 downloader.py --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output -sudo python3 downloader.py --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output -sudo python3 downloader.py --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output -sudo python3 downloader.py --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output -sudo python3 downloader.py --name person-reidentification-retail-0277 --output_dir /opt/openvino_toolkit/models/person-reidentification/output -sudo python3 downloader.py --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output -sudo python3 downloader.py --name face-reidentification-retail-0095 --output_dir /opt/openvino_toolkit/models/face-reidentification-retail/output -sudo python3 downloader.py --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recognition/output -sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output -``` - -* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi)need to be converted to intermediate representation (For example the model for object detection) - * mobilenet-ssd (replaced ssd_mobilenet_v2_coco to mobilenet-ssd for object detection in ov2022) - ``` - cd /openvino/thirdparty/open_model_zoo/tools/model_tools - sudo python3 downloader.py --name mobilenet-ssd --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output - sudo python3 converter.py --name mobilenet-ssd -d /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/ -o /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert - ``` - * vehicle-license-plate-detection-barrier-0123 - ``` - cd /openvino/thirdparty/open_model_zoo/tools/model_tools - sudo python3 downloader.py --name vehicle-license-plate-detection-barrier-0123 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output - sudo python3 converter.py --name vehicle-license-plate-detection-barrier-0123 -d /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output -o /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output/convert - ``` - * deeplabv3 - ``` - cd /openvino/thirdparty/open_model_zoo/tools/model_tools - sudo python3 downloader.py --name deeplabv3 --output_dir /opt/openvino_toolkit/models/deeplabv3/output - sudo python3 converter.py --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert - ``` - -* Copy label files (execute once) -* Before launch, copy label files to the same model path, make sure the model path and label path match the ros_openvino_toolkit/vino_launch/param/xxxx.yaml. -``` - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 - sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels -``` -### Install OpenVINO 2022.1 by PIP -* OMZ tools are provided for downloading and converting OMZ models in ov2022 ([guide](https://pypi.org/project/openvino-dev/)). - -* See all available models -``` -omz_downloader --print_all -``` - -* Download the optimized Intermediate Representation (IR) of model (execute once), for example: -``` -omz_downloader --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output -omz_downloader --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output -omz_downloader --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output -omz_downloader --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output -omz_downloader --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output -omz_downloader --name person-reidentification-retail-0277 --output_dir /opt/openvino_toolkit/models/person-reidentification/output -omz_downloader --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output -omz_downloader --name semantic-segmentation-adas-0001 --output_dir /opt/openvino_toolkit/models/semantic-segmentation/output -omz_downloader --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recognition/output -omz_downloader --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output -``` -* Copy label files (execute once) -``` - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 - sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels -``` - -* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to intermediate representation (such as the model for object detection): - * mobilenet-ssd (replaced ssd_mobilenet_v2_coco to mobilenet-ssd for object detection in ov2022) - ``` - omz_downloader --name mobilenet-ssd --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output - omz_converter --name mobilenet-ssd -d /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output -o /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert - ``` - * vehicle-license-plate-detection-barrier-0123 - ``` - omz_downloader --name vehicle-license-plate-detection-barrier-0123 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output - omz_converter --name vehicle-license-plate-detection-barrier-0123 -d /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output -o /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output/convert - ``` - - * deeplabv3 - ``` - omz_downloader --name deeplabv3 --output_dir /opt/openvino_toolkit/models/deeplabv3/output - omz_converter --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert - ``` - -* Please check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. - * run face detection sample code input from StandardCamera. - ``` - roslaunch vino_launch pipeline_people.launch - ``` - * run person reidentification sample code input from StandardCamera. - ``` - roslaunch vino_launch pipeline_face_reidentification.launch - ``` - * run person reidentification sample code input from StandardCamera - ``` - roslaunch vino_launch pipeline_reidentification.launch - ``` - * run face detection sample code input from Image. - ``` - roslaunch vino_launch pipeline_image.launch - ``` - * run object sample - ``` - roslaunch vino_launch pipeline_object.launch - ``` - * run object topic sample - ``` - roslaunch vino_launch pipeline_object_topic.launch - ``` - * run object segmentation sample code input from RealSenseCamera. - ``` - roslaunch vino_launch pipeline_segmentation.launch - ``` - * run vehicle detection sample code input from StandardCamera. - ``` - roslaunch vino_launch pipeline_vehicle_detection.launch - ``` - * run video sample - ``` - roslaunch vino_launch pipeline_video.launch - ``` - -# More Information - -###### *Any security issue should be reported using process at https://01.org/security* - - - diff --git a/doc/quick_start/getting_started_with_ros_ov2.0.md b/doc/quick_start/getting_started_with_ros_ov2.0.md new file mode 100644 index 00000000..ae0e8987 --- /dev/null +++ b/doc/quick_start/getting_started_with_ros_ov2.0.md @@ -0,0 +1,152 @@ +# ROS_OpenVINO_Toolkit + +**NOTE:** +Below steps have been tested on **Ubuntu 18.04** and **Ubuntu 20.04**. +Supported ROS versions include noetic and melodic. + +## 1. Environment Setup +* For ROS noetic on ubuntu 20.04: + * Install ROS. ([noetic_guide](http://wiki.ros.org/noetic/Installation/Ubuntu)) + + * Install Intel® OpenVINO™ Toolkit Version: 2022.1. ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)) + * Install from an achive file. Both runtime and development tool are needed, `pip` is recommended for installing the development tool. ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)) + + * Install Intel® RealSense™ SDK. ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)) + +* For ROS melodic on ubuntu 18.04: + * Install ROS. ([melodic_guide](http://wiki.ros.org/melodic/Installation/Ubuntu)) + + * Install Intel® OpenVINO™ Toolkit Version: 2022.1. ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)) + * Install from an achive file. Both runtime and development tool are needed, `pip` is recommended for installing the development tool. ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)) + + * Install Intel® RealSense™ SDK. ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)) + +* Install Dependency + * Install gflags + ``` + sudo apt-get install -y libgflags-dev + ``` + +## 2. Build and Installation +* Install ROS_OpenVINO_Toolkit packages +``` +mkdir -p ~/catkin_ws/src +cd ~/catkin_ws/src +git clone https://github.com/intel/ros_openvino_toolkit -b ros +git clone https://github.com/intel/object_msgs +git clone https://github.com/ros-perception/vision_opencv.git -b +git clone https://github.com/IntelRealSense/realsense-ros.git +cd realsense-ros +git checkout `git tag | sort -V | grep -P "^2.\d+\.\d+" | tail -1` +``` +* Install dependencies +``` +sudo apt-get install ros--dynamic-reconfigure +sudo apt install python3-colcon-common-extensions +``` +* Build package +``` +source /opt/ros//setup.bash +source /setupvars.sh +cd ~/catkin_ws +catkin_make_isolated +source ./devel_isolated/setup.bash +``` + +## 3. Running the Demo +### Install OpenVINO 2022.1 by PIP +* OMZ tools are provided for downloading and converting models of open_model_zoo in ov2022.([guide](https://pypi.org/project/openvino-dev/)) + +* See all available models +``` +omz_downloader --print_all +``` + +* Download the optimized Intermediate Representation (IR) of model (execute once), for example: +``` +cd ~/catkin_ws/src/ros_openvino_toolkit/data/model_list +omz_downloader --list download_model.lst -o /opt/openvino_toolkit/models/ +``` + +* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to intermediate representation (such as the model for object detection): +``` +cd ~/catkin_ws/src/ros_openvino_toolkit/data/model_list +omz_converter --list convert_model.lst -o /opt/openvino_toolkit/models/convert +``` +### Install OpenVINO 2022.1 by source code +* See all available models +``` +cd ~/openvino/thirdparty/open_model_zoo/tools/model_tools +sudo python3 downloader.py --print_all +``` + +* Download the optimized Intermediate Representation (IR) of models (execute once), for example: +``` +cd ~/openvino/thirdparty/open_model_zoo/tools/model_tools +sudo python3 downloader.py --list ~/catkin_ws/src/ros_openvino_toolkit/data/model_list/download_model.lst -o /opt/openvino_toolkit/models/ +``` + +* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to Intermediate Representation (such as the model for object detection): +``` +cd ~/openvino/thirdparty/open_model_zoo/tools/model_tools +sudo python3 converter.py --list ~/catkin_ws/src/ros_openvino_toolkit/data/model_list/convert_model.lst -o /opt/openvino_toolkit/models/convert +``` + +* Copy label files (execute once) +**Note**:Need to make label_dirs if skip steps for set output_dirs above. +* Before launch, copy label files to the same model path, make sure the model path and label path match the ros_openvino_toolkit/vino_launch/param/xxxx.yaml. +``` + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 + sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels +``` + +* Please check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. + * run face detection sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_people.launch + ``` + * run person reidentification sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_face_reidentification.launch + ``` + * run person reidentification sample code input from StandardCamera + ``` + roslaunch vino_launch pipeline_reidentification.launch + ``` + * run face detection sample code input from Image. + ``` + roslaunch vino_launch pipeline_image.launch + ``` + * run object sample + ``` + roslaunch vino_launch pipeline_object.launch + ``` + * run object topic sample + ``` + roslaunch vino_launch pipeline_object_topic.launch + ``` + * run object segmentation sample code input from RealSenseCamera. + ``` + roslaunch vino_launch pipeline_segmentation.launch + ``` + * run vehicle detection sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_vehicle_detection.launch + ``` + * run video sample + ``` + roslaunch vino_launch pipeline_video.launch + ``` + +# More Information + +###### *Any security issue should be reported using process at https://01.org/security* + + + From 474c8dd4b7990f8b1155ff6e983dfaf2e0e39c5d Mon Sep 17 00:00:00 2001 From: wujiawei Date: Thu, 8 Dec 2022 13:49:47 +0800 Subject: [PATCH 29/40] add dockerfile and instruction for ros --- .../getting_started_with_ros_ov2.0.md | 2 +- docker/Dockerfile | 80 ++++++++++ docker/docker_instruction.md | 139 ++++++++++++++++++ 3 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile create mode 100644 docker/docker_instruction.md diff --git a/doc/quick_start/getting_started_with_ros_ov2.0.md b/doc/quick_start/getting_started_with_ros_ov2.0.md index ae0e8987..bc4b69a1 100644 --- a/doc/quick_start/getting_started_with_ros_ov2.0.md +++ b/doc/quick_start/getting_started_with_ros_ov2.0.md @@ -41,7 +41,7 @@ git checkout `git tag | sort -V | grep -P "^2.\d+\.\d+" | tail -1` ``` * Install dependencies ``` -sudo apt-get install ros--dynamic-reconfigure +sudo apt-get install ros--ddynamic-reconfigure sudo apt install python3-colcon-common-extensions ``` * Build package diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..58d5a7f7 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,80 @@ +# ros openvino toolkit env master ec5b9b9716e4 + +ARG ROS_VERSION +FROM osrf/ros:${ROS_VERSION} +ARG VERSION + +# setting proxy env --option +# If needed, enable the below ENV setting by correct proxies. +# ENV HTTP_PROXY="your_proxy" +# ENV HTTPS_PROXY="your_proxy" +# ENV FTP_PROXY="your_proxy" + +# maintainer information +LABEL maintainer="Jiawei Wu " + +# default shell type +SHELL ["/bin/bash", "-c"] + +# ignore the warning +ARG DEBIAN_FRONTEND=noninteractive +ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 +RUN apt-get update && apt-get install --assume-yes apt-utils + +#update cmake +WORKDIR /tmp +RUN apt-get install -y wget && wget https://github.com/Kitware/CMake/releases/download/v3.14.4/cmake-3.14.4-Linux-x86_64.sh \ +&& chmod +x cmake-3.14.4-Linux-x86_64.sh \ +&& ./cmake-3.14.4-Linux-x86_64.sh --prefix=/usr/local --exclude-subdir --skip-license \ +&& rm ./cmake-3.14.4-Linux-x86_64.sh + +# install openvino 2022.1 +# https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_apt.html +RUN apt update && apt install --assume-yes curl gnupg2 lsb-release +RUN curl -s https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB |apt-key add - +RUN echo "deb https://apt.repos.intel.com/openvino/2022 $(lsb_release -cs) main" |tee /etc/apt/sources.list.d/intel-openvino-2022.list +RUN apt update && apt-cache search openvino-2022.1.0 +RUN apt install --assume-yes openvino-2022.1.0 +RUN ls -lh /opt/intel/openvino_2022 +RUN source /opt/intel/openvino_2022/setupvars.sh + +# install librealsense2 +RUN apt-get install -y --no-install-recommends \ +software-properties-common +# https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md +# Make sure you set http-proxy in below commands if your environment needs. +# RUN apt-key adv --keyserver-options http-proxy=your_proxy --keyserver keys.gnupg.net --recv-key F6E65AC044F831AC80A06380C8B3A55A6F3EFCDE || apt-key adv --keyserver-options http-proxy=your_proxy --keyserver hkp://keyserver.ubuntu.com:80 --recv-key F6E65AC044F831AC80A06380C8B3A55A6F3EFCDE +RUN apt-key adv --keyserver-options http-proxy=${HTTP_PROXY} --keyserver keyserver.ubuntu.com --recv-key F6E65AC044F831AC80A06380C8B3A55A6F3EFCDE || apt-key adv --keyserver-options http-proxy=${HTTP_PROXY} --keyserver hkp://keyserver.ubuntu.com:80 --recv-key F6E65AC044F831AC80A06380C8B3A55A6F3EFCDE +RUN add-apt-repository "deb https://librealsense.intel.com/Debian/apt-repo $(lsb_release -cs) main" -u \ +&& apt-get install -y --no-install-recommends \ +librealsense2-dkms \ +librealsense2-utils \ +librealsense2-dev \ +librealsense2-dbg \ +libgflags-dev \ +libboost-all-dev \ +&& rm -rf /var/lib/apt/lists/* + +# other dependencies +RUN apt-get update && apt-get install -y python3-pip && python3 -m pip install -U \ +numpy \ +networkx \ +pyyaml \ +requests \ +&& apt-get install -y --no-install-recommends libboost-all-dev +WORKDIR /usr/lib/x86_64-linux-gnu +RUN ln -sf libboost_python-py36.so libboost_python37.so + + +# build ros openvino toolkit +# set env before build ros openvino tolkkit +WORKDIR /root +RUN mkdir -p catkin_ws/src +WORKDIR /root/catkin_ws/src +RUN git init && git clone -b ros https://github.com/intel/ros_openvino_toolkit \ +&& git clone https://github.com/intel/object_msgs.git \ +&& git clone -b ${VERSION} https://github.com/ros-perception/vision_opencv.git \ +&& git clone -b ros1-legacy https://github.com/IntelRealSense/realsense-ros.git +RUN apt-get install ros-${VERSION}-ddynamic-reconfigure +WORKDIR /root/catkin_ws/ +RUN source /opt/ros/${VERSION}/setup.bash && source /opt/intel/openvino_2022/setupvars.sh && catkin_make_isolated diff --git a/docker/docker_instruction.md b/docker/docker_instruction.md new file mode 100644 index 00000000..8cb76715 --- /dev/null +++ b/docker/docker_instruction.md @@ -0,0 +1,139 @@ +# Run Docker Images For ROS_OpenVINO_Toolkit + +**NOTE:** +Below steps have been tested on **Ubuntu 20.04** and **Ubuntu 18.04** . +Supported ros versions include noetic and melodic. + +## 1. Environment Setup +* Install docker ([guide](https://docs.docker.com/engine/install/ubuntu/)) + +## 2. Build docker image by dockerfile +``` +cd ~/ros_openvino_toolkit/docker/Dockerfile +vi ~/ros_openvino_toolkit/docker/Dockerfile +docker build --build-arg ROS_VERSION= --build-arg VERSION= --build-arg "HTTP_PROXY=set_your_proxy" -t ros_openvino_202201 . +``` +For example: +* Build image for ros_noetic +``` +cd ~/ros_openvino_toolkit/docker/Dockerfile +vi ~/ros_openvino_toolkit/docker/Dockerfile +docker build --build-arg ROS_VERSION=noetic-desktop-full --build-arg VERSION=noetic --build-arg "HTTP_PROXY=set_your_proxy" -t ros_noetic_openvino_202201 . +``` +* Build image for ros_melodic +``` +cd ~/ros_openvino_toolkit/docker/Dockerfile +vi ~/ros_openvino_toolkit/docker/Dockerfile +docker build --build-arg ROS_VERSION=melodic-desktop-full --build-arg VERSION=melodic --build-arg "HTTP_PROXY=set_your_proxy" -t ros_melodic_openvino_202201 . +``` + +## 3. Download and load docker image +* Download docker image +``` + # ros_openvino_202201 for demo + cd ~/Downloads/ + wget +``` +* Load docker image +``` +cd ~/Downloads/ +docker load -i +docker images +// (show in the list) +``` + +## 4. Running the Demos +* Install dependency +``` + sudo apt install x11-xserver-utils + xhost + +``` +* Run docker image +``` + docker images + docker run -itd  -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix -v /dev:/dev  --privileged=true --name +``` +* In Docker Container + +* Preparation +``` +source /opt/intel/openvino_2022/setupvars.sh +source /opt/ros//setup.bash +cd ~/catkin_ws +source ./devel_isolated/setup.bash +``` + +* See all available models +OMZ tools are provided for downloading and converting OMZ models in ov2022.([guide](https://pypi.org/project/openvino-dev/)) + +``` +omz_downloader --print_all +``` + +* Download the optimized Intermediate Representation (IR) of model (execute once), for example: +``` +cd ~/catkin_ws/src/ros_openvino_toolkit/data/model_list +omz_downloader --list download_model.lst -o /opt/openvino_toolkit/models/ +``` + +* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to intermediate representation (such as the model for object detection): +``` +cd ~/catkin_ws/src/ros_openvino_toolkit/data/model_list +omz_converter --list convert_model.lst -o /opt/openvino_toolkit/models/convert +``` +* Copy label files (execute once) +**Note**:Need to make label_dirs if skip steps for set output_dirs above. +``` + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 + sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels +``` + +* Before launch, check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml, make sure the paramter like model path, label path, inputs are right. + * run face detection sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_people.launch + ``` + * run person reidentification sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_face_reidentification.launch + ``` + * run person reidentification sample code input from StandardCamera + ``` + roslaunch vino_launch pipeline_reidentification.launch + ``` + * run face detection sample code input from Image. + ``` + roslaunch vino_launch pipeline_image.launch + ``` + * run object sample + ``` + roslaunch vino_launch pipeline_object.launch + ``` + * run object topic sample + ``` + roslaunch vino_launch pipeline_object_topic.launch + ``` + * run object segmentation sample code input from RealSenseCamera. + ``` + roslaunch vino_launch pipeline_segmentation.launch + ``` + * run vehicle detection sample code input from StandardCamera. + ``` + roslaunch vino_launch pipeline_vehicle_detection.launch + ``` + * run video sample + ``` + roslaunch vino_launch pipeline_video.launch + ``` + +# More Information +* ros OpenVINO discription writen in Chinese: https://mp.weixin.qq.com/s/BgG3RGauv5pmHzV_hkVAdw + +###### *Any security issue should be reported using process at https://01.org/security* + From a479937a8bcc621cb3140c4a4f854c54fe97ff22 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 12 Dec 2022 09:46:14 +0800 Subject: [PATCH 30/40] refine dockerfile --- docker/Dockerfile | 3 ++- docker/docker_instruction.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 58d5a7f7..acd9dec8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -64,7 +64,8 @@ requests \ && apt-get install -y --no-install-recommends libboost-all-dev WORKDIR /usr/lib/x86_64-linux-gnu RUN ln -sf libboost_python-py36.so libboost_python37.so - +RUN pip install --upgrade pip +RUN pip install openvino-dev[tensorflow2]==2022.1 # build ros openvino toolkit # set env before build ros openvino tolkkit diff --git a/docker/docker_instruction.md b/docker/docker_instruction.md index 8cb76715..7ffdc7fa 100644 --- a/docker/docker_instruction.md +++ b/docker/docker_instruction.md @@ -79,7 +79,7 @@ omz_downloader --list download_model.lst -o /opt/openvino_toolkit/models/ * If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to intermediate representation (such as the model for object detection): ``` cd ~/catkin_ws/src/ros_openvino_toolkit/data/model_list -omz_converter --list convert_model.lst -o /opt/openvino_toolkit/models/convert +omz_converter --list convert_model.lst -d /opt/openvino_toolkit/models/ -o /opt/openvino_toolkit/models/convert ``` * Copy label files (execute once) **Note**:Need to make label_dirs if skip steps for set output_dirs above. From fed6345d9adc295e8dce9c304a5037005dc37e62 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 12 Dec 2022 14:58:12 +0800 Subject: [PATCH 31/40] add segmentation maskrcnn pipeline --- .../getting_started_with_ros_ov2.0.md | 2 +- vino_core_lib/CMakeLists.txt | 2 + .../inferences/object_segmentation_maskrcnn.h | 145 +++++++++++ .../object_segmentation_maskrcnn_model.h | 61 +++++ .../vino_core_lib/outputs/base_output.h | 7 + .../outputs/image_window_output.h | 7 + .../vino_core_lib/outputs/ros_topic_output.h | 6 + .../vino_core_lib/outputs/rviz_output.h | 6 + .../include/vino_core_lib/pipeline_manager.h | 2 + .../include/vino_core_lib/pipeline_params.h | 1 + .../object_segmentation_maskrcnn.cpp | 237 ++++++++++++++++++ .../object_segmentation_maskrcnn_model.cpp | 229 +++++++++++++++++ .../src/outputs/image_window_output.cpp | 77 +++++- .../src/outputs/ros_topic_output.cpp | 23 ++ vino_core_lib/src/outputs/rviz_output.cpp | 6 + vino_core_lib/src/pipeline_manager.cpp | 24 ++ .../pipeline_segmentation_maskrcnn.launch | 10 + .../param/pipeline_segmentation_maskrcnn.yaml | 22 ++ 18 files changed, 861 insertions(+), 6 deletions(-) create mode 100644 vino_core_lib/include/vino_core_lib/inferences/object_segmentation_maskrcnn.h create mode 100644 vino_core_lib/include/vino_core_lib/models/object_segmentation_maskrcnn_model.h create mode 100644 vino_core_lib/src/inferences/object_segmentation_maskrcnn.cpp create mode 100644 vino_core_lib/src/models/object_segmentation_maskrcnn_model.cpp create mode 100644 vino_launch/launch/pipeline_segmentation_maskrcnn.launch create mode 100644 vino_launch/param/pipeline_segmentation_maskrcnn.yaml diff --git a/doc/quick_start/getting_started_with_ros_ov2.0.md b/doc/quick_start/getting_started_with_ros_ov2.0.md index bc4b69a1..9feb3c8c 100644 --- a/doc/quick_start/getting_started_with_ros_ov2.0.md +++ b/doc/quick_start/getting_started_with_ros_ov2.0.md @@ -71,7 +71,7 @@ omz_downloader --list download_model.lst -o /opt/openvino_toolkit/models/ * If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to intermediate representation (such as the model for object detection): ``` cd ~/catkin_ws/src/ros_openvino_toolkit/data/model_list -omz_converter --list convert_model.lst -o /opt/openvino_toolkit/models/convert +omz_converter --list convert_model.lst -d /opt/openvino_toolkit/models/ -o /opt/openvino_toolkit/models/convert ``` ### Install OpenVINO 2022.1 by source code * See all available models diff --git a/vino_core_lib/CMakeLists.txt b/vino_core_lib/CMakeLists.txt index 726951ec..f9615bd1 100644 --- a/vino_core_lib/CMakeLists.txt +++ b/vino_core_lib/CMakeLists.txt @@ -150,6 +150,7 @@ add_library(${PROJECT_NAME} SHARED src/inferences/head_pose_detection.cpp src/inferences/object_detection.cpp src/inferences/object_segmentation.cpp + src/inferences/object_segmentation_maskrcnn.cpp src/inferences/person_reidentification.cpp src/inferences/face_reidentification.cpp src/inferences/base_reidentification.cpp @@ -171,6 +172,7 @@ add_library(${PROJECT_NAME} SHARED src/models/object_detection_ssd_model.cpp # src/models/object_detection_yolov2voc_model.cpp src/models/object_segmentation_model.cpp + src/models/object_segmentation_maskrcnn_model.cpp src/models/person_reidentification_model.cpp src/models/face_reidentification_model.cpp src/models/person_attribs_detection_model.cpp diff --git a/vino_core_lib/include/vino_core_lib/inferences/object_segmentation_maskrcnn.h b/vino_core_lib/include/vino_core_lib/inferences/object_segmentation_maskrcnn.h new file mode 100644 index 00000000..7e5a53f1 --- /dev/null +++ b/vino_core_lib/include/vino_core_lib/inferences/object_segmentation_maskrcnn.h @@ -0,0 +1,145 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief A header file with declaration for ObjectSegmentationMaskrcnn Class + * @file object_detection.h + */ +#ifndef VINO_CORE_LIB__INFERENCES__OBJECT_SEGMENTATION_MASKRCNN_H +#define VINO_CORE_LIB__INFERENCES__OBJECT_SEGMENTATION_MASKRCNN_H +#include +#include +#include +#include +#include +#include +#include "vino_core_lib/models/object_segmentation_maskrcnn_model.h" +#include "vino_core_lib/engines/engine.h" +#include "vino_core_lib/inferences/base_inference.h" +#include "openvino/openvino.hpp" +#include "opencv2/opencv.hpp" +// namespace +namespace vino_core_lib +{ +/** + * @class ObjectSegmentationMaskrcnnResult + * @brief Class for storing and processing object segmentation result. + */ +class ObjectSegmentationMaskrcnnResult : public Result +{ +public: + friend class ObjectSegmentationMaskrcnn; + explicit ObjectSegmentationMaskrcnnResult(const cv::Rect & location); + std::string getLabel() const + { + return label_; + } + /** + * @brief Get the confidence that the detected area is a face. + * @return The confidence value. + */ + float getConfidence() const + { + return confidence_; + } + cv::Mat getMask() const + { + return mask_; + } + +private: + std::string label_ = ""; + float confidence_ = -1; + cv::Mat mask_; +}; +/** + * @class ObjectSegmentation + * @brief Class to load object segmentation model and perform object segmentation. + */ +class ObjectSegmentationMaskrcnn : public BaseInference +{ +public: + using Result = vino_core_lib::ObjectSegmentationMaskrcnnResult; + explicit ObjectSegmentationMaskrcnn(double); + ~ObjectSegmentationMaskrcnn() override; + /** + * @brief Load the object segmentation model. + */ + void loadNetwork(std::shared_ptr); + /** + * @brief Enqueue a frame to this class. + * The frame will be buffered but not infered yet. + * @param[in] frame The frame to be enqueued. + * @param[in] input_frame_loc The location of the enqueued frame with respect + * to the frame generated by the input device. + * @return Whether this operation is successful. + */ + bool enqueue(const cv::Mat &, const cv::Rect &) override; + + //Deprecated!! + bool enqueue_for_one_input(const cv::Mat &, const cv::Rect &); + + /** + * @brief Start inference for all buffered frames. + * @return Whether this operation is successful. + */ + bool submitRequest() override; + /** + * @brief This function will fetch the results of the previous inference and + * stores the results in a result buffer array. All buffered frames will be + * cleared. + * @return Whether the Inference object fetches a result this time + */ + bool fetchResults() override; + /** + * @brief Get the length of the buffer result array. + * @return The length of the buffer result array. + */ + int getResultsLength() const override; + /** + * @brief Get the location of result with respect + * to the frame generated by the input device. + * @param[in] idx The index of the result. + */ + const vino_core_lib::Result * getLocationResult(int idx) const override; + /** + * @brief Show the observed detection result either through image window + or ROS topic. + */ + void observeOutput(const std::shared_ptr & output); + /** + * @brief Get the name of the Inference instance. + * @return The name of the Inference instance. + */ + const std::string getName() const override; + const std::vector getFilteredROIs( + const std::string filter_conditions) const override; + +private: + std::shared_ptr valid_model_; + std::vector results_; + int width_ = 0; + int height_ = 0; + double show_output_thresh_ = 0; + + std::vector colors_ = { + {128, 64, 128}, {232, 35, 244}, {70, 70, 70}, {156, 102, 102}, {153, 153, 190}, + {153, 153, 153}, {30, 170, 250}, {0, 220, 220}, {35, 142, 107}, {152, 251, 152}, + {180, 130, 70}, {60, 20, 220}, {0, 0, 255}, {142, 0, 0}, {70, 0, 0}, + {100, 60, 0}, {90, 0, 0}, {230, 0, 0}, {32, 11, 119}, {0, 74, 111}, + {81, 0, 81} + }; +}; +} // namespace vino_core_lib +#endif // VINO_CORE_LIB__INFERENCES__OBJECT_SEGMENTATION_H diff --git a/vino_core_lib/include/vino_core_lib/models/object_segmentation_maskrcnn_model.h b/vino_core_lib/include/vino_core_lib/models/object_segmentation_maskrcnn_model.h new file mode 100644 index 00000000..b134bf58 --- /dev/null +++ b/vino_core_lib/include/vino_core_lib/models/object_segmentation_maskrcnn_model.h @@ -0,0 +1,61 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/** + * @brief A header file with declaration for ObjectSegmentationMaskrcnnModel Class + * @file face_detection_model.h + */ +#ifndef VINO_CORE_LIB__MODELS__OBJECT_SEGMENTATION_MASKRCNN_MODEL_H +#define VINO_CORE_LIB__MODELS__OBJECT_SEGMENTATION_MASKRCNN_MODEL_H +#include +#include +#include "vino_core_lib/models/base_model.h" +namespace Models +{ +/** + * @class ObjectSegmentationMaskrcnnModel + * @brief This class generates the object segmentation maskrcnn model. + */ +class ObjectSegmentationMaskrcnnModel : public BaseModel +{ +public: + ObjectSegmentationMaskrcnnModel(const std::string& label_loc, const std::string & model_loc, int batch_size = 1); + inline int getMaxProposalCount() const + { + return max_proposal_count_; + } + inline int getObjectSize() const + { + return object_size_; + } + + bool enqueue(const std::shared_ptr & ,const cv::Mat &, + const cv::Rect & ) override; + + bool matToBlob( + const cv::Mat & , const cv::Rect &, float , + int , const std::shared_ptr & ); + + /** + * @brief Get the name of this segmentation model. + * @return Name of the model. + */ + const std::string getModelCategory() const override; + bool updateLayerProperty(std::shared_ptr&) override; + +private: + int max_proposal_count_; + int object_size_; +}; +} // namespace Models +#endif // VINO_CORE_LIB__MODELS__OBJECT_SEGMENTATION_MASKRCNN_MODEL_H \ No newline at end of file diff --git a/vino_core_lib/include/vino_core_lib/outputs/base_output.h b/vino_core_lib/include/vino_core_lib/outputs/base_output.h index 8270d433..7d823c9f 100644 --- a/vino_core_lib/include/vino_core_lib/outputs/base_output.h +++ b/vino_core_lib/include/vino_core_lib/outputs/base_output.h @@ -31,6 +31,7 @@ #include "vino_core_lib/inferences/face_detection.h" #include "vino_core_lib/inferences/head_pose_detection.h" #include "vino_core_lib/inferences/object_segmentation.h" +#include "vino_core_lib/inferences/object_segmentation_maskrcnn.h" #include "vino_core_lib/inferences/person_reidentification.h" #include "vino_core_lib/inferences/landmarks_detection.h" #include "vino_core_lib/inferences/face_reidentification.h" @@ -133,6 +134,12 @@ class BaseOutput virtual void accept(const std::vector&) { } + /** + * @brief Generate output content according to the object segmentation maskrcnn result. + */ + virtual void accept(const std::vector &) + { + } /** * @brief Generate output content according to the person reidentification result. */ diff --git a/vino_core_lib/include/vino_core_lib/outputs/image_window_output.h b/vino_core_lib/include/vino_core_lib/outputs/image_window_output.h index f514c0cc..16028463 100644 --- a/vino_core_lib/include/vino_core_lib/outputs/image_window_output.h +++ b/vino_core_lib/include/vino_core_lib/outputs/image_window_output.h @@ -119,6 +119,12 @@ class ImageWindowOutput : public BaseOutput * @param[in] An object segmentation result objetc. */ void accept(const std::vector&) override; + /** + * @brief Generate image window output content according to + * the object segmentation maskrcnn result. + * @param[in] An obejct segmentation result objetc. + */ + void accept(const std::vector &) override; /** * @brief Generate image window output content according to * the person re-ID result. @@ -131,6 +137,7 @@ class ImageWindowOutput : public BaseOutput * @param[in] An object segmentation result objetc. */ void mergeMask(const std::vector&); + void mergeMask(const std::vector &); private: unsigned findOutput(const cv::Rect&); diff --git a/vino_core_lib/include/vino_core_lib/outputs/ros_topic_output.h b/vino_core_lib/include/vino_core_lib/outputs/ros_topic_output.h index a2e24385..e61bed09 100644 --- a/vino_core_lib/include/vino_core_lib/outputs/ros_topic_output.h +++ b/vino_core_lib/include/vino_core_lib/outputs/ros_topic_output.h @@ -116,6 +116,12 @@ class RosTopicOutput : public BaseOutput * @param[in] results a bundle of object segmentation results. */ void accept(const std::vector&) override; + /** + * @brief Generate ros topic infomation according to + * the object segmentation result. + * @param[in] results a bundle of object segmentation maskrcnn results. + */ + void accept(const std::vector &) override; /** * @brief Generate ros topic infomation according to * the face detection result. diff --git a/vino_core_lib/include/vino_core_lib/outputs/rviz_output.h b/vino_core_lib/include/vino_core_lib/outputs/rviz_output.h index 6e19d653..79e5267b 100644 --- a/vino_core_lib/include/vino_core_lib/outputs/rviz_output.h +++ b/vino_core_lib/include/vino_core_lib/outputs/rviz_output.h @@ -108,6 +108,12 @@ class RvizOutput : public BaseOutput * @param[in] An object segmentation result objetc. */ void accept(const std::vector&) override; + /** + * @brief Generate rviz output content according to + * the object segmentation result. + * @param[in] results A bundle of object segmentation maskrcnn results. + */ + void accept(const std::vector &) override; /** * @brief Generate rviz output content according to * the person re-ID result. diff --git a/vino_core_lib/include/vino_core_lib/pipeline_manager.h b/vino_core_lib/include/vino_core_lib/pipeline_manager.h index cec91d86..e7703163 100644 --- a/vino_core_lib/include/vino_core_lib/pipeline_manager.h +++ b/vino_core_lib/include/vino_core_lib/pipeline_manager.h @@ -108,6 +108,8 @@ class PipelineManager std::shared_ptr createObjectSegmentation(const Params::ParamManager::InferenceRawData& infer); std::shared_ptr + createObjectSegmentationMaskrcnn(const Params::ParamManager::InferenceRawData & infer); + std::shared_ptr createPersonReidentification(const Params::ParamManager::InferenceRawData& infer); std::shared_ptr createFaceReidentification(const Params::ParamManager::InferenceRawData& infer); diff --git a/vino_core_lib/include/vino_core_lib/pipeline_params.h b/vino_core_lib/include/vino_core_lib/pipeline_params.h index c8c606bf..e28205f2 100644 --- a/vino_core_lib/include/vino_core_lib/pipeline_params.h +++ b/vino_core_lib/include/vino_core_lib/pipeline_params.h @@ -55,6 +55,7 @@ const char kInferTpye_EmotionRecognition[] = "EmotionRecognition"; const char kInferTpye_HeadPoseEstimation[] = "HeadPoseEstimation"; const char kInferTpye_ObjectDetection[] = "ObjectDetection"; const char kInferTpye_ObjectSegmentation[] = "ObjectSegmentation"; +const char kInferTpye_ObjectSegmentationMaskrcnn[] = "ObjectSegmentationMaskrcnn"; const char kInferTpye_PersonReidentification[] = "PersonReidentification"; const char kInferTpye_ObjectDetectionTypeSSD[] = "SSD"; const char kInferTpye_ObjectDetectionTypeYolov2voc[] = "yolov2-voc"; diff --git a/vino_core_lib/src/inferences/object_segmentation_maskrcnn.cpp b/vino_core_lib/src/inferences/object_segmentation_maskrcnn.cpp new file mode 100644 index 00000000..792cc17b --- /dev/null +++ b/vino_core_lib/src/inferences/object_segmentation_maskrcnn.cpp @@ -0,0 +1,237 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of ObjectSegmentation class and + * ObjectSegmentationResult class + * @file object_segmentation.cpp + */ +#include +#include +#include +#include +#include + +#include +#include "vino_core_lib/inferences/object_segmentation_maskrcnn.h" +#include "vino_core_lib/outputs/base_output.h" +#include "vino_core_lib/slog.h" + +// ObjectSegmentationResult +vino_core_lib::ObjectSegmentationMaskrcnnResult::ObjectSegmentationMaskrcnnResult(const cv::Rect &location) + : Result(location) +{ +} + +// ObjectSegmentation +vino_core_lib::ObjectSegmentationMaskrcnn::ObjectSegmentationMaskrcnn(double show_output_thresh) + : show_output_thresh_(show_output_thresh), vino_core_lib::BaseInference() +{ +} + +vino_core_lib::ObjectSegmentationMaskrcnn::~ObjectSegmentationMaskrcnn() = default; + +void vino_core_lib::ObjectSegmentationMaskrcnn::loadNetwork( + const std::shared_ptr network) +{ + slog::info << "Loading Network: " << network->getModelCategory() << slog::endl; + valid_model_ = network; + setMaxBatchSize(network->getMaxBatchSize()); +} + +/** + * Deprecated! + * This function only support OpenVINO version <=2018R5 + */ +bool vino_core_lib::ObjectSegmentationMaskrcnn::enqueue_for_one_input( + const cv::Mat &frame, + const cv::Rect &input_frame_loc) +{ + if (width_ == 0 && height_ == 0) + { + width_ = frame.cols; + height_ = frame.rows; + } + if (!vino_core_lib::BaseInference::enqueue(frame, input_frame_loc, 1, 0, + valid_model_->getInputName())) + { + return false; + } + Result r(input_frame_loc); + results_.clear(); + results_.emplace_back(r); + return true; +} + +bool vino_core_lib::ObjectSegmentationMaskrcnn::enqueue( + const cv::Mat &frame, + const cv::Rect &input_frame_loc) +{ + if (width_ == 0 && height_ == 0) + { + width_ = frame.cols; + height_ = frame.rows; + } + + if (valid_model_ == nullptr || getEngine() == nullptr) + { + throw std::logic_error("Model or Engine is not set correctly!"); + return false; + } + + if (enqueued_frames_ >= valid_model_->getMaxBatchSize()) + { + slog::warn << "Number of " << getName() << "input more than maximum(" << + max_batch_size_ << ") processed by inference" << slog::endl; + return false; + } + + if (!valid_model_->enqueue(getEngine(), frame, input_frame_loc)) + { + return false; + } + + enqueued_frames_ += 1; + return true; +} + +bool vino_core_lib::ObjectSegmentationMaskrcnn::submitRequest() +{ + return vino_core_lib::BaseInference::submitRequest(); +} + +bool vino_core_lib::ObjectSegmentationMaskrcnn::fetchResults() +{ + + bool can_fetch = vino_core_lib::BaseInference::fetchResults(); + if (!can_fetch) + { + return false; + } + bool found_result = false; + results_.clear(); + ov::InferRequest infer_request = getEngine()->getRequest(); + slog::debug << "Analyzing Detection results..." << slog::endl; + std::string detection_output = valid_model_->getOutputName("detection"); + std::string mask_output = valid_model_->getOutputName("masks"); + slog::debug << "Detection_output=" << detection_output << ", Mask_output=" << mask_output << slog::endl; + + //get detection data + ov::Tensor do_tensor = infer_request.get_tensor(detection_output.c_str()); + const auto do_data = do_tensor.data(); + ov::Shape do_shape = do_tensor.get_shape(); + slog::debug << "Detection Blob getDims = " <(); + ov::Shape mask_shape = mask_tensor.get_shape(); + + // determine models + size_t box_description_size = do_shape.back(); + OPENVINO_ASSERT(mask_shape.size() == 4); + size_t box_num = mask_shape[0]; + size_t C = mask_shape[1]; + size_t H = mask_shape[2]; + size_t W = mask_shape[3]; + size_t box_stride = W * H * C; + slog::debug << "box_description is:" << box_description_size << slog::endl; + slog::debug << "box_num is:" << box_num<< slog::endl; + slog::debug << "C is:" << C << slog::endl; + slog::debug << "H is:" << H << slog::endl; + slog::debug << "W is:" << W << slog::endl; + + for (size_t box = 0; box < box_num; ++box) { + // box description: batch, label, prob, x1, y1, x2, y2 + float * box_info = do_data + box * box_description_size; + auto batch = static_cast(box_info[0]); + slog::debug << "batch =" << batch << slog::endl; + if (batch < 0) { + break; + } + float prob = box_info[2]; + if (prob > show_output_thresh_) { + float x1 = std::min(std::max(0.0f, box_info[3] * width_), static_cast(width_)); + float y1 = std::min(std::max(0.0f, box_info[4] * height_), static_cast(height_)); + float x2 = std::min(std::max(0.0f, box_info[5] * width_), static_cast(width_)); + float y2 = std::min(std::max(0.0f, box_info[6] * height_), static_cast(height_)); + int box_width = static_cast(x2 - x1); + int box_height = static_cast(y2 - y1); + slog::debug << "Box[" << box_width << "x" << box_height << "]" << slog::endl; + if (box_width <= 0 || box_height <=0) break; + int class_id = static_cast(box_info[1] + 1e-6f); + float * mask_arr = mask_data + box_stride * box + H * W * (class_id - 1); + slog::info << "Detected class " << class_id << " with probability " << prob << " from batch " << batch + << ": [" << x1 << ", " << y1 << "], [" << x2 << ", " << y2 << "]" << slog::endl; + cv::Mat mask_mat(H, W, CV_32FC1, mask_arr); + cv::Rect roi = cv::Rect(static_cast(x1), static_cast(y1), box_width, box_height); + cv::Mat resized_mask_mat(box_height, box_width, CV_32FC1); + cv::resize(mask_mat, resized_mask_mat, cv::Size(box_width, box_height)); + Result result(roi); + result.confidence_ = prob; + std::vector & labels = valid_model_->getLabels(); + result.label_ = class_id < labels.size() ? labels[class_id] : + std::string("label #") + std::to_string(class_id); + result.mask_ = resized_mask_mat; + found_result = true; + slog::debug << "adding one segmentation Box ..." << slog::endl; + results_.emplace_back(result); + } + } + if (!found_result) { + slog::debug << "No Segmentation Result Found!" << slog::endl; + results_.clear(); + } + return true; +} + +int vino_core_lib::ObjectSegmentationMaskrcnn::getResultsLength() const +{ + return static_cast(results_.size()); +} + +const vino_core_lib::Result * +vino_core_lib::ObjectSegmentationMaskrcnn::getLocationResult(int idx) const +{ + return &(results_[idx]); +} + +const std::string vino_core_lib::ObjectSegmentationMaskrcnn::getName() const +{ + return valid_model_->getModelCategory(); +} + +void vino_core_lib::ObjectSegmentationMaskrcnn::observeOutput( + const std::shared_ptr &output) +{ + if (output != nullptr) + { + output->accept(results_); + } +} + +const std::vector vino_core_lib::ObjectSegmentationMaskrcnn::getFilteredROIs( + const std::string filter_conditions) const +{ + if (!filter_conditions.empty()) + { + slog::err << "Object segmentation does not support filtering now! " + << "Filter conditions: " << filter_conditions << slog::endl; + } + std::vector filtered_rois; + for (auto res : results_) + { + filtered_rois.push_back(res.getLocation()); + } + return filtered_rois; +} diff --git a/vino_core_lib/src/models/object_segmentation_maskrcnn_model.cpp b/vino_core_lib/src/models/object_segmentation_maskrcnn_model.cpp new file mode 100644 index 00000000..cf3d47a0 --- /dev/null +++ b/vino_core_lib/src/models/object_segmentation_maskrcnn_model.cpp @@ -0,0 +1,229 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @brief a header file with declaration of ObjectSegmentationModel class + * @file object_segmentation_model.cpp + */ +#include +#include +#include +#include "vino_core_lib/models/object_segmentation_maskrcnn_model.h" +#include "vino_core_lib/inferences/object_segmentation_maskrcnn.h" +#include "vino_core_lib/slog.h" +#include "vino_core_lib/engines/engine.h" + +// Validated Object Segmentation Network +Models::ObjectSegmentationMaskrcnnModel::ObjectSegmentationMaskrcnnModel( + const std::string & label_loc, + const std::string & model_loc, + int max_batch_size) + : BaseModel(label_loc, model_loc, max_batch_size) +{ +} + +bool Models::ObjectSegmentationMaskrcnnModel::enqueue( + const std::shared_ptr &engine, + const cv::Mat &frame, + const cv::Rect &input_frame_loc) +{ + if (engine == nullptr) + { + slog::err << "A frame is trying to be enqueued in a NULL Engine." << slog::endl; + return false; + } + + for (const auto &inputInfoItem : inputs_info_) + { + // Fill first input tensor with images. First b channel, then g and r channels + auto dims = inputInfoItem.get_shape(); + slog::debug << "input tensor shape is:"<< dims.size() <getRequest().get_tensor(inputInfoItem); + auto data = in_tensor.data(); + data[0] = static_cast(frame.rows); // height + data[1] = static_cast(frame.cols); // width + data[2] = 1; + } + } + + return true; +} + +bool Models::ObjectSegmentationMaskrcnnModel::matToBlob( + const cv::Mat &orig_image, const cv::Rect &, float scale_factor, + int batch_index, const std::shared_ptr &engine) +{ + (void)scale_factor; + (void)batch_index; + + if (engine == nullptr) + { + slog::err << "A frame is trying to be enqueued in a NULL Engine." << slog::endl; + return false; + } + + ov::InferRequest infer_request = engine->getRequest(); + ov::Tensor input_tensor = infer_request.get_tensor(getInputName("input")); + ov::Shape input_shape = input_tensor.get_shape(); + + OPENVINO_ASSERT(input_shape.size() == 4); + // For frozen graph model: + const size_t width = input_shape[3]; + const size_t height = input_shape[2]; + const size_t channels = input_shape[1]; + + slog::debug <<"width is:"<< width << slog::endl; + slog::debug <<"height is:"<< height << slog::endl; + slog::debug <<"channels is:"<< channels << slog::endl; + slog::debug <<"origin channels is:"<< orig_image.channels() << slog::endl; + slog::debug <<"input shape is:"<< input_shape << slog::endl; + + if (static_cast(orig_image.channels()) != channels) { + throw std::runtime_error("The number of channels for net input and image must match"); + } + + const auto input_data = input_tensor.data(); + cv::Mat resized_image(orig_image); + if (static_cast(width) != orig_image.size().width || + static_cast(height) != orig_image.size().height) { + cv::resize(orig_image, resized_image, cv::Size(width, height)); + } + + int batchOffset = batch_index * width * height * channels; + if (channels == 1) { + for (size_t h = 0; h < height; h++) { + for (size_t w = 0; w < width; w++) { + input_data[batchOffset + h * width + w] = resized_image.at(h, w); + } + } + } else if (channels == 3) { + for (size_t c = 0; c < channels; c++) { + for (size_t h = 0; h < height; h++) { + for (size_t w = 0; w < width; w++) { + input_data[batchOffset + c * width * height + h * width + w] = + resized_image.at(h, w)[c]; + } + } + } + } else { + throw std::runtime_error("Unsupported number of channels"); + } + + return true; +} + +const std::string Models::ObjectSegmentationMaskrcnnModel::getModelCategory() const +{ + return "Object Segmentation"; +} + +bool Models::ObjectSegmentationMaskrcnnModel::updateLayerProperty( + std::shared_ptr& model) +{ + slog::info<< "Checking INPUTS for Model" <inputs(); + slog::debug<<"input size="<input("image_tensor").get_shape(); + slog::debug<<"image_tensor shape is:"<< input_shape.size() <input("image_info").get_shape(); + slog::debug<<"image_info shape is:"<< info_shape.size() <output("masks").get_shape(); + slog::debug<<"masks shape is:"<< mask_shape.size() <output("reshape_do_2d").get_shape(); + slog::debug<< "detection shape is:" << detection_shape.size() <& results) { - const float alpha = 0.5f; - cv::Mat roi_img = frame_; - cv::Mat colored_mask = results[0].getMask(); - cv::resize(colored_mask, colored_mask, cv::Size(frame_.size().width, frame_.size().height)); - cv::addWeighted(colored_mask, alpha, roi_img, 1.0f - alpha, 0.0f, roi_img); + std::map class_color; + for (unsigned i = 0; i < results.size(); i++) { + std::string class_label = results[i].getLabel(); + if (class_color.find(class_label) == class_color.end()) { + class_color[class_label] = class_color.size(); + } + auto & color = colors_[class_color[class_label] % colors_.size() ]; + const float alpha = 0.7f; + const float MASK_THRESHOLD = 0.5; + + cv::Rect location = results[i].getLocation(); + cv::Mat roi_img = frame_(location); + cv::Mat mask = results[i].getMask(); + cv::Mat colored_mask(location.height, location.width, frame_.type(), + cv::Scalar(color[2], color[1], color[0]) ); + roi_img.copyTo(colored_mask, mask <= MASK_THRESHOLD); + +/** + for (int h = 0; h < mask.size().height; ++h) { + for (int w = 0; w < mask.size().width; ++w) { + for (int ch = 0; ch < colored_mask.channels(); ++ch) { + colored_mask.at(h, w)[ch] = mask.at(h, w) > MASK_THRESHOLD ? + 255 * color[ch] : + roi_img.at(h, w)[ch]; + } + } + } +*/ + cv::addWeighted(colored_mask, alpha, roi_img, 1.0f - alpha, 0.0f, roi_img); + } } void Outputs::ImageWindowOutput::accept(const std::vector& results) @@ -232,6 +257,48 @@ void Outputs::ImageWindowOutput::accept(const std::vector & results) +{ + std::map class_color; + for (unsigned i = 0; i < results.size(); i++) { + std::string class_label = results[i].getLabel(); + if (class_color.find(class_label) == class_color.end()) { + class_color[class_label] = class_color.size(); + } + auto & color = colors_[class_color[class_label] % colors_.size() ]; + const float alpha = 0.7f; + const float MASK_THRESHOLD = 0.5; + + cv::Rect location = results[i].getLocation(); + cv::Mat roi_img = frame_(location); + cv::Mat mask = results[i].getMask(); + cv::Mat colored_mask(location.height, location.width, frame_.type(), + cv::Scalar(color[2], color[1], color[0]) ); + roi_img.copyTo(colored_mask, mask <= MASK_THRESHOLD); + cv::addWeighted(colored_mask, alpha, roi_img, 1.0f - alpha, 0.0f, roi_img); + } +} +void Outputs::ImageWindowOutput::accept( + const std::vector & results) +{ + for (unsigned i = 0; i < results.size(); i++) { + cv::Rect result_rect = results[i].getLocation(); + unsigned target_index = findOutput(result_rect); + outputs_[target_index].rect = result_rect; + auto fd_conf = results[i].getConfidence(); + if (fd_conf >= 0) { + std::ostringstream ostream; + ostream << "[" << std::fixed << std::setprecision(3) << fd_conf << "]"; + outputs_[target_index].desc += ostream.str(); + } + auto label = results[i].getLabel(); + outputs_[target_index].desc += "[" + label + "]"; + } + mergeMask(results); +} + + void Outputs::ImageWindowOutput::accept(const std::vector& results) { for (unsigned i = 0; i < results.size(); i++) diff --git a/vino_core_lib/src/outputs/ros_topic_output.cpp b/vino_core_lib/src/outputs/ros_topic_output.cpp index fa6fa147..fc917cd5 100644 --- a/vino_core_lib/src/outputs/ros_topic_output.cpp +++ b/vino_core_lib/src/outputs/ros_topic_output.cpp @@ -174,6 +174,29 @@ void Outputs::RosTopicOutput::accept(const std::vector & results) +{ + segmented_objects_topic_ = std::make_shared(); + vino_people_msgs::ObjectInMask object; + for (auto & r : results) { + auto loc = r.getLocation(); + object.roi.x_offset = loc.x; + object.roi.y_offset = loc.y; + object.roi.width = loc.width; + object.roi.height = loc.height; + object.object_name = r.getLabel(); + object.probability = r.getConfidence(); + cv::Mat mask = r.getMask(); + for (int h = 0; h < mask.size().height; ++h) { + for (int w = 0; w < mask.size().width; ++w) { + object.mask_array.push_back(mask.at(h, w)); + } + } + segmented_objects_topic_->objects_vector.push_back(object); + } +} + void Outputs::RosTopicOutput::accept(const std::vector& results) { faces_topic_ = std::make_shared(); diff --git a/vino_core_lib/src/outputs/rviz_output.cpp b/vino_core_lib/src/outputs/rviz_output.cpp index e750d6ac..141d9564 100644 --- a/vino_core_lib/src/outputs/rviz_output.cpp +++ b/vino_core_lib/src/outputs/rviz_output.cpp @@ -85,6 +85,12 @@ void Outputs::RvizOutput::accept(const std::vectoraccept(results); } + +void Outputs::RvizOutput::accept(const std::vector & results) +{ + image_window_output_->accept(results); +} + void Outputs::RvizOutput::accept(const std::vector& results) { image_window_output_->accept(results); diff --git a/vino_core_lib/src/pipeline_manager.cpp b/vino_core_lib/src/pipeline_manager.cpp index c3f72118..3234f2a4 100644 --- a/vino_core_lib/src/pipeline_manager.cpp +++ b/vino_core_lib/src/pipeline_manager.cpp @@ -32,6 +32,7 @@ #include "vino_core_lib/inferences/vehicle_attribs_detection.h" #include "vino_core_lib/inferences/license_plate_detection.h" #include "vino_core_lib/inferences/landmarks_detection.h" +#include "vino_core_lib/inferences/object_segmentation_maskrcnn.h" #include "vino_core_lib/inputs/image_input.h" #include "vino_core_lib/inputs/realsense_camera.h" #include "vino_core_lib/inputs/realsense_camera_topic.h" @@ -49,6 +50,7 @@ #include "vino_core_lib/models/vehicle_attribs_detection_model.h" #include "vino_core_lib/models/license_plate_detection_model.h" #include "vino_core_lib/models/landmarks_detection_model.h" +#include "vino_core_lib/models/object_segmentation_maskrcnn_model.h" #include "vino_core_lib/outputs/image_window_output.h" #include "vino_core_lib/outputs/ros_topic_output.h" #include "vino_core_lib/outputs/rviz_output.h" @@ -247,6 +249,10 @@ PipelineManager::parseInference(const Params::ParamManager::PipelineRawData& par { object = createObjectSegmentation(infer); } + else if (infer.name == kInferTpye_ObjectSegmentationMaskrcnn) + { + object = createObjectSegmentationMaskrcnn(infer); + } else if (infer.name == kInferTpye_PersonReidentification) { object = createPersonReidentification(infer); @@ -370,6 +376,24 @@ PipelineManager::createObjectSegmentation(const Params::ParamManager::InferenceR return segmentation_inference_ptr; } +std::shared_ptr +PipelineManager::createObjectSegmentationMaskrcnn(const Params::ParamManager::InferenceRawData & infer) +{ + auto model = + std::make_shared(infer.label, infer.model, infer.batch); + model->modelInit(); + slog::info << "Segmentation model initialized." << slog::endl; + auto engine = engine_manager_.createEngine(infer.engine, model); + slog::info << "Segmentation Engine initialized." << slog::endl; + auto segmentation_inference_ptr = std::make_shared( + infer.confidence_threshold); + slog::info << "Segmentation Inference instanced." << slog::endl; + segmentation_inference_ptr->loadNetwork(model); + segmentation_inference_ptr->loadEngine(engine); + + return segmentation_inference_ptr; +} + std::shared_ptr PipelineManager::createPersonReidentification(const Params::ParamManager::InferenceRawData& infer) { diff --git a/vino_launch/launch/pipeline_segmentation_maskrcnn.launch b/vino_launch/launch/pipeline_segmentation_maskrcnn.launch new file mode 100644 index 00000000..514cf3cb --- /dev/null +++ b/vino_launch/launch/pipeline_segmentation_maskrcnn.launch @@ -0,0 +1,10 @@ + + + + + + + + + diff --git a/vino_launch/param/pipeline_segmentation_maskrcnn.yaml b/vino_launch/param/pipeline_segmentation_maskrcnn.yaml new file mode 100644 index 00000000..01b33bb6 --- /dev/null +++ b/vino_launch/param/pipeline_segmentation_maskrcnn.yaml @@ -0,0 +1,22 @@ +Pipelines: +- name: segmentation + inputs: [RealSenseCamera] + infers: + - name: ObjectSegmentationMaskrcnn + model: /opt/openvino_toolkit/models/convert/public/mask_rcnn_inception_v2_coco_2018_01_28/OUT/frozen_inference_graph.xml + engine: CPU #"HETERO:CPU,GPU,MYRIAD" + label: to/be/set/xxx.labels + batch: 1 + confidence_threshold: 0.5 + outputs: [ImageWindow, RosTopic, RViz] + connects: + - left: RealSenseCamera + right: [ObjectSegmentationMaskrcnn] + - left: ObjectSegmentationMaskrcnn + right: [ImageWindow] + - left: ObjectSegmentationMaskrcnn + right: [RosTopic] + - left: ObjectSegmentationMaskrcnn + right: [RViz] + +OpenvinoCommon: \ No newline at end of file From 3c23d1527487b7013b5cf36a6bd145bec8a3de76 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 12 Dec 2022 16:03:12 +0800 Subject: [PATCH 32/40] fix vehicle attribs and license plate --- .vscode/settings.json | 70 +++++++++++++++++++ ...icense-plate-detection-barrier-0106.labels | 1 + .../models/vehicle_attribs_detection_model.h | 3 +- .../inferences/license_plate_detection.cpp | 3 +- 4 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..75f35efa --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,70 @@ +{ + "files.associations": { + "cctype": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "csignal": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "array": "cpp", + "atomic": "cpp", + "strstream": "cpp", + "*.tcc": "cpp", + "bitset": "cpp", + "chrono": "cpp", + "complex": "cpp", + "condition_variable": "cpp", + "cstdint": "cpp", + "deque": "cpp", + "forward_list": "cpp", + "list": "cpp", + "unordered_map": "cpp", + "unordered_set": "cpp", + "vector": "cpp", + "exception": "cpp", + "algorithm": "cpp", + "functional": "cpp", + "iterator": "cpp", + "map": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "optional": "cpp", + "random": "cpp", + "ratio": "cpp", + "set": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "utility": "cpp", + "fstream": "cpp", + "future": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "mutex": "cpp", + "new": "cpp", + "ostream": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "cfenv": "cpp", + "cinttypes": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "variant": "cpp", + "bit": "cpp" + } +} \ No newline at end of file diff --git a/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels b/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels index 23d4cd9a..827dc158 100644 --- a/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels +++ b/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels @@ -1,2 +1,3 @@ +background vehicle license diff --git a/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h b/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h index 33051fe0..1ea110da 100644 --- a/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h +++ b/vino_core_lib/include/vino_core_lib/models/vehicle_attribs_detection_model.h @@ -32,7 +32,7 @@ class VehicleAttribsDetectionModel : public BaseModel VehicleAttribsDetectionModel(const std::string& label_loc, const std::string& model_loc, int batch_size = 1); inline const std::string getInputName() { - return input_; + return input_tensor_name_; } inline const std::string getColorOutputName() { @@ -50,7 +50,6 @@ class VehicleAttribsDetectionModel : public BaseModel protected: bool updateLayerProperty(std::shared_ptr&) override; - std::string input_; std::string color_output_; std::string type_output_; }; diff --git a/vino_core_lib/src/inferences/license_plate_detection.cpp b/vino_core_lib/src/inferences/license_plate_detection.cpp index 16076182..37a87d64 100644 --- a/vino_core_lib/src/inferences/license_plate_detection.cpp +++ b/vino_core_lib/src/inferences/license_plate_detection.cpp @@ -48,8 +48,7 @@ void vino_core_lib::LicensePlateDetection::fillSeqBlob() valid_model_->getSeqInputName()); int max_sequence_size = seq_tensor.get_shape()[0]; float * tensor_data = seq_tensor.data(); - tensor_data[0] = 0.0f; - std::fill(tensor_data + 1, tensor_data + max_sequence_size, 1.0f); + std::fill(tensor_data, tensor_data + max_sequence_size, 1.0f); } bool vino_core_lib::LicensePlateDetection::enqueue(const cv::Mat& frame, const cv::Rect& input_frame_loc) From 1158e5cfec13e280bd7bf02f792a3165bf00c6d2 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 12 Dec 2022 17:18:36 +0800 Subject: [PATCH 33/40] fix yaml path --- vino_launch/param/image_object_server.yaml | 4 ++-- vino_launch/param/image_people_server.yaml | 14 +++++++------- vino_launch/param/image_segmentation_server.yaml | 5 +++-- .../param/multi_pipleine_service_object.yaml | 4 ++-- vino_launch/param/pipeline_image.yaml | 2 +- vino_launch/param/pipeline_object.yaml | 2 +- vino_launch/param/pipeline_object_topic.yaml | 2 +- vino_launch/param/pipeline_reidentification.yaml | 2 +- vino_launch/param/pipeline_segmentation.yaml | 4 ++-- vino_launch/param/pipeline_vehicle_detection.yaml | 4 ++-- vino_launch/param/pipeline_video.yaml | 6 +++--- 11 files changed, 25 insertions(+), 24 deletions(-) diff --git a/vino_launch/param/image_object_server.yaml b/vino_launch/param/image_object_server.yaml index b39cfa2d..b43adb19 100644 --- a/vino_launch/param/image_object_server.yaml +++ b/vino_launch/param/image_object_server.yaml @@ -1,9 +1,10 @@ Pipelines: - name: object inputs: [Image] + input_path: to/be/set/image_path infers: - name: ObjectDetection - model: /opt/openvino_toolkit/models/public/mobilenet-ssd/FP16/mobilenet-ssd.xml + model: /opt/openvino_toolkit/models/convert/public/mobilenet-ssd/FP16/mobilenet-ssd.xml engine: CPU label: to/be/set/xxx.labels batch: 1 @@ -15,6 +16,5 @@ Pipelines: right: [ObjectDetection] - left: ObjectDetection right: [RosService] - input_path: "/home/intel/Pictures/car.png" Common: diff --git a/vino_launch/param/image_people_server.yaml b/vino_launch/param/image_people_server.yaml index 22a0b960..21e2a2de 100644 --- a/vino_launch/param/image_people_server.yaml +++ b/vino_launch/param/image_people_server.yaml @@ -1,26 +1,27 @@ Pipelines: - name: people inputs: [Image] + input_path: to/be/set/image_path infers: - name: FaceDetection - model: /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP16/face-detection-adas-0001.xml + model: /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP32/face-detection-adas-0001.xml engine: CPU - label: /to/be/set/xxx.labels + label: /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP16/face-detection-adas-0001.labels batch: 1 confidence_threshold: 0.5 enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame - name: AgeGenderRecognition - model: /opt/openvino_toolkit/models/intel/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013.xml + model: /opt/openvino_toolkit/models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml engine: CPU label: to/be/set/xxx.labels batch: 16 - name: EmotionRecognition - model: /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP16/emotions-recognition-retail-0003.xml + model: /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml engine: CPU - label: to/be/set/xxx.labels + label: /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.labels batch: 16 - name: HeadPoseEstimation - model: /opt/openvino_toolkit/models/intel/head-pose-estimation-adas-0001/FP16/head-pose-estimation-adas-0001.xml + model: /opt/openvino_toolkit/models/intel/head-pose-estimation-adas-0001/FP32/head-pose-estimation-adas-0001.xml engine: CPU label: to/be/set/xxx.labels batch: 16 @@ -36,6 +37,5 @@ Pipelines: right: [RosService, RViz] - left: HeadPoseEstimation right: [RosService, RViz] - input_path: "~/Pictures/face.jpeg" Common: diff --git a/vino_launch/param/image_segmentation_server.yaml b/vino_launch/param/image_segmentation_server.yaml index 6e9d60bf..70b5e217 100644 --- a/vino_launch/param/image_segmentation_server.yaml +++ b/vino_launch/param/image_segmentation_server.yaml @@ -1,9 +1,10 @@ Pipelines: - name: segmentation inputs: [Image] + input_path: to/be/set/image_path infers: - name: ObjectSegmentation - model: /opt/openvino_toolkit/models/public/deeplabv3/FP16/deeplabv3.xml + model: /opt/openvino_toolkit/models/convert/public/deeplabv3/FP16/deeplabv3.xml engine: CPU label: to/be/set/xxx.labels batch: 1 @@ -14,6 +15,6 @@ Pipelines: right: [ObjectSegmentation] - left: ObjectSegmentation right: [RosService] - input_path: "/opt/openvino_toolkit/ros_openvino_toolkit/data/images/car.png" + OpenvinoCommon: diff --git a/vino_launch/param/multi_pipleine_service_object.yaml b/vino_launch/param/multi_pipleine_service_object.yaml index dedebac0..03f41ca8 100644 --- a/vino_launch/param/multi_pipleine_service_object.yaml +++ b/vino_launch/param/multi_pipleine_service_object.yaml @@ -4,7 +4,7 @@ Pipelines: inputs: [StandardCamera] infers: - name: ObjectDetection - model: /opt/openvino_toolkit/models/public/mobilenet-ssd/FP16/mobilenet-ssd.xml + model: /opt/openvino_toolkit/models/convert/public/mobilenet-ssd/FP16/mobilenet-ssd.xml engine: CPU label: to/be/set/xxx.labels batch: 16 @@ -25,7 +25,7 @@ Pipelines: inputs: [RealSenseCamera] infers: - name: ObjectDetection - model: /opt/openvino_toolkit/models/public/mobilenet-ssd/FP16/mobilenet-ssd.xml + model: /opt/openvino_toolkit/models/convert/public/mobilenet-ssd/FP16/mobilenet-ssd.xml engine: CPU label: to/be/set/xxx.labels batch: 16 diff --git a/vino_launch/param/pipeline_image.yaml b/vino_launch/param/pipeline_image.yaml index f633eaae..63652a5a 100644 --- a/vino_launch/param/pipeline_image.yaml +++ b/vino_launch/param/pipeline_image.yaml @@ -1,7 +1,7 @@ Pipelines: - name: people inputs: [Image] - input_path: /home/ubuntu20/jiawei/ros-ov/ros_openvino_ws/src/ros_openvino_toolkit/data/images/team.jpg + input_path: to/be/set/image_path infers: - name: FaceDetection model: /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP16/face-detection-adas-0001.xml diff --git a/vino_launch/param/pipeline_object.yaml b/vino_launch/param/pipeline_object.yaml index cb189084..22e9469c 100644 --- a/vino_launch/param/pipeline_object.yaml +++ b/vino_launch/param/pipeline_object.yaml @@ -3,7 +3,7 @@ Pipelines: inputs: [StandardCamera] infers: - name: ObjectDetection - model: /opt/openvino_toolkit/models/public/ssd_mobilenet_v2_coco/FP16/ssd_mobilenet_v2_coco.xml + model: /opt/openvino_toolkit/models/convert/public/mobilenet-ssd/FP16/mobilenet-ssd.xml engine: CPU label: to/be/set/xxx.labels batch: 1 diff --git a/vino_launch/param/pipeline_object_topic.yaml b/vino_launch/param/pipeline_object_topic.yaml index eb2df545..d61a2b58 100644 --- a/vino_launch/param/pipeline_object_topic.yaml +++ b/vino_launch/param/pipeline_object_topic.yaml @@ -3,7 +3,7 @@ Pipelines: inputs: [RealSenseCamera] #[ImageTopic] infers: - name: ObjectDetection - model: /opt/openvino_toolkit/models/public/ssd_mobilenet_v2_coco/FP16/ssd_mobilenet_v2_coco.xml + model: /opt/openvino_toolkit/models/convert/public/mobilenet-ssd/FP16/mobilenet-ssd.xml engine: CPU label: to/be/set/xxx.labels batch: 1 diff --git a/vino_launch/param/pipeline_reidentification.yaml b/vino_launch/param/pipeline_reidentification.yaml index c543b1ae..7894328b 100644 --- a/vino_launch/param/pipeline_reidentification.yaml +++ b/vino_launch/param/pipeline_reidentification.yaml @@ -3,7 +3,7 @@ Pipelines: inputs: [StandardCamera] infers: - name: ObjectDetection - model: /opt/openvino_toolkit/models/intel/person-detection-retail-0013/FP16/person-detection-retail-0013.xml + model: /opt/openvino_toolkit/models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013.xml engine: CPU label: to/be/set/xxx.labels batch: 1 diff --git a/vino_launch/param/pipeline_segmentation.yaml b/vino_launch/param/pipeline_segmentation.yaml index 97f8c47f..1de763a6 100644 --- a/vino_launch/param/pipeline_segmentation.yaml +++ b/vino_launch/param/pipeline_segmentation.yaml @@ -3,9 +3,9 @@ Pipelines: inputs: [RealSenseCamera] infers: - name: ObjectSegmentation - model: /opt/openvino_toolkit/models/public/deeplabv3/FP16/deeplabv3.xml + model: /opt/openvino_toolkit/models/convert/public/deeplabv3/FP16/deeplabv3.xml engine: CPU - label: /opt/openvino_toolkit/models/public/deeplabv3/FP16/deeplabv3.labels + label: /opt/openvino_toolkit/models/convert/public/deeplabv3/FP16/deeplabv3.labels batch: 1 confidence_threshold: 0.5 outputs: [ImageWindow, RosTopic, RViz] diff --git a/vino_launch/param/pipeline_vehicle_detection.yaml b/vino_launch/param/pipeline_vehicle_detection.yaml index 5c9819bd..363b38e9 100644 --- a/vino_launch/param/pipeline_vehicle_detection.yaml +++ b/vino_launch/param/pipeline_vehicle_detection.yaml @@ -1,13 +1,13 @@ Pipelines: - name: object inputs: [Video] - input_path: /home/ubuntu20/jiawei/ros-ov/ros_openvino_ws/src/ros_openvino_toolkit/data/car_cut.mp4 + input_path: to/be/set/video_path infers: - name: ObjectDetection model: /opt/openvino_toolkit/models/public/vehicle-license-plate-detection-barrier-0123/FP16/vehicle-license-plate-detection-barrier-0123.xml engine: CPU model_type: SSD - label: /opt/openvino_toolkit/models/public/vehicle-license-plate-detection-barrier-0123/FP16/vehicle-license-plate-detection-barrier-0106.labels + label: /opt/openvino_toolkit/models/intel/vehicle-license-plate-detection-barrier-0123/FP16/vehicle-license-plate-detection-barrier-0106.labels batch: 1 enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame - name: VehicleAttribsDetection diff --git a/vino_launch/param/pipeline_video.yaml b/vino_launch/param/pipeline_video.yaml index 007e3970..1712b65e 100644 --- a/vino_launch/param/pipeline_video.yaml +++ b/vino_launch/param/pipeline_video.yaml @@ -1,12 +1,12 @@ Pipelines: - name: segmentation inputs: [Video] - input_path: /home/ubuntu20/jiawei/ros-ov/ros_openvino_ws/src/ros_openvino_toolkit/data/car_cut.mp4 + input_path: to/be/set/video_path infers: - name: ObjectSegmentation - model: /opt/openvino_toolkit/models/public/deeplabv3/FP16/deeplabv3.xml + model: /opt/openvino_toolkit/models/convert/public/deeplabv3/FP16/deeplabv3.xml engine: CPU - label: /opt/openvino_toolkit/models/public/deeplabv3/FP16/deeplabv3.labels + label: /opt/openvino_toolkit/models/convert/public/deeplabv3/FP16/deeplabv3.labels batch: 1 confidence_threshold: 0.5 outputs: [ImageWindow, RosTopic, RViz] From cfea37d6d47d29eea83ed3b86868461535f0c584 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Tue, 13 Dec 2022 14:24:19 +0800 Subject: [PATCH 34/40] fix doc dockerfile and model list --- data/model_list/download_model.lst | 2 ++ doc/quick_start/getting_started_with_ros_ov2.0.md | 2 +- docker/Dockerfile | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/data/model_list/download_model.lst b/data/model_list/download_model.lst index f485052e..50ddc2f4 100644 --- a/data/model_list/download_model.lst +++ b/data/model_list/download_model.lst @@ -14,3 +14,5 @@ human-pose-estimation-0001 semantic-segmentation-adas-0001 mobilenet-ssd deeplabv3 +face-reidentification-retail-0095.xml +vehicle-license-plate-detection-barrier-0123.xml diff --git a/doc/quick_start/getting_started_with_ros_ov2.0.md b/doc/quick_start/getting_started_with_ros_ov2.0.md index 9feb3c8c..6f11a5ce 100644 --- a/doc/quick_start/getting_started_with_ros_ov2.0.md +++ b/doc/quick_start/getting_started_with_ros_ov2.0.md @@ -100,7 +100,7 @@ sudo python3 converter.py --list ~/catkin_ws/src/ros_openvino_toolkit/data/model sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 + sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP16 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels diff --git a/docker/Dockerfile b/docker/Dockerfile index acd9dec8..2fbcd37a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -66,6 +66,7 @@ WORKDIR /usr/lib/x86_64-linux-gnu RUN ln -sf libboost_python-py36.so libboost_python37.so RUN pip install --upgrade pip RUN pip install openvino-dev[tensorflow2]==2022.1 +RUN apt-get install -y libcanberra-gtk* git # build ros openvino toolkit # set env before build ros openvino tolkkit From 3d271201f4e71a7e44f7655b668322d57d3a8d74 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Tue, 13 Dec 2022 14:28:23 +0800 Subject: [PATCH 35/40] fix param --- .vscode/settings.json | 70 ------------------- .../models/attributes/base_attribute.h | 2 +- 2 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 75f35efa..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "files.associations": { - "cctype": "cpp", - "clocale": "cpp", - "cmath": "cpp", - "csignal": "cpp", - "cstdarg": "cpp", - "cstddef": "cpp", - "cstdio": "cpp", - "cstdlib": "cpp", - "cstring": "cpp", - "ctime": "cpp", - "cwchar": "cpp", - "cwctype": "cpp", - "array": "cpp", - "atomic": "cpp", - "strstream": "cpp", - "*.tcc": "cpp", - "bitset": "cpp", - "chrono": "cpp", - "complex": "cpp", - "condition_variable": "cpp", - "cstdint": "cpp", - "deque": "cpp", - "forward_list": "cpp", - "list": "cpp", - "unordered_map": "cpp", - "unordered_set": "cpp", - "vector": "cpp", - "exception": "cpp", - "algorithm": "cpp", - "functional": "cpp", - "iterator": "cpp", - "map": "cpp", - "memory": "cpp", - "memory_resource": "cpp", - "numeric": "cpp", - "optional": "cpp", - "random": "cpp", - "ratio": "cpp", - "set": "cpp", - "string": "cpp", - "string_view": "cpp", - "system_error": "cpp", - "tuple": "cpp", - "type_traits": "cpp", - "utility": "cpp", - "fstream": "cpp", - "future": "cpp", - "initializer_list": "cpp", - "iomanip": "cpp", - "iosfwd": "cpp", - "iostream": "cpp", - "istream": "cpp", - "limits": "cpp", - "mutex": "cpp", - "new": "cpp", - "ostream": "cpp", - "sstream": "cpp", - "stdexcept": "cpp", - "streambuf": "cpp", - "thread": "cpp", - "cfenv": "cpp", - "cinttypes": "cpp", - "typeindex": "cpp", - "typeinfo": "cpp", - "variant": "cpp", - "bit": "cpp" - } -} \ No newline at end of file diff --git a/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h b/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h index d18aa7ed..f2a8bd12 100644 --- a/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h +++ b/vino_core_lib/include/vino_core_lib/models/attributes/base_attribute.h @@ -90,7 +90,7 @@ class ModelAttribute slog::info << "--------------------------------" << slog::endl; } - virtual bool updateLayerProperty(std::shared_ptr&) + virtual bool updateLayerProperty(const std::shared_ptr&) { return false; } From 809ebfb2661314b4ac61c1e115ec76b834f3ee37 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Thu, 15 Dec 2022 12:58:12 +0800 Subject: [PATCH 36/40] fix model download list --- data/model_list/download_model.lst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/model_list/download_model.lst b/data/model_list/download_model.lst index 50ddc2f4..20a8a27d 100644 --- a/data/model_list/download_model.lst +++ b/data/model_list/download_model.lst @@ -14,5 +14,5 @@ human-pose-estimation-0001 semantic-segmentation-adas-0001 mobilenet-ssd deeplabv3 -face-reidentification-retail-0095.xml -vehicle-license-plate-detection-barrier-0123.xml +face-reidentification-retail-0095 +vehicle-license-plate-detection-barrier-0123 From 3d97a619ca9e4f3911d66284c1178d880c15eca2 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 19 Dec 2022 14:53:18 +0800 Subject: [PATCH 37/40] fix label path --- ...ting_started_with_Noetic_Ubuntu20.04_ov2.0.md | 16 ++++++++-------- .../getting_started_with_ros_ov2.0.md | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md b/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md index becfd227..588874d3 100644 --- a/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md +++ b/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md @@ -121,14 +121,14 @@ omz_downloader --name license-plate-recognition-barrier-0001 --output_dir /opt/o ``` * Copy label files (execute once) ``` - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 - sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP32/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP32/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/intel/vehicle-license-plate-detection-barrier-0106/FP32 +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/convert/public/mobilenet-ssd/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/convert/public/deeplabv3/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/intel/semantic-segmentation-adas-0001/FP16/ ``` * If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to intermediate representation (such as the model for object detection): diff --git a/doc/quick_start/getting_started_with_ros_ov2.0.md b/doc/quick_start/getting_started_with_ros_ov2.0.md index 6f11a5ce..37065c40 100644 --- a/doc/quick_start/getting_started_with_ros_ov2.0.md +++ b/doc/quick_start/getting_started_with_ros_ov2.0.md @@ -96,14 +96,14 @@ sudo python3 converter.py --list ~/catkin_ws/src/ros_openvino_toolkit/data/model **Note**:Need to make label_dirs if skip steps for set output_dirs above. * Before launch, copy label files to the same model path, make sure the model path and label path match the ros_openvino_toolkit/vino_launch/param/xxxx.yaml. ``` - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP16 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 - sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP32/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP32/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/intel/vehicle-license-plate-detection-barrier-0106/FP32 +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/convert/public/mobilenet-ssd/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/convert/public/deeplabv3/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/intel/semantic-segmentation-adas-0001/FP16/ ``` * Please check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. From b7f6ee668999d209213c788c1a35d78715c185af Mon Sep 17 00:00:00 2001 From: wujiawei Date: Mon, 19 Dec 2022 15:10:43 +0800 Subject: [PATCH 38/40] fix label path for docker --- docker/docker_instruction.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/docker_instruction.md b/docker/docker_instruction.md index 7ffdc7fa..7e45644d 100644 --- a/docker/docker_instruction.md +++ b/docker/docker_instruction.md @@ -84,14 +84,14 @@ omz_converter --list convert_model.lst -d /opt/openvino_toolkit/models/ -o /opt/ * Copy label files (execute once) **Note**:Need to make label_dirs if skip steps for set output_dirs above. ``` - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 - sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP32/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP32/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/intel/vehicle-license-plate-detection-barrier-0106/FP32 +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/convert/public/mobilenet-ssd/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/convert/public/deeplabv3/FP16/ +sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/intel/semantic-segmentation-adas-0001/FP16/ ``` * Before launch, check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml, make sure the paramter like model path, label path, inputs are right. From 98fb7cedcb85d8c154890770e591db85f4622666 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Wed, 21 Dec 2022 16:22:56 +0800 Subject: [PATCH 39/40] update documents for ros 2.0 --- README.md | 393 +++++++++--------- doc/EN/BINARY_VERSION_README.md | 255 ------------ doc/EN/OBJECT_DETECTION.md | 190 --------- doc/EN/OPEN_SOURCE_CODE_README.md | 152 ------- doc/EN/YAML_CONFIGURATION_GUIDE.md | 119 ------ ...g_started_with_Noetic_Ubuntu20.04_ov2.0.md | 194 --------- .../getting_started_with_ros_ov2.0.md | 33 +- doc/quick_start/yaml_configuration_guide.md | 130 ++++++ 8 files changed, 349 insertions(+), 1117 deletions(-) delete mode 100644 doc/EN/BINARY_VERSION_README.md delete mode 100644 doc/EN/OBJECT_DETECTION.md delete mode 100644 doc/EN/OPEN_SOURCE_CODE_README.md delete mode 100644 doc/EN/YAML_CONFIGURATION_GUIDE.md delete mode 100644 doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md create mode 100644 doc/quick_start/yaml_configuration_guide.md diff --git a/README.md b/README.md index 2163eb28..80fc3e2b 100644 --- a/README.md +++ b/README.md @@ -1,251 +1,254 @@ > **中文文档入口:** -# Introduction - -The OpenVINO™ (Open visual inference and neural network optimization) toolkit provides a ROS-adaptered runtime framework of neural network which quickly deploys applications and solutions for vision inference. By leveraging Intel® OpenVINO™ toolkit and corresponding libraries, this runtime framework extends workloads across Intel® hardware (including accelerators) and maximizes performance. -* Enables CNN-based deep learning inference at the edge -* Supports heterogeneous execution across computer vision accelerators—CPU, GPU, Intel® Movidius™ Neural Compute Stick, and FPGA—using a common API -* Speeds up time to market via a library of functions and preoptimized kernels -* Includes optimized calls for OpenCV and OpenVX* +# ros_openvino_toolkit + +# Table of Contents +* [➤ Overview](#overview) + * [ROS Version Supported](#ros-version-supported) + * [Inference Features Supported](#inference-features-supported) +* [➤ Prerequisite](#prerequisite) +* [➤ Introduction](#introduction) + * [Design Architecture](#design-architecture) + * [Logic Flow](#logic-flow) +* [➤ Supported Features](#supported-features) + * [Multiple Input Components](#multiple-input-components) + * [Inference Implementations](#inference-implementations) + * [ROS Interfaces and Outputs](#ros-interfaces-and-outputs) + * [Demo Result Snapshots](#demo-result-snapshots) +* [➤ Installation & Launching](#installation-and-launching) + * [Deploy in Local Environment](#deploy-in-local-environment) + * [Deploy in Docker](#deploy-in-docker) +* [➤ Reference](#reference) +* [➤ FAQ](#faq) +* [➤ Feedback](#feedback) +* [➤ More Information](#more-information) + +# Overview +## ROS Version Supported + +|Branch Name|ROS Version Supported|Openvino Version|OS Version| +|-----------------------|-----------------------|--------------------------------|----------------------| +|[ros](https://github.com/intel/ros_openvino_toolkit/tree/ros)|Melodic, Noetic|V2022.1, V2022.2|Ubuntu18.04, Ubuntu 20.04| +|[melodic-ov_legacy](https://github.com/intel/ros_openvino_toolkit/tree/melodic-ov_legacy)|Melodic|V2022.1, V2022.2|Ubuntu 18.04| +|[noetic-ov_legacy](https://github.com/intel/ros_openvino_toolkit/tree/noetic-ov_legacy)|Noetic|V2021.4|Ubuntu 20.04| + +## Inference Features Supported +* [x] Object Detection +* [x] Face Detection +* [x] Age Gender Recognition +* [x] Emotion Recognition +* [x] Head Pose Estimation +* [x] Object Segmentation +* [x] Person Re-Identification +* [x] Vehicle Attribute Detection +* [x] Vehicle License Plate Detection + +# Prerequisite + +|Prerequisite|Mandatory?|Description| +|-----------------------|-----------------------|--------------------------------| +|**Processor**|Mandatory|A platform with Intel processors assembled. (Refer to [here](https://software.intel.com/content/www/us/en/develop/articles/openvino-2020-3-lts-relnotes.html) for the full list of Intel processors supported.)| +|**OS**|Mandatory|We only tested this project under Ubuntu distros. It is recommended to install the corresponding Ubuntu Distro according to the ROS distro that you select to use. **For example: Ubuntu 18.04 for Melodic, Ubuntu 20.04 for Noetic.**| +|**ROS**|Mandatory|We have already supported active ROS distros (Humble, Galactic, Foxy and Dashing (deprecated)). Choose the one matching your needs. You may find the corresponding branch from the table above in section [**ROS Version Supported**](#ros-version-supported).| +|**OpenVINO**|Mandatory|The version of OpenVINO toolkit is decided by the OS and ROS distros you use. See the table above in Section [**ROS Version Supported**](#ros-version-supported).| +|**Realsense Camera**|Optional|Realsense Camera is optional, you may choose these alternatives as the input: Standard Camera, ROS Image Topic, Video/Image File or RTSP camera.| +# Introduction ## Design Architecture From the view of hirarchical architecture design, the package is divided into different functional components, as shown in below picture. ![OpenVINO_Architecture](./data/images/design_arch.PNG "OpenVINO RunTime Architecture") -- **Intel® OpenVINO™ toolkit** is leveraged to provide deep learning basic implementation for data inference. is free software that helps developers and data scientists speed up computer vision workloads, streamline deep learning inference and deployments, -and enable easy, heterogeneous execution across Intel® platforms from edge to cloud. It helps to: +

+

+Intel® OpenVINO™ toolkit + +- **Intel® OpenVINO™ toolkit** provides a ROS-adapted runtime framework of neural network which quickly deploys applications and solutions for vision inference. By leveraging Intel® OpenVINO™ toolkit and corresponding libraries, this ROS runtime framework extends workloads across Intel® hardware (including accelerators) and maximizes performance. - Increase deep learning workload performance up to 19x1 with computer vision accelerators from Intel. - Unleash convolutional neural network (CNN)-based deep learning inference using a common API. - Speed development using optimized OpenCV* and OpenVX* functions. -- **ros OpenVINO Runtime Framework** is the main body of this repo. it provides key logic implementation for pipeline lifecycle management, resource management and ROS system adapter, which extends Intel OpenVINO toolkit and libraries. Furthermore, this runtime framework provides ways to ease launching, configuration and data analytics and re-use. -- **Diversal Input resources** are the data resources to be infered and analyzed with the OpenVINO framework. +See more from [here](https://github.com/openvinotoolkit/openvino) for Intel OpenVINO™ introduction. +
+

+ +

+

+ROS OpenVINO Runtime Framework + +- **ROS OpenVINO Runtime Framework** is the main body of this repo. It provides key logic implementation for pipeline lifecycle management, resource management and ROS system adapter, which extends Intel OpenVINO toolkit and libraries. Furthermore, this runtime framework provides ways to simplify launching, configuration, data analysis and re-use. +
+

+ +

+

+ROS Input & Output + +- **Diversal Input resources** are data resources to be infered and analyzed with the OpenVINO framework. - **ROS interfaces and outputs** currently include _Topic_ and _service_. Natively, RViz output and CV image window output are also supported by refactoring topic message and inferrence results. -- **Optimized Models** provides by Model Optimizer component of Intel® OpenVINO™ toolkit. Imports trained models from various frameworks (Caffe*, Tensorflow*, MxNet*, ONNX*, Kaldi*) and converts them to a unified intermediate representation file. It also optimizes topologies through node merging, horizontal fusion, eliminating batch normalization, and quantization.It also supports graph freeze and graph summarize along with dynamic input freezing. +
+

+ +

+

+Optimized Models + +- **Optimized Models** provided by Model Optimizer component of Intel® OpenVINO™ toolkit. Imports trained models from various frameworks (Caffe*, Tensorflow*, MxNet*, ONNX*, Kaldi*) and converts them to a unified intermediate representation file. It also optimizes topologies through node merging, horizontal fusion, eliminating batch normalization, and quantization. It also supports graph freeze and graph summarize along with dynamic input freezing. +
+

## Logic Flow -From the view of logic implementation, the package introduces the definitions of parameter manager, pipeline and pipeline manager. The below picture depicts how these entities co-work together when the corresponding program is launched. +From the view of logic implementation, the package introduces the definitions of parameter manager, pipeline and pipeline manager. The following picture depicts how these entities co-work together when the corresponding program is launched. ![Logic_Flow](./data/images/impletation_logic.PNG "OpenVINO RunTime Logic Flow") Once a corresponding program is launched with a specified .yaml config file passed in the .launch file or via commandline, _**parameter manager**_ analyzes the configurations about pipeline and the whole framework, then shares the parsed configuration information with pipeline procedure. A _**pipeline instance**_ is created by following the configuration info and is added into _**pipeline manager**_ for lifecycle control and inference action triggering. -The contents in **.yaml config file** should be well structured and follow the supported rules and entity names. Please see [the configuration guidance](./doc/EN/YAML_CONFIGURATION_GUIDE.md) for how to create or edit the config files. +The contents in **.yaml config file** should be well structured and follow the supported rules and entity names. Please see [yaml configuration guidance](./doc/quick_start/yaml_configuration_guide.md) for how to create or edit the config files. + +

+

+Pipeline **Pipeline** fulfills the whole data handling process: initiliazing Input Component for image data gathering and formating; building up the structured inference network and passing the formatted data through the inference network; transfering the inference results and handling output, etc. +
+

+ +

+

+Pipeline manager **Pipeline manager** manages all the created pipelines according to the inference requests or external demands (say, system exception, resource limitation, or end user's operation). Because of co-working with resource management and being aware of the whole framework, it covers the ability of performance optimization by sharing system resource between pipelines and reducing the burden of data copy. +
+

# Supported Features -## Diversal Input Components -Currently, the package support several kinds of input resources of gaining image data: +## Multiple Input Components +Currently, the package supports several input resources for acquiring image data. The following tables are listed: + +

+

+Input Resource Table |Input Resource|Description| |--------------------|------------------------------------------------------------------| |StandardCamera|Any RGB camera with USB port supporting. Currently only the first USB camera if many are connected.| |RealSenseCamera| Intel RealSense RGB-D Camera, directly calling RealSense Camera via librealsense plugin of openCV.| -|Image Topic| any ROS topic which is structured in image message.| -|Image File| Any image file which can be parsed by openCV, such as .png, .jpeg.| -|Video File| Any video file which can be parsed by openCV.| +|ImageTopic| Any ROS topic which is structured in image message.| +|Image| Any image file which can be parsed by openCV, such as .png, .jpeg.| +|Video| Any video file which can be parsed by openCV.| +|IpCamera| Any RTSP server which can push video stream.| +
+

## Inference Implementations -Currently, the inference feature list is supported: - -|Inference|Description| -|-----------------------|------------------------------------------------------------------| -|Face Detection|Object Detection task applied to face recognition using a sequence of neural networks.| -|Emotion Recognition| Emotion recognition based on detected face image.| -|Age & Gender Recognition| Age and gener recognition based on detected face image.| -|Head Pose Estimation| Head pose estimation based on detected face image.| -|Object Detection| object detection based on SSD-based trained models.| -|Vehicle Detection| Vehicle and passenger detection based on Intel models.| -|Object Segmentation| object detection and segmentation.| -|Person Reidentification| Person Reidentification based on object detection.| +Currently, the corresponding relation of supported inference features, models used and yaml configurations are listed as follows: + +

+

+Inference Feature Correspondence Table + +|Inference|Description|YAML Configuration|Model Used| +|-----------------------|------------------------------------------------------------------|----------------------|----------------------| +|Face Detection| Object Detection task applied to face recognition using a sequence of neural networks.|[pipeline_image.yaml](./vino_launch/param/pipeline_image.yaml)
[pipeline_people.yaml](./vino_launch/param/pipeline_people.yaml)|[face-detection-adas-0001](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/face-detection-adas-0001)
[age-gender-recognition-retail-0013](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/age-gender-recognition-retail-0013)
[emotions-recognition-retail-0003](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/emotions-recognition-retail-0003)
[head-pose-estimation-adas-0001](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/head-pose-estimation-adas-0001)| +|Emotion Recognition| Emotion recognition based on detected face image.|[pipeline_image.yaml](./vino_launch/param/pipeline_image.yaml)
[pipeline_people.yaml](./vino_launch/param/pipeline_people.yaml)|[emotions-recognition-retail-0003](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/emotions-recognition-retail-0003)| +|Age & Gender Recognition| Age and gender recognition based on detected face image.|[pipeline_image.yaml](./vino_launch/param/pipeline_image.yaml)
[pipeline_people.yaml](./vino_launch/param/pipeline_people.yaml)|[age-gender-recognition-retail-0013](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/age-gender-recognition-retail-0013)| +|Head Pose Estimation| Head pose estimation based on detected face image.|[pipeline_image.yaml](./vino_launch/param/pipeline_image.yaml)
[pipeline_people.yaml](./vino_launch/param/pipeline_people.yaml)|[head-pose-estimation-adas-0001](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/head-pose-estimation-adas-0001)| +|Object Detection| Object detection based on SSD-based trained models.|[pipeline_object.yaml](./vino_launch/param/pipeline_object.yaml)
[pipeline_object_topic.yaml](./vino_launch/param/pipeline_object_topic.yaml)|[mobilenet-ssd](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/public/mobilenet-ssd)| +|Vehicle and License Detection| Vehicle and license detection based on Intel models.|[pipeline_vehicle_detection.yaml](./vino_launch/param/pipeline_vehicle_detection.yaml)|[vehicle-license-plate-detection-barrier-0123](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/public/vehicle-license-plate-detection-barrier-0123)
[vehicle-attributes-recognition-barrier-0039](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/vehicle-attributes-recognition-barrier-0039)
[license-plate-recognition-barrier-0001](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/license-plate-recognition-barrier-0001)| +|Object Segmentation| Object segmentation.|[pipeline_segmentation.yaml](./vino_launch/param/pipeline_segmentation.yaml)
[pipeline_video.yaml](./vino_launch/param/pipeline_video.yaml)|[semantic-segmentation-adas-0001](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/semantic-segmentation-adas-0001)
[deeplabv3](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/public/deeplabv3)| +|Person Reidentification|Person reidentification based on object detection.|[pipeline_person_reidentification.yaml](./vino_launch/param/pipeline_reidentification.yaml)|[person-detection-retail-0013](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/person-detection-retail-0013)
[person-reidentification-retail-0277](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/intel/person-reidentification-retail-0277)| +|Object Segmentation Maskrcnn| Object segmentation and detection based on maskrcnn model.|[pipeline_segmentation_maskrcnn.yaml](./vino_launch/param/pipeline_segmentation_maskrcnn.yaml)|[mask_rcnn_inception_v2_coco_2018_01_28](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1/models/public/mask_rcnn_inception_resnet_v2_atrous_coco)| +
+

## ROS interfaces and outputs +The inference results can be output in several types. One or more types can be enabled for any inference pipeline. ### Topic -#### Subscribed Topic -- Image topic: -```/camera/color/image_raw```([sensor_msgs::Image](http://docs.ros.org/melodic/api/sensor_msgs/html/msg/Image.html)) -#### Published Topic -- Face Detection: -```/ros_openvino_toolkit/face_detection```([object_msgs::ObjectsInBoxes](https://github.com/intel/object_msgs/blob/master/msg/ObjectsInBoxes.msg)) -- Emotion Recognition: -```/ros_openvino_toolkit/emotions_recognition```([people_msgs::EmotionsStamped](./people_msgs/msg/EmotionsStamped.msg)) -- Age and Gender Recognition: -```/ros_openvino_toolkit/age_genders_Recognition```([people_msgs::AgeGenderStamped](./people_msgs/msg/AgeGenderStamped.msg)) -- Head Pose Estimation: -```/ros_openvino_toolkit/headposes_estimation```([people_msgs::HeadPoseStamped](./people_msgs/msg/HeadPoseStamped.msg)) -- Object Detection: -```/ros_openvino_toolkit/detected_objects```([object_msgs::ObjectsInBoxes](https://github.com/intel/object_msgs/blob/master/msg/ObjectsInBoxes.msg)) -- Object Segmentation: -```/ros_openvino_toolkit/segmented_obejcts```([people_msgs::ObjectsInMasks](./people_msgs/msg/ObjectsInMasks.msg)) -- Person Reidentification: -```/ros_openvino_toolkit/reidentified_persons```([people_msgs::ReidentificationStamped](./people_msgs/msg/ReidentificationStamped.msg)) -- Vehicle Detection: -```/ros_openvino_toolkit/detected_license_plates```([people_msgs::VehicleAttribsStamped](./people_msgs/msg/VehicleAttribsStamped.msg) -- Vehicle License Detection: -```/ros_openvino_toolkit/detected_license_plates```([people_msgs::LicensePlateStamped](./people_msgs/msg/LicensePlateStamped.msg) -- Rviz Output: -```/ros_openvino_toolkit/image_rviz```([sensor_msgs::Image](http://docs.ros.org/melodic/api/sensor_msgs/html/msg/Image.html)) +Specific topic(s) can be generated and published according to the given inference functionalities. + +

+

+Published Topic Correspondence Table + +|Inference|Published Topic| +|---|---| +|People Detection|```/ros_openvino_toolkit/face_detection```([object_msgs:msg:ObjectsInBoxes](https://github.com/intel/object_msgs/blob/master/msg/ObjectsInBoxes.msg))| +|Emotion Recognition|```/ros_openvino_toolkit/emotions_recognition```([people_msgs:msg:EmotionsStamped](./vino_people_msgs/msg/EmotionsStamped.msg))| +|Age and Gender Recognition|```/ros_openvino_toolkit/age_genders_Recognition```([people_msgs:msg:AgeGenderStamped](./vino_people_msgs/msg/AgeGenderStamped.msg))| +|Head Pose Estimation|```/ros_openvino_toolkit/headposes_estimation```([people_msgs:msg:HeadPoseStamped](./vino_people_msgs/msg/HeadPoseStamped.msg))| +|Object Detection|```/ros_openvino_toolkit/detected_objects```([object_msgs::msg::ObjectsInBoxes](https://github.com/intel/object_msgs/blob/master/msg/ObjectsInBoxes.msg))| +|Object Segmentation|```/ros_openvino_toolkit/segmented_obejcts```([people_msgs::msg::ObjectsInMasks](./vino_people_msgs/msg/ObjectsInMasks.msg))| +|Object Segmentation Maskrcnn|```/ros_openvino_toolkit/segmented_obejcts```([people_msgs::msg::ObjectsInMasks](./vino_people_msgs/msg/ObjectsInMasks.msg))| +|Person Reidentification|```/ros_openvino_toolkit/reidentified_persons```([people_msgs::msg::ReidentificationStamped](./vino_people_msgs/msg/ReidentificationStamped.msg))| +|Vehicle Detection|```/ros_openvino_toolkit/detected_vehicles_attribs```([people_msgs::msg::VehicleAttribsStamped](./vino_people_msgs/msg/PersonAttributeStamped.msg))| +|Vehicle License Detection|```/ros_openvino_toolkit/detected_license_plates```([people_msgs::msg::LicensePlateStamped](./vino_people_msgs/msg/LicensePlateStamped.msg))| +
+

### Service -- Object Detection Service: -```/detect_object``` ([object_msgs::DetectObject](https://github.com/intel/object_msgs/blob/master/srv/DetectObject.srv)) -- Face Detection Service: -```/detect_face``` ([object_msgs::DetectObject](https://github.com/intel/object_msgs/blob/master/srv/DetectObject.srv)) -- Age & Gender Detection Service: -```/detect_age_gender``` ([people_msgs::AgeGender](./people_msgs/srv/AgeGenderSrv.srv)) -- Headpose Detection Service: -```/detect_head_pose``` ([people_msgs::HeadPose](./people_msgs/srv/HeadPoseSrv.srv)) -- Emotion Detection Service: -```/detect_emotion``` ([people_msgs::Emotion](./people_msgs/srv/EmotionSrv.srv)) - +Several ROS Services are created, expecting to be used in client/server mode, especially when synchronously getting inference results for a given image frame or when managing inference pipeline's lifecycle.
+ +

+

+Service Correspondence Table + +|Inference|Service| +|---|---| +|Object Detection Service|```/detect_object```([object_msgs::srv::DetectObject](https://github.com/intel/object_msgs/blob/master/srv/DetectObject.srv))| +|Face Detection Service|```/detect_face```([object_msgs::srv::DetectObject](https://github.com/intel/object_msgs/blob/master/srv/DetectObject.srv))| +|Age Gender Detection Service|```/detect_age_gender```([vino_people_msgs::srv::AgeGender](./vino_people_msgs/srv/AgeGenderSrv.srv))| +|Headpose Detection Service|```/detect_head_pose```([vino_people_msgs::srv::HeadPose](./vino_people_msgs/srv/HeadPoseSrv.srv))| +|Emotion Detection Service|```/detect_emotion```([vino_people_msgs::srv::Emotion](./vino_people_msgs/srv/EmotionSrv.srv))| +
+

### RViz -RViz dispaly is also supported by the composited topic of original image frame with inference result. +RViz display is also supported by the composited topic of original image frame with inference result. To show in RViz tool, add an image marker with the composited topic: -```/ros_openvino_toolkit/image_rviz```([sensor_msgs::Image](http://docs.ros.org/melodic/api/sensor_msgs/html/msg/Image.html)) +```/ros_openvino_toolkit/image_rviz```([sensor_msgs::Image](https://docs.ros.org/en/api/sensor_msgs/html/msg/Image.html)) ### Image Window OpenCV based image window is natively supported by the package. -To enable window, Image Window output should be added into the output choices in .yaml config file. see [the config file guidance](./doc/EN/YAML_CONFIGURATION_GUIDE.md) for checking/adding this feature in your launching. +To enable window, Image Window output should be added into the output choices in .yaml config file. Refer to [the config file guidance](./doc/quick_start/yaml_configuration_guide.md) for more information about checking/adding this feature in your launching. ## Demo Result Snapshots -See below pictures for the demo result snapshots. -* face detection input from standard camera +For the snapshot of demo results, refer to the following picture. + +* Face detection input from standard camera ![face_detection_demo_image](./data/images/face_detection.png "face detection demo image") -* object detection input from realsense camera +* Object detection input from realsense camera ![object_detection_demo_realsense](./data/images/object_detection.gif "object detection demo realsense") -* object segmentation input from video +* Object segmentation input from video ![object_segmentation_demo_video](./data/images/object_segmentation.gif "object segmentation demo video") -* Person Reidentification input from standard camera +* Person reidentification input from standard camera ![person_reidentification_demo_video](./data/images/person-reidentification.gif "person reidentification demo video") -# Installation & Launching -**NOTE:** Intel releases 2 different series of OpenVINO Toolkit, we call them as [OpenSource Version](https://github.com/opencv/dldt/) and [Tarball Version](https://software.intel.com/en-us/openvino-toolkit). This guidelie uses OpenSource Version as the installation and launching example. **If you want to use Tarball version, please follow [the guide for Tarball Version](./doc/EN/BINARY_VERSION_README.md).** - -## Dependencies Installation -One-step installation scripts are provided for the dependencies' installation. Please see [the guide](./doc/EN/OPEN_SOURCE_CODE_README.md) for details. - -## Launching -* Preparation - * Configure the Neural Compute Stick USB Driver - ```bash - cd ~/Downloads - cat < 97-usbboot.rules - SUBSYSTEM=="usb", ATTRS{idProduct}=="2150", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" - SUBSYSTEM=="usb", ATTRS{idProduct}=="2485", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" - SUBSYSTEM=="usb", ATTRS{idProduct}=="f63b", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" - EOF - sudo cp 97-usbboot.rules /etc/udev/rules.d/ - sudo udevadm control --reload-rules - sudo udevadm trigger - sudo ldconfig - rm 97-usbboot.rules - ``` - * download [Object Detection model](./doc/EN/OBJECT_DETECTION.md) - * download and convert a trained model to produce an optimized Intermediate Representation (IR) of the model - ```bash - #object segmentation model - cd /opt/openvino_toolkit/dldt/model-optimizer/install_prerequisites - sudo ./install_prerequisites.sh - mkdir -p ~/Downloads/models - cd ~/Downloads/models - wget http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_v2_coco_2018_01_28.tar.gz - tar -zxvf mask_rcnn_inception_v2_coco_2018_01_28.tar.gz - cd mask_rcnn_inception_v2_coco_2018_01_28 - #FP32 - sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/openvino_toolkit/dldt/model-optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --output_dir /opt/openvino_toolkit/models/segmentation/output/FP32 - #FP16 - sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/openvino_toolkit/dldt/model-optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --data_type=FP16 --output_dir /opt/openvino_toolkit/models/segmentation/output/FP16 - ``` - * download the optimized Intermediate Representation (IR) of model (excute _once_)
- ```bash - cd /opt/openvino_toolkit/open_model_zoo/tools/downloader - sudo python3 downloader.py --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output - sudo python3 downloader.py --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output - sudo python3 downloader.py --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output - sudo python3 downloader.py --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output - sudo python3 downloader.py --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output - sudo python3 downloader.py --name person-reidentification-retail-0076 --output_dir /opt/openvino_toolkit/models/person-reidentification/output - sudo python3 downloader.py --name vehicle-license-plate-detection-barrier-0106 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection/output - sudo python3 downloader.py --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recongnition/output - sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output - sudo python3 downloader.py --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output - sudo python3 downloader.py --name face-reidentification-retail-0095 --output_dir /opt/openvino_toolkit/models/face-reidentification/output - ``` - * copy label files (excute _once_)
- ```bash - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/segmentation/output/FP32/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/segmentation/output/FP16/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/intel/vehicle-license-plate-detection-barrier-0106/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - - ``` - * set ENV LD_LIBRARY_PATH
- ```bash - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/openvino_toolkit/dldt/inference-engine/bin/intel64/Release/lib - ``` -* run face detection sample code input from StandardCamera. - ```bash - roslaunch vino_launch pipeline_people.launch - ``` -* run face detection sample code input from Image. - ```bash - roslaunch vino_launch pipeline_image.launch - ``` -* run object segmentation sample code input from RealSenseCameraTopic. - ```bash - roslaunch vino_launch pipeline_segmentation.launch - ``` -* run object segmentation sample code input from Video. - ```bash - roslaunch vino_launch pipeline_video.launch - ``` -* run person reidentification sample code input from StandardCamera. - ```bash - roslaunch vino_launch pipeline_reidentification.launch - ``` -* run face re-identification with facial landmarks from realsense camera - ```bash - roslaunch vino_launch pipeline_face_reidentification.launch - ``` -* run vehicle detection sample code input from StandardCamera. - ```bash - roslaunch vino_launch pipeline_vehicle_detection.launch - ``` -* run object detection service sample code input from Image - Run image processing service: - ```bash - roslaunch vino_launch image_object_server.launch - ``` - Run example application with an absolute path of an image on another console: - ```bash - rosrun dynamic_vino_sample image_object_client ~/catkin_ws/src/ros_openvino_toolkit/data/images/car.png - ``` -* run face detection service sample code input from Image - Run image processing service: - ```bash - roslaunch vino_launch image_people_server.launch - ``` - Run example application with an absolute path of an image on another console: - ```bash - rosrun dynamic_vino_sample image_people_client ~/catkin_ws/src/ros_openvino_toolkit/data/images/team.jpg - ``` -# TODO Features -* Support **result filtering** for inference process, so that the inference results can be filtered to different subsidiary inference. For example, given an image, firstly we do Object Detection on it, secondly we pass cars to vehicle brand recognition and pass license plate to license number recognition. -* Design **resource manager** to better use such resources as models, engines, and other external plugins. -* Develop GUI based **configuration and management tools** (and monitoring and diagnose tools), in order to provide easy entry for end users to simplify their operation. +# Installation and Launching +## Deploy in Local Environment +* Refer to the quick start document for [getting_started_with_ros](./doc/quick_start/getting_started_with_ros_ov2.0.md) for detailed installation & lauching instructions. +* Refer to the quick start document for [yaml configuration guidance](./doc/quick_start/yaml_configuration_guide.md) for detailed configuration guidance. -# More Information -* ros OpenVINO discription writen in Chinese: https://mp.weixin.qq.com/s/BgG3RGauv5pmHzV_hkVAdw +## Deploy in Docker +* Refer to the docker instruction for [docker_instructions](./docker/docker_instruction.md) for detailed information about building docker image and launching. +* Refer to the quick start document for [yaml configuration guidance](./doc/quick_start/yaml_configuration_guide.md) for detailed configuration guidance. + +# Reference +* Open_model_zoo: Refer to the OpenVINO document for [open_model_zoo](https://github.com/openvinotoolkit/open_model_zoo/tree/releases/2022/1) for detailed model structure and demo samples. +* OpenVINO api 2.0: Refer to the OpenVINO document for [OpenVINO_api_2.0](https://docs.openvino.ai/latest/openvino_2_0_transition_guide.html) for latest api 2.0 transition guide. + +# FAQ +* [How to build OpenVINO by source?](https://github.com/openvinotoolkit/openvino/wiki#how-to-build) +* [How to build RealSense by source?](https://github.com/IntelRealSense/librealsense/blob/master/doc/installation.md) +* [What is the basic command of Docker CLI?](https://docs.docker.com/engine/reference/commandline/docker/) +* [What is the canonical C++ API for interacting with ROS?](https://wiki.ros.org/APIs) +# Feedback +* Report questions, issues and suggestions, using: [issue](https://github.com/intel/ros_openvino_toolkit/issues). + +# More Information +* ROS OpenVINO discription written in Chinese: https://mp.weixin.qq.com/s/BgG3RGauv5pmHzV_hkVAdw +###### *Any security issue should be reported using process at https://01.org/security* diff --git a/doc/EN/BINARY_VERSION_README.md b/doc/EN/BINARY_VERSION_README.md deleted file mode 100644 index a30ba6b6..00000000 --- a/doc/EN/BINARY_VERSION_README.md +++ /dev/null @@ -1,255 +0,0 @@ -# ros_openvino_toolkit - -## 1. Introduction -The [OpenVINO™](https://software.intel.com/en-us/openvino-toolkit) toolkit quickly deploys applications and solutions that emulate human vision. Based on Convolutional Neural Networks (CNN), the Toolkit extends computer vision (CV) workloads across Intel® hardware, maximizing performance. - -This project is a ROS wrapper for CV API of [OpenVINO™](https://software.intel.com/en-us/openvino-toolkit), providing the following features: -* Support CPU and GPU platforms -* Support standard USB camera and Intel® RealSense™ camera -* Face detection -* Emotion recognition -* Age and gender recognition -* Head pose recognition -* Demo application to show above detection and recognitions - -## 2. Prerequisite -- An x86_64 computer running Ubuntu 18.04. Below processors are supported: - * 6th-8th Generation Intel® Core™ - * Intel® Xeon® v5 family - * Intel® Xeon® v6 family -- ROS [Dashing](https://github.com/ros2/ros2) - -- [OpenVINO™ Toolkit](https://software.intel.com/en-us/openvino-toolkit) -- RGB Camera, e.g. RealSense D400 Series or standard USB camera or Video/Image File -- Graphics are required only if you use a GPU. The official system requirements for GPU are: - * 6th to 8th generation Intel® Core™ processors with Iris® Pro graphics and Intel® HD Graphics - * 6th to 8th generation Intel® Xeon® processors with Iris Pro graphics and Intel HD Graphics (excluding the e5 product family, which does not have graphics) - * Intel® Pentium® processors N4200/5, N3350/5, N3450/5 with Intel HD Graphics - - Use one of the following methods to determine the GPU on your hardware: - 1. [lspci] command: GPU info may lie in the [VGA compatible controller] line. - 2. Ubuntu system: Menu [System Settings] --> [Details] may help you find the graphics information. - 3. Openvino: Download the install package, install_GUI.sh inside will check the GPU information before installation. - -## 3. Environment Setup -**Note**:You can choose to build the environment using *./environment_setup_binary.sh* script in the script subfolder. The *modules.conf* file in the same directory as the .sh file is the configuration file that controls the installation process.You can modify the *modules.conf* to customize your installation process. -```bash -./environment_setup_binary.sh -``` -**Note**:You can also choose to follow the steps below to build the environment step by step. - -- Install ROS Dashing Desktop-Full ([guide](https://index.ros.org/doc/ros2/Installation/Dashing/)) - -- Install [OpenVINO™ Toolkit](https://software.intel.com/en-us/openvino-toolkit) ([guide](https://software.intel.com/en-us/articles/OpenVINO-Install-Linux)). - - **Note**: Please use *root privileges* to run the installer when installing the core components. -- Install OpenCL Driver for GPU -```bash -cd /opt/intel/computer_vision_sdk/install_dependencies -sudo ./install_NEO_OCL_driver.sh -``` -* Install [OpenCV 3.4 or later](https://docs.opencv.org/master/d9/df8/tutorial_root.html)([guide](https://docs.opencv.org/master/d7/d9f/tutorial_linux_install.html)) - ```bash - [compiler] sudo apt-get install build-essential - [required] sudo apt-get install cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev - [optional] sudo apt-get install python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libdc1394-22-dev - mkdir -p ~/code && cd ~/code - git clone https://github.com/opencv/opencv.git - git clone https://github.com/opencv/opencv_contrib.git - cd opencv && git checkout 3.4.2 && cd .. - cd opencv_contrib && git checkout 3.4.2 && cd .. - cd opencv - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -D OPENCV_EXTRA_MODULES_PATH=/home//code/opencv_contrib/modules/ .. - make -j8 - sudo make install - ``` - * Additional steps are required on ubuntu 18.04 - ```bash - sudo add-apt-repository "deb http://security.ubuntu.com/ubuntu xenial-security main" - sudo apt update - sudo apt install libjasper1 libjasper-dev - ``` -- Install Intel® RealSense™ SDK 2.0 [(tag v2.30.0)](https://github.com/IntelRealSense/librealsense/tree/v2.30.0) - * [Install from package](https://github.com/IntelRealSense/librealsense/blob/v2.30.0/doc/distribution_linux.md) - -- Other Dependencies -```bash -# numpy and networkx -pip3 install numpy -pip3 install networkx -# libboost -sudo apt-get install -y --no-install-recommends libboost-all-dev -cd /usr/lib/x86_64-linux-gnu -sudo ln -sf libboost_python-py35.so libboost_python3.so -``` - -## 4. Building and Installation -- Build sample code under openvino toolkit -```bash -# root is required instead of sudo -source /opt/intel/openvino/bin/setupvars.sh -cd /opt/intel/openvino/deployment_tools/inference_engine/samples/ -mkdir build -cd build -cmake .. -make -``` - -- Set Environment CPU_EXTENSION_LIB and GFLAGS_LIB -```bash -export CPU_EXTENSION_LIB=/opt/intel/openvino/deployment_tools/inference_engine/samples/build/intel64/Release/lib/libcpu_extension.so -export GFLAGS_LIB=/opt/intel/openvino/deployment_tools/inference_engine/samples/build/intel64/Release/lib/libgflags_nothreads.a -``` - -- Install ROS_OpenVINO packages -```bash -mkdir -p ~/catkin_ws/src -cd ~/catkin_ws/src -git clone https://github.com/intel/ros_openvino_toolkit -git clone https://github.com/intel/object_msgs -git clone https://github.com/ros-perception/vision_opencv -git clone https://github.com/intel-ros/realsense -cd realsense -git checkout 2.1.3 -``` - -- Build package -``` -# Ubuntu 16.04 -source /opt/ros/kinetic/setup.bash -# Ubuntu 18.04 -source /opt/ros/melodic/setup.bash - -source /opt/intel/openvino/bin/setupvars.sh -export OpenCV_DIR=$HOME/code/opencv/build -cd ~/catkin_ws -catkin_make -source ./devel/setup.bash -sudo mkdir -p /opt/openvino_toolkit -sudo ln -s ~/catkin_ws/src/ros_openvino_toolkit /opt/openvino_toolkit/ros_openvino_toolkit -``` - -## 5. Running the Demo -* Preparation - * Configure the Neural Compute Stick USB Driver - ```bash - cd ~/Downloads - cat < 97-usbboot.rules - SUBSYSTEM=="usb", ATTRS{idProduct}=="2150", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" - SUBSYSTEM=="usb", ATTRS{idProduct}=="2485", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" - SUBSYSTEM=="usb", ATTRS{idProduct}=="f63b", ATTRS{idVendor}=="03e7", GROUP="users", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1" - EOF - sudo cp 97-usbboot.rules /etc/udev/rules.d/ - sudo udevadm control --reload-rules - sudo udevadm trigger - sudo ldconfig - rm 97-usbboot.rules - ``` - * download [Object Detection model](https://github.com/intel/ros_openvino_toolkit/tree/devel/doc/OBJECT_DETECTION.md) - * download and convert a trained model to produce an optimized Intermediate Representation (IR) of the model - ```bash - #object segmentation model - cd /opt/intel/openvino/deployment_tools/model_optimizer/install_prerequisites - sudo ./install_prerequisites.sh - mkdir -p ~/Downloads/models - cd ~/Downloads/models - wget http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_v2_coco_2018_01_28.tar.gz - tar -zxvf mask_rcnn_inception_v2_coco_2018_01_28.tar.gz - cd mask_rcnn_inception_v2_coco_2018_01_28 - #FP32 - sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/intel/openvino/deployment_tools/model_optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --output_dir /opt/openvino_toolkit/models/segmentation/output/FP32 - #FP16 - sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo_tf.py --input_model frozen_inference_graph.pb --tensorflow_use_custom_operations_config /opt/intel/openvino/deployment_tools/model_optimizer/extensions/front/tf/mask_rcnn_support.json --tensorflow_object_detection_api_pipeline_config pipeline.config --reverse_input_channels --data_type=FP16 --output_dir /opt/openvino_toolkit/models/segmentation/output/FP16 - ``` - * download the optimized Intermediate Representation (IR) of model (excute once) - ```bash - cd /opt/intel/openvino/deployment_tools/tools/model_downloader - sudo python3 downloader.py --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output - sudo python3 downloader.py --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output - sudo python3 downloader.py --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output - sudo python3 downloader.py --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output - sudo python3 downloader.py --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output - sudo python3 downloader.py --name person-reidentification-retail-0076 --output_dir /opt/openvino_toolkit/models/person-reidentification/output - sudo python3 downloader.py --name vehicle-license-plate-detection-barrier-0106 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection/output - sudo python3 downloader.py --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recongnition/output - sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output - sudo python3 downloader.py --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output - sudo python3 downloader.py --name face-reidentification-retail-0095 --output_dir /opt/openvino_toolkit/models/face-reidentification/output - ``` - * copy label files (excute _once_)
- ```bash - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/segmentation/output/FP32/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/segmentation/output/FP16/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/intel/vehicle-license-plate-detection-barrier-0106/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - ``` - * set ENV LD_LIBRARY_PATH and environment - ```bash - source /opt/intel/openvino/bin/setupvars.sh - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/openvino/deployment_tools/inference_engine/samples/build/intel64/Release/lib - ``` -* run face detection sample code input from StandardCamera.(connect Intel® Neural Compute Stick 2) - ```bash - roslaunch vino_launch pipeline_people.launch - ``` -* run face detection sample code input from Image. - ```bash - roslaunch vino_launch pipeline_image.launch - ``` -* run object segmentation sample code input from RealSenseCameraTopic. - ```bash - roslaunch vino_launch pipeline_segmentation.launch - ``` -* run object segmentation sample code input from Video. - ```bash - roslaunch vino_launch pipeline_video.launch - ``` -* run person reidentification sample code input from StandardCamera. - ```bash - roslaunch vino_launch pipeline_reidentification.launch - ``` -* run face re-identification with facial landmarks from realsense camera - ```bash - roslaunch vino_launch pipeline_face_reidentification.launch - ``` -* run vehicle detection sample code input from StandardCamera. - ```bash - roslaunch vino_launch pipeline_vehicle_detection.launch - ``` -* run object detection service sample code input from Image - Run image processing service: - ```bash - roslaunch vino_launch image_object_server.launch - ``` - Run example application with an absolute path of an image on another console: - ```bash - rosrun dynamic_vino_sample image_object_client ~/catkin_ws/src/ros_openvino_toolkit/data/images/car.png - ``` -* run people detection service sample code input from Image - Run image processing service: - ```bash - roslaunch vino_launch image_people_server.launch - ``` - Run example application with an absolute path of an image on another console: - ```bash - rosrun dynamic_vino_sample image_people_client ~/catkin_ws/src/ros_openvino_toolkit/data/images/team.jpg - ``` -## 6. Known Issues -* Possible problems - * When running sample with Intel® Neural Compute Stick 2 occurred error: - ```bash - E: [ncAPI] [ 0] ncDeviceOpen:668 failed to find device - # or - E: [ncAPI] [ 0] ncDeviceCreate:324 global mutex initialization failed - ``` - > solution - Please refer to the [guide](https://software.intel.com/en-us/neural-compute-stick/get-started) to set up the environment. - * Segmentation fault occurs occasionally, which is caused by librealsense library. See [Issue #2645](https://github.com/IntelRealSense/librealsense/issues/2645) for more details. - -###### *Any security issue should be reported using process at https://01.org/security* - - diff --git a/doc/EN/OBJECT_DETECTION.md b/doc/EN/OBJECT_DETECTION.md deleted file mode 100644 index 5e23b1ae..00000000 --- a/doc/EN/OBJECT_DETECTION.md +++ /dev/null @@ -1,190 +0,0 @@ -# Object Detection -## Launching -### OpenSource Version -#### mobilenet-ssd -* download and convert a trained model to produce an optimized Intermediate Representation (IR) of the model - ```bash - cd /opt/openvino_toolkit/open_model_zoo/tools/downloader - python3 ./downloader.py --name mobilenet-ssd - #FP32 precision model - sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo.py --input_model /opt/openvino_toolkit/open_model_zoo/tools/downloader/public/mobilenet-ssd/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP32 --mean_values [127.5,127.5,127.5] --scale_values [127.5] - #FP16 precision model - sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo.py --input_model /opt/openvino_toolkit/open_model_zoo/tools/downloader/public/mobilenet-ssd/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16 --data_type=FP16 --mean_values [127.5,127.5,127.5] --scale_values [127.5] - ``` -* copy label files (excute _once_)
- ```bash - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16 - ``` -* run object detection sample code input from RealSenseCamera.(connect Intel® Neural Compute Stick 2) - ```bash - roslaunch vino_launch pipeline_object.launch - ``` -* run object detection sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) - ```bash - roslaunch vino_launch pipeline_object_topic.launch - ``` -#### YOLOv2-voc -* Darkflow to protobuf(.pb) - - install [darkflow](https://github.com/thtrieu/darkflow) - - install prerequsites - ```bash - pip3 install tensorflow opencv-python numpy networkx cython - ``` - - Get darkflow and YOLO-OpenVINO - ```bash - mkdir -p ~/code && cd ~/code - git clone https://github.com/thtrieu/darkflow - git clone https://github.com/chaoli2/YOLO-OpenVINO - sudo ln -sf ~/code/darkflow /opt/openvino_toolkit/ - ``` - - modify the line self.offset = 16 in the ./darkflow/utils/loader.py file and replace with self.offset = 20 - - Install darkflow - ```bash - cd ~/code/darkflow - pip3 install . - ``` - - Copy voc.names in YOLO-OpenVINO/common to labels.txt in darkflow. - ```bash - cp ~/code/YOLO-OpenVINO/common/voc.names ~/code/darkflow/labels.txt - ``` - - Get yolov2 weights and cfg - ```bash - cd ~/code/darkflow - mkdir -p models - cd models - wget -c https://pjreddie.com/media/files/yolov2-voc.weights - wget https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2-voc.cfg - ``` - - Run convert script - ```bash - cd ~/code/darkflow - flow --model models/yolov2-voc.cfg --load models/yolov2-voc.weights --savepb - ``` -* Convert YOLOv2-voc TensorFlow Model to the optimized Intermediate Representation (IR) of model - ```bash - cd ~/code/darkflow - # FP32 precision model - sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo_tf.py \ - --input_model built_graph/yolov2-voc.pb \ - --batch 1 \ - --tensorflow_use_custom_operations_config /opt/openvino_toolkit/dldt/model-optimizer/extensions/front/tf/yolo_v2_voc.json \ - --data_type FP32 \ - --output_dir /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP32 - # FP16 precision model - sudo python3 /opt/openvino_toolkit/dldt/model-optimizer/mo_tf.py \ - --input_model built_graph/yolov2-voc.pb \ - --batch 1 \ - --tensorflow_use_custom_operations_config /opt/openvino_toolkit/dldt/model-optimizer/extensions/front/tf/yolo_v2_voc.json \ - --data_type FP16 \ - --output_dir /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP16 - ``` -* copy label files (excute _once_)
- ```bash - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/yolov2-voc.labels /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/yolov2-voc.labels /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP16 - ``` -* run object detection sample code input from RealSenseCamera.(connect Intel® Neural Compute Stick 2) - ```bash - roslaunch vino_launch pipeline_object_yolo.launch - ``` -* run object detection sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) - ```bash - roslaunch vino_launch pipeline_object_yolo_topic.launch - ``` -### Binary Version -#### mobilenet-ssd -* download and convert a trained model to produce an optimized Intermediate Representation (IR) of the model - ```bash - cd /opt/intel/openvino/deployment_tools/tools/model_downloader - sudo python3 ./downloader.py --name mobilenet-ssd - #FP32 precision model - sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo.py --input_model /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/public/mobilenet-ssd/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP32 --mean_values [127.5,127.5,127.5] --scale_values [127.5] - #FP16 precision model - sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo.py --input_model /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader/public/mobilenet-ssd/mobilenet-ssd.caffemodel --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16 --data_type=FP16 --mean_values [127.5,127.5,127.5] --scale_values [127.5] - ``` -* copy label files (excute _once_)
- ```bash - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet-ssd/caffe/output/FP16 - ``` -* run object detection sample code input from RealSenseCamera.(connect Intel® Neural Compute Stick 2) - ```bash - roslaunch vino_launch pipeline_object.launch - ``` -* run object detection sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) - ```bash - roslaunch vino_launch pipeline_object_topic.launch - ``` - #### YOLOv2-voc -* Darkflow to protobuf(.pb) - - install [darkflow](https://github.com/thtrieu/darkflow) - - install prerequsites - ```bash - pip3 install tensorflow opencv-python numpy networkx cython - ``` - - Get darkflow and YOLO-OpenVINO - ```bash - mkdir -p ~/code && cd ~/code - git clone https://github.com/thtrieu/darkflow - git clone https://github.com/chaoli2/YOLO-OpenVINO - sudo ln -sf ~/code/darkflow /opt/openvino_toolkit/ - ``` - - modify the line self.offset = 16 in the ./darkflow/utils/loader.py file and replace with self.offset = 20 - - Install darkflow - ```bash - cd ~/code/darkflow - pip3 install . - ``` - - Copy voc.names in YOLO-OpenVINO/common to labels.txt in darkflow. - ```bash - cp ~/code/YOLO-OpenVINO/common/voc.names ~/code/darkflow/labels.txt - ``` - - Get yolov2 weights and cfg - ```bash - cd ~/code/darkflow - mkdir -p models - cd models - wget -c https://pjreddie.com/media/files/yolov2-voc.weights - wget https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov2-voc.cfg - ``` - - Run convert script - ```bash - cd ~/code/darkflow - flow --model models/yolov2-voc.cfg --load models/yolov2-voc.weights --savepb - ``` -* Convert YOLOv2-voc TensorFlow Model to the optimized Intermediate Representation (IR) of model - ```bash - cd ~/code/darkflow - # FP32 precision model - sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo_tf.py \ - --input_model built_graph/yolov2-voc.pb \ - --batch 1 \ - --tensorflow_use_custom_operations_config /opt/intel/openvino/deployment_tools/model_optimizer/extensions/front/tf/yolo_v2_voc.json \ - --data_type FP32 \ - --output_dir /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP32 - # FP16 precision model - sudo python3 /opt/intel/openvino/deployment_tools/model_optimizer/mo_tf.py \ - --input_model built_graph/yolov2-voc.pb \ - --batch 1 \ - --tensorflow_use_custom_operations_config /opt/intel/openvino/deployment_tools/model_optimizer/extensions/front/tf/yolo_v2_voc.json \ - --data_type FP16 \ - --output_dir /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP16 - ``` -* copy label files (excute _once_)
- ```bash - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/yolov2-voc.labels /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP32 - sudo cp /opt/openvino_toolkit/ros_openvino_toolkit/data/labels/object_detection/yolov2-voc.labels /opt/openvino_toolkit/models/object_detection/YOLOv2-voc/tf/output/FP16 - ``` -* run object detection sample code input from RealSenseCamera.(connect Intel® Neural Compute Stick 2) - ```bash - roslaunch vino_launch pipeline_object_yolo.launch - ``` -* run object detection sample code input from RealSenseCameraTopic.(connect Intel® Neural Compute Stick 2) - ```bash - roslaunch vino_launch pipeline_object_yolo_topic.launch - ``` - - - - diff --git a/doc/EN/OPEN_SOURCE_CODE_README.md b/doc/EN/OPEN_SOURCE_CODE_README.md deleted file mode 100644 index 48e811ce..00000000 --- a/doc/EN/OPEN_SOURCE_CODE_README.md +++ /dev/null @@ -1,152 +0,0 @@ -# ros_openvino_toolkit - -## 1. Introduction -The [OpenVINO™](https://software.intel.com/en-us/openvino-toolkit) toolkit quickly deploys applications and solutions that emulate human vision. Based on Convolutional Neural Networks (CNN), the Toolkit extends computer vision (CV) workloads across Intel® hardware, maximizing performance. - -This project is a ROS wrapper for CV API of [OpenVINO™](https://software.intel.com/en-us/openvino-toolkit), providing the following features: -* Support CPU and GPU platforms -* Support standard USB camera and Intel® RealSense™ camera -* Face detection -* Emotion recognition -* Age and gender recognition -* Head pose recognition -* Demo application to show above detection and recognitions - -## 2. Prerequisite -- An x86_64 computer running Ubuntu 16.04. Below processors are supported: - * 6th-8th Generation Intel® Core™ - * Intel® Xeon® v5 family - * Intel® Xeon® v6 family -- ROS Kinetic(For Ubuntu16.04) or ROS Melodic(For Ubuntu18.04) - -- OpenVINO™ Toolkit Open Source - * The [Deep Learning Deployment Toolkit](https://github.com/opencv/dldt) that helps to enable fast, heterogeneous deep learning inferencing for Intel® processors (CPU and GPU/Intel® Processor Graphics), and supports more than 100 public and custom models. - * [Open Model Zoo](https://github.com/opencv/open_model_zoo) includes 20+ pre-trained deep learning models to expedite development and improve deep learning inference on Intel® processors (CPU, Intel Processor Graphics, FPGA, VPU), along with many samples to easily get started. -- RGB Camera, e.g. RealSense D400 Series or standard USB camera or Video/Image File - -- Graphics are required only if you use a GPU. The official system requirements for GPU are: - * 6th to 8th generation Intel® Core™ processors with Iris® Pro graphics and Intel® HD Graphics - * 6th to 8th generation Intel® Xeon® processors with Iris Pro graphics and Intel HD Graphics (excluding the e5 product family, which does not have graphics) - * Intel® Pentium® processors N4200/5, N3350/5, N3450/5 with Intel HD Graphics - -- Use one of the following methods to determine the GPU on your hardware: - * [lspci] command: GPU info may lie in the [VGA compatible controller] line. - * Ubuntu system: Menu [System Settings] --> [Details] may help you find the graphics information. - * Openvino: Download the install package, install_GUI.sh inside will check the GPU information before installation. - -## 3. Environment Setup -**Note**:You can choose to build the environment using *./environment_setup.sh* script in the script subfolder. The *modules.conf* file in the same directory as the .sh file is the configuration file that controls the installation process.You can modify the *modules.conf* to customize your installation process. -```bash -./environment_setup.sh -``` -**Note**:You can also choose to follow the steps below to build the environment step by step. -- For Ubuntu16.04, install ROS Kinetic Desktop-Full [(guide)](http://wiki.ros.org/kinetic/Installation/Ubuntu) -- For Ubuntu18.04, install ROS Melodic Desktop-Full [(guide)](http://wiki.ros.org/melodic/Installation/Ubuntu) -- Install OpenVINO™ Toolkit Open Source - * Install [OpenCV 3.x: 3.4 or later](https://docs.opencv.org/master/d9/df8/tutorial_root.html)([guide](https://docs.opencv.org/master/d7/d9f/tutorial_linux_install.html)) - ``` - [compiler] sudo apt-get install build-essential - [required] sudo apt-get install cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev - [optional] sudo apt-get install python-dev python-numpy libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libjasper-dev libdc1394-22-dev - cd ~/code - git clone https://github.com/opencv/opencv.git - git clone https://github.com/opencv/opencv_contrib.git - cd opencv && git checkout 3.4.2 && cd .. - cd opencv_contrib && git checkout 3.4.2 && cd .. - cd opencv - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -D OPENCV_EXTRA_MODULES_PATH=$HOME/code/opencv_contrib/modules/ .. - make -j8 - sudo make install - ``` - - * Install OpenCL Driver for GPU([guide](http://registrationcenter-download.intel.com/akdlm/irc_nas/11396/intel-opencl-4.1-installation.pdf))
- ```bash - cd ~/Downloads - wget http://registrationcenter-download.intel.com/akdlm/irc_nas/11396/SRB5.0_linux64.zip - unzip SRB5.0_linux64.zip -d SRB5.0_linux64 - cd SRB5.0_linux64 - sudo apt-get install xz-utils - mkdir intel-opencl - tar -C intel-opencl -Jxf intel-opencl-r5.0-63503.x86_64.tar.xz - tar -C intel-opencl -Jxf intel-opencl-devel-r5.0-63503.x86_64.tar.xz - tar -C intel-opencl -Jxf intel-opencl-cpu-r5.0-63503.x86_64.tar.xz - sudo cp -R intel-opencl/* / - sudo ldconfig - ``` - * Install [Deep Learning Deployment Toolkit](https://github.com/opencv/dldt)([guide](https://github.com/opencv/dldt/tree/2018/inference-engine))
- ```bash - mkdir ~/code && cd ~/code - git clone https://github.com/opencv/dldt.git - cd dldt/inference-engine/ - git checkout 2019_R3 - ./install_dependencies.sh - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - make -j8 - sudo mkdir -p /opt/openvino_toolkit - sudo ln -s ~/code/dldt /opt/openvino_toolkit/dldt - ``` - * Install [Open Model Zoo](https://github.com/opencv/open_model_zoo)([guide](https://github.com/opencv/open_model_zoo/tree/2018/demos))
- ```bash - cd ~/code - git clone https://github.com/opencv/open_model_zoo.git - cd open_model_zoo/demos/ - git checkout 2019_R3 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release /opt/openvino_toolkit/dldt/inference-engine - make -j8 - sudo mkdir -p /opt/openvino_toolkit - sudo ln -s ~/code/open_model_zoo /opt/openvino_toolkit/open_model_zoo - ``` - -- Install Intel® RealSense™ SDK 2.0 [(tag v2.30.0)](https://github.com/IntelRealSense/librealsense/tree/v2.30.0) - * [Install from package](https://github.com/IntelRealSense/librealsense/blob/v2.30.0/doc/distribution_linux.md) - -- Other Dependencies - ```bash - # numpy and networkx - pip3 install numpy - pip3 install networkx - # libboost - sudo apt-get install -y --no-install-recommends libboost-all-dev - cd /usr/lib/x86_64-linux-gnu - sudo ln -s libboost_python-py35.so libboost_python3.so - ``` - -## 4. Building and Installation - -* set ENV InferenceEngine_DIR, CPU_EXTENSION_LIB and GFLAGS_LIB - ```bash - export InferenceEngine_DIR=/opt/openvino_toolkit/dldt/inference-engine/build/ - export CPU_EXTENSION_LIB=/opt/openvino_toolkit/dldt/inference-engine/bin/intel64/Release/lib/libcpu_extension.so - export GFLAGS_LIB=/opt/openvino_toolkit/dldt/inference-engine/bin/intel64/Release/lib/libgflags_nothreads.a - ``` -* Install ROS_OpenVINO packages - ```bash - mkdir -p ~/catkin_ws/src - cd ~/catkin_ws/src - git clone https://github.com/intel/ros_openvino_toolkit - git clone https://github.com/intel/object_msgs - git clone https://github.com/ros-perception/vision_opencv - git clone https://github.com/intel-ros/realsense - cd realsense - git checkout 2.1.3 - ``` - -* Build package - ``` - # Ubuntu 16.04 - source /opt/ros/kinetic/setup.bash - # Ubuntu 18.04 - source /opt/ros/melodic/setup.bash - - cd ~/catkin_ws - catkin_make - source devel/setup.bash - sudo mkdir -p /opt/openvino_toolkit - sudo ln -s ~/catkin_ws/src/ros_openvino_toolkit /opt/openvino_toolkit/ros_openvino_toolkit - ``` - -###### *Any security issue should be reported using process at https://01.org/security* - diff --git a/doc/EN/YAML_CONFIGURATION_GUIDE.md b/doc/EN/YAML_CONFIGURATION_GUIDE.md deleted file mode 100644 index fb2f78dd..00000000 --- a/doc/EN/YAML_CONFIGURATION_GUIDE.md +++ /dev/null @@ -1,119 +0,0 @@ -# Introduction - -The contents in .yaml config file should be well structured and follow the supported rules and entity names. - -# Sample -## [pipeline_people.yaml](https://github.com/intel/ros_openvino_toolkit/blob/devel/sample/param/pipeline_people.yaml) -```bash -Pipelines: -- name: people - inputs: [StandardCamera] - infers: - - name: FaceDetection - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-detection-adas-0001/FP32/face-detection-adas-0001.xml - engine: CPU - label: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-detection-adas-0001/FP32/face-detection-adas-0001.labels - batch: 1 - - name: AgeGenderRecognition - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - - name: EmotionRecognition - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml - engine: CPU - label: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.labels - batch: 16 - - name: HeadPoseEstimation - model: /opt/intel/computer_vision_sdk/deployment_tools/intel_models/head-pose-estimation-adas-0001/FP32/head-pose-estimation-adas-0001.xml - engine: CPU - label: to/be/set/xxx.labels - batch: 16 - outputs: [ImageWindow, RosTopic, RViz] - confidence_threshold: 0.2 - connects: - - left: StandardCamera - right: [FaceDetection] - - left: FaceDetection - right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, ImageWindow, RosTopic, RViz] - - left: AgeGenderRecognition - right: [ImageWindow, RosTopic, RViz] - - left: EmotionRecognition - right: [ImageWindow, RosTopic, RViz] - - left: HeadPoseEstimation - right: [ImageWindow, RosTopic, RViz] - -Common: -``` -## Interface Description - -### name -The name of this pipeline, its value can be any value other than empty. - -### inputs -**Note**:The value of the input parametar can only have one.
-Currently, options for inputs are: - -|option|Description| -|--------------------|------------------------------------------------------------------| -|StandardCamera|Any RGB camera with USB port supporting. Currently only the first USB camera if many are connected.| -|RealSenseCamera| Intel RealSense RGB-D Camera, directly calling RealSense Camera via librealsense plugin of openCV.| -|RealSenseCameraTopic| any ROS topic which is structured in image message.| -|Image| Any image file which can be parsed by openCV, such as .png, .jpeg.| -|Video| Any video file which can be parsed by openCV.| - -### input_path -When input is Image or Video, need to use input_path to specify the path of the input file. - -### infers -The Inference Engine is a set of C++ classes to provides an API to read the Intermediate Representation, set the input and output formats, and execute the model on devices. - -#### name -The name of the inference engine. Currently, the inference feature list is supported: - -|Inference|Description| -|-----------------------|------------------------------------------------------------------| -|FaceDetection|Object Detection task applied to face recognition using a sequence of neural networks.| -|EmotionRecognition| Emotion recognition based on detected face image.| -|AgeGenderRecognition| Age and gener recognition based on detected face image.| -|HeadPoseEstimation| Head pose estimation based on detected face image.| -|ObjectDetection| object detection based on SSD-based trained models.| -|VehicleDetection| Vehicle and passenger detection based on Intel models.| -|ObjectSegmentation| object detection and segmentation.| - -#### model -The path of the model. The scheme below illustrates the typical workflow for deploying a trained deep learning model. -![trained deep learning model](https://github.com/intel/ros_openvino_toolkit/blob/devel/data/images/CVSDK_Flow.png "trained deep learning model") - -#### engine -**Note**:Currently, only supports CPU and GPU.
-options for target device are: - -|target device| -|-----------------------| -|CPU| -|Intel® Integrated Graphics| -|FPGA| -|Intel® Movidius™ Neural Compute Stick| - -#### label -Currently, This parameter does not work. - -#### batch -Enable dynamic batch size for inference engine net. - -### outputs -**Note**:The value of the output parameter can be selected one or more.
-Currently, options for outputs are: - -|option|Description| -|--------------------|------------------------------------------------------------------| -|ImageWindow| window showing results| -|RosTopic| output the topic| -|RViz| display the result in rviz| - -### confidence_threshold -Probability threshold for detections. - -### connects -The topology of the pipeline, left can only have one value, right can have multiple values. diff --git a/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md b/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md deleted file mode 100644 index 588874d3..00000000 --- a/doc/quick_start/getting_started_with_Noetic_Ubuntu20.04_ov2.0.md +++ /dev/null @@ -1,194 +0,0 @@ -# ROS_NOETIC_OpenVINO_Toolkit - -**NOTE:** -Below steps have been tested on **Ubuntu 20.04**. - -## 1. Environment Setup -* Install ROS Noetic ([guide](http://wiki.ros.org/noetic/Installation/Ubuntu)). - -* Install Intel® OpenVINO™ Toolkit Version: 2022.1 ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)). - * Install from an achive file ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)). -Tips: Both runtime and development tool are needed, `pip` is recommended for installing the development tool ([guide](https://docs.openvino.ai/latest/openvino_docs_install_guides_install_dev_tools.html)). - * Install from source code ([guide](https://github.com/openvinotoolkit/openvino/wiki/BuildingForLinux)). - -* Install Intel® RealSense ™ SDK ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)). - -* Install Dependency - * Install gflags - ``` - sudo apt-get install -y libgflags-dev - ``` - -## 2. Build and Installation -* Install ROS_OpenVINO_Toolkit packages -``` -mkdir -p ~/catkin_ws/src -cd ~/catkin_ws/src -git clone https://github.com/intel/ros_openvino_toolkit -b noetic -git clone https://github.com/intel/object_msgs -git clone https://github.com/IntelRealSense/realsense-ros.git -cd realsense-ros -git checkout `git tag | sort -V | grep -P "^2.\d+\.\d+" | tail -1` -``` -* Install dependencies -``` -sudo apt-get install ros-noetic-ddynamic-reconfigure -``` -* Build package -``` -source /opt/ros/noetic/setup.bash -source /setupvars.sh -cd ~/catkin_ws -catkin_make_isolated -source ./devel_isolated/setup.bash -``` - -## 3. Running the Demo -### Install OpenVINO 2022.1 by source code -* See all available models -``` -cd /openvino/thirdparty/open_model_zoo/tools/model_tools -sudo python3 downloader.py --print_all -``` - -* Download the optimized Intermediate Representation (IR) of model (execute once): -``` -cd /openvino/thirdparty/open_model_zoo/tools/model_tools -sudo python3 downloader.py --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output -sudo python3 downloader.py --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output -sudo python3 downloader.py --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output -sudo python3 downloader.py --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output -sudo python3 downloader.py --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output -sudo python3 downloader.py --name person-reidentification-retail-0277 --output_dir /opt/openvino_toolkit/models/person-reidentification/output -sudo python3 downloader.py --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output -sudo python3 downloader.py --name face-reidentification-retail-0095 --output_dir /opt/openvino_toolkit/models/face-reidentification-retail/output -sudo python3 downloader.py --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recognition/output -sudo python3 downloader.py --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output -``` - -* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi)need to be converted to intermediate representation (For example the model for object detection) - * mobilenet-ssd (replaced ssd_mobilenet_v2_coco to mobilenet-ssd for object detection in ov2022) - ``` - cd /openvino/thirdparty/open_model_zoo/tools/model_tools - sudo python3 downloader.py --name mobilenet-ssd --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output - sudo python3 converter.py --name mobilenet-ssd -d /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/ -o /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert - ``` - * vehicle-license-plate-detection-barrier-0123 - ``` - cd /openvino/thirdparty/open_model_zoo/tools/model_tools - sudo python3 downloader.py --name vehicle-license-plate-detection-barrier-0123 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output - sudo python3 converter.py --name vehicle-license-plate-detection-barrier-0123 -d /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output -o /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output/convert - ``` - * deeplabv3 - ``` - cd /openvino/thirdparty/open_model_zoo/tools/model_tools - sudo python3 downloader.py --name deeplabv3 --output_dir /opt/openvino_toolkit/models/deeplabv3/output - sudo python3 converter.py --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert - ``` - -* Copy label files (execute once) -* Before launch, copy label files to the same model path, make sure the model path and label path match the ros_openvino_toolkit/vino_launch/param/xxxx.yaml. -``` - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/emotions-recognition/output/intel/emotions-recognition-retail-0003/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP32/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/face_detection/output/intel/face-detection-adas-0001/FP16/ - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/vehicle-license-plate-detection/output/convert/intel/vehicle-license-plate-detection-barrier-0106/FP32 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert/public/mobilenet-ssd/FP16 - sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16 - sudo mv /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/frozen_inference_graph.labels /opt/openvino_toolkit/models/deeplabv3/output/convert/public/deeplabv3/FP16/deeplabv3.labels -``` -### Install OpenVINO 2022.1 by PIP -* OMZ tools are provided for downloading and converting OMZ models in ov2022 ([guide](https://pypi.org/project/openvino-dev/)). - -* See all available models -``` -omz_downloader --print_all -``` - -* Download the optimized Intermediate Representation (IR) of model (execute once), for example: -``` -omz_downloader --name face-detection-adas-0001 --output_dir /opt/openvino_toolkit/models/face_detection/output -omz_downloader --name age-gender-recognition-retail-0013 --output_dir /opt/openvino_toolkit/models/age-gender-recognition/output -omz_downloader --name emotions-recognition-retail-0003 --output_dir /opt/openvino_toolkit/models/emotions-recognition/output -omz_downloader --name head-pose-estimation-adas-0001 --output_dir /opt/openvino_toolkit/models/head-pose-estimation/output -omz_downloader --name person-detection-retail-0013 --output_dir /opt/openvino_toolkit/models/person-detection/output -omz_downloader --name person-reidentification-retail-0277 --output_dir /opt/openvino_toolkit/models/person-reidentification/output -omz_downloader --name landmarks-regression-retail-0009 --output_dir /opt/openvino_toolkit/models/landmarks-regression/output -omz_downloader --name semantic-segmentation-adas-0001 --output_dir /opt/openvino_toolkit/models/semantic-segmentation/output -omz_downloader --name vehicle-attributes-recognition-barrier-0039 --output_dir /opt/openvino_toolkit/models/vehicle-attributes-recognition/output -omz_downloader --name license-plate-recognition-barrier-0001 --output_dir /opt/openvino_toolkit/models/license-plate-recognition/output -``` -* Copy label files (execute once) -``` -sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP32/ -sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/emotions-recognition/FP32/emotions-recognition-retail-0003.labels /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP16/ -sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP32/ -sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/face_detection/face-detection-adas-0001.labels /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP16/ -sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/vehicle-license-plate-detection-barrier-0106.labels /opt/openvino_toolkit/models/intel/vehicle-license-plate-detection-barrier-0106/FP32 -sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_detection/mobilenet-ssd.labels /opt/openvino_toolkit/models/convert/public/mobilenet-ssd/FP16/ -sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/convert/public/deeplabv3/FP16/ -sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/intel/semantic-segmentation-adas-0001/FP16/ -``` - -* If the model (tensorflow, caffe, MXNet, ONNX, Kaldi) need to be converted to intermediate representation (such as the model for object detection): - * mobilenet-ssd (replaced ssd_mobilenet_v2_coco to mobilenet-ssd for object detection in ov2022) - ``` - omz_downloader --name mobilenet-ssd --output_dir /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output - omz_converter --name mobilenet-ssd -d /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output -o /opt/openvino_toolkit/models/object_detection/mobilenet_ssd/output/convert - ``` - * vehicle-license-plate-detection-barrier-0123 - ``` - omz_downloader --name vehicle-license-plate-detection-barrier-0123 --output_dir /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output - omz_converter --name vehicle-license-plate-detection-barrier-0123 -d /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output -o /opt/openvino_toolkit/models/vehicle-license-plate-detection-barrier/output/convert - ``` - - * deeplabv3 - ``` - omz_downloader --name deeplabv3 --output_dir /opt/openvino_toolkit/models/deeplabv3/output - omz_converter --name deeplabv3 -d /opt/openvino_toolkit/models/deeplabv3/output -o /opt/openvino_toolkit/models/deeplabv3/output/convert - ``` - -* Please check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. - * run face detection sample code input from StandardCamera. - ``` - roslaunch vino_launch pipeline_people.launch - ``` - * run person reidentification sample code input from StandardCamera. - ``` - roslaunch vino_launch pipeline_face_reidentification.launch - ``` - * run person reidentification sample code input from StandardCamera - ``` - roslaunch vino_launch pipeline_reidentification.launch - ``` - * run face detection sample code input from Image. - ``` - roslaunch vino_launch pipeline_image.launch - ``` - * run object sample - ``` - roslaunch vino_launch pipeline_object.launch - ``` - * run object topic sample - ``` - roslaunch vino_launch pipeline_object_topic.launch - ``` - * run object segmentation sample code input from RealSenseCamera. - ``` - roslaunch vino_launch pipeline_segmentation.launch - ``` - * run vehicle detection sample code input from StandardCamera. - ``` - roslaunch vino_launch pipeline_vehicle_detection.launch - ``` - * run video sample - ``` - roslaunch vino_launch pipeline_video.launch - ``` - -# More Information - -###### *Any security issue should be reported using process at https://01.org/security* - - diff --git a/doc/quick_start/getting_started_with_ros_ov2.0.md b/doc/quick_start/getting_started_with_ros_ov2.0.md index 37065c40..44ce1b4d 100644 --- a/doc/quick_start/getting_started_with_ros_ov2.0.md +++ b/doc/quick_start/getting_started_with_ros_ov2.0.md @@ -5,21 +5,29 @@ Below steps have been tested on **Ubuntu 18.04** and **Ubuntu 20.04**. Supported ROS versions include noetic and melodic. ## 1. Environment Setup -* For ROS noetic on ubuntu 20.04: - * Install ROS. ([noetic_guide](http://wiki.ros.org/noetic/Installation/Ubuntu)) + \For ROS noetic on ubuntu 20.04: + * Install ROS.
+ Refer to: [ROS_noetic_install_guide](http://wiki.ros.org/noetic/Installation/Ubuntu) - * Install Intel® OpenVINO™ Toolkit Version: 2022.1. ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)) - * Install from an achive file. Both runtime and development tool are needed, `pip` is recommended for installing the development tool. ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)) + * Install Intel® OpenVINO™ Toolkit Version: 2022.1.
+ Refer to: [OpenVINO_install_guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html) + * Install from an achive file. Both runtime and development tool are needed, `pip` is recommended for installing the development tool.
+ Refer to: [OpenVINO_devtool_install_guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) - * Install Intel® RealSense™ SDK. ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)) + * Install Intel® RealSense™ SDK.
+ Refer to: [RealSense_install_guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md) -* For ROS melodic on ubuntu 18.04: - * Install ROS. ([melodic_guide](http://wiki.ros.org/melodic/Installation/Ubuntu)) +For ROS melodic on ubuntu 18.04: + * Install ROS.
+ Refer to: [ROS_melodic_install_guide](http://wiki.ros.org/melodic/Installation/Ubuntu)) - * Install Intel® OpenVINO™ Toolkit Version: 2022.1. ([guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html)) - * Install from an achive file. Both runtime and development tool are needed, `pip` is recommended for installing the development tool. ([guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html)) + * Install Intel® OpenVINO™ Toolkit Version: 2022.1.
+ Refer to: [OpenVINO_install_guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html) + * Install from an achive file. Both runtime and development tool are needed, `pip` is recommended for installing the development tool.
+ Refer to: [OpenVINO_devtool_install_guide](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit/download.html) - * Install Intel® RealSense™ SDK. ([guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md)) + * Install Intel® RealSense™ SDK.
+ Refer to: [RealSense_install_guide](https://github.com/IntelRealSense/librealsense/blob/master/doc/distribution_linux.md) * Install Dependency * Install gflags @@ -55,7 +63,8 @@ source ./devel_isolated/setup.bash ## 3. Running the Demo ### Install OpenVINO 2022.1 by PIP -* OMZ tools are provided for downloading and converting models of open_model_zoo in ov2022.([guide](https://pypi.org/project/openvino-dev/)) +* OMZ tools are provided for downloading and converting models of open_model_zoo in ov2022.
+Refer to: [OMZtool_guide](https://pypi.org/project/openvino-dev/) * See all available models ``` @@ -106,7 +115,7 @@ sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/fro sudo cp ~/catkin_ws/src/ros_openvino_toolkit/data/labels/object_segmentation/frozen_inference_graph.labels /opt/openvino_toolkit/models/intel/semantic-segmentation-adas-0001/FP16/ ``` -* Please check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly. +* Please check the parameter configuration in ros_openvino_toolkit/sample/param/xxxx.yaml before lauching, make sure parameters such as model_path, label_path and input_path are set correctly.Please refer to the quick start document for [yaml configuration guidance](./yaml_configuration_guide.md) for detailed configuration guidance. * run face detection sample code input from StandardCamera. ``` roslaunch vino_launch pipeline_people.launch diff --git a/doc/quick_start/yaml_configuration_guide.md b/doc/quick_start/yaml_configuration_guide.md new file mode 100644 index 00000000..44fa89b0 --- /dev/null +++ b/doc/quick_start/yaml_configuration_guide.md @@ -0,0 +1,130 @@ +# Introduction + +The contents in .yaml config file should be well structured and follow the supported rules and entity names. + +# Sample +## [pipeline_people.yaml](../../vino_launch/param/pipeline_people.yaml) +```bash +Pipelines: +- name: people + inputs: [StandardCamera] + infers: + - name: FaceDetection + model: /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP16/face-detection-adas-0001.xml + engine: CPU + label: /opt/openvino_toolkit/models/intel/face-detection-adas-0001/FP16/face-detection-adas-0001.labels + batch: 1 + confidence_threshold: 0.5 + enable_roi_constraint: true # set enable_roi_constraint to false if you don't want to make the inferred ROI (region of interest) constrained into the camera frame + - name: AgeGenderRecognition + model: /opt/openvino_toolkit/models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 16 + - name: EmotionRecognition + model: /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.xml + engine: CPU + label: /opt/openvino_toolkit/models/intel/emotions-recognition-retail-0003/FP32/emotions-recognition-retail-0003.labels + batch: 16 + - name: HeadPoseEstimation + model: /opt/openvino_toolkit/models/intel/head-pose-estimation-adas-0001/FP32/head-pose-estimation-adas-0001.xml + engine: CPU + label: to/be/set/xxx.labels + batch: 16 + outputs: [ImageWindow, RosTopic, RViz] + connects: + - left: StandardCamera + right: [FaceDetection] + - left: FaceDetection + right: [AgeGenderRecognition, EmotionRecognition, HeadPoseEstimation, ImageWindow, RosTopic, RViz] + - left: AgeGenderRecognition + right: [ImageWindow, RosTopic, RViz] + - left: EmotionRecognition + right: [ImageWindow, RosTopic, RViz] + - left: HeadPoseEstimation + right: [ImageWindow, RosTopic, RViz] + +Common: +``` +## Interface Description + +### Specify pipeline name +The name value of this pipeline can be anyone other than null. + +### Specify inputs +**Note:** The input parameter can only have one value.
+Currently, options for inputs are: + +|Input Option|Description|Configuration| +|--------------------|------------------------------------------------------------------|-----------------------------------------| +|StandardCamera|Any RGB camera with USB port supporting. Currently only the first USB camera if many are connected.|```inputs: [StandardCamera]```| +|RealSenseCamera| Intel RealSense RGB-D Camera, directly calling RealSense Camera via librealsense plugin of openCV.|```inputs: [RealSenseCamera]```| +|RealSenseCameraTopic| Any ROS topic which is structured in image message.|```inputs: [RealSenseCameraTopic]```| +|Image| Any image file which can be parsed by openCV, such as .png, .jpeg.|```inputs: [Image]```| +|Video| Any video file which can be parsed by openCV.|```inputs: [Video]```| +|IpCamera| Any RTSP server which can push video stream.|```inputs: [IpCamera]```| + +**Note:** Please refer to this opensource repo [RTSP_server_install_guide](https://github.com/EasyDarwin/EasyDarwin) to install RTSP server for IpCamera input. + +### Specify input_path +The input_path need to be specified when input is Image, Video and Ipcamera. + +|Input Option|Configuration| +|--------------------|------------------------------------------------------------------| +|Image|```input_path: to/be/set/image_path```| +|Video|```input_path: to/be/set/video_path```| +|IpCamera|```input_path: "rtsp://localhost/test"```| + +### Specify infers +The Inference Engine is a set of C++ classes to provides an API to read the Intermediate Representation, set the input and output formats, and execute the model on devices. + +* #### name +The name of inference engine need to be specified here. Currently, the inference feature list is supported: + +|Inference|Description| +|-----------------------|------------------------------------------------------------------| +|FaceDetection|Object Detection task applied to face recognition using a sequence of neural networks.| +|EmotionRecognition| Emotion recognition based on detected face image.| +|AgeGenderRecognition| Age and gener recognition based on detected face image.| +|HeadPoseEstimation| Head pose estimation based on detected face image.| +|ObjectDetection| object detection based on SSD-based trained models.| +|VehicleDetection| Vehicle and passenger detection based on Intel models.| +|ObjectSegmentation| object detection and segmentation.| +|ObjectSegmentationMaskrcnn| object segmentation based on Maskrcnn model.| + +* #### model +The path of model need to be specified here. The scheme below illustrates the typical workflow for deploying a trained deep learning model. +![trained deep learning model](../../data/images/CVSDK_Flow.png "trained deep learning model") + +* #### engine +**Note:** Currently, only CPU and GPU are supported.
+Target device options are: + +|Target Device| +|-----------------------| +|CPU| +|Intel® Integrated Graphics| +|FPGA| +|Intel® Movidius™ Neural Compute Stick| + +* #### label +Currently, this parameter does not work. + +* #### batch +Enable dynamic batch size for the inference engine net. + +### Specify outputs +**Note:** The output parameter can be one or more.
+Currently, the output options are: + +|Option|Description|Configuration| +|--------------------|-----------------------------------------------------|---------------------------------------------| +|ImageWindow| Window showing results|```outputs: [ImageWindow, RosTopic, RViz]```| +|RosTopic| Output the topic|```outputs: [ImageWindow, RosTopic, RViz]```| +|RViz| Display the result in rviz|```outputs: [ImageWindow, RosTopic, RViz]```| + +### Specify confidence_threshold +Set the threshold of detection probability. + +### Specify connects +The topology of a pipe can only have one value on the left and multiple values on the right. The value of the first left node should be the same as the specified **inputs**. From fad9c8b6bd590fe50dd2a6a86689de0546e26c82 Mon Sep 17 00:00:00 2001 From: wujiawei Date: Tue, 27 Dec 2022 15:54:14 +0800 Subject: [PATCH 40/40] fix docker --- docker/Dockerfile | 13 +++++-------- docker/docker_instruction.md | 6 +++--- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2fbcd37a..dd7eba75 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # ros openvino toolkit env master ec5b9b9716e4 -ARG ROS_VERSION -FROM osrf/ros:${ROS_VERSION} +ARG ROS_PRE_INSTALLED_PKG +FROM osrf/ros:${ROS_PRE_INSTALLED_PKG} ARG VERSION # setting proxy env --option @@ -10,8 +10,8 @@ ARG VERSION # ENV HTTPS_PROXY="your_proxy" # ENV FTP_PROXY="your_proxy" -# maintainer information -LABEL maintainer="Jiawei Wu " +# author information +LABEL author="Jiawei Wu " # default shell type SHELL ["/bin/bash", "-c"] @@ -63,7 +63,6 @@ pyyaml \ requests \ && apt-get install -y --no-install-recommends libboost-all-dev WORKDIR /usr/lib/x86_64-linux-gnu -RUN ln -sf libboost_python-py36.so libboost_python37.so RUN pip install --upgrade pip RUN pip install openvino-dev[tensorflow2]==2022.1 RUN apt-get install -y libcanberra-gtk* git @@ -74,9 +73,7 @@ WORKDIR /root RUN mkdir -p catkin_ws/src WORKDIR /root/catkin_ws/src RUN git init && git clone -b ros https://github.com/intel/ros_openvino_toolkit \ -&& git clone https://github.com/intel/object_msgs.git \ -&& git clone -b ${VERSION} https://github.com/ros-perception/vision_opencv.git \ -&& git clone -b ros1-legacy https://github.com/IntelRealSense/realsense-ros.git +&& git clone https://github.com/intel/object_msgs.git RUN apt-get install ros-${VERSION}-ddynamic-reconfigure WORKDIR /root/catkin_ws/ RUN source /opt/ros/${VERSION}/setup.bash && source /opt/intel/openvino_2022/setupvars.sh && catkin_make_isolated diff --git a/docker/docker_instruction.md b/docker/docker_instruction.md index 7e45644d..b5ead669 100644 --- a/docker/docker_instruction.md +++ b/docker/docker_instruction.md @@ -11,20 +11,20 @@ Supported ros versions include noetic and melodic. ``` cd ~/ros_openvino_toolkit/docker/Dockerfile vi ~/ros_openvino_toolkit/docker/Dockerfile -docker build --build-arg ROS_VERSION= --build-arg VERSION= --build-arg "HTTP_PROXY=set_your_proxy" -t ros_openvino_202201 . +docker build --build-arg ROS_PRE_INSTALLED_PKG= --build-arg VERSION= --build-arg "HTTP_PROXY=set_your_proxy" -t ros_openvino_202201 . ``` For example: * Build image for ros_noetic ``` cd ~/ros_openvino_toolkit/docker/Dockerfile vi ~/ros_openvino_toolkit/docker/Dockerfile -docker build --build-arg ROS_VERSION=noetic-desktop-full --build-arg VERSION=noetic --build-arg "HTTP_PROXY=set_your_proxy" -t ros_noetic_openvino_202201 . +docker build --build-arg ROS_PRE_INSTALLED_PKG=noetic-desktop-full --build-arg VERSION=noetic --build-arg "HTTP_PROXY=set_your_proxy" -t ros_noetic_openvino_202201 . ``` * Build image for ros_melodic ``` cd ~/ros_openvino_toolkit/docker/Dockerfile vi ~/ros_openvino_toolkit/docker/Dockerfile -docker build --build-arg ROS_VERSION=melodic-desktop-full --build-arg VERSION=melodic --build-arg "HTTP_PROXY=set_your_proxy" -t ros_melodic_openvino_202201 . +docker build --build-arg ROS_PRE_INSTALLED_PKG=melodic-desktop-full --build-arg VERSION=melodic --build-arg "HTTP_PROXY=set_your_proxy" -t ros_melodic_openvino_202201 . ``` ## 3. Download and load docker image