[WIP] Umat poc

.bazelrc

@@ -121,6 +121,6 @@ build --cxxopt=-DPYTHON_DISABLE=1
 coverage --cxxopt=-DPYTHON_DISABLE=1
 test --cxxopt=-DPYTHON_DISABLE=1
 
-build --cxxopt=-DOVMS_DUMP_TO_FILE=0
-coverage --cxxopt=-DOVMS_DUMP_TO_FILE=0
-test --cxxopt=-DOVMS_DUMP_TO_FILE=0
+build --cxxopt=-DOVMS_DUMP_TO_FILE=1
+coverage --cxxopt=-DOVMS_DUMP_TO_FILE=1
+test --cxxopt=-DOVMS_DUMP_TO_FILE=1

Dockerfile.openvino

@@ -170,6 +170,26 @@ RUN cd flatbuffers && mkdir build && cd build && cmake -G "Unix Makefiles" -DCMA
 
 ## End of base image
 
+## GPU OCL support
+ARG INSTALL_DRIVER_VERSION="23.13.26032"
+ARG GPU=0
+if [ "$GPU" == "1" ] ; then \
+    apt-get update && apt-get install -y ocl-icd-opencl-dev libnuma1 ocl-icd-libopencl1 --no-install-recommends && rm -rf /var/lib/apt/lists/* && \
+        case $INSTALL_DRIVER_VERSION in \
+        "23.22.26516") \
+                        mkdir /tmp/gpu_deps && cd /tmp/gpu_deps ; \
+                        curl -L -O https://github.com/intel/compute-runtime/releases/download/23.22.26516.18/intel-level-zero-gpu_1.3.26516.18_amd64.deb ; \
+                        curl -L -O https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.14062.11/intel-igc-core_1.0.14062.11_amd64.deb ; \
+                        curl -L -O https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.14062.11/intel-igc-opencl_1.0.14062.11_amd64.deb ; \
+                        curl -L -O https://github.com/intel/compute-runtime/releases/download/23.22.26516.18/intel-opencl-icd_23.22.26516.18_amd64.deb ; \
+                        curl -L -O https://github.com/intel/compute-runtime/releases/download/23.22.26516.18/libigdgmm12_22.3.0_amd64.deb ; \
+                        dpkg -i *.deb && rm -Rf /tmp/gpu_deps ; \
+                ;; \
+        esac ; \
+fi ; \
+apt-get clean ; \
+rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* ;
+
 FROM base as build
 
 RUN wget -O video.mp4 "https://www.pexels.com/download/video/3044127/?fps=24.0&h=1080&w=1920"

Makefile

@@ -23,6 +23,8 @@ INPUT_VIDEO_LINK ?= "https://www.pexels.com/download/video/3044127/?fps=24.0&h=1
 OVMS_COMMIT ?="7bf5d7fc217a81ce4b0a089c7679ecb77e6c544c"
 JOBS ?= $(shell python3 -c 'import multiprocessing as mp; print(mp.cpu_count())')
 DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.0/linux/l_openvino_toolkit_ubuntu20_2024.0.0.14509.34caeefd078_x86_64.tgz
+INSTALL_DRIVER_VERSION ?= "23.22.26516"
+GPU ?= 0
 
 # Targets to use outside running mediapipe_ovms container
 docker_build:
@@ -32,6 +34,8 @@ docker_build:
 	--build-arg JOBS=$(JOBS) . \
 	--build-arg OVMS_COMMIT=$(OVMS_COMMIT) \
 	-t $(OVMS_MEDIA_DOCKER_IMAGE):$(OVMS_MEDIA_IMAGE_TAG)
+	--build-arg INSTALL_DRIVER_VERSION=$(INSTALL_DRIVER_VERSION)\
+	--build-arg GPU=$(GPU)\
 
 tests: run_unit_tests run_hello_world run_hello_ovms
 run_hello_ovms:

WORKSPACE

@@ -825,6 +825,12 @@ new_local_repository(
     path = "/opt/intel/openvino/runtime",
 )
 
+new_git_repository(
+    name = "openvino",
+    remote = "https:///github.com/openvinotoolkit/openvino/",
+    commit = "c3c409ee133ffb26bf8fd5570ef50a7c004839a4"
+)
+
 git_repository(
     name = "oneTBB",
     branch = "v2021.10.0",

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -190,6 +190,7 @@ class ImageTransformationCalculator : public CalculatorBase {
  private:
   absl::Status RenderCpu(CalculatorContext* cc);
   absl::Status RenderGpu(CalculatorContext* cc);
+  absl::Status RenderOpenCl(CalculatorContext* cc);
   absl::Status GlSetup();
 
   void ComputeOutputDimensions(int input_width, int input_height,
@@ -411,7 +412,8 @@ absl::Status ImageTransformationCalculator::Process(CalculatorContext* cc) {
     if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
       return absl::OkStatus();
     }
-    return RenderCpu(cc);
+    //return RenderCpu(cc);
+    return RenderOpenCl(cc);
   }
   return absl::OkStatus();
 }
@@ -442,6 +444,137 @@ absl::Status ImageTransformationCalculator::Close(CalculatorContext* cc) {
   return absl::OkStatus();
 }
 
+absl::Status ImageTransformationCalculator::RenderOpenCl(CalculatorContext* cc) {
+  // TODO UMAT
+  cv::UMatUsageFlags usageFlags = cv::USAGE_ALLOCATE_HOST_MEMORY;
+  cv::UMat input_mat = cv::UMat(usageFlags);
+  mediapipe::ImageFormat::Format format;
+
+  const auto& input = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
+  //std::cout << " cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>(); " << std::endl;
+  input_mat = formats::MatView(const_cast<ImageFrame*>(&input), usageFlags);
+  //input_mat = formats::MatView(&input, cv::USAGE_ALLOCATE_SHARED_MEMORY);
+  std::string fileName = std::string("./imageTrans/input") + std::to_string(cc->InputTimestamp().Value());
+  //dumpMatToFile(fileName, input_mat);
+
+  format = input.Format();
+
+  const int input_width = input_mat.cols;
+  const int input_height = input_mat.rows;
+  int output_width;
+  int output_height;
+  ComputeOutputDimensions(input_width, input_height, &output_width,
+                          &output_height);
+
+  if (output_width_ > 0 && output_height_ > 0) {
+    // TODO umat
+    cv::UMat scaled_mat = cv::UMat(usageFlags);
+    if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
+      int scale_flag =
+          input_mat.cols > output_width_ && input_mat.rows > output_height_
+              ? cv::INTER_AREA
+              : cv::INTER_LINEAR;
+      cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
+                 0, 0, scale_flag);
+      // DEFAULT
+      // std::cout << "SCALE - STRETCH" <<std::endl;
+    } else {
+      // std::cout << "SCALE - NON STRETCH" <<std::endl;
+      const float scale =
+          std::min(static_cast<float>(output_width_) / input_width,
+                   static_cast<float>(output_height_) / input_height);
+      const int target_width = std::round(input_width * scale);
+      const int target_height = std::round(input_height * scale);
+      int scale_flag = scale < 1.0f ? cv::INTER_AREA : cv::INTER_LINEAR;
+      if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
+        cv::UMat intermediate_mat = cv::UMat(usageFlags);
+        cv::resize(input_mat, intermediate_mat,
+                   cv::Size(target_width, target_height), 0, 0, scale_flag);
+        const int top = (output_height_ - target_height) / 2;
+        const int bottom = output_height_ - target_height - top;
+        const int left = (output_width_ - target_width) / 2;
+        const int right = output_width_ - target_width - left;
+        cv::copyMakeBorder(intermediate_mat, scaled_mat, top, bottom, left,
+                           right,
+                           options_.constant_padding() ? cv::BORDER_CONSTANT
+                                                       : cv::BORDER_REPLICATE,
+                           padding_color_);
+      } else {
+        cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
+                   0, 0, scale_flag);
+        output_width = target_width;
+        output_height = target_height;
+      }
+    }
+    input_mat = scaled_mat;
+  }
+
+  if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
+    auto padding = absl::make_unique<std::array<float, 4>>();
+    ComputeOutputLetterboxPadding(input_width, input_height, output_width,
+                                  output_height, padding.get());
+    cc->Outputs()
+        .Tag("LETTERBOX_PADDING")
+        .Add(padding.release(), cc->InputTimestamp());
+  }
+
+  //TODO umat
+  cv::UMat rotated_mat = cv::UMat(usageFlags);
+  cv::Size rotated_size(output_width, output_height);
+  if (input_mat.size() == rotated_size) {
+    const int angle = RotationModeToDegrees(rotation_);
+    cv::Point2f src_center(input_mat.cols / 2.0, input_mat.rows / 2.0);
+    // TODO UMAT
+    cv::Mat rotation_mat = cv::getRotationMatrix2D(src_center, angle, 1.0);
+    cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size);
+    // DEFAULT
+    // std::cout << "ROTATE - MATRIX" <<std::endl;
+  } else {
+    switch (rotation_) {
+      case mediapipe::RotationMode_Mode_UNKNOWN:
+      case mediapipe::RotationMode_Mode_ROTATION_0:
+        // std::cout << "ROTATE - COPY" <<std::endl;
+        rotated_mat = input_mat;
+        break;
+      case mediapipe::RotationMode_Mode_ROTATION_90:
+        cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE);
+        break;
+      case mediapipe::RotationMode_Mode_ROTATION_180:
+        cv::rotate(input_mat, rotated_mat, cv::ROTATE_180);
+        break;
+      case mediapipe::RotationMode_Mode_ROTATION_270:
+        cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE);
+        break;
+    }
+  }
+
+  // TODO UMAT
+  cv::UMat flipped_mat = cv::UMat(usageFlags);
+  if (flip_horizontally_ || flip_vertically_) {
+    const int flip_code =
+        flip_horizontally_ && flip_vertically_ ? -1 : flip_horizontally_;
+    cv::flip(rotated_mat, flipped_mat, flip_code);
+    // std::cout << "FLIP - FLIP" <<std::endl;
+  } else {
+    // DEFAULT
+    //std::cout << "FLIP - COPY" <<std::endl;
+    flipped_mat = rotated_mat;
+  }
+
+  // TODO: removed FLIP pass
+  std::unique_ptr<ImageFrame> output_frame(
+      new ImageFrame(flipped_mat, format, output_width, output_height));
+
+  fileName = std::string("./imageTrans/output") + std::to_string(cc->InputTimestamp().Value());
+  //dumpMatToFile(fileName, flipped_mat);
+
+  cc->Outputs()
+      .Tag(kImageFrameTag)
+      .Add(output_frame.release(), cc->InputTimestamp());
+
+  return absl::OkStatus();
+}
+
 absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
   cv::Mat input_mat;
   mediapipe::ImageFormat::Format format;

mediapipe/calculators/util/annotation_overlay_calculator.cc

@@ -143,6 +143,9 @@ class AnnotationOverlayCalculator : public CalculatorBase {
   absl::Status RenderToCpu(CalculatorContext* cc,
                            const ImageFormat::Format& target_format,
                            uchar* data_image);
+  absl::Status RenderToOpencl(
+    CalculatorContext* cc, const ImageFormat::Format& target_format,
+    cv::Mat& input_mat);
 
   absl::Status GlRender(CalculatorContext* cc);
   template <typename Type, const char* Tag>
@@ -345,8 +348,10 @@ absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) {
 #endif  // !MEDIAPIPE_DISABLE_GPU
   } else {
     // Copy the rendered image to output.
-    uchar* image_mat_ptr = image_mat->data;
-    MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr));
+    //uchar* image_mat_ptr = image_mat->data;
+    //MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr));
+
+    MP_RETURN_IF_ERROR(RenderToOpencl(cc, target_format, *image_mat.get()));
   }
 
   return absl::OkStatus();
@@ -365,6 +370,25 @@ absl::Status AnnotationOverlayCalculator::Close(CalculatorContext* cc) {
   return absl::OkStatus();
 }
 
+absl::Status AnnotationOverlayCalculator::RenderToOpencl(
+    CalculatorContext* cc, const ImageFormat::Format& target_format,
+    cv::Mat& input_mat) {
+      // TODO switch to umat
+      cv::UMat copy;
+      input_mat.copyTo(copy);
+
+  std::unique_ptr<ImageFrame> output_frame(
+  new ImageFrame(copy, target_format, renderer_->GetImageWidth(), renderer_->GetImageHeight()));
+
+  if (cc->Outputs().HasTag(kImageFrameTag)) {
+    cc->Outputs()
+        .Tag(kImageFrameTag)
+        .Add(output_frame.release(), cc->InputTimestamp());
+  }
+
+  return absl::OkStatus();
+}
+
 absl::Status AnnotationOverlayCalculator::RenderToCpu(
     CalculatorContext* cc, const ImageFormat::Format& target_format,
     uchar* data_image) {

mediapipe/examples/desktop/demo_run_graph_main.cc

@@ -16,6 +16,8 @@
 #include <cstdlib>
 #include <chrono>
 
+#include <opencv2/core/ocl.hpp>
+
 #include "absl/flags/flag.h"
 #include "absl/flags/parse.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -86,8 +88,14 @@ absl::Status RunMPPGraph() {
   LOG(INFO) << "Start grabbing and processing frames.";
   bool grab_frames = true;
   int count_frames = 0;
-  auto begin = std::chrono::high_resolution_clock::now();
+
 
+  OpenClWrapper ocl;
+  ocl.initOpenCL();
+
+  int max_frame = 0;
+  auto begin = std::chrono::high_resolution_clock::now();
+  cv::UMatUsageFlags usageFlags = cv::USAGE_ALLOCATE_HOST_MEMORY;
   while (grab_frames) {
     // Capture opencv camera or video frame.
     cv::Mat camera_frame_raw;
@@ -101,20 +109,29 @@ absl::Status RunMPPGraph() {
       break;
     }
     count_frames+=1;
-    cv::Mat camera_frame;
-    cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
+    cv::UMat camera_frame = cv::UMat(usageFlags);
+    // = cv::UMat(camera_frame_raw.rows, camera_frame_raw.cols, camera_frame_raw.type ,cv::USAGE_ALLOCATE_SHARED_MEMORY)
+    // SEGFAULT icv_k0_ownsCopy_8u_repE9 cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
+    //cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGBA);
+    camera_frame_raw.copyTo(camera_frame);
     if (!load_video) {
       cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
     }
 
     // Wrap Mat into an ImageFrame.
     auto input_frame = absl::make_unique<mediapipe::ImageFrame>(
+        camera_frame,
         mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
         mediapipe::ImageFrame::kDefaultAlignmentBoundary);
-    cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get());
-    camera_frame.copyTo(input_frame_mat);
 
-    // Send image packet into the graph.
+    // mediapipe::ImageFormat::SRGBA, camera_frame.cols, camera_frame.rows,        
+    //cv::UMat input_frame_mat = mediapipe::formats::MatView(input_frame.get(), cv::USAGE_ALLOCATE_SHARED_MEMORY);
+    //camera_frame.copyTo(input_frame_mat);
+    std::string fileName = std::string("./imageDemo/input") + std::to_string(count_frames);
+    //dumpMatToFile(fileName, camera_frame);
+
+
+    // Send image packet INFO the graph.
     size_t frame_timestamp_us =
         (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
     MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
@@ -127,8 +144,8 @@ absl::Status RunMPPGraph() {
     auto& output_frame = packet.Get<mediapipe::ImageFrame>();
 
     // Convert back to opencv for display or saving.
-    cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame);
-    cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
+    cv::UMat output_frame_mat = mediapipe::formats::MatView(const_cast<mediapipe::ImageFrame*>(&output_frame), usageFlags);
+    // SEGFAULT icv_k0_ownsCopy_8u_repE9 cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
     if (save_video) {
       if (!writer.isOpened()) {
         LOG(INFO) << "Prepare video writer.";
@@ -144,6 +161,10 @@ absl::Status RunMPPGraph() {
       const int pressed_key = cv::waitKey(5);
       if (pressed_key >= 0 && pressed_key != 255) grab_frames = false;
     }
+
+    if (count_frames == max_frame) {
+      break;
+    }
   }
   auto duration = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin);
   auto totalTime = duration.count();

mediapipe/examples/desktop/demo_run_graph_main_gpu.cc

@@ -94,8 +94,11 @@ absl::Status RunMPPGraph() {
 
   LOG(INFO) << "Start grabbing and processing frames.";
   bool grab_frames = true;
+  auto begin = std::chrono::high_resolution_clock::now();
+  int count_frames = 0;
   while (grab_frames) {
     // Capture opencv camera or video frame.
+    count_frames+=1;
     cv::Mat camera_frame_raw;
     capture >> camera_frame_raw;
     if (camera_frame_raw.empty()) {
@@ -183,7 +186,13 @@ absl::Status RunMPPGraph() {
       if (pressed_key >= 0 && pressed_key != 255) grab_frames = false;
     }
   }
+  auto duration = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin);
+  auto totalTime = duration.count();
+  float avgFps = (1000000 * (float)(count_frames) / (float)totalTime);
+  float avgLatencyms = 1000 / avgFps;
 
+  LOG(INFO) << "Frames:" << count_frames << ", Duration [ms]:" << totalTime / 1000 << ", FPS:" << avgFps << ", Avg latency [ms]:" << avgLatencyms;   
+
   LOG(INFO) << "Shutting down.";
   if (writer.isOpened()) writer.release();
   MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream));

mediapipe/examples/desktop/face_detection/BUILD

@@ -40,6 +40,7 @@ cc_binary(
     name = "face_detection_gpu",
     data = ["//mediapipe/modules/face_detection:face_detection_short_range.tflite"],
     deps = [
+        "@ovms//src:ovms_lib",
         "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
         "//mediapipe/graphs/face_detection:desktop_live_gpu_calculators",
     ],