From 3a045b33ca63f3c3e034888ebe4b5086ed135846 Mon Sep 17 00:00:00 2001
From: Mert <101130780+mertalev@users.noreply.github.com>
Date: Sat, 16 Mar 2024 00:04:45 -0400
Subject: [PATCH] chore(deps): update onnxruntime-openvino (#7854)

---
 machine-learning/Dockerfile         | 41 ++++++++++++++++-------------
 machine-learning/app/models/base.py | 18 +++++--------
 machine-learning/app/test_main.py   | 21 ---------------
 machine-learning/poetry.lock        | 17 ++++++------
 machine-learning/pyproject.toml     |  2 +-
 machine-learning/start.sh           |  7 +++--
 6 files changed, 43 insertions(+), 63 deletions(-)

diff --git a/machine-learning/Dockerfile b/machine-learning/Dockerfile
index 1ed90c46c3..2df540f091 100644
--- a/machine-learning/Dockerfile
+++ b/machine-learning/Dockerfile
@@ -2,7 +2,7 @@ ARG DEVICE=cpu
 
 FROM python:3.11-bookworm@sha256:991e20a11120277e977cadbc104e7a9b196a68a346597879821b19034285a403 as builder-cpu
 
-FROM openvino/ubuntu22_runtime:2023.1.0@sha256:002842a9005ba01543b7169ff6f14ecbec82287f09c4d1dd37717f0a8e8754a7 as builder-openvino
+FROM openvino/ubuntu22_runtime:2023.3.0@sha256:176646df619032ea6c10faf842867119c393e7497b7f88b5e307e932a0fd5aa8 as builder-openvino
 USER root
 RUN apt-get update && apt-get install -y --no-install-recommends python3-dev
 
@@ -21,10 +21,12 @@ FROM builder-${DEVICE} as builder
 
 ARG DEVICE
 ENV PYTHONDONTWRITEBYTECODE=1 \
-  PYTHONUNBUFFERED=1 \
-  PIP_NO_CACHE_DIR=true \
-  VIRTUAL_ENV="/opt/venv" \
-  PATH="/opt/venv/bin:${PATH}"
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=true \
+    VIRTUAL_ENV="/opt/venv" \
+    PATH="/opt/venv/bin:${PATH}"
+
+RUN apt-get update && apt-get install -y --no-install-recommends g++
 
 RUN pip install --upgrade pip && pip install poetry
 RUN poetry config installer.max-workers 10 && \
@@ -36,7 +38,7 @@ RUN poetry install --sync --no-interaction --no-ansi --no-root --with ${DEVICE}
 
 FROM python:3.11-slim-bookworm@sha256:a2eb07f336e4f194358382611b4fea136c632b40baa6314cb27a366deeaf0144 as prod-cpu
 
-FROM openvino/ubuntu22_runtime:2023.1.0@sha256:002842a9005ba01543b7169ff6f14ecbec82287f09c4d1dd37717f0a8e8754a7 as prod-openvino
+FROM openvino/ubuntu22_runtime:2023.3.0@sha256:176646df619032ea6c10faf842867119c393e7497b7f88b5e307e932a0fd5aa8 as prod-openvino
 USER root
 
 FROM nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu22.04@sha256:2d913b09e6be8387e1a10976933642c73c840c0b735f0bf3c28d97fc9bc422e0 as prod-cuda
@@ -56,14 +58,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends ocl-icd-libopen
     mkdir /opt/armnn
 
 COPY --from=builder-armnn \
-      /opt/armnn/libarmnn.so.?? \
-      /opt/armnn/libarmnnOnnxParser.so.?? \
-      /opt/armnn/libarmnnDeserializer.so.?? \
-      /opt/armnn/libarmnnTfLiteParser.so.?? \
-      /opt/armnn/libprotobuf.so.?.??.?.? \
-      /opt/ann/libann.s[o] \
-      /opt/ann/build.sh \
-      /opt/armnn/
+    /opt/armnn/libarmnn.so.?? \
+    /opt/armnn/libarmnnOnnxParser.so.?? \
+    /opt/armnn/libarmnnDeserializer.so.?? \
+    /opt/armnn/libarmnnTfLiteParser.so.?? \
+    /opt/armnn/libprotobuf.so.?.??.?.? \
+    /opt/ann/libann.s[o] \
+    /opt/ann/build.sh \
+    /opt/armnn/
 
 FROM prod-${DEVICE} as prod
 
@@ -73,11 +75,12 @@ RUN apt-get update && \
 
 WORKDIR /usr/src/app
 ENV NODE_ENV=production \
-  TRANSFORMERS_CACHE=/cache \
-  PYTHONDONTWRITEBYTECODE=1 \
-  PYTHONUNBUFFERED=1 \
-  PATH="/opt/venv/bin:$PATH" \
-  PYTHONPATH=/usr/src
+    TRANSFORMERS_CACHE=/cache \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PATH="/opt/venv/bin:$PATH" \
+    PYTHONPATH=/usr/src \
+    DEVICE=${DEVICE}
 
 # prevent core dumps
 RUN echo "hard core 0" >> /etc/security/limits.conf && \
diff --git a/machine-learning/app/models/base.py b/machine-learning/app/models/base.py
index ad48624b4e..496012270c 100644
--- a/machine-learning/app/models/base.py
+++ b/machine-learning/app/models/base.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 from abc import ABC, abstractmethod
 from pathlib import Path
 from shutil import rmtree
@@ -115,17 +114,12 @@ class InferenceModel(ABC):
             case ".armnn":
                 session = AnnSession(model_path)
             case ".onnx":
-                cwd = os.getcwd()
-                try:
-                    os.chdir(model_path.parent)
-                    session = ort.InferenceSession(
-                        model_path.as_posix(),
-                        sess_options=self.sess_options,
-                        providers=self.providers,
-                        provider_options=self.provider_options,
-                    )
-                finally:
-                    os.chdir(cwd)
+                session = ort.InferenceSession(
+                    model_path.as_posix(),
+                    sess_options=self.sess_options,
+                    providers=self.providers,
+                    provider_options=self.provider_options,
+                )
             case _:
                 raise ValueError(f"Unsupported model file type: {model_path.suffix}")
         return session
diff --git a/machine-learning/app/test_main.py b/machine-learning/app/test_main.py
index 72cd020ff2..d79da0cbd5 100644
--- a/machine-learning/app/test_main.py
+++ b/machine-learning/app/test_main.py
@@ -262,7 +262,6 @@ class TestBase:
 
         mock_ann = mocker.patch("app.models.base.AnnSession")
         mock_ort = mocker.patch("app.models.base.ort.InferenceSession")
-        mocker.patch("app.models.base.os.chdir")
 
         encoder = OpenCLIPEncoder("ViT-B-32__openai")
         encoder._make_session(mock_armnn_path)
@@ -285,26 +284,6 @@ class TestBase:
         mock_ann.assert_not_called()
         mock_ort.assert_not_called()
 
-    def test_make_session_changes_cwd(self, mocker: MockerFixture) -> None:
-        mock_model_path = mocker.Mock()
-        mock_model_path.is_file.return_value = True
-        mock_model_path.suffix = ".onnx"
-        mock_model_path.parent = "model_parent"
-        mock_model_path.with_suffix.return_value = mock_model_path
-        mock_ort = mocker.patch("app.models.base.ort.InferenceSession")
-        mock_chdir = mocker.patch("app.models.base.os.chdir")
-
-        encoder = OpenCLIPEncoder("ViT-B-32__openai")
-        encoder._make_session(mock_model_path)
-
-        mock_chdir.assert_has_calls(
-            [
-                mock.call(mock_model_path.parent),
-                mock.call(os.getcwd()),
-            ]
-        )
-        mock_ort.assert_called_once()
-
     def test_download(self, mocker: MockerFixture) -> None:
         mock_snapshot_download = mocker.patch("app.models.base.snapshot_download")
 
diff --git a/machine-learning/poetry.lock b/machine-learning/poetry.lock
index a41aed55ec..7a0c66bdcd 100644
--- a/machine-learning/poetry.lock
+++ b/machine-learning/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
 
 [[package]]
 name = "aiocache"
@@ -2064,21 +2064,22 @@ reference = "cuda12"
 
 [[package]]
 name = "onnxruntime-openvino"
-version = "1.15.0"
+version = "1.17.1"
 description = "ONNX Runtime is a runtime accelerator for Machine Learning models"
 optional = false
 python-versions = "*"
 files = [
-    {file = "onnxruntime_openvino-1.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9bfe245312e897f219dfef619c0d98f4797ffb008ad55aa41aedb32b522f72"},
-    {file = "onnxruntime_openvino-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:a31cd9c9848dc196803d74ea46152fe0f3dd876bc5769eff7e3776fef4c654de"},
-    {file = "onnxruntime_openvino-1.15.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c9bc1614f9d267d62023287035d204d9840ac0057d1c7a770a27acdd1642662"},
-    {file = "onnxruntime_openvino-1.15.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5808b7b876e5f6a083228bd43fc1028096cb9b485f466bf980d8f72d8424d"},
+    {file = "onnxruntime_openvino-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ed693011b472f9a617b2d5c4785d5fa1e1b77f7cb2b02e47b899534ec6c6396"},
+    {file = "onnxruntime_openvino-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:5152b5e56e83e022ced2986700d68dd8ba7b1466761725ce774f679c5710ab87"},
+    {file = "onnxruntime_openvino-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ce3b1aa06d6b8b732d314d217028ec4735de5806215c44d3bdbcad03b9260d5"},
+    {file = "onnxruntime_openvino-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:21133a701bb07ea19e01f48b8c23beee575f2e879f49173843f275d7c91a625a"},
+    {file = "onnxruntime_openvino-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76824dac3c392ad4b812f29c18be2055ab3bba2e3c111e44baae847b33d5b081"},
 ]
 
 [package.dependencies]
 coloredlogs = "*"
 flatbuffers = "*"
-numpy = ">=1.21.6"
+numpy = ">=1.25.2"
 packaging = "*"
 protobuf = "*"
 sympy = "*"
@@ -3626,4 +3627,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.12"
-content-hash = "c947090d326e81179054b7ce4dded311df8b7ca5a56680d5e9459cf8ca18df1a"
+content-hash = "1b014276ec94f9389459a70d31f0d96d1dd5a138bcc988900865e5f07a72bc62"
diff --git a/machine-learning/pyproject.toml b/machine-learning/pyproject.toml
index 327f4fd35e..c0e549af54 100644
--- a/machine-learning/pyproject.toml
+++ b/machine-learning/pyproject.toml
@@ -51,7 +51,7 @@ onnxruntime-gpu = {version = "^1.17.0", source = "cuda12"}
 optional = true
 
 [tool.poetry.group.openvino.dependencies]
-onnxruntime-openvino = ">=1.15.0,<1.16.0"
+onnxruntime-openvino = "^1.17.1"
 
 [tool.poetry.group.armnn]
 optional = true
diff --git a/machine-learning/start.sh b/machine-learning/start.sh
index d4a971f0ce..7a5cb919a6 100755
--- a/machine-learning/start.sh
+++ b/machine-learning/start.sh
@@ -1,8 +1,11 @@
 #!/usr/bin/env sh
 
 lib_path="/usr/lib/$(arch)-linux-gnu/libmimalloc.so.2"
-export LD_PRELOAD="$lib_path"
-export LD_BIND_NOW=1
+# mimalloc seems to increase memory usage dramatically with openvino, need to investigate
+if ! [ "$DEVICE" = "openvino" ]; then 
+	export LD_PRELOAD="$lib_path"
+	export LD_BIND_NOW=1
+fi
 
 : "${MACHINE_LEARNING_HOST:=[::]}"
 : "${MACHINE_LEARNING_PORT:=3003}"