[DEPLOYMENT] Jenkins remote deployment

src/benchmark/config_processor.py

@@ -86,6 +86,11 @@ def parse_independent_parameters(self, curr_test):
         batch_size = _batch_size.data if (_batch_size and _batch_size.data != 'None') else None
 
         device = indep_parameters_tag.getElementsByTagName('Device')[0].firstChild.data.strip()
+        python_path = 'python3'
+
+        if indep_parameters_tag.getElementsByTagName('PythonInterpreter')[0].firstChild:
+            python_path = indep_parameters_tag.getElementsByTagName('PythonInterpreter')[0].firstChild.data.strip()
+
         iteration_count = indep_parameters_tag.getElementsByTagName('IterationCount')[0].firstChild.data.strip()
         test_time_limit = int(indep_parameters_tag.getElementsByTagName('TestTimeLimit')[0].firstChild.data)
         timeout_overhead_element = indep_parameters_tag.getElementsByTagName('TimeoutOverhead')
@@ -129,6 +134,7 @@ def parse_independent_parameters(self, curr_test):
             timeout_overhead=timeout_overhead,
             custom_models_links=custom_models_links,
             raw_output=raw_output,
+            python_path=python_path,
         )
 
     def parse_dependent_parameters(self, curr_test, framework):

src/benchmark/frameworks/config_parser/framework_independent_parameters.py

@@ -3,7 +3,8 @@
 
 class FrameworkIndependentParameters(FrameworkParameters):
     def __init__(self, inference_framework, batch_size, device, iterarion_count, test_time_limit,
-                 timeout_overhead, custom_models_links=None, raw_output=True, num_gpu_devices=None):
+                 timeout_overhead, custom_models_links=None, raw_output=True, num_gpu_devices=None,
+                 python_path='python3'):
         self.inference_framework = None
         self.batch_size = None
         self.device = None
@@ -12,6 +13,7 @@ def __init__(self, inference_framework, batch_size, device, iterarion_count, tes
         self.test_time_limit = None
         self.custom_models_links = custom_models_links
         self.raw_output = raw_output
+        self.python_path = 'python3'
         if self._parameter_is_not_none(inference_framework):
             self.inference_framework = inference_framework
         else:
@@ -43,3 +45,4 @@ def __init__(self, inference_framework, batch_size, device, iterarion_count, tes
         else:
             default_timeout_overhead = 300
             self.timeout_overhead = default_timeout_overhead
+        self.python_path = python_path

src/benchmark/frameworks/dgl_pytorch/dgl_pytorch_process.py

@@ -20,7 +20,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_pytorch_script = Path.joinpath(self.inference_script_root, 'inference_dgl_pytorch.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         name = self._test.model.name
         model = self._test.model.model

src/benchmark/frameworks/executorch/executorch_process.py

@@ -19,7 +19,7 @@ def get_performance_metrics(self):
         return self.get_performance_metrics_from_json_report()
 
     def _fill_command_line(self):
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         dataset = self._test.dataset.path
         input_shape = self._test.dep_parameters.input_shape
         layout = self._test.dep_parameters.layout

src/benchmark/frameworks/intel_caffe/intel_caffe_process.py

@@ -19,7 +19,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_intelcaffe_script = Path.joinpath(self.inference_script_root, 'inference_caffe.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         model_prototxt = self._test.model.model
         model_caffemodel = self._test.model.weight

src/benchmark/frameworks/mxnet/mxnet_process.py

@@ -97,7 +97,7 @@ def get_performance_metrics(self):
     def _fill_command_line(self):
         path_to_sync_script = Path.joinpath(self.inference_script_root,
                                             'inference_mxnet_sync_mode.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         time_limit = self._test.indep_parameters.test_time_limit
         common_params = super()._fill_command_line()
         common_params += f' --time {time_limit}'
@@ -116,7 +116,7 @@ def get_performance_metrics(self):
     def _fill_command_line(self):
         path_to_async_script = Path.joinpath(self.inference_script_root,
                                              'inference_mxnet_async_mode.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         common_params = super()._fill_command_line()
         command_line = f'{python} {path_to_async_script} {common_params}'
 

src/benchmark/frameworks/ncnn/ncnn_process.py

@@ -20,7 +20,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_ncnn_script = Path.joinpath(self.inference_script_root, 'inference_ncnn.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         name = self._test.model.name
         model = self._test.model.model

src/benchmark/frameworks/onnx_runtime_python/onnx_runtime_python_process.py

@@ -19,7 +19,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_onnx_script = Path.joinpath(self.inference_script_root, 'inference_onnx_runtime.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         model = self._test.model.model
         dataset = self._test.dataset.path if self._test.dataset else None

src/benchmark/frameworks/opencv_dnn_python/opencv_dnn_python_process.py

@@ -19,7 +19,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_opencv_script = Path.joinpath(self.inference_script_root, 'inference_opencv.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         model = self._test.model.model
         weights = self._test.model.weight

src/benchmark/frameworks/openvino/openvino_python_api_process.py

@@ -54,7 +54,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_async_script = Path.joinpath(self.inference_script_root, 'inference_openvino_async_mode.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         common_params = super()._fill_command_line()
         command_line = f'{python} {path_to_async_script} {common_params}'
@@ -82,7 +82,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_sync_script = Path.joinpath(self.inference_script_root, 'inference_openvino_sync_mode.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         time_limit = self._test.indep_parameters.test_time_limit
 
         common_params = super()._fill_command_line()

src/benchmark/frameworks/paddlepaddle/paddlepaddle_process.py

@@ -19,7 +19,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_paddlepaddle_script = Path.joinpath(self.inference_script_root, 'inference_paddlepaddle.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         model = self._test.model.model
         params = self._test.model.weight

src/benchmark/frameworks/processes.py

@@ -1,7 +1,6 @@
 import abc
 import json
 import os
-import platform
 from datetime import datetime
 from pathlib import Path
 
@@ -34,14 +33,8 @@ def report_path(self):
         return report_path
 
     @staticmethod
-    def get_cmd_python_version():
-        cmd_python_version = ''
-        os_type = platform.system()
-        if os_type == 'Linux':
-            cmd_python_version = 'python3'
-        else:
-            cmd_python_version = 'python'
-
+    def get_cmd_python_version(test):
+        cmd_python_version = test.indep_parameters.python_path
         return cmd_python_version
 
     def get_model_shape(self):

src/benchmark/frameworks/pytorch/pytorch_process.py

@@ -20,7 +20,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_pytorch_script = Path.joinpath(self.inference_script_root, 'inference_pytorch.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         name = self._test.model.name
         model = self._test.model.model

src/benchmark/frameworks/spektral/spektral_process.py

@@ -19,7 +19,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_spektral_script = Path.joinpath(self.inference_script_root, 'inference_spektral.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         model = self._test.model.model
         dataset = self._test.dataset.path if self._test.dataset else None

src/benchmark/frameworks/tensorflow/tensorflow_process.py

@@ -19,7 +19,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_tensorflow_script = Path.joinpath(self.inference_script_root, 'inference_tensorflow.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         model = self._test.model.model
         dataset = self._test.dataset.path if self._test.dataset else None

src/benchmark/frameworks/tensorflow_lite/tensorflow_lite_process.py

@@ -19,7 +19,7 @@ def get_performance_metrics(self):
 
     def _fill_command_line(self):
         path_to_tensorflow_script = Path.joinpath(self.inference_script_root, 'inference_tensorflowlite.py')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
 
         model = self._test.model.model
         dataset = self._test.dataset.path if self._test.dataset else None

src/benchmark/frameworks/tvm/tvm_process.py

@@ -1,5 +1,4 @@
 from pathlib import Path
-
 from ..processes import ProcessHandler
 
 
@@ -108,7 +107,7 @@ def _fill_command_line(self):
         else:
             raise Exception('Incorrect model parameters. Set model name or file names.')
         common_params += '-f mxnet '
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         time_limit = self._test.indep_parameters.test_time_limit
         common_params += super()._fill_command_line()
         common_params += f' --time {time_limit}'
@@ -144,7 +143,7 @@ def _fill_command_line(self):
             common_params += (f'-m {model_pt} ')
         else:
             raise Exception('Incorrect model parameters. Set model name or file names.')
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         time_limit = self._test.indep_parameters.test_time_limit
         common_params += super()._fill_command_line()
         common_params += f' --time {time_limit}'
@@ -163,7 +162,7 @@ def get_performance_metrics(self):
     def _fill_command_line(self):
         model = self._test.model.model
         common_params = f'-m {model} -f onnx '
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         time_limit = self._test.indep_parameters.test_time_limit
         common_params += super()._fill_command_line()
         common_params += f' --time {time_limit}'
@@ -183,7 +182,7 @@ def _fill_command_line(self):
         model = self._test.model.model
         weight = self._test.model.weight
         common_params = f'-m {model} -w {weight} -f caffe '
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         time_limit = self._test.indep_parameters.test_time_limit
         common_params += super()._fill_command_line()
         common_params += f' --time {time_limit}'
@@ -218,7 +217,7 @@ def _fill_command_line(self):
         else:
             raise ValueError('Wrong arguments.')
         common_params += '-f tvm '
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         time_limit = self._test.indep_parameters.test_time_limit
         common_params += super()._fill_command_line()
         common_params += f' --time {time_limit}'
@@ -237,7 +236,7 @@ def get_performance_metrics(self):
     def _fill_command_line(self):
         model = self._test.model.model
         common_params = f'-m {model} -f tflite '
-        python = ProcessHandler.get_cmd_python_version()
+        python = ProcessHandler.get_cmd_python_version(self._test)
         time_limit = self._test.indep_parameters.test_time_limit
         common_params += super()._fill_command_line()
         common_params += f' --time {time_limit}'

src/benchmark/tests/test_processes.py

@@ -16,7 +16,6 @@
                                                                           OpenVINOBenchmarkCppProcess)
 from src.benchmark.frameworks.openvino.openvino_process import OpenVINOProcess
 from src.benchmark.frameworks.openvino.openvino_python_api_process import AsyncOpenVINOProcess, SyncOpenVINOProcess
-from src.benchmark.frameworks.processes import ProcessHandler
 from src.benchmark.frameworks.pytorch.pytorch_process import PyTorchProcess
 from src.benchmark.frameworks.pytorch_cpp.pytorch_cpp_process import PyTorchCppProcess
 from src.benchmark.frameworks.tensorflow.tensorflow_process import TensorFlowProcess
@@ -59,12 +58,6 @@ class DotDict(dict):
 WRAPPER_REGISTRY = FrameworkWrapperRegistry()
 
 
-@pytest.mark.parametrize('os', [['Linux', 'python3'], ['Windows', 'python']])
-def test_python_version(os, mocker):
-    mocker.patch('platform.system', return_value=os[0])
-    assert ProcessHandler.get_cmd_python_version() == os[1]
-
-
 @pytest.mark.parametrize('inference_framework', [['OpenVINO DLDT', OpenVINOProcess],
                                                  ['Caffe', IntelCaffeProcess],
                                                  ['TensorFlow', TensorFlowProcess],

src/deployment/jenkins/logger.py

@@ -0,0 +1,27 @@
+import logging as log
+import sys
+
+
+DEFAULT_FORMATTER = log.Formatter('[ %(levelname)s ] %(message)s')
+
+
+def configure_logger(name='', level=log.INFO, use_default_formatter=False):
+
+    logger = log.getLogger()
+    logger.setLevel(level)
+
+    if logger.hasHandlers():
+        logger.handlers.clear()
+
+    stream_handler = log.StreamHandler(stream=sys.stdout)
+    stream_handler.setLevel(level)
+    stream_handler.setFormatter(DEFAULT_FORMATTER)
+    logger.addHandler(stream_handler)
+
+    file_handler = log.FileHandler(name, 'w')
+    file_handler.setLevel(level)
+    file_handler.setFormatter(DEFAULT_FORMATTER)
+    logger.addHandler(file_handler)
+
+    logger.propagate = False
+    return logger

src/deployment/jenkins/models_template.txt

@@ -0,0 +1,8 @@
+resnet50;;resnet-50-pytorch/resnet50-19c8e357.pth;torch;3 224 224;1,2,8;data
+efficientnet-b0;efficientnet-b0/efficientnet-b0.onnx;;onnx;224 224 3;1,2,8;image
+densenet-121-tf;densenet-121-tf/densenet-121-tf.onnx;;onnx;224 224 3;1,2,8;keras_tensor
+googlenet-v4-tf;googlenet-v4-tf/inception_v4.onnx;;onnx;299 299 3;1,2,8;input
+ssd_512_resnet50_v1_coco;;;mxnet;3 512 512;1,2,8;
+ssd_512_vgg16_atrous_voc;;;mxnet;3 512 512;1,2,8;
+ssd_300_vgg16_atrous_voc;;;mxnet;3 300 300;1,2,8;
+ssd_512_mobilenet1.0_coco;;;mxnet;3 512 512;1,2,8;

src/deployment/jenkins/requirements.txt

@@ -0,0 +1,11 @@
+traitlets==5.9.0
+decorator
+attrs
+typing-extensions
+psutil
+scipy
+pybind11
+numpy
+opencv-python
+scipy
+openvino-dev==2024.3.0