[Executorch][C++] Add low level inference API (#584)

Co-authored-by: Valentina <[email protected]>

Author: ismukhin

src/benchmark/config_parser_factory.py

@@ -13,6 +13,7 @@
 from frameworks.tvm.tvm_parameters_parser import TVMParametersParser
 from frameworks.ncnn.ncnn_parameters_parser import NcnnParametersParser
 from frameworks.spektral.spektral_parameters_parser import SpektralParametersParser
+from frameworks.executorch.executorch_parameters_parser import ExecuTorchParametersParser
 
 
 def get_parameters_parser(framework):
@@ -54,4 +55,6 @@ def get_parameters_parser(framework):
         return CppParametersParser()
     if framework == KnownFrameworks.executorch_cpp:
         return CppParametersParser()
+    if framework == KnownFrameworks.executorch:
+        return ExecuTorchParametersParser()
     raise NotImplementedError(f'Unknown framework {framework}')

src/benchmark/frameworks/executorch/__init__.py

@@ -0,0 +1,69 @@
+from ..config_parser.dependent_parameters_parser import DependentParametersParser
+from ..config_parser.framework_parameters_parser import FrameworkParameters
+
+
+class ExecuTorchParametersParser(DependentParametersParser):
+    def parse_parameters(self, curr_test):
+        CONFIG_FRAMEWORK_DEPENDENT_TAG = 'FrameworkDependent'
+        CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG = 'InputName'
+        CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG = 'InputShape'
+        CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG = 'Normalize'
+        CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG = 'Mean'
+        CONFIG_FRAMEWORK_DEPENDENT_STD_TAG = 'Std'
+        CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG = 'ChannelSwap'
+        CONFIG_FRAMEWORK_DEPENDENT_LAYOUT_TAG = 'Layout'
+
+        dep_parameters_tag = curr_test.getElementsByTagName(CONFIG_FRAMEWORK_DEPENDENT_TAG)[0]
+
+        _input_name = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_INPUT_NAME_TAG)[0].firstChild
+        _input_shape = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_INPUT_SHAPE_TAG)[0].firstChild
+        _normalize = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_NORMALIZE_TAG)[0].firstChild
+        _mean = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_MEAN_TAG)[0].firstChild
+        _std = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_STD_TAG)[0].firstChild
+        _channel_swap = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_CHANNEL_SWAP_TAG)[0].firstChild
+        _layout = dep_parameters_tag.getElementsByTagName(
+            CONFIG_FRAMEWORK_DEPENDENT_LAYOUT_TAG)[0].firstChild
+
+        return ExecuTorchParameters(
+            input_name=_input_name.data if _input_name else None,
+            input_shape=_input_shape.data if _input_shape else None,
+            normalize=_normalize.data if _normalize else None,
+            mean=_mean.data if _mean else None,
+            std=_std.data if _std else None,
+            channel_swap=_channel_swap.data if _channel_swap else None,
+            layout=_layout.data if _layout else None,
+        )
+
+
+class ExecuTorchParameters(FrameworkParameters):
+    def __init__(self, input_name, input_shape,
+                 normalize, mean, std, channel_swap,
+                 layout):
+        self.input_name = None
+        self.input_shape = None
+        self.normalize = None
+        self.mean = None
+        self.std = None
+        self.channel_swap = None
+        self.layout = None
+
+        if self._parameter_is_not_none(input_name):
+            self.input_name = input_name
+        if self._parameter_is_not_none(input_shape):
+            self.input_shape = input_shape
+        if self._parameter_is_not_none(normalize):
+            self.normalize = normalize
+        if self._parameter_is_not_none(mean):
+            self.mean = mean
+        if self._parameter_is_not_none(std):
+            self.std = std
+        if self._parameter_is_not_none(channel_swap):
+            self.channel_swap = channel_swap
+        if self._parameter_is_not_none(layout):
+            self.layout = layout

src/benchmark/frameworks/executorch/executorch_parameters_parser.py

@@ -0,0 +1,67 @@
+from pathlib import Path
+
+from ..processes import ProcessHandler
+
+
+class ExecuTorchProcess(ProcessHandler):
+    benchmark_app_name = 'executorch_python_benchmark'
+    launcher_latency_units = 'seconds'
+
+    def __init__(self, test, executor, log):
+        super().__init__(test, executor, log)
+        self.path_to_script = Path.joinpath(self.inference_script_root, 'inference_executorch.py')
+
+    @staticmethod
+    def create_process(test, executor, log):
+        return ExecuTorchProcess(test, executor, log)
+
+    def get_performance_metrics(self):
+        return self.get_performance_metrics_from_json_report()
+
+    def _fill_command_line(self):
+        python = ProcessHandler.get_cmd_python_version()
+        dataset = self._test.dataset.path
+        input_shape = self._test.dep_parameters.input_shape
+        layout = self._test.dep_parameters.layout
+        batch_size = self._test.indep_parameters.batch_size
+        iteration = self._test.indep_parameters.iteration
+        name = self._test.model.name
+        model = self._test.model.model
+        common_params = (f'-i {dataset} -is {input_shape} -b {batch_size} '
+                         f'-ni {iteration} --report_path {self.report_path} '
+                         f'--layout {layout} ')
+
+        input_name = self._test.dep_parameters.input_name
+        common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
+            common_params, '--input_name', input_name)
+
+        normalize = self._test.dep_parameters.normalize
+        if normalize == 'True':
+            common_params = ExecuTorchProcess._add_flag_to_cmd_line(
+                common_params, '--norm')
+
+        mean = self._test.dep_parameters.mean
+        common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
+            common_params, '--mean', mean)
+
+        std = self._test.dep_parameters.std
+        common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
+            common_params, '--std', std)
+
+        channel_swap = self._test.dep_parameters.channel_swap
+        common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
+            common_params, '--channel_swap', channel_swap)
+
+        device = self._test.indep_parameters.device
+        common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
+            common_params, '--device', device)
+
+        common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
+            common_params, '-m', model)
+
+        common_params = ExecuTorchProcess._add_optional_argument_to_cmd_line(
+            common_params, '-mn', name)
+
+        command_line = f'{python} {self.path_to_script} {common_params}'
+
+        return command_line

src/benchmark/frameworks/executorch/executorch_process.py

@@ -0,0 +1,28 @@
+from collections import OrderedDict
+
+from ..config_parser.test_reporter import Test
+
+
+class ExecuTorchTest(Test):
+    def __init__(self, model, dataset, indep_parameters, dep_parameters):
+        super().__init__(model, dataset, indep_parameters, dep_parameters)
+
+    def get_report(self, process):
+        parameters = OrderedDict()
+        parameters.update({'Device': self.indep_parameters.device})
+        parameters.update({'Iteration count': self.indep_parameters.iteration})
+        other_param = self._get_optional_parameters_string(parameters)
+
+        report_res = {
+            'task': self.model.task,
+            'model': self.model.name,
+            'dataset': self.dataset.name,
+            'source_framework': self.model.source_framework,
+            'inference_framework': self.indep_parameters.inference_framework,
+            'precision': self.model.precision,
+            'batch_size': self.indep_parameters.batch_size,
+            'mode': 'Sync',
+            'framework_params': other_param,
+        }
+
+        return report_res

src/benchmark/frameworks/executorch/executorch_test.py

@@ -0,0 +1,16 @@
+from .executorch_process import ExecuTorchProcess
+from .executorch_test import ExecuTorchTest
+from ..framework_wrapper import FrameworkWrapper
+from ..known_frameworks import KnownFrameworks
+
+
+class ExecuTorchWrapper(FrameworkWrapper):
+    framework_name = KnownFrameworks.executorch
+
+    @staticmethod
+    def create_process(test, executor, log, **kwargs):
+        return ExecuTorchProcess.create_process(test, executor, log)
+
+    @staticmethod
+    def create_test(model, dataset, indep_parameters, dep_parameters):
+        return ExecuTorchTest(model, dataset, indep_parameters, dep_parameters)

src/benchmark/frameworks/executorch/executorch_wrapper.py

@@ -20,6 +20,7 @@
 from .spektral.spektral_wrapper import SpektralWrapper
 from .rknn.rknn_wrapper import RknnWrapper
 from .executorch_cpp.executorch_cpp_wrapper import ExecuTorchCppWrapper
+from .executorch.executorch_wrapper import ExecuTorchWrapper
 
 
 class FrameworkWrapperRegistry(metaclass=Singleton):
@@ -60,3 +61,4 @@ def _get_wrappers(self):
         self._framework_wrappers[SpektralWrapper.framework_name] = SpektralWrapper()
         self._framework_wrappers[RknnWrapper.framework_name] = RknnWrapper()
         self._framework_wrappers[ExecuTorchCppWrapper.framework_name] = ExecuTorchCppWrapper()
+        self._framework_wrappers[ExecuTorchWrapper.framework_name] = ExecuTorchWrapper()

src/benchmark/frameworks/framework_wrapper_registry.py

@@ -18,3 +18,4 @@ class KnownFrameworks:
     rknn = 'RKNN'
     ncnn = 'ncnn'
     executorch_cpp = 'ExecuTorch Cpp'
+    executorch = 'ExecuTorch'

src/benchmark/frameworks/known_frameworks.py

@@ -20,7 +20,8 @@ option(BUILD_TFLITE_GPU_LAUNCHER "Build TensorFlow Lite with GPU Delegate launch
 option(BUILD_PYTORCH_LAUNCHER "Build PyTorch launcher" OFF)
 option(BUILD_PYTORCH_TENSORRT_LAUNCHER "Build PyTorch-TensorRT launcher" OFF)
 option(BUILD_RKNN_LAUNCHER "Build RKNN launcher" OFF)
-option(BUILD_EXECUTORCH_LAUNCHER "Build ExecuTorch launcher" OFF)
+option(BUILD_EXECUTORCH_LAUNCHER "Build ExecuTorch launcher with high level inference API" OFF)
+option(BUILD_EXECUTORCH_LAUNCHER_LOW_LEVEL_INFERENCE_API "Build ExecuTorch launcher with low level inference API")
 
 include(CMakeParseArguments)
 
@@ -112,6 +113,6 @@ if (BUILD_RKNN_LAUNCHER)
     add_subdirectory(rknn_launcher)
 endif()
 
-if(BUILD_EXECUTORCH_LAUNCHER)
+if(BUILD_EXECUTORCH_LAUNCHER OR BUILD_EXECUTORCH_LAUNCHER_LOW_LEVEL_INFERENCE_API)
     add_subdirectory(executorch_launcher)
 endif()

src/cpp_dl_benchmark/CMakeLists.txt

@@ -19,6 +19,12 @@ else()
     set(BACKEND_DEF "XNNPACK")
 endif()
 
+if (BUILD_EXECUTORCH_LAUNCHER)
+    set(MODULE_API "MODULE_API")
+elseif (BUILD_EXECUTORCH_LAUNCHER_LOW_LEVEL_INFERENCE_API)
+    set(MODULE_API "LOW_LEVEL_API")
+endif()
+
 list(REMOVE_ITEM EXECUTORCH_LIBRARIES extension_module) # Dynamic library not working https://github.com/pytorch/executorch/issues/8870
 
 if (NOT ${EXECUTORCH_OPTIMIZED_OPS_FIND} STREQUAL EXECUTORCH_OPTIMIZED_OPS_FIND-NOTFOUND)
@@ -42,4 +48,4 @@ add_launcher(NAME executorch_benchmark
         -Wl,--whole-archive,--allow-multiple-definition
         ${EXECUTORCH_LIBRARIES}
         -Wl,--no-whole-archive
-    DEFINITIONS EXECUTORCH_${BACKEND_DEF})
+    DEFINITIONS EXECUTORCH_${BACKEND_DEF} ${MODULE_API})