From 8e4fb028edd7be67e1c3d38fb3367e110158c8d8 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Fri, 28 Mar 2025 12:10:29 +0530 Subject: [PATCH 1/5] Update backend_onnxruntime.py --- .../python/backend_onnxruntime.py | 7 +++++++ vision/classification_and_detection/python/main.py | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/vision/classification_and_detection/python/backend_onnxruntime.py b/vision/classification_and_detection/python/backend_onnxruntime.py index 787e721f22..58ebe41555 100755 --- a/vision/classification_and_detection/python/backend_onnxruntime.py +++ b/vision/classification_and_detection/python/backend_onnxruntime.py @@ -26,7 +26,14 @@ def image_format(self): def load(self, model_path, inputs=None, outputs=None): """Load model and find input/outputs from the model file.""" + print("************************************************************") + print(">>> Value of num_threads: ", num_threads) + print("************************************************************") + os.environ["OMP_NUM_THREADS"] = str(num_threads) + os.environ["OPENBLAS_NUM_THREADS"] = str(num_threads) + os.environ["MKL_NUM_THREADS"] = str(num_threads) opt = rt.SessionOptions() + opt.intra_op_num_threads = int(num_threads) # By default all optimizations are enabled # https://onnxruntime.ai/docs/performance/graph-optimizations.html diff --git a/vision/classification_and_detection/python/main.py b/vision/classification_and_detection/python/main.py index 44a7b6bf7f..6518e69325 100755 --- a/vision/classification_and_detection/python/main.py +++ b/vision/classification_and_detection/python/main.py @@ -646,7 +646,9 @@ def main(): model = backend.load( args.model, inputs=args.inputs, - outputs=args.outputs) + # outputs=args.output, + num_threads=args.threads + ) final_results = { "runtime": model.name(), "version": model.version(), From 28c3140bed2f30c4c44fea5c67510a9ca5b577c6 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Fri, 28 Mar 2025 14:29:46 +0530 Subject: [PATCH 2/5] Update backend_onnxruntime.py --- .../classification_and_detection/python/backend_onnxruntime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vision/classification_and_detection/python/backend_onnxruntime.py b/vision/classification_and_detection/python/backend_onnxruntime.py index 58ebe41555..8a6b0ac20f 100755 --- a/vision/classification_and_detection/python/backend_onnxruntime.py +++ b/vision/classification_and_detection/python/backend_onnxruntime.py @@ -24,7 +24,7 @@ def image_format(self): """image_format. For onnx it is always NCHW.""" return "NCHW" - def load(self, model_path, inputs=None, outputs=None): + def load(self, model_path, inputs=None, outputs=None, num_threads=None): """Load model and find input/outputs from the model file.""" print("************************************************************") print(">>> Value of num_threads: ", num_threads) From a8b830573cee4739583dacf4e7ac4fe42506e01b Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Fri, 28 Mar 2025 14:57:14 +0530 Subject: [PATCH 3/5] Update main.py --- vision/classification_and_detection/python/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vision/classification_and_detection/python/main.py b/vision/classification_and_detection/python/main.py index 6518e69325..4f2b2de8e8 100755 --- a/vision/classification_and_detection/python/main.py +++ b/vision/classification_and_detection/python/main.py @@ -646,9 +646,9 @@ def main(): model = backend.load( args.model, inputs=args.inputs, - # outputs=args.output, + outputs=args.outputs, num_threads=args.threads - ) + ) final_results = { "runtime": model.name(), "version": model.version(), From 6dc16bf7682e54259541bd9966b534b35c8024a1 Mon Sep 17 00:00:00 2001 From: Sujith Kanakkassery Date: Sun, 30 Mar 2025 11:13:52 +0530 Subject: [PATCH 4/5] load function thread parameter update --- .../classification_and_detection/python/backend.py | 2 +- .../python/backend_ncnn.py | 2 +- .../python/backend_onnxruntime.py | 10 ++++------ .../python/backend_pytorch.py | 4 +++- .../python/backend_pytorch_native.py | 4 +++- .../python/backend_tf.py | 2 +- .../python/backend_tvm.py | 2 +- vision/classification_and_detection/python/main.py | 14 +++++++++++++- 8 files changed, 27 insertions(+), 13 deletions(-) diff --git a/vision/classification_and_detection/python/backend.py b/vision/classification_and_detection/python/backend.py index 6fc13454a8..67d07e39d5 100755 --- a/vision/classification_and_detection/python/backend.py +++ b/vision/classification_and_detection/python/backend.py @@ -16,7 +16,7 @@ def version(self): def name(self): raise NotImplementedError("Backend:name") - def load(self, model_path, inputs=None, outputs=None): + def load(self, model_path, inputs=None, outputs=None, threads=None): raise NotImplementedError("Backend:load") def predict(self, feed): diff --git a/vision/classification_and_detection/python/backend_ncnn.py b/vision/classification_and_detection/python/backend_ncnn.py index 16d44a02fa..07267fc044 100644 --- a/vision/classification_and_detection/python/backend_ncnn.py +++ b/vision/classification_and_detection/python/backend_ncnn.py @@ -18,7 +18,7 @@ def image_format(self): """image_format. For ncnn it is NCHW.""" return "NCHW" - def load(self, model_path, inputs=None, outputs=None): + def load(self, model_path, inputs=None, outputs=None, threads=None): param_file, bin_file = f"{model_path}.param", f"{model_path}.bin" if param_file.endswith("resnet50_v1.param"): # download model files if doesn't diff --git a/vision/classification_and_detection/python/backend_onnxruntime.py b/vision/classification_and_detection/python/backend_onnxruntime.py index 8a6b0ac20f..e28d461421 100755 --- a/vision/classification_and_detection/python/backend_onnxruntime.py +++ b/vision/classification_and_detection/python/backend_onnxruntime.py @@ -24,16 +24,14 @@ def image_format(self): """image_format. For onnx it is always NCHW.""" return "NCHW" - def load(self, model_path, inputs=None, outputs=None, num_threads=None): + def load(self, model_path, inputs=None, outputs=None, threads=None): """Load model and find input/outputs from the model file.""" print("************************************************************") - print(">>> Value of num_threads: ", num_threads) + print(">>> Value of num_threads: ", threads) print("************************************************************") - os.environ["OMP_NUM_THREADS"] = str(num_threads) - os.environ["OPENBLAS_NUM_THREADS"] = str(num_threads) - os.environ["MKL_NUM_THREADS"] = str(num_threads) opt = rt.SessionOptions() - opt.intra_op_num_threads = int(num_threads) + opt.intra_op_num_threads = int(threads) + opt.inter_op_num_threads = int(threads) # By default all optimizations are enabled # https://onnxruntime.ai/docs/performance/graph-optimizations.html diff --git a/vision/classification_and_detection/python/backend_pytorch.py b/vision/classification_and_detection/python/backend_pytorch.py index 02b010ac71..a0d32cf540 100644 --- a/vision/classification_and_detection/python/backend_pytorch.py +++ b/vision/classification_and_detection/python/backend_pytorch.py @@ -30,8 +30,10 @@ def name(self): def image_format(self): return "NCHW" - def load(self, model_path, inputs=None, outputs=None): + def load(self, model_path, inputs=None, outputs=None, threads=None): self.model = onnx.load(model_path) + torch.set_num_threads(threads) + torch.set_num_interop_threads(threads) # find inputs from the model if not passed in by config if inputs: diff --git a/vision/classification_and_detection/python/backend_pytorch_native.py b/vision/classification_and_detection/python/backend_pytorch_native.py index 23a1a7f9ca..69bf19168e 100755 --- a/vision/classification_and_detection/python/backend_pytorch_native.py +++ b/vision/classification_and_detection/python/backend_pytorch_native.py @@ -24,9 +24,11 @@ def name(self): def image_format(self): return "NCHW" - def load(self, model_path, inputs=None, outputs=None): + def load(self, model_path, inputs=None, outputs=None, threads=None): self.model = torch.load(model_path, weights_only=False) self.model.eval() + torch.set_num_threads(threads) + torch.set_num_interop_threads(threads) # find inputs from the model if not passed in by config if inputs: self.inputs = inputs diff --git a/vision/classification_and_detection/python/backend_tf.py b/vision/classification_and_detection/python/backend_tf.py index b53bf0b784..1d8e0ada1e 100755 --- a/vision/classification_and_detection/python/backend_tf.py +++ b/vision/classification_and_detection/python/backend_tf.py @@ -27,7 +27,7 @@ def image_format(self): # NHWC) return "NHWC" - def load(self, model_path, inputs=None, outputs=None): + def load(self, model_path, inputs=None, outputs=None, threads=None): # there is no input/output meta data i the graph so it need to come # from config. if not inputs: diff --git a/vision/classification_and_detection/python/backend_tvm.py b/vision/classification_and_detection/python/backend_tvm.py index 86223b60e5..3a63bc7e28 100644 --- a/vision/classification_and_detection/python/backend_tvm.py +++ b/vision/classification_and_detection/python/backend_tvm.py @@ -200,7 +200,7 @@ def _worker_handler(feed): global global_executor return global_executor.predict(feed) - def load(self, model_path, inputs=None, outputs=None): + def load(self, model_path, inputs=None, outputs=None, threads=None): """Load model and find input/outputs from the model file.""" self.load_impl(model_path, inputs, outputs, self.max_batchsize) diff --git a/vision/classification_and_detection/python/main.py b/vision/classification_and_detection/python/main.py index 4f2b2de8e8..9629e2969b 100755 --- a/vision/classification_and_detection/python/main.py +++ b/vision/classification_and_detection/python/main.py @@ -383,6 +383,18 @@ def get_args(): if args.scenario not in SCENARIO_MAP: parser.error("valid scanarios:" + str(list(SCENARIO_MAP.keys()))) + + if args.threads: + num_threads = args.threads + print("*****>>> Value of num_threads: {num_threads} **********") + os.environ["OMP_NUM_THREADS"] = str(num_threads) + os.environ["OPENBLAS_NUM_THREADS"] = str(num_threads) + os.environ["MKL_NUM_THREADS"] = str(num_threads) + os.environ["TF_NUM_INTRAOP_THREADS"] = str(num_threads) + os.environ["TF_NUM_INTEROP_THREADS"] = str(num_threads) + os.environ["TVM_NUM_THREADS"] = str(num_threads) + os.environ["TFLITE_NUM_THREADS"] = str(num_threads) + os.environ["NCNN_NUM_THREADS"] = str(num_threads) return args @@ -647,7 +659,7 @@ def main(): args.model, inputs=args.inputs, outputs=args.outputs, - num_threads=args.threads + threads=args.threads, ) final_results = { "runtime": model.name(), From edf0ac3d177ef2f502a66e829d42d92bb5721add Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 30 Mar 2025 05:44:26 +0000 Subject: [PATCH 5/5] [Automated Commit] Format Codebase --- .../python/main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/vision/classification_and_detection/python/main.py b/vision/classification_and_detection/python/main.py index 9629e2969b..d2a0cf309e 100755 --- a/vision/classification_and_detection/python/main.py +++ b/vision/classification_and_detection/python/main.py @@ -383,18 +383,18 @@ def get_args(): if args.scenario not in SCENARIO_MAP: parser.error("valid scanarios:" + str(list(SCENARIO_MAP.keys()))) - + if args.threads: num_threads = args.threads print("*****>>> Value of num_threads: {num_threads} **********") - os.environ["OMP_NUM_THREADS"] = str(num_threads) - os.environ["OPENBLAS_NUM_THREADS"] = str(num_threads) - os.environ["MKL_NUM_THREADS"] = str(num_threads) - os.environ["TF_NUM_INTRAOP_THREADS"] = str(num_threads) - os.environ["TF_NUM_INTEROP_THREADS"] = str(num_threads) - os.environ["TVM_NUM_THREADS"] = str(num_threads) - os.environ["TFLITE_NUM_THREADS"] = str(num_threads) - os.environ["NCNN_NUM_THREADS"] = str(num_threads) + os.environ["OMP_NUM_THREADS"] = str(num_threads) + os.environ["OPENBLAS_NUM_THREADS"] = str(num_threads) + os.environ["MKL_NUM_THREADS"] = str(num_threads) + os.environ["TF_NUM_INTRAOP_THREADS"] = str(num_threads) + os.environ["TF_NUM_INTEROP_THREADS"] = str(num_threads) + os.environ["TVM_NUM_THREADS"] = str(num_threads) + os.environ["TFLITE_NUM_THREADS"] = str(num_threads) + os.environ["NCNN_NUM_THREADS"] = str(num_threads) return args