From 13640e3bce54d29be4ce7cc7b8fafc9c105ef3bc Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Wed, 3 Jan 2024 22:29:58 +0000 Subject: [PATCH 1/2] log warning for batch_size 1 --- src/deepsparse/legacy/pipeline.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/deepsparse/legacy/pipeline.py b/src/deepsparse/legacy/pipeline.py index 318433f250..ca3d55e114 100644 --- a/src/deepsparse/legacy/pipeline.py +++ b/src/deepsparse/legacy/pipeline.py @@ -16,6 +16,7 @@ Classes and registry for end to end inference pipelines that wrap an underlying inference engine and include pre/postprocessing """ +import logging import os from abc import ABC, abstractmethod from concurrent.futures import ThreadPoolExecutor @@ -75,6 +76,8 @@ SUPPORTED_PIPELINE_ENGINES = [DEEPSPARSE_ENGINE, ORT_ENGINE] +_LOGGER = logging.getLogger(__name__) + class Pipeline(BasePipeline): """ @@ -191,8 +194,12 @@ def __init__( f"instantiating Pipeline" ) + if self._batch_size == 0: + _LOGGER.warning("Overriding batch_size from 0 to 1") + self._batch_size = 1 + self._engine_args = dict( - batch_size=self._batch_size or 1, # bs=1 for dynamic batch + batch_size=self._batch_size, # bs=1 for dynamic batch num_cores=num_cores, input_shapes=input_shapes, ) @@ -206,7 +213,6 @@ def __init__( self.engine = None else: self.engine = self._initialize_engine() - self._batch_size = self._batch_size or 1 self.log( identifier=f"{SystemGroups.INFERENCE_DETAILS}/num_cores_total", From 71720109418562da8d39b5b8a5526fb34395251d Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Thu, 4 Jan 2024 17:44:19 +0000 Subject: [PATCH 2/2] fix tests, leave batch_size validation as is for no batchsize pathway --- src/deepsparse/legacy/pipeline.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/deepsparse/legacy/pipeline.py b/src/deepsparse/legacy/pipeline.py index ca3d55e114..c9aa87dc05 100644 --- a/src/deepsparse/legacy/pipeline.py +++ b/src/deepsparse/legacy/pipeline.py @@ -194,15 +194,15 @@ def __init__( f"instantiating Pipeline" ) - if self._batch_size == 0: - _LOGGER.warning("Overriding batch_size from 0 to 1") - self._batch_size = 1 - self._engine_args = dict( - batch_size=self._batch_size, # bs=1 for dynamic batch + batch_size=self._batch_size, num_cores=num_cores, input_shapes=input_shapes, ) + if self._engine_args["batch_size"] == 0: + _LOGGER.warning("Overriding batch_size from 0 to 1 for dynamic batch") + self._engine_args["batch_size"] = 1 + if engine_type.lower() == DEEPSPARSE_ENGINE: self._engine_args["scheduler"] = scheduler self._engine_args["num_streams"] = num_streams @@ -213,6 +213,7 @@ def __init__( self.engine = None else: self.engine = self._initialize_engine() + self._batch_size = self._batch_size or 1 self.log( identifier=f"{SystemGroups.INFERENCE_DETAILS}/num_cores_total",