Refactoring of folder - Multiple Bug Fixes

.gitignore

@@ -138,3 +138,6 @@ variables.data*
 
 # Local files
 /*.txt
+
+# Top Level Script Files
+/*.sh

tftrt/examples/benchmark_args.py

@@ -140,13 +140,6 @@ def __init__(self):
             help="If set to True, the benchmark will use XLA JIT Compilation."
         )
 
-        self._add_bool_argument(
-            name="skip_accuracy_testing",
-            default=False,
-            required=False,
-            help="If set to True, accuracy calculation will be skipped."
-        )
-
         self._add_bool_argument(
             name="use_synthetic_data",
             default=False,
@@ -187,6 +180,15 @@ def __init__(self):
             help="Minimum number of TensorFlow ops in a TRT engine."
         )
 
+        self._parser.add_argument(
+            "--num_build_batches",
+            type=int,
+            default=1,
+            help="How many iterations(batches) to use to build the TF-TRT "
+            "engines. If not supplied, only one batch will be used. This "
+            "parameter has only an effect if `--optimize_offline=True`"
+        )
+
         self._parser.add_argument(
             "--num_calib_batches",
             type=int,

tftrt/examples/benchmark_runner.py

@@ -169,7 +169,11 @@ def get_trt_precision(precision):
             print("\n[*] TF-TRT Converter Parameters:")
             print_dict(trt_converter_params)
 
-            converter = trt.TrtGraphConverterV2(**trt_converter_params)
+            try:
+                converter = trt.TrtGraphConverterV2(**trt_converter_params)
+            except TypeError:
+                del trt_converter_params["enable_sparse_compute"]
+                converter = trt.TrtGraphConverterV2(**trt_converter_params)
 
             def engine_build_input_fn(num_batches, model_phase):
                 dataset, _ = self.get_dataset_batches()
@@ -218,14 +222,14 @@ def engine_build_input_fn(num_batches, model_phase):
             except AttributeError:
                 pass
 
-            if strtobool(os.environ.get("TF_TRT_BENCHMARK_QUIT_AFTER_SUMMARY",
-                                        "0")):
+            if strtobool(os.environ.get("TF_TRT_BENCHMARK_EARLY_QUIT", "0")):
                 sys.exit(0)
 
-            if self._args.optimize_offline or self._args.use_dynamic_shape:
+            if self._args.optimize_offline:
 
                 offline_opt_input_fn = lambda: engine_build_input_fn(
-                    num_batches=1, model_phase="Building"
+                    num_batches=self._args.num_build_batches,
+                    model_phase="Building"
                 )
 
                 with timed_section("Building TensorRT engines"):
@@ -248,7 +252,9 @@ def engine_build_input_fn(num_batches, model_phase):
                     del converter
                     del graph_func
                     graph_func = load_model_from_disk(
-                        self._args.output_saved_model_dir
+                        self._args.output_saved_model_dir,
+                        tags=self._args.model_tag.split(","),
+                        signature_key=self._args.input_signature_key
                     )
 
         if isinstance(graph_func.structured_outputs, (tuple, list)):

tftrt/examples/image_classification/scripts/base_script.sh → tftrt/examples/image_classification/base_run_inference.sh

@@ -40,7 +40,7 @@ do
         shift # Remove --output_tensors_name= from processing
         ;;
         --use_xla_auto_jit)
-        TF_AUTO_JIT_XLA_FLAG="TF_XLA_FLAGS=--tf_xla_auto_jit=2"
+        TF_AUTO_JIT_XLA_FLAG="TF_XLA_FLAGS=\"--tf_xla_auto_jit=2 --tf_xla_cpu_global_jit\""
         shift # Remove --use_xla_auto_jit from processing
         ;;
         *)
@@ -54,7 +54,7 @@ done
 INPUT_SIZE=224
 PREPROCESS_METHOD="vgg"
 NUM_CLASSES=1001
-MAX_SAMPLES=49920
+MAX_SAMPLES=50000
 OUTPUT_TENSORS_NAME="logits"
 
 case ${MODEL_NAME} in
@@ -153,7 +153,7 @@ fi
 
 # %%%%%%%%%%%%%%%%%%%%%%% ARGUMENT VALIDATION %%%%%%%%%%%%%%%%%%%%%%% #
 
-BENCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )"
+BENCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd ${BENCH_DIR}
 
 # Execute the example

tftrt/examples/image_classification/models/inception_v3/analysis.txt

@@ -0,0 +1,15 @@
+
+MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
+
+signature_def['serving_default']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['input'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 299, 299, 3)
+        name: input:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['logits'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 1001)
+        name: logits:0
+  Method name is: tensorflow/serving/predict

tftrt/examples/image_classification/models/inception_v3/run_inference.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
+
+bash ${BASE_DIR}/base_run_inference.sh --model_name="inception_v3" ${@}

tftrt/examples/image_classification/models/inception_v4/analysis.txt

@@ -0,0 +1,15 @@
+
+MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
+
+signature_def['serving_default']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['input'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 299, 299, 3)
+        name: input:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['logits'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 1001)
+        name: logits:0
+  Method name is: tensorflow/serving/predict

tftrt/examples/image_classification/models/inception_v4/run_inference.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
+
+bash ${BASE_DIR}/base_run_inference.sh --model_name="inception_v4" ${@}

tftrt/examples/image_classification/models/mobilenet_v1/analysis.txt

@@ -0,0 +1,15 @@
+
+MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
+
+signature_def['serving_default']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['input'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 224, 224, 3)
+        name: input:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['logits'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 1001)
+        name: logits:0
+  Method name is: tensorflow/serving/predict

tftrt/examples/image_classification/models/mobilenet_v1/run_inference.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
+
+bash ${BASE_DIR}/base_run_inference.sh --model_name="mobilenet_v1" ${@}

tftrt/examples/image_classification/models/mobilenet_v2/analysis.txt

@@ -0,0 +1,15 @@
+
+MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
+
+signature_def['serving_default']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['input'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 224, 224, 3)
+        name: input:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['logits'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 1001)
+        name: logits:0
+  Method name is: tensorflow/serving/predict

tftrt/examples/image_classification/models/mobilenet_v2/run_inference.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
+
+bash ${BASE_DIR}/base_run_inference.sh --model_name="mobilenet_v2" ${@}

tftrt/examples/image_classification/models/nasnet_large/analysis.txt

@@ -0,0 +1,15 @@
+
+MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
+
+signature_def['serving_default']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['input'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 331, 331, 3)
+        name: input:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['logits'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 1001)
+        name: logits:0
+  Method name is: tensorflow/serving/predict

tftrt/examples/image_classification/models/nasnet_large/run_inference.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
+
+bash ${BASE_DIR}/base_run_inference.sh --model_name="nasnet_large" ${@}

tftrt/examples/image_classification/models/nasnet_mobile/analysis.txt

@@ -0,0 +1,15 @@
+
+MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
+
+signature_def['serving_default']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['input'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 224, 224, 3)
+        name: input:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['logits'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 1001)
+        name: logits:0
+  Method name is: tensorflow/serving/predict

tftrt/examples/image_classification/models/nasnet_mobile/run_inference.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
+
+bash ${BASE_DIR}/base_run_inference.sh --model_name="nasnet_mobile" ${@}

tftrt/examples/image_classification/models/resnet50-v1.5_tf1_ngc/analysis.txt

@@ -0,0 +1,36 @@
+
+MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
+
+signature_def['predict']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['input'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 224, 224, 3)
+        name: input_tensor:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['classes'] tensor_info:
+        dtype: DT_INT32
+        shape: (-1)
+        name: ArgMax:0
+    outputs['probabilities'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 1001)
+        name: resnet50/output/softmax:0
+  Method name is: tensorflow/serving/predict
+
+signature_def['serving_default']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['input'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 224, 224, 3)
+        name: input_tensor:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['classes'] tensor_info:
+        dtype: DT_INT32
+        shape: (-1)
+        name: ArgMax:0
+    outputs['probabilities'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 1001)
+        name: resnet50/output/softmax:0
+  Method name is: tensorflow/serving/predict

tftrt/examples/image_classification/models/resnet50-v1.5_tf1_ngc/run_inference.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
+
+bash ${BASE_DIR}/base_run_inference.sh --model_name="resnet50-v1.5_tf1_ngc" ${@}

tftrt/examples/image_classification/models/resnet50v2_backbone/analysis.txt

@@ -0,0 +1,15 @@
+
+MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
+
+signature_def['resnet50v2']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['inputs'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 256, 256, 3)
+        name: conv2d_input:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['outputs'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 8, 8, 2048)
+        name: outputs:0
+  Method name is: tensorflow/serving/predict

tftrt/examples/image_classification/models/resnet50v2_backbone/run_inference.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
+
+bash ${BASE_DIR}/base_run_inference.sh \
+    --model_name="resnet50v2_backbone" \
+    --input_signature_key="resnet50v2" \
+    --use_synthetic_data \
+    ${@}

tftrt/examples/image_classification/models/resnet50v2_sparse_backbone/analysis.txt

@@ -0,0 +1,15 @@
+
+MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:
+
+signature_def['resnet50v2']:
+  The given SavedModel SignatureDef contains the following input(s):
+    inputs['inputs'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 256, 256, 3)
+        name: inputs:0
+  The given SavedModel SignatureDef contains the following output(s):
+    outputs['outputs'] tensor_info:
+        dtype: DT_FLOAT
+        shape: (-1, 8, 8, 2048)
+        name: outputs:0
+  Method name is: tensorflow/serving/predict

tftrt/examples/image_classification/models/resnet50v2_sparse_backbone/run_inference.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.."
+
+bash ${BASE_DIR}/base_run_inference.sh \
+    --model_name="resnet50v2_sparse_backbone" \
+    --input_signature_key="resnet50v2" \
+    --use_synthetic_data \
+    ${@}