update

examples/technical_blogs/tartan_blog/build_and_run_networks.py

@@ -1,64 +1,77 @@
+#!/usr/bin/env python
+# Copyright (C) Codeplay Software Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License") with LLVM
+# Exceptions; you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://github.com/codeplaysoftware/oneapi-construction-kit/blob/main/LICENSE.txt
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
 import subprocess
 import os
+import sys
 import shutil
 import requests
+import argparse
+
+parser = argparse.ArgumentParser(description='The purpose of this script is to build and run '
+                                 'VGG16 and Resnet50 networks with oneAPI and oneapi-construction-kit.')
 
 # Prereqs: portBLAS, SYCL-DNN, oneapi oneapi_construction_kit should be cloned and installed.
-portBLAS_source = os.environ.get("portBLAS_source")
-SYCL_DNN_source = os.environ.get("SYCL-DNN_source")
-path_to_oneapi = os.environ.get("path_to_oneapi")
-OCK_libCL_path = os.environ.get("OCK_libCL_path")
-input_image = os.environ.get("input_image")
+parser.add_argument('--portBLAS_source', required=True, help='Path to portBLAS source')
+parser.add_argument('--SYCL_DNN_source', required=True, help='Path to SYCL-DNN source')
+parser.add_argument('--oneapi_construction_kit_build_dir', required=True, help='Path to oneAPI construction kit')
+parser.add_argument('--oneapi_base_toolkit', required=True, help='Path to oneAPI base toolkit')
+parser.add_argument('--input_image', required=True, help='Path to the input image to be processed using the VGG16 and Resnet50.')
+
+args = parser.parse_args()
 
 # Set environment variables
-os.environ["CMAKE_CXX_COMPILER"] = os.path.join(path_to_oneapi, "compiler/linux/bin-llvm/clang++")
-os.environ["LD_LIBRARY_PATH"] = f"{os.path.join(path_to_oneapi, 'compiler/linux/lib/')}:{os.path.join(path_to_oneapi, 'compiler/linux/compiler/lib/intel64_lin')}:{os.environ.get('LD_LIBRARY_PATH', '')}"
-os.environ["CXX"] = os.environ["CMAKE_CXX_COMPILER"]
-os.environ["CC"] = os.path.join(path_to_oneapi, "compiler/linux/bin-llvm/clang")
+os.environ["CMAKE_CXX_COMPILER"] = os.path.join(args.oneapi_base_toolkit, "compiler", "linux", "bin-llvm", "clang++")
+os.environ["CMAKE_C_COMPILER"] = os.path.join(args.oneapi_base_toolkit, "compiler", "linux", "bin-llvm", "clang")
+os.environ["LD_LIBRARY_PATH"] = f"{os.path.join(args.oneapi_base_toolkit, 'compiler', 'linux', 'lib')}:{os.path.join(args.oneapi_base_toolkit, 'compiler', 'linux', 'compiler', 'lib', 'intel64_lin')}:{os.environ.get('LD_LIBRARY_PATH', '')}"
+os.environ["OCL_ICD_FILENAMES"] = f"{os.path.join(args.oneapi_construction_kit_build_dir, 'lib', 'libCL.so')}"
 
-def create_directory(path):
-    if os.path.exists(path):
-        shutil.rmtree(path)
-        print(f"Directory '{path}' removed successfully.")
-    os.mkdir(path)
-    print(f"Directory '{path}' created successfully.")
 
-def run_cmake_and_ninja(cmake_command):
+def run_cmake_and_ninja(cmake_command, build_dir):
     try:
         subprocess.run(cmake_command, check=True)
-        subprocess.run("ninja", check=True)
+        subprocess.run(["ninja", "-C", build_dir], check=True)
     except subprocess.CalledProcessError as e:
-        print(f"Error executing command: {e}")
+        print(f"Error executing command: {e}", file=sys.stderr)
 
 # Build portBLAS
 def build_portBLAS():
     script_directory = os.getcwd()
-    portBlas_build_dir_path = os.path.join(script_directory, "build_portBlas")
-
-    create_directory(portBlas_build_dir_path)
-    os.chdir(portBlas_build_dir_path)
+    portBlas_build_dir = os.path.join(script_directory, "build_portBlas")
 
     cmake_portBlas_command = [
         "cmake",
-        f"{portBLAS_source}",
+        f"-B {portBlas_build_dir}",
+        f"{args.portBLAS_source}",
         "-GNinja",
         "-DSYCL_COMPILER=dpcpp"
     ]
 
-    run_cmake_and_ninja(cmake_portBlas_command)
-    os.chdir("..")
+    run_cmake_and_ninja(cmake_portBlas_command, portBlas_build_dir)
 
 # Build SYCL-DNN
 def build_sycl_dnn():
     script_directory = os.getcwd()
     sycl_dnn_build_dir = os.path.join(script_directory, "build_sycl_dnn")
 
-    create_directory(sycl_dnn_build_dir)
-    os.chdir(sycl_dnn_build_dir)
-
     cmake_sycl_dnn_command = [
         "cmake",
-        f"{SYCL_DNN_source}",
+        f"-B {sycl_dnn_build_dir}",
+        f"{args.SYCL_DNN_source}",
         "-GNinja",
         f"-DCMAKE_CXX_COMPILER={os.environ['CMAKE_CXX_COMPILER']}",
         "-DSNN_BUILD_BENCHMARKS=OFF",
@@ -67,32 +80,46 @@ def build_sycl_dnn():
         f"-DSyclBLAS_DIR='{os.path.join(script_directory, 'build_portBlas')}'"
     ]
 
-    run_cmake_and_ninja(cmake_sycl_dnn_command)
-    os.chdir("..")
+    run_cmake_and_ninja(cmake_sycl_dnn_command, sycl_dnn_build_dir)
 
 
-def download_and_convert_model(file_url, destination_file, h5_to_bin_script):
-    try:
-        response = requests.get(file_url)
-        response.raise_for_status()
-        with open(destination_file, "wb") as file:
-            file.write(response.content)
-        print(f"Downloaded '{destination_file}' successfully.")
+def download_and_convert_model(dir, file_url, destination_file, h5_to_bin_script):
+    #Check if the model dir already exists
+    path = os.path.join(os.getcwd(), dir)
+    if os.path.exists(path):
+        print(f"'{path}' already exists. Skipping creating '{path}'.")
+    else:
+        os.mkdir(path)
+        print(f"Directory '{path}' created successfully.")
+    os.chdir(path)
+
+    # Check if the file already exists
+    if os.path.exists(destination_file):
+        print(f"'{destination_file}' already exists. Skipping download.")
+    else:
+        try:
+            response = requests.get(file_url)
+            response.raise_for_status()
+            with open(destination_file, "wb") as file:
+                file.write(response.content)
+            print(f"Downloaded '{destination_file}' successfully.")
 
-        subprocess.run(["python", h5_to_bin_script, destination_file], check=True)
-        print(f"Converted '{destination_file}' to binary successfully.")
-    except requests.exceptions.RequestException as e:
-        print(f"Error downloading file: {e}")
-    except subprocess.CalledProcessError as e:
-        print(f"Error running h5toBin.py: {e}")
+            subprocess.run(["python", h5_to_bin_script, destination_file], check=True)
+            print(f"Converted '{destination_file}' to binary successfully.")
+            os.chdir("..")  # Move up one level to the parent directory
+        except requests.exceptions.RequestException as e:
+            print(f"Error downloading file: {e}", file=sys.stderr)
+        except subprocess.CalledProcessError as e:
+            print(f"Error running h5toBin.py: {e}", file=sys.stderr)
+    os.chdir("..")
 
 
 def prepare_image(img_script_path, input_image):
     try:
         subprocess.run(["python", img_script_path, input_image], check=True)
         print(f"Image prepared successfully.")
     except subprocess.CalledProcessError as e:
-        print(f"Error executing the script: {e}")
+        print(f"Error executing the script: {e}", file=sys.stderr)
 
 # Test on images
 def test_image(model_script, data_dir, output_file):
@@ -105,52 +132,53 @@ def test_image(model_script, data_dir, output_file):
         subprocess.run(command_args, check=True)
         print(f"Image tested successfully with {os.path.basename(model_script)}.")
     except subprocess.CalledProcessError as e:
-        print(f"Error executing the command: {e}")
+        print(f"Error executing the command: {e}", file=sys.stderr)
 
 
-if __name__ == "__main__":
+def main():
     build_portBLAS()
     build_sycl_dnn()
     sycl_dnn_build_dir = os.path.join(os.getcwd(), "build_sycl_dnn")
+    vgg_dir = os.path.join(os.getcwd(), "vdata")
+    resnet_dir = os.path.join(os.getcwd(), "rdata")
 
     # Set the environment variables
-    os.environ["CA_HAL_DEBUG"] = "1"
-    os.environ["CA_PROFILE_LEVEL"] = "3"
-    os.environ["OCL_ICD_FILENAMES"] = f"{OCK_libCL_path}"
+    os.environ["CA_HAL_DEBUG"] =  os.environ.get("CA_HAL_DEBUG", "1")
+    os.environ["CA_PROFILE_LEVEL"] = os.environ.get("CA_PROFILE_LEVEL", "3")
     os.environ["ONEAPI_DEVICE_SELECTOR"] = "opencl::acc"
     os.environ["SYCL_CONFIG_FILE_NAME"] = "null.cfg"
 
     # VGG16
-    vgg_dir = "vdata"
-    create_directory(vgg_dir)
-    os.chdir(vgg_dir)
-    file_url = "https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5"
-    destination_file = "vgg16_weights_tf_dim_ordering_tf_kernels.h5"
-    h5_to_bin_script = os.path.join(SYCL_DNN_source, "samples", "networks", "vgg", "h5toBin.py")
-
-    download_and_convert_model(file_url, destination_file, h5_to_bin_script)
-    os.chdir("..")  # Move up one level to the parent directory
-
+    vgg_url = "https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5"
+    vgg_dest_file = "vgg16_weights_tf_dim_ordering_tf_kernels.h5"
+    vgg_h5_to_bin_script = os.path.join(args.SYCL_DNN_source, "samples", "networks", "vgg", "h5toBin.py")
+
+    download_and_convert_model(vgg_dir, vgg_url, vgg_dest_file, vgg_h5_to_bin_script)
+
     # Resnet50
-    resnet_dir = "rdata"
-    create_directory(resnet_dir)
-    os.chdir(resnet_dir)
-    url = "https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels.h5"
-    r_dest_file = "resnet50_weights_tf_dim_ordering_tf_kernels.h5"
-    r_h5_to_bin_script = os.path.join(SYCL_DNN_source, "samples", "networks", "resnet50", "h5toBin.py")
-
-    download_and_convert_model(url, r_dest_file, r_h5_to_bin_script)
-    os.chdir("..")
-
+    resnet_url = "https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels.h5"
+    resnet_dest_file = "resnet50_weights_tf_dim_ordering_tf_kernels.h5"
+    resnet_h5_to_bin_script = os.path.join(args.SYCL_DNN_source, "samples", "networks", "resnet50", "h5toBin.py")
+
+    download_and_convert_model(resnet_dir, resnet_url, resnet_dest_file, resnet_h5_to_bin_script)
+
     # Prepare the image
-    img_script_path = os.path.join(SYCL_DNN_source, "samples", "networks", "img2bin.py")
-    
-    prepare_image(img_script_path, input_image)
-    
+    img_script_path = os.path.join(args.SYCL_DNN_source, "samples", "networks", "img2bin.py")
+
+    prepare_image(img_script_path, args.input_image)
+
     # Test images
-    output_file = input_image + ".bin"
-    vgg_script = os.path.join(sycl_dnn_build_dir, "samples/networks/vgg/vgg")
-    resnet_script = os.path.join(sycl_dnn_build_dir, "samples/networks/resnet50/resnet50")
-    
+    output_file = args.input_image + ".bin"
+    vgg_script = os.path.join(sycl_dnn_build_dir, "samples", "networks", "vgg", "vgg")
+    resnet_script = os.path.join(sycl_dnn_build_dir, "samples", "networks", "resnet50", "resnet50")
+
     test_image(vgg_script, vgg_dir, output_file)
-    test_image(resnet_script, resnet_dir, output_file)
+    test_image(resnet_script, resnet_dir, output_file)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)  # Terminate the script with a non-zero exit code