From 2a29ba5d90c480ca42c009ea14ffeb97c1f44717 Mon Sep 17 00:00:00 2001 From: Marya Date: Fri, 29 Sep 2023 13:45:59 +0100 Subject: [PATCH] update --- .../tartan_blog/build_and_run_networks.py | 186 ++++++++++-------- 1 file changed, 107 insertions(+), 79 deletions(-) diff --git a/examples/technical_blogs/tartan_blog/build_and_run_networks.py b/examples/technical_blogs/tartan_blog/build_and_run_networks.py index a252fff83..e2dac0666 100644 --- a/examples/technical_blogs/tartan_blog/build_and_run_networks.py +++ b/examples/technical_blogs/tartan_blog/build_and_run_networks.py @@ -1,64 +1,77 @@ +#!/usr/bin/env python +# Copyright (C) Codeplay Software Limited +# +# Licensed under the Apache License, Version 2.0 (the "License") with LLVM +# Exceptions; you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://github.com/codeplaysoftware/oneapi-construction-kit/blob/main/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + import subprocess import os +import sys import shutil import requests +import argparse + +parser = argparse.ArgumentParser(description='The purpose of this script is to build and run ' + 'VGG16 and Resnet50 networks with oneAPI and oneapi-construction-kit.') # Prereqs: portBLAS, SYCL-DNN, oneapi oneapi_construction_kit should be cloned and installed. -portBLAS_source = os.environ.get("portBLAS_source") -SYCL_DNN_source = os.environ.get("SYCL-DNN_source") -path_to_oneapi = os.environ.get("path_to_oneapi") -OCK_libCL_path = os.environ.get("OCK_libCL_path") -input_image = os.environ.get("input_image") +parser.add_argument('--portBLAS_source', required=True, help='Path to portBLAS source') +parser.add_argument('--SYCL_DNN_source', required=True, help='Path to SYCL-DNN source') +parser.add_argument('--oneapi_construction_kit_build_dir', required=True, help='Path to oneAPI construction kit') +parser.add_argument('--oneapi_base_toolkit', required=True, help='Path to oneAPI base toolkit') +parser.add_argument('--input_image', required=True, help='Path to the input image to be processed using the VGG16 and Resnet50.') + +args = parser.parse_args() # Set environment variables -os.environ["CMAKE_CXX_COMPILER"] = os.path.join(path_to_oneapi, "compiler/linux/bin-llvm/clang++") -os.environ["LD_LIBRARY_PATH"] = f"{os.path.join(path_to_oneapi, 'compiler/linux/lib/')}:{os.path.join(path_to_oneapi, 'compiler/linux/compiler/lib/intel64_lin')}:{os.environ.get('LD_LIBRARY_PATH', '')}" -os.environ["CXX"] = os.environ["CMAKE_CXX_COMPILER"] -os.environ["CC"] = os.path.join(path_to_oneapi, "compiler/linux/bin-llvm/clang") +os.environ["CMAKE_CXX_COMPILER"] = os.path.join(args.oneapi_base_toolkit, "compiler", "linux", "bin-llvm", "clang++") +os.environ["CMAKE_C_COMPILER"] = os.path.join(args.oneapi_base_toolkit, "compiler", "linux", "bin-llvm", "clang") +os.environ["LD_LIBRARY_PATH"] = f"{os.path.join(args.oneapi_base_toolkit, 'compiler', 'linux', 'lib')}:{os.path.join(args.oneapi_base_toolkit, 'compiler', 'linux', 'compiler', 'lib', 'intel64_lin')}:{os.environ.get('LD_LIBRARY_PATH', '')}" +os.environ["OCL_ICD_FILENAMES"] = f"{os.path.join(args.oneapi_construction_kit_build_dir, 'lib', 'libCL.so')}" -def create_directory(path): - if os.path.exists(path): - shutil.rmtree(path) - print(f"Directory '{path}' removed successfully.") - os.mkdir(path) - print(f"Directory '{path}' created successfully.") -def run_cmake_and_ninja(cmake_command): +def run_cmake_and_ninja(cmake_command, build_dir): try: subprocess.run(cmake_command, check=True) - subprocess.run("ninja", check=True) + subprocess.run(["ninja", "-C", build_dir], check=True) except subprocess.CalledProcessError as e: - print(f"Error executing command: {e}") + print(f"Error executing command: {e}", file=sys.stderr) # Build portBLAS def build_portBLAS(): script_directory = os.getcwd() - portBlas_build_dir_path = os.path.join(script_directory, "build_portBlas") - - create_directory(portBlas_build_dir_path) - os.chdir(portBlas_build_dir_path) + portBlas_build_dir = os.path.join(script_directory, "build_portBlas") cmake_portBlas_command = [ "cmake", - f"{portBLAS_source}", + f"-B {portBlas_build_dir}", + f"{args.portBLAS_source}", "-GNinja", "-DSYCL_COMPILER=dpcpp" ] - run_cmake_and_ninja(cmake_portBlas_command) - os.chdir("..") + run_cmake_and_ninja(cmake_portBlas_command, portBlas_build_dir) # Build SYCL-DNN def build_sycl_dnn(): script_directory = os.getcwd() sycl_dnn_build_dir = os.path.join(script_directory, "build_sycl_dnn") - create_directory(sycl_dnn_build_dir) - os.chdir(sycl_dnn_build_dir) - cmake_sycl_dnn_command = [ "cmake", - f"{SYCL_DNN_source}", + f"-B {sycl_dnn_build_dir}", + f"{args.SYCL_DNN_source}", "-GNinja", f"-DCMAKE_CXX_COMPILER={os.environ['CMAKE_CXX_COMPILER']}", "-DSNN_BUILD_BENCHMARKS=OFF", @@ -67,24 +80,38 @@ def build_sycl_dnn(): f"-DSyclBLAS_DIR='{os.path.join(script_directory, 'build_portBlas')}'" ] - run_cmake_and_ninja(cmake_sycl_dnn_command) - os.chdir("..") + run_cmake_and_ninja(cmake_sycl_dnn_command, sycl_dnn_build_dir) -def download_and_convert_model(file_url, destination_file, h5_to_bin_script): - try: - response = requests.get(file_url) - response.raise_for_status() - with open(destination_file, "wb") as file: - file.write(response.content) - print(f"Downloaded '{destination_file}' successfully.") +def download_and_convert_model(dir, file_url, destination_file, h5_to_bin_script): + #Check if the model dir already exists + path = os.path.join(os.getcwd(), dir) + if os.path.exists(path): + print(f"'{path}' already exists. Skipping creating '{path}'.") + else: + os.mkdir(path) + print(f"Directory '{path}' created successfully.") + os.chdir(path) + + # Check if the file already exists + if os.path.exists(destination_file): + print(f"'{destination_file}' already exists. Skipping download.") + else: + try: + response = requests.get(file_url) + response.raise_for_status() + with open(destination_file, "wb") as file: + file.write(response.content) + print(f"Downloaded '{destination_file}' successfully.") - subprocess.run(["python", h5_to_bin_script, destination_file], check=True) - print(f"Converted '{destination_file}' to binary successfully.") - except requests.exceptions.RequestException as e: - print(f"Error downloading file: {e}") - except subprocess.CalledProcessError as e: - print(f"Error running h5toBin.py: {e}") + subprocess.run(["python", h5_to_bin_script, destination_file], check=True) + print(f"Converted '{destination_file}' to binary successfully.") + os.chdir("..") # Move up one level to the parent directory + except requests.exceptions.RequestException as e: + print(f"Error downloading file: {e}", file=sys.stderr) + except subprocess.CalledProcessError as e: + print(f"Error running h5toBin.py: {e}", file=sys.stderr) + os.chdir("..") def prepare_image(img_script_path, input_image): @@ -92,7 +119,7 @@ def prepare_image(img_script_path, input_image): subprocess.run(["python", img_script_path, input_image], check=True) print(f"Image prepared successfully.") except subprocess.CalledProcessError as e: - print(f"Error executing the script: {e}") + print(f"Error executing the script: {e}", file=sys.stderr) # Test on images def test_image(model_script, data_dir, output_file): @@ -105,52 +132,53 @@ def test_image(model_script, data_dir, output_file): subprocess.run(command_args, check=True) print(f"Image tested successfully with {os.path.basename(model_script)}.") except subprocess.CalledProcessError as e: - print(f"Error executing the command: {e}") + print(f"Error executing the command: {e}", file=sys.stderr) -if __name__ == "__main__": +def main(): build_portBLAS() build_sycl_dnn() sycl_dnn_build_dir = os.path.join(os.getcwd(), "build_sycl_dnn") + vgg_dir = os.path.join(os.getcwd(), "vdata") + resnet_dir = os.path.join(os.getcwd(), "rdata") # Set the environment variables - os.environ["CA_HAL_DEBUG"] = "1" - os.environ["CA_PROFILE_LEVEL"] = "3" - os.environ["OCL_ICD_FILENAMES"] = f"{OCK_libCL_path}" + os.environ["CA_HAL_DEBUG"] = os.environ.get("CA_HAL_DEBUG", "1") + os.environ["CA_PROFILE_LEVEL"] = os.environ.get("CA_PROFILE_LEVEL", "3") os.environ["ONEAPI_DEVICE_SELECTOR"] = "opencl::acc" os.environ["SYCL_CONFIG_FILE_NAME"] = "null.cfg" # VGG16 - vgg_dir = "vdata" - create_directory(vgg_dir) - os.chdir(vgg_dir) - file_url = "https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5" - destination_file = "vgg16_weights_tf_dim_ordering_tf_kernels.h5" - h5_to_bin_script = os.path.join(SYCL_DNN_source, "samples", "networks", "vgg", "h5toBin.py") - - download_and_convert_model(file_url, destination_file, h5_to_bin_script) - os.chdir("..") # Move up one level to the parent directory - + vgg_url = "https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels.h5" + vgg_dest_file = "vgg16_weights_tf_dim_ordering_tf_kernels.h5" + vgg_h5_to_bin_script = os.path.join(args.SYCL_DNN_source, "samples", "networks", "vgg", "h5toBin.py") + + download_and_convert_model(vgg_dir, vgg_url, vgg_dest_file, vgg_h5_to_bin_script) + # Resnet50 - resnet_dir = "rdata" - create_directory(resnet_dir) - os.chdir(resnet_dir) - url = "https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels.h5" - r_dest_file = "resnet50_weights_tf_dim_ordering_tf_kernels.h5" - r_h5_to_bin_script = os.path.join(SYCL_DNN_source, "samples", "networks", "resnet50", "h5toBin.py") - - download_and_convert_model(url, r_dest_file, r_h5_to_bin_script) - os.chdir("..") - + resnet_url = "https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels.h5" + resnet_dest_file = "resnet50_weights_tf_dim_ordering_tf_kernels.h5" + resnet_h5_to_bin_script = os.path.join(args.SYCL_DNN_source, "samples", "networks", "resnet50", "h5toBin.py") + + download_and_convert_model(resnet_dir, resnet_url, resnet_dest_file, resnet_h5_to_bin_script) + # Prepare the image - img_script_path = os.path.join(SYCL_DNN_source, "samples", "networks", "img2bin.py") - - prepare_image(img_script_path, input_image) - + img_script_path = os.path.join(args.SYCL_DNN_source, "samples", "networks", "img2bin.py") + + prepare_image(img_script_path, args.input_image) + # Test images - output_file = input_image + ".bin" - vgg_script = os.path.join(sycl_dnn_build_dir, "samples/networks/vgg/vgg") - resnet_script = os.path.join(sycl_dnn_build_dir, "samples/networks/resnet50/resnet50") - + output_file = args.input_image + ".bin" + vgg_script = os.path.join(sycl_dnn_build_dir, "samples", "networks", "vgg", "vgg") + resnet_script = os.path.join(sycl_dnn_build_dir, "samples", "networks", "resnet50", "resnet50") + test_image(vgg_script, vgg_dir, output_file) - test_image(resnet_script, resnet_dir, output_file) \ No newline at end of file + test_image(resnet_script, resnet_dir, output_file) + + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) # Terminate the script with a non-zero exit code \ No newline at end of file