Unify all file handling with pathlib (#512)

* Replace occurences of os.path functions with equivalent functions from
  pathlib library

* Remove unwanted imports of os.path and os

* Add coding guidelines for using pathlib instead of os.path

CONTRIBUTING.md

@@ -51,3 +51,9 @@ Now, when you commit code to the repository you should see something like this:
 ![A screen capture showing terminal output from a pre-commit hook](docs/data/contributing/pre-commit-hook.png)
 
 Please see the [pre-commit documentation](https://pre-commit.com/#quick-start) for additional information.
+
+## Coding guidelines
+
+Below are some repository specific guidelines which are followed througout the repository.
+Any future contributions should adhere to these guidelines:
+* Use pathlib library functions instead of os.path for manipulating file paths

docs/archive/docs-1.x/conf.py

@@ -8,19 +8,20 @@
 
 # -- Path setup --------------------------------------------------------------
 
+import subprocess as sp
+import sys
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
+# documentation root, use str(Path(<rel_path>).absolute().resolve()) to make it absolute, like shown here.
 #
-import os
-import subprocess as sp
-import sys
+from pathlib import Path
 
-sys.path.insert(0, os.path.abspath(".."))
+sys.path.insert(0, str(Path("..").absolute().resolve()))
 
 repo_version = "unknown"
 # Determine short version by file in repo
-if os.path.isfile("./VERSION"):
+if Path("./VERSION").is_file():
     with open("./VERSION") as f:
         repo_version = f.readline().strip()
 

docs/archive/docs-2.x/conf.py

@@ -8,19 +8,20 @@
 
 # -- Path setup --------------------------------------------------------------
 
+import subprocess as sp
+import sys
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
+# documentation root, use str(Path(<rel_path>).absolute().resolve()) to make it absolute, like shown here.
 #
-import os
-import subprocess as sp
-import sys
+from pathlib import Path
 
-sys.path.insert(0, os.path.abspath(".."))
+sys.path.insert(0, str(Path("..").absolute().resolve()))
 
 repo_version = "unknown"
 # Determine short version by file in repo
-if os.path.isfile("./VERSION"):
+if Path("./VERSION").is_file():
     with open("./VERSION") as f:
         repo_version = f.readline().strip()
 

src/argparser.py

@@ -25,6 +25,7 @@
 import argparse
 import os
 import shutil
+from pathlib import Path
 
 
 def print_avail_arch(avail_arch: list):
@@ -123,7 +124,7 @@ def omniarg_parser(
         metavar="",
         type=str,
         dest="path",
-        default=os.path.join(os.getcwd(), "workloads"),
+        default=str(Path(os.getcwd()).joinpath("workloads")),
         required=False,
         help="\t\t\tSpecify path to save workload.\n\t\t\t(DEFAULT: {}/workloads/<name>)".format(
             os.getcwd()

src/rocprof-compute

@@ -26,14 +26,14 @@
 # SOFTWARE.
 ##############################################################################el
 
-# import logging
-import os
 import re
 import sys
 
+# import logging
+from pathlib import Path
+
 try:
     from importlib import metadata
-    from pathlib import Path
 
     from rocprof_compute_base import RocProfCompute
     from utils.utils import console_error
@@ -73,12 +73,12 @@ def verify_deps():
         )
         sys.exit(1)
 
-    bindir = Path(__file__).resolve().parent
+    bindir = str(Path(__file__).resolve().parent)
     depsLocation = ["requirements.txt", "../requirements.txt"]
 
     for location in depsLocation:
-        checkFile = os.path.join(bindir, location)
-        if os.path.exists(checkFile):
+        checkFile = str(Path(bindir).joinpath(location))
+        if Path(checkFile).exists():
             with open(checkFile, "r", encoding="utf-8") as file_in:
                 dependencies = file_in.read().splitlines()
 

src/rocprof_compute_analyze/analysis_base.py

@@ -192,9 +192,9 @@ def sanitize(self):
             )
         # ensure absolute path
         for dir in self.__args.path:
-            full_path = os.path.abspath(dir[0])
+            full_path = str(Path(dir[0]).absolute().resolve())
             dir[0] = full_path
-            if not os.path.isdir(dir[0]):
+            if not Path(dir[0]).is_dir():
                 console_error("Invalid directory {}\nPlease try again.".format(dir[0]))
             # validate profiling data
 

src/rocprof_compute_analyze/analysis_webui.py

@@ -25,6 +25,7 @@
 import copy
 import os
 import random
+from pathlib import Path
 
 import dash
 import dash_bootstrap_components as dbc
@@ -45,7 +46,7 @@ def __init__(self, args, supported_archs):
         self.app = dash.Dash(
             __name__, title=PROJECT_NAME, external_stylesheets=[dbc.themes.CYBORG]
         )
-        self.dest_dir = os.path.abspath(args.path[0][0])
+        self.dest_dir = str(Path(args.path[0][0]).absolute().resolve())
         self.arch = None
 
         self.__hidden_sections = ["Memory Chart", "Roofline"]
@@ -168,7 +169,7 @@ def generate_from_filter(
             div_children.append(
                 get_memchart(panel_configs[300]["data source"], base_data[base_run])
             )
-            has_roofline = os.path.isfile(os.path.join(self.dest_dir, "roofline.csv"))
+            has_roofline = Path(self.dest_dir).joinpath("roofline.csv").is_file()
             if has_roofline and hasattr(self.get_socs()[self.arch], "roofline_obj"):
                 # update roofline for visualization in GUI
                 self.get_socs()[self.arch].analysis_setup(

src/rocprof_compute_base.py

@@ -205,9 +205,13 @@ def parse_args(self):
             # FIXME:
             #     Might want to get host name from detected spec
             if self.__args.subpath == "node_name":
-                self.__args.path = os.path.join(self.__args.path, socket.gethostname())
+                self.__args.path = str(
+                    Path(self.__args.path).joinpath(socket.gethostname())
+                )
             elif self.__args.subpath == "gpu_model":
-                self.__args.path = os.path.join(self.__args.path, self.__mspec.gpu_model)
+                self.__args.path = str(
+                    Path(self.__args.path).joinpath(self.__mspec.gpu_model)
+                )
 
             p = Path(self.__args.path)
             if not p.exists():
@@ -227,9 +231,9 @@ def run_profiler(self):
         #     unless there is a specific reason to do here.
 
         # Update default path
-        if self.__args.path == os.path.join(os.getcwd(), "workloads"):
-            self.__args.path = os.path.join(
-                self.__args.path, self.__args.name, self.__mspec.gpu_model
+        if self.__args.path == str(Path(os.getcwd()).joinpath("workloads")):
+            self.__args.path = str(
+                Path(self.__args.path).joinpath(self.__args.name, self.__mspec.gpu_model)
             )
 
         # instantiate desired profiler

src/rocprof_compute_profile/profiler_base.py

@@ -29,6 +29,7 @@
 import sys
 import time
 from abc import ABC, abstractmethod
+from pathlib import Path
 
 import pandas as pd
 from tqdm import tqdm
@@ -53,8 +54,10 @@ def __init__(self, args, profiler_mode, soc):
         self.__args = args
         self.__profiler = profiler_mode
         self._soc = soc  # OmniSoC obj
-        self.__perfmon_dir = os.path.join(
-            str(config.rocprof_compute_home), "rocprof_compute_soc", "profile_configs"
+        self.__perfmon_dir = str(
+            Path(str(config.rocprof_compute_home)).joinpath(
+                "rocprof_compute_soc", "profile_configs"
+            )
         )
 
     def get_args(self):
@@ -263,7 +266,7 @@ def pre_processing(self):
         # verify correct formatting for application binary
         self.__args.remaining = self.__args.remaining[1:]
         if self.__args.remaining:
-            if not os.path.isfile(self.__args.remaining[0]):
+            if not Path(self.__args.remaining[0]).is_file():
                 console_error(
                     "Your command %s doesn't point to a executable. Please verify."
                     % self.__args.remaining[0]
@@ -290,7 +293,7 @@ def run_profiling(self, version: str, prog: str):
         # log basic info
         console_log(str(prog).title() + " version: " + str(version))
         console_log("Profiler choice: %s" % self.__profiler)
-        console_log("Path: " + str(os.path.abspath(self.__args.path)))
+        console_log("Path: " + str(Path(self.__args.path).absolute().resolve()))
         console_log("Target: " + str(self._soc._mspec.gpu_model))
         console_log("Command: " + str(self.__args.remaining))
         console_log("Kernel Selection: " + str(self.__args.kernel))

src/rocprof_compute_profile/profiler_rocprof_v1.py

@@ -23,6 +23,7 @@
 ##############################################################################el
 
 import os
+from pathlib import Path
 
 from rocprof_compute_profile.profiler_base import RocProfCompute_Base
 from utils.utils import console_log, demarcate, replace_timestamps, store_app_cmd
@@ -33,12 +34,12 @@ def __init__(self, profiling_args, profiler_mode, soc):
         super().__init__(profiling_args, profiler_mode, soc)
         self.ready_to_profile = (
             self.get_args().roof_only
-            and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv"))
+            and not Path(self.get_args().path).joinpath("pmc_perf.csv").is_file()
             or not self.get_args().roof_only
         )
 
     def get_profiler_options(self, fname):
-        fbase = os.path.splitext(os.path.basename(fname))[0]
+        fbase = Path(fname).stem
         app_cmd = self.get_args().remaining
         args = [
             # v1 requires request for timestamps

src/rocprof_compute_profile/profiler_rocprof_v2.py

@@ -24,6 +24,7 @@
 
 import os
 import shlex
+from pathlib import Path
 
 from rocprof_compute_profile.profiler_base import RocProfCompute_Base
 from utils.utils import console_log, demarcate, replace_timestamps, store_app_cmd
@@ -34,12 +35,12 @@ def __init__(self, profiling_args, profiler_mode, soc):
         super().__init__(profiling_args, profiler_mode, soc)
         self.ready_to_profile = (
             self.get_args().roof_only
-            and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv"))
+            and not Path(self.get_args().path).joinpath("pmc_perf.csv").is_file()
             or not self.get_args().roof_only
         )
 
     def get_profiler_options(self, fname):
-        fbase = os.path.splitext(os.path.basename(fname))[0]
+        fbase = Path(fname).stem
         app_cmd = shlex.split(self.get_args().remaining)
         args = [
             # v2 requires output directory argument

src/rocprof_compute_profile/profiler_rocprof_v3.py

@@ -24,6 +24,7 @@
 
 import os
 import shlex
+from pathlib import Path
 
 import config
 from rocprof_compute_profile.profiler_base import RocProfCompute_Base
@@ -35,7 +36,7 @@ def __init__(self, profiling_args, profiler_mode, soc):
         super().__init__(profiling_args, profiler_mode, soc)
         self.ready_to_profile = (
             self.get_args().roof_only
-            and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv"))
+            and not Path(self.get_args().path).joinpath("pmc_perf.csv").is_file()
             or not self.get_args().roof_only
         )
 

src/rocprof_compute_soc/soc_base.py

@@ -53,9 +53,11 @@ def __init__(
         self.populate_mspec()
         # In some cases (i.e. --specs) path will not be given
         if hasattr(self.__args, "path"):
-            if self.__args.path == os.path.join(os.getcwd(), "workloads"):
-                self.__workload_dir = os.path.join(
-                    self.__args.path, self.__args.name, self._mspec.gpu_model
+            if self.__args.path == str(Path(os.getcwd()).joinpath("workloads")):
+                self.__workload_dir = str(
+                    Path(self.__args.path).joinpath(
+                        self.__args.name, self._mspec.gpu_model
+                    )
                 )
             else:
                 self.__workload_dir = self.__args.path
@@ -196,16 +198,17 @@ def populate_mspec(self):
     @demarcate
     def perfmon_filter(self, roofline_perfmon_only: bool):
         """Filter default performance counter set based on user arguments"""
-        if roofline_perfmon_only and os.path.isfile(
-            os.path.join(self.get_args().path, "pmc_perf.csv")
+        if (
+            roofline_perfmon_only
+            and Path(self.get_args().path).joinpath("pmc_perf.csv").is_file()
         ):
             return
         workload_perfmon_dir = self.__workload_dir + "/perfmon"
 
         # Initialize directories
-        if not os.path.isdir(self.__workload_dir):
+        if not Path(self.__workload_dir).is_dir():
             os.makedirs(self.__workload_dir)
-        elif not os.path.islink(self.__workload_dir):
+        elif not Path(self.__workload_dir).is_symlink():
             shutil.rmtree(self.__workload_dir)
         else:
             os.unlink(self.__workload_dir)
@@ -226,7 +229,7 @@ def perfmon_filter(self, roofline_perfmon_only: bool):
 
                 pmc_files_list = []
                 for fname in ref_pmc_files_list:
-                    fbase = os.path.splitext(os.path.basename(fname))[0]
+                    fbase = Path(fname).stem
                     ip = re.match(mpattern, fbase).group(1)
                     if ip in self.__args.ipblocks:
                         pmc_files_list.append(fname)
@@ -469,9 +472,10 @@ def perfmon_coalesce(pmc_files_list, perfmon_config, workload_dir, spatial_multi
         )
 
         for f_idx in range(groups_per_bucket):
-            file_name = os.path.join(
-                workload_perfmon_dir,
-                "pmc_perf_" + "node_" + str(node_idx) + "_" + str(f_idx) + ".txt",
+            file_name = str(
+                Path(workload_perfmon_dir).joinpath(
+                    "pmc_perf_" + "node_" + str(node_idx) + "_" + str(f_idx) + ".txt"
+                )
             )
 
             pmc = []
@@ -494,7 +498,7 @@ def perfmon_coalesce(pmc_files_list, perfmon_config, workload_dir, spatial_multi
     else:
         # Output to files
         for f in output_files:
-            file_name = os.path.join(workload_perfmon_dir, f.file_name)
+            file_name = str(Path(workload_perfmon_dir).joinpath(f.file_name))
 
             pmc = []
             for block_name in f.blocks.keys():
@@ -529,7 +533,7 @@ def perfmon_coalesce(pmc_files_list, perfmon_config, workload_dir, spatial_multi
     # Add a timestamp file
     # TODO: Does v3 need this?
     if not using_v3():
-        fd = open(os.path.join(workload_perfmon_dir, "timestamps.txt"), "w")
+        fd = open(str(Path(workload_perfmon_dir).joinpath("timestamps.txt")), "w")
         fd.write("pmc:\n\n")
         fd.write("gpu:\n")
         fd.write("range:\n")

src/rocprof_compute_soc/soc_gfx906.py

@@ -22,7 +22,7 @@
 # SOFTWARE.
 ##############################################################################el
 
-import os
+from pathlib import Path
 
 import config
 from rocprof_compute_soc.soc_base import OmniSoC_Base
@@ -34,11 +34,12 @@ def __init__(self, args, mspec):
         super().__init__(args, mspec)
         self.set_arch("gfx906")
         self.set_perfmon_dir(
-            os.path.join(
-                str(config.rocprof_compute_home),
-                "rocprof_compute_soc",
-                "profile_configs",
-                self.get_arch(),
+            str(
+                Path(str(config.rocprof_compute_home)).joinpath(
+                    "rocprof_compute_soc",
+                    "profile_configs",
+                    self.get_arch(),
+                )
             )
         )
         self.set_compatible_profilers(["rocprofv1", "rocscope"])