update TACC readme instructions to speed up LLVM build (#46)

README_TACC.md

@@ -86,8 +86,8 @@ module load impi/19.0.9
 module load python3/3.9.7
 module load boost-mpi/1.72
 
-export LLVM_DIR="$SCRATCH/llvm-project/build/cmake/modules/CMakeFiles/"
-export fmt_DIR="$SCRATCH/fmt/build/"
+export LLVM_DIR="$WORK/llvm-project/build/cmake/modules/CMakeFiles/"
+export fmt_DIR="$WORK/fmt/build/"
 ```
 
 ## Building Dependencies
@@ -104,36 +104,29 @@ files then you need to clone and download your dependencies manually.  For
 Galois you need [llvm](https://github.com/llvm/llvm-project) and
 [fmt](https://github.com/fmtlib/fmt).
 
-*Note:* `llvm` is a very heavyweight library that will take hours to pull
-and build.  Galois really only uses this library for parsing arguments in
-programs and it would probably be faster to remove Galois' dependency on
-`llvm` and instead use `boost` to parse arguments than to build `llvm`.
-But it is easier to just build `llvm` and burn a few hours.
-
 The following will build `fmt` properly:
 
 ```shell
-cd $SCRATCH
+cd $WORK
 git clone https://github.com/fmtlib/fmt
 cd fmt
 cmake -B build
 cd build
 make -j4
 ```
 
-If you are still reading this I can see you decided to build `llvm` anyways,
-all I can say is good luck. `llvm` needs to be built with certain `cmake`
-flags in order to work with Galois, the following sequence will build `llvm`
-properly:
+`llvm` needs to be built with certain `cmake` flags in order to work with
+Galois, the following sequence will build `llvm` properly:
 
 ```shell
-cd $SCRATCH
+idev -m 120 # enter an interactive dev machine to use more threads
+cd $WORK
 git clone https://github.com/llvm/llvm-project
 cd llvm-project
 cmake -S llvm -B build -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_RTTI=ON \
   -DLLVM_ENABLE_ZSTD=OFF -DLLVM_ENABLE_ZLIB=OFF -DLLVM_ENABLE_TERMINFO=OFF
 cd build
-make -j4
+make -j
 ```
 
 ## Building Galois
@@ -142,7 +135,7 @@ After you have built Galois' dependencies the process is more familiar.
 
 The important bits are to set env vars to tell `cmake` where to find your
 prebuilt dependencies.  For example:
-`export LLVM_DIR="$SCRATCH/llvm-project/build/cmake/modules/CMakeFiles/"`.
+`export LLVM_DIR="$WORK/llvm-project/build/cmake/modules/CMakeFiles/"`.
 It is recommended to define the env vars as part of your TACC environment
 script for ease of use.
 
@@ -151,10 +144,11 @@ version or compiler, then `cmake` will cache this choice.  In order to rebuild
 with a different library version or compiler you must remove the file
 `build/CMakeCache.txt` and then rebuild with the proper settings.
 
+<!-- note: meyer did not have this issue: -->
 Another caviat is with `llvm`: my builds were unable to find the header files
 for `llvm` despite `cmake` finding the dependency.  In order to resolve this
 I used the following `cmake` hack to include them directly:
-`target_include_directories(<exe> PRIVATE $SCRATCH/llvm-project/llvm/include)`.
+`target_include_directories(<exe> PRIVATE $WORK/llvm-project/llvm/include)`.
 Note that this does not break builds for different machines since `cmake` will
 simply ignore paths that do not exist on the current filesystem.
 

include/scea/graph/csr.hpp

@@ -15,7 +15,8 @@ namespace scea::graph {
 
 class LC_CSR : public MutableGraph {
 private:
-  using GraphTy = galois::graphs::LC_CSR_64_Graph<void, void>;
+  using GraphTy =
+      galois::graphs::LC_CSR_64_Graph<void, void>::with_no_lockable<true>::type;
   std::vector<std::vector<uint64_t>> m_adj;
   std::unique_ptr<GraphTy> m_graph;
   std::atomic<size_t> num_edges = ATOMIC_VAR_INIT(0);
@@ -47,6 +48,7 @@ class LC_CSR : public MutableGraph {
   }
 
   void sort_edges(uint64_t src) override { m_graph->sortEdgesByDst(src); }
+
   bool find_edge_sorted(uint64_t src, uint64_t dst) override {
     return m_graph->findEdgeSortedByDst(src, dst) != m_graph->edge_end(src);
   }

include/scea/stats.hpp

@@ -137,15 +137,15 @@ class ScopeBenchmarker {
     uint64_t minorPageFaults = minorPageFaultsEvent.readValue();
     uint64_t majorPageFaults = majorPageFaultsEvent.readValue();
 
-    std::cout << "Benchmark results for " << scopeName << ":\n";
-    std::cout << "Duration: " << timer.getDurationNano() << " nanoseconds\n";
-    std::cout << "Max RSS: " << max_rss << " KB\n";
-    std::cout << "Cache Miss Rate: "
-              << (static_cast<double>(cacheMisses) / cacheReferences) * 100
-              << "%\n";
-    std::cout << "Instructions: " << instructions << "\n";
-    std::cout << "Minor Page Faults: " << minorPageFaults << "\n";
-    std::cout << "Major Page Faults: " << majorPageFaults << "\n";
+    std::cout << "Benchmark results for " << scopeName << ":" << std::endl //
+              << "Duration: " << timer.getDurationNano() << " nanoseconds"
+              << std::endl                                             //
+              << "Max RSS: " << max_rss << " KB" << std::endl          //
+              << "Cache Misses: " << cacheMisses << std::endl          //
+              << "Cache References: " << cacheReferences << std::endl  //
+              << "Instructions: " << instructions << std::endl         //
+              << "Minor Page Faults: " << minorPageFaults << std::endl //
+              << "Major Page Faults: " << majorPageFaults << std::endl;
   }
 };
 

scripts/tacc_edit_scalability_friendster.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright (c) 2023. University of Texas at Austin. All rights reserved.
+
+GRAPH_PATH="${GRAPH_PATH:-$SCRATCH/graphs/friendster_randomized_25.txt}"
+GRAPH_NUM_VERTICES="${GRAPH_NUM_VERTICES:-124836180}"
+
+# required sbatch parameters
+
+# note that TIME is in the format H:MM:SS
+
+# note that the bash paradigm `HOSTS="${HOSTS:-<default>}`
+# means: set the env var HOSTS equal to $HOSTS if HOSTS is
+# already set, if it is not set then set `HOSTS=default`
+HOSTS="${HOSTS:-1}"
+PROCS="${PROCS:-1}"
+TIME="${TIME:-1:00:00}"
+QUEUE="${QUEUE:-normal}"
+JOBS="${JOBS:-edit-scalability}"
+
+ENV=${WORK}/scea/graph-log-sketch/scripts/tacc_env.sh
+
+# These variables are not necessary but recommended for ease of use
+# The data directory is helpful for storing outputs and is recommended
+# but not necessary
+BUILD="${BUILD:-$WORK/scea/graph-log-sketch/build}"
+DATA="${DATA:-$SCRATCH/scea/graph-log-sketch/data/friendster}"
+
+# Create the data directory, if it does not yet exist:
+mkdir -p $DATA
+
+# print statements to validate input, can and should
+# be extended with application parameters
+echo $HOSTS
+echo $PROCS
+echo $THREADS
+echo $TIME
+echo $DATA
+echo $QUEUE
+echo $JOBN
+
+for algo in bfs tc; do
+	mkdir -p "${DATA}/$algo"
+	for nthreads in 8 16 32 64; do
+		for graph in lscsr lccsr adj; do
+			# JOBN should be parameterized with application parameters as well
+			# possibly time as well time prevent conflicts and overwriting
+			JOBN=${DATA}/$algo/${JOBS}_t=${nthreads}_g=${graph}
+			echo "Submitting job: $JOBN"
+
+			# start of job that runs on the supercomputer
+			sbatch <<-EOT
+				#!/bin/bash
+
+				# special arguments passed to sbatch here instead of by command line
+				# the mail arguments are optional and are just there to send you email
+				# notifications when your jobs are scheduled and complete
+				#SBATCH -J ${JOBN}
+				#SBATCH -o ${JOBN}.out
+				#SBATCH -e ${JOBN}.err
+				#SBATCH -t ${TIME}
+				#SBATCH -N ${HOSTS}
+				#SBATCH -n ${PROCS}
+				#SBATCH --mail-type=none
+				#SBATCH [email protected]
+				#SBATCH -p ${QUEUE}
+
+				# ensure the proper runtime environment is set
+				module purge
+				. ${ENV}
+
+				# actually run the equivalent of $\(mpirun) the $\(--) ensures arguments
+				# are passed to your executable and not $\(ibrun)
+				ibrun -- ${BUILD}/microbench/edit-scalability \
+				                --algo $algo \
+				                --bfs-src 101 \
+				                --graph $graph \
+				                --ingest-threads $nthreads \
+				                --algo-threads $nthreads \
+				                --input-file $GRAPH_PATH \
+				                --num-vertices $GRAPH_NUM_VERTICES
+			EOT
+		done
+	done
+done

scripts/tacc_edit_scalability_rmat18.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright (c) 2023. University of Texas at Austin. All rights reserved.
+
+GRAPH_PATH="${GRAPH_PATH:-$SCRATCH/graphs/rmat18_nV262144_randomized_20.txt}"
+GRAPH_NUM_VERTICES="${GRAPH_NUM_VERTICES:-262145}"
+
+# required sbatch parameters
+
+# note that TIME is in the format H:MM:SS
+
+# note that the bash paradigm `HOSTS="${HOSTS:-<default>}`
+# means: set the env var HOSTS equal to $HOSTS if HOSTS is
+# already set, if it is not set then set `HOSTS=default`
+HOSTS="${HOSTS:-1}"
+PROCS="${PROCS:-1}"
+TIME="${TIME:-1:00:00}"
+QUEUE="${QUEUE:-normal}"
+JOBS="${JOBS:-edit-scalability-rmat18}"
+
+ENV=${WORK}/scea/graph-log-sketch/scripts/tacc_env.sh
+
+# These variables are not necessary but recommended for ease of use
+# The data directory is helpful for storing outputs and is recommended
+# but not necessary
+BUILD="${BUILD:-$WORK/scea/graph-log-sketch/build}"
+DATA="${DATA:-$SCRATCH/scea/graph-log-sketch/data/rmat18}"
+
+# Create the data directory, if it does not yet exist:
+mkdir -p $DATA
+
+# print statements to validate input, can and should
+# be extended with application parameters
+echo $HOSTS
+echo $PROCS
+echo $THREADS
+echo $TIME
+echo $DATA
+echo $QUEUE
+echo $JOBN
+
+for algo in bfs tc; do
+	mkdir -p "${DATA}/$algo"
+	for nthreads in 8 16 32 64; do
+		for graph in lscsr lccsr adj; do
+			# JOBN should be parameterized with application parameters as well
+			# possibly time as well time prevent conflicts and overwriting
+			JOBN=${DATA}/$algo/${JOBS}_t=${nthreads}_g=${graph}
+			echo "Submitting job: $JOBN"
+
+			# start of job that runs on the supercomputer
+			sbatch <<-EOT
+				#!/bin/bash
+
+				# special arguments passed to sbatch here instead of by command line
+				# the mail arguments are optional and are just there to send you email
+				# notifications when your jobs are scheduled and complete
+				#SBATCH -J ${JOBN}
+				#SBATCH -o ${JOBN}.out
+				#SBATCH -e ${JOBN}.err
+				#SBATCH -t ${TIME}
+				#SBATCH -N ${HOSTS}
+				#SBATCH -n ${PROCS}
+				#SBATCH --mail-type=none
+				#SBATCH [email protected]
+				#SBATCH -p ${QUEUE}
+
+				# ensure the proper runtime environment is set
+				module purge
+				. ${ENV}
+
+				# actually run the equivalent of $\(mpirun) the $\(--) ensures arguments
+				# are passed to your executable and not $\(ibrun)
+				ibrun -- ${BUILD}/microbench/edit-scalability \
+				                --algo $algo \
+				                --bfs-src 101 \
+				                --graph $graph \
+				                --ingest-threads $nthreads \
+				                --algo-threads $nthreads \
+				                --input-file $GRAPH_PATH \
+				                --num-vertices $GRAPH_NUM_VERTICES
+			EOT
+		done
+	done
+done

scripts/tacc_env.sh

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright (c) 2023. University of Texas at Austin. All rights reserved.
+
+module load intel/19.1.1
+module load impi/19.0.9
+module load python3/3.9.7
+module load boost-mpi/1.72
+
+export LLVM_DIR="$WORK/llvm-project/build/cmake/modules/CMakeFiles/"
+export fmt_DIR="$WORK/fmt/build/"

scripts/tacc_to_csv.py

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: BSD-2-Clause
+# Copyright (c) 2023. University of Texas at Austin. All rights reserved.
+
+import os
+import re
+from pathlib import Path
+import pandas as pd
+
+data_dir = Path(os.environ["SCRATCH"]) / "scea" / "graph-log-sketch" / "data"
+datasets = os.listdir(data_dir)
+print("datasets:", " ".join(datasets))
+
+df = pd.DataFrame(columns=["Dataset", "Algorithm", "Graph", "Threads", "Batch", "Stage", "Duration (ns)", "Max RSS (KB)", "Cache Misses", "Cache References", "Instructions"])
+
+
+for dataset in datasets:
+    print("processing", dataset)
+    dataset_dir = data_dir / dataset
+    algos = os.listdir(dataset_dir)
+    print("> algos:", " ".join(algos))
+
+    for algo in algos:
+        algo_dir = dataset_dir / algo
+        for file in os.listdir(algo_dir):
+            if file.endswith(".out"):
+                graph = re.search("g=(lscsr|adj|lccsr)", file).group(1)
+                threads = re.search("t=(\d+)", file).group(1)
+                print("> > processing", algo_dir / file)
+                with open(algo_dir / file) as f:
+                    contents = f.read()
+                    for m in re.findall("Benchmark results for (Ingestion|Post-ingest|Algorithm) for Batch (\d+):\nDuration: (\d+) nanoseconds\nMax RSS: (\d+) KB\nCache Misses: (\d+)\nCache References: (\d+)\nInstructions: (\d+)", contents, re.MULTILINE):
+                        stage, batch, duration, max_rss, cache_misses, cache_references, instructions = m
+                        df = df.append(
+                            {
+                                "Dataset": dataset,
+                                "Algorithm": algo,
+                                "Graph": graph,
+                                "Threads": threads,
+                                "Batch": batch,
+                                "Stage": stage,
+                                "Duration (ns)": duration,
+                                "Max RSS (KB)": max_rss,
+                                "Cache Misses": cache_misses,
+                                "Cache References": cache_references,
+                                "Instructions": instructions,
+                            },
+                            ignore_index=True
+                        )
+
+df.to_csv('data.csv', index=False)