Skip to content

Commit

Permalink
chore: Port wf4 over (#33)
Browse files Browse the repository at this point in the history
* chore: Bump clang-format on ci
  • Loading branch information
patrickkenney9801 authored Mar 29, 2024
1 parent cfc7228 commit dcc8625
Show file tree
Hide file tree
Showing 32 changed files with 3,183 additions and 6 deletions.
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ repos:
rev: v2.2.6
hooks:
- id: codespell
args: [-L, orgin]
- repo: https://github.com/pocc/pre-commit-hooks
rev: v1.3.5
hooks:
Expand Down
13 changes: 11 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ project(
LANGUAGES C CXX
)

if(PROJECT_IS_TOP_LEVEL)
set(CMAKE_CXX_FLAGS_COVERAGE "-g -O0 --coverage -fprofile-abs-path")
set(CMAKE_EXE_LINKER_FLAGS_COVERAGE "--coverage")
set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE "--coverage")
set(CMAKE_MODULE_LINKER_FLAGS_COVERAGE "--coverage")
endif()

# helper functions
include(cmake/GaloisCompilerOptions.cmake)

Expand Down Expand Up @@ -37,6 +44,7 @@ if(PROJECT_IS_TOP_LEVEL)
endif()

set(GALOIS_ENABLE_DIST ON CACHE BOOL "" FORCE)
set(GALOIS_ENABLE_WMD ON)

set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_STANDARD 17)
Expand All @@ -57,11 +65,12 @@ set(BUILD_TESTING OFF)
add_subdirectory(galois EXCLUDE_FROM_ALL)
set(BUILD_TESTING "${BUILD_TESTING_SAVED}")

set(PHMAP_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/galois/external/parallel-hashmap)
set(PCG_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/galois/external/pcg-cpp/include)
set(PHMAP_DIRECTORY ${Galois_SOURCE_DIR}/external/parallel-hashmap)
set(PCG_DIRECTORY ${Galois_SOURCE_DIR}/external/pcg-cpp/include)

add_subdirectory(microbench)
add_subdirectory(scripts)
add_subdirectory(wf4)

# tests
option(BUILD_TESTING "Build tests." ON)
Expand Down
5 changes: 3 additions & 2 deletions Dockerfile.dev
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ RUN if [ "${IS_CI}" != "true" ] ; then \
doxygen \
texlive-latex-extra \
texlive-font-utils \
gcovr \
&& apt clean \
&& update-locale; fi

Expand All @@ -121,12 +122,12 @@ USER ${UNAME}
WORKDIR /home/${UNAME}
ENV BUILD_DIR=${BUILD_DIR}

RUN pip3 install compdb pre-commit cpplint "clang-format>=12.0.1"
RUN pip3 install compdb pre-commit cpplint "clang-format>=14.0.0,<17.0.0"

RUN echo "export SRC_DIR=${SRC_DIR}" >> /home/${UNAME}/.profile
RUN echo "export BUILD_DIR=${BUILD_DIR}" >> /home/${UNAME}/.profile
RUN echo "touch ${SRC_DIR}/env-docker.sh" >> /home/${UNAME}/.profile
RUN echo "bash ${SRC_DIR}/env-docker.sh" >> /home/${UNAME}/.profile
RUN echo "source ${SRC_DIR}/env-docker.sh" >> /home/${UNAME}/.profile

RUN echo "PATH=/home/${UNAME}/.local/bin/:\$PATH" >> /home/${UNAME}/.zshenv
WORKDIR ${SRC_DIR}
Expand Down
66 changes: 65 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ CONTAINER_BUILD_DIR ?= /galois/build
CONTAINER_WORKDIR ?= ${CONTAINER_SRC_DIR}
CONTAINER_CONTEXT ?= default
CONTAINER_OPTS ?=
CONTAINER_CMD ?= setarch `uname -m` -R bash -l
CONTAINER_CMD ?= bash -l
INTERACTIVE ?= i

BUILD_TYPE ?= Release
Expand All @@ -33,6 +33,19 @@ GALOIS_CONTAINER_FLAGS ?=
GALOIS_BUILD_TOOL ?= 'Unix Makefiles'
GALOIS_CCACHE_DIR ?= ${SRC_DIR}/.ccache

# Variables for SonarQube
PROFILE ?= observability
CONTAINER_SONAR_DIR ?= /galois/build-sonar
CONTAINER_SONAR_CMD ?= ${CONTAINER_CMD} -c "make sonar-scan"
SONAR_EXE ?= sonar-scanner
SONAR_PROJECT_VERSION ?= -Dsonar.projectVersion=0.1.0
define sonar_ip
$(shell kubectl --context ${PROFILE} get nodes --namespace sonarqube -o jsonpath="{.items[0].status.addresses[0].address}")
endef
define sonar_port
$(shell kubectl --context ${PROFILE} get --namespace sonarqube -o jsonpath="{.spec.ports[0].nodePort}" services sonarqube-sonarqube)
endef

dependencies: dependencies-asdf

dependencies-asdf:
Expand Down Expand Up @@ -129,6 +142,57 @@ test:

run-tests: test

gcovr:
@echo "Should be run outside a container"
@echo file://${SRC_DIR}/build-sonar/coverage/html
@python -mwebbrowser file://${SRC_DIR}/build-sonar/coverage/html

sonar:
@echo "Should be run outside a container"
@echo http://$(call sonar_ip):$(call sonar_port)
@python -mwebbrowser http://$(call sonar_ip):$(call sonar_port)


run-gcovr:
COVERAGE_CMD='echo done' ${MAKE} run-coverage

run-sonar:
COVERAGE_CMD="${SONAR_EXE} -Dsonar.host.url=http://$(call sonar_ip):$(call sonar_port) ${SONAR_PROJECT_VERSION}" ${MAKE} run-coverage

run-coverage:
@docker --context ${CONTAINER_CONTEXT} run --rm \
-v ${SRC_DIR}/:${CONTAINER_SRC_DIR} \
${GALOIS_CONTAINER_MOUNTS} \
${GALOIS_CONTAINER_ENV} \
-e=COVERAGE_CMD="${COVERAGE_CMD}" \
--privileged \
--net=host \
--workdir=${CONTAINER_WORKDIR} ${CONTAINER_OPTS} -${INTERACTIVE}t \
${IMAGE_NAME}:${VERSION} \
${CONTAINER_SONAR_CMD}

sonar-scan:
@mkdir -p ${CONTAINER_SONAR_DIR}/coverage
@cmake \
-S ${SRC_DIR} \
-B ${CONTAINER_SONAR_DIR} \
-DCMAKE_C_COMPILER=gcc-12 \
-DCMAKE_CXX_COMPILER=g++-12 \
-DCMAKE_BUILD_TYPE=Coverage \
-DBUILD_TESTING=ON
@cd ${CONTAINER_SONAR_DIR} && \
make -j8 && \
ctest --verbose --output-junit junit.xml
@cd ${CONTAINER_SONAR_DIR} && \
gcovr \
--gcov-executable gcov-12 \
--root .. \
--xml-pretty --xml cobertura.xml \
--sonarqube sonarqube.xml \
--html-details coverage/html \
-j8 .
@${COVERAGE_CMD}

# this command is slow since hooks are not stored in the container image
# this is mostly for CI use
docker-pre-commit:
Expand Down
131 changes: 131 additions & 0 deletions scripts/wf4/extract.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: BSD-2-Clause
# Copyright (c) 2023. University of Texas at Austin. All rights reserved.

##### Inputs #####
# hosts = [8, 16, 32, 64]
# scales = ["31", "31", "31", "31"]
hosts = [8, 16, 32, 16, 32, 8, 16, 32]
scales = ["1", "1", "1", "2", "2", "3", "3", "3"]



##### Parameters #####
data_size = 8
id_size = 8
addr_size = 8



##### Processing #####
for i, host in enumerate(hosts):
scale = scales[i]

local_read = []
master_read = []
master_write = []
mirror_read = []
mirror_write = []
dirty_mirror_to_remote = []

for host_id in range(host):
local_read.append([])
master_read.append([])
master_write.append([])
mirror_read.append([])
mirror_write.append([])
dirty_mirror_to_remote.append([])

round_curr = -1
round_total = 0

filename = "scale" + scale + "_" + str(host) + "procs_id" + str(host_id)
f = open(filename, "r")
lines = f.readlines()

for row in lines:
if row.find('##### Round') != -1:
row_split = row.strip().split()
round_curr += 1
round_total += 1

local_read[host_id].append([])
master_read[host_id].append([])
master_write[host_id].append([])
mirror_read[host_id].append([])
mirror_write[host_id].append([])
dirty_mirror_to_remote[host_id].append([-1 for i in range(host)])
elif row.find('local read (stream)') != -1:
row_split = row.strip().split()
num = row_split[5]
local_read[host_id][round_curr] = int(num)
elif row.find('master reads') != -1:
row_split = row.strip().split()
num = row_split[4]
master_read[host_id][round_curr] = int(num)
elif row.find('master writes') != -1:
row_split = row.strip().split()
num = row_split[4]
master_write[host_id][round_curr] = int(num)
elif row.find('mirror reads') != -1:
row_split = row.strip().split()
num = row_split[4]
mirror_read[host_id][round_curr] = int(num)
elif row.find('mirror writes') != -1:
row_split = row.strip().split()
num = row_split[4]
mirror_write[host_id][round_curr] = int(num)
elif row.find('remote communication for') != -1:
row_split = row.strip().split()
to_host = int(row_split[6][:-1])
num = row_split[7]
dirty_mirror_to_remote[host_id][round_curr][to_host] = int(num)

f.close()

filename = "GAL_WF4_" + str(host) + "_1_" + scale + ".stats"
f = open(filename, "w")

for round_num in range(round_total):
f.write("BSP " + str(2*round_num) + "\n")

for src in range(host):
for dst in range(host):
if (src == dst): # only local in compute phase`
stream_read = local_read[src][round_num]
stream_read_bytes = stream_read * addr_size

random_read = master_read[src][round_num] + mirror_read[src][round_num]
random_read_bytes = random_read * data_size

random_write = master_write[src][round_num] + mirror_write[src][round_num]
random_write_bytes = random_write * data_size

if stream_read != 0 or random_read != 0 or random_write != 0:
f.write("# " + str(src) + " " + str(dst) + "\n")

if stream_read != 0:
f.write("STR RD " + str(stream_read) + " " + str(stream_read_bytes) + "\n")

if random_read != 0:
f.write("RND RD " + str(random_read) + " " + str(random_read_bytes) + "\n")

if random_write != 0:
f.write("RND WR " + str(random_write) + " " + str(random_write_bytes) + "\n")

f.write("\n")

f.write("BSP " + str(2*round_num+1) + "\n")

for src in range(host):
for dst in range(host):
if (src != dst): # only remote in communication phase
random_write = dirty_mirror_to_remote[src][round_num][dst]
random_write_bytes = random_write * data_size

if random_write != 0:
f.write("# " + str(src) + " " + str(dst) + "\n")
f.write("RND RMW " + str(random_write) + " " + str(random_write_bytes) + "\n")

f.write("\n")

f.close()
Loading

0 comments on commit dcc8625

Please sign in to comment.