Skip to content
tests

misc comment #3174

Sign in to view logs

misc comment

misc comment #3174

Workflow file for this run

.github/workflows/tests.yml at f3e2173
name: tests
on:
push:
branches:
- '**'
tags-ignore:
- '**'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
# don't cancel on main/master/default
cancel-in-progress: ${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref }}
env:
PY_COLORS: "1"
BOT_NAME: wipacdevbot
BOT_EMAIL: [email protected]
CI_TEST_RUN_STDOUT_STDERR_DIR: /home/runner/work/skymap_scanner/testrun_outputs
REALTIME_EVENTS_DIR: /home/runner/work/skymap_scanner/skymap_scanner/tests/data/realtime_events
SKYSCAN_CACHE_DIR: /home/runner/work/skymap_scanner/skymap_scanner/cache
SKYSCAN_OUTPUT_DIR: /home/runner/work/skymap_scanner/skymap_scanner/output
SKYSCAN_DEBUG_DIR: /home/runner/work/skymap_scanner/skymap_scanner/debug
EWMS_PILOT_DUMP_SUBPROC_OUTPUT: False # get logs in "reco-icetray logs" step instead
EWMS_PILOT_STOP_LISTENING_ON_TASK_ERROR: False # allow a reco to fail (if it keeps failing on redelivery/ies, expect it to affect the scanner server's timeout eventually)
# see source tests/env-vars.sh
jobs:
#############################################################################
# PACKAGING & LINTERS
#############################################################################
py-versions:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.versions.outputs.matrix }}
steps:
- uses: actions/checkout@v3
- id: versions
uses: WIPACrepo/[email protected]
# flake8:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-python@v3
# - uses: WIPACrepo/[email protected]
mypy:
needs: [ py-versions ]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v3
with:
python-version: ${{ matrix.py3 }}
- uses: WIPACrepo/[email protected]
py-setup:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v3
with:
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
- uses: WIPACrepo/[email protected]
if: github.actor != ${{ env.BOT_NAME }} # no auto-updates for bots # should match all 'git_committer_name' uses
with:
git_committer_name: ${{ env.BOT_NAME }}
git_committer_email: ${{ env.BOT_EMAIL }}
base-keywords: IceCube
py-dependencies:
runs-on: ubuntu-latest
steps:
- uses: jlumbroso/free-disk-space@main # need space for images to build
with:
tool-cache: 'true'
- name: checkout
uses: actions/checkout@v3
with:
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
- uses: WIPACrepo/[email protected]
if: github.actor != ${{ env.BOT_NAME }} # no auto-updates for bots # should match all 'git_committer_name' uses
with:
git_committer_name: ${{ env.BOT_NAME }}
git_committer_email: ${{ env.BOT_EMAIL }}
dockerfile_nametags: Dockerfile:icecube/skymap_scanner:latest
#############################################################################
# TESTS
#############################################################################
test-build-docker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v3
with:
context: .
cache-from: type=gha
cache-to: type=gha,mode=min
file: Dockerfile
tags: icecube/skymap_scanner:local
test-run-singularity-dummy-reco:
needs: test-build-docker
runs-on: ubuntu-latest
env:
SKYSCAN_BROKER_ADDRESS: user1:password@localhost/test
# SKYSCAN_BROKER_AUTH: user1 # using this would override password in address
services:
rabbitmq:
# see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
image: bitnami/rabbitmq:3.11.15-debian-11-r0
env:
RABBITMQ_USERNAME: user1
RABBITMQ_PASSWORD: password
RABBITMQ_VHOST: test
BITNAMI_DEBUG: true
# Note: `--network` option is not supported.
options: >-
--name rabbitmq
--health-cmd "rabbitmqctl node_health_check"
--health-interval 5s
--health-timeout 5s
--health-retries 10
ports:
- 5672:5672
- 15672:15672
steps:
- uses: jlumbroso/free-disk-space@main # need space for mq broker and image
with:
tool-cache: 'true'
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v3
with:
context: .
cache-from: type=gha
# cache-to: type=gha,mode=min
file: Dockerfile
tags: icecube/skymap_scanner:local
load: true
- uses: eWaterCycle/setup-apptainer@v2
with:
apptainer-version: 1.3.2
- name: build singularity image
run: |
sudo singularity build skymap_scanner.sif docker-daemon://icecube/skymap_scanner:local
ls -lh skymap_scanner.sif
- name: run singularity container
run: |
source tests/env-vars.sh
mkdir $SKYSCAN_CACHE_DIR
mkdir $SKYSCAN_OUTPUT_DIR
# Launch Server
singularity run skymap_scanner.sif \
python -m skymap_scanner.server \
--reco-algo dummy \
--event-file $REALTIME_EVENTS_DIR/hese_event_01.json \
--cache-dir $SKYSCAN_CACHE_DIR \
--output-dir $SKYSCAN_OUTPUT_DIR \
--client-startup-json ./startup.json \
--nsides 1:0 \
--simulated-event \
&
./resources/launch_scripts/wait_for_file.sh ./startup.json $CLIENT_STARTER_WAIT_FOR_STARTUP_JSON
# Launch Clients
nclients=2
echo "Launching $nclients clients"
mkdir $SKYSCAN_DEBUG_DIR
export EWMS_PILOT_TASK_TIMEOUT=1800 # 30 mins
for i in $( seq 1 $nclients ); do
singularity run skymap_scanner.sif \
python -m skymap_scanner.client \
--client-startup-json ./startup.json \
--debug-directory $SKYSCAN_DEBUG_DIR \
&
echo -e "\tclient #$i launched"
done
wait -n # for server
for i in $( seq 1 $nclients ); do
wait -n # for client
done
- name: look at results file (.npz)
run: |
ls .
ls $SKYSCAN_OUTPUT_DIR
outfile=$(ls -d $SKYSCAN_OUTPUT_DIR/*.npz)
echo $outfile
- name: broker docker logs
if: always()
run: |
docker logs rabbitmq
test-run-nsides-thresholds-dummy:
needs: test-build-docker
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
nsides: [
"1:0",
"1:0 2:12",
"1:0 2:12 4:12"
]
predictive_scanning_threshold: [
1.0,
0.65,
]
env:
SKYSCAN_BROKER_ADDRESS: user1@localhost/test
SKYSCAN_BROKER_AUTH: password # using this would override password in address
services:
rabbitmq:
# see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
image: bitnami/rabbitmq:3.11.15-debian-11-r0
env:
RABBITMQ_USERNAME: user1
RABBITMQ_PASSWORD: password
RABBITMQ_VHOST: test
BITNAMI_DEBUG: true
# Note: `--network` option is not supported.
options: >-
--name rabbitmq
--health-cmd "rabbitmqctl node_health_check"
--health-interval 5s
--health-timeout 5s
--health-retries 10
ports:
- 5672:5672
- 15672:15672
steps:
- uses: jlumbroso/free-disk-space@main # need space for mq broker and image
with:
tool-cache: 'true'
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v3
with:
context: .
cache-from: type=gha
# cache-to: type=gha,mode=min
file: Dockerfile
tags: icecube/skymap_scanner:local
load: true
- name: run
timeout-minutes: 10 # on average max~=5min
run: |
set -x
source tests/env-vars.sh
export _RECO_ALGO=dummy
export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
export _NSIDES="${{ matrix.nsides }}"
export _PREDICTIVE_SCANNING_THRESHOLD=${{ matrix.predictive_scanning_threshold }}
cd ./resources/launch_scripts
./local-scan.sh 2 $CI_TEST_RUN_STDOUT_STDERR_DIR
- name: check no nsides skipped
run: |
ls $SKYSCAN_OUTPUT_DIR
# get newest run*.json
export outfile=$(find $SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n' {} + | sort | tail -1 | awk '{print $4}')
echo $outfile
python3 -c '
import json
import os
with open(os.getenv("outfile")) as f:
pydict = json.load(f)
nsides = "${{ matrix.nsides }}"
assert len(pydict) == nsides.count(":")
'
- name: central server stdout/stderr
if: always()
run: |
cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
- name: client worker \#1 stdout/stderr
if: always()
run: |
cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-1.out
- name: client worker \#2 stdout/stderr
if: always()
run: |
cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-2.out
- name: reco-icetray logs
if: always()
run: |
sudo apt install tree
tree $SKYSCAN_DEBUG_DIR
find $SKYSCAN_DEBUG_DIR -type f -not -name "*.i3" -not -name "*.pkl" -exec "more" {} + | cat # recursively cats with filenames (delimited by :::::::)
- name: broker docker logs
if: always()
run: |
docker logs rabbitmq
test-run-crash-dummy:
needs: test-build-docker
runs-on: ubuntu-latest
env:
SKYSCAN_BROKER_ADDRESS: user1@localhost/test
SKYSCAN_BROKER_AUTH: password # using this would override password in address
services:
rabbitmq:
# see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
image: bitnami/rabbitmq:3.11.15-debian-11-r0
env:
RABBITMQ_USERNAME: user1
RABBITMQ_PASSWORD: password
RABBITMQ_VHOST: test
BITNAMI_DEBUG: true
# Note: `--network` option is not supported.
options: >-
--name rabbitmq
--health-cmd "rabbitmqctl node_health_check"
--health-interval 5s
--health-timeout 5s
--health-retries 10
ports:
- 5672:5672
- 15672:15672
steps:
- uses: jlumbroso/free-disk-space@main # need space for mq broker and image
with:
tool-cache: 'true'
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v3
with:
context: .
cache-from: type=gha
# cache-to: type=gha,mode=min
file: Dockerfile
tags: icecube/skymap_scanner:local
load: true
- name: run
timeout-minutes: 10 # on average ~6min # yes, `timeout` is used below but this is insurance
run: |
set -x
source tests/env-vars.sh
export _RECO_ALGO=crash_dummy
export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/hese_event_01.json)
export _NSIDES="1:0 2:12 4:12"
export SKYSCAN_CRASH_DUMMY_PROBABILITY=0.75
export EWMS_PILOT_TASK_TIMEOUT=15
cd ./resources/launch_scripts
# since SKYSCAN_CRASH_DUMMY_PROBABILITY<1, this step will go forever
# so, stop it after some time and chek that it has stderrfiles
timeout 240 ./local-scan.sh 2 $CI_TEST_RUN_STDOUT_STDERR_DIR || true
- name: look at stderrfiles
run: |
set -x
# check for fails
stderrfiles=$(find $SKYSCAN_DEBUG_DIR/ -name "stderrfile" -type f -exec sh -c "tail -1 {} | sed -e 's/^.*DEBUG //p' " \; | sort | uniq -c)
echo $stderrfiles
if [ -z "${stderrfiles}" ]; then
exit 1
fi
- name: central server stdout/stderr
if: always()
run: |
cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
- name: client worker \#1 stdout/stderr
if: always()
run: |
cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-1.out
- name: client worker \#2 stdout/stderr
if: always()
run: |
cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-2.out
test-run-realistic:
needs: test-build-docker
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
reco_algo: [
millipede_original,
millipede_wilks,
splinempe,
splinempe_pointed
]
eventfile: [
hese_event_01.json,
run00136766-evt000007637140-GOLD.pkl,
run00136662-evt000035405932-BRONZE.pkl,
138632_31747601.json
]
exclude:
# splinempe should not run on HESE
- reco_algo: splinempe
eventfile: hese_event_01.json
- reco_algo: splinempe_pointed
eventfile: hese_event_01.json
env:
SKYSCAN_BROKER_ADDRESS: user1@localhost/test
SKYSCAN_BROKER_AUTH: password # using this would override password in address
services:
rabbitmq:
# see image.tag -> https://github.com/Observation-Management-Service/path-kubernetes/blob/main/helm-values-rabbitmq-bitnami.yaml (see https://artifacthub.io/packages/helm/bitnami/rabbitmq/11.14.3)
image: bitnami/rabbitmq:3.11.15-debian-11-r0
env:
RABBITMQ_USERNAME: user1
RABBITMQ_PASSWORD: password
RABBITMQ_VHOST: test
BITNAMI_DEBUG: true
# Note: `--network` option is not supported.
options: >-
--name rabbitmq
--health-cmd "rabbitmqctl node_health_check"
--health-interval 5s
--health-timeout 5s
--health-retries 10
ports:
- 5672:5672
- 15672:15672
steps:
- uses: jlumbroso/free-disk-space@main # need space for mq broker and image
with:
tool-cache: 'true'
- uses: actionhippie/swap-space@v1
with:
size: 10G
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v3
with:
context: .
cache-from: type=gha
# cache-to: type=gha,mode=min
file: Dockerfile
tags: icecube/skymap_scanner:local
load: true
- name: run
timeout-minutes: 35 # on average max~=26min
run: |
set -x
lscpu
python3 --version
source tests/env-vars.sh
export _RECO_ALGO=${{ matrix.reco_algo }}
export _EVENTS_FILE=$(realpath $REALTIME_EVENTS_DIR/${{ matrix.eventfile }})
export _NSIDES="1:0"
cd ./resources/launch_scripts
./local-scan.sh 2 $CI_TEST_RUN_STDOUT_STDERR_DIR
- name: test output against known result (.json)
run: |
ls $SKYSCAN_OUTPUT_DIR
# get newest run*.json
outfile=$(find $SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n' {} + | sort | tail -1 | awk '{print $4}')
echo $outfile
cat $outfile
pip install . # don't need icecube, so no docker container needed
python tests/compare_scan_results.py \
--actual $outfile \
--expected tests/data/results_json/${{ matrix.reco_algo }}/$(basename $outfile) \
--assert \
|| (cat $(ls *.diff.json) && false)
- name: central server stdout/stderr
if: always()
run: |
cat $CI_TEST_RUN_STDOUT_STDERR_DIR/server.out
- name: client worker \#1 stdout/stderr
if: always()
run: |
cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-1.out
- name: client worker \#2 stdout/stderr
if: always()
run: |
cat $CI_TEST_RUN_STDOUT_STDERR_DIR/client-2.out
- name: reco-icetray logs
if: always()
run: |
sudo apt install tree
tree $SKYSCAN_DEBUG_DIR
find $SKYSCAN_DEBUG_DIR -type f -not -name "*.i3" -not -name "*.pkl" -exec "more" {} + | cat # recursively cats with filenames (delimited by :::::::)
- name: broker docker logs
if: always()
run: |
docker logs rabbitmq
test-file-staging:
needs: test-build-docker
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v3
with:
context: .
cache-from: type=gha
# cache-to: type=gha,mode=min
file: Dockerfile
tags: icecube/skymap_scanner:local
load: true
- name: run
run: |
docker run --rm -i \
icecube/skymap_scanner:local \
python tests/file_staging.py
test-run-single-pixel:
needs: test-build-docker
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
dir: [
"BRONZE",
"GOLD",
"JSON",
]
reco_algo:
[
millipede_original,
millipede_wilks,
splinempe,
splinempe_pointed
]
exclude:
# splinempe should not run on HESE
- reco_algo: splinempe
dir: "JSON"
- reco_algo: splinempe_pointed
dir: "JSON"
steps:
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- uses: docker/build-push-action@v3
with:
context: .
cache-from: type=gha
# cache-to: type=gha,mode=min
file: Dockerfile
tags: icecube/skymap_scanner:local
load: true
- name: run
timeout-minutes: 15 # on average max~=7min
run: |
set -e
source tests/env-vars.sh
# grab the GCDQp_packet key and throw into a file
jq '.GCDQp_packet' \
tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}/startup.json > \
tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}//GCDQp_packet.json
# run reco
docker run --network="host" --rm -i \
--shm-size=6gb \
--mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \
--env PY_COLORS=1 \
$(env | grep '^SKYSCAN_' | awk '$0="--env "$0') \
icecube/skymap_scanner:local \
python -m skymap_scanner.client.reco_icetray \
--infile /local/test-data/in.json \
--gcdqp-packet-json /local/test-data/GCDQp_packet.json \
--baseline-gcd-file $(jq -r '.baseline_GCD_file' tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}/startup.json) \
--outfile /local/test-data/out-actual.json
- name: test output against known result
run: |
ls tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}
# need icecube for depickling, so docker container needed
docker run --network="host" --rm -i \
--shm-size=6gb \
--mount type=bind,source=$(readlink -f tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}),target=/local/test-data \
--env PY_COLORS=1 \
$(env | grep '^SKYSCAN_' | awk '$0="--env "$0') \
icecube/skymap_scanner:local \
python tests/compare_reco_pixel_single.py \
--actual /local/test-data/out-actual.json \
--expected /local/test-data/out.json \
--diff-out-dir /local/test-data/ \
--assert \
|| (cat $(ls tests/data/reco_pixel_single/${{ matrix.reco_algo }}/${{ matrix.dir }}/*.diff.json) && false)
#############################################################################
# GITHUB RELEASE
#############################################################################
release:
# only run on main/master/default
if: format('refs/heads/{0}', github.event.repository.default_branch) == github.ref
# have to wait for tests so python-semantic-release can push (branch protection on main don't work w/ the bot)
needs: [
mypy,
py-setup,
py-dependencies,
test-build-docker,
test-run-singularity-dummy-reco,
test-file-staging,
test-run-nsides-thresholds-dummy,
test-run-crash-dummy,
test-run-realistic,
test-run-single-pixel
]
runs-on: ubuntu-latest
concurrency: release
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
- name: Python Semantic Release
uses: python-semantic-release/[email protected]
with:
git_committer_name: ${{ env.BOT_NAME }}
git_committer_email: ${{ env.BOT_EMAIL }}
github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
# repository_username: __token__
# repository_password: ${{ secrets.PYPI_TOKEN }}