diff --git a/.github/scripts/create_readonly_utilix_config.sh b/.github/scripts/create_readonly_utilix_config.sh new file mode 100644 index 00000000..cb1ea03e --- /dev/null +++ b/.github/scripts/create_readonly_utilix_config.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +if [ ! -z "$RUNDB_API_URL" ] +then +cat > $HOME/.xenon_config < these are the "normal" tests and should be run for all +# python versions +# - Coveralls -> this is to see if we are covering all our lines of +# code with our tests. The results get uploaded to +# coveralls.io/github/XENONnT/fuse + +name: Test package + +# Trigger this code when a new release is published +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + runs-on: ${{ matrix.os }} + env: + HAVE_ACCESS_TO_SECRETS: ${{ secrets.RUNDB_API_URL }} + strategy: + fail-fast: false + matrix: + os: [ "ubuntu-latest" ] + python-version: [ "3.9", "3.10" ] + test: [ 'coveralls', 'pytest' ] + + steps: + # Setup and installation + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Install dependencies + # following https://github.com/NESTCollaboration/nestpy/blob/master/README.md + run: | + python -m pip install --upgrade pip + python -m pip install pytest coverage coveralls + git clone https://github.com/NESTCollaboration/nestpy.git + cd nestpy + git submodule update --init --recursive + pip install . + cd .. + rm -rf nestpy + + - name: Start MongoDB + uses: supercharge/mongodb-github-action@1.10.0 + with: + mongodb-version: 4.4.1 + + - name: patch utilix file + # Secrets and required files + # Patch this file if we want to have access to the database + run: bash .github/scripts/create_readonly_utilix_config.sh + env: + # RunDB + RUNDB_API_URL: ${{ secrets.RUNDB_API_URL }} + RUNDB_API_USER_READONLY: ${{ secrets.RUNDB_API_USER_READONLY }} + RUNDB_API_PASSWORD_READONLY: ${{ secrets.RUNDB_API_PASSWORD_READONLY}} + PYMONGO_URL: ${{ secrets.PYMONGO_URL }} + PYMONGO_USER: ${{ secrets.PYMONGO_USER }} + PYMONGO_PASSWORD: ${{ secrets.PYMONGO_PASSWORD }} + PYMONGO_DATABASE: ${{ secrets.PYMONGO_DATABASE }} + # SCADA + SCADA_URL: ${{ secrets.SCADA_URL }} + SCADA_VALUE_URL: ${{ secrets.SCADA_VALUE_URL }} + SCADA_USER: ${{ secrets.SCADA_USER }} + SCADA_LOGIN_URL: ${{ secrets.SCADA_LOGIN_URL }} + SCADA_PWD: ${{ secrets.SCADA_PWD }} + + - name: Install fuse + run: | + pip install . + + - name: Test package + # This is running a normal test + env: + TEST_MONGO_URI: 'mongodb://localhost:27017/' + run: | + coverage run --source=fuse -m pytest --durations 0 + coverage report + + - name: Coveralls + # Make the coverage report and upload + env: + TEST_MONGO_URI: 'mongodb://localhost:27017/' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: matrix.test == 'coveralls' && env.HAVE_ACCESS_TO_SECRETS != null + run: | + coverage run --source=fuse -m pytest -v + coveralls --service=github + + - name: goodbye + run: echo "tests done, bye bye" diff --git a/.gitignore b/.gitignore index 806c5319..d3b7d14f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ - -fuse/__pycache__/* +__pycache__ fuse.egg-info/* .eggs/* build/* -.DS_Store .vscode/* docs/build/* -tests/resource_cache/* \ No newline at end of file +resource_cache +.coverage +.hypothesis +.DS_Store diff --git a/fuse/plugins/detector_physics/s2_photon_propagation.py b/fuse/plugins/detector_physics/s2_photon_propagation.py index eafdb67a..4fcbd03b 100644 --- a/fuse/plugins/detector_physics/s2_photon_propagation.py +++ b/fuse/plugins/detector_physics/s2_photon_propagation.py @@ -343,8 +343,10 @@ def compute(self, individual_electrons, interactions_in_roi, start, end): n_chunks = len(electron_chunks) if n_chunks > 1: - log.info("Chunk size exceeding file size target.") - log.info("Downchunking to %d chunks" % n_chunks) + log.info( + "Chunk size exceeding file size target. " + f"Downchunking to {n_chunks} chunks" + ) last_start = start if n_chunks>1: diff --git a/fuse/plugins/micro_physics/input.py b/fuse/plugins/micro_physics/input.py index 3a5ea731..0e2241d6 100644 --- a/fuse/plugins/micro_physics/input.py +++ b/fuse/plugins/micro_physics/input.py @@ -311,8 +311,10 @@ def output_chunk(self): self.chunk_bounds = np.append(chunk_start[0]-self.first_chunk_left, chunk_bounds) else: - log.warning("Only one Chunk created! Only a few events simulated? If no, your chunking parameters might not be optimal.") - log.warning("Try to decrease the source_rate or decrease the n_interactions_per_chunk") + log.warning( + "Only one Chunk created! Only a few events simulated? If no, your chunking parameters might not be optimal. " + "Try to decrease the source_rate or decrease the n_interactions_per_chunk." + ) self.chunk_bounds = [chunk_start[0] - self.first_chunk_left, chunk_end[0]+self.last_chunk_length] source_done = False diff --git a/fuse/plugins/pmt_and_daq/pmt_response_and_daq.py b/fuse/plugins/pmt_and_daq/pmt_response_and_daq.py index b77b3da1..1d10546e 100644 --- a/fuse/plugins/pmt_and_daq/pmt_response_and_daq.py +++ b/fuse/plugins/pmt_and_daq/pmt_response_and_daq.py @@ -224,8 +224,10 @@ def compute(self, propagated_photons, pulse_windows, start, end): n_chunks = len(pulse_window_chunks) if n_chunks > 1: - log.info("Chunk size exceeding file size target.") - log.info("Downchunking to %d chunks" % n_chunks) + log.info( + "Chunk size exceeding file size target. " + f"Downchunking to {n_chunks} chunks" + ) last_start = start diff --git a/pyproject.toml b/pyproject.toml index 5d8a3758..6e5f6e51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,15 +18,17 @@ classifiers = [ ] dependencies = [ "numpy", - "strax", "pandas", "scipy", + "scikit-learn", "immutabledict", "timeout_decorator", - "nestpy >= 2.0.0", + "nestpy >= 2.0.2", "numba >= 0.57.0", "awkward >= 2.2.1", "uproot >= 5.0.7", + "strax >= 1.6.0", + "straxen >= 2.2.0", ] [project.urls] diff --git a/tests/_utils.py b/tests/_utils.py new file mode 100644 index 00000000..ec2d67f4 --- /dev/null +++ b/tests/_utils.py @@ -0,0 +1 @@ +test_root_file_name = 'test_cryo_neutrons_tpc-nveto.root' diff --git a/tests/test_FullChain.py b/tests/test_FullChain.py index 939d39f6..35587a92 100644 --- a/tests/test_FullChain.py +++ b/tests/test_FullChain.py @@ -1,30 +1,47 @@ +import os +import shutil import unittest -import fuse import tempfile import timeout_decorator +import fuse +import straxen +from _utils import test_root_file_name TIMEOUT = 60 + class TestFullChain(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): - self.temp_dir = tempfile.TemporaryDirectory() + cls.temp_dir = tempfile.TemporaryDirectory() - self.test_context = fuse.context.full_chain_context(output_folder = self.temp_dir.name) + cls.test_context = fuse.context.full_chain_context(output_folder = cls.temp_dir.name) - self.test_context.set_config({"path": "/project2/lgrandi/xenonnt/simulations/testing", - "file_name": "pmt_neutrons_100.root", + cls.test_context.set_config({"path": cls.temp_dir.name, + "file_name": test_root_file_name, "entry_stop": 5, }) - self.run_number = "TestRun_00000" + cls.run_number = "TestRun_00000" @classmethod - def tearDownClass(self): + def tearDownClass(cls): + + cls.temp_dir.cleanup() + + def setUp(self): + downloader = straxen.MongoDownloader(store_files_at=(self.temp_dir.name,)) + downloader.download_single(test_root_file_name, human_readable_file_name=True) + + assert os.path.exists(os.path.join(self.temp_dir.name, test_root_file_name)) + + def tearDown(self): - self.temp_dir.cleanup() + # self.temp_dir.cleanup() + shutil.rmtree(self.temp_dir.name) + os.makedirs(self.temp_dir.name) @timeout_decorator.timeout(TIMEOUT, exception_message='S1PhotonHits timed out') def test_S1PhotonHits(self): diff --git a/tests/test_MicroPhysics.py b/tests/test_MicroPhysics.py index 6d64fa70..807738d1 100644 --- a/tests/test_MicroPhysics.py +++ b/tests/test_MicroPhysics.py @@ -1,30 +1,47 @@ +import os +import shutil import unittest -import fuse import tempfile import timeout_decorator +import fuse +import straxen +from _utils import test_root_file_name TIMEOUT = 60 + class TestMicroPhysics(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): - self.temp_dir = tempfile.TemporaryDirectory() + cls.temp_dir = tempfile.TemporaryDirectory() - self.test_context = fuse.context.microphysics_context(self.temp_dir.name) + cls.test_context = fuse.context.microphysics_context(cls.temp_dir.name) - self.test_context.set_config({"path": "/project2/lgrandi/xenonnt/simulations/testing", - "file_name": "pmt_neutrons_100.root", + cls.test_context.set_config({"path": cls.temp_dir.name, + "file_name": test_root_file_name, "entry_stop": 25, }) - self.run_number = "TestRun_00000" + cls.run_number = "TestRun_00000" @classmethod - def tearDownClass(self): + def tearDownClass(cls): + + cls.temp_dir.cleanup() + + def setUp(self): + downloader = straxen.MongoDownloader(store_files_at=(self.temp_dir.name,)) + downloader.download_single(test_root_file_name, human_readable_file_name=True) + + assert os.path.exists(os.path.join(self.temp_dir.name, test_root_file_name)) + + def tearDown(self): - self.temp_dir.cleanup() + # self.temp_dir.cleanup() + shutil.rmtree(self.temp_dir.name) + os.makedirs(self.temp_dir.name) @timeout_decorator.timeout(TIMEOUT, exception_message='ChunkInput timed out') def test_ChunkInput(self): diff --git a/tests/test_deterministic_seed.py b/tests/test_deterministic_seed.py index a414e56e..b8fadcaf 100644 --- a/tests/test_deterministic_seed.py +++ b/tests/test_deterministic_seed.py @@ -1,10 +1,14 @@ +import os import unittest -import fuse import tempfile -import numpy as np import timeout_decorator +import fuse +import straxen +from numpy.testing import assert_array_equal, assert_raises +from _utils import test_root_file_name + +TIMEOUT = 180 -TIMEOUT = 180 #Use a longer timeout for these tests. class TestDeterministicSeed(unittest.TestCase): @@ -13,17 +17,22 @@ def setUp(self): self.temp_dir_0 = tempfile.TemporaryDirectory() self.temp_dir_1 = tempfile.TemporaryDirectory() + for temp_dir in [self.temp_dir_0, self.temp_dir_1]: + downloader = straxen.MongoDownloader(store_files_at=(temp_dir.name,)) + downloader.download_single(test_root_file_name, human_readable_file_name=True) + assert os.path.exists(os.path.join(temp_dir.name, test_root_file_name)) + self.test_context_0 = fuse.context.full_chain_context(output_folder = self.temp_dir_0.name) - self.test_context_0.set_config({"path": "/project2/lgrandi/xenonnt/simulations/testing", - "file_name": "pmt_neutrons_100.root", + self.test_context_0.set_config({"path": self.temp_dir_0.name, + "file_name": test_root_file_name, "entry_stop": 5, }) self.test_context_1 = fuse.context.full_chain_context(output_folder = self.temp_dir_1.name) - self.test_context_1.set_config({"path": "/project2/lgrandi/xenonnt/simulations/testing", - "file_name": "pmt_neutrons_100.root", + self.test_context_1.set_config({"path": self.temp_dir_1.name, + "file_name": test_root_file_name, "entry_stop": 5, }) @@ -42,10 +51,10 @@ def test_MicroPhysics_SameSeed(self): self.test_context_0.make(self.run_number_0, "microphysics_summary") self.test_context_1.make(self.run_number_1, "microphysics_summary") - output_0 = self.test_context_0.get_array(self.run_number_0, "microphysics_summary") - output_1 = self.test_context_1.get_array(self.run_number_0, "microphysics_summary") + output_0 = self.test_context_0.get_array(self.run_number_0, "microphysics_summary", progress_bar=False) + output_1 = self.test_context_1.get_array(self.run_number_0, "microphysics_summary", progress_bar=False) - self.assertTrue(np.all(output_0 == output_1)) + assert_array_equal(output_0, output_1) @timeout_decorator.timeout(TIMEOUT, exception_message='MicroPhysics_DifferentSeed timed out') def test_MicroPhysics_DifferentSeed(self): @@ -54,10 +63,10 @@ def test_MicroPhysics_DifferentSeed(self): self.test_context_0.make(self.run_number_0, "microphysics_summary") self.test_context_1.make(self.run_number_1, "microphysics_summary") - output_0 = self.test_context_0.get_array(self.run_number_0, "microphysics_summary") - output_1 = self.test_context_1.get_array(self.run_number_1, "microphysics_summary") + output_0 = self.test_context_0.get_array(self.run_number_0, "microphysics_summary", progress_bar=False) + output_1 = self.test_context_1.get_array(self.run_number_1, "microphysics_summary", progress_bar=False) - self.assertFalse(np.all(output_0 == output_1)) + assert_raises(AssertionError, assert_array_equal, output_0, output_1) @timeout_decorator.timeout(TIMEOUT, exception_message='FullChain_SameSeed timed out') def test_FullChain_SameSeed(self): @@ -66,10 +75,10 @@ def test_FullChain_SameSeed(self): self.test_context_0.make(self.run_number_0, "raw_records") self.test_context_1.make(self.run_number_1, "raw_records") - output_0 = self.test_context_0.get_array(self.run_number_0, "raw_records") - output_1 = self.test_context_1.get_array(self.run_number_0, "raw_records") + output_0 = self.test_context_0.get_array(self.run_number_0, "raw_records", progress_bar=False) + output_1 = self.test_context_1.get_array(self.run_number_0, "raw_records", progress_bar=False) - self.assertTrue(np.all(output_0 == output_1)) + assert_array_equal(output_0, output_1) @timeout_decorator.timeout(TIMEOUT, exception_message='FullChain_DifferentSeed timed out') def test_FullChain_DifferentSeed(self): @@ -78,10 +87,10 @@ def test_FullChain_DifferentSeed(self): self.test_context_0.make(self.run_number_0, "raw_records") self.test_context_1.make(self.run_number_1, "raw_records") - output_0 = self.test_context_0.get_array(self.run_number_0, "raw_records") - output_1 = self.test_context_1.get_array(self.run_number_1, "raw_records") + output_0 = self.test_context_0.get_array(self.run_number_0, "raw_records", progress_bar=False) + output_1 = self.test_context_1.get_array(self.run_number_1, "raw_records", progress_bar=False) - self.assertFalse(np.all(output_0 == output_1)) + assert_raises(AssertionError, assert_array_equal, output_0, output_1) if __name__ == '__main__': unittest.main() \ No newline at end of file