Merge branch 'master' into jovoy-mirror_fix

.github/workflows/validation.yml

@@ -213,72 +213,15 @@ jobs:
         with:
           key: ${{ env.BRANCH_NAME }}-${{ github.sha }}
           path: ${{ env.REST_PATH }}
-      - name: 02_PandaXiiiMC
+      - name: PandaXIII Geant4
         run: |
           source ${{ env.REST_PATH }}/thisREST.sh
-          cd framework/pipeline/pandaxiii_MC
           restG4 ${{ env.REST_PATH }}/examples/restG4/05.PandaXIII/Xe136bb0n.rml -o Xe136bb0n.root
-          restRoot -b -q ${{ env.REST_PATH }}/examples/restG4/05.PandaXIII/Validate.C'("Xe136bb0n.root")'
-      #      - name: 02_PandaXiiiMC small for reference # We can use these artifacts to update the validation reference file
-      #        run: |
-      #          source ${{ env.REST_PATH }}/thisREST.sh
-      #          cd framework/pipeline/pandaxiii_MC
-      #          restG4 ${{ env.REST_PATH }}/examples/restG4/05.PandaXIII/Xe136bb0n.rml -o Xe136bb0n_small_reference_pipeline.root -n 5
-      #          restManager --c AllProcesses.rml --i Xe136bb0n_small_reference_pipeline.root --o Xe136bb0n_small_reference_processed_pipeline.root
-      #          restRoot -b -q ../MakeBasicTree.C'("Xe136bb0n_small_reference_processed_pipeline.root", "Xe136bb0n_validation_pipeline.root")'
-      #      - name: Upload Artifacts
-      #        uses: actions/upload-artifact@v3
-      #        with:
-      #          name: Xe136bb
-      #          path: |
-      #            framework/pipeline/pandaxiii_MC/Xe136bb0n.root
-      #            framework/pipeline/pandaxiii_MC/Xe136bb0n_small_reference_pipeline.root
-      #            framework/pipeline/pandaxiii_MC/Xe136bb0n_small_reference_processed_pipeline.root
-      #            framework/pipeline/pandaxiii_MC/Xe136bb0n_validation_pipeline.root
-      #          retention-days: 1
-      - name: PandaXIII Topological
-        run: |
-          source ${{ env.REST_PATH }}/thisREST.sh
-          cd framework/pipeline/pandaxiii_MC
-          restManager --c AllProcesses.rml --i Xe136bb0n_small_reference.root --o Xe136bb0n_small_reference_processed.root
-          restRoot -b -q ../MakeBasicTree.C'("Xe136bb0n_small_reference_processed.root")'
-          restRoot -b -q ../ValidateTrees.C'("Xe136bb0n_validation.root")'
-          # restManager --c plots.rml --i Xe136bb0n_small_reference_processed.root
-      # This command is failing in the docker, reproducible locally, not clear why...
-      #        echo | sleep 5 | restManager --c plots.rml --i testOutput.root
-      #    - name: Upload Artifacts
-      #      uses: actions/upload-artifact@v3
-      #      with:
-      #        name: PandaTrackParam
-      #        path: framework/pipeline/pandaxiii_MC/trackParameter.png
-      #        retention-days: 1
-      - name: PandaXIII Topological from Geant4
-        run: |
-          source ${{ env.REST_PATH }}/thisREST.sh
-          cd framework/pipeline/pandaxiii_MC
-          echo "using just-generated g4 file"
-          restManager --c AllProcesses.rml --i Xe136bb0n.root --o Xe136bb0n_processed.root --j 1 --e 5
-          restRoot -b -q ../MakeBasicTree.C'("Xe136bb0n_processed.root")'
-          restRoot -b -q ../ValidateTrees.C'("Xe136bb0n_validation.root")'
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: PandaTestOutput
-          path: framework/pipeline/pandaxiii_MC/Xe136bb0n_processed.root
-          retention-days: 1
       - name: PandaXIII Data
         run: |
           source ${{ env.REST_PATH }}/thisREST.sh
-          cd framework/pipeline/pandaxiii_data
+          cd framework/pipeline/panda-x
           restManager --c P3AutoChain.rml --i CoBo_AsAd0_2019-03-15.graw --o testOutput.root --j 1
-          restRoot -b -q ../MakeBasicTree.C'("testOutput.root")'
-          restRoot -b -q ../ValidateTrees.C'("validation.root")'
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: PandaTriggerRate
-          path: framework/pipeline/pandaxiii_data/TriggerRate.png
-          retention-days: 1
 
   trex-dm:
     name: "TREX-DM"
@@ -455,77 +398,6 @@ jobs:
           key: ${{ env.BRANCH_NAME }}-${{ github.sha }}
           path: ${{ env.REST_PATH }}
 
-
-  pandax-iii-reference:
-    name: "PandaX-III on reference version"
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/lobis/root-geant4-garfield:rest-for-physics-reference-jun2022
-    needs: [ framework-install-reference ]
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: rest-for-physics/framework
-          path: framework
-      - name: Checkout framework branch
-        run: |
-          cd framework
-          ./scripts/checkoutRemoteBranch.sh ${{ env.BRANCH_NAME }}
-      - name: Restore cache
-        uses: actions/cache@v3
-        id: framework-install-cache-reference
-        with:
-          key: ${{ env.BRANCH_NAME }}-${{ github.sha }}
-          path: ${{ env.REST_PATH }}
-      - name: 02_PandaXiiiMC
-        run: |
-          source ${{ env.REST_PATH }}/thisREST.sh
-          cd framework/pipeline/pandaxiii_MC
-          restG4 ${{ env.REST_PATH }}/examples/restG4/05.PandaXIII/Xe136bb0n.rml -o Xe136bb0n.root
-          restRoot -b -q ${{ env.REST_PATH }}/examples/restG4/05.PandaXIII/Validate.C'("Xe136bb0n.root")'
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: Xe136bbRef
-          path: framework/pipeline/pandaxiii_MC/Xe136bb0n.root
-          retention-days: 1
-      - name: PandaXIII Topological
-        run: |
-          source ${{ env.REST_PATH }}/thisREST.sh
-          cd framework/pipeline/pandaxiii_MC
-          restManager --c AllProcesses.rml --i Xe136bb0n_small_reference.root --o Xe136bb0n_small_reference_processed.root --j 1 --e 5
-          restRoot -b -q ../MakeBasicTree.C'("Xe136bb0n_small_reference_processed.root")'
-          restRoot -b -q ../ValidateTrees.C'("Xe136bb0n_validation.root")'
-          # restManager --c plots.rml --i Xe136bb0n_small_reference_processed.root
-      - name: PandaXIII Topological from Geant4
-        run: |
-          source ${{ env.REST_PATH }}/thisREST.sh
-          cd framework/pipeline/pandaxiii_MC
-          echo "using just-generated g4 file"
-          restManager --c AllProcesses.rml --i Xe136bb0n.root --o Xe136bb0n_processed.root --j 1 --e 5
-          restRoot -b -q ../MakeBasicTree.C'("Xe136bb0n_processed.root")'
-          # TODO: regenerate reference file
-          # restRoot -b -q ../ValidateTrees.C'("Xe136bb0n_validation_G4_v10.4.3.root")'
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: PandaTestOutputRef
-          path: framework/pipeline/pandaxiii_MC/Xe136bb0n_processed.root
-          retention-days: 1
-      - name: PandaXIII Data
-        run: |
-          source ${{ env.REST_PATH }}/thisREST.sh
-          cd framework/pipeline/pandaxiii_data
-          restManager --c P3AutoChain.rml --i CoBo_AsAd0_2019-03-15.graw --o testOutput.root --j 1
-          restRoot -b -q ../MakeBasicTree.C'("testOutput.root")'
-          restRoot -b -q ../ValidateTrees.C'("validation.root")'
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: PandaTriggerRateRef
-          path: framework/pipeline/pandaxiii_data/TriggerRate.png
-          retention-days: 1
-
   examples-reference:
     name: Run examples on reference version
     runs-on: ubuntu-latest

cmake/thisREST.cmake

@@ -16,22 +16,25 @@ execute_process(
 string(REGEX REPLACE "\n$" "" GEANT4_PATH "${GEANT4_PATH}")
 get_filename_component(GEANT4_BIN_DIR "${GEANT4_PATH}/bin/" REALPATH)
 
+set(g4LibPath "")
+set(loadG4 "")
 if (${REST_G4} MATCHES "ON")
     # https://github.com/rest-for-physics/framework/issues/331
     set(g4LibPath ":${GEANT4_PATH}/lib/")
     set(loadG4
         "\# if geant4.sh script is found we load the same Geant4 version as used in compilation\nif [[ -f \\\"${GEANT4_BIN_DIR}/geant4.sh\\\" ]]; then
     [[ -n \\\"\\\${ZSH_VERSION}\\\" ]] && pushd ${GEANT4_BIN_DIR} > /dev/null\n    source ${GEANT4_BIN_DIR}/geant4.sh\n    [[ -n \\\"\\\${ZSH_VERSION}\\\" ]] && popd > /dev/null\nfi\n"
     )
-else ()
-    set(g4LibPath "")
-    set(loadG4 "")
-endif (${REST_G4} MATCHES "ON")
+endif ()
 
+set(loadMPFR "")
 if (DEFINED MPFR_PATH)
     set(loadMPFR "export LD_LIBRARY_PATH=${MPFR_PATH}/lib:\$LD_LIBRARY_PATH")
-else ()
-    set(loadMPFR "")
+endif ()
+
+set(loadCRY "")
+if (DEFINED REST_CRY_PATH)
+    set(loadCRY "export LD_LIBRARY_PATH=${REST_CRY_PATH}/lib:\$LD_LIBRARY_PATH")
 endif ()
 
 set(loadGarfield "")
@@ -111,6 +114,7 @@ fi
 
 ${loadG4}
 ${loadMPFR}
+${loadCRY}
 ${loadGarfield}
 
 if [ \\\$REST_PATH ] ; then
@@ -124,6 +128,8 @@ fi
 
 export REST_SOURCE=${CMAKE_CURRENT_SOURCE_DIR}
 export REST_PATH=\\\${thisdir}
+# REST_HOME is where temporary files are stored
+export REST_HOME=\\\${HOME}
 export ROOT_INCLUDE_PATH=\\\$REST_PATH/include${Garfield_INCLUDE_ENV}:\\\$ROOT_INCLUDE_PATH
 export REST_INPUTDATA=\\\$REST_PATH/data
 export REST_GARFIELD_INCLUDE=${Garfield_INCLUDE_DIRS}
@@ -137,6 +143,7 @@ export PYTHONPATH=${PYTHON_BINDINGS_INSTALL_DIR}:\\\$PYTHONPATH
 
 alias restRoot=\\\"restRoot -l\\\"
 alias restRootMacros=\\\"restRoot -l --m\\\"
+alias restListMacros=\\\"restManager ListMacros\\\"
 
 if [ \\\$(rest-config --flags | grep \\\"REST_WELCOME=ON\\\") ]; then
     rest-config --welcome

examples/01.alphaTrack/README.md

@@ -22,7 +22,7 @@ This will produce an initial ROOT file that contains the simulated Geant4 event
 In order to further process the data we need to execute:
 
 ```
-restManager --c processing --f data/Run_5MeV_1um.root
+restManager --c processing.rml --f data/Run_5MeV_1um.root
 ```
 
 The processing includes the electron diffusion, the readout segmentation, and shaping and noise signal effects.

examples/calibrationCorrection.rml

@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<TRestDataSetGainMap name="calib" verboseLevel="info">
+        <parameter name="calibFileName" value="myDataSet.root"/>
+        <parameter name="outputFileName" value = "myCalibration.root"/>
+        <parameter name = "observable" value="rawAna_ThresholdIntegral"/>
+        <parameter name = "spatialObservableX" value="hitsAna_xMean"/>
+        <parameter name = "spatialObservableY" value="hitsAna_yMean"/>
+        <module planeId="0" moduleId="0"
+            moduleDefinitionCut="TREXsides_tagId==1"
+            numberOfSegmentsX="5"
+            numberOfSegmentsY="5"
+            readoutRange="(-1,246.24)">
+            <peak energy="22.5" range=""/>
+            <peak energy="8.0"  range=""/>
+        </module>
+        <module planeId="1" moduleId="0"
+            moduleDefinitionCut="TREXsides_tagId==2"
+            numberOfSegmentsX="5"
+            numberOfSegmentsY="5"
+            readoutRange="(-1,246.24)">
+            <peak energy="22.5" range=""/>
+            <peak energy="8.0"  range=""/>
+        </module>
+</TRestDataSetGainMap>

macros/REST_CheckRunFileList.C → macros/REST_CheckValidRuns.C

@@ -1,24 +1,46 @@
+#include <filesystem>
 #include <iostream>
 #include <vector>
 
 #include "TGeoManager.h"
 #include "TRestTask.h"
 #include "TSystem.h"
 using namespace std;
+namespace fs = std::filesystem;
 
-#ifndef RESTTask_CheckRunFileList
-#define RESTTask_CheckRunFileList
+#ifndef RESTTask_CheckValidRuns
+#define RESTTask_CheckValidRuns
 
 //*******************************************************************************************************
 //***
-//*** Your HELP is needed to verify, validate and document this macro
-//*** This macro might need update/revision.
+//*** Description: This macro will identify run files that were not properly closed.
+//***
+//*** --------------
+//*** The first and mandatory argument must provide a full data and pattern to filter the files that
+//*** will be checked. E.g. to add all the Hits files from run number 123 the first argument would be
+//*** pattern = "/path/to/data/Run00123*Hits*root".
+//***
+//*** IMPORTANT: The pattern must be given using double quotes ""
+//***
+//*** --------------
+//*** Usage: restManager CheckValidRuns "/full/path/file_*pattern*.root" [purge]
+//*** --------------
+//***
+//*** An optional parameter `purge` can be made true. In that case, this macro will not just provide
+//*** a list of the files not properly closed but it will also remove them!
+//***
+//*** The following command will remove the non-valid runs
+//*** --------------
+//*** Usage: restManager CheckValidRuns "/full/path/file_*pattern*.root" 1
+//*** --------------
+//***
+//*** CAUTION: Be aware that any non-REST file in the list will be removed if you use purge=1
 //***
 //*******************************************************************************************************
-Int_t REST_CheckRunFileList(TString namePattern, Int_t N = 100000) {
+Int_t REST_CheckValidRuns(TString namePattern, Bool_t purge = false) {
     TGeoManager::SetVerboseLevel(0);
 
-    vector<TString> filesNotWellClosed;
+    vector<std::string> filesNotWellClosed;
 
     TRestStringOutput RESTLog;
 
@@ -70,7 +92,16 @@ Int_t REST_CheckRunFileList(TString namePattern, Int_t N = 100000) {
         RESTLog << "---------------------" << RESTendl;
         RESTLog << "Files not well closed" << RESTendl;
         RESTLog << "---------------------" << RESTendl;
-        for (int i = 0; i < filesNotWellClosed.size(); i++) RESTLog << filesNotWellClosed[i] << RESTendl;
+        for (int i = 0; i < filesNotWellClosed.size(); i++) {
+            RESTLog << filesNotWellClosed[i] << RESTendl;
+            if (purge) fs::remove(filesNotWellClosed[i]);
+        }
+
+        if (purge) {
+            RESTLog << "---------------------------" << RESTendl;
+            RESTLog << "The files have been removed" << RESTendl;
+            RESTLog << "---------------------------" << RESTendl;
+        }
     }
 
     RESTLog << "------------------------------" << RESTendl;

macros/REST_GenerateDataSets.C

@@ -15,14 +15,16 @@
 //***
 //*******************************************************************************************************
 
-Int_t REST_GenerateDataSets(const std::string& inputRML, const std::string& datasets) {
+Int_t REST_GenerateDataSets(const std::string& inputRML, const std::string& datasets,
+                            std::string outPath = "") {
     std::vector<std::string> sets = REST_StringHelper::Split(datasets, ",");
 
     for (const auto& set : sets) {
         std::cout << "Set : " << set << std::endl;
         TRestDataSet d(inputRML.c_str(), set.c_str());
         d.GenerateDataSet();
-        d.Export("Dataset_" + set + ".root");
+        if (!outPath.empty() && outPath.back() != '/') outPath += '/';
+        d.Export(outPath + "Dataset_" + set + ".root");
     }
     return 0;
 }

macros/REST_OpenInputFile.C

@@ -46,8 +46,10 @@ void REST_OpenInputFile(const std::string& fileName) {
         printf("\n%s\n", " - dSet->PrintMetadata()");
         printf("%s\n", " - dSet->GetDataFrame().GetColumnNames()");
         printf("%s\n\n", " - dSet->GetTree()->GetEntries()");
+        printf("%s\n\n", " - dSet->GetDataFrame().Display({\"colName1,colName2\"})->Print()");
         if (dSet) delete dSet;
         dSet = new TRestDataSet();
+        dSet->EnableMultiThreading(false);
         dSet->Import(fileName);
     } else {
         printf("\n%s is not a valid TRestRun or TRestDataSet\n", fileName.c_str());