Merge branch 'AliceO2Group:master' into tpc_krypton_new

rmunzer · Mar 26, 2024 · 2eacd3e · 2eacd3e
2 parents 3ef3eba + bb29737
commit 2eacd3e
Show file tree

Hide file tree

Showing 54 changed files with 1,899 additions and 389 deletions.
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -17,11 +17,15 @@
 /DATA/testing/detectors/TRD @martenole
 /DATA/testing/detectors/ZDC
 
-/MC @sawenzel @chiarazampolli @gconesab @benedikt-voelkel
+/MC @sawenzel @chiarazampolli @benedikt-voelkel
 
 # PWG experts
 /MC/*/PWGHF @AliceO2Group/reviewers-pwg-hf @sawenzel @chiarazampolli @benedikt-voelkel
 /MC/*/PWGLF @AliceO2Group/reviewers-pwg-lf @sawenzel @chiarazampolli @benedikt-voelkel
 /MC/*/PWGEM @AliceO2Group/reviewers-pwg-em @sawenzel @chiarazampolli @benedikt-voelkel
+/MC/*/PWGGAJE @AliceO2Group/reviewers-pwg-je @sawenzel @chiarazampolli @benedikt-voelkel
+/MC/*/PWGCF @AliceO2Group/reviewers-pwg-cf @sawenzel @chiarazampolli @benedikt-voelkel
+/MC/*/PWGDQ @AliceO2Group/reviewers-pwg-dq @sawenzel @chiarazampolli @benedikt-voelkel
+/MC/*/PWGUD @AliceO2Group/reviewers-pwg-ud @sawenzel @chiarazampolli @benedikt-voelkel
 
-/RelVal @sawenzel @chiarazampolli @gconesab @benedikt-voelkel
+/RelVal @sawenzel @chiarazampolli @benedikt-voelkel
diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh
@@ -208,6 +208,7 @@ fi
 
 
 if [[ "0${GEN_TOPO_VERBOSE:-}" == "01" ]]; then
+  echo "CALIB_RCT_UPDATER = ${CALIB_RCT_UPDATER:-}" 1>&2
   echo "CALIB_PRIMVTX_MEANVTX = $CALIB_PRIMVTX_MEANVTX" 1>&2
   echo "CALIB_TOF_LHCPHASE = $CALIB_TOF_LHCPHASE" 1>&2
   echo "CALIB_TOF_CHANNELOFFSETS = $CALIB_TOF_CHANNELOFFSETS" 1>&2
@@ -239,6 +240,8 @@ fi
 
 # define spec for proxy for TF-based outputs from BARREL
 if [[ -z ${CALIBDATASPEC_BARREL_TF:-} ]]; then
+  # RCT updater
+  if [[ ${CALIB_RCT_UPDATER:-} == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "calibRCT:CTD/DONE/0"; fi
   # prim vtx
   if [[ $CALIB_PRIMVTX_MEANVTX == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "pvtx:GLO/PVTX/0"; fi
 

diff --git a/DATA/production/calib/phs-led.sh b/DATA/production/calib/phs-led.sh
@@ -4,36 +4,67 @@ source common/setenv.sh
 
 source common/getCommonArgs.sh
 
-if [ -z $PHS_MAX_STATISTICS ]; then
-  PHS_MAX_STATISTICS=1000000
+source common/gen_topo_helper_functions.sh
+
+max_statistics=50000
+if [[ ! -z ${PHS_MAX_STATISTICS:-} ]]; then
+  max_statistics=$PHS_MAX_STATISTICS
 fi
 
-PROXY_INSPEC="A:PHS/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION"
+PROXY_INSPEC="A:PHS/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0"
 
 EXTRA_CONFIG=" "
 
-if [ -z $PHS_CCDB_PATH ]; then
-  PHS_CCDB_PATH="http://o2-ccdb.internal"
+push_ccdb_path="http://o2-ccdb.internal"
+pull_ccdb_path="http://o2-ccdb.internal"
+if [[ ! -z ${PUHS_CCDB_PATH:-} ]]; then
+  push_ccdb_path=$PUHS_CCDB_PATH
+fi
+
+if [[ ! -z ${PULL_CCDB_PATH:-} ]]; then
+  pull_ccdb_path=$PULL_CCDB_PATH
+fi
+
+if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then
+  push_ccdb_path="http://ccdb-test.cern.ch:8080"
+fi
+
+QC_CONFIG="/o2/components/qc/ANY/any/phs-led-qc"
+
+WORKFLOW=
+add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0
+add_W o2-phos-reco-workflow "--input-type raw --output-type cells --disable-root-input --disable-root-output --keepHGLG on --condition-backend ${pull_ccdb_path}"
+add_W o2-phos-calib-workflow "--hglgratio on --statistics ${max_statistics} --forceupdate"
+add_W o2-calibration-ccdb-populator-workflow "--ccdb-path ${push_ccdb_path}"
+workflow_has_parameter QC && add_QC_from_consul "${QC_CONFIG}"
+
+WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"
+
+if [ $WORKFLOWMODE == "print" ]; then
+    echo Workflow command:
+    echo $WORKFLOW | sed "s/| */|\n/g"
+else
+    # Execute the command we have assembled
+    WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}"
+    eval $WORKFLOW
 fi
 
-QC_CONFIG=consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/phs-led-qc
-
-o2-dpl-raw-proxy $ARGS_ALL \
-		 --dataspec "$PROXY_INSPEC" --inject-missing-data \
-		 --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \
-    | o2-phos-reco-workflow $ARGS_ALL \
-			    --input-type raw  \
-			    --output-type cells \
-			    --disable-root-input \
-			    --disable-root-output \
-			    --keepHGLG on \
-    | o2-phos-calib-workflow $ARGS_ALL \
-			     --hglgratio on \
-			     --statistics $PHS_MAX_STATISTICS \
-			     --configKeyValues "NameConf.mCCDBServer=${PHS_CCDB_PATH}" \
-			     --forceupdate \
-    | o2-qc $ARGS_ALL \
-            --config $QC_CONFIG \
-    | o2-calibration-ccdb-populator-workflow $ARGS_ALL \
-					     --ccdb-path $PHS_CCDB_PATH \
-    | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE}
+#o2-dpl-raw-proxy $ARGS_ALL \
+#		 --dataspec "$PROXY_INSPEC" --inject-missing-data \
+#		 --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \
+#    | o2-phos-reco-workflow $ARGS_ALL \
+#			    --input-type raw  \
+#			    --output-type cells \
+#			    --disable-root-input \
+#			    --disable-root-output \
+#			    --keepHGLG on \
+#    | o2-phos-calib-workflow $ARGS_ALL \
+#			     --hglgratio on \
+#			     --statistics $PHS_MAX_STATISTICS \
+#			     --configKeyValues "NameConf.mCCDBServer=${PHS_CCDB_PATH}" \
+#			     --forceupdate \
+#    | o2-qc $ARGS_ALL \
+#            --config $QC_CONFIG \
+#    | o2-calibration-ccdb-populator-workflow $ARGS_ALL \
+#					     --ccdb-path $PHS_CCDB_PATH \
+#    | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE}
diff --git a/DATA/production/calib/phs-pedestal.sh b/DATA/production/calib/phs-pedestal.sh
@@ -4,36 +4,67 @@ source common/setenv.sh
 
 source common/getCommonArgs.sh
 
-if [ -z $PHS_MAX_STATISTICS ] ; then
-  PHS_MAX_STATISTICS=10000
+source common/gen_topo_helper_functions.sh
+
+max_statistics=5000
+if [[ ! -z ${PHS_MAX_STATISTICS:-} ]]; then
+  max_statistics=$PHS_MAX_STATISTICS
+fi
+
+PROXY_INSPEC="A:PHS/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0"
+
+
+push_ccdb_path="http://o2-ccdb.internal"
+pull_ccdb_path="http://o2-ccdb.internal"
+if [[ ! -z ${PUHS_CCDB_PATH:-} ]]; then
+  push_ccdb_path=$PUHS_CCDB_PATH
 fi
 
-PROXY_INSPEC="A:PHS/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION"
+if [[ ! -z ${PULL_CCDB_PATH:-} ]]; then
+  pull_ccdb_path=$PULL_CCDB_PATH
+fi
+
+if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then
+  push_ccdb_path="http://ccdb-test.cern.ch:8080"
+fi
+
+QC_CONFIG="/o2/components/qc/ANY/any/phs-pedestal-qc"
+
+WORKFLOW=
+add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0
+add_W o2-phos-reco-workflow "--input-type raw --output-type cells --pedestal on --disable-root-input --disable-root-output --condition-backend ${pull_ccdb_path}" 
+add_W o2-phos-calib-workflow "--pedestals --statistics ${max_statistics} --forceupdate"
+#add_W o2-calibration-ccdb-populator-workflow "--ccdb-path ${push_ccdb_path}"
+workflow_has_parameter QC && add_QC_from_consul "${QC_CONFIG}"
+add_W o2-calibration-ccdb-populator-workflow "--ccdb-path ${push_ccdb_path}"
 
-EXTRA_CONFIG=" "
+WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}"
 
-if [ -z $PHS_CCDB_PATH ] ; then
-  PHS_CCDB_PATH="http://o2-ccdb.internal"
+if [ $WORKFLOWMODE == "print" ]; then
+    echo Workflow command:
+    echo $WORKFLOW | sed "s/| */|\n/g"
+else
+    # Execute the command we have assembled
+    WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}"
+    eval $WORKFLOW
 fi
 
-QC_CONFIG=consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/phs-pedestal-qc
-
-o2-dpl-raw-proxy $ARGS_ALL \
-		 --dataspec "$PROXY_INSPEC" --inject-missing-data \
-		 --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \
-    | o2-phos-reco-workflow $ARGS_ALL \
-			    --input-type raw  \
-			    --output-type cells \
-			    --pedestal on \
-			    --disable-root-input \
-			    --disable-root-output \
-    | o2-phos-calib-workflow $ARGS_ALL \
-			     --pedestals \
-			     --statistics $PHS_MAX_STATISTICS \
-			     --configKeyValues "NameConf.mCCDBServer=${PHS_CCDB_PATH}" \
-			     --forceupdate \
-    | o2-qc $ARGS_ALL \
-            --config $QC_CONFIG \
-    | o2-calibration-ccdb-populator-workflow $ARGS_ALL \
-					     --ccdb-path $PHS_CCDB_PATH \
-    | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE}
+#o2-dpl-raw-proxy $ARGS_ALL \
+#		 --dataspec "$PROXY_INSPEC" --inject-missing-data \
+#		 --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \
+#    | o2-phos-reco-workflow $ARGS_ALL \
+#			    --input-type raw  \
+#			    --output-type cells \
+#			    --pedestal on \
+#			    --disable-root-input \
+#			    --disable-root-output \
+#    | o2-phos-calib-workflow $ARGS_ALL \
+#			     --pedestals \
+#			     --statistics $PHS_MAX_STATISTICS \
+#			     --configKeyValues "NameConf.mCCDBServer=${PHS_CCDB_PATH}" \
+#			     --forceupdate \
+#    | o2-qc $ARGS_ALL \
+#            --config $QC_CONFIG \
+#    | o2-calibration-ccdb-populator-workflow $ARGS_ALL \
+#					     --ccdb-path $PHS_CCDB_PATH \
+#    | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE}
diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh
@@ -67,11 +67,12 @@ o2-dpl-raw-proxy ${ARGS_ALL} \
     | o2-tpc-reco-workflow ${ARGS_ALL}  ${TPC_CORR_SCALING:-} \
     --input-type digitizer  \
     --output-type "tracks,disable-writer,clusters" \
+    --disable-ctp-lumi-request \
     --disable-mc \
     --pipeline tpc-zsEncoder:20,tpc-tracker:8 \
     ${GPU_CONFIG} \
     --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser=TPC/Calib/LaserTracks" \
-    --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1;GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;GPU_rec_tpc.clustersShiftTimebinsClusterizer=35;GPU_proc.memoryScalingFactor=2" \
+    --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1;GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;GPU_rec_tpc.clustersShiftTimebinsClusterizer=35;GPU_proc.memoryScalingFactor=2;GPU_proc_param.tpcTriggerHandling=0" \
     | o2-tpc-laser-track-filter ${ARGS_ALL}  \
     | o2-dpl-output-proxy ${ARGS_ALL} \
     --dataspec "$PROXY_OUTSPEC" \

diff --git a/DATA/production/common/setVarsFromALIEN_PROC_ID.sh b/DATA/production/common/setVarsFromALIEN_PROC_ID.sh
@@ -3,18 +3,25 @@
 # Script to set a couple of variables that depend on ALIEN_PROC_ID.
 # Since several scripts might need this, it is in a separate script.
 
-# let's get the last 10 (for INT) and 5 (for int16) digits of ALIEN_PROC_ID, to be passed to define NUMAID and shm-segment-id via O2JOBID, which are int and int16 respectively. Then we make them an int or int16
-ALIEN_PROC_ID_MAX_NDIGITS_INT32=10
-ALIEN_PROC_ID_MAX_NDIGITS_INT16=9
-ALIEN_PROC_ID_LENTGH=`echo $ALIEN_PROC_ID | wc -l`
-ALIEN_PROC_ID_OFFSET_INT32=$((ALIEN_PROC_ID_LENTGH - ALIEN_PROC_ID_MAX_NDIGITS_INT32))
-ALIEN_PROC_ID_OFFSET_INT16=$((ALIEN_PROC_ID_LENTGH - ALIEN_PROC_ID_MAX_NDIGITS_INT16))
-# lets's take the last 10 (int) or 5 (int16) digits
-ALIEN_PROC_ID_OFFSET_INT32=${ALIEN_PROC_ID:$((ALIEN_PROC_ID_OFFSET_INT32))}
-ALIEN_PROC_ID_OFFSET_INT16=${ALIEN_PROC_ID:$((ALIEN_PROC_ID_OFFSET_INT16))}
+# let's get the last 9 (for int) and 8 (for int16) digits of ALIEN_PROC_ID, to be passed to define NUMAID and shm-segment-id via O2JOBID, which are int and int16 respectively. Then we make them an int or int16
+ALIEN_PROC_ID_MAX_NDIGITS_INT32=9
+ALIEN_PROC_ID_MAX_NDIGITS_INT16=8
+echo "ALIEN_PROC_ID for current job = ${ALIEN_PROC_ID}"
+
+if [[ -n ${ALIEN_JDL_PACKAGES} ]] && [[ ${#ALIEN_PROC_ID} -lt ${ALIEN_PROC_ID_MAX_NDIGITS_INT32} ]]; then # we are on the grid, and we expect to have the PROC_ID
+  echo "We cannot determine O2JOBID, the job id is too short (${ALIEN_PROC_ID}), we need at least ${ALIEN_PROC_ID_MAX_NDIGITS_INT32} digits, returning error"
+  exit 2
+fi
+
+ALIEN_PROC_ID_OFFSET_INT32=$((10#${ALIEN_PROC_ID: -${ALIEN_PROC_ID_MAX_NDIGITS_INT32}}))
+echo "ALIEN_PROC_ID_OFFSET_INT32 = $ALIEN_PROC_ID_OFFSET_INT32"
+
+ALIEN_PROC_ID_OFFSET_INT16=$((10#${ALIEN_PROC_ID: -${ALIEN_PROC_ID_MAX_NDIGITS_INT16}}))
+echo "ALIEN_PROC_ID_OFFSET_INT16 = $ALIEN_PROC_ID_OFFSET_INT16"
+
 # let's make them int32 or int16, but not with the max possible value (which would be 0x7fffffff and 0x7fff respectively)
 # but a bit less, to allow to add [0, 15] on top afterwards if needed (e.g. we usually add
 # the NUMAID), see https://github.com/AliceO2Group/O2DPG/pull/993#pullrequestreview-1393401475
 export O2JOBID=$(((ALIEN_PROC_ID_OFFSET_INT32 & 0x7ffffff) * 16))
 export O2JOBSHMID=$(((ALIEN_PROC_ID_OFFSET_INT16 & 0x7ff) * 16))
-echo "ALIEN_PROC_ID = $ALIEN_PROC_ID, we will set O2JOBID = $O2JOBID, SHMEMID = $SHMEMID"
+echo "ALIEN_PROC_ID = $ALIEN_PROC_ID, we will set O2JOBID = $O2JOBID, SHMEMID = $O2JOBSHMID"
diff --git a/DATA/production/qc-async/cpv.json b/DATA/production/qc-async/cpv.json
@@ -3,23 +3,14 @@
     "config": {
       "database": {
         "implementation": "CCDB",
-        "host": "alio2-cr1-hv-qcdb1.cern.ch:8083",
-        "username": "not_applicable",
-        "password": "not_applicable",
-        "name": "not_applicable"
-      },
-      "Activity": {
-        "number": "42",
-        "type": "2"
+        "host": "ccdb-test.cern.ch:8080"
       },
+      "Activity": {},
       "monitoring": {
-        "url": "influxdb-unix:///tmp/telegraf.sock"
-      },
-      "consul": {
-          "url": "alio2-cr1-hv-aliecs:8500"
+        "url": "infologger:///debug?qc"
       },
       "conditionDB": {
-        "url": "http://localhost:8084/"
+        "url": "ccdb-test.cern.ch:8080"
       }
     },
     "tasks": {
@@ -30,7 +21,6 @@
         "moduleName": "QcCPV",
         "detectorName": "CPV",
         "cycleDurationSeconds": "120",
-        "maxNumberCycles": "-1",
         "dataSource": {
           "type": "direct",
           "query": "clusters:CPV/CLUSTERS/0;ctrigrec:CPV/CLUSTERTRIGRECS/0"
@@ -40,9 +30,7 @@
           "ccdbCheckInterval": "1000"
         }
       }
-    },
-    "dataSamplingPolicies": [
-
-    ]
-  }
-}
+    }
+  },
+  "dataSamplingPolicies": []
+}
diff --git a/DATA/production/qc-async/its.json b/DATA/production/qc-async/its.json
@@ -44,17 +44,7 @@
 	  "dicttimestamp" : "0",
           "publishDetailedSummary": "1",
 	  "publishSummary1D": "0"
-        },
-	"grpGeomRequest" : {
-          "geomRequest": "Aligned",
-          "askGRPECS": "false",
-          "askGRPLHCIF": "false",
-          "askGRPMagField": "false",
-          "askMatLUT": "false",
-          "askTime": "false",
-          "askOnceAllButField": "true",
-          "needPropagatorD":  "false"
-        }
+        }	
       },
       "ITSTracks": {
         "active": "true",
@@ -80,7 +70,8 @@
 	  "nBCbins" : "103",
 	  "dicttimestamp" : 0,
 	  "doNorm": "1",
-          "InvMasses" : "1"
+          "InvMasses" : "1",
+	  "publishMore": "0"
         }
       }
     },

diff --git a/DATA/production/qc-async/mch-tracks.json b/DATA/production/qc-async/mch-tracks.json
@@ -33,7 +33,7 @@
                 "maxNumberCycles": "-1",
                 "dataSource": {
                     "type": "direct",
-                    "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS"
+                    "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/TRACKDIGITS"
                 },
                 "taskParameters": {
                     "maxTracksPerTF": "600",