From 0b181edf41bb61aeb4d064b8d5348a514d3bb82a Mon Sep 17 00:00:00 2001 From: FedericoDevoto757 <151157377+FedericoDevoto757@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:29:03 +0200 Subject: [PATCH 01/16] Add files via upload --- .../new_nsb_DL1_to_DL2.py | 206 ++++++++++++++++++ .../new_nsb_DL2_to_DL3.py | 201 +++++++++++++++++ 2 files changed, 407 insertions(+) create mode 100644 magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py create mode 100644 magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py new file mode 100644 index 00000000..dd913256 --- /dev/null +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py @@ -0,0 +1,206 @@ +""" +This script creates the bashscripts necessary to apply "lst1_magic_dl1_stereo_to_dl2.py" +to the DL1 stereo data (real and MC). It also creates new subdirectories associated with +the data level 2. The DL2 files are saved at: +WorkingDirectory/DL2/ +and in the subdirectories therein. + +Usage: +$ python DL1_to_DL2.py -c configuration_file.yaml + +""" +import argparse +import glob +import logging +import os +import time +from pathlib import Path +import datetime +import joblib +import numpy as np +import pandas as pd +import yaml +from magicctapipe import __version__ +from magicctapipe.io import resource_file +from magicctapipe.scripts.lst1_magic.semi_automatic_scripts.clusters import ( + rc_lines, + slurm_lines, +) + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) +logger.setLevel(logging.INFO) + + +def ST_NSB_List(target_dir, nsb_list, nsb_limit, source): + """ + This function creates the lists of runs separeted by run period and NSB level. + + Parameters + ---------- + target_dir: str + Path to the working directory + nsb_list: + nsb_limit: + + source: + source name + """ + + "NSB dataframe" + df_LST = pd.read_hdf( + "/fefs/aswg/workspace/elisa.visentin/auto_MCP_PR/observations_LST.h5", + key="joint_obs", + ) + + "Period Lists" + ST_list = ["ST0320A", "ST0319A", "ST0318A", "ST0317A", "ST0316A"] + ST_begin = ["2023_03_10", "2022_12_15", "2022_06_10", "2021_12_30", "2020_10_24"] + ST_end = ["2025_01_30", "2023_03_09", "2022_08_31", "2022_06_09", "2021_09_29"] + # ST0320 ongoing -> 'service' end date + + "Loops over all runs of all nights" + Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/*")) + for night in Nights_list: + "Night period" + night_date=night.split('/')[-1] + print('night',night_date) + date_lst = night_date[:4] + "_" + night_date[4:6] + "_" + night_date[6:8] + print(date_lst) + date_magic = datetime.datetime.strptime(date_lst, "%Y_%m_%d") + datetime.timedelta(days=1) + print('dates', date_lst, str(date_magic)) + for p in range(len(ST_begin)): + if ( + date_magic + >= datetime.datetime.strptime(ST_begin[p], "%Y_%m_%d") + ) and ( + date_magic + <= datetime.datetime.strptime(ST_end[p], "%Y_%m_%d") + ): + period = ST_list[p] + + Run_list = glob.glob(f"{night}/Merged/*.h5") + for Run in Run_list: + "getting the run NSB" + print(Run) + run_str=Run.split('/')[-1].split('.')[1] + print('run', run_str) + run_LST_id = run_str.lstrip('Run') + print('run_lst', run_LST_id) + nsb = df_LST[df_LST["LST1_run"] == run_LST_id]["nsb"].tolist()[0] + print('nsb', nsb) + "rounding the NSB to the nearest MC nsb value" + for j in range(0, len(nsb_list)-1): + if (nsb < nsb_limit[j + 1]) & (nsb > nsb_limit[j]): + nsb = nsb_list[j] + "Writing on output .txt file" + if (nsb <= 3.1): + with open(f"{night}/Merged/logs/{period}_{nsb}.txt","a+") as file: + file.write(f"{Run}\n") + + +def bash_DL1Stereo_to_DL2(target_dir, source, env_name): + """ + This function generates the bashscript for running the DL1Stereo to DL2 analisys. + + Parameters + ---------- + target_dir : str + Path to the working directory + source: + source name + env_name: + conda enviroment name + """ + + process_name = source + Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/*")) + for night in Nights_list: + File_list = glob.glob(f"{night}/Merged/logs/ST*.txt") + night_date=night.split('/')[-1] + os.makedirs(f'{target_dir}/v{__version__}/{source}/DL2/{night_date}/logs',exist_ok=True) + for file in File_list: + with open(file, "r") as f: + process_size = len(f.readlines()) - 1 + if process_size < 0: + continue + nsb=file.split("/")[-1].split("_")[-1][:3] + p=file.split("/")[-1].split("_")[0] + RFdir=f'/fefs/aswg/LST1MAGIC/mc/models/{p}/NSB{nsb}/v01.2/dec_2276/' + with open(f'{source}_DL1_to_DL2_{night_date}_{file.split("/")[-1].rstrip("txt")}sh', "w") as f: + f.write("#!/bin/sh\n\n") + f.write("#SBATCH -p long\n") + f.write("#SBATCH -J " + process_name + "\n") + f.write(f"#SBATCH --array=0-{process_size}%100\n") + f.write("#SBATCH --mem=90g\n") + f.write("#SBATCH -N 1\n\n") + f.write("ulimit -l unlimited\n") + f.write("ulimit -s unlimited\n") + f.write("ulimit -a\n\n") + + f.write( + f"SAMPLE_LIST=($(<{file}))\n" + ) + f.write("SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n") + f.write( + f"export LOG={target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]}/logs" + + "/DL1_to_DL2_${SLURM_ARRAY_TASK_ID}.log\n" + ) + f.write( + f"conda run -n {env_name} lst1_magic_dl1_stereo_to_dl2 --input-file-dl1 $SAMPLE --input-dir-rfs {RFdir} --output-dir {target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]} >$LOG 2>&1\n\n" + ) + + + +def main(): + """ + Here we read the config_general.yaml file and call the functions defined above. + """ + + parser = argparse.ArgumentParser() + parser.add_argument( + "--config-file", + "-c", + dest="config_file", + type=str, + default="./config_general.yaml", + help="Path to a configuration file", + ) + + args = parser.parse_args() + with open( + args.config_file, "rb" + ) as f: # "rb" mode opens the file in binary format for reading + config = yaml.safe_load(f) + + target_dir = Path(config["directories"]["workspace_dir"]) + env_name = config["general"]["env_name"] + nsb_list = config["general"]["nsb"] + width = [a / 2 - b / 2 for a, b in zip(nsb_list[1:], nsb_list[:-1])] + width.append(0.25) + nsb_limit = [a + b for a, b in zip(nsb_list[:], width[:])] + nsb_limit.insert(0, 0) + source_in = config["data_selection"]["source_name_database"] + source = config["data_selection"]["source_name_output"] + + cluster = config["general"]["cluster"] + + if source_in is None: + source_list = joblib.load("list_sources.dat") + else: + source_list = [source] + for source_name in source_list: + ST_NSB_List(target_dir, nsb_list, nsb_limit, source_name) + + bash_DL1Stereo_to_DL2(target_dir,source_name, env_name) + list_of_stereo_scripts = np.sort(glob.glob(f'{source_name}_DL1_to_DL2*.sh')) + print(list_of_stereo_scripts) + for n, run in enumerate(list_of_stereo_scripts): + if n == 0: + launch_jobs = f"stereo{n}=$(sbatch --parsable {run})" + else: + launch_jobs = ( + f"{launch_jobs} && stereo{n}=$(sbatch --parsable {run})" + ) + print(launch_jobs) + os.system(launch_jobs) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py new file mode 100644 index 00000000..7b791024 --- /dev/null +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py @@ -0,0 +1,201 @@ +""" +This script creates the bashscripts necessary to apply "lst1_magic_dl2_to_dl3.py" +to the DL2. It also creates new subdirectories associated with +the data level 3. + + +Usage: +$ python DL2_to_DL3.py -c configuration_file.yaml + +""" +import argparse +import os +import numpy as np +import glob +import yaml +import logging +import joblib +from pathlib import Path +from magicctapipe import __version__ +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler()) +logger.setLevel(logging.INFO) + + +def configuration_DL3(ids, target_dir, target_coords,source): + """ + This function creates the configuration file needed for the DL2 to DL3 conversion + + Parameters + ---------- + ids: list + list of telescope IDs + target_dir: str + Path to the working directory + target_coords: + sorce coordinates + source: str + source name + """ + + f = open(str(target_dir) + "/config_DL3.yaml", "w") + f.write( + "mc_tel_ids:\n LST-1: " + + str(ids[0]) + + "\n LST-2: " + + str(ids[1]) + + "\n LST-3: " + + str(ids[2]) + + "\n LST-4: " + + str(ids[3]) + + "\n MAGIC-I: " + + str(ids[4]) + + "\n MAGIC-II: " + + str(ids[5]) + + "\n\n" + ) + f.write( + f'dl2_to_dl3:\n interpolation_method: "linear" # select "nearest", "linear" or "cubic"\n interpolation_scheme: "cosZd" # select "cosZdAz" or "cosZd"\n max_distance: "90. deg"\n source_name: "{source}"\n source_ra: "{target_coords[0]} deg" # used when the source name cannot be resolved\n source_dec: "{target_coords[1]} deg" # used when the source name cannot be resolved\n\n' + ) + + f.close() + + +def DL2_to_DL3(target_dir, source, env_name): + """ + This function creates the bash scripts to run lst1_magic_dl2_to_dl3.py on the real data. + + Parameters + ---------- + target_dir: str + Path to the working directory + source: str + source name + env_name: str + conda enviroment name + """ + + target_dir = str(target_dir) + if not os.path.exists(target_dir + f"/v{__version__}/{source}/DL3"): + os.mkdir(target_dir + f"/v{__version__}/{source}/DL3") + if not os.path.exists(target_dir + f"/v{__version__}/{source}/DL3/logs"): + os.mkdir(target_dir + f"/v{__version__}/{source}/DL3/logs") + + #Loop over all nights + Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL2/*")) + for night in Nights_list: + #Loop over every run: + File_list = np.sort(glob.glob(f"{night}/*.txt")) + for file in File_list: + with open(file, "r") as f: + runs = f.readlines() + process_size = len(runs) - 1 + run_new = [run.split("DL1Stereo")[0] + "DL2" + run.split("DL1Stereo")[1].replace("dl1_stereo", "dl2").replace("/Merged", "") for run in runs] + with open(file, "w") as g: + g.writelines(run_new) + + nsb = file.split("/")[-1].split("_")[-1][:3] + print("nsb = ", nsb) + period = file.split("/")[-1].split("_")[0] + print("period = ", period) + + IRF_dir = ( + f"/fefs/aswg/LST1MAGIC/mc/IRF/{period}/NSB{nsb}/GammaTest/v01.2/g_dyn_0.9_th_glo_0.2/dec_2276/" + ) + + process_name = "DL3_" + target_dir.split("/")[-2:][1] + str(nsb) + output = target_dir + f"/v{__version__}/{source}/DL3" + + f = open(f'{source}_DL3_{nsb}_{period}_{night.split("/")[-1]}.sh', "w") + f.write("#!/bin/sh\n\n") + f.write("#SBATCH -p short\n") + f.write("#SBATCH -J " + process_name + "\n") + f.write("#SBATCH --mem=10g\n") + f.write(f"#SBATCH --array=0-{process_size}%100\n") + f.write("#SBATCH -N 1\n\n") + f.write("ulimit -l unlimited\n") + f.write("ulimit -s unlimited\n") + f.write("ulimit -a\n\n") + + f.write( + f"SAMPLE_LIST=($(<{file}))\n" + ) + f.write("SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n") + + f.write(f'export LOG={output}/logs/DL3_{nsb}_{period}_{night.split("/")[-1]}.log\n') + f.write( + f"conda run -n {env_name} lst1_magic_dl2_to_dl3 --input-file-dl2 $SAMPLE --input-dir-irf {IRF_dir} --output-dir {output} --config-file {target_dir}/config_DL3.yaml >$LOG 2>&1\n\n" + ) + + f.close() + + +def main(): + """ + Here we read the config_general.yaml file and call the functions defined above. + """ + + parser = argparse.ArgumentParser() + parser.add_argument( + "--config-file", + "-c", + dest="config_file", + type=str, + default="./config_general.yaml", + help="Path to a configuration file", + ) + + args = parser.parse_args() + with open( + args.config_file, "rb" + ) as f: # "rb" mode opens the file in binary format for reading + config = yaml.safe_load(f) + + telescope_ids = list(config["mc_tel_ids"].values()) + target_dir = Path(config["directories"]["workspace_dir"]) + + target_coords = [ + config["data_selection"]["target_RA_deg"], + config["data_selection"]["target_Dec_deg"], + ] + + print("***** Generating file config_DL3.yaml...") + print("***** This file can be found in ", target_dir) + + source_in = config["data_selection"]["source_name_database"] + source = config["data_selection"]["source_name_output"] + env_name = config["general"]["env_name"] + cluster = config["general"]["cluster"] + + #cp the .txt files from DL1 stereo anaysis to be used again. + DL1stereo_Nihts = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/*")) + for night in DL1stereo_Nihts: + File_list = glob.glob(f"{night}/Merged/logs/ST*.txt") + night_date = night.split("/")[-1] + print("night date ", night_date) + for file in File_list: + cp_dir = f"{target_dir}/v{__version__}/{source}/DL2/{night_date}" + os.system(f"cp {file} {cp_dir}") + + + if source_in is None: + source_list = joblib.load("list_sources.dat") + else: + source_list = [source] + for source_name in source_list: + configuration_DL3(telescope_ids, target_dir, target_coords, source_name) + DL2_to_DL3(target_dir, source_name, env_name) + list_of_stereo_scripts = np.sort(glob.glob(f'{source_name}_DL3*.sh')) + print(list_of_stereo_scripts) + for n, run in enumerate(list_of_stereo_scripts): + if n == 0: + launch_jobs = f"stereo{n}=$(sbatch --parsable {run})" + else: + launch_jobs = ( + f"{launch_jobs} && stereo{n}=$(sbatch --parsable {run})" + ) + print(launch_jobs) + os.system(launch_jobs) + +if __name__ == "__main__": + main() From 27d2f094d9e22a5670007ffab2aee990f5401d35 Mon Sep 17 00:00:00 2001 From: FedericoDevoto757 <151157377+FedericoDevoto757@users.noreply.github.com> Date: Thu, 12 Sep 2024 14:29:13 +0200 Subject: [PATCH 02/16] Add files via upload --- .../lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py | 5 ++++- .../lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py index dd913256..a79aef2b 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py @@ -6,7 +6,7 @@ and in the subdirectories therein. Usage: -$ python DL1_to_DL2.py -c configuration_file.yaml +$ python new_DL1_to_DL2.py -c configuration_file.yaml """ import argparse @@ -204,3 +204,6 @@ def main(): ) print(launch_jobs) os.system(launch_jobs) + +if __name__ == "__main__": + main() diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py index 7b791024..4a5ee6f6 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py @@ -5,7 +5,7 @@ Usage: -$ python DL2_to_DL3.py -c configuration_file.yaml +$ python new_DL2_to_DL3.py -c configuration_file.yaml """ import argparse From fffd6b6c7836397d0c457e5f8a15c323f99058d7 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Sun, 17 Nov 2024 14:06:03 +0100 Subject: [PATCH 03/16] some fixes --- .../{new_nsb_DL1_to_DL2.py => DL1_to_DL2.py} | 70 +++++----- .../{new_nsb_DL2_to_DL3.py => DL2_to_DL3.py} | 131 ++++++++++-------- 2 files changed, 107 insertions(+), 94 deletions(-) rename magicctapipe/scripts/lst1_magic/semi_automatic_scripts/{new_nsb_DL1_to_DL2.py => DL1_to_DL2.py} (81%) rename magicctapipe/scripts/lst1_magic/semi_automatic_scripts/{new_nsb_DL2_to_DL3.py => DL2_to_DL3.py} (60%) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py similarity index 81% rename from magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py rename to magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py index a79aef2b..64c10695 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL1_to_DL2.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py @@ -60,7 +60,7 @@ def ST_NSB_List(target_dir, nsb_list, nsb_limit, source): # ST0320 ongoing -> 'service' end date "Loops over all runs of all nights" - Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/*")) + Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*")) for night in Nights_list: "Night period" night_date=night.split('/')[-1] @@ -79,7 +79,7 @@ def ST_NSB_List(target_dir, nsb_list, nsb_limit, source): ): period = ST_list[p] - Run_list = glob.glob(f"{night}/Merged/*.h5") + Run_list = glob.glob(f"{night}/*.h5") for Run in Run_list: "getting the run NSB" print(Run) @@ -95,7 +95,7 @@ def ST_NSB_List(target_dir, nsb_list, nsb_limit, source): nsb = nsb_list[j] "Writing on output .txt file" if (nsb <= 3.1): - with open(f"{night}/Merged/logs/{period}_{nsb}.txt","a+") as file: + with open(f"{night}/logs/{period}_{nsb}.txt","a+") as file: file.write(f"{Run}\n") @@ -114,9 +114,9 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name): """ process_name = source - Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/*")) + Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*")) for night in Nights_list: - File_list = glob.glob(f"{night}/Merged/logs/ST*.txt") + File_list = glob.glob(f"{night}/logs/ST*.txt") night_date=night.split('/')[-1] os.makedirs(f'{target_dir}/v{__version__}/{source}/DL2/{night_date}/logs',exist_ok=True) for file in File_list: @@ -127,29 +127,33 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name): nsb=file.split("/")[-1].split("_")[-1][:3] p=file.split("/")[-1].split("_")[0] RFdir=f'/fefs/aswg/LST1MAGIC/mc/models/{p}/NSB{nsb}/v01.2/dec_2276/' - with open(f'{source}_DL1_to_DL2_{night_date}_{file.split("/")[-1].rstrip("txt")}sh', "w") as f: - f.write("#!/bin/sh\n\n") - f.write("#SBATCH -p long\n") - f.write("#SBATCH -J " + process_name + "\n") - f.write(f"#SBATCH --array=0-{process_size}%100\n") - f.write("#SBATCH --mem=90g\n") - f.write("#SBATCH -N 1\n\n") - f.write("ulimit -l unlimited\n") - f.write("ulimit -s unlimited\n") - f.write("ulimit -a\n\n") - - f.write( - f"SAMPLE_LIST=($(<{file}))\n" - ) - f.write("SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n") - f.write( - f"export LOG={target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]}/logs" - + "/DL1_to_DL2_${SLURM_ARRAY_TASK_ID}.log\n" - ) - f.write( + slurm = slurm_lines( + queue="short", + job_name=f"{process_name}_DL1_to_DL2", + array=process_size, + mem="2g", + out_name=f"{target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]}/logs/slurm-%x.%A_%a", + ) + rc = rc_lines( + store="$SAMPLE ${SLURM_ARRAY_JOB_ID} ${SLURM_ARRAY_TASK_ID}", + out="$OUTPUTDIR/logs/list", + ) + + lines = ( + slurm + + [ + f"SAMPLE_LIST=($(<{file}))\n", + "SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n", + f"export LOG={target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]}/logs", + "/DL1_to_DL2_${SLURM_ARRAY_TASK_ID}.log\n", f"conda run -n {env_name} lst1_magic_dl1_stereo_to_dl2 --input-file-dl1 $SAMPLE --input-dir-rfs {RFdir} --output-dir {target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]} >$LOG 2>&1\n\n" - ) - + ] + + rc + ) + with open(f'{source}_DL1_to_DL2_{night_date}_{file.split("/")[-1].rstrip("txt")}sh', "w") as f: + f.writelines(lines) + + def main(): @@ -195,14 +199,12 @@ def main(): bash_DL1Stereo_to_DL2(target_dir,source_name, env_name) list_of_stereo_scripts = np.sort(glob.glob(f'{source_name}_DL1_to_DL2*.sh')) print(list_of_stereo_scripts) + if len(list_of_stereo_scripts) < 1: + logger.warning("No bash scripts for real data") + continue + launch_jobs = "" for n, run in enumerate(list_of_stereo_scripts): - if n == 0: - launch_jobs = f"stereo{n}=$(sbatch --parsable {run})" - else: - launch_jobs = ( - f"{launch_jobs} && stereo{n}=$(sbatch --parsable {run})" - ) - print(launch_jobs) + launch_jobs += (" && " if n > 0 else "") + f"sbatch {run}" os.system(launch_jobs) if __name__ == "__main__": diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py similarity index 60% rename from magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py rename to magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py index 4a5ee6f6..9798f84d 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/new_nsb_DL2_to_DL3.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py @@ -17,12 +17,15 @@ import joblib from pathlib import Path from magicctapipe import __version__ +from magicctapipe.io import resource_file + + logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.INFO) -def configuration_DL3(ids, target_dir, target_coords,source): +def configuration_DL3(target_dir, source_name, config_file): """ This function creates the configuration file needed for the DL2 to DL3 conversion @@ -38,28 +41,34 @@ def configuration_DL3(ids, target_dir, target_coords,source): source name """ - f = open(str(target_dir) + "/config_DL3.yaml", "w") - f.write( - "mc_tel_ids:\n LST-1: " - + str(ids[0]) - + "\n LST-2: " - + str(ids[1]) - + "\n LST-3: " - + str(ids[2]) - + "\n LST-4: " - + str(ids[3]) - + "\n MAGIC-I: " - + str(ids[4]) - + "\n MAGIC-II: " - + str(ids[5]) - + "\n\n" - ) - f.write( - f'dl2_to_dl3:\n interpolation_method: "linear" # select "nearest", "linear" or "cubic"\n interpolation_scheme: "cosZd" # select "cosZdAz" or "cosZd"\n max_distance: "90. deg"\n source_name: "{source}"\n source_ra: "{target_coords[0]} deg" # used when the source name cannot be resolved\n source_dec: "{target_coords[1]} deg" # used when the source name cannot be resolved\n\n' - ) - f.close() + if config_file == "": + config_file = resource_file("config.yaml") + + with open( + config_file, "rb" + ) as fc: # "rb" mode opens the file in binary format for reading + config_dict = yaml.safe_load(fc) + DL3_config = config_dict['dl2_to_dl3'] + DL3_config['source_name']=source_name + #DL3_config['source_ra']=ra + #DL3_config['source_dec']=dec + conf = { + "mc_tel_ids": config_dict["mc_tel_ids"], + "dl2_to_dl3": DL3_config, + } + + conf_dir = f"{target_dir}/v{__version__}/{source_name}" + os.makedirs(conf_dir, exist_ok=True) + + file_name = f"{conf_dir}/config_DL3.yaml" + + with open(file_name, "w") as f: + + yaml.dump(conf, f, default_flow_style=False) + + def DL2_to_DL3(target_dir, source, env_name): """ @@ -76,10 +85,8 @@ def DL2_to_DL3(target_dir, source, env_name): """ target_dir = str(target_dir) - if not os.path.exists(target_dir + f"/v{__version__}/{source}/DL3"): - os.mkdir(target_dir + f"/v{__version__}/{source}/DL3") - if not os.path.exists(target_dir + f"/v{__version__}/{source}/DL3/logs"): - os.mkdir(target_dir + f"/v{__version__}/{source}/DL3/logs") + + os.makedirs(target_dir + f"/v{__version__}/{source}/DL3/logs", exist_ok=True) #Loop over all nights Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL2/*")) @@ -103,33 +110,41 @@ def DL2_to_DL3(target_dir, source, env_name): f"/fefs/aswg/LST1MAGIC/mc/IRF/{period}/NSB{nsb}/GammaTest/v01.2/g_dyn_0.9_th_glo_0.2/dec_2276/" ) - process_name = "DL3_" + target_dir.split("/")[-2:][1] + str(nsb) + process_name = source output = target_dir + f"/v{__version__}/{source}/DL3" - f = open(f'{source}_DL3_{nsb}_{period}_{night.split("/")[-1]}.sh', "w") - f.write("#!/bin/sh\n\n") - f.write("#SBATCH -p short\n") - f.write("#SBATCH -J " + process_name + "\n") - f.write("#SBATCH --mem=10g\n") - f.write(f"#SBATCH --array=0-{process_size}%100\n") - f.write("#SBATCH -N 1\n\n") - f.write("ulimit -l unlimited\n") - f.write("ulimit -s unlimited\n") - f.write("ulimit -a\n\n") - - f.write( - f"SAMPLE_LIST=($(<{file}))\n" - ) - f.write("SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n") - - f.write(f'export LOG={output}/logs/DL3_{nsb}_{period}_{night.split("/")[-1]}.log\n') - f.write( - f"conda run -n {env_name} lst1_magic_dl2_to_dl3 --input-file-dl2 $SAMPLE --input-dir-irf {IRF_dir} --output-dir {output} --config-file {target_dir}/config_DL3.yaml >$LOG 2>&1\n\n" + + slurm = slurm_lines( + queue="short", + job_name=f"{process_name}_DL2_to_DL3", + array=process_size, + mem="1g", + out_name=f"{target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]}/logs/slurm-%x.%A_%a", ) + rc = rc_lines( + store="$SAMPLE ${SLURM_ARRAY_JOB_ID} ${SLURM_ARRAY_TASK_ID}", + out="$OUTPUTDIR/logs/list", + ) + + lines = ( + slurm + + [ + f"SAMPLE_LIST=($(<{file}))\n", + "SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n", + f"export LOG={output}/logs", + "/DL2_to_DL3_${SLURM_ARRAY_TASK_ID}.log\n", + f"conda run -n {env_name} lst1_magic_dl2_to_dl3 --input-file-dl2 $SAMPLE --input-dir-irf {IRF_dir} --output-dir {output} --config-file {target_dir}/config_DL3.yaml >$LOG 2>&1\n\n" + ] + + rc + ) + with open(f'{source}_DL3_{nsb}_{period}_{night.split("/")[-1]}.sh', "w") as f: + f.writelines(lines) - f.close() + + + def main(): """ Here we read the config_general.yaml file and call the functions defined above. @@ -153,11 +168,7 @@ def main(): telescope_ids = list(config["mc_tel_ids"].values()) target_dir = Path(config["directories"]["workspace_dir"]) - - target_coords = [ - config["data_selection"]["target_RA_deg"], - config["data_selection"]["target_Dec_deg"], - ] + print("***** Generating file config_DL3.yaml...") print("***** This file can be found in ", target_dir) @@ -165,12 +176,13 @@ def main(): source_in = config["data_selection"]["source_name_database"] source = config["data_selection"]["source_name_output"] env_name = config["general"]["env_name"] + config_file = config["general"]["base_config_file"] cluster = config["general"]["cluster"] #cp the .txt files from DL1 stereo anaysis to be used again. - DL1stereo_Nihts = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/*")) + DL1stereo_Nihts = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*")) for night in DL1stereo_Nihts: - File_list = glob.glob(f"{night}/Merged/logs/ST*.txt") + File_list = glob.glob(f"{night}/logs/ST*.txt") night_date = night.split("/")[-1] print("night date ", night_date) for file in File_list: @@ -187,15 +199,14 @@ def main(): DL2_to_DL3(target_dir, source_name, env_name) list_of_stereo_scripts = np.sort(glob.glob(f'{source_name}_DL3*.sh')) print(list_of_stereo_scripts) + if len(list_of_stereo_scripts) < 1: + logger.warning("No bash scripts for real data") + continue + launch_jobs = "" for n, run in enumerate(list_of_stereo_scripts): - if n == 0: - launch_jobs = f"stereo{n}=$(sbatch --parsable {run})" - else: - launch_jobs = ( - f"{launch_jobs} && stereo{n}=$(sbatch --parsable {run})" - ) - print(launch_jobs) + launch_jobs += (" && " if n > 0 else "") + f"sbatch {run}" os.system(launch_jobs) + if __name__ == "__main__": main() From 1f932b2ef681edc72eb557c4038024f4d2ff7f2f Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Mon, 18 Nov 2024 09:53:06 +0100 Subject: [PATCH 04/16] some fixes --- .../semi_automatic_scripts/DL1_to_DL2.py | 138 ++++++++++-------- .../semi_automatic_scripts/DL2_to_DL3.py | 80 +++++----- .../config_auto_MCP.yaml | 3 +- 3 files changed, 120 insertions(+), 101 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py index 64c10695..d013594c 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py @@ -1,25 +1,25 @@ """ This script creates the bashscripts necessary to apply "lst1_magic_dl1_stereo_to_dl2.py" -to the DL1 stereo data (real and MC). It also creates new subdirectories associated with -the data level 2. The DL2 files are saved at: -WorkingDirectory/DL2/ -and in the subdirectories therein. +to the DL1 stereo data. It also creates new subdirectories associated with +the data level 2. Usage: -$ python new_DL1_to_DL2.py -c configuration_file.yaml +$ DL1_to_DL2 -c configuration_file.yaml """ import argparse +import datetime import glob import logging import os import time from pathlib import Path -import datetime + import joblib import numpy as np import pandas as pd import yaml + from magicctapipe import __version__ from magicctapipe.io import resource_file from magicctapipe.scripts.lst1_magic.semi_automatic_scripts.clusters import ( @@ -32,7 +32,7 @@ logger.setLevel(logging.INFO) -def ST_NSB_List(target_dir, nsb_list, nsb_limit, source): +def ST_NSB_List(target_dir, nsb_list, nsb_limit, source, df_LST): """ This function creates the lists of runs separeted by run period and NSB level. @@ -41,65 +41,64 @@ def ST_NSB_List(target_dir, nsb_list, nsb_limit, source): target_dir: str Path to the working directory nsb_list: + nsb_limit: - + source: - source name - """ + source name + df_LST: - "NSB dataframe" - df_LST = pd.read_hdf( - "/fefs/aswg/workspace/elisa.visentin/auto_MCP_PR/observations_LST.h5", - key="joint_obs", - ) + """ + + "Period Lists" - ST_list = ["ST0320A", "ST0319A", "ST0318A", "ST0317A", "ST0316A"] - ST_begin = ["2023_03_10", "2022_12_15", "2022_06_10", "2021_12_30", "2020_10_24"] - ST_end = ["2025_01_30", "2023_03_09", "2022_08_31", "2022_06_09", "2021_09_29"] - # ST0320 ongoing -> 'service' end date + ST_list = ["ST0321A", "ST0320A", "ST0319A", "ST0318A", "ST0317A", "ST0316A"] + ST_begin = ["2024_05_19","2023_03_10", "2022_12_15", "2022_06_10", "2021_12_30", "2020_10_24"] + ST_end = ["2026_01_01","2024_05_18", "2023_03_09", "2022_08_31", "2022_06_09", "2021_09_29"] + # ST0321 ongoing -> 'service' end date "Loops over all runs of all nights" - Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*")) + Nights_list = np.sort( + glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*") + ) for night in Nights_list: "Night period" - night_date=night.split('/')[-1] - print('night',night_date) + night_date = night.split("/")[-1] + print("night", night_date) date_lst = night_date[:4] + "_" + night_date[4:6] + "_" + night_date[6:8] print(date_lst) - date_magic = datetime.datetime.strptime(date_lst, "%Y_%m_%d") + datetime.timedelta(days=1) - print('dates', date_lst, str(date_magic)) + date_magic = datetime.datetime.strptime( + date_lst, "%Y_%m_%d" + ) + datetime.timedelta(days=1) + print("dates", date_lst, str(date_magic)) for p in range(len(ST_begin)): - if ( - date_magic - >= datetime.datetime.strptime(ST_begin[p], "%Y_%m_%d") - ) and ( - date_magic - <= datetime.datetime.strptime(ST_end[p], "%Y_%m_%d") - ): + if (date_magic >= datetime.datetime.strptime(ST_begin[p], "%Y_%m_%d")) and ( + date_magic <= datetime.datetime.strptime(ST_end[p], "%Y_%m_%d") + ): period = ST_list[p] Run_list = glob.glob(f"{night}/*.h5") for Run in Run_list: "getting the run NSB" print(Run) - run_str=Run.split('/')[-1].split('.')[1] - print('run', run_str) - run_LST_id = run_str.lstrip('Run') - print('run_lst', run_LST_id) + run_str = Run.split("/")[-1].split(".")[1] + print("run", run_str) + run_LST_id = run_str.lstrip("Run") + print("run_lst", run_LST_id) nsb = df_LST[df_LST["LST1_run"] == run_LST_id]["nsb"].tolist()[0] - print('nsb', nsb) + print("nsb", nsb) "rounding the NSB to the nearest MC nsb value" - for j in range(0, len(nsb_list)-1): - if (nsb < nsb_limit[j + 1]) & (nsb > nsb_limit[j]): + for j in range(0, len(nsb_list) - 1): + if (nsb <= nsb_limit[j + 1]) & (nsb > nsb_limit[j]): nsb = nsb_list[j] "Writing on output .txt file" - if (nsb <= 3.1): - with open(f"{night}/logs/{period}_{nsb}.txt","a+") as file: + if nsb <= nsb_limit[-1]: + with open(f"{night}/logs/{period}_{nsb}.txt", "a+") as file: file.write(f"{Run}\n") -def bash_DL1Stereo_to_DL2(target_dir, source, env_name): +def bash_DL1Stereo_to_DL2(target_dir, source, env_name, cluster, RF_dir, df_LST): """ This function generates the bashscript for running the DL1Stereo to DL2 analisys. @@ -108,25 +107,36 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name): target_dir : str Path to the working directory source: - source name + source name env_name: - conda enviroment name + conda enviroment name """ - + if cluster != "SLURM": + logger.warning( + "Automatic processing not implemented for the cluster indicated in the config file" + ) + return process_name = source - Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*")) + Nights_list = np.sort( + glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*") + ) for night in Nights_list: File_list = glob.glob(f"{night}/logs/ST*.txt") - night_date=night.split('/')[-1] - os.makedirs(f'{target_dir}/v{__version__}/{source}/DL2/{night_date}/logs',exist_ok=True) + night_date = night.split("/")[-1] + os.makedirs( + f"{target_dir}/v{__version__}/{source}/DL2/{night_date}/logs", exist_ok=True + ) for file in File_list: with open(file, "r") as f: process_size = len(f.readlines()) - 1 if process_size < 0: continue - nsb=file.split("/")[-1].split("_")[-1][:3] - p=file.split("/")[-1].split("_")[0] - RFdir=f'/fefs/aswg/LST1MAGIC/mc/models/{p}/NSB{nsb}/v01.2/dec_2276/' + nsb = file.split("/")[-1].split("_")[-1][:3] + p = file.split("/")[-1].split("_")[0] + dec=df_LST[df_LST.source == source].iloc[0]['MC_dec'] + RFdir = f"{RF_dir}/{p}/NSB{nsb}/dec_{dec}/" + if (not os.path.isdir(RFdir)) or (len(os.listdir(RFdir)) == 0): + continue slurm = slurm_lines( queue="short", job_name=f"{process_name}_DL1_to_DL2", @@ -138,7 +148,7 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name): store="$SAMPLE ${SLURM_ARRAY_JOB_ID} ${SLURM_ARRAY_TASK_ID}", out="$OUTPUTDIR/logs/list", ) - + lines = ( slurm + [ @@ -146,14 +156,15 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name): "SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n", f"export LOG={target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]}/logs", "/DL1_to_DL2_${SLURM_ARRAY_TASK_ID}.log\n", - f"conda run -n {env_name} lst1_magic_dl1_stereo_to_dl2 --input-file-dl1 $SAMPLE --input-dir-rfs {RFdir} --output-dir {target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]} >$LOG 2>&1\n\n" + f"conda run -n {env_name} lst1_magic_dl1_stereo_to_dl2 --input-file-dl1 $SAMPLE --input-dir-rfs {RFdir} --output-dir {target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]} >$LOG 2>&1\n\n", ] + rc ) - with open(f'{source}_DL1_to_DL2_{night_date}_{file.split("/")[-1].rstrip("txt")}sh', "w") as f: + with open( + f'{source}_DL1_to_DL2_{night_date}_{file.split("/")[-1].rstrip("txt")}sh', + "w", + ) as f: f.writelines(lines) - - def main(): @@ -178,6 +189,7 @@ def main(): config = yaml.safe_load(f) target_dir = Path(config["directories"]["workspace_dir"]) + RF_dir=config["directories"]["RF"] env_name = config["general"]["env_name"] nsb_list = config["general"]["nsb"] width = [a / 2 - b / 2 for a, b in zip(nsb_list[1:], nsb_list[:-1])] @@ -188,24 +200,30 @@ def main(): source = config["data_selection"]["source_name_output"] cluster = config["general"]["cluster"] + "LST dataframe" + df_LST = pd.read_hdf( + "/fefs/aswg/workspace/elisa.visentin/auto_MCP_PR/observations_LST.h5", + key="joint_obs", + ) if source_in is None: source_list = joblib.load("list_sources.dat") else: source_list = [source] for source_name in source_list: - ST_NSB_List(target_dir, nsb_list, nsb_limit, source_name) + ST_NSB_List(target_dir, nsb_list, nsb_limit, source_name, df_LST) - bash_DL1Stereo_to_DL2(target_dir,source_name, env_name) - list_of_stereo_scripts = np.sort(glob.glob(f'{source_name}_DL1_to_DL2*.sh')) + bash_DL1Stereo_to_DL2(target_dir, source_name, env_name, cluster, RF_dir) + list_of_stereo_scripts = np.sort(glob.glob(f"{source_name}_DL1_to_DL2*.sh")) print(list_of_stereo_scripts) if len(list_of_stereo_scripts) < 1: - logger.warning("No bash scripts for real data") + logger.warning(f"No bash scripts for {source_name}") continue launch_jobs = "" for n, run in enumerate(list_of_stereo_scripts): launch_jobs += (" && " if n > 0 else "") + f"sbatch {run}" os.system(launch_jobs) - + + if __name__ == "__main__": main() diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py index 9798f84d..36a86943 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py @@ -9,17 +9,18 @@ """ import argparse -import os -import numpy as np import glob -import yaml import logging -import joblib +import os from pathlib import Path + +import joblib +import numpy as np +import yaml + from magicctapipe import __version__ from magicctapipe.io import resource_file - logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.INFO) @@ -36,13 +37,11 @@ def configuration_DL3(target_dir, source_name, config_file): target_dir: str Path to the working directory target_coords: - sorce coordinates + sorce coordinates source: str - source name + source name """ - - if config_file == "": config_file = resource_file("config.yaml") @@ -50,10 +49,10 @@ def configuration_DL3(target_dir, source_name, config_file): config_file, "rb" ) as fc: # "rb" mode opens the file in binary format for reading config_dict = yaml.safe_load(fc) - DL3_config = config_dict['dl2_to_dl3'] - DL3_config['source_name']=source_name - #DL3_config['source_ra']=ra - #DL3_config['source_dec']=dec + DL3_config = config_dict["dl2_to_dl3"] + DL3_config["source_name"] = source_name + # DL3_config['source_ra']=ra + # DL3_config['source_dec']=dec conf = { "mc_tel_ids": config_dict["mc_tel_ids"], "dl2_to_dl3": DL3_config, @@ -68,7 +67,6 @@ def configuration_DL3(target_dir, source_name, config_file): yaml.dump(conf, f, default_flow_style=False) - def DL2_to_DL3(target_dir, source, env_name): """ @@ -79,25 +77,32 @@ def DL2_to_DL3(target_dir, source, env_name): target_dir: str Path to the working directory source: str - source name + source name env_name: str - conda enviroment name + conda enviroment name """ target_dir = str(target_dir) - - os.makedirs(target_dir + f"/v{__version__}/{source}/DL3/logs", exist_ok=True) - #Loop over all nights + os.makedirs(target_dir + f"/v{__version__}/{source}/DL3/logs", exist_ok=True) + + # Loop over all nights Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL2/*")) for night in Nights_list: - #Loop over every run: + # Loop over every run: File_list = np.sort(glob.glob(f"{night}/*.txt")) for file in File_list: - with open(file, "r") as f: + with open(file, "r") as f: runs = f.readlines() process_size = len(runs) - 1 - run_new = [run.split("DL1Stereo")[0] + "DL2" + run.split("DL1Stereo")[1].replace("dl1_stereo", "dl2").replace("/Merged", "") for run in runs] + run_new = [ + run.split("DL1Stereo")[0] + + "DL2" + + run.split("DL1Stereo")[1] + .replace("dl1_stereo", "dl2") + .replace("/Merged", "") + for run in runs + ] with open(file, "w") as g: g.writelines(run_new) @@ -106,13 +111,10 @@ def DL2_to_DL3(target_dir, source, env_name): period = file.split("/")[-1].split("_")[0] print("period = ", period) - IRF_dir = ( - f"/fefs/aswg/LST1MAGIC/mc/IRF/{period}/NSB{nsb}/GammaTest/v01.2/g_dyn_0.9_th_glo_0.2/dec_2276/" - ) + IRF_dir = f"/fefs/aswg/LST1MAGIC/mc/IRF/{period}/NSB{nsb}/GammaTest/v01.2/g_dyn_0.9_th_glo_0.2/dec_2276/" process_name = source - output = target_dir + f"/v{__version__}/{source}/DL3" - + output = target_dir + f"/v{__version__}/{source}/DL3" slurm = slurm_lines( queue="short", @@ -125,7 +127,7 @@ def DL2_to_DL3(target_dir, source, env_name): store="$SAMPLE ${SLURM_ARRAY_JOB_ID} ${SLURM_ARRAY_TASK_ID}", out="$OUTPUTDIR/logs/list", ) - + lines = ( slurm + [ @@ -133,18 +135,16 @@ def DL2_to_DL3(target_dir, source, env_name): "SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n", f"export LOG={output}/logs", "/DL2_to_DL3_${SLURM_ARRAY_TASK_ID}.log\n", - f"conda run -n {env_name} lst1_magic_dl2_to_dl3 --input-file-dl2 $SAMPLE --input-dir-irf {IRF_dir} --output-dir {output} --config-file {target_dir}/config_DL3.yaml >$LOG 2>&1\n\n" + f"conda run -n {env_name} lst1_magic_dl2_to_dl3 --input-file-dl2 $SAMPLE --input-dir-irf {IRF_dir} --output-dir {output} --config-file {target_dir}/config_DL3.yaml >$LOG 2>&1\n\n", ] + rc ) - with open(f'{source}_DL3_{nsb}_{period}_{night.split("/")[-1]}.sh', "w") as f: + with open( + f'{source}_DL3_{nsb}_{period}_{night.split("/")[-1]}.sh', "w" + ) as f: f.writelines(lines) - - - - def main(): """ Here we read the config_general.yaml file and call the functions defined above. @@ -168,7 +168,6 @@ def main(): telescope_ids = list(config["mc_tel_ids"].values()) target_dir = Path(config["directories"]["workspace_dir"]) - print("***** Generating file config_DL3.yaml...") print("***** This file can be found in ", target_dir) @@ -179,8 +178,10 @@ def main(): config_file = config["general"]["base_config_file"] cluster = config["general"]["cluster"] - #cp the .txt files from DL1 stereo anaysis to be used again. - DL1stereo_Nihts = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*")) + # cp the .txt files from DL1 stereo anaysis to be used again. + DL1stereo_Nihts = np.sort( + glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*") + ) for night in DL1stereo_Nihts: File_list = glob.glob(f"{night}/logs/ST*.txt") night_date = night.split("/")[-1] @@ -189,7 +190,6 @@ def main(): cp_dir = f"{target_dir}/v{__version__}/{source}/DL2/{night_date}" os.system(f"cp {file} {cp_dir}") - if source_in is None: source_list = joblib.load("list_sources.dat") else: @@ -197,7 +197,7 @@ def main(): for source_name in source_list: configuration_DL3(telescope_ids, target_dir, target_coords, source_name) DL2_to_DL3(target_dir, source_name, env_name) - list_of_stereo_scripts = np.sort(glob.glob(f'{source_name}_DL3*.sh')) + list_of_stereo_scripts = np.sort(glob.glob(f"{source_name}_DL3*.sh")) print(list_of_stereo_scripts) if len(list_of_stereo_scripts) < 1: logger.warning("No bash scripts for real data") @@ -206,7 +206,7 @@ def main(): for n, run in enumerate(list_of_stereo_scripts): launch_jobs += (" && " if n > 0 else "") + f"sbatch {run}" os.system(launch_jobs) - + if __name__ == "__main__": main() diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml index 769ef909..a2ebac74 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml @@ -2,7 +2,8 @@ directories: workspace_dir: "/fefs/aswg/workspace/elisa.visentin/auto_MCP_PR/" # Output directory where all the data products will be saved. MC_version: "" #MCP version used to process MC; if not provided, current MCP version will be used real_input_version: "" #MCP version used to process input real data (to be used in case you want to run only the last steps of the analysis, using lower-level data previously processed); if not provided, current MCP version will be used - + RF: '' + IRF: '' data_selection: source_name_database: "CrabNebula" # MUST BE THE SAME AS IN THE DATABASE; Set to null to process all sources in the given time range. source_name_output: 'Crabtest' # Name tag of your target. Used only if source_name_database != null. From 3d7d79ab9565c583b9c3d467fdede3d3c21581e3 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Mon, 18 Nov 2024 11:02:12 +0100 Subject: [PATCH 05/16] fixes config --- .../semi_automatic_scripts/DL1_to_DL2.py | 19 ++++++++------- .../config_auto_MCP.yaml | 14 +++++++---- .../database_production/LSTnsb.py | 6 ++--- .../database_production/__init__.py | 3 +-- .../database_production/lstchain_version.py | 23 +++++++++++++++---- .../database_production/nsb_level.py | 2 +- 6 files changed, 43 insertions(+), 24 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py index d013594c..8c39e2fb 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py @@ -32,7 +32,7 @@ logger.setLevel(logging.INFO) -def ST_NSB_List(target_dir, nsb_list, nsb_limit, source, df_LST): +def ST_NSB_List(target_dir, nsb_list, nsb_limit, source, df_LST, ST_list, ST_begin, ST_end): """ This function creates the lists of runs separeted by run period and NSB level. @@ -47,16 +47,12 @@ def ST_NSB_List(target_dir, nsb_list, nsb_limit, source, df_LST): source: source name df_LST: - """ - "Period Lists" - ST_list = ["ST0321A", "ST0320A", "ST0319A", "ST0318A", "ST0317A", "ST0316A"] - ST_begin = ["2024_05_19","2023_03_10", "2022_12_15", "2022_06_10", "2021_12_30", "2020_10_24"] - ST_end = ["2026_01_01","2024_05_18", "2023_03_09", "2022_08_31", "2022_06_09", "2021_09_29"] - # ST0321 ongoing -> 'service' end date + + "Loops over all runs of all nights" Nights_list = np.sort( @@ -172,7 +168,7 @@ def main(): Here we read the config_general.yaml file and call the functions defined above. """ - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser()["2026_01_01","2024_05_18", "2023_03_09", "2022_08_31", "2022_06_09", "2021_09_29"] parser.add_argument( "--config-file", "-c", @@ -191,7 +187,10 @@ def main(): target_dir = Path(config["directories"]["workspace_dir"]) RF_dir=config["directories"]["RF"] env_name = config["general"]["env_name"] - nsb_list = config["general"]["nsb"] + ST_list= config["needed_parameters"]['ST_list'] + ST_begin= config["needed_parameters"]['ST_begin'] + ST_end= config["needed_parameters"]['ST_end'] + nsb_list = config["needed_parameters"]["nsb"] width = [a / 2 - b / 2 for a, b in zip(nsb_list[1:], nsb_list[:-1])] width.append(0.25) nsb_limit = [a + b for a, b in zip(nsb_list[:], width[:])] @@ -211,7 +210,7 @@ def main(): else: source_list = [source] for source_name in source_list: - ST_NSB_List(target_dir, nsb_list, nsb_limit, source_name, df_LST) + ST_NSB_List(target_dir, nsb_list, nsb_limit, source_name, df_LST, ST_list, ST_begin, ST_end) bash_DL1Stereo_to_DL2(target_dir, source_name, env_name, cluster, RF_dir) list_of_stereo_scripts = np.sort(glob.glob(f"{source_name}_DL1_to_DL2*.sh")) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml index a2ebac74..d9c8e9c0 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml @@ -16,11 +16,17 @@ data_selection: general: base_config_file: '' # path + name to a custom MCP config file. If not provided, the default config.yaml file will be used - LST_version: "v0.10" # check the `processed_lstchain_file` version in the LST database! + LST_version: "v0.10" # check the `processed_lstchain_file` version in the LST database! + env_name: magic-lst # name of the conda environment to be used to process data. + cluster: "SLURM" # cluster management system on which data are processed. At the moment we have only SLURM available, in the future maybe also condor (PIC, CNAF). + +needed_parameters: # DON'T CHANGE THESE VALUES, UNLESS YOU REALLY KNOW WHAT YOU ARE DOING! They are needed for a meaningful and complete analysis LST_tailcut: "tailcut84" simtel_nsb: "/fefs/aswg/data/mc/DL0/LSTProd2/TestDataset/sim_telarray/node_theta_14.984_az_355.158_/output_v1.4/simtel_corsika_theta_14.984_az_355.158_run10.simtel.gz" # simtel file (DL0) to evaluate NSB lstchain_modified_config: true # use_flatfield_heuristic = True to evaluate NSB nsb: [0.5, 1.0, 1.5, 2.0, 2.5, 3.0] - env_name: magic-lst # name of the conda environment to be used to process data. - cluster: "SLURM" # cluster management system on which data are processed. At the moment we have only SLURM available, in the future maybe also condor (PIC, CNAF). - + lstchain_versions: ["v0.9", "v0.10"] + ST_list: ["ST0321A", "ST0320A", "ST0319A", "ST0318A", "ST0317A", "ST0316A"] + ST_begin: ["2024_05_19","2023_03_10", "2022_12_15", "2022_06_10", "2021_12_30", "2020_10_24"] + ST_end: ["2026_01_01","2024_05_18", "2023_03_09", "2022_08_31", "2022_06_09", "2021_09_29"] + # ST0321 ongoing -> 'service' end date \ No newline at end of file diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/LSTnsb.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/LSTnsb.py index 955e00bd..90e06cd3 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/LSTnsb.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/LSTnsb.py @@ -181,10 +181,10 @@ def main(): date = args.day denominator = args.denominator lst_config = args.lst_conf - simtel = config["general"]["simtel_nsb"] - nsb_list = config["general"]["nsb"] + simtel = config["needed_parameters"]["simtel_nsb"] + nsb_list = config["needed_parameters"]["nsb"] lst_version = config["general"]["LST_version"] - lst_tailcut = config["general"]["LST_tailcut"] + lst_tailcut = config["needed_parameters"]["LST_tailcut"] width = [a / 2 - b / 2 for a, b in zip(nsb_list[1:], nsb_list[:-1])] width.append(0.25) nsb_limit = [a + b for a, b in zip(nsb_list[:], width[:])] diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/__init__.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/__init__.py index 9957c9db..dc7cbea6 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/__init__.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/__init__.py @@ -1,4 +1,4 @@ -from .lstchain_version import lstchain_versions, version_lstchain +from .lstchain_version import version_lstchain from .LSTnsb import nsb from .nsb_level import bash_scripts from .nsb_to_h5 import collect_nsb @@ -12,7 +12,6 @@ "bash_scripts", "collect_nsb", "fix_lists_and_convert", - "lstchain_versions", "nsb", "table_magic_runs", "update_tables", diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/lstchain_version.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/lstchain_version.py index d3cb1935..39a27137 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/lstchain_version.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/lstchain_version.py @@ -10,17 +10,17 @@ import glob import os - +import argparse import pandas as pd import yaml from magicctapipe.io import resource_file -lstchain_versions = ["v0.9", "v0.10"] + __all__ = ["version_lstchain"] -def version_lstchain(df_LST): +def version_lstchain(df_LST, lstchain_versions): """ Evaluates (and store in the database) all the versions used to process a given file and the last version of a file @@ -69,6 +69,21 @@ def main(): """ Main function """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--config-file", + "-c", + dest="config_auto", + type=str, + default="../config_auto_MCP.yaml", + help="Path to a configuration file", + ) + args = parser.parse_args() + with open( + args.config_auto, "rb" + ) as f: # "rb" mode opens the file in binary format for reading + config = yaml.safe_load(f) + lstchain_versions = config['needed_parameters']['lstchain_versions'] config_file = resource_file("database_config.yaml") with open( @@ -80,7 +95,7 @@ def main(): LST_key = config_dict["database_keys"]["LST"] df_LST = pd.read_hdf(LST_h5, key=LST_key) - version_lstchain(df_LST) + version_lstchain(df_LST, lstchain_versions) df_LST.to_hdf( LST_h5, diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/nsb_level.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/nsb_level.py index 802b9964..f483f8bb 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/nsb_level.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/nsb_level.py @@ -120,7 +120,7 @@ def main(): key=LST_key, ) lstchain_v = config["general"]["LST_version"] - lstchain_modified = config["general"]["lstchain_modified_config"] + lstchain_modified = config["needed_parameters"]["lstchain_modified_config"] conda_path = os.environ["CONDA_PREFIX"] lst_config_orig = ( str(conda_path) From 38d065c9e1d82bc7718b6b93b56ce6f742bd1930 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Mon, 18 Nov 2024 14:16:34 +0100 Subject: [PATCH 06/16] some fixes --- .../semi_automatic_scripts/DL1_to_DL2.py | 96 +++++++++++++------ .../semi_automatic_scripts/__init__.py | 6 ++ .../config_auto_MCP.yaml | 6 +- .../database_production/lstchain_version.py | 8 +- setup.cfg | 1 + 5 files changed, 81 insertions(+), 36 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py index 8c39e2fb..bb8eb677 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py @@ -1,18 +1,16 @@ """ This script creates the bashscripts necessary to apply "lst1_magic_dl1_stereo_to_dl2.py" to the DL1 stereo data. It also creates new subdirectories associated with -the data level 2. +the data level 2. Usage: $ DL1_to_DL2 -c configuration_file.yaml - """ import argparse import datetime import glob import logging import os -import time from pathlib import Path import joblib @@ -27,33 +25,39 @@ slurm_lines, ) +__all__ = ["ST_NSB_List", "bash_DL1Stereo_to_DL2"] + logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.INFO) -def ST_NSB_List(target_dir, nsb_list, nsb_limit, source, df_LST, ST_list, ST_begin, ST_end): +def ST_NSB_List( + target_dir, nsb_list, nsb_limit, source, df_LST, ST_list, ST_begin, ST_end +): """ This function creates the lists of runs separeted by run period and NSB level. Parameters ---------- - target_dir: str + target_dir : str Path to the working directory - nsb_list: - - nsb_limit: - - source: - source name - df_LST: + nsb_list : list + List of the MC NSB values + nsb_limit : list + Edges of the NSB binning + source : str + Source name + df_LST : :class:`pandas.DataFrame` + Dataframe collecting the LST1 runs (produced by the create_LST_table script) + ST_list : list + List of the observastion periods + ST_begin : list + List of beginning dates for the observation periods + ST_end : list + List of ending dates for the observation periods """ - - - - - "Loops over all runs of all nights" Nights_list = np.sort( glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*") @@ -102,10 +106,16 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name, cluster, RF_dir, df_LST) ---------- target_dir : str Path to the working directory - source: - source name - env_name: - conda enviroment name + source : str + Source name + env_name : str + Conda enviroment name + cluster : str + Cluster system + RF_dir : str + Path to the RFs + df_LST : :class:`pandas.DataFrame` + Dataframe collecting the LST1 runs (produced by the create_LST_table script) """ if cluster != "SLURM": logger.warning( @@ -129,7 +139,7 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name, cluster, RF_dir, df_LST) continue nsb = file.split("/")[-1].split("_")[-1][:3] p = file.split("/")[-1].split("_")[0] - dec=df_LST[df_LST.source == source].iloc[0]['MC_dec'] + dec = df_LST[df_LST.source == source].iloc[0]["MC_dec"] RFdir = f"{RF_dir}/{p}/NSB{nsb}/dec_{dec}/" if (not os.path.isdir(RFdir)) or (len(os.listdir(RFdir)) == 0): continue @@ -168,7 +178,14 @@ def main(): Here we read the config_general.yaml file and call the functions defined above. """ - parser = argparse.ArgumentParser()["2026_01_01","2024_05_18", "2023_03_09", "2022_08_31", "2022_06_09", "2021_09_29"] + parser = argparse.ArgumentParser()[ + "2026_01_01", + "2024_05_18", + "2023_03_09", + "2022_08_31", + "2022_06_09", + "2021_09_29", + ] parser.add_argument( "--config-file", "-c", @@ -185,11 +202,11 @@ def main(): config = yaml.safe_load(f) target_dir = Path(config["directories"]["workspace_dir"]) - RF_dir=config["directories"]["RF"] + RF_dir = config["directories"]["RF"] env_name = config["general"]["env_name"] - ST_list= config["needed_parameters"]['ST_list'] - ST_begin= config["needed_parameters"]['ST_begin'] - ST_end= config["needed_parameters"]['ST_end'] + ST_list = config["needed_parameters"]["ST_list"] + ST_begin = config["needed_parameters"]["ST_begin"] + ST_end = config["needed_parameters"]["ST_end"] nsb_list = config["needed_parameters"]["nsb"] width = [a / 2 - b / 2 for a, b in zip(nsb_list[1:], nsb_list[:-1])] width.append(0.25) @@ -199,10 +216,20 @@ def main(): source = config["data_selection"]["source_name_output"] cluster = config["general"]["cluster"] + "LST dataframe" + config_db = resource_file("database_config.yaml") + + with open( + config_db, "rb" + ) as fc: # "rb" mode opens the file in binary format for reading + config_dict_db = yaml.safe_load(fc) + + LST_h5 = config_dict_db["database_paths"]["LST"] + LST_key = config_dict_db["database_keys"]["LST"] df_LST = pd.read_hdf( - "/fefs/aswg/workspace/elisa.visentin/auto_MCP_PR/observations_LST.h5", - key="joint_obs", + LST_h5, + key=LST_key, ) if source_in is None: @@ -210,7 +237,16 @@ def main(): else: source_list = [source] for source_name in source_list: - ST_NSB_List(target_dir, nsb_list, nsb_limit, source_name, df_LST, ST_list, ST_begin, ST_end) + ST_NSB_List( + target_dir, + nsb_list, + nsb_limit, + source_name, + df_LST, + ST_list, + ST_begin, + ST_end, + ) bash_DL1Stereo_to_DL2(target_dir, source_name, env_name, cluster, RF_dir) list_of_stereo_scripts = np.sort(glob.glob(f"{source_name}_DL1_to_DL2*.sh")) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/__init__.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/__init__.py index d153a2dd..0b49b415 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/__init__.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/__init__.py @@ -11,6 +11,8 @@ directories_generator_real, lists_and_bash_gen_MAGIC, ) +from .DL1_to_DL2 import ST_NSB_List, bash_DL1Stereo_to_DL2 +from .DL2_to_DL3 import DL2_to_DL3, configuration_DL3 from .job_accounting import run_shell from .list_from_h5 import clear_files, list_run, magic_date, split_lst_date from .merge_stereo import MergeStereo @@ -18,12 +20,15 @@ from .stereo_events import bash_stereo, configfile_stereo __all__ = [ + "bash_DL1Stereo_to_DL2", "bash_stereo", "clear_files", "configfile_coincidence", "configfile_stereo", + "configuration_DL3", "config_file_gen", "directories_generator_real", + "DL2_to_DL3", "existing_files", "fix_lists_and_convert", "linking_bash_lst", @@ -37,5 +42,6 @@ "run_shell", "slurm_lines", "split_lst_date", + "ST_NSB_List", "table_magic_runs", ] diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml index d9c8e9c0..0275b724 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/config_auto_MCP.yaml @@ -26,7 +26,7 @@ needed_parameters: # DON'T CHANGE THESE VALUES, UNLESS YOU REALLY KNOW WHAT YOU lstchain_modified_config: true # use_flatfield_heuristic = True to evaluate NSB nsb: [0.5, 1.0, 1.5, 2.0, 2.5, 3.0] lstchain_versions: ["v0.9", "v0.10"] - ST_list: ["ST0321A", "ST0320A", "ST0319A", "ST0318A", "ST0317A", "ST0316A"] - ST_begin: ["2024_05_19","2023_03_10", "2022_12_15", "2022_06_10", "2021_12_30", "2020_10_24"] - ST_end: ["2026_01_01","2024_05_18", "2023_03_09", "2022_08_31", "2022_06_09", "2021_09_29"] + ST_list: ["ST0321A", "ST0320A", "ST0319A", "ST0318A", "M20318A", "ST0318A", "ST0317A", "ST0316A"] + ST_begin: ["2024_05_19","2023_03_10","2023_01_07", "2022_12_15", "2022_09_04", "2022_06_10", "2021_12_30", "2020_10_24"] + ST_end: ["2026_01_01","2024_05_18", "2023_03_09", "2023_01_06", "2022_12_14", "2022_08_31", "2022_06_09", "2021_09_29"] # ST0321 ongoing -> 'service' end date \ No newline at end of file diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/lstchain_version.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/lstchain_version.py index 39a27137..e98fcafb 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/lstchain_version.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/lstchain_version.py @@ -8,15 +8,15 @@ """ +import argparse import glob import os -import argparse + import pandas as pd import yaml from magicctapipe.io import resource_file - __all__ = ["version_lstchain"] @@ -28,6 +28,8 @@ def version_lstchain(df_LST, lstchain_versions): ---------- df_LST : :class:`pandas.DataFrame` Dataframe of the LST-1 observations. + lstchain_versions : list + List of the available lstchain varsions that can be processed by MCP (from older to newer) """ for i, row in df_LST.iterrows(): @@ -83,7 +85,7 @@ def main(): args.config_auto, "rb" ) as f: # "rb" mode opens the file in binary format for reading config = yaml.safe_load(f) - lstchain_versions = config['needed_parameters']['lstchain_versions'] + lstchain_versions = config["needed_parameters"]["lstchain_versions"] config_file = resource_file("database_config.yaml") with open( diff --git a/setup.cfg b/setup.cfg index 35cfa835..b1d595c9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,6 +100,7 @@ console_scripts = coincident_events = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.coincident_events:main create_LST_table = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.database_production.create_LST_table:main dl1_production = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.dl1_production:main + DL1_to_DL2 = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.DL1_to_DL2:main job_accounting = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.job_accounting:main list_from_h5 = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.list_from_h5:main lstchain_version = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.database_production.lstchain_version:main From e64a44c7f6ab5f0fd839b733bb71bc2abf4dfe0f Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Mon, 18 Nov 2024 15:01:51 +0100 Subject: [PATCH 07/16] some fixes --- .../semi_automatic_scripts/DL2_to_DL3.py | 95 +++++++++++++------ setup.cfg | 1 + 2 files changed, 67 insertions(+), 29 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py index 36a86943..434a91cd 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py @@ -3,10 +3,8 @@ to the DL2. It also creates new subdirectories associated with the data level 3. - Usage: $ python new_DL2_to_DL3.py -c configuration_file.yaml - """ import argparse import glob @@ -16,30 +14,39 @@ import joblib import numpy as np +import pandas as pd import yaml from magicctapipe import __version__ from magicctapipe.io import resource_file +from magicctapipe.scripts.lst1_magic.semi_automatic_scripts.clusters import ( + rc_lines, + slurm_lines, +) + +__all__ = ["configuration_DL3", "DL2_to_DL3"] logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.INFO) -def configuration_DL3(target_dir, source_name, config_file): +def configuration_DL3(target_dir, source_name, config_file, ra, dec): """ This function creates the configuration file needed for the DL2 to DL3 conversion Parameters ---------- - ids: list - list of telescope IDs - target_dir: str + target_dir : str Path to the working directory - target_coords: - sorce coordinates - source: str - source name + source_name : str + Source name + config_file : str + Path to MCP configuration file (e.g., resources/config.yaml) + ra : float + Source RA + dec : float + Source Dec """ if config_file == "": @@ -51,8 +58,8 @@ def configuration_DL3(target_dir, source_name, config_file): config_dict = yaml.safe_load(fc) DL3_config = config_dict["dl2_to_dl3"] DL3_config["source_name"] = source_name - # DL3_config['source_ra']=ra - # DL3_config['source_dec']=dec + DL3_config["source_ra"] = ra + DL3_config["source_dec"] = dec conf = { "mc_tel_ids": config_dict["mc_tel_ids"], "dl2_to_dl3": DL3_config, @@ -68,20 +75,30 @@ def configuration_DL3(target_dir, source_name, config_file): yaml.dump(conf, f, default_flow_style=False) -def DL2_to_DL3(target_dir, source, env_name): +def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster): """ This function creates the bash scripts to run lst1_magic_dl2_to_dl3.py on the real data. Parameters ---------- - target_dir: str + target_dir : str Path to the working directory - source: str - source name - env_name: str - conda enviroment name + source : str + Source name + env_name : str + Conda enviroment name + IRF_dir : str + Path to the IRFs + df_LST : :class:`pandas.DataFrame` + Dataframe collecting the LST1 runs (produced by the create_LST_table script) + cluster : str + Cluster system """ - + if cluster != "SLURM": + logger.warning( + "Automatic processing not implemented for the cluster indicated in the config file" + ) + return target_dir = str(target_dir) os.makedirs(target_dir + f"/v{__version__}/{source}/DL3/logs", exist_ok=True) @@ -110,9 +127,10 @@ def DL2_to_DL3(target_dir, source, env_name): print("nsb = ", nsb) period = file.split("/")[-1].split("_")[0] print("period = ", period) - - IRF_dir = f"/fefs/aswg/LST1MAGIC/mc/IRF/{period}/NSB{nsb}/GammaTest/v01.2/g_dyn_0.9_th_glo_0.2/dec_2276/" - + dec = df_LST[df_LST.source == source].iloc[0]["MC_dec"] + IRFdir = f"{IRF_dir}/{period}/NSB{nsb}/dec_{dec}/" + if (not os.path.isdir(IRFdir)) or (len(os.listdir(IRFdir)) == 0): + continue process_name = source output = target_dir + f"/v{__version__}/{source}/DL3" @@ -135,12 +153,12 @@ def DL2_to_DL3(target_dir, source, env_name): "SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n", f"export LOG={output}/logs", "/DL2_to_DL3_${SLURM_ARRAY_TASK_ID}.log\n", - f"conda run -n {env_name} lst1_magic_dl2_to_dl3 --input-file-dl2 $SAMPLE --input-dir-irf {IRF_dir} --output-dir {output} --config-file {target_dir}/config_DL3.yaml >$LOG 2>&1\n\n", + f"conda run -n {env_name} lst1_magic_dl2_to_dl3 --input-file-dl2 $SAMPLE --input-dir-irf {IRFdir} --output-dir {output} --config-file {target_dir}/config_DL3.yaml >$LOG 2>&1\n\n", ] + rc ) with open( - f'{source}_DL3_{nsb}_{period}_{night.split("/")[-1]}.sh', "w" + f'{source}_DL2_to_DL3_{nsb}_{period}_{night.split("/")[-1]}.sh', "w" ) as f: f.writelines(lines) @@ -166,8 +184,8 @@ def main(): ) as f: # "rb" mode opens the file in binary format for reading config = yaml.safe_load(f) - telescope_ids = list(config["mc_tel_ids"].values()) target_dir = Path(config["directories"]["workspace_dir"]) + IRF_dir = config["directories"]["IRF"] print("***** Generating file config_DL3.yaml...") print("***** This file can be found in ", target_dir) @@ -178,6 +196,20 @@ def main(): config_file = config["general"]["base_config_file"] cluster = config["general"]["cluster"] + config_db = resource_file("database_config.yaml") + + with open( + config_db, "rb" + ) as fc: # "rb" mode opens the file in binary format for reading + config_dict_db = yaml.safe_load(fc) + + LST_h5 = config_dict_db["database_paths"]["LST"] + LST_key = config_dict_db["database_keys"]["LST"] + df_LST = pd.read_hdf( + LST_h5, + key=LST_key, + ) + # cp the .txt files from DL1 stereo anaysis to be used again. DL1stereo_Nihts = np.sort( glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*") @@ -195,12 +227,17 @@ def main(): else: source_list = [source] for source_name in source_list: - configuration_DL3(telescope_ids, target_dir, target_coords, source_name) - DL2_to_DL3(target_dir, source_name, env_name) - list_of_stereo_scripts = np.sort(glob.glob(f"{source_name}_DL3*.sh")) + wobble_offset = df_LST[df_LST.source == source].iloc[0]["wobble_offset"] + if str(wobble_offset) != "0.4": + continue + ra = df_LST[df_LST.source == source].iloc[0]["ra"] + dec = df_LST[df_LST.source == source].iloc[0]["dec"] + configuration_DL3(target_dir, source_name, config_file, ra, dec) + DL2_to_DL3(target_dir, source_name, env_name, IRF_dir, df_LST, cluster) + list_of_stereo_scripts = np.sort(glob.glob(f"{source_name}_DL2_to_DL3*.sh")) print(list_of_stereo_scripts) if len(list_of_stereo_scripts) < 1: - logger.warning("No bash scripts for real data") + logger.warning(f"No bash scripts for {source_name}") continue launch_jobs = "" for n, run in enumerate(list_of_stereo_scripts): diff --git a/setup.cfg b/setup.cfg index b1d595c9..b306d5e2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -101,6 +101,7 @@ console_scripts = create_LST_table = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.database_production.create_LST_table:main dl1_production = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.dl1_production:main DL1_to_DL2 = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.DL1_to_DL2:main + DL2_to_DL3 = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.DL2_to_DL3:main job_accounting = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.job_accounting:main list_from_h5 = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.list_from_h5:main lstchain_version = magicctapipe.scripts.lst1_magic.semi_automatic_scripts.database_production.lstchain_version:main From 3d4cc911b90d2a6d787baf6e85b75c73b3bc95b2 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Wed, 27 Nov 2024 15:04:16 +0100 Subject: [PATCH 08/16] fixes --- .../semi_automatic_scripts/DL1_to_DL2.py | 39 ++++++------------- .../semi_automatic_scripts/DL2_to_DL3.py | 21 ++++------ 2 files changed, 20 insertions(+), 40 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py index bb8eb677..da4ab5cd 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py @@ -58,20 +58,17 @@ def ST_NSB_List( List of ending dates for the observation periods """ - "Loops over all runs of all nights" + # Loops over all runs of all nights Nights_list = np.sort( glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*") ) for night in Nights_list: - "Night period" + # Night period night_date = night.split("/")[-1] - print("night", night_date) date_lst = night_date[:4] + "_" + night_date[4:6] + "_" + night_date[6:8] - print(date_lst) date_magic = datetime.datetime.strptime( date_lst, "%Y_%m_%d" ) + datetime.timedelta(days=1) - print("dates", date_lst, str(date_magic)) for p in range(len(ST_begin)): if (date_magic >= datetime.datetime.strptime(ST_begin[p], "%Y_%m_%d")) and ( date_magic <= datetime.datetime.strptime(ST_end[p], "%Y_%m_%d") @@ -80,19 +77,15 @@ def ST_NSB_List( Run_list = glob.glob(f"{night}/*.h5") for Run in Run_list: - "getting the run NSB" - print(Run) + # getting the run NSB run_str = Run.split("/")[-1].split(".")[1] - print("run", run_str) run_LST_id = run_str.lstrip("Run") - print("run_lst", run_LST_id) nsb = df_LST[df_LST["LST1_run"] == run_LST_id]["nsb"].tolist()[0] - print("nsb", nsb) - "rounding the NSB to the nearest MC nsb value" + # rounding the NSB to the nearest MC nsb value for j in range(0, len(nsb_list) - 1): if (nsb <= nsb_limit[j + 1]) & (nsb > nsb_limit[j]): nsb = nsb_list[j] - "Writing on output .txt file" + # Writing on output .txt file if nsb <= nsb_limit[-1]: with open(f"{night}/logs/{period}_{nsb}.txt", "a+") as file: file.write(f"{Run}\n") @@ -138,9 +131,9 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name, cluster, RF_dir, df_LST) if process_size < 0: continue nsb = file.split("/")[-1].split("_")[-1][:3] - p = file.split("/")[-1].split("_")[0] + period = file.split("/")[-1].split("_")[0] dec = df_LST[df_LST.source == source].iloc[0]["MC_dec"] - RFdir = f"{RF_dir}/{p}/NSB{nsb}/dec_{dec}/" + RFdir = f"{RF_dir}/{period}/NSB{nsb}/dec_{dec}/" if (not os.path.isdir(RFdir)) or (len(os.listdir(RFdir)) == 0): continue slurm = slurm_lines( @@ -178,14 +171,7 @@ def main(): Here we read the config_general.yaml file and call the functions defined above. """ - parser = argparse.ArgumentParser()[ - "2026_01_01", - "2024_05_18", - "2023_03_09", - "2022_08_31", - "2022_06_09", - "2021_09_29", - ] + parser = argparse.ArgumentParser() parser.add_argument( "--config-file", "-c", @@ -217,7 +203,7 @@ def main(): cluster = config["general"]["cluster"] - "LST dataframe" + # LST dataframe config_db = resource_file("database_config.yaml") with open( @@ -249,13 +235,12 @@ def main(): ) bash_DL1Stereo_to_DL2(target_dir, source_name, env_name, cluster, RF_dir) - list_of_stereo_scripts = np.sort(glob.glob(f"{source_name}_DL1_to_DL2*.sh")) - print(list_of_stereo_scripts) - if len(list_of_stereo_scripts) < 1: + list_of_dl2_scripts = np.sort(glob.glob(f"{source_name}_DL1_to_DL2*.sh")) + if len(list_of_dl2_scripts) < 1: logger.warning(f"No bash scripts for {source_name}") continue launch_jobs = "" - for n, run in enumerate(list_of_stereo_scripts): + for n, run in enumerate(list_of_dl2_scripts): launch_jobs += (" && " if n > 0 else "") + f"sbatch {run}" os.system(launch_jobs) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py index 434a91cd..6835269b 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py @@ -99,9 +99,8 @@ def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster): "Automatic processing not implemented for the cluster indicated in the config file" ) return - target_dir = str(target_dir) - - os.makedirs(target_dir + f"/v{__version__}/{source}/DL3/logs", exist_ok=True) + + os.makedirs(f"{target_dir}/v{__version__}/{source}/DL3/logs", exist_ok=True) # Loop over all nights Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL2/*")) @@ -124,15 +123,13 @@ def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster): g.writelines(run_new) nsb = file.split("/")[-1].split("_")[-1][:3] - print("nsb = ", nsb) period = file.split("/")[-1].split("_")[0] - print("period = ", period) dec = df_LST[df_LST.source == source].iloc[0]["MC_dec"] IRFdir = f"{IRF_dir}/{period}/NSB{nsb}/dec_{dec}/" if (not os.path.isdir(IRFdir)) or (len(os.listdir(IRFdir)) == 0): continue process_name = source - output = target_dir + f"/v{__version__}/{source}/DL3" + output = f"{target_dir}/v{__version__}/{source}/DL3" slurm = slurm_lines( queue="short", @@ -211,13 +208,12 @@ def main(): ) # cp the .txt files from DL1 stereo anaysis to be used again. - DL1stereo_Nihts = np.sort( + DL1stereo_Nights = np.sort( glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*") ) - for night in DL1stereo_Nihts: + for night in DL1stereo_Nights: File_list = glob.glob(f"{night}/logs/ST*.txt") night_date = night.split("/")[-1] - print("night date ", night_date) for file in File_list: cp_dir = f"{target_dir}/v{__version__}/{source}/DL2/{night_date}" os.system(f"cp {file} {cp_dir}") @@ -234,13 +230,12 @@ def main(): dec = df_LST[df_LST.source == source].iloc[0]["dec"] configuration_DL3(target_dir, source_name, config_file, ra, dec) DL2_to_DL3(target_dir, source_name, env_name, IRF_dir, df_LST, cluster) - list_of_stereo_scripts = np.sort(glob.glob(f"{source_name}_DL2_to_DL3*.sh")) - print(list_of_stereo_scripts) - if len(list_of_stereo_scripts) < 1: + list_of_dl3_scripts = np.sort(glob.glob(f"{source_name}_DL2_to_DL3*.sh")) + if len(list_of_dl3_scripts) < 1: logger.warning(f"No bash scripts for {source_name}") continue launch_jobs = "" - for n, run in enumerate(list_of_stereo_scripts): + for n, run in enumerate(list_of_dl3_scripts): launch_jobs += (" && " if n > 0 else "") + f"sbatch {run}" os.system(launch_jobs) From d4375a6ffa2174a23f9287af079bf1092f4c7f11 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Wed, 27 Nov 2024 15:07:43 +0100 Subject: [PATCH 09/16] lint --- .../scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py index 6835269b..59fc4612 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py @@ -99,7 +99,7 @@ def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster): "Automatic processing not implemented for the cluster indicated in the config file" ) return - + os.makedirs(f"{target_dir}/v{__version__}/{source}/DL3/logs", exist_ok=True) # Loop over all nights From c4dd28c70305422155d53b45c98fb750f53bb1b5 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin <121040436+Elisa-Visentin@users.noreply.github.com> Date: Thu, 28 Nov 2024 15:41:38 +0100 Subject: [PATCH 10/16] Update nsb_level.py --- .../semi_automatic_scripts/database_production/nsb_level.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/nsb_level.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/nsb_level.py index f483f8bb..aeabe82e 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/nsb_level.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/database_production/nsb_level.py @@ -21,8 +21,6 @@ from magicctapipe.io import resource_file from magicctapipe.scripts.lst1_magic.semi_automatic_scripts.clusters import slurm_lines -from .lstchain_version import lstchain_versions - __all__ = ["bash_scripts"] logger = logging.getLogger(__name__) @@ -114,7 +112,7 @@ def main(): env_name = config["general"]["env_name"] cluster = config["general"]["cluster"] - + lstchain_versions = config["needed_parameters"]["lstchain_versions"] df_LST = pd.read_hdf( LST_h5, key=LST_key, From 7165212f0fba8abe86a0907af685ab2a7dea24d4 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin <121040436+Elisa-Visentin@users.noreply.github.com> Date: Thu, 28 Nov 2024 16:08:47 +0100 Subject: [PATCH 11/16] fix bug --- .../lst1_magic/semi_automatic_scripts/DL1_to_DL2.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py index da4ab5cd..37c6da16 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py @@ -91,7 +91,7 @@ def ST_NSB_List( file.write(f"{Run}\n") -def bash_DL1Stereo_to_DL2(target_dir, source, env_name, cluster, RF_dir, df_LST): +def bash_DL1Stereo_to_DL2(target_dir, source, env_name, cluster, RF_dir, df_LST, MC_v): """ This function generates the bashscript for running the DL1Stereo to DL2 analisys. @@ -109,6 +109,8 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name, cluster, RF_dir, df_LST) Path to the RFs df_LST : :class:`pandas.DataFrame` Dataframe collecting the LST1 runs (produced by the create_LST_table script) + MC_v : str + Version of MC processing """ if cluster != "SLURM": logger.warning( @@ -133,7 +135,8 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name, cluster, RF_dir, df_LST) nsb = file.split("/")[-1].split("_")[-1][:3] period = file.split("/")[-1].split("_")[0] dec = df_LST[df_LST.source == source].iloc[0]["MC_dec"] - RFdir = f"{RF_dir}/{period}/NSB{nsb}/dec_{dec}/" + dec = str(dec).replace(".", "") + RFdir = f"{RF_dir}/{period}/NSB{nsb}/{MC_v}/dec_{dec}/" if (not os.path.isdir(RFdir)) or (len(os.listdir(RFdir)) == 0): continue slurm = slurm_lines( @@ -200,6 +203,7 @@ def main(): nsb_limit.insert(0, 0) source_in = config["data_selection"]["source_name_database"] source = config["data_selection"]["source_name_output"] + MC_v = config["directories"]["MC_version"] cluster = config["general"]["cluster"] @@ -234,7 +238,7 @@ def main(): ST_end, ) - bash_DL1Stereo_to_DL2(target_dir, source_name, env_name, cluster, RF_dir) + bash_DL1Stereo_to_DL2(target_dir, source_name, env_name, cluster, RF_dir, df_LST, MC_v) list_of_dl2_scripts = np.sort(glob.glob(f"{source_name}_DL1_to_DL2*.sh")) if len(list_of_dl2_scripts) < 1: logger.warning(f"No bash scripts for {source_name}") From 13de13eea38cfa6462c93087f134da489432b27a Mon Sep 17 00:00:00 2001 From: Elisa-Visentin <121040436+Elisa-Visentin@users.noreply.github.com> Date: Sat, 30 Nov 2024 17:07:49 +0100 Subject: [PATCH 12/16] Mem. RF --- .../scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py index 37c6da16..4af45637 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py @@ -143,7 +143,7 @@ def bash_DL1Stereo_to_DL2(target_dir, source, env_name, cluster, RF_dir, df_LST, queue="short", job_name=f"{process_name}_DL1_to_DL2", array=process_size, - mem="2g", + mem="50g", out_name=f"{target_dir}/v{__version__}/{source}/DL2/{night.split('/')[-1]}/logs/slurm-%x.%A_%a", ) rc = rc_lines( From 42171662d0308a3d727f25c8cc015aae33a32dd6 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin <121040436+Elisa-Visentin@users.noreply.github.com> Date: Sat, 30 Nov 2024 17:14:52 +0100 Subject: [PATCH 13/16] Bug fixes --- .../semi_automatic_scripts/DL2_to_DL3.py | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py index 59fc4612..ec9394c8 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py @@ -58,8 +58,8 @@ def configuration_DL3(target_dir, source_name, config_file, ra, dec): config_dict = yaml.safe_load(fc) DL3_config = config_dict["dl2_to_dl3"] DL3_config["source_name"] = source_name - DL3_config["source_ra"] = ra - DL3_config["source_dec"] = dec + DL3_config["source_ra"] = float(ra) + DL3_config["source_dec"] = float(dec) conf = { "mc_tel_ids": config_dict["mc_tel_ids"], "dl2_to_dl3": DL3_config, @@ -75,7 +75,7 @@ def configuration_DL3(target_dir, source_name, config_file, ra, dec): yaml.dump(conf, f, default_flow_style=False) -def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster): +def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster, MC_v): """ This function creates the bash scripts to run lst1_magic_dl2_to_dl3.py on the real data. @@ -93,6 +93,8 @@ def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster): Dataframe collecting the LST1 runs (produced by the create_LST_table script) cluster : str Cluster system + MC_v : str + Version of MC processing """ if cluster != "SLURM": logger.warning( @@ -106,7 +108,7 @@ def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster): Nights_list = np.sort(glob.glob(f"{target_dir}/v{__version__}/{source}/DL2/*")) for night in Nights_list: # Loop over every run: - File_list = np.sort(glob.glob(f"{night}/*.txt")) + File_list = np.sort(glob.glob(f"{night}/logs/ST*.txt")) for file in File_list: with open(file, "r") as f: runs = f.readlines() @@ -125,7 +127,8 @@ def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster): nsb = file.split("/")[-1].split("_")[-1][:3] period = file.split("/")[-1].split("_")[0] dec = df_LST[df_LST.source == source].iloc[0]["MC_dec"] - IRFdir = f"{IRF_dir}/{period}/NSB{nsb}/dec_{dec}/" + dec = str(dec).replace(".", "") + IRFdir = f"{IRF_dir}/{period}/NSB{nsb}/GammaTest/{MC_v}/g_dyn_0.9_th_glo_0.2/dec_{dec}/" if (not os.path.isdir(IRFdir)) or (len(os.listdir(IRFdir)) == 0): continue process_name = source @@ -150,7 +153,7 @@ def DL2_to_DL3(target_dir, source, env_name, IRF_dir, df_LST, cluster): "SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n", f"export LOG={output}/logs", "/DL2_to_DL3_${SLURM_ARRAY_TASK_ID}.log\n", - f"conda run -n {env_name} lst1_magic_dl2_to_dl3 --input-file-dl2 $SAMPLE --input-dir-irf {IRFdir} --output-dir {output} --config-file {target_dir}/config_DL3.yaml >$LOG 2>&1\n\n", + f"conda run -n {env_name} lst1_magic_dl2_to_dl3 --input-file-dl2 $SAMPLE --input-dir-irf {IRFdir} --output-dir {output} --config-file {target_dir}/v{__version__}/{source}/config_DL3.yaml >$LOG 2>&1\n\n", ] + rc ) @@ -192,6 +195,7 @@ def main(): env_name = config["general"]["env_name"] config_file = config["general"]["base_config_file"] cluster = config["general"]["cluster"] + MC_v = config["directories"]["MC_version"] config_db = resource_file("database_config.yaml") @@ -207,29 +211,29 @@ def main(): key=LST_key, ) - # cp the .txt files from DL1 stereo anaysis to be used again. - DL1stereo_Nights = np.sort( - glob.glob(f"{target_dir}/v{__version__}/{source}/DL1Stereo/Merged/*") - ) - for night in DL1stereo_Nights: - File_list = glob.glob(f"{night}/logs/ST*.txt") - night_date = night.split("/")[-1] - for file in File_list: - cp_dir = f"{target_dir}/v{__version__}/{source}/DL2/{night_date}" - os.system(f"cp {file} {cp_dir}") - if source_in is None: source_list = joblib.load("list_sources.dat") else: source_list = [source] for source_name in source_list: - wobble_offset = df_LST[df_LST.source == source].iloc[0]["wobble_offset"] - if str(wobble_offset) != "0.4": + wobble_offset = df_LST[df_LST.source == source_name].iloc[0]["wobble_offset"] + if str(wobble_offset) != "[0.40]": continue - ra = df_LST[df_LST.source == source].iloc[0]["ra"] - dec = df_LST[df_LST.source == source].iloc[0]["dec"] + # cp the .txt files from DL1 stereo anaysis to be used again. + DL1stereo_Nights = np.sort( + glob.glob(f"{target_dir}/v{__version__}/{source_name}/DL1Stereo/Merged/*") + ) + for night in DL1stereo_Nights: + File_list = glob.glob(f"{night}/logs/ST*.txt") + night_date = night.split("/")[-1] + for file in File_list: + cp_dir = f"{target_dir}/v{__version__}/{source_name}/DL2/{night_date}" + os.system(f"cp {file} {cp_dir}") + + ra = df_LST[df_LST.source == source_name].iloc[0]["ra"] + dec = df_LST[df_LST.source == source_name].iloc[0]["dec"] configuration_DL3(target_dir, source_name, config_file, ra, dec) - DL2_to_DL3(target_dir, source_name, env_name, IRF_dir, df_LST, cluster) + DL2_to_DL3(target_dir, source_name, env_name, IRF_dir, df_LST, cluster, MC_v) list_of_dl3_scripts = np.sort(glob.glob(f"{source_name}_DL2_to_DL3*.sh")) if len(list_of_dl3_scripts) < 1: logger.warning(f"No bash scripts for {source_name}") From 7d34d8691121a067d74e28b60a76ef7c71d897e0 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin <121040436+Elisa-Visentin@users.noreply.github.com> Date: Sat, 30 Nov 2024 20:29:19 +0100 Subject: [PATCH 14/16] Update DL1_to_DL2.py --- .../scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py index 4af45637..a1515138 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL1_to_DL2.py @@ -238,7 +238,9 @@ def main(): ST_end, ) - bash_DL1Stereo_to_DL2(target_dir, source_name, env_name, cluster, RF_dir, df_LST, MC_v) + bash_DL1Stereo_to_DL2( + target_dir, source_name, env_name, cluster, RF_dir, df_LST, MC_v + ) list_of_dl2_scripts = np.sort(glob.glob(f"{source_name}_DL1_to_DL2*.sh")) if len(list_of_dl2_scripts) < 1: logger.warning(f"No bash scripts for {source_name}") From 711db046de50c36f5072f6856ab16a1503973f9e Mon Sep 17 00:00:00 2001 From: Elisa-Visentin <121040436+Elisa-Visentin@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:39:46 +0100 Subject: [PATCH 15/16] bug fix --- .../scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py index ec9394c8..e6d6f9ee 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py @@ -227,7 +227,7 @@ def main(): File_list = glob.glob(f"{night}/logs/ST*.txt") night_date = night.split("/")[-1] for file in File_list: - cp_dir = f"{target_dir}/v{__version__}/{source_name}/DL2/{night_date}" + cp_dir = f"{target_dir}/v{__version__}/{source_name}/DL2/{night_date}/logs" os.system(f"cp {file} {cp_dir}") ra = df_LST[df_LST.source == source_name].iloc[0]["ra"] From d74aa42c495cb3dc2c89129b90cbf22b04a1f6f8 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin <121040436+Elisa-Visentin@users.noreply.github.com> Date: Fri, 6 Dec 2024 11:49:51 +0100 Subject: [PATCH 16/16] Update DL2_to_DL3.py --- .../scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py index e6d6f9ee..28bb0a48 100644 --- a/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py +++ b/magicctapipe/scripts/lst1_magic/semi_automatic_scripts/DL2_to_DL3.py @@ -58,8 +58,8 @@ def configuration_DL3(target_dir, source_name, config_file, ra, dec): config_dict = yaml.safe_load(fc) DL3_config = config_dict["dl2_to_dl3"] DL3_config["source_name"] = source_name - DL3_config["source_ra"] = float(ra) - DL3_config["source_dec"] = float(dec) + DL3_config["source_ra"] = f'{ra} deg' + DL3_config["source_dec"] = f'{dec} deg' conf = { "mc_tel_ids": config_dict["mc_tel_ids"], "dl2_to_dl3": DL3_config,