diff --git a/.gitignore b/.gitignore index f327264..31704f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc plot/ -.env \ No newline at end of file +.env +data/ \ No newline at end of file diff --git a/README.md b/README.md index 00fe81c..d542fc7 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ DGMR_PLOT_PATH="" # If empty, forecast GIFs will be sav ### Downloading Data with Meteo-France's API -Simply run the script `./dgmr/download_data.sh` to download the latest radar images. +Simply run the script `python download_data.py` to download the latest radar image. ### Setting up a Cron Job for Automated Data Download @@ -81,9 +81,9 @@ To automate the data downloading process using a cron job, follow these steps: 1. Open your terminal and type `crontab -e` to edit the cron table. -2. Add a new line at the end of the file to define the cron job. The general syntax for a cron task is: +2. Add a new line at the end of the file to define the cron job. The syntax for the cron task is: ```bash - */5 * * * * /dgmr/download_data.sh + */5 * * * * micromamba activate dgmr && cd && python download_data.py >> cron_output.txt 2>&1 ``` ### Making Forecasts diff --git a/dgmr/data.py b/dgmr/data.py index f2276a7..b68ccc5 100644 --- a/dgmr/data.py +++ b/dgmr/data.py @@ -12,7 +12,7 @@ def get_list_files(date: dt.datetime) -> List[Path]: delta = dt.timedelta(minutes=TIMESTEP) dates = [date + i * delta for i in range(-INPUT_STEPS + 1, 1)] - filenames = [d.strftime("%Y_%m_%d_%H_%M.h5") for d in dates] + filenames = [d.strftime("%Y-%m-%d_%Hh%M.h5") for d in dates] return [DATA_PATH / f for f in filenames] diff --git a/download_data.py b/download_data.py new file mode 100644 index 0000000..d071fca --- /dev/null +++ b/download_data.py @@ -0,0 +1,58 @@ +import datetime as dt +import os +import time +from pathlib import Path + +import requests +from dotenv import load_dotenv + +# Load environment variables from .env file if it exists +dotenv_path = Path(".env") +if dotenv_path.is_file(): + load_dotenv(dotenv_path) + +METEO_FRANCE_DATA_PATH = Path(os.getenv("METEO_FRANCE_DATA_PATH", "data")) +METEO_FRANCE_DATA_PATH.mkdir(parents=True, exist_ok=True) + +METEO_FRANCE_API_URL = ( + "https://public-api.meteofrance.fr/public/DPRadar/v1/mosaiques/" + "METROPOLE/observations/LAME_D_EAU/produit?maille=500" +) +METEO_FRANCE_API_KEY = os.getenv("METEO_FRANCE_API_KEY") + + +max_retries = 20 +retry_delay = 3 +retry_count = 0 +success = False + +while retry_count < max_retries: + response = requests.get( + METEO_FRANCE_API_URL, + headers={ + "accept": "application/octet-stream+gzip", + "apikey": METEO_FRANCE_API_KEY, + }, + ) + + if response.status_code == 200: + now = dt.datetime.now(dt.timezone.utc) + now = now - dt.timedelta( # round date to 5 minutes + minutes=now.minute % 5, + seconds=now.second, + microseconds=now.microsecond, + ) + output_file = METEO_FRANCE_DATA_PATH / now.strftime("%Y-%m-%d_%Hh%M.h5") + with open(output_file, "wb") as file: + file.write(response.content) + success = True + break + else: + print(f"Attempt {retry_count + 1} failed. Retry in {retry_delay} seconds...") + retry_count += 1 + time.sleep(retry_delay) + +if success: + print(f"Downloaded successfully {output_file.name}.") +else: + print(f"Download failed after {max_retries} attempts.") diff --git a/run_download.sh b/run_download.sh new file mode 100755 index 0000000..ebebe13 --- /dev/null +++ b/run_download.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Activate the micromamba environment +source /usr/local/bin/micromamba +micromamba activate dgmr + +python download_data.py \ No newline at end of file