Download data with python instead of bash

.gitignore

@@ -1,3 +1,4 @@
 *.pyc
 plot/
-.env
+.env
+data/

README.md

@@ -71,7 +71,7 @@ DGMR_PLOT_PATH="<path_to_save_the_plots>"  # If empty, forecast GIFs will be sav
 
 ### Downloading Data with Meteo-France's API
 
-Simply run the script `./dgmr/download_data.sh` to download the latest radar images.
+Simply run the script `python download_data.py` to download the latest radar image.
 
 ### Setting up a Cron Job for Automated Data Download
 
@@ -81,9 +81,9 @@ To automate the data downloading process using a cron job, follow these steps:
 
 1. Open your terminal and type `crontab -e` to edit the cron table.
 
-2. Add a new line at the end of the file to define the cron job. The general syntax for a cron task is:
+2. Add a new line at the end of the file to define the cron job. The syntax for the cron task is:
     ```bash
-    */5 * * * * <ABSOLUTE_PATH_TO_REPO>/dgmr/download_data.sh
+    */5 * * * * micromamba activate dgmr && cd <ABSOLUTE_PATH_TO_REPO> && python download_data.py >> cron_output.txt 2>&1
     ```
 
 ### Making Forecasts

dgmr/data.py

@@ -12,7 +12,7 @@
 def get_list_files(date: dt.datetime) -> List[Path]:
     delta = dt.timedelta(minutes=TIMESTEP)
     dates = [date + i * delta for i in range(-INPUT_STEPS + 1, 1)]
-    filenames = [d.strftime("%Y_%m_%d_%H_%M.h5") for d in dates]
+    filenames = [d.strftime("%Y-%m-%d_%Hh%M.h5") for d in dates]
     return [DATA_PATH / f for f in filenames]
 
 

download_data.py

@@ -0,0 +1,58 @@
+import datetime as dt
+import os
+import time
+from pathlib import Path
+
+import requests
+from dotenv import load_dotenv
+
+# Load environment variables from .env file if it exists
+dotenv_path = Path(".env")
+if dotenv_path.is_file():
+    load_dotenv(dotenv_path)
+
+METEO_FRANCE_DATA_PATH = Path(os.getenv("METEO_FRANCE_DATA_PATH", "data"))
+METEO_FRANCE_DATA_PATH.mkdir(parents=True, exist_ok=True)
+
+METEO_FRANCE_API_URL = (
+    "https://public-api.meteofrance.fr/public/DPRadar/v1/mosaiques/"
+    "METROPOLE/observations/LAME_D_EAU/produit?maille=500"
+)
+METEO_FRANCE_API_KEY = os.getenv("METEO_FRANCE_API_KEY")
+
+
+max_retries = 20
+retry_delay = 3
+retry_count = 0
+success = False
+
+while retry_count < max_retries:
+    response = requests.get(
+        METEO_FRANCE_API_URL,
+        headers={
+            "accept": "application/octet-stream+gzip",
+            "apikey": METEO_FRANCE_API_KEY,
+        },
+    )
+
+    if response.status_code == 200:
+        now = dt.datetime.now(dt.timezone.utc)
+        now = now - dt.timedelta(  # round date to 5 minutes
+            minutes=now.minute % 5,
+            seconds=now.second,
+            microseconds=now.microsecond,
+        )
+        output_file = METEO_FRANCE_DATA_PATH / now.strftime("%Y-%m-%d_%Hh%M.h5")
+        with open(output_file, "wb") as file:
+            file.write(response.content)
+        success = True
+        break
+    else:
+        print(f"Attempt {retry_count + 1} failed. Retry in {retry_delay} seconds...")
+        retry_count += 1
+        time.sleep(retry_delay)
+
+if success:
+    print(f"Downloaded successfully {output_file.name}.")
+else:
+    print(f"Download failed after {max_retries} attempts.")

run_download.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Activate the micromamba environment
+source /usr/local/bin/micromamba
+micromamba activate dgmr
+
+python download_data.py