From 673faed075b6f7771e650b5314f250cf8d30e9a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michael=20J=C3=A4hn?= <mjaehn@ethz.ch>
Date: Fri, 21 Jun 2024 15:30:28 +0200
Subject: [PATCH] Initial commit

---
 check_datasets.py | 32 ++++++++++++++++++++++++++++++++
 requirements.txt  |  2 ++
 2 files changed, 34 insertions(+)
 create mode 100644 check_datasets.py
 create mode 100644 requirements.txt

diff --git a/check_datasets.py b/check_datasets.py
new file mode 100644
index 00000000..c39c0e27
--- /dev/null
+++ b/check_datasets.py
@@ -0,0 +1,32 @@
+import requests
+import json
+import re
+import boto3
+
+def download_json(url):
+    response = requests.get(url)
+    return json.loads(response.text)
+
+def read_markdown(file_path):
+    with open(file_path, 'r') as file:
+        return file.read()
+
+def write_markdown(file_path, content):
+    with open(file_path, 'w') as file:
+        file.write(content)
+
+def update_markdown(markdown, json_data):
+    # This is a placeholder. You'll need to replace this with the actual logic
+    # to update the markdown content based on the JSON data.
+    return re.sub(r'(Size: )(\d+ TB)', r'\1' + str(json_data['size']) + ' TB', markdown)
+
+# Main part of the script
+json_data1 = download_json('https://zephyr-c2sm.s3.eu-central-1.amazonaws.com/datasets.json')
+json_data2 = download_json('https://zephyr-c2sm.s3.eu-central-1.amazonaws.com/file_tree_cordex_noindent.json')
+
+markdown = read_markdown('climate_model_data.md')
+
+updated_markdown = update_markdown(markdown, json_data1)
+updated_markdown = update_markdown(updated_markdown, json_data2)
+
+write_markdown('climate_model_data.md', updated_markdown)
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 00000000..ed7d4492
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,2 @@
+requests==2.26.0
+boto3==1.18.33