janitor: automatically refresh branches.json

For SNAPSHOTS the targets and architectures likely change over time.
Automatically create the overview file to lower manual maintenance.

Signed-off-by: Paul Spooren <[email protected]>

asu/api.py

@@ -59,6 +59,11 @@ def api_latest():
     return redirect("/json/v1/latest.json")
 
 
+@bp.route("/overview")
+def api_v1_overview():
+    return redirect("/json/v1/overview.json")
+
+
 def validate_packages(req):
     if req.get("packages_versions") and not req.get("packages"):
         req["packages"] = req["packages_versions"].keys()
@@ -235,11 +240,6 @@ def return_job_v1(job):
 
 
 # legacy offering /api/overview
-@bp.route("/overview")
-def api_v1_overview():
-    return jsonify(current_app.config["OVERVIEW"])
-
-
 def api_v1_build_get(request_hash):
     job = get_queue().fetch_job(request_hash)
     if not job:

asu/asu.py

@@ -1,6 +1,3 @@
-import json
-
-# from msilib.schema import Registry
 from os import getenv
 from pathlib import Path
 
@@ -95,48 +92,13 @@ def store_path(path="index.html"):
             filter(lambda b: b.get("enabled"), app.config["BRANCHES"].values()),
         )
     )
-    latest = list(
-        map(
-            lambda b: b["versions"][0],
-            filter(
-                lambda b: b.get("enabled"),
-                app.config["BRANCHES"].values(),
-            ),
-        )
-    )
-
-    app.config["OVERVIEW"] = {
-        "latest": latest,
-        "branches": branches,
-        "server": {
-            "version": __version__,
-            "contact": "[email protected]",
-            "allow_defaults": app.config["ALLOW_DEFAULTS"],
-        },
-    }
-
-    # legacy
-    (app.config["JSON_PATH"] / "branches.json").write_text(
-        json.dumps(list(branches.values()))
-    )
-
-    # tdb
-    (app.config["JSON_PATH"] / "latest.json").write_text(json.dumps({"latest": latest}))
-
-    (app.config["JSON_PATH"] / "overview.json").write_text(
-        json.dumps(
-            app.config["OVERVIEW"],
-            indent=2,
-        )
-    )
 
     @app.route("/")
     def overview():
         return render_template(
             "overview.html",
             branches=branches,
             defaults=app.config["ALLOW_DEFAULTS"],
-            latest=latest,
             version=__version__,
         )
 

asu/janitor.py

@@ -11,6 +11,7 @@
 from rq.exceptions import NoSuchJobError
 from rq.registry import FinishedJobRegistry
 
+from asu import __version__
 from asu.common import get_redis, is_modified
 
 bp = Blueprint("janitor", __name__)
@@ -86,8 +87,6 @@ def get_packages_arch_repo(branch, arch, repo):
 
 
 def update_branch(branch):
-    r = get_redis()
-
     version_path = branch["path"].format(version=branch["versions"][0])
     targets = list(
         filter(
@@ -374,6 +373,64 @@ def update_target_profiles(branch: dict, version: str, target: str) -> str:
     return metadata["arch_packages"]
 
 
+def update_meta_json():
+    latest = list(
+        map(
+            lambda b: b["versions"][0],
+            filter(
+                lambda b: b.get("enabled"),
+                current_app.config["BRANCHES"].values(),
+            ),
+        )
+    )
+
+    branches = dict(
+        map(
+            lambda b: (
+                b["name"],
+                {
+                    **b,
+                    "targets": dict(
+                        map(
+                            lambda a: (a[0].decode(), a[1].decode()),
+                            get_redis().hgetall(f"architecture:{b['name']}").items(),
+                        )
+                    ),
+                },
+            ),
+            filter(
+                lambda b: b.get("enabled"),
+                current_app.config["BRANCHES"].values(),
+            ),
+        )
+    )
+
+    current_app.config["OVERVIEW"] = {
+        "latest": latest,
+        "branches": branches,
+        "server": {
+            "version": __version__,
+            "contact": "[email protected]",
+            "allow_defaults": current_app.config["ALLOW_DEFAULTS"],
+        },
+    }
+
+    (current_app.config["JSON_PATH"] / "overview.json").write_text(
+        json.dumps(
+            current_app.config["OVERVIEW"],
+            indent=2,
+        )
+    )
+
+    (current_app.config["JSON_PATH"] / "branches.json").write_text(
+        json.dumps(list(branches.values()))
+    )
+
+    (current_app.config["JSON_PATH"] / "latest.json").write_text(
+        json.dumps({"latest": latest})
+    )
+
+
 @bp.cli.command("update")
 @click.option("-i", "--interval", default=10, type=int)
 def update(interval):
@@ -397,6 +454,8 @@ def update(interval):
             current_app.logger.info(f"Update {branch['name']}")
             update_branch(branch)
 
+        update_meta_json()
+
         if interval > 0:
             current_app.logger.info(f"Next reload in { interval } minutes")
             sleep(interval * 60)

asu/templates/overview.html

@@ -66,7 +66,6 @@ <h2>Sysupgrade Server Configuration:</h2>
 
             <p>
                 <b>Allow custom UCI defaults:</b> {{ defaults }} <br/>
-                <b>Latest versions:</b> {{ latest }}
             </p>
         </div>
     </div>

tests/test_api.py

@@ -1,11 +1,6 @@
 import pytest
 
 
-def test_api_version(client, app):
-    response = client.get("/api/branches")
-    assert response.status == "200 OK"
-
-
 def test_api_build(client, upstream):
     response = client.post(
         "/api/v1/build",
@@ -62,6 +57,7 @@ def test_api_build_filesystem_squashfs(app, upstream):
     assert "# CONFIG_TARGET_ROOTFS_EXT4FS is not set" in config
     assert "CONFIG_TARGET_ROOTFS_SQUASHFS=y" in config
 
+
 def test_api_build_filesystem_empty(app, upstream):
     client = app.test_client()
     response = client.post(

tests/test_factory.py

@@ -12,21 +12,6 @@ def test_other(app):
     assert app.config["UPSTREAM_URL"] == "http://localhost:8001"
 
 
-def test_json_path_latest(client):
-    response = client.get("/json/latest.json")
-    assert "19.07.7" in response.json["latest"]
-    assert "21.02.0" in response.json["latest"]
-    assert "SNAPSHOT" in response.json["latest"]
-    assert response.status == "200 OK"
-
-
-def test_json_path_branches(client):
-    response = client.get("/json/branches.json")
-    assert "19.07" == response.json[3]["name"]
-    assert "SNAPSHOT" == response.json[0]["name"]
-    assert response.status == "200 OK"
-
-
 def test_json_store(client):
     response = client.get("/store/")
     assert response.status == "404 NOT FOUND"

tests/test_janitor.py

@@ -33,6 +33,15 @@ def test_update_branch(app, upstream):
     assert (app.config["JSON_PATH"] / "snapshots/overview.json").is_file()
 
 
+def test_update_meta_json(app):
+    with app.app_context():
+        update_meta_json()
+    latest_json = json.loads((app.config["JSON_PATH"] / "latest.json").read_text())
+    assert "19.07.7" in latest_json["latest"]
+    assert "21.02.0" in latest_json["latest"]
+    assert "SNAPSHOT" in latest_json["latest"]
+
+
 def test_parse_packages_file(app, upstream):
     url = (
         app.config["UPSTREAM_URL"]

tests/test_stats.py

@@ -131,6 +131,7 @@ def test_stats_clients_auc(client, upstream):
         as_text=True
     )
 
+
 def test_stats_clients_auc_possible_new_format(client, upstream):
     response = client.post(
         "/api/v1/build",