api: add handling of package changes

If a package is dropped or renamed the server should handle that
transparently. The information is available in the overview.json and
branches.json file for downstream to process.

Signed-off-by: Paul Spooren <[email protected]>

asu/api.py

@@ -86,6 +86,18 @@ def validate_packages(req):
 
     req["packages"] = tr
 
+    if req.get("diff_packages", False):
+        for package, change in (
+            current_app.config["BRANCHES"][req["branch"]]
+            .get("package_changes", {})
+            .items()
+        ):
+            if package in req["packages"]:
+                current_app.logger.debug("changes to package %s", package)
+                req["packages"].remove(package)
+                if isinstance(change, str):
+                    req["packages"].add(change)
+
     # store request packages temporary in Redis and create a diff
     temp = str(uuid4())
     pipeline = r.pipeline(True)

asu/asu.py

@@ -57,6 +57,13 @@ def create_app(test_config: dict = None) -> Flask:
             app.config[option] = Path(value)
             app.config[option].mkdir(parents=True, exist_ok=True)
 
+    if not "BRANCHES" in app.config:
+        if "BRANCHES_FILE" not in app.config:
+            app.config["BRANCHES_FILE"] = resource_filename(__name__, "branches.yml")
+
+        with open(app.config["BRANCHES_FILE"], "r") as branches:
+            app.config["BRANCHES"] = safe_load(branches)["branches"]
+
     app.wsgi_app = DispatcherMiddleware(
         app.wsgi_app, {"/metrics": make_wsgi_app(app.config["REGISTRY"])}
     )

tests/conftest.py

@@ -35,6 +35,7 @@ def redis_load_mock_data(redis):
         "test1",
         "test2",
         "test3",
+        "valid_new_package",
     )
     redis.sadd(
         "profiles:TESTVERSION:TESTVERSION:testtarget/testsubtarget", "testprofile"
@@ -113,6 +114,10 @@ def app(test_path, redis_server):
                         "testtarget/testsubtarget": "testarch",
                         "x86/64": "x86_64",
                     },
+                    "package_changes": {
+                        "package_to_remove": None,
+                        "package_to_replace": "valid_new_package",
+                    },
                 },
                 "21.02": {
                     "name": "21.02",
@@ -146,6 +151,51 @@ def app(test_path, redis_server):
     return mock_app
 
 
+@pytest.fixture
+def app_using_branches_yml(test_path, redis_server):
+    redis_load_mock_data(redis_server)
+
+    registry = prometheus_client.CollectorRegistry(auto_describe=True)
+
+    mock_app = create_app(
+        {
+            "REGISTRY": registry,
+            "ASYNC_QUEUE": False,
+            "JSON_PATH": test_path + "/json",
+            "REDIS_CONN": redis_server,
+            "STORE_PATH": test_path + "/store",
+            "CACHE_PATH": test_path,
+            "TESTING": True,
+            "UPSTREAM_URL": "http://localhost:8001",
+            "BRANCHES_FILE": "./asu/branches.yml",
+        }
+    )
+
+    return mock_app
+
+
+@pytest.fixture
+def app_using_default_branches(test_path, redis_server):
+    redis_load_mock_data(redis_server)
+
+    registry = prometheus_client.CollectorRegistry(auto_describe=True)
+
+    mock_app = create_app(
+        {
+            "REGISTRY": registry,
+            "ASYNC_QUEUE": False,
+            "JSON_PATH": test_path + "/json",
+            "REDIS_CONN": redis_server,
+            "STORE_PATH": test_path + "/store",
+            "CACHE_PATH": test_path,
+            "TESTING": True,
+            "UPSTREAM_URL": "http://localhost:8001",
+        }
+    )
+
+    return mock_app
+
+
 @pytest.fixture
 def client(app):
     return app.test_client()

tests/test_api.py

@@ -474,3 +474,47 @@ def test_api_build_bad_packages(client):
     )
     assert response.json.get("detail") == "Unsupported package(s): test4"
     assert response.status == "422 UNPROCESSABLE ENTITY"
+
+
+def test_api_build_package_to_remove_diff_packages_true(client, upstream):
+    response = client.post(
+        "/api/v1/build",
+        json=dict(
+            version="TESTVERSION",
+            target="testtarget/testsubtarget",
+            profile="testprofile",
+            packages=["test1", "test2", "package_to_remove"],
+            diff_packages=True,
+        ),
+    )
+    assert response.status == "200 OK"
+    assert response.json.get("request_hash") == "b7b991edc9a53302df840f842603288e"
+
+
+def test_api_build_package_to_remove_diff_packages_false(client, upstream):
+    response = client.post(
+        "/api/v1/build",
+        json=dict(
+            version="TESTVERSION",
+            target="testtarget/testsubtarget",
+            profile="testprofile",
+            packages=["test1", "test2", "package_to_remove"],
+            diff_packages=False,
+        ),
+    )
+    assert response.status == "422 UNPROCESSABLE ENTITY"
+
+
+def test_api_build_package_to_replace(client, upstream):
+    response = client.post(
+        "/api/v1/build",
+        json=dict(
+            version="TESTVERSION",
+            target="testtarget/testsubtarget",
+            profile="testprofile",
+            packages=["test1", "test2", "package_to_replace"],
+            diff_packages=True,
+        ),
+    )
+    assert response.status == "200 OK"
+    assert response.json.get("request_hash") == "8dd81d51057d322b46aa3739ccf4a8a6"

tests/test_factory.py

@@ -8,6 +8,18 @@ def test_pathlib(app):
     assert app.config["JSON_PATH"].is_dir()
 
 
+def test_branches_yaml(app_using_branches_yml):
+    assert isinstance(app_using_branches_yml.config["BRANCHES"], dict)
+    assert "SNAPSHOT" in app_using_branches_yml.config["BRANCHES"]
+    assert "22.03" in app_using_branches_yml.config["BRANCHES"]
+
+
+def test_branches_default(app_using_default_branches):
+    assert isinstance(app_using_default_branches.config["BRANCHES"], dict)
+    assert "SNAPSHOT" in app_using_default_branches.config["BRANCHES"]
+    assert "22.03" in app_using_default_branches.config["BRANCHES"]
+
+
 def test_other(app):
     assert app.config["UPSTREAM_URL"] == "http://localhost:8001"
 

tests/test_janitor.py

@@ -33,7 +33,7 @@ def test_update_branch(app, upstream):
     assert (app.config["JSON_PATH"] / "snapshots/overview.json").is_file()
 
 
-def test_update_meta_json(app):
+def test_update_meta_latest_json(app):
     with app.app_context():
         update_meta_json()
     latest_json = json.loads((app.config["JSON_PATH"] / "latest.json").read_text())
@@ -42,6 +42,13 @@ def test_update_meta_json(app):
     assert "SNAPSHOT" in latest_json["latest"]
 
 
+def test_update_meta_overview_json(app):
+    with app.app_context():
+        update_meta_json()
+    overview_json = json.loads((app.config["JSON_PATH"] / "overview.json").read_text())
+    assert "package_changes" in overview_json["branches"]["TESTVERSION"]
+
+
 def test_parse_packages_file(app, upstream):
     url = (
         app.config["UPSTREAM_URL"]