Fix invalid links not replaced

Co-authored-by: David Venhoff <[email protected]>
digitalfabrik · Sep 23, 2024 · d83f2ae · d83f2ae
1 parent 127d22a
commit d83f2ae
Show file tree

Hide file tree

Showing 7 changed files with 215 additions and 7 deletions.
diff --git a/integreat_cms/cms/fixtures/test_data.json b/integreat_cms/cms/fixtures/test_data.json
@@ -2219,6 +2219,54 @@
       "redirect_to": "https://integreat.app/augsburg/de/test-links/"
     }
   },
+  {
+    "model": "linkcheck.url",
+    "pk": 18,
+    "fields": {
+      "url": "https://tuerantuer.de/",
+      "last_checked": "2023-08-23T10:19:32Z",
+      "anchor_status": null,
+      "ssl_status": null,
+      "status": true,
+      "status_code": 302,
+      "redirect_status_code": 200,
+      "message": "200 OK",
+      "error_message": "",
+      "redirect_to": ""
+    }
+  },
+  {
+    "model": "linkcheck.url",
+    "pk": 19,
+    "fields": {
+      "url": "https://integreat.app/fake-region/de/willkommen/",
+      "last_checked": "2023-08-23T10:19:32Z",
+      "anchor_status": null,
+      "ssl_status": null,
+      "status": false,
+      "status_code": null,
+      "redirect_status_code": null,
+      "message": "",
+      "error_message": "Diese Region existiert nicht oder ist nicht aktiv.",
+      "redirect_to": ""
+    }
+  },
+  {
+    "model": "linkcheck.url",
+    "pk": 20,
+    "fields": {
+      "url": "https://tuerantuer.dx/",
+      "last_checked": "2023-08-23T10:19:32Z",
+      "anchor_status": null,
+      "ssl_status": null,
+      "status": false,
+      "status_code": null,
+      "redirect_status_code": null,
+      "message": "Name Resolution Error: Failed to resolve 'tuerantuer.dx' ([Errno -2] Name or service not known)",
+      "error_message": "Name Resolution Error: Failed to resolve 'tuerantuer.dx' ([Errno -2] Name or service not known)",
+      "redirect_to": ""
+    }
+  },
   {
     "model": "linkcheck.link",
     "pk": 1,
@@ -2627,6 +2675,54 @@
       "ignore": false
     }
   },
+  {
+    "model": "linkcheck.link",
+    "pk": 35,
+    "fields": {
+      "content_type": ["cms", "pagetranslation"],
+      "object_id": 99,
+      "field": "content",
+      "url": 7,
+      "text": "Internal valid link",
+      "ignore": false
+    }
+  },
+  {
+    "model": "linkcheck.link",
+    "pk": 36,
+    "fields": {
+      "content_type": ["cms", "pagetranslation"],
+      "object_id": 99,
+      "field": "content",
+      "url": 19,
+      "text": "Internal invalid link",
+      "ignore": false
+    }
+  },
+  {
+    "model": "linkcheck.link",
+    "pk": 37,
+    "fields": {
+      "content_type": ["cms", "pagetranslation"],
+      "object_id": 99,
+      "field": "content",
+      "url": 18,
+      "text": "External valid link",
+      "ignore": false
+    }
+  },
+  {
+    "model": "linkcheck.link",
+    "pk": 38,
+    "fields": {
+      "content_type": ["cms", "pagetranslation"],
+      "object_id": 99,
+      "field": "content",
+      "url": 20,
+      "text": "External invalid link",
+      "ignore": false
+    }
+  },
   {
     "model": "cms.page",
     "pk": 1,
@@ -3271,6 +3367,29 @@
       "embedded_offers": []
     }
   },
+  {
+    "model": "cms.page",
+    "pk": 29,
+    "fields": {
+      "created_date": "2023-08-18T10:31:48.247Z",
+      "lft": 1,
+      "rgt": 2,
+      "tree_id": 10,
+      "depth": 1,
+      "parent": null,
+      "region": 2,
+      "explicitly_archived": false,
+      "icon": null,
+      "mirrored_page": null,
+      "mirrored_page_first": true,
+      "organization": null,
+      "api_token": "",
+      "hix_ignore": false,
+      "authors": [],
+      "editors": [],
+      "embedded_offers": []
+    }
+  },
   { "model": "cms.regionfeedback", "pk": 1, "fields": { "feedback_ptr": 1 } },
   { "model": "cms.regionfeedback", "pk": 4, "fields": { "feedback_ptr": 4 } },
   { "model": "cms.regionfeedback", "pk": 7, "fields": { "feedback_ptr": 7 } },
@@ -5336,6 +5455,27 @@
       "hix_feedback": null
     }
   },
+  {
+    "model": "cms.pagetranslation",
+    "pk": 99,
+    "fields": {
+      "title": "Test links",
+      "slug": "test_links",
+      "status": "PUBLIC",
+      "content": "<div><p><a href=\"https://integreat.app/augsburg/de/willkommen/\">Internal valid</a></p><p><a href=\"https://integreat.app/fake-region/de/willkommen/\">Internal invalid</a></p><p> </p><p><a class=\"link-external\" href=\"https://tuerantuer.de/\">External valid</a></p><p><a class=\"link-external\" href=\"https://tuerantuer.dx/\">External invalid</a></p></div>",
+      "language": 1,
+      "currently_in_translation": false,
+      "machine_translated": false,
+      "version": 1,
+      "minor_edit": false,
+      "last_updated": "2022-12-30T11:11:11.111Z",
+      "creator": ["root"],
+      "automatic_translation": false,
+      "page": 29,
+      "hix_score": null,
+      "hix_feedback": null
+    }
+  },
   {
     "model": "cms.imprintpagetranslation",
     "pk": 1,

diff --git a/integreat_cms/cms/utils/linkcheck_utils.py b/integreat_cms/cms/utils/linkcheck_utils.py
@@ -243,7 +243,15 @@ def find_target_url_per_content(
     )
 
     links_to_replace = (
-        [link for link in links if link.url.type in link_types] if link_types else links
+        (
+            link
+            for link in links
+            if link.url.type in link_types
+            or link.url.status is False
+            and "invalid" in link_types
+        )
+        if link_types
+        else links
     )
 
     content_objects: DefaultDict[AbstractContentTranslation, dict[str, str]] = (

diff --git a/integreat_cms/release_notes/current/unreleased/3058.yml b/integreat_cms/release_notes/current/unreleased/3058.yml
@@ -0,0 +1,2 @@
+en: Fix the bug invalid links are not replaced
+de: Behebe den Fehler, dass ungültige Links nicht ersetzt werden
diff --git a/tests/cms/views/link_replace/test_link_replace.py b/tests/cms/views/link_replace/test_link_replace.py
@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+from django.contrib.contenttypes.models import ContentType
+
+from integreat_cms.cms.models import PageTranslation, Region
+from integreat_cms.cms.utils.linkcheck_utils import find_target_url_per_content
+
+REGION: str = "nurnberg"
+
+# list of tuple (<link types to replace>, <URLs that should be replaced>, <type and id of content that have URLs listed before>)
+parameters = [
+    (
+        ["invalid"],
+        ["https://tuerantuer.dx/", "https://integreat.app/fake-region/de/willkommen/"],
+        (PageTranslation, 99),
+    ),
+    (
+        ["internal"],
+        [
+            "https://integreat.app/fake-region/de/willkommen/",
+            "https://integreat.app/augsburg/de/willkommen/",
+        ],
+        (PageTranslation, 99),
+    ),
+    (
+        ["external"],
+        ["https://tuerantuer.dx/", "https://tuerantuer.de/"],
+        (PageTranslation, 99),
+    ),
+]
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("parameter", parameters)
+def test_find_target_url_per_content_invalid_links(
+    load_test_data: None,
+    parameter: tuple[list[str], list[str], tuple[ContentType, int]],
+) -> None:
+    """
+    Test link filter per selected link type(s) is working correctly.
+    """
+    link_types, target_urls, target_content_detail = parameter
+
+    # "d" is something many of our links can contain thanks to ".de" or language slug "de"
+    result = find_target_url_per_content(
+        "d", "d", Region.objects.filter(slug=REGION).first(), link_types
+    )
+
+    content_type, content_id = target_content_detail
+    target_content = content_type.objects.filter(id=content_id).first()
+    for content, urls in result.items():
+        assert content == target_content
+        assert len(urls) == len(target_urls)
+        for url in urls:
+            assert url in target_urls
diff --git a/tests/core/management/commands/test_replace_links.py b/tests/core/management/commands/test_replace_links.py
@@ -83,7 +83,7 @@ def test_replace_links_dry_run(load_test_data_transactional: Any | None) -> None
         url=test_url
     ).exists(), "Test URL should exist in test data"
     assert (
-        Link.objects.filter(url__url=test_url).count() == 3
+        Link.objects.filter(url__url=test_url).count() == 4
     ), "Test link should exist in test data"
     assert not Url.objects.filter(
         url=replaced_url
@@ -103,7 +103,7 @@ def test_replace_links_dry_run(load_test_data_transactional: Any | None) -> None
         url=test_url
     ).exists(), "Test URL should not be removed during dry run"
     assert (
-        Link.objects.filter(url__url=test_url).count() == 3
+        Link.objects.filter(url__url=test_url).count() == 4
     ), "Test link should not be removed during dry run"
     assert not Url.objects.filter(
         url=replaced_url
@@ -129,7 +129,7 @@ def test_replace_links_commit(load_test_data_transactional: Any | None) -> None:
         url=test_url
     ).exists(), "Test URL should not be removed during dry run"
     assert (
-        Link.objects.filter(url__url=test_url).count() == 3
+        Link.objects.filter(url__url=test_url).count() == 4
     ), "Test link should not be removed during dry run"
     assert not Url.objects.filter(
         url=replaced_url
@@ -154,5 +154,5 @@ def test_replace_links_commit(load_test_data_transactional: Any | None) -> None:
         url=replaced_url
     ).exists(), "Replaced URL should exist after replacement"
     assert (
-        Link.objects.filter(url__url=replaced_url).count() == 3
+        Link.objects.filter(url__url=replaced_url).count() == 4
     ), "Replaced link should exist after replacement"
diff --git a/tests/sitemap/expected-sitemaps/sitemap-nurnberg-de.xml b/tests/sitemap/expected-sitemaps/sitemap-nurnberg-de.xml
@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">
-<url><loc>https://integreat.app/nurnberg/de/willkommen/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/welcome/"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/الترحيب/"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/خوش-آمدید/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/willkommen-in-nurnberg/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/الترحيب/مرحبا-بكم-في-نورنبيرغ/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/zusammenleben-in-deutschland/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/welcome/community-life-in-germany/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/wissenswertes-uber-nurnberg/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/welcome/trivia-about-nuremberg/"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/الترحيب/معلومات-قيمة-عن-نورنبيرغ/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/stadtplan/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority></url><url><loc>https://integreat.app/nurnberg/de/willkommen/uber-integreat/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/welcome/about-integreat/"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/خوش-آمدید/درباره-integreat/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/aufenthaltstitel/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority></url><url><loc>https://integreat.app/nurnberg/de/locations/test-ort/</loc><lastmod>2020-01-22</lastmod><changefreq>monthly</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/locations/test-location/"/></url><url><loc>https://integreat.app/nurnberg/de/offers/ihk-lehrstellenboerse</loc><lastmod>2019-10-09</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/offers/ihk-lehrstellenboerse"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/offers/ihk-lehrstellenboerse"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/offers/ihk-lehrstellenboerse"/></url><url><loc>https://integreat.app/nurnberg/de/offers/ihk-praktikumsboerse</loc><lastmod>2019-10-09</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/offers/ihk-praktikumsboerse"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/offers/ihk-praktikumsboerse"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/offers/ihk-praktikumsboerse"/></url><url><loc>https://integreat.app/nurnberg/de/offers/lehrstellen-radar</loc><lastmod>2019-10-09</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/offers/lehrstellen-radar"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/offers/lehrstellen-radar"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/offers/lehrstellen-radar"/></url><url><loc>https://integreat.app/nurnberg/de/events/test-veranstaltung/</loc><lastmod>2020-01-22</lastmod><changefreq>daily</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/events/test-event/"/></url>
+<url><loc>https://integreat.app/nurnberg/de/willkommen/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/welcome/"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/الترحيب/"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/خوش-آمدید/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/willkommen-in-nurnberg/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/الترحيب/مرحبا-بكم-في-نورنبيرغ/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/zusammenleben-in-deutschland/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/welcome/community-life-in-germany/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/wissenswertes-uber-nurnberg/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/welcome/trivia-about-nuremberg/"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/الترحيب/معلومات-قيمة-عن-نورنبيرغ/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/stadtplan/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority></url><url><loc>https://integreat.app/nurnberg/de/willkommen/uber-integreat/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/welcome/about-integreat/"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/خوش-آمدید/درباره-integreat/"/></url><url><loc>https://integreat.app/nurnberg/de/willkommen/aufenthaltstitel/</loc><lastmod>2019-08-12</lastmod><changefreq>monthly</changefreq><priority>1.0</priority></url><url><loc>https://integreat.app/nurnberg/de/test_links/</loc><lastmod>2022-12-30</lastmod><changefreq>monthly</changefreq><priority>1.0</priority></url><url><loc>https://integreat.app/nurnberg/de/locations/test-ort/</loc><lastmod>2020-01-22</lastmod><changefreq>monthly</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/locations/test-location/"/></url><url><loc>https://integreat.app/nurnberg/de/offers/ihk-lehrstellenboerse</loc><lastmod>2019-10-09</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/offers/ihk-lehrstellenboerse"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/offers/ihk-lehrstellenboerse"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/offers/ihk-lehrstellenboerse"/></url><url><loc>https://integreat.app/nurnberg/de/offers/ihk-praktikumsboerse</loc><lastmod>2019-10-09</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/offers/ihk-praktikumsboerse"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/offers/ihk-praktikumsboerse"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/offers/ihk-praktikumsboerse"/></url><url><loc>https://integreat.app/nurnberg/de/offers/lehrstellen-radar</loc><lastmod>2019-10-09</lastmod><changefreq>monthly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/offers/lehrstellen-radar"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/ar/offers/lehrstellen-radar"/><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/fa/offers/lehrstellen-radar"/></url><url><loc>https://integreat.app/nurnberg/de/events/test-veranstaltung/</loc><lastmod>2020-01-22</lastmod><changefreq>daily</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="" href="https://integreat.app/nurnberg/en/events/test-event/"/></url>
 </urlset>
diff --git a/tests/sitemap/sitemap_config.py b/tests/sitemap/sitemap_config.py
@@ -39,7 +39,7 @@
     (
         "/nurnberg/de/sitemap.xml",
         "tests/sitemap/expected-sitemaps/sitemap-nurnberg-de.xml",
-        73,
+        78,
     ),
     (
         "/nurnberg/en/sitemap.xml",