Merge pull request #66 from GSA/fix-sqlite-fq-constraints

sqlite foreign key & db fixes

.github/workflows/deploy.yml

@@ -33,6 +33,11 @@
             cat requirements.txt
         - name: deploy DHL
           uses: cloud-gov/cg-cli-tools@main
+          env:
+              CF_API_URL: https://api.fr.cloud.gov
+              CF_SERVICE_USER: ${{secrets.CF_SERVICE_USER}}
+              CF_SERVICE_AUTH: ${{secrets.CF_SERVICE_AUTH}}
+              LM_RUNNER_APP_GUID: ${{secrets.LM_RUNNER_APP_GUID}}
           with:
             command: cf push --vars-file vars.staging.yml --strategy rolling --no-wait
             cf_org: gsa-datagov
@@ -91,6 +96,11 @@
             cat requirements.txt
         - name: deploy DHL
           uses: cloud-gov/cg-cli-tools@main
+          env:
+              CF_API_URL: https://api.fr.cloud.gov
+              CF_SERVICE_USER: ${{secrets.CF_SERVICE_USER}}
+              CF_SERVICE_AUTH: ${{secrets.CF_SERVICE_AUTH}}
+              LM_RUNNER_APP_GUID: ${{secrets.LM_RUNNER_APP_GUID}}          
           with:
             command: cf push --vars-file vars.prod.yml --strategy rolling --no-wait
             cf_org: gsa-datagov
@@ -125,7 +135,7 @@
             LAST_COMMIT: ${{ github.sha }}
             LAST_RUN_BY: ${{ github.actor }}
             RUN_ID: ${{ github.run_id }}
-            REPO: ${{ github.repository }}
+            REPO: ${{ github.repository }}         
           with:
             filename: .github/deploy_failure.md
             update_existing: true

Makefile

@@ -3,7 +3,7 @@ pypi-upload: build-dist  ## Uploads new package to PyPi after clean, build
 
 deps-update: ## Updates requirements.txt and requirements_dev.txt from pyproject.toml
 	poetry export --without-hashes --without=dev --format=requirements.txt > requirements.txt
-	poetry export --without-hashes --only=dev --format=requirements-dev.txt > requirements-dev.txt 
+	poetry export --without-hashes --only=dev --format=requirements.txt > requirements-dev.txt 
 
 # pypi-upload-test: build-dist  ## Uploads new package to TEST PyPi after clean, build
 # 	twine upload -r testpypi dist/*	

app/__init__.py

@@ -34,6 +34,7 @@ def create_app(testing=False):
         register_routes(app)
 
     with app.app_context():
+        db.create_all()
         load_manager()
 
     return app

requirements-dev.txt

@@ -1,11 +1,18 @@
-colorama==0.4.6 ; python_version >= "3.10" and sys_platform == "win32"
-coverage[toml]==7.4.3 ; python_version >= "3.10"
+black==24.4.2 ; python_version >= "3.10"
+click==8.1.7 ; python_version >= "3.10"
+colorama==0.4.6 ; python_version >= "3.10" and (sys_platform == "win32" or platform_system == "Windows")
+coverage[toml]==7.5.0 ; python_version >= "3.10"
 debugpy==1.8.1 ; python_version >= "3.10"
-exceptiongroup==1.2.0 ; python_version < "3.11" and python_version >= "3.10"
+exceptiongroup==1.2.1 ; python_version < "3.11" and python_version >= "3.10"
 iniconfig==2.0.0 ; python_version >= "3.10"
-packaging==23.2 ; python_version >= "3.10"
-pluggy==1.4.0 ; python_version >= "3.10"
+isort==5.13.2 ; python_version >= "3.10"
+mypy-extensions==1.0.0 ; python_version >= "3.10"
+packaging==24.0 ; python_version >= "3.10"
+pathspec==0.12.1 ; python_version >= "3.10"
+platformdirs==4.2.1 ; python_version >= "3.10"
+pluggy==1.5.0 ; python_version >= "3.10"
 pytest-cov==4.1.0 ; python_version >= "3.10"
-pytest==7.4.4 ; python_version >= "3.10"
-ruff==0.0.261 ; python_version >= "3.10"
+pytest==8.1.1 ; python_version >= "3.10"
+ruff==0.0.291 ; python_version >= "3.10"
 tomli==2.0.1 ; python_full_version <= "3.11.0a6" and python_version >= "3.10"
+typing-extensions==4.11.0 ; python_version < "3.11" and python_version >= "3.10"

requirements.txt

@@ -1,61 +1,56 @@
-aiohttp==3.9.3 ; python_version >= "3.10"
+aiohttp==3.9.5 ; python_version >= "3.10"
 aiosignal==1.3.1 ; python_version >= "3.10"
 alembic==1.13.1 ; python_version >= "3.10"
 async-timeout==4.0.3 ; python_version < "3.11" and python_version >= "3.10"
 attrs==23.2.0 ; python_version >= "3.10"
 beautifulsoup4==4.12.3 ; python_version >= "3.10"
 blinker==1.7.0 ; python_version >= "3.10"
-boto3==1.34.54 ; python_version >= "3.10"
-botocore==1.34.54 ; python_version >= "3.10"
 certifi==2024.2.2 ; python_version >= "3.10"
 charset-normalizer==3.3.2 ; python_version >= "3.10"
-ckanapi==4.7 ; python_version >= "3.10"
+ckanapi==4.8 ; python_version >= "3.10"
 click==8.1.7 ; python_version >= "3.10"
 cloudfoundry-client==1.36.0 ; python_version >= "3.10"
 colorama==0.4.6 ; python_version >= "3.10" and platform_system == "Windows"
-deepdiff==6.7.1 ; python_version >= "3.10"
+deepdiff==7.0.1 ; python_version >= "3.10"
 docopt==0.6.2 ; python_version >= "3.10"
 dominate==2.9.1 ; python_version >= "3.10"
 flask-bootstrap==3.3.7.1 ; python_version >= "3.10"
 flask-migrate==4.0.7 ; python_version >= "3.10"
 flask-sqlalchemy==3.1.1 ; python_version >= "3.10"
 flask-wtf==1.2.1 ; python_version >= "3.10"
-flask==3.0.2 ; python_version >= "3.10"
+flask==3.0.3 ; python_version >= "3.10"
 frozenlist==1.4.1 ; python_version >= "3.10"
 greenlet==3.0.3 ; (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version >= "3.10"
-idna==3.6 ; python_version >= "3.10"
-itsdangerous==2.1.2 ; python_version >= "3.10"
+idna==3.7 ; python_version >= "3.10"
+itsdangerous==2.2.0 ; python_version >= "3.10"
 jinja2==3.1.3 ; python_version >= "3.10"
-jmespath==1.0.1 ; python_version >= "3.10"
 jsonschema-specifications==2023.12.1 ; python_version >= "3.10"
 jsonschema==4.21.1 ; python_version >= "3.10"
-mako==1.3.2 ; python_version >= "3.10"
+mako==1.3.3 ; python_version >= "3.10"
 markupsafe==2.1.5 ; python_version >= "3.10"
 multidict==6.0.5 ; python_version >= "3.10"
 oauth2-client==1.4.2 ; python_version >= "3.10"
 ordered-set==4.1.0 ; python_version >= "3.10"
 polling2==0.5.0 ; python_version >= "3.10"
 protobuf==4.25.3 ; python_version >= "3.10"
 psycopg2-binary==2.9.9 ; python_version >= "3.10"
-python-dateutil==2.9.0.post0 ; python_version >= "3.10"
 python-dotenv==1.0.1 ; python_version >= "3.10"
 python-slugify==8.0.4 ; python_version >= "3.10"
 pyyaml==6.0.1 ; python_version >= "3.10"
-referencing==0.33.0 ; python_version >= "3.10"
+referencing==0.35.0 ; python_version >= "3.10"
 requests==2.31.0 ; python_version >= "3.10"
 rpds-py==0.18.0 ; python_version >= "3.10"
-s3transfer==0.10.0 ; python_version >= "3.10"
 sansjson==0.3.0 ; python_version >= "3.10"
-setuptools==69.1.1 ; python_version >= "3.10"
+setuptools==69.5.1 ; python_version >= "3.10"
 simplejson==3.19.2 ; python_version >= "3.10"
 six==1.16.0 ; python_version >= "3.10"
 soupsieve==2.5 ; python_version >= "3.10"
-sqlalchemy==2.0.28 ; python_version >= "3.10"
+sqlalchemy==2.0.29 ; python_version >= "3.10"
 text-unidecode==1.3 ; python_version >= "3.10"
-typing-extensions==4.10.0 ; python_version >= "3.10"
-urllib3==2.0.7 ; python_version >= "3.10"
+typing-extensions==4.11.0 ; python_version >= "3.10"
+urllib3==2.2.1 ; python_version >= "3.10"
 visitor==0.1.3 ; python_version >= "3.10"
 websocket-client==1.7.0 ; python_version >= "3.10"
-werkzeug==3.0.1 ; python_version >= "3.10"
+werkzeug==3.0.2 ; python_version >= "3.10"
 wtforms==3.1.2 ; python_version >= "3.10"
 yarl==1.9.4 ; python_version >= "3.10"

tests/conftest.py

@@ -4,7 +4,7 @@
 import pytest
 from dotenv import load_dotenv
 from sqlalchemy.orm import scoped_session, sessionmaker
-
+from sqlalchemy.sql import text
 from app import create_app
 from flask import Flask
 from database.interface import HarvesterDBInterface
@@ -36,6 +36,7 @@ def session(app) -> scoped_session:
 
         SessionLocal = sessionmaker(bind=connection, autocommit=False, autoflush=False)
         session = scoped_session(SessionLocal)
+        session.execute(text("pragma foreign_keys=on"))
         yield session
 
         session.remove()
@@ -190,16 +191,19 @@ def source_data_dcatus_invalid_records_job(
 
 @pytest.fixture
 def interface_with_multiple_jobs(
-    interface,
-    source_data_dcatus: dict,
+    interface, organization_data, source_data_dcatus, source_data_waf
 ):
     statuses = ["pending", "pending_manual", "in_progress", "complete"]
-    source_ids = ["1234", "abcd"]
+    source_ids = [source_data_dcatus["id"], source_data_waf["id"]]
     jobs = [
         {"status": status, "harvest_source_id": source}
         for status in statuses
         for source in source_ids
     ]
+    interface.add_organization(organization_data)
+    interface.add_harvest_source(source_data_dcatus)
+    interface.add_harvest_source(source_data_waf)
+
     for job in jobs:
         interface.add_harvest_job(job)
 

tests/unit/database/test_db.py

@@ -121,13 +121,22 @@ def test_add_harvest_records(
         assert success is True
         assert len(interface.get_all_harvest_records()) == 10
 
-    def test_add_harvest_job_with_id(self, interface, job_data_dcatus):
+    def test_add_harvest_job_with_id(
+        self, interface, organization_data, source_data_dcatus, job_data_dcatus
+    ):
+        interface.add_organization(organization_data)
+        interface.add_harvest_source(source_data_dcatus)
         job = interface.add_harvest_job(job_data_dcatus)
         assert job.id == job_data_dcatus["id"]
         assert job.status == job_data_dcatus["status"]
         assert job.harvest_source_id == job_data_dcatus["harvest_source_id"]
 
-    def test_add_harvest_job_without_id(self, interface, job_data_dcatus):
+    def test_add_harvest_job_without_id(
+        self, interface, organization_data, source_data_dcatus, job_data_dcatus
+    ):
+        interface.add_organization(organization_data)
+        interface.add_harvest_source(source_data_dcatus)
+
         job_data_dcatus_id = job_data_dcatus["id"]
         del job_data_dcatus["id"]
         job = interface.add_harvest_job(job_data_dcatus)
@@ -136,17 +145,33 @@ def test_add_harvest_job_without_id(self, interface, job_data_dcatus):
         assert job.status == job_data_dcatus["status"]
         assert job.harvest_source_id == job_data_dcatus["harvest_source_id"]
 
-    def test_get_harvest_jobs_by_filter(self, interface_with_multiple_jobs):
-        filters = {"status": "pending", "harvest_source_id": "1234"}
+    def test_get_harvest_jobs_by_filter(
+        self, source_data_dcatus, interface_with_multiple_jobs
+    ):
+        filters = {
+            "status": "pending",
+            "harvest_source_id": f"{source_data_dcatus['id']}",
+        }
         filtered_list = interface_with_multiple_jobs.get_harvest_jobs_by_filter(filters)
         assert len(filtered_list) == 1
         assert filtered_list[0]["status"] == "pending"
-        assert filtered_list[0]["harvest_source_id"] == "1234"
+        assert filtered_list[0]["harvest_source_id"] == source_data_dcatus["id"]
 
-    def test_filter_jobs_by_faceted_filter(self, interface_with_multiple_jobs):
+    def test_filter_jobs_by_faceted_filter(
+        self, source_data_dcatus, interface_with_multiple_jobs
+    ):
         faceted_list = interface_with_multiple_jobs.get_harvest_jobs_by_faceted_filter(
             "status", ["pending", "pending_manual"]
         )
         assert len(faceted_list) == 4
         assert len([x for x in faceted_list if x["status"] == "pending"]) == 2
-        assert len([x for x in faceted_list if x["harvest_source_id"] == "1234"]) == 2
+        assert (
+            len(
+                [
+                    x
+                    for x in faceted_list
+                    if x["harvest_source_id"] == source_data_dcatus["id"]
+                ]
+            )
+            == 2
+        )