diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 74b493b3..e29ab9c6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -33,6 +33,11 @@ cat requirements.txt - name: deploy DHL uses: cloud-gov/cg-cli-tools@main + env: + CF_API_URL: https://api.fr.cloud.gov + CF_SERVICE_USER: ${{secrets.CF_SERVICE_USER}} + CF_SERVICE_AUTH: ${{secrets.CF_SERVICE_AUTH}} + LM_RUNNER_APP_GUID: ${{secrets.LM_RUNNER_APP_GUID}} with: command: cf push --vars-file vars.staging.yml --strategy rolling --no-wait cf_org: gsa-datagov @@ -91,6 +96,11 @@ cat requirements.txt - name: deploy DHL uses: cloud-gov/cg-cli-tools@main + env: + CF_API_URL: https://api.fr.cloud.gov + CF_SERVICE_USER: ${{secrets.CF_SERVICE_USER}} + CF_SERVICE_AUTH: ${{secrets.CF_SERVICE_AUTH}} + LM_RUNNER_APP_GUID: ${{secrets.LM_RUNNER_APP_GUID}} with: command: cf push --vars-file vars.prod.yml --strategy rolling --no-wait cf_org: gsa-datagov @@ -125,7 +135,7 @@ LAST_COMMIT: ${{ github.sha }} LAST_RUN_BY: ${{ github.actor }} RUN_ID: ${{ github.run_id }} - REPO: ${{ github.repository }} + REPO: ${{ github.repository }} with: filename: .github/deploy_failure.md update_existing: true diff --git a/Makefile b/Makefile index 21876599..d111d6d5 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ pypi-upload: build-dist ## Uploads new package to PyPi after clean, build deps-update: ## Updates requirements.txt and requirements_dev.txt from pyproject.toml poetry export --without-hashes --without=dev --format=requirements.txt > requirements.txt - poetry export --without-hashes --only=dev --format=requirements-dev.txt > requirements-dev.txt + poetry export --without-hashes --only=dev --format=requirements.txt > requirements-dev.txt # pypi-upload-test: build-dist ## Uploads new package to TEST PyPi after clean, build # twine upload -r testpypi dist/* diff --git a/app/__init__.py b/app/__init__.py index 5633a245..102d42e2 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -34,6 +34,7 @@ def create_app(testing=False): register_routes(app) with app.app_context(): + db.create_all() load_manager() return app diff --git a/poetry.lock b/poetry.lock index 3275f0b8..766a2207 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2059,4 +2059,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10" -content-hash = "3dfb2da517ae5d4f623bdf3abcf83d5a8855939ca72dbfc10f2367d1a30e9039" +content-hash = "c8b0230cc631d5340413d64bbb575772126d497af909800b1a265444413c3163" diff --git a/requirements-dev.txt b/requirements-dev.txt index aa9d6045..011b1a30 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,11 +1,18 @@ -colorama==0.4.6 ; python_version >= "3.10" and sys_platform == "win32" -coverage[toml]==7.4.3 ; python_version >= "3.10" +black==24.4.2 ; python_version >= "3.10" +click==8.1.7 ; python_version >= "3.10" +colorama==0.4.6 ; python_version >= "3.10" and (sys_platform == "win32" or platform_system == "Windows") +coverage[toml]==7.5.0 ; python_version >= "3.10" debugpy==1.8.1 ; python_version >= "3.10" -exceptiongroup==1.2.0 ; python_version < "3.11" and python_version >= "3.10" +exceptiongroup==1.2.1 ; python_version < "3.11" and python_version >= "3.10" iniconfig==2.0.0 ; python_version >= "3.10" -packaging==23.2 ; python_version >= "3.10" -pluggy==1.4.0 ; python_version >= "3.10" +isort==5.13.2 ; python_version >= "3.10" +mypy-extensions==1.0.0 ; python_version >= "3.10" +packaging==24.0 ; python_version >= "3.10" +pathspec==0.12.1 ; python_version >= "3.10" +platformdirs==4.2.1 ; python_version >= "3.10" +pluggy==1.5.0 ; python_version >= "3.10" pytest-cov==4.1.0 ; python_version >= "3.10" -pytest==7.4.4 ; python_version >= "3.10" -ruff==0.0.261 ; python_version >= "3.10" +pytest==8.1.1 ; python_version >= "3.10" +ruff==0.0.291 ; python_version >= "3.10" tomli==2.0.1 ; python_full_version <= "3.11.0a6" and python_version >= "3.10" +typing-extensions==4.11.0 ; python_version < "3.11" and python_version >= "3.10" diff --git a/requirements.txt b/requirements.txt index 1afdea48..a601758b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,35 +1,32 @@ -aiohttp==3.9.3 ; python_version >= "3.10" +aiohttp==3.9.5 ; python_version >= "3.10" aiosignal==1.3.1 ; python_version >= "3.10" alembic==1.13.1 ; python_version >= "3.10" async-timeout==4.0.3 ; python_version < "3.11" and python_version >= "3.10" attrs==23.2.0 ; python_version >= "3.10" beautifulsoup4==4.12.3 ; python_version >= "3.10" blinker==1.7.0 ; python_version >= "3.10" -boto3==1.34.54 ; python_version >= "3.10" -botocore==1.34.54 ; python_version >= "3.10" certifi==2024.2.2 ; python_version >= "3.10" charset-normalizer==3.3.2 ; python_version >= "3.10" -ckanapi==4.7 ; python_version >= "3.10" +ckanapi==4.8 ; python_version >= "3.10" click==8.1.7 ; python_version >= "3.10" cloudfoundry-client==1.36.0 ; python_version >= "3.10" colorama==0.4.6 ; python_version >= "3.10" and platform_system == "Windows" -deepdiff==6.7.1 ; python_version >= "3.10" +deepdiff==7.0.1 ; python_version >= "3.10" docopt==0.6.2 ; python_version >= "3.10" dominate==2.9.1 ; python_version >= "3.10" flask-bootstrap==3.3.7.1 ; python_version >= "3.10" flask-migrate==4.0.7 ; python_version >= "3.10" flask-sqlalchemy==3.1.1 ; python_version >= "3.10" flask-wtf==1.2.1 ; python_version >= "3.10" -flask==3.0.2 ; python_version >= "3.10" +flask==3.0.3 ; python_version >= "3.10" frozenlist==1.4.1 ; python_version >= "3.10" greenlet==3.0.3 ; (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version >= "3.10" -idna==3.6 ; python_version >= "3.10" -itsdangerous==2.1.2 ; python_version >= "3.10" +idna==3.7 ; python_version >= "3.10" +itsdangerous==2.2.0 ; python_version >= "3.10" jinja2==3.1.3 ; python_version >= "3.10" -jmespath==1.0.1 ; python_version >= "3.10" jsonschema-specifications==2023.12.1 ; python_version >= "3.10" jsonschema==4.21.1 ; python_version >= "3.10" -mako==1.3.2 ; python_version >= "3.10" +mako==1.3.3 ; python_version >= "3.10" markupsafe==2.1.5 ; python_version >= "3.10" multidict==6.0.5 ; python_version >= "3.10" oauth2-client==1.4.2 ; python_version >= "3.10" @@ -37,25 +34,23 @@ ordered-set==4.1.0 ; python_version >= "3.10" polling2==0.5.0 ; python_version >= "3.10" protobuf==4.25.3 ; python_version >= "3.10" psycopg2-binary==2.9.9 ; python_version >= "3.10" -python-dateutil==2.9.0.post0 ; python_version >= "3.10" python-dotenv==1.0.1 ; python_version >= "3.10" python-slugify==8.0.4 ; python_version >= "3.10" pyyaml==6.0.1 ; python_version >= "3.10" -referencing==0.33.0 ; python_version >= "3.10" +referencing==0.35.0 ; python_version >= "3.10" requests==2.31.0 ; python_version >= "3.10" rpds-py==0.18.0 ; python_version >= "3.10" -s3transfer==0.10.0 ; python_version >= "3.10" sansjson==0.3.0 ; python_version >= "3.10" -setuptools==69.1.1 ; python_version >= "3.10" +setuptools==69.5.1 ; python_version >= "3.10" simplejson==3.19.2 ; python_version >= "3.10" six==1.16.0 ; python_version >= "3.10" soupsieve==2.5 ; python_version >= "3.10" -sqlalchemy==2.0.28 ; python_version >= "3.10" +sqlalchemy==2.0.29 ; python_version >= "3.10" text-unidecode==1.3 ; python_version >= "3.10" -typing-extensions==4.10.0 ; python_version >= "3.10" -urllib3==2.0.7 ; python_version >= "3.10" +typing-extensions==4.11.0 ; python_version >= "3.10" +urllib3==2.2.1 ; python_version >= "3.10" visitor==0.1.3 ; python_version >= "3.10" websocket-client==1.7.0 ; python_version >= "3.10" -werkzeug==3.0.1 ; python_version >= "3.10" +werkzeug==3.0.2 ; python_version >= "3.10" wtforms==3.1.2 ; python_version >= "3.10" yarl==1.9.4 ; python_version >= "3.10" diff --git a/tests/conftest.py b/tests/conftest.py index 109d13a0..37a7f0bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ import pytest from dotenv import load_dotenv from sqlalchemy.orm import scoped_session, sessionmaker - +from sqlalchemy.sql import text from app import create_app from flask import Flask from database.interface import HarvesterDBInterface @@ -36,6 +36,7 @@ def session(app) -> scoped_session: SessionLocal = sessionmaker(bind=connection, autocommit=False, autoflush=False) session = scoped_session(SessionLocal) + session.execute(text("pragma foreign_keys=on")) yield session session.remove() @@ -190,16 +191,19 @@ def source_data_dcatus_invalid_records_job( @pytest.fixture def interface_with_multiple_jobs( - interface, - source_data_dcatus: dict, + interface, organization_data, source_data_dcatus, source_data_waf ): statuses = ["pending", "pending_manual", "in_progress", "complete"] - source_ids = ["1234", "abcd"] + source_ids = [source_data_dcatus["id"], source_data_waf["id"]] jobs = [ {"status": status, "harvest_source_id": source} for status in statuses for source in source_ids ] + interface.add_organization(organization_data) + interface.add_harvest_source(source_data_dcatus) + interface.add_harvest_source(source_data_waf) + for job in jobs: interface.add_harvest_job(job) diff --git a/tests/unit/database/test_db.py b/tests/unit/database/test_db.py index da23fb3e..7f20949a 100644 --- a/tests/unit/database/test_db.py +++ b/tests/unit/database/test_db.py @@ -121,13 +121,22 @@ def test_add_harvest_records( assert success is True assert len(interface.get_all_harvest_records()) == 10 - def test_add_harvest_job_with_id(self, interface, job_data_dcatus): + def test_add_harvest_job_with_id( + self, interface, organization_data, source_data_dcatus, job_data_dcatus + ): + interface.add_organization(organization_data) + interface.add_harvest_source(source_data_dcatus) job = interface.add_harvest_job(job_data_dcatus) assert job.id == job_data_dcatus["id"] assert job.status == job_data_dcatus["status"] assert job.harvest_source_id == job_data_dcatus["harvest_source_id"] - def test_add_harvest_job_without_id(self, interface, job_data_dcatus): + def test_add_harvest_job_without_id( + self, interface, organization_data, source_data_dcatus, job_data_dcatus + ): + interface.add_organization(organization_data) + interface.add_harvest_source(source_data_dcatus) + job_data_dcatus_id = job_data_dcatus["id"] del job_data_dcatus["id"] job = interface.add_harvest_job(job_data_dcatus) @@ -136,17 +145,33 @@ def test_add_harvest_job_without_id(self, interface, job_data_dcatus): assert job.status == job_data_dcatus["status"] assert job.harvest_source_id == job_data_dcatus["harvest_source_id"] - def test_get_harvest_jobs_by_filter(self, interface_with_multiple_jobs): - filters = {"status": "pending", "harvest_source_id": "1234"} + def test_get_harvest_jobs_by_filter( + self, source_data_dcatus, interface_with_multiple_jobs + ): + filters = { + "status": "pending", + "harvest_source_id": f"{source_data_dcatus['id']}", + } filtered_list = interface_with_multiple_jobs.get_harvest_jobs_by_filter(filters) assert len(filtered_list) == 1 assert filtered_list[0]["status"] == "pending" - assert filtered_list[0]["harvest_source_id"] == "1234" + assert filtered_list[0]["harvest_source_id"] == source_data_dcatus["id"] - def test_filter_jobs_by_faceted_filter(self, interface_with_multiple_jobs): + def test_filter_jobs_by_faceted_filter( + self, source_data_dcatus, interface_with_multiple_jobs + ): faceted_list = interface_with_multiple_jobs.get_harvest_jobs_by_faceted_filter( "status", ["pending", "pending_manual"] ) assert len(faceted_list) == 4 assert len([x for x in faceted_list if x["status"] == "pending"]) == 2 - assert len([x for x in faceted_list if x["harvest_source_id"] == "1234"]) == 2 + assert ( + len( + [ + x + for x in faceted_list + if x["harvest_source_id"] == source_data_dcatus["id"] + ] + ) + == 2 + )