SFR-2741: Run CI via Github Actions

.github/workflows/build-qa.yaml

@@ -36,3 +36,7 @@ jobs:
       - name: Force ECS Update
         run: |
           aws ecs update-service --cluster sfr-pipeline-qa-tf --service sfr-pipeline-qa-tf --force-new-deployment
+  run_ci:
+    needs: publish_qa
+    uses: NYPL/drb-etl-pipeline/.github/workflows/ci.yml@main
+    secrets: inherit

.github/workflows/ci.yaml

@@ -0,0 +1,42 @@
+name: 'Continuous Integration'
+
+on:
+  workflow_call:
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      ENVIRONMENT: qa
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip wheel
+          pip install -r dev-requirements.txt
+          pip install -r requirements.txt
+      - name: Run API tests
+        run: |
+          pytest tests/integration/api

Makefile

@@ -7,13 +7,13 @@ help:
 	@echo "make help"
 
 unit: 
-	python -m pytest --cov-report term-missing --cov=. tests/unit
+	python -m pytest --cov-report term-missing --cov=. tests/unit --env=$(ENV)
 
-allure-test:
-	python -m pytest --alluredir=./allure-results ./tests/unit
+functional:
+	python -m pytest tests/functional --env=$(ENV)
 
 integration: 
-	python -m pytest tests/integration
+	python -m pytest tests/integration --env=$(ENV)
 
 up:
 	$(compose_command) up -d

tests/conftest.py

@@ -0,0 +1,16 @@
+import os
+import pytest
+
+from load_env import load_env_file
+
+
+def pytest_addoption(parser):
+    parser.addoption('--env', action='store', default='local', help='Environment to use for tests')
+
+
+@pytest.fixture(scope='session', autouse=True)
+def setup_env(pytestconfig):
+    environment = os.environ.get('ENVIRONMENT') or pytestconfig.getoption('--env')
+
+    if environment in ['local', 'qa']:
+        load_env_file(environment, file_string=f'config/{environment}.yaml')

tests/functional/processes/ingest/assert_ingested_records.py

@@ -9,11 +9,11 @@ def assert_ingested_records(source_name: str):
     db_manager.generateEngine()
     db_manager.createSession()
 
-    doab_records = (
+    records = (
         db_manager.session.query(Record)
             .filter(Record.source == source_name)
             .filter(Record.date_modified > datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(minutes=5))
             .all()
     )
 
-    assert len(doab_records) > 1
+    assert len(records) > 1

tests/functional/processes/ingest/test_chicago_isac_process.py

@@ -1,9 +1,6 @@
 from processes import ChicagoISACProcess
-from load_env import load_env_file
 from .assert_ingested_records import assert_ingested_records
 
-load_env_file('local', file_string='config/local.yaml')
-
 
 def test_chigaco_isac_process():
     isac_process = ChicagoISACProcess('complete', None, None, None, 5, None)

tests/functional/processes/ingest/test_doab_process.py

@@ -1,9 +1,6 @@
 from processes import DOABProcess
-from load_env import load_env_file
 from .assert_ingested_records import assert_ingested_records
 
-load_env_file('local', file_string='config/local.yaml')
-
 
 def test_doab_process():
     doab_process = DOABProcess('complete', None, None, None, 5, None)

tests/functional/processes/ingest/test_gutenberg_process.py

@@ -1,9 +1,6 @@
 from processes import GutenbergProcess
-from load_env import load_env_file
 from .assert_ingested_records import assert_ingested_records
 
-load_env_file('local', file_string='config/local.yaml')
-
 
 def test_gutenberg_process():
     gutenberg_process = GutenbergProcess('complete', None, None, None, 5, None)

tests/functional/processes/ingest/test_loc_process.py

@@ -1,9 +1,6 @@
 from processes import LOCProcess
-from load_env import load_env_file
 from .assert_ingested_records import assert_ingested_records
 
-load_env_file('local', file_string='config/local.yaml')
-
 
 def test_loc_process():
     loc_process = LOCProcess('complete', None, None, None, 5, None)

tests/functional/processes/ingest/test_met_process.py

@@ -1,9 +1,6 @@
 from processes import LOCProcess
-from load_env import load_env_file
 from .assert_ingested_records import assert_ingested_records
 
-load_env_file('local', file_string='config/local.yaml')
-
 
 def test_met_process():
     met_process = LOCProcess('complete', None, None, None, 5, None)

tests/functional/processes/ingest/test_muse_process.py

@@ -1,8 +1,6 @@
 from processes import MUSEProcess
-from load_env import load_env_file
 from .assert_ingested_records import assert_ingested_records
 
-load_env_file('local', file_string='config/local.yaml')
 
 
 def test_muse_process():

tests/functional/processes/ingest/test_nypl_process.py

@@ -1,9 +1,6 @@
 from processes import NYPLProcess
-from load_env import load_env_file
 from .assert_ingested_records import assert_ingested_records
 
-load_env_file('local', file_string='config/local.yaml')
-
 
 def test_nypl_process():
     nypl_process = NYPLProcess('complete', None, None, None, 5, None)

tests/functional/processes/ingest/test_publisher_backlist_process.py

@@ -1,9 +1,6 @@
 from processes import PublisherBacklistProcess
-from load_env import load_env_file
 from .assert_ingested_records import assert_ingested_records
 
-load_env_file('local', file_string='config/local.yaml')
-
 
 def test_publisher_backlist_process():
     publisher_backlist_project = PublisherBacklistProcess('complete', None, None, None, 5, None)

tests/integration/services/sources/test_nypl_bib_service.py

@@ -1,13 +1,11 @@
 from datetime import datetime, timezone, timedelta
 import pytest
 
-from load_env import load_env_file
 from services import NYPLBibService
 
 class TestNYPLBibService:
     @pytest.fixture
     def test_instance(self):
-        load_env_file('local', file_string='config/local.yaml')
         return NYPLBibService()
 
     def test_get_records(self, test_instance: NYPLBibService):

tests/integration/services/sources/test_publisher_backlist_service.py

@@ -1,13 +1,11 @@
 from datetime import datetime, timezone, timedelta
 import pytest
 
-from load_env import load_env_file
 from services import PublisherBacklistService
 
 class TestPublisherBacklistService:
     @pytest.fixture
     def test_instance(self):
-        load_env_file('local', file_string='config/local.yaml')
         return PublisherBacklistService()
 
     def test_get_records(self, test_instance: PublisherBacklistService):

tests/integration/test_google_integration.py

@@ -1,14 +1,11 @@
 import os
 import pytest
-from load_env import load_env_file
 from services import GoogleDriveService
 
-load_env_file('local-compose', file_string='config/local-compose.yaml')
 
 class TestGoogleDriveService:
     @pytest.fixture
     def test_instance(self):
-        load_env_file('local', file_string='config/local.yaml')
         return GoogleDriveService()
 
     def test_get_drive_file(self, test_instance: GoogleDriveService):

tests/integration/test_oclc_auth_manager.py

@@ -1,7 +1,4 @@
 from managers.oclc_auth import OCLCAuthManager
-from load_env import load_env_file
-
-load_env_file('local-compose', file_string='config/local-compose.yaml')
 
 
 def test_get_search_token():

tests/integration/test_oclc_catalog_manager.py

@@ -1,11 +1,9 @@
 import pytest
-from load_env import load_env_file
 from managers import OCLCCatalogManager
 
 class TestOCLCCatalogManager:
     @pytest.fixture
     def test_instance(self):
-        load_env_file('local-compose', file_string='config/local-compose.yaml')
         return OCLCCatalogManager()
 
     def test_query_bibs(self, test_instance: OCLCCatalogManager):

tests/integration/test_ssm_service.py

@@ -1,13 +1,10 @@
-import os
 import pytest
-from load_env import load_env_file
 
 from services.ssm_service import SSMService
 
 class TestSSMService:
     @pytest.fixture
     def test_instance(self):
-        load_env_file('local', file_string='config/local.yaml')
         return SSMService()
 
     def test_get_parameter(self, test_instance):