From d3c7c4d1b1cee6db1979d52c44d3460486f6d44d Mon Sep 17 00:00:00 2001 From: kyle Date: Thu, 9 Jan 2025 10:59:51 -0500 Subject: [PATCH 1/8] SFR-2741: Run CI via Github Actions --- .github/workflows/ci.yaml | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..af588f22e5 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,46 @@ +name: 'Continuous Integration' + +on: + workflow_call: + # TODO: determine what secrets we need to add to Github or get them from parameter store + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + NYPL_API_CLIENT_ID: + required: true + NYPL_API_CLIENT_SECRET: + required: true + NYPL_API_TOKEN_URL: + required: true + +permissions: + contents: read + +jobs: + integration-tests: + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + NYPL_API_CLIENT_ID: ${{ secrets.NYPL_API_CLIENT_ID }} + NYPL_API_CLIENT_SECRET: ${{ secrets.NYPL_API_CLIENT_SECRET }} + NYPL_API_TOKEN_URL: ${{ secrets.NYPL_API_TOKEN_URL }} + ENVIRONMENT: qa + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install dependencies + run: | + make unit-deps + # TODO: is this the right flow? + - name: Run functional tests + run: | + make functional + - name: Run integration tests + run: | + make integration From 521a120aa06708a3d7488d5ffa3c9810b1574b66 Mon Sep 17 00:00:00 2001 From: kyle Date: Fri, 10 Jan 2025 15:18:48 -0500 Subject: [PATCH 2/8] configuring tests to load env vars --- .github/workflows/ci.yaml | 8 ++------ Makefile | 8 ++++---- tests/conftest.py | 16 ++++++++++++++++ .../processes/ingest/assert_ingested_records.py | 4 ++-- .../ingest/test_chicago_isac_process.py | 3 --- .../processes/ingest/test_doab_process.py | 3 --- .../processes/ingest/test_gutenberg_process.py | 3 --- .../processes/ingest/test_loc_process.py | 3 --- .../processes/ingest/test_met_process.py | 3 --- .../processes/ingest/test_muse_process.py | 2 -- .../processes/ingest/test_nypl_process.py | 3 --- .../ingest/test_publisher_backlist_process.py | 3 --- .../services/sources/test_nypl_bib_service.py | 2 -- .../sources/test_publisher_backlist_service.py | 2 -- tests/integration/test_google_integration.py | 3 --- tests/integration/test_oclc_auth_manager.py | 3 --- tests/integration/test_oclc_catalog_manager.py | 2 -- tests/integration/test_ssm_service.py | 3 --- 18 files changed, 24 insertions(+), 50 deletions(-) create mode 100644 tests/conftest.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index af588f22e5..fd3f398811 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,11 +22,8 @@ jobs: integration-tests: runs-on: ubuntu-latest env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - NYPL_API_CLIENT_ID: ${{ secrets.NYPL_API_CLIENT_ID }} - NYPL_API_CLIENT_SECRET: ${{ secrets.NYPL_API_CLIENT_SECRET }} - NYPL_API_TOKEN_URL: ${{ secrets.NYPL_API_TOKEN_URL }} + AWS_ACCESS: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ENVIRONMENT: qa steps: - uses: actions/checkout@v3 @@ -37,7 +34,6 @@ jobs: - name: Install dependencies run: | make unit-deps - # TODO: is this the right flow? - name: Run functional tests run: | make functional diff --git a/Makefile b/Makefile index fd0b938e80..2590a2dbd9 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,13 @@ help: @echo "make help" unit: - python -m pytest --cov-report term-missing --cov=. tests/unit + python -m pytest --cov-report term-missing --cov=. tests/unit --env=$(ENV) -allure-test: - python -m pytest --alluredir=./allure-results ./tests/unit +functional: + python -m pytest tests/functional --env=$(ENV) integration: - python -m pytest tests/integration + python -m pytest tests/integration --env=$(ENV) up: $(compose_command) up -d diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..4f0087eb0a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,16 @@ +import os +import pytest + +from load_env import load_env_file + + +def pytest_addoption(parser): + parser.addoption('--env', action='store', default='local', help='Environment to use for tests') + + +@pytest.fixture(scope='session', autouse=True) +def setup_env(pytestconfig): + environment = os.environ.get('ENVIRONMENT') or pytestconfig.getoption('--env') + + if environment in ['local', 'qa']: + load_env_file(environment, file_string=f'config/{environment}.yaml') diff --git a/tests/functional/processes/ingest/assert_ingested_records.py b/tests/functional/processes/ingest/assert_ingested_records.py index 76b69822e8..be9aa5b361 100644 --- a/tests/functional/processes/ingest/assert_ingested_records.py +++ b/tests/functional/processes/ingest/assert_ingested_records.py @@ -9,11 +9,11 @@ def assert_ingested_records(source_name: str): db_manager.generateEngine() db_manager.createSession() - doab_records = ( + records = ( db_manager.session.query(Record) .filter(Record.source == source_name) .filter(Record.date_modified > datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(minutes=5)) .all() ) - assert len(doab_records) > 1 + assert len(records) > 1 diff --git a/tests/functional/processes/ingest/test_chicago_isac_process.py b/tests/functional/processes/ingest/test_chicago_isac_process.py index 20a9c5dc52..264a0715e5 100644 --- a/tests/functional/processes/ingest/test_chicago_isac_process.py +++ b/tests/functional/processes/ingest/test_chicago_isac_process.py @@ -1,9 +1,6 @@ from processes import ChicagoISACProcess -from load_env import load_env_file from .assert_ingested_records import assert_ingested_records -load_env_file('local', file_string='config/local.yaml') - def test_chigaco_isac_process(): isac_process = ChicagoISACProcess('complete', None, None, None, 5, None) diff --git a/tests/functional/processes/ingest/test_doab_process.py b/tests/functional/processes/ingest/test_doab_process.py index 6e98049906..b923c1bab4 100644 --- a/tests/functional/processes/ingest/test_doab_process.py +++ b/tests/functional/processes/ingest/test_doab_process.py @@ -1,9 +1,6 @@ from processes import DOABProcess -from load_env import load_env_file from .assert_ingested_records import assert_ingested_records -load_env_file('local', file_string='config/local.yaml') - def test_doab_process(): doab_process = DOABProcess('complete', None, None, None, 5, None) diff --git a/tests/functional/processes/ingest/test_gutenberg_process.py b/tests/functional/processes/ingest/test_gutenberg_process.py index ebf9466651..c5660bcb89 100644 --- a/tests/functional/processes/ingest/test_gutenberg_process.py +++ b/tests/functional/processes/ingest/test_gutenberg_process.py @@ -1,9 +1,6 @@ from processes import GutenbergProcess -from load_env import load_env_file from .assert_ingested_records import assert_ingested_records -load_env_file('local', file_string='config/local.yaml') - def test_gutenberg_process(): gutenberg_process = GutenbergProcess('complete', None, None, None, 5, None) diff --git a/tests/functional/processes/ingest/test_loc_process.py b/tests/functional/processes/ingest/test_loc_process.py index e496ef8704..e836da66c4 100644 --- a/tests/functional/processes/ingest/test_loc_process.py +++ b/tests/functional/processes/ingest/test_loc_process.py @@ -1,9 +1,6 @@ from processes import LOCProcess -from load_env import load_env_file from .assert_ingested_records import assert_ingested_records -load_env_file('local', file_string='config/local.yaml') - def test_loc_process(): loc_process = LOCProcess('complete', None, None, None, 5, None) diff --git a/tests/functional/processes/ingest/test_met_process.py b/tests/functional/processes/ingest/test_met_process.py index 7ac0e1676d..cb95e73994 100644 --- a/tests/functional/processes/ingest/test_met_process.py +++ b/tests/functional/processes/ingest/test_met_process.py @@ -1,9 +1,6 @@ from processes import LOCProcess -from load_env import load_env_file from .assert_ingested_records import assert_ingested_records -load_env_file('local', file_string='config/local.yaml') - def test_met_process(): met_process = LOCProcess('complete', None, None, None, 5, None) diff --git a/tests/functional/processes/ingest/test_muse_process.py b/tests/functional/processes/ingest/test_muse_process.py index 6e093db1b7..3530056ed2 100644 --- a/tests/functional/processes/ingest/test_muse_process.py +++ b/tests/functional/processes/ingest/test_muse_process.py @@ -1,8 +1,6 @@ from processes import MUSEProcess -from load_env import load_env_file from .assert_ingested_records import assert_ingested_records -load_env_file('local', file_string='config/local.yaml') def test_muse_process(): diff --git a/tests/functional/processes/ingest/test_nypl_process.py b/tests/functional/processes/ingest/test_nypl_process.py index 9baf954e9b..5a3b7f52bb 100644 --- a/tests/functional/processes/ingest/test_nypl_process.py +++ b/tests/functional/processes/ingest/test_nypl_process.py @@ -1,9 +1,6 @@ from processes import NYPLProcess -from load_env import load_env_file from .assert_ingested_records import assert_ingested_records -load_env_file('local', file_string='config/local.yaml') - def test_nypl_process(): nypl_process = NYPLProcess('complete', None, None, None, 5, None) diff --git a/tests/functional/processes/ingest/test_publisher_backlist_process.py b/tests/functional/processes/ingest/test_publisher_backlist_process.py index 41297f8a73..ad7b9f205e 100644 --- a/tests/functional/processes/ingest/test_publisher_backlist_process.py +++ b/tests/functional/processes/ingest/test_publisher_backlist_process.py @@ -1,9 +1,6 @@ from processes import PublisherBacklistProcess -from load_env import load_env_file from .assert_ingested_records import assert_ingested_records -load_env_file('local', file_string='config/local.yaml') - def test_publisher_backlist_process(): publisher_backlist_project = PublisherBacklistProcess('complete', None, None, None, 5, None) diff --git a/tests/integration/services/sources/test_nypl_bib_service.py b/tests/integration/services/sources/test_nypl_bib_service.py index 730523a268..4d71cf5574 100644 --- a/tests/integration/services/sources/test_nypl_bib_service.py +++ b/tests/integration/services/sources/test_nypl_bib_service.py @@ -1,13 +1,11 @@ from datetime import datetime, timezone, timedelta import pytest -from load_env import load_env_file from services import NYPLBibService class TestNYPLBibService: @pytest.fixture def test_instance(self): - load_env_file('local', file_string='config/local.yaml') return NYPLBibService() def test_get_records(self, test_instance: NYPLBibService): diff --git a/tests/integration/services/sources/test_publisher_backlist_service.py b/tests/integration/services/sources/test_publisher_backlist_service.py index b493832417..7f5af69b1a 100644 --- a/tests/integration/services/sources/test_publisher_backlist_service.py +++ b/tests/integration/services/sources/test_publisher_backlist_service.py @@ -1,13 +1,11 @@ from datetime import datetime, timezone, timedelta import pytest -from load_env import load_env_file from services import PublisherBacklistService class TestPublisherBacklistService: @pytest.fixture def test_instance(self): - load_env_file('local', file_string='config/local.yaml') return PublisherBacklistService() def test_get_records(self, test_instance: PublisherBacklistService): diff --git a/tests/integration/test_google_integration.py b/tests/integration/test_google_integration.py index 600500cd80..1fa2d89c71 100644 --- a/tests/integration/test_google_integration.py +++ b/tests/integration/test_google_integration.py @@ -1,14 +1,11 @@ import os import pytest -from load_env import load_env_file from services import GoogleDriveService -load_env_file('local-compose', file_string='config/local-compose.yaml') class TestGoogleDriveService: @pytest.fixture def test_instance(self): - load_env_file('local', file_string='config/local.yaml') return GoogleDriveService() def test_get_drive_file(self, test_instance: GoogleDriveService): diff --git a/tests/integration/test_oclc_auth_manager.py b/tests/integration/test_oclc_auth_manager.py index ccfcd8a7b4..a7ef149ab6 100644 --- a/tests/integration/test_oclc_auth_manager.py +++ b/tests/integration/test_oclc_auth_manager.py @@ -1,7 +1,4 @@ from managers.oclc_auth import OCLCAuthManager -from load_env import load_env_file - -load_env_file('local-compose', file_string='config/local-compose.yaml') def test_get_search_token(): diff --git a/tests/integration/test_oclc_catalog_manager.py b/tests/integration/test_oclc_catalog_manager.py index c02ec5037a..853ff9c99a 100644 --- a/tests/integration/test_oclc_catalog_manager.py +++ b/tests/integration/test_oclc_catalog_manager.py @@ -1,11 +1,9 @@ import pytest -from load_env import load_env_file from managers import OCLCCatalogManager class TestOCLCCatalogManager: @pytest.fixture def test_instance(self): - load_env_file('local-compose', file_string='config/local-compose.yaml') return OCLCCatalogManager() def test_query_bibs(self, test_instance: OCLCCatalogManager): diff --git a/tests/integration/test_ssm_service.py b/tests/integration/test_ssm_service.py index 98fda0a726..5bd8eb1eff 100644 --- a/tests/integration/test_ssm_service.py +++ b/tests/integration/test_ssm_service.py @@ -1,13 +1,10 @@ -import os import pytest -from load_env import load_env_file from services.ssm_service import SSMService class TestSSMService: @pytest.fixture def test_instance(self): - load_env_file('local', file_string='config/local.yaml') return SSMService() def test_get_parameter(self, test_instance): From 5704fd6f9b3a37857aafaed469b3166183cd48c3 Mon Sep 17 00:00:00 2001 From: kyle Date: Fri, 10 Jan 2025 15:22:57 -0500 Subject: [PATCH 3/8] tests --- .github/workflows/ci.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fd3f398811..817071b55b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,18 +2,11 @@ name: 'Continuous Integration' on: workflow_call: - # TODO: determine what secrets we need to add to Github or get them from parameter store secrets: AWS_ACCESS_KEY_ID: required: true AWS_SECRET_ACCESS_KEY: required: true - NYPL_API_CLIENT_ID: - required: true - NYPL_API_CLIENT_SECRET: - required: true - NYPL_API_TOKEN_URL: - required: true permissions: contents: read From 5298d18de50b7f81b86134cd33094b464a53dd37 Mon Sep 17 00:00:00 2001 From: kyle Date: Fri, 10 Jan 2025 15:25:54 -0500 Subject: [PATCH 4/8] let's try --- .github/workflows/ci.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 817071b55b..4183fdbd35 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,6 +7,8 @@ on: required: true AWS_SECRET_ACCESS_KEY: required: true + pull_request: + types: [opened, synchronize] permissions: contents: read @@ -27,9 +29,6 @@ jobs: - name: Install dependencies run: | make unit-deps - - name: Run functional tests + - name: Run API tests run: | - make functional - - name: Run integration tests - run: | - make integration + pytest tests/integration/api From 88a9161e68668162a6ece7f8391a4db28c2503c3 Mon Sep 17 00:00:00 2001 From: kyle Date: Fri, 10 Jan 2025 15:26:48 -0500 Subject: [PATCH 5/8] bloop --- .github/workflows/ci.yaml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4183fdbd35..5c24d17879 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,14 +21,24 @@ jobs: AWS_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ENVIRONMENT: qa steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.9' + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- - name: Install dependencies run: | - make unit-deps + python -m pip install --upgrade pip wheel + pip install -r dev-requirements.txt + pip install -r requirements.txt - name: Run API tests run: | pytest tests/integration/api From 8274a84b35729fd477f6945088cc25f829893685 Mon Sep 17 00:00:00 2001 From: kyle Date: Fri, 10 Jan 2025 15:30:15 -0500 Subject: [PATCH 6/8] got it --- .github/workflows/ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5c24d17879..5175023327 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,8 +7,6 @@ on: required: true AWS_SECRET_ACCESS_KEY: required: true - pull_request: - types: [opened, synchronize] permissions: contents: read From d6afe0991c5353ec30fc2760b2fb2e1115ee4e12 Mon Sep 17 00:00:00 2001 From: kyle Date: Mon, 13 Jan 2025 09:48:59 -0500 Subject: [PATCH 7/8] adding ci after deploy to QA --- .github/workflows/build-qa.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-qa.yaml b/.github/workflows/build-qa.yaml index a8e29af27c..457c15924a 100644 --- a/.github/workflows/build-qa.yaml +++ b/.github/workflows/build-qa.yaml @@ -36,3 +36,7 @@ jobs: - name: Force ECS Update run: | aws ecs update-service --cluster sfr-pipeline-qa-tf --service sfr-pipeline-qa-tf --force-new-deployment + run_ci: + needs: publish_qa + uses: NYPL/drb-etl-pipeline/.github/workflows/ci.yml@main + secrets: inherit From 56ece262d852fd4213604eff25634785cb54b6f8 Mon Sep 17 00:00:00 2001 From: kyle Date: Mon, 13 Jan 2025 11:02:39 -0500 Subject: [PATCH 8/8] upgrading marketplace actions --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5175023327..159f554b76 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,13 +19,13 @@ jobs: AWS_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ENVIRONMENT: qa steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.9 uses: actions/setup-python@v2 with: python-version: '3.9' - name: Cache dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}