FE-231 upgrade aiohttp to 3.9.4 (#337)

* add todo for later * Update to aiotthp to 3.9.4 FE-231 * Add note about prod project for google auth * skipping all e2e tests - see FE-203 & FE-204 - by not running them, rather than using decorators or config files. * Update requirements.txt --------- Co-authored-by: dsp-fieldeng-bot <[email protected]>
DataBiosphere · Jun 5, 2024 · eef9c3a · eef9c3a
1 parent bd34278
commit eef9c3a
Show file tree

Hide file tree

Showing 8 changed files with 18 additions and 14 deletions.
diff --git a/.github/workflows/validate_pull_request_main.yaml b/.github/workflows/validate_pull_request_main.yaml
@@ -51,6 +51,9 @@ jobs:
         run: sbt "set ThisBuild/coverageEnabled := true" test IntegrationTest/test coverageAggregate
       - name: Publish Scala Test coverage
         uses: codecov/codecov-action@v1
-      - name: Run E2E test suite
-        run: poetry run pytest -v -m e2e
-        working-directory: ${{ github.workspace }}/orchestration
+# We are skipping the E2E tests during PR validation until we refactor to test against TDR prod
+# See FE-203 and FE-204 for complete details
+# See the e2e README for instructions to run the tests locally (orchestration/hca_orchestration/tests/e2e/README.md)
+#      - name: Run E2E test suite
+#        run: poetry run pytest -v -m e2e
+#        working-directory: ${{ github.workspace }}/orchestration
diff --git a/README.md b/README.md
@@ -14,6 +14,7 @@ system design information.
       the docker compose command, as this will pull the latest image from Artifact Registry._
 * Authenticate with gcloud using your Broad credentials `gcloud auth login`
 * Then set up your billing project `gcloud config set project PROJECT_ID`
+  * For prod this is `mystical-slate-284720`
 * Then set up your default login for applications `gcloud auth application-default login`
 * Build and run the dataflow tests
   * From the repository/image root: `sbt test` 

diff --git a/orchestration/hca_manage/manifest.py b/orchestration/hca_manage/manifest.py
@@ -109,7 +109,7 @@ def _parse_csv(csv_path: str, env: str, project_id_only: bool = False,
                 project_id = row[1]
                 key = project_id
             else:
-
+                # TODO check for all caps - change to all caps if not, then match
                 if institution not in STAGING_AREA_BUCKETS[env]:
                     raise Exception(f"Unknown institution {institution} found")
 

diff --git a/orchestration/hca_orchestration/tests/e2e/README.md b/orchestration/hca_orchestration/tests/e2e/README.md
@@ -5,7 +5,9 @@ They run against a real dataset, using a local beam runner and staged fixture da
 GCS. They are excluded from the default unit test suite given how long they take to run,
 but are run as part of CI (and are completely runnable locally).
 
-Note that test_load_hca.py is currently being skipped - see FE-203 for more details.
+Note that these currently being skipped in CI (validate_pull_request_main.yaml)\
+and will be re-enabled when either there is time to refactor to use TDR prod or we've migrated (and will use TDR prod)\
+See FE-203 & FE-204 for more details.
 
 ### To run locally:
 

diff --git a/orchestration/hca_orchestration/tests/e2e/test_load_hca.py b/orchestration/hca_orchestration/tests/e2e/test_load_hca.py
@@ -9,10 +9,7 @@
     assert_metadata_loaded,
 )
 
-logging.info("Skipping test_load_hca.py - FE-203")
 
-
-@pytest.mark.skip(reason="This test is failing against TDR/BQ dev env - FE-203")
 @pytest.mark.e2e
 def test_load_hca(load_hca_run_config, dataset_name, tdr_bigquery_client, dataset_info):
     job = load_hca_job()

diff --git a/orchestration/poetry.lock b/orchestration/poetry.lock
diff --git a/orchestration/pyproject.toml b/orchestration/pyproject.toml
@@ -35,7 +35,7 @@ sentry-sdk = "^1.39.2"
 typing-extensions = "^3.7.4"
 # werkzeug = "2.2.3"
 # will have to update dagit which means updating broad-dagster-utils - FE-36
-aiohttp = "3.9.2"
+aiohttp = "3.9.4"
 
 [tool.poetry.dev-dependencies]
 # NB this notation is not preferred after poetry 1.2.0 https://python-poetry.org/docs/master/managing-dependencies/

diff --git a/orchestration/requirements.txt b/orchestration/requirements.txt
@@ -1,4 +1,4 @@
-aiohttp==3.9.2; python_version >= "3.8"
+aiohttp==3.9.4; python_version >= "3.8"
 aiosignal==1.3.1; python_version >= "3.9" and python_version < "3.10" and python_full_version >= "3.6.0"
 alembic==1.6.5; python_version >= "3.9" and python_full_version < "3.0.0" and python_version < "3.10" or python_version >= "3.9" and python_version < "3.10" and python_full_version >= "3.6.0"
 argo-workflows==5.0.0
@@ -23,7 +23,7 @@ dagster==0.12.14
 data-repo-client==1.542.0
 docstring-parser==0.15; python_version >= "3.9" and python_version < "3.10"
 frozenlist==1.4.0; python_version >= "3.9" and python_version < "3.10" and python_full_version >= "3.6.0"
-google-api-core==2.17.1; python_version >= "3.9" and python_version < "3.10" and (python_version >= "3.9" and python_full_version < "3.0.0" and python_version < "3.10" or python_full_version >= "3.6.0" and python_version >= "3.9" and python_version < "3.10") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7")
+google-api-core==2.19.0; python_version >= "3.9" and python_version < "3.10" and (python_version >= "3.9" and python_full_version < "3.0.0" and python_version < "3.10" or python_full_version >= "3.6.0" and python_version >= "3.9" and python_version < "3.10") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7")
 google-api-python-client==1.12.11; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 google-auth-httplib2==0.1.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
 google-auth==2.23.3; python_version >= "3.9" and python_full_version < "3.0.0" and python_version < "3.10" or python_full_version >= "3.6.0" and python_version >= "3.9" and python_version < "3.10"