From 523b5ccdd288754651bf59fd6c987da51e722b5f Mon Sep 17 00:00:00 2001 From: Joe Flack Date: Wed, 4 Sep 2024 18:32:20 -0400 Subject: [PATCH] Disabling OAuth - It has been causing problems, the latest of which was in deployment. --- .github/workflows/backend_dev.yml | 5 +++-- .github/workflows/backend_prod.yml | 5 +++-- .github/workflows/kill_idle_cons.yml | 5 +++-- .github/workflows/refresh_counts.yml | 5 +++-- .github/workflows/refresh_db.yml | 5 +++-- .github/workflows/refresh_from_datasets.yml | 5 +++-- .github/workflows/refresh_voc.yml | 5 +++-- .../workflows/resolve_fetch_failures_0_members.yml | 5 +++-- .../resolve_fetch_failures_excess_items.yml | 5 +++-- .../workflows/test_backend_e2e_and_unit_and_qc.yml | 7 +++---- enclave_wrangler/utils.py | 13 ++++++++----- install-requirements.sh | 2 -- requirements-unlocked.txt | 2 +- requirements.txt | 1 - 14 files changed, 39 insertions(+), 31 deletions(-) delete mode 100644 install-requirements.sh diff --git a/.github/workflows/backend_dev.yml b/.github/workflows/backend_dev.yml index 597a6f310..ed7906ed7 100644 --- a/.github/workflows/backend_dev.yml +++ b/.github/workflows/backend_dev.yml @@ -53,9 +53,10 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" # todo: optional: run tests # - name: Run tests # run: python -m unittest discover -v diff --git a/.github/workflows/backend_prod.yml b/.github/workflows/backend_prod.yml index 069133f19..1ab0b53f2 100644 --- a/.github/workflows/backend_prod.yml +++ b/.github/workflows/backend_prod.yml @@ -53,9 +53,10 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" # todo: optional: run tests # - name: Run tests # run: python -m unittest discover -v diff --git a/.github/workflows/kill_idle_cons.yml b/.github/workflows/kill_idle_cons.yml index 0545d9d7e..c283b7c71 100644 --- a/.github/workflows/kill_idle_cons.yml +++ b/.github/workflows/kill_idle_cons.yml @@ -33,9 +33,10 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" # Run the action - name: Kill idle connections diff --git a/.github/workflows/refresh_counts.yml b/.github/workflows/refresh_counts.yml index 5c670485e..110ef9313 100644 --- a/.github/workflows/refresh_counts.yml +++ b/.github/workflows/refresh_counts.yml @@ -40,9 +40,10 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" # Refresh - name: Refresh counts diff --git a/.github/workflows/refresh_db.yml b/.github/workflows/refresh_db.yml index bedfb96f7..042069192 100644 --- a/.github/workflows/refresh_db.yml +++ b/.github/workflows/refresh_db.yml @@ -55,9 +55,10 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" # Refresh - name: Refresh db diff --git a/.github/workflows/refresh_from_datasets.yml b/.github/workflows/refresh_from_datasets.yml index 1928aef9f..e46141018 100644 --- a/.github/workflows/refresh_from_datasets.yml +++ b/.github/workflows/refresh_from_datasets.yml @@ -37,9 +37,10 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" # todo: Do we actually need this here? # i. termhub-vocab: Not needed for this i think diff --git a/.github/workflows/refresh_voc.yml b/.github/workflows/refresh_voc.yml index db3cd471a..328db41bf 100644 --- a/.github/workflows/refresh_voc.yml +++ b/.github/workflows/refresh_voc.yml @@ -40,9 +40,10 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" # Refresh - name: Refresh vocab diff --git a/.github/workflows/resolve_fetch_failures_0_members.yml b/.github/workflows/resolve_fetch_failures_0_members.yml index 717dca0f1..6ee188fa7 100644 --- a/.github/workflows/resolve_fetch_failures_0_members.yml +++ b/.github/workflows/resolve_fetch_failures_0_members.yml @@ -43,9 +43,10 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" - name: Resolve fetch failures run: | diff --git a/.github/workflows/resolve_fetch_failures_excess_items.yml b/.github/workflows/resolve_fetch_failures_excess_items.yml index eeed375bd..fabceae75 100644 --- a/.github/workflows/resolve_fetch_failures_excess_items.yml +++ b/.github/workflows/resolve_fetch_failures_excess_items.yml @@ -40,9 +40,10 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" - name: Resolve fetch failures run: | diff --git a/.github/workflows/test_backend_e2e_and_unit_and_qc.yml b/.github/workflows/test_backend_e2e_and_unit_and_qc.yml index 4fbf3805f..d85ff3fd3 100644 --- a/.github/workflows/test_backend_e2e_and_unit_and_qc.yml +++ b/.github/workflows/test_backend_e2e_and_unit_and_qc.yml @@ -42,10 +42,9 @@ jobs: # python -m pip install --upgrade pip # pip install --upgrade wheel # pip install --upgrade setuptools + # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) + # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" pip install -r requirements.txt - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" - TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) - python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" - name: Run tests run: make test-backend diff --git a/enclave_wrangler/utils.py b/enclave_wrangler/utils.py index e587dd8ea..19eb817c7 100644 --- a/enclave_wrangler/utils.py +++ b/enclave_wrangler/utils.py @@ -13,7 +13,10 @@ from datetime import datetime, timezone, timedelta from http.client import HTTPConnection from requests import Response -from vshub_sdk.core.api import UserTokenAuth +try: + from vshub_sdk.core.api import UserTokenAuth +except ModuleNotFoundError: # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 + pass from enclave_wrangler.config import OUTDIR_OBJECTS, config, TERMHUB_VERSION, CSET_VERSION_MIN_ID from backend.utils import dump @@ -48,7 +51,7 @@ class ActionValidateError(RuntimeError): """Wrapper just to handle errors from this module""" -def get_headers(personal=False, content_type="application/json", for_curl=False, oauth=True): +def get_headers(personal=False, content_type="application/json", for_curl=False, oauth=False): """Format headers for enclave calls todo: fix all this -- we've been switching back and forth between service token and personal because some APIs are @@ -116,7 +119,7 @@ def get_oauth_token() -> str: return OAUTH_TOKEN if OAUTH_TOKEN_EXPIRES > datetime.now() else refresh_oauth_token() -def get_auth_token(oauth=True) -> str: +def get_auth_token(oauth=False) -> str: """Returns the auth token for the N3C Palantir Foundry data enclave.""" service_user_token: str = config[TOKEN_KEY] return get_oauth_token() if oauth else service_user_token @@ -237,7 +240,7 @@ def handle_response_error( }) -def enclave_get(url: str, verbose: bool = True, args: Dict = {}, error_dir: str = None, oauth=True) -> Response: +def enclave_get(url: str, verbose: bool = True, args: Dict = {}, error_dir: str = None, oauth=False) -> Response: """Get from the enclave and print curl""" if verbose: print_curl(url, args=args) @@ -502,7 +505,7 @@ def process_validate_errors(response: Response, err_type: Exception=None, print_ return out_errors -def enclave_post(url: str, data: Union[List, Dict], raise_validate_error: bool=False, verbose=True, oauth=True) -> Response: +def enclave_post(url: str, data: Union[List, Dict], raise_validate_error: bool=False, verbose=True, oauth=False) -> Response: """Post to the enclave and handle / report on some common issues""" if verbose: print_curl(url, data) diff --git a/install-requirements.sh b/install-requirements.sh deleted file mode 100644 index 615c649c7..000000000 --- a/install-requirements.sh +++ /dev/null @@ -1,2 +0,0 @@ -python3 -m pip install -r requirements.txt -python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" diff --git a/requirements-unlocked.txt b/requirements-unlocked.txt index 4e61a8438..512858e4e 100755 --- a/requirements-unlocked.txt +++ b/requirements-unlocked.txt @@ -37,4 +37,4 @@ setuptools # non-PyPi packages # - For the command to install these, consult: install-requirements.sh # foundry-sdk-runtime==1.113.0 -# vshub_sdk==0.1.0 +# vshub_sdk==0.1.0 # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 diff --git a/requirements.txt b/requirements.txt index fc72c64ce..0efae5da5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -211,7 +211,6 @@ validators==0.20.0 virtualenv==20.15.1 virtualenv-clone==0.5.7 virtualenvwrapper==4.8.4 -# vshub_sdk==0.1.0 watchdog==3.0.0 watchfiles==0.15.0 webcolors==1.12