feat(schematic): integration testing (#2398)

* changed authenticication so that only endpoints that need it have it * updated schematic * add patch for access token * schema endpoints no longer mockeed * added tests for handle exceptions * added integration tests * marked synapse tests * added error handling for bad schema urls * fix error message * add workflow for end to end testing * fix some test results * add unit mark * add unit mark * add workflow for testing with secrets * rename file * fix synapse test file when secrets file doesnt exists * fix test workflows * turned synapse ids into secrets in workflow * turned synapse ids into secrets in workflow
Sage-Bionetworks · Dec 8, 2023 · e82e87f · e82e87f
1 parent c0c7adf
commit e82e87f
Show file tree

Hide file tree

Showing 13 changed files with 797 additions and 511 deletions.
diff --git a/.github/workflows/schematic-api-ci.yml b/.github/workflows/schematic-api-ci.yml
@@ -0,0 +1,265 @@
+name: Schematic API CI
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  NX_BRANCH: ${{ github.event.number }}
+  NX_RUN_GROUP: ${{ github.run_id }}
+  NX_CLOUD_AUTH_TOKEN: ${{ secrets.NX_CLOUD_AUTH_TOKEN }}
+  NX_CLOUD_ENCRYPTION_KEY: ${{ secrets.NX_CLOUD_ENCRYPTION_KEY }}
+  NX_CLOUD_ENV_NAME: 'linux'
+  SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+  DOCKER_USERNAME: ${{ github.actor }}
+  DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  push:
+    runs-on: ubuntu-22.04-4core-16GBRAM-150GBSSD
+    if: ${{ github.event_name != 'pull_request' }}
+    environment: schematic
+    steps:
+      - uses: actions/checkout@v3
+        name: Checkout [${{ github.ref_name }}]
+        with:
+          fetch-depth: 0
+
+      - name: Derive appropriate SHAs for base and head for `nx affected` commands
+        uses: nrwl/nx-set-shas@v3
+
+      - name: Set up Yarn cache
+        uses: actions/cache@v3
+        with:
+          path: '/tmp/.yarn/cache'
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Set up Renv cache
+        uses: actions/cache@v3
+        with:
+          path: '/tmp/.cache/R/renv/cache'
+          key: ${{ runner.os }}-renv-cache-${{ hashFiles('**/renv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-renv-cache-
+
+      - name: Set up Poetry cache
+        uses: actions/cache@v3
+        with:
+          path: '/tmp/.cache/pypoetry'
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Set up venv cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            /tmp/.local/share/virtualenv
+            **/.venv
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Set up Gradle cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            /tmp/.gradle/caches
+            /tmp/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      - name: Install the Dev Container CLI
+        run: npm install -g @devcontainers/[email protected]
+
+      - name: Start the dev container
+        run: |
+          mkdir -p \
+            /tmp/.yarn/cache \
+            /tmp/.cache/R/renv/cache \
+            /tmp/.cache/pypoetry \
+            /tmp/.local/share/virtualenv \
+            /tmp/.gradle/caches \
+            /tmp/.gradle/wrapper
+
+          devcontainer up \
+            --mount type=bind,source=/tmp/.yarn/cache,target=/workspaces/sage-monorepo/.yarn/cache \
+            --mount type=bind,source=/tmp/.cache/R/renv/cache,target=/home/vscode/.cache/R/renv/cache \
+            --mount type=bind,source=/tmp/.cache/pypoetry,target=/home/vscode/.cache/pypoetry \
+            --mount type=bind,source=/tmp/.local/share/virtualenv,target=/home/vscode/.local/share/virtualenv \
+            --mount type=bind,source=/tmp/.gradle/caches,target=/home/vscode/.gradle/caches \
+            --mount type=bind,source=/tmp/.gradle/wrapper,target=/home/vscode/.gradle/wrapper \
+            --workspace-folder ../sage-monorepo
+
+      - name: Prepare the workspace
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c "
+            sudo chown -R vscode:vscode \
+              /workspaces/sage-monorepo \
+              /home/vscode/.cache \
+              /home/vscode/.local \
+              /home/vscode/.gradle \
+            && . ./dev-env.sh \
+            && workspace-install-affected"
+
+      - name: Lint the affected projects
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=lint"
+
+      - name: Build the affected projects
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=build,server"
+
+      - name: Test the affected projects (unit)
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=test"
+
+      - name: Test the affected projects (integration)
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=integration-test"
+
+      - name: Scan the affected projects with Sonar
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=sonar"
+
+      - name: Publish the images of the affected projects
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin ghcr.io \
+            && nx affected --target=publish-image"
+
+      - name: Remove the dev container
+        run: docker rm -f sage_devcontainer
+
+  pr:
+    runs-on: ubuntu-22.04-4core-16GBRAM-150GBSSD
+    # Runs this job if triggered by a PR and if at least one of these conditions are true:
+    # - the PR originate from the Schematic-API-Staging branch and targets the main branch
+    if: |
+      github.event_name == 'pull_request' && 
+      github.event.pull_request.base.ref == 'main' &&
+      github.event.pull_request.head.ref == 'Schematic-API-Staging'
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          fetch-depth: 0
+
+      - name: Derive appropriate SHAs for base and head for `nx affected` commands
+        uses: nrwl/nx-set-shas@v3
+
+      - name: Set up Yarn cache
+        uses: actions/cache@v3
+        with:
+          path: '/tmp/.yarn/cache'
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Set up Renv cache
+        uses: actions/cache@v3
+        with:
+          path: '/tmp/.cache/R/renv/cache'
+          key: ${{ runner.os }}-renv-cache-${{ hashFiles('**/renv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-renv-cache-
+
+      - name: Set up Poetry cache
+        uses: actions/cache@v3
+        with:
+          path: '/tmp/.cache/pypoetry'
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Set up venv cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            /tmp/.local/share/virtualenv
+            **/.venv
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Set up Gradle cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            /tmp/.gradle/caches
+            /tmp/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      - name: Install the Dev Container CLI
+        run: npm install -g @devcontainers/[email protected]
+
+      - name: Start the dev container
+        run: |
+          mkdir -p \
+            /tmp/.yarn/cache \
+            /tmp/.cache/R/renv/cache \
+            /tmp/.cache/pypoetry \
+            /tmp/.local/share/virtualenv \
+            /tmp/.gradle/caches \
+            /tmp/.gradle/wrapper
+
+          devcontainer up \
+            --mount type=bind,source=/tmp/.yarn/cache,target=/workspaces/sage-monorepo/.yarn/cache \
+            --mount type=bind,source=/tmp/.cache/R/renv/cache,target=/home/vscode/.cache/R/renv/cache \
+            --mount type=bind,source=/tmp/.cache/pypoetry,target=/home/vscode/.cache/pypoetry \
+            --mount type=bind,source=/tmp/.local/share/virtualenv,target=/home/vscode/.local/share/virtualenv \
+            --mount type=bind,source=/tmp/.gradle/caches,target=/home/vscode/.gradle/caches \
+            --mount type=bind,source=/tmp/.gradle/wrapper,target=/home/vscode/.gradle/wrapper \
+            --workspace-folder ../sage-monorepo
+
+      - name: Prepare the workspace
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c "
+            sudo chown -R vscode:vscode \
+              /workspaces/sage-monorepo \
+              /home/vscode/.cache \
+              /home/vscode/.local \
+              /home/vscode/.gradle \
+            && . ./dev-env.sh \
+            && workspace-install"
+
+      - name: Lint the affected projects
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=lint"
+
+      - name: Build the affected projects
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=build,server"
+
+      - name: create synapse config
+        run: |
+          import yaml
+          dict = {
+            "synapse_token": "${{secrets.SCHEMATIC_SYNAPSE_TOKEN}}",
+            "test_project": "${{secrets.SCHEMATIC_TEST_PROJECT}}",
+            "test_dataset": "${{secrets.SCHEMATIC_TEST_DATASET}}",
+            "test_manifest": "${{secrets.SCHEMATIC_TEST_MANIFEST}}",
+            "test_asset_view": "${{secrets.SCHEMATIC_TEST_ASSET_VIEW}}",
+          }
+          with open('apps/schematic/api/schematic_api/test/data/synapse_config.yaml', 'w') as file:
+              yaml.dump(dict, file)
+        shell: python
+
+      - name: Test the affected projects (all)
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=test-all"
+
+      - name: Build the images of the affected projects
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=build-image"
+
+      - name: Remove the dev container
+        run: docker rm -f sage_devcontainer
diff --git a/apps/schematic/api/.gitignore b/apps/schematic/api/.gitignore
@@ -69,6 +69,7 @@ target/
 
 #secrets
 *secrets*
+synapse_config.yaml
 
 #schematic downloaded files
 manifests

diff --git a/apps/schematic/api/project.json b/apps/schematic/api/project.json
@@ -64,10 +64,7 @@
     "generate": {
       "executor": "nx:run-commands",
       "options": {
-        "commands": [
-          "xargs rm -fr <.openapi-generator/FILES", 
-          "openapi-generator-cli generate"
-        ],
+        "commands": ["xargs rm -fr <.openapi-generator/FILES", "openapi-generator-cli generate"],
         "cwd": "{projectRoot}",
         "parallel": false
       },
@@ -108,7 +105,21 @@
     "test": {
       "executor": "nx:run-commands",
       "options": {
-        "command": "poetry run tox",
+        "command": "poetry run pytest -m 'not secrets'",
+        "cwd": "apps/schematic/api"
+      }
+    },
+    "test-integration": {
+      "executor": "nx:run-commands",
+      "options": {
+        "command": "poetry run pytest -m 'secrets'",
+        "cwd": "apps/schematic/api"
+      }
+    },
+    "test-all": {
+      "executor": "nx:run-commands",
+      "options": {
+        "command": "poetry run pytest",
         "cwd": "apps/schematic/api"
       }
     },

diff --git a/apps/schematic/api/pytest.ini b/apps/schematic/api/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+markers =
+    synapse: tests that interact with synapse
+    secrets: tests that require secrets
diff --git a/apps/schematic/api/schematic_api/controllers/schema_controller_impl.py b/apps/schematic/api/schematic_api/controllers/schema_controller_impl.py
@@ -73,8 +73,8 @@ def get_connected_nodes_from_schematic(
 
 @handle_exceptions
 def get_connected_nodes(
-    relationship_type: str,
     schema_url: str,
+    relationship_type: str,
 ) -> tuple[Union[ConnectedNodesPage, BasicError], int]:
     """Gets a list of connected node pairs via the provide relationship