feat(schematic): integration testing (Sage-Bionetworks#2400)

* feat(schematic): integration testing (Sage-Bionetworks#2398)

* changed authenticication so that only endpoints that need it have it

* updated schematic

* add patch for access token

* schema endpoints no longer mockeed

* added tests for handle exceptions

* added integration tests

* marked synapse tests

* added error handling for bad schema urls

* fix error message

* add workflow for end to end testing

* fix some test results

* add unit mark

* add unit mark

* add workflow for testing with secrets

* rename file

* fix synapse test file when secrets file doesnt exists

* fix test workflows

* turned synapse ids into secrets in workflow

* turned synapse ids into secrets in workflow

* Update schematic-api-ci.yml

* Update schematic-api-ci.yml

* Update schematic-api-ci.yml

* Update schematic-api-ci.yml

* Update schematic-api-ci.yml

* Update schematic-api-ci.yml

* Update schematic-api-ci.yml

* Update schematic-api-ci.yml

* Update schematic-api-ci.yml

* Update schematic-api-ci.yml

.github/workflows/schematic-api-ci.yml

@@ -0,0 +1,128 @@
+name: Schematic API CI
+on:
+  pull_request:
+    branches:
+      - main
+
+env:
+  NX_BRANCH: ${{ github.event.number }}
+  NX_RUN_GROUP: ${{ github.run_id }}
+  NX_CLOUD_AUTH_TOKEN: ${{ secrets.NX_CLOUD_AUTH_TOKEN }}
+  NX_CLOUD_ENCRYPTION_KEY: ${{ secrets.NX_CLOUD_ENCRYPTION_KEY }}
+  NX_CLOUD_ENV_NAME: 'linux'
+  SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+  DOCKER_USERNAME: ${{ github.actor }}
+  DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  pr:
+    runs-on: ubuntu-22.04-4core-16GBRAM-150GBSSD
+    # Runs this job if:
+    # 1. Triggered by a PR 
+    # 2. The PR originate from the Schematic-API-Staging branch
+    # 3. Targets the main branch
+    if: |
+      github.event_name == 'pull_request' && 
+      github.event.pull_request.base.ref == 'main' &&
+      github.event.pull_request.head.ref == 'Schematic-API-Staging'
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          fetch-depth: 0
+
+      - name: Derive appropriate SHAs for base and head for `nx affected` commands
+        uses: nrwl/nx-set-shas@v3
+
+      - name: Set up Yarn cache
+        uses: actions/cache@v3
+        with:
+          path: '/tmp/.yarn/cache'
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Set up Poetry cache
+        uses: actions/cache@v3
+        with:
+          path: '/tmp/.cache/pypoetry'
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Set up venv cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            /tmp/.local/share/virtualenv
+            **/.venv
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Set up Gradle cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            /tmp/.gradle/caches
+            /tmp/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      - name: Install the Dev Container CLI
+        run: npm install -g @devcontainers/[email protected]
+
+      - name: Start the dev container
+        run: |
+          mkdir -p \
+            /tmp/.yarn/cache \
+            /tmp/.cache/R/renv/cache \
+            /tmp/.cache/pypoetry \
+            /tmp/.local/share/virtualenv \
+            /tmp/.gradle/caches \
+            /tmp/.gradle/wrapper
+
+          devcontainer up \
+            --mount type=bind,source=/tmp/.yarn/cache,target=/workspaces/sage-monorepo/.yarn/cache \
+            --mount type=bind,source=/tmp/.cache/R/renv/cache,target=/home/vscode/.cache/R/renv/cache \
+            --mount type=bind,source=/tmp/.cache/pypoetry,target=/home/vscode/.cache/pypoetry \
+            --mount type=bind,source=/tmp/.local/share/virtualenv,target=/home/vscode/.local/share/virtualenv \
+            --mount type=bind,source=/tmp/.gradle/caches,target=/home/vscode/.gradle/caches \
+            --mount type=bind,source=/tmp/.gradle/wrapper,target=/home/vscode/.gradle/wrapper \
+            --workspace-folder ../sage-monorepo
+
+      - name: Prepare the workspace
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c "
+            sudo chown -R vscode:vscode \
+              /workspaces/sage-monorepo \
+              /home/vscode/.cache \
+              /home/vscode/.local \
+              /home/vscode/.gradle \
+            && . ./dev-env.sh \
+            && workspace-install"
+
+      - name: create synapse config
+        run: |
+          import yaml
+          secrets = {
+              "synapse_token": "${{ secrets.SCHEMATIC_SYNAPSE_TOKEN }}",
+              "test_project": "${{ secrets.SCHEMATIC_TEST_PROJECT }}",
+              "test_dataset": "${{ secrets.SCHEMATIC_TEST_DATASET }}",
+              "test_manifest": "${{ secrets.SCHEMATIC_TEST_MANIFEST }}",
+              "test_asset_view": "${{ secrets.SCHEMATIC_TEST_ASSET_VIEW }}"
+          }
+          for key, secret in secrets.items():
+              assert secret is not None
+              assert isinstance(secret, str)
+              assert len(secret) > 0
+          with open('apps/schematic/api/schematic_api/test/data/synapse_config.yaml', 'w') as file:
+              yaml.dump(secrets, file)
+        shell: python
+
+      - name: Test the affected projects (all)
+        run: |
+          devcontainer exec --workspace-folder ../sage-monorepo bash -c ". ./dev-env.sh \
+            && nx affected --target=test-all"
+
+      - name: Remove the dev container
+        run: docker rm -f sage_devcontainer

apps/schematic/api/.gitignore

@@ -69,6 +69,7 @@ target/
 
 #secrets
 *secrets*
+synapse_config.yaml
 
 #schematic downloaded files
 manifests

apps/schematic/api/project.json

@@ -64,10 +64,7 @@
     "generate": {
       "executor": "nx:run-commands",
       "options": {
-        "commands": [
-          "xargs rm -fr <.openapi-generator/FILES", 
-          "openapi-generator-cli generate"
-        ],
+        "commands": ["xargs rm -fr <.openapi-generator/FILES", "openapi-generator-cli generate"],
         "cwd": "{projectRoot}",
         "parallel": false
       },
@@ -108,7 +105,21 @@
     "test": {
       "executor": "nx:run-commands",
       "options": {
-        "command": "poetry run tox",
+        "command": "poetry run pytest -m 'not secrets'",
+        "cwd": "apps/schematic/api"
+      }
+    },
+    "test-integration": {
+      "executor": "nx:run-commands",
+      "options": {
+        "command": "poetry run pytest -m 'secrets'",
+        "cwd": "apps/schematic/api"
+      }
+    },
+    "test-all": {
+      "executor": "nx:run-commands",
+      "options": {
+        "command": "poetry run pytest",
         "cwd": "apps/schematic/api"
       }
     },

apps/schematic/api/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+markers =
+    synapse: tests that interact with synapse
+    secrets: tests that require secrets

apps/schematic/api/schematic_api/controllers/schema_controller_impl.py

@@ -73,8 +73,8 @@ def get_connected_nodes_from_schematic(
 
 @handle_exceptions
 def get_connected_nodes(
-    relationship_type: str,
     schema_url: str,
+    relationship_type: str,
 ) -> tuple[Union[ConnectedNodesPage, BasicError], int]:
     """Gets a list of connected node pairs via the provide relationship
 

apps/schematic/api/schematic_api/controllers/utils.py

@@ -3,6 +3,7 @@
 import urllib.request
 import shutil
 import tempfile
+from urllib.error import HTTPError
 
 from flask import request  # type: ignore
 from synapseclient.core.exceptions import (  # type: ignore
@@ -59,6 +60,11 @@ def func(*args: Any, **kwargs: Any) -> tuple[Union[Any, BasicError], int]:
             res = BasicError("Synapse entity access error", status, str(error))
             return res, status
 
+        except InvalidSchemaURL as error:
+            status = 404
+            res = BasicError("Invalid URL", status, str(error))
+            return res, status
+
         except Exception as error:  # pylint: disable=broad-exception-caught
             status = 500
             res = BasicError("Internal error", status, str(error))
@@ -67,18 +73,52 @@ def func(*args: Any, **kwargs: Any) -> tuple[Union[Any, BasicError], int]:
     return func
 
 
+class InvalidSchemaURL(Exception):
+    """Raised when a provided url for a schema is incorrect"""
+
+    def __init__(self, message: str, url: str):
+        """
+        Args:
+            message (str): The error message
+            url (str): The provided incorrect URL
+        """
+        self.message = message
+        self.url = url
+        super().__init__(self.message)
+
+    def __str__(self) -> str:
+        return f"{self.message}: {self.url}"
+
+
 def download_schema_file_as_jsonld(schema_url: str) -> str:
     """Downloads a schema and saves it as temp file
 
     Args:
         schema_url (str): The URL of the schema
 
+    Raises:
+        InvalidSchemaURL: When the schema url doesn't exist or is badly formatted
+
     Returns:
         str: The path fo the schema jsonld file
     """
-    with urllib.request.urlopen(schema_url) as response:
-        with tempfile.NamedTemporaryFile(
-            delete=False, suffix=".model.jsonld"
-        ) as tmp_file:
-            shutil.copyfileobj(response, tmp_file)
-            return tmp_file.name
+    try:
+        with urllib.request.urlopen(schema_url) as response:
+            with tempfile.NamedTemporaryFile(
+                delete=False, suffix=".model.jsonld"
+            ) as tmp_file:
+                shutil.copyfileobj(response, tmp_file)
+                return tmp_file.name
+    except ValueError as error:
+        # checks for specific ValueError where the url isn't correctly formatted
+        if str(error).startswith("unknown url type"):
+            raise InvalidSchemaURL(
+                "The provided URL is incorrectly formatted", schema_url
+            ) from error
+        # reraises the ValueError if it isn't the specific type above
+        else:
+            raise
+    except HTTPError as error:
+        raise InvalidSchemaURL(
+            "The provided URL could not be found", schema_url
+        ) from error

apps/schematic/api/schematic_api/test/conftest.py

@@ -8,7 +8,7 @@
 def csv_to_bytes(path: str) -> str:
     """reads in a csv file and returns as bytes"""
     dataframe = pd.read_csv(path)
-    csv_string = dataframe.to_csv(line_terminator="\r\n", index=False)
+    csv_string = dataframe.to_csv(lineterminator="\r\n", index=False)
     return bytes(csv_string, encoding="utf-8")
 
 

apps/schematic/api/schematic_api/test/data/synapse_config_example.yaml

@@ -0,0 +1,5 @@
+synapse_token: xxx
+test_project: syn53072007
+test_dataset: syn53072009
+test_manifest: syn53072014
+test_asset_view: syn53072011