feat(binding/nodejs): upgrade test behavior and infra (#3297)

* feat: upgrade unit test * chore: update ci file * fix: cr & ci problems * fix: ci * feat: add file header * fix: ci * fix: ci * fix: ci fail * fix: upgrade test * fix: ci * fix: add test * fix: test case name * fix: cr * fix: cr * feat: add ci test config * fix: test logic * fix: test logic * Generate plan for nodejs Signed-off-by: Xuanwo <[email protected]> * Fix typo Signed-off-by: Xuanwo <[email protected]> * Fix working directory Signed-off-by: Xuanwo <[email protected]> * Fix config loading Signed-off-by: Xuanwo <[email protected]> * polish Signed-off-by: Xuanwo <[email protected]> * Fix format Signed-off-by: Xuanwo <[email protected]> * FIx dotenv Signed-off-by: Xuanwo <[email protected]> * Fix tests Signed-off-by: Xuanwo <[email protected]> --------- Signed-off-by: Xuanwo <[email protected]> Co-authored-by: Xuanwo <[email protected]>
apache · Nov 1, 2023 · 8618b57 · 8618b57
1 parent a08e30c
commit 8618b57
Show file tree

Hide file tree

Showing 21 changed files with 2,838 additions and 1,431 deletions.
diff --git a/.github/actions/behavior_test_binding_nodejs/action.yaml b/.github/actions/behavior_test_binding_nodejs/action.yaml
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Test Core
+description: 'Test Core with given setup and service'
+inputs:
+  setup:
+    description: "The setup action for test"
+  service:
+    description: "The service to test"
+  feature:
+    description: "The feature to test"
+
+runs:
+  using: "composite"
+  steps:
+      - name: Setup
+        shell: bash
+        run: |
+            mkdir -p ./dynamic_test_binding_nodejs &&
+            cat <<EOF >./dynamic_test_binding_nodejs/action.yml
+            runs:
+              using: composite
+              steps:
+              - name: Setup Test
+                uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
+              - name: Run Test Binding NodeJS
+                shell: bash
+                working-directory: bindings/nodejs
+                run: |
+                    yarn build:debug
+                    yarn test
+                env:
+                  NAPI_FEATURES: ${{ inputs.feature }}
+                  OPENDAL_TEST: ${{ inputs.service }}
+            EOF
+      - name: Run
+        uses: ./dynamic_test_binding_nodejs
diff --git a/.github/scripts/behavior_test/plan.py b/.github/scripts/behavior_test/plan.py
@@ -77,6 +77,8 @@ class Hint:
     binding_java: bool = field(default=False, init=False)
     # Is binding python affected?
     binding_python: bool = field(default=False, init=False)
+    # Is binding nodejs affected?
+    binding_nodejs: bool = field(default=False, init=False)
 
     # Should we run all services test?
     all_service: bool = field(default=False, init=False)
@@ -99,14 +101,20 @@ def calculate_hint(changed_files: list[str]) -> Hint:
             hint.core = True
             hint.binding_java = True
             hint.binding_python = True
+            hint.binding_nodejs = True
             hint.all_service = True
         if p == ".github/workflows/behavior_test_core.yml":
             hint.core = True
             hint.all_service = True
         if p == ".github/workflows/behavior_test_binding_java.yml":
             hint.binding_java = True
+            hint.all_service = True
+        if p == ".github/workflows/behavior_test_binding_python.yml":
             hint.binding_python = True
             hint.all_service = True
+        if p == ".github/workflows/behavior_test_binding_nodejs.yml":
+            hint.binding_nodejs = True
+            hint.all_service = True
 
         # core affected
         if (
@@ -119,6 +127,7 @@ def calculate_hint(changed_files: list[str]) -> Hint:
             hint.core = True
             hint.binding_java = True
             hint.binding_python = True
+            hint.binding_nodejs = True
             hint.all_service = True
 
         # binding java affected.
@@ -131,12 +140,18 @@ def calculate_hint(changed_files: list[str]) -> Hint:
             hint.binding_python = True
             hint.all_service = True
 
+        # binding nodejs affected.
+        if p.startswith("bindings/python/"):
+            hint.binding_nodejs = True
+            hint.all_service = True
+
         # core service affected
         match = re.search(service_pattern, p)
         if match:
             hint.core = True
             hint.binding_java = True
             hint.binding_python = True
+            hint.binding_nodejs = True
             hint.services.add(match.group(1))
 
         # core test affected
@@ -145,6 +160,7 @@ def calculate_hint(changed_files: list[str]) -> Hint:
             hint.core = True
             hint.binding_java = True
             hint.binding_python = True
+            hint.binding_nodejs = True
             hint.services.add(match.group(1))
 
     return hint
@@ -230,23 +246,47 @@ def generate_binding_python_cases(
     return cases
 
 
+def generate_binding_nodejs_cases(
+    cases: list[dict[str, str]], hint: Hint
+) -> list[dict[str, str]]:
+    cases = unique_cases(cases)
+
+    if os.getenv("GITHUB_IS_PUSH") == "true":
+        return cases
+
+    # Return empty if core is False
+    if not hint.binding_nodejs:
+        return []
+
+    # Return all services if all_service is True
+    if hint.all_service:
+        return cases
+
+    # Filter all cases that not shown un in changed files
+    cases = [v for v in cases if v["service"] in hint.services]
+    return cases
+
+
 def plan(changed_files: list[str]) -> dict[str, Any]:
     cases = provided_cases()
     hint = calculate_hint(changed_files)
 
     core_cases = generate_core_cases(cases, hint)
     binding_java_cases = generate_binding_java_cases(cases, hint)
     binding_python_cases = generate_binding_python_cases(cases, hint)
+    binding_nodejs_cases = generate_binding_nodejs_cases(cases, hint)
 
     jobs = {
         "components": {
             "core": False,
             "binding_java": False,
             "binding_python": False,
+            "binding_nodejs": False,
         },
         "core": [],
         "binding_java": [],
         "binding_python": [],
+        "binding_nodejs": [],
     }
 
     if len(core_cases) > 0:
@@ -274,6 +314,11 @@ def plan(changed_files: list[str]) -> dict[str, Any]:
         jobs["binding_python"].append(
             {"os": "ubuntu-latest", "cases": binding_python_cases}
         )
+    if len(binding_nodejs_cases) > 0:
+        jobs["components"]["binding_nodejs"] = True
+        jobs["binding_nodejs"].append(
+            {"os": "ubuntu-latest", "cases": binding_nodejs_cases}
+        )
 
     return jobs
 

diff --git a/.github/workflows/behavior_test.yml b/.github/workflows/behavior_test.yml
@@ -115,3 +115,16 @@ jobs:
     with:
       os: ${{ matrix.os }}
       cases: ${{ toJson(matrix.cases) }}
+
+  test_binding_nodejs:
+    name: binding_nodejs / ${{ matrix.os }}
+    needs: [plan]
+    if: fromJson(needs.plan.outputs.plan).components.binding_nodejs
+    secrets: inherit
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.plan.outputs.plan).binding_nodejs }}
+    uses: ./.github/workflows/behavior_test_binding_nodejs.yml
+    with:
+      os: ${{ matrix.os }}
+      cases: ${{ toJson(matrix.cases) }}
diff --git a/.github/workflows/behavior_test_binding_nodejs.yml b/.github/workflows/behavior_test_binding_nodejs.yml
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Behavior Test Binding NodeJS
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        required: true
+        type: string
+      cases:
+        required: true
+        type: string
+
+jobs:
+  test:
+    name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }}
+    runs-on: ${{ inputs.os }}
+    strategy:
+      matrix:
+        cases: ${{ fromJson(inputs.cases) }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+        with:
+          need-nextest: true
+          need-protoc: true
+          need-rocksdb: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # TODO: 1password is only supported on linux
+      #
+      # Waiting for https://github.com/1Password/load-secrets-action/issues/46
+      - name: Setup 1Password Connect
+        if: runner.os == 'Linux'
+        uses: 1password/load-secrets-action/configure@v1
+        with:
+          connect-host: ${{ secrets.OP_CONNECT_HOST }}
+          connect-token: ${{ secrets.OP_CONNECT_TOKEN }}
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+          cache: yarn
+          cache-dependency-path: "bindings/nodejs/yarn.lock"
+      - name: Corepack
+        working-directory: bindings/nodejs
+        run: corepack enable
+      - name: Install dependencies
+        working-directory: bindings/nodejs
+        run: yarn install --immutable
+
+      - name: Test Core
+        uses: ./.github/actions/behavior_test_binding_nodejs
+        with:
+          setup: ${{ matrix.cases.setup }}
+          service: ${{ matrix.cases.service }}
+          feature: ${{ matrix.cases.feature }}
diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml
@@ -69,4 +69,4 @@ jobs:
           OPENDAL_S3_ROOT: CI/
           OPENDAL_S3_BUCKET: opendal-testing
           OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
-          OPENDAL_S3_REGION: ap-northeast-1
+          OPENDAL_S3_REGION: ap-northeast-1
diff --git a/Cargo.lock b/Cargo.lock