Merge pull request #4092 from DataDog/vpellan/system-tests-force-mult…

…iple-scenario

.github/forced-tests-list.json

@@ -0,0 +1,3 @@
+{
+
+}

.github/forced-tests-list.json.sample

@@ -0,0 +1,11 @@
+{
+  "DEFAULT":
+    [
+      "tests/appsec/waf/test_miscs.py::Test_CorrectOptionProcessing",
+      "tests/test_semantic_conventions.py::Test_Meta::test_meta_span_kind"
+    ],
+  "APPSEC_STANDALONE":
+    [
+      "tests/appsec/test_asm_standalone.py"
+    ]
+}

.github/workflows/system-tests.yml

@@ -11,9 +11,7 @@ on:
 env:
   REGISTRY: ghcr.io
   REPO: ghcr.io/datadog/dd-trace-rb
-  ST_REF: main
-  FORCE_TESTS:
-  FORCE_TESTS_SCENARIO:
+  SYSTEM_TESTS_REF: main # This must always be set to `main` on dd-trace-rb's master branch
 
 jobs:
   build-harness:
@@ -36,7 +34,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: 'DataDog/system-tests'
-          ref: ${{ env.ST_REF }}
+          ref: ${{ env.SYSTEM_TESTS_REF }}
           persist-credentials: false
       - name: Pull released image
         run: |
@@ -111,12 +109,14 @@ jobs:
     name: Build (${{ matrix.app }})
     permissions:
       packages: write
+    outputs:
+      FORCED_TESTS_LIST: ${{steps.read_forced_tests_list.outputs.FORCED_TESTS_LIST}}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           repository: 'DataDog/system-tests'
-          ref: ${{ env.ST_REF }}
+          ref: ${{ env.SYSTEM_TESTS_REF }}
           persist-credentials: false
       - name: Checkout ${{ matrix.library.repository }}
         uses: actions/checkout@v4
@@ -125,6 +125,12 @@ jobs:
           path: 'binaries/${{ matrix.library.path }}'
           fetch-depth: 2
           persist-credentials: false
+      - name: Read forced-tests-list.json file
+        id: read_forced_tests_list
+        run: |
+          echo "FORCED_TESTS_LIST<<EOF" >> $GITHUB_OUTPUT
+          echo "$(cat binaries/dd-trace-rb/.github/forced-tests-list.json)" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
       - name: Pull released image
         run: |
           if docker pull ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest; then
@@ -277,7 +283,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: 'DataDog/system-tests'
-          ref: ${{ env.ST_REF }}
+          ref: ${{ env.SYSTEM_TESTS_REF }}
           persist-credentials: false
       - name: Pull runner image
         run: |
@@ -299,12 +305,17 @@ jobs:
         run: |
           docker image list
       - name: Run scenario
+        # The last escape sequence check if the JSON string exists,
+        # then check if the current scenario is contained in it, then create a string by chaining -F and forced tests.
+        # A potential error might come from an empty array in a scenario in the json file.
         run: |
-          ./run.sh ++docker ${{ matrix.scenario }} ${{matrix.scenario == env.FORCE_TESTS_SCENARIO && env.FORCE_TESTS || ''}}
+          ./run.sh ++docker ${{ matrix.scenario }} \
+            ${{ env.FORCED_TESTS_LIST && fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario] && format('-F {0}', join(fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario], ' -F ')) || ''}}
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
           SYSTEM_TESTS_AWS_ACCESS_KEY_ID: ${{ secrets.SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID }}
           SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: ${{ secrets.SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY }}
+          FORCED_TESTS_LIST: ${{ needs.build-apps.outputs.FORCED_TESTS_LIST }}
       - name: Archive logs
         uses: actions/upload-artifact@v4
         if: ${{ always() }}
@@ -353,7 +364,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: 'DataDog/system-tests'
-          ref: ${{ env.ST_REF }}
+          ref: ${{ env.SYSTEM_TESTS_REF }}
           persist-credentials: false
       - name: Retrieve logs
         uses: actions/download-artifact@v4

docs/ForcingSystemTests.md

@@ -0,0 +1,44 @@
+# Forcing System Tests
+
+During development, you may want to force-execute specific system-tests on dd-trace-rb CI while their declaration aren't merged yet on system-tests side.
+To do so, you can complete the `.github/forced-tests-list.json` file by following this template:
+
+```json
+{
+  "SYSTEM_TEST_SCENARIO_1":
+    [
+      "tests/test_forced_file.py",
+      "tests/test_forced_file.py::Test_ForcedClass",
+      "tests/test_forced_file.py::Test_ForcedClass::test_forced_method"
+    ],
+  "SYSTEM_TEST_SCENARIO_2":
+    [
+      ...
+    ],
+    ...
+}
+```
+
+## Example
+
+```json
+{
+  "DEFAULT":
+    [
+      "tests/appsec/waf/test_miscs.py::Test_CorrectOptionProcessing",
+      "tests/test_semantic_conventions.py::Test_Meta::test_meta_span_kind"
+    ],
+  "APPSEC_STANDALONE":
+    [
+      "tests/appsec/test_asm_standalone.py"
+    ]
+}
+```
+
+## Cleanup
+
+You can leave other force-executed tests added by other developers and append yours to the .json file, there is a cleanup task in the release process.
+
+## Reference
+
+System-tests documentation on [force-executing tests](https://github.com/DataDog/system-tests/blob/main/docs/execute/force-execute.md)