Testing: Implements new Mocha/Chai based API tests. (NUWCDIVNPT#1382)

.github/workflows/api-audit-test-coverage-response.yml

@@ -64,6 +64,9 @@ jobs:
       - name: Install app dependencies
         working-directory: ./api/source
         run: npm ci
+      - name: Install test dependencies
+        run: npm ci
+        working-directory: ./test/api/
       - name: Run mock Keycloak
         id: idp-run
         working-directory: ./test/api/mock-keycloak
@@ -82,10 +85,6 @@ jobs:
       - name: Install c8 coverage reporter
         run: |
           sudo npm install -g c8
-      - name: Install Newman reporter
-        id: newman-reporter-install
-        run: |
-          sudo npm install -g newman-reporter-htmlextra
       - name: Run app and echo pid
         working-directory: ./api/source
         run: sh -c 'echo $$ > /tmp/test.pid ; exec c8 -r html node index.js > api-log.json 2>&1' &
@@ -94,62 +93,24 @@ jobs:
         run: cat /tmp/test.pid
       - name: Wait for bootstrap
         run: for i in {1..10}; do [ $i -gt 1 ] && sleep 5; curl --output /dev/null --silent --fail http://localhost:64001/api/op/configuration && s=0 && break || s=$?; printf '.'; done; (exit $s)
-      - name: Run Newman Collection LoadTestData
-        id: newman-run-loadTestData
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 1 --folder LoadTestData -r cli,htmlextra  --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showOnlyFails --reporter-htmlextra-export ./newman/dataPreloadReport.html | grep -A18 '┌─────'
-      - name: Run Newman Collection GETS
-        id: newman-run-gets
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 6 --folder GETs -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showOnlyFails --reporter-htmlextra-export ./newman/GetsReport.html | grep -A18 '┌─────'
-      - name: Run Newman Collection Posts, Puts, Patches, Deletes
-        id: newman-run-pppd
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 6 --folder "POSTS, Puts, Patches, and Deletes" -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showOnlyFails --reporter-htmlextra-export ./newman/PPPDReport.html | grep -A18 '┌─────'
-      - name: Run Newman Collection STIGS
-        id: newman-run-stigs
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 2 --folder "STIGS" -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showOnlyFails --reporter-htmlextra-export ./newman/stigsReport.html | grep -A18 '┌─────'
-      - name: Run Newman Collection LVL1 Cross-Boundary Tests
-        id: newman-run-lvl1-cross-boundary
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 1 --folder "LVL1 cross-boundary tests" -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showEnvironmentData --reporter-htmlextra-export ./newman/lvl1Report.html | grep -A18 '┌─────'
-      - name: Run Newman Collection Additional Sundry Tests
-        id: newman-run-additional-sundry
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 1 --folder "Additional sundry tests" -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showEnvironmentData --reporter-htmlextra-export ./newman/AdditionalSundryReport.html | grep -A18 '┌─────'
+      - name: Run tests with coverage
+        working-directory: ./test/api/
+        run: npm test
+      - name: Upload mocha test report
+        id: artifact-upload-mocha
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: mocha-report
+          path: ./test/api/mochawesome-report
       - name: Kill child pid
         working-directory: ./api/source
         run: kill -SIGINT $(ps -ef --ppid `cat /tmp/test.pid` -o pid= )
       - name: Wait until c8 process ends, or 60 seconds, whichever is shorter.
         run: timeout 60 tail --pid=`cat /tmp/test.pid` -f /dev/null
       - name: run c8 text/lcov report
         working-directory: ./api/source
-        run: c8 report -r lcov -r text
-      - name: Upload Newman artifact
-        id: artifact-upload-newman
-        uses: actions/upload-artifact@v4
-        if: ${{ always() }}
-        with:
-          name: newman-htmlextra
-          path: ./test/api/newman
+        run: c8 report -r lcov -r text -r html
       - name: Upload coverage artifact
         id: artifact-upload-coverage
         uses: actions/upload-artifact@v4

.github/workflows/api-container-tests.yml

@@ -24,7 +24,7 @@ jobs:
 
   test_api:
     needs: fetch-mysql-versions
-    name: Postman tests with ${{ matrix.container.name }} and MySQL ${{ matrix.mysql_version }}
+    name: Mocha tests with ${{ matrix.container.name }} and MySQL ${{ matrix.mysql_version }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -67,60 +67,23 @@ jobs:
           -e STIGMAN_DEV_RESPONSE_VALIDATION=logOnly \
           ${{ matrix.container.name }}
 
-      - name: Install Newman reporter
-        id: newman-reporter-install
-        run: |
-          sudo npm install -g newman-reporter-htmlextra
+      - name: Install test dependencies
+        run: npm ci
+        working-directory: ./test/api/
+
       - name: Wait for bootstrap
         run: for i in {1..10}; do [ $i -gt 1 ] && sleep 5; curl --output /dev/null --silent --fail http://localhost:64001/api/op/configuration && s=0 && break || s=$?; printf '.'; done; (exit $s)
-      - name: Run Newman Collection LoadTestData
-        id: newman-run-loadTestData
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 1 --folder LoadTestData -r cli,htmlextra  --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showOnlyFails --reporter-htmlextra-export ./newman/dataPreloadReport-${{ matrix.container.name }}-${{ matrix.mysql_version }}-${{ github.run_id }}.html | grep -A18 '┌─────'
-      - name: Run Newman Collection GETS
-        id: newman-run-gets
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 6 --folder GETs -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showOnlyFails --reporter-htmlextra-export ./newman/GetsReport-${{ matrix.container.name }}-${{ matrix.mysql_version }}-${{ github.run_id }}.html | grep -A18 '┌─────'
-      - name: Run Newman Collection Posts, Puts, Patches, Deletes
-        id: newman-run-pppd
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 6 --folder "POSTS, Puts, Patches, and Deletes" -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showOnlyFails --reporter-htmlextra-export ./newman/PPPDReport-${{ matrix.container.name }}-${{ matrix.mysql_version }}-${{ github.run_id }}.html | grep -A18 '┌─────'
-      - name: Run Newman Collection STIGS
-        id: newman-run-stigs
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 2 --folder "STIGS" -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showOnlyFails --reporter-htmlextra-export ./newman/stigsReport-${{ matrix.container.name }}-${{ matrix.mysql_version }}-${{ github.run_id }}.html | grep -A18 '┌─────'
-      - name: Run Newman Collection LVL1 Cross-Boundary Tests
-        id: newman-run-lvl1-cross-boundary
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 1 --folder "LVL1 cross-boundary tests" -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showEnvironmentData --reporter-htmlextra-export ./newman/lvl1Report-${{ matrix.container.name }}-${{ matrix.mysql_version }}-${{ github.run_id }}.html | grep -A18 '┌─────'
-      - name: Run Newman Collection Additional Sundry Tests
-        id: newman-run-additional-sundry
-        if: steps.newman-run-loadTestData.conclusion == 'success'
-        working-directory: ./test/api
-        run: |
-          set -o pipefail
-          newman run postman_collection.json -e postman_environment.json -d collectionRunnerData.json -n 1 --folder "Additional sundry tests" -r cli,htmlextra --reporter-cli-no-assertions --reporter-cli-no-console --reporter-htmlextra-showEnvironmentData --reporter-htmlextra-export ./newman/AdditionalSundryReport-${{ matrix.container.name }}-${{ matrix.mysql_version }}-${{ github.run_id }}.html | grep -A18 '┌─────'
-      - name: Upload Newman artifact
-        id: artifact-upload
+      - name: Run tests with coverage
+        working-directory: ./test/api/
+        run: npm test
+      - name: Upload mocha test report
+        id: artifact-upload-mocha
         uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
-          name: newman-htmlextra-${{ matrix.container.name }}-${{ matrix.mysql_version }}-${{ github.run_id }}
-          path: ./test/api/newman
+          name: mocha-report-${{ matrix.container.name }}-${{ matrix.mysql_version }}
+          path: ./test/api/mochawesome-report
+
       - name: Extract API container log
         id: api-log-extract
         if: ${{ always() }}

.gitignore

@@ -31,6 +31,7 @@ api/source/tls/mysql2/tls.cnf
 clients/extjs/js/keycloak.json
 .gitignore
 newman/
+mochawesome-report/
 /docs/_build/doctrees/
 
 # macOS file

docs/the-project/contributing.rst

@@ -121,18 +121,16 @@ This section outlines the main components of the STIG Manager project, their tec
      - | `DB <https://github.com/NUWCDIVNPT/stig-manager/issues?q=is%3Aopen+is%3Aissue+label%3ADB>`__
        | `API <https://github.com/NUWCDIVNPT/stig-manager/issues?q=is%3Aopen+is%3Aissue+label%3AAPI>`__
    * - Test suites
-     - | Postman
-       | newman
-       | Mocha
+     - | Mocha
        | Chai
+       | Chai-http
      - @cd-rite
      - - Automated UI Testing
        - Help would be appreciated identifying additional test cases. 
      - `tests <https://github.com/NUWCDIVNPT/stig-manager/issues?q=is%3Aopen+is%3Aissue+label%3Atests>`__
    * - CI/CD pipelines
      - | GitHub Actions
        | Docker
-       | newman
      - | @cd-rite
        | @csmig
      - - Integration of additional automated security scanning. 
@@ -199,7 +197,7 @@ The team regularly uses these tools:
   - Docker: For containerization and testing
   - VS Code: Recommended IDE for development
   - Node.js: For running the application and tests
-  - Mocha, Chai, Postman: For writing and running tests
+  - Mocha, Chai, Chai-http: For writing and running tests
 
 
 Software Components

docs/the-project/project-description.rst

@@ -132,7 +132,9 @@ Products, Systems, Tools and Methods Used
 -  OpenAPI 3.0+
 -  Azure
 -  Keycloak
--  Postman
+-  Mocha
+-  Chai
+-  Chai-http
 -  DoD Iron Bank
 -  npm
 -  git

docs/the-project/testing.rst

@@ -5,9 +5,9 @@ API Testing Guide
 ########################################
 
 
-The STIG Manager project currently tests its API using a Postman Collection and specific test data, which can be found in the repo.
+The STIG Manager project currently tests its API using a Mocha/Chai/Chai-HTTP and specific test data, which can be found in the repo.
 
-The Postman Collection tests are run automatically with Newman whenever a Pull Request is made to the project.
+The tests are run automatically with Mocha via github actions whenever a Pull Request is made to the project.
 
 The tests focus on proper functioning of the API, and verification that appropriate data is returned to Users.
 The tests run in several iterations, simulating Users accessing the system with varying privileges, Access Levels and Assignments, and checks that they can only receive and alter appropriate data.

test/api/README.md

@@ -1,13 +1,5 @@
-# API testing with Postman
-
-## Required tooling
-- Node.js
-- [newman](https://www.npmjs.com/package/newman) (global install)
-- [newman-reporter-htmlextra](https://www.npmjs.com/package/newman-reporter-htmlextra)
-
-
-## Optional tooling
-- [Postman](https://www.postman.com/downloads/)
+# API testing
+This project contains a set of Mocha and Chai tests for stig-manager.
 
 ## Runtime environment
 ### Authentication Server
@@ -50,30 +42,59 @@ Run ***ONE*** of the following:
   docker run --name stig-manager-api \
   -p 64001:54000 \
   nuwcdivnpt/stig-manager
-   ```   
+   ```
+
+
+
+## Installation
+
+To install the dependencies required to run the test suite, run this command from the test folder:
+
+```
+npm install
+```
+
+Ensure that testConfig.json is configured correctly. This file contains the base URL for the API and access token for the admin test user. 
+```test/api/mocha/testConfig.json```
+
+
+## Usage
+
+The test suite uses Mocha as the test runner and Chai-http to call endpoints and Chai for assertions. 
+
+To run the tests for local development, use the following bash script:
+
+```test/api/runMocha.sh``` (use -h flag for help)
+
+In CI/CD use ```npm test``` to run tests. 
+
+
 
-## Running the Tests
 
-### From the Command Line Using newman
-- Ensure the newman npm module is installed. If not, run `npm install -g newman`
-- From the /test/api folder of the project repo, run the `runFailsOnly.sh` bash script.
-- Test result summaries are output to the console, and detailed test reports are output to the `/test/api/newman`` directory.
+## How to Write Tests
 
-### From Postman UI
+The test suite follows these conventions:
 
-- Open Postman and import the collection and environment files from the `/test/api` directory of the project repo.
-- Run requests individually, or as part of a Collection or Folder "run" using the `collectionRunnerData.json` file, if user iterations are needed.
+- All access tokens used in the tests are valid when using the ["test" keycloak container maintained here.](https://github.com/NUWCDIVNPT/stig-manager-auth). 
+- The main directory for all testing files is located at ```test/api/mocha```.
+- Tests validating the basic functionality of our endpoints are found in ```test/api/mocha/data```.
+- Each subdirectory within ```test/api/mocha/data``` is organized by API tag
+- Test files generally adhere to the naming convention ```<apiTag><HTTPMethod>.test.js``` (e.g., ```assetPatch.test.js```).
+- The ```test/api/mocha/crossBoundary``` directory contains tests for Level 1 cross-boundary scenarios.
+- Integration tests are located in ```test/api/mocha/integration```. Integration tests, as defined here, involve calling a set of related endpoints together to validate major application functionalities. These differ from the more focused, unit-like data tests that target individual API endpoints.
+- ```iterations.js``` defines the various iterations a test or group of tests will execute. This structure supports running the same test across multiple scenarios. Iterations contain an iteration name (often the test user name), a user Id as found in the test data set, and a test access token for that user.
+- Most tests reference corresponding ```referenceData.js``` and ```expectations.js``` files. These files contain the "answers" or expected data against which the API responses are validated.
+  - ```referenceData.js``` typically contains static or more global data about the tests or API paths.
+   - ```expectations.js``` contains data specific to the current test iterations (e.g., different user scenarios) and controls whether a test should run for a particular iteration.
 
 
-## Test Components
+#### Test Naming conventions
 
-Located in the `/test/api directory of the project repo:
+- top-level describe: ```describe('<HTTPMethod> - <APITag>', function () ``` Example: ```describe('DELETE - Asset', function ()```
+- Iteration-specific describe (used by runMocha.sh to run for a specific iteration): ```describe(iteration:${iteration.name}`, function () ```
+- Endpoint-level describe: ```describe('<operationId> - <endpointPath>', function ()``` Example: ```describe('deleteAssetMetadataKey - /assets/{assetId}/metadata/keys/{key}', function ()```
 
-- `postman_collection.json`  The Postman Collection of API tests.
-- `postman_environment.json`  The Postman Environment for the API tests.
-- `collectionRunnerData.json`  The data file used by the newman/Postman Collection Runner to run iterations of the tests. Each iteration is specific to a user with different levels of access and grants to Collections maintained by the API. 
-- `runFailsOnly.sh`  A bash script that runs the tests using newman, and outputs a summary of the results to the console. Detailed test reports are output to the /test/api/newman directory.  Tests are run in groups defined by the top-level folders of the Postman Collection. 
-- `form-data-files/*`  Test data files sent by Postman/newman to the API. Includes several sets of data to populate the API with data the tests expect, and several reference STIGs to use in the tests.  If using the Postman UI, you may need to adjust Postman settings to allow access to this folder locally. 
+Make sure these files are correctly set up before running the tests.
 
 ## Test Policy