chore: add a certification run script for submission

.github/workflows/conformance.yml

@@ -87,7 +87,6 @@ jobs:
           PLAN_NAME: ${{ matrix.setup.plan }}
           VARIANT: ${{ toJSON(matrix.setup.variant) }}
       - run: node ./conformance/.parse-logs.mjs capture.txt
-      - run: rm capture.txt
       - name: Upload test artifacts
         id: artifact-upload-step
         uses: actions/upload-artifact@v4

conformance/.parse-logs.mjs

@@ -1,20 +1,48 @@
 import * as events from 'node:events'
 import * as fs from 'node:fs'
 import * as readline from 'node:readline'
+import { parseArgs } from 'node:util'
 
-const input = process.argv.reverse()[0]
+import archiver from 'archiver'
+
+const {
+  values: { submission },
+  positionals: [input],
+} = parseArgs({
+  options: {
+    submission: {
+      type: 'boolean',
+      default: false,
+    },
+  },
+  allowPositionals: true,
+})
 
 const rl = readline.createInterface({
   input: fs.createReadStream(input),
   crlfDelay: Infinity,
 })
 
+let planName
+let planId
 let currentFile
 let testName
 let testId
 
+const files = []
+
 rl.on('line', (line) => {
+  if (line.includes('- ID')) {
+    planId = line.slice(6)
+    return
+  }
+  if (line.includes('- Name')) {
+    planName = line.slice(8)
+    return
+  }
+
   line = line.substring(4)
+
   if (currentFile && line.includes('Test ID')) {
     throw new Error()
   }
@@ -38,9 +66,27 @@ rl.on('line', (line) => {
   fs.writeFileSync(currentFile, `${line}\n`, { flag: 'a' })
 
   if (line.includes('Test Finished') || line.includes('Test result is SKIPPED')) {
-    fs.renameSync(currentFile, `${testName}-${testId}.txt`)
+    const fullname = `${testName}-${testId}.txt`
+    files.push(fullname)
+    fs.renameSync(currentFile, fullname)
     currentFile = testName = testId = null
   }
 })
 
 await events.once(rl, 'close')
+
+if (submission) {
+  const archive = archiver('zip')
+  const zip = fs.createWriteStream(`${planId}-client-data.zip`)
+  archive.pipe(zip)
+  for (const file of files) {
+    archive.file(file, { name: file })
+  }
+  await archive.finalize()
+  for (const file of files) {
+    fs.unlinkSync(file)
+  }
+  await events.once(zip, 'close')
+}
+
+fs.unlinkSync(input)

conformance/ava.config.ts

@@ -184,17 +184,21 @@ export default async () => {
 
   const { certificationProfileName } = await api.getTestPlanInfo(plan)
 
-  logToActions('Test Plan Details')
-  logToActions('')
-  logToActions(`- Name: **${PLAN_NAME}**`)
-  logToActions(`- ID: **\`${plan.id}\`**`)
-  logToActions(`- Variant`)
+  function logBoth(input: string) {
+    console.log(input.replaceAll('`', '').replaceAll('**', ''))
+    logToActions(input)
+  }
+
+  logBoth('Test Plan Details')
+  logBoth('')
+  logBoth(`- Name: **${PLAN_NAME}**`)
+  logBoth(`- ID: **\`${plan.id}\`**`)
+  logBoth('- Variant')
   for (const [key, value] of Object.entries(variant)) {
-    logToActions(`  - ${key}: ${value}`)
+    logBoth(`  - ${key}: ${value}`)
   }
   if (certificationProfileName) {
-    console.log('CERTIFICATION PROFILE NAME:', certificationProfileName)
-    logToActions(`- Certification Profile Name: **${certificationProfileName}**`)
+    logBoth(`- Certification Profile Name: **${certificationProfileName}**`)
   }
 
   const files: Set<string> = new Set()

conformance/run-certification.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+
+# Helper function to run a conformance test plan
+run_conformance() {
+  local plan_name=$1
+  local variant=$2
+  local capture_file="capture-$(uuidgen).txt" # Use a unique capture filename
+  echo "Running conformance test with PLAN_NAME=$plan_name, VARIANT=$variant"
+  npm run conformance | tee "$capture_file"
+  node ./conformance/.parse-logs.mjs --submission "$capture_file"
+  echo "===================================================================="
+}
+
+# Basic RP
+export PLAN_NAME=oidcc-client-basic-certification-test-plan
+export VARIANT='{}'
+run_conformance "$PLAN_NAME" "$VARIANT" &
+
+export CLIENT_AUTH_TYPES=("mtls" "private_key_jwt")
+export FAPI_CLIENT_TYPES=("oidc" "plain_oauth")
+
+# FAPI 1.0 Advanced
+export PLAN_NAME=fapi1-advanced-final-client-test-plan
+
+for CLIENT_AUTH_TYPE in "${CLIENT_AUTH_TYPES[@]}"; do
+  for FAPI_CLIENT_TYPE in "${FAPI_CLIENT_TYPES[@]}"; do
+    export VARIANT="{\"client_auth_type\":\"$CLIENT_AUTH_TYPE\",\"fapi_client_type\":\"$FAPI_CLIENT_TYPE\"}"
+    run_conformance "$PLAN_NAME" "$VARIANT" &
+  done
+done
+
+# FAPI 2.0
+export PLAN_NAMES=("fapi2-security-profile-id2-client-test-plan" "fapi2-message-signing-id1-client-test-plan")
+export SENDER_CONSTRAINS=("mtls" "dpop")
+
+for PLAN_NAME in "${PLAN_NAMES[@]}"; do
+  for CLIENT_AUTH_TYPE in "${CLIENT_AUTH_TYPES[@]}"; do
+    for SENDER_CONSTRAIN in "${SENDER_CONSTRAINS[@]}"; do
+      for FAPI_CLIENT_TYPE in "${FAPI_CLIENT_TYPES[@]}"; do
+        export VARIANT="{\"client_auth_type\":\"$CLIENT_AUTH_TYPE\",\"sender_constrain\":\"$SENDER_CONSTRAIN\",\"fapi_client_type\":\"$FAPI_CLIENT_TYPE\"}"
+        run_conformance "$PLAN_NAME" "$VARIANT" &
+      done
+    done
+  done
+done
+
+# Wait for all runs to finish
+wait

conformance/runner.ts

@@ -152,7 +152,7 @@ export const green = test.macro({
       .discoveryRequest(issuer)
       .then((response) => oauth.processDiscoveryResponse(issuer, response))
 
-    t.log('AS Metadata', as)
+    t.log('AS Metadata discovered for', as.issuer)
 
     const client: oauth.Client = {
       client_id: configuration.client.client_id,