Add releasetest (open-quantum-safe#281)

* add releasetest for all algs/combinations

.gitignore

@@ -16,6 +16,7 @@ tmp
 interop.log
 # pycache
 oqs-template/__pycache__
+scripts/__pycache__
 
 # Visual Studio Code
 .vscode

oqs-template/generate.py

@@ -244,6 +244,7 @@ def load_config(include_disabled_sigs=False):
 populate('oqsprov/oqs_encode_key2any.c', config, '/////')
 populate('oqsprov/oqs_decode_der2key.c', config, '/////')
 populate('oqsprov/oqsprov_keys.c', config, '/////')
+populate('scripts/common.py', config, '#####')
 
 config2 = load_config(include_disabled_sigs=True)
 config2 = complete_config(config2)

oqs-template/scripts/common.py/kex_algs.fragment

@@ -0,0 +1,10 @@
+
+    # post-quantum key exchanges
+    {% for kem in config['kems'] %}'{{ kem['name_group'] }}',  {%- endfor %}
+    # post-quantum + classical key exchanges
+    {% for kem in config['kems'] -%}
+       {%- for hybrid in kem['hybrids'] -%}
+          '{{ hybrid['hybrid_group'] }}_{{kem['name_group']}}', 
+       {%- endfor -%}
+    {% endfor %}
+

oqs-template/scripts/common.py/sig_algs.fragment

@@ -0,0 +1,12 @@
+
+    # post-quantum signatures
+    {% for sig in config['sigs'] %}{% for variant in sig['variants'] %}'{{ variant['name'] }}',
+    {%- endfor %} {%- endfor %}
+    # post-quantum + classical signatures
+    {% for sig in config['sigs'] -%}
+        {%- for variant in sig['variants'] -%}
+            {%- for classical_alg in variant['mix_with'] -%}
+            '{{ classical_alg['name'] }}_{{ variant['name'] }}',
+            {%- endfor -%}
+    {%- endfor %} {%- endfor %}
+

scripts/README.md

@@ -0,0 +1,21 @@
+# Build and test support scripts
+
+This directory contains various scripts aiming to ease build and test of `oqsprovider`.
+
+## Building
+
+The key file is [fullbuild.sh](fullbuild.sh) with options documented [here](https://github.com/open-quantum-safe/oqs-provider/blob/main/CONFIGURE.md#convenience-build-script-options).
+
+## Testing
+
+### API testing
+
+All features and enabled algorithms are API tested by `ctest` driven code contained in the [test directory](https://github.com/open-quantum-safe/oqs-provider/tree/main/test).
+
+### Command line testing
+
+All features and enabled algorithms are tested via `openssl` command line instructions via the [runtests.sh](runtests.sh) script with options documented [here](https://github.com/open-quantum-safe/oqs-provider/blob/main/CONFIGURE.md#convenience-build-script-options).
+
+### Release testing
+
+All features and all algorithms can be tested in a full matrix running all possible signature and KEM algorithms in client/server setup via the corresponding `openssl s_server/s_client` commands via the [release-test.sh](release-test.sh) script. To run this test successfully, installation of `python3` and `pytest` with `xdist` extension is required, e.g., via `sudo apt install python3 python3-pytest python3-pytest-xdist python3-psutil`. The test must be executed within the main project directory, e.g., as such `./scripts/release-test.sh`. For full operation, a local and up-to-date (release) installation of `openssl` and `liboqs` (e.g., built via `scripts/fulltest.sh`) is recommended.

scripts/common.py

@@ -0,0 +1,165 @@
+import os
+import subprocess
+import pathlib
+import psutil
+import time
+
+key_exchanges = [
+##### OQS_TEMPLATE_FRAGMENT_KEX_ALGS_START
+    # post-quantum key exchanges
+    'frodo640aes','frodo640shake','frodo976aes','frodo976shake','frodo1344aes','frodo1344shake','kyber512','kyber768','kyber1024','bikel1','bikel3','bikel5','hqc128','hqc192','hqc256',
+    # post-quantum + classical key exchanges
+    'p256_frodo640aes','x25519_frodo640aes','p256_frodo640shake','x25519_frodo640shake','p384_frodo976aes','x448_frodo976aes','p384_frodo976shake','x448_frodo976shake','p521_frodo1344aes','p521_frodo1344shake','p256_kyber512','x25519_kyber512','p384_kyber768','x448_kyber768','x25519_kyber768','p256_kyber768','p521_kyber1024','p256_bikel1','x25519_bikel1','p384_bikel3','x448_bikel3','p521_bikel5','p256_hqc128','x25519_hqc128','p384_hqc192','x448_hqc192','p521_hqc256',
+##### OQS_TEMPLATE_FRAGMENT_KEX_ALGS_END
+]
+signatures = [
+    'ecdsap256', 'rsa3072',
+##### OQS_TEMPLATE_FRAGMENT_SIG_ALGS_START
+    # post-quantum signatures
+    'dilithium2','dilithium3','dilithium5','falcon512','falcon1024','sphincssha2128fsimple','sphincssha2128ssimple','sphincssha2192fsimple','sphincsshake128fsimple',
+    # post-quantum + classical signatures
+    'p256_dilithium2','rsa3072_dilithium2','p384_dilithium3','p521_dilithium5','p256_falcon512','rsa3072_falcon512','p521_falcon1024','p256_sphincssha2128fsimple','rsa3072_sphincssha2128fsimple','p256_sphincssha2128ssimple','rsa3072_sphincssha2128ssimple','p384_sphincssha2192fsimple','p256_sphincsshake128fsimple','rsa3072_sphincsshake128fsimple',
+##### OQS_TEMPLATE_FRAGMENT_SIG_ALGS_END
+]
+
+SERVER_START_ATTEMPTS = 10
+
+def all_pq_groups():
+    ag = ""
+    for kex in key_exchanges:
+        if len(ag)==0:
+           ag = kex 
+        else:
+           ag = ag + ":" + kex
+    return ag
+
+def run_subprocess(command, working_dir='.', expected_returncode=0, input=None, env=os.environ):
+    """
+    Helper function to run a shell command and report success/failure
+    depending on the exit status of the shell command.
+    """
+
+    # Note we need to capture stdout/stderr from the subprocess,
+    # then print it, which pytest will then capture and
+    # buffer appropriately
+    print(working_dir + " > " + " ".join(command))
+    result = subprocess.run(
+        command,
+        input=input,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        cwd=working_dir,
+        env=env
+    )
+    if result.returncode != expected_returncode:
+        print(result.stdout.decode('utf-8'))
+        assert False, "Got unexpected return code {}".format(result.returncode)
+    return result.stdout.decode('utf-8')
+
+def start_server(ossl, test_artifacts_dir, sig_alg, worker_id):
+    command = [ossl, 's_server',
+                      '-cert', os.path.join(test_artifacts_dir, '{}_{}_srv.crt'.format(worker_id, sig_alg)),
+                      '-key', os.path.join(test_artifacts_dir, '{}_{}_srv.key'.format(worker_id, sig_alg)),
+                      '-CAfile', os.path.join(test_artifacts_dir, '{}_{}_CA.crt'.format(worker_id, sig_alg)),
+                      '-tls1_3',
+                      '-quiet',
+# add X25519 for baseline server test and all PQ KEMs for single PQ KEM tests:
+                      '-groups', "x25519:"+all_pq_groups(),
+                      # On UNIX-like systems, binding to TCP port 0
+                      # is a request to dynamically generate an unused
+                      # port number.
+                      # TODO: Check if Windows behaves similarly
+                      '-accept', '0']
+
+    print(" > " + " ".join(command))
+    server = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    server_info = psutil.Process(server.pid)
+
+    # Try SERVER_START_ATTEMPTS times to see
+    # what port the server is bound to.
+    server_start_attempt = 1
+    while server_start_attempt <= SERVER_START_ATTEMPTS:
+        if server_info.connections():
+            break
+        else:
+            server_start_attempt += 1
+            time.sleep(2)
+    server_port = str(server_info.connections()[0].laddr.port)
+
+    # Check SERVER_START_ATTEMPTS times to see
+    # if the server is responsive.
+    server_start_attempt = 1
+    while server_start_attempt <= SERVER_START_ATTEMPTS:
+        result = subprocess.run([ossl, 's_client', '-connect', 'localhost:{}'.format(server_port)],
+                                input='Q'.encode(),
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.STDOUT)
+        if result.returncode == 0:
+            break
+        else:
+            server_start_attempt += 1
+            time.sleep(2)
+
+    if server_start_attempt > SERVER_START_ATTEMPTS:
+        raise Exception('Cannot start OpenSSL server')
+
+    return server, server_port
+
+def gen_keys(ossl, ossl_config, sig_alg, test_artifacts_dir, filename_prefix):
+    pathlib.Path(test_artifacts_dir).mkdir(parents=True, exist_ok=True)
+    if sig_alg == 'ecdsap256':
+        run_subprocess([ossl, 'ecparam',
+                              '-name', 'prime256v1',
+                              '-out', os.path.join(test_artifacts_dir, '{}_prime256v1.pem'.format(filename_prefix))])
+        run_subprocess([ossl, 'req', '-x509', '-new',
+                                     '-newkey', 'ec:{}'.format(os.path.join(test_artifacts_dir, '{}_prime256v1.pem'.format(filename_prefix))),
+                                     '-keyout', os.path.join(test_artifacts_dir, '{}_ecdsap256_CA.key'.format(filename_prefix)),
+                                     '-out', os.path.join(test_artifacts_dir, '{}_ecdsap256_CA.crt'.format(filename_prefix)),
+                                     '-nodes',
+                                         '-subj', '/CN=oqstest_CA',
+                                         '-days', '365',
+                                     '-config', ossl_config])
+        run_subprocess([ossl, 'req', '-new',
+                                     '-newkey', 'ec:{}'.format(os.path.join(test_artifacts_dir, '{}_prime256v1.pem'.format(filename_prefix))),
+                                     '-keyout', os.path.join(test_artifacts_dir, '{}_ecdsap256_srv.key'.format(filename_prefix)),
+                                     '-out', os.path.join(test_artifacts_dir, '{}_ecdsap256_srv.csr'.format(filename_prefix)),
+                                     '-nodes',
+                                         '-subj', '/CN=oqstest_server',
+                                     '-config', ossl_config])
+    else:
+        if sig_alg == 'rsa3072':
+            ossl_sig_alg_arg = 'rsa:3072'
+        else:
+            ossl_sig_alg_arg = sig_alg
+        run_subprocess([ossl, 'req', '-x509', '-new',
+                                     '-newkey', ossl_sig_alg_arg,
+                                     '-keyout', os.path.join(test_artifacts_dir, '{}_{}_CA.key'.format(filename_prefix, sig_alg)),
+                                     '-out', os.path.join(test_artifacts_dir, '{}_{}_CA.crt'.format(filename_prefix, sig_alg)),
+                                     '-nodes',
+                                         '-subj', '/CN=oqstest_CA',
+                                         '-days', '365',
+                                     '-config', ossl_config])
+        run_subprocess([ossl, 'req', '-new',
+                              '-newkey', ossl_sig_alg_arg,
+                              '-keyout', os.path.join(test_artifacts_dir, '{}_{}_srv.key'.format(filename_prefix, sig_alg)),
+                              '-out', os.path.join(test_artifacts_dir, '{}_{}_srv.csr'.format(filename_prefix, sig_alg)),
+                              '-nodes',
+                                  '-subj', '/CN=oqstest_server',
+                              '-config', ossl_config])
+
+    run_subprocess([ossl, 'x509', '-req',
+                                  '-in', os.path.join(test_artifacts_dir, '{}_{}_srv.csr'.format(filename_prefix, sig_alg)),
+                                  '-out', os.path.join(test_artifacts_dir, '{}_{}_srv.crt'.format(filename_prefix, sig_alg)),
+                                  '-CA', os.path.join(test_artifacts_dir, '{}_{}_CA.crt'.format(filename_prefix, sig_alg)),
+                                  '-CAkey', os.path.join(test_artifacts_dir, '{}_{}_CA.key'.format(filename_prefix, sig_alg)),
+                                  '-CAcreateserial',
+                                  '-days', '365'])
+
+    # also create pubkeys from certs for dgst verify tests:
+    env = os.environ
+    #env["OPENSSL_CONF"]=os.path.join("scripts", "openssl.cnf")
+    #env["OPENSSL_MODULES"]=os.path.join("_build", "lib")
+    run_subprocess([ossl, 'req',
+                                  '-in', os.path.join(test_artifacts_dir, '{}_{}_srv.csr'.format(filename_prefix, sig_alg)),
+                                  '-pubkey', '-out', os.path.join(test_artifacts_dir, '{}_{}_srv.pubk'.format(filename_prefix, sig_alg)) ],
+                   env=env)

scripts/conftest.py

@@ -0,0 +1,20 @@
+import os
+import pytest
+import subprocess
+
+def pytest_addoption(parser):
+    parser.addoption("--ossl", action="store", help="ossl: Path to standalone OpenSSL executable.")
+    parser.addoption("--ossl-config", action="store", help="ossl-config: Path to openssl.cnf file.")
+    parser.addoption("--test-artifacts-dir", action="store", help="test-artifacts-dir: Path to directory containing files generated during the testing process.")
+
+@pytest.fixture
+def ossl_config(request):
+    return os.path.normpath(request.config.getoption("--ossl-config"))
+
+@pytest.fixture
+def ossl(request):
+    return os.path.normpath(request.config.getoption("--ossl"))
+
+@pytest.fixture
+def test_artifacts_dir(request):
+    return os.path.normpath(request.config.getoption("--test-artifacts-dir"))

scripts/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = --verbose --ossl=.local/bin/openssl --ossl-config=scripts/openssl-ca.cnf --test-artifacts-dir=tmp

scripts/release-test.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Stop in case of error
+set -e
+
+# To be run as part of a release test only on Linux
+# requires python, pytest, xdist; install e.g. via
+# sudo apt install python3 python3-pytest python3-pytest-xdist python3-psutil
+
+# must be run in main folder
+# multicore machine recommended for fast execution
+
+# expect (ideally latest/release-test) liboqs to be already build and present
+if [ -d liboqs ]; then
+   export LIBOQS_SRC_DIR=`pwd`/liboqs
+else
+   echo "liboqs not found. Exiting."
+   exit 1
+fi
+
+if [ -d oqs-template ]; then
+    # just a temp setup
+    git checkout -b reltest
+    # Activate all algorithms
+    sed -i "s/enable\: false/enable\: true/g" oqs-template/generate.yml
+    python3 oqs-template/generate.py
+    rm -rf _build
+    ./scripts/fullbuild.sh
+    ./scripts/runtests.sh
+    if [ -f .local/bin/openssl ]; then
+        OPENSSL_MODULES=`pwd`/_build/lib OPENSSL_CONF=`pwd`/scripts/openssl-ca.cnf python3 -m pytest --numprocesses=auto scripts/test_tls_full.py
+    else
+        echo "For full TLS PQ SIG/KEM matrix test, build (latest) openssl locally."
+    fi
+    git reset --hard && git checkout main && git branch -D reltest
+else
+    echo "$0 must be run in main oqs-provider folder. Exiting."
+fi
+

scripts/test_tls_full.py

@@ -0,0 +1,30 @@
+import common
+import pytest
+import sys
+import os
+
+@pytest.fixture(params=common.signatures)
+def server(ossl, ossl_config, test_artifacts_dir, request, worker_id):
+    # Setup: start ossl server
+    common.gen_keys(ossl, ossl_config, request.param, test_artifacts_dir, worker_id)
+    server, port = common.start_server(ossl, test_artifacts_dir, request.param, worker_id)
+    # Run tests
+    yield (request.param, port)
+    # Teardown: stop ossl server
+    server.kill()
+
+@pytest.mark.parametrize('kex_name', common.key_exchanges)
+def test_sig_kem_pair(ossl, server, test_artifacts_dir, kex_name, worker_id):
+    client_output = common.run_subprocess([ossl, 's_client',
+                                                  '-groups', kex_name,
+                                                  '-CAfile', os.path.join(test_artifacts_dir, '{}_{}_CA.crt'.format(worker_id, server[0])),
+                                                  '-verify_return_error',
+                                                  '-connect', 'localhost:{}'.format(server[1])],
+                                    input='Q'.encode())
+# OpenSSL3 by default does not output KEM used; so rely on forced client group and OK handshake completion:
+    if not "SSL handshake has read" in client_output:
+        assert False, "Handshake failure."
+
+if __name__ == "__main__":
+    import sys
+    pytest.main(sys.argv)