diff --git a/.gitignore b/.gitignore index 4c607356..003421d7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ tmp interop.log # pycache oqs-template/__pycache__ +scripts/__pycache__ # Visual Studio Code .vscode diff --git a/oqs-template/generate.py b/oqs-template/generate.py index e25fe300..03271e8f 100644 --- a/oqs-template/generate.py +++ b/oqs-template/generate.py @@ -244,6 +244,7 @@ def load_config(include_disabled_sigs=False): populate('oqsprov/oqs_encode_key2any.c', config, '/////') populate('oqsprov/oqs_decode_der2key.c', config, '/////') populate('oqsprov/oqsprov_keys.c', config, '/////') +populate('scripts/common.py', config, '#####') config2 = load_config(include_disabled_sigs=True) config2 = complete_config(config2) diff --git a/oqs-template/scripts/common.py/kex_algs.fragment b/oqs-template/scripts/common.py/kex_algs.fragment new file mode 100644 index 00000000..c8805d32 --- /dev/null +++ b/oqs-template/scripts/common.py/kex_algs.fragment @@ -0,0 +1,10 @@ + + # post-quantum key exchanges + {% for kem in config['kems'] %}'{{ kem['name_group'] }}', {%- endfor %} + # post-quantum + classical key exchanges + {% for kem in config['kems'] -%} + {%- for hybrid in kem['hybrids'] -%} + '{{ hybrid['hybrid_group'] }}_{{kem['name_group']}}', + {%- endfor -%} + {% endfor %} + diff --git a/oqs-template/scripts/common.py/sig_algs.fragment b/oqs-template/scripts/common.py/sig_algs.fragment new file mode 100644 index 00000000..2541638a --- /dev/null +++ b/oqs-template/scripts/common.py/sig_algs.fragment @@ -0,0 +1,12 @@ + + # post-quantum signatures + {% for sig in config['sigs'] %}{% for variant in sig['variants'] %}'{{ variant['name'] }}', + {%- endfor %} {%- endfor %} + # post-quantum + classical signatures + {% for sig in config['sigs'] -%} + {%- for variant in sig['variants'] -%} + {%- for classical_alg in variant['mix_with'] -%} + '{{ classical_alg['name'] }}_{{ variant['name'] }}', + {%- endfor -%} + {%- endfor %} {%- endfor %} + diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..40bb7936 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,21 @@ +# Build and test support scripts + +This directory contains various scripts aiming to ease build and test of `oqsprovider`. + +## Building + +The key file is [fullbuild.sh](fullbuild.sh) with options documented [here](https://github.com/open-quantum-safe/oqs-provider/blob/main/CONFIGURE.md#convenience-build-script-options). + +## Testing + +### API testing + +All features and enabled algorithms are API tested by `ctest` driven code contained in the [test directory](https://github.com/open-quantum-safe/oqs-provider/tree/main/test). + +### Command line testing + +All features and enabled algorithms are tested via `openssl` command line instructions via the [runtests.sh](runtests.sh) script with options documented [here](https://github.com/open-quantum-safe/oqs-provider/blob/main/CONFIGURE.md#convenience-build-script-options). + +### Release testing + +All features and all algorithms can be tested in a full matrix running all possible signature and KEM algorithms in client/server setup via the corresponding `openssl s_server/s_client` commands via the [release-test.sh](release-test.sh) script. To run this test successfully, installation of `python3` and `pytest` with `xdist` extension is required, e.g., via `sudo apt install python3 python3-pytest python3-pytest-xdist python3-psutil`. The test must be executed within the main project directory, e.g., as such `./scripts/release-test.sh`. For full operation, a local and up-to-date (release) installation of `openssl` and `liboqs` (e.g., built via `scripts/fulltest.sh`) is recommended. diff --git a/scripts/common.py b/scripts/common.py new file mode 100644 index 00000000..7b936214 --- /dev/null +++ b/scripts/common.py @@ -0,0 +1,165 @@ +import os +import subprocess +import pathlib +import psutil +import time + +key_exchanges = [ +##### OQS_TEMPLATE_FRAGMENT_KEX_ALGS_START + # post-quantum key exchanges + 'frodo640aes','frodo640shake','frodo976aes','frodo976shake','frodo1344aes','frodo1344shake','kyber512','kyber768','kyber1024','bikel1','bikel3','bikel5','hqc128','hqc192','hqc256', + # post-quantum + classical key exchanges + 'p256_frodo640aes','x25519_frodo640aes','p256_frodo640shake','x25519_frodo640shake','p384_frodo976aes','x448_frodo976aes','p384_frodo976shake','x448_frodo976shake','p521_frodo1344aes','p521_frodo1344shake','p256_kyber512','x25519_kyber512','p384_kyber768','x448_kyber768','x25519_kyber768','p256_kyber768','p521_kyber1024','p256_bikel1','x25519_bikel1','p384_bikel3','x448_bikel3','p521_bikel5','p256_hqc128','x25519_hqc128','p384_hqc192','x448_hqc192','p521_hqc256', +##### OQS_TEMPLATE_FRAGMENT_KEX_ALGS_END +] +signatures = [ + 'ecdsap256', 'rsa3072', +##### OQS_TEMPLATE_FRAGMENT_SIG_ALGS_START + # post-quantum signatures + 'dilithium2','dilithium3','dilithium5','falcon512','falcon1024','sphincssha2128fsimple','sphincssha2128ssimple','sphincssha2192fsimple','sphincsshake128fsimple', + # post-quantum + classical signatures + 'p256_dilithium2','rsa3072_dilithium2','p384_dilithium3','p521_dilithium5','p256_falcon512','rsa3072_falcon512','p521_falcon1024','p256_sphincssha2128fsimple','rsa3072_sphincssha2128fsimple','p256_sphincssha2128ssimple','rsa3072_sphincssha2128ssimple','p384_sphincssha2192fsimple','p256_sphincsshake128fsimple','rsa3072_sphincsshake128fsimple', +##### OQS_TEMPLATE_FRAGMENT_SIG_ALGS_END +] + +SERVER_START_ATTEMPTS = 10 + +def all_pq_groups(): + ag = "" + for kex in key_exchanges: + if len(ag)==0: + ag = kex + else: + ag = ag + ":" + kex + return ag + +def run_subprocess(command, working_dir='.', expected_returncode=0, input=None, env=os.environ): + """ + Helper function to run a shell command and report success/failure + depending on the exit status of the shell command. + """ + + # Note we need to capture stdout/stderr from the subprocess, + # then print it, which pytest will then capture and + # buffer appropriately + print(working_dir + " > " + " ".join(command)) + result = subprocess.run( + command, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=working_dir, + env=env + ) + if result.returncode != expected_returncode: + print(result.stdout.decode('utf-8')) + assert False, "Got unexpected return code {}".format(result.returncode) + return result.stdout.decode('utf-8') + +def start_server(ossl, test_artifacts_dir, sig_alg, worker_id): + command = [ossl, 's_server', + '-cert', os.path.join(test_artifacts_dir, '{}_{}_srv.crt'.format(worker_id, sig_alg)), + '-key', os.path.join(test_artifacts_dir, '{}_{}_srv.key'.format(worker_id, sig_alg)), + '-CAfile', os.path.join(test_artifacts_dir, '{}_{}_CA.crt'.format(worker_id, sig_alg)), + '-tls1_3', + '-quiet', +# add X25519 for baseline server test and all PQ KEMs for single PQ KEM tests: + '-groups', "x25519:"+all_pq_groups(), + # On UNIX-like systems, binding to TCP port 0 + # is a request to dynamically generate an unused + # port number. + # TODO: Check if Windows behaves similarly + '-accept', '0'] + + print(" > " + " ".join(command)) + server = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + server_info = psutil.Process(server.pid) + + # Try SERVER_START_ATTEMPTS times to see + # what port the server is bound to. + server_start_attempt = 1 + while server_start_attempt <= SERVER_START_ATTEMPTS: + if server_info.connections(): + break + else: + server_start_attempt += 1 + time.sleep(2) + server_port = str(server_info.connections()[0].laddr.port) + + # Check SERVER_START_ATTEMPTS times to see + # if the server is responsive. + server_start_attempt = 1 + while server_start_attempt <= SERVER_START_ATTEMPTS: + result = subprocess.run([ossl, 's_client', '-connect', 'localhost:{}'.format(server_port)], + input='Q'.encode(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if result.returncode == 0: + break + else: + server_start_attempt += 1 + time.sleep(2) + + if server_start_attempt > SERVER_START_ATTEMPTS: + raise Exception('Cannot start OpenSSL server') + + return server, server_port + +def gen_keys(ossl, ossl_config, sig_alg, test_artifacts_dir, filename_prefix): + pathlib.Path(test_artifacts_dir).mkdir(parents=True, exist_ok=True) + if sig_alg == 'ecdsap256': + run_subprocess([ossl, 'ecparam', + '-name', 'prime256v1', + '-out', os.path.join(test_artifacts_dir, '{}_prime256v1.pem'.format(filename_prefix))]) + run_subprocess([ossl, 'req', '-x509', '-new', + '-newkey', 'ec:{}'.format(os.path.join(test_artifacts_dir, '{}_prime256v1.pem'.format(filename_prefix))), + '-keyout', os.path.join(test_artifacts_dir, '{}_ecdsap256_CA.key'.format(filename_prefix)), + '-out', os.path.join(test_artifacts_dir, '{}_ecdsap256_CA.crt'.format(filename_prefix)), + '-nodes', + '-subj', '/CN=oqstest_CA', + '-days', '365', + '-config', ossl_config]) + run_subprocess([ossl, 'req', '-new', + '-newkey', 'ec:{}'.format(os.path.join(test_artifacts_dir, '{}_prime256v1.pem'.format(filename_prefix))), + '-keyout', os.path.join(test_artifacts_dir, '{}_ecdsap256_srv.key'.format(filename_prefix)), + '-out', os.path.join(test_artifacts_dir, '{}_ecdsap256_srv.csr'.format(filename_prefix)), + '-nodes', + '-subj', '/CN=oqstest_server', + '-config', ossl_config]) + else: + if sig_alg == 'rsa3072': + ossl_sig_alg_arg = 'rsa:3072' + else: + ossl_sig_alg_arg = sig_alg + run_subprocess([ossl, 'req', '-x509', '-new', + '-newkey', ossl_sig_alg_arg, + '-keyout', os.path.join(test_artifacts_dir, '{}_{}_CA.key'.format(filename_prefix, sig_alg)), + '-out', os.path.join(test_artifacts_dir, '{}_{}_CA.crt'.format(filename_prefix, sig_alg)), + '-nodes', + '-subj', '/CN=oqstest_CA', + '-days', '365', + '-config', ossl_config]) + run_subprocess([ossl, 'req', '-new', + '-newkey', ossl_sig_alg_arg, + '-keyout', os.path.join(test_artifacts_dir, '{}_{}_srv.key'.format(filename_prefix, sig_alg)), + '-out', os.path.join(test_artifacts_dir, '{}_{}_srv.csr'.format(filename_prefix, sig_alg)), + '-nodes', + '-subj', '/CN=oqstest_server', + '-config', ossl_config]) + + run_subprocess([ossl, 'x509', '-req', + '-in', os.path.join(test_artifacts_dir, '{}_{}_srv.csr'.format(filename_prefix, sig_alg)), + '-out', os.path.join(test_artifacts_dir, '{}_{}_srv.crt'.format(filename_prefix, sig_alg)), + '-CA', os.path.join(test_artifacts_dir, '{}_{}_CA.crt'.format(filename_prefix, sig_alg)), + '-CAkey', os.path.join(test_artifacts_dir, '{}_{}_CA.key'.format(filename_prefix, sig_alg)), + '-CAcreateserial', + '-days', '365']) + + # also create pubkeys from certs for dgst verify tests: + env = os.environ + #env["OPENSSL_CONF"]=os.path.join("scripts", "openssl.cnf") + #env["OPENSSL_MODULES"]=os.path.join("_build", "lib") + run_subprocess([ossl, 'req', + '-in', os.path.join(test_artifacts_dir, '{}_{}_srv.csr'.format(filename_prefix, sig_alg)), + '-pubkey', '-out', os.path.join(test_artifacts_dir, '{}_{}_srv.pubk'.format(filename_prefix, sig_alg)) ], + env=env) diff --git a/scripts/conftest.py b/scripts/conftest.py new file mode 100644 index 00000000..758d1e23 --- /dev/null +++ b/scripts/conftest.py @@ -0,0 +1,20 @@ +import os +import pytest +import subprocess + +def pytest_addoption(parser): + parser.addoption("--ossl", action="store", help="ossl: Path to standalone OpenSSL executable.") + parser.addoption("--ossl-config", action="store", help="ossl-config: Path to openssl.cnf file.") + parser.addoption("--test-artifacts-dir", action="store", help="test-artifacts-dir: Path to directory containing files generated during the testing process.") + +@pytest.fixture +def ossl_config(request): + return os.path.normpath(request.config.getoption("--ossl-config")) + +@pytest.fixture +def ossl(request): + return os.path.normpath(request.config.getoption("--ossl")) + +@pytest.fixture +def test_artifacts_dir(request): + return os.path.normpath(request.config.getoption("--test-artifacts-dir")) diff --git a/scripts/pytest.ini b/scripts/pytest.ini new file mode 100644 index 00000000..ef297a1d --- /dev/null +++ b/scripts/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --verbose --ossl=.local/bin/openssl --ossl-config=scripts/openssl-ca.cnf --test-artifacts-dir=tmp diff --git a/scripts/release-test.sh b/scripts/release-test.sh new file mode 100755 index 00000000..df3a60b2 --- /dev/null +++ b/scripts/release-test.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Stop in case of error +set -e + +# To be run as part of a release test only on Linux +# requires python, pytest, xdist; install e.g. via +# sudo apt install python3 python3-pytest python3-pytest-xdist python3-psutil + +# must be run in main folder +# multicore machine recommended for fast execution + +# expect (ideally latest/release-test) liboqs to be already build and present +if [ -d liboqs ]; then + export LIBOQS_SRC_DIR=`pwd`/liboqs +else + echo "liboqs not found. Exiting." + exit 1 +fi + +if [ -d oqs-template ]; then + # just a temp setup + git checkout -b reltest + # Activate all algorithms + sed -i "s/enable\: false/enable\: true/g" oqs-template/generate.yml + python3 oqs-template/generate.py + rm -rf _build + ./scripts/fullbuild.sh + ./scripts/runtests.sh + if [ -f .local/bin/openssl ]; then + OPENSSL_MODULES=`pwd`/_build/lib OPENSSL_CONF=`pwd`/scripts/openssl-ca.cnf python3 -m pytest --numprocesses=auto scripts/test_tls_full.py + else + echo "For full TLS PQ SIG/KEM matrix test, build (latest) openssl locally." + fi + git reset --hard && git checkout main && git branch -D reltest +else + echo "$0 must be run in main oqs-provider folder. Exiting." +fi + diff --git a/scripts/test_tls_full.py b/scripts/test_tls_full.py new file mode 100644 index 00000000..a1639140 --- /dev/null +++ b/scripts/test_tls_full.py @@ -0,0 +1,30 @@ +import common +import pytest +import sys +import os + +@pytest.fixture(params=common.signatures) +def server(ossl, ossl_config, test_artifacts_dir, request, worker_id): + # Setup: start ossl server + common.gen_keys(ossl, ossl_config, request.param, test_artifacts_dir, worker_id) + server, port = common.start_server(ossl, test_artifacts_dir, request.param, worker_id) + # Run tests + yield (request.param, port) + # Teardown: stop ossl server + server.kill() + +@pytest.mark.parametrize('kex_name', common.key_exchanges) +def test_sig_kem_pair(ossl, server, test_artifacts_dir, kex_name, worker_id): + client_output = common.run_subprocess([ossl, 's_client', + '-groups', kex_name, + '-CAfile', os.path.join(test_artifacts_dir, '{}_{}_CA.crt'.format(worker_id, server[0])), + '-verify_return_error', + '-connect', 'localhost:{}'.format(server[1])], + input='Q'.encode()) +# OpenSSL3 by default does not output KEM used; so rely on forced client group and OK handshake completion: + if not "SSL handshake has read" in client_output: + assert False, "Handshake failure." + +if __name__ == "__main__": + import sys + pytest.main(sys.argv)