Init SoftHSMv2 support

- wolfSSL_EVP_PKEY_set1_DH: If both private and public present, output private key
- ToTraditionalInline_ex2: Add DH checking
- wc_ecc_get_curve_id: check index is not negative
- Fix i2d_PKCS8_PRIV_KEY_INFO to actually output pkcs8 instead of just der
- wolfSSL_EVP_PKEY2PKCS8: Create duplicate to avoid double free
- wolfSSL_DH_generate_key: Fix case where not enough buffer was allocated for 128 bit case
- pkcs8_encode: Add DSA and DH support
- wolfSSL_d2i_PKCS8_PKEY: Correctly advance buffer
- RSA_LOW_MEM: export all integers in compat layer
- Add softhsm action
- Define
  - OPENSSL_DH_MAX_MODULUS_BITS
  - OPENSSL_DSA_MAX_MODULUS_BITS
  - OPENSSL_RSA_MAX_MODULUS_BITS
- Implement
  - BN_mul_word
  - i2d_ECPKParameters
  - PEM_write_bio_PKCS8_PRIV_KEY_INFO
  - PEM_read_bio_PKCS8_PRIV_KEY_INFO
  - i2d_PKCS8_PRIV_KEY_INFO
  - RSA_padding_add_PKCS1_PSS_mgf1
  - RSA_verify_PKCS1_PSS_mgf1

.github/workflows/softhsm.yml

@@ -0,0 +1,96 @@
+name: SoftHSMv2 Tests
+
+# START OF COMMON SECTION
+on:
+  push:
+    # TODO uncomment
+    #branches: [ 'master', 'main', 'release/**' ]
+  pull_request:
+    branches: [ '*' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+# END OF COMMON SECTION
+
+jobs:
+  build_wolfssl:
+    name: Build wolfSSL
+    # Just to keep it the same as the testing target
+    runs-on: ubuntu-latest
+    # This should be a safe limit for the tests to run.
+    timeout-minutes: 10
+    steps:
+      - name: Build wolfSSL
+        uses: wolfSSL/actions-build-autotools-project@v1
+        with:
+          path: wolfssl
+          configure: --enable-all
+          install: true
+          check: false
+
+      - name: tar build-dir
+        run: tar -zcf build-dir.tgz build-dir
+
+      - name: Upload built lib
+        uses: actions/upload-artifact@v4
+        with:
+          name: wolf-install-softhsm
+          path: build-dir.tgz
+          retention-days: 5
+
+  softhsm_check:
+    strategy:
+      fail-fast: false
+      matrix:
+        # List of releases to test
+        ref: [ 2.6.1 ]
+    name: ${{ matrix.ref }}
+    runs-on: ubuntu-latest
+    # This should be a safe limit for the tests to run.
+    timeout-minutes: 20
+    needs: build_wolfssl
+    steps:
+      - name: Install dependencies
+        run: |
+          # Don't prompt for anything
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get update
+          sudo apt-get install -y libcppunit-dev
+
+      - name: Download lib
+        uses: actions/download-artifact@v4
+        with:
+          name: wolf-install-softhsm
+
+      - name: untar build-dir
+        run: tar -xf build-dir.tgz
+
+      - name: Checkout OSP
+        uses: actions/checkout@v4
+        with:
+          # TODO point to wolfssl/osp
+          repository: julek-wolfssl/osp
+          path: osp
+          # TODO remove ref
+          ref: softhsm
+
+      - name: Checkout SoftHSMv2
+        uses: actions/checkout@v4
+        with:
+          repository: opendnssec/SoftHSMv2
+          path: softhsm
+          ref: ${{ matrix.ref }}
+
+      # Not using wolfSSL/actions-build-autotools-project@v1 because autogen.sh doesn't work
+      - name: Build softhsm
+        working-directory: softhsm
+        run: |
+          patch -p1 < $GITHUB_WORKSPACE/osp/softhsm/${{ matrix.ref }}.patch
+          autoreconf -if
+          ./configure --with-crypto-backend=wolfssl WOLFSSL_INSTALL_DIR=$GITHUB_WORKSPACE/build-dir
+          make -j
+
+      - name: Test softhsm
+        working-directory: softhsm
+        run: make -j check