Fix build errors and some potential leaks in EVP_* failure cases

Signed-off-by: Songling Han <[email protected]>

src/common/aes/aes_ossl.c

@@ -305,19 +305,19 @@ static void AES256_CTR_inc_stream_iv(const uint8_t *iv, size_t iv_len, const voi
 		return; /* TODO: better error handling */
 	}
 	const struct key_schedule *ks = (const struct key_schedule *) schedule;
-	if (OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_EncryptInit_ex)(ctr_ctx, oqs_aes_256_ctr(), NULL, ks->key, iv_ctr)) != 1) {
+	if ((OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_EncryptInit_ex)(ctr_ctx, oqs_aes_256_ctr(), NULL, ks->key, iv_ctr))) != 1) {
 		OSSL_FUNC(EVP_CIPHER_CTX_free)(ctr_ctx);
 		return;
 	}
 
 	SIZE_T_TO_INT_OR_EXIT(out_len, out_len_input_int)
 	memset(out, 0, (size_t)out_len_input_int);
 	int out_len_output;
-	if (OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_EncryptUpdate)(ctr_ctx, out, &out_len_output, out, out_len_input_int)) != 1) {
+	if ((OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_EncryptUpdate)(ctr_ctx, out, &out_len_output, out, out_len_input_int))) != 1) {
 		OSSL_FUNC(EVP_CIPHER_CTX_free)(ctr_ctx);
 		return;
 	}
-	if (OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_EncryptFinal_ex)(ctr_ctx, out + out_len_output, &out_len_output)) != 1) {
+	if ((OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_EncryptFinal_ex)(ctr_ctx, out + out_len_output, &out_len_output))) != 1) {
 		OSSL_FUNC(EVP_CIPHER_CTX_free)(ctr_ctx);
 		return;
 	}
@@ -333,7 +333,10 @@ static void AES256_CTR_inc_stream_blks(void *schedule, uint8_t *out, size_t out_
 	int out_len_output;
 	SIZE_T_TO_INT_OR_EXIT(out_len, out_len_input_int);
 	memset(out, 0, (size_t)out_len_input_int);
-	OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_EncryptUpdate)(ks->ctx, out, &out_len_output, out, (int) out_len));
+	if ((OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_EncryptUpdate)(ks->ctx, out, &out_len_output, out, out_len_input_int))) != 1) {
+		OSSL_FUNC(EVP_CIPHER_CTX_free)(ks->ctx);
+		return;
+	}
 }
 
 struct OQS_AES_callbacks aes_default_callbacks = {

src/common/common.c

@@ -12,7 +12,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-
+#include <stddef.h>
 #if defined(OQS_DIST_BUILD) && defined(OQS_USE_PTHREADS)
 #include <pthread.h>
 #endif

src/common/sha2/sha2_ossl.c

@@ -20,9 +20,12 @@ static void do_hash(uint8_t *output, const uint8_t *input, size_t inplen, const
 	if (mdctx == NULL) {
 		OQS_EXIT("OpenSSL");
 	}
-	OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_DigestInit_ex)(mdctx, md, NULL));
-	OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_DigestUpdate)(mdctx, input, inplen));
-	OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_DigestFinal_ex)(mdctx, output, &outlen));
+	if (OSSL_FUNC(EVP_DigestInit_ex)(mdctx, md, NULL) != 1 ||
+	        OSSL_FUNC(EVP_DigestUpdate)(mdctx, input, inplen) != 1 ||
+	        OSSL_FUNC(EVP_DigestFinal_ex)(mdctx, output, &outlen) != 1) {
+		OSSL_FUNC(EVP_MD_CTX_free)(mdctx);
+		OQS_EXIT("OpenSSL");
+	}
 	OSSL_FUNC(EVP_MD_CTX_free)(mdctx);
 }
 
@@ -66,7 +69,10 @@ static void SHA2_sha256_inc_init(OQS_SHA2_sha256_ctx *state) {
 	if (mdctx == NULL) {
 		OQS_EXIT("OpenSSL");
 	}
-	OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_DigestInit_ex)(mdctx, md, NULL));
+	if (OSSL_FUNC(EVP_DigestInit_ex)(mdctx, md, NULL) != 1) {
+		OSSL_FUNC(EVP_MD_CTX_free)(mdctx);
+		OQS_EXIT("OpenSSL");
+	}
 	state->ctx = mdctx;
 }
 
@@ -111,7 +117,10 @@ static void SHA2_sha384_inc_init(OQS_SHA2_sha384_ctx *state) {
 	if (mdctx == NULL) {
 		OQS_EXIT("OpenSSL");
 	}
-	OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_DigestInit_ex)(mdctx, md, NULL));
+	if (OSSL_FUNC(EVP_DigestInit_ex)(mdctx, md, NULL) != 1) {
+		OSSL_FUNC(EVP_MD_CTX_free)(mdctx);
+		OQS_EXIT("OpenSSL");
+	}
 	state->ctx = mdctx;
 }
 
@@ -135,7 +144,6 @@ static void SHA2_sha384_inc_ctx_release(OQS_SHA2_sha384_ctx *state) {
 		state->ctx = NULL;
 	}
 }
-
 static void SHA2_sha384_inc_ctx_clone(OQS_SHA2_sha384_ctx *dest, const OQS_SHA2_sha384_ctx *src) {
 	SHA2_sha384_inc_init(dest);
 	OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_MD_CTX_copy_ex)((EVP_MD_CTX *) dest->ctx, (EVP_MD_CTX *) src->ctx));
@@ -152,7 +160,10 @@ static void SHA2_sha512_inc_init(OQS_SHA2_sha512_ctx *state) {
 	if (mdctx == NULL) {
 		OQS_EXIT("OpenSSL");
 	}
-	OQS_OPENSSL_GUARD(OSSL_FUNC(EVP_DigestInit_ex)(mdctx, md, NULL));
+	if (OSSL_FUNC(EVP_DigestInit_ex)(mdctx, md, NULL) != 1) {
+		OSSL_FUNC(EVP_MD_CTX_free)(mdctx);
+		OQS_EXIT("OpenSSL");
+	}
 	state->ctx = mdctx;
 }
 
@@ -203,5 +214,4 @@ struct OQS_SHA2_callbacks sha2_default_callbacks = {
 	SHA2_sha512_inc_finalize,
 	SHA2_sha512_inc_ctx_release,
 };
-
 #endif