From 47e51400bb04571ed0d2848029253518969313fb Mon Sep 17 00:00:00 2001
From: John Safranek <john@wolfssl.com>
Date: Tue, 17 Sep 2024 10:55:11 -0700
Subject: [PATCH] FP SmallStack Fix

The function _fp_exptmod_nct() is using WOLFSSL_NO_MALLOC to guard
using stack allocation vs malloc. It's twin function _fp_exptmod_ct()
is using WOLFSSL_SMALL_STACK for this. This is causing inappropriate use
of malloc() in a small stack environment. The no-malloc case will also
be kept so static memory and no-malloc fix still works.

1. Change the guards for `#ifndef WOLFSSL_NO_MALLOC` in the function
   `_fp_exptmod_nct()` to `#if defined(WOLFSSL_SMALL_STACK) &&
   !defined(WOLFSSL_NO_MALLOC)`.
---
 wolfcrypt/src/tfm.c | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/wolfcrypt/src/tfm.c b/wolfcrypt/src/tfm.c
index fc8578569a..50952f9591 100644
--- a/wolfcrypt/src/tfm.c
+++ b/wolfcrypt/src/tfm.c
@@ -2430,7 +2430,7 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
   fp_int  *res;
   fp_digit buf, mp;
   int      err, bitbuf, bitcpy, bitcnt, mode, digidx, x, y, winsize;
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
   fp_int  *M;
 #else
   fp_int   M[(1 << 6) + 1];
@@ -2455,7 +2455,7 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
      return err;
   }
 
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
   /* only allocate space for what's needed for window plus res */
   M = (fp_int*)XMALLOC(sizeof(fp_int)*((1 << winsize) + 1), NULL,
                                                            DYNAMIC_TYPE_BIGINT);
@@ -2482,7 +2482,7 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
   /* now we need R mod m */
   err = fp_montgomery_calc_normalization (res, P);
   if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
     XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
     return err;
@@ -2493,7 +2493,7 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
      /* G > P so we reduce it first */
      err = fp_mod(G, P, &M[1]);
      if (err != FP_OKAY) {
-     #ifndef WOLFSSL_NO_MALLOC
+     #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
         XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
      #endif
         return err;
@@ -2503,7 +2503,7 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
   }
   err = fp_mulmod (&M[1], res, P, &M[1]);
   if (err != FP_OKAY) {
-  #ifndef WOLFSSL_NO_MALLOC
+  #if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
      XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
   #endif
      return err;
@@ -2516,14 +2516,14 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
     err = fp_sqr (&M[(word32)(1 << (winsize - 1))],
                   &M[(word32)(1 << (winsize - 1))]);
     if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
       XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
       return err;
     }
     err = fp_montgomery_reduce_ex(&M[(word32)(1 << (winsize - 1))], P, mp, 0);
     if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
       XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
       return err;
@@ -2534,14 +2534,14 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
   for (x = (1 << (winsize - 1)) + 1; x < (1 << winsize); x++) {
     err = fp_mul(&M[x - 1], &M[1], &M[x]);
     if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
       XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
       return err;
     }
     err = fp_montgomery_reduce_ex(&M[x], P, mp, 0);
     if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
       XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
       return err;
@@ -2585,14 +2585,14 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
     if (mode == 1 && y == 0) {
       err = fp_sqr(res, res);
       if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
         XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
         return err;
       }
       err = fp_montgomery_reduce_ex(res, P, mp, 0);
       if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
         XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
         return err;
@@ -2610,14 +2610,14 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
       for (x = 0; x < winsize; x++) {
         err = fp_sqr(res, res);
         if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
           XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
           return err;
         }
         err = fp_montgomery_reduce_ex(res, P, mp, 0);
         if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
           XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
           return err;
@@ -2627,14 +2627,14 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
       /* then multiply */
       err = fp_mul(res, &M[bitbuf], res);
       if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
         XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
         return err;
       }
       err = fp_montgomery_reduce_ex(res, P, mp, 0);
       if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
         XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
         return err;
@@ -2653,14 +2653,14 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
     for (x = 0; x < bitcpy; x++) {
       err = fp_sqr(res, res);
       if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
         XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
         return err;
       }
       err = fp_montgomery_reduce_ex(res, P, mp, 0);
       if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
         XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
         return err;
@@ -2672,14 +2672,14 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
         /* then multiply */
         err = fp_mul(res, &M[1], res);
         if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
           XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
           return err;
         }
         err = fp_montgomery_reduce_ex(res, P, mp, 0);
         if (err != FP_OKAY) {
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
           XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
           return err;
@@ -2699,7 +2699,7 @@ static int _fp_exptmod_nct(fp_int * G, fp_int * X, fp_int * P, fp_int * Y)
   /* swap res with Y */
   fp_copy (res, Y);
 
-#ifndef WOLFSSL_NO_MALLOC
+#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_NO_MALLOC)
   XFREE(M, NULL, DYNAMIC_TYPE_BIGINT);
 #endif
   return err;