Update return type for EVP_EncodeUpdate (#1481)

tpm2-tools expects EVP_EncodeUpdate to return an integer, instead of
being a void function: tpm2-software/tpm2-tools@54a4178.

Apparently the return type of this has been updated since OpenSSL
1.1.0: openssl/openssl@cf3404f. It seems worth updating to align
with upstream.

crypto/base64/base64.c

@@ -134,21 +134,21 @@ void EVP_EncodeInit(EVP_ENCODE_CTX *ctx) {
   OPENSSL_memset(ctx, 0, sizeof(EVP_ENCODE_CTX));
 }
 
-void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, uint8_t *out, int *out_len,
-                      const uint8_t *in, size_t in_len) {
+int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, uint8_t *out, int *out_len,
+                     const uint8_t *in, size_t in_len) {
   size_t total = 0;
 
   *out_len = 0;
   if (in_len == 0) {
-    return;
+    return 0;
   }
 
   assert(ctx->data_used < sizeof(ctx->data));
 
   if (sizeof(ctx->data) - ctx->data_used > in_len) {
     OPENSSL_memcpy(&ctx->data[ctx->data_used], in, in_len);
     ctx->data_used += (unsigned)in_len;
-    return;
+    return 1;
   }
 
   if (ctx->data_used != 0) {
@@ -178,7 +178,7 @@ void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, uint8_t *out, int *out_len,
 
     if (total + encoded + 1 < total) {
       *out_len = 0;
-      return;
+      return 0;
     }
 
     total += encoded + 1;
@@ -194,8 +194,11 @@ void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, uint8_t *out, int *out_len,
     // We cannot signal an error, but we can at least avoid making *out_len
     // negative.
     total = 0;
+    return 0;
   }
   *out_len = (int)total;
+
+  return 1;
 }
 
 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, uint8_t *out, int *out_len) {

crypto/base64/base64_test.cc

@@ -206,9 +206,10 @@ TEST_P(Base64Test, EncodeDecode) {
     EVP_EncodeInit(&ctx);
 
     int out_len;
-    EVP_EncodeUpdate(&ctx, out, &out_len,
-                     reinterpret_cast<const uint8_t *>(t.decoded),
-                     decoded_len);
+    int ret = EVP_EncodeUpdate(&ctx, out, &out_len,
+                               reinterpret_cast<const uint8_t *>(t.decoded),
+                               decoded_len);
+    EXPECT_EQ(ret, (strlen(t.encoded) > 0 ? 1 : 0));
     size_t total = out_len;
 
     EVP_EncodeFinal(&ctx, out + total, &out_len);

crypto/decrepit/bio/base64_bio.c

@@ -396,8 +396,10 @@ static int b64_write(BIO *b, const char *in, int inl) {
         ret += n;
       }
     } else {
-      EVP_EncodeUpdate(&(ctx->base64), (uint8_t *)ctx->buf, &ctx->buf_len,
-                       (uint8_t *)in, n);
+      if(!EVP_EncodeUpdate(&(ctx->base64), (uint8_t *)ctx->buf, &ctx->buf_len,
+                       (uint8_t *)in, n)) {
+        return ((ret == 0) ? -1 : ret);
+      }
       assert(ctx->buf_len <= (int)sizeof(ctx->buf));
       assert(ctx->buf_len >= ctx->buf_off);
       ret += n;

crypto/pem/pem_lib.c

@@ -561,7 +561,9 @@ int PEM_write_bio(BIO *bp, const char *name, const char *header,
   i = j = 0;
   while (len > 0) {
     n = (int)((len > (PEM_BUFSIZE * 5)) ? (PEM_BUFSIZE * 5) : len);
-    EVP_EncodeUpdate(&ctx, buf, &outl, &(data[j]), n);
+    if(!EVP_EncodeUpdate(&ctx, buf, &outl, &(data[j]), n)) {
+      goto err;
+    }
     if ((outl) && (BIO_write(bp, (char *)buf, outl) != outl)) {
       goto err;
     }

include/openssl/base64.h

@@ -131,9 +131,9 @@ OPENSSL_EXPORT void EVP_EncodeInit(EVP_ENCODE_CTX *ctx);
 // version of them to |out| and sets |*out_len| to the number of bytes written.
 // Some state may be contained in |ctx| so |EVP_EncodeFinal| must be used to
 // flush it before using the encoded data.
-OPENSSL_EXPORT void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, uint8_t *out,
-                                     int *out_len, const uint8_t *in,
-                                     size_t in_len);
+OPENSSL_EXPORT int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, uint8_t *out,
+                                    int *out_len, const uint8_t *in,
+                                    size_t in_len);
 
 // EVP_EncodeFinal flushes any remaining output bytes from |ctx| to |out| and
 // sets |*out_len| to the number of bytes written.