From cb10b576d41705984244071ef040acfe95ca3632 Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Wed, 29 Nov 2023 12:30:58 -0800 Subject: [PATCH 1/3] chore: update test vectors native so they are positive negative tests --- .../TestVectors/TestVectors.cs | 251 +++++++++++------- 1 file changed, 152 insertions(+), 99 deletions(-) diff --git a/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/TestVectors.cs b/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/TestVectors.cs index dea6891cd..85b70a324 100644 --- a/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/TestVectors.cs +++ b/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/TestVectors.cs @@ -13,6 +13,8 @@ using AWS.Cryptography.EncryptionSDK; using AWS.Cryptography.MaterialProviders; using Exception = System.Exception; +using OpaqueError = AWS.Cryptography.Primitives.OpaqueError; + // ReSharper disable SuggestVarOrType_SimpleTypes // ReSharper disable SuggestVarOrType_Elsewhere // ReSharper disable SuggestVarOrType_BuiltInTypes @@ -133,8 +135,8 @@ public TestVectorDecryptTests(ITestOutputHelper testLogging) } [SkippableTheory] - [ClassData (typeof(DecryptTestVectors))] - public void CanDecryptTestVector( + [ClassData(typeof(DecryptTestVectors))] + public void PositiveDecryptTestVector( string vectorId, DecryptVector vector, Dictionary keyMap, @@ -142,135 +144,186 @@ public void CanDecryptTestVector( string expectedError, MemoryStream ciphertextStream, NetV4_0_0_RetryPolicy _netV400RetryPolicy - ) { + ) + { if (expectedPlaintext != null && expectedError != null) { throw new ArgumentException( $"Test vector {vectorId} has both plaintext and error in its expected result, this is not possible" ); } - Exception exceptionHolder = null; - bool exceptionHasBeenCaught = false; - try + AwsEncryptionSdkConfig config = new AwsEncryptionSdkConfig { - AwsEncryptionSdkConfig config = new AwsEncryptionSdkConfig - { - CommitmentPolicy = ESDKCommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, - NetV4_0_0_RetryPolicy = _netV400RetryPolicy - }; - ESDK encryptionSdk = new ESDK(config); + CommitmentPolicy = ESDKCommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + NetV4_0_0_RetryPolicy = _netV400RetryPolicy + }; + ESDK encryptionSdk = new ESDK(config); - ICryptographicMaterialsManager cmm = MaterialProviderFactory.CreateDecryptCmm(vector, keyMap, vectorId); + ICryptographicMaterialsManager cmm = MaterialProviderFactory.CreateDecryptCmm(vector, keyMap, vectorId); - DecryptInput decryptInput = new DecryptInput + DecryptInput decryptInput = new DecryptInput + { + Ciphertext = ciphertextStream, + MaterialsManager = cmm, + }; + if (vector.CMM is "RequiredEncryptionContext") + { + decryptInput = new DecryptInput { Ciphertext = ciphertextStream, MaterialsManager = cmm, + EncryptionContext = vector.EncryptionContext }; - if (vector.CMM is "RequiredEncryptionContext") - { - decryptInput = new DecryptInput - { - Ciphertext = ciphertextStream, - MaterialsManager = cmm, - EncryptionContext = vector.EncryptionContext - }; - } - AWS.Cryptography.EncryptionSDK.DecryptOutput decryptOutput = encryptionSdk.Decrypt(decryptInput); - if (expectedError != null) - { - throw new TestVectorShouldHaveFailedException( - $"Test vector {vectorId} succeeded when it shouldn't have" - ); - } - - byte[] result = decryptOutput.Plaintext.ToArray(); - Assert.Equal(expectedPlaintext, result); } - // Ensure Test Failure is not caught - catch (TestVectorShouldHaveFailedException) + AWS.Cryptography.EncryptionSDK.DecryptOutput decryptOutput = encryptionSdk.Decrypt(decryptInput); + if (expectedError != null) { - throw; + throw new TestVectorShouldHaveFailedException( + $"Test vector {vectorId} succeeded when it shouldn't have" + ); } - catch (Exception e) when ( - e is AWS.Cryptography.Primitives.CollectionOfErrors - or AWS.Cryptography.KeyStore.CollectionOfErrors - or AWS.Cryptography.MaterialProviders.CollectionOfErrors - or AWS.Cryptography.EncryptionSDK.CollectionOfErrors - ) - { - exceptionHasBeenCaught = true; - exceptionHolder = e; - // Use Reflection to get the common list field - Type collectionType = e.GetType(); - FieldInfo listFieldInfo = collectionType.GetField("list"); - List list = (List)listFieldInfo?.GetValue(e); - List debugList = new List(); - if (vector.MasterKeys != null) debugList.AddRange(vector.MasterKeys.Select(keyInfo => $"Key: {keyInfo.Key}, Type: {keyInfo.Type}")); - testLogging.WriteLine($"CollectionOfErrors Logging. List:\n{string.Join("\n\t", list!)}"); - testLogging.WriteLine($"CollectionOfErrors Logging. master-keys:\n{string.Join("\n\t", debugList)}"); - } - catch (Exception e) when ( - e is AWS.Cryptography.Primitives.OpaqueError - or AWS.Cryptography.KeyStore.OpaqueError - or AWS.Cryptography.MaterialProviders.OpaqueError - or AWS.Cryptography.EncryptionSDK.OpaqueError - ) + byte[] result = decryptOutput.Plaintext.ToArray(); + Assert.Equal(expectedPlaintext, result); + } + + [SkippableTheory] + [ClassData (typeof(DecryptTestVectors))] + public void NegativeDecryptTestVector( + string vectorId, + DecryptVector vector, + Dictionary keyMap, + byte[] expectedPlaintext, + string expectedError, + MemoryStream ciphertextStream, + NetV4_0_0_RetryPolicy _netV400RetryPolicy + ) { + if (expectedPlaintext != null && expectedError != null) { - exceptionHasBeenCaught = true; - // Use Reflection to get the common Obj field - Type opaqueType = e.GetType(); - FieldInfo objFieldInfo = opaqueType.GetField("obj"); - object obj = objFieldInfo?.GetValue(e); - switch (obj) - { - case null when e.Message.Equals(OPAQUE_ERROR_NULL_OBJ_MSG): - testLogging.WriteLine($"OpaqueError Logging: Obj was null. Error Type is {opaqueType}"); - exceptionHolder = e; - break; - case Exception nestedException: - testLogging.WriteLine($"OpaqueError Logging: Obj is an Exception. " + - $"Error Type is {opaqueType}.\n" + - $"Nested Exception is {nestedException.GetType()}.\n" + - $"Nested Exceptions message is: \n\t{nestedException.Message}\n" - // + $"Nested Exceptions StackTrace is: \n\t{nestedException.StackTrace}" - ); - exceptionHolder = nestedException; - break; - default: - testLogging.WriteLine($"OpaqueError Logging: Obj is an arbitrary object. " + - $"Error Type is {opaqueType}. " + - $"Type of Obj is {obj!.GetType()}."); - exceptionHolder = e; - break; - } + throw new ArgumentException( + $"Test vector {vectorId} has both plaintext and error in its expected result, this is not possible" + ); } - catch (Exception e) + + AwsEncryptionSdkConfig config = new AwsEncryptionSdkConfig + { + CommitmentPolicy = ESDKCommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + NetV4_0_0_RetryPolicy = _netV400RetryPolicy + }; + ESDK encryptionSdk = new ESDK(config); + + ICryptographicMaterialsManager cmm = MaterialProviderFactory.CreateDecryptCmm(vector, keyMap, vectorId); + + DecryptInput decryptInput = new DecryptInput { - exceptionHasBeenCaught = true; - exceptionHolder = e; - testLogging.WriteLine($"Unexpected Exception: {e}"); - testLogging.WriteLine($"Unexpected Exception: Error Type is {e.GetType()}. " + - $"Exception's message is: {e.Message}"); + Ciphertext = ciphertextStream, + MaterialsManager = cmm, + }; + if (vector.CMM is "RequiredEncryptionContext") + { + decryptInput = new DecryptInput + { + Ciphertext = ciphertextStream, + MaterialsManager = cmm, + EncryptionContext = vector.EncryptionContext + }; } - finally + // if no exception is thrown this will error - this is good because we are not supposed to succeed here. + // ensures we erroring the expected way + var ex = Assert.Throws(() =>encryptionSdk.Decrypt(decryptInput)); + + // if it is an unknown error we will fail otherwise move on. + // if this is not an error we expect we will fail + Assert.Throws(() => EvaluateThrownError(ex, vector, expectedPlaintext)); + } + + private void EvaluateThrownError(Exception e, DecryptVector vector, byte[] expectedPlaintext) + { + Exception exceptionHolder = null; + bool exceptionHasBeenCaught = false; + + switch (e) { - if (expectedPlaintext == null && exceptionHasBeenCaught) + case AWS.Cryptography.Primitives.CollectionOfErrors + or AWS.Cryptography.KeyStore.CollectionOfErrors + or AWS.Cryptography.MaterialProviders.CollectionOfErrors + or AWS.Cryptography.EncryptionSDK.CollectionOfErrors: { - testLogging.WriteLine("Decrypt Failed, possibly correctly?"); + exceptionHasBeenCaught = true; + exceptionHolder = e; + // Use Reflection to get the common list field + Type collectionType = e.GetType(); + FieldInfo listFieldInfo = collectionType.GetField("list"); + List list = (List)listFieldInfo?.GetValue(e); + List debugList = new List(); + if (vector.MasterKeys != null) debugList.AddRange(vector.MasterKeys.Select(keyInfo => $"Key: {keyInfo.Key}, Type: {keyInfo.Type}")); + + testLogging.WriteLine($"CollectionOfErrors Logging. List:\n{string.Join("\n\t", list!)}"); + testLogging.WriteLine($"CollectionOfErrors Logging. master-keys:\n{string.Join("\n\t", debugList)}"); + break; } - if (expectedPlaintext != null && exceptionHasBeenCaught) + case AWS.Cryptography.Primitives.OpaqueError + or AWS.Cryptography.KeyStore.OpaqueError + or AWS.Cryptography.MaterialProviders.OpaqueError + or AWS.Cryptography.EncryptionSDK.OpaqueError: { - testLogging.WriteLine("Decrypt Failed, and should not have!"); + exceptionHasBeenCaught = true; + // Use Reflection to get the common Obj field + Type opaqueType = e.GetType(); + FieldInfo objFieldInfo = opaqueType.GetField("obj"); + object obj = objFieldInfo?.GetValue(e); + switch (obj) + { + case null when e.Message.Equals(OPAQUE_ERROR_NULL_OBJ_MSG): + testLogging.WriteLine($"OpaqueError Logging: Obj was null. Error Type is {opaqueType}"); + exceptionHolder = e; + break; + case Exception nestedException: + testLogging.WriteLine($"OpaqueError Logging: Obj is an Exception. " + + $"Error Type is {opaqueType}.\n" + + $"Nested Exception is {nestedException.GetType()}.\n" + + $"Nested Exceptions message is: \n\t{nestedException.Message}\n" + // + $"Nested Exceptions StackTrace is: \n\t{nestedException.StackTrace}" + ); + exceptionHolder = nestedException; + break; + default: + testLogging.WriteLine($"OpaqueError Logging: Obj is an arbitrary object. " + + $"Error Type is {opaqueType}. " + + $"Type of Obj is {obj!.GetType()}."); + exceptionHolder = e; + break; + } + + break; } + default: + exceptionHasBeenCaught = true; + exceptionHolder = e; + testLogging.WriteLine($"Unexpected Exception: {e}"); + testLogging.WriteLine($"Unexpected Exception: Error Type is {e.GetType()}. " + + $"Exception's message is: {e.Message}"); + break; + } + + if (expectedPlaintext != null && exceptionHasBeenCaught) + { + testLogging.WriteLine("Decrypt Failed, and should not have!"); } + if (exceptionHolder != null && expectedPlaintext != null) { - // Should succeed but did not, throw exception + if (exceptionHolder is System.Security.Cryptography.CryptographicException + or Org.BouncyCastle.Crypto.InvalidCipherTextException) + { + // Should succeed but did not, throw exception + throw new TestVectorShouldHaveFailedException(exceptionHolder.Message); + } + throw exceptionHolder; } } + private class TestVectorShouldHaveFailedException : Exception { From 22164a217795590327fe817384dc3f09c51ed1db Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Wed, 29 Nov 2023 12:40:02 -0800 Subject: [PATCH 2/3] update gha --- .github/workflows/library_net_tests.yml | 37 ++++++++----------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/.github/workflows/library_net_tests.yml b/.github/workflows/library_net_tests.yml index 1d4a3876f..ddd7ef354 100644 --- a/.github/workflows/library_net_tests.yml +++ b/.github/workflows/library_net_tests.yml @@ -204,18 +204,17 @@ jobs: DOWNLOAD_NAME=valid-Net-4.0.0.zip unzip -o -qq $DOWNLOAD_NAME -d $NET_400_VALID_VECTORS - - name: Run ESDK-NET @ v4.0.0 Valid Vectors expect success + - name: Run ESDK-NET @ v4.0.0 Valid Vectors Positive Decrypt Test working-directory: ./AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors - continue-on-error: true shell: bash run: | NET_400_VALID_VECTORS=$GITHUB_WORKSPACE/v4Net400Valid/vectors ESDK_NET_V400_POLICY="forbid" \ DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$NET_400_VALID_VECTORS/manifest.json" \ - dotnet test --framework net48 + dotnet test --framework net48 --filter PositiveDecryptTestVector ESDK_NET_V400_POLICY="forbid" \ DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$NET_400_VALID_VECTORS/manifest.json" \ - dotnet test --framework net6.0 --logger "console;verbosity=quiet" + dotnet test --framework net6.0 --logger "console;verbosity=quiet" --filter PositiveDecryptTestVector - name: Unzip ESDK-NET @ v4.0.0 Invalid Vectors working-directory: ./AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources @@ -226,24 +225,17 @@ jobs: DOWNLOAD_NAME=invalid-Net-4.0.0.zip unzip -o -qq $DOWNLOAD_NAME -d $NET_400_INVALID_VECTORS - - name: Run ESDK-NET @ v4.0.0 Invalid Vectors .NET 48 expect failure + - name: Run ESDK-NET @ v4.0.0 Invalid Vectors .NET 48 Negative Decrypt Test working-directory: ./AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors - continue-on-error: true shell: bash run: | NET_400_INVALID_VECTORS=$GITHUB_WORKSPACE/v4Net400Invalid/vectors ESDK_NET_V400_POLICY="forbid" \ DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$NET_400_INVALID_VECTORS/manifest.json" \ - dotnet test --framework net48 - # Dotnet test returns 1 for failure. - TEMP=$?; if [[ "$TEMP" -eq 1 ]]; then true; else false; fi; - # We want this to fail, so if it returned 1, step passes, else it fails - # TODO Post-#619: Refactor Test Vectors to expect failure, - # as I doubt this true false logic works + dotnet test --framework net48 --filter NegativeDecryptTestVector - - name: Run ESDK-NET @ v4.0.0 Invalid Vectors .NET 6.0 expect failure + - name: Run ESDK-NET @ v4.0.0 Invalid Vectors .NET 6.0 Negative Decrypt Test working-directory: ./AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors - continue-on-error: true shell: bash run: | NET_400_INVALID_VECTORS=$GITHUB_WORKSPACE/v4Net400Invalid/vectors @@ -251,30 +243,25 @@ jobs: ESDK_NET_V400_POLICY="forbid" \ DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$NET_400_INVALID_VECTORS/manifest.json" \ DYLD_LIBRARY_PATH="/usr/local/opt/openssl@1.1/lib" \ - dotnet test --framework net6.0 + dotnet test --framework net6.0 --filter NegativeDecryptTestVector else ESDK_NET_V400_POLICY="forbid" \ DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$NET_400_INVALID_VECTORS/manifest.json" \ - dotnet test --framework net6.0 + dotnet test --framework net6.0 --filter NegativeDecryptTestVector fi - # Dotnet test returns 1 for failure. - TEMP=$?; if [[ "$TEMP" -eq 1 ]]; then true; else false; fi; - # We want this to fail, so if it returned 1, step passes, else it fails - # TODO Post-#619: Refactor Test Vectors to expect failure, - # as I doubt this true false logic works - - name: Run ESDK-NET @ v4.0.0 Invalid Vectors .NET expect Success + - name: Run ESDK-NET @ v4.0.0 Invalid Vectors .NET Positive Decrypt Test working-directory: ./AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors shell: bash run: | NET_400_INVALID_VECTORS=$GITHUB_WORKSPACE/v4Net400Invalid/vectors DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$NET_400_INVALID_VECTORS/manifest.json" \ - dotnet test --framework net48 --logger "console;verbosity=quiet" + dotnet test --framework net48 --logger "console;verbosity=quiet" --filter PositiveDecryptTestVector if [ "$RUNNER_OS" == "macOS" ]; then DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$NET_400_INVALID_VECTORS/manifest.json" \ DYLD_LIBRARY_PATH="/usr/local/opt/openssl@1.1/lib" \ - dotnet test --framework net6.0 --logger "console;verbosity=quiet" + dotnet test --framework net6.0 --logger "console;verbosity=quiet" --filter PositiveDecryptTestVector else DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$NET_400_INVALID_VECTORS/manifest.json" \ - dotnet test --framework net6.0 --logger "console;verbosity=quiet" + dotnet test --framework net6.0 --logger "console;verbosity=quiet" --filter PositiveDecryptTestVector fi From faf3db7913213bc5749beafa5a42c890a7f5bdd2 Mon Sep 17 00:00:00 2001 From: Jose Corella Date: Wed, 29 Nov 2023 12:42:49 -0800 Subject: [PATCH 3/3] fix other gha --- .github/workflows/library_net_tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/library_net_tests.yml b/.github/workflows/library_net_tests.yml index ddd7ef354..3ce053407 100644 --- a/.github/workflows/library_net_tests.yml +++ b/.github/workflows/library_net_tests.yml @@ -140,7 +140,7 @@ jobs: run: | PYTHON_23_VECTOR_PATH=$GITHUB_WORKSPACE/python23/vectors DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$PYTHON_23_VECTOR_PATH/manifest.json" \ - dotnet test --framework net48 + dotnet test --framework net48 --filter PositiveDecryptTestVector - name: Run Decrypt Test Vectors on .NET net6.0 working-directory: ./AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors @@ -150,10 +150,10 @@ jobs: if [ "$RUNNER_OS" == "macOS" ]; then DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$PYTHON_23_VECTOR_PATH/manifest.json" \ DYLD_LIBRARY_PATH="/usr/local/opt/openssl@1.1/lib" \ - dotnet test --framework net6.0 + dotnet test --framework net6.0 --filter PositiveDecryptTestVector else DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="$PYTHON_23_VECTOR_PATH/manifest.json" \ - dotnet test --framework net6.0 + dotnet test --framework net6.0 --filter PositiveDecryptTestVector fi - name: Generate Test Vectors with .NET Framework net6.0