From 37d6746a7d2d52dabd0697dcc178067fad814cea Mon Sep 17 00:00:00 2001
From: Steve Lawrence <slawrence@apache.org>
Date: Tue, 12 Dec 2023 08:08:19 -0500
Subject: [PATCH] Use encodingErrorPolicy replace, even when set to error

If encodingErrorPolicy="error" we print a warning letting the user know
it is not supported and that we will use "replace" instead. This is so
that we can support schemas written for IBM DFDL (which only supports
"error") and have the exact same behavior as IBM DFDL except for when
data has encoding errors, which is usually rare.

However, even though we warn that we will use "replace", we do not
actually do so, which leads to an assertion exception if there are
encoding errors. This fixes that so our behavior matches the warning
message and we always use "replace".

DAFFODIL-2861
---
 .../core/dsom/TermEncodingMixin.scala         | 22 +++++++++++--------
 .../daffodil/section00/general/tunables.tdml  |  2 ++
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/TermEncodingMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/TermEncodingMixin.scala
index 4d972e0d68..df5e993206 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/TermEncodingMixin.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/TermEncodingMixin.scala
@@ -59,15 +59,19 @@ trait TermEncodingMixin extends KnownEncodingMixin { self: Term =>
           )
         optionEncodingErrorPolicy.getOrElse(EncodingErrorPolicy.Replace)
       }
-    if (policy == EncodingErrorPolicy.Error) {
-      // DFDL-935 to enable
-      SDW(
-        WarnID.EncodingErrorPolicyError,
-        "dfdl:encodingErrorPolicy=\"error\" is not yet implemented. The 'replace' value will be used.",
-      )
-      EncodingErrorPolicy.Replace
-    }
-    policy
+
+    // DFDL-935 to enable
+    schemaDefinitionWarningWhen(
+      WarnID.EncodingErrorPolicyError,
+      policy == EncodingErrorPolicy.Error,
+      "dfdl:encodingErrorPolicy=\"error\" is not yet implemented. The 'replace' value will be used.",
+    )
+
+    // The encodingErrorPolicy property is essentially ignored. We always use "replace", but
+    // warn if it is "error". This allows for compatibility with schemas written for IBM DFDL
+    // (which only supports "error") and has the same behavior as IBM DFDL as long as data does
+    // not contain encoding errors
+    EncodingErrorPolicy.Replace
   }
 
   /**
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/tunables.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/tunables.tdml
index e53276a29a..86d56f194f 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/tunables.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section00/general/tunables.tdml
@@ -678,6 +678,7 @@
 
 		<tdml:document>
 			<tdml:documentPart type="text">1234</tdml:documentPart>
+			<tdml:documentPart type="byte">80</tdml:documentPart> <!-- invalid in US-ASCII -->
 		</tdml:document>
 
     <tdml:warnings>
@@ -693,6 +694,7 @@
           <ex:char>2</ex:char>
           <ex:char>3</ex:char>
           <ex:char>4</ex:char>
+          <ex:char>&#xFFFD;</ex:char> <!-- unicode replacement character -->
         </ex:root>
       </tdml:dfdlInfoset>
     </tdml:infoset>