Connector: Update encoding/decoding to use replace error handling

Recent Issue#10 shows that the connector does not handle decoding
of some versioned data. Use replace errors rather than strict errors
to allow the connector to work with this data

Change-Id: I3e3e4ab58e9c160acad576a870af3dc3936f2ff0

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 max-line-length = 120
 exclude = gen,build/*,*_pb2.py,*_pb2_grpc.py
-ignore = H301,F403,F401
+ignore = H301,F403,F401,W503
 per-file-ignores =
     # Excludes undefined variable for validators as ctx is defined at runtime
     actionpacks/*:F821

cloudvision/Connector/codec/decoder.py

@@ -42,7 +42,7 @@ def decode_map(self, m):
 
     def __postProcess(self, b):
         if isinstance(b, bytes):
-            return b.decode('utf-8')
+            return b.decode('utf-8', 'replace')
         elif isinstance(b, list):
             return self.decode_array(b)
         elif isinstance(b, (dict, FrozenDict)):

cloudvision/Connector/codec/encoder.py

@@ -15,7 +15,7 @@ def __init__(self):
         self.__buffer = io.BytesIO()
 
     def encode_string(self, s):
-        return self.__packer.pack(bytes(s, 'utf-8'))
+        return self.__packer.pack(bytes(s, 'utf-8', 'replace'))
 
     def encode_array(self, a):
         res = b""

test/test_codec.py

@@ -12,7 +12,7 @@
 import numpy as np
 
 import pytest
-
+import json
 
 import yaml
 
@@ -85,3 +85,28 @@ def test_encode_decode(name, inp, expected):
             failed = rev != inp
 
         assert not failed, "decoding value did not match input value"
+
+
+def test_decode_breaking_values():
+    decodeTcs = [
+        {
+            # Github Issue 10 BGP field decoding 'sentOpenMsg' decoding example
+            "in": b'\xc4$\x04\xfeW\x00\xb4\xac\x14\xfcK\x1a\x02\x18\x01\x04\x00\x01\x00\x01'
+            + b'\x02\x00@\x02A,A\x04\x00\x00\xfeWE\x04\x00\x01\x01\x01',
+            "exp": '\x04�W\x00��\x14�K\x1a\x02\x18\x01\x04\x00\x01\x00\x01'
+            + '\x02\x00@\x02A,A\x04\x00\x00�WE\x04\x00\x01\x01\x01',
+            # Expected json is the golang encoded value taken from the same path using apish
+            # to ensure that the outputs are consistent
+            "exp_json": "{\"value\": \"\\u0004\\ufffdW\\u0000\\ufffd\\ufffd\\u0014\\ufffdK\\u001a"
+            + "\\u0002\\u0018\\u0001\\u0004\\u0000\\u0001\\u0000\\u0001\\u0002\\u0000@\\u0002A,A"
+            + "\\u0004\\u0000\\u0000\\ufffdWE\\u0004\\u0000\\u0001\\u0001\\u0001\"}"
+        },
+    ]
+    for tc in decodeTcs:
+        input = tc["in"]
+        exp = tc["exp"]
+        out = decoder.decode(bytes(input))
+        assert out == exp, f"Bad decoding of {input}\nGot: '{out}'\nExp: '{exp}'"
+        out_json = json.JSONEncoder(ensure_ascii=True).encode({"value": out})
+        exp_json = tc["exp_json"]
+        assert out_json == exp_json, f"Unexpected json decoding of {input}\nGot: '{out_json}'\nExp: '{exp_json}'"