updated code and tests

cultureamp · Sep 12, 2023 · 250fd45 · 250fd45
1 parent 4097a4a
commit 250fd45
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 13 deletions.
diff --git a/...ns/transforms/RedShiftArrayTransformer.kt → ...rms/RedShiftComplexDataTypeTransformer.kt b/...ns/transforms/RedShiftArrayTransformer.kt → ...rms/RedShiftComplexDataTypeTransformer.kt
@@ -26,7 +26,7 @@ import java.util.Collections
  * @param R is ConnectRecord<R>.
  * @constructor Creates a RedShiftArrayTransformer Transformation<R> for a given ConnectRecord<T>
  */
-class RedShiftArrayTransformer<R : ConnectRecord<R>> : Transformation<R> {
+class RedShiftComplexDataTypeTransformer<R : ConnectRecord<R>> : Transformation<R> {
     private val logger = LoggerFactory.getLogger(this::class.java.canonicalName)
     private val purpose = "RedShift™ JSON Array to String Transform"
 
@@ -60,7 +60,7 @@ class RedShiftArrayTransformer<R : ConnectRecord<R>> : Transformation<R> {
     }
 
     private fun updateSchema(field: Field): Schema {
-        if (field.schema().type() == Schema.Type.ARRAY) {
+        if (field.schema().type() == Schema.Type.ARRAY || field.schema().type() == Schema.Type.MAP) {
             return SchemaBuilder.string().build()
         }
         return field.schema()

diff --git a/...ransforms/RedShiftArrayTransformerTest.kt → ...RedShiftComplexDataTypeTransformerTest.kt b/...ransforms/RedShiftArrayTransformerTest.kt → ...RedShiftComplexDataTypeTransformerTest.kt
@@ -26,27 +26,27 @@ import kotlin.test.assertTrue
  * See https://docs.confluent.io/platform/current/connect/javadocs/javadoc/org/apache/kafka/connect/transforms/Transformation.html.
  *
  * @param R is ConnectRecord<R>.
- * @constructor Creates a RedShiftArrayTransformer Transformation<R> for a given ConnectRecord<T>
+ * @constructor Creates a RedShiftComplexDataTypeTransformer Transformation<R> for a given ConnectRecord<T>
  *
  */
-class RedShiftArrayTransformerTest {
+class RedShiftComplexDataTypeTransformerTest {
 
-    private lateinit var transformer: RedShiftArrayTransformer<SourceRecord>
+    private lateinit var transformer: RedShiftComplexDataTypeTransformer<SourceRecord>
 
-    private fun hasNoArrays(obj: SourceRecord): Boolean {
+    private fun hasNoComplexTypes(obj: SourceRecord): Boolean {
 
-        var hasNoArray = true
+        var hasNoComplexTypes = true
         for (field in obj.valueSchema().fields()) {
-            if (field.schema().type() == Schema.Type.ARRAY) {
-                hasNoArray = false
+            if (field.schema().type() == Schema.Type.ARRAY || field.schema().type() == Schema.Type.MAP) {
+                hasNoComplexTypes = false
             }
         }
-        return hasNoArray
+        return hasNoComplexTypes
     }
 
     @Before
     fun setUp() {
-        transformer = RedShiftArrayTransformer()
+        transformer = RedShiftComplexDataTypeTransformer()
     }
 
     @Test
@@ -62,8 +62,8 @@ class RedShiftArrayTransformerTest {
         )
 
         val transformedRecord = transformer.apply(sourceRecord)
-        hasNoArrays(sourceRecord)
-        assertTrue(hasNoArrays(transformedRecord))
+        hasNoComplexTypes(sourceRecord)
+        assertTrue(hasNoComplexTypes(transformedRecord))
     }
 
     private val sourceSchema = AvroSchema.fromJson(fileContent("com/cultureamp/employee-data.employees-value-v1.avsc"))

diff --git a/src/test/resources/com/cultureamp/employee-data.employees-v1.json b/src/test/resources/com/cultureamp/employee-data.employees-v1.json
@@ -57,6 +57,15 @@
 			}
 		}
 	],
+	"test_map": {
+		"added_users_count": 0,
+		"ignored_new_demographics_count": 0,
+		"ignored_users_count": 0,
+		"inactive_updated_users_count": 0,
+		"reactivated_users_count": 0,
+		"removed_users_count": 0,
+		"updated_users_count": 0
+	},
 	"test_string_array": [
 		"a", "b", "c"
 	],

diff --git a/src/test/resources/com/cultureamp/employee-data.employees-value-v1.avsc b/src/test/resources/com/cultureamp/employee-data.employees-value-v1.avsc
@@ -265,6 +265,17 @@
         }
       ],
       "default": null
+    },
+    {
+      "name": "test_map",
+      "type": [
+        "null",
+        {
+          "type": "map",
+          "values": "int"
+        }
+      ],
+      "default": null
     }
   ]
 }