Merge pull request #10 from teamclairvoyant/staging

Staging

.gitignore

@@ -2,6 +2,8 @@
 .metals
 .venv
 .vscode
+.bsp
+.idea
 
 project
 target

build.sbt

@@ -2,7 +2,7 @@ ThisBuild / scalaVersion := "3.3.0"
 
 ThisBuild / organization := "com.clairvoyant.data.scalaxy"
 
-ThisBuild / version := "1.1.0"
+ThisBuild / version := "1.2.0"
 
 ThisBuild / resolvers ++= Seq(
   "DataScalaxyTestUtil Repo" at "https://maven.pkg.github.com/teamclairvoyant/data-scalaxy-test-util"

src/main/scala/com/clairvoyant/data/scalaxy/transformer/DataFrameTransformerImplicits.scala

@@ -448,18 +448,30 @@ object DataFrameTransformerImplicits {
      *
      * @param columnName
      *   Name of the column to be converted
+     * @param schemaDDL
+     *   The Data Definition Language (DDL) for the column
      * @return
      *   DataFrame with the column converted to struct type
      */
     def convertJSONStringToStruct(
-        columnName: String
+        columnName: String,
+        schemaDDL: Option[String] = None
     ): DataFrame =
       import df.sparkSession.implicits.*
+
+      val schema =
+        schemaDDL match {
+          case Some(schemaDDL) =>
+            DataType.fromDDL(schemaDDL)
+          case None =>
+            df.sparkSession.read.json(df.select(columnName).as[String]).schema
+        }
+
       df.withColumn(
         columnName,
         from_json(
           col(columnName),
-          df.sparkSession.read.json(df.select(columnName).as[String]).schema
+          schema
         )
       )
 

src/test/scala/com/clairvoyant/data/scalaxy/transformer/DataFrameTransformerImplicitsSpec.scala

@@ -1024,6 +1024,46 @@ class DataFrameTransformerImplicitsSpec extends DataFrameReader with DataFrameMa
     actualDF should matchExpectedDataFrame(expectedDF)
   }
 
+  "convertJSONStringToStruct() - with columnName and schemaDDL" should "convert the specified column to Struct Type" in {
+    val df = readJSONFromText(
+      """
+        |{
+        |  "col_A": "val_A",
+        |  "col_B": "{\"col_C\": \"val_C\",\"col_D\": 5}"
+        |}
+        |""".stripMargin
+    )
+
+    val actualDF = df.convertJSONStringToStruct(
+      columnName = "col_B",
+      schemaDDL = Some("col_C STRING, col_D STRING")
+    )
+
+    val expectedDF = readJSONFromText(
+      """
+        |{
+        |  "col_A": "val_A",
+        |  "col_B": {
+        |    "col_C": "val_C",
+        |    "col_D": "5"
+        |  }
+        |}
+        |""".stripMargin
+    )
+
+    actualDF.schema.fields
+      .filter(_.name == "col_B")
+      .head
+      .dataType shouldBe StructType(
+      List(
+        StructField("col_C", StringType),
+        StructField("col_D", StringType)
+      )
+    )
+
+    actualDF should matchExpectedDataFrame(expectedDF)
+  }
+
   "flattenSchema()" should "flatten the dataframe" in {
     val df = readJSONFromText(
       """