Merge pull request #242 from Ferlab-Ste-Justine/fix/clin-2119

fix: CLIN-2119 manage column mc with multiple entries
Ferlab-Ste-Justine · Oct 21, 2024 · 71838f7 · 71838f7
2 parents fbb7c58 + f37cada
commit 71838f7
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 3 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -15,6 +15,7 @@ jobs:
         distribution: 'adopt'
         java-version: '11'
         cache: 'sbt'
+    - uses: sbt/setup-sbt@v1
     - name: Run datalake-commons tests
       run: sbt 'project datalake-commons' 'test'
     - name: Run datalake-spark3 tests

diff --git a/...ke-spark3/src/main/scala/bio/ferlab/datalake/spark3/publictables/normalized/Clinvar.scala b/...ke-spark3/src/main/scala/bio/ferlab/datalake/spark3/publictables/normalized/Clinvar.scala
@@ -69,7 +69,7 @@ case class Clinvar(rc: RuntimeETLContext) extends SimpleETLP(rc) {
       )
       .withColumn("clndisdbincl", split(concat_ws("", col("clndisdbincl")), "\\|"))
       .withColumn("clndnincl", split(concat_ws("", col("clndnincl")), "\\|"))
-      .withColumn("mc", split(concat_ws("|", col("mc")), "\\|"))
+      .withColumn("mc", split(array_join(col("mc"), "|"), "\\|"))
       .withColumn("inheritance", inheritance_udf(col("origin")))
       .drop("clin_sig_original", "clndn")
 

diff --git a/...ls/src/main/scala/bio/ferlab/datalake/testutils/models/normalized/NormalizedClinvar.scala b/...ls/src/main/scala/bio/ferlab/datalake/testutils/models/normalized/NormalizedClinvar.scala
@@ -25,7 +25,7 @@ case class NormalizedClinvar(chromosome: String = "2",
                              af_tgp: Double = 0.01118,
                              clnvc: String = "single_nucleotide_variant",
                              clnhgvs: List[String] = List("NC_000002.12:g.69359261T>A"),
-                             mc: List[String] = List("SO:0001627", "intron_variant"),
+                             mc: List[String] = List("SO:0001627", "intron_variant", "SO:0001589", "frameshift_variant"),
                              af_esp: Double = 0.01415,
                              clndisdbincl: List[String] = List(""),
                              conditions: List[String] = List("Congenital myasthenic syndrome 12", "not specified", "not provided"),

diff --git a/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/raw/RawClinvar.scala b/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/raw/RawClinvar.scala
@@ -31,7 +31,7 @@ case class RawClinvar(contigName: String = "2",
                       INFO_AF_TGP: Double = 0.01118,
                       INFO_CLNVC: String = "single_nucleotide_variant",
                       INFO_CLNHGVS: List[String] = List("NC_000002.12:g.69359261T>A"),
-                      INFO_MC: List[String] = List("SO:0001627|intron_variant"),
+                      INFO_MC: List[String] = List("SO:0001627|intron_variant", "SO:0001589|frameshift_variant"),
                       INFO_CLNSIGCONF: Option[List[String]] = None,
                       INFO_AF_ESP: Double = 0.01415,
                       INFO_CLNDISDBINCL: Option[List[String]] = None,