diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e29ce82f..280a269e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,7 @@ jobs: distribution: 'adopt' java-version: '11' cache: 'sbt' + - uses: sbt/setup-sbt@v1 - name: Run datalake-commons tests run: sbt 'project datalake-commons' 'test' - name: Run datalake-spark3 tests diff --git a/datalake-spark3/src/main/scala/bio/ferlab/datalake/spark3/publictables/normalized/Clinvar.scala b/datalake-spark3/src/main/scala/bio/ferlab/datalake/spark3/publictables/normalized/Clinvar.scala index 9f49d62a..4cfc2e54 100644 --- a/datalake-spark3/src/main/scala/bio/ferlab/datalake/spark3/publictables/normalized/Clinvar.scala +++ b/datalake-spark3/src/main/scala/bio/ferlab/datalake/spark3/publictables/normalized/Clinvar.scala @@ -69,7 +69,7 @@ case class Clinvar(rc: RuntimeETLContext) extends SimpleETLP(rc) { ) .withColumn("clndisdbincl", split(concat_ws("", col("clndisdbincl")), "\\|")) .withColumn("clndnincl", split(concat_ws("", col("clndnincl")), "\\|")) - .withColumn("mc", split(concat_ws("|", col("mc")), "\\|")) + .withColumn("mc", split(array_join(col("mc"), "|"), "\\|")) .withColumn("inheritance", inheritance_udf(col("origin"))) .drop("clin_sig_original", "clndn") diff --git a/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/normalized/NormalizedClinvar.scala b/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/normalized/NormalizedClinvar.scala index 896e5080..7514f97c 100644 --- a/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/normalized/NormalizedClinvar.scala +++ b/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/normalized/NormalizedClinvar.scala @@ -25,7 +25,7 @@ case class NormalizedClinvar(chromosome: String = "2", af_tgp: Double = 0.01118, clnvc: String = "single_nucleotide_variant", clnhgvs: List[String] = List("NC_000002.12:g.69359261T>A"), - mc: List[String] = List("SO:0001627", "intron_variant"), + mc: List[String] = List("SO:0001627", "intron_variant", "SO:0001589", "frameshift_variant"), af_esp: Double = 0.01415, clndisdbincl: List[String] = List(""), conditions: List[String] = List("Congenital myasthenic syndrome 12", "not specified", "not provided"), diff --git a/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/raw/RawClinvar.scala b/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/raw/RawClinvar.scala index 1a2b805c..5ac97f25 100644 --- a/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/raw/RawClinvar.scala +++ b/datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/raw/RawClinvar.scala @@ -31,7 +31,7 @@ case class RawClinvar(contigName: String = "2", INFO_AF_TGP: Double = 0.01118, INFO_CLNVC: String = "single_nucleotide_variant", INFO_CLNHGVS: List[String] = List("NC_000002.12:g.69359261T>A"), - INFO_MC: List[String] = List("SO:0001627|intron_variant"), + INFO_MC: List[String] = List("SO:0001627|intron_variant", "SO:0001589|frameshift_variant"), INFO_CLNSIGCONF: Option[List[String]] = None, INFO_AF_ESP: Double = 0.01415, INFO_CLNDISDBINCL: Option[List[String]] = None,