From e5bbf1903817a47c479328edd9910ef938ef990e Mon Sep 17 00:00:00 2001 From: omaus Date: Fri, 27 Jan 2023 16:58:33 +0100 Subject: [PATCH 1/6] Add `Study.list` function from ArcCommander API --- src/arcIO.NET/Study.fs | 84 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/src/arcIO.NET/Study.fs b/src/arcIO.NET/Study.fs index 3da4170..0d069f4 100644 --- a/src/arcIO.NET/Study.fs +++ b/src/arcIO.NET/Study.fs @@ -16,11 +16,36 @@ module Study = module StudyFolder = - /// Checks if an study folder exists in the ARC. + /// Checks if a Study folder exists in the ARC. let exists (arc : string) (identifier : string) = Path.Combine([|arc;rootFolderName;identifier|]) |> System.IO.Directory.Exists + /// Returns the Study identifiers of the Study Files located in each Study's folder of a given path to an ARC. + let findStudyIdentifiers arcDir = + let log = Logging.createLogger "findStudyIdentifiersLog" + let studiesPath = Path.Combine(arcDir, rootFolderName) + try + let studyFolders = Directory.GetDirectories studiesPath + studyFolders + |> Array.collect ( + fun sf -> + let studyIdentifier = (DirectoryInfo sf).Name + Directory.GetFiles sf + |> Array.choose (fun s -> + if s.EndsWith "isa.study.xlsx" then + Some studyIdentifier + elif s.EndsWith "_isa.study.xlsx" then + log.Warn $"The Study File of Study {studyIdentifier} has a deprecated File Name: {s}" + Some studyIdentifier + else + None + ) + ) + with e -> + log.Error e.Message + [||] + let readFromFolder (arc : string) (folderPath : string) = let sp = Path.Combine(folderPath,studyFileName).Replace(@"\","/") let study = StudyFile.Study.fromFile sp @@ -89,4 +114,59 @@ module Study = let study = Study.create(FileName = studyFileName, Identifier = studyName) - init arc study \ No newline at end of file + init arc study + + /// Takes the path to an ARC and lists all study identifiers registered in this ARC's investigation file. + let list (arcDir : string) = + + let log = Logging.createLogger "StudyListLog" + + log.Info("Start Study List") + + (* the following part is _not nice_: The functionality for this already exists in Investigation.fs. Unfortunately, Investigation.fs is compiled *AFTER* + Study.fs and, thus, we cannot use functions from there. Moving Study.fs after Investigation.fs in compilation order also does not work due to + Investigation.fs using functions from Study.fs. + If someone finds a solution for this F#-specific compilation problem, feel free to fix :) *) + let investigationFilePath = Path.Combine(arcDir, "isa.investigation.xlsx") + log.Trace($"InvestigationFile: {investigationFilePath}") + let investigation = Investigation.fromFile investigationFilePath + // end of part + + let studyFileIdentifiers = set (StudyFolder.findStudyIdentifiers arcDir) + + let studyIdentifiers = + investigation.Studies + |> Option.defaultValue [] + |> List.choose (fun s -> + match s.Identifier with + | None | Some "" -> + log.Warn("Study does not have identifier") + None + | Some i -> Some i + ) + |> set + + let onlyRegistered = Set.difference studyIdentifiers studyFileIdentifiers + let onlyInitialized = Set.difference studyFileIdentifiers studyIdentifiers + let combined = Set.union studyIdentifiers studyFileIdentifiers + + if not onlyRegistered.IsEmpty then + log.Warn("The ARC contains following registered studies that have no associated file:") + onlyRegistered + |> Seq.iter ((sprintf "%s") >> log.Warn) + log.Info($"You can init the study file using \"arc s init\"") + + if not onlyInitialized.IsEmpty then + log.Warn("The ARC contains study files with the following identifiers not registered in the investigation:") + onlyInitialized + |> Seq.iter ((sprintf "%s") >> log.Warn) + log.Info($"You can register the study using \"arc s register\"") + + if combined.IsEmpty then + log.Error("The ARC does not contain any studies.") + + combined + |> Seq.map (fun identifier -> + //log.Debug(sprintf "Study: %s" identifier) + sprintf "Study: %s" identifier + ) \ No newline at end of file From c7a7fb50a7e08f64f5fd2e34f5b4872aaa9e371a Mon Sep 17 00:00:00 2001 From: Heinrich Lukas Weil Date: Thu, 9 Feb 2023 17:20:25 +0100 Subject: [PATCH 2/6] update ISADotNet reference --- src/arcIO.NET/arcIO.NET.fsproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arcIO.NET/arcIO.NET.fsproj b/src/arcIO.NET/arcIO.NET.fsproj index 92d923d..ba9894f 100644 --- a/src/arcIO.NET/arcIO.NET.fsproj +++ b/src/arcIO.NET/arcIO.NET.fsproj @@ -23,7 +23,7 @@ - + From a94c9fc4c5923721a6ba28c89b540a3d0bec0ec6 Mon Sep 17 00:00:00 2001 From: Heinrich Lukas Weil Date: Wed, 22 Feb 2023 14:48:07 +0100 Subject: [PATCH 3/6] add overwrite functions --- src/arcIO.NET/Investigation.fs | 41 +++++++++++++++-- src/arcIO.NET/Study.fs | 83 ++++++++++++++++++++++++++++++++-- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/src/arcIO.NET/Investigation.fs b/src/arcIO.NET/Investigation.fs index 47ca40e..b38f72c 100644 --- a/src/arcIO.NET/Investigation.fs +++ b/src/arcIO.NET/Investigation.fs @@ -22,6 +22,30 @@ module Investigation = let investigationFilePath = Path.Combine(arcDir,investigationFileName) Investigation.toFile investigationFilePath investigation + /// Creates an investigation file in the ARC. + let overWrite (arcDir : string) (investigation : ISADotNet.Investigation) = + + let log = Logging.createLogger "InvestigationWriteLog" + log.Info("Start Investigation Write") + + let investigationFilePath = Path.Combine(arcDir,investigationFileName) + + if System.IO.File.Exists(investigationFilePath) then + try + let cache = File.ReadAllBytes(investigationFilePath) + File.Delete(investigationFilePath) + try + Investigation.toFile investigationFilePath investigation + with + | err -> + File.WriteAllBytes(investigationFilePath,cache) + log.Error($"Investigation file could not be overwritten: {err.Message}") + with + | err -> + log.Error($"Investigation file could not be overwritten: {err.Message}") + else + Investigation.toFile investigationFilePath investigation + /// Reads an investigation from the ARC. let read (arcDir : string) = @@ -37,6 +61,7 @@ module Investigation = log.Error("Investigation file does not exist.") raise (System.SystemException()) + /// Reads and combines all ISA components of the ARC into the ISA object let fromArcFolder (arcDir : string) = let log = Logging.createLogger "InvestigationFromArcFolderLog" @@ -52,6 +77,7 @@ module Investigation = match study.Identifier with | Some id -> let studyFromFile = Study.readByIdentifier arcDir id + let mergedStudy = API.Update.UpdateByExistingAppendLists.updateRecordType study studyFromFile // update study assays and contacts with information from assay files match study.Assays with | Some assays -> @@ -65,13 +91,13 @@ module Investigation = | None -> log.Warn("Study \'" + id + "\' contains Assay without filename.") cl, al @ [assay] - ) (studyFromFile.Contacts |> Option.defaultValue [],[]) - {studyFromFile with + ) (mergedStudy.Contacts |> Option.defaultValue [],[]) + {mergedStudy with Contacts = Some (scontacts |> List.distinct) Assays = Some sassays } | None -> - studyFromFile + mergedStudy | None -> log.Warn("Investigation file contains study without identifier.") study @@ -106,7 +132,9 @@ module Investigation = // fill investigation with information from study files and assay files {i with Studies = istudies'} + |> API.Investigation.update + /// Registers an assay to the investigation arc registry let registerAssay arcDir studyName (assayName) = let log = Logging.createLogger "RegisterAssayLog" @@ -142,4 +170,9 @@ module Investigation = Study.init arcDir study [study] |> API.Investigation.setStudies investigation - |> write arcDir \ No newline at end of file + |> write arcDir + + /// Update investigation file with information from the different ISA components of the ARC + let updateRegistry arcDir = + fromArcFolder arcDir + |> overWrite arcDir \ No newline at end of file diff --git a/src/arcIO.NET/Study.fs b/src/arcIO.NET/Study.fs index 0d069f4..c814d5f 100644 --- a/src/arcIO.NET/Study.fs +++ b/src/arcIO.NET/Study.fs @@ -75,16 +75,89 @@ module Study = Path.Combine ([|arc;rootFolderName;studyIdentifier|]) |> readFromFolder arc + /// Writes a study to the given folder. Fails, if the file already exists let writeToFolder (folderPath : string) (study : Study) = - let sp = Path.Combine (folderPath,studyFileName) - StudyFile.Study.toFile sp study + let log = Logging.createLogger "StudyWriteLog" + + log.Info("Start Study Write") + + let studyFilePath = Path.Combine (folderPath,studyFileName) + + if System.IO.File.Exists(studyFilePath) then + log.Error("Study file does already exist.") + + else + StudyFile.Study.toFile studyFilePath study + + /// Writes a study to the given folder. Overwrites it, if the file already exists + let overWriteToFolder (folderPath : string) (study : Study) = + + let log = Logging.createLogger "StudyWriteLog" + log.Info("Start Study Write") + + let studyFilePath = Path.Combine (folderPath,studyFileName) + + if System.IO.File.Exists(studyFilePath) then + try + let cache = File.ReadAllBytes(studyFilePath) + File.Delete(studyFilePath) + try + StudyFile.Study.toFile studyFilePath study + with + | err -> + File.WriteAllBytes(studyFilePath,cache) + log.Error($"Study file could not be overwritten: {err.Message}") + with + | err -> + log.Error($"Study file could not be overwritten: {err.Message}") + else + StudyFile.Study.toFile studyFilePath study + + /// Writes a study to the arc. Fails, if the file already exists let write (arc : string) (study : Study) = + + let log = Logging.createLogger "StudyWriteLog" + + log.Info("Start Study Write") + if study.FileName.IsNone then - failwith "Cannot write study to arc, as it has no filename" - let sp = Path.Combine ([|arc;rootFolderName;study.FileName.Value|]) - StudyFile.Study.toFile sp study + log.Error("Cannot write study to arc, as it has no filename") + else + + let studyFilePath = Path.Combine ([|arc;rootFolderName;study.FileName.Value|]) + if System.IO.File.Exists(studyFilePath) then + log.Error("Study file does already exist.") + + else + StudyFile.Study.toFile studyFilePath study + + /// Writes a study to the arc. Overwrites it, if the file already exists + let overWrite (arc : string) (study : Study) = + let log = Logging.createLogger "StudyWriteLog" + + log.Info("Start Study Write") + + if study.FileName.IsNone then + log.Error("Cannot write study to arc, as it has no filename") + else + let studyFilePath = Path.Combine ([|arc;rootFolderName;study.FileName.Value|]) + if System.IO.File.Exists(studyFilePath) then + try + let cache = File.ReadAllBytes(studyFilePath) + File.Delete(studyFilePath) + try + StudyFile.Study.toFile studyFilePath study + with + | err -> + File.WriteAllBytes(studyFilePath,cache) + log.Error($"Study file could not be overwritten: {err.Message}") + with + | err -> + log.Error($"Study file could not be overwritten: {err.Message}") + else + StudyFile.Study.toFile studyFilePath study let init (arc : string) (study : Study) = From d204adce0562fdb67c8376500c9a5ce909ad631c Mon Sep 17 00:00:00 2001 From: Heinrich Lukas Weil Date: Wed, 22 Feb 2023 14:54:50 +0100 Subject: [PATCH 4/6] update depencdencies --- src/arcIO.NET/arcIO.NET.fsproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/arcIO.NET/arcIO.NET.fsproj b/src/arcIO.NET/arcIO.NET.fsproj index ba9894f..1716f12 100644 --- a/src/arcIO.NET/arcIO.NET.fsproj +++ b/src/arcIO.NET/arcIO.NET.fsproj @@ -19,12 +19,12 @@ - + - - + + From 00f17b695f7d394b03e80e6de077a8ff44345b52 Mon Sep 17 00:00:00 2001 From: Heinrich Lukas Weil Date: Thu, 23 Feb 2023 16:59:45 +0100 Subject: [PATCH 5/6] add converter type --- src/arcIO.NET/Converter.fs | 14 ++++++++++++++ src/arcIO.NET/arcIO.NET.fsproj | 2 ++ 2 files changed, 16 insertions(+) create mode 100644 src/arcIO.NET/Converter.fs diff --git a/src/arcIO.NET/Converter.fs b/src/arcIO.NET/Converter.fs new file mode 100644 index 0000000..77e4106 --- /dev/null +++ b/src/arcIO.NET/Converter.fs @@ -0,0 +1,14 @@ +namespace arcIO.NET.Converter + +open ISADotNet +open ISADotNet.QueryModel +open FsSpreadsheet.DSL +open LitXml + + +type ARCconverter = +| ARCtoCSV of (QInvestigation -> QStudy -> QAssay -> SheetEntity) +| ARCtoTSV of (QInvestigation -> QStudy -> QAssay -> SheetEntity) +| ARCtoXLSX of (QInvestigation -> QStudy -> QAssay -> SheetEntity) +| ARCtoXML of (QInvestigation -> QStudy -> QAssay -> LitXml.XmlPart) +//| ARCtoJSON of QInvestigation -> QStudy -> QAssay -> \ No newline at end of file diff --git a/src/arcIO.NET/arcIO.NET.fsproj b/src/arcIO.NET/arcIO.NET.fsproj index 1716f12..8fd5de0 100644 --- a/src/arcIO.NET/arcIO.NET.fsproj +++ b/src/arcIO.NET/arcIO.NET.fsproj @@ -16,6 +16,7 @@ + @@ -24,6 +25,7 @@ + From e42fd9dabcc325b1c888995fa11563ec426679f2 Mon Sep 17 00:00:00 2001 From: Heinrich Lukas Weil Date: Fri, 24 Feb 2023 16:15:32 +0100 Subject: [PATCH 6/6] add converter helper function --- src/arcIO.NET/Converter.fs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/arcIO.NET/Converter.fs b/src/arcIO.NET/Converter.fs index 77e4106..5f5e26b 100644 --- a/src/arcIO.NET/Converter.fs +++ b/src/arcIO.NET/Converter.fs @@ -11,4 +11,26 @@ type ARCconverter = | ARCtoTSV of (QInvestigation -> QStudy -> QAssay -> SheetEntity) | ARCtoXLSX of (QInvestigation -> QStudy -> QAssay -> SheetEntity) | ARCtoXML of (QInvestigation -> QStudy -> QAssay -> LitXml.XmlPart) -//| ARCtoJSON of QInvestigation -> QStudy -> QAssay -> \ No newline at end of file +//| ARCtoJSON of QInvestigation -> QStudy -> QAssay -> + + member this.ConvertCSV(i,s,a) = + match this with + | ARCtoCSV f -> f i s a + | _ -> failwith "could not convert to csv" + + member this.ConvertTSV(i,s,a) = + match this with + | ARCtoTSV f -> f i s a + | _ -> failwith "could not convert to tsv" + + member this.ConvertXLSX(i,s,a) = + match this with + | ARCtoXLSX f -> f i s a + | _ -> failwith "could not convert to xlsx" + + member this.ConvertXML(i,s,a) = + match this with + | ARCtoXML f -> f i s a + | _ -> failwith "could not convert to xml" + +