Merge pull request #7 from nfdi4plants/arcCommanderMigration

Small fixes and additions

src/arcIO.NET/Converter.fs

@@ -0,0 +1,36 @@
+namespace arcIO.NET.Converter
+
+open ISADotNet
+open ISADotNet.QueryModel
+open FsSpreadsheet.DSL
+open LitXml
+
+
+type ARCconverter =
+| ARCtoCSV      of (QInvestigation -> QStudy -> QAssay -> SheetEntity<Workbook>)
+| ARCtoTSV      of (QInvestigation -> QStudy -> QAssay -> SheetEntity<Workbook>)
+| ARCtoXLSX     of (QInvestigation -> QStudy -> QAssay -> SheetEntity<Workbook>)
+| ARCtoXML      of (QInvestigation -> QStudy -> QAssay -> LitXml.XmlPart)
+//| ARCtoJSON     of QInvestigation -> QStudy -> QAssay -> 
+
+    member this.ConvertCSV(i,s,a) = 
+        match this with
+        | ARCtoCSV f -> f i s a
+        | _ -> failwith "could not convert to csv"
+
+    member this.ConvertTSV(i,s,a) = 
+        match this with
+        | ARCtoTSV f -> f i s a
+        | _ -> failwith "could not convert to tsv"
+
+    member this.ConvertXLSX(i,s,a) = 
+        match this with
+        | ARCtoXLSX f -> f i s a
+        | _ -> failwith "could not convert to xlsx"
+
+    member this.ConvertXML(i,s,a) = 
+        match this with
+        | ARCtoXML f -> f i s a
+        | _ -> failwith "could not convert to xml"
+
+

src/arcIO.NET/Investigation.fs

@@ -22,6 +22,30 @@ module Investigation =
             let investigationFilePath = Path.Combine(arcDir,investigationFileName)    
             Investigation.toFile investigationFilePath investigation
 
+    /// Creates an investigation file in the ARC.
+    let overWrite (arcDir : string) (investigation : ISADotNet.Investigation) =
+
+        let log = Logging.createLogger "InvestigationWriteLog"
+        log.Info("Start Investigation Write")
+
+        let investigationFilePath = Path.Combine(arcDir,investigationFileName)
+
+        if System.IO.File.Exists(investigationFilePath) then
+            try 
+                let cache = File.ReadAllBytes(investigationFilePath)
+                File.Delete(investigationFilePath)
+                try                     
+                    Investigation.toFile investigationFilePath investigation
+                with
+                | err -> 
+                    File.WriteAllBytes(investigationFilePath,cache)
+                    log.Error($"Investigation file could not be overwritten: {err.Message}")
+            with 
+            | err -> 
+                log.Error($"Investigation file could not be overwritten: {err.Message}")
+        else 
+            Investigation.toFile investigationFilePath investigation
+
     /// Reads an investigation from the ARC.
     let read (arcDir : string) =
 
@@ -37,6 +61,7 @@ module Investigation =
             log.Error("Investigation file does not exist.")
             raise (System.SystemException())
 
+    /// Reads and combines all ISA components of the ARC into the ISA object
     let fromArcFolder (arcDir : string) =
         let log = Logging.createLogger "InvestigationFromArcFolderLog"
 
@@ -52,6 +77,7 @@ module Investigation =
                 match study.Identifier with
                 | Some id ->
                     let studyFromFile = Study.readByIdentifier arcDir id
+                    let mergedStudy = API.Update.UpdateByExistingAppendLists.updateRecordType study studyFromFile
                     // update study assays and contacts with information from assay files
                     match study.Assays with
                     | Some assays ->
@@ -65,13 +91,13 @@ module Investigation =
                                 | None ->
                                     log.Warn("Study \'" + id + "\' contains Assay without filename.")
                                     cl, al @ [assay]
-                            ) (studyFromFile.Contacts |> Option.defaultValue [],[])
-                        {studyFromFile with                        
+                            ) (mergedStudy.Contacts |> Option.defaultValue [],[])
+                        {mergedStudy with
                             Contacts = Some (scontacts |> List.distinct)
                             Assays = Some sassays 
                         }
                     | None -> 
-                        studyFromFile
+                        mergedStudy
                 | None ->
                     log.Warn("Investigation file contains study without identifier.")
                     study
@@ -106,7 +132,9 @@ module Investigation =
 
         // fill investigation with information from study files and assay files
         {i with Studies = istudies'}
+        |> API.Investigation.update
 
+    /// Registers an assay to the investigation arc registry
     let registerAssay arcDir studyName (assayName) =
 
         let log = Logging.createLogger "RegisterAssayLog"
@@ -142,4 +170,9 @@ module Investigation =
             Study.init arcDir study
             [study]
         |> API.Investigation.setStudies investigation
-        |> write arcDir
+        |> write arcDir
+
+    /// Update investigation file with information from the different ISA components of the ARC
+    let updateRegistry arcDir =
+        fromArcFolder arcDir
+        |> overWrite arcDir

src/arcIO.NET/Study.fs

@@ -16,11 +16,36 @@ module Study =
 
     module StudyFolder =
 
-        /// Checks if an study folder exists in the ARC.
+        /// Checks if a Study folder exists in the ARC.
         let exists (arc : string) (identifier : string) =
             Path.Combine([|arc;rootFolderName;identifier|])
             |> System.IO.Directory.Exists
 
+        /// Returns the Study identifiers of the Study Files located in each Study's folder of a given path to an ARC.
+        let findStudyIdentifiers arcDir =
+            let log = Logging.createLogger "findStudyIdentifiersLog"
+            let studiesPath = Path.Combine(arcDir, rootFolderName)
+            try
+                let studyFolders = Directory.GetDirectories studiesPath
+                studyFolders
+                |> Array.collect (
+                    fun sf ->
+                        let studyIdentifier = (DirectoryInfo sf).Name
+                        Directory.GetFiles sf
+                        |> Array.choose (fun s -> 
+                            if s.EndsWith "isa.study.xlsx" then
+                                Some studyIdentifier
+                            elif s.EndsWith "_isa.study.xlsx" then
+                                log.Warn $"The Study File of Study {studyIdentifier} has a deprecated File Name: {s}"
+                                Some studyIdentifier
+                            else 
+                                None
+                        )
+                )
+            with e -> 
+                log.Error e.Message
+                [||]
+
     let readFromFolder (arc : string) (folderPath : string) =
         let sp = Path.Combine(folderPath,studyFileName).Replace(@"\","/")
         let study = StudyFile.Study.fromFile sp
@@ -50,16 +75,89 @@ module Study =
         Path.Combine ([|arc;rootFolderName;studyIdentifier|])
         |> readFromFolder arc
 
+    /// Writes a study to the given folder. Fails, if the file already exists
     let writeToFolder (folderPath : string) (study : Study) =
-        let sp = Path.Combine (folderPath,studyFileName)
-        StudyFile.Study.toFile sp study        
 
+        let log = Logging.createLogger "StudyWriteLog"
+
+        log.Info("Start Study Write")
+
+        let studyFilePath = Path.Combine (folderPath,studyFileName)
+
+        if System.IO.File.Exists(studyFilePath) then
+            log.Error("Study file does already exist.")
+
+        else 
+            StudyFile.Study.toFile studyFilePath study 
+
+    /// Writes a study to the given folder. Overwrites it, if the file already exists
+    let overWriteToFolder (folderPath : string) (study : Study) =
+
+        let log = Logging.createLogger "StudyWriteLog"
+        log.Info("Start Study Write")
+
+        let studyFilePath = Path.Combine (folderPath,studyFileName)
+
+        if System.IO.File.Exists(studyFilePath) then
+            try 
+                let cache = File.ReadAllBytes(studyFilePath)
+                File.Delete(studyFilePath)
+                try                     
+                    StudyFile.Study.toFile studyFilePath study 
+                with
+                | err -> 
+                    File.WriteAllBytes(studyFilePath,cache)
+                    log.Error($"Study file could not be overwritten: {err.Message}")
+            with 
+            | err -> 
+                log.Error($"Study file could not be overwritten: {err.Message}")
+        else 
+            StudyFile.Study.toFile studyFilePath study 
+
+    /// Writes a study to the arc. Fails, if the file already exists
     let write (arc : string) (study : Study) =
+
+        let log = Logging.createLogger "StudyWriteLog"
+
+        log.Info("Start Study Write")
+
         if study.FileName.IsNone then
-            failwith "Cannot write study to arc, as it has no filename"
-        let sp = Path.Combine ([|arc;rootFolderName;study.FileName.Value|])
-        StudyFile.Study.toFile sp study    
+            log.Error("Cannot write study to arc, as it has no filename")
+        else 
+
+            let studyFilePath = Path.Combine ([|arc;rootFolderName;study.FileName.Value|])
+
+            if System.IO.File.Exists(studyFilePath) then
+                log.Error("Study file does already exist.")
 
+            else 
+                StudyFile.Study.toFile studyFilePath study 
+
+    /// Writes a study to the arc. Overwrites it, if the file already exists
+    let overWrite (arc : string) (study : Study) =
+        let log = Logging.createLogger "StudyWriteLog"
+
+        log.Info("Start Study Write")
+
+        if study.FileName.IsNone then
+            log.Error("Cannot write study to arc, as it has no filename")
+        else 
+            let studyFilePath = Path.Combine ([|arc;rootFolderName;study.FileName.Value|])
+            if System.IO.File.Exists(studyFilePath) then
+                try 
+                    let cache = File.ReadAllBytes(studyFilePath)
+                    File.Delete(studyFilePath)
+                    try                     
+                        StudyFile.Study.toFile studyFilePath study 
+                    with
+                    | err -> 
+                        File.WriteAllBytes(studyFilePath,cache)
+                        log.Error($"Study file could not be overwritten: {err.Message}")
+                with 
+                | err -> 
+                    log.Error($"Study file could not be overwritten: {err.Message}")
+            else 
+                StudyFile.Study.toFile studyFilePath study 
 
     let init (arc : string) (study : Study) =
 
@@ -89,4 +187,59 @@ module Study =
 
         let study = Study.create(FileName = studyFileName, Identifier = studyName)
 
-        init arc study
+        init arc study
+
+    /// Takes the path to an ARC and lists all study identifiers registered in this ARC's investigation file.
+    let list (arcDir : string) =
+
+        let log = Logging.createLogger "StudyListLog"
+
+        log.Info("Start Study List")
+
+        (* the following part is _not nice_: The functionality for this already exists in Investigation.fs. Unfortunately, Investigation.fs is compiled *AFTER*
+        Study.fs and, thus, we cannot use functions from there. Moving Study.fs after Investigation.fs in compilation order also does not work due to 
+        Investigation.fs using functions from Study.fs.
+        If someone finds a solution for this F#-specific compilation problem, feel free to fix :) *)
+        let investigationFilePath = Path.Combine(arcDir, "isa.investigation.xlsx")
+        log.Trace($"InvestigationFile: {investigationFilePath}")
+        let investigation = Investigation.fromFile investigationFilePath
+        // end of part
+
+        let studyFileIdentifiers = set (StudyFolder.findStudyIdentifiers arcDir)
+
+        let studyIdentifiers = 
+            investigation.Studies
+            |> Option.defaultValue []
+            |> List.choose (fun s -> 
+                match s.Identifier with
+                | None | Some "" -> 
+                    log.Warn("Study does not have identifier")
+                    None
+                | Some i -> Some i
+            ) 
+            |> set
+
+        let onlyRegistered = Set.difference studyIdentifiers studyFileIdentifiers
+        let onlyInitialized = Set.difference studyFileIdentifiers studyIdentifiers
+        let combined = Set.union studyIdentifiers studyFileIdentifiers
+
+        if not onlyRegistered.IsEmpty then
+            log.Warn("The ARC contains following registered studies that have no associated file:")
+            onlyRegistered
+            |> Seq.iter ((sprintf "%s") >> log.Warn) 
+            log.Info($"You can init the study file using \"arc s init\"")
+
+        if not onlyInitialized.IsEmpty then
+            log.Warn("The ARC contains study files with the following identifiers not registered in the investigation:")
+            onlyInitialized
+            |> Seq.iter ((sprintf "%s") >> log.Warn) 
+            log.Info($"You can register the study using \"arc s register\"")
+
+        if combined.IsEmpty then
+            log.Error("The ARC does not contain any studies.")
+
+        combined
+        |> Seq.map (fun identifier ->
+            //log.Debug(sprintf "Study: %s" identifier)
+            sprintf "Study: %s" identifier
+        )

src/arcIO.NET/arcIO.NET.fsproj

@@ -16,15 +16,17 @@
     <Compile Include="Study.fs" />
     <Compile Include="Investigation.fs" />
     <Compile Include="Arc.fs" />
+    <Compile Include="Converter.fs" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Update="FSharp.Core" Version="7.0.0" />
+    <PackageReference Update="FSharp.Core" Version="7.0.300-beta.23114.3" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="ISADotNet.XLSX" Version="0.7.0-preview.5" />
-    <PackageReference Include="NLog" Version="5.1.1" />
+    <PackageReference Include="ISADotNet.XLSX" Version="0.7.0-preview.8" />
+    <PackageReference Include="LitXml" Version="0.1.0" />
+    <PackageReference Include="NLog" Version="5.1.2" />
   </ItemGroup>
 
   <PropertyGroup>