Add extra metadata to workflow runs exported files (#364)

* extend workflow export format with virtual fields * runs dump cmd, extend workflow export fields * use tower's object mapper for virtual extension formats * clean json nodes handling in export formats * export workflow metadata to separate file * reflection files * comment for user info filtering * add workspace name to metadata * add workflow run URL to metadata * add workflow labels to metadata * fix reflection files * remove user email from metadata
seqeralabs · Dec 5, 2023 · 3e14dca · 3e14dca
1 parent 48f4afa
commit 3e14dca
Show file tree

Hide file tree

Showing 6 changed files with 244 additions and 21 deletions.
diff --git a/conf/reflect-config.json b/conf/reflect-config.json
@@ -1969,6 +1969,13 @@
   "name":"io.seqera.tower.cli.shared.ComputeEnvExportFormat$ComputeConfigMixin",
   "queryAllDeclaredMethods":true
 },
+{
+  "name":"io.seqera.tower.cli.shared.WorkflowMetadata",
+  "allDeclaredFields":true,
+  "queryAllDeclaredMethods":true,
+  "queryAllDeclaredConstructors":true,
+  "methods":[{"name":"getLabels","parameterTypes":[] }, {"name":"getPipelineId","parameterTypes":[] }, {"name":"getRunUrl","parameterTypes":[] }, {"name":"getUserId","parameterTypes":[] }, {"name":"getWorkspaceId","parameterTypes":[] }, {"name":"getWorkspaceName","parameterTypes":[] }]
+},
 {
   "name":"io.seqera.tower.cli.utils.VersionProvider",
   "allDeclaredFields":true,
@@ -2150,13 +2157,13 @@
   "allDeclaredFields":true,
   "allDeclaredMethods":true,
   "allDeclaredConstructors":true,
-  "methods":[{"name":"getConfig","parameterTypes":[] }, {"name":"getCredentialsId","parameterTypes":[] }, {"name":"getDateCreated","parameterTypes":[] }, {"name":"getDeleted","parameterTypes":[] }, {"name":"getDescription","parameterTypes":[] }, {"name":"getId","parameterTypes":[] }, {"name":"getLastUpdated","parameterTypes":[] }, {"name":"getLastUsed","parameterTypes":[] }, {"name":"getMessage","parameterTypes":[] }, {"name":"getName","parameterTypes":[] }, {"name":"getOrgId","parameterTypes":[] }, {"name":"getPlatform","parameterTypes":[] }, {"name":"getPrimary","parameterTypes":[] }, {"name":"getStatus","parameterTypes":[] }, {"name":"getWorkspaceId","parameterTypes":[] }]
+  "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"getConfig","parameterTypes":[] }, {"name":"getCredentialsId","parameterTypes":[] }, {"name":"getDateCreated","parameterTypes":[] }, {"name":"getDeleted","parameterTypes":[] }, {"name":"getDescription","parameterTypes":[] }, {"name":"getId","parameterTypes":[] }, {"name":"getLastUpdated","parameterTypes":[] }, {"name":"getLastUsed","parameterTypes":[] }, {"name":"getMessage","parameterTypes":[] }, {"name":"getName","parameterTypes":[] }, {"name":"getOrgId","parameterTypes":[] }, {"name":"getPlatform","parameterTypes":[] }, {"name":"getPrimary","parameterTypes":[] }, {"name":"getStatus","parameterTypes":[] }, {"name":"getWorkspaceId","parameterTypes":[] }, {"name":"setConfig","parameterTypes":["io.seqera.tower.model.ComputeConfig"] }, {"name":"setCredentialsId","parameterTypes":["java.lang.String"] }, {"name":"setDescription","parameterTypes":["java.lang.String"] }, {"name":"setMessage","parameterTypes":["java.lang.String"] }, {"name":"setName","parameterTypes":["java.lang.String"] }, {"name":"setPlatform","parameterTypes":["io.seqera.tower.model.ComputeEnv$PlatformEnum"] }, {"name":"setStatus","parameterTypes":["io.seqera.tower.model.ComputeEnvStatus"] }]
 },
 {
   "name":"io.seqera.tower.model.ComputeEnv$PlatformEnum",
   "allDeclaredFields":true,
   "allDeclaredMethods":true,
-  "methods":[{"name":"getValue","parameterTypes":[] }]
+  "methods":[{"name":"fromValue","parameterTypes":["java.lang.String"] }, {"name":"getValue","parameterTypes":[] }]
 },
 {
   "name":"io.seqera.tower.model.ComputeEnvDbDto",
@@ -2181,13 +2188,14 @@
   "name":"io.seqera.tower.model.ComputeEnvStatus",
   "allDeclaredFields":true,
   "allDeclaredMethods":true,
-  "methods":[{"name":"fromValue","parameterTypes":["java.lang.String"] }]
+  "methods":[{"name":"fromValue","parameterTypes":["java.lang.String"] }, {"name":"getValue","parameterTypes":[] }]
 },
 {
   "name":"io.seqera.tower.model.ComputePlatformDto",
   "allDeclaredFields":true,
   "allDeclaredMethods":true,
-  "allDeclaredConstructors":true
+  "allDeclaredConstructors":true,
+  "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setId","parameterTypes":["java.lang.String"] }, {"name":"setName","parameterTypes":["java.lang.String"] }]
 },
 {
   "name":"io.seqera.tower.model.ConfigEnvVariable",
@@ -2395,7 +2403,8 @@
   "name":"io.seqera.tower.model.DescribeLaunchResponse",
   "allDeclaredFields":true,
   "allDeclaredMethods":true,
-  "allDeclaredConstructors":true
+  "allDeclaredConstructors":true,
+  "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setLaunch","parameterTypes":["io.seqera.tower.model.Launch"] }]
 },
 {
   "name":"io.seqera.tower.model.DescribeOrganizationResponse",
@@ -2547,7 +2556,8 @@
   "name":"io.seqera.tower.model.JobInfoDto",
   "allDeclaredFields":true,
   "allDeclaredMethods":true,
-  "allDeclaredConstructors":true
+  "allDeclaredConstructors":true,
+  "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setExitCode","parameterTypes":["java.lang.Integer"] }, {"name":"setId","parameterTypes":["java.lang.Long"] }, {"name":"setMessage","parameterTypes":["java.lang.String"] }, {"name":"setOperationId","parameterTypes":["java.lang.String"] }, {"name":"setStatus","parameterTypes":["java.lang.String"] }]
 },
 {
   "name":"io.seqera.tower.model.K8sComputeConfig",
@@ -2578,7 +2588,8 @@
   "name":"io.seqera.tower.model.Launch",
   "allDeclaredFields":true,
   "allDeclaredMethods":true,
-  "allDeclaredConstructors":true
+  "allDeclaredConstructors":true,
+  "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"getComputeEnv_JsonNullable","parameterTypes":[] }, {"name":"getConfigProfiles","parameterTypes":[] }, {"name":"getConfigText","parameterTypes":[] }, {"name":"getDateCreated","parameterTypes":[] }, {"name":"getEntryName","parameterTypes":[] }, {"name":"getHeadJobCpus","parameterTypes":[] }, {"name":"getHeadJobMemoryMb","parameterTypes":[] }, {"name":"getId","parameterTypes":[] }, {"name":"getLastUpdated","parameterTypes":[] }, {"name":"getMainScript","parameterTypes":[] }, {"name":"getOptimizationId","parameterTypes":[] }, {"name":"getOptimizationTargets","parameterTypes":[] }, {"name":"getParamsText","parameterTypes":[] }, {"name":"getPipeline","parameterTypes":[] }, {"name":"getPostRunScript","parameterTypes":[] }, {"name":"getPreRunScript","parameterTypes":[] }, {"name":"getPullLatest","parameterTypes":[] }, {"name":"getResume","parameterTypes":[] }, {"name":"getResumeLaunchId","parameterTypes":[] }, {"name":"getRevision","parameterTypes":[] }, {"name":"getRunName","parameterTypes":[] }, {"name":"getSchemaName","parameterTypes":[] }, {"name":"getSessionId","parameterTypes":[] }, {"name":"getStubRun","parameterTypes":[] }, {"name":"getTowerConfig","parameterTypes":[] }, {"name":"getUserSecrets","parameterTypes":[] }, {"name":"getWorkDir","parameterTypes":[] }, {"name":"getWorkspaceSecrets","parameterTypes":[] }, {"name":"setComputeEnv_JsonNullable","parameterTypes":["org.openapitools.jackson.nullable.JsonNullable"] }, {"name":"setConfigProfiles","parameterTypes":["java.util.List"] }, {"name":"setConfigText","parameterTypes":["java.lang.String"] }, {"name":"setDateCreated","parameterTypes":["java.time.OffsetDateTime"] }, {"name":"setEntryName","parameterTypes":["java.lang.String"] }, {"name":"setHeadJobCpus","parameterTypes":["java.lang.Integer"] }, {"name":"setHeadJobMemoryMb","parameterTypes":["java.lang.Integer"] }, {"name":"setId","parameterTypes":["java.lang.String"] }, {"name":"setLastUpdated","parameterTypes":["java.time.OffsetDateTime"] }, {"name":"setMainScript","parameterTypes":["java.lang.String"] }, {"name":"setOptimizationId","parameterTypes":["java.lang.String"] }, {"name":"setOptimizationTargets","parameterTypes":["java.lang.String"] }, {"name":"setParamsText","parameterTypes":["java.lang.String"] }, {"name":"setPipeline","parameterTypes":["java.lang.String"] }, {"name":"setPostRunScript","parameterTypes":["java.lang.String"] }, {"name":"setPreRunScript","parameterTypes":["java.lang.String"] }, {"name":"setPullLatest","parameterTypes":["java.lang.Boolean"] }, {"name":"setResume","parameterTypes":["java.lang.Boolean"] }, {"name":"setResumeLaunchId","parameterTypes":["java.lang.String"] }, {"name":"setRevision","parameterTypes":["java.lang.String"] }, {"name":"setRunName","parameterTypes":["java.lang.String"] }, {"name":"setSchemaName","parameterTypes":["java.lang.String"] }, {"name":"setSessionId","parameterTypes":["java.lang.String"] }, {"name":"setStubRun","parameterTypes":["java.lang.Boolean"] }, {"name":"setTowerConfig","parameterTypes":["java.lang.String"] }, {"name":"setUserSecrets","parameterTypes":["java.util.List"] }, {"name":"setWorkDir","parameterTypes":["java.lang.String"] }, {"name":"setWorkspaceSecrets","parameterTypes":["java.util.List"] }]
 },
 {
   "name":"io.seqera.tower.model.LaunchActionRequest",
@@ -2656,7 +2667,8 @@
   "name":"io.seqera.tower.model.ListParticipantsResponse",
   "allDeclaredFields":true,
   "allDeclaredMethods":true,
-  "allDeclaredConstructors":true
+  "allDeclaredConstructors":true,
+  "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setParticipants","parameterTypes":["java.util.List"] }, {"name":"setTotalSize","parameterTypes":["java.lang.Long"] }]
 },
 {
   "name":"io.seqera.tower.model.ListPipelineSecretsResponse",
@@ -2747,7 +2759,8 @@
 {
   "name":"io.seqera.tower.model.OrgRole",
   "allDeclaredFields":true,
-  "allDeclaredMethods":true
+  "allDeclaredMethods":true,
+  "methods":[{"name":"fromValue","parameterTypes":["java.lang.String"] }]
 },
 {
   "name":"io.seqera.tower.model.Organization",
@@ -2765,7 +2778,8 @@
   "name":"io.seqera.tower.model.ParticipantDbDto",
   "allDeclaredFields":true,
   "allDeclaredMethods":true,
-  "allDeclaredConstructors":true
+  "allDeclaredConstructors":true,
+  "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setEmail","parameterTypes":["java.lang.String"] }, {"name":"setFirstName","parameterTypes":["java.lang.String"] }, {"name":"setLastName","parameterTypes":["java.lang.String"] }, {"name":"setMemberId","parameterTypes":["java.lang.Long"] }, {"name":"setOrgRole","parameterTypes":["io.seqera.tower.model.OrgRole"] }, {"name":"setParticipantId","parameterTypes":["java.lang.Long"] }, {"name":"setTeamAvatarUrl","parameterTypes":["java.lang.String"] }, {"name":"setTeamId","parameterTypes":["java.lang.Long"] }, {"name":"setTeamName","parameterTypes":["java.lang.String"] }, {"name":"setType","parameterTypes":["io.seqera.tower.model.ParticipantType"] }, {"name":"setUserAvatarUrl","parameterTypes":["java.lang.String"] }, {"name":"setUserName","parameterTypes":["java.lang.String"] }, {"name":"setWspRole","parameterTypes":["io.seqera.tower.model.WspRole"] }]
 },
 {
   "name":"io.seqera.tower.model.ParticipantType",
@@ -3042,7 +3056,7 @@
   "allDeclaredFields":true,
   "queryAllDeclaredMethods":true,
   "queryAllDeclaredConstructors":true,
-  "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setComputeEnv","parameterTypes":["io.seqera.tower.model.ComputeEnv"] }, {"name":"setConfigProfiles","parameterTypes":["java.util.List"] }, {"name":"setConfigText","parameterTypes":["java.lang.String"] }, {"name":"setDateCreated","parameterTypes":["java.time.OffsetDateTime"] }, {"name":"setEntryName","parameterTypes":["java.lang.String"] }, {"name":"setId","parameterTypes":["java.lang.String"] }, {"name":"setMainScript","parameterTypes":["java.lang.String"] }, {"name":"setParamsText","parameterTypes":["java.lang.String"] }, {"name":"setPipeline","parameterTypes":["java.lang.String"] }, {"name":"setPostRunScript","parameterTypes":["java.lang.String"] }, {"name":"setPreRunScript","parameterTypes":["java.lang.String"] }, {"name":"setPullLatest","parameterTypes":["java.lang.Boolean"] }, {"name":"setResume","parameterTypes":["java.lang.Boolean"] }, {"name":"setResumeCommitId","parameterTypes":["java.lang.String"] }, {"name":"setResumeDir","parameterTypes":["java.lang.String"] }, {"name":"setRevision","parameterTypes":["java.lang.String"] }, {"name":"setSchemaName","parameterTypes":["java.lang.String"] }, {"name":"setSessionId","parameterTypes":["java.lang.String"] }, {"name":"setStubRun","parameterTypes":["java.lang.Boolean"] }, {"name":"setTowerConfig","parameterTypes":["java.lang.String"] }, {"name":"setUserSecrets","parameterTypes":["java.util.List"] }, {"name":"setWorkDir","parameterTypes":["java.lang.String"] }, {"name":"setWorkspaceSecrets","parameterTypes":["java.util.List"] }]
+  "methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setComputeEnv","parameterTypes":["io.seqera.tower.model.ComputeEnv"] }, {"name":"setConfigProfiles","parameterTypes":["java.util.List"] }, {"name":"setConfigText","parameterTypes":["java.lang.String"] }, {"name":"setDateCreated","parameterTypes":["java.time.OffsetDateTime"] }, {"name":"setEntryName","parameterTypes":["java.lang.String"] }, {"name":"setHeadJobCpus","parameterTypes":["java.lang.Integer"] }, {"name":"setHeadJobMemoryMb","parameterTypes":["java.lang.Integer"] }, {"name":"setId","parameterTypes":["java.lang.String"] }, {"name":"setMainScript","parameterTypes":["java.lang.String"] }, {"name":"setOptimizationId","parameterTypes":["java.lang.String"] }, {"name":"setOptimizationTargets","parameterTypes":["java.lang.String"] }, {"name":"setParamsText","parameterTypes":["java.lang.String"] }, {"name":"setPipeline","parameterTypes":["java.lang.String"] }, {"name":"setPipelineId","parameterTypes":["java.lang.Long"] }, {"name":"setPostRunScript","parameterTypes":["java.lang.String"] }, {"name":"setPreRunScript","parameterTypes":["java.lang.String"] }, {"name":"setPullLatest","parameterTypes":["java.lang.Boolean"] }, {"name":"setResume","parameterTypes":["java.lang.Boolean"] }, {"name":"setResumeCommitId","parameterTypes":["java.lang.String"] }, {"name":"setResumeDir","parameterTypes":["java.lang.String"] }, {"name":"setRevision","parameterTypes":["java.lang.String"] }, {"name":"setSchemaName","parameterTypes":["java.lang.String"] }, {"name":"setSessionId","parameterTypes":["java.lang.String"] }, {"name":"setStubRun","parameterTypes":["java.lang.Boolean"] }, {"name":"setTowerConfig","parameterTypes":["java.lang.String"] }, {"name":"setUserSecrets","parameterTypes":["java.util.List"] }, {"name":"setWorkDir","parameterTypes":["java.lang.String"] }, {"name":"setWorkspaceSecrets","parameterTypes":["java.util.List"] }]
 },
 {
   "name":"io.seqera.tower.model.WorkflowLoad",
@@ -3073,7 +3087,8 @@
 {
   "name":"io.seqera.tower.model.WspRole",
   "allDeclaredFields":true,
-  "allDeclaredMethods":true
+  "allDeclaredMethods":true,
+  "methods":[{"name":"fromValue","parameterTypes":["java.lang.String"] }]
 },
 {
   "name":"java.io.Console",
@@ -3085,6 +3100,10 @@
   "allDeclaredMethods":true,
   "allDeclaredConstructors":true
 },
+{
+  "name":"java.io.FileNotFoundException",
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String"] }]
+},
 {
   "name":"java.io.IOException",
   "methods":[{"name":"<init>","parameterTypes":["java.lang.String"] }]

diff --git a/src/main/java/io/seqera/tower/cli/commands/runs/AbstractRunsCmd.java b/src/main/java/io/seqera/tower/cli/commands/runs/AbstractRunsCmd.java
@@ -23,6 +23,7 @@
 import io.seqera.tower.cli.exceptions.RunNotFoundException;
 import io.seqera.tower.cli.exceptions.WorkflowProgressNotFoundException;
 import io.seqera.tower.model.DescribeLaunchResponse;
+import io.seqera.tower.model.DescribeWorkflowLaunchResponse;
 import io.seqera.tower.model.DescribeWorkflowResponse;
 import io.seqera.tower.model.GetProgressResponse;
 import io.seqera.tower.model.Launch;
@@ -45,6 +46,16 @@ protected DescribeWorkflowResponse workflowById(Long workspaceId, String id) thr
         return workflowResponse;
     }
 
+    protected DescribeWorkflowLaunchResponse workflowLaunchById(Long workspaceId, String workflowId) throws ApiException {
+        DescribeWorkflowLaunchResponse wfLaunchResponse = api().describeWorkflowLaunch(workflowId, workspaceId);
+
+        if (wfLaunchResponse == null) {
+            throw new ApiException(String.format("Workflow '%s' launch not found at %d workspace", workflowId, workspaceId));
+        }
+
+        return wfLaunchResponse;
+    }
+
     protected Launch launchById(Long workspaceId, String id) throws ApiException {
         DescribeLaunchResponse launchResponse = api().describeLaunch(id, workspaceId);
 

diff --git a/src/main/java/io/seqera/tower/cli/commands/runs/DumpCmd.java b/src/main/java/io/seqera/tower/cli/commands/runs/DumpCmd.java
@@ -24,8 +24,10 @@
 import io.seqera.tower.cli.exceptions.TowerException;
 import io.seqera.tower.cli.responses.Response;
 import io.seqera.tower.cli.responses.runs.RunDump;
+import io.seqera.tower.cli.shared.WorkflowMetadata;
 import io.seqera.tower.cli.utils.SilentPrintWriter;
 import io.seqera.tower.model.DescribeTaskResponse;
+import io.seqera.tower.model.DescribeWorkflowLaunchResponse;
 import io.seqera.tower.model.DescribeWorkflowResponse;
 import io.seqera.tower.model.Launch;
 import io.seqera.tower.model.ListTasksResponse;
@@ -167,15 +169,41 @@ private void dumpTasks(PrintWriter progress, TarArchiveOutputStream out, Long ws
     private void dumpWorkflowDetails(PrintWriter progress, TarArchiveOutputStream out, Long wspId) throws ApiException, IOException {
         progress.println(ansi("- Workflow details"));
 
+        // General workflow info
         DescribeWorkflowResponse workflowResponse = workflowById(wspId, id);
         Workflow workflow = workflowResponse.getWorkflow();
         if (workflow == null) {
             throw new TowerException("Unknown workflow");
         }
+
+        // Launch info
+        Launch launch = null;
+        Long pipelineId = null;
+        if (workflow.getLaunchId() != null) {
+
+            launch = launchById(wspId, workflow.getLaunchId());
+
+            DescribeWorkflowLaunchResponse wfLaunchResponse = workflowLaunchById(wspId, workflow.getId());
+            if (wfLaunchResponse != null && wfLaunchResponse.getLaunch() != null) {
+                pipelineId = wfLaunchResponse.getLaunch().getPipelineId();
+            }
+        }
+
+        // Load and metrics info
         WorkflowLoad workflowLoad = workflowLoadByWorkflowId(wspId, id);
-        Launch launch = workflow.getLaunchId() != null ? launchById(wspId, workflow.getLaunchId()) : null;
         List<WorkflowMetrics> metrics = api().describeWorkflowMetrics(workflow.getId(), wspId).getMetrics();
+
+        WorkflowMetadata wfMetadata = new WorkflowMetadata(
+                pipelineId,
+                wspId,
+                workflowResponse.getWorkspaceName(),
+                workflow.getOwnerId(),
+                generateUrl(wspId, workflow.getUserName(), workflow.getId()),
+                workflowResponse.getLabels()
+        );
+
         addEntry(out, "workflow.json", Workflow.class, workflow);
+        addEntry(out, "workflow-metadata.json", WorkflowMetadata.class, wfMetadata);
         addEntry(out, "workflow-load.json", WorkflowLoad.class, workflowLoad);
         addEntry(out, "workflow-launch.json", Launch.class, launch);
         addEntry(out, "workflow-metrics.json", List.class, metrics);
@@ -285,8 +313,14 @@ private <T> void addEntry(TarArchiveOutputStream out, String fileName, Class<T>
         if (value == null) {
             return;
         }
+        addEntry(out, fileName, toJSON(type, value));
+    }
+
+    private void addEntry(TarArchiveOutputStream out, String fileName, byte[] data) throws IOException {
+        if (data == null) {
+            return;
+        }
         TarArchiveEntry entry = new TarArchiveEntry(fileName);
-        byte[] data = toJSON(type, value);
         entry.setSize(data.length);
         out.putArchiveEntry(entry);
         out.write(data);
@@ -306,5 +340,13 @@ private <T> byte[] toJSON(Class<T> type, T value) throws JsonProcessingException
                 .writerWithDefaultPrettyPrinter()
                 .writeValueAsBytes(value);
     }
+
+    private String generateUrl(Long wspId, String userName, String wfId) throws ApiException {
+        if (wspId == null) {
+            return String.format("%s/user/%s/watch/%s", serverUrl(), userName, wfId);
+        }
+        return String.format("%s/orgs/%s/workspaces/%s/watch/%s", serverUrl(), orgName(wspId), workspaceName(wspId), wfId);
+    }
+
 }