From 5a99e71f92b6fcc212fdd8814c40c21214f38563 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 28 Jul 2020 14:42:20 -0400 Subject: [PATCH 1/2] Feat/model updates (#91) * update gql schema types to include Long scalar * update index model to reflect changes in relay * update converter and other models * uncomment future things * add fields to gql schema --- .../search/index/model/TaskDocument.java | 38 ++++++++++++++---- .../search/index/model/WorkflowDocument.java | 2 +- .../workflow/search/model/graphql/Run.java | 2 +- .../workflow/search/model/graphql/Task.java | 34 ++++++++++++++-- .../workflow/search/model/wes/RunLog.java | 7 ++-- .../workflow/search/model/wes/TaskLog.java | 39 ++++++++++++++++--- .../workflow/search/util/Converter.java | 24 +++++++----- src/main/resources/schema.graphqls | 15 +++++-- .../search/service/ConverterTest.java | 28 +++++++++++-- 9 files changed, 152 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/icgc_argo/workflow/search/index/model/TaskDocument.java b/src/main/java/org/icgc_argo/workflow/search/index/model/TaskDocument.java index 0dd11ee..ffd3de9 100644 --- a/src/main/java/org/icgc_argo/workflow/search/index/model/TaskDocument.java +++ b/src/main/java/org/icgc_argo/workflow/search/index/model/TaskDocument.java @@ -75,15 +75,39 @@ public class TaskDocument { /** Task filesystem working directory */ @NonNull private String workdir; - /** Task cpu usage */ + /** The cpus number request for the task execution */ private Integer cpus; - /** Task memory usage */ - private Integer memory; + /** The memory request for the task execution */ + private Long memory; - /** Task duration (ms) */ - private Integer duration; + /** Time elapsed to complete since the submission */ + private Long duration; - /** Task real execution time (ms) */ - private Integer realtime; + /** Task execution time i.e. delta between completion and start timestamp */ + private Long realtime; + + /** Real memory (resident set) size of the process. Equivalent to `ps -o rss` */ + private Long rss; + + /** Peak of real memory. This data is read from field `VmHWM` in `/proc/$pid/status file` */ + private Long peakRss; + + /** Virtual memory size of the process. Equivalent to `ps -o vsize` */ + private Long vmem; + + /** Peak of virtual memory. This data is read from field `VmPeak` in `/proc/$pid/status file` */ + private Long peakVmem; + + /** + * Number of bytes the process directly read from disk. This data is read from file + * `/proc/$pid/io` + */ + private Long readBytes; + + /** + * Number of bytes the process originally dirtied in the page-cache (assuming they will go to disk + * later). This data is read from file `/proc/$pid/io` + */ + private Long writeBytes; } diff --git a/src/main/java/org/icgc_argo/workflow/search/index/model/WorkflowDocument.java b/src/main/java/org/icgc_argo/workflow/search/index/model/WorkflowDocument.java index e47721e..8084cc8 100644 --- a/src/main/java/org/icgc_argo/workflow/search/index/model/WorkflowDocument.java +++ b/src/main/java/org/icgc_argo/workflow/search/index/model/WorkflowDocument.java @@ -64,7 +64,7 @@ public class WorkflowDocument { private String errorReport; /** Workflow duration */ - private Integer duration; + private Long duration; /** The command line that was executed */ @NonNull private String commandLine; diff --git a/src/main/java/org/icgc_argo/workflow/search/model/graphql/Run.java b/src/main/java/org/icgc_argo/workflow/search/model/graphql/Run.java index 17a5b4b..ceef43f 100644 --- a/src/main/java/org/icgc_argo/workflow/search/model/graphql/Run.java +++ b/src/main/java/org/icgc_argo/workflow/search/model/graphql/Run.java @@ -54,7 +54,7 @@ public class Run { private String errorReport; - private String duration; + private Long duration; private String commandLine; diff --git a/src/main/java/org/icgc_argo/workflow/search/model/graphql/Task.java b/src/main/java/org/icgc_argo/workflow/search/model/graphql/Task.java index 4fadac2..9db2d8d 100644 --- a/src/main/java/org/icgc_argo/workflow/search/model/graphql/Task.java +++ b/src/main/java/org/icgc_argo/workflow/search/model/graphql/Task.java @@ -61,13 +61,41 @@ public class Task { private String workdir; + /** The cpus number request for the task execution */ private Integer cpus; - private Integer memory; + /** The memory request for the task execution */ + private Long memory; - private Integer duration; + /** Time elapsed to complete since the submission */ + private Long duration; - private Integer realtime; + /** Task execution time i.e. delta between completion and start timestamp */ + private Long realtime; + + /** Real memory (resident set) size of the process. Equivalent to `ps -o rss` */ + private Long rss; + + /** Peak of real memory. This data is read from field `VmHWM` in `/proc/$pid/status file` */ + private Long peakRss; + + /** Virtual memory size of the process. Equivalent to `ps -o vsize` */ + private Long vmem; + + /** Peak of virtual memory. This data is read from field `VmPeak` in `/proc/$pid/status file` */ + private Long peakVmem; + + /** + * Number of bytes the process directly read from disk. This data is read from file + * `/proc/$pid/io` + */ + private Long readBytes; + + /** + * Number of bytes the process originally dirtied in the page-cache (assuming they will go to disk + * later). This data is read from file `/proc/$pid/io` + */ + private Long writeBytes; @SneakyThrows public static Task parse(@NonNull Map sourceMap) { diff --git a/src/main/java/org/icgc_argo/workflow/search/model/wes/RunLog.java b/src/main/java/org/icgc_argo/workflow/search/model/wes/RunLog.java index fe7916c..668b8c5 100644 --- a/src/main/java/org/icgc_argo/workflow/search/model/wes/RunLog.java +++ b/src/main/java/org/icgc_argo/workflow/search/model/wes/RunLog.java @@ -21,10 +21,11 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; import io.swagger.annotations.ApiModel; -import java.util.List; -import javax.validation.Valid; import lombok.*; +import javax.validation.Valid; +import java.util.List; + /** Log and other info */ @ApiModel(description = "Log and other info") @Data @@ -52,5 +53,5 @@ public class RunLog { private Boolean success; - private Integer duration; + private Long duration; } diff --git a/src/main/java/org/icgc_argo/workflow/search/model/wes/TaskLog.java b/src/main/java/org/icgc_argo/workflow/search/model/wes/TaskLog.java index b0a4513..44fdf79 100644 --- a/src/main/java/org/icgc_argo/workflow/search/model/wes/TaskLog.java +++ b/src/main/java/org/icgc_argo/workflow/search/model/wes/TaskLog.java @@ -21,10 +21,11 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; import io.swagger.annotations.ApiModel; -import java.util.List; -import javax.validation.Valid; import lombok.*; +import javax.validation.Valid; +import java.util.List; + /** Log and other info */ @ApiModel(description = "Log and other info") @Data @@ -66,11 +67,39 @@ public class TaskLog { private String workdir; + /** The cpus number request for the task execution */ private Integer cpus; - private Integer memory; + /** The memory request for the task execution */ + private Long memory; + + /** Time elapsed to complete since the submission */ + private Long duration; + + /** Task execution time i.e. delta between completion and start timestamp */ + private Long realtime; + + /** Real memory (resident set) size of the process. Equivalent to `ps -o rss` */ + private Long rss; + + /** Peak of real memory. This data is read from field `VmHWM` in `/proc/$pid/status file` */ + private Long peakRss; + + /** Virtual memory size of the process. Equivalent to `ps -o vsize` */ + private Long vmem; + + /** Peak of virtual memory. This data is read from field `VmPeak` in `/proc/$pid/status file` */ + private Long peakVmem; - private Integer duration; + /** + * Number of bytes the process directly read from disk. This data is read from file + * `/proc/$pid/io` + */ + private Long readBytes; - private Integer realtime; + /** + * Number of bytes the process originally dirtied in the page-cache (assuming they will go to disk + * later). This data is read from file `/proc/$pid/io` + */ + private Long writeBytes; } diff --git a/src/main/java/org/icgc_argo/workflow/search/util/Converter.java b/src/main/java/org/icgc_argo/workflow/search/util/Converter.java index e6aea83..91da635 100644 --- a/src/main/java/org/icgc_argo/workflow/search/util/Converter.java +++ b/src/main/java/org/icgc_argo/workflow/search/util/Converter.java @@ -18,17 +18,9 @@ package org.icgc_argo.workflow.search.util; -import static org.icgc_argo.workflow.search.model.SearchFields.RUN_ID; -import static org.icgc_argo.workflow.search.model.SearchFields.STATE; -import static org.icgc_argo.workflow.search.model.wes.State.fromValue; - import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import java.time.Instant; -import java.util.Arrays; -import java.util.List; -import java.util.Map; import lombok.NonNull; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; @@ -39,6 +31,15 @@ import org.icgc_argo.workflow.search.index.model.WorkflowDocument; import org.icgc_argo.workflow.search.model.wes.*; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.icgc_argo.workflow.search.model.SearchFields.RUN_ID; +import static org.icgc_argo.workflow.search.model.SearchFields.STATE; +import static org.icgc_argo.workflow.search.model.wes.State.fromValue; + @Slf4j @UtilityClass public class Converter { @@ -73,7 +74,12 @@ public static TaskLog taskDocumentToLog(@NonNull TaskDocument task) { .memory(task.getMemory()) .duration(task.getDuration()) .realtime(task.getRealtime()) - // todo ticket #24 workflow-relay + .rss(task.getRss()) + .peakRss(task.getPeakRss()) + .vmem(task.getVmem()) + .peakVmem(task.getPeakVmem()) + .readBytes(task.getReadBytes()) + .writeBytes(task.getWriteBytes()) .stderr("") .stdout("") .build(); diff --git a/src/main/resources/schema.graphqls b/src/main/resources/schema.graphqls index bd3b270..df03905 100644 --- a/src/main/resources/schema.graphqls +++ b/src/main/resources/schema.graphqls @@ -1,4 +1,5 @@ scalar JSON +scalar Long type Run @key(fields: "runId") { runId: ID! @@ -11,7 +12,7 @@ type Run @key(fields: "runId") { success: Boolean exitStatus: Int errorReport: String - duration: String + duration: Long commandLine: String engineParameters: EngineParameters tasks(taskId: String, state: String, tag: String): [Task] @@ -42,9 +43,15 @@ type Task { script: String workdir: String cpus: Int - memory: Int - duration: Int - realtime: Int + memory: Long + duration: Long + realtime: Long + rss: Long + peakRss: Long + vmem: Long + peakVmem: Long + readBytes: Long + writeBytes: Long run: Run } diff --git a/src/test/java/org/icgc_argo/workflow/search/service/ConverterTest.java b/src/test/java/org/icgc_argo/workflow/search/service/ConverterTest.java index 049426d..9376a52 100644 --- a/src/test/java/org/icgc_argo/workflow/search/service/ConverterTest.java +++ b/src/test/java/org/icgc_argo/workflow/search/service/ConverterTest.java @@ -54,9 +54,17 @@ public class ConverterTest { private static final Integer TASK_ATTEMPT = 1; private static final String TASK_WORKDIR = "/this/is/dir"; private static final Integer TASK_CPUS = 4; - private static final Integer TASK_MEM = 1024; - private static final Integer TASK_DURATION = 2000; - private static final Integer TASK_REALTIME = 1098; + private static final Long TASK_MEM = 1024L; + private static final Long TASK_DURATION = 2000L; + private static final Long TASK_REALTIME = 1098L; + private static final Long TASK_RSS = 1234L; + private static final Long TASK_PEAK_RSS = 4567L; + private static final Long TASK_VMEM = 8901L; + private static final Long TASK_PEAK_VMEM = 2345L; + private static final Long TASK_READ_BYTES = 6789L; + private static final Long TASK_WRITE_BYTES = 10123L; + + @Test public void TestConvertSourceMapToRunStatus() { @@ -90,6 +98,12 @@ public void testTaskDocumentToLog() { .memory(TASK_MEM) .duration(TASK_DURATION) .realtime(TASK_REALTIME) + .rss(TASK_RSS) + .peakRss(TASK_PEAK_RSS) + .vmem(TASK_VMEM) + .peakVmem(TASK_PEAK_VMEM) + .readBytes(TASK_READ_BYTES) + .writeBytes(TASK_WRITE_BYTES) .build(); val log = Converter.taskDocumentToLog(taskDocument); @@ -113,6 +127,12 @@ public void testTaskDocumentToLog() { assertEquals(log.getMemory(), taskDocument.getMemory()); assertEquals(log.getDuration(), taskDocument.getDuration()); assertEquals(log.getRealtime(), taskDocument.getRealtime()); + assertEquals(log.getRss(), taskDocument.getRss()); + assertEquals(log.getPeakRss(), taskDocument.getPeakRss()); + assertEquals(log.getVmem(), taskDocument.getVmem()); + assertEquals(log.getPeakVmem(), taskDocument.getPeakVmem()); + assertEquals(log.getReadBytes(), taskDocument.getReadBytes()); + assertEquals(log.getWriteBytes(), taskDocument.getWriteBytes()); } @Test @@ -167,7 +187,7 @@ private WorkflowDocument buildWorkflowDoc() { .parameters(params) .engineParameters(engineParams) .success(true) - .duration(1000) + .duration(1000L) .build(); } } From 055bc6ae692a6efe8a60d0d8909d33ccd81aa638 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 28 Jul 2020 14:44:16 -0400 Subject: [PATCH 2/2] Release 2.2.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eb121a0..89ded9f 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.icgc_argo workflow-search - 2.0.2-SNAPSHOT + 2.2.0 workflow-search Demo project for Spring Boot