Add developer docs (#4065)

Signed-off-by: Ben Sherman <[email protected]>
Signed-off-by: Paolo Di Tommaso <[email protected]>
Co-authored-by: Paolo Di Tommaso <[email protected]>

docs/.dockerignore

@@ -1 +1,2 @@
+**
 !requirements.txt

docs/_static/theme.css

@@ -1,6 +1,18 @@
 
 /* navbar -- Seqera logo */
 
+.wy-side-nav-search {
+    background-color: #27ae60;
+}
+
+.wy-menu-vertical p.caption {
+    color: #27ae60;
+}
+
+.wy-menu-vertical a:active {
+    background-color: #27ae60;
+}
+
 .nav-footer-logo {
     width: 300px;
     margin-top: 2rem;

docs/developer/diagrams/.gitignore

@@ -0,0 +1 @@
+nextflow-merged.mmd

docs/developer/diagrams/README.md

@@ -0,0 +1,17 @@
+# Class Diagrams
+
+This directory contains class diagrams of the Nextflow source code, abridged and annotated for relevance and ease of use.
+
+Each node is a class. Fields are selectively documented in order to show only core data structures and the classes that "own" them. Methods are not explicitly documented, but they are mentioned in certain links where appropriate.
+
+Links between classes denote one of the following relationships:
+
+- Inheritance (`A <|-- B`): `B` is a subclass of `A`
+- Composition (`A --* B`): `A` contains `B`
+- Instantiation (`A --> B : f`): `A` creates instance(s) of `B` at runtime via `A::f()`
+
+Some links are commented out or not included at all, in order to focus on the most important classes and relationships. You can view these "hidden" links by simply uncommenting them, but I have found that their significance is sufficiently clear from the description files.
+
+A separate diagram description is provided for each package. These files are interoperable, which means that you can combine any subset of files into a larger diagram description. The `merge-diagrams.sh` can create a merged file for you automatically, and it includes a sensible default set of packages.
+
+You can use the [Mermaid Live Editor](https://mermaid-js.github.io/mermaid-live-editor/edit) or the [Mermaid CLI](https://github.com/mermaid-js/mermaid-cli) to render the diagram in a variety of image formats.

docs/developer/diagrams/merge-diagrams.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+packages=()
+packages+=("nextflow")
+# packages+=("nextflow.ast")
+packages+=("nextflow.cache")
+packages+=("nextflow.cli")
+# packages+=("nextflow.cloud.aws")
+# packages+=("nextflow.cloud.aws.nio")
+# packages+=("nextflow.cloud.azure")
+# packages+=("nextflow.cloud.google")
+# packages+=("nextflow.config")
+# packages+=("nextflow.container")
+packages+=("nextflow.dag")
+# packages+=("nextflow.executor")
+# packages+=("nextflow.extension")
+# packages+=("nextflow.ga4gh")
+# packages+=("nextflow.k8s")
+# packages+=("nextflow.plugin")
+packages+=("nextflow.processor")
+# packages+=("nextflow.scm")
+packages+=("nextflow.script")
+# packages+=("nextflow.secret")
+# packages+=("nextflow.trace")
+
+outfile="nextflow-merged.mmd"
+
+echo "classDiagram" > ${outfile}
+
+for package in "${packages[@]}"; do
+    echo "${package}"
+
+    tail -n +2 "${package}.mmd" >> ${outfile}
+    echo >> ${outfile}
+done

docs/developer/diagrams/nextflow.ast.mmd

@@ -0,0 +1,7 @@
+classDiagram
+    %%
+    %% nextflow.ast
+    %%
+    ScriptParser --> NextflowDSLImpl : parse
+    ScriptParser --> NextflowXformImpl : parse
+    ScriptParser --> OpXformImpl : parse

docs/developer/diagrams/nextflow.cache.mmd

@@ -0,0 +1,23 @@
+classDiagram
+    %%
+    %% nextflow.cache
+    %%
+    Session --* CacheDB
+    %% CacheFactory --> CacheDB : createInstance
+
+    CacheDB --* CacheStore
+
+    CacheStore <|-- DefaultCacheStore
+    CacheStore <|-- CloudCacheStore
+
+    class DefaultCacheStore {
+        uniqueId : UUID
+        runName : String
+        baseDir : Path
+    }
+
+    class CloudCacheStore {
+        uniqueId : UUID
+        runName : String
+        basePath : Path
+    }

docs/developer/diagrams/nextflow.cli.mmd

@@ -0,0 +1,29 @@
+classDiagram
+    %%
+    %% nextflow.cli
+    %%
+    class Launcher {
+        cliOptions : CliOptions
+        command : CmdBase
+    } 
+    Launcher --* CliOptions
+    Launcher --* CmdBase
+
+    %% CmdBase <|-- CmdClean
+    %% CmdBase <|-- CmdClone
+    %% CmdBase <|-- CmdConfig
+    CmdBase <|-- CmdConsole
+    %% CmdBase <|-- CmdDrop
+    %% CmdBase <|-- CmdFs
+    CmdBase <|-- CmdHelp
+    CmdBase <|-- CmdInfo
+    %% CmdBase <|-- CmdKubeRun
+    %% CmdBase <|-- CmdList
+    %% CmdBase <|-- CmdLog
+    %% CmdBase <|-- CmdNode
+    %% CmdBase <|-- CmdPlugin
+    %% CmdBase <|-- CmdPull
+    CmdBase <|-- CmdRun
+    %% CmdBase <|-- CmdSecret
+    %% CmdBase <|-- CmdSelfUpdate
+    %% CmdBase <|-- CmdView

docs/developer/diagrams/nextflow.cloud.aws.mmd

@@ -0,0 +1,21 @@
+classDiagram
+    %%
+    %% nextflow.cloud.aws
+    %%
+    Executor <|-- AwsBatchExecutor
+    TaskHandler <|-- AwsBatchTaskHandler
+    BashWrapperBuilder <|-- AwsBatchScriptLauncher
+
+    AwsBatchExecutor --* AwsOptions
+    AwsOptions --* AwsConfig
+    AwsConfig --* AwsBatchConfig
+    AwsConfig --* AwsS3Config
+
+    AwsBatchExecutor --> ParallelPollingMonitor : init
+    AwsBatchExecutor --> AwsBatchTaskHandler : submit
+    AwsBatchTaskHandler --> AwsBatchScriptLauncher : submit
+
+    %% TaskPollingMonitor <|-- ParallelPollingMonitor
+
+    SimpleFileCopyStrategy <|-- AwsBatchFileCopyStrategy
+    AwsBatchScriptLauncher --* AwsBatchFileCopyStrategy

docs/developer/diagrams/nextflow.cloud.aws.nio.mmd

@@ -0,0 +1,31 @@
+classDiagram
+    %%
+    %% nextflow.cloud.aws.nio
+    %%
+    S3FileSystemProvider --> S3FileSystem : newFileSystem
+
+    class S3FileSystem {
+        client : S3Client
+        endpoint : String
+    }
+    S3FileSystem --* S3Client
+
+    class S3Client {
+        client : AmazonS3
+        cannedAcl : CannedAccessControlList
+        kmsKeyId : String
+        storageEncryption : SSEAlgorithm
+        transferManager : TransferManager
+        transferPool : ExecutorService
+        uploadChunkSize : Long
+        uploadMaxThreads : Integer
+    }
+
+    Path <|-- S3Path
+
+    class S3Path {
+        bucket : String
+        parts : List~String~
+        fileSystem : S3FileSystem
+    }
+    S3Path --* S3FileSystem

docs/developer/diagrams/nextflow.cloud.azure.mmd

@@ -0,0 +1,17 @@
+classDiagram
+    %%
+    %% nextflow.cloud.azure
+    %%
+    Executor <|-- AzBatchExecutor
+    TaskHandler <|-- AzBatchTaskHandler
+    BashWrapperBuilder <|-- AzBatchScriptLauncher
+
+    AzBatchExecutor --* AzConfig
+    AzBatchExecutor --> AzBatchService : register
+
+    AzBatchExecutor --> TaskPollingMonitor : init
+    AzBatchExecutor --> AzBatchTaskHandler : submit
+    AzBatchTaskHandler --> AzBatchScriptLauncher : submit
+
+    SimpleFileCopyStrategy <|-- AzFileCopyStrategy
+    AzBatchScriptLauncher --* AzFileCopyStrategy

docs/developer/diagrams/nextflow.cloud.google.mmd

@@ -0,0 +1,13 @@
+classDiagram
+    %%
+    %% nextflow.cloud.google
+    %%
+    Executor <|-- GoogleBatchExecutor
+    TaskHandler <|-- GoogleBatchTaskHandler
+    BashWrapperBuilder <|-- GoogleBatchScriptLauncher
+
+    GoogleBatchExecutor --* BatchConfig
+
+    GoogleBatchExecutor --> TaskPollingMonitor : init
+    GoogleBatchExecutor --> GoogleBatchTaskHandler : submit
+    GoogleBatchTaskHandler --> GoogleBatchScriptLauncher : submit

docs/developer/diagrams/nextflow.config.mmd

@@ -0,0 +1,10 @@
+classDiagram
+    %%
+    %% nextflow.config
+    %%
+    CmdRun --> ConfigMap : run
+    Session --* ConfigMap
+
+    ConfigBuilder --> ConfigParser : build
+    ConfigBuilder --> ConfigMap : build
+    ConfigParser --> ConfigBase : parse

docs/developer/diagrams/nextflow.container.mmd

@@ -0,0 +1,16 @@
+classDiagram
+    %%
+    %% nextflow.container
+    %%
+    direction LR
+
+    BashWrapperBuilder --> ContainerBuilder : build
+
+    ContainerBuilder <|-- CharliecloudBuilder
+    ContainerBuilder <|-- DockerBuilder
+    ContainerBuilder <|-- PodmanBuilder
+    ContainerBuilder <|-- ShifterBuilder
+    ContainerBuilder <|-- SingularityBuilder
+    ContainerBuilder <|-- UdockerBuilder
+
+    SingularityBuilder <|-- ApptainerBuilder

docs/developer/diagrams/nextflow.dag.mmd

@@ -0,0 +1,33 @@
+classDiagram
+    %%
+    %% nextflow.dag
+    %%
+    Session --* DAG
+
+    class DAG {
+        vertices : List~Vertex~
+        edges : List~Edge~
+    }
+    DAG "1" --* "*" Vertex
+    DAG "1" --* "*" Edge
+
+    class Vertex {
+        label : String
+        type : Type
+        operators : List~DataflowProcessor~
+        process : TaskProcessor
+    }
+
+    class Edge {
+        channel : Object
+        from : Vertex
+        to : Vertex
+        label : String
+    }
+
+    %% DagRenderer <|-- CytoscapeHtmlRenderer
+    %% DagRenderer <|-- CytoscapeJsRenderer
+    %% DagRenderer <|-- DotRenderer
+    %% DagRenderer <|-- GexfRenderer
+    %% DagRenderer <|-- GraphvizRenderer
+    %% DagRenderer <|-- MermaidRenderer

docs/developer/diagrams/nextflow.executor.mmd

@@ -0,0 +1,82 @@
+classDiagram
+    %%
+    %% nextflow.executor
+    %%
+    ProcessDef --> Executor : run
+    %% ExecutorFactory --> Executor : getExecutor
+
+    TaskProcessor --* Executor
+
+    %% class Executor {
+    %%     name : String
+    %%     monitor : TaskMonitor
+    %% }
+    %% Executor --* TaskMonitor
+    %% Executor --> TaskHandler : submit
+
+    %% TaskMonitor <|-- TaskPollingMonitor
+
+    class TaskPollingMonitor {
+        capacity : int
+        submitRateLimit : RateLimiter
+        pollIntervalMillis : long
+        dumpInterval : Duration
+    }
+
+    %% TaskPollingMonitor <|-- LocalPollingMonitor
+
+    class LocalPollingMonitor {
+        maxCpus : int
+        maxMemory : long
+    }
+
+    %% class TaskHandler {
+    %%     task : TaskRun
+    %% }
+
+    Executor <|-- AbstractGridExecutor
+    Executor <|-- LocalExecutor
+    %% Executor <|-- NopeExecutor
+    %% AbstractGridExecutor <|-- CondorExecutor
+    %% AbstractGridExecutor <|-- HyperQueueExecutor
+    %% AbstractGridExecutor <|-- LsfExecutor
+    %% AbstractGridExecutor <|-- MoabExecutor
+    %% AbstractGridExecutor <|-- NqsiiExecutor
+    %% AbstractGridExecutor <|-- OarExecutor
+    %% AbstractGridExecutor <|-- PbsExecutor
+    %% AbstractGridExecutor <|-- SgeExecutor
+    %% AbstractGridExecutor <|-- SlurmExecutor
+    %% PbsExecutor <|-- PbsProExecutor
+    %% SgeExecutor <|-- CrgExecutor
+
+    LocalExecutor --> LocalPollingMonitor : init
+    LocalExecutor --> LocalTaskHandler : submit
+    LocalExecutor --> NativeTaskHandler : submit
+    LocalTaskHandler --> BashWrapperBuilder : submit
+
+    AbstractGridExecutor --> TaskPollingMonitor : init
+    AbstractGridExecutor --> GridTaskHandler : submit
+    GridTaskHandler --> BashWrapperBuilder : submit
+
+    %% TaskHandler <|-- CachedTaskHandler
+    %% TaskHandler <|-- GridTaskHandler
+    %% TaskHandler <|-- LocalTaskHandler
+    %% TaskHandler <|-- NativeTaskHandler
+    %% TaskHandler <|-- NopeTaskHandler
+    %% TaskHandler <|-- StoredTaskHandler
+
+    class BashWrapperBuilder {
+        bean : TaskBean
+        copyStrategy : ScriptFileCopyStrategy
+    }
+    BashWrapperBuilder --* TaskBean
+    BashWrapperBuilder --* ScriptFileCopyStrategy
+
+    ScriptFileCopyStrategy <|-- SimpleFileCopyStrategy
+
+    class SimpleFileCopyStrategy {
+        stageinMode : String
+        stageoutMode : String
+        targetDir : Path
+        workDir : Path
+    }