Merge branch 'master'

.air.toml

@@ -0,0 +1,10 @@
+# Config file for [Air](https://github.com/cosmtrek/air) in TOML format
+# e.g. https://github.com/cosmtrek/air/blob/master/air_example.toml
+
+root = "."
+tmp_dir = ".tmp"
+
+[build]
+cmd = ".assets/scripts/build.sh --outdir=.assets/bin"
+bin = ".assets/bin/sts"
+full_bin = ".assets/bin/sts $STS_ARGS"

.assets/dist.arm.yaml

@@ -0,0 +1,37 @@
+OUT: # Outgoing configuration
+  dirs: # Outgoing directory configuration
+    cache : /data/tmp/.sts             # Used to store internal caches
+    out   : /data/transfer/outgoing_to # Directory to watch for files to send; appends "/{target}"
+    logs  : /data/logs/sts             # For log files; appends "outgoing_to/{target}" and "messages"
+  sources: # Supports multiple sources where omitted entries will inherit from previous sources hierarchically
+    - name          : ...      # Name of the source
+      threads       : 8        # Maximum number of concurrent connections
+      bin-size      : 10MB     # The generally-desired size for a given HTTP request (BEFORE any compression)
+      compress      : 0        # Use GZIP compression of level 0-9 (0 = no compression, 9 = best but slowest)
+      min-age       : 15s      # How old a file must be before being added to the "outgoing" queue
+      max-age       : 12h      # How old a file can be before getting logged as "stale" (remains in the queue)
+      timeout       : 30m      # The HTTP timeout for a single request
+      poll-delay    : 5s       # How long to wait after file sent before final validation
+      poll-interval : 1m       # How long to wait between polling requests
+      poll-attempts : 10       # How many times to "poll" for the successful reception of a file before re-sending
+      target: # Target-specific configuration
+        name          : ...      # Name of the target
+        http-host     : ...:1992 # Target host, including port
+        http-tls-cert : /apps/transfer/conf/server.pem  # Public server certificate
+      tags: # Tags are for configuration based on file patterns (omitted attributes are inherited)
+        - pattern   : DEFAULT # The file "tag" pattern
+          priority  : 0       # Relative importance (higher the number, greater the importance)
+          order     : fifo    # File order ("fifo" (first in, first out) or "none")
+          delete    : true    # Whether or not to delete files after reception confirmation
+          method    : http    # Transfer method ("http", "disk", or "none")
+
+IN: # Incoming configuration.
+  dirs: # Incoming directory configuration.
+    stage : /data/transfer/stage         # Directory to stage data as it comes in; appends "/{source}"
+    final : /data/transfer/incoming_from # Final location for incoming files; appends "/{source}"
+    logs  : /data/logs/sts               # For log files; appends "incoming_from/{source}" and "messages"
+  server: # Server configuration.
+    http-port     : 1992 # What port to listen on
+    http-tls      : true # Whether or not to use HTTPS
+    http-tls-cert : /apps/transfer/conf/server.pem # Server certificate
+    http-tls-key  : /apps/transfer/conf/server.key # Server key

.assets/package.yaml

@@ -0,0 +1,40 @@
+comp:           dataflow
+package-name:   sts
+default-prefix: /apps/transfer
+
+build-script:       .assets/scripts/build-local.sh
+package-script-dir: .assets/scripts
+
+include-items:
+   - (.assets/scripts/.godev)/bin
+   - (.assets/scripts/.godev)/conf
+
+author:
+  name: Sherman Beus
+  organization: PNNL
+  email: [email protected]
+  phone: 509-375-2662
+
+special-instructions: >
+  For first-time install, run "systemctl daemon-reload"
+  To start the daemon, run "systemctl start stsd"
+  To restart the daemon, run "systemctl restart stsd"
+
+synopsis: >
+  A self-contained client/server application for reliable file movement
+
+notification-list:
+  - N/A
+
+obsoletes: sts-transfer, sts-delivery
+
+scriptlet-installer: root
+
+#------------------------------------------------------------------------------
+# Required fields that do not need to be set for this release
+
+build-target:   ''
+privileged:     ''
+exclude-items:  []
+package-dependency: []
+file-dependency: []

.assets/scripts/README.md

@@ -0,0 +1,7 @@
+# Building for 32-bit Windows
+
+Assuming the 32-bit version of Go was downloaded to `C:\Go32`, using MinGW console from this directory:
+
+```shell
+GOROOT=/c/Go32 PATH=/c/Go32/bin:$PATH ./build.sh
+```

.assets/scripts/build-local.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+proj=$root/../../
+
+goversion=1.20.2
+gotarget=`uname -s | awk '{print tolower($0)}'`
+gobundle=go${goversion}.${gotarget}-amd64.tar.gz
+gourl=https://storage.googleapis.com/golang/$gobundle
+GOROOT=$root/.go
+GOPATH=$root/.godev
+
+# Download Go
+if [ ! -f $GOROOT/$gobundle ]; then
+    rm -rf $GOROOT
+    mkdir -p $GOROOT
+    curl -o $GOROOT/$gobundle $gourl
+    tar -C $GOROOT -xzf $GOROOT/$gobundle
+fi
+
+export GOROOT=$GOROOT/go
+export PATH=$GOROOT/bin:$PATH
+export GOPATH
+
+# Clean up previous build
+chmod -R u+w $GOPATH  # Directories with 555 perms make cleanup difficult
+rm -rf $GOPATH
+mkdir -p $GOPATH
+
+src=$GOPATH/src/code.arm.gov/dataflow
+mkdir -p $src
+ln -s $proj $src/sts
+
+# Build
+$root/build.sh
+if [ ! -f $GOPATH/bin/sts ]; then
+    echo "Build failed"
+    exit 1
+fi
+
+# Run all tests
+cd $root; go test ../../...
+
+# Copy conf files
+mkdir -p $GOPATH/conf
+cp $root/../dist.arm.yaml $GOPATH/conf/sts.yaml.example
+cp $root/stsd.service $GOPATH/conf

.assets/scripts/build-ms.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+GOOS=windows GOARCH=386 $root/build.sh
+cp $GOPATH/bin/sts.exe $root/sts-32.exe
+
+GOOS=windows GOARCH=amd64 $root/build.sh
+cp $GOPATH/bin/sts.exe $root/sts-64.exe
+
+zip ~/sts-ms.zip sts-32.exe sts-64.exe
+rm sts*.exe

.assets/scripts/build.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+race=""
+ldflags=""
+outdir=$GOPATH/bin
+outname="sts"
+
+for i in "$@"; do
+    case $i in
+        -r|--race)
+        race="-race"
+        shift
+        ;;
+        --ldflags=*)
+        ldflags="${i#*=}"
+        shift
+        ;;
+        --os=*)
+        export GOOS="${i#*=}"
+        shift
+        ;;
+        --outdir=*)
+        outdir="${i#*=}"
+        shift
+        ;;
+        --outname=*)
+        outname="${i#*=}"
+        shift
+        ;;
+    esac
+done
+
+if [[ "$GOOS" == "windows" ]]; then
+    outname="$outname.exe"
+fi
+
+basedir=$(dirname $0)
+rootdir="$basedir/../../"
+
+cd $rootdir
+
+if [ -f $outdir/$outname ]; then
+    echo "-- Cleaning Old Build"
+    rm $outdir/$outname
+fi
+
+echo "-- Preparing"
+mkdir -p $outdir
+
+echo "-- Building Dependencies"
+go mod download
+
+date=`date -u '+%Y-%m-%dT%H:%M:%SZ'`
+vers=$APR_VERSION
+if [ -z "$vers" ]; then
+    vers=`git describe --tags 2>/dev/null`
+    if [ -z "$vers" ]; then
+        vers="v0.0.0"
+    fi
+fi
+
+echo "-- Building Executable (GOOS='$GOOS', GOARCH='$GOARCH')"
+go build -o $outdir/$outname $race \
+    -ldflags="-X 'main.BuildTime=$date UTC' -X 'main.Version=$vers' $ldflags" \
+    ./main/*.go

.assets/scripts/sts.post

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ -z "$DATAFLOW_HOME" ]; then
+    export DATAFLOW_HOME="/apps/transfer"
+fi
+confdir=$DATAFLOW_HOME/conf
+svcname=stsd
+systemdir=/lib/systemd/system
+systemctl=/bin/systemctl
+if [ -w $systemdir ]; then
+    cp $confdir/$svcname.service $systemdir
+    if [ -x $systemctl ]; then
+        $systemctl daemon-reload
+        if [[ $($systemctl is-active $svcname) == "active" ]]; then
+            $systemctl restart $svcname
+        fi
+    fi
+fi

.assets/scripts/stsd.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Site Transfer Software Daemon
+After=syslog.target network.target
+RequiresMountsFor=/data/logs
+
+[Service]
+Type=simple
+User=dsmgr
+Group=data
+RestartSec=5
+ExecStart=/apps/transfer/bin/sts -loop -conf /apps/transfer/conf/sts.yaml
+WatchdogSec=600s
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

.assets/test/done.sh

@@ -0,0 +1,25 @@
+# TODO: automate log/dir checking to confirm:
+# 1) files sent in order
+# 2) no file left behind
+# 3) no duplicates
+
+echo "-------------------------------------------------------------------------"
+echo "ERRORS:"
+grep "ERROR" $STS_HOME/data/log/messages/*/*
+i_sorted=`cut -d ":" -f 1 $STS_HOME/data*/log/incoming_from/*/*/* | sort`
+o_sorted=`cut -d ":" -f 1 $STS_HOME/data*/log/outgoing_to/*/*/* | sort`
+echo "-------------------------------------------------------------------------"
+echo "OUT DUPLICATES:"
+uniq -c <(cat <(echo "$o_sorted")) | grep -v " 1"
+echo "-------------------------------------------------------------------------"
+echo "IN DUPLICATES:"
+uniq -c <(cat <(echo "$i_sorted")) | grep -v " 1"
+echo "-------------------------------------------------------------------------"
+echo "OUT DIR:"
+find $STS_HOME/data/out -type f | sort
+echo "-------------------------------------------------------------------------"
+echo "STAGE DIR:"
+find $STS_HOME/data/stage -type f | sort
+echo "-------------------------------------------------------------------------"
+echo "IN DIR:"
+find $STS_HOME/data/in -type f | sort

.assets/test/makedata.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import random
+import datetime
+import subprocess
+
+
+def makedata(name, prefix, size, count):
+    now = datetime.datetime.fromtimestamp(
+        time.time()
+    ).strftime('%Y%m%d.%H%M%S')
+    root = '%s/data/out/%s' % (os.getenv('STS_HOME', '/var/tmp/sts'), name)
+    root = os.path.abspath(root)
+    tdir = '%s/%s' % (root, prefix)
+    if not os.path.exists(tdir):
+        os.makedirs(tdir)
+    c = int(count)
+    s = int(size)
+    count = random.randint(int(c-c*0.5), int(c+c*0.5))
+    for i in range(count):
+        size = random.randint(int(s-s*0.5), int(s+s*0.5))
+        name = "%s.%s.%06d" % (prefix, now, i)
+        path = "%s/%s" % (tdir, name)
+        with open(os.devnull, 'w') as devnull:
+            subprocess.call([
+                "dd",
+                "if=/dev/urandom",
+                "of=%s.lck" % path,
+                "bs=%d" % size, "count=1"
+            ], stdout=devnull, stderr=devnull)
+        os.rename("%s.lck" % path, path)
+
+
+def main(argv):
+    if len(argv) < 5:
+        print("Expected 5 args")
+        return 1
+    (name, prefix, size, count, sleep) = tuple(argv)
+    sleep = int(sleep)
+    while True:
+        makedata(name, prefix, size, count)
+        if not sleep:
+            break
+        time.sleep(sleep)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])