generated from ARM-Development/arm-repository-template
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
68 changed files
with
12,700 additions
and
162 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
# Config file for [Air](https://github.com/cosmtrek/air) in TOML format | ||
# e.g. https://github.com/cosmtrek/air/blob/master/air_example.toml | ||
|
||
root = "." | ||
tmp_dir = ".tmp" | ||
|
||
[build] | ||
cmd = ".assets/scripts/build.sh --outdir=.assets/bin" | ||
bin = ".assets/bin/sts" | ||
full_bin = ".assets/bin/sts $STS_ARGS" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
OUT: # Outgoing configuration | ||
dirs: # Outgoing directory configuration | ||
cache : /data/tmp/.sts # Used to store internal caches | ||
out : /data/transfer/outgoing_to # Directory to watch for files to send; appends "/{target}" | ||
logs : /data/logs/sts # For log files; appends "outgoing_to/{target}" and "messages" | ||
sources: # Supports multiple sources where omitted entries will inherit from previous sources hierarchically | ||
- name : ... # Name of the source | ||
threads : 8 # Maximum number of concurrent connections | ||
bin-size : 10MB # The generally-desired size for a given HTTP request (BEFORE any compression) | ||
compress : 0 # Use GZIP compression of level 0-9 (0 = no compression, 9 = best but slowest) | ||
min-age : 15s # How old a file must be before being added to the "outgoing" queue | ||
max-age : 12h # How old a file can be before getting logged as "stale" (remains in the queue) | ||
timeout : 30m # The HTTP timeout for a single request | ||
poll-delay : 5s # How long to wait after file sent before final validation | ||
poll-interval : 1m # How long to wait between polling requests | ||
poll-attempts : 10 # How many times to "poll" for the successful reception of a file before re-sending | ||
target: # Target-specific configuration | ||
name : ... # Name of the target | ||
http-host : ...:1992 # Target host, including port | ||
http-tls-cert : /apps/transfer/conf/server.pem # Public server certificate | ||
tags: # Tags are for configuration based on file patterns (omitted attributes are inherited) | ||
- pattern : DEFAULT # The file "tag" pattern | ||
priority : 0 # Relative importance (higher the number, greater the importance) | ||
order : fifo # File order ("fifo" (first in, first out) or "none") | ||
delete : true # Whether or not to delete files after reception confirmation | ||
method : http # Transfer method ("http", "disk", or "none") | ||
|
||
IN: # Incoming configuration. | ||
dirs: # Incoming directory configuration. | ||
stage : /data/transfer/stage # Directory to stage data as it comes in; appends "/{source}" | ||
final : /data/transfer/incoming_from # Final location for incoming files; appends "/{source}" | ||
logs : /data/logs/sts # For log files; appends "incoming_from/{source}" and "messages" | ||
server: # Server configuration. | ||
http-port : 1992 # What port to listen on | ||
http-tls : true # Whether or not to use HTTPS | ||
http-tls-cert : /apps/transfer/conf/server.pem # Server certificate | ||
http-tls-key : /apps/transfer/conf/server.key # Server key |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
comp: dataflow | ||
package-name: sts | ||
default-prefix: /apps/transfer | ||
|
||
build-script: .assets/scripts/build-local.sh | ||
package-script-dir: .assets/scripts | ||
|
||
include-items: | ||
- (.assets/scripts/.godev)/bin | ||
- (.assets/scripts/.godev)/conf | ||
|
||
author: | ||
name: Sherman Beus | ||
organization: PNNL | ||
email: [email protected] | ||
phone: 509-375-2662 | ||
|
||
special-instructions: > | ||
For first-time install, run "systemctl daemon-reload" | ||
To start the daemon, run "systemctl start stsd" | ||
To restart the daemon, run "systemctl restart stsd" | ||
synopsis: > | ||
A self-contained client/server application for reliable file movement | ||
notification-list: | ||
- N/A | ||
|
||
obsoletes: sts-transfer, sts-delivery | ||
|
||
scriptlet-installer: root | ||
|
||
#------------------------------------------------------------------------------ | ||
# Required fields that do not need to be set for this release | ||
|
||
build-target: '' | ||
privileged: '' | ||
exclude-items: [] | ||
package-dependency: [] | ||
file-dependency: [] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
# Building for 32-bit Windows | ||
|
||
Assuming the 32-bit version of Go was downloaded to `C:\Go32`, using MinGW console from this directory: | ||
|
||
```shell | ||
GOROOT=/c/Go32 PATH=/c/Go32/bin:$PATH ./build.sh | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
#!/bin/bash | ||
|
||
root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" | ||
proj=$root/../../ | ||
|
||
goversion=1.20.2 | ||
gotarget=`uname -s | awk '{print tolower($0)}'` | ||
gobundle=go${goversion}.${gotarget}-amd64.tar.gz | ||
gourl=https://storage.googleapis.com/golang/$gobundle | ||
GOROOT=$root/.go | ||
GOPATH=$root/.godev | ||
|
||
# Download Go | ||
if [ ! -f $GOROOT/$gobundle ]; then | ||
rm -rf $GOROOT | ||
mkdir -p $GOROOT | ||
curl -o $GOROOT/$gobundle $gourl | ||
tar -C $GOROOT -xzf $GOROOT/$gobundle | ||
fi | ||
|
||
export GOROOT=$GOROOT/go | ||
export PATH=$GOROOT/bin:$PATH | ||
export GOPATH | ||
|
||
# Clean up previous build | ||
chmod -R u+w $GOPATH # Directories with 555 perms make cleanup difficult | ||
rm -rf $GOPATH | ||
mkdir -p $GOPATH | ||
|
||
src=$GOPATH/src/code.arm.gov/dataflow | ||
mkdir -p $src | ||
ln -s $proj $src/sts | ||
|
||
# Build | ||
$root/build.sh | ||
if [ ! -f $GOPATH/bin/sts ]; then | ||
echo "Build failed" | ||
exit 1 | ||
fi | ||
|
||
# Run all tests | ||
cd $root; go test ../../... | ||
|
||
# Copy conf files | ||
mkdir -p $GOPATH/conf | ||
cp $root/../dist.arm.yaml $GOPATH/conf/sts.yaml.example | ||
cp $root/stsd.service $GOPATH/conf |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
#!/bin/bash | ||
|
||
root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" | ||
|
||
GOOS=windows GOARCH=386 $root/build.sh | ||
cp $GOPATH/bin/sts.exe $root/sts-32.exe | ||
|
||
GOOS=windows GOARCH=amd64 $root/build.sh | ||
cp $GOPATH/bin/sts.exe $root/sts-64.exe | ||
|
||
zip ~/sts-ms.zip sts-32.exe sts-64.exe | ||
rm sts*.exe |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
#!/bin/bash | ||
|
||
race="" | ||
ldflags="" | ||
outdir=$GOPATH/bin | ||
outname="sts" | ||
|
||
for i in "$@"; do | ||
case $i in | ||
-r|--race) | ||
race="-race" | ||
shift | ||
;; | ||
--ldflags=*) | ||
ldflags="${i#*=}" | ||
shift | ||
;; | ||
--os=*) | ||
export GOOS="${i#*=}" | ||
shift | ||
;; | ||
--outdir=*) | ||
outdir="${i#*=}" | ||
shift | ||
;; | ||
--outname=*) | ||
outname="${i#*=}" | ||
shift | ||
;; | ||
esac | ||
done | ||
|
||
if [[ "$GOOS" == "windows" ]]; then | ||
outname="$outname.exe" | ||
fi | ||
|
||
basedir=$(dirname $0) | ||
rootdir="$basedir/../../" | ||
|
||
cd $rootdir | ||
|
||
if [ -f $outdir/$outname ]; then | ||
echo "-- Cleaning Old Build" | ||
rm $outdir/$outname | ||
fi | ||
|
||
echo "-- Preparing" | ||
mkdir -p $outdir | ||
|
||
echo "-- Building Dependencies" | ||
go mod download | ||
|
||
date=`date -u '+%Y-%m-%dT%H:%M:%SZ'` | ||
vers=$APR_VERSION | ||
if [ -z "$vers" ]; then | ||
vers=`git describe --tags 2>/dev/null` | ||
if [ -z "$vers" ]; then | ||
vers="v0.0.0" | ||
fi | ||
fi | ||
|
||
echo "-- Building Executable (GOOS='$GOOS', GOARCH='$GOARCH')" | ||
go build -o $outdir/$outname $race \ | ||
-ldflags="-X 'main.BuildTime=$date UTC' -X 'main.Version=$vers' $ldflags" \ | ||
./main/*.go |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
#!/bin/bash | ||
|
||
if [ -z "$DATAFLOW_HOME" ]; then | ||
export DATAFLOW_HOME="/apps/transfer" | ||
fi | ||
confdir=$DATAFLOW_HOME/conf | ||
svcname=stsd | ||
systemdir=/lib/systemd/system | ||
systemctl=/bin/systemctl | ||
if [ -w $systemdir ]; then | ||
cp $confdir/$svcname.service $systemdir | ||
if [ -x $systemctl ]; then | ||
$systemctl daemon-reload | ||
if [[ $($systemctl is-active $svcname) == "active" ]]; then | ||
$systemctl restart $svcname | ||
fi | ||
fi | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
[Unit] | ||
Description=Site Transfer Software Daemon | ||
After=syslog.target network.target | ||
RequiresMountsFor=/data/logs | ||
|
||
[Service] | ||
Type=simple | ||
User=dsmgr | ||
Group=data | ||
RestartSec=5 | ||
ExecStart=/apps/transfer/bin/sts -loop -conf /apps/transfer/conf/sts.yaml | ||
WatchdogSec=600s | ||
Restart=on-failure | ||
|
||
[Install] | ||
WantedBy=multi-user.target |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
# TODO: automate log/dir checking to confirm: | ||
# 1) files sent in order | ||
# 2) no file left behind | ||
# 3) no duplicates | ||
|
||
echo "-------------------------------------------------------------------------" | ||
echo "ERRORS:" | ||
grep "ERROR" $STS_HOME/data/log/messages/*/* | ||
i_sorted=`cut -d ":" -f 1 $STS_HOME/data*/log/incoming_from/*/*/* | sort` | ||
o_sorted=`cut -d ":" -f 1 $STS_HOME/data*/log/outgoing_to/*/*/* | sort` | ||
echo "-------------------------------------------------------------------------" | ||
echo "OUT DUPLICATES:" | ||
uniq -c <(cat <(echo "$o_sorted")) | grep -v " 1" | ||
echo "-------------------------------------------------------------------------" | ||
echo "IN DUPLICATES:" | ||
uniq -c <(cat <(echo "$i_sorted")) | grep -v " 1" | ||
echo "-------------------------------------------------------------------------" | ||
echo "OUT DIR:" | ||
find $STS_HOME/data/out -type f | sort | ||
echo "-------------------------------------------------------------------------" | ||
echo "STAGE DIR:" | ||
find $STS_HOME/data/stage -type f | sort | ||
echo "-------------------------------------------------------------------------" | ||
echo "IN DIR:" | ||
find $STS_HOME/data/in -type f | sort |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
#!/usr/bin/env python3 | ||
|
||
import os | ||
import sys | ||
import time | ||
import random | ||
import datetime | ||
import subprocess | ||
|
||
|
||
def makedata(name, prefix, size, count): | ||
now = datetime.datetime.fromtimestamp( | ||
time.time() | ||
).strftime('%Y%m%d.%H%M%S') | ||
root = '%s/data/out/%s' % (os.getenv('STS_HOME', '/var/tmp/sts'), name) | ||
root = os.path.abspath(root) | ||
tdir = '%s/%s' % (root, prefix) | ||
if not os.path.exists(tdir): | ||
os.makedirs(tdir) | ||
c = int(count) | ||
s = int(size) | ||
count = random.randint(int(c-c*0.5), int(c+c*0.5)) | ||
for i in range(count): | ||
size = random.randint(int(s-s*0.5), int(s+s*0.5)) | ||
name = "%s.%s.%06d" % (prefix, now, i) | ||
path = "%s/%s" % (tdir, name) | ||
with open(os.devnull, 'w') as devnull: | ||
subprocess.call([ | ||
"dd", | ||
"if=/dev/urandom", | ||
"of=%s.lck" % path, | ||
"bs=%d" % size, "count=1" | ||
], stdout=devnull, stderr=devnull) | ||
os.rename("%s.lck" % path, path) | ||
|
||
|
||
def main(argv): | ||
if len(argv) < 5: | ||
print("Expected 5 args") | ||
return 1 | ||
(name, prefix, size, count, sleep) = tuple(argv) | ||
sleep = int(sleep) | ||
while True: | ||
makedata(name, prefix, size, count) | ||
if not sleep: | ||
break | ||
time.sleep(sleep) | ||
|
||
|
||
if __name__ == '__main__': | ||
main(sys.argv[1:]) |
Oops, something went wrong.