forked from jjbot/picasclient
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #13 from sara-nl/add-examples
Add examples This branch adds the examples from the documentation at https://doc.grid.surfsara.nl/en/latest/Pages/Practices/picas/picas_overview.html# to the repository. There are 3 examples. Local, slurm and Grid. They are based on running on the SURF instance of CouchDB as PiCaS backend. There are some small but crucial changes, especially in the decoder of the tokens. This is done with jsons, and in this version of PiCaS the syntax changed, which is visible in the change in documents.py and examples having a open().read(). Before it was just open() and the decoder could handle it, but not any more for unclear reasons. In any case: the examples work in all cases and the CICD runs properly.
- Loading branch information
Showing
20 changed files
with
657 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
#!/bin/bash | ||
|
||
PARAM_Q_MIN=100 | ||
PARAM_Q_MAX=260 | ||
PARAM_Q_STEP=84 | ||
|
||
PARAM_D_MIN=256 | ||
PARAM_D_MAX=8192 | ||
PARAM_D_STEP=2024 | ||
|
||
PARAM_M_MIN=400 | ||
PARAM_M_MAX=10000 | ||
PARAM_M_STEP=4000 | ||
|
||
function testNumerical { | ||
if [ $1 -eq $1 2> /dev/null ]; then | ||
return | ||
fi | ||
echo "$1 is not numerical!" | ||
exit 1 | ||
} | ||
|
||
while getopts "q:d:m:" opt; do | ||
testNumerical $OPTARG | ||
case $opt in | ||
q) | ||
PARAM_Q_STEP=$OPTARG | ||
;; | ||
d) | ||
PARAM_D_STEP=$OPTARG | ||
;; | ||
m) | ||
PARAM_M_STEP=$OPTARG | ||
;; | ||
esac | ||
done | ||
|
||
PARAMFILE=$( mktemp ) | ||
|
||
PARAM_Q=$PARAM_Q_MIN | ||
PARAM_D=$PARAM_D_MIN | ||
PARAM_M=$PARAM_M_MIN | ||
|
||
STR_PARAM_Q="" | ||
|
||
while [ $PARAM_Q_MAX -ge $PARAM_Q ]; do | ||
# Ugly hack in order not to have to use bc | ||
if [ $[ $PARAM_Q / 10 ] -eq 0 ]; then | ||
STR_PARAM_Q="0.00${PARAM_Q}" | ||
elif [ $[ $PARAM_Q / 100 ] -eq 0 ]; then | ||
STR_PARAM_Q="0.0${PARAM_Q}" | ||
else | ||
STR_PARAM_Q="0.${PARAM_Q}" | ||
fi | ||
while [ $PARAM_D_MAX -ge $PARAM_D ]; do | ||
while [ $PARAM_M_MAX -ge $PARAM_M ]; do | ||
echo "-q ${STR_PARAM_Q} -d ${PARAM_D} -m ${PARAM_M}" >> $PARAMFILE | ||
PARAM_M=$[ $PARAM_M+$PARAM_M_STEP ] | ||
done | ||
PARAM_M=$PARAM_M_MIN | ||
PARAM_D=$[ $PARAM_D+$PARAM_D_STEP ] | ||
done | ||
PARAM_D=$PARAM_D_MIN | ||
PARAM_Q=$[ $PARAM_Q+$PARAM_Q_STEP ] | ||
done | ||
|
||
echo ${PARAMFILE} |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
[ | ||
ParameterStart=0; | ||
ParameterStep=1; | ||
Parameters=5; | ||
|
||
Executable = "/bin/sh"; | ||
Arguments = "startpilot.sh"; | ||
Stdoutput = "parametricjob.out"; | ||
StdError = "parametricjob.err"; | ||
InputSandbox = {"grid-sandbox/CouchDB-1.2.tar.gz", "grid-sandbox/picas.tar", "grid-sandbox/startpilot.sh", "grid-sandbox/grid-example.py", "grid-sandbox/process_task.sh", "bin/fractals", "picasconfig.py"}; | ||
OutputSandbox = {"parametricjob.out", "parametricjob.err"}; | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
''' | ||
@helpdesk: SURF helpdesk <[email protected]> | ||
usage: python grid-example.py | ||
description: | ||
Connect to PiCaS server | ||
Get the next token in todo View | ||
Fetch the token parameters, e.g. input value | ||
Run main job (process_task.sh) with the input argument | ||
When done, return the exit code to the token | ||
Attach the logs to the token | ||
''' | ||
|
||
#python imports | ||
import os | ||
import time | ||
import couchdb | ||
import picasconfig | ||
|
||
#picas imports | ||
from picas.actors import RunActor | ||
from picas.clients import CouchDB | ||
from picas.iterators import TaskViewIterator | ||
from picas.modifiers import BasicTokenModifier | ||
from picas.executers import execute | ||
|
||
class ExampleActor(RunActor): | ||
def __init__(self, db, modifier, view="todo", **viewargs): | ||
super(ExampleActor, self).__init__(db, view=view, **viewargs) | ||
self.modifier = modifier | ||
self.client = db | ||
|
||
def process_task(self, token): | ||
# Print token information | ||
print("-----------------------") | ||
print("Working on token: " +token['_id']) | ||
for key, value in token.doc.items(): | ||
print(key, value) | ||
print("-----------------------") | ||
|
||
# Start running the main job | ||
# /usr/bin/time -v ./process_task.sh [input] [tokenid] 2> logs_[token_id].err 1> logs_[token_id].out | ||
command = "/usr/bin/time -v ./process_task.sh " + "\"" +token['input'] + "\" " + token['_id'] + " 2> logs_" + str(token['_id']) + ".err 1> logs_" + str(token['_id']) + ".out" | ||
|
||
out = execute(command,shell=True) | ||
|
||
## Get the job exit code in the token | ||
token['exit_code'] = out[0] | ||
token = self.modifier.close(token) | ||
#self.client.db[token['_id']] = token # necessary? | ||
|
||
# Attach logs in token | ||
curdate = time.strftime("%d/%m/%Y_%H:%M:%S_") | ||
try: | ||
logsout = "logs_" + str(token['_id']) + ".out" | ||
log_handle = open(logsout, 'rb') | ||
token.put_attachment(logsout, log_handle.read()) | ||
|
||
logserr = "logs_" + str(token['_id']) + ".err" | ||
log_handle = open(logserr, 'rb') | ||
token.put_attachment(logserr, log_handle.read()) | ||
except: | ||
print("excepted attachemnt") | ||
pass | ||
|
||
def main(): | ||
# setup connection to db | ||
client = CouchDB(url=picasconfig.PICAS_HOST_URL, db=picasconfig.PICAS_DATABASE, username=picasconfig.PICAS_USERNAME, password=picasconfig.PICAS_PASSWORD) | ||
print("Connected to the database %s sucessfully. Now starting work..." %(picasconfig.PICAS_DATABASE)) | ||
# Create token modifier | ||
modifier = BasicTokenModifier() | ||
# Create actor | ||
actor = ExampleActor(client, modifier) | ||
# Start work! | ||
actor.run() | ||
|
||
if __name__ == '__main__': | ||
main() |
Oops, something went wrong.