run_child_poet_saga_cluster.sh

#!/bin/bash
# Job name:
#SBATCH --job-name=child_poet
#
# Project:
#SBATCH --account=nn9740k
#
# Wall clock limit (hh:mm:ss):
#SBATCH --time=96:00:00
#
## Allocates cpus
#SBATCH --ntasks=1 --cpus-per-task=20
#
## allocates ram per cpu
#SBATCH --mem-per-cpu=2G

## Set up job environment:
module --quiet purge   # clear any inherited modules
set -o errexit # exit on errors
set -o nounset # treat unset variables as error

## Load python (make sure you load the same python version on saga before creating your virtual environment!!!!!!!!)
module load Python/3.6.6-foss-2018b
export PS1=\$
source child_poet_virtualenv/bin/activate

## Copy input files to the work directory:
## copy a file
#cp $SUBMITDIR/poet_test.py $SCRATCH

## copy a directory
cp -r $SUBMITDIR/child_poet $SCRATCH
cp -r $SUBMITDIR/tmp $SCRATCH

## Go to your work folder
cd $SCRATCH

## Use this to save files generated by the program back to your home area
savefile tmp
savefile child_poet/mlruns

## Run your program
if [ -z "$1" ]
then
    echo "Missing an experiment id"
    exit 1
fi

experiment=poet_$1

mkdir -p $SCRATCH/tmp/logs/$experiment
mkdir -p $SCRATCH/tmp/niche_encodings/$experiment

cd $SCRATCH/child_poet

srun python -u child_poet_master_script.py \
  $SCRATCH/tmp/logs/$experiment \
  $SCRATCH/tmp/niche_encodings/$experiment \
  --init=random \
  --learning_rate=0.01 \
  --lr_decay=0.9999 \
  --lr_limit=0.001 \
  --batch_size=1 \
  --batches_per_chunk=256 \
  --eval_batch_size=1 \
  --eval_batches_per_step=5 \
  --master_seed=24582922 \
  --repro_threshold=200 \
  --mc_lower=25 \
  --mc_upper=340 \
  --noise_std=0.1 \
  --noise_decay=0.999 \
  --noise_limit=0.01 \
  --normalize_grads_by_noise_std \
  --returns_normalization=centered_ranks \
  --envs stump pit roughness \
  --max_num_envs=40 \
  --adjust_interval=2 \
  --propose_with_adam \
  --steps_before_transfer=25 \
  --max_children=16 \
  --max_admitted=1 \
  --num_workers 20 \
  --run_child_poet \
  --child_success_reward=0.5 \
  --agent_tracker_certainty_threshold=0.8 \
  --start_from=$SCRATCH/tmp/logs/poet_dec2_168h/poet_dec2_168h.flat.best.json \
  --n_iterations=60000 2>&1 | tee ~/tmp/ipp/$experiment/run.log

## Exit
exit 0