preprocess.sh

#!/bin/bash

DIR=$1  # take a parameter from the command line
NAME=$2  # take a parameter from the command line
NAMELOG=$NAME-log.txt
NAMEENG=$NAME-eng.txt

if [ ! -d "$1" ] || [ -z "$1/$NAMELOG" ] || [ -z "$1/$NAMEENG" ]
then
      echo "$1 is empty and/or $1/$NAMELOG is missing"
      echo "usage:  > ./preprocess.sh Dir Filename "
      exit 1
else
      echo "Running preprocessing script with parameters $1 and $2."
      echo "This script expects to find $1/$2-log.txt and $1/$2-eng.txt"
      echo "as files of logical expressions and their English"
      echo "equivalents, respectively."
fi

echo "starting at date/time"
date

pushd .
cd $DIR

if [ ! -d "data" ]
then
      mkdir data
fi

cd data

cp ../$NAMELOG .
cp ../$NAMEENG .

perl -pe 's/,/ , /g; s/ +/ /g;' $NAMEENG > $NAMEENG-0
   # need to split 's and n't off from words into separate tokens

perl -pe 's/([()?])/ $1 /g; s/ +/ /g;' $NAMELOG > $NAMELOG-0

paste -d\# $NAMELOG-0 $NAMEENG-0 > out.merged  # put english and its logic on the same line

echo "finished horizontal merge"

shuf < out.merged > out.mergedR  # shuffles the lines in the file into random order
rm out.merged
echo "finished shuffle"

#NUM_LINES=$(wc -l < out.mergedR)

# reduce the total size of the data set

#head -n $((NUM_LINES/1)) out.mergedR > temp
#rm out.mergedR
#mv temp out.mergedR

NUM_LINES=$(wc -l < out.mergedR)

#TRAIN_L=$((NUM_LINES * 99 / 100))
#TRAIN_L=$((NUM_LINES * 99 / 100))

#split -l $NUMLINES out.mergedR

#mv xaa train.mergedR   # xaa is the name of the first file generated by split

#mv xab bigtest.mergedR # xab is the name of the second file generated by split

#BIGTEST_LINES=$(wc -l < bigtest.mergedR)
# TEST_L=$((BIGTEST_LINES / 2))
TEST_L=100
REMAIN=$((NUM_LINES - $TEST_L))

head -n $TEST_L out.mergedR > dev.mergedR
tail -n $REMAIN out.mergedR > temp.mergedR
REMAIN=$((NUM_LINES - 2 * $TEST_L))
head -n $TEST_L temp.mergedR > test.mergedR
tail -n $REMAIN temp.mergedR > train.mergedR
echo "$TEST_L lines in dev and test, $REMAIN lines in train"
rm temp.mergedR

for i in dev test train; do cut -f1 -d\# $i.mergedR > $i.sum1; done  # make sumo test and train files by extracting the first field (before the #)

for i in dev test train; do cut -f2- -d\# $i.mergedR > $i.nat; done  # make english test and train files by extracting the second field (after the #)

echo "finished field extraction"

perl -pe 's/ +/\n/g' $NAMELOG-0 | sort -u > vocab.sum1
perl -pe 's/ +/\n/g' $NAMEENG-0 | sort -u > vocab.nat

sed -i '1d' vocab.sum1 # delete space character from vocab
sed -i '1d' vocab.nat

echo "run script training.sh to train, then test.sh to test"

popd