test.sh.in

#!/bin/bash -ex
export VCOMPUTE=("${NODE_NAME}-Rinjin1","${NODE_NAME}-Rinjin2","${NODE_NAME}-Quanta")
MODIFY_API_PACKAGE="${MODIFY_API_PACKAGE}"
source ${WORKSPACE}/build-config/shareMethod.sh

cleanUpDockerImages(){
    echo $SUDO_PASSWORD |sudo -S docker rmi $(echo $SUDO_PASSWORD |sudo -S docker images -q)
}

cleanUpDockerContainer(){
    echo $SUDO_PASSWORD |sudo -S docker rm $(echo $SUDO_PASSWORD |sudo -S docker ps -a -q)
}

cleanUp(){
    set +e
    cleanUpDockerContainer
    cleanUpDockerImages
    echo $SUDO_PASSWORD |sudo -S service mongodb stop
    echo $SUDO_PASSWORD |sudo -S service rabbitmq-server stop
    set -e
}

apiPackageModify() {
    pushd ${WORKSPACE}/build-deps/on-http/extra
    sed -i "s/.*git symbolic-ref.*/ continue/g" make-deb.sh
    sed -i "/build-package.bash/d" make-deb.sh
    sed -i "/GITCOMMITDATE/d" make-deb.sh
    sed -i "/mkdir/d" make-deb.sh
    bash make-deb.sh
    popd
    for package in ${API_PACKAGE_LIST}; do
      sudo pip uninstall -y ${package//./-} || true
      pushd ${WORKSPACE}/build-deps/on-http/$package
        fail=true
        while $fail; do
          python setup.py install
          if [ $? -eq 0 ];then
        	  fail=false
          fi
        done
      popd
    done
}

VCOMPUTE="${VCOMPUTE}"
TOTAL_VCOMPUTES=3
if [ -z "${VCOMPUTE}" ]; then
  VCOMPUTE=("jvm-Quanta_T41-1" "jvm-vRinjin-1" "jvm-vRinjin-2")
fi

TEST_GROUP="${TEST_GROUP}"
if [ -z "${TEST_GROUP}" ]; then
   TEST_GROUP="smoke-tests"
fi

nodesOff() {
  cd ${WORKSPACE}/build-config/deployment/
  if [ "${USE_VCOMPUTE}" != "false" ]; then
    for i in ${VCOMPUTE[@]}; do
      ./vm_control.sh "${ESXI_HOST},${ESXI_USER},${ESXI_PASS},power_off,1,${i}_*"
    done
  else
     ./telnet_sentry.exp ${SENTRY_HOST} ${SENTRY_USER} ${SENTRY_PASS} off ${OUTLET_NAME}
     sleep 5
  fi
}

nodesOn() {
  cd ${WORKSPACE}/build-config/deployment/
  if [ "${USE_VCOMPUTE}" != "false" ]; then
    for i in ${VCOMPUTE[@]}; do
      ./vm_control.sh "${ESXI_HOST},${ESXI_USER},${ESXI_PASS},power_on,1,${i}_*"
    done
  else
     ./telnet_sentry.exp ${SENTRY_HOST} ${SENTRY_USER} ${SENTRY_PASS} on ${OUTLET_NAME}
     sleep 5
  fi
}

nodesDelete() {
  cd ${WORKSPACE}/build-config/deployment/
  if [ "${USE_VCOMPUTE}" != "false" ]; then
    if [ ${OVA_POST_TEST} == "true" ]; then
      VCOMPUTE+=("${NODE_NAME}-ova-for-post-test")
    fi
    for i in ${VCOMPUTE[@]}; do
      ./vm_control.sh "${ESXI_HOST},${ESXI_USER},${ESXI_PASS},delete,1,${i}_*"
    done
  fi
}

nodesCreate() {
  cd ${WORKSPACE}/build-config/deployment/
  if [ "${USE_VCOMPUTE}" != "false" ]; then
    for i in {1..2}
    do
      execWithTimeout "ovftool --overwrite --noSSLVerify --diskMode=${DISKMODE} --datastore=${DATASTORE}  --name='${NODE_NAME}-Rinjin${i}' --net:'${NIC}=${NODE_NAME}-switch' '${HOME}/isofarm/OVA/vRinjin-Haswell.ova'   vi://${ESXI_USER}:${ESXI_PASS}@${ESXI_HOST}"
    done
    execWithTimeout "ovftool --overwrite --noSSLVerify --diskMode=${DISKMODE} --datastore=${DATASTORE} --name='${NODE_NAME}-Quanta' --net:'${NIC}=${NODE_NAME}-switch' '${HOME}/isofarm/OVA/vQuanta-T41-Haswell.ova'   vi://${ESXI_USER}:${ESXI_PASS}@${ESXI_HOST}"
  else
    nodesOff
  fi
}

vnc_record_start(){
  mkdir -p ${WORKSPACE}/build-log
  pushd ${WORKSPACE}/build-config 
  export fname_prefix="vNode"
  if [ ! -z $BUILD_ID ]; then
      export fname_prefix=${fname_prefix}_b${BUILD_ID}
  fi
  bash vnc_record.sh ${WORKSPACE}/build-log $fname_prefix &
}

vnc_record_stop(){
  #sleep 2 sec to ensure FLV finishes the disk I/O before VM destroyed
  set +e
  pkill -f flvrec.py
  sleep 2
  set -e
}

generateSolLog(){
  pushd ${WORKSPACE}/build-config
  bash generate-sol-log.sh > ${WORKSPACE}/sol_script.log &
}

generateSolLogStop(){
  set +e
  pkill -f SCREEN
}

generateSysLog(){
  set +e
  containerId=$( echo $SUDO_PASSWORD |sudo -S docker ps|grep "my/test" | awk '{print $1}' )
  echo $SUDO_PASSWORD |sudo -S docker exec -it $containerId dmesg > ${WORKSPACE}/build-log/dmesg.log
}

generateMongoLog(){
  set +e
  containerId=$( echo $SUDO_PASSWORD |sudo -S docker ps|grep "my/test" | awk '{print $1}' )
  echo $SUDO_PASSWORD |sudo -S docker cp $containerId:/var/log/mongodb ${WORKSPACE}/build-log
  echo $SUDO_PASSWORD |sudo -S chown -R $USER:$USER ${WORKSPACE}/build-log/mongodb
}

generateRackHDLog(){
  set +e
  containerId=$( echo $SUDO_PASSWORD |sudo -S docker ps|grep "my/test" | awk '{print $1}' )
  echo $SUDO_PASSWORD |sudo -S docker cp $containerId:/var/log/rackhd.log ${WORKSPACE}/build-log
  echo $SUDO_PASSWORD |sudo -S chown -R $USER:$USER ${WORKSPACE}/build-log/logs
  mv ${WORKSPACE}/build-log/logs/*.log ${WORKSPACE}/build-log
}

setupVirtualEnv(){
  pushd ${WORKSPACE}/RackHD/test
  rm -rf .venv/on-build-config
  ./mkenv.sh on-build-config
  source myenv_on-build-config
  popd
  if [ "$MODIFY_API_PACKAGE" == true ] ; then
      apiPackageModify
  fi
}

BASE_REPO_URL="${BASE_REPO_URL}"
runTests() {
  set +e
  netstat -ntlp
  args=()
  if [ ! -z "$1" ];then
      args+="$1"
  fi
  fitSmokeTest "${args}"
  set -e
}

waitForAPI() {
  netstat -ntlp
  timeout=0
  maxto=60
  set +e
  url=http://localhost:9090/api/2.0/nodes
  while [ ${timeout} != ${maxto} ]; do
    wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 1 --continue ${url}
    if [ $? = 0 ]; then 
      break
    fi
    sleep 10
    timeout=`expr ${timeout} + 1`
  done
  set -e
  if [ ${timeout} == ${maxto} ]; then
    echo "Timed out waiting for RackHD API service (duration=`expr $maxto \* 10`s)."
    exit 1
  fi
}

waitForNodes() {
  # Wait for virtual nodes to be discovered, this expects exactly the number of virtual
  # compute nodes defined by TOTAL_VCOMPUTES at the beginning of this script
  netstat -ntlp
  timeout=0
  maxto=60
  set +e
  url=http://localhost:9090/api/2.0/nodes
  # check if nodeids have been created for the virtual nodes 
  sleep 20
  while [ ${timeout} != ${maxto} ]; do
    echo "Current node list: "
    wget -SO- -T 1 -t 1 --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 --continue ${url} 
    wget -SO- -T 1 -t 1 --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 --continue ${url} | grep -o "compute" | wc -l | grep ${TOTAL_VCOMPUTES}
    if [ $? = 0 ]; then
      break
    fi
    sleep 10
    timeout=`expr ${timeout} + 1`
  done
  set -e
  if [ ${timeout} == ${maxto} ]; then
    echo "Timed out waiting for RackHD virtual node discovery (duration=`expr $maxto \* 10`s)."
    exit 1
  fi
}

######################################
#    OVA POST SMOKE TEST RELATED     #
######################################
portForwarding(){
    # forward ova to localhost
    # according to vagrant/mongo/config.json and cit/fit config
    socat TCP4-LISTEN:9091,forever,reuseaddr,fork TCP4:$1:5672 &
    socat TCP4-LISTEN:9090,forever,reuseaddr,fork TCP4:$1:8080 &
    socat TCP4-LISTEN:9092,forever,reuseaddr,fork TCP4:$1:9080 &
    socat TCP4-LISTEN:9093,forever,reuseaddr,fork TCP4:$1:8443 &
    socat TCP4-LISTEN:2222,forever,reuseaddr,fork TCP4:$1:22 &
    socat TCP4-LISTEN:37017,forever,reuseaddr,fork TCP4:$1:27017 &
    echo "Finished ova -> localhost port forwarding"
    echo "5672->9091"
    echo "8080->9090"
    echo "9080->9092"
    echo "8443->9093"
    echo "22->2222"
    echo "27017->37017"
}

fetchOVALog(){
    ansible_workspace=${WORKSPACE}/build-config/jobs/build_ova/ansible
    # fetch rackhd log
    pushd $ansible_workspace
      echo "ova-post-test ansible_host=$OVA_INTERNAL_IP ansible_user=$OVA_USER ansible_ssh_pass=$OVA_PASSWORD ansible_become_pass=$OVA_PASSWORD" > hosts
      ansible-playbook -i hosts main.yml --tags "after-test"
      mkdir -p ${WORKSPACE}/build-log
      for log in `ls *.log *.flv *sol.log.raw | xargs` ; do
        cp $log ${WORKSPACE}/build-log
      done
    popd
}

dockerUp(){
    ifconfig
    netstat -ntlp 
    pushd $WORKSPACE
    echo $SUDO_PASSWORD |sudo -S docker load -i rackhd_pipeline_docker.tar
    popd
    cp -r ${WORKSPACE}/build-deps ${WORKSPACE}/build-config/jobs/pr_gate/docker
    pushd ${WORKSPACE}/build-config/jobs/pr_gate/docker
    #cp -r ${WORKSPACE}/build-config/jobs/pr_gate/docker/* .
    echo $SUDO_PASSWORD |sudo -S docker build -t my/test .
    echo $SUDO_PASSWORD |sudo -S docker run --net=host -v /etc/localtime:/etc/localtime:ro -d -t my/test
    popd
}

setupTestsConfig(){
    RACKHD_DHCP_HOST_IP=$(ifconfig | awk '/inet addr/{print substr($2,6)}' |grep 172.31.128)
    if [ -n "$RACKHD_CONFIG_FILE_URL" ]; then
      wget --tries=3 $RACKHD_CONFIG_FILE_URL -O ${WORKSPACE}/build-config/jobs/pr_gate/docker/monorail/config.json
    fi
    sed -i "s/172.31.128.1/${RACKHD_DHCP_HOST_IP}/g" ${WORKSPACE}/build-config/jobs/pr_gate/docker/monorail/config.json

    pushd ${WORKSPACE}/RackHD/test/config
    sed -i "s/\"username\": \"vagrant\"/\"username\": \"${SUDO_USER}\"/g" credentials_default.json
    sed -i "s/\"password\": \"vagrant\"/\"password\": \"$SUDO_PASSWORD\"/g" credentials_default.json
    popd
    pushd ${WORKSPACE}/RackHD
    find ./ -type f -exec sed -i -e "s/172.31.128.1/${RACKHD_DHCP_HOST_IP}/g" {} \;
    popd
}

collectTestReport()
{
    pushd ${WORKSPACE}/RackHD/test
    mkdir -p ${WORKSPACE}/xunit-reports
    cp *.xml ${WORKSPACE}/xunit-reports
    popd
}

fitSmokeTest()
{
    set +e
    echo "########### Run FIT Stack Init #############"
    pushd ${WORKSPACE}/RackHD/test
    #TODO Parameterize FIT args
    tstack="${TEST_STACK}"
    args=()
    if [ ! -z "$1" ];then
        args+="$1"
    fi
    waitForNodes
    python run_tests.py -test deploy/rackhd_stack_init.py ${tstack} ${args} -xunit
    if [ $? -ne 0 ]; then
        echo "Test FIT failed running deploy/rackhd_stack_init.py"
        collectTestReport
        exit 1
    fi
    echo "########### Run FIT Smoke Test #############"
    python run_tests.py ${TEST_GROUP} ${tstack} ${args} -v 4 -xunit
    if [ $? -ne 0 ]; then
        echo "Test FIT failed running smoke test"
        collectTestReport
        exit 1
    fi
    collectTestReport
    popd
    set -e
}

exportLog(){
    set +e
    mkdir -p ${WORKSPACE}/build-log   
    vnc_record_stop
    generateSolLogStop
    generateRackHDLog
    generateMongoLog
    echo $SUDO_PASSWORD| sudo -S chown -R $USER:$USER ${WORKSPACE}
    set -e
}
######################################
#  OVA POST SMOKE TEST RELATED END   #
######################################

  if [ "$TEST_TYPE" == "ova" ]; then
    # based on the assumption that in the same folder, the VMs has been exist normally. so don't destroy VM here.
    
    nodesCreate
    
    # Prepare RackHD
    # Forward local host port to ova
    portForwarding ${OVA_INTERNAL_IP}

    # We setup the virtual-environment here, since once we
    # call "nodesOn", it's a race to get to the first test
    # before the nodes get booted far enough to start being
    # seen by RackHD. Logically, it would be better IN runTests.
    # We do it between the vagrant and waitForAPI to use the
    # time to make the env instead of doing sleeps...
    setupVirtualEnv
    waitForAPI
    nodesOn &

    # signal handler
    trap "deactivate && fetchOVALog" SIGINT SIGTERM SIGKILL EXIT

    # Run tests
    runTests

    # Clean Up below

    #shutdown vagrant box and delete all resource (like removing vm disk files in "~/VirtualBox VMs/")
    #cleanupVMs
    #nodesDelete
  elif [ "$TEST_TYPE" == "docker" ]; then
    # based on the assumption that in the same folder, the VMs has been exist normally. so don't destroy VM here.
    
    nodesCreate
    
    # Prepare RackHD
    # Forward local host port to ova
    portForwarding localhost
    trap exportLog SIGINT SIGTERM SIGKILL EXIT

    # We setup the virtual-environment here, since once we
    # call "nodesOn", it's a race to get to the first test
    # before the nodes get booted far enough to start being
    # seen by RackHD. Logically, it would be better IN runTests.
    # We do it between the vagrant and waitForAPI to use the
    # time to make the env instead of doing sleeps...
    setupVirtualEnv
    waitForAPI
    nodesOn &

    generateSolLog
    vnc_record_start
    # Run tests
    runTests
    # exit venv
    deactivate

    # Clean Up below

    #shutdown vagrant box and delete all resource (like removing vm disk files in "~/VirtualBox VMs/")
    #cleanupVMs
    #nodesDelete
  else
    cleanUp
    # register the signal handler to export log
    trap exportLog SIGINT SIGTERM SIGKILL EXIT
    nodesCreate
    setupTestsConfig
    dockerUp
    # Setup the virtual-environment
    setupVirtualEnv
    waitForAPI
    nodesOn

    generateSolLog
    vnc_record_start
    # Run tests
    runTests " --sm-amqp-use-user guest"
  fi