#!/bin/bash
####################################################################################
# Script name: buildproblem_bh
# Created:  10-14-2011 by Matthew Noyes
# Modified: 11-14-2012 by Baowei Liu
# Purpose:  Compile and run modules in AstroBEAR on bluehive 
####################################################################################
    
TITLE="AstroBEAR Testing on BlueHive ALERT"
ADDRS="bliu@pas.rochester.edu"

# Declare the base scratch directory.  This is where all of the problem directories
# will be set up.  This is currently set up to run in the group space on bluehive.

tests=( BasicDisk BonnorEbertSphere Bondi BrioWuShockTube CorotatingBinary EinfeldtRarefaction FieldLoopAdvection FieldLoopRestart FillingFraction GravitationalCascade HydroWaves IsoHydroWaves IsoMHDWaves IsotropicTurbulence Marquee MHDWaves MolecularCloudFormation MomentumConservation MultiClumps OrbitingParticles RadiativeInstability0 RadiativeInstability05 RadiativeInstability1 RadShock RTInstability SingleClump SlowMolecularCloudFormation SodShockTube ThermalInstability UniformCollapse )


trac_host=clover.pas.rochester.edu
trac_astrobear=/data/trac/astrobear
test_repo=/data/repositories/tests
changeset=`hg heads | grep changeset | awk -F' ' '{ print $2 }'`


CODE_DIR=`pwd`/
MODULE=""
NUM_PROC=8
buildtests=off
uploadflag=off
resultsflag=off
cleanmake=on
runflag=on
firsttest=0

USAGE_ERR="usage: $0 [-d code_dir] [-m module] [-np processors]"

output_dir=/data/scrambler_tests/bliu/bluehive/${changeset}
output_server=clover.pas.rochester.edu
run_dir=${CODE_DIR}modules/Problem

while [ $# -gt 0 ]
do
    case "$1" in
        -d)  CODE_DIR="$2"; shift;;
	-m)  MODULE="$2"; shift;;
	-np) NUM_PROC="$2"; shift;;
	-set) changeset="$2"; shift;;
	-t)  buildtests=on;;
	-u)  uploadflag=on;;
	-td) output_server=${localhost}; output_dir="$2"; shift;;
        -c) cleanmake=off;;
        -r) runflag="$2";  shift;;
        -s) firsttest="$2";  shift;;
	-results) resultsflag=on; shift;;
	--) shift; break;;
	-*)
	    echo >&2 ${USAGE_ERR}
		    
    exit 1;;
*)  break;;# terminate while loop
    esac
    shift
done


echo "OUTPUT_SERVER: " ${output_server}

#if [ ! -d ${CODE_DIR} ]; then
    #hg clone ssh://`whoami`@clover.pas.rochester.edu//data/repositories/scrambler ${CODE_DIR}
#    hg clone ssh://bliu@clover.pas.rochester.edu//data/repositories/scrambler ${CODE_DIR}
#fi


# all command line switches are processed,
# "$@" contains all file names
 
echo ""

fetch_tests () {
# Retrieve a list of tests to execute (directories in modules/tests)
#    tests=( $( ls -l ${CODE_DIR}modules/tests | grep ^d | awk '{print $8}' ) )
 
    numtests=${#tests[@]}

    echo "Num Tests:" ${numtests}
}

fetch_test_results() {
    echo "fetch test results"
    echo -e "\n\n*****TEST RESULTS*****"
   #===================
    ssh bliu@${output_server} "rm -f ${output_dir}/tests/testlog; rm -f ${output_dir}/tests/timeslog"
   #===================
    for (( i=0;i<$numtests;i++)); do
        test=${tests[${i}]}
#=============================
    ssh bliu@${output_server} "ls ${output_dir}/tests/${test}/logs"
    ssh bliu@${output_server} "echo ${test}......... >> ${output_dir}/tests/testlog; cat ${output_dir}/tests/${test}/logs/testlog | grep '^Test Result:' | sed -e 's/^Test Result: //' >> ${output_dir}/tests/testlog"
    ssh bliu@${output_server} "echo ${test}......... >> ${output_dir}/tests/timeslog; cat ${output_dir}/tests/${test}/logs/testlog | grep 'Runtime' >> ${output_dir}/tests/timeslog"
    done

    scp bliu@${output_server}:${output_dir}/tests/testlog ${run_dir}/ 
    scp bliu@${output_server}:${output_dir}/tests/timeslog ${run_dir}/
#============================
    cp ${run_dir}/testlog ./
    cp ${run_dir}/timeslog ./
#    cat ${run_dir}/testlog | mail -s "bearTest" $ADDRS
#    cat ${run_dir}/timeslog | mail -s "bearTest" $ADDRS
}

if [ ${resultsflag} == on ]; then  # If only results are requested then gather them and exit
    fetch_tests
    fetch_test_results
    exit 0
fi


if [ ${buildtests} == on ]; then

    fetch_tests

    echo "NUMPROCS is:" ${NUM_PROC}
    echo "Output Directory:" ${output_dir}
    # setup test repository
    echo '*** creating output_dir on remote host'
    ssh bliu@${output_server} "if [ ! -d ${output_dir} ]; then mkdir ${output_dir}; fi"
    echo '*** checking out test repository from svn'
    ssh bliu@${output_server} "svn co -q svn+ssh://bliu@clover.pas.rochester.edu/data/repositories/tests ${output_dir}/tests; rm -f ${output_dir}/tests/*/images/sim.png" 
    ssh bliu@clover.pas.rochester.edu "rm -f /data/trac/astrobear/doc/bliu_bluehive_current_test; ln -s ${output_dir}/tests /data/trac/astrobear/doc/bliu_bluehive_current_test; chmod 775 /data/trac/astrobear/doc/bliu_bluehive_current_test; chgrp orda /data/trac/astrobear/doc/bliu_bluehive_current_test"

    # setup bear2fix
    echo '*** compiling bear2fix'
    ssh bliu@{$output_server} "cd ~/test_suite_run/; rm -rf *"
    scp -r ${CODE_DIR}/bear2fix bliu@${output_server}:~/test_suite_run/
    ssh bliu@${output_server} "cd ~/test_suite_run/bear2fix; rm -f Makefile.inc; ln -s Makefile.inc.clover Makefile.inc && make -s clean && make -s > /dev/null; cp -R ~/test_suite_run/bear2fix/ ${output_dir}/tests"
    
    cd ${CODE_DIR}
    # Link to the test makefile.
    rm -f ${CODE_DIR}Makefile.inc
    #ln -s ${CODE_DIR}Makefile.inc.`hostname | cut -f1 -d.`  ${CODE_DIR}Makefile.inc
    ln -s ${CODE_DIR}Makefile.inc.bluehive  ${CODE_DIR}Makefile.inc
    # Link to the selected test's problem directory.
    #ls -l
    # Compile the code using the specified problem.
    if [ ${cleanmake} == on ]; then
	make -s clean
    else
	rm astrobear
    fi
    
    # Execute all of the tests
    for (( i=$firsttest;i<$numtests;i++)); do
	test=${tests[${i}]}
        rm -f ${CODE_DIR}modules/Problem
        ln -s ${CODE_DIR}modules/${test} ${CODE_DIR}modules/Problem
	echo "${CODE_DIR}buildproblem_bh -d ${CODE_DIR} -m ${test} -np ${NUM_PROC} -set ${changeset} -u -c -r ${runflag}"
        ${CODE_DIR}buildproblem_bh -d ${CODE_DIR} -m ${test} -np ${NUM_PROC} -set ${changeset} -u -c -r ${runflag} 

    done
    
    fetch_test_results
    if [ `cat testlog | grep PASS | wc -l` == ${numtests} ]; then 
        cat ${run_dir}/testlog | mail -s "BlueHive: All Tests Passed" $ADDRS
        if [ ${uploadflag} == on ]; then
		ssh bliu@${output_server} "cd ${output_dir}/tests && svn ci -m 'commited after successfully passing tests with AstroBEAR changeset ${changeset}' && svn update"
		ssh bliu@${output_server} "exit `svn info | grep '^Revision:' | sed -e 's/^Revision: //'`"
		REVISION=$?
		if [ `localhost` == ${trac_host} ]; then
		    trac-admin ${trac_astrobear} changeset added ${test_repo} ${REVISION}
		else
		    ssh `whoami`@${trac_host} "trac-admin ${trac_astrobear} changeset added ${test_repo} ${REVISION}"
		fi
	fi
    else
	#echo -e "`cat testlog | grep PASS | wc -l` out of ${numtests} passed"
        #testsPassed=`cat testlog | grep PASS | wc -l`
        cat ${run_dir}/testlog | mail -s "BlueHive: Tests Failed" $ADDRS
	exit 1
    fi

    exit 0
fi #if buildtests=on


while [ ! -d "${CODE_DIR}modules/${MODULE}" ]
do
    echo Please select one of these modules in ${CODE_DIR}modules:
    echo ""

    ls -l ${CODE_DIR}modules | grep ^d | awk '{print $8}'

    echo ""
    echo -n "Module Name: " 
    read MODULE

done

cd ${CODE_DIR}


# Link to the test makefile.
rm -f ${CODE_DIR}Makefile.inc
ln -s ${CODE_DIR}Makefile.inc.bluehive  ${CODE_DIR}Makefile.inc
ls -l Makefile.inc

# Link to the selected test's problem directory.
rm -f ${CODE_DIR}modules/Problem
ln -s ${CODE_DIR}modules/${MODULE} ${CODE_DIR}modules/Problem

# Compile the code using the specified problem.

if [ ${cleanmake} == on ]; then
    make -s clean
else
    touch modules/Problem/problem.f90
fi
make -s > /dev/null

if [ -f astrobear ];  then
    echo ${MODULE} "make done."
else
    echo -e ${MODULE} "failed to compile.  Aborting"
    exit 1
fi

nprocs=${NUM_PROC}

# Specify the name of the problem directory for this test.  The directory name will include
# the test's name, plus the number of processors and the number of levels of refinement.

# If this problem directory does not exist, then create it.  This includes creating the
# out/ sub directory and copying over the cooling tables.

if [ ! -e ${run_dir}/out ]; then
   mkdir ${run_dir}/out
fi

if [ ! -e ${run_dir}/TABLES ]; then
    cp -r ${CODE_DIR}source/TABLES ${run_dir}
fi

# Copy over the newly-compiled executable.
#cp ${CODE_DIR}astrobear ${run_dir}

# Copy the problem data files to the run directory.
# Copy the bear2fix.data files to the run directory.
# Copy the script files to the run directory.
#if [ ${CODE_DIR}modules/${MODULE} != ${run_dir} ]; then
#    echo *** 'copying over data files to run directory'
#    cp ${CODE_DIR}modules/${MODULE}/*.data ${run_dir}
#    cp ${CODE_DIR}modules/${MODULE}/bear2fix.* ${run_dir}
#    cp ${CODE_DIR}modules/${MODULE}/*.s* ${run_dir}
#    if [ ${MODULE} == BrioWuShockTube ]; then
#    cp ${CODE_DIR}modules/${MODULE}/exact.curve ${run_dir}
#    fi
#fi

echo ${MODULE} "copy done."

echo -e "\n"

rm ${run_dir}/runTestscript_pbs.sh
cat >> ${run_dir}/runTestscript_pbs.sh <<EOF
#!/bin/bash
#PBS -N bearTest
EOF

echo '#PBS -l nodes=1:ppn='$NUM_PROC',pvmem=2000mb' >> ${run_dir}/runTestscript_pbs.sh 

cat >> ${run_dir}/runTestscript_pbs.sh <<EOF
#PBS -l walltime=1:00:00 
#PBS -q debug 
#PBS -j oe
EOF
echo '#PBS -o '${MODULE}'.log' >> ${run_dir}/runTestscript_pbs.sh
cat >> ${run_dir}/runTestscript_pbs.sh <<EOF
cd \$PBS_O_WORKDIR
ulimit -s unlimited
export LD_LIBRARY_PATH=/home/jcarrol5/local/mvapich2_hdf5/lib:\$LD_LIBRARY_PATH

EOF
echo 'mpirun -np '${NUM_PROC}' ../../astrobear' >> ${run_dir}/runTestscript_pbs.sh

if [ ${NUM_PROC} -gt 0 ]; then

    echo 'Removing old processing files'
    ssh bliu@${output_server} "rm -f ${output_dir}/tests/${MODULE}/images/sim.png; rm -f ${output_dir}/tests/${MODULE}/logs/*; echo 'Test Result: FAIL' > ${output_dir}/tests/${MODULE}/logs/testlog"

    echo -e Running ${run_dir}/astrobear "...\c"
    pushd ${run_dir}
    echo "Run Directory is:" ${run_dir}
    log_file=${MODULE}.log

    if [ ${runflag} == on ]; then
      rm profile.data
      if [ -e testrun.s.bh ]; then
        mkdir ${run_dir}/testdata
        #./testrun.s.q  > ${run_dir}/${log_file}
        ./testrun.s.bh
      else
          qsub runTestscript_pbs.sh 
          sleep 4
      fi
    else
	echo 'skipping running'
    #wait
    fi
  fi

  if [ $? != 0 ]; then
    echo "submiting jobs returned $?"
  else
    jobRunning=1
    jobQueue=0
    while [[ ${jobRunning} -ge 1 || ${jobQueue} -ge 1 ]]
    do
    jobRunning=`qstat | grep bearTest | awk '{print $5}' |  grep "R" | wc | awk '{print $1}'`
    jobQueue=`qstat | grep bearTest | awk '{print $5}' |  grep "Q" | wc | awk '{print $1}'`
    sleep 4
    done
  fi

  jobSuccess=`grep "stats" ${run_dir}/$log_file | wc | awk '{print $1}'` 

  if [[  -e out/chombo00000.hdf && -e out/chombo00001.hdf ]]; then
     chomboFiles=2
  fi
  
  if [ ${MODULE} == FieldLoopRestart ]; then
     if [[  -e out/chombo00001.hdf && -e out/chombo00002.hdf ]]; then
        chomboFiles=2
     fi
  fi

  if [[ ${jobSuccess} -ge 2  || chomboFiles -eq 2 ]]; then 
      if [ -e ${run_dir}/testprocess.s ]; then
          files="${run_dir}/testprocess.s"
          outfiles="${run_dir}/testdata/*"

      else
           FRAME=`cat ${run_dir}/bear2fix.data.img | grep -w FRAME | tr -dc '[0-9]'`
           if [ ! -e ${run_dir}/out/chombo0000${FRAME}.hdf ]; then
               echo -e '*** Unable to finding expected output frame for post processing.  ${MODULE} likely failed.'
               exit 1
           else
               echo '*** uploading run results to server'
               outfiles=${run_dir}/out/chombo0000${FRAME}.hdf
               files=${run_dir}/bear2fix.data.*
           fi
       fi

       outfiles=${outfiles}" ${run_dir}/${log_file}"
       files=${files}" ${run_dir}/scales.data"
      
       scp ${files} bliu@${output_server}:${output_dir}/tests/${MODULE}
       ssh bliu@${output_server} "rm -rf ${output_dir}/tests/${MODULE}/out && mkdir -p ${output_dir}/tests/${MODULE}/out"
       scp ${outfiles} bliu@${output_server}:${output_dir}/tests/${MODULE}/out
       ssh bliu@${output_server} "${output_dir}/tests/process_chombos -m tests/${MODULE} -c ${changeset} -d ${output_dir}"
       if [ $? != 0 ]; then
           echo "${output_dir}/tests/process_chombos -m tests/${MODULE} -c ${changeset} -d ${output_dir} returned $?"
           exit 1
       fi
  else
      echo -e "failed. check ${run_dir}/${log_file} for details "
      exit 1   
  fi
echo "done."

