#!/bin/bash
####################################################################################
# Script name: buildproblem_bgq
# Modified: 10-14-2012 by Baowei Liu from buildproblem
# Purpose:  Compile and run modules in AstroBEAR on blue streak
####################################################################################
    
TITLE="AstroBEAR Testing on Blue Streak ALERT"
ADDRS="bliu@pas.rochester.edu"

# Declare the base scratch directory.  This is where all of the problem directories
# will be set up.  This is currently set up to run in the group space on bluehive.

tests=( BasicDisk BonnorEbertSphere Bondi BrioWuShockTube CorotatingBinary EinfeldtRarefaction FieldLoopAdvection FieldLoopRestart FillingFraction GravitationalCascade HydroWaves IsoHydroWaves IsoMHDWaves IsotropicTurbulence Marquee MHDWaves MolecularCloudFormation MomentumConservation MultiClumps OrbitingParticles RadiativeInstability0 RadiativeInstability05 RadiativeInstability1 RadShock RTInstability SingleClump SlowMolecularCloudFormation SodShockTube ThermalInstability UniformCollapse )

trac_host=clover.pas.rochester.edu
trac_astrobear=/data/trac/astrobear
test_repo=/data/repositories/tests
changeset=`hg heads | grep changeset | awk -F' ' '{ print $2 }'`


CODE_DIR=`pwd`/
MODULE=""
NUM_PROC=16
buildtests=off
uploadflag=off
resultsflag=off
cleanmake=on
runflag=on
firsttest=0

USAGE_ERR="usage: $0 [-d code_dir] [-m module] [-np processors]"

output_dir=/data/scrambler_tests/bliu/bluestreak/${changeset}
output_server=clover.pas.rochester.edu
run_dir=/scratch/bliu17/bearTest

while [ $# -gt 0 ]
do
    case "$1" in
        -d)  CODE_DIR="$2"; shift;;
	-m)  MODULE="$2"; shift;;
	-np) NUM_PROC="$2"; shift;;
	-set) changeset="$2"; shift;;
	-t)  buildtests=on;;
	-u)  uploadflag=on;;
	-td) output_server=${localhost}; output_dir="$2"; shift;;
        -c) cleanmake=off;;
        -r) runflag="$2";  shift;;
        -s) firsttest="$2";  shift;;
	-results) resultsflag=on; shift;;
	--) shift; break;;
	-*)
	    echo >&2 ${USAGE_ERR}
		    
    exit 1;;
*)  break;;# terminate while loop
    esac
    shift
done


echo "OUTPUT_SERVER: " ${output_server}

#if [ ! -d ${CODE_DIR} ]; then
    #hg clone ssh://`whoami`@clover.pas.rochester.edu//data/repositories/scrambler ${CODE_DIR}
#    hg clone ssh://bliu@clover.pas.rochester.edu//data/repositories/scrambler ${CODE_DIR}
#fi


# all command line switches are processed,
# "$@" contains all file names
 
echo ""

fetch_tests () {
# Retrieve a list of tests to execute (directories in modules/tests)
#    tests=( $( ls -l ${CODE_DIR}modules/tests | grep ^d | awk '{print $8}' ) )
 
    numtests=${#tests[@]}

    echo "Num Tests:" ${numtests}
}

fetch_test_results() {
    echo "fetch test results"
    echo -e "\n\n*****TEST RESULTS*****"
   #===================
    ssh bliu@${output_server} "rm -f ${output_dir}/tests/testlog; rm -f ${output_dir}/tests/timeslog"
   #===================
    for (( i=0;i<$numtests;i++)); do
        test=${tests[${i}]}
#=============================
    ssh bliu@${output_server} "ls ${output_dir}/tests/${test}/logs"
    ssh bliu@${output_server} "echo ${test}......... >> ${output_dir}/tests/testlog; cat ${output_dir}/tests/${test}/logs/testlog | grep '^Test Result:' | sed -e 's/^Test Result: //' >> ${output_dir}/tests/testlog"
    ssh bliu@${output_server} "echo ${test}......... >> ${output_dir}/tests/timeslog; cat ${output_dir}/tests/${test}/logs/testlog | grep 'Runtime' >> ${output_dir}/tests/timeslog"
    done

    scp bliu@${output_server}:${output_dir}/tests/testlog ${run_dir}/
    scp bliu@${output_server}:${output_dir}/tests/timeslog ${run_dir}/
#============================
    cp ${run_dir}/testlog ./
    cp ${run_dir}/timeslog ./
#    cat ${run_dir}/testlog | mail -s "bearTest" $ADDRS
#    cat ${run_dir}/timeslog | mail -s "bearTest" $ADDRS
}

if [ ${resultsflag} == on ]; then  # If only results are requested then gather them and exit
    fetch_tests
    fetch_test_results
    exit 0
fi


if [ ${buildtests} == on ]; then

    fetch_tests

    echo "NUMPROCS is:" ${NUM_PROC}
    echo "Output Directory:" ${output_dir}
    # setup test repository
    echo '*** creating output_dir on remote host'
    ssh bliu@${output_server} "if [ ! -d ${output_dir} ]; then mkdir ${output_dir}; fi"
    echo '*** checking out test repository from svn'
    ssh bliu@${output_server} "svn co -q svn+ssh://bliu@clover.pas.rochester.edu/data/repositories/tests ${output_dir}/tests; rm -f ${output_dir}/tests/*/images/sim.png" 
    ssh bliu@clover.pas.rochester.edu "rm -f /data/trac/astrobear/doc/bliu_bluestreak_current_test; ln -s ${output_dir}/tests /data/trac/astrobear/doc/bliu_bluestreak_current_test; chmod 775 /data/trac/astrobear/doc/bliu_bluestreak_current_test; chgrp orda /data/trac/astrobear/doc/bliu_bluestreak_current_test"

    # setup bear2fix
    echo '*** compiling bear2fix'
    ssh bliu@${output_server} "cd ~/test_suite_run_bgq/; rm -rf *"
    scp -r ${CODE_DIR}/bear2fix bliu@${output_server}:~/test_suite_run_bgq/
    ssh bliu@${output_server} "cd ~/test_suite_run_bgq/bear2fix; rm -f Makefile.inc; ln -s Makefile.inc.clover Makefile.inc && make -s clean && make -s > /dev/null; cp -R ~/test_suite_run_bgq/bear2fix/ ${output_dir}/tests"
    
    cd ${CODE_DIR}
    # Link to the test makefile.
    rm -f ${CODE_DIR}Makefile.inc
    #ln -s ${CODE_DIR}Makefile.inc.`hostname | cut -f1 -d.`  ${CODE_DIR}Makefile.inc
    ln -s ${CODE_DIR}Makefile.inc.bluestreak  ${CODE_DIR}Makefile.inc
    # Link to the selected test's problem directory.
    rm -f ${CODE_DIR}modules/Problem
    ln -s ${CODE_DIR}modules/${MODULE} ${CODE_DIR}modules/Problem

    # Compile the code using the specified problem.
    if [ ${cleanmake} == on ]; then
	make -s clean
    else
	rm astrobear
    fi
    
    # Execute all of the tests
    for (( i=$firsttest;i<$numtests;i++)); do
	test=${tests[${i}]}
	echo "${CODE_DIR}buildproblem_bgq -d ${CODE_DIR} -m ${test} -np ${NUM_PROC} -set ${changeset} -u -c -r ${runflag}"
        ${CODE_DIR}buildproblem_bgq -d ${CODE_DIR} -m ${test} -np ${NUM_PROC} -set ${changeset} -u -c -r ${runflag} >/dev/null

    done
    
    fetch_test_results
    if [ `cat testlog | grep PASS | wc -l` == ${numtests} ]; then 
        cat ${run_dir}/testlog | mail -s "Blue Streak: All Tests Passed" $ADDRS
	if [ ${uploadflag} == on ]; then
		ssh bliu@${output_server} "cd ${output_dir}/tests && svn ci -m 'commited after successfully passing tests with AstroBEAR changeset ${changeset}' && svn update"
		ssh bliu@${output_server} "exit `svn info | grep '^Revision:' | sed -e 's/^Revision: //'`"
		REVISION=$?
		if [ `localhost` == ${trac_host} ]; then
	    	trac-admin ${trac_astrobear} changeset added ${test_repo} ${REVISION}
		else
	    	ssh `whoami`@${trac_host} "trac-admin ${trac_astrobear} changeset added ${test_repo} ${REVISION}"
		fi
	fi
    else
	#echo -e "`cat testlog | grep PASS | wc -l` out of ${numtests} passed"
        #testsPassed=`cat testlog | grep PASS | wc -l`
        cat ${run_dir}/testlog | mail -s "Blue Streak: Tests Failed" $ADDRS
	exit 1
    fi

    exit 0
fi #if buildtests=on


while [ ! -d "${CODE_DIR}modules/${MODULE}" ]
do
    echo Please select one of these modules in ${CODE_DIR}modules:
    echo ""

    ls -l ${CODE_DIR}modules | grep ^d | awk '{print $8}'

    echo ""
    echo -n "Module Name: " 
    read MODULE

done

cd ${CODE_DIR}

# Link to the test makefile.
rm -f ${CODE_DIR}Makefile.inc
ln -s ${CODE_DIR}Makefile.inc.bluestreak  ${CODE_DIR}Makefile.inc
ls -l Makefile.inc

# Link to the selected test's problem directory.
rm -f ${CODE_DIR}modules/Problem
ln -s ${CODE_DIR}modules/${MODULE} ${CODE_DIR}modules/Problem

# Compile the code using the specified problem.

if [ ${cleanmake} == on ]; then
    make -s clean
else
    touch modules/Problem/problem.f90
fi
make -s > /dev/null

if [ -f astrobear ];  then
    echo ${MODULE} "make done."
else
    echo -e ${MODULE} "failed to compile.  Aborting"
    exit 1
fi

nprocs=${NUM_PROC}

# Specify the name of the problem directory for this test.  The directory name will include
# the test's name, plus the number of processors and the number of levels of refinement.

# If this problem directory does not exist, then create it.  This includes creating the
# out/ sub directory and copying over the cooling tables.
if [ ! -e ${run_dir} ]; then
    mkdir ${run_dir}
else
   cd ${run_dir}
   rm -rf *
fi

if [ ! -e ${run_dir}/TABLES ]; then
    cp -r ${CODE_DIR}source/TABLES ${run_dir}
fi 

# Copy over the newly-compiled executable.
cp ${CODE_DIR}astrobear ${run_dir}

# Copy the problem data files to the run directory.
# Copy the bear2fix.data files to the run directory.
# Copy the script files to the run directory.
if [ ${CODE_DIR}modules/${MODULE} != ${run_dir} ]; then
    echo *** 'copying over data files to run directory'
    cp ${CODE_DIR}modules/${MODULE}/*.data ${run_dir}

    cp ${CODE_DIR}modules/${MODULE}/bear2fix.* ${run_dir}
    cp ${CODE_DIR}modules/${MODULE}/*.s* ${run_dir}
    if [ ${MODULE} == BrioWuShockTube ]; then
    cp ${CODE_DIR}modules/${MODULE}/exact.curve ${run_dir}
    fi
    if [ ${MODULE} == FieldLoopRestart ]; then
    cp -r ${CODE_DIR}modules/${MODULE}/out ${run_dir}
    fi
fi

if [ ! -e ${run_dir}/out ]; then
    mkdir ${run_dir}/out
fi

echo ${MODULE} "copy done."

echo -e "\n"

rm ${run_dir}/runTestscript_slurm.sh
cat >> ${run_dir}/runTestscript_slurm.sh <<EOF
#!/bin/bash
EOF
echo '#SBATCH -J bearTest' >> ${run_dir}/runTestscript_slurm.sh
cat >> ${run_dir}/runTestscript_slurm.sh <<EOF
#SBATCH -N 1
EOF

echo '#SBATCH -n '$NUM_PROC >> ${run_dir}/runTestscript_slurm.sh 

cat >> ${run_dir}/runTestscript_slurm.sh <<EOF
#SBATCH -c 1 
#SBATCH -p debug 
#SBATCH -t 01:00:00

module purge
module load mpi-xl
module load hdf5-1.8.8-MPI-XL 
module load fftw-3.3.2-MPI-XL
module load hypre-2.8.0b-MPI-XL

ulimit -s unlimited

#1node 8 processors
EOF
echo 'srun astrobear > '${MODULE}.log >> ${run_dir}/runTestscript_slurm.sh


if [ ${NUM_PROC} -gt 0 ]; then

    echo 'Removing old processing files'
    ssh bliu@${output_server} "rm -f ${output_dir}/tests/${MODULE}/images/sim.png; rm -f ${output_dir}/tests/${MODULE}/logs/*; echo 'Test Result: FAIL' > ${output_dir}/tests/${MODULE}/logs/testlog"

    echo -e Running ${run_dir}/astrobear "...\c"
    pushd ${run_dir}
    echo "Run Directory is:" ${run_dir}
    log_file=${MODULE}.log

    if [ ${runflag} == on ]; then
      rm profile.data
      if [ -e testrun.s.q ]; then
        mkdir ${run_dir}/testdata
        #./testrun.s.q  > ${run_dir}/${log_file}
        ./testrun.s.q
      else
          sbatch runTestscript_slurm.sh 
          sleep 4
      fi
    else
	echo 'skipping running'
    #wait
    fi
  fi

  if [ $? != 0 ]; then
    echo "submiting jobs returned $?"
  else
    jobRunning=1
    jobQueue=0
    #numChomb=0
    while [[ ${jobRunning} -ge 1 || ${jobQueue} -ge 1 ]] 
    do
    jobRunning=`qstat | grep bearTest | awk '{print $5}' |  grep "R" | wc | awk '{print $1}'`
    jobQueue=`qstat | grep bearTest | awk '{print $5}' |  grep "Q" | wc | awk '{print $1}'`
    #numChomb=`ls ${run_dir}/out/chombo*.hdf | wc | awk '{print $1}'`
    sleep 4
    done
  fi

  jobSuccess=`grep "stats" ${run_dir}/$log_file | wc | awk '{print $1}'` 

  if [[  -e out/chombo00000.hdf && -e out/chombo00001.hdf ]]; then
     chomboFiles=2
  fi
  
  if [ ${MODULE} == FieldLoopRestart ]; then
     if [[  -e out/chombo00001.hdf && -e out/chombo00002.hdf ]]; then
        chomboFiles=2
     fi
  fi

  if [[ ${jobSuccess} -ge 2 || chomboFiles -ge 2 ]]; then 
      if [ -e ${run_dir}/testprocess.s ]; then
          files="${run_dir}/testprocess.s"
          outfiles="${run_dir}/testdata/*"

      else
          FRAME=`cat ${run_dir}/bear2fix.data.img | grep -w FRAME | tr -dc '[0-9]'`
          if [ ! -e ${run_dir}/out/chombo0000${FRAME}.hdf ]; then
              echo -e '*** Unable to finding expected output frame for post processing.  ${MODULE} likely failed.'
              exit 1
          else
              echo '*** uploading run results to server'
              outfiles=${run_dir}/out/chombo0000${FRAME}.hdf
              files=${run_dir}/bear2fix.data.*
          fi
      fi

      outfiles=${outfiles}" ${run_dir}/${log_file}"
      files=${files}" ${run_dir}/scales.data"
      
      scp ${files} bliu@${output_server}:${output_dir}/tests/${MODULE}
      ssh bliu@${output_server} "rm -rf ${output_dir}/tests/${MODULE}/out && mkdir -p ${output_dir}/tests/${MODULE}/out"
      scp ${outfiles} bliu@${output_server}:${output_dir}/tests/${MODULE}/out
      ssh bliu@${output_server} "${output_dir}/tests/process_chombos -m tests/${MODULE} -c ${changeset} -d ${output_dir}"
      if [ $? != 0 ]; then
          echo "${output_dir}/tests/process_chombos -m tests/${MODULE} -c ${changeset} -d ${output_dir} returned $?"
          exit 1
      fi
  else
      echo -e "failed. check ${run_dir}/${log_file} for details "
      exit 1   
  fi
echo "done."

