#!/bin/bash
####################################################################################
# Script name: buildproblem
# Created:  10-14-2011 by Matthew Noyes
# Purpose:  Compile and run modules in AstroBEAR
####################################################################################


# Declare the base scratch directory.  This is where all of the problem directories
# will be set up.  This is currently set up to run in the group space on bluehive.

trac_host=clover.pas.rochester.edu
trac_astrobear=/data/trac/astrobear
test_repo=/data/repositories/tests
localhost=`hostname | cut -f1 -d.`
#changeset=`hg heads | grep changeset | awk -F' ' '{ print $2 }'`
changeset=988:af45abad226a


CODE_DIR=`pwd`/
MODULE=""
NUM_PROC=8
buildtests=off
uploadflag=off
resultsflag=off
cleanmake=on
runflag=on
firsttest=0
RES_ID=bgpfen.235.r

USAGE_ERR="usage: $0 [-d code_dir] [-m module] [-np processors]"

output_dir=/data/scrambler_tests/${changeset}/tests
output_server=clover.pas.rochester.edu


while [ $# -gt 0 ]
do
    case "$1" in
        -d)  CODE_DIR="$2"; shift;;
	-m)  MODULE="$2"; shift;;
	-np) NUM_PROC="$2"; shift;;
	-t)  buildtests=on;;
	-u)  uploadflag=on;;
	-td) output_server=${localhost}; output_dir="$2"; shift;;
        -c) cleanmake=off;;
        -r) runflag="$2";  shift;;
        -rid) RES_ID="$2"; shift;;
	-results) resultsflag=on; shift;;
	--) shift; break;;
	-*)
	    echo >&2 ${USAGE_ERR}
		    
    exit 1;;
*)  break;;# terminate while loop
    esac
    shift
done


echo "OUTPUT_SERVER: " ${output_server}

if [ ! -d ${CODE_DIR} ]; then
    #hg clone ssh://`whoami`@clover.pas.rochester.edu//data/repositories/scrambler ${CODE_DIR}
    hg clone ssh://bliu@clover.pas.rochester.edu//data/repositories/scrambler ${CODE_DIR}
fi


# all command line switches are processed,
# "$@" contains all file names
 
echo ""

fetch_tests () {
# Retrieve a list of tests to execute (directories in modules/tests)
#    tests=( $( ls -l ${CODE_DIR}modules/tests | grep ^d | awk '{print $8}' ) )
#    tests=RTInstability
#     tests=( Bondi FieldLoopAdvection FieldLoopRestart OrbitingParticles RadiativeInstability0 RadiativeInstability05 RadiativeInstability1 RTInstability UniformCollapse )
tests=( BasicDisk BonnorEbertSphere Bondi BrioWuShockTube CorotatingBinary EinfeldtRarefaction FieldLoopAdvection FieldLoopRestart FillingFraction GravitationalCascade HydroWaves IsoHydroWaves IsoMHDWaves IsotropicTurbulence Marquee MolecularCloudFormation MomentumConservation MultiClumps OrbitingParticles RadiativeInstability0 RadiativeInstability05 RadiativeInstability1 RadShock RTInstability SingleClump SlowMolecularCloudFormation SodShockTube ThermalInstability UniformCollapse )
    numtests=${#tests[@]}

    echo "Num Tests:" ${numtests}
}

fetch_test_results() {
echo "fetch test results"
echo -e "\n\n*****TEST RESULTS*****"
ssh bliu@${output_server} "rm -f ${output_dir}/tests/testlog"
   #===================
for (( i=$firsttest;i<$numtests;i++)); do
    test=${tests[${i}]}
#=============================
    ssh bliu@${output_server} "ls ${output_dir}/tests/${test}/logs"
    ssh bliu@${output_server} "echo ${test}......... >> ${output_dir}/tests/testlog; cat ${output_dir}/tests/${test}/logs/testlog | grep '^Test Result:' | sed -e 's/^Test Result: //' >> ${output_dir}/tests/testlog"
#============================
#	    ssh `whoami`@${output_server} "echo ${test}......... >> ${output_dir}/tests/timeslog; cat ${output_dir}/tests/${test}/logs/testlog | grep 'Runtime' >> ${output_dir}/tests/times"
#	done
#	scp `whoami`@${output_server}:${output_dir}/tests/testlog ./
#	scp `whoami`@${output_server}:${output_dir}/tests/timeslog ./
#============================
    ssh bliu@${output_server} "echo ${test}......... >> ${output_dir}/tests/timeslog; cat ${output_dir}/tests/${test}/logs/testlog | grep 'Runtime' >> ${output_dir}/tests/times"
done
scp bliu@${output_server}:${output_dir}/tests/testlog ./
scp bliu@${output_server}:${output_dir}/tests/timeslog ./
#============================
cat ./testlog
cat ./timeslog
}

if [ ${resultsflag} == on ]; then  # If only results are requested then gather them and exit
    fetch_tests
    fetch_test_results
    exit 0
fi

if [ ${buildtests} == on ]; then

    fetch_tests

    echo "NUMPROCS is:" ${NUM_PROC}
    echo "Output Directory:" ${output_dir}
    # setup test repository
    echo '*** creating output_dir on remote host'
    ssh bliu@${output_server} "if [ ! -d ${output_dir} ]; then mkdir ${output_dir}; fi"
    echo '*** checking out test repository from svn'
    ssh bliu@${output_server} "svn co -q svn+ssh://bliu@clover.pas.rochester.edu/data/repositories/tests ${output_dir}/tests; rm -f ${output_dir}/tests/*/images/sim.png" 
    ssh bliu@clover.pas.rochester.edu "rm -f /data/trac/astrobear/doc/current_test; ln -s ${output_dir}/tests /data/trac/astrobear/doc/current_test; chmod 775 /data/trac/astrobear/doc/current_test; chgrp orda /data/trac/astrobear/doc/current_test"

    # setup bear2fix
    echo '*** compiling bear2fix'
    scp -r ${CODE_DIR}/bear2fix bliu@${output_server}:~/test_suite_run/
    ssh bliu@${output_server} "cd ~/test_suite_run/bear2fix; rm -f Makefile.inc; ln -s Makefile.inc.clover Makefile.inc && make -s clean && make -s > /dev/null; cp -R ~/test_suite_run/bear2fix/ ${output_dir}/tests"
    
    # Link to the selected test's problem directory.
    cd ${CODE_DIR}
    # Link to the test makefile.
    rm -f ${CODE_DIR}Makefile.inc
    #ln -s ${CODE_DIR}Makefile.inc.`hostname`  ${CODE_DIR}Makefile.inc
    ln -s ${CODE_DIR}Makefile.inc.bluegene  ${CODE_DIR}Makefile.inc
    # Link to the selected test's problem directory.
    rm -f ${CODE_DIR}modules/Problem
    ln -s ${CODE_DIR}modules/${MODULE} ${CODE_DIR}modules/Problem

    # Compile the code using the specified problem.
    if [ ${cleanmake} == on ]; then
	make -s clean
    else
	rm astrobear
    fi

    echo "before Execute all of the tests:"${buildtests}
      
    # Execute all of the tests
    for (( i=$firsttest;i<$numtests;i++)); do
	test=${tests[${i}]}
	echo "${CODE_DIR}buildproblem_bg -d ${CODE_DIR} -m tests/${test} -np ${NUM_PROC} -u -c -r ${runflag}"
	${CODE_DIR}buildproblem_bg -d ${CODE_DIR} -m ${test} -np ${NUM_PROC} -u -c -r ${runflag}
	if [ $? != 0 ]; then
          echo -e "${test} failed... "
	  read -p "Continue/Retry/Abort/reProcess (C/R/A/P)? "
	  if [ "$REPLY" == "A" ]; then
	      exit 1
	  else
	      if [ "$REPLY" == "P" ]; then
		  runflag=off
		  i=$(( i - 1 ))
	      else
		  runflag=on
		  if [ "$REPLY" == "R" ]; then
                      i=$(( i - 1 ))
		  fi
	      fi
	  fi
	else
	    runflag=on
        fi
    done
    
    fetch_test_results

    if [ `cat testlog | grep PASS | wc -l` == ${numtests} ]; then 
	if [ ${uploadflag} == on ]; then
	   ssh bliu@${output_server} "cd ${output_dir}/tests && svn ci -m 'commited after successfully passing tests with AstroBEAR changeset ${changeset}' && svn update"
	   ssh bliu@${output_server} "exit `svn info | grep '^Revision:' | sed -e 's/^Revision: //'`"
		REVISION=$?
	   ssh bliu@${trac_host} "trac-admin ${trac_astrobear} changeset added ${test_repo} ${REVISION}"
	fi
    else
	echo -e "only `cat testlog | grep PASS | wc -l` out of ${numtests} passed"
	exit 1
    fi
    
    exit 0
fi #if buildtests=on


while [ ! -d "${CODE_DIR}modules/${MODULE}" ]
do
    echo Please select one of these modules in ${CODE_DIR}modules:
    echo ""

    ls -l ${CODE_DIR}modules | grep ^d | awk '{print $8}'

    echo ""
    echo -n "Module Name: " 
    read MODULE

done

cd ${CODE_DIR}

# Link to the test makefile.
rm -f ${CODE_DIR}Makefile.inc
ln -s ${CODE_DIR}Makefile.inc.bluegene  ${CODE_DIR}Makefile.inc
ls -l Makefile.inc

# Link to the selected test's problem directory.
rm -f ${CODE_DIR}modules/Problem
ln -s ${CODE_DIR}modules/${MODULE} ${CODE_DIR}modules/Problem

# Compile the code using the specified problem.

if [ ${cleanmake} == on ]; then
    make -s clean
else
    touch modules/Problem/problem.f90
fi
make -s > /dev/null

if [ -f astrobear ];  then
    echo ${MODULE} "make done."
else
    echo -e ${MODULE} "failed to compile.  Aborting"
    exit 1
fi

nprocs=${NUM_PROC}

# Specify the name of the problem directory for this test.  The directory name will include
# the test's name, plus the number of processors and the number of levels of refinement.
run_dir="${CODE_DIR}modules/${MODULE}"

# If this problem directory does not exist, then create it.  This includes creating the
# out/ sub directory and copying over the cooling tables.
if [ ! -e ${run_dir} ]; then
    mkdir ${run_dir}
fi

if [ ! -e ${run_dir}/TABLES ]; then
    cp -r ${CODE_DIR}source/TABLES ${run_dir}
fi 

if [ ! -e ${run_dir}/out ]; then
    mkdir ${run_dir}/out
fi

# Copy over the newly-compiled executable.
cp ${CODE_DIR}astrobear ${run_dir}

# Copy the problem data files to the run directory.
if [ ${CODE_DIR}modules/${MODULE} != ${run_dir} ]; then
    echo *** 'copying over data files to run directory'
    cp ${CODE_DIR}modules/${MODULE}/*.data ${run_dir}
fi

echo ${MODULE} "copy done."

echo -e "\n"

rm ${run_dir}/runTestscript.bg
cat >> ${run_dir}/runTestscript.bg <<EOF
#@ job_type = bluegene
#@ job_name = bearTest 
#@ error = \$(job_name).\$(jobid).err 
#@ output = \$(job_name).\$(jobid).out
#@ environment = COPY_ALL;
#@ notification = never
#@ wall_clock_limit = 00:19:59
#@ bg_size = 64
#@ class = small 
#@ queue
export BG_MAXALIGNEXP=-1
EOF

echo "output_server=`echo $output_server`" >> ${run_dir}/runTestscript.bg
echo "CODE_DIR=`echo $CODE_DIR`" >> ${run_dir}/runTestscript.bg
echo "run_dir=`echo $run_dir`" >> ${run_dir}/runTestscript.bg
echo "runflag=`echo $runflag`" >> ${run_dir}/runTestscript.bg
echo "uploadflag=`echo $uploadflag`" >> ${run_dir}/runTestscript.bg
echo "MODULE=`echo $MODULE`" >> ${run_dir}/runTestscript.bg
echo "NUM_PROC=`echo $NUM_PROC`" >> ${run_dir}/runTestscript.bg 

if [ ${NUM_PROC} -gt 0 ]; then

    echo -e Running ${run_dir}/astrobear "...\c"
    pushd ${run_dir}
    echo "Run Directory is:" ${run_dir}
    log_file=`date -I`.log

    if [ ${runflag} == on ]; then
      rm profile.data
      echo "mpirun -exp_env BG_MAXALIGNEXP -np ${NUM_PROC} -mode VN -cwd $PWD -exe ${run_dir}/astrobear > ${run_dir}/${log_file}" >> runTestscript.bg
      chmod +x runTestscript.bg
      export LL_RES_ID=$RES_ID
      llsubmit runTestscript.bg > qsubOut
      jobID=`awk -F\" '{print $(NF-1)}' qsubOut | cut -f5 -d.`   
      #echo "mpiexec -np $NUM_PROC $run_dir/astrobear > $run_dir/$log_file" >> runTestscript.bg
    else
	echo 'skipping mpiexec'
    #wait
    fi
    jobStat=`llq | grep ${jobID} | wc -l`

    echo "running: "
    while [ $jobStat -ne 0 ]
    do
       printf "%c" "|"
       sleep 10
       jobStat=`llq | grep ${jobID} | wc -l`
    done

    #if [ $? == 0 ]; then
    if [ $jobStat == 0 ]; then

	if [ ${uploadflag} == on ]; then
	    FRAME=`cat ${run_dir}/bear2fix.data.img | grep -w FRAME | tr -dc '[0-9]'`
	    if [ ! -e ${run_dir}/out/chombo0000${FRAME}.hdf ]; then
		echo -e '*** Unable to finding expected output frame for post processing.  ${MODULE} likely failed.'
		exit 1
	    else
 	        echo ssh bliu@${output_server} "mkdir -p ${output_dir}/${MODULE}/out"
                ssh bliu@${output_server} "rm -rf ${output_dir}/${MODULE}/out && mkdir -p ${output_dir}/${MODULE}/out"

		echo '*** uploading run results to server'
                scp ${run_dir}/out/chombo0000${FRAME}.hdf bliu@${output_server}:${output_dir}/${MODULE}/out
                scp ${run_dir}/bear2fix.data.* bliu@${output_server}:${output_dir}/${MODULE}
                scp ${run_dir}/${log_file} bliu@${output_server}:${output_dir}/${MODULE}/out
                echo 'finish uploading'

                echo '*** running process_chombos'
                echo "running ${output_dir}/tests/process_chombos -m ${MODULE} -c ${changeset} -d ${output_dir}"
	#rsync --progress --partial -arvz ${run_dir}/out  `whoami`@${output_server}:${output_dir}	
	        ssh bliu@${output_server} "${output_dir}/tests/process_chombos -m ${MODULE} -c ${changeset} -d ${output_dir}"
   	        if [ $? != 0 ]; then
	            echo "${output_dir}/tests/process_chombos -m ${MODULE} -c ${changeset} -d ${output_dir} returned $?"
		    exit 1
	        fi
            fi
       fi
    fi
fi
echo "done."

