#!/bin/bash
####################################################################################
# Script name: buildproblem
# Created:  10-14-2011 by Matthew Noyes
# Purpose:  Compile and run modules in AstroBEAR
####################################################################################


# Declare the base scratch directory.  This is where all of the problem directories
# will be set up.  This is currently set up to run in the group space on bluehive.
tests=( BasicDisk BonnorEbertSphere Bondi BrioWuShockTube CorotatingBinary EinfeldtRarefaction FieldLoopAdvection FieldLoopRestart FillingFraction GravitationalCascade HydroWaves IsoHydroWaves IsoMHDWaves IsotropicTurbulence Marquee MHDWaves MolecularCloudFormation MomentumConservation MultiClumps OrbitingParticles RadiativeInstability0 RadiativeInstability05 RadiativeInstability1 RadShock RTInstability SingleClump SlowMolecularCloudFormation SodShockTube ThermalInstability UniformCollapse )

#tests=( HydroWaves )

trac_host=clover.pas.rochester.edu
trac_astrobear=/data/trac/astrobear
test_repo=/data/repositories/tests
localhost=`hostname`
changeset=`hg heads | grep changeset | awk -F' ' '{ print $2 }'`


CODE_DIR=`pwd`/
MODULE=""
NUM_PROC=1
buildtests=off
uploadflag=off
resultsflag=off
cleanmake=on
runflag=on
runAll=off
firsttest=0
USAGE_ERR="usage: $0 [-d code_dir] [-m module] [-np processors]"

case $localhost in
    "grass")
	output_dir=/cloverdata/scrambler_tests/$USER/$localhost/${changeset}
	output_server=$localhost;;
    "alfalfa")
	output_dir=/cloverdata/scrambler_tests/$USER/$localhost/${changeset}
	output_server=$localhost;;
    "clover")
	output_dir=/data/scrambler_tests/$USER/$localhost/${changeset}
	output_server=$localhost;;
    "bamboo")
	output_dir=/cloverdata/scrambler_tests/$USER/$localhost/${changeset}
	output_server=$localhost;;
    *)
	output_dir=/data/scrambler_tests/$USER/$localhost/${changeset}
	output_server=clover.pas.rochester.edu;;
esac


while [ $# -gt 0 ]
do
    case "$1" in
        -d)  CODE_DIR="$2"; shift;;
	-m)  MODULE="$2"; shift;;
	-np) NUM_PROC="$2"; shift;;
	-t)  buildtests=on;;
	-u)  uploadflag=on;;
	-td) output_server=${localhost}; output_dir="$2"; shift;;
        -s) firsttest="$2"; shift;;
        -c) cleanmake=off;;
        -r) runflag="$2";  shift;;
	-results) resultsflag=on; shift;;
	-a) runAll="$2";  shift;;
	--) shift; break;;
	-*)
	    echo >&2 ${USAGE_ERR}
		    
    exit 1;;
*)  break;;# terminate while loop
    esac
    shift
done


echo "OUTPUT_SERVER: " ${output_server}

if [ ! -d ${CODE_DIR} ]; then
    hg clone ssh://`whoami`@clover.pas.rochester.edu//data/repositories/scrambler ${CODE_DIR}
fi


# all command line switches are processed,
# "$@" contains all file names
 
echo ""

fetch_tests () {
# Retrieve a list of tests to execute (directories in modules/tests)
#    tests=( $( ls -l ${CODE_DIR}modules/tests | grep ^d | awk '{print $8}' ) )
#    tests=RTInstability

    numtests=${#tests[@]}

    echo "Num Tests:" ${numtests}
}

fetch_test_results() {
    echo "fetch test results"
    if [ ${output_server} == ${localhost} ]; then
	echo -e "\n\n*****TEST RESULTS*****"
	rm -f ./testlog
	rm -f ./timeslog
	for (( i=0;i<$numtests;i++)); do
            test=${tests[${i}]}
	    result=`cat ${output_dir}/tests/${test}/logs/testlog |grep '^Test Result:' | sed -e 's/^Test Result: //'`
	    echo "${test}..........${result}" >> testlog
	    result=`cat ${output_dir}/tests/${test}/logs/testlog |grep 'Runtime'`
	    echo "${test}..........${result}" >> timeslog

	done
    else
	echo -e "\n\n*****TEST RESULTS*****"
	ssh `whoami`@${output_server} "rm -f ${output_dir}/tests/testlog"
	for (( i=0;i<$numtests;i++)); do
            test=${tests[${i}]}
#	    echo ssh `whoami`@${output_server} "echo ${test}......... >> ${output_dir}/tests/testlog; cat ${output_dir}/test/${test}/logs/testlog | grep '^Test Result:' | sed -e 's/^Test Result: //' >> ${output_dir}/tests/testlog"
	    ssh `whoami`@${output_server} "ls ${output_dir}/tests/${test}/logs"
	    ssh `whoami`@${output_server} "echo ${test}......... >> ${output_dir}/tests/testlog; cat ${output_dir}/tests/${test}/logs/testlog | grep '^Test Result:' | sed -e 's/^Test Result: //' >> ${output_dir}/tests/testlog"
	    ssh `whoami`@${output_server} "echo ${test}......... >> ${output_dir}/tests/timeslog; cat ${output_dir}/tests/${test}/logs/testlog | grep 'Runtime' >> ${output_dir}/tests/timeslog"
	done
	scp `whoami`@${output_server}:${output_dir}/tests/testlog ./
	scp `whoami`@${output_server}:${output_dir}/tests/timeslog ./
    fi
    cat ./testlog
    echo '*****Run Times*****'
    cat ./timeslog
}

if [ ${resultsflag} == on ]; then  # If only results are requested then gather them and exit
    fetch_tests
    fetch_test_results
    exit 0
fi


if [ ${buildtests} == on ]; then

    fetch_tests

    echo "NUMPROCS is:" ${NUM_PROC}
    echo "Output Directory:" ${output_dir}
    # setup test repository
    if [ ${output_server} == ${localhost} ]; then
	if [ ! -d ${output_dir} ]; then
	    mkdir ${output_dir};
	    svn co svn+ssh://`whoami`@clover.pas.rochester.edu/data/repositories/tests ${output_dir}/tests 
	else
	    svn update ${output_dir}
	fi
	for (( i=0;i<$firsttest;i++)); do
	    test=${tests[${i}]}
	    echo "skipping ${test}"
	done
	for (( i=$firsttest;i<$numtests;i++)); do
	    test=${tests[${i}]}
	    rm -f ${output_dir}/tests/$test/images/sim.png
	done
    else
	echo '*** creating output_dir on remote host'
	ssh `whoami`@${output_server} "if [ ! -d ${output_dir} ]; then mkdir ${output_dir}; fi"
	echo '*** checking out test repository from svn'
 	ssh `whoami`@${output_server} "svn co -q svn+ssh://`whoami`@clover.pas.rochester.edu/data/repositories/tests ${output_dir}/tests; rm -f ${output_dir}/tests/*/images/sim.png" 
    fi
    ssh `whoami`@clover.pas.rochester.edu "rm -f /data/trac/astrobear/doc/${USER}_${localhost}_current_test; ln -s ${output_dir}/tests /data/trac/astrobear/doc/${USER}_${localhost}_current_test;"
#chmod 775 /data/trac/astrobear/doc/current_test; chgrp orda /data/trac/astrobear/doc/current_test"



    # setup bear2fix
    echo '*** compiling bear2fix'
    if [ ${output_server} == ${localhost} ]; then
	cd ${CODE_DIR}bear2fix
	rm -f Makefile.inc; ln -s Makefile.inc.`hostname` Makefile.inc
	if [ ${cleanmake} == on ]; then
	    make -s clean
	fi
	make -s > /dev/null
	echo "copying bear2fix to ${output_dir}/tests"
	cp -R ${CODE_DIR}/bear2fix/ ${output_dir}/tests
    else
	scp -r ${CODE_DIR}/bear2fix `whoami`@${output_server}:${output_dir}/tests/
	if [ ${cleanmake} == on ]; then
	    ssh `whoami`@${output_server} "cd ${output_dir}/tests/bear2fix; rm -f Makefile.inc; ln -s Makefile.inc.`hostname` Makefile.inc && make -s clean && make -s > /dev/null"

	else
	    ssh `whoami`@${output_server} "cd ${output_dir}/tests/bear2fix; rm -f Makefile.inc; ln -s Makefile.inc.`hostname` Makefile.inc && make -s > /dev/null"
	fi

    fi
    
    # Link to the selected test's problem directory.
    rm -f ${CODE_DIR}modules/Problem
    ln -s ${CODE_DIR}modules/${MODULE} ${CODE_DIR}modules/Problem
    cd ${CODE_DIR}
    if [ ${cleanmake} == on ]; then
	make -s clean
    else
	rm astrobear
    fi
    
    # Execute all of the tests
    for (( i=$firsttest;i<$numtests;i++)); do
	test=${tests[${i}]}
	if [ ${output_server} == ${localhost} ]; then
	    if [[ -e substscript.s ]]; then
		echo ./substscript.s modules/${test}
		./substscript.s  modules/${test}
            fi
	    echo "${CODE_DIR}buildproblem -d ${CODE_DIR} -m ${test} -np ${NUM_PROC} -u -c -td ${output_dir} -r ${runflag} -a ${runAll}"
	    ${CODE_DIR}buildproblem -d ${CODE_DIR} -m ${test} -np ${NUM_PROC} -u -c -td ${output_dir} -r ${runflag} -a ${runAll}
	else
	    echo "${CODE_DIR}buildproblem -d ${CODE_DIR} -m ${test} -np ${NUM_PROC} -u -c -r ${runflag} -a ${runAll}"
	    ${CODE_DIR}buildproblem -d ${CODE_DIR} -m ${test} -np ${NUM_PROC} -u -c -r ${runflag} -a ${runAll} 
	fi
	if [ $? != 0 ]; then
            if [ ${runAll} == "off" ]; then
		echo -e "${test} failed... "
		read -p "Continue/Retry/Abort/reProcess (C/R/A/P)? "
		if [ "$REPLY" == "A" ]; then
		    exit 1
		else
		    if [ "$REPLY" == "P" ]; then
			runflag=off
			i=$(( i - 1 ))
		    else
			runflag=on
			if [ "$REPLY" == "R" ]; then
			    i=$(( i - 1 ))
			fi
		    fi
		fi
            else #runAll=on
		if [ $? != 0 ]; then
		    echo -e "${test} failed... "
		    runflag=on
		fi
            fi 
	else
            runflag=on
	fi
    done
    
    fetch_test_results
    
    if [ `cat testlog | grep PASS | wc -l` == ${numtests} ]; then 
	if [ ${uploadflag} == on ]; then
	    if [ ${output_server} == ${localhost} ]; then
		cd ${output_dir}/tests
		svn ci -m "commited after successfully passing tests with AstroBEAR changeset ${changeset}"
		svn update
		REVISION=`svn info | grep '^Revision:' | sed -e 's/^Revision: //'`
		if [ ${localhost} == ${trac_host} ]; then
		    trac-admin ${trac_astrobear} changeset added ${test_repo} ${REVISION}
		else
		    ssh `whoami`@${trac_host} "trac-admin ${trac_astrobear} changeset added ${test_repo} ${REVISION}"
		fi
	    else
		ssh `whoami`@${output_server} "cd ${output_dir}/tests && svn ci -m 'commited after successfully passing tests with AstroBEAR changeset ${changeset}' && svn update"
		ssh `whoami`@${output_server} "exit `svn info | grep '^Revision:' | sed -e 's/^Revision: //'`"
		REVISION=$?
		if [ `localhost` == ${trac_host} ]; then
		    trac-admin ${trac_astrobear} changeset added ${test_repo} ${REVISION}
		else
		    ssh `whoami`@${trac_host} "trac-admin ${trac_astrobear} changeset added ${test_repo} ${REVISION}"
		fi
	    fi
	fi
    else
	echo -e "only `cat testlog | grep PASS | wc -l` out of ${numtests} passed"
	exit 1
    fi
    
    exit 0
fi


while [ ! -d "${CODE_DIR}modules/${MODULE}" ]
do
    echo Please select one of these modules in ${CODE_DIR}modules:
    echo ""

    ls -l ${CODE_DIR}modules | grep ^d | awk '{print $8}'

    echo ""
    echo -n "Module Name: " 
    read MODULE

done

echo 'Code Directory:' ${CODE_DIR}
echo 'Module:' ${MODULE}
echo 'Processors:' ${NUM_PROC}


# A list of the basic tests to be run.  These tests should match the names of the
# problem directories in CODE_DIRmodules.

cd ${CODE_DIR}

# Link to the test makefile.
rm -f ${CODE_DIR}Makefile.inc
ln -s ${CODE_DIR}Makefile.inc.`hostname`  ${CODE_DIR}Makefile.inc
#ls -l

# Link to the selected test's problem directory.
rm -f ${CODE_DIR}modules/Problem
ln -s ${CODE_DIR}modules/${MODULE} ${CODE_DIR}modules/Problem

# Compile the code using the specified problem.

if [ ${cleanmake} == on ]; then
    make -s clean
else
    rm astrobear
    touch modules/Problem/problem.f90
fi
make -s > /dev/null

if [ -f astrobear ];  then
    echo ${MODULE} "make done."
else
    echo -e ${MODULE} "failed to compile.  Aborting"
    exit 1
fi

nprocs=${NUM_PROC}

# Specify the name of the problem directory for this test.  The directory name will include
# the test's name, plus the number of processors and the number of levels of refinement.
run_dir="${CODE_DIR}modules/${MODULE}"

# If this problem directory does not exist, then create it.  This includes creating the
# out/ sub directory and copying over the cooling tables.
if [ ! -e ${run_dir} ]; then
    mkdir ${run_dir}
fi

if [ ! -e ${run_dir}/TABLES ]; then
    cp -r ${CODE_DIR}source/TABLES ${run_dir}
fi 

if [ ! -e ${run_dir}/out ]; then
    mkdir ${run_dir}/out
fi

# Copy over the newly-compiled executable.
cp ${CODE_DIR}astrobear ${run_dir}

# Copy the problem data files to the run directory.
if [ ${CODE_DIR}modules/${MODULE} != ${run_dir} ]; then
    echo *** 'copying over data files to run directory'
    cp ${CODE_DIR}modules/${MODULE}/*.data ${run_dir}
fi

echo ${MODULE} "copy done."

echo -e "\n"

if [ ${NUM_PROC} -gt 0 ]; then


    echo 'Removing old processing files'
    if [ ${output_server} == ${localhost} ]; then
	rm -f ${output_dir}/tests/${MODULE}/images/sim.png;
	rm -f ${output_dir}/tests/${MODULE}/logs/*
	echo 'Test Result: FAIL' > ${output_dir}/tests/${MODULE}/logs/testlog
    else 			
	ssh `whoami`@${output_server} "rm -f ${output_dir}/tests/${MODULE}/images/sim.png; rm -f ${output_dir}/tests/${MODULE}/logs/*; echo 'Test Result: FAIL' > ${output_dir}/tests/${MODULE}/logs/testlog"
    fi


    echo -e Running ${run_dir}/astrobear "...\c"
    pushd ${run_dir}
    echo "Run Directory is:" ${run_dir}
    log_file=`date -I`.log

    if [ ${runflag} == on ]; then
      rm profile.data
      if [ -e testrun.s ]; then
        ./testrun.s ${NUM_PROC} > ${run_dir}/${log_file}
      else
	  mpirun -np ${NUM_PROC} ${run_dir}/astrobear > ${run_dir}/${log_file}
      fi
    else
	echo 'skipping mpirun'
    #wait
    fi

    if [ $? == 0 ]; then
	if [ ${uploadflag} == on ]; then
	    if [ -e ${run_dir}/testprocess.s ]; then 
		files="${run_dir}/testprocess.s"
		outfiles="${run_dir}/testdata/*"
	    else
		FRAME=`cat ${run_dir}/bear2fix.data.img | grep -w FRAME | tr -dc '[0-9]'`
		if [ ! -e ${run_dir}/out/chombo0000${FRAME}.hdf ]; then
		    echo -e '*** Unable to finding expected output frame for post processing.  ${MODULE} likely failed.'
		    exit 1
		else
		    echo '*** uploading run results to server'
		    outfiles=${run_dir}/out/chombo0000${FRAME}.hdf
		    files=${run_dir}/bear2fix.data.*
		fi
	    fi
	    outfiles=${outfiles}" ${run_dir}/${log_file}"
	    files=${files}" ${run_dir}/scales.data"

	    echo "outfiles = ${outfiles}"
	    

#	    echo ssh `whoami`@${output_server} "mkdir -p ${output_dir}/${MODULE}/out"
		
	#rsync --progress --partial -arvz ${run_dir}/out  `whoami`@${output_server}:${output_dir}	
	    if [ ${output_server} == ${localhost} ]; then
#		    echo "cp ${outfiles} ${output_dir}/tests/${MODULE}/out"
		echo "running ${output_dir}/tests/process_chombos -m ${MODULE} -c ${changeset} -d ${output_dir}"
		rm -rf ${output_dir}/tests/${MODULE}/out && mkdir -p ${output_dir}/tests/${MODULE}/out
		cp ${files} ${output_dir}/tests/${MODULE}
		cp ${outfiles} ${output_dir}/tests/${MODULE}/out
		
		echo '*** running process_chombos'
		${output_dir}/tests/process_chombos -m tests/${MODULE} -c ${changeset} -d ${output_dir}

		echo ${output_dir}/tests/process_chombos -m tests/${MODULE} -c ${changeset} -d ${output_dir}
		if [ $? != 0 ]; then
		    echo "${output_dir}/tests/process_chombos -m tests/${MODULE} -c ${changeset} -d ${output_dir}"
		    exit 1
		fi
	    else 			
		scp ${files} `whoami`@${output_server}:${output_dir}/tests/${MODULE}
		scp ${outfiles} `whoami`@${output_server}:${output_dir}/tests/${MODULE}/out
		ssh `whoami`@${output_server} "${output_dir}/tests/process_chombos -m tests/${MODULE} -c ${changeset} -d ${output_dir}"
		if [ $? != 0 ]; then
		    echo "${output_dir}/tests/process_chombos -m tests/${MODULE} -c ${changeset} -d ${output_dir} returned $?"
		    exit 1
		fi
	    fi
	fi
    else

	echo -e "*** the command mpirun -np ${NUM_PROC} ${run_dir}/astrobear > ${run_dir}/${log_file} exited with code $? "
	exit 1
    fi
fi
popd



echo "done."
