diff --git a/testing/test-examples-lassen.sh b/testing/test-examples-lassen.sh new file mode 100755 index 000000000..aecb92277 --- /dev/null +++ b/testing/test-examples-lassen.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +origin=$(pwd) +EXAMPLES_HOME=$2 +# examples_dir=${1-$origin} +BATCH_SCRIPT_NAME="examples-lassen-batch.sh" +examples_dir="${EXAMPLES_HOME}" + +rm -rf ${BATCH_SCRIPT_NAME} +cat < ${BATCH_SCRIPT_NAME} +#!/bin/bash + +#BSUB -nnodes 1 +#BSUB -G uiuc +#BSUB -W 120 +#BSUB -q pdebug + +printf "Running with EMIRGE_HOME=${EMIRGE_HOME}\n" + +source "${EMIRGE_HOME}/config/activate_env.sh" +export PYOPENCL_CTX="port:tesla" +export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch" +rm -rf \$XDG_CACHE_HOME +rm -f timing-run-done +which python +conda env list +env +env | grep LSB_MCPU_HOSTS + +serial_spawner_cmd="jsrun -g 1 -a 1 -n 1" +parallel_spawner_cmd="jsrun -g 1 -a 1 -n 2" + +set -o nounset + +rm -f *.vtu *.pvtu + +declare -i numfail=0 +declare -i numsuccess=0 +echo "*** Running examples in $examples_dir ..." +failed_examples="" +succeeded_examples="" + +for example in $examples_dir/*.py +do + if [[ "\$example" == *"-mpi-lazy.py" ]] + then + echo "*** Running parallel lazy example (1 rank): \$example" + \$serial_spawner_cmd python -O -m mpi4py \${example} --lazy + elif [[ "\$example" == *"-mpi.py" ]]; then + echo "*** Running parallel example (2 ranks): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} + elif [[ "\$example" == *"-lazy.py" ]]; then + echo "*** Running serial lazy example: \$example" + python -O \${example} --lazy + else + echo "*** Running serial example: \$example" + python -O \${example} + fi + if [[ \$? -eq 0 ]] + then + ((numsuccess=numsuccess+1)) + echo "*** Example \$example succeeded." + succeeded_examples="\$succeeded_examples \$example" + else + ((numfail=numfail+1)) + echo "*** Example \$example failed." + failed_examples="\$failed_examples \$example" + fi + rm -rf *vtu *sqlite *pkl *-journal restart_data +done +((numtests=numsuccess+numfail)) +echo "*** Done running examples!" +if [[ \$numfail -eq 0 ]] +then + echo "*** No errors." +else + echo "*** Errors detected." + echo "*** Failed tests: (\$numfail/\$numtests): \$failed_examples" +fi +echo "*** Successful tests: (\$numsuccess/\$numtests): \$succeeded_examples" + +rm -rf example-testing-results +printf "\$numfail\n" > example-testing-results +touch example-testing-done +exit \$numfail + +EOF + +rm -f example-testing-done +chmod +x ${BATCH_SCRIPT_NAME} +# ---- Submit the batch script and wait for the job to finish +bsub ${BATCH_SCRIPT_NAME} +# ---- Wait 25 minutes right off the bat +sleep 1500 +iwait=0 +while [ ! -f ./example-testing-done ]; do + iwait=$((iwait+1)) + if [ "$iwait" -gt 89 ]; then # give up after almost 2 hours + printf "Timed out waiting on batch job.\n" + exit 1 # skip the rest of the script + fi + sleep 60 +done +sleep 30 # give the batch system time to spew its junk into the log +cat *.out > example-testing-output +date >> example-testing-output +rm *.out +date diff --git a/testing/test-examples-linux.sh b/testing/test-examples-linux.sh new file mode 100644 index 000000000..f72c9408a --- /dev/null +++ b/testing/test-examples-linux.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +origin=$(pwd) +EXAMPLES_HOME=$2 +BATCH_SCRIPT_NAME="run-examples-linux.sh" +examples_dir="${EXAMPLES_HOME}" + +rm -rf ${BATCH_SCRIPT_NAME} +cat < ${BATCH_SCRIPT_NAME} +#!/bin/bash + +printf "Running with EMIRGE_HOME=${EMIRGE_HOME}\n" + +source "${EMIRGE_HOME}/config/activate_env.sh" +# export PYOPENCL_CTX="port:tesla" +export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch" +rm -rf \$XDG_CACHE_HOME +rm -f examples-run-done +which python +conda env list +env + +parallel_spawner_cmd="mpiexec -n 2" + +set -o nounset + +rm -f *.vtu *.pvtu + +declare -i numfail=0 +declare -i numsuccess=0 +echo "*** Running examples in $examples_dir ..." +failed_examples="" +succeeded_examples="" + +for example in $examples_dir/*.py +do + if [[ "\$example" == *"-mpi-lazy.py" ]] + then + echo "*** Running parallel lazy example (2 rank): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} --lazy + elif [[ "\$example" == *"-mpi.py" ]]; then + echo "*** Running parallel example (2 ranks): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} + elif [[ "\$example" == *"-lazy.py" ]]; then + echo "*** Running serial lazy example: \$example" + python -O \${example} --lazy + else + echo "*** Running serial example: \$example" + python -O \${example} + fi + if [[ \$? -eq 0 ]] + then + ((numsuccess=numsuccess+1)) + echo "*** Example \$example succeeded." + succeeded_examples="\$succeeded_examples \$example" + else + ((numfail=numfail+1)) + echo "*** Example \$example failed." + failed_examples="\$failed_examples \$example" + fi + rm -rf *vtu *sqlite *pkl *-journal restart_data +done +((numtests=numsuccess+numfail)) +echo "*** Done running examples!" +if [[ \$numfail -eq 0 ]] +then + echo "*** No errors." +else + echo "*** Errors detected." + echo "*** Failed tests: (\$numfail/\$numtests): \$failed_examples" +fi +echo "*** Successful tests: (\$numsuccess/\$numtests): \$succeeded_examples" + +rm -rf example-testing-results +printf "\$numfail\n" > example-testing-results +touch example-testing-done +exit \$numfail + +EOF + +rm -f example-testing-done +chmod +x ${BATCH_SCRIPT_NAME} +# ---- Submit the batch script and wait for the job to finish +EXAMPLES_RUN_OUTPUT=$(${BATCH_SCRIPT_NAME}) + +# ---- Wait 25 minutes right off the bat +sleep 1500 +iwait=0 +while [ ! -f ./example-testing-done ]; do + iwait=$((iwait+1)) + if [ "$iwait" -gt 89 ]; then # give up after almost 2 hours + printf "Timed out waiting on batch job.\n" + exit 1 # skip the rest of the script + fi + sleep 60 +done +sleep 30 # give the batch system time to spew its junk into the log +printf "${EXAMPLSE_RUN_OUTPUT}\n" > example-testing-output +date >> example-testing-output +date diff --git a/testing/test-examples-quartz.sh b/testing/test-examples-quartz.sh new file mode 100755 index 000000000..9cfd1382a --- /dev/null +++ b/testing/test-examples-quartz.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +origin=$(pwd) +EXAMPLES_HOME=$2 +# examples_dir=${1-$origin} +BATCH_SCRIPT_NAME="examples-quartz-batch.sh" +examples_dir="${EXAMPLES_HOME}" + +rm -rf ${BATCH_SCRIPT_NAME} +cat < ${BATCH_SCRIPT_NAME} +#!/bin/bash + +#SBATCH -N 2 +#SBATCH -J mirgecom-examples-test +#SBATCH -t 120 +#SBATCH -p pbatch +#SBATCH -A uiuc + +printf "Running with EMIRGE_HOME=${EMIRGE_HOME}\n" + +source "${EMIRGE_HOME}/config/activate_env.sh" +# export PYOPENCL_CTX="port:tesla" +export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch" +rm -rf \$XDG_CACHE_HOME +rm -f timing-run-done +which python +conda env list +env +env | grep LSB_MCPU_HOSTS + +serial_spawner_cmd="srun -n 1" +parallel_spawner_cmd="srun -n 2" + +set -o nounset + +rm -f *.vtu *.pvtu + +declare -i numfail=0 +declare -i numsuccess=0 +echo "*** Running examples in $examples_dir ..." +failed_examples="" +succeeded_examples="" + +for example in $examples_dir/*.py +do + if [[ "\$example" == *"-mpi-lazy.py" ]] + then + echo "*** Running parallel lazy example (2 rank): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} --lazy + elif [[ "\$example" == *"-mpi.py" ]]; then + echo "*** Running parallel example (2 ranks): \$example" + \$parallel_spawner_cmd python -O -m mpi4py \${example} + elif [[ "\$example" == *"-lazy.py" ]]; then + echo "*** Running serial lazy example: \$example" + python -O \${example} --lazy + else + echo "*** Running serial example: \$example" + python -O \${example} + fi + if [[ \$? -eq 0 ]] + then + ((numsuccess=numsuccess+1)) + echo "*** Example \$example succeeded." + succeeded_examples="\$succeeded_examples \$example" + else + ((numfail=numfail+1)) + echo "*** Example \$example failed." + failed_examples="\$failed_examples \$example" + fi + rm -rf *vtu *sqlite *pkl *-journal restart_data +done +((numtests=numsuccess+numfail)) +echo "*** Done running examples!" +if [[ \$numfail -eq 0 ]] +then + echo "*** No errors." +else + echo "*** Errors detected." + echo "*** Failed tests: (\$numfail/\$numtests): \$failed_examples" +fi +echo "*** Successful tests: (\$numsuccess/\$numtests): \$succeeded_examples" + +rm -rf example-testing-results +printf "\$numfail\n" > example-testing-results +touch example-testing-done +exit \$numfail + +EOF + +rm -f example-testing-done +chmod +x ${BATCH_SCRIPT_NAME} +# ---- Submit the batch script and wait for the job to finish +sbatch ${BATCH_SCRIPT_NAME} +# ---- Wait 60 minutes right off the bat +printf "Waiting for the batch job to finish." +sleep 3600 +printf "." +iwait=0 +while [ ! -f ./example-testing-done ]; do + iwait=$((iwait+1)) + if [ "$iwait" -gt 180 ]; then # give up after 4 hours + printf "\nTimed out waiting on batch job, aborting tests.\n" + exit 1 # skip the rest of the script + fi + sleep 60 + printf "." +done +printf "(finished)\n" +sleep 30 # give the batch system time to spew its junk into the log +cat *.out > example-testing-output +date >> example-testing-output +rm *.out +date diff --git a/testing/test-lassen.sh b/testing/test-lassen.sh new file mode 100755 index 000000000..6e6620c96 --- /dev/null +++ b/testing/test-lassen.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +TESTING_RESULTS_FILE=$2 +TESTING_LOG_FILE=$3 + +printf "Testing examples.\n" +./test-examples-lassen.sh ${EMIRGE_HOME} ../examples +examples_script_result=$? +printf "Examples script result: ${examples_result}" +cat example-testing-output >> ${TESTING_LOG_FILE} +examples_testing_result=$(cat example-testing-results) +printf "mirgecom-examples: ${examples_testing_result}\n" >> ${TESTING_RESULTS_FILE} diff --git a/testing/test-linux.sh b/testing/test-linux.sh new file mode 100644 index 000000000..686561639 --- /dev/null +++ b/testing/test-linux.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +TESTING_RESULTS_FILE=$2 +TESTING_LOG_FILE=$3 + +printf "Testing examples.\n" +./test-examples-linux.sh ${EMIRGE_HOME} ../examples +examples_script_result=$? +printf "Examples script result: ${examples_result}" +cat example-testing-output >> ${TESTING_LOG_FILE} +examples_testing_result=$(cat example-testing-results) +printf "mirgecom-examples: ${examples_testing_result}\n" >> ${TESTING_RESULTS_FILE} diff --git a/testing/test-quartz.sh b/testing/test-quartz.sh new file mode 100755 index 000000000..7c366c4a5 --- /dev/null +++ b/testing/test-quartz.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +EMIRGE_HOME=$1 +TESTING_RESULTS_FILE=$2 +TESTING_LOG_FILE=$3 + +printf "Testing examples.\n" +./test-examples-quartz.sh ${EMIRGE_HOME} ../examples +examples_script_result=$? +printf "Examples script result: ${examples_result}" +cat example-testing-output >> ${TESTING_LOG_FILE} +examples_testing_result=$(cat example-testing-results) +printf "mirgecom-examples: ${examples_testing_result}\n" >> ${TESTING_RESULTS_FILE}