66 lines
2.3 KiB
Bash
Executable File
66 lines
2.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# set -vx
|
|
# Simple script to robustly run a full suite of SETTE tests
|
|
#
|
|
########################################
|
|
function wait_on_q()
|
|
{
|
|
# SETTE testing on ARCHER2 uses the test queue in which users are limited to
|
|
# 16 queued jobs (including a maximum of 4 running). To prevent sette testing
|
|
# breaching this limit each configuration is processed separately and
|
|
# processing only begins when the user has no more than 12 jobs already queued.
|
|
# The supposition here is that each config forks no more than 4 tests - may need
|
|
# re-revaluating if used for PHYSICS tests.
|
|
#
|
|
# This function checks the queue usage and waits if necessary until the queue
|
|
# has drained sufficiently for the next test.
|
|
NRUN=999
|
|
NIT=0
|
|
BATCH_STAT="squeue -u $USER"
|
|
BATCH_NAME=sette
|
|
echo "Checking queues"
|
|
while [[ $NRUN -gt 12 && $nit -le 1080 ]]; do
|
|
nit=$((nit+1))
|
|
NRUN=$( ${BATCH_STAT} | grep ${BATCH_NAME} | wc -l )
|
|
if [[ $NRUN -gt 12 ]]; then
|
|
printf "%-3d %s\r" $NRUN 'nemo_sette runs still in queue or running ...';
|
|
else
|
|
printf "%-50s\n" "Queues sufficiently drained"
|
|
return 99
|
|
fi
|
|
sleep 10
|
|
done
|
|
echo "Something has gone wrong. Excessive wait time has been exceeded"
|
|
exit
|
|
}
|
|
#
|
|
########################################
|
|
# Start of main script
|
|
########################################
|
|
FULLSET=( ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 AGRIF WED025 GYRE_PISCES SAS ORCA2_ICE_OBS SWG ICE_AGRIF OVERFLOW LOCK_EXCHANGE VORTEX ISOMIP+ )
|
|
#
|
|
GROUP_SETS=( "-r" )
|
|
#
|
|
# These groups sets correspond to the following test regimes:
|
|
#
|
|
# A. complete sets with various combinations of options:
|
|
#
|
|
printf "%-93s %s\n" "Full tests - <branch_name> (using *_ST config dirs) : " "${GROUP_SETS[0]}"
|
|
#
|
|
# A. Full tests
|
|
for gs in 0
|
|
do
|
|
for n in `seq 0 1 $(( ${#FULLSET[@]} - 1 ))`
|
|
do
|
|
confstr="${FULLSET[$n]}"
|
|
# compile seperately since the final link sometimes has a bus error on ARCHER2 (which never happens on the 2nd attempt)
|
|
echo ./sette.sh ${GROUP_SETS[$gs]} -x "COMPILE" -n "$confstr"
|
|
./sette.sh ${GROUP_SETS[$gs]} -x "COMPILE" -n "$confstr"
|
|
# Now run the test (and finish linking if necessary)
|
|
echo ./sette.sh ${GROUP_SETS[$gs]} -x "RESTART REPRO CORRUPT" -n "$confstr"
|
|
./sette.sh ${GROUP_SETS[$gs]} -x "RESTART REPRO CORRUPT" -n "$confstr"
|
|
wait_on_q
|
|
done
|
|
done
|
|
exit
|