277 lines
6.6 KiB
Bash
Executable File
277 lines
6.6 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# BENCH launching scripts for beaufix, Meteo-France
|
|
# and curie, TGCC
|
|
# To be modified for other machines
|
|
#
|
|
set -u
|
|
#set -vx
|
|
#
|
|
cores=$1
|
|
ncore_node=$2
|
|
resolution=$3
|
|
dateref=$4
|
|
machine=$( hostname | sed -e "s/[0-9]*//g" )
|
|
#
|
|
# number of processes for each executable
|
|
nproc_exe1=$( echo $cores | bc )
|
|
nproc=$nproc_exe1
|
|
nnode=$(( ( $nproc + $ncore_node - 1 ) / $ncore_node ))
|
|
|
|
nproc5=$( printf "%05d\n" ${nproc_exe1} )
|
|
|
|
case ${resolution} in
|
|
"1")
|
|
if [ $nproc_exe1 -lt 50 ]
|
|
then
|
|
timejob=3600
|
|
elif [ $nproc_exe1 -lt 100 ]
|
|
then
|
|
timejob=1800
|
|
elif [ $nproc_exe1 -lt 200 ]
|
|
then
|
|
timejob=900
|
|
else
|
|
timejob=600
|
|
fi
|
|
;;
|
|
"025")
|
|
if [ $nproc_exe1 -lt 50 ]
|
|
then
|
|
timejob=15000
|
|
elif [ $nproc_exe1 -lt 100 ]
|
|
then
|
|
timejob=7000
|
|
elif [ $nproc_exe1 -lt 200 ]
|
|
then
|
|
timejob=3600
|
|
elif [ $nproc_exe1 -lt 400 ]
|
|
then
|
|
timejob=2000
|
|
elif [ $nproc_exe1 -lt 800 ]
|
|
then
|
|
timejob=1000
|
|
else
|
|
timejob=600
|
|
fi
|
|
;;
|
|
"12")
|
|
if [ $nproc_exe1 -lt 200 ]
|
|
then
|
|
timejob=30000
|
|
elif [ $nproc_exe1 -lt 400 ]
|
|
then
|
|
timejob=15000
|
|
elif [ $nproc_exe1 -lt 800 ]
|
|
then
|
|
timejob=20000
|
|
elif [ $nproc_exe1 -lt 1600 ]
|
|
then
|
|
timejob=15000
|
|
elif [ $nproc_exe1 -lt 3200 ]
|
|
then
|
|
timejob=7500
|
|
elif [ $nproc_exe1 -lt 10000 ]
|
|
then
|
|
timejob=5000
|
|
elif [ $nproc_exe1 -lt 20000 ]
|
|
then
|
|
timejob=2500
|
|
else
|
|
timejob=1200
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
|
|
######################################################################
|
|
### beaufixlogin
|
|
######################################################################
|
|
|
|
if [ "$machine" == "beaufixlogin" ]
|
|
then
|
|
|
|
cat > Log/run_bench << EOF
|
|
#!/bin/bash
|
|
#SBATCH --time=00:1:00
|
|
#SBATCH -p normal64 # partition/queue
|
|
#SBATCH --job-name=bench # job name
|
|
#SBATCH -N $nnode # number of nodes
|
|
#SBATCH -n $nproc # number of procs
|
|
#SBATCH -o /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/Log/job.out%j
|
|
#SBATCH -e /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/Log/job.out%j
|
|
#SBATCH --exclusive
|
|
|
|
module unload intelmpi intel grib_api
|
|
module load intel/16.1.150 intelmpi/5.1.2.150
|
|
|
|
itac=0
|
|
xpmpi=0
|
|
|
|
if [ \$xpmpi == 1 ]; then
|
|
module load bullxde
|
|
module load xPMPI/1.1_intelmpi
|
|
fi
|
|
|
|
[ \$itac == 1 ] && module load itac/2017.2.028
|
|
|
|
set -vx
|
|
|
|
cd \${TMPDIR}
|
|
cp /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/* .
|
|
|
|
# Best decompositions BENCH-1
|
|
jpni=${cores/\**/}
|
|
jpnj=${cores/?*\*/}
|
|
|
|
sed -e "s/jpni *=.*/jpni = \${jpni}/" -e "s/jpnj *=.*/jpnj = \${jpnj}/" namelist_cfg_orca${resolution}_like > namelist_cfg
|
|
|
|
export OMP_NUM_THREADS=1
|
|
ulimit -s unlimited
|
|
#
|
|
if [ \$itac == 1 ]; then
|
|
source /opt/softs/intel/2017/update_1/itac_latest/bin/itacvars.sh
|
|
time mpirun -ordered-output -prepend-rank -trace -np $nproc_exe1 ./nemo > jobout 2>joberr
|
|
else
|
|
time mpirun -ordered-output -prepend-rank -np $nproc_exe1 ./nemo > jobout_${resolution}_${nproc5}
|
|
fi
|
|
/opt/softs/bin/ja
|
|
|
|
if [ \$xpmpi == 1 ]; then
|
|
module unload xPMPI/1.1_intelmpi
|
|
module unload bullxde
|
|
fi
|
|
#
|
|
EOF
|
|
|
|
|
|
### 4. Execute the model
|
|
|
|
echo 'Submitting the job to queue using sbatch'
|
|
sbatch Log/run_bench
|
|
squeue -u cglo315
|
|
|
|
echo 'is executed or submitted to queue.'
|
|
|
|
|
|
fi
|
|
|
|
######################################################################
|
|
### curie or irene
|
|
######################################################################
|
|
|
|
if [[ ( "$machine" == "curie" ) || ( "$machine" == "irene" ) ]]
|
|
then
|
|
|
|
[ "$machine" == "curie" ] && queuename=standard || queuename=skylake
|
|
|
|
EXPjob=../EXP_${resolution}_${nproc5}_${dateref}
|
|
mkdir -p ${EXPjob}
|
|
cd ${EXPjob}
|
|
jobname=jobbench
|
|
cat > $jobname << EOF
|
|
#!/bin/bash
|
|
#MSUB -r bench${nproc5}
|
|
#MSUB -n ${nproc_exe1}
|
|
#MSUB -T $timejob
|
|
#MSUB -e bench_${resolution}_${nproc5}_%I.eo
|
|
#MSUB -o bench_${resolution}_${nproc5}_%I.eo
|
|
#MSUB -j oe
|
|
#MSUB -x
|
|
#MSUB -q ${queuename}
|
|
#MSUB -A gen6895
|
|
#==========================================
|
|
set -u
|
|
#
|
|
|
|
cd \${BRIDGE_MSUB_PWD}
|
|
|
|
for ff in \${BRIDGE_MSUB_PWD}/../EXPREF/namelist_*cfg \${BRIDGE_MSUB_PWD}/../EXPREF/namelist_*ref \${BRIDGE_MSUB_PWD}/../BLD/bin/nemo.exe
|
|
do
|
|
cp \$ff .
|
|
done
|
|
|
|
jpni=${cores/\**/}
|
|
jpnj=${cores/?*\*/}
|
|
|
|
sed -e "s/jpni *=.*/jpni = \${jpni}/" \
|
|
-e "s/jpnj *=.*/jpnj = \${jpnj}/"\
|
|
-e "s/ln_timing *= *.false./ln_timing = .true./" \
|
|
\${BRIDGE_MSUB_PWD}/../EXPREF/namelist_cfg_orca${resolution}_like > namelist_cfg
|
|
|
|
time ccc_mprun -n \${BRIDGE_MSUB_NPROC} ./nemo.exe > jobout_${resolution}_${nproc5}_${dateref} 2>&1
|
|
|
|
EOF
|
|
|
|
ccc_msub $jobname
|
|
|
|
fi
|
|
|
|
######################################################################
|
|
### Jean-Zay
|
|
######################################################################
|
|
|
|
if [ "$machine" == "jean-zay" ]
|
|
then
|
|
hh=$( printf "%02d\n" $(( ${timejob} / 3600 )) )
|
|
mm=$( printf "%02d\n" $(( ( ${timejob} % 3600 ) / 60 )) )
|
|
ss=$( printf "%02d\n" $(( ( ${timejob} % 3600 ) % 60 )) )
|
|
|
|
EXPjob=../EXP_${resolution}_${nproc5}_${dateref}
|
|
mkdir -p ${EXPjob}
|
|
cd ${EXPjob}
|
|
jobname=jobbench
|
|
cat > $jobname << EOF
|
|
#!/bin/bash
|
|
#SBATCH --job-name=Seq # nom du job
|
|
#SBATCH --partition=cpu_gct3 # demande d'allocation sur la partition CPU
|
|
#SBATCH --nodes=${nnode} # nombre de noeuds
|
|
#SBATCH --ntasks-per-node=${ncore_node} # nombre de taches MPI par noeud
|
|
#SBATCH --ntasks-per-core=1 # 1 processus MPI par coeur physique (pas d'hyperthreading)
|
|
#SBATCH --time=${hh}:${mm}:${ss} # temps d execution maximum demande (HH:MM:SS)
|
|
#SBATCH --output=bench_${resolution}_${nproc5}_%j.eo # nom du fichier de sortie
|
|
#SBATCH --error=bench_${resolution}_${nproc5}_%j.eo # nom du fichier d'erreur (ici en commun avec la sortie)
|
|
#==========================================
|
|
set -u
|
|
#set -xv
|
|
#
|
|
#cd \${SLURM_SUBMIT_DIR}
|
|
cd \${JOBSCRATCH}
|
|
pwd
|
|
|
|
for ff in \${SLURM_SUBMIT_DIR}/../EXPREF/namelist_*cfg \${SLURM_SUBMIT_DIR}/../EXPREF/namelist_*ref \${SLURM_SUBMIT_DIR}/../BLD/bin/nemo.exe
|
|
do
|
|
cp \$ff .
|
|
done
|
|
|
|
jpni=${cores/\**/}
|
|
jpnj=${cores/?*\*/}
|
|
|
|
sed -e "s/jpni *=.*/jpni = \${jpni}/" \
|
|
-e "s/jpnj *=.*/jpnj = \${jpnj}/" \
|
|
-e "s/ln_timing *= *.false./ln_timing = .true./" \
|
|
\${SLURM_SUBMIT_DIR}/../EXPREF/namelist_cfg_orca${resolution}_like > namelist_cfg
|
|
|
|
ls -l
|
|
|
|
echo
|
|
echo
|
|
echo " =========== start the model ==========="
|
|
echo
|
|
echo
|
|
|
|
time srun --mpi=pmi2 --cpu-bind=cores -K1 -n ${nproc} ./nemo.exe > jobout_${resolution}_${nproc5}_${dateref} 2>&1
|
|
|
|
ls -l
|
|
|
|
if [ "\$( pwd )" != "\${SLURM_SUBMIT_DIR}" ]
|
|
then
|
|
rsync -av namelist_cfg time.step ocean.output jobout_${resolution}_${nproc5}_${dateref} communication_report.txt layout.dat timing.output output.namelist* \${SLURM_SUBMIT_DIR}
|
|
fi
|
|
|
|
EOF
|
|
|
|
sbatch $jobname
|
|
|
|
fi
|