Index: benchmarks/benchmarks =================================================================== --- benchmarks/benchmarks (.../trunk) (revision 12000) +++ benchmarks/benchmarks (.../branches/tech_clustr) (revision 12000) @@ -31,10 +31,11 @@ # Informs the user of how to use the script, and then exits. function print_usage_and_exit { error "Usage: benchmarks [ -s ] [ -c ] [ -i TESTS | -x TESTS ] " + error "Pwd needs to be the directory of this benchmark script." error error "Options" error "=======" - error "-s Single machine mode, don't use PBS" + error "-s Single machine mode, don't use PBS/SLURM" error "-c Compile only, don't run the tests" error "-p Post processing only" error "-r Generate printer friendly report" @@ -248,7 +249,7 @@ if has_pbs ; then # Submit as a PBS job - qsub -l walltime=$pp_walltime $dependency < /dev/null + qsub -l walltime=$pp_walltime $dependency < /dev/null #!/bin/sh #PBS -N bm_${test_name}_pp cd $workspace @@ -256,11 +257,14 @@ EOF else # Submit as a SLURM job - sbatch -n 1 --time=$pp_walltime $dependency < /dev/null + sbatch -n 1 --time=$pp_walltime $dependency < /dev/null #!/bin/sh #SBATCH -J pp_${test_name} #SBATCH --mail-type=FAIL #SBATCH -o "slurm-%j.log" +$SBATCH_COMPUTEACCOUNT +module purge >/dev/null +module load foss/2022a Ghostscript/9.56.1 Python/2.7.18 $(cat $common_script) EOF fi @@ -282,9 +286,9 @@ pushd $workspace &> /dev/null if [ -f $submit_vars ]; then - jobid=$(./submit.sh -i $insfile -n "bm_$test_name" -s $submit_vars 2> /dev/null) + jobid=$(./submit.sh -i $insfile -n "bm_$test_name" -s $submit_vars) else - jobid=$(./submit.sh -i $insfile -n "bm_$test_name" 2> /dev/null) + jobid=$(./submit.sh -i $insfile -n "bm_$test_name") fi local submit_exit_status=$? @@ -426,6 +430,10 @@ echo "Local WC path:" > $file2 pwd -P >> $file2 echo >> $file2 + if has_slurm ; then + module list 2>&1 | grep -v '^[[:space:]]*$' >> $file2 + echo >> $file2 + fi echo "$changesdiff" >> $file2 fi else @@ -437,22 +445,25 @@ # Ok, let's start! -# Create the progress.txt file, overwriting if it existed -echo "The following benchmarks are done:" > progress.txt - # If necessary, create the working directory where all the benchmarks will be # run from. mkdir -p $work_dir - # Convert the work_dir variable to an absolute path if it isn't already work_dir=`cd $1; pwd` +# Create the progress.txt file, or write to the pre-existing one if available. +touch progress.txt +echo >> progress.txt +echo "Benchmarks submitted $(date)" >> progress.txt +echo "$work_dir" >> progress.txt +echo "The following benchmarks have been submitted:" >> progress.txt + if (( $printer_friendly == 1 )); then ./printerfriendly $work_dir exit 0 fi -# Generate meta data about this run +# Generate meta data about this run # (such as time stamp and version control information). if (( $pp_only == 0 )); then meta_data $work_dir @@ -498,6 +509,13 @@ fi done + +# Read the compute account number from the environment, if available. +# Only used on a SLURM system. +if [ -n "$COMPUTEACCOUNT" ]; then + SBATCH_COMPUTEACCOUNT="#SBATCH -A $COMPUTEACCOUNT" +fi + # Call run_benchmark for each test for test_name in "${test_names[@]}" do Index: benchmarks/crop_global/submit_vars.sh =================================================================== --- benchmarks/crop_global/submit_vars.sh (.../trunk) (revision 12000) +++ benchmarks/crop_global/submit_vars.sh (.../branches/tech_clustr) (revision 12000) @@ -1,4 +1,4 @@ NPROCESS=30 if [[ $ARCH == "aurora" ]]; then - NPROCESS=100 + NPROCESS=96 fi Index: benchmarks/europe/submit_vars.sh =================================================================== --- benchmarks/europe/submit_vars.sh (.../trunk) (revision 12000) +++ benchmarks/europe/submit_vars.sh (.../branches/tech_clustr) (revision 12000) @@ -1,5 +1,5 @@ NPROCESS=12 if [[ $ARCH == "aurora" ]] then - NPROCESS=40 + NPROCESS=48 fi Index: benchmarks/global/submit_vars.sh =================================================================== --- benchmarks/global/submit_vars.sh (.../trunk) (revision 12000) +++ benchmarks/global/submit_vars.sh (.../branches/tech_clustr) (revision 12000) @@ -1,5 +1,5 @@ NPROCESS=24 if [[ $ARCH == "aurora" ]] then - NPROCESS=80 + NPROCESS=96 fi Index: benchmarks/panarctic/submit_vars.sh =================================================================== --- benchmarks/panarctic/submit_vars.sh (.../trunk) (revision 12000) +++ benchmarks/panarctic/submit_vars.sh (.../branches/tech_clustr) (revision 12000) @@ -1,4 +1,4 @@ NPROCESS=12 if [[ $ARCH == "aurora" ]]; then - NPROCESS=40 + NPROCESS=48 fi Index: benchmarks/report2html =================================================================== --- benchmarks/report2html (.../trunk) (revision 12000) +++ benchmarks/report2html (.../branches/tech_clustr) (revision 12000) @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/sw/easybuild_milan/software/Python/2.7.18-GCCcore-11.3.0-bare/bin/python """Small script to convert a report in the form of an XML file, as generated in the post processing step of a benchmark, into HTML. Index: benchmarks/wetland_global/submit_vars.sh =================================================================== --- benchmarks/wetland_global/submit_vars.sh (.../trunk) (revision 12000) +++ benchmarks/wetland_global/submit_vars.sh (.../branches/tech_clustr) (revision 12000) @@ -1,5 +1,5 @@ NPROCESS=24 if [[ $ARCH == "aurora" ]] then - NPROCESS=80 + NPROCESS=96 fi Index: data/ins/europe.ins =================================================================== --- data/ins/europe.ins (.../trunk) (revision 12000) +++ data/ins/europe.ins (.../branches/tech_clustr) (revision 12000) Property changes on: data/ins/europe.ins ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,2 ## Merged /LPJ-GUESS/trunk/data/ins/europe.ins:r11165-11964 Merged /LPJ-GUESS/branches/tech_submit/data/ins/europe.ins:r10149-11956 Index: data/ins/global.ins =================================================================== --- data/ins/global.ins (.../trunk) (revision 12000) +++ data/ins/global.ins (.../branches/tech_clustr) (revision 12000) Property changes on: data/ins/global.ins ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,2 ## Merged /LPJ-GUESS/branches/tech_submit/data/ins/global.ins:r10149-11956 Merged /LPJ-GUESS/trunk/data/ins/global.ins:r11165-11964 Index: parallel_version/aurora.tmpl =================================================================== --- parallel_version/aurora.tmpl (.../trunk) (revision 12000) +++ parallel_version/aurora.tmpl (.../branches/tech_clustr) (revision 12000) @@ -7,15 +7,15 @@ # as a parallel job using SLURM on Aurora. # # Created automatically on DATE -# -# Usage: # +# Usage: +# # 1. Copy script to the directory where you want output written. # This will be called the RUN DIRECTORY. # 2. In an editor, set appropriate values for the variables NPROCESS, # INSFILE, GRIDLIST and OUTFILES (NB: no space after the = sign): -NPROCESS=20 # NB: Should be multiple of 20 on Aurora! +NPROCESS=48 # NB: Should be multiple of 48 on Cosmos! WALLTIME=150:00:00 WALLTIME_APPEND=01:30:00 # Does not need to be as long as guess walltime INSFILE=guess.ins @@ -33,7 +33,14 @@ # and separated by spaces (filenames only, including # extension, no directory.) Shell wildcards are allowed. # -# 3. Run the script using the command: +# 3. Optionally specify an compute account nmumber (i.e. a code) by +# exporting the following variable to the environment. +# On your command line or in your .bashrc, do: +# export COMPUTEACCOUNT= +# An account number is necessary on lunarc systems if you are +# member of more than one compute project. +# +# 4. Run the script using the command: # ./submit.sh # or: # ./submit.sh [-n ] [-s ] [-i ] @@ -70,6 +77,12 @@ INSFILE=$ins fi +# Read the compute account number from the environment, if available. +if [ -n "$COMPUTEACCOUNT" ]; then + SBATCH_COMPUTEACCOUNT="#SBATCH -A $COMPUTEACCOUNT" +fi + + # On Aurora, the recommendation is to submit jobs with the --exclusive # option, so we get exclusive nodes. Since each node has 20 cores, we # should set NPROCESS to a multiple of 20 to avoid waste. @@ -76,10 +89,10 @@ # If you really want to, you could remove this check and the --exclusive # option below, but your jobs might then be disturbed by other jobs # sharing your nodes. -CORES_PER_NODE=20 +CORES_PER_NODE=48 if [[ $((NPROCESS%CORES_PER_NODE)) != 0 ]]; then - echo "Please set NPROCESS to a multiple of 20 on Aurora!" >&2 - exit 1 + echo "Please set NPROCESS to a multiple of 48 on Cosmos!" >&2 +# exit 1 fi # Convert INSFILE to an absolute path since we will be starting the @@ -145,6 +158,9 @@ #SBATCH --time=$WALLTIME #SBATCH --exclusive #SBATCH -o "slurm-%j.log" +$SBATCH_COMPUTEACCOUNT +module purge >/dev/null +module load foss/2022a netCDF/4.9.0 set -e if ! type -P mpirun &> /dev/null; then @@ -177,8 +193,8 @@ #!/bin/bash #SBATCH -n 1 #SBATCH --time=$WALLTIME_APPEND -#SBATCH --mem-per-cpu=12000 #SBATCH -o "slurm-%j.log" +$SBATCH_COMPUTEACCOUNT set -e function append_files { Index: . =================================================================== --- . (.../trunk) (revision 12000) +++ . (.../branches/tech_clustr) (revision 12000) Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,2 ## Merged /LPJ-GUESS/trunk:r11165-11964 Merged /LPJ-GUESS/branches/tech_submit:r10149-11956