slurm.knl

#!/bin/bash -l
#SBATCH -A m2887
#SBATCH -p debug
#SBATCH -N 8
#SBATCH -t 00:05:00
#SBATCH -o qout.%j
#SBATCH -e qout.%j
#SBATCH -L SCRATCH
#SBATCH -C knl,quad,cache

cd $SLURM_SUBMIT_DIR

# display the names and values of all MPI-IO hints
export MPICH_MPIIO_HINTS_DISPLAY=1
# display the MPI rank IDs of all I/O aggregators
export MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1
# display file write and read access patterns
export MPICH_MPIIO_STATS=1
# display timing data for different phases in MPI-IO
export MPICH_MPIIO_TIMERS=1

# specifiy the number of MPI process per compute node
nprocs_per_node=64

# calculate the number of logical cores to allocate per MPI process
cpus_per_task=$((68 / $nprocs_per_node))
cpus_per_task=$(($cpus_per_task * 4))

# NP is the total number of MPI processes to run
NP=$(($nprocs_per_node * $SLURM_JOB_NUM_NODES))

KNL_OPTS="-c $cpus_per_task --cpu_bind=cores"

echo "----------------------------------------------------"
echo "---- Running $NP MPI processes on Cori NKL nodes ---"
echo "---- SLURM_JOB_NUM_NODES = $SLURM_JOB_NUM_NODES"
echo "---- Running $nprocs_per_node MPI processes per KNL node"
echo "---- command: srun -n $NP $KNL_OPTS"
echo "----------------------------------------------------"
echo ""

# input/output files
OUTDIR=$SCRATCH/FS_1M_64
INFILE=$SCRATCH/f_case_48602x72_512p.nc

# run command
RUN_OPTS="-k -o $OUTDIR $INFILE"

# A trick for faster loading the executable by copying the executable to
# the local disks of all compute nodes. See
# https://docs.nersc.gov/jobs/best-practices/#large-jobs
sbcast src/e3sm_io /tmp/${USER}_e3sm_io
RUN_CMMD=/tmp/${USER}_e3sm_io

echo "---- KNL $RUN_CMMD ----"
command="srun -n $NP $KNL_OPTS $RUN_CMMD $RUN_OPTS"
echo "command=$command"
$command