#!/bin/bash

# example to use use 2 x (2 x 4) = 16 MPI processes, each assigned 
# to one of the two tiles (stacks) of an PVC GPU

#SBATCH --partition=gpu-pvc
#SBATCH --nodes=2
#SBATCH --ntasks-per-node=8
#SBATCH --job-name=pin-check

# required for usage of Intel GPUs 
module load intel
# required for MPI, apparently
module load impi/2021.11

# required for GPU usage with MPI
export FI_PROVIDER=psm3

# to enable GPU support in Intel MPI
export I_MPI_OFFLOAD=1

# assign each rank a tile of a GPU
export I_MPI_OFFLOAD_CELL=tile

# for checking the process pinning
export I_MPI_DEBUG=3
export I_MPI_OFFLOAD_PRINT_TOPOLOGY=1

mpirun ./application