Vergleich von Inhalten

#!/bin/bash

#SBATCH --partition=cpu-genoa:test
#SBATCH --nodes=2
#SBATCH --tasks-per-node=3
#SBATCH --job-name=auto_loc_io_test
#SBATCH --output=%x.%j.out

echo ""
echo "Used environment variables:"
echo "WORK" $WORK
echo "LOCAL_TMPDIR" $LOCAL_TMPDIR
# Auto set by Slurm prolog: /local/$USER_$SLURM_JOB_ID
echo ""
echo "Used slurm variables:"
echo "SLURM_JOB_NAME" $SLURM_JOB_NAME
echo "SLURM_JOB_ID" $SLURM_JOB_ID
echo "SLURM_NODELIST" $SLURM_NODELIST
echo ""

### prepare case

# master dir. on global filesystem
# can be created by master node (= directly this script)
MASTERDIR=$WORK/$SLURM_JOB_NAME"."$SLURM_JOB_ID
echo "All job data is (collected) here: $MASTERDIR"

# subdirectories of master dir.
# can be created by master node (= directly this script)
INPUT_DIR=$MASTERDIR/in
OUTPUT_DIR=$MASTERDIR/allout
mkdir -p $INPUT_DIR
mkdir -p $OUTPUT_DIR
echo "example data inside input file" > $INPUT_DIR/input_data.dat

# $LOCAL_TMPDIR (aka /local/$USER_$SLURM_JOB_ID)
# is dir. on node-local ssd or ram if no ssd is present
# only exist during job lifetime

# subdirectories of $LOCAL_TMPDIR 
# can only be created node-locally (= "child" srun -w node)
LOC_OUT_DIR=$LOCAL_TMPDIR/out


# prepare script with empty 10 second job
cat <<EOF > $INPUT_DIR/dummyjob.sh
#!/bin/bash
echo "Main job started on \$SLURMD_NODENAME in \$PWD."
sleep 10
mkdir -p $LOC_OUT_DIR
hostname > $LOC_OUT_DIR/data.\$SLURMD_NODENAME
# important to use unique local name in order to mv it to global dir without overwriting
echo "\$SLURMD_NODENAME main job finished. Data written to $LOC_OUT_DIR."
EOF
chmod u+x $INPUT_DIR/dummyjob.sh

# command to cp data from global master to tempory local
# before mv -> check if dir is present and if not empty (=data to copy inside)
loc2mst_cmd_string='if [ -d '$LOC_OUT_DIR' ]; then if ! [ $(find '$LOC_OUT_DIR' -maxdepth 0 -empty) ]; then mv '$LOC_OUT_DIR'/* '$OUTPUT_DIR'; fi; fi'
# All ifs/checks are only needed to avoid warnings triggered for example if no data to mv is present...

### cp input files to node-local directories (in parallel)
pdcp -r -w $SLURM_NODELIST $INPUT_DIR $LOCAL_TMPDIR

echo "Relevant data is copied to node-local locations. Main job is starting..."

### execute main job itself: e.g. mpirun or srun
srun $INPUT_DIR/dummyjob.sh &
main_pid=$!

echo "Main job running. Start copying node-local data back to global filesystem in the background..."
while ps -p $main_pid &>/dev/null; do
	echo "Capacity of \$LOCAL_TMPDIR is:"
	pdsh -w $SLURM_NODELIST "$loc2mst_cmd_string"
	echo "loc2mst cmd executed. New capacity of \$LOCAL_TMPDIR is:"
	df -h $LOCAL_TMPDIR
	sleep 3
done

wait
echo "Main job finished. Copying remaining node-local data (if any)."
pdsh -w $SLURM_NODELIST "$loc2mst_cmd_string"

echo "All (parallel) copy jobs are done. All data is here: $MASTERDIR"