From bf36d63fa69e19c58734fb7531652ea6757fc03b Mon Sep 17 00:00:00 2001 From: Robin Rombach Date: Thu, 28 Jul 2022 00:12:03 +0200 Subject: [PATCH] launchers --- .../launcher.sh | 27 ++++++++++++ .../sbatch.sh | 42 +++++++++++++++++++ .../launcher.sh | 27 ++++++++++++ .../sbatch.sh | 42 +++++++++++++++++++ 4 files changed, 138 insertions(+) create mode 100755 scripts/slurm/v1-upscaling-f16-pretraining-512-aesthetics/launcher.sh create mode 100755 scripts/slurm/v1-upscaling-f16-pretraining-512-aesthetics/sbatch.sh create mode 100755 scripts/slurm/v1_inpainting_aesthetics-larger-masks/launcher.sh create mode 100755 scripts/slurm/v1_inpainting_aesthetics-larger-masks/sbatch.sh diff --git a/scripts/slurm/v1-upscaling-f16-pretraining-512-aesthetics/launcher.sh b/scripts/slurm/v1-upscaling-f16-pretraining-512-aesthetics/launcher.sh new file mode 100755 index 0000000..c37f762 --- /dev/null +++ b/scripts/slurm/v1-upscaling-f16-pretraining-512-aesthetics/launcher.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# mpi version for node rank +H=`hostname` +THEID=`echo -e $HOSTNAMES | python3 -c "import sys;[sys.stdout.write(str(i)) for i,line in enumerate(next(sys.stdin).split(' ')) if line.strip() == '$H'.strip()]"` +export NODE_RANK=${THEID} +echo THEID=$THEID + +echo "##########################################" +echo MASTER_ADDR=${MASTER_ADDR} +echo MASTER_PORT=${MASTER_PORT} +echo NODE_RANK=${NODE_RANK} +echo WORLD_SIZE=${WORLD_SIZE} +echo "##########################################" +# debug environment worked great so we stick with it +# no magic there, just a miniconda python=3.9, pytorch=1.12, cudatoolkit=11.3 +# env with pip dependencies from stable diffusion's requirements.txt +eval "$(/fsx/stable-diffusion/debug/miniconda3/bin/conda shell.bash hook)" +conda activate torch111 +cd /fsx/robin/stable-diffusion/stable-diffusion + +CONFIG="/fsx/robin/stable-diffusion/stable-diffusion/configs/stable-diffusion/upscaling/upscale-v1-with-f16.yaml" + +# debugging +#EXTRA="${EXTRA} -d True lightning.callbacks.image_logger.params.batch_frequency=50" + +python main.py --base $CONFIG --gpus 0,1,2,3,4,5,6,7 -t --num_nodes ${WORLD_SIZE} --scale_lr False #$EXTRA diff --git a/scripts/slurm/v1-upscaling-f16-pretraining-512-aesthetics/sbatch.sh b/scripts/slurm/v1-upscaling-f16-pretraining-512-aesthetics/sbatch.sh new file mode 100755 index 0000000..ef402e0 --- /dev/null +++ b/scripts/slurm/v1-upscaling-f16-pretraining-512-aesthetics/sbatch.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#SBATCH --partition=compute-od-gpu +#SBATCH --job-name=stable-diffusion-v1-upscaling-f16-pretraining-512-aesthetics +#SBATCH --nodes 8 +#SBATCH --ntasks-per-node 1 +#SBATCH --cpus-per-gpu=4 +#SBATCH --gres=gpu:8 +#SBATCH --exclusive +#SBATCH --output=%x_%j.out +#SBATCH --comment "Key=Monitoring,Value=ON" + +module load intelmpi +source /opt/intel/mpi/latest/env/vars.sh +export LD_LIBRARY_PATH=/opt/aws-ofi-nccl/lib:/opt/amazon/efa/lib64:/usr/local/cuda-11.0/efa/lib:/usr/local/cuda-11.0/lib:/usr/local/cuda-11.0/lib64:/usr/local/cuda-11.0:/opt/nccl/build/lib:/opt/aws-ofi-nccl-inst +all/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH +export NCCL_PROTO=simple +export PATH=/opt/amazon/efa/bin:$PATH +export LD_PRELOAD="/opt/nccl/build/lib/libnccl.so" +export FI_EFA_FORK_SAFE=1 +export FI_LOG_LEVEL=1 +export FI_EFA_USE_DEVICE_RDMA=1 # use for p4dn +export NCCL_DEBUG=info +export PYTHONFAULTHANDLER=1 +export CUDA_LAUNCH_BLOCKING=0 +export OMPI_MCA_mtl_base_verbose=1 +export FI_EFA_ENABLE_SHM_TRANSFER=0 +export FI_PROVIDER=efa +export FI_EFA_TX_MIN_CREDITS=64 +export NCCL_TREE_THRESHOLD=0 + +# sent to sub script +export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"` +export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1) +export MASTER_PORT=12802 +export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l` +export WORLD_SIZE=$COUNT_NODE + +echo go $COUNT_NODE +echo $HOSTNAMES +echo $WORLD_SIZE + +mpirun -n $COUNT_NODE -perhost 1 /fsx/robin/stable-diffusion/stable-diffusion/scripts/slurm/v1-upscaling-f16-pretraining-512-aesthetics/launcher.sh diff --git a/scripts/slurm/v1_inpainting_aesthetics-larger-masks/launcher.sh b/scripts/slurm/v1_inpainting_aesthetics-larger-masks/launcher.sh new file mode 100755 index 0000000..349a00d --- /dev/null +++ b/scripts/slurm/v1_inpainting_aesthetics-larger-masks/launcher.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# mpi version for node rank +H=`hostname` +THEID=`echo -e $HOSTNAMES | python3 -c "import sys;[sys.stdout.write(str(i)) for i,line in enumerate(next(sys.stdin).split(' ')) if line.strip() == '$H'.strip()]"` +export NODE_RANK=${THEID} +echo THEID=$THEID + +echo "##########################################" +echo MASTER_ADDR=${MASTER_ADDR} +echo MASTER_PORT=${MASTER_PORT} +echo NODE_RANK=${NODE_RANK} +echo WORLD_SIZE=${WORLD_SIZE} +echo "##########################################" +# debug environment worked great so we stick with it +# no magic there, just a miniconda python=3.9, pytorch=1.12, cudatoolkit=11.3 +# env with pip dependencies from stable diffusion's requirements.txt +eval "$(/fsx/stable-diffusion/debug/miniconda3/bin/conda shell.bash hook)" +conda activate torch111 +cd /fsx/robin/stable-diffusion/stable-diffusion + +CONFIG="/fsx/robin/stable-diffusion/stable-diffusion/configs/stable-diffusion/inpainting/v1-finetune-for-inpainting-laion-aesthetic-larger-masks.yaml" + +# debugging +#EXTRA="${EXTRA} -d True lightning.callbacks.image_logger.params.batch_frequency=50" + +python main.py --base $CONFIG --gpus 0,1,2,3,4,5,6,7 -t --num_nodes ${WORLD_SIZE} --scale_lr False diff --git a/scripts/slurm/v1_inpainting_aesthetics-larger-masks/sbatch.sh b/scripts/slurm/v1_inpainting_aesthetics-larger-masks/sbatch.sh new file mode 100755 index 0000000..f04318c --- /dev/null +++ b/scripts/slurm/v1_inpainting_aesthetics-larger-masks/sbatch.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#SBATCH --partition=compute-od-gpu +#SBATCH --job-name=stable-diffusion-v1-v1_inpainting_aesthetics-larger-masks +#SBATCH --nodes 24 +#SBATCH --ntasks-per-node 1 +#SBATCH --cpus-per-gpu=4 +#SBATCH --gres=gpu:8 +#SBATCH --exclusive +#SBATCH --output=%x_%j.out +#SBATCH --comment "Key=Monitoring,Value=ON" + +module load intelmpi +source /opt/intel/mpi/latest/env/vars.sh +export LD_LIBRARY_PATH=/opt/aws-ofi-nccl/lib:/opt/amazon/efa/lib64:/usr/local/cuda-11.0/efa/lib:/usr/local/cuda-11.0/lib:/usr/local/cuda-11.0/lib64:/usr/local/cuda-11.0:/opt/nccl/build/lib:/opt/aws-ofi-nccl-inst +all/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH +export NCCL_PROTO=simple +export PATH=/opt/amazon/efa/bin:$PATH +export LD_PRELOAD="/opt/nccl/build/lib/libnccl.so" +export FI_EFA_FORK_SAFE=1 +export FI_LOG_LEVEL=1 +export FI_EFA_USE_DEVICE_RDMA=1 # use for p4dn +export NCCL_DEBUG=info +export PYTHONFAULTHANDLER=1 +export CUDA_LAUNCH_BLOCKING=0 +export OMPI_MCA_mtl_base_verbose=1 +export FI_EFA_ENABLE_SHM_TRANSFER=0 +export FI_PROVIDER=efa +export FI_EFA_TX_MIN_CREDITS=64 +export NCCL_TREE_THRESHOLD=0 + +# sent to sub script +export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"` +export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1) +export MASTER_PORT=12802 +export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l` +export WORLD_SIZE=$COUNT_NODE + +echo go $COUNT_NODE +echo $HOSTNAMES +echo $WORLD_SIZE + +mpirun -n $COUNT_NODE -perhost 1 /fsx/robin/stable-diffusion/stable-diffusion/scripts/slurm/v1_inpainting_aesthetics-larger-masks/launcher.sh \ No newline at end of file