diff --git a/configs/stable-diffusion/inpainting/v1-finetune-for-inpainting-laion-iaesthe.yaml b/configs/stable-diffusion/inpainting/v1-finetune-for-inpainting-laion-iaesthe.yaml new file mode 100644 index 0000000..af9c568 --- /dev/null +++ b/configs/stable-diffusion/inpainting/v1-finetune-for-inpainting-laion-iaesthe.yaml @@ -0,0 +1,144 @@ +model: + base_learning_rate: 7.5e-05 + target: ldm.models.diffusion.ddpm.LatentInpaintDiffusion + params: + linear_start: 0.00085 + linear_end: 0.0120 + num_timesteps_cond: 1 + log_every_t: 200 + timesteps: 1000 + first_stage_key: "jpg" + cond_stage_key: "txt" + image_size: 64 + channels: 4 + cond_stage_trainable: false # Note: different from the one we trained before + conditioning_key: hybrid # important + monitor: val/loss_simple_ema + scale_factor: 0.18215 + ckpt_path: "/fsx/stable-diffusion/stable-diffusion/checkpoints2/v1pp/v1pp-flatline-pruned.ckpt" + + scheduler_config: # 10000 warmup steps + target: ldm.lr_scheduler.LambdaLinearScheduler + params: + warm_up_steps: [ 2500 ] # NOTE for resuming. use 10000 if starting from scratch + cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases + f_start: [ 1.e-6 ] + f_max: [ 1. ] + f_min: [ 1. ] + + unet_config: + target: ldm.modules.diffusionmodules.openaimodel.UNetModel + params: + image_size: 32 # unused + in_channels: 9 # 4 data + 4 downscaled image + 1 mask + out_channels: 4 + model_channels: 320 + attention_resolutions: [ 4, 2, 1 ] + num_res_blocks: 2 + channel_mult: [ 1, 2, 4, 4 ] + num_heads: 8 + use_spatial_transformer: True + transformer_depth: 1 + context_dim: 768 + use_checkpoint: True + legacy: False + + first_stage_config: + target: ldm.models.autoencoder.AutoencoderKL + params: + embed_dim: 4 + monitor: val/rec_loss + ddconfig: + double_z: true + z_channels: 4 + resolution: 256 + in_channels: 3 + out_ch: 3 + ch: 128 + ch_mult: + - 1 + - 2 + - 4 + - 4 + num_res_blocks: 2 + attn_resolutions: [] + dropout: 0.0 + lossconfig: + target: torch.nn.Identity + + cond_stage_config: + target: ldm.modules.encoders.modules.FrozenCLIPEmbedder + + +data: + target: ldm.data.laion.WebDataModuleFromConfig + params: + tar_base: "pipe:aws s3 cp s3://s-datasets/laion-high-resolution/" + batch_size: 4 + num_workers: 4 + multinode: True + min_size: 512 + train: + shards: '{00000..17279}.tar -' + shuffle: 10000 + image_key: jpg + image_transforms: + - target: torchvision.transforms.Resize + params: + size: 512 + interpolation: 3 + - target: torchvision.transforms.RandomCrop + params: + size: 512 + postprocess: + target: ldm.data.laion.AddMask + + # NOTE use enough shards to avoid empty validation loops in workers + validation: + shards: '{17280..17535}.tar -' + shuffle: 0 + image_key: jpg + image_transforms: + - target: torchvision.transforms.Resize + params: + size: 512 + interpolation: 3 + - target: torchvision.transforms.CenterCrop + params: + size: 512 + postprocess: + target: ldm.data.laion.AddMask + + +lightning: + find_unused_parameters: False + + modelcheckpoint: + params: + every_n_train_steps: 2000 + + callbacks: + image_logger: + target: main.ImageLogger + params: + disabled: False + batch_frequency: 1000 + max_images: 4 + increase_log_steps: False + log_first_step: False + log_images_kwargs: + use_ema_scope: False + inpaint: False + plot_progressive_rows: False + plot_diffusion_rows: False + N: 4 + unconditional_guidance_scale: 3.0 + unconditional_guidance_label: [""] + ddim_steps: 100 # todo check these out for inpainting, + ddim_eta: 1.0 # todo check these out for inpainting, + + trainer: + benchmark: True + val_check_interval: 5000000 # really sorry + num_sanity_val_steps: 0 + accumulate_grad_batches: 2 diff --git a/ldm/data/base.py b/ldm/data/base.py index b196c2f..742794e 100644 --- a/ldm/data/base.py +++ b/ldm/data/base.py @@ -1,3 +1,5 @@ +import os +import numpy as np from abc import abstractmethod from torch.utils.data import Dataset, ConcatDataset, ChainDataset, IterableDataset @@ -20,4 +22,19 @@ class Txt2ImgIterableBaseDataset(IterableDataset): @abstractmethod def __iter__(self): - pass \ No newline at end of file + pass + + +class PRNGMixin(object): + """ + Adds a prng property which is a numpy RandomState which gets + reinitialized whenever the pid changes to avoid synchronized sampling + behavior when used in conjunction with multiprocessing. + """ + @property + def prng(self): + currentpid = os.getpid() + if getattr(self, "_initpid", None) != currentpid: + self._initpid = currentpid + self._prng = np.random.RandomState() + return self._prng diff --git a/ldm/data/inpainting/__init__.py b/ldm/data/inpainting/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ldm/data/inpainting/synthetic_mask.py b/ldm/data/inpainting/synthetic_mask.py new file mode 100644 index 0000000..9dcc3d6 --- /dev/null +++ b/ldm/data/inpainting/synthetic_mask.py @@ -0,0 +1,150 @@ +from PIL import Image, ImageDraw +import numpy as np + +settings = { + "256narrow": { + "p_irr": 1, + "min_n_irr": 4, + "max_n_irr": 50, + "max_l_irr": 40, + "max_w_irr": 10, + "min_n_box": None, + "max_n_box": None, + "min_s_box": None, + "max_s_box": None, + "marg": None, + }, + "256train": { + "p_irr": 0.5, + "min_n_irr": 1, + "max_n_irr": 5, + "max_l_irr": 200, + "max_w_irr": 100, + "min_n_box": 1, + "max_n_box": 4, + "min_s_box": 30, + "max_s_box": 150, + "marg": 10, + }, + "512train": { # TODO: experimental + "p_irr": 0.5, + "min_n_irr": 1, + "max_n_irr": 5, + "max_l_irr": 450, + "max_w_irr": 250, + "min_n_box": 1, + "max_n_box": 4, + "min_s_box": 30, + "max_s_box": 300, + "marg": 10, + }, +} + + +def gen_segment_mask(mask, start, end, brush_width): + mask = mask > 0 + mask = (255 * mask).astype(np.uint8) + mask = Image.fromarray(mask) + draw = ImageDraw.Draw(mask) + draw.line([start, end], fill=255, width=brush_width, joint="curve") + mask = np.array(mask) / 255 + return mask + + +def gen_box_mask(mask, masked): + x_0, y_0, w, h = masked + mask[y_0:y_0 + h, x_0:x_0 + w] = 1 + return mask + + +def gen_round_mask(mask, masked, radius): + x_0, y_0, w, h = masked + xy = [(x_0, y_0), (x_0 + w, y_0 + w)] + + mask = mask > 0 + mask = (255 * mask).astype(np.uint8) + mask = Image.fromarray(mask) + draw = ImageDraw.Draw(mask) + draw.rounded_rectangle(xy, radius=radius, fill=255) + mask = np.array(mask) / 255 + return mask + + +def gen_large_mask(prng, img_h, img_w, + marg, p_irr, min_n_irr, max_n_irr, max_l_irr, max_w_irr, + min_n_box, max_n_box, min_s_box, max_s_box): + """ + img_h: int, an image height + img_w: int, an image width + marg: int, a margin for a box starting coordinate + p_irr: float, 0 <= p_irr <= 1, a probability of a polygonal chain mask + + min_n_irr: int, min number of segments + max_n_irr: int, max number of segments + max_l_irr: max length of a segment in polygonal chain + max_w_irr: max width of a segment in polygonal chain + + min_n_box: int, min bound for the number of box primitives + max_n_box: int, max bound for the number of box primitives + min_s_box: int, min length of a box side + max_s_box: int, max length of a box side + """ + + mask = np.zeros((img_h, img_w)) + uniform = prng.randint + + if np.random.uniform(0, 1) < p_irr: # generate polygonal chain + n = uniform(min_n_irr, max_n_irr) # sample number of segments + + for _ in range(n): + y = uniform(0, img_h) # sample a starting point + x = uniform(0, img_w) + + a = uniform(0, 360) # sample angle + l = uniform(10, max_l_irr) # sample segment length + w = uniform(5, max_w_irr) # sample a segment width + + # draw segment starting from (x,y) to (x_,y_) using brush of width w + x_ = x + l * np.sin(a) + y_ = y + l * np.cos(a) + + mask = gen_segment_mask(mask, start=(x, y), end=(x_, y_), brush_width=w) + x, y = x_, y_ + else: # generate Box masks + n = uniform(min_n_box, max_n_box) # sample number of rectangles + + for _ in range(n): + h = uniform(min_s_box, max_s_box) # sample box shape + w = uniform(min_s_box, max_s_box) + + x_0 = uniform(marg, img_w - marg - w) # sample upper-left coordinates of box + y_0 = uniform(marg, img_h - marg - h) + + if np.random.uniform(0, 1) < 0.5: + mask = gen_box_mask(mask, masked=(x_0, y_0, w, h)) + else: + r = uniform(0, 60) # sample radius + mask = gen_round_mask(mask, masked=(x_0, y_0, w, h), radius=r) + return mask + + +make_lama_mask = lambda prng, h, w: gen_large_mask(prng, h, w, + **settings["256train"]) + +make_narrow_lama_mask = lambda prng, h, w: gen_large_mask(prng, h, w, + **settings["256narrow"]) + +make_512_lama_mask = lambda prng, h, w: gen_large_mask(prng, h, w, + **settings["512train"]) + +if __name__ == "__main__": + import sys + + out = sys.argv[1] + + prng = np.random.RandomState(1) + kwargs = settings["256train"] + mask = gen_large_mask(prng, 256, 256, **kwargs) + mask = (255 * mask).astype(np.uint8) + mask = Image.fromarray(mask) + mask.save(out) diff --git a/ldm/data/laion.py b/ldm/data/laion.py index e3e14a3..4c9b73d 100644 --- a/ldm/data/laion.py +++ b/ldm/data/laion.py @@ -16,6 +16,8 @@ from webdataset.handlers import warn_and_continue from ldm.util import instantiate_from_config +from ldm.data.inpainting.synthetic_mask import gen_large_mask, make_lama_mask, make_narrow_lama_mask, make_512_lama_mask +from ldm.data.base import PRNGMixin class DataWithWings(torch.utils.data.IterableDataset): @@ -229,6 +231,23 @@ class AddLR(object): return sample +class AddMask(PRNGMixin): + def __init__(self, size=512): + super().__init__() + self.make_mask = make_512_lama_mask if size == 512 else make_lama_mask + + def __call__(self, sample): + # sample['jpg'] is tensor hwc in [-1, 1] at this point + x = sample['jpg'] + mask = self.make_mask(self.prng, x.shape[0], x.shape[1]) + mask[mask < 0.5] = 0 + mask[mask > 0.5] = 1 + mask = torch.from_numpy(mask[..., None]) + sample['mask'] = mask + sample['masked_image'] = x * (mask < 0.5) + return sample + + def example00(): url = "pipe:aws s3 cp s3://s-datasets/laion5b/laion2B-data/000000.tar -" dataset = wds.WebDataset(url) diff --git a/ldm/models/diffusion/ddpm.py b/ldm/models/diffusion/ddpm.py index 8d53d4c..0b9bc98 100644 --- a/ldm/models/diffusion/ddpm.py +++ b/ldm/models/diffusion/ddpm.py @@ -1260,7 +1260,6 @@ class LatentDiffusion(DDPM): use_ema_scope=True, **kwargs): ema_scope = self.ema_scope if use_ema_scope else nullcontext - use_ddim = ddim_steps is not None log = dict() @@ -1582,6 +1581,168 @@ class LatentUpscaleDiffusion(LatentDiffusion): return log +class LatentInpaintDiffusion(LatentDiffusion): + """ + can either run as pure inpainting model (only concat mode) or with mixed conditionings, + e.g. mask as concat and text via cross-attn. + To disable finetuning mode, set finetune_keys to None + """ + def __init__(self, + finetune_keys=("model.diffusion_model.input_blocks.0.0.weight", "model_ema.diffusion_modelinput_blocks00weight"), + concat_keys=("mask", "masked_image"), + masked_image_key="masked_image", + keep_finetune_dims=4, # if model was trained without concat mode before and we would like to keep these channels + *args, **kwargs + ): + ckpt_path = kwargs.pop("ckpt_path", None) + ignore_keys = kwargs.pop("ignore_keys", list()) + super().__init__(*args, **kwargs) + self.masked_image_key = masked_image_key + assert self.masked_image_key in concat_keys + self.finetune_keys = finetune_keys + self.concat_keys = concat_keys + self.keep_dims = keep_finetune_dims + if exists(self.finetune_keys): assert exists(ckpt_path), 'can only finetune from a given checkpoint' + if exists(ckpt_path): + self.init_from_ckpt(ckpt_path, ignore_keys) + + def init_from_ckpt(self, path, ignore_keys=list(), only_model=False): + sd = torch.load(path, map_location="cpu") + if "state_dict" in list(sd.keys()): + sd = sd["state_dict"] + keys = list(sd.keys()) + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print("Deleting key {} from state_dict.".format(k)) + del sd[k] + + # make it explicit, finetune by including extra input channels + if exists(self.finetune_keys) and k in self.finetune_keys: + new_entry = None + for name, param in self.named_parameters(): + if name in self.finetune_keys: + print(f"modifying key '{name}' and keeping its original {self.keep_dims} (channels) dimensions only") + new_entry = torch.zeros_like(param) # zero init + assert exists(new_entry), 'did not find matching parameter to modify' + new_entry[:, :self.keep_dims, ...] = sd[k] + sd[k] = new_entry + + missing, unexpected = self.load_state_dict(sd, strict=False) if not only_model else self.model.load_state_dict( + sd, strict=False) + print(f"Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys") + if len(missing) > 0: + print(f"Missing Keys: {missing}") + if len(unexpected) > 0: + print(f"Unexpected Keys: {unexpected}") + + @torch.no_grad() + def get_input(self, batch, k, cond_key=None, bs=None, return_first_stage_outputs=False): + # note: restricted to non-trainable encoders currently + assert not self.cond_stage_trainable, 'trainable cond stages not yet supported for inpaiting' + z, c, x, xrec, xc = super().get_input(batch, self.first_stage_key, return_first_stage_outputs=True, + force_c_encode=True, return_original_cond=True, bs=bs) + + assert exists(self.concat_keys) + c_cat = list() + for ck in self.concat_keys: + cc = rearrange(batch[ck], 'b h w c -> b c h w').to(memory_format=torch.contiguous_format).float() + if bs is not None: + cc = cc[:bs] + cc = cc.to(self.device) + bchw = z.shape + if ck != self.masked_image_key: + cc = torch.nn.functional.interpolate(cc, size=bchw[-2:]) + else: + cc = self.get_first_stage_encoding(self.encode_first_stage(cc)) + c_cat.append(cc) + c_cat = torch.cat(c_cat, dim=1) + all_conds = {"c_concat": [c_cat], "c_crossattn": [c]} + if return_first_stage_outputs: + return z, all_conds, x, xrec, xc + return z, all_conds + + @torch.no_grad() + def log_images(self, batch, N=8, n_row=4, sample=True, ddim_steps=200, ddim_eta=1., return_keys=None, + quantize_denoised=True, inpaint=True, plot_denoise_rows=False, plot_progressive_rows=True, + plot_diffusion_rows=True, unconditional_guidance_scale=1., unconditional_guidance_label=None, + use_ema_scope=True, + **kwargs): + ema_scope = self.ema_scope if use_ema_scope else nullcontext + use_ddim = ddim_steps is not None + + log = dict() + z, c, x, xrec, xc = self.get_input(batch, self.first_stage_key, bs=N, return_first_stage_outputs=True) + c_cat, c = c["c_concat"][0], c["c_crossattn"][0] + N = min(x.shape[0], N) + n_row = min(x.shape[0], n_row) + log["inputs"] = x + log["reconstruction"] = xrec + if self.model.conditioning_key is not None: + if hasattr(self.cond_stage_model, "decode"): + xc = self.cond_stage_model.decode(c) + log["conditioning"] = xc + elif self.cond_stage_key in ["caption", "txt"]: + xc = log_txt_as_img((x.shape[2], x.shape[3]), batch[self.cond_stage_key], size=x.shape[2] // 25) + log["conditioning"] = xc + elif self.cond_stage_key == 'class_label': + xc = log_txt_as_img((x.shape[2], x.shape[3]), batch["human_label"], size=x.shape[2] // 25) + log['conditioning'] = xc + elif isimage(xc): + log["conditioning"] = xc + if ismap(xc): + log["original_conditioning"] = self.to_rgb(xc) + + if plot_diffusion_rows: + # get diffusion row + diffusion_row = list() + z_start = z[:n_row] + for t in range(self.num_timesteps): + if t % self.log_every_t == 0 or t == self.num_timesteps - 1: + t = repeat(torch.tensor([t]), '1 -> b', b=n_row) + t = t.to(self.device).long() + noise = torch.randn_like(z_start) + z_noisy = self.q_sample(x_start=z_start, t=t, noise=noise) + diffusion_row.append(self.decode_first_stage(z_noisy)) + + diffusion_row = torch.stack(diffusion_row) # n_log_step, n_row, C, H, W + diffusion_grid = rearrange(diffusion_row, 'n b c h w -> b n c h w') + diffusion_grid = rearrange(diffusion_grid, 'b n c h w -> (b n) c h w') + diffusion_grid = make_grid(diffusion_grid, nrow=diffusion_row.shape[0]) + log["diffusion_row"] = diffusion_grid + + if sample: + # get denoise row + with ema_scope("Sampling"): + samples, z_denoise_row = self.sample_log(cond={"c_concat": [c_cat], "c_crossattn": [c]}, + batch_size=N, ddim=use_ddim, + ddim_steps=ddim_steps, eta=ddim_eta) + # samples, z_denoise_row = self.sample(cond=c, batch_size=N, return_intermediates=True) + x_samples = self.decode_first_stage(samples) + log["samples"] = x_samples + if plot_denoise_rows: + denoise_grid = self._get_denoise_row_from_list(z_denoise_row) + log["denoise_row"] = denoise_grid + + if unconditional_guidance_scale > 1.0: + uc_cross = self.get_unconditional_conditioning(N, unconditional_guidance_label) + uc_cat = c_cat + uc_full = {"c_concat": [uc_cat], "c_crossattn": [uc_cross]} + with ema_scope("Sampling with classifier-free guidance"): + samples_cfg, _ = self.sample_log(cond={"c_concat": [c_cat], "c_crossattn": [c]}, + batch_size=N, ddim=use_ddim, + ddim_steps=ddim_steps, eta=ddim_eta, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=uc_full, + ) + x_samples_cfg = self.decode_first_stage(samples_cfg) + log[f"samples_cfg_scale_{unconditional_guidance_scale:.2f}"] = x_samples_cfg + + log["masked_image"] = rearrange(batch["masked_image"], + 'b h w c -> b c h w').to(memory_format=torch.contiguous_format).float() + return log + + class Layout2ImgDiffusion(LatentDiffusion): # TODO: move all layout-specific hacks to this class def __init__(self, cond_stage_key, *args, **kwargs): diff --git a/scripts/slurm/v1_inpainting_improvedaesthetics_torch111/launcher.sh b/scripts/slurm/v1_inpainting_improvedaesthetics_torch111/launcher.sh new file mode 100755 index 0000000..d507c53 --- /dev/null +++ b/scripts/slurm/v1_inpainting_improvedaesthetics_torch111/launcher.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# mpi version for node rank +H=`hostname` +THEID=`echo -e $HOSTNAMES | python3 -c "import sys;[sys.stdout.write(str(i)) for i,line in enumerate(next(sys.stdin).split(' ')) if line.strip() == '$H'.strip()]"` +export NODE_RANK=${THEID} +echo THEID=$THEID + +echo "##########################################" +echo MASTER_ADDR=${MASTER_ADDR} +echo MASTER_PORT=${MASTER_PORT} +echo NODE_RANK=${NODE_RANK} +echo WORLD_SIZE=${WORLD_SIZE} +echo "##########################################" +# debug environment worked great so we stick with it +# no magic there, just a miniconda python=3.9, pytorch=1.12, cudatoolkit=11.3 +# env with pip dependencies from stable diffusion's requirements.txt +eval "$(/fsx/stable-diffusion/debug/miniconda3/bin/conda shell.bash hook)" +conda activate torch111 +cd /fsx/robin/stable-diffusion/stable-diffusion + +CONFIG="/fsx/robin/stable-diffusion/stable-diffusion/configs/stable-diffusion/inpainting/v1-finetune-for-inpainting-laion-iaesthe.yaml" + +# debugging +#EXTRA="${EXTRA} -d True lightning.callbacks.image_logger.params.batch_frequency=50" + +python main.py --base $CONFIG --gpus 0,1,2,3,4,5,6,7 -t --num_nodes ${WORLD_SIZE} --scale_lr False #$EXTRA diff --git a/scripts/slurm/v1_inpainting_improvedaesthetics_torch111/sbatch.sh b/scripts/slurm/v1_inpainting_improvedaesthetics_torch111/sbatch.sh new file mode 100755 index 0000000..cc714b0 --- /dev/null +++ b/scripts/slurm/v1_inpainting_improvedaesthetics_torch111/sbatch.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#SBATCH --partition=compute-od-gpu +#SBATCH --job-name=stable-diffusion-v1-inpainting-improvedaesthetics-torch111 +#SBATCH --nodes 8 +#SBATCH --ntasks-per-node 1 +#SBATCH --cpus-per-gpu=4 +#SBATCH --gres=gpu:8 +#SBATCH --exclusive +#SBATCH --output=%x_%j.out +#SBATCH --comment "Key=Monitoring,Value=ON" + +module load intelmpi +source /opt/intel/mpi/latest/env/vars.sh +export LD_LIBRARY_PATH=/opt/aws-ofi-nccl/lib:/opt/amazon/efa/lib64:/usr/local/cuda-11.0/efa/lib:/usr/local/cuda-11.0/lib:/usr/local/cuda-11.0/lib64:/usr/local/cuda-11.0:/opt/nccl/build/lib:/opt/aws-ofi-nccl-inst +all/lib:/opt/aws-ofi-nccl/lib:$LD_LIBRARY_PATH +export NCCL_PROTO=simple +export PATH=/opt/amazon/efa/bin:$PATH +export LD_PRELOAD="/opt/nccl/build/lib/libnccl.so" +export FI_EFA_FORK_SAFE=1 +export FI_LOG_LEVEL=1 +export FI_EFA_USE_DEVICE_RDMA=1 # use for p4dn +export NCCL_DEBUG=info +export PYTHONFAULTHANDLER=1 +export CUDA_LAUNCH_BLOCKING=0 +export OMPI_MCA_mtl_base_verbose=1 +export FI_EFA_ENABLE_SHM_TRANSFER=0 +export FI_PROVIDER=efa +export FI_EFA_TX_MIN_CREDITS=64 +export NCCL_TREE_THRESHOLD=0 + +# sent to sub script +export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"` +export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1) +export MASTER_PORT=12802 +export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l` +export WORLD_SIZE=$COUNT_NODE + +echo go $COUNT_NODE +echo $HOSTNAMES +echo $WORLD_SIZE + +mpirun -n $COUNT_NODE -perhost 1 /fsx/robin/stable-diffusion/stable-diffusion/scripts/slurm/v1_inpainting_improvedaesthetics_torch111/launcher.sh