From abd9fb7af1d9623fcd5dce59f96adac334805fcd Mon Sep 17 00:00:00 2001 From: rromb Date: Tue, 31 May 2022 22:54:59 +0200 Subject: [PATCH] handle it --- ldm/data/laion.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ldm/data/laion.py b/ldm/data/laion.py index 92f91c5..ddba0bb 100644 --- a/ldm/data/laion.py +++ b/ldm/data/laion.py @@ -145,11 +145,12 @@ class WebDataModuleFromConfig(pl.LightningDataModule): dset = wds.WebDataset( tars, nodesplitter=nodesplitter, - shardshuffle=shardshuffle).shuffle(shuffle) + shardshuffle=shardshuffle, + handler=wds.warn_and_continue).shuffle(shuffle) print(f'Loading webdataset with {len(dset.pipeline[0].urls)} shards.') dset = (dset - .decode('pil', handler=warn_and_continue) + .decode('pil', handler=wds.warn_and_continue) .map_dict(**transform_dict, handler=wds.warn_and_continue) .batched(self.batch_size, partial=False, collation_fn=dict_collation_fn)