never run out of data

2022-06-13 10:59:50 +00:00 · 2022-06-13 10:59:50 +00:00 · 6f604acf0a
parent 046410167f
commit 6f604acf0a
1 changed files with 1 additions and 1 deletions
--- a/ldm/data/laion.py
+++ b/ldm/data/laion.py
@ -153,7 +153,7 @@ class WebDataModuleFromConfig(pl.LightningDataModule):
                tars,
                nodesplitter=nodesplitter,
                shardshuffle=shardshuffle,
-                handler=wds.warn_and_continue).shuffle(shuffle)
+                handler=wds.warn_and_continue).repeat().shuffle(shuffle)
        print(f'Loading webdataset with {len(dset.pipeline[0].urls)} shards.')

        dset = (dset