Chunksize 40

pytorch · Feb 6, 2025 · 88ab3c7 · 88ab3c7
1 parent 65744ac
commit 88ab3c7
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/examples/ibm_rescaling/rescaling_demo.py b/examples/ibm_rescaling/rescaling_demo.py
@@ -77,9 +77,9 @@
         time.sleep(5)
 
 # Build dataloader
-data = ScalableReader(datapath, rank, world_size, ArrowHandler, -1, seed=args.seed, max_chunksize=30, n_logical_shards=args.logical_shards)
+data = ScalableReader(datapath, rank, world_size, ArrowHandler, -1, seed=args.seed, max_chunksize=40, n_logical_shards=args.logical_shards)
 # Pad entries to make them batch-able
-data = PreprocessDataset(data, lambda x: x + [-1]*(30-len(x)))
+data = PreprocessDataset(data, lambda x: x + [-1]*(40-len(x)))
 # Statelessly convert all outputs to tensors
 data = PreprocessDataset(data, torch.tensor)
 # Wrap in StatefulDataLoader