Set shuffle=True by default in data_sampler

deepspeed/runtime/engine.py

@@ -1784,7 +1784,7 @@ def deepspeed_io(self,
                 dataset,
                 num_replicas=data_parallel_world_size,
                 rank=data_parallel_rank,
-                shuffle=False,
+                shuffle=True,
             )
 
         deepspeed_dataloader_config = {}

deepspeed/runtime/pipe/engine.py

@@ -266,7 +266,7 @@ def _build_data_iter(self, dataset):
         sampler = torch.utils.data.distributed.DistributedSampler(dataset,
                                                                   num_replicas=self.dp_world_size,
                                                                   rank=self.mpu.get_data_parallel_rank(),
-                                                                  shuffle=False)
+                                                                  shuffle=True)
         # Build a loader and make it repeating.
         pipe_dataloader = self.deepspeed_io(dataset, data_sampler=sampler)
         pipe_dataloader = RepeatingLoader(pipe_dataloader)