Skip to content

Commit

Permalink
cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
dlwh committed Oct 14, 2024
1 parent 2747705 commit 538f0ed
Showing 1 changed file with 0 additions and 8 deletions.
8 changes: 0 additions & 8 deletions src/levanter/data/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,14 +263,6 @@ def _batchified_shape(Batch, leaf: hax.NamedArray | Array) -> ShapeSpec | NamedS
return ShapeSpec((Batch.size,) + leaf.shape, leaf.dtype)


def _pspec_for(self, shape_spec: ShapeSpec | NamedShapeSpec) -> PartitionSpec:
if isinstance(shape_spec, ShapeSpec): # type: ignore
batch_name = hax.partitioning.physical_axis_name(self.Batch, self.axis_resources)
return PartitionSpec(batch_name, *((None,) * (len(shape_spec.shape) - 1)))
else:
return hax.partitioning.pspec_for_axis(shape_spec.shape, self.axis_resources) # type: ignore


@functools.partial(jax.jit, static_argnums=(0,))
def _stack_tree(batch_name, individual_datums):
def _stack_leaves_unchecked(*leaves):
Expand Down

0 comments on commit 538f0ed

Please sign in to comment.