Skip to content

Commit

Permalink
small fix
Browse files Browse the repository at this point in the history
  • Loading branch information
blahBlahhhJ committed May 25, 2024
1 parent abcd5d2 commit 3313002
Showing 1 changed file with 5 additions and 4 deletions.
9 changes: 5 additions & 4 deletions src/levanter/mesh.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,15 @@ def process_mesh_mapping(mesh) -> dict[int, int]:
leftmost2uid = {}

for i in range(jax.process_count()):
upper_left_position = tuple([np.min(axis) for axis in local_device_grid_positions(mesh, i)])
upper_left_position[2][...] = 0 # we want the device with TP group index 0 in the same DP/FSDP group
tmp = [np.min(axis) for axis in local_device_grid_positions(mesh, i)]
tmp[-1] = 0 # we want the device with TP group index 0 in the same DP/FSDP group
upper_left_position = tuple(tmp) # in order to index into devices
upper_left_process = devices[upper_left_position].process_index
# assign uid to each process that has a device with TP group index 0
if upper_left_process not in leftmost2uid:
leftmost2uid[upper_left_position] = i
leftmost2uid[upper_left_process] = i
i += 1
uid = leftmost2uid[upper_left_position]
uid = leftmost2uid[upper_left_process]
result[i] = uid

return result

0 comments on commit 3313002

Please sign in to comment.