From 1ad4ae6367399e4b41ad6611c57696429424d44b Mon Sep 17 00:00:00 2001 From: Tarun Karuturi Date: Thu, 30 May 2024 03:13:11 -0700 Subject: [PATCH] Handle time wrap around in inspector (#3766) Summary: Pull Request resolved: https://github.com/pytorch/executorch/pull/3766 In platforms which represent time using 32 bits there might be a wrap around and we handle that now in inspector.py. Ideally we should source this max value from the platform using etdump, but for now it should be fine to assume 32 bits if there's a wrap around. Reviewed By: abhiag-git, Jack-Khuu, Olivia-liu Differential Revision: D57925542 fbshipit-source-id: 69886ca1f09074d1b16f125540a41d780d6c2972 --- sdk/inspector/_inspector.py | 41 +++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/sdk/inspector/_inspector.py b/sdk/inspector/_inspector.py index 45fe272cbb..b0201c0d7b 100644 --- a/sdk/inspector/_inspector.py +++ b/sdk/inspector/_inspector.py @@ -432,6 +432,25 @@ def _gen_from_inference_events( return ret_event + @staticmethod + def _calculate_elapsed_time(start_time, end_time): + # We're assuming if there's a wraparound in the time values, then + # the time representation of that platform only contains 32 bits. + # This should be fine for now, but ideally we should source the max + # time value from the platform using etdump. + max_uint32 = 2**32 - 1 + if start_time > end_time: + if (start_time > max_uint32) or (end_time > max_uint32): + raise ValueError( + f"Expected start_time ({start_time}) and end_time ({end_time}) to be less than {max_uint32} for cases where there is wrap-around of time values." + ) + # Handle wraparound + elapsed_time = (max_uint32 - start_time) + end_time + else: + # Normal case + elapsed_time = end_time - start_time + return elapsed_time + @staticmethod def _populate_profiling_related_fields( ret_event: "Event", @@ -488,27 +507,31 @@ def _populate_profiling_related_fields( # Scale factor should only be applied to non-delegated ops if ( ret_event.is_delegated_op - and ret_event._delegate_time_scale_converter is not None + and (convert_time_scale := ret_event._delegate_time_scale_converter) + is not None ): - scaled_time = ret_event._delegate_time_scale_converter( - ret_event.name, - profile_event.end_time, - # pyre-ignore - ) - ret_event._delegate_time_scale_converter( - ret_event.name, profile_event.start_time + scaled_time = Event._calculate_elapsed_time( + convert_time_scale(ret_event.name, profile_event.start_time), + convert_time_scale(ret_event.name, profile_event.end_time), ) # If it's not a delegated op then we can just use the raw time values # and then scale them according to the scale factor that was passed in. elif not ret_event.is_delegated_op: scaled_time = ( - float(profile_event.end_time - profile_event.start_time) + float( + Event._calculate_elapsed_time( + profile_event.start_time, profile_event.end_time + ) + ) / scale_factor ) # If there was no scale factor passed in just take a difference of the # end and start times. else: scaled_time = float( - profile_event.end_time - profile_event.start_time + Event._calculate_elapsed_time( + profile_event.start_time, profile_event.end_time + ) ) data.append(scaled_time)