diff --git a/benchmarks/decoders/benchmark_decoders_library.py b/benchmarks/decoders/benchmark_decoders_library.py index 4646d80a..f5f2fe8a 100644 --- a/benchmarks/decoders/benchmark_decoders_library.py +++ b/benchmarks/decoders/benchmark_decoders_library.py @@ -123,7 +123,7 @@ def get_consecutive_frames_from_video(self, video_file, numFramesToDecode): def decode_and_transform(self, video_file, pts_list, height, width, device): self.torchvision.set_video_backend(self._backend) - reader = self.torchvision.io.VideoReader(video_file, "video") + reader = self.torchvision.io.VideoReader(video_file, "video", num_threads=1) frames = [] for pts in pts_list: reader.seek(pts) @@ -177,15 +177,20 @@ def get_consecutive_frames_from_video(self, video_file, numFramesToDecode): class TorchCodecCoreNonBatch(AbstractDecoder): def __init__(self, num_threads=None, color_conversion_library=None, device="cpu"): - self._num_threads = int(num_threads) if num_threads else None + self._num_threads = num_threads self._color_conversion_library = color_conversion_library self._device = device + from torchvision.transforms import v2 as transforms_v2 + + self.transforms_v2 = transforms_v2 + def get_frames_from_video(self, video_file, pts_list): decoder = create_from_file(video_file) + num_threads = int(self._num_threads) if self._num_threads else 0 _add_video_stream( decoder, - num_threads=self._num_threads, + num_threads=num_threads, color_conversion_library=self._color_conversion_library, device=self._device, ) @@ -199,10 +204,11 @@ def get_frames_from_video(self, video_file, pts_list): return frames def get_consecutive_frames_from_video(self, video_file, numFramesToDecode): + num_threads = int(self._num_threads) if self._num_threads else 0 decoder = create_from_file(video_file) _add_video_stream( decoder, - num_threads=self._num_threads, + num_threads=num_threads, color_conversion_library=self._color_conversion_library, device=self._device, ) @@ -214,6 +220,29 @@ def get_consecutive_frames_from_video(self, video_file, numFramesToDecode): return frames + def decode_and_transform(self, video_file, pts_list, height, width, device): + num_threads = int(self._num_threads) if self._num_threads else 1 + decoder = create_from_file(video_file) + _add_video_stream( + decoder, + num_threads=num_threads, + color_conversion_library=self._color_conversion_library, + device=self._device, + ) + + frames = [] + for pts in pts_list: + seek_to_pts(decoder, pts) + frame, *_ = get_next_frame(decoder) + frames.append(frame) + + frames = [ + self.transforms_v2.functional.resize(frame.to(device), (height, width)) + for frame in frames + ] + + return frames + class TorchCodecCoreBatch(AbstractDecoder): def __init__(self, num_threads=None, color_conversion_library=None, device="cpu"): @@ -258,7 +287,7 @@ def get_consecutive_frames_from_video(self, video_file, numFramesToDecode): class TorchCodecPublic(AbstractDecoder): def __init__(self, num_ffmpeg_threads=None, device="cpu"): - self._num_ffmpeg_threads = int(num_ffmpeg_threads) if num_ffmpeg_threads else 1 + self._num_ffmpeg_threads = num_ffmpeg_threads self._device = device from torchvision.transforms import v2 as transforms_v2 @@ -266,14 +295,20 @@ def __init__(self, num_ffmpeg_threads=None, device="cpu"): self.transforms_v2 = transforms_v2 def get_frames_from_video(self, video_file, pts_list): + num_ffmpeg_threads = ( + int(self._num_ffmpeg_threads) if self._num_ffmpeg_threads else 0 + ) decoder = VideoDecoder( - video_file, num_ffmpeg_threads=self._num_ffmpeg_threads, device=self._device + video_file, num_ffmpeg_threads=num_ffmpeg_threads, device=self._device ) return decoder.get_frames_played_at(pts_list) def get_consecutive_frames_from_video(self, video_file, numFramesToDecode): + num_ffmpeg_threads = ( + int(self._num_ffmpeg_threads) if self._num_ffmpeg_threads else 0 + ) decoder = VideoDecoder( - video_file, num_ffmpeg_threads=self._num_ffmpeg_threads, device=self._device + video_file, num_ffmpeg_threads=num_ffmpeg_threads, device=self._device ) frames = [] count = 0 @@ -285,8 +320,11 @@ def get_consecutive_frames_from_video(self, video_file, numFramesToDecode): return frames def decode_and_transform(self, video_file, pts_list, height, width, device): + num_ffmpeg_threads = ( + int(self._num_ffmpeg_threads) if self._num_ffmpeg_threads else 1 + ) decoder = VideoDecoder( - video_file, num_ffmpeg_threads=self._num_ffmpeg_threads, device=self._device + video_file, num_ffmpeg_threads=num_ffmpeg_threads, device=self._device ) frames = decoder.get_frames_played_at(pts_list) frames = self.transforms_v2.functional.resize(frames.data, (height, width)) @@ -339,7 +377,9 @@ def __init__(self): def get_frames_from_video(self, video_file, pts_list): stream_reader = self.torchaudio.io.StreamReader(src=video_file) - stream_reader.add_basic_video_stream(frames_per_chunk=1) + stream_reader.add_basic_video_stream( + frames_per_chunk=1, decoder_option={"threads": "0"} + ) frames = [] for pts in pts_list: stream_reader.seek(pts) @@ -350,7 +390,9 @@ def get_frames_from_video(self, video_file, pts_list): def get_consecutive_frames_from_video(self, video_file, numFramesToDecode): stream_reader = self.torchaudio.io.StreamReader(src=video_file) - stream_reader.add_basic_video_stream(frames_per_chunk=1) + stream_reader.add_basic_video_stream( + frames_per_chunk=1, decoder_option={"threads": "0"} + ) frames = [] frame_cnt = 0 for vframe in stream_reader.stream(): @@ -363,7 +405,9 @@ def get_consecutive_frames_from_video(self, video_file, numFramesToDecode): def decode_and_transform(self, video_file, pts_list, height, width, device): stream_reader = self.torchaudio.io.StreamReader(src=video_file) - stream_reader.add_basic_video_stream(frames_per_chunk=1) + stream_reader.add_basic_video_stream( + frames_per_chunk=1, decoder_option={"threads": "1"} + ) frames = [] for pts in pts_list: stream_reader.seek(pts) @@ -634,7 +678,10 @@ def run_benchmarks( }, label=f"video={video_file_path} {metadata_label}", sub_label=decoder_name, - description=f"dataloader[threads={bp.num_threads},batch_size={bp.batch_size}] {num_samples} decode_and_transform()", + description=f"concurrent[threads={bp.num_threads},batch_size={bp.batch_size}] {num_samples} decode_and_transform()", + ) + print( + f"{decoder_name} concurrent[threads={bp.num_threads} batch_size={bp.batch_size}]" ) results.append( dataloader_result.blocked_autorange( @@ -647,7 +694,7 @@ def run_benchmarks( decoder_name, video_file_path, num_samples * dataloader_parameters.batch_parameters.batch_size, - f"dataloader[threads={bp.num_threads} batch_size={bp.batch_size}] {num_samples} x decode_and_transform()", + f"concurrent[threads={bp.num_threads} batch_size={bp.batch_size}] {num_samples} x decode_and_transform()", ) ) @@ -670,6 +717,7 @@ def run_benchmarks( sub_label=decoder_name, description=f"{kind} {num_samples} seek()+next()", ) + print(f"{decoder_name} {kind} {num_samples} seek()+next()") results.append( seeked_result.blocked_autorange(min_run_time=min_runtime_seconds) ) @@ -697,6 +745,7 @@ def run_benchmarks( sub_label=decoder_name, description=f"batch {kind} {num_samples} seek()+next()", ) + print(f"{decoder_name} batch {kind} {num_samples} seek()+next()") results.append( seeked_result.blocked_autorange( min_run_time=min_runtime_seconds @@ -724,6 +773,7 @@ def run_benchmarks( sub_label=decoder_name, description=f"{num_consecutive_nexts} next()", ) + print(f"{decoder_name} {num_consecutive_nexts} next()") results.append( consecutive_frames_result.blocked_autorange( min_run_time=min_runtime_seconds @@ -753,6 +803,7 @@ def run_benchmarks( sub_label=decoder_name, description=f"batch {num_consecutive_nexts} next()", ) + print(f"{decoder_name} batch {num_consecutive_nexts} next()") results.append( consecutive_frames_result.blocked_autorange( min_run_time=min_runtime_seconds diff --git a/benchmarks/decoders/benchmark_readme_chart.png b/benchmarks/decoders/benchmark_readme_chart.png index 2ead24b5..64c99ff1 100644 Binary files a/benchmarks/decoders/benchmark_readme_chart.png and b/benchmarks/decoders/benchmark_readme_chart.png differ diff --git a/benchmarks/decoders/benchmark_readme_data.json b/benchmarks/decoders/benchmark_readme_data.json index 3bfa1f18..7269802f 100644 --- a/benchmarks/decoders/benchmark_readme_data.json +++ b/benchmarks/decoders/benchmark_readme_data.json @@ -1,397 +1,395 @@ { - "experiments": - [ - { - "decoder": "torchcodec", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 2.1909987258958803, - "fps_p25": 2.1909987258958803, - "fps_p75": 2.1909987258958803, - "frame_count": 500, - "iqr": 0.0, - "median": 228.2064311997965, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "uniform 10 seek()+next()", - "fps_median": 0.22373366699708405, - "fps_p25": 0.22373366699708405, - "fps_p75": 0.22373366699708405, - "frame_count": 10, - "iqr": 0.0, - "median": 44.69600008893758, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "random 10 seek()+next()", - "fps_median": 0.2870396511009182, - "fps_p25": 0.2870396511009182, - "fps_p75": 0.2870396511009182, - "frame_count": 10, - "iqr": 0.0, - "median": 34.83839240204543, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "100 next()", - "fps_median": 79.89466672412033, - "fps_p25": 99.18266043890266, - "fps_p75": 79.07153270302595, - "frame_count": 100, - "iqr": 0.2564368974417448, - "median": 1.2516480023041368, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 7.324784294018666, - "fps_p25": 7.324784294018666, - "fps_p75": 7.324784294018666, - "frame_count": 500, - "iqr": 0.0, - "median": 68.26139582134783, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "uniform 10 seek()+next()", - "fps_median": 1.232367350732705, - "fps_p25": 1.233843742370965, - "fps_p75": 1.2314023481824525, - "frame_count": 10, - "iqr": 0.016068589873611927, - "median": 8.114463592413813, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "random 10 seek()+next()", - "fps_median": 1.5437990877014915, - "fps_p25": 1.5438508087203877, - "fps_p75": 1.5435480997081767, - "frame_count": 10, - "iqr": 0.0012702811509370804, - "median": 6.477526823058724, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "100 next()", - "fps_median": 179.5141829898575, - "fps_p25": 182.56822073549603, - "fps_p75": 124.4290827477999, - "frame_count": 100, - "iqr": 0.25593018205836415, - "median": 0.5570590486750007, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 1.951097912335046, - "fps_p25": 1.951097912335046, - "fps_p75": 1.951097912335046, - "frame_count": 500, - "iqr": 0.0, - "median": 256.265970476903, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "uniform 10 seek()+next()", - "fps_median": 0.23515671574954122, - "fps_p25": 0.23515671574954122, - "fps_p75": 0.23515671574954122, - "frame_count": 10, - "iqr": 0.0, - "median": 42.524832719005644, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "random 10 seek()+next()", - "fps_median": 0.17733950193054088, - "fps_p25": 0.17733950193054088, - "fps_p75": 0.17733950193054088, - "frame_count": 10, - "iqr": 0.0, - "median": 56.38901593349874, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "100 next()", - "fps_median": 91.93830916027736, - "fps_p25": 93.48146430884644, - "fps_p75": 89.72122041281625, - "frame_count": 100, - "iqr": 0.04483274556696415, - "median": 1.087685872334987, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 0.23115768079016835, - "fps_p25": 0.23115768079016835, - "fps_p75": 0.23115768079016835, - "frame_count": 500, - "iqr": 0.0, - "median": 2163.0256813913584, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "uniform 10 seek()+next()", - "fps_median": 0.26119132784303123, - "fps_p25": 0.26119132784303123, - "fps_p75": 0.26119132784303123, - "frame_count": 10, - "iqr": 0.0, - "median": 38.2861103489995, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "random 10 seek()+next()", - "fps_median": 0.19600716639142407, - "fps_p25": 0.19600716639142407, - "fps_p75": 0.19600716639142407, - "frame_count": 10, - "iqr": 0.0, - "median": 51.01854276098311, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "100 next()", - "fps_median": 100.42696877778134, - "fps_p25": 101.82432674120878, - "fps_p75": 98.63909948929518, - "frame_count": 100, - "iqr": 0.0317131788469851, - "median": 0.995748464949429, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 78.89304868408422, - "fps_p25": 79.36504926194816, - "fps_p75": 78.80019881505761, - "frame_count": 500, - "iqr": 0.04515926633030176, - "median": 6.3376939836889505, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "uniform 10 seek()+next()", - "fps_median": 9.854385050842327, - "fps_p25": 10.094368141227985, - "fps_p75": 9.656371003428172, - "frame_count": 10, - "iqr": 0.04493432166054845, - "median": 1.014776665251702, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "random 10 seek()+next()", - "fps_median": 7.965955631552031, - "fps_p25": 8.052124079150976, - "fps_p75": 7.759671780042043, - "frame_count": 10, - "iqr": 0.04680596780963242, - "median": 1.2553421664051712, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec", - "description": "100 next()", - "fps_median": 250.74790033552227, - "fps_p25": 255.02885647654145, - "fps_p75": 248.55035150151298, - "frame_count": 100, - "iqr": 0.010220475262030959, - "median": 0.3988069286569953, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 14.558551600810196, - "fps_p25": 14.558551600810196, - "fps_p75": 14.558551600810196, - "frame_count": 500, - "iqr": 0.0, - "median": 34.34407581947744, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "uniform 10 seek()+next()", - "fps_median": 3.3993119043827935, - "fps_p25": 3.4033747004472383, - "fps_p75": 3.3871671818375297, - "frame_count": 10, - "iqr": 0.014059508219361305, - "median": 2.9417718295007944, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "random 10 seek()+next()", - "fps_median": 4.208422451807454, - "fps_p25": 4.228530445598264, - "fps_p75": 4.202620082236087, - "frame_count": 10, - "iqr": 0.014580214396119118, - "median": 2.376187304034829, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchcodec[cuda]", - "description": "100 next()", - "fps_median": 108.81798607716902, - "fps_p25": 109.2651801306662, - "fps_p75": 107.98428953245536, - "frame_count": 100, - "iqr": 0.010855996049940586, - "median": 0.9189657298848033, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 41.39450037405011, - "fps_p25": 41.84854039364475, - "fps_p75": 41.139148433617976, - "frame_count": 500, - "iqr": 0.20602534664794803, - "median": 12.078899261541665, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "uniform 10 seek()+next()", - "fps_median": 4.743936824381266, - "fps_p25": 4.806402420821976, - "fps_p75": 4.66718304034089, - "frame_count": 10, - "iqr": 0.06206185230985284, - "median": 2.1079538725316525, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "random 10 seek()+next()", - "fps_median": 2.9809550025126472, - "fps_p25": 3.038024440532653, - "fps_p75": 2.95329673866747, - "frame_count": 10, - "iqr": 0.09443371556699276, - "median": 3.354629637673497, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchvision[video_reader]", - "description": "100 next()", - "fps_median": 177.20109315629767, - "fps_p25": 182.07930273828418, - "fps_p75": 173.29419285866405, - "frame_count": 100, - "iqr": 0.027842139592394233, - "median": 0.5643306043930352, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "dataloader[threads=10,batch_size=50] 10 decode_and_transform()", - "fps_median": 9.23025646940316, - "fps_p25": 9.23025646940316, - "fps_p75": 9.23025646940316, - "frame_count": 500, - "iqr": 0.0, - "median": 54.1696757460013, - "type": "dataloader[threads=10,batch_size=50] 10 x decode_and_transform()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "uniform 10 seek()+next()", - "fps_median": 10.100628756440091, - "fps_p25": 10.212753705185772, - "fps_p75": 9.780241639161853, - "frame_count": 10, - "iqr": 0.0433017835021019, - "median": 0.9900373769924045, - "type": "10 x uniform seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "random 10 seek()+next()", - "fps_median": 6.6581950176290565, - "fps_p25": 6.769915708964532, - "fps_p75": 6.587147793106109, - "frame_count": 10, - "iqr": 0.04098446690477431, - "median": 1.5019085463136435, - "type": "10 x random seek()+next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - }, - { - "decoder": "torchaudio", - "description": "100 next()", - "fps_median": 263.36187377747655, - "fps_p25": 272.8191596128445, - "fps_p75": 254.92052325953637, - "frame_count": 100, - "iqr": 0.02573595428839326, - "median": 0.37970568239688873, - "type": "100 next()", - "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" - } - ], - "system_metadata": - { - "cpu_count": 22, - "machine": "x86_64", - "python_version": "3.12.5", - "system": "Linux", - "cuda": "NVIDIA PG509-210" - } + "experiments": [ + { + "decoder": "torchcodec", + "description": "concurrent[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 2.1743115488659117, + "fps_p25": 2.1743115488659117, + "fps_p75": 2.1743115488659117, + "frame_count": 500, + "iqr": 0.0, + "median": 229.95784585736692, + "type": "concurrent[threads=10 batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "uniform 10 seek()+next()", + "fps_median": 1.4197235571038909, + "fps_p25": 1.4299150228184128, + "fps_p75": 1.418538963182993, + "frame_count": 10, + "iqr": 0.056084178388118744, + "median": 7.043624760583043, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "random 10 seek()+next()", + "fps_median": 1.7206641646083398, + "fps_p25": 1.7417559798368405, + "fps_p75": 1.7134928198772545, + "frame_count": 10, + "iqr": 0.09470024821348488, + "median": 5.811709342058748, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "100 next()", + "fps_median": 110.86217475983852, + "fps_p25": 128.7595368347069, + "fps_p75": 108.57353226978415, + "frame_count": 100, + "iqr": 0.1443932841066271, + "median": 0.9020209121517837, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "concurrent[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 7.372989435697209, + "fps_p25": 7.372989435697209, + "fps_p75": 7.372989435697209, + "frame_count": 500, + "iqr": 0.0, + "median": 67.8150978460908, + "type": "concurrent[threads=10 batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "uniform 10 seek()+next()", + "fps_median": 1.2310264801210358, + "fps_p25": 1.2323052993790233, + "fps_p75": 1.2217488626929376, + "frame_count": 10, + "iqr": 0.07011599279940128, + "median": 8.123302107211202, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "random 10 seek()+next()", + "fps_median": 1.5450999514041033, + "fps_p25": 1.5466979702262287, + "fps_p75": 1.5413175350595594, + "frame_count": 10, + "iqr": 0.02256938721984625, + "median": 6.472073208540678, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "100 next()", + "fps_median": 124.30621009600625, + "fps_p25": 175.020342530923, + "fps_p75": 122.92145945236358, + "frame_count": 100, + "iqr": 0.2421654462814331, + "median": 0.8044650377705693, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "concurrent[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 1.962498268414507, + "fps_p25": 1.962498268414507, + "fps_p75": 1.962498268414507, + "frame_count": 500, + "iqr": 0.0, + "median": 254.7772948630154, + "type": "concurrent[threads=10 batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "uniform 10 seek()+next()", + "fps_median": 0.21549066350606524, + "fps_p25": 0.21549066350606524, + "fps_p75": 0.21549066350606524, + "frame_count": 10, + "iqr": 0.0, + "median": 46.40572281554341, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "random 10 seek()+next()", + "fps_median": 0.14958701949104536, + "fps_p25": 0.14958701949104536, + "fps_p75": 0.14958701949104536, + "frame_count": 10, + "iqr": 0.0, + "median": 66.85072029661387, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "100 next()", + "fps_median": 91.66328273271024, + "fps_p25": 92.88764930899312, + "fps_p75": 70.55172438906548, + "frame_count": 100, + "iqr": 0.3408304089680314, + "median": 1.0909493640065193, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "concurrent[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 0.23053130741893438, + "fps_p25": 0.23053130741893438, + "fps_p75": 0.23053130741893438, + "frame_count": 500, + "iqr": 0.0, + "median": 2168.9028080310673, + "type": "concurrent[threads=10 batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "uniform 10 seek()+next()", + "fps_median": 1.4976354877500007, + "fps_p25": 1.4994125488709173, + "fps_p75": 1.4899736083055237, + "frame_count": 10, + "iqr": 0.04224969074130058, + "median": 6.677192201837897, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "random 10 seek()+next()", + "fps_median": 1.1657519746939773, + "fps_p25": 1.171936101561095, + "fps_p75": 1.1544982783997169, + "frame_count": 10, + "iqr": 0.12888282490894198, + "median": 8.578154030255973, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "100 next()", + "fps_median": 246.353856352369, + "fps_p25": 249.17051134616946, + "fps_p75": 239.5810914306213, + "frame_count": 100, + "iqr": 0.016063610091805458, + "median": 0.4059201730415225, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/mandelbrot_1920x1080_120s_60fps_600gop_libx264_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "concurrent[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 77.9962515152657, + "fps_p25": 82.87312977603095, + "fps_p75": 77.73715330753502, + "frame_count": 500, + "iqr": 0.39861226361244917, + "median": 6.410564485937357, + "type": "concurrent[threads=10 batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "uniform 10 seek()+next()", + "fps_median": 31.48413804512292, + "fps_p25": 32.25486550195806, + "fps_p75": 30.361247196570773, + "frame_count": 10, + "iqr": 0.019336488097906113, + "median": 0.3176202564500272, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "random 10 seek()+next()", + "fps_median": 30.247269728247023, + "fps_p25": 31.424954708064178, + "fps_p75": 29.112848192603753, + "frame_count": 10, + "iqr": 0.025272516068071127, + "median": 0.3306083520874381, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec", + "description": "100 next()", + "fps_median": 616.6502704404603, + "fps_p25": 629.9302485611195, + "fps_p75": 604.7070901928084, + "frame_count": 100, + "iqr": 0.006621584761887789, + "median": 0.1621664739213884, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "concurrent[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 14.668088342064335, + "fps_p25": 14.668088342064335, + "fps_p75": 14.668088342064335, + "frame_count": 500, + "iqr": 0.0, + "median": 34.08760489709675, + "type": "concurrent[threads=10 batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "uniform 10 seek()+next()", + "fps_median": 3.3783752118675854, + "fps_p25": 3.3819478268521985, + "fps_p75": 3.366496472046222, + "frame_count": 10, + "iqr": 0.013571301475167274, + "median": 2.960002774372697, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "random 10 seek()+next()", + "fps_median": 4.2479605640225895, + "fps_p25": 4.255711626335866, + "fps_p75": 4.215874977198029, + "frame_count": 10, + "iqr": 0.02220357395708561, + "median": 2.354070818051696, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchcodec[cuda]", + "description": "100 next()", + "fps_median": 109.70223507344751, + "fps_p25": 110.20567249824958, + "fps_p75": 108.71415524502414, + "frame_count": 100, + "iqr": 0.012449108064174652, + "median": 0.9115584557875991, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "concurrent[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 46.15328022790921, + "fps_p25": 46.290790395468754, + "fps_p75": 46.06453008771282, + "frame_count": 500, + "iqr": 0.053053878247737885, + "median": 10.833466170355678, + "type": "concurrent[threads=10 batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "uniform 10 seek()+next()", + "fps_median": 4.999047713150926, + "fps_p25": 5.067322573561908, + "fps_p75": 4.931835999191345, + "frame_count": 10, + "iqr": 0.05421370454132557, + "median": 2.000380987301469, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "random 10 seek()+next()", + "fps_median": 3.14103808091942, + "fps_p25": 3.161504271012246, + "fps_p75": 3.121023210766595, + "frame_count": 10, + "iqr": 0.04102618247270584, + "median": 3.1836608606390655, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchvision[video_reader]", + "description": "100 next()", + "fps_median": 200.40285788502993, + "fps_p25": 202.4248314443549, + "fps_p75": 196.41718398457368, + "frame_count": 100, + "iqr": 0.015109885483980179, + "median": 0.4989948798902333, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "concurrent[threads=10,batch_size=50] 10 decode_and_transform()", + "fps_median": 10.024318990916512, + "fps_p25": 10.024318990916512, + "fps_p75": 10.024318990916512, + "frame_count": 500, + "iqr": 0.0, + "median": 49.87870003469288, + "type": "concurrent[threads=10 batch_size=50] 10 x decode_and_transform()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "uniform 10 seek()+next()", + "fps_median": 35.769066009258374, + "fps_p25": 36.774760546622694, + "fps_p75": 34.801270494753894, + "frame_count": 10, + "iqr": 0.015420199371874332, + "median": 0.2795711802318692, + "type": "10 x uniform seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "random 10 seek()+next()", + "fps_median": 28.915163916539186, + "fps_p25": 29.427930394196714, + "fps_p75": 28.119357363602, + "frame_count": 10, + "iqr": 0.01581367664039135, + "median": 0.3458392983302474, + "type": "10 x random seek()+next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + }, + { + "decoder": "torchaudio", + "description": "100 next()", + "fps_median": 815.6591709670496, + "fps_p25": 863.3320758405556, + "fps_p75": 792.7637904590958, + "frame_count": 100, + "iqr": 0.010310693643987179, + "median": 0.12260022759437561, + "type": "100 next()", + "video": "/tmp/torchcodec_benchmarking_videos/nasa_960x540_206s_30fps_yuv420p.mp4" + } + ], + "system_metadata": { + "cpu_count": 22, + "cuda": "NVIDIA PG509-210", + "machine": "x86_64", + "python_version": "3.12.5", + "system": "Linux" + } }