diff --git a/source/extract_srt_subtitles_to_files/lib/ffmpeg/README.md b/source/extract_srt_subtitles_to_files/lib/ffmpeg/README.md index f1209743e..894d8afa8 100644 --- a/source/extract_srt_subtitles_to_files/lib/ffmpeg/README.md +++ b/source/extract_srt_subtitles_to_files/lib/ffmpeg/README.md @@ -3,14 +3,427 @@ This python module is a helper library for any Unmanic plugin that needs to build FFmpeg commands to be executed. -## Using the module +# Using the module -### Adding it to your project -It should be included in your plugin project as a submodule. +## Adding it to your project +```bash +└── my_plugin_id/ + ├── changelog.md + ├── description.md + ├── .gitignore + ├── icon.png + ├── info.json + ├── lib/ + | └── ffmpeg/ + | ├── __init__.py + | ├── LICENSE + | ├── mimetype_overrides.py + | ├── parser.py + | ├── probe.py + | ├── README.md + | └── stream_mapper.py + ├── LICENSE + ├── plugin.py + └── requirements.txt +``` + +### Git Submodule +It can be included in your plugin project as a submodule. ``` git submodule add https://github.com/Josh5/unmanic.plugin.helpers.ffmpeg.git ./lib/ffmpeg ``` +If you use it sure to include all files in the lib directory when publishing your project to the Unmanic plugin repository. + +### Project source download +Download the git repository as zip file and extract it to `lib` directory. +``` +mkdir -p ./lib +curl -L "https://github.com/Josh5/unmanic.plugin.helpers.ffmpeg/archive/refs/heads/master.zip" --output /tmp/unmanic.plugin.helpers.ffmpeg.zip +unzip /tmp/unmanic.plugin.helpers.ffmpeg.zip -d ./lib/ +mv -v ./lib/unmanic.plugin.helpers.ffmpeg-master ./lib/ffmpeg +``` + +--- + +## Importing it in your project + +This module comes with x3 classes to assist in generating FFmpeg commands for your Unmanic plugin. + +You can import all 3 classes into your plugin like this: + +```python +from my_plugin_id.lib.ffmpeg import Parser, Probe, StreamMapper +``` +> **Note** +> Be sure to rename 'my_plugin_id' in the example above. + +--- + +## Using the `Probe` class + +The Probe class is a wrapper around the `ffprobe` cli. This can be used to generate a file probe object containing file format and stream info. + +Add this to your plugin runner function: +```python + # Get file probe + probe = Probe.init_probe(data, logger, allowed_mimetypes=['video', 'audio']) + if not probe: + # File not able to be probed by ffprobe. The file is probably not a audio/video file. + return +``` + +You can then use this newly created Probe object in your plugin. To read the FFprobe data, add this: +```python + ffprobe_data = probe.get_probe() +``` + +### FFprobe Example +
+ Show + + ```json +{ + "streams": [ + { + "index": 0, + "codec_name": "hevc", + "codec_long_name": "H.265 / HEVC (High Efficiency Video Coding)", + "profile": "Main", + "codec_type": "video", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "width": 1920, + "height": 1080, + "coded_width": 1920, + "coded_height": 1080, + "closed_captions": 0, + "film_grain": 0, + "has_b_frames": 2, + "sample_aspect_ratio": "1:1", + "display_aspect_ratio": "16:9", + "pix_fmt": "yuv420p", + "level": 120, + "color_range": "tv", + "color_space": "bt709", + "color_transfer": "bt709", + "color_primaries": "bt709", + "chroma_location": "left", + "refs": 1, + "r_frame_rate": "24000/1001", + "avg_frame_rate": "24000/1001", + "time_base": "1/1000", + "start_pts": 21, + "start_time": "0.021000", + "extradata_size": 2471, + "disposition": { + "default": 1, + "dub": 0, + "original": 0, + "comment": 0, + "lyrics": 0, + "karaoke": 0, + "forced": 0, + "hearing_impaired": 0, + "visual_impaired": 0, + "clean_effects": 0, + "attached_pic": 0, + "timed_thumbnails": 0, + "captions": 0, + "descriptions": 0, + "metadata": 0, + "dependent": 0, + "still_image": 0 + }, + "tags": { + "DURATION": "00:00:10.239000000" + } + }, + { + "index": 1, + "codec_name": "aac", + "codec_long_name": "AAC (Advanced Audio Coding)", + "profile": "LC", + "codec_type": "audio", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "sample_fmt": "fltp", + "sample_rate": "48000", + "channels": 6, + "channel_layout": "5.1", + "bits_per_sample": 0, + "r_frame_rate": "0/0", + "avg_frame_rate": "0/0", + "time_base": "1/1000", + "start_pts": 0, + "start_time": "0.000000", + "extradata_size": 5, + "disposition": { + "default": 1, + "dub": 0, + "original": 0, + "comment": 0, + "lyrics": 0, + "karaoke": 0, + "forced": 0, + "hearing_impaired": 0, + "visual_impaired": 0, + "clean_effects": 0, + "attached_pic": 0, + "timed_thumbnails": 0, + "captions": 0, + "descriptions": 0, + "metadata": 0, + "dependent": 0, + "still_image": 0 + }, + "tags": { + "language": "eng", + "title": "Surround", + "DURATION": "00:00:10.005000000" + } + }, + { + "index": 2, + "codec_name": "ass", + "codec_long_name": "ASS (Advanced SSA) subtitle", + "codec_type": "subtitle", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "r_frame_rate": "0/0", + "avg_frame_rate": "0/0", + "time_base": "1/1000", + "start_pts": 0, + "start_time": "0.000000", + "duration_ts": 10614, + "duration": "10.614000", + "extradata_size": 487, + "disposition": { + "default": 0, + "dub": 0, + "original": 0, + "comment": 0, + "lyrics": 0, + "karaoke": 0, + "forced": 0, + "hearing_impaired": 0, + "visual_impaired": 0, + "clean_effects": 0, + "attached_pic": 0, + "timed_thumbnails": 0, + "captions": 0, + "descriptions": 0, + "metadata": 0, + "dependent": 0, + "still_image": 0 + }, + "tags": { + "language": "bul", + "DURATION": "00:00:10.614000000" + } + } + ], + "chapters": [ + { + "id": 1, + "time_base": "1/1000000000", + "start": 0, + "start_time": "0.000000", + "end": 10000000000, + "end_time": "10.000000", + "tags": { + "title": "Chapter 1" + } + } + ], + "format": { + "filename": "TEST_FILE.mkv", + "nb_streams": 3, + "nb_programs": 0, + "format_name": "matroska,webm", + "format_long_name": "Matroska / WebM", + "start_time": "0.000000", + "duration": "10.614000", + "size": "1280059", + "bit_rate": "964807", + "probe_score": 100, + "tags": { + "ENCODER": "Lavf59.27.100" + } + } +} + ``` +
+ +--- + +## Using the `StreamMapper` class + +The StreamMapper class is used to simplify building a ffmpeg command. It uses a previously initialised probe object as an input and uses it to define stream mapping from the input file to the output. + +This class should be extended with a child class to configure it and implement the custom functions required to manage streams that will need to be processed. + +```python +class PluginStreamMapper(StreamMapper): + def __init__(self): + super(PluginStreamMapper, self).__init__(logger, ['video']) + self.settings = None + + def set_settings(self, settings): + self.settings = settings + + def test_stream_needs_processing(self, stream_info: dict): + """ + Run through a set of test against the given stream_info. + + Return 'True' if it needs to be process. + Return 'False' if it should just be copied over to the new file. + + :param stream_info: + :return: bool + """ + if stream_info.get('codec_name').lower() in ['h264']: + return False + return True + + def custom_stream_mapping(self, stream_info: dict, stream_id: int): + """ + Will be provided with stream_info and the stream_id of a stream that has been + determined to need processing by the `test_stream_needs_processing` function. + + Use this function to `-map` (select) an input stream to be included in the output file + and apply a `-c` (codec) selection and encoder arguments to the command. + + This function must return a dictionary containing 2 key values: + { + 'stream_mapping': [], + 'stream_encoding': [], + } + + Where: + - 'stream_mapping' is a list of arguments for input streams to map. Eg. ['-map', '0:v:1'] + - 'stream_encoding' is a list of encoder arguments. Eg. ['-c:v:1', 'libx264', '-preset', 'slow'] + + + :param stream_info: + :param stream_id: + :return: dict + """ + if self.settings.get_setting('advanced'): + stream_encoding = ['-c:v:{}'.format(stream_id), 'libx264'] + stream_encoding += self.settings.get_setting('custom_options').split() + else: + stream_encoding = [ + '-c:v:{}'.format(stream_id), 'libx264', + '-preset', str(self.settings.get_setting('preset')), + '-crf', str(self.settings.get_setting('crf')), + ] + + return { + 'stream_mapping': ['-map', '0:v:{}'.format(stream_id)], + 'stream_encoding': stream_encoding, + } +``` + +Once you have created your stream mapper class, you can use it to determine if a file needs a FFmpeg command executed against it using its `streams_need_processing` function. + +```python +def on_library_management_file_test(data): + + ... + + # Get plugin settings + settings = Settings(library_id=data.get('library_id')) + + # Get file probe + probe = Probe.init_probe(data, logger, allowed_mimetypes=['audio']) + if not probe: + # File not able to be probed by FFprobe. The file is probably not a audio/video file. + return + + # Get stream mapper + mapper = PluginStreamMapper() + mapper.set_settings(settings) + mapper.set_probe(probe) + + # Check if file needs a FFmpeg command run against it + if mapper.streams_need_processing(): + # Mark this file to be added to the pending tasks + data['add_file_to_pending_tasks'] = True + + +def on_worker_process(data): + + ... + + # Get plugin settings + settings = Settings(library_id=data.get('library_id')) + + # Get file probe + probe = Probe.init_probe(data, logger, allowed_mimetypes=['audio']) + if not probe: + # File not able to be probed by FFprobe. The file is probably not a audio/video file. + return + + # Get stream mapper + mapper = PluginStreamMapper() + mapper.set_settings(settings) + mapper.set_probe(probe) + + # Check if file needs a FFmpeg command run against it + if mapper.streams_need_processing(): + + """ + HERE: Configure FFmpeg command args as required for this plugin + """ + + # Set the input and output file + mapper.set_input_file(data.get('file_in')) + mapper.set_output_file(data.get('file_out')) + + # Get final generated FFmpeg args + ffmpeg_args = mapper.get_ffmpeg_args() + + # Apply FFmpeg args to command for Unmanic to execute + data['exec_command'] = ['ffmpeg'] + data['exec_command'] += ffmpeg_args + +``` + +--- + +## Using the `Parser` class + +Unmanic has the ability to execute a command provided by a plugin and display a output of that command's progress. As Unmanic is able to execute any command a plugin provides it, we need a way + +This progress is only possible if the provided command is accompanied with a progress parser function. If such a function is not provided to Unmanic, then the command will still be executed, but the Unmanic worker will only report an indeterminate progress status with the logs. + +This python module provides a function for parsing the output of a FFmpeg command to determine progress of that command's execution. + +This should be returned with the built command in the `on_worker_process` plugin function: + + +```python +def on_worker_process(data): + + ... + + # Get file probe + probe = Probe.init_probe(data, logger, allowed_mimetypes=['audio']) + if not probe: + # File not able to be probed by FFprobe. The file is probably not a audio/video file. + return + + # Set the parser + parser = Parser(logger) + parser.set_probe(probe) + data['command_progress_parser'] = parser.parse_progress +``` + +--- -### Calling it in your project -For an example of how to use this module, see the [libx264 encoder plugin](https://github.com/Josh5/unmanic.plugin.encoder_video_h264_libx264). +## Examples +For examples of how to use this module, see these plugin sources: +- [Limit Library Search by FFprobe Data](https://github.com/Unmanic/plugin.limit_library_search_by_ffprobe_data/blob/master/plugin.py) +- [Re-order audio streams by language](https://github.com/Unmanic/plugin.reorder_audio_streams_by_language/blob/master/plugin.py) +- [Transcode Video Files](https://github.com/Unmanic/plugin.video_transcoder/blob/master/plugin.py) diff --git a/source/extract_srt_subtitles_to_files/lib/ffmpeg/parser.py b/source/extract_srt_subtitles_to_files/lib/ffmpeg/parser.py index ba46cb813..d7394070e 100644 --- a/source/extract_srt_subtitles_to_files/lib/ffmpeg/parser.py +++ b/source/extract_srt_subtitles_to_files/lib/ffmpeg/parser.py @@ -97,7 +97,7 @@ def parse_progress(self, line_text): :return: """ # Fetch data from line text - if line_text and 'frame=' in line_text: + if line_text and ('frame=' in line_text or 'time=' in line_text): # Update time _time = self.get_progress_from_regex_of_string(line_text, r"time=(\s+|)(\d+:\d+:\d+\.\d+)", self.time) if _time: diff --git a/source/extract_srt_subtitles_to_files/lib/ffmpeg/probe.py b/source/extract_srt_subtitles_to_files/lib/ffmpeg/probe.py index c4790a7e4..9f3ff51a0 100644 --- a/source/extract_srt_subtitles_to_files/lib/ffmpeg/probe.py +++ b/source/extract_srt_subtitles_to_files/lib/ffmpeg/probe.py @@ -60,7 +60,13 @@ def ffprobe_cmd(params): raw_output = out.decode("utf-8") except Exception as e: raise FFProbeError(command, str(e)) - if pipe.returncode == 1 or 'error' in raw_output: + + if 'error' in raw_output: + try: + info = json.loads(raw_output) + except Exception as e: + raise FFProbeError(command, raw_output) + if pipe.returncode == 1: raise FFProbeError(command, raw_output) if not raw_output: raise FFProbeError(command, 'No info found') @@ -84,6 +90,7 @@ def ffprobe_file(vid_file_path): "-show_format", "-show_streams", "-show_error", + "-show_chapters", vid_file_path ] @@ -149,6 +156,37 @@ class variable, it will fail this test. return True + @staticmethod + def init_probe(data, logger, allowed_mimetypes=None): + """ + Fetch the Probe object given a plugin's data object + + :param data: + :param logger: + :param allowed_mimetypes: + :return: + """ + probe = Probe(logger, allowed_mimetypes=allowed_mimetypes) + # Start by fetching probe data from 'shared_info'. + ffprobe_data = data.get('shared_info', {}).get('ffprobe') + if ffprobe_data: + if not probe.set_probe(ffprobe_data): + # Failed to set ffprobe from 'shared_info'. + # Probably due to it being for an incompatible mimetype declared above. + return + return probe + # No 'shared_info' ffprobe exists. Attempt to probe file. + if not probe.file(data.get('path')): + # File probe failed, skip the rest of this test. + # Again, probably due to it being for an incompatible mimetype. + return + # Successfully probed file. + # Set file probe to 'shared_info' for subsequent file test runners. + if 'shared_info' not in data: + data['shared_info'] = {} + data['shared_info']['ffprobe'] = probe.get_probe() + return probe + def file(self, file_path): """ Sets the 'probe' dict by probing the given file path. @@ -180,6 +218,7 @@ def set_probe(self, probe_info): """Sets the probe dictionary""" file_path = probe_info.get('format', {}).get('filename') if not file_path: + self.logger.error("Provided file probe information does not contain the expected 'filename' key.") return if not self.__test_valid_mimetype(file_path): return diff --git a/source/extract_srt_subtitles_to_files/lib/ffmpeg/stream_mapper.py b/source/extract_srt_subtitles_to_files/lib/ffmpeg/stream_mapper.py index e8fb8f8e6..a4182caac 100644 --- a/source/extract_srt_subtitles_to_files/lib/ffmpeg/stream_mapper.py +++ b/source/extract_srt_subtitles_to_files/lib/ffmpeg/stream_mapper.py @@ -37,6 +37,14 @@ class StreamMapper(object): probe: Probe = None + stream_type_idents = { + 'video': 'v', + 'audio': 'a', + 'subtitle': 's', + 'data': 'd', + 'attachment': 't' + } + processing_stream_type = '' found_streams_to_encode = False stream_mapping = [] @@ -129,7 +137,7 @@ def test_stream_needs_processing(self, stream_info: dict): """ Overwrite this function to test a stream. Return 'True' if it needs to be process. - Return 'False' if it should just be copied over to the new file + Return 'False' if it should just be copied over to the new file. :param stream_info: :return: bool @@ -194,10 +202,12 @@ def __set_stream_mapping(self): self.video_stream_count += 1 continue else: - found_streams_to_process = True - self.__apply_custom_stream_mapping( - self.custom_stream_mapping(stream_info, self.video_stream_count) - ) + mapping = self.custom_stream_mapping(stream_info, self.video_stream_count) + if mapping: + found_streams_to_process = True + self.__apply_custom_stream_mapping(mapping) + else: + self.__copy_stream_mapping('v', self.video_stream_count) self.video_stream_count += 1 continue else: @@ -214,15 +224,28 @@ def __set_stream_mapping(self): self.audio_stream_count += 1 continue else: - found_streams_to_process = True - self.__apply_custom_stream_mapping( - self.custom_stream_mapping(stream_info, self.audio_stream_count) - ) + mapping = self.custom_stream_mapping(stream_info, self.audio_stream_count) + if mapping: + found_streams_to_process = True + self.__apply_custom_stream_mapping(mapping) + else: + self.__copy_stream_mapping('a', self.audio_stream_count) self.audio_stream_count += 1 continue else: - self.__copy_stream_mapping('a', self.audio_stream_count) - self.audio_stream_count += 1 + if self.settings.get_setting('mode') == 'advanced': + amaps = self.settings.get_setting('custom_options').split() + self.logger.debug("Advanced Mode Video Settings with custom audio encoding: '%s'", amaps) + if '-c:a' not in amaps: + self.logger.debug("-c:a not detected in custom mappings: '%s'", amaps) + self.__copy_stream_mapping('a', self.audio_stream_count) + else: + self.logger.debug("-c:a detected in custom mappings: '%s'", amaps) + self.stream_mapping += ['-map', '0:{}:{}'.format('a', self.audio_stream_count)] + self.audio_stream_count += 1 + else: + self.__copy_stream_mapping('a', self.audio_stream_count) + self.audio_stream_count += 1 continue # If this is a subtitle stream? @@ -234,15 +257,28 @@ def __set_stream_mapping(self): self.subtitle_stream_count += 1 continue else: - found_streams_to_process = True - self.__apply_custom_stream_mapping( - self.custom_stream_mapping(stream_info, self.subtitle_stream_count) - ) + mapping = self.custom_stream_mapping(stream_info, self.subtitle_stream_count) + if mapping: + found_streams_to_process = True + self.__apply_custom_stream_mapping(mapping) + else: + self.__copy_stream_mapping('s', self.subtitle_stream_count) self.subtitle_stream_count += 1 continue else: - self.__copy_stream_mapping('s', self.subtitle_stream_count) - self.subtitle_stream_count += 1 + if self.settings.get_setting('mode') == 'advanced': + submaps = self.settings.get_setting('custom_options').split() + self.logger.debug("Advanced Mode Video Settings with custom subtitle encoding: '%s'", submaps) + if '-c:s' not in submaps: + self.logger.debug("-c:s not detected in custom mappings: '%s'", submaps) + self.__copy_stream_mapping('s', self.subtitle_stream_count) + else: + self.logger.debug("-c:s detected in custom mappings: '%s'", submaps) + self.stream_mapping += ['-map', '0:{}:{}'.format('s', self.subtitle_stream_count)] + self.subtitle_stream_count += 1 + else: + self.__copy_stream_mapping('s', self.subtitle_stream_count) + self.subtitle_stream_count += 1 continue # If this is a data stream? @@ -254,10 +290,12 @@ def __set_stream_mapping(self): self.data_stream_count += 1 continue else: - found_streams_to_process = True - self.__apply_custom_stream_mapping( - self.custom_stream_mapping(stream_info, self.data_stream_count) - ) + mapping = self.custom_stream_mapping(stream_info, self.data_stream_count) + if mapping: + found_streams_to_process = True + self.__apply_custom_stream_mapping(mapping) + else: + self.__copy_stream_mapping('d', self.data_stream_count) self.data_stream_count += 1 continue else: @@ -274,10 +312,12 @@ def __set_stream_mapping(self): self.attachment_stream_count += 1 continue else: - found_streams_to_process = True - self.__apply_custom_stream_mapping( - self.custom_stream_mapping(stream_info, self.attachment_stream_count) - ) + mapping = self.custom_stream_mapping(stream_info, self.attachment_stream_count) + if mapping: + found_streams_to_process = True + self.__apply_custom_stream_mapping(mapping) + else: + self.__copy_stream_mapping('t', self.attachment_stream_count) self.attachment_stream_count += 1 continue else: @@ -349,6 +389,9 @@ def set_output_file(self, path): def set_output_null(self): """Set the output container to NULL for the FFmpeg args""" self.output_file = '-' + if os.name == "nt": + # Windows uses NUL instead + self.output_file = 'NUL' main_options = { "-f": 'null', } @@ -434,15 +477,15 @@ def get_ffmpeg_args(self): # Add generic options first args += self.generic_options + # Add other main options + args += self.main_options + # Add the input file # This class requires at least one input file specified with the input_file attribute if not self.input_file: raise Exception("Input file has not been set") args += ['-i', self.input_file] - # Add other main options - args += self.main_options - # Add advanced options. This includes the stream mapping and the encoding args args += self.advanced_options args += self.stream_mapping diff --git a/source/extract_srt_subtitles_to_files/lib/ffmpeg/tools.py b/source/extract_srt_subtitles_to_files/lib/ffmpeg/tools.py new file mode 100644 index 000000000..c4d9c5475 --- /dev/null +++ b/source/extract_srt_subtitles_to_files/lib/ffmpeg/tools.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" + plugins.tools.py + + Written by: Josh.5 + Date: 17 Feb 2023, (12:07 PM) + + Copyright: + Copyright (C) 2021 Josh Sunnex + + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General + Public License as published by the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License along with this program. + If not, see . + +""" + +image_video_codecs = [ + 'alias_pix', + 'apng', + 'brender_pix', + 'dds', + 'dpx', + 'exr', + 'fits', + 'gif', + 'mjpeg', + 'mjpegb', + 'pam', + 'pbm', + 'pcx', + 'pfm', + 'pgm', + 'pgmyuv', + 'pgx', + 'photocd', + 'pictor', + 'pixlet', + 'png', + 'ppm', + 'ptx', + 'sgi', + 'sunrast', + 'tiff', + 'vc1image', + 'wmv3image', + 'xbm', + 'xface', + 'xpm', + 'xwd', +] + +resolution_map = { + '480p_sdtv': { + 'width': 854, + 'height': 480, + 'label': "480p (SDTV)", + }, + '576p_sdtv': { + 'width': 1024, + 'height': 576, + 'label': "576p (SDTV)", + }, + '720p_hdtv': { + 'width': 1280, + 'height': 720, + 'label': "720p (HDTV)", + }, + '1080p_hdtv': { + 'width': 1920, + 'height': 1080, + 'label': "1080p (HDTV)", + }, + 'dci_2k_hdtv': { + 'width': 2048, + 'height': 1080, + 'label': "DCI 2K (HDTV)", + }, + '1440p': { + 'width': 2560, + 'height': 1440, + 'label': "1440p (WQHD)", + }, + '4k_uhd': { + 'width': 3840, + 'height': 2160, + 'label': "4K (UHD)", + }, + 'dci_4k': { + 'width': 4096, + 'height': 2160, + 'label': "DCI 4K", + }, + '8k_uhd': { + 'width': 8192, + 'height': 4608, + 'label': "8k (UHD)", + }, +} + + +def get_video_stream_resolution(streams: list) -> object: + """ + Given a list of streams from a video file, returns the first video + stream's resolution and index. + + :param streams: The list of streams for the video file. + :type streams: list + :return: A tuple of the (width, height, stream_index,) + :rtype: object + """ + width = 0 + height = 0 + video_stream_index = 0 + + for stream in streams: + if stream.get('codec_type', '') == 'video': + width = stream.get('width', stream.get('coded_width', 0)) + height = stream.get('height', stream.get('coded_height', 0)) + video_stream_index = stream.get('index') + break + + return width, height, video_stream_index