From 99c9d16d4c3f05924c8148d68a6e777356f44d26 Mon Sep 17 00:00:00 2001 From: Jules-A <1760158+Jules-A@users.noreply.github.com> Date: Sat, 30 Sep 2023 19:19:07 +0800 Subject: [PATCH] Add files via upload --- yt_dlp/downloader/fragment.py | 64 +++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py index b4b680dae119..c212ad25cbfd 100644 --- a/yt_dlp/downloader/fragment.py +++ b/yt_dlp/downloader/fragment.py @@ -12,8 +12,9 @@ from ..compat import compat_os_name from ..networking import Request from ..networking.exceptions import HTTPError, IncompleteRead -from ..utils import DownloadError, RetryManager, encodeFilename, traverse_obj +from ..utils import DownloadError, RetryManager, encodeFilename, TimeKeeper, traverse_obj, TsTracker from ..utils.networking import HTTPHeaderDict +from ..utils.progress import ProgressCalculator class HttpQuietDownloader(HttpFD): @@ -226,8 +227,7 @@ def _start_frag_download(self, ctx, info_dict): resume_len = ctx['complete_frags_downloaded_bytes'] total_frags = ctx['total_frags'] ctx_id = ctx.get('ctx_id') - # This dict stores the download progress, it's updated by the progress - # hook + # Stores the download progress, updated by the progress hook state = { 'status': 'downloading', 'downloaded_bytes': resume_len, @@ -237,14 +237,12 @@ def _start_frag_download(self, ctx, info_dict): 'tmpfilename': ctx['tmpfilename'], } - start = time.time() - ctx.update({ - 'started': start, - 'fragment_started': start, - # Amount of fragment's bytes downloaded by the time of the previous - # frag progress hook invocation - 'prev_frag_downloaded_bytes': 0, - }) + ctx['started'] = time.time() + progress = ProgressCalculator(resume_len) + update_rate = 0.1 + tracker = TsTracker(max(round(5 / update_rate), 1), initial_bytes=resume_len) + speed_timer = TimeKeeper() + progress_timer = TimeKeeper() def frag_progress_hook(s): if s['status'] not in ('downloading', 'finished'): @@ -259,38 +257,45 @@ def frag_progress_hook(s): state['max_progress'] = ctx.get('max_progress') state['progress_idx'] = ctx.get('progress_idx') - time_now = time.time() - state['elapsed'] = time_now - start + state['elapsed'] = progress.elapsed frag_total_bytes = s.get('total_bytes') or 0 s['fragment_info_dict'] = s.pop('info_dict', {}) + if not ctx['live']: estimated_size = ( (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) / (state['fragment_index'] + 1) * total_frags) - state['total_bytes_estimate'] = estimated_size + progress.total = estimated_size + progress.update(s['downloaded_bytes']) + state['total_bytes_estimate'] = progress.total + else: + progress.update(s['downloaded_bytes']) if s['status'] == 'finished': state['fragment_index'] += 1 ctx['fragment_index'] = state['fragment_index'] - state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes'] - ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes'] - ctx['speed'] = state['speed'] = self.calc_speed( - ctx['fragment_started'], time_now, frag_total_bytes) - ctx['fragment_started'] = time.time() - ctx['prev_frag_downloaded_bytes'] = 0 + state['downloaded_bytes'] = ctx['complete_frags_downloaded_bytes'] = progress.downloaded + #ctx['speed'] = state['speed'] = progress.speed else: - frag_downloaded_bytes = s['downloaded_bytes'] - state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes'] - ctx['speed'] = state['speed'] = self.calc_speed( - ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx.get('frag_resume_len', 0)) - if not ctx['live']: - state['eta'] = self.calc_eta(state['speed'], estimated_size - state['downloaded_bytes']) - ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes - self._hook_progress(state, info_dict) + state['downloaded_bytes'] = progress.downloaded + #ctx['speed'] = state['speed'] = progress.speed + #state['eta'] = progress.eta + + if speed_timer.elapsed(update_rate): + state['speed'] = tracker.update(state['downloaded_bytes']) + if state.get('speed') and state.get('total_bytes_estimate'): + remaining_est = state['total_bytes_estimate'] - state['downloaded_bytes'] + if remaining_est > 0 and state.get('speed') > 0: + state['eta'] = remaining_est / state['speed'] + else: + state['eta'] = None + + if progress_timer.elapsed(0.1) or state['status'] == 'finished': + self._hook_progress(state, info_dict) ctx['dl'].add_progress_hook(frag_progress_hook) - return start + return ctx['started'] def _finish_frag_download(self, ctx, info_dict): ctx['dest_stream'].close() @@ -500,7 +505,6 @@ def _download_fragment(fragment): download_fragment(fragment, ctx_copy) return fragment, fragment['frag_index'], ctx_copy.get('fragment_filename_sanitized') - self.report_warning('The download speed shown is only of one thread. This is a known issue') with tpe or concurrent.futures.ThreadPoolExecutor(max_workers) as pool: try: for fragment, frag_index, frag_filename in pool.map(_download_fragment, fragments):