Skip to content

Commit

Permalink
docker-ce: code fix
Browse files Browse the repository at this point in the history
1. Fix retry logic. Setting DEFAULT_RETRIES is incorrect. See https://github.com/psf/requests/blob/881281250f74549f560408e5546d95a8cd73ce28/src/requests/adapters.py#L143
2. Adjust timeout value and set timeout for requests_download()
  • Loading branch information
taoky committed Sep 14, 2023
1 parent c151dc3 commit 75489ae
Showing 1 changed file with 13 additions and 6 deletions.
19 changes: 13 additions & 6 deletions docker-ce/tunasync/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
from email.utils import parsedate_to_datetime
import re
import traceback
import signal

import requests
from pyquery import PyQuery as pq
Expand All @@ -19,12 +18,20 @@
WORKING_DIR = os.getenv("TO")
WORKERS = os.getenv("SYNC_WORKERS", 1)

# connect and read timeout value
TIMEOUT_OPTION = (7, 10)
# (connect, read) timeout value
TIMEOUT_OPTION = (7, 30)
# user agent
requests.utils.default_user_agent = lambda: SYNC_USER_AGENT
# retries
requests.adapters.DEFAULT_RETRIES = 3
# retries, see https://stackoverflow.com/a/39050757/8460426
def set_retry(max_retries):
def decorate(func):
def wrapper(self, *args, **kwargs):
func(self, *args, **kwargs)
self.max_retries = max_retries
return wrapper
return decorate
func = requests.adapters.HTTPAdapter.__init__
requests.adapters.HTTPAdapter.__init__ = set_retry(3)(func)

REL_URL_RE = re.compile(r"https?:\/\/.+?\/(.+?)(\/index\.html)?$")

Expand Down Expand Up @@ -112,7 +119,7 @@ def files(self):

def requests_download(remote_url: str, dst_file: Path):
# NOTE the stream=True parameter below
with requests.get(remote_url, stream=True) as r:
with requests.get(remote_url, timeout=TIMEOUT_OPTION, stream=True) as r:
r.raise_for_status()
remote_ts = parsedate_to_datetime(
r.headers['last-modified']).timestamp()
Expand Down

0 comments on commit 75489ae

Please sign in to comment.