Skip to content

Commit

Permalink
Switched file part uploading to pycurl_requests (#32)
Browse files Browse the repository at this point in the history
Switched file part uploading to pycurl_requests due to bugs with certain S3 implementations.
Download might be done in a later PR
  • Loading branch information
MoritzHahn1337 authored Aug 4, 2022
1 parent 0483519 commit 230f21d
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 16 deletions.
2 changes: 1 addition & 1 deletion ghga_connector/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@
CLI - Client to perform up- and download operations to and from a local ghga instance
"""

__version__ = "0.1.3"
__version__ = "0.1.4"
18 changes: 3 additions & 15 deletions ghga_connector/core/file_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from typing import Iterator, Sequence

import pycurl
import pycurl_requests as requests

from ghga_connector.core import exceptions
from ghga_connector.core.retry import WithRetry
Expand Down Expand Up @@ -138,21 +139,8 @@ def read_file_parts(
def upload_file_part(*, presigned_url: str, part: bytes) -> None:
"""Upload File"""

body = BytesIO(part)

curl = pycurl.Curl()
curl.setopt(curl.URL, presigned_url)

curl.setopt(curl.UPLOAD, 1)
curl.setopt(curl.READDATA, body)

try:
curl.perform()
status_code = curl.getinfo(pycurl.RESPONSE_CODE)
except pycurl.error as pycurl_error:
raise exceptions.RequestFailedError(url=presigned_url) from pycurl_error
finally:
curl.close()
response = requests.put(presigned_url, data=part)
status_code = response.status_code

if status_code == 200:
return
Expand Down
1 change: 1 addition & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ install_requires =
httpyexpect==0.2.2
typer==0.4.1
pycurl==7.44.1
pycurl_requests==0.2.1

python_requires = >= 3.9.10

Expand Down

0 comments on commit 230f21d

Please sign in to comment.