Local file to S3 upload.

kghub_downloader/download_utils.py

@@ -1,25 +1,30 @@
-import os, pathlib, re
+import json
 import logging
+import os
+import pathlib
+import re
+from multiprocessing.sharedctypes import Value
+from typing import List, Optional
+from urllib.error import URLError
+from urllib.request import Request, urlopen
 
-import json, yaml
+import boto3
 import compress_json  # type: ignore
+import elasticsearch
+import elasticsearch.helpers
+import gdown
+import yaml
+from botocore.exceptions import NoCredentialsError
+from google.cloud import storage
+from google.cloud.storage.blob import Blob
+from tqdm.auto import tqdm  # type: ignore
 
 # from compress_json import compress_json
 
-from multiprocessing.sharedctypes import Value
 
-from urllib.error import URLError
-from urllib.request import Request, urlopen
 
 
-import elasticsearch
-import elasticsearch.helpers
 
-from tqdm.auto import tqdm  # type: ignore
-from google.cloud import storage
-from google.cloud.storage.blob import Blob
-from typing import List, Optional
-import gdown
 
 GDOWN_MAP = {"gdrive": "https://drive.google.com/uc?id="}
 
@@ -157,6 +162,8 @@ def download_from_yaml(
 
 
 def mirror_to_bucket(local_file, bucket_url, remote_file) -> None:
+    bucket_split = bucket_url.split("/")
+    bucket_name = bucket_split[2]
     with open(local_file, "rb"):
         if bucket_url.startswith("gs://"):
 
@@ -165,8 +172,6 @@ def mirror_to_bucket(local_file, bucket_url, remote_file) -> None:
 
             # Connect to GCS Bucket
             storage_client = storage.Client()
-            bucket_split = bucket_url.split("/")
-            bucket_name = bucket_split[2]
             bucket = storage_client.bucket(bucket_name)
 
             # Upload blob from local file
@@ -188,12 +193,27 @@ def mirror_to_bucket(local_file, bucket_url, remote_file) -> None:
             blob.upload_from_filename(local_file)
 
         elif bucket_url.startswith("s3://"):
-            raise ValueError("Currently, only Google Cloud storage is supported.")
-            # bashCommand = f"aws s3 cp {outfile} {mirror}"
-            # subprocess.run(bashCommand.split())
+            # Create an S3 client
+            s3 = boto3.client("s3")
+
+            try:
+                # Upload the file
+                # ! This will only work if the user has the AWS IAM user
+                # ! access keys set up as environment variables.
+                s3.upload_file(local_file, bucket_name, remote_file)
+                print(f"File {local_file} uploaded to {bucket_name}/{remote_file}")
+                return True
+            except FileNotFoundError:
+                print(f"The file {local_file} was not found")
+                return False
+            except NoCredentialsError:
+                print("Credentials not available")
+                return False
 
         else:
-            raise ValueError("Currently, only Google Cloud storage is supported.")
+            raise ValueError(
+                "Currently, only Google Cloud and S3 storage is supported."
+            )
 
     return None
 

kghub_downloader/main.py

@@ -1,6 +1,7 @@
-from typing import Optional, List
+from typing import List, Optional
 
 import typer
+
 from kghub_downloader.download_utils import download_from_yaml
 
 typer_app = typer.Typer()

pyproject.toml

@@ -12,17 +12,21 @@ python = "^3.8"
 elasticsearch = ">7.0, <9.0"
 compress-json = ">1.0, <2.0"
 PyYAML = ">5.0,<7.0"
-tqdm = "^4.62.3"
+tqdm = ">=4.62.3"
 google-cloud-storage = "^2.1.0"
 typer = "^0.7.0"
 mkdocs = "^1.4.0"
 mkdocs-material = "^8.5.6"
 mkdocstrings = {extras = ["python"], version = "^0.20.0"}
-gdown = "^4.7.1"
+gdown = ">=4.7.1"
+boto3 = ">=1.34.35"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.5"
 
+[tool.poetry.group.dev.dependencies]
+moto = "^5.0.1"
+
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"

test/integration/test_download.py

@@ -1,6 +1,7 @@
-from kghub_downloader.download_utils import download_from_yaml
-from os.path import exists
 import os
+from os.path import exists
+
+from kghub_downloader.download_utils import download_from_yaml
 
 
 # Integration test using example configuration
@@ -24,10 +25,7 @@ def test_download():
 
 
 def test_tag():
-    files = [
-        "test/output/zfin/fish_phenotype.txt",
-        "test/output/test_file.yaml"
-    ]
+    files = ["test/output/zfin/fish_phenotype.txt", "test/output/test_file.yaml"]
     tagged_files = ["test/output/gdrive_test_1.txt"]
 
     for file in files:

test/unit/test_mirror.py

@@ -1,7 +1,13 @@
-from kghub_downloader.download_utils import *
-
+import os
 from unittest import mock
 
+import boto3
+import moto
+import pytest
+
+from kghub_downloader.download_utils import mirror_to_bucket
+
+
 # proper test
 @mock.patch("google.cloud.storage.Client")
 def test_mirror(client):
@@ -16,3 +22,41 @@ def test_mirror(client):
 
     blob = bucket().blob
     blob.assert_called_with("kghub_test_upload.txt")
+
+
+@pytest.fixture(scope="session")
+def aws_credentials():
+    """Mocked AWS Credentials for moto."""
+    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
+    os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
+    os.environ["AWS_SECURITY_TOKEN"] = "testing"
+    os.environ["AWS_SESSION_TOKEN"] = "testing"
+    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
+
+
+@pytest.fixture(scope="function")
+def empty_bucket(aws_credentials):
+    moto_fake = moto.mock_aws()
+    try:
+        moto_fake.start()
+        conn = boto3.resource("s3")
+        conn.create_bucket(Bucket="monarch-test")  # or the name of the bucket you use
+        yield conn
+    finally:
+        moto_fake.stop()
+
+
+def test_mirror_to_bucket_s3(empty_bucket):
+    # Call the function under test
+    result = mirror_to_bucket(
+        local_file="test/resources/testfile.txt",
+        bucket_url="s3://monarch-test/",
+        remote_file="kghub_test_upload.txt",
+    )
+
+    # Check if the file was created in the bucket
+    bucket = empty_bucket.Bucket("monarch-test")
+    files_in_bucket = list(bucket.objects.all())
+    assert len(files_in_bucket) == 1
+    assert files_in_bucket[0].key == "kghub_test_upload.txt"
+    assert result is True

test/unit/test_parse_url.py

@@ -1,6 +1,8 @@
-from kghub_downloader.download_utils import *
+import os
+
+import pytest
 
-import os, pytest
+from kghub_downloader.download_utils import *
 
 os.environ["TEST_URL"] = "https://testurl.com/fake/download/rootpath"
 os.environ["TEST_KEY"] = "123456789987654321"