Move content of library from resource registration repo

README.md

@@ -1,2 +1,69 @@
-# registration-library
-Python library with common functions for resource registration
+# EOEPCA+ Resource Registration Common Library
+
+<!-- PROJECT SHIELDS -->
+<!--
+*** See the bottom of this document for the declaration of the reference variables
+*** for contributors-url, forks-url, etc. This is an optional, concise syntax you may use.
+*** https://www.markdownguide.org/basic-syntax/#reference-style-links
+-->
+
+[![MIT License][license-shield]][license-url]
+
+<!-- PROJECT LOGO -->
+<br />
+<p align="center">
+  <a href="https://github.com/EOEPCA">
+    <img src="https://raw.githubusercontent.com/EOEPCA/resource-registration/main/docs/img/eoepca-logo.png" alt="Logo" width="80" height="80">
+  </a>
+
+  <h3 align="center">EOEPCA+ Resource Registration Common Library</h3>
+
+  <p align="center">
+    Python library with common functions for resource registration
+    <br />
+    <a href="https://eoepca.readthedocs.io/projects/resource-registration/en/latest/design/common-library/design/"><strong>Explore the docs »</strong></a>
+    <br />
+  </p>
+</p>
+
+<!-- TABLE OF CONTENTS -->
+
+<!-- ABOUT THE PROJECT -->
+
+## Description
+
+## Getting Started
+
+## Documentation
+
+The Registration Library documentation can be found [here](https://eoepca.readthedocs.io/projects/resource-registration/en/latest/design/common-library/design/).
+
+<!-- LICENSE -->
+
+## License
+
+The EOEPCA building blocks are distributed under the Apache-2.0 License. See `LICENSE` for more information.
+
+<!-- CONTACT -->
+
+## Contact
+
+Project Link: [https://github.com/EOEPCA/registration-library](https://github.com/EOEPCA/registration-library)
+
+<!-- ACKNOWLEDGEMENTS -->
+
+- README.md is based on [this template](https://github.com/othneildrew/Best-README-Template) by [Othneil Drew](https://github.com/othneildrew).
+
+<!-- MARKDOWN LINKS & IMAGES -->
+<!-- https://www.markdownguide.org/basic-syntax/#reference-style-links -->
+
+[contributors-shield]: https://img.shields.io/github/contributors/EOEPCA/registration-library.svg?style=flat-square
+[contributors-url]: https://github.com/EOEPCA/registration-library/graphs/contributors
+[forks-shield]: https://img.shields.io/github/forks/EOEPCA/registration-library.svg?style=flat-square
+[forks-url]: https://github.com/EOEPCA/registration-library/network/members
+[stars-shield]: https://img.shields.io/github/stars/EOEPCA/registration-library.svg?style=flat-square
+[stars-url]: https://github.com/EOEPCA/registration-library/stargazers
+[issues-shield]: https://img.shields.io/github/issues/EOEPCA/registration-library.svg?style=flat-square
+[issues-url]: https://github.com/EOEPCA/registration-library/issues
+[license-shield]: https://img.shields.io/github/license/EOEPCA/registration-library.svg?style=flat-square
+[license-url]: https://github.com/EOEPCA/registration-library/blob/master/LICENSE

pyproject.toml

@@ -0,0 +1,64 @@
+[project]
+name = "registration-library"
+version = "0.0.1"
+authors = [
+    {name = "Mario Winkler", email="[email protected]"},
+    {name = "Jonas Eberle", email="[email protected]"}    
+]
+description = "Common Library of the Resource Registration Building Block to harvest and register resources"
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "pystac",
+    "pystac_client",
+    "stactools_sentinel2",
+    "rio-cogeo",
+    "requests",
+    "psycopg2-binary",
+    "pika"
+]
+license = {file = "LICENSE"}
+
+[project.optional-dependencies]
+dev = [
+    "black",
+    "flake8"
+]
+test = [
+    "pytest"
+]
+docs = [
+    "mkdocs"
+]
+
+[project.urls]
+Homepage = "https://github.com/EOEPCA/registration-library"
+Issues = "https://github.com/EOEPCA/registration-library/issues"
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = "-q --import-mode=importlib --no-header -rfE --disable-warnings --log-level=INFO"
+testpaths = [
+    "tests",
+    "integration",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/registration-library"]
+
+
+[tool.black]
+line-length = 120
+target-version = ["py38"]

src/registration-library/base/download.py

@@ -0,0 +1,122 @@
+import os
+import time
+import netrc
+import requests
+from datetime import datetime
+
+
+def access_token():
+    """
+    Description...
+
+    Returns:
+        (str): ...
+    """
+    if "token_expire_time" in os.environ and time.time() <= (float(os.environ["token_expire_time"]) - 5):
+        return os.environ["s3_access_key"]
+
+    print("Need to get a new access token")
+    # todo: added try block & Exception raise for testing
+    try:
+        auth = netrc.netrc().authenticators("dataspace.copernicus.eu")
+        username = auth[0]
+        password = auth[2]
+    except Exception as e:
+        raise Exception("Failed to get credentials from netrc: %s" % e)
+    auth_server_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token"
+    data = {
+        "client_id": "cdse-public",
+        "grant_type": "password",
+        "username": username,
+        "password": password,
+    }
+
+    # response = requests.post(auth_server_url, data=data, verify=True, allow_redirects=False).json()
+    # todo: added try block for testing
+    try:
+        response = requests.post(auth_server_url, data=data, verify=True, allow_redirects=False)
+        response.raise_for_status()
+        response_json = response.json()
+    except requests.exceptions.RequestException as e:
+        raise Exception("Failed to get access token: %s" % e)
+
+    token_time = time.time()
+    os.environ["token_expire_time"] = str(token_time + response_json["expires_in"])
+    print(
+        "New expiration tme for access token: %s"
+        % datetime.fromtimestamp(float(os.environ["token_expire_time"])).strftime("%m/%d/%Y, %H:%M:%S")
+    )
+    os.environ["s3_access_key"] = response_json["access_token"]
+    # () gelöscht
+    return os.environ["s3_access_key"]
+
+
+def download_data(
+    url, output_dir, file_name=None, chunk_size=1024 * 1000, timeout=300, auth=None, check_size=True, overwrite=False
+):
+    """
+    Download single file from USGS M2M by download url
+
+    Parameters:
+        url: x
+        output_dir: x
+        file_name: x
+        chunk_size: x
+        timeout: x
+        auth: x
+        check_size: x
+        overwrite: x
+
+    Returns:
+        (str|bool): filepath or False
+    """
+
+    try:
+        print("Waiting for server response...")
+        if auth:
+            r = requests.get(url, stream=True, allow_redirects=True, timeout=timeout, auth=auth)
+        else:
+            r = requests.get(url, stream=True, allow_redirects=True, timeout=timeout)
+        expected_file_size = int(r.headers.get("content-length", -1))
+        if file_name is None:
+            try:
+                file_name = r.headers["Content-Disposition"].split('"')[1]
+            except Exception:
+                file_name = os.path.basename(url)
+                # raise Exception("Can not automatically identify file_name.")
+
+        print(f"Filename: {file_name}")
+        file_path = os.path.join(output_dir, file_name)
+        # TODO: Check for existing files and whether they have the correct file size
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        if os.path.exists(file_path) and overwrite is False:
+            return file_path
+        elif os.path.exists(file_path) and overwrite is True:
+            print("Removing old file")
+            os.remove(file_path)
+
+        with open(file_path, "wb") as f:
+            start = time.perf_counter()
+            print(f"Download of {file_name} in progress...")
+            for chunk in r.iter_content(chunk_size=chunk_size):
+                f.write(chunk)
+            duration = time.perf_counter() - start
+
+        file_size = os.stat(file_path).st_size
+        speed = round((file_size / duration) / (1000 * 1000), 2)
+
+        if check_size:
+            if expected_file_size != file_size:
+                os.remove(file_path)
+                print(f"Failed to download from {url}")
+                return False
+
+        print(f"Download of {file_name} successful. Average download speed: {speed} MB/s")
+        return file_path
+
+    except Exception as e:
+        print(e)
+        print(f"Failed to download from {url}.")
+        return False