diff --git a/README.md b/README.md index 1830c8d..55d1690 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,13 @@ git2s3 start - **GIT_TOKEN** - GitHub token to get ALL repos (including private). - **GIT_IGNORE** - List of repositories/gists to ignore. Defaults to `[]` - **SOURCE** - Source options `[repo, gist, wiki]` to back up. Defaults to all. -- **LOG** - Log options to log to a ``file`` or ``stdout``. _Does not apply when custom logger is used_ +- **LOG** - Log options to log to a `file` or `stdout`. _Does not apply when custom logger is used_ - **DEBUG** - Boolean flag to enable debug level logging. _Does not apply when custom logger is used_ -- **AWS_PROFILE_NAME** - AWS profile name. Uses the CLI config value ``AWS_DEFAULT_PROFILE`` by default. -- **AWS_ACCESS_KEY_ID** - AWS access key ID. Uses the CLI config value ``AWS_ACCESS_KEY_ID`` by default. -- **AWS_SECRET_ACCESS_KEY** - AWS secret key. Uses the CLI config value ``AWS_SECRET_ACCESS_KEY`` by default. -- **AWS_REGION_NAME** - S3 bucket's region. Uses the CLI config value ``AWS_DEFAULT_REGION`` by default. +- **STORE_LOCAL** - Boolean flag to store the backup locally. Defaults to `False` +- **AWS_PROFILE_NAME** - AWS profile name. Uses the CLI config value `AWS_DEFAULT_PROFILE` by default. +- **AWS_ACCESS_KEY_ID** - AWS access key ID. Uses the CLI config value `AWS_ACCESS_KEY_ID` by default. +- **AWS_SECRET_ACCESS_KEY** - AWS secret key. Uses the CLI config value `AWS_SECRET_ACCESS_KEY` by default. +- **AWS_REGION_NAME** - S3 bucket's region. Uses the CLI config value `AWS_DEFAULT_REGION` by default. - **AWS_BUCKET_NAME** - AWS bucket name to store the backups. - **AWS_S3_PREFIX** - S3 prefix _(folder like)_ for the backup. Defaults to `github` - **BOTO3_RETRY_ATTEMPTS** - Number of retries for Boto3 client config. Defaults to `10` diff --git a/docs/README.html b/docs/README.html index 17686e5..c74cf79 100644 --- a/docs/README.html +++ b/docs/README.html @@ -99,6 +99,7 @@

Environment Variables

SOURCE - Source options [repo, gist, wiki] to back up. Defaults to all.

  • LOG - Log options to log to a file or stdout. Does not apply when custom logger is used

  • DEBUG - Boolean flag to enable debug level logging. Does not apply when custom logger is used

    • +

  • STORE_LOCAL - Boolean flag to store the backup locally. Defaults to False

  • AWS_PROFILE_NAME - AWS profile name. Uses the CLI config value AWS_DEFAULT_PROFILE by default.

  • AWS_ACCESS_KEY_ID - AWS access key ID. Uses the CLI config value AWS_ACCESS_KEY_ID by default.

  • AWS_SECRET_ACCESS_KEY - AWS secret key. Uses the CLI config value AWS_SECRET_ACCESS_KEY by default.

    • diff --git a/docs/README.md b/docs/README.md index 1830c8d..55d1690 100644 --- a/docs/README.md +++ b/docs/README.md @@ -61,12 +61,13 @@ git2s3 start - **GIT_TOKEN** - GitHub token to get ALL repos (including private). - **GIT_IGNORE** - List of repositories/gists to ignore. Defaults to `[]` - **SOURCE** - Source options `[repo, gist, wiki]` to back up. Defaults to all. -- **LOG** - Log options to log to a ``file`` or ``stdout``. _Does not apply when custom logger is used_ +- **LOG** - Log options to log to a `file` or `stdout`. _Does not apply when custom logger is used_ - **DEBUG** - Boolean flag to enable debug level logging. _Does not apply when custom logger is used_ -- **AWS_PROFILE_NAME** - AWS profile name. Uses the CLI config value ``AWS_DEFAULT_PROFILE`` by default. -- **AWS_ACCESS_KEY_ID** - AWS access key ID. Uses the CLI config value ``AWS_ACCESS_KEY_ID`` by default. -- **AWS_SECRET_ACCESS_KEY** - AWS secret key. Uses the CLI config value ``AWS_SECRET_ACCESS_KEY`` by default. -- **AWS_REGION_NAME** - S3 bucket's region. Uses the CLI config value ``AWS_DEFAULT_REGION`` by default. +- **STORE_LOCAL** - Boolean flag to store the backup locally. Defaults to `False` +- **AWS_PROFILE_NAME** - AWS profile name. Uses the CLI config value `AWS_DEFAULT_PROFILE` by default. +- **AWS_ACCESS_KEY_ID** - AWS access key ID. Uses the CLI config value `AWS_ACCESS_KEY_ID` by default. +- **AWS_SECRET_ACCESS_KEY** - AWS secret key. Uses the CLI config value `AWS_SECRET_ACCESS_KEY` by default. +- **AWS_REGION_NAME** - S3 bucket's region. Uses the CLI config value `AWS_DEFAULT_REGION` by default. - **AWS_BUCKET_NAME** - AWS bucket name to store the backups. - **AWS_S3_PREFIX** - S3 prefix _(folder like)_ for the backup. Defaults to `github` - **BOTO3_RETRY_ATTEMPTS** - Number of retries for Boto3 client config. Defaults to `10` diff --git a/docs/_sources/README.md.txt b/docs/_sources/README.md.txt index 1830c8d..55d1690 100644 --- a/docs/_sources/README.md.txt +++ b/docs/_sources/README.md.txt @@ -61,12 +61,13 @@ git2s3 start - **GIT_TOKEN** - GitHub token to get ALL repos (including private). - **GIT_IGNORE** - List of repositories/gists to ignore. Defaults to `[]` - **SOURCE** - Source options `[repo, gist, wiki]` to back up. Defaults to all. -- **LOG** - Log options to log to a ``file`` or ``stdout``. _Does not apply when custom logger is used_ +- **LOG** - Log options to log to a `file` or `stdout`. _Does not apply when custom logger is used_ - **DEBUG** - Boolean flag to enable debug level logging. _Does not apply when custom logger is used_ -- **AWS_PROFILE_NAME** - AWS profile name. Uses the CLI config value ``AWS_DEFAULT_PROFILE`` by default. -- **AWS_ACCESS_KEY_ID** - AWS access key ID. Uses the CLI config value ``AWS_ACCESS_KEY_ID`` by default. -- **AWS_SECRET_ACCESS_KEY** - AWS secret key. Uses the CLI config value ``AWS_SECRET_ACCESS_KEY`` by default. -- **AWS_REGION_NAME** - S3 bucket's region. Uses the CLI config value ``AWS_DEFAULT_REGION`` by default. +- **STORE_LOCAL** - Boolean flag to store the backup locally. Defaults to `False` +- **AWS_PROFILE_NAME** - AWS profile name. Uses the CLI config value `AWS_DEFAULT_PROFILE` by default. +- **AWS_ACCESS_KEY_ID** - AWS access key ID. Uses the CLI config value `AWS_ACCESS_KEY_ID` by default. +- **AWS_SECRET_ACCESS_KEY** - AWS secret key. Uses the CLI config value `AWS_SECRET_ACCESS_KEY` by default. +- **AWS_REGION_NAME** - S3 bucket's region. Uses the CLI config value `AWS_DEFAULT_REGION` by default. - **AWS_BUCKET_NAME** - AWS bucket name to store the backups. - **AWS_S3_PREFIX** - S3 prefix _(folder like)_ for the backup. Defaults to `github` - **BOTO3_RETRY_ATTEMPTS** - Number of retries for Boto3 client config. Defaults to `10` diff --git a/docs/genindex.html b/docs/genindex.html index d1350d6..2386cab 100644 --- a/docs/genindex.html +++ b/docs/genindex.html @@ -243,10 +243,12 @@

    I

    L

    diff --git a/docs/index.html b/docs/index.html index d82d4fe..c4012ea 100644 --- a/docs/index.html +++ b/docs/index.html @@ -251,8 +251,16 @@

    Welcome to Git2S3’s documentation!
    -trigger() → None
    +trigger() → bool

    Trigger to upload all file objects concurrently to S3.

    +
    +
    Returns:
    +

    Returns a boolean flag to indicate completion status.

    +
    +
    Return type:
    +

    bool

    +
    +
    @@ -428,6 +436,11 @@

    Configurationdebug: bool
    +
    +
    +local_store: bool
    +
    +
    aws_profile_name: str | None
    diff --git a/docs/objects.inv b/docs/objects.inv index 918b8eb..2256500 100644 Binary files a/docs/objects.inv and b/docs/objects.inv differ diff --git a/docs/searchindex.js b/docs/searchindex.js index 24ce467..7bd189c 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["README", "index"], "filenames": ["README.md", "index.rst"], "titles": ["Git2S3", "Welcome to Git2S3\u2019s documentation!"], "terms": {"backup": 0, "github": [0, 1], "project": 0, "aw": 0, "s3": 0, "platform": 0, "support": [0, 1], "deploy": 0, "recommend": 0, "instal": [0, 1], "python": 0, "3": 0, "10": 0, "11": 0, "us": [0, 1], "dedic": 0, "virtual": 0, "m": 0, "pip": 0, "initi": 0, "id": 0, "import": 0, "__name__": 0, "__main__": 0, "git": [0, 1], "start": [0, 1], "cli": [0, 1], "help": 0, "usag": 0, "instruct": 0, "sourc": [0, 1], "from": [0, 1], "an": [0, 1], "env": [0, 1], "file": [0, 1], "By": 0, "default": [0, 1], "look": 0, "current": 0, "work": 0, "directori": [0, 1], "refer": [0, 1], "sampl": 0, "exampl": 0, "_": 0, "api": 0, "url": [0, 1], "endpoint": 0, "http": [0, 1], "com": [0, 1], "owner": [0, 1], "profil": [0, 1], "organ": [0, 1], "name": [0, 1], "token": 0, "get": [0, 1], "all": [0, 1], "repo": [0, 1], "includ": 0, "privat": [0, 1], "ignor": 0, "list": [0, 1], "repositori": [0, 1], "gist": [0, 1], "option": [0, 1], "wiki": [0, 1], "back": 0, "up": 0, "log": [0, 1], "stdout": [0, 1], "doe": 0, "appli": 0, "when": [0, 1], "custom": 0, "logger": [0, 1], "i": [0, 1], "debug": [0, 1], "boolean": [0, 1], "flag": [0, 1], "enabl": 0, "level": 0, "config": [0, 1], "valu": [0, 1], "aws_default_profil": 0, "access": 0, "kei": 0, "aws_access_key_id": [0, 1], "secret": 0, "aws_secret_access_kei": [0, 1], "region": 0, "bucket": 0, "": 0, "aws_default_region": 0, "store": [0, 1], "prefix": 0, "folder": 0, "like": 0, "boto3": 0, "retri": 0, "attempt": 0, "number": [0, 1], "client": 0, "mode": 0, "configur": 0, "docstr": 0, "format": 0, "googl": 0, "style": 0, "convent": 0, "pep": 0, "8": 0, "isort": 0, "requir": [0, 1], "gitvers": 0, "revers": 0, "f": 0, "release_not": 0, "rst": 0, "t": [0, 1], "pre": 0, "commit": 0, "ensur": 0, "run": [0, 1], "pytest": 0, "gener": [0, 1], "valid": [0, 1], "hyperlink": 0, "markdown": 0, "page": [0, 1], "sphinx": 0, "5": 0, "1": 0, "recommonmark": 0, "org": [0, 1], "thevickypedia": 0, "io": 0, "vignesh": 0, "rao": 0, "under": 0, "mit": 0, "kick": 1, "off": 1, "environ": 1, "variabl": 1, "code": 1, "standard": 1, "releas": 1, "note": 1, "lint": 1, "pypi": 1, "packag": 1, "runbook": 1, "licens": 1, "copyright": 1, "class": 1, "env_fil": 1, "str": 1, "o": 1, "pathlik": 1, "none": 1, "max_per_pag": 1, "int": 1, "100": 1, "instanti": 1, "object": 1, "clone": 1, "upload": 1, "keyword": 1, "argument": 1, "bring": 1, "your": 1, "own": 1, "maximum": 1, "fetch": 1, "per": 1, "profile_typ": 1, "type": 1, "return": 1, "cmd": 1, "fail": 1, "bool": 1, "true": 1, "command": 1, "paramet": 1, "error": 1, "after": 1, "get_al": 1, "sourcecontrol": 1, "dict": 1, "iter": 1, "through": 1, "target": 1, "avail": 1, "yield": 1, "dictionari": 1, "each": 1, "inform": 1, "set_pat": 1, "union": 1, "creat": 1, "authent": 1, "updat": 1, "netloc": 1, "set": 1, "origin": 1, "take": 1, "input": 1, "thi": 1, "step": 1, "public": 1, "clone_wiki": 1, "datastor": 1, "model": 1, "worker": 1, "json": 1, "payload": 1, "rais": 1, "If": 1, "thread": 1, "cloner": 1, "concurr": 1, "threadpoolexecutor": 1, "doesn": 1, "have": 1, "rate": 1, "limit": 1, "so": 1, "multi": 1, "safe": 1, "make": 1, "depend": 1, "host": 1, "machin": 1, "commun": 1, "discuss": 1, "44515": 1, "ani": 1, "process": 1, "onc": 1, "complet": 1, "successfulli": 1, "envconfig": 1, "upload_fil": 1, "local_file_path": 1, "s3_file_path": 1, "local": 1, "path": 1, "trigger": 1, "archer": 1, "destin": 1, "archiv": 1, "given": 1, "delet": 1, "while": 1, "retain": 1, "zipfil": 1, "assertionerror": 1, "present": 1, "env_load": 1, "filenam": 1, "load": 1, "base": 1, "filetyp": 1, "where": 1, "var": 1, "source_detector": 1, "detect": 1, "default_logg": 1, "consol": 1, "check_file_pres": 1, "source_dir": 1, "subdirectori": 1, "check": 1, "presenc": 1, "root": 1, "ar": 1, "basemodel": 1, "clone_url": 1, "descript": 1, "baseset": 1, "pydant": 1, "git_api_url": 1, "git_own": 1, "git_token": 1, "git_ignor": 1, "logopt": 1, "aws_profile_nam": 1, "aws_region_nam": 1, "aws_bucket_nam": 1, "aws_s3_prefix": 1, "boto3_retry_attempt": 1, "boto3_retry_mod": 1, "boto3retrymod": 1, "classmethod": 1, "from_env_fil": 1, "instanc": 1, "addit": 1, "featur": 1, "both": 1, "system": 1, "session": 1, "parse_sourc": 1, "pars": 1, "remov": 1, "parse_git_api_url": 1, "strip": 1, "end": 1, "parse_git_ignor": 1, "convert": 1, "lowercas": 1, "env_prefix": 1, "extra": 1, "allow": 1, "hide_input_in_error": 1, "strenum": 1, "control": 1, "exc": 1, "directoryexist": 1, "warn": 1, "alreadi": 1, "exist": 1, "unsupportedsourc": 1, "git2s3error": 1, "githubapierror": 1, "invalidown": 1, "invalid": 1, "invalidsourc": 1, "archiveerror": 1, "uploaderror": 1, "index": 1, "modul": 1, "search": 1}, "objects": {"git2s3.config": [[1, 0, 1, "", "DataStore"], [1, 0, 1, "", "EnvConfig"], [1, 0, 1, "", "LogOptions"], [1, 0, 1, "", "SourceControl"]], "git2s3.config.DataStore": [[1, 1, 1, "", "clone_url"], [1, 1, 1, "", "description"], [1, 1, 1, "", "name"], [1, 1, 1, "", "private"], [1, 1, 1, "", "source"]], "git2s3.config.EnvConfig": [[1, 0, 1, "", "Config"], [1, 1, 1, "", "aws_access_key_id"], [1, 1, 1, "", "aws_bucket_name"], [1, 1, 1, "", "aws_profile_name"], [1, 1, 1, "", "aws_region_name"], [1, 1, 1, "", "aws_s3_prefix"], [1, 1, 1, "", "aws_secret_access_key"], [1, 1, 1, "", "boto3_retry_attempts"], [1, 1, 1, "", "boto3_retry_mode"], [1, 1, 1, "", "debug"], [1, 2, 1, "", "from_env_file"], [1, 1, 1, "", "git_api_url"], [1, 1, 1, "", "git_ignore"], [1, 1, 1, "", "git_owner"], [1, 1, 1, "", "git_token"], [1, 1, 1, "", "log"], [1, 2, 1, "", "parse_git_api_url"], [1, 2, 1, "", "parse_git_ignore"], [1, 2, 1, "", "parse_source"], [1, 1, 1, "", "source"]], "git2s3.config.EnvConfig.Config": [[1, 1, 1, "", "env_prefix"], [1, 1, 1, "", "extra"], [1, 1, 1, "", "hide_input_in_errors"]], "git2s3.config.LogOptions": [[1, 1, 1, "", "file"], [1, 1, 1, "", "stdout"]], "git2s3.config.SourceControl": [[1, 1, 1, "", "all"], [1, 1, 1, "", "gist"], [1, 1, 1, "", "repo"], [1, 1, 1, "", "wiki"]], "git2s3": [[1, 3, 0, "-", "exc"], [1, 3, 0, "-", "main"], [1, 3, 0, "-", "s3"], [1, 3, 0, "-", "squire"]], "git2s3.exc": [[1, 4, 1, "", "ArchiveError"], [1, 4, 1, "", "DirectoryExists"], [1, 4, 1, "", "Git2S3Error"], [1, 4, 1, "", "GitHubAPIError"], [1, 4, 1, "", "InvalidOwner"], [1, 4, 1, "", "InvalidSource"], [1, 4, 1, "", "UnsupportedSource"], [1, 4, 1, "", "UploadError"]], "git2s3.main": [[1, 0, 1, "", "Git2S3"]], "git2s3.main.Git2S3": [[1, 2, 1, "", "cli"], [1, 2, 1, "", "clone_wiki"], [1, 2, 1, "", "cloner"], [1, 2, 1, "", "get_all"], [1, 2, 1, "", "profile_type"], [1, 2, 1, "", "set_pat"], [1, 2, 1, "", "start"], [1, 2, 1, "", "worker"]], "git2s3.s3": [[1, 0, 1, "", "Uploader"]], "git2s3.s3.Uploader": [[1, 2, 1, "", "trigger"], [1, 2, 1, "", "upload_file"]], "git2s3.squire": [[1, 5, 1, "", "archer"], [1, 5, 1, "", "check_file_presence"], [1, 5, 1, "", "default_logger"], [1, 5, 1, "", "env_loader"], [1, 5, 1, "", "source_detector"]]}, "objtypes": {"0": "py:class", "1": "py:attribute", "2": "py:method", "3": "py:module", "4": "py:exception", "5": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "attribute", "Python attribute"], "2": ["py", "method", "Python method"], "3": ["py", "module", "Python module"], "4": ["py", "exception", "Python exception"], "5": ["py", "function", "Python function"]}, "titleterms": {"git2s3": [0, 1], "kick": 0, "off": 0, "environ": 0, "variabl": 0, "code": 0, "standard": 0, "releas": 0, "note": 0, "lint": 0, "pypi": 0, "packag": 0, "runbook": 0, "licens": 0, "copyright": 0, "welcom": 1, "": 1, "document": 1, "content": 1, "main": 1, "s3": 1, "squir": 1, "configur": 1, "except": 1, "indic": 1, "tabl": 1}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 56}}) \ No newline at end of file +Search.setIndex({"docnames": ["README", "index"], "filenames": ["README.md", "index.rst"], "titles": ["Git2S3", "Welcome to Git2S3\u2019s documentation!"], "terms": {"backup": 0, "github": [0, 1], "project": 0, "aw": 0, "s3": 0, "platform": 0, "support": [0, 1], "deploy": 0, "recommend": 0, "instal": [0, 1], "python": 0, "3": 0, "10": 0, "11": 0, "us": [0, 1], "dedic": 0, "virtual": 0, "m": 0, "pip": 0, "initi": 0, "id": 0, "import": 0, "__name__": 0, "__main__": 0, "git": [0, 1], "start": [0, 1], "cli": [0, 1], "help": 0, "usag": 0, "instruct": 0, "sourc": [0, 1], "from": [0, 1], "an": [0, 1], "env": [0, 1], "file": [0, 1], "By": 0, "default": [0, 1], "look": 0, "current": 0, "work": 0, "directori": [0, 1], "refer": [0, 1], "sampl": 0, "exampl": 0, "_": 0, "api": 0, "url": [0, 1], "endpoint": 0, "http": [0, 1], "com": [0, 1], "owner": [0, 1], "profil": [0, 1], "organ": [0, 1], "name": [0, 1], "token": 0, "get": [0, 1], "all": [0, 1], "repo": [0, 1], "includ": 0, "privat": [0, 1], "ignor": 0, "list": [0, 1], "repositori": [0, 1], "gist": [0, 1], "option": [0, 1], "wiki": [0, 1], "back": 0, "up": 0, "log": [0, 1], "stdout": [0, 1], "doe": 0, "appli": 0, "when": [0, 1], "custom": 0, "logger": [0, 1], "i": [0, 1], "debug": [0, 1], "boolean": [0, 1], "flag": [0, 1], "enabl": 0, "level": 0, "store": [0, 1], "local": [0, 1], "fals": 0, "config": [0, 1], "valu": [0, 1], "aws_default_profil": 0, "access": 0, "kei": 0, "aws_access_key_id": [0, 1], "secret": 0, "aws_secret_access_kei": [0, 1], "region": 0, "bucket": 0, "": 0, "aws_default_region": 0, "prefix": 0, "folder": 0, "like": 0, "boto3": 0, "retri": 0, "attempt": 0, "number": [0, 1], "client": 0, "mode": 0, "configur": 0, "docstr": 0, "format": 0, "googl": 0, "style": 0, "convent": 0, "pep": 0, "8": 0, "isort": 0, "requir": [0, 1], "gitvers": 0, "revers": 0, "f": 0, "release_not": 0, "rst": 0, "t": [0, 1], "pre": 0, "commit": 0, "ensur": 0, "run": [0, 1], "pytest": 0, "gener": [0, 1], "valid": [0, 1], "hyperlink": 0, "markdown": 0, "page": [0, 1], "sphinx": 0, "5": 0, "1": 0, "recommonmark": 0, "org": [0, 1], "thevickypedia": 0, "io": 0, "vignesh": 0, "rao": 0, "under": 0, "mit": 0, "kick": 1, "off": 1, "environ": 1, "variabl": 1, "code": 1, "standard": 1, "releas": 1, "note": 1, "lint": 1, "pypi": 1, "packag": 1, "runbook": 1, "licens": 1, "copyright": 1, "class": 1, "env_fil": 1, "str": 1, "o": 1, "pathlik": 1, "none": 1, "max_per_pag": 1, "int": 1, "100": 1, "instanti": 1, "object": 1, "clone": 1, "upload": 1, "keyword": 1, "argument": 1, "bring": 1, "your": 1, "own": 1, "maximum": 1, "fetch": 1, "per": 1, "profile_typ": 1, "type": 1, "return": 1, "cmd": 1, "fail": 1, "bool": 1, "true": 1, "command": 1, "paramet": 1, "error": 1, "after": 1, "get_al": 1, "sourcecontrol": 1, "dict": 1, "iter": 1, "through": 1, "target": 1, "avail": 1, "yield": 1, "dictionari": 1, "each": 1, "inform": 1, "set_pat": 1, "union": 1, "creat": 1, "authent": 1, "updat": 1, "netloc": 1, "set": 1, "origin": 1, "take": 1, "input": 1, "thi": 1, "step": 1, "public": 1, "clone_wiki": 1, "datastor": 1, "model": 1, "worker": 1, "json": 1, "payload": 1, "rais": 1, "If": 1, "thread": 1, "cloner": 1, "concurr": 1, "threadpoolexecutor": 1, "doesn": 1, "have": 1, "rate": 1, "limit": 1, "so": 1, "multi": 1, "safe": 1, "make": 1, "depend": 1, "host": 1, "machin": 1, "commun": 1, "discuss": 1, "44515": 1, "ani": 1, "process": 1, "onc": 1, "complet": 1, "successfulli": 1, "envconfig": 1, "upload_fil": 1, "local_file_path": 1, "s3_file_path": 1, "path": 1, "trigger": 1, "statu": 1, "archer": 1, "destin": 1, "archiv": 1, "given": 1, "delet": 1, "while": 1, "retain": 1, "zipfil": 1, "assertionerror": 1, "present": 1, "env_load": 1, "filenam": 1, "load": 1, "base": 1, "filetyp": 1, "where": 1, "var": 1, "source_detector": 1, "detect": 1, "default_logg": 1, "consol": 1, "check_file_pres": 1, "source_dir": 1, "subdirectori": 1, "check": 1, "presenc": 1, "root": 1, "ar": 1, "basemodel": 1, "clone_url": 1, "descript": 1, "baseset": 1, "pydant": 1, "git_api_url": 1, "git_own": 1, "git_token": 1, "git_ignor": 1, "logopt": 1, "local_stor": 1, "aws_profile_nam": 1, "aws_region_nam": 1, "aws_bucket_nam": 1, "aws_s3_prefix": 1, "boto3_retry_attempt": 1, "boto3_retry_mod": 1, "boto3retrymod": 1, "classmethod": 1, "from_env_fil": 1, "instanc": 1, "addit": 1, "featur": 1, "both": 1, "system": 1, "session": 1, "parse_sourc": 1, "pars": 1, "remov": 1, "parse_git_api_url": 1, "strip": 1, "end": 1, "parse_git_ignor": 1, "convert": 1, "lowercas": 1, "env_prefix": 1, "extra": 1, "allow": 1, "hide_input_in_error": 1, "strenum": 1, "control": 1, "exc": 1, "directoryexist": 1, "warn": 1, "alreadi": 1, "exist": 1, "unsupportedsourc": 1, "git2s3error": 1, "githubapierror": 1, "invalidown": 1, "invalid": 1, "invalidsourc": 1, "archiveerror": 1, "uploaderror": 1, "index": 1, "modul": 1, "search": 1}, "objects": {"git2s3.config": [[1, 0, 1, "", "DataStore"], [1, 0, 1, "", "EnvConfig"], [1, 0, 1, "", "LogOptions"], [1, 0, 1, "", "SourceControl"]], "git2s3.config.DataStore": [[1, 1, 1, "", "clone_url"], [1, 1, 1, "", "description"], [1, 1, 1, "", "name"], [1, 1, 1, "", "private"], [1, 1, 1, "", "source"]], "git2s3.config.EnvConfig": [[1, 0, 1, "", "Config"], [1, 1, 1, "", "aws_access_key_id"], [1, 1, 1, "", "aws_bucket_name"], [1, 1, 1, "", "aws_profile_name"], [1, 1, 1, "", "aws_region_name"], [1, 1, 1, "", "aws_s3_prefix"], [1, 1, 1, "", "aws_secret_access_key"], [1, 1, 1, "", "boto3_retry_attempts"], [1, 1, 1, "", "boto3_retry_mode"], [1, 1, 1, "", "debug"], [1, 2, 1, "", "from_env_file"], [1, 1, 1, "", "git_api_url"], [1, 1, 1, "", "git_ignore"], [1, 1, 1, "", "git_owner"], [1, 1, 1, "", "git_token"], [1, 1, 1, "", "local_store"], [1, 1, 1, "", "log"], [1, 2, 1, "", "parse_git_api_url"], [1, 2, 1, "", "parse_git_ignore"], [1, 2, 1, "", "parse_source"], [1, 1, 1, "", "source"]], "git2s3.config.EnvConfig.Config": [[1, 1, 1, "", "env_prefix"], [1, 1, 1, "", "extra"], [1, 1, 1, "", "hide_input_in_errors"]], "git2s3.config.LogOptions": [[1, 1, 1, "", "file"], [1, 1, 1, "", "stdout"]], "git2s3.config.SourceControl": [[1, 1, 1, "", "all"], [1, 1, 1, "", "gist"], [1, 1, 1, "", "repo"], [1, 1, 1, "", "wiki"]], "git2s3": [[1, 3, 0, "-", "exc"], [1, 3, 0, "-", "main"], [1, 3, 0, "-", "s3"], [1, 3, 0, "-", "squire"]], "git2s3.exc": [[1, 4, 1, "", "ArchiveError"], [1, 4, 1, "", "DirectoryExists"], [1, 4, 1, "", "Git2S3Error"], [1, 4, 1, "", "GitHubAPIError"], [1, 4, 1, "", "InvalidOwner"], [1, 4, 1, "", "InvalidSource"], [1, 4, 1, "", "UnsupportedSource"], [1, 4, 1, "", "UploadError"]], "git2s3.main": [[1, 0, 1, "", "Git2S3"]], "git2s3.main.Git2S3": [[1, 2, 1, "", "cli"], [1, 2, 1, "", "clone_wiki"], [1, 2, 1, "", "cloner"], [1, 2, 1, "", "get_all"], [1, 2, 1, "", "profile_type"], [1, 2, 1, "", "set_pat"], [1, 2, 1, "", "start"], [1, 2, 1, "", "worker"]], "git2s3.s3": [[1, 0, 1, "", "Uploader"]], "git2s3.s3.Uploader": [[1, 2, 1, "", "trigger"], [1, 2, 1, "", "upload_file"]], "git2s3.squire": [[1, 5, 1, "", "archer"], [1, 5, 1, "", "check_file_presence"], [1, 5, 1, "", "default_logger"], [1, 5, 1, "", "env_loader"], [1, 5, 1, "", "source_detector"]]}, "objtypes": {"0": "py:class", "1": "py:attribute", "2": "py:method", "3": "py:module", "4": "py:exception", "5": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "attribute", "Python attribute"], "2": ["py", "method", "Python method"], "3": ["py", "module", "Python module"], "4": ["py", "exception", "Python exception"], "5": ["py", "function", "Python function"]}, "titleterms": {"git2s3": [0, 1], "kick": 0, "off": 0, "environ": 0, "variabl": 0, "code": 0, "standard": 0, "releas": 0, "note": 0, "lint": 0, "pypi": 0, "packag": 0, "runbook": 0, "licens": 0, "copyright": 0, "welcom": 1, "": 1, "document": 1, "content": 1, "main": 1, "s3": 1, "squir": 1, "configur": 1, "except": 1, "indic": 1, "tabl": 1}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 56}}) \ No newline at end of file diff --git a/git2s3/__init__.py b/git2s3/__init__.py index a010c04..930ae7c 100644 --- a/git2s3/__init__.py +++ b/git2s3/__init__.py @@ -6,7 +6,7 @@ from git2s3.main import Git2S3 -version = "0.0.2" +version = "0.0.3" @click.command() diff --git a/git2s3/config.py b/git2s3/config.py index 086ebca..73dc3da 100644 --- a/git2s3/config.py +++ b/git2s3/config.py @@ -81,6 +81,7 @@ class EnvConfig(BaseSettings): source: SourceControl | List[SourceControl] = SourceControl.all log: LogOptions = LogOptions.stdout debug: bool = False + local_store: bool = False aws_profile_name: str | None = None aws_access_key_id: str | None = None diff --git a/git2s3/main.py b/git2s3/main.py index 5c1d623..2de9d5e 100644 --- a/git2s3/main.py +++ b/git2s3/main.py @@ -217,12 +217,17 @@ def clone_wiki(self, datastore: config.DataStore) -> None: if datastore.private: wiki_dest = str( os.path.join( - self.clone_dir, datastore.source, "private", datastore.name + self.clone_dir, + datastore.source, + "private", + f"{datastore.name}.wiki", ) ) else: wiki_dest = str( - os.path.join(self.clone_dir, datastore.source, "public", datastore.name) + os.path.join( + self.clone_dir, datastore.source, "public", f"{datastore.name}.wiki" + ) ) if not os.path.isdir(wiki_dest): os.makedirs(wiki_dest) @@ -376,7 +381,12 @@ def start(self) -> None: if squire.check_file_presence(self.clone_dir): self.logger.info("Initiating S3 upload process...") s3_upload = s3.Uploader(self.env, self.logger) - s3_upload.trigger() - self.logger.info("S3 upload process completed.") + if s3_upload.trigger(): + self.logger.error("Some objects failed to upload.") + else: + self.logger.info("S3 upload process completed.") + if not self.env.local_store: + self.logger.info("Deleting local copy!") + shutil.rmtree(self.clone_dir) else: self.logger.warning("No files found for S3 upload process.") diff --git a/git2s3/s3.py b/git2s3/s3.py index 1d4fb13..413ee48 100644 --- a/git2s3/s3.py +++ b/git2s3/s3.py @@ -61,8 +61,13 @@ def upload_file( except (FileNotFoundError, BotoCoreError, ClientError) as error: raise exc.UploadError(error) - def trigger(self) -> None: - """Trigger to upload all file objects concurrently to S3.""" + def trigger(self) -> bool: + """Trigger to upload all file objects concurrently to S3. + + Returns: + bool: + Returns a boolean flag to indicate completion status. + """ futures = {} with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: for root, dirs, files in os.walk(self.base_path): @@ -74,10 +79,13 @@ def trigger(self) -> None: self.upload_file, local_file_path, s3_file_path ) futures[future] = s3_file_path + exception = False for future in as_completed(futures): if future.exception(): + exception = True self.logger.error( "Thread processing '%s' received an exception: %s", futures[future], future.exception(), ) + return exception