From 36ac157856801a623c7e5f9423a709aa8d1be8af Mon Sep 17 00:00:00 2001 From: dzaslavskiy Date: Wed, 5 Apr 2023 12:07:45 -0400 Subject: [PATCH] Update service to use shared drives (#5) * Update service to use shared drives * Add delete endpoint --- .github/dependabot.yaml | 18 +++ .github/workflows/README.md | 38 +++++ .github/workflows/codeql-analysis.yaml | 52 +++++++ .github/workflows/deploy-app.yaml | 33 ++++ .github/workflows/python-checks.yaml | 46 ++++++ .github/workflows/stale-items.yaml | 23 +++ .github/workflows/unit-tests.yaml | 36 +++++ gdrive/api.py | 43 +++++- gdrive/client.py | 202 +++++++++++++++---------- gdrive/settings.py | 40 ++--- requirements.txt | 8 +- 11 files changed, 428 insertions(+), 111 deletions(-) create mode 100644 .github/dependabot.yaml create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/codeql-analysis.yaml create mode 100644 .github/workflows/deploy-app.yaml create mode 100644 .github/workflows/python-checks.yaml create mode 100644 .github/workflows/stale-items.yaml create mode 100644 .github/workflows/unit-tests.yaml diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..a195221 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + labels: + - dependencies + - python + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + labels: + - dependencies + - github-actions diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..4c70917 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,38 @@ +# GitHub Actions CI/CD workflows + +## Python Checks +The Python-Checks workflow will run a series of checks on the python code +in this repository. + +### Bandit +The Bandit workflow will run the Bandit security linter tool against this +project. A failed run indicates that Bandit found at least one vulnerability. + +### Black +The workflow outlined in `black.yml` checks to ensure that the Python style +for this project is consistent and fully implemented in all Python files. +For more information about this workflow, see +https://black.readthedocs.io/en/stable/github_actions.html + +## CodeQL-Analysis +The codeql-analysis workflow the CodeQL semantic code analysis engine to help +find security issues very early on in the development process. See +[CodeQL](https://securitylab.github.com/tools/codeql) for more details. + +## Deploy +Deploys the project to the correct GIVE environment within Cloud.gov. The +deploy workflow will run unit-tests and only deploy if those test are +successful. Deployment will also only be triggered in the 18F repository. This +will prevent forks from needlessly running workflows that will always fail +(forks won't be able to authenticate into the dev environment). + +## Stale Items +The stale-items workflow will run once per day and mark issues and PR's as +stale if they have not seen any activity over the last 30 days. After being +marked stale for 5 days, the workflow will close the item. + +## Unit Tests +The unit-tests workflow will install the project runtime dependencies and run +the unit test suite against the code. This workflow is used to run unit tests +for the application against pull requests before merging takes place. Additional +unit testing will take place on merging. diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml new file mode 100644 index 0000000..cf1841a --- /dev/null +++ b/.github/workflows/codeql-analysis.yaml @@ -0,0 +1,52 @@ +--- +name: "CodeQL" + +on: + push: + branches: [main] + paths-ignore: + - '**.md' # All markdown files in the repository + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + paths-ignore: + - '**.md' + schedule: + # weekly run at arbitrary time + - cron: '43 22 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: ['python'] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java) + # If this step fails, then remove it and run the build manually. See below + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + # - run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/deploy-app.yaml b/.github/workflows/deploy-app.yaml new file mode 100644 index 0000000..f3598de --- /dev/null +++ b/.github/workflows/deploy-app.yaml @@ -0,0 +1,33 @@ +--- +# This workflow will run unit tests and deploy the application to a +# target environment + +name: Deploy + +on: + push: + branches: + - main + tags: + - "*" + paths-ignore: + - "**.md" # All markdown files in the repository + release: + types: [released] + +jobs: + deploy: + if: github.repository_owner == 'GSA-TTS' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: GSA-TTS/identity-idva-cf-setup@v2 + id: cf-setup + with: + cf-username: ${{ secrets.CF_USERNAME }} + cf-password: ${{ secrets.CF_PASSWORD }} + cf-org: ${{ secrets.CF_ORG }} + + - name: Deploy application + run: cf push --vars-file vars.yaml --var ENVIRONMENT=${{ steps.cf-setup.outputs.target-environment }} --strategy rolling diff --git a/.github/workflows/python-checks.yaml b/.github/workflows/python-checks.yaml new file mode 100644 index 0000000..a2b2113 --- /dev/null +++ b/.github/workflows/python-checks.yaml @@ -0,0 +1,46 @@ +--- +# This workflow will run the Black Python formatter as well as the +# Bandit security linter. See the following pages for details: +# See https://black.readthedocs.io/en/stable/github_actions.html +# https://github.com/PyCQA/bandit +name: Python-Checks + +on: + push: + branches: + - main + paths: + - '**.py' # All python files in the repository + pull_request: + paths: + - '**.py' + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - uses: psf/black@stable + + bandit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Scan + run: | + pip install -r requirements-dev.txt + bandit --exclude ./.venv/,./test -r . diff --git a/.github/workflows/stale-items.yaml b/.github/workflows/stale-items.yaml new file mode 100644 index 0000000..7097f18 --- /dev/null +++ b/.github/workflows/stale-items.yaml @@ -0,0 +1,23 @@ +--- +name: 'Stale-Items' +on: + schedule: + # daily run at arbitrary time + - cron: '30 1 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v7 + with: + stale-issue-message: >- + This issue has been automatically marked as stale because it has + not had any activity in the last 30 days. Remove stale label or + comment or this will be closed in 5 days. + stale-pr-message: >- + This issue has been automatically marked as stale because it has + not had any activity in the last 30 days. Remove stale label or + comment or this will be closed in 5 days. + days-before-stale: 30 + days-before-close: 5 diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml new file mode 100644 index 0000000..2699419 --- /dev/null +++ b/.github/workflows/unit-tests.yaml @@ -0,0 +1,36 @@ +--- +# This workflow will install Python dependencies and run tests so that +# unit tests can be run against pull requests. + +name: Unit-Tests + +on: + pull_request: + paths-ignore: + - '**.md' # All markdown files in the repository + workflow_call: + +jobs: + unit-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Set up Python 3.9 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest + - name: Test with Pytest unit tests + run: | + export DEBUG=True + python -m pytest diff --git a/gdrive/api.py b/gdrive/api.py index 4f7f77b..ff786eb 100644 --- a/gdrive/api.py +++ b/gdrive/api.py @@ -7,7 +7,8 @@ import logging import fastapi -from fastapi import Body +from fastapi import Response, status +from googleapiclient.http import HttpError from starlette.requests import Request from . import client, settings @@ -16,18 +17,44 @@ router = fastapi.APIRouter() +client.init() @router.post("/upload") -async def upload_file(id, filename, request: Request, base64: bool = False): +async def upload_file(id, filename, request: Request, response: Response, base64: bool = False): """ Upload file to gdrive. """ - body = await request.body() - if base64: - body = base64decoder.b64decode(body) + try: + body = await request.body() - stream = io.BytesIO(body) + if base64: + body = base64decoder.b64decode(body) - parent = client.create_folder(id, settings.ROOT_DIRECTORY) - client.upload_basic(filename, parent, stream) + stream = io.BytesIO(body) + + parent = client.create_folder(id, settings.ROOT_DIRECTORY) + client.upload_basic(filename, parent, stream) + + except HttpError as error: + log.error(f"An error occurred: {error}") + response.status_code = error.status_code + +@router.delete("/upload") +async def delete_file(filename, response: Response): + """ + Delete file from gdrive. + """ + + try: + files = client.get_files(filename) + if files: + for file in files: + client.delete_file(file['id']) + else: + response.status_code = status.HTTP_404_NOT_FOUND + + except HttpError as error: + log.error(f"An error occurred: {error}") + response.status_code = error.status_code + \ No newline at end of file diff --git a/gdrive/client.py b/gdrive/client.py index 68b25fa..b86db94 100644 --- a/gdrive/client.py +++ b/gdrive/client.py @@ -1,126 +1,164 @@ import io -import mimetypes import logging +import mimetypes +from typing import List from google.oauth2 import service_account from googleapiclient.discovery import build -from googleapiclient.errors import HttpError from googleapiclient.http import MediaIoBaseUpload from gdrive import settings log = logging.getLogger(__name__) -if settings.CREDENTIALS != None: - log.info("Loading credentials from env var") - creds = service_account.Credentials.from_service_account_info( - settings.CREDENTIALS, scopes=settings.SCOPES - ) -else: - log.info("Loading credentials from file") - creds = service_account.Credentials.from_service_account_file( - settings.SERVICE_ACCOUNT_FILE, scopes=settings.SCOPES +creds = service_account.Credentials.from_service_account_info( + settings.CREDENTIALS, scopes=settings.SCOPES +) +service = build("drive", "v3", credentials=creds) + +def init(): + drive = drives_list() + result = ( + service.files() + .get(fileId=settings.ROOT_DIRECTORY, supportsAllDrives=True) + .execute() ) + driveId = result["id"] + log.info(f"Connected to Root Directory {driveId}") + -def list(count: int): +def list(count: int = 10, shared: bool = True) -> None: """ Prints the names and ids of the first files the user has access to. """ - try: - service = build("drive", "v3", credentials=creds) - - results = ( - service.files() - .list( - pageSize=count, fields="nextPageToken, files(id, name, parents, trashed)" - ) - .execute() + results = ( + service.files() + .list( + pageSize=count, + fields="*", + supportsAllDrives=shared, + includeItemsFromAllDrives=shared, ) - items = results.get("files", []) - - if not items: - print("No files found.") - return - print("Files:") - print("name (id) parents trashed") - for item in items: - try: - print( - "{0} ({1}) {2} {3}".format( - item["name"], item["id"], item["parents"], item["trashed"] - ) + .execute() + ) + items = results.get("files", []) + + if not items: + log.info("No files found.") + return + log.info("Files:") + log.info("name (id) parents trashed") + for item in items: + try: + log.info( + "{0} ({1}) {2} {3}".format( + item["name"], item["id"], item["parents"], item["trashed"] ) - except KeyError as error: - print(f"No such key: {error} in {item}") - except HttpError as error: - # TODO(developer) - Handle errors from drive API. - print(f"An error occurred: {error}") + ) + except KeyError as error: + log.info(f"No such key: {error} in {item}") -def upload_basic(filename: str, parent_id: str, bytes: io.BytesIO): - """Insert new file. - Returns : Id's of the file uploaded +def drives_list(): """ + List available shared drives + """ + + result = service.drives().list().execute() + return result - try: - service = build("drive", "v3", credentials=creds) - file_metadata = {"name": filename, "parents": [parent_id]} +def upload_basic(filename: str, parent_id: str, bytes: io.BytesIO) -> str: + """ + Upload new file to given parent folder + Returns : Id of the file uploaded + """ - mimetype, _ = mimetypes.guess_type(filename) - if mimetype is None: - # Guess failed, use octet-stream. - mimetype = "application/octet-stream" + file_metadata = {"name": filename, "parents": [parent_id]} - media = MediaIoBaseUpload(bytes, mimetype=mimetype) + mimetype, _ = mimetypes.guess_type(filename) + if mimetype is None: + # Guess failed, use octet-stream. + mimetype = "application/octet-stream" - file = service.files().create(body=file_metadata, media_body=media, fields="id").execute() - - print(f'File ID: {file.get("id")}') + media = MediaIoBaseUpload(bytes, mimetype=mimetype) - except HttpError as error: - print(f"An error occurred: {error}") - file = None + file = ( + service.files() + .create( + body=file_metadata, + media_body=media, + fields="id", + supportsAllDrives=True, + ) + .execute() + ) + + log.debug(f'File ID: {file.get("id")}') return file.get("id") -def create_folder(name: str, parent_id: str): - """Create a folder and prints the folder ID +def create_folder(name: str, parent_id: str) -> str: + """ + Create a folder and prints the folder ID Returns : Folder Id """ - try: - service = build("drive", "v3", credentials=creds) - file_metadata = { - "name": name, - "parents": [parent_id], - "mimeType": "application/vnd.google-apps.folder", - } + file_metadata = { + "name": name, + "parents": [parent_id], + "mimeType": "application/vnd.google-apps.folder", + } + + existing = ( + service.files() + .list( + q=f"name='{name}' and '{parent_id}' in parents", + fields="files(id, name)", + supportsAllDrives=True, + includeItemsFromAllDrives=True, + ) + .execute() + .get("files", []) + ) - existing = service.files().list(q=f"name='{name}' and '{parent_id}' in parents", fields="files(id, name)").execute().get("files", []) + if not existing: + file = ( + service.files() + .create(body=file_metadata, fields="id", supportsAllDrives=True) + .execute() + ) + log.debug(f'Folder has created with ID: "{file.get("id")}".') + else: + file = existing[0] + log.debug("Folder already exists") - if not existing: - file = service.files().create(body=file_metadata, fields="id").execute() - print(f'Folder has created with ID: "{file.get("id")}".') - else: - file = existing[0] - print("Folder already exists") + return file.get("id") - except HttpError as error: - print(f"An error occurred: {error}") - file = None - return file.get("id") +def get_files(filename: str) -> List: + """ + Get list of files by filename + """ + results = ( + service.files() + .list( + q=f"name = '{filename}'", + supportsAllDrives=True, + includeItemsFromAllDrives=True, + ) + .execute() + ) -def delete_file(id: str): + return results["files"] - try: - service = build("drive", "v3", credentials=creds) - service.files().delete(fileId=id).execute() +def delete_file(id: str) -> None: + """ + Delete file by id + """ - except HttpError as error: - print(f"An error occurred: {error}") + service.files().delete(fileId=id, supportsAllDrives=True).execute() diff --git a/gdrive/settings.py b/gdrive/settings.py index 8c39585..b262936 100644 --- a/gdrive/settings.py +++ b/gdrive/settings.py @@ -8,21 +8,6 @@ log = logging.getLogger(__name__) -def load_credentials(): - vcap_services = os.getenv("VCAP_SERVICES") - if vcap_services is not None: - try: - gdrive_service = json.loads(vcap_services)["user-provided"][0] - if gdrive_service["name"] == "gdrive": - return gdrive_service["credentials"] - else: - return None - except (json.JSONDecodeError, KeyError) as err: - log.warning("Unable to load credentials from VCAP_SERVICES") - log.debug("Error: %s", str(err)) - return None - return None - # SECURITY WARNING: don't run with debug turned on in production! # DEBUG set is set to True if env var is "True" DEBUG = os.getenv("DEBUG", "False") == "True" @@ -31,5 +16,26 @@ def load_credentials(): SCOPES = ["https://www.googleapis.com/auth/drive"] SERVICE_ACCOUNT_FILE = "credentials.json" -ROOT_DIRECTORY = "1hIo6ynbn2ErRIWF4RqMQB6QDcebJ4R5E" -CREDENTIALS = load_credentials() +ROOT_DIRECTORY = "" +CREDENTIALS = None + + +try: + vcap_services = os.getenv("VCAP_SERVICES") + config = {} + if vcap_services: + user_services = json.loads(vcap_services)["user-provided"] + for service in user_services: + if service["name"] == "gdrive": + log.info("Loading credentials from env var") + config = service["credentials"] + break + else: + with open(SERVICE_ACCOUNT_FILE) as file: + log.info("Loading credentials from creds file") + config = json.load(file) + CREDENTIALS = config["credentials"] + ROOT_DIRECTORY = config["root_directory"] +except (json.JSONDecodeError, KeyError) as err: + log.warning("Unable to load credentials from VCAP_SERVICES") + log.debug("Error: %s", str(err)) diff --git a/requirements.txt b/requirements.txt index 2ac0bbc..9d72cad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -fastapi==0.78.0 -uvicorn==0.17.6 +fastapi==0.95.0 +uvicorn==0.21.1 starlette-prometheus==0.9.0 -google-api-python-client==2.55.0 +google-api-python-client==2.83.0 google-auth-httplib2==0.1.0 -google-auth-oauthlib==0.5.2 +google-auth-oauthlib==1.0.0