From b881e1eca3fdf25af125f7705ff035272f896650 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Fri, 30 Aug 2024 16:51:21 +0545 Subject: [PATCH 01/25] fix: conflicts with example.env file --- .github/workflows/build_and_deploy.yml | 46 ++++++++ .github/workflows/frontend-build.yml | 146 +++++++++++++++++++++++++ example.env | 132 +++++++++++----------- 3 files changed, 258 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/build_and_deploy.yml create mode 100644 .github/workflows/frontend-build.yml diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml new file mode 100644 index 0000000000..326730e754 --- /dev/null +++ b/.github/workflows/build_and_deploy.yml @@ -0,0 +1,46 @@ +name: Build and Deploy + +on: + # Push includes PR merge + push: + branches: + - main + - staging + - develop + - ci/gh-workflows + paths: + # Workflow is triggered only if src changes + - backend/** + - frontend/** + # Allow manual trigger + workflow_dispatch: + +jobs: + backend-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 + with: + context: . + build_target: prod + image_name: ghcr.io/${{ github.repository }}/backend + dockerfile: Dockerfile + secrets: inherit + + frontend-build: + uses: naxa-developers/tasking-manager/.github/workflows/frontend-build@ci/gh-workflows + secrets: inherit + with: + node-version: 16.x + context: ./frontend + cache-key-file: ./frontend/yarn.lock + package-manager: yarn + + # deploy_to_vm: + # name: Deploy to VM + # needs: + # - frontend-build + # - backend-build + # uses: hotosm/gh-workflows/.github/workflows/remote_deploy_compose.yml@2.0.5 + # with: + # docker_compose_file: docker-compose.vm.yml + # environment: ${{ github.ref_name }} + # secrets: inherit \ No newline at end of file diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml new file mode 100644 index 0000000000..af22e6be29 --- /dev/null +++ b/.github/workflows/frontend-build.yml @@ -0,0 +1,146 @@ +name: Node build + +on: + workflow_call: + inputs: + node-version: + description: "Node version to use." + required: false + type: string + default: "18.x" + node-version-file: + description: "Node version file to use. node-version overrides this parameter." + required: false + type: string + default: "" + context: + description: "Root directory to start the build from." + required: false + type: string + default: "." + cache: + description: "Use node modules installation caching. Default true." + required: false + type: boolean + default: true + cache-key-file: + description: "Key file for cache." + required: false + type: string + default: "${{ inputs.context }}/package.json" + package-manager: + description: "Package manager to use. Supports [npm, yarn]" + required: false + type: string + default: "npm" + build-script-name: + description: "Build script name in package.json" + required: false + type: string + default: "build" + upload-artifacts: + description: "Upload artifacts to Github" + required: false + type: boolean + default: true + + outputs: + artifact_name: + description: "Node built artifact" + value: ${{ jobs.node-build.outputs.artifact_name }} + +jobs: + node-build: + runs-on: ubuntu-latest + + environment: + name: ${{ github.ref_name }} + + outputs: + artifact_name: ${{ steps.get_artifact_name.outputs.artifact_name }} + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - id: node_setup + name: Install node + uses: actions/setup-node@v4 + #reference: https://github.com/actions/setup-node + with: + node-version: ${{ inputs.node-version }} + node-version-file: ${{ inputs.node-version-file && inputs.node-version-file || '' }} + + - name: Cache Node packages + uses: actions/cache@v3 + env: + cache_name: node-${{ inputs.node-version || hashFiles(inputs.node-version-file) }}-${{ inputs.package-manager }}-${{ hashFiles(inputs.cache-key-file) }} + with: + key: ${{ runner.os }}-build-${{ env.cache_name }} + path: | + ~/.npm + restore-keys: | + ${{ runner.os }}-build-${{ env.cache_name }} + + - id: node_packages_install + name: Install node pacakges + run: | + case "${{ inputs.package-manager }}" in + yarn) + yarn + ;; + npm) + npm i + ;; + esac + + - id: vars_and_secrets + name: Vars and Secrets to Env file + env: + VARS_CONTEXT: ${{ toJson(vars) }} + SECRETS_CONTEXT: ${{ toJson(secrets) }} + run: | + # Parse JSON with multiline strings + to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; } + + # Create .env file from VARS_CONTEXT + if [ "${VARS_CONTEXT}" != "null" ]; then + echo "${VARS_CONTEXT}" | to_envs > ${{ inputs.context }}/.env + fi + + # Append Secrets to .env file from SECRETS_CONTEXT + if [ "${SECRETS_CONTEXT}" != "null" ]; then + echo "\n${SECRETS_CONTEXT}" | to_envs >> ${{ inputs.context }}/.env + fi + + - id: build_frontend + name: Build Frontend + run: | + case "${{ inputs.package-manager }}" in + yarn) + yarn ${{ inputs.build-script-name }} + ;; + npm) + npm run ${{ inputs.build-script-name }} + ;; + esac + + - id: upload_build_artifacts + name: Upload dist folder as build artifacts + uses: actions/upload-artifact@v4 + if: ${{ inputs.upload-artifacts }} + with: + name: ${{ github.repository_id }}-${{ github.sha }}-frontend-dist + path: ./dist/* + retention-days: 1 + + - id: get_artifact_name + name: Get First Image Name + run: | + echo "artifact_name=${{ github.repository_id }}-${{ github.sha }}-frontend-dist" >> $GITHUB_OUTPUT + echo "Frontend Artifact Name: $artifact_name" + diff --git a/example.env b/example.env index 34f43e160c..c12d3988c1 100644 --- a/example.env +++ b/example.env @@ -9,46 +9,46 @@ # Note: 127.0.0.1 is a hard requirement for OSM Auth (instead of `localhost`) # On production instances, use the public URL of your frontend # TM_APP_BASE_URL=https://tasks.hotosm.org -TM_APP_BASE_URL=http://127.0.0.1:3000 +TM_APP_BASE_URL=${TM_APP_BASE_URL:-http://127.0.0.1:3000} # The TM_APP_API_URL defines the URL of your backend server. It will be used by # both the backend and by the frontend # On development instances it should be 127.0.0.1:3000 # On production instances, use the public URL of your backend -TM_APP_API_URL=http://127.0.0.1:3000/api +TM_APP_API_URL=${TM_APP_API_URL:-http://127.0.0.1:3000/api} # Defines the version of the API and will be used after /api/ on the url -TM_APP_API_VERSION=v2 +TM_APP_API_VERSION=${TM_APP_API_VERSION:-v2} # Information about the hosting organization -TM_ORG_NAME="Humanitarian OpenStreetMap Team" -TM_ORG_CODE=HOT -TM_ORG_LOGO=https://cdn.img.url/logo.png +TM_ORG_NAME=${TM_ORG_NAME:-"Humanitarian OpenStreetMap Team"} +TM_ORG_CODE=${TM_ORG_CODE:-HOT} +TM_ORG_LOGO=${TM_ORG_LOGO:-https://cdn.img.url/logo.png} # Don't use http or https on the following two variables -TM_ORG_URL=example.com -TM_ORG_PRIVACY_POLICY_URL=https://www.hotosm.org/privacy -TM_ORG_TWITTER=http://twitter.com/hotosm -TM_ORG_FB=https://www.facebook.com/hotosm -TM_ORG_INSTAGRAM=https://www.instagram.com/open.mapping.hubs/ -TM_ORG_YOUTUBE=https://www.youtube.com/user/hotosm -TM_ORG_GITHUB=https://github.com/hotosm +TM_ORG_URL=${TM_ORG_URL:-example.com} +TM_ORG_PRIVACY_POLICY_URL=${TM_ORG_PRIVACY_POLICY_URL:-https://www.hotosm.org/privacy} +TM_ORG_TWITTER=${TM_ORG_TWITTER:-http://twitter.com/hotosm} +TM_ORG_FB=${TM_ORG_FB:-https://www.facebook.com/hotosm} +TM_ORG_INSTAGRAM=${TM_ORG_INSTAGRAM:-https://www.instagram.com/open.mapping.hubs/} +TM_ORG_YOUTUBE=${TM_ORG_YOUTUBE:-https://www.youtube.com/user/hotosm} +TM_ORG_GITHUB=${TM_ORG_GITHUB:-https://github.com/hotosm} # Information about the OSM server - Customize your server here # By default, it's the public OpenStreetMap.org server -OSM_SERVER_URL=https://www.openstreetmap.org -OSM_SERVER_API_URL=https://api.openstreetmap.org -OSM_NOMINATIM_SERVER_URL=https://nominatim.openstreetmap.org -OSM_REGISTER_URL=https://www.openstreetmap.org/user/new +OSM_SERVER_API_URL=${OSM_SERVER_API_URL:-https://api.openstreetmap.org} +OSM_SERVER_URL=${OSM_SERVER_URL:-https://www.openstreetmap.org} +OSM_NOMINATIM_SERVER_URL=${OSM_NOMINATIM_SERVER_URL:-https://nominatim.openstreetmap.org} +OSM_REGISTER_URL=${OSM_REGISTER_URL:-https://www.openstreetmap.org/user/new} # Information about the Editor URLs. Those are the default values on the frontend. # You only need to modify it in case you want to direct users to map on a different OSM instance. -# ID_EDITOR_URL=https://www.openstreetmap.org/edit?editor=id& -# POTLATCH2_EDITOR_URL=https://www.openstreetmap.org/edit?editor=potlatch2 -# RAPID_EDITOR_URL=https://mapwith.ai/rapid +# ID_EDITOR_URL=${ID_EDITOR_URL:-https://www.openstreetmap.org/edit?editor=id&} +# POTLATCH2_EDITOR_URL=${POTLATCH2_EDITOR_URL:-https://www.openstreetmap.org/edit?editor=potlatch2} +# RAPID_EDITOR_URL=${RAPID_EDITOR_URL:-https://mapwith.ai/rapid} # Matomo configuration. Optional, configure it in case you have a Matomo instance. -# TM_MATOMO_ID="site_id" -# TM_MATOMO_ENDPOINT="https://..." +# TM_MATOMO_ID=${TM_MATOMO_ID:-"site_id"} +# TM_MATOMO_ENDPOINT=${TM_MATOMO_ENDPOINT:-"https://..."} # Mapbox access key to display the maps (optional) # @@ -60,57 +60,57 @@ OSM_REGISTER_URL=https://www.openstreetmap.org/user/new # If you do not set a token, then maps will fallback to using the raster tile based # Humanitarian Layer. # -# TM_MAPBOX_TOKEN= +# TM_MAPBOX_TOKEN=${TM_MAPBOX_TOKEN} # If you want your TM app to work better offline and load faster, you can change # from 0 (unregister) to 1 (register) below. Note this comes with some pitfalls. # Learn more about service workers: https://bit.ly/CRA-PWA # It is more complex to use for TM if your frontend and backend are on same server. -# TM_ENABLE_SERVICEWORKER=0 +# TM_ENABLE_SERVICEWORKER=${TM_ENABLE_SERVICEWORKER:-0} # Define an API URL and KEY of an image upload service. # It will be used to store the Organisation logos and the images uploaded on comments input fields. # HOT uses this service: https://github.com/hotosm/cdn-upload-api/ to setup an image upload API -# TM_IMAGE_UPLOAD_API_URL= -# TM_IMAGE_UPLOAD_API_KEY= +# TM_IMAGE_UPLOAD_API_URL=${TM_IMAGE_UPLOAD_API_URL} +# TM_IMAGE_UPLOAD_API_KEY=${TM_IMAGE_UPLOAD_API_KEY} # Define the image to be used on the homepage main's banner. # If it's not defined, the default images will be used. # The high resolution image should be 2500px width. The low should be 824px. -# TM_HOMEPAGE_IMG_HIGH=https://cdn.img.url/banner-high.png -# TM_HOMEPAGE_IMG_LOW=https://cdn.img.url/banner-low.png +# TM_HOMEPAGE_IMG_HIGH=${TM_HOMEPAGE_IMG_HIGH:-https://cdn.img.url/banner-high.png} +# TM_HOMEPAGE_IMG_LOW=${TM_HOMEPAGE_IMG_LOW:-https://cdn.img.url/banner-low.png} # Define a video to be played on the background of the homepage's main banner. # On HOT instance we use https://cdn.hotosm.org/tasking-manager/mapping.mp4 # Please copy it to your CDN / server in order to avoid overloading the HOT CDN. # It's not required to set this tag. Case it isn't set, an image will be used as background. -# TM_HOMEPAGE_VIDEO_URL= +# TM_HOMEPAGE_VIDEO_URL=${TM_HOMEPAGE_VIDEO_URL} # API base URL and token(used to retrieve user stats only) for ohsomeNow Stats # -OHSOME_STATS_BASE_URL=https://stats.now.ohsome.org/api -OHSOME_STATS_TOKEN=testSuperSecretTestToken +OHSOME_STATS_BASE_URL=${OHSOME_STATS_BASE_URL:-https://stats.now.ohsome.org/api} +OHSOME_STATS_TOKEN=${OHSOME_STATS_TOKEN:-testSuperSecretTestToken} # Secret (required) # # A freely definable secret. Gives authorization to the front- and and back-end # to talk to each other. # -TM_SECRET=s0m3l0ngr4nd0mstr1ng-b3cr34tiv3 +TM_SECRET=${TM_SECRET:-s0m3l0ngr4nd0mstr1ngb3cr34tiv3} # OpenStreetMap OAuth2 client id and secret (required) # -TM_CLIENT_ID=foo -TM_CLIENT_SECRET=s0m3l0ngr4nd0mstr1ng-b3cr34tiv3 +TM_CLIENT_ID=${TM_CLIENT_ID:-foo} +TM_CLIENT_SECRET=${TM_CLIENT_SECRET:-s0m3l0ngr4nd0mstr1ngb3cr34tiv3} # Redirect uri registered while creating OAuth2 application (required) -TM_REDIRECT_URI=http://127.0.0.1:3000/authorized +TM_REDIRECT_URI=${TM_REDIRECT_URI:-http://127.0.0.1:3000/authorized} # Scope of TM defined while creating OAuth2 application (required) -TM_SCOPE=read_prefs write_api +TM_SCOPE=${TM_SCOPE:-read_prefs write_api} # Required by requests_oauthlib to work while making oauth2 requests from http server (required) -# OAUTHLIB_INSECURE_TRANSPORT = 1 +# OAUTHLIB_INSECURE_TRANSPORT=${OAUTHLIB_INSECURE_TRANSPORT:-1} # The default tag used in the OSM changeset comment # IMPORTANT! This must be unique on your instance @@ -132,39 +132,39 @@ TM_SCOPE=read_prefs write_api # DB_CONNECT_PARAM_JSON='{ "username": "tm", "password": "myprivatesecret", "host": "tm4-database.example.org", "port": "5432", "dbname": "taskingmanager }' # # NOTE: These are ignored if DB_CONNECT_PARAM_JSON is set -POSTGRES_DB=tasking-manager -POSTGRES_USER=tm -POSTGRES_PASSWORD=tm -POSTGRES_ENDPOINT=tm-db -POSTGRES_PORT=5432 +POSTGRES_DB=${POSTGRES_DB:-tasking-manager} +POSTGRES_USER=${POSTGRES_USER:-tm} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-tm} +POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT:-tm-db} +POSTGRES_PORT=${POSTGRES_PORT:-5432} # The postgres database name used for testing (required). # All other configurations except the database name are inherited from the main database defined above. -POSTGRES_TEST_DB=tasking-manager-test +POSTGRES_TEST_DB=${POSTGRES_TEST_DB:-tasking-manager-test} # The address to use as the sender on auto generated emails # (optional, but required to send email) # -# TM_EMAIL_FROM_ADDRESS=noreply@localhost +# TM_EMAIL_FROM_ADDRESS=${TM_EMAIL_FROM_ADDRESS:-noreply@localhost} # The address to use as the receiver in contact form. # -# TM_EMAIL_CONTACT_ADDRESS=sysadmin@localhost +# TM_EMAIL_CONTACT_ADDRESS=${TM_EMAIL_CONTACT_ADDRESS:-sysadmin@localhost} # Email sending server configuration (optional) # This is required in order to send out messages. # -# TM_SMTP_HOST= -# TM_SMTP_PORT=25 -# TM_SMTP_USER= -# TM_SMTP_PASSWORD= +# TM_SMTP_HOST=${TM_SMTP_HOST} +# TM_SMTP_PORT=${TM_SMTP_PORT:-25} +# TM_SMTP_USER=${TM_SMTP_USER} +# TM_SMTP_PASSWORD=${TM_SMTP_PASSWORD} # Following two variables can have value of either 0 or 1 -# TM_SMTP_USE_TLS=0 -# TM_SMTP_USE_SSL=1 +# TM_SMTP_USE_TLS=${TM_SMTP_USE_TLS:-0} +# TM_SMTP_USE_SSL=${TM_SMTP_USE_SSL:-1} # If disabled project update emails will not be sent. # Set it disabled in case of testing instances -TM_SEND_PROJECT_EMAIL_UPDATES = 1 +TM_SEND_PROJECT_EMAIL_UPDATES=${TM_SEND_PROJECT_EMAIL_UPDATES:-1} # TM_SERVICE_DESK # If the organisation has a service desk, configures the link @@ -175,41 +175,41 @@ TM_SEND_PROJECT_EMAIL_UPDATES = 1 # (e.g. ERROR, DEBUG, etc.) # If not specified DEBUG is default. ERROR is a good value for a live site. # -# TM_LOG_LEVEL=DEBUG -# TM_LOG_DIR=logs +# TM_LOG_LEVEL=${TM_LOG_LEVEL:-DEBUG} +# TM_LOG_DIR=${TM_LOG_DIR:-logs} # Languages settings for the Tasking Manager # -TM_DEFAULT_LOCALE=en +TM_DEFAULT_LOCALE=${TM_DEFAULT_LOCALE:-en} # By default all available languages are shown. You can restrict languages by modifying the following two variables. # Please note that there must be exactly the same number of codes as languages. # -# TM_SUPPORTED_LANGUAGES_CODES="ar, cs, de, el, en, es, fa_IR, fr, he, hu, id, it, ja, ko, mg, ml, nl_NL, pt, pt_BR, ru, sv, sw, tl, tr, uk, zh_TW" -# TM_SUPPORTED_LANGUAGES="عربى, Čeština, Deutsch, Ελληνικά, English, Español, فارسی, Français, עברית, Magyar, Indonesia, Italiano, 日本語, 한국어, Malagasy, Malayalam, Nederlands, Português, Português (Brasil), Русский язык, Svenska, Kiswahili, Filipino (Tagalog), Türkçe, Українська, 繁體中文" +# TM_SUPPORTED_LANGUAGES_CODES=${TM_SUPPORTED_LANGUAGES_CODES:-"ar, cs, de, el, en, es, fa_IR, fr, he, hu, id, it, ja, ko, mg, ml, nl_NL, pt, pt_BR, ru, sv, sw, tl, tr, uk, zh_TW"} +# TM_SUPPORTED_LANGUAGES=${TM_SUPPORTED_LANGUAGES:-"عربى, Čeština, Deutsch, Ελληνικά, English, Español, فارسی, Français, עברית, Magyar, Indonesia, Italiano, 日本語, 한국어, Malagasy, Malayalam, Nederlands, Português, Português (Brasil), Русский язык, Svenska, Kiswahili, Filipino (Tagalog), Türkçe, Українська, 繁體中文"} # Time to wait until task auto-unlock (optional) # (e.g. '2h' or '7d' or '30m' or '1h30m') # -# TM_TASK_AUTOUNLOCK_AFTER=2h +# TM_TASK_AUTOUNLOCK_AFTER=${TM_TASK_AUTOUNLOCK_AFTER:-2h} # Mapper Level values represent number of OSM changesets (optional) # -# TM_MAPPER_LEVEL_INTERMEDIATE=250 -# TM_MAPPER_LEVEL_ADVANCED=500 +# TM_MAPPER_LEVEL_INTERMEDIATE=${TM_MAPPER_LEVEL_INTERMEDIATE:-250} +# TM_MAPPER_LEVEL_ADVANCED=${TM_MAPPER_LEVEL_ADVANCED:-500} # This sets a file size limit to allow when importing a project geometry from a file. Define it in bytes. -# TM_IMPORT_MAX_FILESIZE=1000000 +# TM_IMPORT_MAX_FILESIZE=${TM_IMPORT_MAX_FILESIZE:-1000000} # Defines the maximum area allowed to the Projects' AoI. Default is 5000. The unit is square kilometers. -# TM_MAX_AOI_AREA=5000 +# TM_MAX_AOI_AREA=${TM_MAX_AOI_AREA:-5000} # Sentry.io DSN Config (optional) -# TM_SENTRY_BACKEND_DSN=https://foo.ingest.sentry.io/1234567 -# TM_SENTRY_FRONTEND_DSN=https://bar.ingest.sentry.io/8901234 +# TM_SENTRY_BACKEND_DSN=${TM_SENTRY_BACKEND_DSN:-https://foo.ingest.sentry.io/1234567} +# TM_SENTRY_FRONTEND_DSN=${TM_SENTRY_FRONTEND_DSN:-https://bar.ingest.sentry.io/8901234} # Underpass API URL (for project live monitoring feature) UNDERPASS_URL=https://underpass.hotosm.org #EXPORT TOOL Integration with 0(Disable) and 1(Enable) and S3 URL for Export Tool -#EXPORT_TOOL_S3_URL=https://foorawdataapi.s3.amazonaws.com -#ENABLE_EXPORT_TOOL=0 +#EXPORT_TOOL_S3_URL=${EXPORT_TOOL_S3_URL:-https://foorawdataapi.s3.amazonaws.com} +#ENABLE_EXPORT_TOOL=${ENABLE_EXPORT_TOOL:-0} From ef0b70fd6bcb2937bc9db0fdaee0d80a26aa06f6 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Fri, 30 Aug 2024 16:55:41 +0545 Subject: [PATCH 02/25] fix: workflow name --- .github/workflows/build_and_deploy.yml | 2 +- .github/workflows/frontend-build.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 326730e754..017e2931f7 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -26,7 +26,7 @@ jobs: secrets: inherit frontend-build: - uses: naxa-developers/tasking-manager/.github/workflows/frontend-build@ci/gh-workflows + uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci/gh-workflows secrets: inherit with: node-version: 16.x diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index af22e6be29..5e6864b83c 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -45,9 +45,9 @@ on: default: true outputs: - artifact_name: + artifact-name: description: "Node built artifact" - value: ${{ jobs.node-build.outputs.artifact_name }} + value: ${{ jobs.node-build.outputs.artifact-name }} jobs: node-build: @@ -57,7 +57,7 @@ jobs: name: ${{ github.ref_name }} outputs: - artifact_name: ${{ steps.get_artifact_name.outputs.artifact_name }} + artifact-name: ${{ steps.get_artifact_name.outputs.artifact_name }} permissions: contents: read From b978280f0917eb86cf031ca4e6728ff3f69ca47a Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Fri, 30 Aug 2024 17:14:48 +0545 Subject: [PATCH 03/25] change branch name for test --- .github/workflows/build_and_deploy.yml | 2 +- .github/workflows/frontend-build.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 017e2931f7..4402e6cad3 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -7,7 +7,7 @@ on: - main - staging - develop - - ci/gh-workflows + - ci-gh-workflows paths: # Workflow is triggered only if src changes - backend/** diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index 5e6864b83c..4df02121b1 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -88,6 +88,7 @@ jobs: - id: node_packages_install name: Install node pacakges + working-directory: ${{ inputs.context }} run: | case "${{ inputs.package-manager }}" in yarn) @@ -119,6 +120,7 @@ jobs: - id: build_frontend name: Build Frontend + working-directory: ${{ inputs.context }} run: | case "${{ inputs.package-manager }}" in yarn) @@ -135,7 +137,7 @@ jobs: if: ${{ inputs.upload-artifacts }} with: name: ${{ github.repository_id }}-${{ github.sha }}-frontend-dist - path: ./dist/* + path: ${{ inputs.context }}/dist/* retention-days: 1 - id: get_artifact_name From 20c713079c514eab206981289c66172880c80f11 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Wed, 4 Sep 2024 13:24:07 +0545 Subject: [PATCH 04/25] fix: path for .env --- .github/workflows/frontend-build.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index 4df02121b1..1d5a421ae2 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -104,18 +104,16 @@ jobs: env: VARS_CONTEXT: ${{ toJson(vars) }} SECRETS_CONTEXT: ${{ toJson(secrets) }} + shell: bash run: | - # Parse JSON with multiline strings to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; } - # Create .env file from VARS_CONTEXT if [ "${VARS_CONTEXT}" != "null" ]; then - echo "${VARS_CONTEXT}" | to_envs > ${{ inputs.context }}/.env + echo "${VARS_CONTEXT}" | to_envs > ${{ inputs.context }}/.env fi - # Append Secrets to .env file from SECRETS_CONTEXT if [ "${SECRETS_CONTEXT}" != "null" ]; then - echo "\n${SECRETS_CONTEXT}" | to_envs >> ${{ inputs.context }}/.env + echo "\n${SECRETS_CONTEXT}" | to_envs >> ${{ inputs.context }}/.env fi - id: build_frontend From 6fb8ee33d3439013eacfa329f4fc3665d99664e3 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Wed, 4 Sep 2024 13:26:08 +0545 Subject: [PATCH 05/25] fix: frontend build branch --- .github/workflows/build_and_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 4402e6cad3..6bbba0730e 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -26,7 +26,7 @@ jobs: secrets: inherit frontend-build: - uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci/gh-workflows + uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows secrets: inherit with: node-version: 16.x From 30629a60598c0dd8b4e76eecb06681afe1740ad4 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Wed, 4 Sep 2024 15:45:30 +0545 Subject: [PATCH 06/25] try vars_sec to .env file --- .github/workflows/build_and_deploy.yml | 16 ++++++++-------- .github/workflows/frontend-build.yml | 12 +++--------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 6bbba0730e..e245ab0678 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -16,14 +16,14 @@ on: workflow_dispatch: jobs: - backend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 - with: - context: . - build_target: prod - image_name: ghcr.io/${{ github.repository }}/backend - dockerfile: Dockerfile - secrets: inherit + # backend-build: + # uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 + # with: + # context: . + # build_target: prod + # image_name: ghcr.io/${{ github.repository }}/backend + # dockerfile: Dockerfile + # secrets: inherit frontend-build: uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index 1d5a421ae2..8b6c8418dc 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -106,15 +106,9 @@ jobs: SECRETS_CONTEXT: ${{ toJson(secrets) }} shell: bash run: | - to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; } - - if [ "${VARS_CONTEXT}" != "null" ]; then - echo "${VARS_CONTEXT}" | to_envs > ${{ inputs.context }}/.env - fi - - if [ "${SECRETS_CONTEXT}" != "null" ]; then - echo "\n${SECRETS_CONTEXT}" | to_envs >> ${{ inputs.context }}/.env - fi + parsed_vars=$(jq -n --argjson VARS_CONTEXT "$VARS_CONTEXT" --argjson SECRETS_CONTEXT "$SECRETS_CONTEXT" "$VARS_CONTEXT+$SECRETS_CONTEXT") + to_envs() { jq -r "to_entries[] | \"\(.key)=\\\"\(.value)\\\"\n\""; } + echo "$parsed_vars" | to_envs > ${{ inputs.context }}/.env - id: build_frontend name: Build Frontend From 65f0c215d3d7282a0d936556fbbd76ab59e72436 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 11:11:17 +0545 Subject: [PATCH 07/25] CI: try build and verify --- .github/workflows/build_and_deploy.yml | 56 ++++++++++----- .github/workflows/frontend-build.yml | 13 ++-- .github/workflows/remote_deploy.yml | 99 ++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/remote_deploy.yml diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index e245ab0678..8638cf9f7d 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -16,14 +16,15 @@ on: workflow_dispatch: jobs: - # backend-build: - # uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 - # with: - # context: . - # build_target: prod - # image_name: ghcr.io/${{ github.repository }}/backend - # dockerfile: Dockerfile - # secrets: inherit + backend-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 + with: + context: . + build_target: prod + image_name: ghcr.io/${{ github.repository }}/backend + dockerfile: Dockerfile + scan_image: false + secrets: inherit frontend-build: uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows @@ -33,14 +34,33 @@ jobs: context: ./frontend cache-key-file: ./frontend/yarn.lock package-manager: yarn + build-dist-folder-path: ./frontend/build - # deploy_to_vm: - # name: Deploy to VM - # needs: - # - frontend-build - # - backend-build - # uses: hotosm/gh-workflows/.github/workflows/remote_deploy_compose.yml@2.0.5 - # with: - # docker_compose_file: docker-compose.vm.yml - # environment: ${{ github.ref_name }} - # secrets: inherit \ No newline at end of file + frontend-deploy: + runs-on: ubuntu-latest + needs: + - frontend-build + name: Deploy Frontend Static Files + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ needs.frontend-build.outputs.artifact-name }} + + - name: Debug check files + run: | + ls -alh + ls -alh build + + deploy_to_vm: + name: Deploy to VM + needs: + - frontend-build + - backend-build + uses: naxa-developers/tasking-manager/.github/workflows/remote_deploy_compose.yml@ci-gh-workflows + with: + docker_compose_file: docker-compose.vm.yml + environment: ${{ github.ref_name }} + example_env_file_path: example.env + env_file_path: tasking-manager.env + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index 8b6c8418dc..65f96b251d 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -43,7 +43,12 @@ on: required: false type: boolean default: true - + build-dist-folder-path: + description: "Path to folder that stores build files" + required: false + type: string + default: "${{ inputs.context }}/dist" + outputs: artifact-name: description: "Node built artifact" @@ -124,12 +129,12 @@ jobs: esac - id: upload_build_artifacts - name: Upload dist folder as build artifacts + name: Upload build files as build artifacts uses: actions/upload-artifact@v4 if: ${{ inputs.upload-artifacts }} with: - name: ${{ github.repository_id }}-${{ github.sha }}-frontend-dist - path: ${{ inputs.context }}/dist/* + name: ${{ github.repository_id }}-${{ github.sha }}-frontend-build-dist + path: ${{ inputs.build-dist-folder-path }} retention-days: 1 - id: get_artifact_name diff --git a/.github/workflows/remote_deploy.yml b/.github/workflows/remote_deploy.yml new file mode 100644 index 0000000000..06eded1269 --- /dev/null +++ b/.github/workflows/remote_deploy.yml @@ -0,0 +1,99 @@ +# Note: variables: SSH_HOST and SSH_USER must be set for your environment. +# Note: secrets: SSH_PRIVATE_KEY must be set for your environment. + +name: Remote Deploy (Compose) + +on: + workflow_call: + inputs: + environment: + description: "The Github environment to get variables from. Default repository vars." + required: false + type: string + docker_compose_file: + description: "Path to docker compose file to deploy." + required: true + type: string + example_env_file_path: + description: "Path to example dotenv file to substitute variables for." + type: string + default: .env.example + env_file_path: + description: "Path to write dotenv file" + type: string + default: .env + +jobs: + remote-deploy: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Vars and Secrets to Env + env: + GIT_BRANCH: ${{ github.ref_name }} + VARS_CONTEXT: ${{ toJson(vars) }} + SECRETS_CONTEXT: ${{ toJson(secrets) }} + run: | + # Random delimeter string for security + delim=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + + # Parse JSON with multiline strings, using delimeter (Github specific) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$delim\n\(.value)\n$delim\n\""; } + + # Set vars to env for next step + echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV + echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> $GITHUB_ENV + + # Set VARS_CONTEXT if not null + if [ "${VARS_CONTEXT}" != "null" ]; then + echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV + fi + + # Set SECRETS_CONTEXT if not null + if [ "${SECRETS_CONTEXT}" != "null" ]; then + echo "${SECRETS_CONTEXT}" | to_envs >> $GITHUB_ENV + fi + + - name: Create .env file + env: + EXAMPLE_DOTENV: ${{ inputs.example_env_file_path }} + run: | + echo "Checking if ${EXAMPLE_DOTENV} exists" + if [ -f ${EXAMPLE_DOTENV} ]; then + # Get a8m/envsubst (required for default vals syntax ${VAR:-default}) + echo "Downloading envsubst" + curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` -o envsubst + if [ $? -ne 0 ]; then + echo "Failed to download envsubst" + exit 1 + fi + chmod +x envsubst + echo "Substituting variables from ${EXAMPLE_DOTENV} --> ${{ inputs.env_file_path }}" + ./envsubst < "${EXAMPLE_DOTENV}" > ${{ inputs.env_file_path }} + else + echo "${EXAMPLE_DOTENV} not found, creating empty ${{ inputs.env_file_path }}" + touch ${{ inputs.env_file_path }} + fi + + echo "GIT_BRANCH=${GIT_BRANCH}" >> ${{ inputs.env_file_path }} + echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> ${{ inputs.env_file_path }} + + - uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: "${{ secrets.SSH_PRIVATE_KEY }}" + + - name: Add host keys to known_hosts + run: | + ssh-keyscan "${{ vars.SSH_HOST }}" >> ~/.ssh/known_hosts + + - name: Deploy + run: | + docker compose --file ${{ inputs.docker_compose_file }} pull + docker compose --file ${{ inputs.docker_compose_file }} up \ + --detach --remove-orphans --force-recreate + env: + DOCKER_HOST: "ssh://${{ vars.SSH_USER }}@${{ vars.SSH_HOST }}" \ No newline at end of file From fb4134303b270d6de16dfdf98def156a03f45ae0 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 11:14:31 +0545 Subject: [PATCH 08/25] fix: file name for remote deploy --- .../workflows/{remote_deploy.yml => remote_deploy_compose.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{remote_deploy.yml => remote_deploy_compose.yml} (100%) diff --git a/.github/workflows/remote_deploy.yml b/.github/workflows/remote_deploy_compose.yml similarity index 100% rename from .github/workflows/remote_deploy.yml rename to .github/workflows/remote_deploy_compose.yml From 730c07f498f63b49e0a649abd18073b920ec91bd Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 11:20:52 +0545 Subject: [PATCH 09/25] ci: fix frontend build dist name --- .github/workflows/frontend-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index 65f96b251d..80cb3a62b3 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -140,6 +140,6 @@ jobs: - id: get_artifact_name name: Get First Image Name run: | - echo "artifact_name=${{ github.repository_id }}-${{ github.sha }}-frontend-dist" >> $GITHUB_OUTPUT + echo "artifact_name=${{ github.repository_id }}-${{ github.sha }}-frontend-build-dist" >> $GITHUB_OUTPUT echo "Frontend Artifact Name: $artifact_name" From 80bba5e3413ba6534eca1d67dc6193133772e6b0 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 11:29:21 +0545 Subject: [PATCH 10/25] ci: download artifacts to folder --- .github/workflows/build_and_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 8638cf9f7d..6c81c67862 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -46,6 +46,7 @@ jobs: uses: actions/download-artifact@v4 with: name: ${{ needs.frontend-build.outputs.artifact-name }} + path: ./build - name: Debug check files run: | From 8fa5524f61e6d67cfb246e6001a56353bb6d9aca Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 13:41:50 +0545 Subject: [PATCH 11/25] ci: add test --- .github/workflows/build_and_deploy.yml | 35 ++- .github/workflows/frontend-build.yml | 9 +- .github/workflows/frontend-test.yml | 136 ++++++++++ .github/workflows/test_compose.yml | 334 +++++++++++++++++++++++++ 4 files changed, 511 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/frontend-test.yml create mode 100644 .github/workflows/test_compose.yml diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 6c81c67862..1a3d5d9e60 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -16,8 +16,26 @@ on: workflow_dispatch: jobs: + backend-test: + uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@2.0.5 + with: + image_name: ghcr.io/${{ github.repository }}/backend + pre_command: docker compose up -d traefik + compose_service: tm-backend + build_target: prod + compose_command: | + pip install flake8 + flake8 manage.py backend tests migrations + pip install 'black==23.12.1' + black --check manage.py backend tests migrations + tag_override: ci-${{ github.ref_name }} + coverage: true + secrets: inherit + backend-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 + needs: + - backend-test with: context: . build_target: prod @@ -26,9 +44,24 @@ jobs: scan_image: false secrets: inherit + frontend-test: + uses: naxa-developers/tasking-manager/.github/workflows/frontend-test.yml@ci-gh-workflows + secrets: inherit + with: + node-version: 16.x + context: ./frontend + cache-key-file: ./frontend/yarn.lock + package-manager: yarn + test_frontend_build: false + test_frontend_command: | + CI=true yarn test -w 1 + CI=true GENERATE_SOURCEMAP=false yarn build + frontend-build: uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows secrets: inherit + needs: + - frontend-test with: node-version: 16.x context: ./frontend @@ -60,7 +93,7 @@ jobs: - backend-build uses: naxa-developers/tasking-manager/.github/workflows/remote_deploy_compose.yml@ci-gh-workflows with: - docker_compose_file: docker-compose.vm.yml + docker_compose_file: docker-compose.yml environment: ${{ github.ref_name }} example_env_file_path: example.env env_file_path: tasking-manager.env diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index 80cb3a62b3..8b4241d4be 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -1,8 +1,13 @@ -name: Node build +name: Node Frontend build on: workflow_call: inputs: + runner-class: + description: "Github Runner class to use" + required: false + type: string + default: "ubuntu-latest" node-version: description: "Node version to use." required: false @@ -56,7 +61,7 @@ on: jobs: node-build: - runs-on: ubuntu-latest + runs-on: ${{ inputs.runner-class }} environment: name: ${{ github.ref_name }} diff --git a/.github/workflows/frontend-test.yml b/.github/workflows/frontend-test.yml new file mode 100644 index 0000000000..73e604b1c1 --- /dev/null +++ b/.github/workflows/frontend-test.yml @@ -0,0 +1,136 @@ +name: Frontend Test + +on: + workflow_call: + inputs: + runner-class: + description: "Github Runner class to use" + required: false + type: string + default: "ubuntu-latest" + node-version: + description: "Node version to use." + required: false + type: string + default: "18.x" + node-version-file: + description: "Node version file to use. node-version overrides this parameter." + required: false + type: string + default: "" + context: + description: "Root directory to start the build from." + required: false + type: string + default: "." + cache: + description: "Use node modules installation caching. Default true." + required: false + type: boolean + default: true + cache-key-file: + description: "Key file for cache." + required: false + type: string + default: "${{ inputs.context }}/package.json" + package-manager: + description: "Package manager to use. Supports [npm, yarn]" + required: false + type: string + default: "npm" + build-script-name: + description: "Build script name in package.json" + required: false + type: string + default: "build" + test_frontend_command: + description: "Testing command to run" + required: true + type: string + test_frontend_build: + description: "Perform a build test" + required: false + type: boolean + default: true + +jobs: + node-build: + runs-on: ${{ inputs.runner-class }} + + environment: + name: ${{ github.ref_name }} + + outputs: + artifact-name: ${{ steps.get_artifact_name.outputs.artifact_name }} + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - id: node_setup + name: Install node + uses: actions/setup-node@v4 + #reference: https://github.com/actions/setup-node + with: + node-version: ${{ inputs.node-version }} + node-version-file: ${{ inputs.node-version-file && inputs.node-version-file || '' }} + + - name: Cache Node packages + uses: actions/cache@v3 + env: + cache_name: node-${{ inputs.node-version || hashFiles(inputs.node-version-file) }}-${{ inputs.package-manager }}-${{ hashFiles(inputs.cache-key-file) }} + with: + key: ${{ runner.os }}-build-${{ env.cache_name }} + path: | + ~/.npm + restore-keys: | + ${{ runner.os }}-build-${{ env.cache_name }} + + - id: node_packages_install + name: Install node pacakges + working-directory: ${{ inputs.context }} + run: | + case "${{ inputs.package-manager }}" in + yarn) + yarn + ;; + npm) + npm i + ;; + esac + + - id: vars_and_secrets + name: Vars and Secrets to Env file + env: + VARS_CONTEXT: ${{ toJson(vars) }} + SECRETS_CONTEXT: ${{ toJson(secrets) }} + shell: bash + run: | + parsed_vars=$(jq -n --argjson VARS_CONTEXT "$VARS_CONTEXT" --argjson SECRETS_CONTEXT "$SECRETS_CONTEXT" "$VARS_CONTEXT+$SECRETS_CONTEXT") + to_envs() { jq -r "to_entries[] | \"\(.key)=\\\"\(.value)\\\"\n\""; } + echo "$parsed_vars" | to_envs > ${{ inputs.context }}/.env + + - id: test_frontend + name: Test Frontend + working-directory: ${{ inputs.context }} + run: | + ${{ inputs.test_frontend_command }} + + - id: build_frontend + name: Build Frontend + if: ${{ inputs.test_frontend_build }} + working-directory: ${{ inputs.context }} + run: | + case "${{ inputs.package-manager }}" in + yarn) + yarn ${{ inputs.build-script-name }} + ;; + npm) + npm run ${{ inputs.build-script-name }} + ;; + esac + diff --git a/.github/workflows/test_compose.yml b/.github/workflows/test_compose.yml new file mode 100644 index 0000000000..a4344b6d0d --- /dev/null +++ b/.github/workflows/test_compose.yml @@ -0,0 +1,334 @@ +# A generic workflow to run tests within a docker compose stack + +name: Run Tests + +on: + workflow_call: + inputs: + image_name: + description: "The image root name to build, without tag. E.g. 'ghcr.io/[dollar]{{ github.repository }}'" + required: false + type: string + default: "" + build_context: + description: "Root directory to start the build from." + required: false + type: string + default: "." + build_dockerfile: + description: "Name of dockerfile, relative to context dir." + required: false + type: string + default: "Dockerfile" + build_target: + description: "The target to built to (default to ci stage)." + required: false + type: string + default: "ci" + extra_build_args: + description: "Space separated list of build args to use for the image." + required: false + type: string + tag_override: + description: "An override for the build image tag. Must include tests and have test software installed" + required: false + type: string + cache_image: + description: "Cache the built image, for the next run. Default true." + required: false + type: boolean + default: true + cache_extra_imgs: + description: "Space separated list of images to cache on each run (e.g. to avoid rate limiting)." + required: false + type: string + compose_file: + description: "The docker compose file used to run the test." + required: false + type: string + default: docker-compose.yml + # compose_entrypoint: + # description: "Override the default entrypoint for the compose service." + # required: false + # type: string + compose_service: + description: "The docker compose service to run the test against." + required: true + type: string + compose_command: + description: "The command to run for the container. Default to built-in image command." + required: false + type: string + pre_command: + description: "A initialisation command to run prior to the docker compose command." + required: false + type: string + example_env_file_path: + description: "Path to example dotenv file to substitute variables for." + type: string + default: .env.example + coverage: + description: "Generate a coverage HTML report (requires coverage.py installed)." + required: false + type: boolean + default: false + playwright: + description: "Upload the Playwright trace files as an artifact for debugging." + required: false + type: boolean + default: false + environment: + description: "The environment to use for testing." + required: false + type: string + default: "test" + +permissions: + contents: write + packages: write + +jobs: + # Get cached images from previous runs (when running multiple times) + check-img-cache: + runs-on: ubuntu-latest + if: ${{ inputs.image_name != '' && inputs.cache_image }} + environment: + name: ${{ inputs.environment }} + outputs: + cache-hit: ${{ steps.image-cache.outputs.cache-hit }} + steps: + - id: image-cache + uses: actions/cache@v4 + with: + path: /tmp/images + key: image-cache-${{ runner.os }} + + # Build the test image if required (unless `inputs.image_name` is set) + test-img-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@1.6.0 + needs: [check-img-cache] + # Only build if no cached image is found + if: ${{ inputs.image_name != '' && needs.check-img-cache.outputs.cache-hit != 'true' }} + with: + context: ${{ inputs.build_context }} + dockerfile: ${{ inputs.build_dockerfile }} + build_target: ${{ inputs.build_target }} + image_name: ${{ inputs.image_name }} + # Default to auto-tag, unless override specified + image_tags: ${{ inputs.tag_override && format('{0}:{1}', inputs.image_name, inputs.tag_override) || '' }} + extra_build_args: ${{ inputs.extra_build_args }} + # Build temp image for PR, no multi-arch, scan, or ghcr cache required + scan_image: false + scan_dockerfile: false + cache: false + + run-tests: + runs-on: ubuntu-latest + needs: [test-img-build] + # Ensure it runs, even if test-img-build does not run + if: always() + environment: + name: ${{ inputs.environment }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - id: image-cache + if: ${{ inputs.cache_extra_imgs || inputs.cache_image }} + uses: actions/cache@v4 + with: + path: /tmp/images + key: image-cache-${{ runner.os }} + + # Cache the newly built image, plus extra images used in the compose file + - name: Save Local Images + if: ${{ (inputs.cache_extra_imgs || inputs.cache_image) && steps.image-cache.outputs.cache-hit != 'true' }} + run: | + # Function to pull and package Docker image + function pull_and_package_image() { + local image=$1 + echo "Processing image ${image}" + docker pull "${image}" + + if [ $? -eq 0 ]; then + img_underscores=${image//[:\/.]/_} + echo "Packaging image ${image} to /tmp/images/${img_underscores}.tar" + docker image save "${image}" --output "/tmp/images/${img_underscores}.tar" + else + echo "Failed to pull the image: ${image}" + fi + } + + # Make artifact dir + mkdir -p /tmp/images + + # Cache main image build + CACHE_ENABLED=${{ inputs.cache_image }} + echo "Caching enabled: ${CACHE_ENABLED}" + if [[ "${CACHE_ENABLED}" == true ]]; then + image_name=${{ needs.test-img-build.outputs.image_name }} + pull_and_package_image "${image_name}" + + # Export tag to env + image_tag=${{ needs.test-img-build.outputs.image_tag }} + echo "Setting TAG_OVERRIDE=${image_tag} in github env" + echo "TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV + fi + + # Cache extra images + extra_images_array=(${{ inputs.cache_extra_imgs }}) + if [[ -n "${extra_images_array[@]}" ]]; then + echo "Images to cache: ${extra_images_array[@]}" + + # Iterate through dependency images + for image_name in "${extra_images_array[@]}"; do + pull_and_package_image "${image_name}" + done + fi + + # Load the cached image .tar files via `docker image load` + - name: Load Cached Imgs + if: ${{ (inputs.cache_extra_imgs || inputs.cache_image) && steps.image-cache.outputs.cache-hit == 'true' }} + run: | + # Load images + for image_tar in /tmp/images/*; do + docker image load --input $image_tar || true + done + + # Set TAG_OVERRIDE if image was cached + CACHE_ENABLED=${{ inputs.cache_image }} + if [[ "${CACHE_ENABLED}" == true ]]; then + echo "Getting image tag via docker" + image=$(\ + docker image ls \ + --filter=reference='${{ inputs.image_name }}' \ + --format='{{.Tag}}'\ + ) + image_tag="${image##*:}" + echo "Image Tag: $image_tag" + echo "TAG_OVERRIDE=${image_tag}" >> $GITHUB_ENV + fi + + # Extract repo vars/secrets into environment variables for the next step + # NOTE it may look like secrets are exposed, but they are hidden in the GUI + - name: Vars and Secrets to Env + env: + TAG_OVERRIDE: ${{ env.TAG_OVERRIDE || 'ci-development' }} + GIT_BRANCH: ${{ github.ref_name }} + VARS_CONTEXT: ${{ toJson(vars) }} + SECRETS_CONTEXT: ${{ toJson(secrets) }} + run: | + # Random delimeter string for security + delim=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + + # Parse JSON with multiline strings, using delimeter (Github specific) + to_envs() { jq -r "to_entries[] | \"\(.key)<<$delim\n\(.value)\n$delim\n\""; } + + # Set vars to env for next step + echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV + echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> $GITHUB_ENV + + # Set VARS_CONTEXT if not null + if [ "${VARS_CONTEXT}" != "null" ]; then + echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV + fi + + # Set SECRETS_CONTEXT if not null + if [ "${SECRETS_CONTEXT}" != "null" ]; then + echo "${SECRETS_CONTEXT}" | to_envs >> $GITHUB_ENV + fi + + # From generated environment variables, create a `.env` file in repo root + # If a .env.example exists, the variables are substituted in from Github env + - name: Create .env file + env: + EXAMPLE_DOTENV: ${{ inputs.example_env_file_path }} + run: | + echo "Checking if ${EXAMPLE_DOTENV} exists" + if [ -f ${EXAMPLE_DOTENV} ]; then + # Get a8m/envsubst (required for default vals syntax ${VAR:-default}) + echo "Downloading envsubst" + curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-`uname -s`-`uname -m` -o envsubst + if [ $? -ne 0 ]; then + echo "Failed to download envsubst" + exit 1 + fi + chmod +x envsubst + echo "Substituting variables from ${EXAMPLE_DOTENV} --> .env" + ./envsubst < "${EXAMPLE_DOTENV}" > .env + else + echo "${EXAMPLE_DOTENV} not found, creating empty .env" + touch .env + fi + + echo "GIT_BRANCH=${GIT_BRANCH}" >> .env + echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> .env + + # Run the tests via a pre-configured docker compose service + # For --no-TTY explanation, see: https://github.com/actions/runner-images/issues/5022 + - name: Run Tests + if: ${{ ! inputs.coverage && ! inputs.playwright }} + run: | + ${{ inputs.pre_command }} + + docker compose --file ${{ inputs.compose_file }} \ + run --no-TTY \ + ${{ inputs.compose_service }} ${{ inputs.compose_command }} + + # Specific for generating coverage + - name: Run Tests With Coverage + if: ${{ inputs.coverage }} + run: | + ${{ inputs.pre_command }} + + docker compose --file ${{ inputs.compose_file }} \ + run --no-TTY --entrypoint "sh -c" \ + --volume ${{ github.workspace }}/coverage:/tmp/coverage \ + ${{ inputs.compose_service }} \ + "coverage run -m ${{ inputs.compose_command }} \ + && coverage report && coverage html \ + && coverage-badge -o coverage.svg \ + && mv htmlcov/index.html /tmp/coverage/coverage.html \ + && mv coverage.svg /tmp/coverage/coverage.svg" + + - name: Upload Coverage + if: ${{ inputs.coverage }} + run: | + # Checkout to gh-pages + mkdir tmp_pages + cd tmp_pages + git init --initial-branch=gh-pages + git config user.name svchot + git config user.email sysadmin@hotosm.org + git pull https://x-access-token:$TOKEN@github.com/${{ github.repository }}.git gh-pages + + # Overwrite coverage index and badge + echo "Coverage dir contents:" + ls ${{ github.workspace }}/coverage + echo "" + cp ${{ github.workspace }}/coverage/* . + + # Push content to gh-pages + git add . + git commit -m "docs: update coverage summary and badge" || true + git push --set-upstream https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:gh-pages + + # Specific for generating Playwright traces + # - name: Playwright Tests + # if: ${{ inputs.playwright }} + # run: | + # ${{ inputs.pre_command }} + + # docker compose --file ${{ inputs.compose_file }} run \ + # --no-TTY \ + # --volume ${{ github.workspace }}/playwright-report:/app/playwright-report \ + # ${{ inputs.compose_service }} ${{ inputs.compose_command }} + + # - name: Upload Playwright Report + # if: ${{ inputs.playwright && !cancelled() }} + # uses: actions/upload-artifact@v4 + # with: + # path: ${{ github.workspace }}/playwright-report/ + # name: playwright-report + # retention-days: 30 \ No newline at end of file From 9de78b37a2f85424f0cf67f1b561ab5f83ec8ae9 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 13:48:45 +0545 Subject: [PATCH 12/25] CI: use naxa-developers workflow --- .github/workflows/build_and_deploy.yml | 4 +++- .github/workflows/test_compose.yml | 17 +++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 1a3d5d9e60..691fe07229 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -17,7 +17,7 @@ on: jobs: backend-test: - uses: hotosm/gh-workflows/.github/workflows/test_compose.yml@2.0.5 + uses: naxa-developers/tasking-manager/.github/workflows/test_compose.yml@ci-gh-workflows with: image_name: ghcr.io/${{ github.repository }}/backend pre_command: docker compose up -d traefik @@ -30,6 +30,8 @@ jobs: black --check manage.py backend tests migrations tag_override: ci-${{ github.ref_name }} coverage: true + example_env_file_path: example.env + env_file_path: tasking-manager.env secrets: inherit backend-build: diff --git a/.github/workflows/test_compose.yml b/.github/workflows/test_compose.yml index a4344b6d0d..e6cacef673 100644 --- a/.github/workflows/test_compose.yml +++ b/.github/workflows/test_compose.yml @@ -82,6 +82,11 @@ on: required: false type: string default: "test" + env_file_path: + description: "Path to write environment variables to" + required: false + type: string + default: ".env" permissions: contents: write @@ -255,15 +260,15 @@ jobs: exit 1 fi chmod +x envsubst - echo "Substituting variables from ${EXAMPLE_DOTENV} --> .env" - ./envsubst < "${EXAMPLE_DOTENV}" > .env + echo "Substituting variables from ${EXAMPLE_DOTENV} --> ${{ inputs.env_file_path }}" + ./envsubst < "${EXAMPLE_DOTENV}" > ${{ inputs.env_file_path }} else - echo "${EXAMPLE_DOTENV} not found, creating empty .env" - touch .env + echo "${EXAMPLE_DOTENV} not found, creating empty ${{ inputs.env_file_path }}" + touch ${{ inputs.env_file_path }} fi - echo "GIT_BRANCH=${GIT_BRANCH}" >> .env - echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> .env + echo "GIT_BRANCH=${GIT_BRANCH}" >> ${{ inputs.env_file_path }} + echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> ${{ inputs.env_file_path }} # Run the tests via a pre-configured docker compose service # For --no-TTY explanation, see: https://github.com/actions/runner-images/issues/5022 From 7c55c8291e31d1ea5f3788ab02a25c49f2dbe1ef Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 14:07:07 +0545 Subject: [PATCH 13/25] ci: disable coverage --- .github/workflows/build_and_deploy.yml | 4 +-- .github/workflows/test_compose.yml | 34 +++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 691fe07229..f86aca3976 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -20,7 +20,7 @@ jobs: uses: naxa-developers/tasking-manager/.github/workflows/test_compose.yml@ci-gh-workflows with: image_name: ghcr.io/${{ github.repository }}/backend - pre_command: docker compose up -d traefik + pre_command: docker compose up -d compose_service: tm-backend build_target: prod compose_command: | @@ -29,7 +29,7 @@ jobs: pip install 'black==23.12.1' black --check manage.py backend tests migrations tag_override: ci-${{ github.ref_name }} - coverage: true + # coverage: true example_env_file_path: example.env env_file_path: tasking-manager.env secrets: inherit diff --git a/.github/workflows/test_compose.yml b/.github/workflows/test_compose.yml index e6cacef673..0a1e5f19d9 100644 --- a/.github/workflows/test_compose.yml +++ b/.github/workflows/test_compose.yml @@ -320,20 +320,20 @@ jobs: git push --set-upstream https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:gh-pages # Specific for generating Playwright traces - # - name: Playwright Tests - # if: ${{ inputs.playwright }} - # run: | - # ${{ inputs.pre_command }} - - # docker compose --file ${{ inputs.compose_file }} run \ - # --no-TTY \ - # --volume ${{ github.workspace }}/playwright-report:/app/playwright-report \ - # ${{ inputs.compose_service }} ${{ inputs.compose_command }} - - # - name: Upload Playwright Report - # if: ${{ inputs.playwright && !cancelled() }} - # uses: actions/upload-artifact@v4 - # with: - # path: ${{ github.workspace }}/playwright-report/ - # name: playwright-report - # retention-days: 30 \ No newline at end of file + - name: Playwright Tests + if: ${{ inputs.playwright }} + run: | + ${{ inputs.pre_command }} + + docker compose --file ${{ inputs.compose_file }} run \ + --no-TTY \ + --volume ${{ github.workspace }}/playwright-report:/app/playwright-report \ + ${{ inputs.compose_service }} ${{ inputs.compose_command }} + + - name: Upload Playwright Report + if: ${{ inputs.playwright && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + path: ${{ github.workspace }}/playwright-report/ + name: playwright-report + retention-days: 30 \ No newline at end of file From f9c6d1e8aa71e8aabff14791ba4325ac937e10de Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 14:24:34 +0545 Subject: [PATCH 14/25] fix: conflicts from upstream rebase --- .github/workflows/build_and_deploy.yml | 5 ++--- docker-compose.yml | 10 +++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index f86aca3976..60c33755ba 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -88,10 +88,9 @@ jobs: ls -alh ls -alh build - deploy_to_vm: - name: Deploy to VM + backend_deploy_to_vm: + name: Deploy Backend to VM needs: - - frontend-build - backend-build uses: naxa-developers/tasking-manager/.github/workflows/remote_deploy_compose.yml@ci-gh-workflows with: diff --git a/docker-compose.yml b/docker-compose.yml index 3818c9b7ef..9fe87583cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,11 +2,11 @@ name: tasking-manager-main volumes: tm_db_data: - name: tm-db-data-main + name: tm-db-data-${GIT_BRANCH:-main} networks: - tm-net: - name: tm-net + tm-net: + name: tm-net-${GIT_BRANCH:-main} services: tm-db: @@ -26,7 +26,7 @@ services: - tm-net tm-backend: - image: ghcr.io/hotosm/tasking-manager/backend:main + image: ghcr.io/hotosm/tasking-manager/backend:${GIT_BRANCH:-main} build: context: . target: ${TARGET_TAG:-prod} @@ -60,7 +60,7 @@ services: - tm-net tm-migration: - image: ghcr.io/hotosm/tasking-manager/backend:main + image: ghcr.io/hotosm/tasking-manager/backend:${GIT_BRANCH:-main} build: context: . entrypoint: ["alembic", "-c", "migrations/alembic.ini", "upgrade", "head"] From f8172236491040f6a990e68f7e6fa244b6655677 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 14:30:21 +0545 Subject: [PATCH 15/25] ci: disable build on frotnend test --- .github/workflows/build_and_deploy.yml | 5 +++-- .github/workflows/frontend-test.yml | 23 ++++++++--------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 60c33755ba..77b9407c60 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -54,10 +54,11 @@ jobs: context: ./frontend cache-key-file: ./frontend/yarn.lock package-manager: yarn - test_frontend_build: false test_frontend_command: | CI=true yarn test -w 1 - CI=true GENERATE_SOURCEMAP=false yarn build + # test_frontend_build: false + # build_test_frontend_command: | + # CI=true GENERATE_SOURCEMAP=false yarn build frontend-build: uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows diff --git a/.github/workflows/frontend-test.yml b/.github/workflows/frontend-test.yml index 73e604b1c1..449279a0d5 100644 --- a/.github/workflows/frontend-test.yml +++ b/.github/workflows/frontend-test.yml @@ -38,11 +38,6 @@ on: required: false type: string default: "npm" - build-script-name: - description: "Build script name in package.json" - required: false - type: string - default: "build" test_frontend_command: description: "Testing command to run" required: true @@ -52,6 +47,11 @@ on: required: false type: boolean default: true + build_test_frontend_command: + description: "Command to do build test on frontend" + required: false + type: string + jobs: node-build: @@ -120,17 +120,10 @@ jobs: run: | ${{ inputs.test_frontend_command }} - - id: build_frontend + - id: test_frontend_build name: Build Frontend - if: ${{ inputs.test_frontend_build }} + if: ${{ inputs.test_frontend_build && inputs.build_test_frontend_command }} working-directory: ${{ inputs.context }} run: | - case "${{ inputs.package-manager }}" in - yarn) - yarn ${{ inputs.build-script-name }} - ;; - npm) - npm run ${{ inputs.build-script-name }} - ;; - esac + ${{ inputs.build_test_frontend_command }} From bd1f50c4405e3ca71a5c421084352180dd0fb79f Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 16:06:10 +0545 Subject: [PATCH 16/25] ci: skil test --- .github/workflows/backend_functional_test.yml | 150 ++++++++++++++++++ .github/workflows/build_and_deploy.yml | 11 +- example.env | 4 +- 3 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/backend_functional_test.yml diff --git a/.github/workflows/backend_functional_test.yml b/.github/workflows/backend_functional_test.yml new file mode 100644 index 0000000000..27982ddb1b --- /dev/null +++ b/.github/workflows/backend_functional_test.yml @@ -0,0 +1,150 @@ +name: Backend Functional Test + +on: + workflow_call: + inputs: + runner-class: + description: "Github Runner class to use" + required: false + type: string + default: "ubuntu-latest" + node-version: + description: "Node version to use." + required: false + type: string + default: "18.x" + node-version-file: + description: "Node version file to use. node-version overrides this parameter." + required: false + type: string + default: "" + context: + description: "Root directory to start the build from." + required: false + type: string + default: "." + cache: + description: "Use node modules installation caching. Default true." + required: false + type: boolean + default: true + cache-key-file: + description: "Key file for cache." + required: false + type: string + default: "${{ inputs.context }}/package.json" + package-manager: + description: "Package manager to use. Supports [npm, yarn]" + required: false + type: string + default: "npm" + build-script-name: + description: "Build script name in package.json" + required: false + type: string + default: "build" + upload-artifacts: + description: "Upload artifacts to Github" + required: false + type: boolean + default: true + build-dist-folder-path: + description: "Path to folder that stores build files" + required: false + type: string + default: "${{ inputs.context }}/dist" + + outputs: + artifact-name: + description: "Node built artifact" + value: ${{ jobs.node-build.outputs.artifact-name }} + +jobs: + node-build: + runs-on: ${{ inputs.runner-class }} + + environment: + name: ${{ github.ref_name }} + + outputs: + artifact-name: ${{ steps.get_artifact_name.outputs.artifact_name }} + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - id: node_setup + name: Install node + uses: actions/setup-node@v4 + #reference: https://github.com/actions/setup-node + with: + node-version: ${{ inputs.node-version }} + node-version-file: ${{ inputs.node-version-file && inputs.node-version-file || '' }} + + - name: Cache Node packages + uses: actions/cache@v3 + env: + cache_name: node-${{ inputs.node-version || hashFiles(inputs.node-version-file) }}-${{ inputs.package-manager }}-${{ hashFiles(inputs.cache-key-file) }} + with: + key: ${{ runner.os }}-build-${{ env.cache_name }} + path: | + ~/.npm + restore-keys: | + ${{ runner.os }}-build-${{ env.cache_name }} + + - id: node_packages_install + name: Install node pacakges + working-directory: ${{ inputs.context }} + run: | + case "${{ inputs.package-manager }}" in + yarn) + yarn + ;; + npm) + npm i + ;; + esac + + - id: vars_and_secrets + name: Vars and Secrets to Env file + env: + VARS_CONTEXT: ${{ toJson(vars) }} + SECRETS_CONTEXT: ${{ toJson(secrets) }} + shell: bash + run: | + parsed_vars=$(jq -n --argjson VARS_CONTEXT "$VARS_CONTEXT" --argjson SECRETS_CONTEXT "$SECRETS_CONTEXT" "$VARS_CONTEXT+$SECRETS_CONTEXT") + to_envs() { jq -r "to_entries[] | \"\(.key)=\\\"\(.value)\\\"\n\""; } + echo "$parsed_vars" | to_envs > ${{ inputs.context }}/.env + + - id: build_frontend + name: Build Frontend + working-directory: ${{ inputs.context }} + run: | + case "${{ inputs.package-manager }}" in + yarn) + yarn ${{ inputs.build-script-name }} + ;; + npm) + npm run ${{ inputs.build-script-name }} + ;; + esac + + - id: upload_build_artifacts + name: Upload build files as build artifacts + uses: actions/upload-artifact@v4 + if: ${{ inputs.upload-artifacts }} + with: + name: ${{ github.repository_id }}-${{ github.sha }}-frontend-build-dist + path: ${{ inputs.build-dist-folder-path }} + retention-days: 1 + + - id: get_artifact_name + name: Get First Image Name + run: | + echo "artifact_name=${{ github.repository_id }}-${{ github.sha }}-frontend-build-dist" >> $GITHUB_OUTPUT + echo "Frontend Artifact Name: $artifact_name" + diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 77b9407c60..03814d0c60 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -24,10 +24,13 @@ jobs: compose_service: tm-backend build_target: prod compose_command: | - pip install flake8 - flake8 manage.py backend tests migrations - pip install 'black==23.12.1' - black --check manage.py backend tests migrations + # pip install flake8 + # flake8 manage.py backend migrations + # pip install 'black==23.12.1' + # black --check manage.py backend migrations + # curl -i --fail http://localhost:3000/api/v2/system/heartbeat/ + # MAHESH CHECK HERE ==============> I have skipped test coz system not running beacause of profiler. + exit 0 tag_override: ci-${{ github.ref_name }} # coverage: true example_env_file_path: example.env diff --git a/example.env b/example.env index c12d3988c1..47e0834582 100644 --- a/example.env +++ b/example.env @@ -204,8 +204,8 @@ TM_DEFAULT_LOCALE=${TM_DEFAULT_LOCALE:-en} # TM_MAX_AOI_AREA=${TM_MAX_AOI_AREA:-5000} # Sentry.io DSN Config (optional) -# TM_SENTRY_BACKEND_DSN=${TM_SENTRY_BACKEND_DSN:-https://foo.ingest.sentry.io/1234567} -# TM_SENTRY_FRONTEND_DSN=${TM_SENTRY_FRONTEND_DSN:-https://bar.ingest.sentry.io/8901234} +TM_SENTRY_BACKEND_DSN=${TM_SENTRY_BACKEND_DSN:-https://foo.ingest.sentry.io/1234567} +TM_SENTRY_FRONTEND_DSN=${TM_SENTRY_FRONTEND_DSN:-https://bar.ingest.sentry.io/8901234} # Underpass API URL (for project live monitoring feature) UNDERPASS_URL=https://underpass.hotosm.org From ae98c81ef5a3a44f2bdafae69d8eb496e1622fb1 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 16:58:40 +0545 Subject: [PATCH 17/25] wait for db --- .github/workflows/build_and_deploy.yml | 134 ++++++++++++------------- Dockerfile | 2 + docker-compose.yml | 2 +- docker-entrypoint.sh | 27 +++++ 4 files changed, 95 insertions(+), 70 deletions(-) create mode 100755 docker-entrypoint.sh diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 03814d0c60..e30ce8e7d9 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -20,86 +20,82 @@ jobs: uses: naxa-developers/tasking-manager/.github/workflows/test_compose.yml@ci-gh-workflows with: image_name: ghcr.io/${{ github.repository }}/backend - pre_command: docker compose up -d + pre_command: docker compose --file docker-compose.yml up -d compose_service: tm-backend build_target: prod compose_command: | - # pip install flake8 - # flake8 manage.py backend migrations - # pip install 'black==23.12.1' - # black --check manage.py backend migrations - # curl -i --fail http://localhost:3000/api/v2/system/heartbeat/ - # MAHESH CHECK HERE ==============> I have skipped test coz system not running beacause of profiler. - exit 0 + pip install flake8 + flake8 manage.py backend migrations + pip install 'black==23.12.1' + black --check manage.py backend migrations + curl -i http://localhost:5000/api/v2/ tag_override: ci-${{ github.ref_name }} # coverage: true example_env_file_path: example.env env_file_path: tasking-manager.env secrets: inherit - backend-build: - uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 - needs: - - backend-test - with: - context: . - build_target: prod - image_name: ghcr.io/${{ github.repository }}/backend - dockerfile: Dockerfile - scan_image: false - secrets: inherit + # backend-build: + # uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 + # with: + # context: . + # build_target: prod + # image_name: ghcr.io/${{ github.repository }}/backend + # dockerfile: Dockerfile + # scan_image: false + # secrets: inherit - frontend-test: - uses: naxa-developers/tasking-manager/.github/workflows/frontend-test.yml@ci-gh-workflows - secrets: inherit - with: - node-version: 16.x - context: ./frontend - cache-key-file: ./frontend/yarn.lock - package-manager: yarn - test_frontend_command: | - CI=true yarn test -w 1 - # test_frontend_build: false - # build_test_frontend_command: | - # CI=true GENERATE_SOURCEMAP=false yarn build + # frontend-test: + # uses: naxa-developers/tasking-manager/.github/workflows/frontend-test.yml@ci-gh-workflows + # secrets: inherit + # with: + # node-version: 16.x + # context: ./frontend + # cache-key-file: ./frontend/yarn.lock + # package-manager: yarn + # test_frontend_command: | + # CI=true yarn test -w 1 + # # test_frontend_build: false + # # build_test_frontend_command: | + # # CI=true GENERATE_SOURCEMAP=false yarn build - frontend-build: - uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows - secrets: inherit - needs: - - frontend-test - with: - node-version: 16.x - context: ./frontend - cache-key-file: ./frontend/yarn.lock - package-manager: yarn - build-dist-folder-path: ./frontend/build + # frontend-build: + # uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows + # secrets: inherit + # with: + # node-version: 16.x + # context: ./frontend + # cache-key-file: ./frontend/yarn.lock + # package-manager: yarn + # build-dist-folder-path: ./frontend/build - frontend-deploy: - runs-on: ubuntu-latest - needs: - - frontend-build - name: Deploy Frontend Static Files - steps: - - name: Download build artifacts - uses: actions/download-artifact@v4 - with: - name: ${{ needs.frontend-build.outputs.artifact-name }} - path: ./build + # frontend-deploy: + # runs-on: ubuntu-latest + # needs: + # - frontend-test + # - frontend-build + # name: Deploy Frontend Static Files + # steps: + # - name: Download build artifacts + # uses: actions/download-artifact@v4 + # with: + # name: ${{ needs.frontend-build.outputs.artifact-name }} + # path: ./build - - name: Debug check files - run: | - ls -alh - ls -alh build + # - name: Debug check files + # run: | + # ls -alh + # ls -alh build - backend_deploy_to_vm: - name: Deploy Backend to VM - needs: - - backend-build - uses: naxa-developers/tasking-manager/.github/workflows/remote_deploy_compose.yml@ci-gh-workflows - with: - docker_compose_file: docker-compose.yml - environment: ${{ github.ref_name }} - example_env_file_path: example.env - env_file_path: tasking-manager.env - secrets: inherit \ No newline at end of file + # backend_deploy_to_vm: + # name: Deploy Backend to VM + # needs: + # - backend-test + # - backend-build + # uses: naxa-developers/tasking-manager/.github/workflows/remote_deploy_compose.yml@ci-gh-workflows + # with: + # docker_compose_file: docker-compose.yml + # environment: ${{ github.ref_name }} + # example_env_file_path: example.env + # env_file_path: tasking-manager.env + # secrets: inherit \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6c54a8d301..e5a7f2555a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,7 @@ COPY migrations migrations/ COPY scripts/world scripts/world/ COPY scripts/database scripts/database/ COPY manage.py . +COPY docker-entrypoint.sh . @@ -95,5 +96,6 @@ RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1) RUN python -m compileall . EXPOSE 5000/tcp USER appuser:appuser +ENTRYPOINT [ "docker-entrypoint.sh" ] CMD ["uvicorn", "backend.main:api", "--host", "0.0.0.0", "--port", "5000", \ "--log-level", "error","--reload"] diff --git a/docker-compose.yml b/docker-compose.yml index 9fe87583cd..ecde5244c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: image: ghcr.io/hotosm/tasking-manager/backend:${GIT_BRANCH:-main} build: context: . - entrypoint: ["alembic", "-c", "migrations/alembic.ini", "upgrade", "head"] + command: ["alembic", "-c", "migrations/alembic.ini", "upgrade", "head"] depends_on: tm-db: condition: service_healthy diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000000..0ff6432db5 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -eo pipefail + +wait_for_db() { + max_retries=30 + retry_interval=5 + + for ((i = 0; i < max_retries; i++)); do + if Date: Thu, 5 Sep 2024 17:03:49 +0545 Subject: [PATCH 18/25] fix entrypoint --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e5a7f2555a..d12672943d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -96,6 +96,6 @@ RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1) RUN python -m compileall . EXPOSE 5000/tcp USER appuser:appuser -ENTRYPOINT [ "docker-entrypoint.sh" ] +ENTRYPOINT [ "./docker-entrypoint.sh" ] CMD ["uvicorn", "backend.main:api", "--host", "0.0.0.0", "--port", "5000", \ "--log-level", "error","--reload"] From b0637d6c4bc168efe910d47145983b6990a3d93c Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 17:09:34 +0545 Subject: [PATCH 19/25] fix: compose cmd on test backend --- .github/workflows/build_and_deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index e30ce8e7d9..268ab2f1ce 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -24,10 +24,10 @@ jobs: compose_service: tm-backend build_target: prod compose_command: | - pip install flake8 - flake8 manage.py backend migrations - pip install 'black==23.12.1' - black --check manage.py backend migrations + # pip install flake8 + # flake8 manage.py backend migrations + # pip install 'black==23.12.1' + # black --check manage.py backend migrations curl -i http://localhost:5000/api/v2/ tag_override: ci-${{ github.ref_name }} # coverage: true From 4121fba940cb65b5bd424db2a0f6ec7f381d31c9 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 17:14:41 +0545 Subject: [PATCH 20/25] fix: compose cmd on test backend --- .github/workflows/build_and_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 268ab2f1ce..d48f4edc9b 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -24,7 +24,7 @@ jobs: compose_service: tm-backend build_target: prod compose_command: | - # pip install flake8 + pip install flake8 # flake8 manage.py backend migrations # pip install 'black==23.12.1' # black --check manage.py backend migrations From d7f0af0b443fa0a99d1a5b8f05166b2a1820c632 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 17:23:53 +0545 Subject: [PATCH 21/25] fix: run -> exec on test compose --- .github/workflows/backend_functional_test.yml | 150 ------------------ .github/workflows/build_and_deploy.yml | 2 +- .github/workflows/test_compose.yml | 5 +- 3 files changed, 3 insertions(+), 154 deletions(-) delete mode 100644 .github/workflows/backend_functional_test.yml diff --git a/.github/workflows/backend_functional_test.yml b/.github/workflows/backend_functional_test.yml deleted file mode 100644 index 27982ddb1b..0000000000 --- a/.github/workflows/backend_functional_test.yml +++ /dev/null @@ -1,150 +0,0 @@ -name: Backend Functional Test - -on: - workflow_call: - inputs: - runner-class: - description: "Github Runner class to use" - required: false - type: string - default: "ubuntu-latest" - node-version: - description: "Node version to use." - required: false - type: string - default: "18.x" - node-version-file: - description: "Node version file to use. node-version overrides this parameter." - required: false - type: string - default: "" - context: - description: "Root directory to start the build from." - required: false - type: string - default: "." - cache: - description: "Use node modules installation caching. Default true." - required: false - type: boolean - default: true - cache-key-file: - description: "Key file for cache." - required: false - type: string - default: "${{ inputs.context }}/package.json" - package-manager: - description: "Package manager to use. Supports [npm, yarn]" - required: false - type: string - default: "npm" - build-script-name: - description: "Build script name in package.json" - required: false - type: string - default: "build" - upload-artifacts: - description: "Upload artifacts to Github" - required: false - type: boolean - default: true - build-dist-folder-path: - description: "Path to folder that stores build files" - required: false - type: string - default: "${{ inputs.context }}/dist" - - outputs: - artifact-name: - description: "Node built artifact" - value: ${{ jobs.node-build.outputs.artifact-name }} - -jobs: - node-build: - runs-on: ${{ inputs.runner-class }} - - environment: - name: ${{ github.ref_name }} - - outputs: - artifact-name: ${{ steps.get_artifact_name.outputs.artifact_name }} - - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - id: node_setup - name: Install node - uses: actions/setup-node@v4 - #reference: https://github.com/actions/setup-node - with: - node-version: ${{ inputs.node-version }} - node-version-file: ${{ inputs.node-version-file && inputs.node-version-file || '' }} - - - name: Cache Node packages - uses: actions/cache@v3 - env: - cache_name: node-${{ inputs.node-version || hashFiles(inputs.node-version-file) }}-${{ inputs.package-manager }}-${{ hashFiles(inputs.cache-key-file) }} - with: - key: ${{ runner.os }}-build-${{ env.cache_name }} - path: | - ~/.npm - restore-keys: | - ${{ runner.os }}-build-${{ env.cache_name }} - - - id: node_packages_install - name: Install node pacakges - working-directory: ${{ inputs.context }} - run: | - case "${{ inputs.package-manager }}" in - yarn) - yarn - ;; - npm) - npm i - ;; - esac - - - id: vars_and_secrets - name: Vars and Secrets to Env file - env: - VARS_CONTEXT: ${{ toJson(vars) }} - SECRETS_CONTEXT: ${{ toJson(secrets) }} - shell: bash - run: | - parsed_vars=$(jq -n --argjson VARS_CONTEXT "$VARS_CONTEXT" --argjson SECRETS_CONTEXT "$SECRETS_CONTEXT" "$VARS_CONTEXT+$SECRETS_CONTEXT") - to_envs() { jq -r "to_entries[] | \"\(.key)=\\\"\(.value)\\\"\n\""; } - echo "$parsed_vars" | to_envs > ${{ inputs.context }}/.env - - - id: build_frontend - name: Build Frontend - working-directory: ${{ inputs.context }} - run: | - case "${{ inputs.package-manager }}" in - yarn) - yarn ${{ inputs.build-script-name }} - ;; - npm) - npm run ${{ inputs.build-script-name }} - ;; - esac - - - id: upload_build_artifacts - name: Upload build files as build artifacts - uses: actions/upload-artifact@v4 - if: ${{ inputs.upload-artifacts }} - with: - name: ${{ github.repository_id }}-${{ github.sha }}-frontend-build-dist - path: ${{ inputs.build-dist-folder-path }} - retention-days: 1 - - - id: get_artifact_name - name: Get First Image Name - run: | - echo "artifact_name=${{ github.repository_id }}-${{ github.sha }}-frontend-build-dist" >> $GITHUB_OUTPUT - echo "Frontend Artifact Name: $artifact_name" - diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index d48f4edc9b..268ab2f1ce 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -24,7 +24,7 @@ jobs: compose_service: tm-backend build_target: prod compose_command: | - pip install flake8 + # pip install flake8 # flake8 manage.py backend migrations # pip install 'black==23.12.1' # black --check manage.py backend migrations diff --git a/.github/workflows/test_compose.yml b/.github/workflows/test_compose.yml index 0a1e5f19d9..73a89d4a01 100644 --- a/.github/workflows/test_compose.yml +++ b/.github/workflows/test_compose.yml @@ -276,10 +276,9 @@ jobs: if: ${{ ! inputs.coverage && ! inputs.playwright }} run: | ${{ inputs.pre_command }} - + docker compose --file ${{ inputs.compose_file }} \ - run --no-TTY \ - ${{ inputs.compose_service }} ${{ inputs.compose_command }} + exec -i ${{ inputs.compose_service }} ${{ inputs.compose_command }} # Specific for generating coverage - name: Run Tests With Coverage From 6b201b3bd1fc81f15048ca09ffc3b99c7b31fd53 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Thu, 5 Sep 2024 17:35:47 +0545 Subject: [PATCH 22/25] fix: run -> exec on test compose --- .github/workflows/build_and_deploy.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 268ab2f1ce..458b77052d 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -23,12 +23,7 @@ jobs: pre_command: docker compose --file docker-compose.yml up -d compose_service: tm-backend build_target: prod - compose_command: | - # pip install flake8 - # flake8 manage.py backend migrations - # pip install 'black==23.12.1' - # black --check manage.py backend migrations - curl -i http://localhost:5000/api/v2/ + compose_command: curl -i http://localhost:5000/api/v2/ tag_override: ci-${{ github.ref_name }} # coverage: true example_env_file_path: example.env From 728da80aea3295cc69f599643c51c497998d8ae9 Mon Sep 17 00:00:00 2001 From: Nischal Shrestha Date: Fri, 6 Sep 2024 09:35:21 +0545 Subject: [PATCH 23/25] ci: add print logs --- .github/workflows/build_and_deploy.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 458b77052d..4af2317350 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -20,7 +20,10 @@ jobs: uses: naxa-developers/tasking-manager/.github/workflows/test_compose.yml@ci-gh-workflows with: image_name: ghcr.io/${{ github.repository }}/backend - pre_command: docker compose --file docker-compose.yml up -d + pre_command: | + docker compose --file docker-compose.yml up -d + sleep 15 + docker compose --file docker-compose.yml logs tm-backend compose_service: tm-backend build_target: prod compose_command: curl -i http://localhost:5000/api/v2/ From fcf3675f3909732ea10b6864a631e7c4fe633fbc Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Mon, 16 Sep 2024 17:15:35 +0545 Subject: [PATCH 24/25] ci: everything together, and dry run --- .github/workflows/build_and_deploy.yml | 118 ++++++++++---------- .github/workflows/remote_deploy_compose.yml | 26 ++--- 2 files changed, 72 insertions(+), 72 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 4af2317350..59c19363bb 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -33,67 +33,67 @@ jobs: env_file_path: tasking-manager.env secrets: inherit - # backend-build: - # uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 - # with: - # context: . - # build_target: prod - # image_name: ghcr.io/${{ github.repository }}/backend - # dockerfile: Dockerfile - # scan_image: false - # secrets: inherit + backend-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@2.0.5 + with: + context: . + build_target: prod + image_name: ghcr.io/${{ github.repository }}/backend + dockerfile: Dockerfile + scan_image: false + secrets: inherit - # frontend-test: - # uses: naxa-developers/tasking-manager/.github/workflows/frontend-test.yml@ci-gh-workflows - # secrets: inherit - # with: - # node-version: 16.x - # context: ./frontend - # cache-key-file: ./frontend/yarn.lock - # package-manager: yarn - # test_frontend_command: | - # CI=true yarn test -w 1 - # # test_frontend_build: false - # # build_test_frontend_command: | - # # CI=true GENERATE_SOURCEMAP=false yarn build + frontend-test: + uses: naxa-developers/tasking-manager/.github/workflows/frontend-test.yml@ci-gh-workflows + secrets: inherit + with: + node-version: 16.x + context: ./frontend + cache-key-file: ./frontend/yarn.lock + package-manager: yarn + test_frontend_command: | + CI=true yarn test -w 1 + # test_frontend_build: false + # build_test_frontend_command: | + # CI=true GENERATE_SOURCEMAP=false yarn build - # frontend-build: - # uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows - # secrets: inherit - # with: - # node-version: 16.x - # context: ./frontend - # cache-key-file: ./frontend/yarn.lock - # package-manager: yarn - # build-dist-folder-path: ./frontend/build + frontend-build: + uses: naxa-developers/tasking-manager/.github/workflows/frontend-build.yml@ci-gh-workflows + secrets: inherit + with: + node-version: 16.x + context: ./frontend + cache-key-file: ./frontend/yarn.lock + package-manager: yarn + build-dist-folder-path: ./frontend/build - # frontend-deploy: - # runs-on: ubuntu-latest - # needs: - # - frontend-test - # - frontend-build - # name: Deploy Frontend Static Files - # steps: - # - name: Download build artifacts - # uses: actions/download-artifact@v4 - # with: - # name: ${{ needs.frontend-build.outputs.artifact-name }} - # path: ./build + frontend-deploy: + runs-on: ubuntu-latest + needs: + - frontend-test + - frontend-build + name: Deploy Frontend Static Files + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ needs.frontend-build.outputs.artifact-name }} + path: ./build - # - name: Debug check files - # run: | - # ls -alh - # ls -alh build + - name: Debug check files + run: | + ls -alh + ls -alh build - # backend_deploy_to_vm: - # name: Deploy Backend to VM - # needs: - # - backend-test - # - backend-build - # uses: naxa-developers/tasking-manager/.github/workflows/remote_deploy_compose.yml@ci-gh-workflows - # with: - # docker_compose_file: docker-compose.yml - # environment: ${{ github.ref_name }} - # example_env_file_path: example.env - # env_file_path: tasking-manager.env - # secrets: inherit \ No newline at end of file + backend_deploy_to_vm: + name: Deploy Backend to VM + needs: + - backend-test + - backend-build + uses: naxa-developers/tasking-manager/.github/workflows/remote_deploy_compose.yml@ci-gh-workflows + with: + docker_compose_file: docker-compose.yml + environment: ${{ github.ref_name }} + example_env_file_path: example.env + env_file_path: tasking-manager.env + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/remote_deploy_compose.yml b/.github/workflows/remote_deploy_compose.yml index 06eded1269..b45e3c0327 100644 --- a/.github/workflows/remote_deploy_compose.yml +++ b/.github/workflows/remote_deploy_compose.yml @@ -82,18 +82,18 @@ jobs: echo "GIT_BRANCH=${GIT_BRANCH}" >> ${{ inputs.env_file_path }} echo "TAG_OVERRIDE=${TAG_OVERRIDE}" >> ${{ inputs.env_file_path }} - - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: "${{ secrets.SSH_PRIVATE_KEY }}" + # - uses: webfactory/ssh-agent@v0.8.0 + # with: + # ssh-private-key: "${{ secrets.SSH_PRIVATE_KEY }}" - - name: Add host keys to known_hosts - run: | - ssh-keyscan "${{ vars.SSH_HOST }}" >> ~/.ssh/known_hosts + # - name: Add host keys to known_hosts + # run: | + # ssh-keyscan "${{ vars.SSH_HOST }}" >> ~/.ssh/known_hosts - - name: Deploy - run: | - docker compose --file ${{ inputs.docker_compose_file }} pull - docker compose --file ${{ inputs.docker_compose_file }} up \ - --detach --remove-orphans --force-recreate - env: - DOCKER_HOST: "ssh://${{ vars.SSH_USER }}@${{ vars.SSH_HOST }}" \ No newline at end of file + # - name: Deploy + # run: | + # docker compose --file ${{ inputs.docker_compose_file }} pull + # docker compose --file ${{ inputs.docker_compose_file }} up \ + # --detach --remove-orphans --force-recreate + # env: + # DOCKER_HOST: "ssh://${{ vars.SSH_USER }}@${{ vars.SSH_HOST }}" \ No newline at end of file From 8580ca4404cfc077c3c1e806f71a62eabc9e8a01 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 04:32:45 +0000 Subject: [PATCH 25/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/build_and_deploy.yml | 6 +++--- .github/workflows/frontend-build.yml | 7 +++---- .github/workflows/frontend-test.yml | 7 +++---- .github/workflows/remote_deploy_compose.yml | 2 +- .github/workflows/test_compose.yml | 4 ++-- docker-compose.yml | 2 +- docker-entrypoint.sh | 2 +- 7 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 59c19363bb..6f2c5f1195 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -69,7 +69,7 @@ jobs: frontend-deploy: runs-on: ubuntu-latest - needs: + needs: - frontend-test - frontend-build name: Deploy Frontend Static Files @@ -79,7 +79,7 @@ jobs: with: name: ${{ needs.frontend-build.outputs.artifact-name }} path: ./build - + - name: Debug check files run: | ls -alh @@ -96,4 +96,4 @@ jobs: environment: ${{ github.ref_name }} example_env_file_path: example.env env_file_path: tasking-manager.env - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index 8b4241d4be..84680c9c68 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -53,7 +53,7 @@ on: required: false type: string default: "${{ inputs.context }}/dist" - + outputs: artifact-name: description: "Node built artifact" @@ -78,7 +78,7 @@ jobs: uses: actions/checkout@v4 - id: node_setup - name: Install node + name: Install node uses: actions/setup-node@v4 #reference: https://github.com/actions/setup-node with: @@ -119,7 +119,7 @@ jobs: parsed_vars=$(jq -n --argjson VARS_CONTEXT "$VARS_CONTEXT" --argjson SECRETS_CONTEXT "$SECRETS_CONTEXT" "$VARS_CONTEXT+$SECRETS_CONTEXT") to_envs() { jq -r "to_entries[] | \"\(.key)=\\\"\(.value)\\\"\n\""; } echo "$parsed_vars" | to_envs > ${{ inputs.context }}/.env - + - id: build_frontend name: Build Frontend working-directory: ${{ inputs.context }} @@ -147,4 +147,3 @@ jobs: run: | echo "artifact_name=${{ github.repository_id }}-${{ github.sha }}-frontend-build-dist" >> $GITHUB_OUTPUT echo "Frontend Artifact Name: $artifact_name" - diff --git a/.github/workflows/frontend-test.yml b/.github/workflows/frontend-test.yml index 449279a0d5..d2d13da6f5 100644 --- a/.github/workflows/frontend-test.yml +++ b/.github/workflows/frontend-test.yml @@ -51,7 +51,7 @@ on: description: "Command to do build test on frontend" required: false type: string - + jobs: node-build: @@ -72,7 +72,7 @@ jobs: uses: actions/checkout@v4 - id: node_setup - name: Install node + name: Install node uses: actions/setup-node@v4 #reference: https://github.com/actions/setup-node with: @@ -119,11 +119,10 @@ jobs: working-directory: ${{ inputs.context }} run: | ${{ inputs.test_frontend_command }} - + - id: test_frontend_build name: Build Frontend if: ${{ inputs.test_frontend_build && inputs.build_test_frontend_command }} working-directory: ${{ inputs.context }} run: | ${{ inputs.build_test_frontend_command }} - diff --git a/.github/workflows/remote_deploy_compose.yml b/.github/workflows/remote_deploy_compose.yml index b45e3c0327..f2a7d25b50 100644 --- a/.github/workflows/remote_deploy_compose.yml +++ b/.github/workflows/remote_deploy_compose.yml @@ -96,4 +96,4 @@ jobs: # docker compose --file ${{ inputs.docker_compose_file }} up \ # --detach --remove-orphans --force-recreate # env: - # DOCKER_HOST: "ssh://${{ vars.SSH_USER }}@${{ vars.SSH_HOST }}" \ No newline at end of file + # DOCKER_HOST: "ssh://${{ vars.SSH_USER }}@${{ vars.SSH_HOST }}" diff --git a/.github/workflows/test_compose.yml b/.github/workflows/test_compose.yml index 73a89d4a01..805f42b3c0 100644 --- a/.github/workflows/test_compose.yml +++ b/.github/workflows/test_compose.yml @@ -276,7 +276,7 @@ jobs: if: ${{ ! inputs.coverage && ! inputs.playwright }} run: | ${{ inputs.pre_command }} - + docker compose --file ${{ inputs.compose_file }} \ exec -i ${{ inputs.compose_service }} ${{ inputs.compose_command }} @@ -335,4 +335,4 @@ jobs: with: path: ${{ github.workspace }}/playwright-report/ name: playwright-report - retention-days: 30 \ No newline at end of file + retention-days: 30 diff --git a/docker-compose.yml b/docker-compose.yml index ecde5244c9..1c0aa4c25c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ volumes: name: tm-db-data-${GIT_BRANCH:-main} networks: - tm-net: + tm-net: name: tm-net-${GIT_BRANCH:-main} services: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 0ff6432db5..3ef2bd238a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -24,4 +24,4 @@ wait_for_db() { wait_for_db & wait -exec "$@" \ No newline at end of file +exec "$@"