From b1b542dd0b54b8d826401c53076aa09b89440dad Mon Sep 17 00:00:00 2001 From: royallsilwallz Date: Mon, 22 Jan 2024 14:00:46 +0545 Subject: [PATCH 01/29] Fix `Project Stats` page crash issue for archived projects - Register `CategoryScale` & `LinearScale` in ChartJS - Fixes #6131 --- frontend/src/components/projectStats/contributorsStats.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/projectStats/contributorsStats.js b/frontend/src/components/projectStats/contributorsStats.js index 5facb55904..1d6aa41bb9 100644 --- a/frontend/src/components/projectStats/contributorsStats.js +++ b/frontend/src/components/projectStats/contributorsStats.js @@ -1,5 +1,5 @@ import React from 'react'; -import { Chart as ChartJS, ArcElement, BarElement } from 'chart.js'; +import { Chart as ChartJS, ArcElement, BarElement, CategoryScale, LinearScale } from 'chart.js'; import { Doughnut, Bar } from 'react-chartjs-2'; import { FormattedMessage, useIntl } from 'react-intl'; @@ -10,7 +10,7 @@ import { formatChartData, formatTooltip } from '../../utils/formatChartJSData'; import { useContributorStats } from '../../hooks/UseContributorStats'; import { StatsCardContent } from '../statsCard'; -ChartJS.register(ArcElement, BarElement); +ChartJS.register(ArcElement, BarElement, CategoryScale, LinearScale); export default function ContributorsStats({ contributors }) { const intl = useIntl(); From 86dcf94ace8ae9a8b56b02389ec8b11bc0f63cf6 Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Wed, 24 Jan 2024 17:08:33 -0700 Subject: [PATCH 02/29] fix: Add deoc on validation --- docs/validation.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 docs/validation.md diff --git a/docs/validation.md b/docs/validation.md new file mode 100644 index 0000000000..63c170143d --- /dev/null +++ b/docs/validation.md @@ -0,0 +1,46 @@ +# Validation + +Validation is the final step of the mapping process. It is necessary +to ensure good quality data. Validation requires a slightly different +skill set than just being a good mapper. Being an experienced mapper is +a good start of course. + +For Tasking Manager, validators review the work of the remote +mappers. It is not unusual that features get misidentified by the +mappers due to poor imagery. Validators have much experience with +identifying obscure buildings types in different countries. + +Being a validator is more than being a mapper. As a person dealing +with quality issues, a validator is also a mentor to other mappers, +improving their skills as well. Mentoring of course requires an +ability to communicate mistakes with empathy and understanding, as we +all make mistakes... + +## References + +There are multiple good sources for information on validation for more detail. + +* After you’ve had some experience mapping, you may find yourself +correcting other contributors work. [/MissingMaps_validation_faq_en.pdf](https://www.missingmaps.org/assets/downloads/MissingMaps_validation_faq_en.pdf) + +* Before validating the work of others, you should be an experienced + mapper yourself. [Validating_Data_EN.pdf](https://www.missingmaps.org/assets/downloads/Validating_Data_EN.pdf) + +* Mappers wishing to become validators should take time to become + experienced at mapping and learn what validation entails before + submitting an application. [validators-experienced-mappers-making-an-outsized-difference/](https://www.hotosm.org/updates/validators-experienced-mappers-making-an-outsized-difference/) + +* Validation is the process of an experienced mapper reviewing the + mapping of the initial mapper of a project task. [Validation](https://learnosm.org/en/coordination/validation/) + +* In OpenStreetMap anyone can validate as long as they do not validate + their own work. However, they can validate another mapper's + contributions. [Validating Data](https://wiki.openstreetmap.org/wiki/Tasking_Manager/Validating_data) + +* Mappers that wish to become validators must have experience with + tracing and tagging complex features in OSM, outstanding image + interpretation skills, a positive attitude and manner towards other + users, and a willingness to learn new software. [Validation + hub](https://www.youthmappers.org/validation-hub) + + From ff7489fa274b3cdb6bda7e191b77f22c106186fe Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Sun, 28 Jan 2024 17:10:00 -0700 Subject: [PATCH 03/29] fix: Add validation doc --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index db971e1eb6..65a590de3c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -88,5 +88,6 @@ nav: - Customize: sysadmins/customize.md - CI: sysadmins/ci-cd.md - Working Groups: working-groups.md + - Validation: validation.md - License: LICENSE.md # - Changelog: CHANGELOG.md From bfb80ac42026e7ad740a9b26a640c391eede7198 Mon Sep 17 00:00:00 2001 From: DK Benjamin Date: Mon, 29 Jan 2024 09:48:44 -0500 Subject: [PATCH 04/29] Fix version of black to last working version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a57248f6da..3f4e3d146c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ jobs: - run: name: black tests command: | - pip install black + pip install 'black==23.12.1' ## TODO: Update to version 24 black --check manage.py backend tests migrations backend-functional-tests: From ca4ec2bf1dad22791e5cd66d35e3cdf6ca64e83c Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Mon, 8 Jan 2024 17:11:08 +0545 Subject: [PATCH 05/29] infra: added terraform snippets for tm-extrator deployment --- .gitignore | 8 +++ .../lambda/TM-Extractor/_envcommon/lambda.hcl | 46 +++++++++++++ scripts/aws/lambda/TM-Extractor/dev/env.hcl | 12 ++++ .../dev/tm-extractor/terragrunt.hcl | 34 +++++++++ .../aws/lambda/TM-Extractor/terragrunt.hcl | 69 +++++++++++++++++++ 5 files changed, 169 insertions(+) create mode 100644 scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl create mode 100644 scripts/aws/lambda/TM-Extractor/dev/env.hcl create mode 100644 scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl create mode 100644 scripts/aws/lambda/TM-Extractor/terragrunt.hcl diff --git a/.gitignore b/.gitignore index d2ab1428d5..298875d50d 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,11 @@ venv_aws/ # Backend coverage generated files htmlcov/ .coverage + +# Terraform +.terraform +*.tfstate +*.tfstate.backup + +# Terragrunt +.terragrunt-cache diff --git a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl new file mode 100644 index 0000000000..45773918e4 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl @@ -0,0 +1,46 @@ +# --------------------------------------------------------------------------------------------------------------------- +# COMMON TERRAGRUNT CONFIGURATION +# This is the common component configuration for mysql. The common variables for each environment to +# deploy mysql are defined here. This configuration will be merged into the environment configuration +# via an include block. +# --------------------------------------------------------------------------------------------------------------------- + +# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the +# working directory, into a temporary folder, and execute your Terraform commands in that folder. If any environment +# needs to deploy a different module version, it should redefine this block with a different ref to override the +# deployed version. +terraform { + #source = "${local.base_source_url}?ref=feat-terraform" + source = "/Users/invoron/NAXA/tm-extrator-hotosm/" +} + +# --------------------------------------------------------------------------------------------------------------------- +# Locals are named constants that are reusable within the configuration. +# --------------------------------------------------------------------------------------------------------------------- +locals { + # Automatically load environment-level variables + environment_vars = read_terragrunt_config(find_in_parent_folders("env.hcl")) + + # Extract out common variables for reuse + env = local.environment_vars.locals.environment + + # Expose the base source URL so different versions of the module can be deployed in different environments. This will + # be used to construct the terraform block in the child terragrunt configurations. + base_source_url = "git::git@github.com:mahesh-naxa/TM-Extractor.git//lambda" +} + +# --------------------------------------------------------------------------------------------------------------------- +# MODULE PARAMETERS +# These are the variables we have to pass in to use the module. This defines the parameters that are common across all +# environments. +# --------------------------------------------------------------------------------------------------------------------- +# Defaults, overridden by env.hcl + +inputs = { + iam_lambda_role_name = "tm-extractor-role" + cw_retention_in_days = "14" + lambda_environment = "dev" + lambda_memory_size = "128" + lambda_timeout = "14" + lambda_cron_expression = "cron(0 0 * * ? *)" +} diff --git a/scripts/aws/lambda/TM-Extractor/dev/env.hcl b/scripts/aws/lambda/TM-Extractor/dev/env.hcl new file mode 100644 index 0000000000..58c33315ab --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/dev/env.hcl @@ -0,0 +1,12 @@ +# Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to +# feed forward to the child modules. + +locals { + environment = "dev" + account_name = "naxadevelopers" + aws_region = "ap-south-1" + application = "tasking-manager" + team = "HOTOSM" + creator = "HOTOSM" + owner = "HOTOSM" +} \ No newline at end of file diff --git a/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl new file mode 100644 index 0000000000..6660750c3c --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl @@ -0,0 +1,34 @@ +# --------------------------------------------------------------------------------------------------------------------- +# TERRAGRUNT CONFIGURATION +# This is the configuration for Terragrunt, a thin wrapper for Terraform that helps keep your code DRY and +# maintainable: https://github.com/gruntwork-io/terragrunt +# --------------------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------------------- +# Include configurations that are common used across multiple environments. +# --------------------------------------------------------------------------------------------------------------------- + +# Include the root `terragrunt.hcl` configuration. The root configuration contains settings that are common across all +# components and environments, such as how to configure remote state. +include "root" { + path = find_in_parent_folders() +} + +# Include the envcommon configuration for the component. The envcommon configuration contains settings that are common +# for the component across all environments. +include "envcommon" { + path = "${dirname(find_in_parent_folders())}/_envcommon/lambda.hcl" +} + +# --------------------------------------------------------------------------------------------------------------------- +# Override parameters for this environment +# --------------------------------------------------------------------------------------------------------------------- + +# For production, we want to specify bigger instance classes and storage, so we specify override parameters here. These +# inputs get merged with the common inputs from the root and the envcommon lambda.hcl +inputs = { + zip_output_dir = "files" + active_projects_api_base_url = "https://tm.naxa.com.np/api/v2" + raw_data_api = "https://api-prod.raw-data.hotosm.org/v1" + config_json = "config.json" +} diff --git a/scripts/aws/lambda/TM-Extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/terragrunt.hcl new file mode 100644 index 0000000000..9c6b18d006 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/terragrunt.hcl @@ -0,0 +1,69 @@ +# --------------------------------------------------------------------------------------------------------------------- +# TERRAGRUNT CONFIGURATION +# Terragrunt is a thin wrapper for Terraform that provides extra tools for working with multiple Terraform modules, +# remote state, and locking: https://github.com/gruntwork-io/terragrunt +# --------------------------------------------------------------------------------------------------------------------- + +locals { + # Automatically load environment-level variables + environment_vars = read_terragrunt_config(find_in_parent_folders("env.hcl")) + + # Extract the variables we need for easy access + account_name = local.environment_vars.locals.account_name + aws_region = local.environment_vars.locals.aws_region + environment = local.environment_vars.locals.environment + application = local.environment_vars.locals.application + team = local.environment_vars.locals.team + creator = local.environment_vars.locals.creator + owner = local.environment_vars.locals.owner +} + +# Generate an AWS provider block +generate "provider" { + path = "provider.tf" + if_exists = "overwrite_terragrunt" + contents = < Date: Mon, 8 Jan 2024 17:37:04 +0545 Subject: [PATCH 06/29] docs: updated readme for tm-extrctor lambda setup. --- scripts/aws/lambda/TM-Extractor/README.md | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 scripts/aws/lambda/TM-Extractor/README.md diff --git a/scripts/aws/lambda/TM-Extractor/README.md b/scripts/aws/lambda/TM-Extractor/README.md new file mode 100644 index 0000000000..77d93be5fa --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/README.md @@ -0,0 +1,36 @@ + +## TM-Extractor with Terragrunt + +This directory contains infrastructure config for deploying [tm-extractor](https://github.com/hotosm/TM-Extractor) on AWS Lambda. + +Check [tm-extractor](https://github.com/hotosm/TM-Extractor) for resources created and exporter script. + +To get it running on your environment follow the following. + +### Prerequisites + +This are list of things you will need. +* AWS IAM User with permission for creating resources such as lambda function, cloudwatch group, cloudwatch event etc. +See all resource [here](https://github.com/hotosm/TM-Extractor/blob/ec37c1318325c534b4ac47f057263050e6e92f03/main.tf). +* [Terraform](https://www.terraform.io/) +* [Terragrunt](https://terragrunt.gruntwork.io/) + +### Environments + +Before running deployment, make sure you have the following environment variables exported. +- `TF_VAR_rawdata_api_auth_token`, Auth token for raw data api, Request [here](https://github.com/hotosm/raw-data-api/). +- `TF_VAR_raw_data_api`, API endpoint. Defaults to https://api-prod.raw-data.hotosm.org/v1 +- `TF_VAR_active_projects_api_base_url`, Your [tasking-manager](https://github.com/hotosm/tasking-manager) instance. +Defaults to https://tasking-manager-staging-api.hotosm.org/api/v2 + + +## Plan and Apply +- `cd` into `scripts/aws/lambda/TM-Extractor//tm-extractor`. +- `terragrunt init`, initializes providers and resources. +- `terragrunt plan`, run a plan to check for infrastructure changes. +- `terragrunt apply`, applies the configuration. + + +_For more information, please refer to the [TM-Extractor](https://github.com/hotosm/TM-Extractor)_ repository. + +

(back to top)

\ No newline at end of file From d13d070a69efb687792a335cef998c7c146ad763 Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Mon, 8 Jan 2024 17:53:54 +0545 Subject: [PATCH 07/29] infra: added terragrunt.hcl for stag setup --- .gitignore | 1 + scripts/aws/lambda/TM-Extractor/stag/env.hcl | 12 +++++++ .../stag/tm-extractor/terragrunt.hcl | 34 +++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 scripts/aws/lambda/TM-Extractor/stag/env.hcl create mode 100644 scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl diff --git a/.gitignore b/.gitignore index 298875d50d..40b0a6a6e8 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ htmlcov/ .terraform *.tfstate *.tfstate.backup +.terraform.lock.hcl # Terragrunt .terragrunt-cache diff --git a/scripts/aws/lambda/TM-Extractor/stag/env.hcl b/scripts/aws/lambda/TM-Extractor/stag/env.hcl new file mode 100644 index 0000000000..88c99ef25e --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/stag/env.hcl @@ -0,0 +1,12 @@ +# Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to +# feed forward to the child modules. + +locals { + environment = "stag" + account_name = "hotosm" + aws_region = "ap-south-1" + application = "tasking-manager" + team = "HOTOSM" + creator = "HOTOSM" + owner = "HOTOSM" +} \ No newline at end of file diff --git a/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl new file mode 100644 index 0000000000..337d70ca34 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl @@ -0,0 +1,34 @@ +# --------------------------------------------------------------------------------------------------------------------- +# TERRAGRUNT CONFIGURATION +# This is the configuration for Terragrunt, a thin wrapper for Terraform that helps keep your code DRY and +# maintainable: https://github.com/gruntwork-io/terragrunt +# --------------------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------------------- +# Include configurations that are common used across multiple environments. +# --------------------------------------------------------------------------------------------------------------------- + +# Include the root `terragrunt.hcl` configuration. The root configuration contains settings that are common across all +# components and environments, such as how to configure remote state. +include "root" { + path = find_in_parent_folders() +} + +# Include the envcommon configuration for the component. The envcommon configuration contains settings that are common +# for the component across all environments. +include "envcommon" { + path = "${dirname(find_in_parent_folders())}/_envcommon/lambda.hcl" +} + +# --------------------------------------------------------------------------------------------------------------------- +# Override parameters for this environment +# --------------------------------------------------------------------------------------------------------------------- + +# For production, we want to specify bigger instance classes and storage, so we specify override parameters here. These +# inputs get merged with the common inputs from the root and the envcommon lambda.hcl +inputs = { + zip_output_dir = "files" + active_projects_api_base_url = "https://tasking-manager-staging-api.hotosm.org/api/v2" + raw_data_api = "https://api-prod.raw-data.hotosm.org/v1" + config_json = "config.json" +} From 0fbc7a887c92d78cd7125edbfa126af1db540800 Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Mon, 8 Jan 2024 20:45:56 +0545 Subject: [PATCH 08/29] infra: added lambda conf and env.hcl for prod --- scripts/aws/lambda/TM-Extractor/prod/env.hcl | 12 +++++++ .../prod/tm-extractor/terragrunt.hcl | 34 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 scripts/aws/lambda/TM-Extractor/prod/env.hcl create mode 100644 scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl diff --git a/scripts/aws/lambda/TM-Extractor/prod/env.hcl b/scripts/aws/lambda/TM-Extractor/prod/env.hcl new file mode 100644 index 0000000000..a7c2790117 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/prod/env.hcl @@ -0,0 +1,12 @@ +# Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to +# feed forward to the child modules. + +locals { + environment = "prod" + account_name = "hotosm" + aws_region = "ap-south-1" + application = "tasking-manager" + team = "HOTOSM" + creator = "HOTOSM" + owner = "HOTOSM" +} \ No newline at end of file diff --git a/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl new file mode 100644 index 0000000000..132bbc5ff9 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl @@ -0,0 +1,34 @@ +# --------------------------------------------------------------------------------------------------------------------- +# TERRAGRUNT CONFIGURATION +# This is the configuration for Terragrunt, a thin wrapper for Terraform that helps keep your code DRY and +# maintainable: https://github.com/gruntwork-io/terragrunt +# --------------------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------------------- +# Include configurations that are common used across multiple environments. +# --------------------------------------------------------------------------------------------------------------------- + +# Include the root `terragrunt.hcl` configuration. The root configuration contains settings that are common across all +# components and environments, such as how to configure remote state. +include "root" { + path = find_in_parent_folders() +} + +# Include the envcommon configuration for the component. The envcommon configuration contains settings that are common +# for the component across all environments. +include "envcommon" { + path = "${dirname(find_in_parent_folders())}/_envcommon/lambda.hcl" +} + +# --------------------------------------------------------------------------------------------------------------------- +# Override parameters for this environment +# --------------------------------------------------------------------------------------------------------------------- + +# For production, we want to specify bigger instance classes and storage, so we specify override parameters here. These +# inputs get merged with the common inputs from the root and the envcommon lambda.hcl +inputs = { + zip_output_dir = "files" + active_projects_api_base_url = "https://tasking-manager-tm4-production-api.hotosm.org/api/v2" + raw_data_api = "https://api-prod.raw-data.hotosm.org/v1" + config_json = "config.json" +} From ce94e336a40d1d5555c2052bec95d0878f2d3be8 Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Mon, 8 Jan 2024 20:59:11 +0545 Subject: [PATCH 09/29] infra: terragrunt now sources from git repository. --- scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl index 45773918e4..830ad6d5f8 100644 --- a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl +++ b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl @@ -10,8 +10,8 @@ # needs to deploy a different module version, it should redefine this block with a different ref to override the # deployed version. terraform { - #source = "${local.base_source_url}?ref=feat-terraform" - source = "/Users/invoron/NAXA/tm-extrator-hotosm/" + # Sourcing from here rn, updating to a release from https://github.com/hotosm/TM-Extractor/ later. + source = "${local.base_source_url}?ref=feat-terraform" } # --------------------------------------------------------------------------------------------------------------------- @@ -26,7 +26,7 @@ locals { # Expose the base source URL so different versions of the module can be deployed in different environments. This will # be used to construct the terraform block in the child terragrunt configurations. - base_source_url = "git::git@github.com:mahesh-naxa/TM-Extractor.git//lambda" + base_source_url = "git::https://github.com/naxa-developers/TM-Extractor/" } # --------------------------------------------------------------------------------------------------------------------- From 2b2eb0ea913f8e64ee4c0d14b96fd6da5d4910dc Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Mon, 8 Jan 2024 21:35:31 +0545 Subject: [PATCH 10/29] infra: terraform now sources modules from hotosm repo v1.0.0 --- scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl index 830ad6d5f8..670d9b2a80 100644 --- a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl +++ b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl @@ -11,7 +11,7 @@ # deployed version. terraform { # Sourcing from here rn, updating to a release from https://github.com/hotosm/TM-Extractor/ later. - source = "${local.base_source_url}?ref=feat-terraform" + source = "${local.base_source_url}?ref=v1.0.0" } # --------------------------------------------------------------------------------------------------------------------- @@ -26,7 +26,7 @@ locals { # Expose the base source URL so different versions of the module can be deployed in different environments. This will # be used to construct the terraform block in the child terragrunt configurations. - base_source_url = "git::https://github.com/naxa-developers/TM-Extractor/" + base_source_url = "git::https://github.com/hotosm/TM-Extractor/" } # --------------------------------------------------------------------------------------------------------------------- From 4170cc76949d08dd1af1e97437691af102111a53 Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Thu, 11 Jan 2024 09:19:28 +0545 Subject: [PATCH 11/29] readme: updated terraform lambda deployment docs --- scripts/aws/lambda/TM-Extractor/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/aws/lambda/TM-Extractor/README.md b/scripts/aws/lambda/TM-Extractor/README.md index 77d93be5fa..0fcae55da8 100644 --- a/scripts/aws/lambda/TM-Extractor/README.md +++ b/scripts/aws/lambda/TM-Extractor/README.md @@ -12,8 +12,10 @@ To get it running on your environment follow the following. This are list of things you will need. * AWS IAM User with permission for creating resources such as lambda function, cloudwatch group, cloudwatch event etc. See all resource [here](https://github.com/hotosm/TM-Extractor/blob/ec37c1318325c534b4ac47f057263050e6e92f03/main.tf). -* [Terraform](https://www.terraform.io/) -* [Terragrunt](https://terragrunt.gruntwork.io/) +* [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) or (OpenTofu)[https://opentofu.org/docs/intro/install/] +* [Terragrunt](https://terragrunt.gruntwork.io/docs/getting-started/install/#install-terragrunt) +> **_NOTE:_** Please check compatibility note. https://terragrunt.gruntwork.io/docs/getting-started/supported-versions/ +Tested on: Terraform v1.5.5 & terragrunt version v0.54.9. ### Environments From 42ee293a3d7199d9b9ffebf4c340331a22aa694f Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Thu, 11 Jan 2024 13:50:54 +0545 Subject: [PATCH 12/29] infra: modified terragrunt config to use aws_profile as defined --- scripts/aws/lambda/TM-Extractor/dev/env.hcl | 1 + scripts/aws/lambda/TM-Extractor/prod/env.hcl | 1 + scripts/aws/lambda/TM-Extractor/stag/env.hcl | 1 + .../aws/lambda/TM-Extractor/terragrunt.hcl | 42 ++++++++++++++----- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/scripts/aws/lambda/TM-Extractor/dev/env.hcl b/scripts/aws/lambda/TM-Extractor/dev/env.hcl index 58c33315ab..2b6e88ef37 100644 --- a/scripts/aws/lambda/TM-Extractor/dev/env.hcl +++ b/scripts/aws/lambda/TM-Extractor/dev/env.hcl @@ -4,6 +4,7 @@ locals { environment = "dev" account_name = "naxadevelopers" + aws_profile = "default" aws_region = "ap-south-1" application = "tasking-manager" team = "HOTOSM" diff --git a/scripts/aws/lambda/TM-Extractor/prod/env.hcl b/scripts/aws/lambda/TM-Extractor/prod/env.hcl index a7c2790117..9257d1b15b 100644 --- a/scripts/aws/lambda/TM-Extractor/prod/env.hcl +++ b/scripts/aws/lambda/TM-Extractor/prod/env.hcl @@ -4,6 +4,7 @@ locals { environment = "prod" account_name = "hotosm" + aws_profile = "default" aws_region = "ap-south-1" application = "tasking-manager" team = "HOTOSM" diff --git a/scripts/aws/lambda/TM-Extractor/stag/env.hcl b/scripts/aws/lambda/TM-Extractor/stag/env.hcl index 88c99ef25e..268873331c 100644 --- a/scripts/aws/lambda/TM-Extractor/stag/env.hcl +++ b/scripts/aws/lambda/TM-Extractor/stag/env.hcl @@ -4,6 +4,7 @@ locals { environment = "stag" account_name = "hotosm" + aws_profile = "default" aws_region = "ap-south-1" application = "tasking-manager" team = "HOTOSM" diff --git a/scripts/aws/lambda/TM-Extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/terragrunt.hcl index 9c6b18d006..fca61f78ac 100644 --- a/scripts/aws/lambda/TM-Extractor/terragrunt.hcl +++ b/scripts/aws/lambda/TM-Extractor/terragrunt.hcl @@ -10,6 +10,7 @@ locals { # Extract the variables we need for easy access account_name = local.environment_vars.locals.account_name + aws_profile = local.environment_vars.locals.aws_profile aws_region = local.environment_vars.locals.aws_region environment = local.environment_vars.locals.environment application = local.environment_vars.locals.application @@ -23,21 +24,39 @@ generate "provider" { path = "provider.tf" if_exists = "overwrite_terragrunt" contents = < Date: Mon, 29 Jan 2024 15:20:46 +0545 Subject: [PATCH 13/29] infra: moved raw_data_api & config_json to common inputs --- scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl | 4 +++- .../aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl | 2 -- .../aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl | 3 --- .../aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl | 2 -- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl index 670d9b2a80..9589d5712b 100644 --- a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl +++ b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl @@ -10,7 +10,7 @@ # needs to deploy a different module version, it should redefine this block with a different ref to override the # deployed version. terraform { - # Sourcing from here rn, updating to a release from https://github.com/hotosm/TM-Extractor/ later. + # Sourcing from https://github.com/hotosm/TM-Extractor/ source = "${local.base_source_url}?ref=v1.0.0" } @@ -43,4 +43,6 @@ inputs = { lambda_memory_size = "128" lambda_timeout = "14" lambda_cron_expression = "cron(0 0 * * ? *)" + raw_data_api = "https://api-prod.raw-data.hotosm.org/v1" + config_json = "config.json" } diff --git a/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl index 6660750c3c..ddb6710a01 100644 --- a/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl +++ b/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl @@ -29,6 +29,4 @@ include "envcommon" { inputs = { zip_output_dir = "files" active_projects_api_base_url = "https://tm.naxa.com.np/api/v2" - raw_data_api = "https://api-prod.raw-data.hotosm.org/v1" - config_json = "config.json" } diff --git a/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl index 132bbc5ff9..a189413910 100644 --- a/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl +++ b/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl @@ -27,8 +27,5 @@ include "envcommon" { # For production, we want to specify bigger instance classes and storage, so we specify override parameters here. These # inputs get merged with the common inputs from the root and the envcommon lambda.hcl inputs = { - zip_output_dir = "files" active_projects_api_base_url = "https://tasking-manager-tm4-production-api.hotosm.org/api/v2" - raw_data_api = "https://api-prod.raw-data.hotosm.org/v1" - config_json = "config.json" } diff --git a/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl index 337d70ca34..567fc8049b 100644 --- a/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl +++ b/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl @@ -29,6 +29,4 @@ include "envcommon" { inputs = { zip_output_dir = "files" active_projects_api_base_url = "https://tasking-manager-staging-api.hotosm.org/api/v2" - raw_data_api = "https://api-prod.raw-data.hotosm.org/v1" - config_json = "config.json" } From e3055fa5dc8a1c0ef4daebca8535aa620c24df6b Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Mon, 29 Jan 2024 15:40:35 +0545 Subject: [PATCH 14/29] readme: Updated readme with input documentation --- scripts/aws/lambda/TM-Extractor/README.md | 23 ++++++++++++++++++++ scripts/aws/lambda/TM-Extractor/dev/env.hcl | 1 + scripts/aws/lambda/TM-Extractor/prod/env.hcl | 1 + scripts/aws/lambda/TM-Extractor/stag/env.hcl | 1 + 4 files changed, 26 insertions(+) diff --git a/scripts/aws/lambda/TM-Extractor/README.md b/scripts/aws/lambda/TM-Extractor/README.md index 0fcae55da8..10315c0239 100644 --- a/scripts/aws/lambda/TM-Extractor/README.md +++ b/scripts/aws/lambda/TM-Extractor/README.md @@ -25,6 +25,29 @@ Before running deployment, make sure you have the following environment variable - `TF_VAR_active_projects_api_base_url`, Your [tasking-manager](https://github.com/hotosm/tasking-manager) instance. Defaults to https://tasking-manager-staging-api.hotosm.org/api/v2 +## Inputs and Local Variables +`env.hcl` within each deployment environment overrides the input vars. +``` +locals { + environment = "stag" + # Appends to all the resource names eg: tm-extrator-hotosm-lambda-stag + + account_name = "hotosm" + # Also appends to the resource names for better naming + + aws_profile = "default" + # Acts as AWS_PROFILE environment var, If you have multiple profile defined in ~/.aws/credentials, you can set these so each envionment deployment uses its own set of profiles. + + aws_region = "ap-south-1" + # The AWS region the resources should be created on. + + application = "tasking-manager" + team = "HOTOSM" + creator = "HOTOSM" + owner = "HOTOSM" + # Above are used for generating tags for AWS resources. +} +``` ## Plan and Apply - `cd` into `scripts/aws/lambda/TM-Extractor//tm-extractor`. diff --git a/scripts/aws/lambda/TM-Extractor/dev/env.hcl b/scripts/aws/lambda/TM-Extractor/dev/env.hcl index 2b6e88ef37..5d5d8c4cab 100644 --- a/scripts/aws/lambda/TM-Extractor/dev/env.hcl +++ b/scripts/aws/lambda/TM-Extractor/dev/env.hcl @@ -1,5 +1,6 @@ # Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to # feed forward to the child modules. +# Check ../README.md ## Inputs and Local Variables for details. locals { environment = "dev" diff --git a/scripts/aws/lambda/TM-Extractor/prod/env.hcl b/scripts/aws/lambda/TM-Extractor/prod/env.hcl index 9257d1b15b..f36d715ca1 100644 --- a/scripts/aws/lambda/TM-Extractor/prod/env.hcl +++ b/scripts/aws/lambda/TM-Extractor/prod/env.hcl @@ -1,5 +1,6 @@ # Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to # feed forward to the child modules. +# Check ../README.md ## Inputs and Local Variables for details. locals { environment = "prod" diff --git a/scripts/aws/lambda/TM-Extractor/stag/env.hcl b/scripts/aws/lambda/TM-Extractor/stag/env.hcl index 268873331c..bf584eb7ed 100644 --- a/scripts/aws/lambda/TM-Extractor/stag/env.hcl +++ b/scripts/aws/lambda/TM-Extractor/stag/env.hcl @@ -1,5 +1,6 @@ # Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to # feed forward to the child modules. +# Check ../README.md ## Inputs and Local Variables for details. locals { environment = "stag" From 740e3a282604a03230759b9b3707d6f624a700e7 Mon Sep 17 00:00:00 2001 From: mahesh-naxa Date: Mon, 29 Jan 2024 16:26:54 +0545 Subject: [PATCH 15/29] infra: bumped tm-extrator tag to v1.0.2 --- scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl index 9589d5712b..6f477a8855 100644 --- a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl +++ b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl @@ -11,7 +11,7 @@ # deployed version. terraform { # Sourcing from https://github.com/hotosm/TM-Extractor/ - source = "${local.base_source_url}?ref=v1.0.0" + source = "${local.base_source_url}?ref=v1.0.1" } # --------------------------------------------------------------------------------------------------------------------- From 80bff8cd50c415b8499e5d8ecadb4def76013f2b Mon Sep 17 00:00:00 2001 From: Deepak Pradhan Date: Tue, 30 Jan 2024 12:34:41 +0545 Subject: [PATCH 16/29] fix: footer dynamic on visibility with boolean key --- .../src/components/projectDetail/footer.js | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/projectDetail/footer.js b/frontend/src/components/projectDetail/footer.js index a14f13abd6..c3c5c9899c 100644 --- a/frontend/src/components/projectDetail/footer.js +++ b/frontend/src/components/projectDetail/footer.js @@ -10,39 +10,48 @@ import { AddToFavorites } from './favorites'; import { HorizontalScroll } from '../horizontalScroll'; import './styles.scss'; +import { ENABLE_EXPORT_TOOL } from '../../config'; const menuItems = [ { href: '#top', label: , + isVisibleCondition: true, }, { href: '#description', label: , + isVisibleCondition: true, }, { href: '#coordination', label: , + isVisibleCondition: true, }, { href: '#teams', label: , + isVisibleCondition: true, }, { href: '#questionsAndComments', label: , + isVisibleCondition: true, }, { href: '#contributions', label: , + isVisibleCondition: true, }, { href: '#downloadOsmData', label: , + isVisibleCondition: +ENABLE_EXPORT_TOOL === 1, }, { href: '#similarProjects', label: , + isVisibleCondition: true, }, ]; @@ -64,14 +73,20 @@ export const ProjectDetailFooter = ({ className, projectId }) => { containerClass={'.menu-items-container'} >
- {menuItems.map((menuItem, index) => ( - - - {menuItem.label} - - {index < menuItems.length - 1 && ·} - - ))} + {menuItems.map((menuItem, index) => { + if (menuItem.isVisibleCondition) { + return ( + + + {menuItem.label} + + {index < menuItems.length - 1 && ·} + + ); + } else { + return null; + } + })}
From abb9cf83855d5e4d14c9c5c7d3405f7579444e4f Mon Sep 17 00:00:00 2001 From: balex Date: Wed, 8 Nov 2023 11:09:13 -0700 Subject: [PATCH 17/29] Removed link and fixed tests --- frontend/src/components/userDetail/headerProfile.js | 7 ------- .../userDetail/tests/headerProfile.test.js | 12 ++++-------- frontend/src/views/tests/contributions.test.js | 2 +- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/frontend/src/components/userDetail/headerProfile.js b/frontend/src/components/userDetail/headerProfile.js index c366184efc..af57d13bd5 100644 --- a/frontend/src/components/userDetail/headerProfile.js +++ b/frontend/src/components/userDetail/headerProfile.js @@ -9,7 +9,6 @@ import { NextMappingLevel } from '../user/topBar'; import { UserOrganisations } from './userTeamsOrgs'; import { SectionMenu } from '../menu'; import OsmLogo from '../../assets/img/osm_logo.png'; -import MissingMapsLogo from '../../assets/img/organizations/missingmaps.png'; import SlackLogo from '../../assets/img/icons/slack.png'; import OsmChaLogo from '../../assets/img/icons/osm-cha.png'; import HdycNeisOneLogo from '../../assets/img/icons/hdyc-neis-one.png'; @@ -68,12 +67,6 @@ export const SocialMedia = ({ data }) => { {createLink('osm', data.username)}
-
  • -
    - Missing Maps{' '} - {createLink('missingmaps', data.username)} -
    -
  • OSM Cha Logo{' '} diff --git a/frontend/src/components/userDetail/tests/headerProfile.test.js b/frontend/src/components/userDetail/tests/headerProfile.test.js index b62cf98022..216289476a 100644 --- a/frontend/src/components/userDetail/tests/headerProfile.test.js +++ b/frontend/src/components/userDetail/tests/headerProfile.test.js @@ -27,8 +27,8 @@ describe('Social Media component', () => { , ); - // img tag for OSM, Missng Maps, Slack and HDYC Neis One - expect(screen.getAllByRole('img')).toHaveLength(5); + // img tag for OSM, Slack and HDYC Neis One + expect(screen.getAllByRole('img')).toHaveLength(4); // SVGs for Facebook, Linkedin, Twitter expect(container.querySelectorAll('svg').length).toBe(3); }); @@ -43,20 +43,16 @@ describe('Social Media component', () => { screen.getAllByRole('link', { name: 'johndoe', }), - ).toHaveLength(4); + ).toHaveLength(3); expect(screen.queryAllByRole('link', { name: 'johndoe' })[0]).toHaveAttribute( 'href', 'https://www.openstreetmap.org/user/johndoe', ); expect(screen.queryAllByRole('link', { name: 'johndoe' })[1]).toHaveAttribute( - 'href', - 'https://www.missingmaps.org/users/#/johndoe', - ); - expect(screen.queryAllByRole('link', { name: 'johndoe' })[2]).toHaveAttribute( 'href', 'https://osmcha.org/?filters={"users":[{"label":"johndoe","value":"johndoe"}]}', ); - expect(screen.getAllByRole('link', { name: 'johndoe' })[3]).toHaveAttribute( + expect(screen.getAllByRole('link', { name: 'johndoe' })[2]).toHaveAttribute( 'href', 'https://hdyc.neis-one.org/?johndoe', ); diff --git a/frontend/src/views/tests/contributions.test.js b/frontend/src/views/tests/contributions.test.js index d82f7f7425..cdac0760a6 100644 --- a/frontend/src/views/tests/contributions.test.js +++ b/frontend/src/views/tests/contributions.test.js @@ -69,7 +69,7 @@ describe('Contributions Page Index', () => { , ); - expect(screen.getAllByRole('link', { name: 'test_user' }).length).toBe(4); + expect(screen.getAllByRole('link', { name: 'test_user' }).length).toBe(3); }); }); From 5fd6442b4a5aac211e3c6d17c7f1b62673ed236b Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 12:45:35 +0000 Subject: [PATCH 18/29] build: add TM_DEV_PORT variable for traefik port change --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 459512fdd6..2978165f32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: image: traefik:v2.10 restart: always ports: - - "80:80" + - "${TM_DEV_PORT:-80}:80" volumes: - /var/run/docker.sock:/var/run/docker.sock:ro command: From aa25b3ad732c528d1f4d301d41b5cc8b53f4ba5d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 12:45:53 +0000 Subject: [PATCH 19/29] build: add container_name for all compose services consistency --- docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2978165f32..62ddffca73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: # Main application backend: <<: *backend - container_name: backend + container_name: tm-backend restart: always labels: - traefik.http.routers.backend.rule=(Host(`127.0.0.1`) || Host(`localhost`)) && PathPrefix(`/api/`) @@ -23,12 +23,13 @@ services: migration: <<: *backend - container_name: migration + container_name: tm-migration restart: on-failure command: flask db upgrade frontend: image: hotosm-tasking-manager:frontend + container_name: tm-frontend restart: always networks: - tm-web @@ -38,7 +39,7 @@ services: postgresql: image: postgis/postgis:14-3.3 - container_name: postgresql + container_name: tm-postgresql restart: always env_file: ${ENV_FILE:-tasking-manager.env} networks: @@ -46,6 +47,7 @@ services: traefik: image: traefik:v2.10 + container_name: tm-traefik restart: always ports: - "${TM_DEV_PORT:-80}:80" From ec4b363844370faeb6d9a48fb5c655d137012c42 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 12:46:11 +0000 Subject: [PATCH 20/29] docs: add TM_DEV_PORT to example.env --- example.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example.env b/example.env index 7425d3523a..dd5a77dfa0 100644 --- a/example.env +++ b/example.env @@ -19,6 +19,9 @@ TM_APP_API_URL=http://127.0.0.1:5000 # Defines the version of the API and will be used after /api/ on the url TM_APP_API_VERSION=v2 +# Override default port 80 during development +# TM_DEV_PORT=880 + # Information about the hosting organization TM_ORG_NAME="Humanitarian OpenStreetMap Team" TM_ORG_CODE=HOT From 28eb6db484c9d967a2b0849ea4081078d0943679 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 12:46:29 +0000 Subject: [PATCH 21/29] docs: update development setup info --- docs/developers/development-setup.md | 128 ++++++++++++++++++++++----- 1 file changed, 107 insertions(+), 21 deletions(-) diff --git a/docs/developers/development-setup.md b/docs/developers/development-setup.md index b941656d76..817053a168 100644 --- a/docs/developers/development-setup.md +++ b/docs/developers/development-setup.md @@ -1,7 +1,7 @@ -## Development setup +# Development setup -### Architecture +## Architecture The Tasking Manager is composed of two parts: @@ -10,13 +10,104 @@ The Tasking Manager is composed of two parts: The two parts can be developed independently of each other. +## OSM Auth + +The Tasking Manager uses OAuth2 with OSM to authenticate users. + +In order to use the frontend, you may need to create keys for OSM: + +1. [Login to OSM][1] + (_If you do not have an account yet, click the signup + button at the top navigation bar to create one_). + + Click the drop down arrow on the top right of the navigation bar + and select My Settings. + +2. Register your Tasking Manager instance to OAuth 2 applications. + + Put your login redirect url as `http://127.0.0.1:880/authorized/` + + > Note: `127.0.0.1` is required for debugging instead of `localhost` + > due to OSM restrictions. + +3. Permissions required: + - Read user preferences (read_prefs). + - Modify the map (write_api). + +4. Now save your Client ID and Client Secret for the next step. + +## Configure The Dot Env File + +1. Copy the `example.env` to `tasking-manager.env`. + + ```bash + cp example.env tasking-manager.env + ``` + +2. Uncomment or update the following variables + + ```dotenv + TM_DEV_PORT=880 + TM_APP_BASE_URL=http://127.0.0.1:880 + TM_APP_API_URL=http://127.0.0.1:880/api + # 'postgresql' if using docker, else 'localhost' or comment out + POSTGRES_ENDPOINT=postgresql + TM_REDIRECT_URI=http://127.0.0.1:880/authorized + TM_CLIENT_ID=from-previous-step + TM_CLIENT_SECRET=from-previous-step + ``` + + - Note that the port 880 can be swapped to any available port on + your system. + - If you change this, don't forget to update the OAuth login redirect + URL from the step above. + +> If you are a frontend developer and do not wish to configure the +> backend, you can use our staging server API. +> +> Update the variable: +> +>    `TM_APP_API_URL='https://tasking-manager-staging-api.hotosm.org'` +> +> before running the `yarn start` command. +> +> Be aware that the staging API can be offline while we are deploying +> newer versions to the staging server and that you'll not have access +> to some management views due to permissions. Check the +> [configuration](#configuration) section to learn more about how +> to configure Tasking Manager. + +For more details see the [configuration section](#configuration). + +## Docker + +The easiest option to get started with all components may be using Docker. + +### Requirements + +[Docker Engine](https://docs.docker.com/engine/install/) must be +available locally. + +### Running Tasking Manager + +Once the steps above have been complete, simply run: + +```bash +docker compose pull +docker compose build +docker compose --env-file tasking-manager.env up --detach +``` + +Tasking Manager should be available from: +[http://127.0.0.1:880](http://127.0.0.1:880) + +## Running Components Standalone + ### Frontend The client is the front-end user interface of the Tasking Manager. It is based on the React framework and you can find all files in the `frontend` directory. -If you don't want to setup a backend server, you can work on frontend development using our staging server API. Execute `export TM_APP_API_URL='https://tasking-manager-staging-api.hotosm.org'` before running the `yarn start` command. Be aware that the staging API can be offline while we are deploying newer versions to the staging server and that you'll not have access to some management views due to permissions. Check the [configuration](#configuration) section to learn more about how to configure Tasking Manager. - -**Dependencies** +#### Dependencies The following dependencies must be available _globally_ on your system: * Download and install [NodeJS LTS v12+](https://nodejs.org/en/) and [yarn](https://classic.yarnpkg.com/en/docs/install) @@ -149,7 +240,7 @@ We use create the database from the migrations directory. Check the instructions on how to setup a PostGIS database with [docker](#creating-a-local-postgis-database-with-docker) or on your -[local system](#non-docker). Then you can execute the following +[local system](#creating-a-local-postgis-database-without-docker). Then you can execute the following command to apply the migrations: ``` @@ -169,7 +260,7 @@ on your terminal (with the OS user that is the owner of the database): `psql -d -c "UPDATE users set role = 1 where username = ''"` -### API +## API If you plan to only work on the API you only have to build the backend architecture. Install the backend dependencies, and run the server: @@ -183,7 +274,7 @@ docs is also available on our [production](https://tasks.hotosm.org/api-docs) and [staging](https://tasks-stage.hotosm.org/api-docs/) instances. -#### API Authentication +### API Authentication In order to authenticate on the API, you need to have an Authorization Token. @@ -217,7 +308,7 @@ edit history of your user, selecting a changeset from the list, and then at the bottom link `Changeset XML` and it will be in the `uid` field of the XML returned. -#### API Authentication on remote instance +### API Authentication on remote instance To get your token on the production or staging Tasking Manager instances, sign in in the browser and then either: @@ -227,20 +318,14 @@ instances, sign in in the browser and then either: - inspect a network request and search for the `Authorization` field in the request headers section. -# Docker +## Additional Info -## Creating a local PostGIS database with Docker +### Creating a local PostGIS database with Docker If you're not able to connect to an existing tasking-manager DB, we have a [Dockerfile]() that will allow you to run PostGIS locally as follows. -### Dependencies - -Following must be available locally: - -* [Docker CE](https://www.docker.com/community-edition#/download) - ### Build & Run the PostGIS dockerfile 1. From the root of the project: @@ -279,21 +364,21 @@ export TM_DB=postgresql://hottm:hottm@localhost/tasking-manager 6. Refer to the rest of the instructions in the README to setup the DB and run the app. -# Non-Docker +## Creating a local PostGIS database without Docker -## Creating the PostGIS database +### Creating the PostGIS database It may be the case you would like to set up the database without using Docker for one reason or another. This provides you with a set of commands to create the database and export the database address to allow you to dive into backend development. -### Dependencies +#### Dependencies First, ensure that Postgresql and PostGIS are installed and running on your computer. -### Create the database user and database +#### Create the database user and database Assuming you have sudo access and the unix Postgresql owner is `postgres`: @@ -325,3 +410,4 @@ get a working version of the API running. * [Managing CI/CD with CircleCI](../sysadmins/ci-cd.md) * [Deployment Guide](../sysadmins/deployment.md) +[1]: https://www.openstreetmap.org/login "Login to OSM" From 576d007517d7c8fb79cc97f5f7d05ba8d8d12264 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 15:33:50 +0000 Subject: [PATCH 22/29] refactor: remove support for pep582 (PEP blocked) --- scripts/install/install_on_debian_10_buster.sh | 2 -- scripts/install/install_on_ubuntu_18_04.sh | 2 -- 2 files changed, 4 deletions(-) diff --git a/scripts/install/install_on_debian_10_buster.sh b/scripts/install/install_on_debian_10_buster.sh index 6590350837..a447c2bf42 100755 --- a/scripts/install/install_on_debian_10_buster.sh +++ b/scripts/install/install_on_debian_10_buster.sh @@ -32,8 +32,6 @@ git clone https://github.com/hotosm/tasking-manager.git && cd tasking-manager/ && pip install --upgrade pip && pip install --upgrade pdm && -pdm config --global python.use_venv False && -eval "$(pdm --pep582)" && pdm install && # Set up configuration diff --git a/scripts/install/install_on_ubuntu_18_04.sh b/scripts/install/install_on_ubuntu_18_04.sh index 1f4ce43b41..c7fa3a0540 100755 --- a/scripts/install/install_on_ubuntu_18_04.sh +++ b/scripts/install/install_on_ubuntu_18_04.sh @@ -32,8 +32,6 @@ git clone https://github.com/hotosm/tasking-manager.git && cd tasking-manager/ && pip install --upgrade pip && pip install --upgrade pdm && -pdm config --global python.use_venv False && -eval "$(pdm --pep582)" && pdm install && echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p && From a1a43856419212ae1e8ea87bec8eeb0018654f33 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 15:34:24 +0000 Subject: [PATCH 23/29] ci: pin pdm version to ==2.7.4 during functional tests --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f4e3d146c..a67393bb63 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,8 +81,7 @@ jobs: - run: sudo apt-get update - run: sudo apt-get -y install libgeos-dev # Required for shapely - run: sudo apt-get -y install proj-bin libproj-dev - - run: pip install --upgrade pip pdm - - run: pdm config --global python.use_venv False + - run: pip install --upgrade pip pdm==2.7.4 - run: pdm export --dev --without-hashes > requirements.txt - run: pip install -r requirements.txt - run: mkdir --mode 766 -p /tmp/logs From 9e96f8f401a57e24687199c50fc5b88ffe9caa42 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 2 Feb 2024 15:34:39 +0000 Subject: [PATCH 24/29] docs: clarify info about running backend api only --- docs/developers/development-setup.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/developers/development-setup.md b/docs/developers/development-setup.md index 817053a168..bbc95807fd 100644 --- a/docs/developers/development-setup.md +++ b/docs/developers/development-setup.md @@ -187,6 +187,7 @@ directory. To use that last option, follow the below instructions: - `TM_CLIENT_SECRET`=oauth-client-secret-key-from-openstreetmap - `TM_REDIRECT_URI`=oauth-client-redirect_uri - `TM_SCOPE`=oauth-client-scopes + - `TM_LOG_DIR=logs` In order to send email correctly, set these variables as well: - `TM_SMTP_HOST` @@ -196,20 +197,12 @@ In order to send email correctly, set these variables as well: - `TM_SMTP_USE_TLS=0` - `TM_SMTP_USE_SSL=1` (Either TLS or SSL can be set to 1 but not both) -#### Build +#### Install Dependencies * Install project dependencies: - * Linux/Mac (Option 1: pep582): - * First ensure the Python version in `pyproject.toml:requires-python` is installed on your system. - * ```pip install --upgrade pdm``` - * ```pdm config --global python.use_venv False``` - * ```pdm --pep582 >> ~/.bash_profile``` - * ```source ~/.bash_profile``` - * ```pdm install``` - * Linux/Mac (Option 2: pip (system/venv)): - * ```pip install --upgrade pdm``` - * ```pdm export --without-hashes > requirements.txt``` - * ```pip install requirements.txt``` + * First ensure the Python version in `pyproject.toml:requires-python` is installed on your system. + * ```pip install --upgrade pdm``` + * ```pdm install``` #### Tests @@ -265,7 +258,16 @@ on your terminal (with the OS user that is the owner of the database): If you plan to only work on the API you only have to build the backend architecture. Install the backend dependencies, and run the server: -`flask run --debug --reload` or `pdm run start` +```bash +# Install dependencies +pdm install + +# Run (Option 1) +pdm run start + +# Run (Option 2) +pdm run flask run --debug --reload +``` You can access the API documentation on [http://localhost:5000/api-docs](http://localhost:5000/api-docs), it From b72457476dfc6ae682b5a3c8da1f7f2eb7835548 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Sun, 11 Feb 2024 09:30:12 +0545 Subject: [PATCH 25/29] Added matomo trackevent option --- frontend/src/components/projectDetail/downloadOsmData.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/components/projectDetail/downloadOsmData.js b/frontend/src/components/projectDetail/downloadOsmData.js index d8cdfcc994..0d7ba1e619 100644 --- a/frontend/src/components/projectDetail/downloadOsmData.js +++ b/frontend/src/components/projectDetail/downloadOsmData.js @@ -83,6 +83,10 @@ export const DownloadOsmData = ({ projectMappingTypes, project }) => { var handle = window.open(downloadUrl); handle.blur(); window.focus(); + if (window._paq) { + // Check if Matomo tracking array (_paq) exists + window._paq.push(['trackEvent', 'OSMDownloads', 'Click', `${title}_${fileFormat}`]); + } // Check if the request was successful if (responsehead.ok) { setIsDownloadingState({ title: title, fileFormat: fileFormat, isDownloading: false }); From 96160f9da16a0323533ede553f08fe6474f7a6dc Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Sun, 11 Feb 2024 16:56:22 +0545 Subject: [PATCH 26/29] Update matomo configuration --- frontend/public/index.html | 56 +++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/frontend/public/index.html b/frontend/public/index.html index 58af457272..8d5db278e6 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -6,20 +6,23 @@ - - + + - - - - - + + + + + - - - - + + + + + style="top: 122px; bottom: 0; left: 0; right: 0; z-index: -1; display: none" + >
    +