Merge pull request #940 from GFDRR/fix-transifex

Use new transifex API in build
GFDRR · Jun 27, 2023 · dbb037a · dbb037a
2 parents 5d3dbc3 + 7cb0383
commit dbb037a
Show file tree

Hide file tree

Showing 12 changed files with 42 additions and 39 deletions.
diff --git a/.env.tmpl b/.env.tmpl
@@ -16,6 +16,5 @@ AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
 AWS_BUCKET_NAME=${AWS_BUCKET_NAME}
 ANALYTICS=${ANALYTICS}
 BROKER_URL=${BROKER_URL}
-TX_USR=${TX_USR}
-TX_PWD=${TX_PWD}
+TX_TOKEN=${TX_TOKEN}
 HTPASSWORDS=${HTPASSWORDS}
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -13,8 +13,7 @@ jobs:
     env:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
       DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-      TX_USR: ${{ secrets.TX_USR }}
-      TX_PWD: ${{ secrets.TX_PWD }}
+      TX_TOKEN: ${{ secrets.TX_TOKEN }}
     steps:
       - uses: actions/checkout@v3
       - name: Build

diff --git a/.tx/config b/.tx/config
@@ -1,16 +1,20 @@
 [main]
 host = https://www.transifex.com
 
-[gfdrr-thinkhazard.ui]
-source_file = /tmp/thinkhazard.pot
+[o:gfdrr-labs:p:gfdrr-thinkhazard:r:ui]
 type = PO
+source_file = /tmp/thinkhazard.pot
 source_lang = en
-trans.es = thinkhazard/locale/es/LC_MESSAGES/thinkhazard.po
-trans.fr = thinkhazard/locale/fr/LC_MESSAGES/thinkhazard.po
+file_filter = thinkhazard/locale/<lang>/LC_MESSAGES/thinkhazard.po
 
-[gfdrr-thinkhazard.database]
+[o:gfdrr-labs:p:gfdrr-thinkhazard:r:database]
+type = PO
 source_file = /tmp/thinkhazard-database.pot
+source_lang = en
+file_filter = /tmp/thinkhazard-database-<lang>.po
+
+[o:gfdrr-labs:p:gfdrr-thinkhazard:r:test--database]
 type = PO
+source_file = /tmp/thinkhazard-database.pot
 source_lang = en
-trans.es = /tmp/thinkhazard-database-es.po
-trans.fr = /tmp/thinkhazard-database-fr.po
+file_filter = /tmp/thinkhazard-database-<lang>.po
diff --git a/Dockerfile b/Dockerfile
@@ -12,14 +12,19 @@ ENV PYTHONUNBUFFERED 1
 RUN apt-get update && apt-get install -y \
     # Install postgis for shp2pgsql as ogr2ogr from distrib is not compatible with PostgreSQL 12
     postgis \
-    git python-numpy gdal-bin libgdal-dev tidy gnupg2 unzip \
+    curl git python-numpy gdal-bin libgdal-dev tidy gnupg2 unzip \
     # pyppeteer dependencies (cf https://github.com/puppeteer/puppeteer/issues/1345)
     gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget \
     && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
     && echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" >  /etc/apt/sources.list.d/pgdg.list \
     && apt-get update && apt-get install -y postgresql-client-12 \
     && rm -rf /var/lib/apt/lists/*
 
+RUN curl -L -o /tmp/tx-linux-amd64.tar.gz https://github.com/transifex/cli/releases/download/v1.6.7/tx-linux-amd64.tar.gz \
+    && tar -C /usr/local/bin -xf /tmp/tx-linux-amd64.tar.gz tx \
+    && chmod +x /usr/local/bin/tx \
+    && rm -f /tmp/tx-linux-amd64.tar.gz
+
 ENV HOME=/home/user \
     NODE_PATH=/opt/thinkhazard/node_modules
 
@@ -91,10 +96,8 @@ RUN pip install --no-cache-dir -r /app/requirements-dev.txt \
 WORKDIR /app
 COPY . /app/
 
-ARG TX_USR
-ARG TX_PWD
-RUN TX_USR=$TX_USR \
-    TX_PWD=$TX_PWD \
+ARG TX_TOKEN
+RUN TX_TOKEN=$TX_TOKEN \
     make -f docker.mk build
 
 RUN pip install --no-deps -e .

diff --git a/Makefile b/Makefile
@@ -95,7 +95,8 @@ compile_catalog:
 
 .PHONY: test
 test: ## Run automated tests
-	$(DOCKER_CMD) nosetests -v
+	# $(DOCKER_CMD) nosetests -v
+	$(DOCKER_CMD) pytest -vv --cov=thinkhazard tests
 
 .PHONY: bash
 test-bash: ## Open bash in a test container
@@ -133,15 +134,13 @@ cleanall: clean
 .PHONY: docker_build_thinkhazard
 docker_build_thinkhazard:
 	docker build \
-		--build-arg TX_USR=${TX_USR} \
-		--build-arg TX_PWD=${TX_PWD} \
+		--build-arg TX_TOKEN=${TX_TOKEN} \
 		--target app -t camptocamp/thinkhazard .
 
 .PHONY: docker_build_builder
 docker_build_builder:
 	docker build \
-		--build-arg TX_USR=${TX_USR} \
-		--build-arg TX_PWD=${TX_PWD} \
+		--build-arg TX_TOKEN=${TX_TOKEN} \
 		--target builder -t camptocamp/thinkhazard-builder .
 
 .PHONY: docker_build_testdb

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -25,8 +25,7 @@ services:
       - hazardsets:/tmp/hazardsets
     environment: &env
       - INI_FILE
-      - TX_USR
-      - TX_PWD
+      - TX_TOKEN
       - PGHOST
       - PGPORT
       - PGUSER_PUBLIC

diff --git a/docker.mk b/docker.mk
@@ -52,16 +52,15 @@ compile_catalog: \
 	msgfmt -o $@ $<
 
 thinkhazard/locale/%/LC_MESSAGES/thinkhazard.po: $(HOME)/.transifexrc
-	tx pull -r gfdrr-thinkhazard.ui
+	tx pull --translations --languages=$* --resources=gfdrr-thinkhazard.ui --force
 	touch `find thinkhazard/locale/ -name '*.po' 2> /dev/null`
 
 .INTERMEDIATE: $(HOME)/.transifexrc
 $(HOME)/.transifexrc:
 	echo "[https://www.transifex.com]" > $@
-	echo "hostname = https://www.transifex.com" >> $@
-	echo "username = $(TX_USR)" >> $@
-	@echo "password = $(TX_PWD)" >> $@
-	echo "token =" >> $@
+	echo "rest_hostname = https://rest.api.transifex.com" >> $@
+	@echo "token = $(TX_TOKEN)" >> $@
+	cat $@
 
 
 check: flake8 jshint

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -7,4 +7,6 @@ mock
 nose
 pep8-naming
 pydevd
+pytest
+pytest-cov
 WebTest
diff --git a/requirements.txt b/requirements.txt
@@ -1,17 +1,19 @@
 APScheduler
 Babel
 PyPDF2
-SQLAlchemy
+SQLAlchemy<2
 alembic
 asyncio
 boto3
+chameleon  # needed by lingua
 colorlog
 geoalchemy2
 gunicorn
 httplib2
-jinja2
+jinja2<3.1
 lingua
 markdown
+numpy<1.24  # for compatibility with rasterio 1.3.7
 papyrus
 passlib
 paste
@@ -35,7 +37,6 @@ shapely
 shapely
 simplejson
 transaction
-transifex-client
 waitress
 zope.sqlalchemy
 celery

diff --git a/thinkhazard/scripts/tx-init b/thinkhazard/scripts/tx-init
@@ -2,8 +2,6 @@
 
 cat <<EOF > ${HOME}/.transifexrc
 [https://www.transifex.com]
-hostname = https://www.transifex.com
-username = ${TX_USR}
-password = ${TX_PWD}
-token =
+rest_hostname = https://rest.api.transifex.com
+token = ${TX_TOKEN}
 EOF
diff --git a/thinkhazard/scripts/tx-pull-db b/thinkhazard/scripts/tx-pull-db
@@ -1,7 +1,7 @@
-#!/bin/bash
+#!/bin/bash -e
 
 $(dirname $0)/tx-init
 
-tx pull -s -r gfdrr-thinkhazard.database ${TX_BRANCH:+-b "$TX_BRANCH"}
+tx pull --translations --languages=es,fr --resources=gfdrr-thinkhazard.database --force
 
 import_l10n -v
diff --git a/thinkhazard/scripts/tx-push-db b/thinkhazard/scripts/tx-push-db
@@ -1,7 +1,7 @@
-#!/bin/bash
+#!/bin/bash -e
 
 $(dirname $0)/tx-init
 
 pot-create -c lingua.cfg -o /tmp/thinkhazard-database.pot thinkhazard/dont_remove_me.db-i18n
 
-tx push -s -r gfdrr-thinkhazard.database ${TX_BRANCH:+-b "$TX_BRANCH"}
+tx push --source --resources=gfdrr-thinkhazard.database