Ckan 2.10 upgrade

.docker-compose-db.yaml

@@ -1,50 +1,47 @@
-version: '3.2'
-
 services:
-
   db:
     image: viderum/ckan-cloud-docker:db-latest
     build:
       context: db
     restart: always
     expose:
-    - "5432"
+      - "5432"
     env_file:
-    - docker-compose/db-secrets.sh
+      - docker-compose/db-secrets.sh
     volumes:
-    - db:/var/lib/postgresql/data
+      - db:/var/lib/postgresql/data
     networks:
-    - ckan-multi
+      - ckan-multi
 
   jobs-db:
     image: postgres
     restart: always
     expose:
-    - "5432"
+      - "5432"
     env_file:
-    - docker-compose/db-secrets.sh
+      - docker-compose/db-secrets.sh
     volumes:
-    - jobs-db:/var/lib/postgresql/data
+      - jobs-db:/var/lib/postgresql/data
     networks:
-    - ckan-multi
+      - ckan-multi
 
   datastore-db:
     image: viderum/ckan-cloud-docker:db-latest
     restart: always
     expose:
-    - "5432"
+      - "5432"
     env_file:
-    - docker-compose/datastore-db-secrets.sh
+      - docker-compose/datastore-db-secrets.sh
     volumes:
-    - datastore-db:/var/lib/postgresql/data
+      - datastore-db:/var/lib/postgresql/data
     networks:
-    - ckan-multi
+      - ckan-multi
 
   ckan:
     depends_on:
-    - db
-    - jobs-db
-    - datastore-db
+      - db
+      - jobs-db
+      - datastore-db
 
 volumes:
   db:

.docker-compose.vital-strategies-theme.yaml

@@ -1,5 +1,3 @@
-version: '3.2'
-
 services:
 
   proxy:
@@ -21,34 +19,35 @@ services:
     build:
       context: ckan
       args:
-        CKAN_BRANCH: ckan-2.7.3
+        CKAN_BRANCH: ckan-2.10.4
         EXTRA_PACKAGES: cron
         EXTRA_FILESYSTEM: "./overrides/vital-strategies/filesystem/"
-        PRE_INSTALL: "sed  -i -e 's/psycopg2==2.4.5/psycopg2==2.7.7/g' ~/venv/src/ckan/requirements.txt"
         POST_INSTALL: |
-          install_standard_ckan_extension_github -r ViderumGlobal/ckanext-querytool -b v2.1.2 &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b dev/python3-upgrade &&\
           install_standard_ckan_extension_github -r ckan/ckanext-geoview && \
-          install_standard_ckan_extension_github -r okfn/ckanext-sentry && \
-          install_standard_ckan_extension_github -r ckan/ckanext-googleanalytics -b v2.0.2 && \
-          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b fix-null-content-type && \
+          #install_standard_ckan_extension_github -r datopian/ckanext-sentry -b 2.10 && \
+          install_standard_ckan_extension_github -r datopian/ckanext-gtm && \
+          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l en -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l es -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l fr -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l km -f && \
           cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l pt_BR -f && \
-          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l zh_CN -f
+          cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l zh_Hans_CN -f
     environment:
       - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme-
+    #ports: # Uncomment to expose CKAN on localhost for development
+    #  - 5000:5000
 
   jobs:
     image: viderum/ckan-cloud-docker:ckan-latest-vital-strategies-theme
     build:
       context: ckan
       args:
-        CKAN_BRANCH: ckan-2.7.3
+        CKAN_BRANCH: ckan-2.10.4
         POST_INSTALL: |
-          install_standard_ckan_extension_github -r keitaroinc/ckanext-s3filestore -b main &&\
-          install_standard_ckan_extension_github -r datopian/ckanext-querytool &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 &&\
+          install_standard_ckan_extension_github -r datopian/ckanext-querytool -b dev/python3-upgrade &&\
           install_standard_ckan_extension_github -r ckan/ckanext-geoview
     environment:
       - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme-

DEPLOYING.md

@@ -115,7 +115,7 @@ In addition to SSL specific configuration, there is one more line you need to ad
 
 This should be enough for the basic installation. In case you need to tweak versions or other initialization parameters for CKAN, you need these two files:
 
-* `docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini`
+* `docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini`
   This is the file used to generate the CKAN main configuration file.
 
 * `.docker-compose.{instance-id}-theme.yaml`
@@ -196,7 +196,7 @@ bash migrate_filestorage.sh $HOST $ACCESS_KEY $SECRET_KEY $BUCKET $STORAGE_PATH
 
 After migration rebuild the SOLR search index.
 ```
-sudo make shell O=<<instance-id>> S=ckan C='/usr/local/bin/ckan-paster --plugin=ckan search-index rebuild  -c /etc/ckan/production.ini'
+sudo make shell O=<<instance-id>> S=ckan C='/usr/local/bin/ckan-paster --plugin=ckan search-index rebuild  -c /etc/ckan/ckan.ini'
 ```
 
 ## Debugging
@@ -255,10 +255,10 @@ POST_INSTALL: |
   install_standard_ckan_extension_github -r datopian/ckanext-s3filestore &&\
 ```
 
-And add extension to the list of plugins in `docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini.template`
+And add extension to the list of plugins in `docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini.template`
 
 ```
-# in docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini.template
+# in docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini.template
 ckan.plugins = image_view
    ...
    stats
@@ -268,7 +268,7 @@ ckan.plugins = image_view
 Note: depending on extension you might also need to update extensions related configurations in the same file. If needed this type of information is ussually included in extension REAMDE.
 
 ```
-# in docker-compose/ckan-conf-templates/{instance-id}-theme-production.ini.template
+# in docker-compose/ckan-conf-templates/{instance-id}-theme-ckan.ini.template
 ckanext.s3filestore.aws_access_key_id = Your-Access-Key-ID
 ckanext.s3filestore.aws_secret_access_key = Your-Secret-Access-Key
 ckanext.s3filestore.aws_bucket_name = a-bucket-to-store-your-stuff

Makefile

@@ -2,20 +2,24 @@
 
 COMPOSE_FILES = -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.$O-theme.yaml
 
+DATAPUSHER_TYPE ?= datapusher
+
 start:
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
 	docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron
 
 stop:
 	docker-compose $(COMPOSE_FILES) stop
 
 build:
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
 	docker-compose $(COMPOSE_FILES) build
 
 pull:
 	docker-compose $(COMPOSE_FILES) pull
 
 shell:
-	docker-compose $(COMPOSE_FILES) exec $S $C
+	docker-compose $(COMPOSE_FILES) exec -it $S sh -c 'if command -v bash > /dev/null 2>&1; then exec bash; else exec sh; fi'
 
 down:
 	docker-compose $(COMPOSE_FILES) down
@@ -36,10 +40,10 @@ exec:
 	docker-compose $(COMPOSE_FILES) exec $S $C
 
 user:
-	docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan user add $U password=$P email=$E -c /etc/ckan/production.ini
+	docker-compose $(COMPOSE_FILES) exec ckan ckan -c /etc/ckan/ckan.ini user add $U password=$P email=$E
 
 sysadmin:
-	docker-compose $(COMPOSE_FILES) exec ckan /usr/local/bin/ckan-paster --plugin=ckan sysadmin add $U -c /etc/ckan/production.ini
+	docker-compose $(COMPOSE_FILES) exec ckan ckan -c /etc/ckan/ckan.ini sysadmin add $U
 
 secret:
 	python create_secrets.py
@@ -49,6 +53,17 @@ cron:
 
 clean-rebuild:
 	docker-compose $(COMPOSE_FILES) down -v
-	docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs docker rmi -f
+	docker images -a | grep "ckan-cloud-docker" | awk '{print $$3}' | xargs -r docker rmi -f
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
 	docker-compose $(COMPOSE_FILES) build --no-cache
-	docker-compose $(COMPOSE_FILES) up -d --build nginx && make cron
+	@export DATAPUSHER_DIRECTORY=$(DATAPUSHER_TYPE) && \
+	docker-compose $(COMPOSE_FILES) up -d nginx && make cron
+
+backup-db:
+	docker-compose $(COMPOSE_FILES) exec -T db pg_dump -U postgres --format=custom -d ckan > ckan.dump
+	docker-compose ${COMPOSE_FILES} exec -T ckan sh -c "cd /var/lib/ckan && tar -czf /tmp/ckan_data.tar.gz data"
+	docker cp $$(docker-compose ${COMPOSE_FILES} ps -q ckan):/tmp/ckan_data.tar.gz ckan_data.tar.gz
+	docker-compose $(COMPOSE_FILES) exec -T datastore-db pg_dump -U postgres --format=custom -d datastore > datastore.dump
+
+upgrade-db:
+	./db/migration/upgrade_databases.sh "$(COMPOSE_FILES)"

README.md

@@ -92,7 +92,7 @@ In your project specific `docker-compose` file, you must expose port 5000 for CK
 
 ### Remove unused plugins from CKAN
 
-Before building and starting the environment, make sure you only have the required plugins enabled. If you're using a pre-defined project template for local testing, you might not need some of the included extensions, such as `ckanext-googleanalytics` or `ckanext-sentry`. For example, if you want to use the `vital-strategies` project template, you should remove the following plugins from the `.ini` file (found in `docker-compose/ckan-conf-templates/vital-strategies-theme-production.ini`) to avoid issues (unless you want to properly configure them):
+Before building and starting the environment, make sure you only have the required plugins enabled. If you're using a pre-defined project template for local testing, you might not need some of the included extensions, such as `ckanext-googleanalytics` or `ckanext-sentry`. For example, if you want to use the `vital-strategies` project template, you should remove the following plugins from the `.ini` file (found in `docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini`) to avoid issues (unless you want to properly configure them):
 
 ```
 ckan.plugins = image_view
@@ -154,7 +154,7 @@ Create a CKAN admin user
 
 ```
 docker-compose exec ckan ckan-paster --plugin=ckan \
-    sysadmin add -c /etc/ckan/production.ini admin password=12345678 email=admin@localhost
+    sysadmin add -c /etc/ckan/ckan.ini admin password=12345678 email=admin@localhost
 ```
 
 Login to CKAN at http://nginx:8080 with username `admin` and password `12345678`
@@ -187,6 +187,12 @@ Stop the environment:
 make stop O=vital-strategies
 ```
 
+Enter a container:
+
+```
+make shell O=vital-strategies S=SERVICE_NAME
+```
+
 Make a user:
 
 ```
@@ -250,8 +256,8 @@ This allows to test different CKAN configurations and extension combinations
 Duplicate the CKAN default configuration:
 
 ```
-cp docker-compose/ckan-conf-templates/production.ini.template \
-   docker-compose/ckan-conf-templates/my-ckan-production.ini.template
+cp docker-compose/ckan-conf-templates/ckan.ini.template \
+   docker-compose/ckan-conf-templates/my-ckan-ckan.ini.template
 ```
 
 Edit the duplicated file and modify the settings, e.g. add the extensions to the `plugins` configuration and any additional required extension configurations.
@@ -398,3 +404,87 @@ You might need to reload the solr collection after recreate:
 ```
 curl "http://localhost:8983/solr/admin/collections?action=RELOAD&name=${INSTANCE_ID}&wt=json"
 ```
+
+## Migrating to CKAN 2.10 and Python 3
+
+>**Note**: As of January 1, 2020, Python 2 is no longer officially supported. If you're running CKAN 2.7 with Python 2, it's highly recommended to migrate to CKAN 2.10 with Python 3. The latest version of this repo also no longer supports CKAN < 2.10 and Python < 3. If you must stick with those versions for now, you will need to maintain your local copy of this repo yourself.
+
+All of the following commands should be run in `ckan-cloud-docker` (unless stated otherwise). In the examples below, I'm using the `vital-strategies` project template as an example. Replace `vital-strategies` with the name of your project template.
+
+>**Note**: Depending on any custom configurations you have, you might need to adjust the variables in `db/migration/upgrade_databases.sh` (and others, such as your custom `docker-compose` file, or your custom `.ini` file) to match your setup.
+
+1. Start up your _current_ instance (if it's not running already—**don't pull the latest changes yet**): `make start O=vital-strategies`
+2. Backup the DBs: `make backup-db O=vital-strategies` (confirm that you have `ckan.dump`, `datastore.dump` and `ckan_data.tar.gz` in the current directory after running this command)
+3. Stop the containers: `make stop O=vital-strategies`
+4. Pull the latest changes: `git pull`
+5. Specify if you want to use `datapusher-plus` (you can skip this step to stick with vanilla `datapusher`): `export DATAPUSHER_TYPE=datapusher-plus` (always do this in the current terminal session before starting up CKAN 2.10/Python 3 with `ckan-cloud-docker`)
+6. Create secrets: `make secret` (follow the prompts)
+7. Clean and rebuild: `make clean-rebuild O=vital-strategies`
+8. Run the upgrade script: `make upgrade-db O=vital-strategies`
+9. Stop the containers: `make stop O=vital-strategies` (copy the API token that's output at the end, for the next step)
+10. Run `make secret` again and paste the token when prompted (step 13—"Enter Datapusher API token")
+11. Start the containers: `make start O=vital-strategies`
+12. Test and confirm that the migration was successful
+
+>**Note**: The first time you visit the DataStore tab for a given resource, it will say "Error: cannot connect to datapusher". If you click "Upload to DataStore", this error will go away and the process will complete as expected.
+
+>**Important**: It's recommended to make copies of `ckan.dump`, `datastore.dump` and `ckan_data.tar.gz` and move them off of the server, if possible. If anything goes wrong, and you must revert to the old CKAN 2.7 instance, you can restore it by following the steps below:
+
+1. Stop the containers: `make stop O=vital-strategies`
+2. Checkout the last CKAN 2.7 commit: `git checkout d3bdc178a1726ada331b47157b92123cdec82b12`
+3. Create secrets (you probably don't need to do this, but go through the process and make sure your previously entered values are correct): `make secret` (follow the prompts)
+4. Clean and rebuild: `make clean-rebuild O=vital-strategies`
+5. Restore the DBs (_note_: the prior version of this repo doesn't have a command for this—you must do it manually):
+    1. Restore the CKAN DB: `docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.<YOUR_PROJECT>-theme.yaml exec -T db pg_restore -U postgres --verbose --create --clean --if-exists -d postgres < ckan.dump`
+    2. Restore the DataStore DB: `docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.<YOUR_PROJECT>-theme.yaml exec -T datastore-db pg_restore -U postgres --verbose --create --clean --if-exists -d postgres < datastore.dump`
+    3. Restore the CKAN data:
+        1. `docker cp ckan_data.tar.gz $(docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.<YOUR_PROJECT>-theme.yaml ps -q ckan):/tmp/ckan_data.tar.gz`
+        2. `docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.<YOUR_PROJECT>-theme.yaml exec -T ckan bash -c "tar -xzf /tmp/ckan_data.tar.gz -C /tmp/ && cp -r /tmp/data/* /var/lib/ckan/data/ && chown -R ckan:ckan /var/lib/ckan/data"`
+    4. Set datastore permissions:
+        1. Enter your `ckan` container: `docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.<YOUR_PROJECT>-theme.yaml exec ckan bash`
+        2. Create a new file in your `ckan` container, `ckan.sql`, with the following contents:
+           ```
+           \connect "datastore"
+
+           -- revoke permissions for the read-only user
+           REVOKE CREATE ON SCHEMA public FROM PUBLIC;
+           REVOKE USAGE ON SCHEMA public FROM PUBLIC;
+
+           GRANT CREATE ON SCHEMA public TO "postgres";
+           GRANT USAGE ON SCHEMA public TO "postgres";
+
+           -- grant select permissions for read-only user
+           GRANT CONNECT ON DATABASE "datastore" TO "readonly";
+           GRANT USAGE ON SCHEMA public TO "readonly";
+
+           -- grant access to current tables and views to read-only user
+           GRANT SELECT ON ALL TABLES IN SCHEMA public TO "readonly";
+
+           -- grant access to new tables and views by default
+           ALTER DEFAULT PRIVILEGES FOR USER "postgres" IN SCHEMA public
+              GRANT SELECT ON TABLES TO "readonly";
+
+           -- a view for listing valid table (resource id) and view names
+           CREATE OR REPLACE VIEW "_table_metadata" AS
+               SELECT DISTINCT
+                   substr(md5(dependee.relname || COALESCE(dependent.relname, '')), 0, 17) AS "_id",
+                   dependee.relname AS name,
+                   dependee.oid AS oid,
+                   dependent.relname AS alias_of
+                   -- dependent.oid AS oid
+               FROM
+                   pg_class AS dependee
+                   LEFT OUTER JOIN pg_rewrite AS r ON r.ev_class = dependee.oid
+                   LEFT OUTER JOIN pg_depend AS d ON d.objid = r.oid
+                   LEFT OUTER JOIN pg_class AS dependent ON d.refobjid = dependent.oid
+               WHERE
+                   (dependee.oid != dependent.oid OR dependent.oid IS NULL) AND
+                   (dependee.relname IN (SELECT tablename FROM pg_catalog.pg_tables)
+                       OR dependee.relname IN (SELECT viewname FROM pg_catalog.pg_views)) AND
+                   dependee.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname='public')
+                       ORDER BY dependee.oid DESC;
+           ALTER VIEW "_table_metadata" OWNER TO "postgres";
+           GRANT SELECT ON "_table_metadata" TO "readonly";
+           ```
+        3. While still in your `ckan` container, get your `sqlalchemy.url`: `cat /etc/ckan/production.ini | grep sqlalchemy.url` (for example, `postgresql://ckan:123456@db/ckan`)
+        4. Set the permissions by running: `cat ckan.sql | psql <YOUR_SQLALCHEMY_URL>` (for example, `cat ckan.sql | psql postgresql://ckan:123456@db/ckan`)

TESTING.md

@@ -56,7 +56,7 @@ Once you see a successful response, create a CKAN admin user:
 ```
 $ docker-compose -f docker-compose.yaml -f .docker-compose-db.yaml -f .docker-compose.datagov-theme.yaml \
 	  exec ckan ckan-paster --plugin=ckan \
-      sysadmin add -c /etc/ckan/production.ini admin password=12345678 email=admin@localhost
+      sysadmin add -c /etc/ckan/ckan.ini admin password=12345678 email=admin@localhost
 ```
 
 You should see the following prompt:
@@ -153,7 +153,7 @@ $ paster datastore set-permissions -c test-core.ini | psql -h datastore-db -U po
 
 Solr is already configured as 'multi-core'. To verify it, you may run the following command inside the `ckan` container:
 ```
-$ grep solr_url /etc/ckan/production.ini
+$ grep solr_url /etc/ckan/ckan.ini
 # Possible outputs:
 # single-core: solr_url = http://solr:8983/solr
 # multi-core:  solr_url = http://solr:8983/solr/ckan

cca-operator/update-instance.sh

@@ -213,7 +213,7 @@ else
     CKAN_ADMIN_PASSWORD=$(python3 -c "import binascii,os;print(binascii.hexlify(os.urandom(12)).decode())")
     echo y \
         | kubectl $KUBECTL_GLOBAL_ARGS -n ${INSTANCE_NAMESPACE} exec -it ${CKAN_POD_NAME} -- bash -c \
-            "ckan-paster --plugin=ckan sysadmin -c /etc/ckan/production.ini add admin password=${CKAN_ADMIN_PASSWORD} email=${CKAN_ADMIN_EMAIL}" \
+            "ckan -c /etc/ckan/ckan.ini sysadmin add admin password=${CKAN_ADMIN_PASSWORD} email=${CKAN_ADMIN_EMAIL}" \
                 > /dev/stderr &&\
     kubectl $KUBECTL_GLOBAL_ARGS -n "${INSTANCE_NAMESPACE}" \
         create secret generic ckan-admin-password "--from-literal=CKAN_ADMIN_PASSWORD=${CKAN_ADMIN_PASSWORD}"