From d887e89291adfeb17d38f0dcd47e6a3b1bd3188a Mon Sep 17 00:00:00 2001 From: Alicia Date: Wed, 1 Jun 2022 10:50:05 +0200 Subject: [PATCH 1/5] ETL Pipe for download and ingest EEZ data into the geoprocessingdb --- data/data_download/Makefile | 7 ++++- data/data_download/eez/Makefile | 50 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 data/data_download/eez/Makefile diff --git a/data/data_download/Makefile b/data/data_download/Makefile index 156895651c..ce45f4da9c 100644 --- a/data/data_download/Makefile +++ b/data/data_download/Makefile @@ -1,11 +1,16 @@ .DEFAULT_GOAL := seed-data -seed-data: seed-gadm seed-wdpa seed-demo-features-species seed-demo-features-bioregion seed-ecosystems +seed-data: seed-gadm seed-eez seed-wdpa seed-demo-features-species seed-demo-features-bioregion seed-ecosystems + seed-gadm: @echo "Starting seeding gadm data... " @time $(MAKE) -C ./gadm_3.6 import +seed-eez: + @echo "Starting seeding gadm data... " + @time $(MAKE) -C ./eez import + seed-wdpa: @echo "Starting seeding wdpa data... " @time $(MAKE) -C ./wdpa import diff --git a/data/data_download/eez/Makefile b/data/data_download/eez/Makefile new file mode 100644 index 0000000000..2785a14e95 --- /dev/null +++ b/data/data_download/eez/Makefile @@ -0,0 +1,50 @@ +.PHONY: import +# MAKEFLAGS := --jobs=$(shell nproc) +# MAKEFLAGS += --output-sync=target + +MarxanUser:=$(shell psql -X -A -t "postgresql://$$API_POSTGRES_USER:$$API_POSTGRES_PASSWORD@$$API_POSTGRES_HOST:$$API_POSTGRES_PORT/$$API_POSTGRES_DB" -c "select id from users limit 1") +import: data/eez/eez_v11_simp.geojson + ogr2ogr -makevalid \ + -update -append \ + -geomfield the_geom \ + --config OGR_TRUNCATE NO \ + -nln admin_regions -nlt PROMOTE_TO_MULTI \ + -t_srs EPSG:4326 -a_srs EPSG:4326 \ + -f PostgreSQL PG:"dbname=$$GEO_POSTGRES_DB host=$$GEO_POSTGRES_HOST \ + port=$$GEO_POSTGRES_PORT user=$$GEO_POSTGRES_USER password=$$GEO_POSTGRES_PASSWORD" $< \ + -sql "select *,'$(MarxanUser)' as created_by from \"$$(basename -s .geojson "$<")\""; + +data/eez/eez_v11_simp.geojson: data/eez/eez_v11.shp + mapshaper-xl -i $< snap \ + -simplify 25% planar keep-shapes \ + -filter-islands min-vertices=3 min-area=10000m2 remove-empty \ + -filter-slivers min-area=10000m2 remove-empty \ + -clean rewind \ + -each 'level= "eez"' \ + -rename-fields GEONAME=NAME_0,ISO_SOV1=ISO3 \ + -drop fields=MRGID,MRGID_TER1,MRGID_SOV1,TERRITORY1,\ + ISO_TER1,SOVEREIGN1,MRGID_TER2,MRGID_SOV2,TERRITORY2,ISO_TER2,SOVEREIGN2,\ + MRGID_TER3,MRGID_SOV3,TERRITORY3,ISO_TER3,SOVEREIGN3,X_1,Y_1,AREA_KM2,ISO_SOV2,\ + ISO_SOV3,UN_SOV1,UN_SOV2,UN_SOV3,UN_TER1,UN_TER2,UN_TER3 \ + -o $@ format=geojson force ndjson + +data/eez/eez_v11.shp: data/eez/World_EEZ_v11_20191118.zip + unzip -u $< -d data/eez + +data/eez/World_EEZ_v11_20191118.zip: | data/eez + cd data/gadm && \ + curl --location --request POST 'https://www.marineregions.org/download_file.php?name=World_EEZ_v11_20191118.zip' \ + --header 'Cookie: PHPSESSID=870e305efbc0519d59b361427dbd8336; vliz_webc=vliz_webc1' \ + --form 'name="Jen"' \ + --form 'organisation="TNC"' \ + --form 'email="admin@marxan.com"' \ + --form 'country="EEUU"' \ + --form 'user_category="academia"' \ + --form 'purpose_category="Conservation"' \ + --form 'agree="1"' \ + +data/eez: + mkdir -p $@ + +clean: + rm -rf data/eez/ From 5d518eff2126c0b9116beee1632c223cee9e7801 Mon Sep 17 00:00:00 2001 From: Alicia Date: Wed, 1 Jun 2022 11:11:08 +0200 Subject: [PATCH 2/5] improved instructions in the master makefile doc --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index d6d86c058e..85328e79a7 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,8 @@ generate-geo-test-data: extract-geo-test-data # Don't forget to run make clean-slate && make start-api before repopulating the whole db # This will delete all existing data and create tables/views/etc. through the migrations that # run when starting up the API service. +# if you want to test or run an individual pipe please do like this: +# docker-compose --project-name marxan-cloud -f ./data/docker-compose-data_management.yml run marxan-seed-data make seed-eez # Also, be sure to create a user before importing the geodata, otherwise it will fail with an # unrelated error message seed-geodb-data: From 86d3623c389a93941b18b7b469a81d37a0900bd3 Mon Sep 17 00:00:00 2001 From: Alicia Date: Wed, 1 Jun 2022 12:34:15 +0200 Subject: [PATCH 3/5] finish pipe --- Makefile | 3 ++- data/data_download/eez/Makefile | 19 +++++++++---------- data/docker-compose-data_management.yml | 1 + 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 85328e79a7..d9e73fe0c7 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,8 @@ generate-geo-test-data: extract-geo-test-data # This will delete all existing data and create tables/views/etc. through the migrations that # run when starting up the API service. # if you want to test or run an individual pipe please do like this: -# docker-compose --project-name marxan-cloud -f ./data/docker-compose-data_management.yml run marxan-seed-data make seed-eez +# docker-compose -p marxan-cloud -f ./data/docker-compose-data_management.yml up --no-start --build marxan-seed-data marxan-seed-data +# docker-compose -p marxan-cloud -f ./data/docker-compose-data_management.yml run marxan-seed-data make seed-eez # Also, be sure to create a user before importing the geodata, otherwise it will fail with an # unrelated error message seed-geodb-data: diff --git a/data/data_download/eez/Makefile b/data/data_download/eez/Makefile index 2785a14e95..1060a428ec 100644 --- a/data/data_download/eez/Makefile +++ b/data/data_download/eez/Makefile @@ -14,25 +14,23 @@ import: data/eez/eez_v11_simp.geojson port=$$GEO_POSTGRES_PORT user=$$GEO_POSTGRES_USER password=$$GEO_POSTGRES_PASSWORD" $< \ -sql "select *,'$(MarxanUser)' as created_by from \"$$(basename -s .geojson "$<")\""; -data/eez/eez_v11_simp.geojson: data/eez/eez_v11.shp +data/eez/eez_v11_simp.geojson: data/eez/World_EEZ_v11_20191118/eez_v11.shp mapshaper-xl -i $< snap \ -simplify 25% planar keep-shapes \ -filter-islands min-vertices=3 min-area=10000m2 remove-empty \ -filter-slivers min-area=10000m2 remove-empty \ -clean rewind \ -each 'level= "eez"' \ - -rename-fields GEONAME=NAME_0,ISO_SOV1=ISO3 \ - -drop fields=MRGID,MRGID_TER1,MRGID_SOV1,TERRITORY1,\ - ISO_TER1,SOVEREIGN1,MRGID_TER2,MRGID_SOV2,TERRITORY2,ISO_TER2,SOVEREIGN2,\ - MRGID_TER3,MRGID_SOV3,TERRITORY3,ISO_TER3,SOVEREIGN3,X_1,Y_1,AREA_KM2,ISO_SOV2,\ - ISO_SOV3,UN_SOV1,UN_SOV2,UN_SOV3,UN_TER1,UN_TER2,UN_TER3 \ - -o $@ format=geojson force ndjson + -rename-fields NAME_0=GEONAME,ISO3=ISO_SOV1 \ + -drop fields=MRGID,MRGID_TER1,MRGID_SOV1,TERRITORY1,ISO_TER1,SOVEREIGN1,MRGID_TER2,MRGID_SOV2,TERRITORY2,ISO_TER2,SOVEREIGN2,MRGID_TER3,MRGID_SOV3,TERRITORY3,ISO_TER3,SOVEREIGN3,X_1,Y_1,AREA_KM2,ISO_SOV2,ISO_SOV3,UN_SOV1,UN_SOV2,UN_SOV3,UN_TER1,UN_TER2,UN_TER3 \ + -o $@ format=geojson force ndjson && \ + rm -rf data/eez/World_EEZ_v11_20191118 -data/eez/eez_v11.shp: data/eez/World_EEZ_v11_20191118.zip +data/eez/World_EEZ_v11_20191118/eez_v11.shp: data/eez/World_EEZ_v11_20191118.zip unzip -u $< -d data/eez -data/eez/World_EEZ_v11_20191118.zip: | data/eez - cd data/gadm && \ +data/eez/World_EEZ_v11_20191118.zip: data/eez + cd $< && \ curl --location --request POST 'https://www.marineregions.org/download_file.php?name=World_EEZ_v11_20191118.zip' \ --header 'Cookie: PHPSESSID=870e305efbc0519d59b361427dbd8336; vliz_webc=vliz_webc1' \ --form 'name="Jen"' \ @@ -42,6 +40,7 @@ data/eez/World_EEZ_v11_20191118.zip: | data/eez --form 'user_category="academia"' \ --form 'purpose_category="Conservation"' \ --form 'agree="1"' \ + --output './World_EEZ_v11_20191118.zip' data/eez: mkdir -p $@ diff --git a/data/docker-compose-data_management.yml b/data/docker-compose-data_management.yml index 59f8042846..b6f609a2e1 100644 --- a/data/docker-compose-data_management.yml +++ b/data/docker-compose-data_management.yml @@ -6,6 +6,7 @@ services: dockerfile: Dockerfile volumes: - './data/seed/gadm_3.6/:/gadm_3.6/data/' + - './data/seed/eez/:/eez/data/' - './data/seed/wdpa/:/wdpa/data/' - './data/seed/iucn/:/iucn/data/' - './data/seed/world_terrestrial_ecosystems/:/world_terrestrial_ecosystems/data/' From d8dc2ae2a32cdc7201b1ffd2461d21a0f0375e59 Mon Sep 17 00:00:00 2001 From: Alicia Arenzana Gil de muro Date: Wed, 1 Jun 2022 16:17:11 +0200 Subject: [PATCH 4/5] Update data/data_download/Makefile Co-authored-by: andrea rota <47385021+hotzevzl@users.noreply.github.com> --- data/data_download/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/data_download/Makefile b/data/data_download/Makefile index ce45f4da9c..6a11a3cd11 100644 --- a/data/data_download/Makefile +++ b/data/data_download/Makefile @@ -8,7 +8,7 @@ seed-gadm: @time $(MAKE) -C ./gadm_3.6 import seed-eez: - @echo "Starting seeding gadm data... " + @echo "Starting seeding EEZ data... " @time $(MAKE) -C ./eez import seed-wdpa: From aae5e93bf877e39b0f3a401446e21110272e1223 Mon Sep 17 00:00:00 2001 From: Alicia Date: Wed, 1 Jun 2022 16:29:24 +0200 Subject: [PATCH 5/5] added index and new columns to geoentity --- .../1654090110000-AddEezToAdminRegions.ts | 37 +++++++++++++++++++ .../src/admin-areas.geo.entity.ts | 17 ++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 api/apps/geoprocessing/src/migrations/geoprocessing/1654090110000-AddEezToAdminRegions.ts diff --git a/api/apps/geoprocessing/src/migrations/geoprocessing/1654090110000-AddEezToAdminRegions.ts b/api/apps/geoprocessing/src/migrations/geoprocessing/1654090110000-AddEezToAdminRegions.ts new file mode 100644 index 0000000000..24e155fe93 --- /dev/null +++ b/api/apps/geoprocessing/src/migrations/geoprocessing/1654090110000-AddEezToAdminRegions.ts @@ -0,0 +1,37 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddEezToAdminRegions1654090110000 + + implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TYPE adm_level ADD VALUE IF NOT EXISTS 'eez'; + + CREATE TABLE IF NOT EXISTS admin_regions_eez PARTITION OF admin_regions FOR VALUES IN ('eez'); + + ALTER TABLE admin_regions ADD COLUMN IF NOT EXISTS pol_type text, + ADD COLUMN IF NOT EXISTS mrgid_eez integer; + + CREATE UNIQUE INDEX IF NOT EXISTS unique_eez_regions ON admin_regions (mrgid_eez, level) where level = 'eez'; + + DROP INDEX admin_regions_l012_ids; + + CREATE INDEX admin_regions_l012_ids ON admin_regions (gid_0, gid_1, gid_2, mrgid_eez, level); + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE admin_regions + DROP COLUMN pol_type, + DROP COLUMN mrgid_eez; + + ALTER TABLE admin_regions DROP PARTITION admin_regions_eez; + + ALTER TYPE adm_level DROP VALUE IF EXISTS 'eez'; + DROP INDEX unique_eez_regions; + CREATE INDEX admin_regions_l012_ids ON admin_regions (gid_0, gid_1, gid_2, level); + + `); + } +} diff --git a/api/libs/admin-regions/src/admin-areas.geo.entity.ts b/api/libs/admin-regions/src/admin-areas.geo.entity.ts index d233129fbc..8675407e3d 100644 --- a/api/libs/admin-regions/src/admin-areas.geo.entity.ts +++ b/api/libs/admin-regions/src/admin-areas.geo.entity.ts @@ -15,13 +15,18 @@ import { defaultSrid } from '@marxan/utils/geo'; export enum AdminLevel { Country = 'country', Adm1 = 'adm_1', - Amd2 = 'adm_2', + Adm2 = 'adm_2', + Eez = 'eez', } export const adminAreaTableName = 'admin_regions' as const; // property names, not db names in indexes: https://github.com/typeorm/typeorm/issues/930 @Entity(adminAreaTableName) -@Index('admin_regions_l012_ids', ['gid0', 'gid1', 'gid2', 'level']) +@Index('admin_regions_l012_ids', ['gid0', 'gid1', 'gid2', 'mrgidEez', 'level']) +@Index('unique_eez_regions', ['mrgidEez', 'level'], { + unique: true, + where: "level = 'eez'::adm_level", +}) @Index('unique_l2_regions', ['gid2', 'level'], { unique: true, where: "level = 'adm_2'::adm_level", @@ -71,6 +76,14 @@ export class AdminArea { @Column('character varying', { name: 'name_2', nullable: true }) name2?: string | null; + @ApiPropertyOptional() + @Column('character varying', { name: 'pol_type', nullable: true }) + polType?: string | null; + + @ApiPropertyOptional() + @Column('integer', { name: 'mrgid_eez', nullable: true }) + mrgidEez?: BigInteger | null; + @ApiPropertyOptional() @Column('character varying', { length: 3,