ETL Pipe for download and ingest EEZ [MARXAN-1618]

Makefile

@@ -114,6 +114,9 @@ generate-geo-test-data: extract-geo-test-data
 # Don't forget to run make clean-slate && make start-api before repopulating the whole db
 # This will delete all existing data and create tables/views/etc. through the migrations that
 # run when starting up the API service.
+# if you want to test or run an individual pipe please do like this:
+# docker-compose -p marxan-cloud -f ./data/docker-compose-data_management.yml up --no-start --build marxan-seed-data marxan-seed-data
+# docker-compose -p marxan-cloud -f ./data/docker-compose-data_management.yml run marxan-seed-data make seed-eez
 # Also, be sure to create a user before importing the geodata, otherwise it will fail with an
 # unrelated error message
 seed-geodb-data:

api/apps/geoprocessing/src/migrations/geoprocessing/1654090110000-AddEezToAdminRegions.ts

@@ -0,0 +1,37 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddEezToAdminRegions1654090110000
+
+  implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+        ALTER TYPE adm_level ADD VALUE IF NOT EXISTS 'eez';
+
+        CREATE TABLE IF NOT EXISTS admin_regions_eez PARTITION OF admin_regions FOR VALUES IN ('eez');
+
+        ALTER TABLE admin_regions ADD COLUMN IF NOT EXISTS pol_type text,
+                                  ADD COLUMN IF NOT EXISTS mrgid_eez integer;
+
+        CREATE UNIQUE INDEX IF NOT EXISTS unique_eez_regions ON admin_regions (mrgid_eez, level) where level = 'eez';
+
+        DROP INDEX admin_regions_l012_ids;
+
+        CREATE INDEX admin_regions_l012_ids ON admin_regions (gid_0, gid_1, gid_2, mrgid_eez, level);
+          `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+          ALTER TABLE admin_regions
+              DROP COLUMN pol_type,
+              DROP COLUMN mrgid_eez;
+
+          ALTER TABLE admin_regions DROP PARTITION admin_regions_eez;
+
+          ALTER TYPE adm_level DROP VALUE IF EXISTS 'eez';
+          DROP INDEX unique_eez_regions;
+          CREATE INDEX admin_regions_l012_ids ON admin_regions (gid_0, gid_1, gid_2, level);
+
+          `);
+  }
+}

api/libs/admin-regions/src/admin-areas.geo.entity.ts

@@ -15,13 +15,18 @@ import { defaultSrid } from '@marxan/utils/geo';
 export enum AdminLevel {
   Country = 'country',
   Adm1 = 'adm_1',
-  Amd2 = 'adm_2',
+  Adm2 = 'adm_2',
+  Eez = 'eez',
 }
 
 export const adminAreaTableName = 'admin_regions' as const;
 // property names, not db names in indexes: https://github.com/typeorm/typeorm/issues/930
 @Entity(adminAreaTableName)
-@Index('admin_regions_l012_ids', ['gid0', 'gid1', 'gid2', 'level'])
+@Index('admin_regions_l012_ids', ['gid0', 'gid1', 'gid2', 'mrgidEez', 'level'])
+@Index('unique_eez_regions', ['mrgidEez', 'level'], {
+  unique: true,
+  where: "level = 'eez'::adm_level",
+})
 @Index('unique_l2_regions', ['gid2', 'level'], {
   unique: true,
   where: "level = 'adm_2'::adm_level",
@@ -71,6 +76,14 @@ export class AdminArea {
   @Column('character varying', { name: 'name_2', nullable: true })
   name2?: string | null;
 
+  @ApiPropertyOptional()
+  @Column('character varying', { name: 'pol_type', nullable: true })
+  polType?: string | null;
+
+  @ApiPropertyOptional()
+  @Column('integer', { name: 'mrgid_eez', nullable: true })
+  mrgidEez?: BigInteger | null;
+
   @ApiPropertyOptional()
   @Column('character varying', {
     length: 3,

data/data_download/Makefile

@@ -1,11 +1,16 @@
 .DEFAULT_GOAL := seed-data
 
 
-seed-data: seed-gadm seed-wdpa seed-demo-features-species seed-demo-features-bioregion seed-ecosystems
+seed-data: seed-gadm seed-eez seed-wdpa seed-demo-features-species seed-demo-features-bioregion seed-ecosystems
+
 seed-gadm:
 	@echo "Starting seeding gadm data... "
 	@time $(MAKE) -C ./gadm_3.6 import
 
+seed-eez:
+	@echo "Starting seeding EEZ data... "
+	@time $(MAKE) -C ./eez import
+
 seed-wdpa:
 	@echo "Starting seeding wdpa data... "
 	@time $(MAKE) -C ./wdpa import

data/data_download/eez/Makefile

@@ -0,0 +1,49 @@
+.PHONY: import
+# MAKEFLAGS := --jobs=$(shell nproc)
+# MAKEFLAGS += --output-sync=target
+
+MarxanUser:=$(shell psql -X -A -t "postgresql://$$API_POSTGRES_USER:$$API_POSTGRES_PASSWORD@$$API_POSTGRES_HOST:$$API_POSTGRES_PORT/$$API_POSTGRES_DB" -c "select id from users limit 1")
+import: data/eez/eez_v11_simp.geojson
+	ogr2ogr -makevalid \
+		-update -append \
+		-geomfield the_geom \
+		--config OGR_TRUNCATE NO \
+		-nln admin_regions -nlt PROMOTE_TO_MULTI \
+		-t_srs EPSG:4326 -a_srs EPSG:4326 \
+		-f PostgreSQL PG:"dbname=$$GEO_POSTGRES_DB host=$$GEO_POSTGRES_HOST \
+		port=$$GEO_POSTGRES_PORT user=$$GEO_POSTGRES_USER password=$$GEO_POSTGRES_PASSWORD" $< \
+		-sql "select *,'$(MarxanUser)' as created_by from \"$$(basename -s .geojson "$<")\"";
+
+data/eez/eez_v11_simp.geojson: data/eez/World_EEZ_v11_20191118/eez_v11.shp
+	mapshaper-xl -i $< snap \
+				-simplify 25% planar keep-shapes \
+				-filter-islands min-vertices=3 min-area=10000m2 remove-empty \
+				-filter-slivers min-area=10000m2 remove-empty \
+				-clean rewind \
+				-each 'level= "eez"' \
+				-rename-fields NAME_0=GEONAME,ISO3=ISO_SOV1 \
+				-drop fields=MRGID,MRGID_TER1,MRGID_SOV1,TERRITORY1,ISO_TER1,SOVEREIGN1,MRGID_TER2,MRGID_SOV2,TERRITORY2,ISO_TER2,SOVEREIGN2,MRGID_TER3,MRGID_SOV3,TERRITORY3,ISO_TER3,SOVEREIGN3,X_1,Y_1,AREA_KM2,ISO_SOV2,ISO_SOV3,UN_SOV1,UN_SOV2,UN_SOV3,UN_TER1,UN_TER2,UN_TER3 \
+				-o $@ format=geojson force ndjson && \
+	rm -rf data/eez/World_EEZ_v11_20191118
+
+data/eez/World_EEZ_v11_20191118/eez_v11.shp: data/eez/World_EEZ_v11_20191118.zip
+	unzip -u $< -d data/eez
+
+data/eez/World_EEZ_v11_20191118.zip: data/eez
+	cd $< && \
+	curl --location --request POST 'https://www.marineregions.org/download_file.php?name=World_EEZ_v11_20191118.zip' \
+					--header 'Cookie: PHPSESSID=870e305efbc0519d59b361427dbd8336; vliz_webc=vliz_webc1' \
+					--form 'name="Jen"' \
+					--form 'organisation="TNC"' \
+					--form 'email="[email protected]"' \
+					--form 'country="EEUU"' \
+					--form 'user_category="academia"' \
+					--form 'purpose_category="Conservation"' \
+					--form 'agree="1"' \
+					--output './World_EEZ_v11_20191118.zip'
+
+data/eez:
+	mkdir -p $@
+
+clean:
+	rm -rf data/eez/

data/docker-compose-data_management.yml

@@ -6,6 +6,7 @@ services:
         dockerfile: Dockerfile
     volumes:
     - './data/seed/gadm_3.6/:/gadm_3.6/data/'
+    - './data/seed/eez/:/eez/data/'
     - './data/seed/wdpa/:/wdpa/data/'
     - './data/seed/iucn/:/iucn/data/'
     - './data/seed/world_terrestrial_ecosystems/:/world_terrestrial_ecosystems/data/'