diff --git a/Makefile b/Makefile index d6d86c058e..d9e73fe0c7 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,9 @@ generate-geo-test-data: extract-geo-test-data # Don't forget to run make clean-slate && make start-api before repopulating the whole db # This will delete all existing data and create tables/views/etc. through the migrations that # run when starting up the API service. +# if you want to test or run an individual pipe please do like this: +# docker-compose -p marxan-cloud -f ./data/docker-compose-data_management.yml up --no-start --build marxan-seed-data marxan-seed-data +# docker-compose -p marxan-cloud -f ./data/docker-compose-data_management.yml run marxan-seed-data make seed-eez # Also, be sure to create a user before importing the geodata, otherwise it will fail with an # unrelated error message seed-geodb-data: diff --git a/api/apps/geoprocessing/src/migrations/geoprocessing/1654090110000-AddEezToAdminRegions.ts b/api/apps/geoprocessing/src/migrations/geoprocessing/1654090110000-AddEezToAdminRegions.ts new file mode 100644 index 0000000000..24e155fe93 --- /dev/null +++ b/api/apps/geoprocessing/src/migrations/geoprocessing/1654090110000-AddEezToAdminRegions.ts @@ -0,0 +1,37 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddEezToAdminRegions1654090110000 + + implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TYPE adm_level ADD VALUE IF NOT EXISTS 'eez'; + + CREATE TABLE IF NOT EXISTS admin_regions_eez PARTITION OF admin_regions FOR VALUES IN ('eez'); + + ALTER TABLE admin_regions ADD COLUMN IF NOT EXISTS pol_type text, + ADD COLUMN IF NOT EXISTS mrgid_eez integer; + + CREATE UNIQUE INDEX IF NOT EXISTS unique_eez_regions ON admin_regions (mrgid_eez, level) where level = 'eez'; + + DROP INDEX admin_regions_l012_ids; + + CREATE INDEX admin_regions_l012_ids ON admin_regions (gid_0, gid_1, gid_2, mrgid_eez, level); + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE admin_regions + DROP COLUMN pol_type, + DROP COLUMN mrgid_eez; + + ALTER TABLE admin_regions DROP PARTITION admin_regions_eez; + + ALTER TYPE adm_level DROP VALUE IF EXISTS 'eez'; + DROP INDEX unique_eez_regions; + CREATE INDEX admin_regions_l012_ids ON admin_regions (gid_0, gid_1, gid_2, level); + + `); + } +} diff --git a/api/libs/admin-regions/src/admin-areas.geo.entity.ts b/api/libs/admin-regions/src/admin-areas.geo.entity.ts index d233129fbc..8675407e3d 100644 --- a/api/libs/admin-regions/src/admin-areas.geo.entity.ts +++ b/api/libs/admin-regions/src/admin-areas.geo.entity.ts @@ -15,13 +15,18 @@ import { defaultSrid } from '@marxan/utils/geo'; export enum AdminLevel { Country = 'country', Adm1 = 'adm_1', - Amd2 = 'adm_2', + Adm2 = 'adm_2', + Eez = 'eez', } export const adminAreaTableName = 'admin_regions' as const; // property names, not db names in indexes: https://github.com/typeorm/typeorm/issues/930 @Entity(adminAreaTableName) -@Index('admin_regions_l012_ids', ['gid0', 'gid1', 'gid2', 'level']) +@Index('admin_regions_l012_ids', ['gid0', 'gid1', 'gid2', 'mrgidEez', 'level']) +@Index('unique_eez_regions', ['mrgidEez', 'level'], { + unique: true, + where: "level = 'eez'::adm_level", +}) @Index('unique_l2_regions', ['gid2', 'level'], { unique: true, where: "level = 'adm_2'::adm_level", @@ -71,6 +76,14 @@ export class AdminArea { @Column('character varying', { name: 'name_2', nullable: true }) name2?: string | null; + @ApiPropertyOptional() + @Column('character varying', { name: 'pol_type', nullable: true }) + polType?: string | null; + + @ApiPropertyOptional() + @Column('integer', { name: 'mrgid_eez', nullable: true }) + mrgidEez?: BigInteger | null; + @ApiPropertyOptional() @Column('character varying', { length: 3, diff --git a/data/data_download/Makefile b/data/data_download/Makefile index 156895651c..6a11a3cd11 100644 --- a/data/data_download/Makefile +++ b/data/data_download/Makefile @@ -1,11 +1,16 @@ .DEFAULT_GOAL := seed-data -seed-data: seed-gadm seed-wdpa seed-demo-features-species seed-demo-features-bioregion seed-ecosystems +seed-data: seed-gadm seed-eez seed-wdpa seed-demo-features-species seed-demo-features-bioregion seed-ecosystems + seed-gadm: @echo "Starting seeding gadm data... " @time $(MAKE) -C ./gadm_3.6 import +seed-eez: + @echo "Starting seeding EEZ data... " + @time $(MAKE) -C ./eez import + seed-wdpa: @echo "Starting seeding wdpa data... " @time $(MAKE) -C ./wdpa import diff --git a/data/data_download/eez/Makefile b/data/data_download/eez/Makefile new file mode 100644 index 0000000000..1060a428ec --- /dev/null +++ b/data/data_download/eez/Makefile @@ -0,0 +1,49 @@ +.PHONY: import +# MAKEFLAGS := --jobs=$(shell nproc) +# MAKEFLAGS += --output-sync=target + +MarxanUser:=$(shell psql -X -A -t "postgresql://$$API_POSTGRES_USER:$$API_POSTGRES_PASSWORD@$$API_POSTGRES_HOST:$$API_POSTGRES_PORT/$$API_POSTGRES_DB" -c "select id from users limit 1") +import: data/eez/eez_v11_simp.geojson + ogr2ogr -makevalid \ + -update -append \ + -geomfield the_geom \ + --config OGR_TRUNCATE NO \ + -nln admin_regions -nlt PROMOTE_TO_MULTI \ + -t_srs EPSG:4326 -a_srs EPSG:4326 \ + -f PostgreSQL PG:"dbname=$$GEO_POSTGRES_DB host=$$GEO_POSTGRES_HOST \ + port=$$GEO_POSTGRES_PORT user=$$GEO_POSTGRES_USER password=$$GEO_POSTGRES_PASSWORD" $< \ + -sql "select *,'$(MarxanUser)' as created_by from \"$$(basename -s .geojson "$<")\""; + +data/eez/eez_v11_simp.geojson: data/eez/World_EEZ_v11_20191118/eez_v11.shp + mapshaper-xl -i $< snap \ + -simplify 25% planar keep-shapes \ + -filter-islands min-vertices=3 min-area=10000m2 remove-empty \ + -filter-slivers min-area=10000m2 remove-empty \ + -clean rewind \ + -each 'level= "eez"' \ + -rename-fields NAME_0=GEONAME,ISO3=ISO_SOV1 \ + -drop fields=MRGID,MRGID_TER1,MRGID_SOV1,TERRITORY1,ISO_TER1,SOVEREIGN1,MRGID_TER2,MRGID_SOV2,TERRITORY2,ISO_TER2,SOVEREIGN2,MRGID_TER3,MRGID_SOV3,TERRITORY3,ISO_TER3,SOVEREIGN3,X_1,Y_1,AREA_KM2,ISO_SOV2,ISO_SOV3,UN_SOV1,UN_SOV2,UN_SOV3,UN_TER1,UN_TER2,UN_TER3 \ + -o $@ format=geojson force ndjson && \ + rm -rf data/eez/World_EEZ_v11_20191118 + +data/eez/World_EEZ_v11_20191118/eez_v11.shp: data/eez/World_EEZ_v11_20191118.zip + unzip -u $< -d data/eez + +data/eez/World_EEZ_v11_20191118.zip: data/eez + cd $< && \ + curl --location --request POST 'https://www.marineregions.org/download_file.php?name=World_EEZ_v11_20191118.zip' \ + --header 'Cookie: PHPSESSID=870e305efbc0519d59b361427dbd8336; vliz_webc=vliz_webc1' \ + --form 'name="Jen"' \ + --form 'organisation="TNC"' \ + --form 'email="admin@marxan.com"' \ + --form 'country="EEUU"' \ + --form 'user_category="academia"' \ + --form 'purpose_category="Conservation"' \ + --form 'agree="1"' \ + --output './World_EEZ_v11_20191118.zip' + +data/eez: + mkdir -p $@ + +clean: + rm -rf data/eez/ diff --git a/data/docker-compose-data_management.yml b/data/docker-compose-data_management.yml index 59f8042846..b6f609a2e1 100644 --- a/data/docker-compose-data_management.yml +++ b/data/docker-compose-data_management.yml @@ -6,6 +6,7 @@ services: dockerfile: Dockerfile volumes: - './data/seed/gadm_3.6/:/gadm_3.6/data/' + - './data/seed/eez/:/eez/data/' - './data/seed/wdpa/:/wdpa/data/' - './data/seed/iucn/:/iucn/data/' - './data/seed/world_terrestrial_ecosystems/:/world_terrestrial_ecosystems/data/'