added daylight, improved tiles, updated demo page

.gitignore

@@ -1,4 +1,5 @@
 /data/overture_maps/*
+/data/daylight/*
 /data/output/geojson/*
 /data/output/geoparquet/*
 /data/output/pmtiles/*
@@ -14,6 +15,7 @@ lerna-debug.log*
 
 viewer/node_modules
 viewer/dist/overture.pmtiles
+viewer/dist/daylight.pmtiles
 viewer/dist-ssr
 *.local
 

README.md

@@ -2,8 +2,8 @@
 
 Playing around with data from OvertureMaps.
 
--   Download OvertureMaps data
--   Extract data from OvertureMaps for a country and convert to geoparquet
+-   Download OvertureMaps and Daylight data
+-   Extract data from OvertureMaps and Daylight for a country and convert to geoparquet
 -   Create PMTiles from extracted data
 -   Notebook with some queries and a map to display some data
 
@@ -70,23 +70,23 @@ In our test we don't want to do the whole world so we create some bounds, in thi
 The following script will create geoparquet files for all overture-maps themes with only data inside our given bounds, for pmtiles we don't need geoparquet but it's nice to have anyways. This can take some time, no fancy things are done such as spatial partitioning/indexing.
 
 ```sh
-./scripts/convert/to_geoparquet.sh
+./scripts/convert/overture/to_geoparquet.sh
 ```
 
 ### Create GeoJSON
 
 For the PMTiles creation we need GeoJSON files, the script `parquet_to_geojson.sh` creates geojson files for each theme from our geoparquet files which than can feeded into tippecanoe. Not every field is added to the GeoJSON feature properties at the moment.
 
 ```sh
-./scripts/convert/parquet_to_geojson.sh
+./scripts/convert/overture/parquet_to_geojson.sh
 ```
 
 ### Create PMTiles
 
 When we have the GeoJSON files we can create our PMTiles using tippecanoe and pmtiles.
 
 ```sh
-./scripts/convert/geojson_to_pmtiles.sh
+./scripts/convert/overture/geojson_to_pmtiles.sh
 ```
 
 This will create mbtiles for each theme, merge them and convert to PMTiles. Directly creating pmtiles with tippecanoe resulted in a PMTiles V2 file which could not be converted to v3, therefore mbtiles are created and later converted into PMTiles using `pmtiles convert`

scripts/convert/daylight/geojson_to_pmtiles.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+INPUT_PATH="./data/output/geojson"
+OUTPUT_PATH="./data/output/pmtiles"
+
+# When creating pmtiles with tippecanoe we get Version 2 of PMTiles
+# converting using pmtiles convert gives us an error
+# first create mbtiles using tippecanoe and convert mbtiles to pmtiles
+# using pmtiles CLI, this results in PMTiles V3
+
+#############################
+## WATER
+#############################
+echo "Generating water.mbtiles"
+tippecanoe -Z6 -z12 --projection=EPSG:4326 -o $OUTPUT_PATH/water.mbtiles -l water \
+    --detect-shared-borders \
+    --drop-smallest-as-needed \
+    -j '{ "*": [ "any", [ "in", "class", "river", "ocean", "lake" ], [ "all", [ ">=", "$zoom", 9 ], [ "in", "class", "stream" ]], [ ">=", "$zoom", 12]] }' \
+    $INPUT_PATH/water.geojson \
+    --force
+
+#############################
+## LAND
+#############################
+echo "Generating land.mbtiles"
+tippecanoe -Z4 -z12 --projection=EPSG:4326 -o $OUTPUT_PATH/land.mbtiles -l land \
+    --detect-shared-borders \
+    --drop-smallest-as-needed \
+    -j '{ "*": [ "any", [ "in", "class", "glacier" ], [ "all", [ ">=", "$zoom", 5 ], [ "in", "class", "forest", "sand" ]], [ "all", [ ">=", "$zoom", 7 ], [ "in", "class", "reef", "wetland" ]], [ "all", [ ">=", "$zoom", 9 ], [ "in", "class", "grass", "scrub" ]], [ ">=", "$zoom", 12]] }' \
+    $INPUT_PATH/land.geojson \
+    --force
+
+#############################
+## LANDUSE
+#############################
+echo "Generating landuse.mbtiles"
+tippecanoe -Z10 -z12 --projection=EPSG:4326 -o $OUTPUT_PATH/landuse.mbtiles -l landuse \
+    --detect-shared-borders \
+    --drop-smallest-as-needed \
+    -j '{ "*": [ "any", [ "all", [ ">=", "$zoom", 10 ], [ "in", "class", "agriculture", "protected" ]], [ "all", [ ">=", "$zoom", 11 ], [ "in", "class", "park", "airport" ]], [ ">=", "$zoom", 12]] }' \
+    $INPUT_PATH/landuse.geojson \
+    --force
+
+#############################
+## PLACENAME
+#############################
+echo "Generating placename.mbtiles"
+tippecanoe -Z2 -z12 -B6 --projection=EPSG:4326 -o $OUTPUT_PATH/placename.mbtiles -l placename \
+    -j '{ "*": [ "any", [ "in", "subclass", "megacity", "metropolis" ], [ "all", [ ">=", "$zoom", 3 ], [ "in", "subclass", "city", "municipality" ]],  [ "all", [ ">=", "$zoom", 9 ], [ "in", "subclass", "town", "hamlet", "vilage" ]], [ ">=", "$zoom", 12]] }' \
+    $INPUT_PATH/placename.geojson \
+    --force
+
+# join all mbtiles into 1 big file
+echo "merge tiles"
+tile-join -pk -o $OUTPUT_PATH/daylight.mbtiles $OUTPUT_PATH/water.mbtiles $OUTPUT_PATH/land.mbtiles $OUTPUT_PATH/landuse.mbtiles $OUTPUT_PATH/placename.mbtiles --force
+
+# tippecanoe outputs pmtiles V2, we want v3
+echo "convert mbtiles to pmtiles"
+pmtiles convert $OUTPUT_PATH/daylight.mbtiles $OUTPUT_PATH/daylight.pmtiles
+
+echo "Cleaning up intermediate files"
+# remove mbtiles file
+rm $OUTPUT_PATH/water.mbtiles
+rm $OUTPUT_PATH/land.mbtiles
+rm $OUTPUT_PATH/landuse.mbtiles
+rm $OUTPUT_PATH/placename.mbtiles
+rm $OUTPUT_PATH/daylight.mbtiles
+
+# copy pmtiles to the viewer
+cp $OUTPUT_PATH/daylight.pmtiles ./viewer/dist/daylight.pmtiles
+
+echo "Finished!"

scripts/convert/daylight/parquet_to_geojson.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Create GeoJSON files from our geoparquet files which we can than
+# use to generate PMTiles. Currently not all metadata is written
+# to the GeoJSON files.
+
+INPUT_PATH="./data/output/geoparquet"
+OUTPUT_PATH="./data/output/geojson"
+
+#############################
+## WATER
+#############################
+# Very nice strings can be found in names...
+duckdb -c """
+    LOAD spatial; 
+    COPY (
+        SELECT
+            class,
+            subclass,
+            COALESCE(replace(json_extract(replace(replace(names, '\\\\\\\\', ''), '\\\\\"', '''')::json, '$.local'), '\"', '')::varchar, '') as name,
+            --COALESCE(replace(json_extract(metadata::json,'\$.wikidata'), '\"', '')::varchar, '') as wikidata,
+            ST_GeomFromWkb(geometry) AS geometry
+        FROM read_parquet('$INPUT_PATH/water_geo.parquet')
+    ) 
+    TO '$OUTPUT_PATH/water.geojson'
+    WITH (FORMAT GDAL, DRIVER 'GeoJSON');
+"""
+duckdb -c ""
+
+#############################
+## PLACENAME
+#############################
+# Very nice strings can be found in names...
+duckdb -c """
+    LOAD spatial; 
+    COPY (
+        SELECT
+            class,
+            subclass,
+            COALESCE(replace(json_extract(names::json,'\$.local'), '\"', '')::varchar, '') as name,
+            ST_GeomFromWkb(geometry) AS geometry
+        FROM read_parquet('$INPUT_PATH/placename_geo.parquet')
+    ) 
+    TO '$OUTPUT_PATH/placename.geojson'
+    WITH (FORMAT GDAL, DRIVER 'GeoJSON');
+"""
+duckdb -c ""
+
+#############################
+## LANDUSE
+#############################
+duckdb -c """
+    LOAD spatial; 
+    COPY (
+        SELECT
+            class,
+            subclass,
+            COALESCE(replace(json_extract(replace(replace(names, '\\\\\\\\', ''), '\\\\\"', '''')::json, '$.local'), '\"', '')::varchar, '') as name,
+            ST_GeomFromWkb(geometry) AS geometry
+        FROM read_parquet('$INPUT_PATH/landuse_geo.parquet')
+    ) 
+    TO '$OUTPUT_PATH/landuse.geojson'
+    WITH (FORMAT GDAL, DRIVER 'GeoJSON');
+"""
+duckdb -c ""
+
+#############################
+## LAND
+#############################
+duckdb -c """
+    LOAD spatial; 
+    COPY (
+        SELECT
+            class,
+            subclass,
+            COALESCE(replace(json_extract(replace(replace(names, '\\\\\\\\', ''), '\\\\\"', '''')::json, '$.local'), '\"', '')::varchar, '') as name,
+            ST_GeomFromWkb(geometry) AS geometry
+        FROM read_parquet('$INPUT_PATH/land_geo.parquet')
+    ) 
+    TO '$OUTPUT_PATH/land.geojson'
+    WITH (FORMAT GDAL, DRIVER 'GeoJSON');
+"""
+duckdb -c ""

scripts/convert/daylight/parquet_to_geoparquet.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+# Extract data from overture-maps data based on bounds and convert to geoparquet using gpq
+# The extracts can be used for example in QGIS or as input to create GeoJSON files.
+#
+# ToDo: spatial partitioning?
+
+DAYLIGHT_DATA_PATH="./data/daylight"
+OUTPUT_PATH="./data/output/geoparquet"
+COUNTRY_BOUNDS_FILE="./scripts/convert/country_bounds.parquet"
+
+#############################
+## THEME: WATER
+#############################
+
+duckdb -c """
+    INSTALL spatial; 
+    LOAD spatial; 
+    COPY (
+        SELECT 
+            a.geometry_id,
+            a.class,
+            a.subclass,
+            a.metadata,
+            a.original_source_tags,
+            a.names,
+            a.quadkey,
+            ST_AsWKB(ST_GeomFromText(a.wkt)) as geometry,
+            a.theme
+        FROM read_parquet('$DAYLIGHT_DATA_PATH/theme=water/*', hive_partitioning=1) AS a, read_parquet('$COUNTRY_BOUNDS_FILE') AS b
+        WHERE ST_Intersects(ST_GeomFromText(a.wkt), ST_GeomFromWkb(b.geometry))
+    ) 
+    TO 'water.parquet' (FORMAT 'parquet');
+"""
+gpq convert water.parquet $OUTPUT_PATH/water_geo.parquet --from=parquet --to=geoparquet
+rm water.parquet
+
+#############################
+## THEME: PLACENAME
+#############################
+
+duckdb -c """
+    INSTALL spatial; 
+    LOAD spatial; 
+    COPY (
+        SELECT
+            a.geometry_id,
+            a.class,
+            a.subclass,
+            a.metadata,
+            a.original_source_tags,
+            a.names,
+            a.quadkey,
+            ST_AsWKB(ST_GeomFromText(a.wkt)) as geometry,
+            a.theme
+        FROM read_parquet('$DAYLIGHT_DATA_PATH/theme=placename/*', hive_partitioning=1) AS a, read_parquet('$COUNTRY_BOUNDS_FILE') AS b
+        WHERE ST_Intersects(ST_GeomFromText(a.wkt), ST_GeomFromWkb(b.geometry))
+    ) 
+    TO 'placename.parquet' (FORMAT 'parquet');
+"""
+gpq convert placename.parquet $OUTPUT_PATH/placename_geo.parquet --from=parquet --to=geoparquet
+rm placename.parquet
+
+#############################
+## THEME: LANDUSE
+#############################
+
+duckdb -c """
+    INSTALL spatial; 
+    LOAD spatial; 
+    COPY (
+        SELECT
+            a.geometry_id,
+            a.class,
+            a.subclass,
+            a.metadata,
+            a.original_source_tags,
+            a.names,
+            a.quadkey,
+            ST_AsWKB(ST_GeomFromText(a.wkt)) as geometry,
+            a.theme
+        FROM read_parquet('$DAYLIGHT_DATA_PATH/theme=landuse/*', hive_partitioning=1) AS a, read_parquet('$COUNTRY_BOUNDS_FILE') AS b
+        WHERE ST_Intersects(ST_GeomFromText(a.wkt), ST_GeomFromWkb(b.geometry))
+    ) 
+    TO 'landuse.parquet' (FORMAT 'parquet');
+"""
+gpq convert landuse.parquet $OUTPUT_PATH/landuse_geo.parquet --from=parquet --to=geoparquet
+rm landuse.parquet
+
+#############################
+## THEME: LAND
+#############################
+
+duckdb -c """
+    INSTALL spatial; 
+    LOAD spatial; 
+    COPY (
+        SELECT
+            a.geometry_id,
+            a.class,
+            a.subclass,
+            a.metadata,
+            a.original_source_tags,
+            a.names,
+            a.quadkey,
+            ST_AsWKB(ST_GeomFromText(a.wkt)) as geometry,
+            a.theme
+        FROM read_parquet('$DAYLIGHT_DATA_PATH/theme=land/*', hive_partitioning=1) AS a, read_parquet('$COUNTRY_BOUNDS_FILE') AS b
+        WHERE ST_Intersects(ST_GeomFromText(a.wkt), ST_GeomFromWkb(b.geometry))
+    ) 
+    TO 'land.parquet' (FORMAT 'parquet');
+"""
+gpq convert land.parquet $OUTPUT_PATH/land_geo.parquet --from=parquet --to=geoparquet
+rm land.parquet

scripts/convert/geojson_to_pmtiles.sh → scripts/convert/overture/geojson_to_pmtiles.sh

@@ -12,13 +12,19 @@ OUTPUT_PATH="./data/output/pmtiles"
 ## ADMINS
 #############################
 echo "Generating admins.mbtiles"
-tippecanoe -zg -z20 --projection=EPSG:4326 -o $OUTPUT_PATH/admins.mbtiles -l admins --drop-densest-as-needed $INPUT_PATH/admins.geojson --force
+tippecanoe -Z0 -z16 --projection=EPSG:4326 -o $OUTPUT_PATH/admins.mbtiles -l admins \
+    --detect-shared-borders \
+    --drop-smallest-as-needed \
+    $INPUT_PATH/admins.geojson --force
 
 #############################
 ## BUILDINGS
 #############################
 echo "Generating buildings.mbtiles"
-tippecanoe -Z13 -z20 --projection=EPSG:4326 -o $OUTPUT_PATH/buildings.mbtiles -l buildings --drop-densest-as-needed $INPUT_PATH/buildings.geojson --force
+tippecanoe -Z13 -z16 --projection=EPSG:4326 -o $OUTPUT_PATH/buildings.mbtiles -l buildings \
+    --detect-shared-borders \
+    --drop-smallest-as-needed \
+    $INPUT_PATH/buildings.geojson --force
 
 #############################
 ## ROADS
@@ -33,13 +39,16 @@ tippecanoe -Z13 -z20 --projection=EPSG:4326 -o $OUTPUT_PATH/buildings.mbtiles -l
 # 3) [ ">=", "$zoom", 15]]
 #    If zoom is greater than or equals to 15 return true = render everything after and at zoom 15
 echo "Generating roads.mbtiles"
-tippecanoe -Z5 -z20 -o $OUTPUT_PATH/roads.mbtiles --coalesce-smallest-as-needed -j '{ "*": [ "any", [ "in", "class", "motorway" ], [ "all", [ ">=", "$zoom", 9 ], [ "in", "class", "primary", "secondary" ]],  [ "all", [ ">=", "$zoom", 11 ], [ "in", "class", "tertiary", "residential", "livingStreet" ]], [ ">=", "$zoom", 12]] }' -l roads $INPUT_PATH/roads.geojson --force
+tippecanoe -Z5 -z16 -o $OUTPUT_PATH/roads.mbtiles -l roads \
+    --detect-shared-borders \
+    --drop-smallest-as-needed \
+    --coalesce-smallest-as-needed -j '{ "*": [ "any", [ "in", "class", "motorway" ], [ "all", [ ">=", "$zoom", 9 ], [ "in", "class", "primary", "secondary" ]],  [ "all", [ ">=", "$zoom", 11 ], [ "in", "class", "tertiary", "residential", "livingStreet" ]], [ ">=", "$zoom", 12]] }' -l roads $INPUT_PATH/roads.geojson --force
 
 #############################
 ## PLACES
 #############################
 echo "Generating places.mbtiles"
-tippecanoe -Z14 -z20 --projection=EPSG:4326 -o $OUTPUT_PATH/places.mbtiles -l places $INPUT_PATH/places.geojson --force
+tippecanoe -Z14 -z16 -B16 --projection=EPSG:4326 -o $OUTPUT_PATH/places.mbtiles -l places $INPUT_PATH/places.geojson --force
 
 
 # join all mbtiles into 1 big file

scripts/download_daylight_data.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Set the local directory path
+local_directory="./data/daylight"
+
+# Create the local directory if it doesn't exist
+if [ ! -d "$local_directory" ]; then
+  mkdir -p "$local_directory"
+  echo "Created directory: $local_directory"
+fi
+
+# Download using AWS CLI and S3
+source_bucket_water="s3://daylight-openstreetmap/earth/release=v1.29/theme=water"
+source_bucket_placename="s3://daylight-openstreetmap/earth/release=v1.29/theme=placename"
+source_bucket_land="s3://daylight-openstreetmap/earth/release=v1.29/theme=land"
+source_bucket_landuse="s3://daylight-openstreetmap/earth/release=v1.29/theme=landuse"
+
+aws s3 cp --region us-west-2 --no-sign-request --recursive "$source_bucket_water" "$local_directory/theme=water"
+aws s3 cp --region us-west-2 --no-sign-request --recursive "$source_bucket_placename" "$local_directory/theme=placename"
+aws s3 cp --region us-west-2 --no-sign-request --recursive "$source_bucket_land" "$local_directory/theme=land"
+aws s3 cp --region us-west-2 --no-sign-request --recursive "$source_bucket_landuse" "$local_directory/theme=landuse"
+
+# Check if the download was successful
+if [ $? -eq 0 ]; then
+  echo "Download completed successfully."
+else
+  echo "Download failed. Please check the source S3 bucket and try again."
+  exit 1
+fi