Load sources with explicit geometry srids and update tax_lot table im…

…port (#16)
NYCPlanning · Mar 18, 2024 · e07ce1c · e07ce1c
1 parent 3b16b80
commit e07ce1c
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 68 deletions.
diff --git a/README.md b/README.md
@@ -105,4 +105,16 @@ psql ${BUILD_ENGINE_URI} \
 dbt test --select "source:*"
 ```
 
+### Create tables
+psql ${BUILD_ENGINE_URI} \
+  --set ON_ERROR_STOP=1 --single-transaction --quiet \
+  --file create_tables.sql \
+  --variable SCHEMA_NAME=${BUILD_ENGINE_SCHEMA}
+
+### Populate tables
+psql ${BUILD_ENGINE_URI} \
+  --set ON_ERROR_STOP=1 --single-transaction --quiet \
+  --file import_tables.sql \
+  --variable SCHEMA_NAME=${BUILD_ENGINE_SCHEMA}
+
 ### TBD
diff --git a/create_tables.sql b/create_tables.sql
@@ -73,13 +73,3 @@ DO $$ BEGIN
 EXCEPTION
  WHEN duplicate_object THEN null;
 END $$;
-
-CREATE TABLE IF NOT EXISTS "pluto" (
-	"bbl" text PRIMARY KEY NOT NULL,
-	"borough_id" char(2) NOT NULL,
-	"block" text NOT NULL,
-	"lot" text NOT NULL,
-	"address" text,
-	"land_use_id" char(2),
-	"wkt" text
-);
diff --git a/import_tables.sql b/import_tables.sql
@@ -1,18 +1,13 @@
-BEGIN;
-	COPY borough ("id", "title", "abbr")
-		FROM '../borough.csv'
-		DELIMITER ','
-		CSV HEADER;
-
-COMMIT;
+COPY borough ("id", "title", "abbr")
+	FROM '../borough.csv'
+	DELIMITER ','
+	CSV HEADER;
 
-BEGIN;
-	COPY land_use ("id", "description", "color")
-		FROM '../land_use.csv'
-		DELIMITER ','
-		CSV HEADER;
 
-COMMIT;
+COPY land_use ("id", "description", "color")
+	FROM '../land_use.csv'
+	DELIMITER ','
+	CSV HEADER;
 
 INSERT INTO tax_lot
 SELECT
@@ -21,46 +16,37 @@ SELECT
 	block,
 	lot,
 	address,
-	land_use_id,
-	ST_GeomFromText(wkt, 4326) as wgs84,
-	ST_GeomFromText(wkt, 2263) as li_ft
-FROM pluto
-INNER JOIN borough ON pluto.borough_id=borough.abbr;
-
-BEGIN;
-	COPY zoning_district (
-		"id",
-		"label",
-		"wgs84",
-		"li_ft"
-	)
-		FROM '../zoning_district.csv'
-		DELIMITER ','
-		CSV HEADER;
-
-COMMIT;
-
-BEGIN;
-	COPY zoning_district_class (
-		"id",
-		"category",
-		"description",
-		"url",
-		"color"
-	)
-		FROM '../zoning_district_class.csv'
-		DELIMITER ','
-		CSV HEADER;
-
-COMMIT;
-
-BEGIN;
-	COPY zoning_district_zoning_district_class (
-		"zoning_district_id",
-		"zoning_district_class_id"
-	)
-		FROM '../zoning_district_zoning_district_class.csv'
-		DELIMITER ','
-		CSV HEADER;
-
-COMMIT;
+	land_use as land_use_id,
+	ST_Transform(wkt, 4326) as wgs84,
+	wkt as li_ft
+FROM source_pluto
+INNER JOIN borough ON source_pluto.borough=borough.abbr;
+
+COPY zoning_district (
+	"id",
+	"label",
+	"wgs84",
+	"li_ft"
+)
+	FROM '../zoning_district.csv'
+	DELIMITER ','
+	CSV HEADER;
+
+COPY zoning_district_class (
+	"id",
+	"category",
+	"description",
+	"url",
+	"color"
+)
+	FROM '../zoning_district_class.csv'
+	DELIMITER ','
+	CSV HEADER;
+
+COPY zoning_district_zoning_district_class (
+	"zoning_district_id",
+	"zoning_district_class_id"
+)
+	FROM '../zoning_district_zoning_district_class.csv'
+	DELIMITER ','
+	CSV HEADER;
diff --git a/sql/load_sources.sql b/sql/load_sources.sql
@@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS "source_pluto" (
 	"lot" text NOT NULL,
 	"address" text,
 	"land_use" char(2),
-	"wkt" geometry
+	"wkt" geometry(MULTIPOLYGON, 2263)
 );
 CREATE INDEX pluto_geom_idx
   ON source_pluto
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS "source_zoning_districts" (
 	"zonedist" text NOT NULL,
 	"shape_leng" float,
 	"shape_area" float,
-	"wkt" geometry
+	"wkt" geometry(MULTIPOLYGON, 4326)
 );
 CREATE INDEX zoning_districts_geom_idx
   ON source_zoning_districts