diff --git a/products/cdbg/bash/export.sh b/products/cdbg/bash/export.sh index 1650a4476..7a7dbf3e9 100755 --- a/products/cdbg/bash/export.sh +++ b/products/cdbg/bash/export.sh @@ -17,6 +17,9 @@ mkdir -p output && ( echo "export cdbg_tracts.csv ..." csv_export cdbg_tracts cdbg_tracts + + echo "export cdbg_boroughs.csv ..." + csv_export cdbg_boroughs cdbg_boroughs ) zip -r output/output.zip output diff --git a/products/cdbg/models/intermediate/int__block_groups.sql b/products/cdbg/models/intermediate/int__block_groups.sql index 469c03d4f..f821d990a 100644 --- a/products/cdbg/models/intermediate/int__block_groups.sql +++ b/products/cdbg/models/intermediate/int__block_groups.sql @@ -39,6 +39,7 @@ block_group_details AS ( SELECT block_groups_floor_area.geoid, block_groups_demographics.borough_name, + block_groups_demographics.borough_code, block_groups_demographics.tract, block_groups_demographics.block_group, block_groups_floor_area.total_floor_area, diff --git a/products/cdbg/models/intermediate/int__boros.sql b/products/cdbg/models/intermediate/int__boros.sql index 482e0c3c1..0e7a99d99 100644 --- a/products/cdbg/models/intermediate/int__boros.sql +++ b/products/cdbg/models/intermediate/int__boros.sql @@ -5,12 +5,15 @@ WITH tracts AS ( boros AS ( SELECT borough_name, + borough_code, sum(total_floor_area) AS total_floor_area, sum(residential_floor_area) AS residential_floor_area, sum(total_population) AS total_population, sum(low_mod_income_population) AS low_mod_income_population FROM tracts - GROUP BY borough_name + GROUP BY + borough_name, + borough_code ), boro_calculation AS ( diff --git a/products/cdbg/models/intermediate/int__tracts.sql b/products/cdbg/models/intermediate/int__tracts.sql index 760783824..9c8578d7a 100644 --- a/products/cdbg/models/intermediate/int__tracts.sql +++ b/products/cdbg/models/intermediate/int__tracts.sql @@ -8,13 +8,17 @@ WITH block_groups AS ( tracts AS ( SELECT tract_id AS geoid, - max(borough_name) AS borough_name, + borough_name, + borough_code, sum(total_floor_area) AS total_floor_area, sum(residential_floor_area) AS residential_floor_area, sum(total_population) AS total_population, sum(low_mod_income_population) AS low_mod_income_population FROM block_groups - GROUP BY tract_id + GROUP BY + tract_id, + borough_name, + borough_code ), tracts_calculation AS ( diff --git a/products/cdbg/models/product/_product_models.yml b/products/cdbg/models/product/_product_models.yml index b34a99a61..090c748e0 100644 --- a/products/cdbg/models/product/_product_models.yml +++ b/products/cdbg/models/product/_product_models.yml @@ -5,7 +5,7 @@ models: description: Census block groups and their Community Development Block Grant (CDBG) eligibility details config: contract: - enforced: false + enforced: true columns: - name: geoid @@ -18,25 +18,17 @@ models: data_type: string tests: [not_null] -# - name: borough_code -# data_type: integer -# tests: [not_null] -# -# - name: borough_tract -# data_type: string -# tests: [not_null] -# -# - name: tract -# data_type: string -# tests: [not_null] -# -# - name: borough_tract_block_group -# data_type: string -# tests: [not_null] -# -# - name: block_group -# data_type: string -# tests: [not_null] + - name: borough_code + data_type: integer + tests: [not_null] + + - name: tract + data_type: string + tests: [not_null] + + - name: block_group + data_type: string + tests: [not_null] - name: total_floor_area data_type: integer @@ -47,7 +39,7 @@ models: tests: [not_null] - name: residential_floor_area_percentage - data_type: float + data_type: decimal tests: [not_null] - name: low_mod_income_population @@ -55,7 +47,7 @@ models: tests: [not_null] - name: low_mod_income_population_percentage - data_type: float + data_type: decimal tests: [not_null] - name: eligibility_flag @@ -66,7 +58,7 @@ models: data_type: string tests: [not_null] - - name: cdbg_borough + - name: cdbg_boroughs description: Borough and city-wide Community Development Block Grant (CDBG) details config: contract: @@ -75,7 +67,11 @@ models: columns: - name: borough_name data_type: string - tests: [not_null] + tests: [not_null, unique] + + - name: borough_code + data_type: integer + tests: [not_null, unique] - name: total_floor_area data_type: integer @@ -86,22 +82,30 @@ models: tests: [not_null] - name: residential_floor_area_percentage - data_type: float + data_type: decimal tests: [not_null] - name: low_mod_income_population - data_type: integer + data_type: bigint tests: [not_null] - name: low_mod_income_population_percentage - data_type: float + data_type: decimal + tests: [not_null] + + - name: eligibility_flag + data_type: boolean + tests: [not_null] + + - name: eligibility + data_type: string tests: [not_null] - name: cdbg_tracts description: Census tracts and their Community Development Block Grant (CDBG) eligibility details config: contract: - enforced: false + enforced: true columns: - name: geoid @@ -114,17 +118,9 @@ models: data_type: string tests: [not_null] -# - name: borough_code -# data_type: integer -# tests: [not_null] -# -# - name: borough_tract -# data_type: string -# tests: [not_null] -# -# - name: tract -# data_type: string -# tests: [not_null] + - name: borough_code + data_type: integer + tests: [not_null] - name: total_floor_area data_type: integer @@ -135,15 +131,15 @@ models: tests: [not_null] - name: residential_floor_area_percentage - data_type: float + data_type: decimal tests: [not_null] - name: low_mod_income_population - data_type: integer + data_type: bigint tests: [not_null] - name: low_mod_income_population_percentage - data_type: float + data_type: decimal tests: [not_null] - name: eligibility_flag diff --git a/products/cdbg/models/product/cdbg_block_groups.sql b/products/cdbg/models/product/cdbg_block_groups.sql index 1934d6034..2a98bc3f5 100644 --- a/products/cdbg/models/product/cdbg_block_groups.sql +++ b/products/cdbg/models/product/cdbg_block_groups.sql @@ -4,7 +4,16 @@ WITH block_groups AS ( eligibility_calculation AS ( SELECT - *, + geoid, + borough_name, + borough_code, + tract, + block_group, + round(total_floor_area::numeric)::integer AS total_floor_area, + round(residential_floor_area::numeric)::integer AS residential_floor_area, + round(residential_floor_area_percentage::numeric, 2) AS residential_floor_area_percentage, + low_mod_income_population, + round(low_mod_income_population_percentage::numeric, 2) AS low_mod_income_population_percentage, low_mod_income_population_percentage > 51 AND residential_floor_area_percentage > 50 AS eligibility_flag FROM block_groups ), diff --git a/products/cdbg/models/product/cdbg_boros.sql b/products/cdbg/models/product/cdbg_boroughs.sql similarity index 50% rename from products/cdbg/models/product/cdbg_boros.sql rename to products/cdbg/models/product/cdbg_boroughs.sql index 5a686bd0f..e1401def0 100644 --- a/products/cdbg/models/product/cdbg_boros.sql +++ b/products/cdbg/models/product/cdbg_boroughs.sql @@ -4,7 +4,13 @@ WITH boros AS ( eligibility_calculation AS ( SELECT - *, + borough_name, + borough_code, + round(total_floor_area::numeric)::integer AS total_floor_area, + round(residential_floor_area::numeric)::integer AS residential_floor_area, + round(residential_floor_area_percentage::numeric, 2) AS residential_floor_area_percentage, + low_mod_income_population::bigint, + round(low_mod_income_population_percentage::numeric, 2) AS low_mod_income_population_percentage, low_mod_income_population_percentage > 51 AND residential_floor_area_percentage > 50 AS eligibility_flag FROM boros ), diff --git a/products/cdbg/models/product/cdbg_tracts.sql b/products/cdbg/models/product/cdbg_tracts.sql index 374b14005..0fa9371cf 100644 --- a/products/cdbg/models/product/cdbg_tracts.sql +++ b/products/cdbg/models/product/cdbg_tracts.sql @@ -4,7 +4,14 @@ WITH tracts AS ( eligibility_calculation AS ( SELECT - *, + geoid, + borough_name, + borough_code, + round(total_floor_area::numeric)::integer AS total_floor_area, + round(residential_floor_area::numeric)::integer AS residential_floor_area, + round(residential_floor_area_percentage::numeric, 2) AS residential_floor_area_percentage, + low_mod_income_population, + round(low_mod_income_population_percentage::numeric, 2) AS low_mod_income_population_percentage, low_mod_income_population_percentage > 51 AND residential_floor_area_percentage > 50 AS eligibility_flag FROM tracts ), diff --git a/products/cdbg/models/staging/stg__census_data_blocks.sql b/products/cdbg/models/staging/stg__census_data_blocks.sql index 5f2860891..abc8617fe 100644 --- a/products/cdbg/models/staging/stg__census_data_blocks.sql +++ b/products/cdbg/models/staging/stg__census_data_blocks.sql @@ -1,6 +1,6 @@ SELECT geoid20 AS bctcb2020, - borocode AS borough_code, + borocode::integer AS borough_code, geogname AS borough_name, "pop1.1"::numeric AS total_population -- noqa: RF01 FROM {{ source("recipe_sources", "dcp_censusdata_blocks") }} diff --git a/products/cdbg/models/staging/stg__low_mod_by_block_group.sql b/products/cdbg/models/staging/stg__low_mod_by_block_group.sql index b3f26d002..75a457232 100644 --- a/products/cdbg/models/staging/stg__low_mod_by_block_group.sql +++ b/products/cdbg/models/staging/stg__low_mod_by_block_group.sql @@ -6,9 +6,9 @@ SELECT "GEOID"::text AS geoid, -- TODO: coerce to text in ingest "BORO" AS boro, - "TRACT" AS tract, - "BLKGRP" AS block_group, - REPLACE("LOWMODUNIV", ',', '')::numeric AS potential_lowmod_population, - REPLACE("LOWMOD", ',', '')::numeric AS low_mod_income_population, + "TRACT"::text AS tract, + "BLKGRP"::text AS block_group, + REPLACE("LOWMODUNIV", ',', '')::integer AS potential_lowmod_population, + REPLACE("LOWMOD", ',', '')::integer AS low_mod_income_population, RTRIM("LOWMOD_PCT", '%')::numeric AS low_mod_income_population_percentage FROM {{ source("recipe_sources", "hud_lowmodincomebyblockgroup") }} diff --git a/products/cdbg/recipe.yml b/products/cdbg/recipe.yml index d1a61f7aa..731291c70 100644 --- a/products/cdbg/recipe.yml +++ b/products/cdbg/recipe.yml @@ -1,5 +1,7 @@ name: Community Development Block Grant product: db-cdbg +version_strategy: + pin_to_source_dataset: hud_lowmodincomebyblockgroup inputs: missing_versions_strategy: find_latest datasets: