From f032107966b938e94a353e2fed274cdcdbfaa45f Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Fri, 3 Jan 2025 09:16:45 -0500 Subject: [PATCH 1/6] use boroughs instead of boros --- products/cdbg/models/product/_product_models.yml | 4 ++-- .../cdbg/models/product/{cdbg_boros.sql => cdbg_boroughs.sql} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename products/cdbg/models/product/{cdbg_boros.sql => cdbg_boroughs.sql} (100%) diff --git a/products/cdbg/models/product/_product_models.yml b/products/cdbg/models/product/_product_models.yml index b34a99a61..ea804e969 100644 --- a/products/cdbg/models/product/_product_models.yml +++ b/products/cdbg/models/product/_product_models.yml @@ -66,11 +66,11 @@ models: data_type: string tests: [not_null] - - name: cdbg_borough + - name: cdbg_boroughs description: Borough and city-wide Community Development Block Grant (CDBG) details config: contract: - enforced: true + enforced: false columns: - name: borough_name diff --git a/products/cdbg/models/product/cdbg_boros.sql b/products/cdbg/models/product/cdbg_boroughs.sql similarity index 100% rename from products/cdbg/models/product/cdbg_boros.sql rename to products/cdbg/models/product/cdbg_boroughs.sql From f3f2fd08159fe1fd2f321887d31e74a75b9eef2e Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Fri, 3 Jan 2025 09:03:14 -0500 Subject: [PATCH 2/6] export cdbg_boroughs --- products/cdbg/bash/export.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/products/cdbg/bash/export.sh b/products/cdbg/bash/export.sh index 1650a4476..7a7dbf3e9 100755 --- a/products/cdbg/bash/export.sh +++ b/products/cdbg/bash/export.sh @@ -17,6 +17,9 @@ mkdir -p output && ( echo "export cdbg_tracts.csv ..." csv_export cdbg_tracts cdbg_tracts + + echo "export cdbg_boroughs.csv ..." + csv_export cdbg_boroughs cdbg_boroughs ) zip -r output/output.zip output From 320a936ab7be88a054e6dfa4d7c957130dd19220 Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Fri, 3 Jan 2025 12:43:37 -0500 Subject: [PATCH 3/6] enfore `cdbg_block_groups` contract --- .../models/intermediate/int__block_groups.sql | 1 + .../cdbg/models/product/_product_models.yml | 36 ++++++++----------- .../cdbg/models/product/cdbg_block_groups.sql | 11 +++++- .../staging/stg__census_data_blocks.sql | 2 +- .../staging/stg__low_mod_by_block_group.sql | 8 ++--- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/products/cdbg/models/intermediate/int__block_groups.sql b/products/cdbg/models/intermediate/int__block_groups.sql index 469c03d4f..f821d990a 100644 --- a/products/cdbg/models/intermediate/int__block_groups.sql +++ b/products/cdbg/models/intermediate/int__block_groups.sql @@ -39,6 +39,7 @@ block_group_details AS ( SELECT block_groups_floor_area.geoid, block_groups_demographics.borough_name, + block_groups_demographics.borough_code, block_groups_demographics.tract, block_groups_demographics.block_group, block_groups_floor_area.total_floor_area, diff --git a/products/cdbg/models/product/_product_models.yml b/products/cdbg/models/product/_product_models.yml index ea804e969..285c9f974 100644 --- a/products/cdbg/models/product/_product_models.yml +++ b/products/cdbg/models/product/_product_models.yml @@ -5,7 +5,7 @@ models: description: Census block groups and their Community Development Block Grant (CDBG) eligibility details config: contract: - enforced: false + enforced: true columns: - name: geoid @@ -18,25 +18,17 @@ models: data_type: string tests: [not_null] -# - name: borough_code -# data_type: integer -# tests: [not_null] -# -# - name: borough_tract -# data_type: string -# tests: [not_null] -# -# - name: tract -# data_type: string -# tests: [not_null] -# -# - name: borough_tract_block_group -# data_type: string -# tests: [not_null] -# -# - name: block_group -# data_type: string -# tests: [not_null] + - name: borough_code + data_type: integer + tests: [not_null] + + - name: tract + data_type: string + tests: [not_null] + + - name: block_group + data_type: string + tests: [not_null] - name: total_floor_area data_type: integer @@ -47,7 +39,7 @@ models: tests: [not_null] - name: residential_floor_area_percentage - data_type: float + data_type: decimal tests: [not_null] - name: low_mod_income_population @@ -55,7 +47,7 @@ models: tests: [not_null] - name: low_mod_income_population_percentage - data_type: float + data_type: decimal tests: [not_null] - name: eligibility_flag diff --git a/products/cdbg/models/product/cdbg_block_groups.sql b/products/cdbg/models/product/cdbg_block_groups.sql index 1934d6034..2a98bc3f5 100644 --- a/products/cdbg/models/product/cdbg_block_groups.sql +++ b/products/cdbg/models/product/cdbg_block_groups.sql @@ -4,7 +4,16 @@ WITH block_groups AS ( eligibility_calculation AS ( SELECT - *, + geoid, + borough_name, + borough_code, + tract, + block_group, + round(total_floor_area::numeric)::integer AS total_floor_area, + round(residential_floor_area::numeric)::integer AS residential_floor_area, + round(residential_floor_area_percentage::numeric, 2) AS residential_floor_area_percentage, + low_mod_income_population, + round(low_mod_income_population_percentage::numeric, 2) AS low_mod_income_population_percentage, low_mod_income_population_percentage > 51 AND residential_floor_area_percentage > 50 AS eligibility_flag FROM block_groups ), diff --git a/products/cdbg/models/staging/stg__census_data_blocks.sql b/products/cdbg/models/staging/stg__census_data_blocks.sql index 5f2860891..abc8617fe 100644 --- a/products/cdbg/models/staging/stg__census_data_blocks.sql +++ b/products/cdbg/models/staging/stg__census_data_blocks.sql @@ -1,6 +1,6 @@ SELECT geoid20 AS bctcb2020, - borocode AS borough_code, + borocode::integer AS borough_code, geogname AS borough_name, "pop1.1"::numeric AS total_population -- noqa: RF01 FROM {{ source("recipe_sources", "dcp_censusdata_blocks") }} diff --git a/products/cdbg/models/staging/stg__low_mod_by_block_group.sql b/products/cdbg/models/staging/stg__low_mod_by_block_group.sql index b3f26d002..75a457232 100644 --- a/products/cdbg/models/staging/stg__low_mod_by_block_group.sql +++ b/products/cdbg/models/staging/stg__low_mod_by_block_group.sql @@ -6,9 +6,9 @@ SELECT "GEOID"::text AS geoid, -- TODO: coerce to text in ingest "BORO" AS boro, - "TRACT" AS tract, - "BLKGRP" AS block_group, - REPLACE("LOWMODUNIV", ',', '')::numeric AS potential_lowmod_population, - REPLACE("LOWMOD", ',', '')::numeric AS low_mod_income_population, + "TRACT"::text AS tract, + "BLKGRP"::text AS block_group, + REPLACE("LOWMODUNIV", ',', '')::integer AS potential_lowmod_population, + REPLACE("LOWMOD", ',', '')::integer AS low_mod_income_population, RTRIM("LOWMOD_PCT", '%')::numeric AS low_mod_income_population_percentage FROM {{ source("recipe_sources", "hud_lowmodincomebyblockgroup") }} From eb0e84c1e5e31950180cb8aefd971fbb3dd15c0c Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Sun, 5 Jan 2025 22:03:23 -0500 Subject: [PATCH 4/6] enfore `cdbg_tracts` contract --- .../cdbg/models/intermediate/int__tracts.sql | 8 +++++-- .../cdbg/models/product/_product_models.yml | 22 ++++++------------- products/cdbg/models/product/cdbg_tracts.sql | 9 +++++++- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/products/cdbg/models/intermediate/int__tracts.sql b/products/cdbg/models/intermediate/int__tracts.sql index 760783824..9c8578d7a 100644 --- a/products/cdbg/models/intermediate/int__tracts.sql +++ b/products/cdbg/models/intermediate/int__tracts.sql @@ -8,13 +8,17 @@ WITH block_groups AS ( tracts AS ( SELECT tract_id AS geoid, - max(borough_name) AS borough_name, + borough_name, + borough_code, sum(total_floor_area) AS total_floor_area, sum(residential_floor_area) AS residential_floor_area, sum(total_population) AS total_population, sum(low_mod_income_population) AS low_mod_income_population FROM block_groups - GROUP BY tract_id + GROUP BY + tract_id, + borough_name, + borough_code ), tracts_calculation AS ( diff --git a/products/cdbg/models/product/_product_models.yml b/products/cdbg/models/product/_product_models.yml index 285c9f974..4aa053bbf 100644 --- a/products/cdbg/models/product/_product_models.yml +++ b/products/cdbg/models/product/_product_models.yml @@ -93,7 +93,7 @@ models: description: Census tracts and their Community Development Block Grant (CDBG) eligibility details config: contract: - enforced: false + enforced: true columns: - name: geoid @@ -106,17 +106,9 @@ models: data_type: string tests: [not_null] -# - name: borough_code -# data_type: integer -# tests: [not_null] -# -# - name: borough_tract -# data_type: string -# tests: [not_null] -# -# - name: tract -# data_type: string -# tests: [not_null] + - name: borough_code + data_type: integer + tests: [not_null] - name: total_floor_area data_type: integer @@ -127,15 +119,15 @@ models: tests: [not_null] - name: residential_floor_area_percentage - data_type: float + data_type: decimal tests: [not_null] - name: low_mod_income_population - data_type: integer + data_type: bigint tests: [not_null] - name: low_mod_income_population_percentage - data_type: float + data_type: decimal tests: [not_null] - name: eligibility_flag diff --git a/products/cdbg/models/product/cdbg_tracts.sql b/products/cdbg/models/product/cdbg_tracts.sql index 374b14005..0fa9371cf 100644 --- a/products/cdbg/models/product/cdbg_tracts.sql +++ b/products/cdbg/models/product/cdbg_tracts.sql @@ -4,7 +4,14 @@ WITH tracts AS ( eligibility_calculation AS ( SELECT - *, + geoid, + borough_name, + borough_code, + round(total_floor_area::numeric)::integer AS total_floor_area, + round(residential_floor_area::numeric)::integer AS residential_floor_area, + round(residential_floor_area_percentage::numeric, 2) AS residential_floor_area_percentage, + low_mod_income_population, + round(low_mod_income_population_percentage::numeric, 2) AS low_mod_income_population_percentage, low_mod_income_population_percentage > 51 AND residential_floor_area_percentage > 50 AS eligibility_flag FROM tracts ), From 574830ffe9c5079ae84f779948bdda3605fcd864 Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Sun, 5 Jan 2025 22:03:33 -0500 Subject: [PATCH 5/6] enfore `cdbg_boroughs` contract --- .../cdbg/models/intermediate/int__boros.sql | 5 ++++- .../cdbg/models/product/_product_models.yml | 22 ++++++++++++++----- .../cdbg/models/product/cdbg_boroughs.sql | 8 ++++++- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/products/cdbg/models/intermediate/int__boros.sql b/products/cdbg/models/intermediate/int__boros.sql index 482e0c3c1..0e7a99d99 100644 --- a/products/cdbg/models/intermediate/int__boros.sql +++ b/products/cdbg/models/intermediate/int__boros.sql @@ -5,12 +5,15 @@ WITH tracts AS ( boros AS ( SELECT borough_name, + borough_code, sum(total_floor_area) AS total_floor_area, sum(residential_floor_area) AS residential_floor_area, sum(total_population) AS total_population, sum(low_mod_income_population) AS low_mod_income_population FROM tracts - GROUP BY borough_name + GROUP BY + borough_name, + borough_code ), boro_calculation AS ( diff --git a/products/cdbg/models/product/_product_models.yml b/products/cdbg/models/product/_product_models.yml index 4aa053bbf..090c748e0 100644 --- a/products/cdbg/models/product/_product_models.yml +++ b/products/cdbg/models/product/_product_models.yml @@ -62,12 +62,16 @@ models: description: Borough and city-wide Community Development Block Grant (CDBG) details config: contract: - enforced: false + enforced: true columns: - name: borough_name data_type: string - tests: [not_null] + tests: [not_null, unique] + + - name: borough_code + data_type: integer + tests: [not_null, unique] - name: total_floor_area data_type: integer @@ -78,15 +82,23 @@ models: tests: [not_null] - name: residential_floor_area_percentage - data_type: float + data_type: decimal tests: [not_null] - name: low_mod_income_population - data_type: integer + data_type: bigint tests: [not_null] - name: low_mod_income_population_percentage - data_type: float + data_type: decimal + tests: [not_null] + + - name: eligibility_flag + data_type: boolean + tests: [not_null] + + - name: eligibility + data_type: string tests: [not_null] - name: cdbg_tracts diff --git a/products/cdbg/models/product/cdbg_boroughs.sql b/products/cdbg/models/product/cdbg_boroughs.sql index 5a686bd0f..e1401def0 100644 --- a/products/cdbg/models/product/cdbg_boroughs.sql +++ b/products/cdbg/models/product/cdbg_boroughs.sql @@ -4,7 +4,13 @@ WITH boros AS ( eligibility_calculation AS ( SELECT - *, + borough_name, + borough_code, + round(total_floor_area::numeric)::integer AS total_floor_area, + round(residential_floor_area::numeric)::integer AS residential_floor_area, + round(residential_floor_area_percentage::numeric, 2) AS residential_floor_area_percentage, + low_mod_income_population::bigint, + round(low_mod_income_population_percentage::numeric, 2) AS low_mod_income_population_percentage, low_mod_income_population_percentage > 51 AND residential_floor_area_percentage > 50 AS eligibility_flag FROM boros ), From c199425a2e917ecb5da2a998648e7233bb6f1318 Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Sun, 5 Jan 2025 22:13:25 -0500 Subject: [PATCH 6/6] set a version strategy for cdbg pinning to HUD source data version was recommended by OMB --- products/cdbg/recipe.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/products/cdbg/recipe.yml b/products/cdbg/recipe.yml index d1a61f7aa..731291c70 100644 --- a/products/cdbg/recipe.yml +++ b/products/cdbg/recipe.yml @@ -1,5 +1,7 @@ name: Community Development Block Grant product: db-cdbg +version_strategy: + pin_to_source_dataset: hud_lowmodincomebyblockgroup inputs: missing_versions_strategy: find_latest datasets: