data formatting and tract fixes

uchicago-dsi · May 15, 2024 · 3d469e7 · 3d469e7
1 parent e94ebc2
commit 3d469e7
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 42 deletions.
diff --git a/app/tract/[tract]/page.tsx b/app/tract/[tract]/page.tsx
@@ -2,7 +2,6 @@ import { ArrowLeftIcon } from "@radix-ui/react-icons"
 import dynamic from "next/dynamic"
 import React from "react"
 import { TinaMarkdown, TinaMarkdownContent } from "tinacms/dist/rich-text"
-import path from "path"
 import PercentileLineChart from "components/PercentileLineChart"
 import TimeseriesChart from "components/TimeseriesChart"
 import Tooltip from "components/Tooltip"
@@ -11,11 +10,6 @@ import { getContentDirs } from "utils/contentDirs"
 import { getThresholdValue } from "utils/data/formatDataTemplate"
 import { getSummaryStats } from "utils/data/summaryStats"
 import StoreList from "components/StoreList"
-import wasmInit, {readParquet} from "parquet-wasm";
-
-import { parquetRead } from "hyparquet"
-import { readFileSync } from "fs"
-
 const Map = dynamic(() => import("components/Map/Map"), { ssr: false })
 
 type TractRouteProps = {
@@ -146,7 +140,7 @@ const TractPage: React.FC<TractRouteProps> = async ({ params }) => {
     100 - +data[segregationTemplate.column as keyof typeof data],
     +data[economicAdvantageTemplate.column as keyof typeof data]
   ]
-  console.log('food access', data)
+
   const foodAccessText = getThresholdValue(foodAccess, data, foodAccesstemplate)
   const marketPowerText = getThresholdValue(marketPower, data, marketPowerTemplate)
   const segregationText = getThresholdValue(segregation, data, segregationTemplate)

diff --git a/components/StoreTest/index.tsx b/components/StoreTest/index.tsx
@@ -0,0 +1,17 @@
+"use client"
+import { Provider } from "react-redux";
+import { store, useAppSelector } from "utils/state/store";
+
+const Outer = () => {
+  return <Provider store={store}>
+    <StoreTest />
+  </Provider>
+}
+
+const StoreTest = () => {
+  const store = useAppSelector(s => s)
+  console.log(store)
+  return null
+}
+
+export default Outer
diff --git a/content/statistics/tract.mdx b/content/statistics/tract.mdx
@@ -85,7 +85,7 @@ stat:
           * Non-hispanic American Indian: %%100|\*|PCT NH AMERICAN INDIAN%%%
           * Non-hispanic Asian: %%100|\*|PCT NH ASIAN%%%
           * Non-hispanic Pacific Islander: %%100|\*|PCT NH PACIFIC ISLANDER%%%
-          * Non-hispanic Some Another race: %%%100|\*|PCT NH SOME OTHER%%%
+          * Non-hispanic Some Another race: %%100|\*|PCT NH SOME OTHER%%%
           * Non-hispanic Two or more races: %%100|\*|PCT NH TWO OR MORE%%%
 overview:
   - title: Economic Advantage

diff --git a/scripts/precompute_summary_stats.py b/scripts/precompute_summary_stats.py
@@ -200,6 +200,8 @@ def split_df_and_msgpack(df, id_col, outpath, compress=False):
   .merge(county_adi, how='left', left_on="county", right_on="COUNTY")\
   .drop(columns=["COUNTY"])
 # %%
+# %%
+
 split_df_and_msgpack(
   county_joined,
   'county',
@@ -307,16 +309,16 @@ def get_full_data(path, column_dict, out_cols=["GEOID"]):
 full_out.to_parquet(path.join(data_dir, 'full_tract.parquet'), compression='gzip')
 # %%
 columnarize_msgpack(
-  tract_joined.to_dict(orient="records"), 
+  full_out.to_dict(orient="records"), 
   "GEOID", 
-  path.join(data_dir, f'tract_full.msgpack'), 
-  list(tract_joined.columns), 
+  path.join(data_dir, f'tract_full.msgpack.gz'), 
+  list(df_full.columns), 
   compress=True
 )
 # %%
 # %%
 split_df_and_msgpack(
-  tract_joined,
+  full_out,
   'GEOID',
   path.join(data_dir, 'summary', 'tract'),
   compress=True

diff --git a/utils/data/config.ts b/utils/data/config.ts
@@ -3,35 +3,39 @@ import { ColumnConfig, ColumnGroups } from "./config.types"
 
 export const idColumn = "GEOID"
 
-const generateHhiConfig = (year: number) =>
+const generateHhiConfig = (year: number, ds: boolean =false) =>
   ({
     name: `Concentration Index ${year} (No Dollar Stores)`,
-    column: `hhi_${year}`, // hhi_2000, hhi_2010
+    column: `hhi_${ds ? 'ds_' : ''}${year}`, // hhi_2000, hhi_2010
     bivariate: false,
     description: `Herfindahl-Hirschman Index for ${year}`,
     colorScheme: "schemeGreens"
   }) as ColumnConfig
 
-const generateGravityConfig = (year: number, dollar_stores: boolean) =>
+const generateGravityConfig = (year: number, ds: boolean = false) =>
   ({
     name: `Gravity ${year}`,
-    column: `gravity_${year}`,
+    column: `gravity_${ds ? 'ds_' : ''}${year}`,
     bivariate: false,
     nBins: 9,
     colorScheme: "schemeSpectral",
     description: `Gravity for ${year}`,
   }) as ColumnConfig
 
 export const columnsDict = {
-  "Concentration Index 2000 (No Dollar Stores)": generateHhiConfig(2000),
-  "Concentration Index 2010 (No Dollar Stores)": generateHhiConfig(2010),
-  "Concentration Index 2020 (No Dollar Stores)": generateHhiConfig(2020),
-  "Gravity 2000": generateGravityConfig(2000, false),
-  "Gravity 2010": generateGravityConfig(2010, false),
-  "Gravity 2020": generateGravityConfig(2020, false),
-  "Gravity 2000 (With Dollar Stores)": generateGravityConfig(2000, true),
-  "Gravity 2010 (With Dollar Stores)": generateGravityConfig(2010, true),
-  "Gravity 2020 (With Dollar Stores)": generateGravityConfig(2020, true),
+  "Market Concentration - 2000": generateHhiConfig(2000),
+  "Market Concentration - 2010": generateHhiConfig(2010),
+  "Market Concentration - 2020": generateHhiConfig(2020),
+  "Market Concentration - 2023 (Most Recent)": generateHhiConfig(2023),
+  "Market Concentration - 2000 (With Dollar Stores)": generateHhiConfig(2000, true),
+  "Market Concentration - 2010 (With Dollar Stores)": generateHhiConfig(2010, true),
+  "Market Concentration - 2020 (With Dollar Stores)": generateHhiConfig(2020, true),
+  "Food Access Supply - 2000": generateGravityConfig(2000),
+  "Food Access Supply - 2010": generateGravityConfig(2010),
+  "Food Access Supply - 2020": generateGravityConfig(2020),
+  "Food Access Supply - 2000 (With Dollar Stores)": generateGravityConfig(2000, true),
+  "Food Access Supply - 2010 (With Dollar Stores)": generateGravityConfig(2010, true),
+  "Food Access Supply - 2020 (With Dollar Stores)": generateGravityConfig(2020, true),
   "Measure of Segregation (Black/African American and White)": {
     name: "Segregation Factor ICE Hispanic NH White Alone",
     bivariate: false,
@@ -189,20 +193,24 @@ export const columnGroups: ColumnGroups<typeof columnsDict> = {
   "Market Concentration": {
     description: "Concentration Metrics",
     columns: [
-      "Concentration Index 2020 (No Dollar Stores)",
-      "Concentration Index 2010 (No Dollar Stores)",
-      "Concentration Index 2000 (No Dollar Stores)",
+        "Market Concentration - 2023 (Most Recent)",
+        "Market Concentration - 2020",
+        "Market Concentration - 2010",
+        "Market Concentration - 2000",
+        "Market Concentration - 2020 (With Dollar Stores)",
+        "Market Concentration - 2010 (With Dollar Stores)",
+        "Market Concentration - 2000 (With Dollar Stores)",
     ],
   },
   "Food Access": {
     description: "Gravity",
     columns: [
-      "Gravity 2020",
-      "Gravity 2010",
-      "Gravity 2000",
-      "Gravity 2020 (With Dollar Stores)",
-      "Gravity 2010 (With Dollar Stores)",
-      "Gravity 2000 (With Dollar Stores)",
+      "Food Access Supply - 2020",
+      "Food Access Supply - 2010",
+      "Food Access Supply - 2000",
+      "Food Access Supply - 2020 (With Dollar Stores)",
+      "Food Access Supply - 2010 (With Dollar Stores)",
+      "Food Access Supply - 2000 (With Dollar Stores)",
     ],
   },
   "Racial Equity": {
@@ -256,15 +264,15 @@ export const tooltipConfig: Array<{
     section: "Food Access",
     columns: [
       {
-        col: "Gravity 2020",
+        col: "Food Access Supply - 2020",
         label: "2020",
       },
       {
-        col: "Gravity 2010",
+        col: "Food Access Supply - 2010",
         label: "2010",
       },
       {
-        col: "Gravity 2000",
+        col: "Food Access Supply - 2000",
         label: "2000",
       },
     ],
@@ -273,15 +281,19 @@ export const tooltipConfig: Array<{
     section: "Market Concentration",
     columns: [
       {
-        col: "Concentration Index 2020 (No Dollar Stores)",
+        col: "Market Concentration - 2023 (Most Recent)",
+        label: "2023",
+      },
+      {
+        col: "Market Concentration - 2020",
         label: "2020",
       },
       {
-        col: "Concentration Index 2010 (No Dollar Stores)",
+        col: "Market Concentration - 2010",
         label: "2010",
       },
       {
-        col: "Concentration Index 2000 (No Dollar Stores)",
+        col: "Market Concentration - 2000",
         label: "2000",
       },
     ],

diff --git a/utils/data/formatDataTemplate.ts b/utils/data/formatDataTemplate.ts
@@ -39,8 +39,11 @@ export const formatDataTemplate = <T extends Record<string, any>>(_template: str
           const value = data[part] as any
           return value || null
         })
-        const [value1, operator, value2] = parts
-        if (!value1 || !value2 || !operator) return
+        const [value1, operator, value2] = parts.map(part => part === null ? 0 : part)
+        if (!value1 === null || !value2 === null || !operator) {
+          template = ''
+          return 
+        }
         const result = formatter.format(handleOperator(operator, value1, value2))
         template = template.replace(match, `${result}`)
       } else {