From e520b022bf0f53d5af7b2794941909082ea8b427 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Thu, 5 Sep 2024 11:19:54 +0200 Subject: [PATCH 01/11] feat: Extract useGeoData --- .../components/Generator/internal/Preview.tsx | 116 ++++++------------ website/src/domain/geodata.ts | 68 ++++++++++ 2 files changed, 103 insertions(+), 81 deletions(-) create mode 100644 website/src/domain/geodata.ts diff --git a/website/src/components/Generator/internal/Preview.tsx b/website/src/components/Generator/internal/Preview.tsx index ba314770..39a4b511 100644 --- a/website/src/components/Generator/internal/Preview.tsx +++ b/website/src/components/Generator/internal/Preview.tsx @@ -4,15 +4,12 @@ import DeckGL from "@deck.gl/react"; import * as MUI from "@material-ui/core"; import clsx from "clsx"; import * as d3 from "d3"; -import cityData from "public/swiss-city-topo.json"; import * as React from "react"; -import { useQuery } from "react-query"; import { COLOR_SCHEMA_MAP } from "src/domain/color-schema"; -import { previewSourceUrl } from "src/shared"; -import * as topojson from "topojson"; import { useImmer } from "use-immer"; -import { useContext } from "../context"; +import { useContext, Value } from "../context"; import { CH_BBOX, constrainZoom, LINE_COLOR } from "../domain/deck-gl"; +import { useGeoData } from "src/domain/geodata"; interface Props {} @@ -33,58 +30,11 @@ export const Preview = React.forwardRef(({}: Props, deckRef: any) => { const { options } = ctx.state; const [state, mutate] = useImmer({ - fetching: false, viewState: INITIAL_VIEW_STATE, - geoData: { - country: undefined as any, - cantons: undefined as any, - neighbors: undefined as Array | undefined, - municipalities: undefined as any, - lakes: undefined as any, - city: undefined as any, - }, }); - const { data: json, isFetching } = useQuery( - ["preview", options.year, options.simplify, ...options.shapes], - () => fetch(previewSourceUrl(options, "v0")).then((res) => res.json()) - ); - - React.useEffect(() => { - if (!json) { - return; - } - mutate((draft) => { - if (cityData) { - draft.geoData.city = topojson.feature( - cityData as any, - cityData.objects["swiss-city"] as any - ); - } - - if (json.objects?.country) { - draft.geoData.country = topojson.feature(json, json.objects.country); - } - - if (json.objects?.cantons) { - draft.geoData.cantons = topojson.feature(json, json.objects.cantons); - draft.geoData.neighbors = topojson.neighbors( - json.objects.cantons.geometries - ); - } - - if (json.objects?.municipalities) { - draft.geoData.municipalities = topojson.feature( - json, - json.objects.municipalities - ); - } - - if (json.objects?.lakes) { - draft.geoData.lakes = topojson.feature(json, json.objects.lakes); - } - }); - }, [json]); + const query = useGeoData(options); + const { data: geoData, isFetching } = query; /* const onViewStateChange = React.useCallback( @@ -118,11 +68,11 @@ export const Preview = React.forwardRef(({}: Props, deckRef: any) => { * See https://observablehq.com/@mbostock/map-coloring * */ const colorIndex = (() => { - const { cantons, neighbors } = state.geoData; + const { cantons, neighbors } = geoData; if (!neighbors) { return undefined; } - const index = new Int32Array(cantons.features.length); + const index = new Int32Array(cantons ? cantons.features.length : 0); for (let i = 0; i < index.length; ++i) { index[i] = ((d3.max(neighbors[i], (j) => index[j]) as number) + 1) | 0; } @@ -139,10 +89,15 @@ export const Preview = React.forwardRef(({}: Props, deckRef: any) => { // domain is decided by coloring item size // currently only support cantons // if not exist, a random number 30 is assigned - .domain(["1", state.geoData?.cantons?.length ?? "30"]) + .domain([ + "1", + geoData?.cantons?.features?.length + ? `${geoData?.cantons?.features?.length}` + : "30", + ]) .range(color) ); - }, [options.color, state.geoData.cantons]); + }, [options.color, geoData.cantons]); return (
@@ -165,7 +120,7 @@ export const Preview = React.forwardRef(({}: Props, deckRef: any) => { {options.shapes.has("country") && ( { {options.shapes.has("cantons") && ( { /> )} - {state.geoData.municipalities && - options.shapes.has("municipalities") && ( - - )} + {geoData.municipalities && options.shapes.has("municipalities") && ( + + )} {options.shapes.has("lakes") && ( { {options.withName && ( { {ctx.state.highlightedShape && options.shapes.has(ctx.state.highlightedShape) && (() => { - const data = state.geoData[ - ctx.state.highlightedShape as keyof typeof state.geoData + const data = geoData[ + ctx.state.highlightedShape as keyof typeof geoData ] as $FixMe; if (ctx.state.highlightedShape === "lakes") { diff --git a/website/src/domain/geodata.ts b/website/src/domain/geodata.ts new file mode 100644 index 00000000..d1a333fa --- /dev/null +++ b/website/src/domain/geodata.ts @@ -0,0 +1,68 @@ +import cityData from "public/swiss-city-topo.json"; +import { useQuery } from "react-query"; +import { previewSourceUrl } from "src/shared"; +import * as topojson from "topojson"; +import { Value } from "../components/Generator/context"; +import { MultiPolygon } from "geojson"; + +const getGeoData = (json: any) => { + const geoData = { + country: json.objects?.country + ? topojson.feature(json, json.objects.country) + : undefined, + cantons: json.objects?.cantons + ? topojson.feature(json, json.objects.cantons) + : undefined, + neighbors: json.objects?.cantons + ? topojson.neighbors(json.objects.cantons.geometries) + : undefined, + municipalities: json.objects?.municipalities + ? topojson.feature(json, json.objects.municipalities as MultiPolygon) + : undefined, + lakes: json.objects?.lakes + ? topojson.feature(json, json.objects.lakes) + : undefined, + city: cityData + ? topojson.feature(cityData as any, cityData.objects["swiss-city"] as any) + : undefined, + }; + return geoData; +}; + +const fetchGeoData = (options: Value["state"]["options"]) => { + return fetch(previewSourceUrl(options, "v0")) + .then((res) => res.json()) + .then((json) => { + return getGeoData(json); + }); +}; + +export const useGeoData = ( + options: Omit & { + year: undefined | string; + }, + queryOptions?: { + enabled?: boolean; + } +) => { + const query = useQuery({ + queryKey: ["geoData", options.year!, options.simplify, ...options.shapes], + queryFn: () => + options ? fetchGeoData(options as Value["state"]["options"]) : undefined, + enabled: options && options.year ? queryOptions?.enabled : false, + }); + + return { + ...query, + // TODO replace by initialData once react-query is upgraded to v4 + // https://github.com/TanStack/query/discussions/1331#discussioncomment-4802682 + data: query.data ?? { + country: undefined, + cantons: undefined, + neighbors: undefined as Array | undefined, + municipalities: undefined, + lakes: undefined, + city: undefined, + }, + }; +}; From f427a21408d8bbee8836a1ba4551e5ee068aa551 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Thu, 5 Sep 2024 14:00:37 +0200 Subject: [PATCH 02/11] feat: Add mutations page to review mutations of municipality --- .prettierrc | 4 + website/package.json | 4 +- website/src/components/Mutations/Map.tsx | 49 + website/src/domain/geodata.ts | 23 +- website/src/pages/mutations.tsx | 354 +++++ website/yarn.lock | 1598 +++++++++++++++++++++- 6 files changed, 2024 insertions(+), 8 deletions(-) create mode 100644 .prettierrc create mode 100644 website/src/components/Mutations/Map.tsx create mode 100644 website/src/pages/mutations.tsx diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..222861c3 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 2, + "useTabs": false +} diff --git a/website/package.json b/website/package.json index b130107c..ee1f871a 100644 --- a/website/package.json +++ b/website/package.json @@ -10,6 +10,7 @@ "@deck.gl/layers": "^8.9.4", "@deck.gl/react": "^8.9.4", "@material-ui/core": "^4.12.4", + "@turf/turf": "^7.1.0", "@types/cors": "^2.8.17", "@types/d3": "^7.4.0", "@types/node": "^20.11.30", @@ -32,7 +33,8 @@ "swiss-maps": "^4.5.0", "topojson": "^3.0.2", "typescript": "^5.4.3", - "use-immer": "^0.10.0" + "use-immer": "^0.10.0", + "zod": "^3.23.8" }, "devDependencies": {} } diff --git a/website/src/components/Mutations/Map.tsx b/website/src/components/Mutations/Map.tsx new file mode 100644 index 00000000..994bb534 --- /dev/null +++ b/website/src/components/Mutations/Map.tsx @@ -0,0 +1,49 @@ +import { GeoJsonLayer } from "@deck.gl/layers"; +import DeckGL from "@deck.gl/react"; +import React, { ComponentProps } from "react"; +import { useGeoData } from "src/domain/geodata"; +import { MapController } from "@deck.gl/core"; + +export const LINE_COLOR = [100, 100, 100, 127] as const; + +const MutationsMap = ({ + highlightedMunicipalities, + geoData, + ...props +}: { + highlightedMunicipalities: { + added: number[]; + removed: number[]; + }; + geoData: ReturnType["data"]; +} & ComponentProps) => { + return ( + + {geoData.municipalities && ( + { + return highlightedMunicipalities.added.includes(d.properties.id) + ? [0, 255, 0, 100] + : highlightedMunicipalities.removed.includes(d.properties.id) + ? [255, 0, 0, 100] + : [255, 255, 255]; + }} + /> + )} + + ); +}; + +export default MutationsMap; diff --git a/website/src/domain/geodata.ts b/website/src/domain/geodata.ts index d1a333fa..522b70d6 100644 --- a/website/src/domain/geodata.ts +++ b/website/src/domain/geodata.ts @@ -4,26 +4,38 @@ import { previewSourceUrl } from "src/shared"; import * as topojson from "topojson"; import { Value } from "../components/Generator/context"; import { MultiPolygon } from "geojson"; +import { Feature, Geometry, GeoJsonProperties } from "geojson"; + +export type GeoDataFeature = Feature; + +const castFeatures = (d: any) => { + return d as { features: GeoDataFeature[] }; +}; const getGeoData = (json: any) => { const geoData = { country: json.objects?.country - ? topojson.feature(json, json.objects.country) + ? castFeatures(topojson.feature(json, json.objects.country)) : undefined, cantons: json.objects?.cantons - ? topojson.feature(json, json.objects.cantons) + ? castFeatures(topojson.feature(json, json.objects.cantons)) : undefined, neighbors: json.objects?.cantons ? topojson.neighbors(json.objects.cantons.geometries) : undefined, municipalities: json.objects?.municipalities - ? topojson.feature(json, json.objects.municipalities as MultiPolygon) + ? castFeatures(topojson.feature(json, json.objects.municipalities)) : undefined, lakes: json.objects?.lakes - ? topojson.feature(json, json.objects.lakes) + ? castFeatures(topojson.feature(json, json.objects.lakes)) : undefined, city: cityData - ? topojson.feature(cityData as any, cityData.objects["swiss-city"] as any) + ? castFeatures( + topojson.feature( + cityData as any, + cityData.objects["swiss-city"] as any + ) + ) : undefined, }; return geoData; @@ -33,6 +45,7 @@ const fetchGeoData = (options: Value["state"]["options"]) => { return fetch(previewSourceUrl(options, "v0")) .then((res) => res.json()) .then((json) => { + console.log({ json }); return getGeoData(json); }); }; diff --git a/website/src/pages/mutations.tsx b/website/src/pages/mutations.tsx new file mode 100644 index 00000000..8933a0da --- /dev/null +++ b/website/src/pages/mutations.tsx @@ -0,0 +1,354 @@ +import { z } from "zod"; +import { useEffect, useState } from "react"; +import { + Box, + Button, + Link, + List, + ListItem, + ListItemText, + Typography, +} from "@material-ui/core"; +import dynamic from "next/dynamic"; +import { groupBy } from "fp-ts/lib/NonEmptyArray"; +import { GeoDataFeature, useGeoData } from "src/domain/geodata"; +import * as turf from "@turf/turf"; +import { FlyToInterpolator } from "@deck.gl/core"; +import { parse } from "path"; + +const row = z.object({ + "N° d'hist.": z.string(), + Canton: z.string(), + "N° du district": z.string(), + "Nom du district": z.string(), + "N° OFS commune": z.string().transform((x) => Number(x)), + "Nom de la commune": z.string(), + "Raison de la radiation": z.string(), + "Raison de l'inscription": z.string(), + + "Numéro de mutation": z.string(), + "Type de mutation": z.string(), + "Entrée en vigueur": z.string(), +}); + +interface CsvRow { + "N° d'hist.": string; + Canton: string; + "N° du district": string; + "Nom du district": string; + "N° OFS commune": string; + "Nom de la commune": string; + "Raison de la radiation": string; + "Raison de l'inscription": string; + "Numéro de mutation": string; + "Type de mutation": string; + "Entrée en vigueur": string; +} + +const parseHTML = (htmlContent: string) => { + const parser = new DOMParser(); + const doc = parser.parseFromString( + `${htmlContent}`, + "text/html" + ); + const table = doc.querySelector("table"); + if (!table) { + throw new Error("Could not find table"); + } + const rows = table.querySelectorAll("tr"); + const headers = Array.from(rows[0].querySelectorAll("th")).map( + (header) => header.textContent + ); + const data: CsvRow[] = []; + for (let i = 1; i < rows.length; i++) { + const cells = rows[i].querySelectorAll("td"); + const rowData: CsvRow = {} as CsvRow; + for (let j = 0; j < cells.length; j++) { + const header = headers[j]; + if (!header) { + continue; + } + rowData[header as NonNullable] = cells[j] + .textContent as string; + } + data.push(rowData); + } + + // Filter out special lines and extract mutation information + const filteredData: CsvRow[] = []; + let mutationInfo: { num: string; type: string; date: string } | null = null; + + data.forEach((row) => { + if (row["N° d'hist."].includes("Numéro de mutation :")) { + mutationInfo = { + num: "", + type: "", + date: "", + }; + const rx = + /Numéro de mutation : (.*), Type de mutation : (.*), Entrée en vigueur : (.*)/; + const match = row["N° d'hist."].match(rx); + if (match) { + mutationInfo.num = match[1]; + mutationInfo.type = match[2]; + mutationInfo.date = match[3]; + } + } else { + if (mutationInfo) { + row["Numéro de mutation"] = mutationInfo.num; + row["Type de mutation"] = mutationInfo.type; + row["Entrée en vigueur"] = mutationInfo.date; + } + filteredData.push(row); + } + }); + + const mutations = z.array(row).parse(filteredData); + const groupedMutations = groupBy( + (mutation) => mutation["Numéro de mutation"] + )(mutations); + + return groupedMutations; +}; + +type Row = z.infer; + +const MutationsMap = dynamic(() => import("../components/Mutations/Map"), { + ssr: false, +}); + +const parseMutationRows = (rows: Row[]) => { + const removed = rows.filter((r) => r["Raison de la radiation"]); + const added = rows.filter((r) => r["Raison de l'inscription"]); + return { + label: `+ ${added + .map((x) => x["Nom de la commune"]) + .join(", ")} / - ${removed + .map((x) => x["Nom de la commune"]) + .join(", ")}`, + added, + removed, + year: Number(rows[0]?.["Entrée en vigueur"].split(".")[2]), + "Entrée en vigueur": rows[0]?.["Entrée en vigueur"], + "Numéro de mutation": rows[0]?.["Numéro de mutation"], + }; +}; + +type Parsed = ReturnType; + +const INITIAL_VIEW_STATE = { + latitude: 46.8182, + longitude: 8.2275, + zoom: 7, + maxZoom: 16, + minZoom: 2, + pitch: 0, + bearing: 0, + transitionInterpolator: new FlyToInterpolator(), + transitionDuration: 1000, +}; + +export default function Page() { + const [groupedMutations, setGroupedMutations] = useState< + ReturnType[] + >([]); + const [highlightedMunicipalities, setHighlightedMunicipalities] = + useState(); + + const handleMutationSelect = (parsed: Parsed) => { + setHighlightedMunicipalities(parsed); + setYear1(`${parsed.year - 1}`); + setYear2(`${parsed.year}`); + }; + + const [viewState, setViewState] = useState(INITIAL_VIEW_STATE); + + const [year1, setYear1] = useState("2022"); + const [year2, setYear2] = useState("2024"); + const { data: geoData1 } = useGeoData({ + year: year1, + format: "topojson", + simplify: 0.02, + shapes: new Set(["municipalities"]), + projection: "wgs84", + color: "default", + dimensions: { width: 800, height: 600 }, + withName: false, + }); + + const { data: geoData2 } = useGeoData({ + year: year2, + format: "topojson", + simplify: 0.02, + shapes: new Set(["municipalities"]), + projection: "wgs84", + color: "default", + dimensions: { width: 800, height: 600 }, + withName: false, + }); + useEffect(() => { + const { added = [], removed = [] } = highlightedMunicipalities ?? {}; + const all = [...added, ...removed].map((x) => x["N° OFS commune"]); + const findFeature = (x: GeoDataFeature) => all.includes(x.properties?.id); + const municipality = + geoData1.municipalities?.features.find(findFeature) ?? + geoData2.municipalities?.features.find(findFeature); + if (municipality) { + const mbbox = turf.center(municipality); + setViewState((prev) => ({ + ...prev, + longitude: mbbox.geometry.coordinates[0], + latitude: mbbox.geometry.coordinates[1], + zoom: 9, + + transitionInterpolator: new FlyToInterpolator(), + transitionDuration: 300, + })); + } else { + console.log( + "Cannot find municipality", + municipality, + highlightedMunicipalities + ); + } + }, [highlightedMunicipalities, geoData1, geoData2]); + + const handleChangeContent = (html: string) => { + try { + const data = Object.values(parseHTML(html)); + const groupedMutations = data.map(parseMutationRows); + setGroupedMutations(groupedMutations); + setHighlightedMunicipalities(groupedMutations[0]); + } catch (e) { + alert(e instanceof Error ? e.message : `${e}`); + } + }; + + return ( + + + + + Copy/paste here the HTML content of the mutation table, see for{" "} + + here + + . Choose display "100" elements to be able to copy all mutations. . + +