From 5b2a100aaee7e86a2ad6bdd6a35ec23561606dca Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 11:23:58 +0100 Subject: [PATCH 01/11] refactor: Extract generation from handler --- website/src/pages/api/generate.ts | 97 +++++++++++++++++++------------ 1 file changed, 59 insertions(+), 38 deletions(-) diff --git a/website/src/pages/api/generate.ts b/website/src/pages/api/generate.ts index 5a8773a7..ddfaa1c3 100644 --- a/website/src/pages/api/generate.ts +++ b/website/src/pages/api/generate.ts @@ -42,6 +42,60 @@ const Query = t.type({ download: t.union([t.undefined, t.string]), }); +const generateFromShapefiles = async ({ + format, + year, + shapes, + simplify +} : { + format: 'topojson' | 'svg', + year: string, + shapes: string[] + simplify?: number, +}) => { + + const input = await (async () => { + const props = shapes.flatMap((shape) => { + return ["shp", "dbf", "prj"].map( + async (ext) => + [ + `${shape}.${ext}`, + await fs.readFile( + path.join( + process.cwd(), + "public", + "swiss-maps", + year, + `${shape}.${ext}` + ) + ), + ] as const + ); + }); + return Object.fromEntries(await Promise.all(props)); + })(); + + const inputFiles = shapes.map((shape) => `${shape}.shp`).join(" "); + + const commands = [ + `-i ${inputFiles} combine-files string-fields=*`, + // `-rename-layers ${shp.join(",")}`, + simplify ? `-simplify ${simplify} keep-shapes` : "", + "-clean", + `-proj ${format === "topojson" ? "wgs84" : "somerc"}`, + `-o output.${format} format=${format} drop-table id-field=id`, + ].join("\n"); + + console.log("### Mapshaper commands ###"); + console.log(commands); + + const output = await mapshaper.applyCommands(commands, input); + + return format === "topojson" + ? output["output.topojson"] + : output["output.svg"]; +}; + export default async function handler( req: NextApiRequest, res: NextApiResponse @@ -72,45 +126,12 @@ export default async function handler( draft.shapes = new Set(query.shapes.split(",") as $FixMe); } }); - const { format, year, shapes } = options; - // console.log(year, shapes); - - const input = await (async () => { - const props = [...shapes].flatMap((shape) => { - return ["shp", "dbf", "prj"].map( - async (ext) => - [ - `${shape}.${ext}`, - await fs.readFile( - path.join( - process.cwd(), - "public", - "swiss-maps", - year, - `${shape}.${ext}` - ) - ), - ] as const - ); - }); - return Object.fromEntries(await Promise.all(props)); - })(); - - const inputFiles = [...shapes].map((shape) => `${shape}.shp`).join(" "); - - const commands = [ - `-i ${inputFiles} combine-files string-fields=*`, - // `-rename-layers ${shp.join(",")}`, - query.simplify ? `-simplify ${query.simplify} keep-shapes` : "", - "-clean", - `-proj ${format === "topojson" ? "wgs84" : "somerc"}`, - `-o output.${format} format=${format} drop-table id-field=id`, - ].join("\n"); - - console.log("### Mapshaper commands ###"); - console.log(commands); - const output = await mapshaper.applyCommands(commands, input); + const { format } = options + const output = await generateFromShapefiles({ + ...options, + shapes: [...options.shapes] + }) if (format === "topojson") { if (query.download !== undefined) { From 776995c3d333deaf471b6195981152ddce0eed60 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 11:39:45 +0100 Subject: [PATCH 02/11] refactor: Extract generation from HTTP handler --- website/src/pages/api/v0.ts | 101 +++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 43 deletions(-) diff --git a/website/src/pages/api/v0.ts b/website/src/pages/api/v0.ts index 1e700a6a..0816f9a4 100644 --- a/website/src/pages/api/v0.ts +++ b/website/src/pages/api/v0.ts @@ -41,6 +41,60 @@ const Query = t.type({ download: t.union([t.undefined, t.string]), }); +const generate = async ({ + format, + shapes, + year, + simplify +}: { + format: 'topojson' | 'svg' + shapes: Set, + year: string, + simplify: number +}) => { + const input = await (async () => { + const props = [...shapes].flatMap((shape) => { + return ["shp", "dbf", "prj"].map( + async (ext) => + [ + `${shape}.${ext}`, + await fs.readFile( + path.join( + process.cwd(), + "public/swiss-maps", + year, + `${shape}.${ext}` + ) + ), + ] as const + ); + }); + return Object.fromEntries(await Promise.all(props)); + })(); + + const inputFiles = [...shapes].map((shape) => `${shape}.shp`).join(" "); + + const commands = [ + `-i ${inputFiles} combine-files string-fields=*`, + simplify ? `-simplify ${simplify} keep-shapes` : "", + "-clean", + `-proj ${format === "topojson" ? "wgs84" : "somerc"}`, + // svg coloring, otherwise is all bblack + shapes.has("cantons") + ? `-style fill='#e6e6e6' stroke='#999' target='cantons'` + : "", + shapes.has("lakes") ? `-style fill='#a1d0f7' target='lakes'` : "", + `-o output.${format} format=${format} target=*`, + ].join("\n"); + + console.log("### Mapshaper commands ###"); + console.log(commands); + + const output = await mapshaper.applyCommands(commands, input); + + return format === 'topojson' ? output['output.topojson'] : output['output.svg'] +} + export default async function handler( req: NextApiRequest, res: NextApiResponse @@ -76,47 +130,8 @@ export default async function handler( draft.shapes = new Set(query.shapes.split(",") as $FixMe); } }); - const { format, year, shapes } = options; - - const input = await (async () => { - const props = [...shapes].flatMap((shape) => { - return ["shp", "dbf", "prj"].map( - async (ext) => - [ - `${shape}.${ext}`, - await fs.readFile( - path.join( - process.cwd(), - "public/swiss-maps", - year, - `${shape}.${ext}` - ) - ), - ] as const - ); - }); - return Object.fromEntries(await Promise.all(props)); - })(); - - const inputFiles = [...shapes].map((shape) => `${shape}.shp`).join(" "); - - const commands = [ - `-i ${inputFiles} combine-files string-fields=*`, - query.simplify ? `-simplify ${query.simplify} keep-shapes` : "", - "-clean", - `-proj ${format === "topojson" ? "wgs84" : "somerc"}`, - // svg coloring, otherwise is all bblack - shapes.has("cantons") - ? `-style fill='#e6e6e6' stroke='#999' target='cantons'` - : "", - shapes.has("lakes") ? `-style fill='#a1d0f7' target='lakes'` : "", - `-o output.${format} format=${format} target=*`, - ].join("\n"); - - console.log("### Mapshaper commands ###"); - console.log(commands); - - const output = await mapshaper.applyCommands(commands, input); + const { format } = options; + const output = await generate(options) switch (format) { case "topojson": { @@ -128,7 +143,7 @@ export default async function handler( } res.setHeader("Content-Type", "application/json"); - res.status(200).send(output["output.topojson"]); + res.status(200).send(output); break; } @@ -141,7 +156,7 @@ export default async function handler( `attachment; filename="swiss-maps.svg"` ); } - res.status(200).send(output["output.svg"]); + res.status(200).send(output); break; } default: From 8fa674d35fc80988fe0a81729ba27581348ea473 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 11:46:40 +0100 Subject: [PATCH 03/11] refactor: Simplify format handling in HTTP handlers --- website/src/pages/api/_utils.ts | 74 ++++++++++++++++++++++++ website/src/pages/api/generate.ts | 86 ++++++--------------------- website/src/pages/api/v0.ts | 96 +++++++------------------------ 3 files changed, 112 insertions(+), 144 deletions(-) create mode 100644 website/src/pages/api/_utils.ts diff --git a/website/src/pages/api/_utils.ts b/website/src/pages/api/_utils.ts new file mode 100644 index 00000000..52b765b8 --- /dev/null +++ b/website/src/pages/api/_utils.ts @@ -0,0 +1,74 @@ +import Cors from "cors"; +import { either } from "fp-ts"; +import { promises as fs } from "fs"; +import { enableMapSet, produce } from "immer"; +import * as t from "io-ts"; +import * as mapshaper from "mapshaper"; +import { NextApiRequest, NextApiResponse } from "next"; +import * as path from "path"; +import { defaultOptions, Shape } from "src/shared"; + +type Format = "topojson" | "svg"; + +export const formatExtensions = { + topojson: "json", + svg: "svg", +} as Record; + +export const formatContentTypes = { + topojson: "application/json", + svg: "image/svg+xml", +} as Record; + +const Query = t.type({ + format: t.union([t.undefined, t.literal("topojson"), t.literal("svg")]), + year: t.union([t.undefined, t.string]), + shapes: t.union([t.undefined, t.string]), + simplify: t.union([t.undefined, t.string]), + download: t.union([t.undefined, t.string]), +}); + +export const parseOptions = (req: NextApiRequest, res: NextApiResponse) => { + const query = either.getOrElseW(() => undefined)( + Query.decode(req.query) + ); + + if (!query) { + res.statusCode = 200; + res.setHeader("Content-Type", "text/plain"); + res.end("Failed to decode query"); + return; + } + + if (!query.shapes) { + res.setHeader("Content-Type", "text/plain"); + res.status(204).send("0"); + return; + } + + const options = produce(defaultOptions, (draft) => { + if (query.year) { + draft.year = query.year; + } + if (query.format) { + draft.format = query.format; + } + if (query.shapes) { + draft.shapes = new Set(query.shapes.split(",") as $FixMe); + } + }); + + return options; +}; + +export function initMiddleware(middleware: $FixMe) { + return (req: NextApiRequest, res: NextApiResponse) => + new Promise((resolve, reject) => { + middleware(req, res, (result: unknown) => { + if (result instanceof Error) { + return reject(result); + } + return resolve(result); + }); + }); +} diff --git a/website/src/pages/api/generate.ts b/website/src/pages/api/generate.ts index ddfaa1c3..28bfc9e0 100644 --- a/website/src/pages/api/generate.ts +++ b/website/src/pages/api/generate.ts @@ -7,41 +7,21 @@ import * as mapshaper from "mapshaper"; import { NextApiRequest, NextApiResponse } from "next"; import * as path from "path"; import { defaultOptions, Shape } from "src/shared"; +import { + formatContentTypes, + formatExtensions, + initMiddleware, + parseOptions, +} from "./_utils"; enableMapSet(); -async function get(url: string) { - return fetch(url) - .then((res) => res.arrayBuffer()) - .then((ab) => Buffer.from(ab)); -} - -function initMiddleware(middleware: $FixMe) { - return (req: NextApiRequest, res: NextApiResponse) => - new Promise((resolve, reject) => { - middleware(req, res, (result: unknown) => { - if (result instanceof Error) { - return reject(result); - } - return resolve(result); - }); - }); -} - const cors = initMiddleware( Cors({ methods: ["GET", "POST", "OPTIONS"], }) ); -const Query = t.type({ - format: t.union([t.undefined, t.literal("topojson"), t.literal("svg")]), - year: t.union([t.undefined, t.string]), - shapes: t.union([t.undefined, t.string]), - simplify: t.union([t.undefined, t.string]), - download: t.union([t.undefined, t.string]), -}); - const generateFromShapefiles = async ({ format, year, @@ -103,57 +83,25 @@ export default async function handler( try { await cors(req, res); - const query = either.getOrElseW(() => undefined)( - Query.decode(req.query) - ); - - if (!query) { - res.statusCode = 200; - res.setHeader("Content-Type", "text/plain"); - res.end("Failed to decode query"); - return; - } - - const options = produce(defaultOptions, (draft) => { - if (query.year) { - draft.year = query.year; - } - if (query.format) { - draft.format = query.format; - } + const { query } = req; + const options = parseOptions(req, res)!; - if (query.shapes) { - draft.shapes = new Set(query.shapes.split(",") as $FixMe); - } - }); + const { format, year } = options; - const { format } = options const output = await generateFromShapefiles({ ...options, shapes: [...options.shapes] }) - if (format === "topojson") { - if (query.download !== undefined) { - res.setHeader( - "Content-Disposition", - `attachment; filename="swiss-maps.json"` - ); - } - - res.setHeader("Content-Type", "application/json"); - res.status(200).send(output["output.topojson"]); - } else if (format === "svg") { - res.setHeader("Content-Type", "image/svg+xml"); - - if (query.download !== undefined) { - res.setHeader( - "Content-Disposition", - `attachment; filename="swiss-maps.svg"` - ); - } - res.status(200).send(output["output.svg"]); + if (query.download !== undefined) { + res.setHeader( + "Content-Disposition", + `attachment; filename="swiss-maps.${formatExtensions[format]}"` + ); } + + res.setHeader("Content-Type", formatContentTypes[format]); + res.status(200).send(output); } catch (e) { console.error(e); res.status(500).json({ message: "Internal error" }); diff --git a/website/src/pages/api/v0.ts b/website/src/pages/api/v0.ts index 0816f9a4..f5a7fd8c 100644 --- a/website/src/pages/api/v0.ts +++ b/website/src/pages/api/v0.ts @@ -7,6 +7,12 @@ import * as mapshaper from "mapshaper"; import { NextApiRequest, NextApiResponse } from "next"; import * as path from "path"; import { defaultOptions, Shape } from "src/shared"; +import { + formatContentTypes, + formatExtensions, + initMiddleware, + parseOptions, +} from "./_utils"; /** * Difference from `generate` api @@ -15,31 +21,12 @@ import { defaultOptions, Shape } from "src/shared"; enableMapSet(); -function initMiddleware(middleware: $FixMe) { - return (req: NextApiRequest, res: NextApiResponse) => - new Promise((resolve, reject) => { - middleware(req, res, (result: unknown) => { - if (result instanceof Error) { - return reject(result); - } - return resolve(result); - }); - }); -} - const cors = initMiddleware( Cors({ methods: ["GET", "POST", "OPTIONS"], }) ); -const Query = t.type({ - format: t.union([t.undefined, t.literal("topojson"), t.literal("svg")]), - year: t.union([t.undefined, t.string]), - shapes: t.union([t.undefined, t.string]), - simplify: t.union([t.undefined, t.string]), - download: t.union([t.undefined, t.string]), -}); const generate = async ({ format, @@ -95,6 +82,8 @@ const generate = async ({ return format === 'topojson' ? output['output.topojson'] : output['output.svg'] } + + export default async function handler( req: NextApiRequest, res: NextApiResponse @@ -102,67 +91,24 @@ export default async function handler( try { await cors(req, res); - const query = either.getOrElseW(() => undefined)( - Query.decode(req.query) - ); + const { query } = req; + const options = parseOptions(req, res)!; + const { format } = options; - if (!query) { - res.statusCode = 200; - res.setHeader("Content-Type", "text/plain"); - res.end("Failed to decode query"); - return; + if (!formatExtensions[format]) { + res.status(500).json({ message: `Unsupported format ${format}` }); } - if (!query.shapes) { - res.setHeader("Content-Type", "text/plain"); - res.status(204).send("0"); - return; - } + const output = await generate(options); - const options = produce(defaultOptions, (draft) => { - if (query.year) { - draft.year = query.year; - } - if (query.format) { - draft.format = query.format; - } - if (query.shapes) { - draft.shapes = new Set(query.shapes.split(",") as $FixMe); - } - }); - const { format } = options; - const output = await generate(options) - - switch (format) { - case "topojson": { - if (query.download !== undefined) { - res.setHeader( - "Content-Disposition", - `attachment; filename="swiss-maps.json"` - ); - } - - res.setHeader("Content-Type", "application/json"); - res.status(200).send(output); - break; - } - - case "svg": { - res.setHeader("Content-Type", "image/svg+xml"); - - if (query.download !== undefined) { - res.setHeader( - "Content-Disposition", - `attachment; filename="swiss-maps.svg"` - ); - } - res.status(200).send(output); - break; - } - default: - res.status(500).json({ message: "Unsupported format" }); - break; + if (query.download !== undefined) { + res.setHeader( + "Content-Disposition", + `attachment; filename="swiss-maps.${formatExtensions[format]}"` + ); } + res.setHeader("Content-Type", formatContentTypes[format]); + res.status(200).send(output); } catch (e) { console.error(e); res.status(500).json({ message: "Internal error" }); From 3fa7c262c127d42d8e21fd21739bc593fbd05935 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 12:10:04 +0100 Subject: [PATCH 04/11] chore: Format --- website/src/pages/api/v0.ts | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/website/src/pages/api/v0.ts b/website/src/pages/api/v0.ts index f5a7fd8c..fb7bea53 100644 --- a/website/src/pages/api/v0.ts +++ b/website/src/pages/api/v0.ts @@ -27,17 +27,16 @@ const cors = initMiddleware( }) ); - const generate = async ({ format, shapes, year, - simplify + simplify, }: { - format: 'topojson' | 'svg' - shapes: Set, - year: string, - simplify: number + format: "topojson" | "svg"; + shapes: Set; + year: string; + simplify: number; }) => { const input = await (async () => { const props = [...shapes].flatMap((shape) => { @@ -79,10 +78,10 @@ const generate = async ({ const output = await mapshaper.applyCommands(commands, input); - return format === 'topojson' ? output['output.topojson'] : output['output.svg'] -} - - + return format === "topojson" + ? output["output.topojson"] + : output["output.svg"]; +}; export default async function handler( req: NextApiRequest, From 9d1c72146f194c684afffdbecd11d960897ad748 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 11:49:21 +0100 Subject: [PATCH 05/11] feat: Refactor to accept filenames --- website/src/pages/api/generate.ts | 61 +++++++++++++++---------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/website/src/pages/api/generate.ts b/website/src/pages/api/generate.ts index 28bfc9e0..d8e8d479 100644 --- a/website/src/pages/api/generate.ts +++ b/website/src/pages/api/generate.ts @@ -24,38 +24,34 @@ const cors = initMiddleware( const generateFromShapefiles = async ({ format, - year, - shapes, - simplify -} : { - format: 'topojson' | 'svg', - year: string, - shapes: string[] - simplify?: number, + shpFilenames, + simplify, +}: { + format: "topojson" | "svg"; + year: string; + shpFilenames: string[]; + simplify?: number; }) => { - - const input = await (async () => { - const props = shapes.flatMap((shape) => { - return ["shp", "dbf", "prj"].map( - async (ext) => - [ - `${shape}.${ext}`, - await fs.readFile( - path.join( - process.cwd(), - "public", - "swiss-maps", - year, - `${shape}.${ext}` - ) - ), - ] as const - ); + const input = await (async () => { + const props = shpFilenames.flatMap((shpFilename) => { + const shape = path.basename(shpFilename, ".shp"); + const fullPathWithoutExt = shpFilename.substring( + 0, + shpFilename.length - ".shp".length + ); + return ["shp", "dbf", "prj"].map(async (ext) => { + return [ + `${shape}.${ext}`, + await fs.readFile(`${fullPathWithoutExt}.${ext}`), + ] as const; }); - return Object.fromEntries(await Promise.all(props)); - })(); + }); + return Object.fromEntries(await Promise.all(props)); + })(); - const inputFiles = shapes.map((shape) => `${shape}.shp`).join(" "); + const inputFiles = shpFilenames + .map((shpFilename) => `${path.basename(shpFilename)}`) + .join(" "); const commands = [ `-i ${inputFiles} combine-files string-fields=*`, @@ -87,11 +83,14 @@ export default async function handler( const options = parseOptions(req, res)!; const { format, year } = options; + const cwd = process.cwd(); const output = await generateFromShapefiles({ ...options, - shapes: [...options.shapes] - }) + shpFilenames: [...options.shapes].map((shapeName) => { + return path.join(cwd, "public", "swiss-maps", year, `${shapeName}.shp`); + }), + }); if (query.download !== undefined) { res.setHeader( From 4c81a266c0a34d5b6e5210c2f8ef02c89bb4a86d Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 12:10:07 +0100 Subject: [PATCH 06/11] chore: Add prettier --- .prettierrc | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..222861c3 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 2, + "useTabs": false +} From 10f2570f035cceaf475ed3d99b3a3e32a2f495f9 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 12:15:56 +0100 Subject: [PATCH 07/11] refactor: Remove unused imports --- website/src/pages/api/_utils.ts | 6 +----- website/src/pages/api/generate.ts | 4 +--- website/src/pages/api/v0.ts | 4 +--- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/website/src/pages/api/_utils.ts b/website/src/pages/api/_utils.ts index 52b765b8..2c99443e 100644 --- a/website/src/pages/api/_utils.ts +++ b/website/src/pages/api/_utils.ts @@ -1,11 +1,7 @@ -import Cors from "cors"; import { either } from "fp-ts"; -import { promises as fs } from "fs"; -import { enableMapSet, produce } from "immer"; +import produce from "immer"; import * as t from "io-ts"; -import * as mapshaper from "mapshaper"; import { NextApiRequest, NextApiResponse } from "next"; -import * as path from "path"; import { defaultOptions, Shape } from "src/shared"; type Format = "topojson" | "svg"; diff --git a/website/src/pages/api/generate.ts b/website/src/pages/api/generate.ts index d8e8d479..834292e5 100644 --- a/website/src/pages/api/generate.ts +++ b/website/src/pages/api/generate.ts @@ -1,12 +1,10 @@ import Cors from "cors"; -import { either } from "fp-ts"; import { promises as fs } from "fs"; -import { enableMapSet, produce } from "immer"; +import { enableMapSet } from "immer"; import * as t from "io-ts"; import * as mapshaper from "mapshaper"; import { NextApiRequest, NextApiResponse } from "next"; import * as path from "path"; -import { defaultOptions, Shape } from "src/shared"; import { formatContentTypes, formatExtensions, diff --git a/website/src/pages/api/v0.ts b/website/src/pages/api/v0.ts index fb7bea53..9eb8e369 100644 --- a/website/src/pages/api/v0.ts +++ b/website/src/pages/api/v0.ts @@ -1,12 +1,10 @@ import Cors from "cors"; import { either } from "fp-ts"; import { promises as fs } from "fs"; -import { enableMapSet, produce } from "immer"; -import * as t from "io-ts"; +import { enableMapSet } from "immer"; import * as mapshaper from "mapshaper"; import { NextApiRequest, NextApiResponse } from "next"; import * as path from "path"; -import { defaultOptions, Shape } from "src/shared"; import { formatContentTypes, formatExtensions, From 1a31cdde6710397b76de4e79d37049d96b253fd2 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 12:22:51 +0100 Subject: [PATCH 08/11] feat: Extract generate function + Better styling --- website/src/pages/api/_generate.ts | 56 +++++++++++ website/src/pages/api/generate.ts | 66 ++----------- website/src/pages/api/v0.ts | 149 +++++++++++++++++------------ 3 files changed, 154 insertions(+), 117 deletions(-) create mode 100644 website/src/pages/api/_generate.ts diff --git a/website/src/pages/api/_generate.ts b/website/src/pages/api/_generate.ts new file mode 100644 index 00000000..4a7cf5a1 --- /dev/null +++ b/website/src/pages/api/_generate.ts @@ -0,0 +1,56 @@ +import { promises as fs } from "fs"; +import * as mapshaper from "mapshaper"; +import * as path from "path"; + +export const generate = async ({ + format, + shapes, + simplify, + year, + mapshaperCommands, +}: { + format: "topojson" | "svg"; + shapes: string[]; + simplify: number; + mapshaperCommands?: string[]; + year: string; +}) => { + const input = await (async () => { + const props = [...shapes].flatMap((shape) => { + return ["shp", "dbf", "prj"].map( + async (ext) => + [ + `${shape}.${ext}`, + await fs.readFile( + path.join( + process.cwd(), + "public/swiss-maps", + year, + `${shape}.${ext}` + ) + ), + ] as const + ); + }); + return Object.fromEntries(await Promise.all(props)); + })(); + + const inputFiles = [...shapes].map((shape) => `${shape}.shp`).join(" "); + + const commands = [ + `-i ${inputFiles} combine-files string-fields=*`, + simplify ? `-simplify ${simplify} keep-shapes` : "", + "-clean", + `-proj ${format === "topojson" ? "wgs84" : "somerc"}`, + ...(mapshaperCommands || []), + ].join("\n"); + + console.log("### Mapshaper commands ###"); + console.log(commands); + + const output = await mapshaper.applyCommands(commands, input); + + return format === "topojson" + ? output["output.topojson"] + : output["output.svg"]; +}; diff --git a/website/src/pages/api/generate.ts b/website/src/pages/api/generate.ts index 834292e5..98ffb79d 100644 --- a/website/src/pages/api/generate.ts +++ b/website/src/pages/api/generate.ts @@ -1,10 +1,8 @@ import Cors from "cors"; -import { promises as fs } from "fs"; import { enableMapSet } from "immer"; -import * as t from "io-ts"; -import * as mapshaper from "mapshaper"; import { NextApiRequest, NextApiResponse } from "next"; import * as path from "path"; +import { generate } from "./_generate"; import { formatContentTypes, formatExtensions, @@ -20,56 +18,6 @@ const cors = initMiddleware( }) ); -const generateFromShapefiles = async ({ - format, - shpFilenames, - simplify, -}: { - format: "topojson" | "svg"; - year: string; - shpFilenames: string[]; - simplify?: number; -}) => { - const input = await (async () => { - const props = shpFilenames.flatMap((shpFilename) => { - const shape = path.basename(shpFilename, ".shp"); - const fullPathWithoutExt = shpFilename.substring( - 0, - shpFilename.length - ".shp".length - ); - return ["shp", "dbf", "prj"].map(async (ext) => { - return [ - `${shape}.${ext}`, - await fs.readFile(`${fullPathWithoutExt}.${ext}`), - ] as const; - }); - }); - return Object.fromEntries(await Promise.all(props)); - })(); - - const inputFiles = shpFilenames - .map((shpFilename) => `${path.basename(shpFilename)}`) - .join(" "); - - const commands = [ - `-i ${inputFiles} combine-files string-fields=*`, - // `-rename-layers ${shp.join(",")}`, - simplify ? `-simplify ${simplify} keep-shapes` : "", - "-clean", - `-proj ${format === "topojson" ? "wgs84" : "somerc"}`, - `-o output.${format} format=${format} drop-table id-field=id`, - ].join("\n"); - - console.log("### Mapshaper commands ###"); - console.log(commands); - - const output = await mapshaper.applyCommands(commands, input); - - return format === "topojson" - ? output["output.topojson"] - : output["output.svg"]; -}; - export default async function handler( req: NextApiRequest, res: NextApiResponse @@ -80,14 +28,16 @@ export default async function handler( const { query } = req; const options = parseOptions(req, res)!; - const { format, year } = options; + const { format, year, shapes } = options; const cwd = process.cwd(); - const output = await generateFromShapefiles({ + const output = await generate({ ...options, - shpFilenames: [...options.shapes].map((shapeName) => { - return path.join(cwd, "public", "swiss-maps", year, `${shapeName}.shp`); - }), + year, + shapes: [...shapes], + mapshaperCommands: [ + `-o output.${format} format=${format} drop-table id-field=id target=*`, + ], }); if (query.download !== undefined) { diff --git a/website/src/pages/api/v0.ts b/website/src/pages/api/v0.ts index 9eb8e369..71feff63 100644 --- a/website/src/pages/api/v0.ts +++ b/website/src/pages/api/v0.ts @@ -1,8 +1,5 @@ import Cors from "cors"; -import { either } from "fp-ts"; -import { promises as fs } from "fs"; import { enableMapSet } from "immer"; -import * as mapshaper from "mapshaper"; import { NextApiRequest, NextApiResponse } from "next"; import * as path from "path"; import { @@ -11,6 +8,7 @@ import { initMiddleware, parseOptions, } from "./_utils"; +import { generate } from "./_generate"; /** * Difference from `generate` api @@ -25,60 +23,48 @@ const cors = initMiddleware( }) ); -const generate = async ({ - format, - shapes, - year, - simplify, -}: { - format: "topojson" | "svg"; - shapes: Set; - year: string; - simplify: number; -}) => { - const input = await (async () => { - const props = [...shapes].flatMap((shape) => { - return ["shp", "dbf", "prj"].map( - async (ext) => - [ - `${shape}.${ext}`, - await fs.readFile( - path.join( - process.cwd(), - "public/swiss-maps", - year, - `${shape}.${ext}` - ) - ), - ] as const - ); - }); - return Object.fromEntries(await Promise.all(props)); - })(); - - const inputFiles = [...shapes].map((shape) => `${shape}.shp`).join(" "); - - const commands = [ - `-i ${inputFiles} combine-files string-fields=*`, - simplify ? `-simplify ${simplify} keep-shapes` : "", - "-clean", - `-proj ${format === "topojson" ? "wgs84" : "somerc"}`, - // svg coloring, otherwise is all bblack - shapes.has("cantons") - ? `-style fill='#e6e6e6' stroke='#999' target='cantons'` - : "", - shapes.has("lakes") ? `-style fill='#a1d0f7' target='lakes'` : "", - `-o output.${format} format=${format} target=*`, - ].join("\n"); - - console.log("### Mapshaper commands ###"); - console.log(commands); - - const output = await mapshaper.applyCommands(commands, input); - - return format === "topojson" - ? output["output.topojson"] - : output["output.svg"]; +const truthy = (x: T): x is Exclude => { + return Boolean(x); +}; + +const makeMapshaperStyleCommands = ( + shapeStyles: Record< + string, + null | { + fill?: string; + stroke?: string; + } + > +) => { + return Object.entries(shapeStyles) + .map(([shapeName, style]) => { + if (style === null) { + return style; + } + return `-style target='${shapeName}' ${Object.entries(style) + .map(([propName, propValue]) => { + return `${propName}='${propValue}'`; + }) + .join(" ")}`; + }) + .filter(truthy); +}; + +const getShapeZIndex = (shape: string) => { + if (shape.includes("country")) { + return 3; + } else if (shape.includes("cantons")) { + return 2; + } else if (shape.includes("lakes")) { + return 1; + } + return 0; +}; + +const shapeIndexComparator = (a: string, b: string) => { + const za = getShapeZIndex(a); + const zb = getShapeZIndex(b); + return za === zb ? 0 : za < zb ? -1 : 1; }; export default async function handler( @@ -90,13 +76,58 @@ export default async function handler( const { query } = req; const options = parseOptions(req, res)!; - const { format } = options; + const { format, shapes, year } = options; if (!formatExtensions[format]) { res.status(500).json({ message: `Unsupported format ${format}` }); } - const output = await generate(options); + const cwd = process.cwd(); + const shpFilenames = [...options.shapes] + .map((shapeName) => { + return path.join(cwd, "public", "swiss-maps", year, `${shapeName}.shp`); + }) + .sort(shapeIndexComparator); + + const hasCantons = shapes.has("cantons"); + const hasMunicipalities = shapes.has("municipalities"); + const hasLakes = shapes.has("lakes"); + + const shapeStyles = { + country: { + fill: hasCantons || hasMunicipalities ? "transparent" : "#eee", + stroke: "#111", + }, + lakes: hasLakes + ? { + fill: "#a1d0f7", + } + : null, + cantons: hasCantons + ? { + fill: hasMunicipalities ? "transparent" : "#eee", + stroke: "#666", + } + : null, + municipalities: hasMunicipalities + ? { + fill: "#eee", + stroke: hasCantons ? "#bbb" : "#666", + } + : null, + }; + + const styleCommands = makeMapshaperStyleCommands(shapeStyles); + + const output = await generate({ + ...options, + year, + shapes: [...shapes], + mapshaperCommands: [ + ...styleCommands, + `-o output.${format} format=${format} target=*`, + ], + }); if (query.download !== undefined) { res.setHeader( From dbe65a9d859e4a44ce995aa425aeeb94a1af6017 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 16:46:08 +0100 Subject: [PATCH 09/11] fix: Repair simplify Types is still not super good though --- website/src/pages/api/_generate.ts | 2 +- website/src/pages/api/generate.ts | 1 + website/src/pages/api/v0.ts | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/website/src/pages/api/_generate.ts b/website/src/pages/api/_generate.ts index 4a7cf5a1..2dc60e09 100644 --- a/website/src/pages/api/_generate.ts +++ b/website/src/pages/api/_generate.ts @@ -11,7 +11,7 @@ export const generate = async ({ }: { format: "topojson" | "svg"; shapes: string[]; - simplify: number; + simplify: string; mapshaperCommands?: string[]; year: string; }) => { diff --git a/website/src/pages/api/generate.ts b/website/src/pages/api/generate.ts index 98ffb79d..48099662 100644 --- a/website/src/pages/api/generate.ts +++ b/website/src/pages/api/generate.ts @@ -34,6 +34,7 @@ export default async function handler( const output = await generate({ ...options, year, + simplify: query.simplify as string, shapes: [...shapes], mapshaperCommands: [ `-o output.${format} format=${format} drop-table id-field=id target=*`, diff --git a/website/src/pages/api/v0.ts b/website/src/pages/api/v0.ts index 71feff63..ee43107d 100644 --- a/website/src/pages/api/v0.ts +++ b/website/src/pages/api/v0.ts @@ -121,6 +121,7 @@ export default async function handler( const output = await generate({ ...options, + simplify: query.simplify as string, year, shapes: [...shapes], mapshaperCommands: [ From 30dd70aefbd5e4bf74b17af02641d097c7a0d812 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 16:55:51 +0100 Subject: [PATCH 10/11] feat: Extract shape utils --- website/src/pages/api/_utils.ts | 44 +++++++++++++++++++++++++++++++ website/src/pages/api/v0.ts | 46 ++------------------------------- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/website/src/pages/api/_utils.ts b/website/src/pages/api/_utils.ts index 2c99443e..7891d81c 100644 --- a/website/src/pages/api/_utils.ts +++ b/website/src/pages/api/_utils.ts @@ -68,3 +68,47 @@ export function initMiddleware(middleware: $FixMe) { }); }); } + +const truthy = (x: T): x is Exclude => { + return Boolean(x); +}; + +export const makeMapshaperStyleCommands = ( + shapeStyles: Record< + string, + null | { + fill?: string; + stroke?: string; + } + > +) => { + return Object.entries(shapeStyles) + .map(([shapeName, style]) => { + if (style === null) { + return style; + } + return `-style target='${shapeName}' ${Object.entries(style) + .map(([propName, propValue]) => { + return `${propName}='${propValue}'`; + }) + .join(" ")}`; + }) + .filter(truthy); +}; + +const getShapeZIndex = (shape: string) => { + if (shape.includes("country")) { + return 3; + } else if (shape.includes("cantons")) { + return 2; + } else if (shape.includes("lakes")) { + return 1; + } + return 0; +}; + +export const shapeIndexComparator = (a: string, b: string) => { + const za = getShapeZIndex(a); + const zb = getShapeZIndex(b); + return za === zb ? 0 : za < zb ? -1 : 1; +}; diff --git a/website/src/pages/api/v0.ts b/website/src/pages/api/v0.ts index ee43107d..00197f79 100644 --- a/website/src/pages/api/v0.ts +++ b/website/src/pages/api/v0.ts @@ -6,7 +6,9 @@ import { formatContentTypes, formatExtensions, initMiddleware, + makeMapshaperStyleCommands, parseOptions, + shapeIndexComparator, } from "./_utils"; import { generate } from "./_generate"; @@ -23,50 +25,6 @@ const cors = initMiddleware( }) ); -const truthy = (x: T): x is Exclude => { - return Boolean(x); -}; - -const makeMapshaperStyleCommands = ( - shapeStyles: Record< - string, - null | { - fill?: string; - stroke?: string; - } - > -) => { - return Object.entries(shapeStyles) - .map(([shapeName, style]) => { - if (style === null) { - return style; - } - return `-style target='${shapeName}' ${Object.entries(style) - .map(([propName, propValue]) => { - return `${propName}='${propValue}'`; - }) - .join(" ")}`; - }) - .filter(truthy); -}; - -const getShapeZIndex = (shape: string) => { - if (shape.includes("country")) { - return 3; - } else if (shape.includes("cantons")) { - return 2; - } else if (shape.includes("lakes")) { - return 1; - } - return 0; -}; - -const shapeIndexComparator = (a: string, b: string) => { - const za = getShapeZIndex(a); - const zb = getShapeZIndex(b); - return za === zb ? 0 : za < zb ? -1 : 1; -}; - export default async function handler( req: NextApiRequest, res: NextApiResponse From 37a7091985d71d2886594576d9142401ae14134e Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Fri, 17 Feb 2023 17:06:37 +0100 Subject: [PATCH 11/11] feat: Reuse shape --- website/src/pages/api/generate.ts | 4 ++-- website/src/pages/api/v0.ts | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/website/src/pages/api/generate.ts b/website/src/pages/api/generate.ts index 48099662..c057f3c9 100644 --- a/website/src/pages/api/generate.ts +++ b/website/src/pages/api/generate.ts @@ -8,6 +8,7 @@ import { formatExtensions, initMiddleware, parseOptions, + shapeIndexComparator, } from "./_utils"; enableMapSet(); @@ -29,13 +30,12 @@ export default async function handler( const options = parseOptions(req, res)!; const { format, year, shapes } = options; - const cwd = process.cwd(); const output = await generate({ ...options, year, simplify: query.simplify as string, - shapes: [...shapes], + shapes: [...shapes].sort(shapeIndexComparator), mapshaperCommands: [ `-o output.${format} format=${format} drop-table id-field=id target=*`, ], diff --git a/website/src/pages/api/v0.ts b/website/src/pages/api/v0.ts index 00197f79..a0d13258 100644 --- a/website/src/pages/api/v0.ts +++ b/website/src/pages/api/v0.ts @@ -40,13 +40,6 @@ export default async function handler( res.status(500).json({ message: `Unsupported format ${format}` }); } - const cwd = process.cwd(); - const shpFilenames = [...options.shapes] - .map((shapeName) => { - return path.join(cwd, "public", "swiss-maps", year, `${shapeName}.shp`); - }) - .sort(shapeIndexComparator); - const hasCantons = shapes.has("cantons"); const hasMunicipalities = shapes.has("municipalities"); const hasLakes = shapes.has("lakes"); @@ -81,7 +74,7 @@ export default async function handler( ...options, simplify: query.simplify as string, year, - shapes: [...shapes], + shapes: [...shapes].sort(shapeIndexComparator), mapshaperCommands: [ ...styleCommands, `-o output.${format} format=${format} target=*`,