From 0afd3e8434b299498e7c3a47dc5f8e04bac51f00 Mon Sep 17 00:00:00 2001 From: Chad Burt Date: Thu, 5 Sep 2024 15:41:13 -0700 Subject: [PATCH] WIP --- packages/api/generated-schema.gql | 1 + packages/api/package-lock.json | 208 ++++++++- packages/api/package.json | 4 + packages/api/src/graphileOptions.ts | 2 + .../api/src/plugins/metadataParserPlugin.ts | 55 +++ packages/client/package-lock.json | 13 + packages/client/package.json | 1 + packages/client/src/admin/MetadataEditor.tsx | 1 + .../src/admin/data/OverlayMetadataEditor.tsx | 2 + .../admin/data/TableOfContentsItemMenu.tsx | 2 +- .../client/src/dataLayers/MetadataModal.tsx | 14 +- .../TableOfContentsMetadataModal.tsx | 17 +- packages/client/src/editor/EditorMenuBar.tsx | 56 +++ packages/client/src/editor/config.ts | 25 +- .../src/formElements/prosemirror-body.css | 20 +- packages/client/src/generated/graphql.ts | 74 ++++ packages/client/src/generated/queries.ts | 47 ++ .../src/queries/DraftTableOfContents.graphql | 15 + ...ta.xml => Coral-fgdc-example-metadata.xml} | 0 .../__tests__/metadata-parser.test.js | 7 - .../__tests__/metadata-parser.test.ts | 32 +- .../output/3nm_Polyline-iso-metadata.md | 34 ++ .../__tests__/output/Coral-esri-metadata.md | 1 - .../__tests__/output/Coral-fgcd-metadata.md | 43 ++ packages/metadata-parser/package.json | 11 +- .../metadata-parser/src/esriToMarkdown.ts | 110 ----- .../metadata-parser/src/fgdcToMarkdown.ts | 157 +++++++ .../metadata-parser/src/fgdcToProsemirror.ts | 250 +++++++++++ packages/metadata-parser/src/iso19139.ts | 22 +- .../src/iso19139ToProseMirror.ts | 409 ++++++++++++++++++ .../metadata-parser/src/metadata-parser.ts | 27 +- packages/metadata-parser/tsconfig.json | 2 +- 32 files changed, 1478 insertions(+), 184 deletions(-) create mode 100644 packages/api/src/plugins/metadataParserPlugin.ts rename packages/metadata-parser/__tests__/{Coral-esri-example-metadata.xml => Coral-fgdc-example-metadata.xml} (100%) delete mode 100644 packages/metadata-parser/__tests__/metadata-parser.test.js create mode 100644 packages/metadata-parser/__tests__/output/3nm_Polyline-iso-metadata.md delete mode 100644 packages/metadata-parser/__tests__/output/Coral-esri-metadata.md create mode 100644 packages/metadata-parser/__tests__/output/Coral-fgcd-metadata.md delete mode 100644 packages/metadata-parser/src/esriToMarkdown.ts create mode 100644 packages/metadata-parser/src/fgdcToMarkdown.ts create mode 100644 packages/metadata-parser/src/fgdcToProsemirror.ts create mode 100644 packages/metadata-parser/src/iso19139ToProseMirror.ts diff --git a/packages/api/generated-schema.gql b/packages/api/generated-schema.gql index 7db352d4d..ab59c8542 100644 --- a/packages/api/generated-schema.gql +++ b/packages/api/generated-schema.gql @@ -9574,6 +9574,7 @@ type Mutation { """ input: UpdateTableOfContentsItemChildrenInput! ): UpdateTableOfContentsItemChildrenPayload + updateTocMetadataFromXML(id: Int!, xmlMetadata: String!): TableOfContentsItem! """Updates a single `Topic` using a unique key and a patch.""" updateTopic( diff --git a/packages/api/package-lock.json b/packages/api/package-lock.json index b22474634..1cb755097 100644 --- a/packages/api/package-lock.json +++ b/packages/api/package-lock.json @@ -34,6 +34,7 @@ "@types/pg-format": "^1.0.2", "@types/pluralize": "^0.0.29", "@types/pngjs": "^6.0.4", + "@types/prosemirror-markdown": "^1.9.0", "@types/prosemirror-model": "^1.11.2", "@types/redis": "^2.8.25", "@types/sanitize-html": "^1.27.0", @@ -72,8 +73,10 @@ "pngjs": "^7.0.0", "postgraphile": "^4.14.0", "postgraphile-plugin-upload-field": "^1.0.0-alpha.10", + "prosemirror-markdown": "^1.13.0", "prosemirror-model": "^1.19.3", "prosemirror-schema-list": "^1.1.4", + "prosemirror-tables": "^1.5.0", "puppeteer": "^19.7.2", "pureimage": "^0.4.13", "rate-limiter-flexible": "^2.4.1", @@ -5091,6 +5094,15 @@ "integrity": "sha512-IkVfat549ggtkZUthUzEX49562eGikhSYeVGX97SkMFn+sTZrgRewXjQ4tPKFPCykZHkX1Zfd9OoELGqKU2jJA==", "dev": true }, + "node_modules/@types/prosemirror-markdown": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@types/prosemirror-markdown/-/prosemirror-markdown-1.9.0.tgz", + "integrity": "sha512-6ghbzhx/JshByvYfqQmknSkth03Pf+78TGLB555kTnL6AxuJeL2zOx1EAg6kVRSFv6eckLt1dZs984ioWeD9Bg==", + "deprecated": "This is a stub types definition. prosemirror-markdown provides its own type definitions, so you do not need this installed.", + "dependencies": { + "prosemirror-markdown": "*" + } + }, "node_modules/@types/prosemirror-model": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/@types/prosemirror-model/-/prosemirror-model-1.17.0.tgz", @@ -14626,6 +14638,14 @@ "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz", "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=" }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, "node_modules/load-json-file": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", @@ -15045,6 +15065,38 @@ "resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.1.tgz", "integrity": "sha512-nMkuDXFv60aBr9soUG5q+GvZYL+2KZHVvsqFCzqnkGEf46U2fvmytHaEVc1/YZbiLn8X+eR3QzX1+dwDO1lxlw==" }, + "node_modules/markdown-it": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, + "node_modules/markdown-it/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/markdown-it/node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/md5": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", @@ -15055,6 +15107,11 @@ "is-buffer": "~1.1.6" } }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==" + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -17047,10 +17104,28 @@ "node": ">= 6" } }, + "node_modules/prosemirror-keymap": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/prosemirror-keymap/-/prosemirror-keymap-1.2.2.tgz", + "integrity": "sha512-EAlXoksqC6Vbocqc0GtzCruZEzYgrn+iiGnNjsJsH4mrnIGex4qbLdWWNza3AW5W36ZRrlBID0eM6bdKH4OStQ==", + "dependencies": { + "prosemirror-state": "^1.0.0", + "w3c-keyname": "^2.2.0" + } + }, + "node_modules/prosemirror-markdown": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/prosemirror-markdown/-/prosemirror-markdown-1.13.0.tgz", + "integrity": "sha512-UziddX3ZYSYibgx8042hfGKmukq5Aljp2qoBiJRejD/8MH70siQNz5RB1TrdTPheqLMy4aCe4GYNF10/3lQS5g==", + "dependencies": { + "markdown-it": "^14.0.0", + "prosemirror-model": "^1.20.0" + } + }, "node_modules/prosemirror-model": { - "version": "1.19.3", - "resolved": "https://registry.npmjs.org/prosemirror-model/-/prosemirror-model-1.19.3.tgz", - "integrity": "sha512-tgSnwN7BS7/UM0sSARcW+IQryx2vODKX4MI7xpqY2X+iaepJdKBPc7I4aACIsDV/LTaTjt12Z56MhDr9LsyuZQ==", + "version": "1.22.3", + "resolved": "https://registry.npmjs.org/prosemirror-model/-/prosemirror-model-1.22.3.tgz", + "integrity": "sha512-V4XCysitErI+i0rKFILGt/xClnFJaohe/wrrlT2NSZ+zk8ggQfDH4x2wNK7Gm0Hp4CIoWizvXFP7L9KMaCuI0Q==", "dependencies": { "orderedmap": "^2.0.0" } @@ -17075,6 +17150,18 @@ "prosemirror-view": "^1.27.0" } }, + "node_modules/prosemirror-tables": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/prosemirror-tables/-/prosemirror-tables-1.5.0.tgz", + "integrity": "sha512-VMx4zlYWm7aBlZ5xtfJHpqa3Xgu3b7srV54fXYnXgsAcIGRqKSrhiK3f89omzzgaAgAtDOV4ImXnLKhVfheVNQ==", + "dependencies": { + "prosemirror-keymap": "^1.1.2", + "prosemirror-model": "^1.8.1", + "prosemirror-state": "^1.3.1", + "prosemirror-transform": "^1.2.1", + "prosemirror-view": "^1.13.3" + } + }, "node_modules/prosemirror-transform": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/prosemirror-transform/-/prosemirror-transform-1.7.0.tgz", @@ -17180,6 +17267,14 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "engines": { + "node": ">=6" + } + }, "node_modules/pupa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/pupa/-/pupa-2.0.1.tgz", @@ -20237,6 +20332,11 @@ "node": ">=14.17" } }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==" + }, "node_modules/ulid": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/ulid/-/ulid-2.3.0.tgz", @@ -20656,6 +20756,11 @@ "browser-process-hrtime": "^1.0.0" } }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==" + }, "node_modules/w3c-xmlserializer": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz", @@ -25460,6 +25565,14 @@ "integrity": "sha512-IkVfat549ggtkZUthUzEX49562eGikhSYeVGX97SkMFn+sTZrgRewXjQ4tPKFPCykZHkX1Zfd9OoELGqKU2jJA==", "dev": true }, + "@types/prosemirror-markdown": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@types/prosemirror-markdown/-/prosemirror-markdown-1.9.0.tgz", + "integrity": "sha512-6ghbzhx/JshByvYfqQmknSkth03Pf+78TGLB555kTnL6AxuJeL2zOx1EAg6kVRSFv6eckLt1dZs984ioWeD9Bg==", + "requires": { + "prosemirror-markdown": "*" + } + }, "@types/prosemirror-model": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/@types/prosemirror-model/-/prosemirror-model-1.17.0.tgz", @@ -33210,6 +33323,14 @@ "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz", "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=" }, + "linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "requires": { + "uc.micro": "^2.0.0" + } + }, "load-json-file": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", @@ -33576,6 +33697,31 @@ } } }, + "markdown-it": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "requires": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "dependencies": { + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==" + } + } + }, "md5": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", @@ -33586,6 +33732,11 @@ "is-buffer": "~1.1.6" } }, + "mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==" + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -35163,10 +35314,28 @@ "sisteransi": "^1.0.4" } }, + "prosemirror-keymap": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/prosemirror-keymap/-/prosemirror-keymap-1.2.2.tgz", + "integrity": "sha512-EAlXoksqC6Vbocqc0GtzCruZEzYgrn+iiGnNjsJsH4mrnIGex4qbLdWWNza3AW5W36ZRrlBID0eM6bdKH4OStQ==", + "requires": { + "prosemirror-state": "^1.0.0", + "w3c-keyname": "^2.2.0" + } + }, + "prosemirror-markdown": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/prosemirror-markdown/-/prosemirror-markdown-1.13.0.tgz", + "integrity": "sha512-UziddX3ZYSYibgx8042hfGKmukq5Aljp2qoBiJRejD/8MH70siQNz5RB1TrdTPheqLMy4aCe4GYNF10/3lQS5g==", + "requires": { + "markdown-it": "^14.0.0", + "prosemirror-model": "^1.20.0" + } + }, "prosemirror-model": { - "version": "1.19.3", - "resolved": "https://registry.npmjs.org/prosemirror-model/-/prosemirror-model-1.19.3.tgz", - "integrity": "sha512-tgSnwN7BS7/UM0sSARcW+IQryx2vODKX4MI7xpqY2X+iaepJdKBPc7I4aACIsDV/LTaTjt12Z56MhDr9LsyuZQ==", + "version": "1.22.3", + "resolved": "https://registry.npmjs.org/prosemirror-model/-/prosemirror-model-1.22.3.tgz", + "integrity": "sha512-V4XCysitErI+i0rKFILGt/xClnFJaohe/wrrlT2NSZ+zk8ggQfDH4x2wNK7Gm0Hp4CIoWizvXFP7L9KMaCuI0Q==", "requires": { "orderedmap": "^2.0.0" } @@ -35191,6 +35360,18 @@ "prosemirror-view": "^1.27.0" } }, + "prosemirror-tables": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/prosemirror-tables/-/prosemirror-tables-1.5.0.tgz", + "integrity": "sha512-VMx4zlYWm7aBlZ5xtfJHpqa3Xgu3b7srV54fXYnXgsAcIGRqKSrhiK3f89omzzgaAgAtDOV4ImXnLKhVfheVNQ==", + "requires": { + "prosemirror-keymap": "^1.1.2", + "prosemirror-model": "^1.8.1", + "prosemirror-state": "^1.3.1", + "prosemirror-transform": "^1.2.1", + "prosemirror-view": "^1.13.3" + } + }, "prosemirror-transform": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/prosemirror-transform/-/prosemirror-transform-1.7.0.tgz", @@ -35289,6 +35470,11 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, + "punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==" + }, "pupa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/pupa/-/pupa-2.0.1.tgz", @@ -37765,6 +37951,11 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==" }, + "uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==" + }, "ulid": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/ulid/-/ulid-2.3.0.tgz", @@ -38119,6 +38310,11 @@ "browser-process-hrtime": "^1.0.0" } }, + "w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==" + }, "w3c-xmlserializer": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz", diff --git a/packages/api/package.json b/packages/api/package.json index 41eb8413e..e6748260d 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -55,6 +55,7 @@ "@seasketch/geostats-types": "^1.0.0", "@seasketch/map-tile-cache-calculator": "^1.0.0", "@seasketch/mapbox-gl-esri-sources": "^0.9.0", + "@seasketch/metadata-parser": "^1.0.0", "@sentry/browser": "^7.44.2", "@sentry/node": "^7.111.0", "@turf/bbox": "^6.5.0", @@ -73,6 +74,7 @@ "@types/pg-format": "^1.0.2", "@types/pluralize": "^0.0.29", "@types/pngjs": "^6.0.4", + "@types/prosemirror-markdown": "^1.9.0", "@types/prosemirror-model": "^1.11.2", "@types/redis": "^2.8.25", "@types/sanitize-html": "^1.27.0", @@ -111,8 +113,10 @@ "pngjs": "^7.0.0", "postgraphile": "^4.14.0", "postgraphile-plugin-upload-field": "^1.0.0-alpha.10", + "prosemirror-markdown": "^1.13.0", "prosemirror-model": "^1.19.3", "prosemirror-schema-list": "^1.1.4", + "prosemirror-tables": "^1.5.0", "puppeteer": "^19.7.2", "pureimage": "^0.4.13", "rate-limiter-flexible": "^2.4.1", diff --git a/packages/api/src/graphileOptions.ts b/packages/api/src/graphileOptions.ts index a50a1c45d..956a6db83 100644 --- a/packages/api/src/graphileOptions.ts +++ b/packages/api/src/graphileOptions.ts @@ -42,6 +42,7 @@ import DraftTocStatusPlugin from "./plugins/projectDraftTableOfContentsStatusSub import ComputedMetadataPlugin from "./plugins/computedMetadataPlugin"; import SearchOverlaysRateLimiterPlugin from "./plugins/searchOverlaysRateLimiterPlugin"; import ProjectBackgroundJobSubscriptionPlugin from "./plugins/projectBackgroundJobSubscriptionPlugin"; +import MetadataParserPlugin from "./plugins/metadataParserPlugin"; const pluginHook = makePluginHook([{ ...PgPubsub, ...SentryPlugin }]); @@ -94,6 +95,7 @@ export default function graphileOptions(): PostGraphileOptions { ComputedMetadataPlugin, SearchOverlaysRateLimiterPlugin, ProjectBackgroundJobSubscriptionPlugin, + MetadataParserPlugin, // reorderSchemaFields(graphqlSchemaModifiers.fieldOrder), // extraDocumentationPlugin(graphqlSchemaModifiers.documentation), ], diff --git a/packages/api/src/plugins/metadataParserPlugin.ts b/packages/api/src/plugins/metadataParserPlugin.ts new file mode 100644 index 000000000..173b43e50 --- /dev/null +++ b/packages/api/src/plugins/metadataParserPlugin.ts @@ -0,0 +1,55 @@ +import { makeExtendSchemaPlugin, gql } from "graphile-utils"; +import { metadataToProseMirror } from "@seasketch/metadata-parser"; + +const MetadataParserPlugin = makeExtendSchemaPlugin((build) => { + const { pgSql: sql } = build; + return { + typeDefs: gql` + extend type Mutation { + updateTocMetadataFromXML( + id: Int! + xmlMetadata: String! + ): TableOfContentsItem! + } + `, + resolvers: { + Mutation: { + updateTocMetadataFromXML: async ( + _query, + args, + context, + resolveInfo + ) => { + const { pgClient } = context; + const data = await metadataToProseMirror(args.xmlMetadata); + if (data?.doc) { + const { rows } = await pgClient.query( + `update public.table_of_contents_items set metadata = $1 where id = $2 returning data_layer_id`, + [data.doc, args.id] + ); + if (rows?.[0].data_layer_id && data.attribution) { + await pgClient.query( + `update public.data_sources set attribution = $1 where id = any(select data_source_id from data_layers where id = $2)`, + [data.attribution, rows[0].data_layer_id] + ); + } + const [row] = + await resolveInfo.graphile.selectGraphQLResultFromTable( + sql.fragment`public.table_of_contents_items`, + (tableAlias, queryBuilder) => { + queryBuilder.where( + sql.fragment`${tableAlias}.id = ${sql.value(args.id)}` + ); + } + ); + return row; + } else { + throw new Error("Invalid metadata"); + } + }, + }, + }, + }; +}); + +export default MetadataParserPlugin; diff --git a/packages/client/package-lock.json b/packages/client/package-lock.json index 1e36cc4e5..fc3f1d54b 100644 --- a/packages/client/package-lock.json +++ b/packages/client/package-lock.json @@ -191,6 +191,7 @@ "prosemirror-schema-basic": "^1.1.2", "prosemirror-schema-list": "^1.1.4", "prosemirror-state": "^1.3.4", + "prosemirror-tables": "^1.5.0", "prosemirror-view": "^1.17.2", "qrcode": "^1.5.0", "rbush": "^3.0.1", @@ -39115,6 +39116,18 @@ "prosemirror-view": "^1.27.0" } }, + "node_modules/prosemirror-tables": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/prosemirror-tables/-/prosemirror-tables-1.5.0.tgz", + "integrity": "sha512-VMx4zlYWm7aBlZ5xtfJHpqa3Xgu3b7srV54fXYnXgsAcIGRqKSrhiK3f89omzzgaAgAtDOV4ImXnLKhVfheVNQ==", + "dependencies": { + "prosemirror-keymap": "^1.1.2", + "prosemirror-model": "^1.8.1", + "prosemirror-state": "^1.3.1", + "prosemirror-transform": "^1.2.1", + "prosemirror-view": "^1.13.3" + } + }, "node_modules/prosemirror-transform": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/prosemirror-transform/-/prosemirror-transform-1.8.0.tgz", diff --git a/packages/client/package.json b/packages/client/package.json index 40ebf9039..73884e004 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -190,6 +190,7 @@ "prosemirror-schema-basic": "^1.1.2", "prosemirror-schema-list": "^1.1.4", "prosemirror-state": "^1.3.4", + "prosemirror-tables": "^1.5.0", "prosemirror-view": "^1.17.2", "qrcode": "^1.5.0", "rbush": "^3.0.1", diff --git a/packages/client/src/admin/MetadataEditor.tsx b/packages/client/src/admin/MetadataEditor.tsx index 186833d58..0a8ecfe5c 100644 --- a/packages/client/src/admin/MetadataEditor.tsx +++ b/packages/client/src/admin/MetadataEditor.tsx @@ -124,6 +124,7 @@ export default function MetadataEditor({ }`} > { - metadataContext.open(firstItem.id); + metadataContext.open(firstItem.id, firstItem.title); }} > View metadata diff --git a/packages/client/src/dataLayers/MetadataModal.tsx b/packages/client/src/dataLayers/MetadataModal.tsx index 23eda26ac..985870b28 100644 --- a/packages/client/src/dataLayers/MetadataModal.tsx +++ b/packages/client/src/dataLayers/MetadataModal.tsx @@ -1,5 +1,5 @@ import { DOMSerializer, Node } from "prosemirror-model"; -import { useEffect, useRef } from "react"; +import { useEffect, useMemo, useRef } from "react"; import Modal from "../components/Modal"; import Spinner from "../components/Spinner"; import { metadata as editorConfig } from "../editor/config"; @@ -22,6 +22,15 @@ export default function MetadataModal({ const target = useRef(null); const serializer = useRef(DOMSerializer.fromSchema(schema)); + const showTitle = useMemo(() => { + return ( + !loading && + !document?.content?.find( + (node: any) => node.type === "heading" && node.attrs?.level === 1 + ) + ); + }, [document, loading]); + useEffect(() => { if (target.current && document) { target.current.innerHTML = ""; @@ -64,6 +73,9 @@ export default function MetadataModal({ )} + {showTitle && title && ( +

{title}

+ )}
diff --git a/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx b/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx index bda1020c7..77926da73 100644 --- a/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx +++ b/packages/client/src/dataLayers/TableOfContentsMetadataModal.tsx @@ -5,7 +5,8 @@ import MetadataModal from "./MetadataModal"; export const TableOfContentsMetadataModalContext = createContext<{ id?: number; onRequestClose: () => void; - open: (id: number) => void; + open: (id: number, title?: string) => void; + title?: string; }>({ onRequestClose: () => {}, open: () => {}, @@ -14,9 +15,11 @@ export const TableOfContentsMetadataModalContext = createContext<{ export default function TableOfContentsMetadataModal({ id, onRequestClose, + title, }: { id: number; onRequestClose: () => void; + title?: string; }) { const { data, loading, error } = useGetMetadataQuery({ variables: { @@ -31,6 +34,7 @@ export default function TableOfContentsMetadataModal({ loading={loading} error={error} onRequestClose={onRequestClose} + title={title} /> ); } @@ -41,6 +45,7 @@ export function TableOfContentsMetadataModalProvider({ children: React.ReactNode; }) { const [id, setId] = useState(); + const [title, setTitle] = useState(); return ( { setId(undefined); + setTitle(undefined); }, - open: (id: number) => { + open: (id: number, title?: string) => { setId(id); + setTitle(title); }, }} > @@ -58,7 +65,11 @@ export function TableOfContentsMetadataModalProvider({ {id && ( setId(undefined)} + onRequestClose={() => { + setId(undefined); + setTitle(undefined); + }} + title={title} /> )} diff --git a/packages/client/src/editor/EditorMenuBar.tsx b/packages/client/src/editor/EditorMenuBar.tsx index ec48e94ab..c56d07fa1 100644 --- a/packages/client/src/editor/EditorMenuBar.tsx +++ b/packages/client/src/editor/EditorMenuBar.tsx @@ -30,6 +30,7 @@ import { SketchTocDetailsFragment, FileUploadDetailsFragment, UploaderResponse, + useUpdateMetadataFromXmlMutation, } from "../generated/graphql"; import { sketchType } from "./config"; import { ChevronDownIcon } from "@heroicons/react/outline"; @@ -45,6 +46,7 @@ import { MenuBarContentClasses, MenuBarItemClasses, } from "../components/Menubar"; +import { UploadIcon } from "@radix-ui/react-icons"; require("../admin/data/GLStyleEditor/RadixDropdown.css"); interface EditorMenuBarProps { @@ -61,7 +63,9 @@ interface EditorMenuBarProps { ) => Promise; onUseServiceMetadata?: () => void; dynamicMetadataAvailable?: boolean; + showUploadOption?: boolean; children?: ReactNode; + tocId?: number; } export default function EditorMenuBar(props: EditorMenuBarProps) { @@ -150,6 +154,48 @@ export default function EditorMenuBar(props: EditorMenuBarProps) { [props.view, sketchingContext] ); + const [uploadXMLMutation, uploadXMLMutationState] = + useUpdateMetadataFromXmlMutation(); + + const onUploadButtonClick = useCallback(() => { + // create an input element to trigger the file upload dialog + var input = document.createElement("input"); + input.type = "file"; + // only accept xml files + input.accept = ".xml"; + input.onchange = (e: any) => { + if (e.target.files.length > 0 && props.tocId) { + const file = e.target.files[0]; + // verify that the file is an xml file + if (file.type !== "text/xml") { + alert("Please upload an XML file"); + return; + } + // read the xml file as a string + const reader = new FileReader(); + reader.onload = async (e) => { + let xml = e.target?.result; + if (xml) { + xml = xml.toString(); + try { + const response = await uploadXMLMutation({ + variables: { + itemId: props.tocId!, + xml, + }, + }); + } catch (e) { + console.error(e); + onError(e); + } + } + }; + reader.readAsText(file); + } + }; + input.click(); + }, [props.tocId, uploadXMLMutation]); + const [contextMenuTarget, setContextMenuTarget] = useState(null); @@ -403,6 +449,16 @@ export default function EditorMenuBar(props: EditorMenuBarProps) { /> + {props.showUploadOption && ( + + )} + {(schema.nodes.sketches || schema.marks.attachmentLink) && ( diff --git a/packages/client/src/editor/config.ts b/packages/client/src/editor/config.ts index 38c402dbb..e598231a9 100644 --- a/packages/client/src/editor/config.ts +++ b/packages/client/src/editor/config.ts @@ -8,6 +8,7 @@ import { schema as baseSchema } from "./basicSchema"; import { exampleSetup } from "prosemirror-example-setup"; import { addListNodes } from "prosemirror-schema-list"; import QuestionPlaceholderPlugin from "./QuestionPlaceholderPlugin"; +import { tableNodes } from "prosemirror-tables"; let spec = baseSchema.spec; @@ -63,7 +64,29 @@ const questionSchema: Schema = new Schema({ const metadataSchema = new Schema({ // @ts-ignore - nodes: addListNodes(baseSchema.spec.nodes, "paragraph block*", "block"), + nodes: addListNodes( + baseSchema.spec.nodes, + "paragraph block*", + "block" + ).append( + tableNodes({ + tableGroup: "block", + cellContent: "block+", + cellAttributes: { + background: { + default: null, + getFromDOM(dom) { + return (dom.style && dom.style.backgroundColor) || null; + }, + setDOMAttr(value, attrs) { + if (value) + // eslint-disable-next-line i18next/no-literal-string + attrs.style = (attrs.style || "") + `background-color: ${value};`; + }, + }, + }, + }) + ), // @ts-ignore marks: baseMarks, }); diff --git a/packages/client/src/formElements/prosemirror-body.css b/packages/client/src/formElements/prosemirror-body.css index fb0c09aba..3ebf4a3fa 100644 --- a/packages/client/src/formElements/prosemirror-body.css +++ b/packages/client/src/formElements/prosemirror-body.css @@ -133,18 +133,26 @@ div[attachments="forumAttachments"] { display: none; } -.metadata.small-variant .ProseMirror h1 { +.metadata .ProseMirror h1 { @apply text-2xl; } -.metadata.small-variant .ProseMirror h2 { - @apply text-xl; +.metadata .ProseMirror h2 { + @apply text-xl pt-3 pb-0.5; } -.metadata.small-variant .ProseMirror h3 { - @apply text-lg; +.metadata .ProseMirror h3 { + @apply text-lg pt-2 pb-0.5; } -.metadata.small-variant .ProseMirror p { +.metadata .ProseMirror p { @apply text-sm; } + +.metadata .ProseMirror table { + @apply text-sm mt-1 mb-2; +} + +.metadata .ProseMirror td { + @apply border p-1; +} diff --git a/packages/client/src/generated/graphql.ts b/packages/client/src/generated/graphql.ts index b5384d3ad..683d049a6 100644 --- a/packages/client/src/generated/graphql.ts +++ b/packages/client/src/generated/graphql.ts @@ -7150,6 +7150,7 @@ export type Mutation = { /** Updates a single `TableOfContentsItem` using its globally unique id and a patch. */ updateTableOfContentsItemByNodeId?: Maybe; updateTableOfContentsItemChildren?: Maybe; + updateTocMetadataFromXML: TableOfContentsItem; /** Updates a single `Topic` using a unique key and a patch. */ updateTopic?: Maybe; /** Updates a single `Topic` using its globally unique id and a patch. */ @@ -8607,6 +8608,13 @@ export type MutationUpdateTableOfContentsItemChildrenArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationUpdateTocMetadataFromXmlArgs = { + id: Scalars['Int']; + xmlMetadata: Scalars['String']; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationUpdateTopicArgs = { input: UpdateTopicInput; @@ -17413,6 +17421,28 @@ export type UpdateMetadataMutation = ( )> } ); +export type UpdateMetadataFromXmlMutationVariables = Exact<{ + itemId: Scalars['Int']; + xml: Scalars['String']; +}>; + + +export type UpdateMetadataFromXmlMutation = ( + { __typename?: 'Mutation' } + & { updateTocMetadataFromXML: ( + { __typename?: 'TableOfContentsItem' } + & Pick + & { dataLayer?: Maybe<( + { __typename?: 'DataLayer' } + & Pick + & { dataSource?: Maybe<( + { __typename?: 'DataSource' } + & Pick + )> } + )> } + ) } +); + export type ProjectHostingQuotaQueryVariables = Exact<{ slug: Scalars['String']; }>; @@ -26249,6 +26279,49 @@ export function useUpdateMetadataMutation(baseOptions?: Apollo.MutationHookOptio export type UpdateMetadataMutationHookResult = ReturnType; export type UpdateMetadataMutationResult = Apollo.MutationResult; export type UpdateMetadataMutationOptions = Apollo.BaseMutationOptions; +export const UpdateMetadataFromXmlDocument = gql` + mutation UpdateMetadataFromXML($itemId: Int!, $xml: String!) { + updateTocMetadataFromXML(id: $itemId, xmlMetadata: $xml) { + id + metadata + computedMetadata + dataLayer { + id + dataSource { + id + attribution + } + } + } +} + `; +export type UpdateMetadataFromXmlMutationFn = Apollo.MutationFunction; + +/** + * __useUpdateMetadataFromXmlMutation__ + * + * To run a mutation, you first call `useUpdateMetadataFromXmlMutation` within a React component and pass it any options that fit your needs. + * When your component renders, `useUpdateMetadataFromXmlMutation` returns a tuple that includes: + * - A mutate function that you can call at any time to execute the mutation + * - An object with fields that represent the current status of the mutation's execution + * + * @param baseOptions options that will be passed into the mutation, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options-2; + * + * @example + * const [updateMetadataFromXmlMutation, { data, loading, error }] = useUpdateMetadataFromXmlMutation({ + * variables: { + * itemId: // value for 'itemId' + * xml: // value for 'xml' + * }, + * }); + */ +export function useUpdateMetadataFromXmlMutation(baseOptions?: Apollo.MutationHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useMutation(UpdateMetadataFromXmlDocument, options); + } +export type UpdateMetadataFromXmlMutationHookResult = ReturnType; +export type UpdateMetadataFromXmlMutationResult = Apollo.MutationResult; +export type UpdateMetadataFromXmlMutationOptions = Apollo.BaseMutationOptions; export const ProjectHostingQuotaDocument = gql` query ProjectHostingQuota($slug: String!) { projectBySlug(slug: $slug) { @@ -33291,6 +33364,7 @@ export const namedOperations = { UpdateFetchStrategy: 'UpdateFetchStrategy', UpdateEnableHighDPIRequests: 'UpdateEnableHighDPIRequests', UpdateMetadata: 'UpdateMetadata', + UpdateMetadataFromXML: 'UpdateMetadataFromXML', PublishTableOfContents: 'PublishTableOfContents', ImportArcGISService: 'ImportArcGISService', SetMaxZoom: 'SetMaxZoom', diff --git a/packages/client/src/generated/queries.ts b/packages/client/src/generated/queries.ts index b70a69d12..43819073c 100644 --- a/packages/client/src/generated/queries.ts +++ b/packages/client/src/generated/queries.ts @@ -7148,6 +7148,7 @@ export type Mutation = { /** Updates a single `TableOfContentsItem` using its globally unique id and a patch. */ updateTableOfContentsItemByNodeId?: Maybe; updateTableOfContentsItemChildren?: Maybe; + updateTocMetadataFromXML: TableOfContentsItem; /** Updates a single `Topic` using a unique key and a patch. */ updateTopic?: Maybe; /** Updates a single `Topic` using its globally unique id and a patch. */ @@ -8605,6 +8606,13 @@ export type MutationUpdateTableOfContentsItemChildrenArgs = { }; +/** The root mutation type which contains root level fields which mutate data. */ +export type MutationUpdateTocMetadataFromXmlArgs = { + id: Scalars['Int']; + xmlMetadata: Scalars['String']; +}; + + /** The root mutation type which contains root level fields which mutate data. */ export type MutationUpdateTopicArgs = { input: UpdateTopicInput; @@ -17411,6 +17419,28 @@ export type UpdateMetadataMutation = ( )> } ); +export type UpdateMetadataFromXmlMutationVariables = Exact<{ + itemId: Scalars['Int']; + xml: Scalars['String']; +}>; + + +export type UpdateMetadataFromXmlMutation = ( + { __typename?: 'Mutation' } + & { updateTocMetadataFromXML: ( + { __typename?: 'TableOfContentsItem' } + & Pick + & { dataLayer?: Maybe<( + { __typename?: 'DataLayer' } + & Pick + & { dataSource?: Maybe<( + { __typename?: 'DataSource' } + & Pick + )> } + )> } + ) } +); + export type ProjectHostingQuotaQueryVariables = Exact<{ slug: Scalars['String']; }>; @@ -23873,6 +23903,22 @@ export const UpdateMetadataDocument = /*#__PURE__*/ gql` } } `; +export const UpdateMetadataFromXmlDocument = /*#__PURE__*/ gql` + mutation UpdateMetadataFromXML($itemId: Int!, $xml: String!) { + updateTocMetadataFromXML(id: $itemId, xmlMetadata: $xml) { + id + metadata + computedMetadata + dataLayer { + id + dataSource { + id + attribution + } + } + } +} + `; export const ProjectHostingQuotaDocument = /*#__PURE__*/ gql` query ProjectHostingQuota($slug: String!) { projectBySlug(slug: $slug) { @@ -26377,6 +26423,7 @@ export const namedOperations = { UpdateFetchStrategy: 'UpdateFetchStrategy', UpdateEnableHighDPIRequests: 'UpdateEnableHighDPIRequests', UpdateMetadata: 'UpdateMetadata', + UpdateMetadataFromXML: 'UpdateMetadataFromXML', PublishTableOfContents: 'PublishTableOfContents', ImportArcGISService: 'ImportArcGISService', SetMaxZoom: 'SetMaxZoom', diff --git a/packages/client/src/queries/DraftTableOfContents.graphql b/packages/client/src/queries/DraftTableOfContents.graphql index 4abda9abd..5b846b479 100644 --- a/packages/client/src/queries/DraftTableOfContents.graphql +++ b/packages/client/src/queries/DraftTableOfContents.graphql @@ -638,6 +638,21 @@ mutation UpdateMetadata($itemId: Int!, $metadata: JSON) { } } +mutation UpdateMetadataFromXML($itemId: Int!, $xml: String!) { + updateTocMetadataFromXML(id: $itemId, xmlMetadata: $xml) { + id + metadata + computedMetadata + dataLayer { + id + dataSource { + id + attribution + } + } + } +} + query ProjectHostingQuota($slug: String!) { projectBySlug(slug: $slug) { id diff --git a/packages/metadata-parser/__tests__/Coral-esri-example-metadata.xml b/packages/metadata-parser/__tests__/Coral-fgdc-example-metadata.xml similarity index 100% rename from packages/metadata-parser/__tests__/Coral-esri-example-metadata.xml rename to packages/metadata-parser/__tests__/Coral-fgdc-example-metadata.xml diff --git a/packages/metadata-parser/__tests__/metadata-parser.test.js b/packages/metadata-parser/__tests__/metadata-parser.test.js deleted file mode 100644 index af115118f..000000000 --- a/packages/metadata-parser/__tests__/metadata-parser.test.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict'; - -const metadataParser = require('..'); -const assert = require('assert').strict; - -assert.strictEqual(metadataParser(), 'Hello from metadataParser'); -console.info('metadataParser tests passed'); diff --git a/packages/metadata-parser/__tests__/metadata-parser.test.ts b/packages/metadata-parser/__tests__/metadata-parser.test.ts index b9aa38ec5..3a3e0fd43 100644 --- a/packages/metadata-parser/__tests__/metadata-parser.test.ts +++ b/packages/metadata-parser/__tests__/metadata-parser.test.ts @@ -1,14 +1,17 @@ import { describe, it, expect, beforeAll } from "vitest"; +// @ts-ignore import fs from "fs"; +// @ts-ignore import path from "path"; import { - esriToMarkdown, - getAttribution as getEsriAttribution, -} from "../src/esriToMarkdown"; + fgdcToMarkdown, + getAttribution as getFGDCAttribution, +} from "../src/fgdcToMarkdown"; import { iso19139ToMarkdown, getAttribution as getIsoAttribution, } from "../src/iso19139"; +// @ts-ignore import xml2js from "xml2js"; // Helper to load XML files @@ -24,24 +27,23 @@ function writeMarkdownFile(fileName: string, markdown: string) { } describe("Metadata Parsers", () => { - describe("Esri Metadata", () => { - let esriMetadata: any; + describe("FGDC Metadata", () => { + let fgdcMetadata: any; beforeAll(async () => { const filePath = path.resolve( __dirname, - "./Coral-esri-example-metadata.xml" + "./Coral-fgdc-example-metadata.xml" ); - esriMetadata = await loadXml(filePath); + fgdcMetadata = await loadXml(filePath); }); - it("parses Esri metadata to Markdown", () => { - expect(esriMetadata).toBeTruthy(); - const markdown = esriToMarkdown(esriMetadata); + it("parses fgdc metadata to Markdown", () => { + expect(fgdcMetadata).toBeTruthy(); + const markdown = fgdcToMarkdown(fgdcMetadata); // Write the markdown file - writeMarkdownFile("Coral-esri-metadata.md", markdown); + writeMarkdownFile("Coral-fgcd-metadata.md", markdown); - console.log("markdown", markdown); expect(markdown).toContain("# Resilience Metrics from FRRP Coral Data"); expect(markdown).toContain( "**Abstract:** The dataset includes Taxonomic richness" @@ -54,8 +56,8 @@ describe("Metadata Parsers", () => { expect(markdown).toContain("- **Organization:** The Nature Conservancy"); }); - it("returns the correct Esri attribution", () => { - const attribution = getEsriAttribution(esriMetadata); + it("returns the correct FGDC attribution", () => { + const attribution = getFGDCAttribution(fgdcMetadata); expect(attribution).toBe("The Nature Conservancy"); }); }); @@ -82,7 +84,7 @@ describe("Metadata Parsers", () => { expect(markdown).toContain("**Keywords:** FAUNA, FISHERIES"); expect(markdown).toContain("## Contact Information"); expect(markdown).toContain("- **Organization:** Geoscience_MFMRD"); - expect(markdown).toContain("- **Contact Person:** Catherine Paul"); + expect(markdown).toContain("**Name:** Catherine Paul"); }); it("returns the correct ISO attribution", () => { diff --git a/packages/metadata-parser/__tests__/output/3nm_Polyline-iso-metadata.md b/packages/metadata-parser/__tests__/output/3nm_Polyline-iso-metadata.md new file mode 100644 index 000000000..24bff5c3e --- /dev/null +++ b/packages/metadata-parser/__tests__/output/3nm_Polyline-iso-metadata.md @@ -0,0 +1,34 @@ +# 3nm_Polyline + +**Abstract:** This is a 3nm demarcation within the Polyline for Kiribati + + +**Keywords:** FAUNA, FISHERIES, GEOSCIENCES, MARINE, PHOTOGRAPHY-AND-IMAGERY + + +**Topic Categories:** boundaries, planningCadastre, oceans, geoscientificInformation + +## Data Quality + +**Lineage:** Using the 3 nautical mile shapefile as a reference to create its 3nm using creating new shapefile and buffer tool on QGIS. + +## Contact Information + +- **Name:** Catherine Paul +- **Organization:** Geoscience_MFMRD +- **Position:** Principal Mineral Officer +- **Phone:** 75021099 +- **Address:** P.O.Box 64 +- **City:** Tarawa +- **Postal Code:** KI0107 +- **Country:** Kiribati +- **Email:** catherinepaul@mfmrd.gov.ki + +## Resource Constraints +- **Use Limitations:** open access + +- **Access Constraints:** copyright + +- **Use Constraints:** copyright + + diff --git a/packages/metadata-parser/__tests__/output/Coral-esri-metadata.md b/packages/metadata-parser/__tests__/output/Coral-esri-metadata.md deleted file mode 100644 index 4f9f41e84..000000000 --- a/packages/metadata-parser/__tests__/output/Coral-esri-metadata.md +++ /dev/null @@ -1 +0,0 @@ -# Untitled \ No newline at end of file diff --git a/packages/metadata-parser/__tests__/output/Coral-fgcd-metadata.md b/packages/metadata-parser/__tests__/output/Coral-fgcd-metadata.md new file mode 100644 index 000000000..9ecf69a16 --- /dev/null +++ b/packages/metadata-parser/__tests__/output/Coral-fgcd-metadata.md @@ -0,0 +1,43 @@ +# Resilience Metrics from FRRP Coral Data +**Abstract:** The dataset includes Taxonomic richness (# of genera and species combined), intensity of bleaching, and disease prevalence, and an index of reef resilience (see VanWoesik and Burman 2012) of benthic stony corals collected from multiple habitat types across the south Florida shelf from St. Lucie Inlet to Biscayne Bay for years 2005 through 2014. The data were collected during synoptic broad-scale surveys of coral reef and hard-bottom habitats that were stratified into sub-regions or along-shelf positions (e.g., Biscayne, SEFCRI and various habitat types (e.g., patch reefs, low relief hard-bottom, high-relief spur and groove, etc.). A 200m x 200m polygon grid was used to overlay onto existing bathymetry and benthic habitat maps of the study area, and a two-stage stratified random design was used to select sites for sampling from various strata combinations of cross-shelf habitat type, along-shelf position (i.e. region), and management zone (http://frrp.org/). + +**Purpose:** This dataset was compiled to 1) describe the spatial distribution patterns of stony coral density, bleaching intensity, and disease prevalence along the Florida Coral Reef Tract (FCRT) of the SEFCRI Region for the Nature Conservancy's Florida Reef Resilience Program. This dataset is intended to inform the Southeast Florida Coral Reef Initiative "Our Florida Reefs" management process. + +**Keywords:** Florida Reef Resiliency Program, The Nature Conservancy, Florida Keys, Southeast Florida, SEFCRI, Resilience, Coral Richness, Disease, Bleaching, Coral Density + +**Use Constraints:** Data is open access. Contact James Byrne (TNC) for more information + +**Temporal Coverage:** +- **Start Date:** 20050101 +- **End Date:** 20111231 + +## Attribute Information +| Attribute Label | Definition | Domain | +|-----------------|-------------|--------| +| OBJECTID | Internal feature number. | Sequential unique whole numbers that are automatically generated. | +| Shape | Feature geometry. | Coordinates defining the features. | +| Batch | | | +| Latitude | latitude | | +| Longitude | longitude | | +| Subregion | subregion of study area | | +| Zone | zone | | +| Depth | Depth, in meters | | +| Habitat | habitat (RVC classifications) | | +| Site | FRRP site of survey | | +| Region | | | +| N_Taxa | | | +| NC_Sum | | | +| BI_Mean | | | +| DisTot_Sum | | | +| ColDens | | | +| N_Trans | | | +| SBII | | | +| SDII | | | +| Area_m2 | | | +| DnClSEFCRI | | | +| SBl_SEFCRI | | | +| SDI_SEFCRI | | | +| RRI_SEFCRI | | | + +## Contact Information +- **Organization:** The Nature Conservancy diff --git a/packages/metadata-parser/package.json b/packages/metadata-parser/package.json index a9e8bf5b6..9cb1d283a 100644 --- a/packages/metadata-parser/package.json +++ b/packages/metadata-parser/package.json @@ -1,5 +1,5 @@ { - "name": "metadata-parser", + "name": "@seasketch/metadata-parser", "version": "1.0.0", "description": "Parses metadata in esri, iso19139, and possibly other formats and converts them to markdown summaries for use in SeaSketch", "author": "Chad Burt ", @@ -7,7 +7,7 @@ "license": "BSD-3-Clause", "main": "dist/metadata-parser.js", "directories": { - "lib": "src", + "src": "src", "test": "__tests__" }, "files": [ @@ -19,14 +19,17 @@ }, "scripts": { "build": "tsc", - "watch": "tsc --watch" + "watch": "tsc --watch", + "test": "vitest" }, "bugs": { "url": "https://github.com/seasketch/next/issues" }, "dependencies": { "@types/xml2js": "^0.4.14", - "vitest": "^2.0.5", "xml2js": "^0.6.2" + }, + "devDependencies": { + "vitest": "^2.0.5" } } diff --git a/packages/metadata-parser/src/esriToMarkdown.ts b/packages/metadata-parser/src/esriToMarkdown.ts deleted file mode 100644 index 98b3389cc..000000000 --- a/packages/metadata-parser/src/esriToMarkdown.ts +++ /dev/null @@ -1,110 +0,0 @@ -export function esriToMarkdown(metadata: any): string { - const md: string[] = []; - - // Title - const title = - metadata?.metadata?.idinfo?.citation?.citeinfo?.title?.[0] || "Untitled"; - md.push(`# ${title}`); - - // Abstract - const abstract = metadata?.metadata?.idinfo?.descript?.abstract?.[0]; - if (abstract) { - md.push(`\n**Abstract:** ${abstract}\n`); - } - - // Purpose - const purpose = metadata?.metadata?.idinfo?.descript?.purpose?.[0]; - if (purpose) { - md.push(`\n**Purpose:** ${purpose}\n`); - } - - // Keywords - const themeKeywords = - metadata?.metadata?.idinfo?.keywords?.theme?.[0]?.themekey - ?.map((key: any) => key) - ?.join(", "); - if (themeKeywords) { - md.push(`\n**Keywords:** ${themeKeywords}\n`); - } - - // Temporal Coverage - const beginDate = - metadata?.metadata?.idinfo?.timeperd?.timeinfo?.rngdates?.begdate?.[0]; - const endDate = - metadata?.metadata?.idinfo?.timeperd?.timeinfo?.rngdates?.enddate?.[0]; - if (beginDate || endDate) { - md.push(`\n**Temporal Coverage:**`); - if (beginDate) md.push(`\n- **Start Date:** ${beginDate}`); - if (endDate) md.push(`\n- **End Date:** ${endDate}`); - md.push("\n"); - } - - // Spatial Extent (Bounding Box) - const west = metadata?.metadata?.spdom?.bounding?.westbc?.[0]; - const east = metadata?.metadata?.spdom?.bounding?.eastbc?.[0]; - const north = metadata?.metadata?.spdom?.bounding?.northbc?.[0]; - const south = metadata?.metadata?.spdom?.bounding?.southbc?.[0]; - if (west || east || north || south) { - md.push(`\n**Spatial Extent (Bounding Box):**`); - if (west) md.push(`\n- **West:** ${west}`); - if (east) md.push(`\n- **East:** ${east}`); - if (north) md.push(`\n- **North:** ${north}`); - if (south) md.push(`\n- **South:** ${south}`); - md.push("\n"); - } - - // Attribute Information (Table) - const attributes = metadata?.metadata?.eainfo?.detailed?.attr || []; - if (attributes.length > 0) { - md.push(`\n## Attribute Information\n`); - md.push( - "| Attribute Label | Definition | Domain |\n|-----------------|-------------|--------|\n" - ); - attributes.forEach((attr: any) => { - const label = attr?.attrlabl?.[0] || "Unknown"; - const definition = attr?.attrdef?.[0] || "No definition"; - const domain = attr?.attrdomv?.udom?.[0] || "No domain"; - md.push(`| ${label} | ${definition} | ${domain} |\n`); - }); - } - - // Contact Information (Enhanced Parsing without placeholders) - const contactInfo = metadata?.metadata?.metainfo?.metc?.cntinfo || {}; - const contactOrg = contactInfo?.cntorgp?.cntorg?.[0]; - const contactPerson = contactInfo?.cntper?.[0]; - const contactPhone = contactInfo?.cntvoice?.[0]; - const contactEmail = contactInfo?.cntemail?.[0]; - const contactAddress = contactInfo?.cntaddr?.[0]?.address?.[0]; - - if ( - contactOrg || - contactPerson || - contactPhone || - contactEmail || - contactAddress - ) { - md.push(`\n## Contact Information\n`); - if (contactOrg) md.push(`- **Organization:** ${contactOrg}\n`); - if (contactPerson) md.push(`- **Contact Person:** ${contactPerson}\n`); - if (contactPhone) md.push(`- **Phone:** ${contactPhone}\n`); - if (contactEmail) md.push(`- **Email:** ${contactEmail}\n`); - if (contactAddress) md.push(`- **Address:** ${contactAddress}\n`); - } - - return md.join(""); -} - -// Get attribution from Esri metadata -export function getAttribution(metadata: any): string | null { - // Try to retrieve the organization from the citation info - const organization = - metadata?.metadata?.idinfo?.citation?.citeinfo?.origin?.[0]; - if (organization) { - return organization; - } - - // Fallback to contact organization if citation doesn't have origin - const contactOrg = - metadata?.metadata?.metainfo?.metc?.cntinfo?.cntorgp?.cntorg?.[0]; - return contactOrg || null; -} diff --git a/packages/metadata-parser/src/fgdcToMarkdown.ts b/packages/metadata-parser/src/fgdcToMarkdown.ts new file mode 100644 index 000000000..fd4a3b2df --- /dev/null +++ b/packages/metadata-parser/src/fgdcToMarkdown.ts @@ -0,0 +1,157 @@ +export function fgdcToMarkdown(metadata: any): string { + const md: string[] = []; + + // Helper function to safely access arrays + const getFirst = (value: any) => + Array.isArray(value) && value.length > 0 ? value[0] : ""; + + // Title + const idinfo = getFirst(metadata?.metadata?.idinfo); + const citation = getFirst(idinfo?.citation); + const citeinfo = getFirst(citation?.citeinfo); + const title = getFirst(citeinfo?.title); + md.push(`# ${title || "Untitled"}`); + + // Abstract + const descript = getFirst(idinfo?.descript); + const abstract = getFirst(descript?.abstract); + if (abstract) { + md.push(`\n**Abstract:** ${abstract.split("Attribute Fields:")[0]}\n`); + } + + // Purpose + const purpose = getFirst(descript?.purpose); + if (purpose) { + md.push(`\n**Purpose:** ${purpose}\n`); + } + + // Keywords + const keywords = getFirst(idinfo?.keywords); + const theme = getFirst(keywords?.theme); + const themeKeywords = theme?.themekey.filter(Boolean).join(", "); + if (themeKeywords) { + md.push(`\n**Keywords:** ${themeKeywords}\n`); + } + + // Use Constraints (added section) + const useConstraints = getFirst(idinfo?.useconst); + if (useConstraints) { + md.push( + `\n**Use Constraints:** ${ + useConstraints.split("Downloaded data filename:")[0] + }\n` + ); + } + + // Temporal Coverage + const timeperd = getFirst(idinfo?.timeperd); + const timeinfo = getFirst(timeperd?.timeinfo); + const rngdates = getFirst(timeinfo?.rngdates); + const beginDate = getFirst(rngdates?.begdate); + const endDate = getFirst(rngdates?.enddate); + if (beginDate || endDate) { + md.push(`\n**Temporal Coverage:**`); + if (beginDate) md.push(`\n- **Start Date:** ${beginDate}`); + if (endDate) md.push(`\n- **End Date:** ${endDate}`); + md.push("\n"); + } + + // Spatial Extent (Bounding Box) + const spdom = getFirst(metadata?.metadata?.spdom); + const bounding = getFirst(spdom?.bounding); + const west = getFirst(bounding?.westbc); + const east = getFirst(bounding?.eastbc); + const north = getFirst(bounding?.northbc); + const south = getFirst(bounding?.southbc); + if (west || east || north || south) { + md.push(`\n**Spatial Extent (Bounding Box):**`); + if (west) md.push(`\n- **West:** ${west}`); + if (east) md.push(`\n- **East:** ${east}`); + if (north) md.push(`\n- **North:** ${north}`); + if (south) md.push(`\n- **South:** ${south}`); + md.push("\n"); + } + + // Attribute Information (Table) + const eainfo = getFirst(metadata?.metadata?.eainfo); + const detailed = eainfo?.detailed?.[0] || {}; + const attributes = detailed?.attr || []; + if (attributes.length > 0) { + md.push(`\n## Attribute Information\n`); + md.push( + "| Attribute Label | Definition | Domain |\n|-----------------|-------------|--------|\n" + ); + attributes.forEach((attr: any) => { + const label = getFirst(attr?.attrlabl) || "Unknown"; + const definition = getFirst(attr?.attrdef) || ""; + + // Enhanced Domain Parsing + let domain = ""; + const attrdomv = getFirst(attr?.attrdomv); + if (attrdomv?.udom) { + domain = getFirst(attrdomv?.udom); + } else if (attrdomv?.rdom) { + const rdommin = getFirst(attrdomv?.rdom?.rdommin); + const rdommax = getFirst(attrdomv?.rdom?.rdommax); + if (rdommin && rdommax) { + domain = `${rdommin} to ${rdommax}`; + } else if (rdommin || rdommax) { + domain = `Value: ${rdommin || rdommax}`; + } + } + + md.push(`| ${label} | ${definition} | ${domain} |\n`); + }); + } + + // Contact Information (Enhanced Parsing without placeholders) + const metainfo = getFirst(metadata?.metadata?.metainfo); + const metc = getFirst(metainfo?.metc); + const cntinfo = getFirst(metc?.cntinfo); + const cntorgp = getFirst(cntinfo?.cntorgp); + const contactOrg = getFirst(cntorgp?.cntorg); + const contactPerson = getFirst(cntinfo?.cntper); + const contactPhone = getFirst(cntinfo?.cntvoice); + const contactEmail = getFirst(cntinfo?.cntemail); + const cntaddr = getFirst(cntinfo?.cntaddr); + const contactAddress = getFirst(cntaddr?.address); + + if ( + contactOrg || + contactPerson || + contactPhone || + contactEmail || + contactAddress + ) { + md.push(`\n## Contact Information\n`); + if (contactOrg) md.push(`- **Organization:** ${contactOrg}\n`); + if (contactPerson) md.push(`- **Contact Person:** ${contactPerson}\n`); + if (contactPhone) md.push(`- **Phone:** ${contactPhone}\n`); + if (contactEmail) md.push(`- **Email:** ${contactEmail}\n`); + if (contactAddress) md.push(`- **Address:** ${contactAddress}\n`); + } + + return md.join(""); +} + +export function getAttribution(metadata: any): string { + const getFirst = (value: any) => + Array.isArray(value) && value.length > 0 ? value[0] : ""; + + // Try to retrieve the organization from the citation info + const idinfo = getFirst(metadata?.metadata?.idinfo); + const citation = getFirst(idinfo?.citation); + const citeinfo = getFirst(citation?.citeinfo); + const organization = getFirst(citeinfo?.origin); + if (organization) { + return organization; + } + + // Fallback to contact organization if citation doesn't have origin + const metainfo = getFirst(metadata?.metadata?.metainfo); + const metc = getFirst(metainfo?.metc); + const cntinfo = getFirst(metc?.cntinfo); + const cntorgp = getFirst(cntinfo?.cntorgp); + const contactOrg = getFirst(cntorgp?.cntorg); + return contactOrg || null; +} diff --git a/packages/metadata-parser/src/fgdcToProsemirror.ts b/packages/metadata-parser/src/fgdcToProsemirror.ts new file mode 100644 index 000000000..23f200453 --- /dev/null +++ b/packages/metadata-parser/src/fgdcToProsemirror.ts @@ -0,0 +1,250 @@ +// Helper function to safely access arrays +const getFirst = (value: any) => + Array.isArray(value) && value.length > 0 ? value[0] : ""; + +// Function to create a text node +function createTextNode(content: string) { + if (content === "") { + content = " "; + } + return { + type: "text", + text: content, + }; +} + +// Function to create a paragraph node +function createParagraphNode(content: any[]) { + return { + type: "paragraph", + content: content, + }; +} + +// Function to create a heading node +function createHeadingNode(content: any[], level: number) { + return { + type: "heading", + attrs: { level }, + content: content, + }; +} + +// Function to create a table cell node +function createTableCellNode(content: any[]) { + return { + type: "table_cell", + content: [createParagraphNode(content)], + }; +} + +// Function to create a table row node +function createTableRowNode(cells: any[]) { + return { + type: "table_row", + content: cells, + }; +} + +// Function to create a table node +function createTableNode(rows: any[]) { + return { + type: "table", + content: rows, + }; +} + +// Function to create a bullet list node +function createBulletListNode(items: any[]) { + return { + type: "bullet_list", + content: items, + }; +} + +// Function to create list item node +function createListItemNode(content: any[]) { + return { + type: "list_item", + content: [createParagraphNode(content)], + }; +} + +// Main function to generate ProseMirror nodes from Esri metadata +export function fgdcToProseMirror(metadata: any) { + const doc: any = { type: "doc", content: [] }; + + // Title + const idinfo = getFirst(metadata?.metadata?.idinfo); + const citation = getFirst(idinfo?.citation); + const citeinfo = getFirst(citation?.citeinfo); + const title = getFirst(citeinfo?.title); + // if (title) { + // doc.content.push(createHeadingNode([createTextNode(title)], 1)); + // } + + // Abstract + const descript = getFirst(idinfo?.descript); + const abstract = getFirst(descript?.abstract); + if (abstract) { + doc.content.push(createHeadingNode([createTextNode("Abstract")], 2)); + doc.content.push( + createParagraphNode([ + createTextNode(abstract.split("Attribute Fields:")[0]), + ]) + ); + } + + // Purpose + const purpose = getFirst(descript?.purpose); + if (purpose) { + doc.content.push(createHeadingNode([createTextNode("Purpose")], 2)); + doc.content.push(createParagraphNode([createTextNode(purpose)])); + } + + // Keywords + const keywords = getFirst(idinfo?.keywords); + const theme = getFirst(keywords?.theme); + const themeKeywords = theme?.themekey.filter(Boolean).join(", "); + if (themeKeywords) { + doc.content.push(createHeadingNode([createTextNode("Keywords")], 2)); + doc.content.push(createParagraphNode([createTextNode(themeKeywords)])); + } + + // Use Constraints + const useConstraints = getFirst(idinfo?.useconst); + if (useConstraints) { + doc.content.push(createHeadingNode([createTextNode("Use Constraints")], 2)); + doc.content.push( + createParagraphNode([ + createTextNode(useConstraints.split("Downloaded data filename:")[0]), + ]) + ); + } + + // Attribute Information (Table using ProseMirror Table Schema) + const eainfo = getFirst(metadata?.metadata?.eainfo); + const detailed = eainfo?.detailed?.[0] || {}; + const attributes = detailed?.attr || []; + if (attributes.length > 0) { + doc.content.push( + createHeadingNode([createTextNode("Attribute Information")], 2) + ); + + const tableRows: any[] = []; + + // Create header row + const headerRow = createTableRowNode([ + createTableCellNode([createTextNode("Attribute Label")]), + createTableCellNode([createTextNode("Definition")]), + createTableCellNode([createTextNode("Domain")]), + ]); + tableRows.push(headerRow); + + // Create data rows + attributes.forEach((attr: any) => { + const label = getFirst(attr?.attrlabl) || "Unknown"; + const definition = getFirst(attr?.attrdef); + + let domain = " "; + const attrdomv = getFirst(attr?.attrdomv); + if (attrdomv?.udom) { + domain = getFirst(attrdomv?.udom); + } else if (attrdomv?.rdom) { + const rdommin = getFirst(attrdomv?.rdom?.rdommin); + const rdommax = getFirst(attrdomv?.rdom?.rdommax); + if (rdommin && rdommax) { + domain = `${rdommin} to ${rdommax}`; + } else if (rdommin || rdommax) { + domain = `Value: ${rdommin || rdommax}`; + } + } + + const dataRow = createTableRowNode([ + createTableCellNode([createTextNode(label)]), + createTableCellNode([createTextNode(definition || " ")]), + createTableCellNode([createTextNode(domain)]), + ]); + tableRows.push(dataRow); + }); + + const tableNode = createTableNode(tableRows); + doc.content.push(tableNode); + } + + // Contact Information + const metainfo = getFirst(metadata?.metadata?.metainfo); + const metc = getFirst(metainfo?.metc); + const cntinfo = getFirst(metc?.cntinfo); + const cntorgp = getFirst(cntinfo?.cntorgp); + const contactOrg = getFirst(cntorgp?.cntorg); + const contactPerson = getFirst(cntinfo?.cntper); + const contactPhone = getFirst(cntinfo?.cntvoice); + const contactEmail = getFirst(cntinfo?.cntemail); + const cntaddr = getFirst(cntinfo?.cntaddr); + const contactAddress = getFirst(cntaddr?.address); + + if ( + contactOrg || + contactPerson || + contactPhone || + contactEmail || + contactAddress + ) { + const contactInfoContent = []; + doc.content.push( + createHeadingNode([createTextNode("Contact Information")], 2) + ); + if (contactOrg) + contactInfoContent.push( + createListItemNode([createTextNode(`Organization: ${contactOrg}`)]) + ); + if (contactPerson) + contactInfoContent.push( + createListItemNode([createTextNode(`Contact Person: ${contactPerson}`)]) + ); + if (contactPhone) + contactInfoContent.push( + createListItemNode([createTextNode(`Phone: ${contactPhone}`)]) + ); + if (contactEmail) + contactInfoContent.push( + createListItemNode([createTextNode(`Email: ${contactEmail}`)]) + ); + if (contactAddress) + contactInfoContent.push( + createListItemNode([createTextNode(`Address: ${contactAddress}`)]) + ); + doc.content.push(createBulletListNode(contactInfoContent)); + } + + const attribution = getAttribution(metadata); + + return { + title, + doc, + attribution, + }; +} + +export function getAttribution(metadata: any): string { + const getFirst = (value: any) => + Array.isArray(value) && value.length > 0 ? value[0] : ""; + + // Try to retrieve the organization from the citation info + const idinfo = getFirst(metadata?.metadata?.idinfo); + const citation = getFirst(idinfo?.citation); + const citeinfo = getFirst(citation?.citeinfo); + const organization = getFirst(citeinfo?.origin); + if (organization) { + return organization; + } + + // Fallback to contact organization if citation doesn't have origin + const metainfo = getFirst(metadata?.metadata?.metainfo); + const metc = getFirst(metainfo?.metc); + const cntinfo = getFirst(metc?.cntinfo); + const cntorgp = getFirst(cntinfo?.cntorgp); + const contactOrg = getFirst(cntorgp?.cntorg); + return contactOrg || "Unknown attribution"; +} diff --git a/packages/metadata-parser/src/iso19139.ts b/packages/metadata-parser/src/iso19139.ts index ef616653c..892196335 100644 --- a/packages/metadata-parser/src/iso19139.ts +++ b/packages/metadata-parser/src/iso19139.ts @@ -109,6 +109,9 @@ const parseDataQuality = (dataQualityInfo: any): string => { }; export function iso19139ToMarkdown(metadata: any) { + if ("gmd:MD_Metadata" in metadata) { + metadata = metadata["gmd:MD_Metadata"]; + } const md: string[] = []; // Title @@ -225,18 +228,21 @@ export function iso19139ToMarkdown(metadata: any) { // Get attribution from ISO 19139 metadata export function getAttribution(metadata: any): string | null { + if ("gmd:MD_Metadata" in metadata) { + metadata = metadata["gmd:MD_Metadata"]; + } + const contact = metadata?.["gmd:contact"]?.[0]; + + const responsibleParty = contact?.["gmd:CI_ResponsibleParty"]?.[0]; // Try to retrieve the responsible organization const organization = - metadata?.gmd?.contact?.[0]?.CI_ResponsibleParty?.organisationName - ?.CharacterString?.[0]; + responsibleParty["gmd:organisationName"]?.[0]["gco:CharacterString"]?.[0]; if (organization) { return organization; } - // Fallback to other contact organization info if available - const contactOrg = - metadata?.gmd?.identificationInfo?.[0]?.MD_DataIdentification?.citation - ?.CI_Citation?.citedResponsibleParty?.[0]?.CI_ResponsibleParty - ?.organisationName?.CharacterString?.[0]; - return contactOrg || null; + // Fallback to other contact individual info if available + const individual = + responsibleParty?.["gmd:individualName"]?.[0]["gco:CharacterString"]?.[0]; + return individual || null; } diff --git a/packages/metadata-parser/src/iso19139ToProseMirror.ts b/packages/metadata-parser/src/iso19139ToProseMirror.ts new file mode 100644 index 000000000..c908a6eed --- /dev/null +++ b/packages/metadata-parser/src/iso19139ToProseMirror.ts @@ -0,0 +1,409 @@ +// Helper function to safely access arrays +const getFirst = (value: any) => + Array.isArray(value) && value.length > 0 ? value[0] : ""; + +// Function to create a text node +function createTextNode(content: string) { + return { + type: "text", + text: content, + }; +} + +// Function to create a paragraph node +function createParagraphNode(content: any[]) { + return { + type: "paragraph", + content: content, + }; +} + +// Function to create a heading node +function createHeadingNode(content: any[], level: number) { + return { + type: "heading", + attrs: { level }, + content: content, + }; +} + +// Function to create a table cell node +function createTableCellNode(content: any[]) { + return { + type: "table_cell", + content: [createParagraphNode(content)], + }; +} + +// Function to create a table row node +function createTableRowNode(cells: any[]) { + return { + type: "table_row", + content: cells, + }; +} + +// Function to create a table node +function createTableNode(rows: any[]) { + return { + type: "table", + content: rows, + }; +} + +// Function to create a bullet list node +function createBulletListNode(items: any[]) { + return { + type: "bullet_list", + content: items, + }; +} + +// Function to create list item node +function createListItemNode(content: any[]) { + return { + type: "list_item", + content: [createParagraphNode(content)], + }; +} + +const parseTopicCategories = (categories: any): string => { + const topicCategories = + categories + ?.map((category: any) => category["gmd:MD_TopicCategoryCode"]?.[0]) + .join(", ") || ""; + return topicCategories; +}; + +const parseDataQuality = (dataQualityInfo: any): string => { + const lineage = + dataQualityInfo["gmd:lineage"]?.[0]["gmd:LI_Lineage"]?.[0][ + "gmd:statement" + ]?.[0]["gco:CharacterString"]?.[0] || ""; + const report = dataQualityInfo["gmd:report"]?.[0]; + let dataQuality = ""; + + if (lineage) { + dataQuality += `\n**Lineage:** ${lineage}\n`; + } + + if (report) { + const explanation = + report["gmd:DQ_Element"]?.[0]["gmd:measureDescription"]?.[0][ + "gco:CharacterString" + ]?.[0] || ""; + if (explanation) { + dataQuality += `\n**Quality Report:** ${explanation}\n`; + } + } + + return dataQuality ? `## Data Quality\n${dataQuality}` : ""; +}; + +// Main function to generate ProseMirror nodes from ISO 19139 metadata +export function iso19139ToProseMirror(metadata: any) { + if ("gmd:MD_Metadata" in metadata) { + metadata = metadata["gmd:MD_Metadata"]; + } + + const doc: any = { type: "doc", content: [] }; + + // Title + const title = getFirst( + metadata["gmd:identificationInfo"][0]["gmd:MD_DataIdentification"][0][ + "gmd:citation" + ][0]["gmd:CI_Citation"][0]["gmd:title"][0]["gco:CharacterString"] + ); + // if (title) { + // doc.content.push(createHeadingNode([createTextNode(title)], 1)); + // } + + // Abstract + const abstract = getFirst( + metadata["gmd:identificationInfo"][0]["gmd:MD_DataIdentification"][0][ + "gmd:abstract" + ][0]["gco:CharacterString"] + ); + if (abstract) { + doc.content.push(createHeadingNode([createTextNode("Abstract")], 2)); + doc.content.push(createParagraphNode([createTextNode(abstract)])); + } + + // Keywords + const keywords = metadata["gmd:identificationInfo"][0][ + "gmd:MD_DataIdentification" + ][0]["gmd:descriptiveKeywords"]?.[0]["gmd:MD_Keywords"][0]["gmd:keyword"] + ?.map((kw: any) => kw["gco:CharacterString"][0]) + .join(", "); + if (keywords) { + doc.content.push(createHeadingNode([createTextNode("Keywords")], 2)); + doc.content.push(createParagraphNode([createTextNode(keywords)])); + } + + const topicCategories = + metadata["gmd:identificationInfo"][0]["gmd:MD_DataIdentification"][0][ + "gmd:topicCategory" + ]; + if (topicCategories) { + doc.content.push( + createHeadingNode([createTextNode("Topic Categories")], 2) + ); + doc.content.push( + createParagraphNode([ + createTextNode(parseTopicCategories(topicCategories)), + ]) + ); + } + + // Use Constraints + const resourceConstraints = + metadata["gmd:identificationInfo"][0]["gmd:MD_DataIdentification"][0][ + "gmd:resourceConstraints" + ]; + if (resourceConstraints) { + const useConstraints = resourceConstraints + .map((constraint: any) => + parseConstraints( + constraint["gmd:MD_LegalConstraints"]?.[0] || + constraint["gmd:MD_SecurityConstraints"]?.[0] || + constraint["gmd:MD_Constraints"]?.[0] + ) + ) + .join("\n"); + + if (useConstraints) { + console.log("use constraints", useConstraints); + doc.content.push( + createHeadingNode([createTextNode("Use Constraints")], 2) + ); + doc.content.push(createParagraphNode([createTextNode(useConstraints)])); + } + } + + const dataQualityInfo = + metadata["gmd:dataQualityInfo"]?.[0]["gmd:DQ_DataQuality"]?.[0]; + if (dataQualityInfo) { + const lineage = + dataQualityInfo["gmd:lineage"]?.[0]["gmd:LI_Lineage"]?.[0][ + "gmd:statement" + ]?.[0]["gco:CharacterString"]?.[0] || ""; + const report = dataQualityInfo["gmd:report"]?.[0]; + if (lineage || report) { + doc.content.push(createHeadingNode([createTextNode("Data Quality")], 2)); + if (lineage) { + doc.content.push(createParagraphNode([createTextNode(lineage)])); + } + if (report) { + const explanation = + report["gmd:DQ_Element"]?.[0]["gmd:measureDescription"]?.[0][ + "gco:CharacterString" + ]?.[0] || ""; + if (explanation) { + doc.content.push(createParagraphNode([createTextNode(explanation)])); + } + } + } + } + + // Attribute Information (Table using ProseMirror Table Schema) + const eainfo = getFirst(metadata?.metadata?.eainfo); + const detailed = eainfo?.detailed?.[0] || {}; + const attributes = detailed?.attr || []; + if (attributes.length > 0) { + doc.content.push( + createHeadingNode([createTextNode("Attribute Information")], 2) + ); + + const tableRows: any[] = []; + + // Create header row + const headerRow = createTableRowNode([ + createTableCellNode([createTextNode("Attribute Label")]), + createTableCellNode([createTextNode("Definition")]), + createTableCellNode([createTextNode("Domain")]), + ]); + tableRows.push(headerRow); + + // Create data rows + attributes.forEach((attr: any) => { + const label = getFirst(attr?.attrlabl) || "Unknown"; + const definition = getFirst(attr?.attrdef) || "No definition"; + + let domain = "No domain"; + const attrdomv = getFirst(attr?.attrdomv); + if (attrdomv?.udom) { + domain = getFirst(attrdomv?.udom); + } else if (attrdomv?.rdom) { + const rdommin = getFirst(attrdomv?.rdom?.rdommin); + const rdommax = getFirst(attrdomv?.rdom?.rdommax); + if (rdommin && rdommax) { + domain = `Range: ${rdommin} to ${rdommax}`; + } else if (rdommin || rdommax) { + domain = `Value: ${rdommin || rdommax}`; + } + } + + const dataRow = createTableRowNode([ + createTableCellNode([createTextNode(label)]), + createTableCellNode([createTextNode(definition)]), + createTableCellNode([createTextNode(domain)]), + ]); + tableRows.push(dataRow); + }); + + const tableNode = createTableNode(tableRows); + doc.content.push(tableNode); + } + + // Contact Information + const contact = metadata["gmd:contact"][0]["gmd:CI_ResponsibleParty"][0]; + const contactInfo = parseContact(contact); + const contactInfoContent = []; + doc.content.push( + createHeadingNode([createTextNode("Contact Information")], 2) + ); + if (contactInfo.name) + contactInfoContent.push( + createListItemNode([createTextNode(`Name: ${contactInfo.name}`)]) + ); + if (contactInfo.organization) + contactInfoContent.push( + createListItemNode([ + createTextNode(`Organization: ${contactInfo.organization}`), + ]) + ); + if (contactInfo.position) + contactInfoContent.push( + createListItemNode([createTextNode(`Position: ${contactInfo.position}`)]) + ); + if (contactInfo.phone) + contactInfoContent.push( + createListItemNode([createTextNode(`Phone: ${contactInfo.phone}`)]) + ); + if (contactInfo.address) + contactInfoContent.push( + createListItemNode([createTextNode(`Address: ${contactInfo.address}`)]) + ); + if (contactInfo.city) + contactInfoContent.push( + createListItemNode([createTextNode(`City: ${contactInfo.city}`)]) + ); + if (contactInfo.postalCode) + contactInfoContent.push( + createListItemNode([ + createTextNode(`Postal Code: ${contactInfo.postalCode}`), + ]) + ); + if (contactInfo.country) + contactInfoContent.push( + createListItemNode([createTextNode(`Country: ${contactInfo.country}`)]) + ); + if (contactInfo.email) + contactInfoContent.push( + createListItemNode([createTextNode(`Email: ${contactInfo.email}`)]) + ); + doc.content.push(createBulletListNode(contactInfoContent)); + + return { + title, + doc, + attribution: getAttribution(metadata), + }; +} + +// Get attribution from ISO 19139 metadata +export function getAttribution(metadata: any): string | null { + if ("gmd:MD_Metadata" in metadata) { + metadata = metadata["gmd:MD_Metadata"]; + } + const contact = metadata?.["gmd:contact"]?.[0]; + + const responsibleParty = contact?.["gmd:CI_ResponsibleParty"]?.[0]; + // Try to retrieve the responsible organization + const organization = + responsibleParty["gmd:organisationName"]?.[0]["gco:CharacterString"]?.[0]; + if (organization) { + return organization; + } + + // Fallback to other contact individual info if available + const individual = + responsibleParty?.["gmd:individualName"]?.[0]["gco:CharacterString"]?.[0]; + return individual || null; +} + +// Helper functions for parsing contact and constraints +function parseContact(contact: any) { + const contactInfo = contact["gmd:contactInfo"]?.[0]["gmd:CI_Contact"]?.[0]; + + return { + name: contact["gmd:individualName"]?.[0]["gco:CharacterString"]?.[0] || "", + organization: + contact["gmd:organisationName"]?.[0]["gco:CharacterString"]?.[0] || "", + position: + contact["gmd:positionName"]?.[0]["gco:CharacterString"]?.[0] || "", + phone: + contactInfo?.["gmd:phone"]?.[0]["gmd:CI_Telephone"]?.[0]["gmd:voice"] + ?.map((p: any) => p["gco:CharacterString"]?.[0]) + .join(", ") || "", + address: + contactInfo?.["gmd:address"]?.[0]["gmd:CI_Address"]?.[0][ + "gmd:deliveryPoint" + ] + ?.map((p: any) => p["gco:CharacterString"]?.[0]) + .join(", ") || "", + city: + contactInfo?.["gmd:address"]?.[0]["gmd:CI_Address"]?.[0]["gmd:city"]?.[0][ + "gco:CharacterString" + ]?.[0] || "", + postalCode: + contactInfo?.["gmd:address"]?.[0]["gmd:CI_Address"]?.[0][ + "gmd:postalCode" + ]?.[0]["gco:CharacterString"]?.[0] || "", + country: + contactInfo?.["gmd:address"]?.[0]["gmd:CI_Address"]?.[0][ + "gmd:country" + ]?.[0]["gco:CharacterString"]?.[0] || "", + email: + contactInfo?.["gmd:address"]?.[0]["gmd:CI_Address"]?.[0][ + "gmd:electronicMailAddress" + ] + ?.map((e: any) => e["gco:CharacterString"]?.[0]) + .join(", ") || "", + }; +} + +function parseConstraints(constraints: any): string { + const useLimitations = + constraints["gmd:useLimitation"] + ?.map((limitation: any) => limitation["gco:CharacterString"][0]) + .join("; ") || ""; + const accessConstraints = + constraints["gmd:accessConstraints"] + ?.map( + (constraint: any) => + constraint["gmd:MD_RestrictionCode"]?.[0]?.["$"]?.["codeListValue"] + ) + .join(", ") || ""; + const useConstraints = + constraints["gmd:useConstraints"] + ?.map( + (constraint: any) => + constraint["gmd:MD_RestrictionCode"]?.[0]?.["$"]?.["codeListValue"] + ) + .join(", ") || ""; + + let constraintsSummary = ""; + + if (useLimitations) { + constraintsSummary += `Use Limitations: ${useLimitations}\n`; + } + if (accessConstraints) { + constraintsSummary += `Access Constraints: ${accessConstraints}\n`; + } + if (useConstraints) { + constraintsSummary += `Use Constraints: ${useConstraints}\n`; + } + + return constraintsSummary; +} diff --git a/packages/metadata-parser/src/metadata-parser.ts b/packages/metadata-parser/src/metadata-parser.ts index 4181878ba..2b02250d2 100644 --- a/packages/metadata-parser/src/metadata-parser.ts +++ b/packages/metadata-parser/src/metadata-parser.ts @@ -1,14 +1,11 @@ -import { getAttribution, iso19139ToMarkdown } from "./iso19139"; -import { - esriToMarkdown, - getAttribution as getEsriAttribution, -} from "./esriToMarkdown"; // @ts-ignore import { parseStringPromise } from "xml2js"; +import { fgdcToProseMirror } from "./fgdcToProsemirror"; +import { iso19139ToProseMirror } from "./iso19139ToProseMirror"; export enum MetadataType { ISO19139, - ESRI, + FGDC, } export interface Contact { @@ -23,26 +20,16 @@ export interface Contact { email: string; } -export async function metadataToMarkdown(xmlString: string) { +export async function metadataToProseMirror(xmlString: string) { try { const data = await parseStringPromise(xmlString); - let markdown = ""; - let attribution: string | null = null; - let type = MetadataType.ISO19139; if (data["gmd:MD_Metadata"]) { - markdown = iso19139ToMarkdown(data); - attribution = getAttribution(data); - } else if (data["metadata"]) { - markdown = esriToMarkdown(data); - attribution = getEsriAttribution(data); + return iso19139ToProseMirror(data); + } else if (data["metadata"] && data["metadata"]["idinfo"]) { + return fgdcToProseMirror(data); } else { return null; } - return { - type, - markdown, - attribution, - }; } catch (error: any) { throw new Error(`Error processing XML: ${error.message}`); } diff --git a/packages/metadata-parser/tsconfig.json b/packages/metadata-parser/tsconfig.json index ca050f0ee..19c2cbd4d 100644 --- a/packages/metadata-parser/tsconfig.json +++ b/packages/metadata-parser/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES2023", + "target": "ES5", "declaration": true, "outDir": "./dist", "strict": true,