Set up RPPD transformation and Strapi import (RPB-81)

.gitignore

@@ -14,3 +14,5 @@ application-log-*.gz
 RPB-Export_HBZ_SW.txt
 RPB-Export_HBZ_Tit.txt
 RPB-Export_HBZ_Tit_hbzIds.txt
+RPB-Export_HBZ_Bio.txt
+nohup.out*

README.md

@@ -71,6 +71,22 @@ sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb.ndjson PATH=rpb-n
 sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-spatial.ndjson PATH=rpb-spatials"
 ```
 
+### Run RPPD transformation to strapi data
+
+```bash
+sbt "runMain rpb.ETL conf/rppd-to-strapi.flux"
+```
+
+Writes output to `output/output-rppd-strapi.ndjson`.
+
+### Import RPPD strapi data
+
+```bash
+sbt "runMain rpb.ETL conf/rppd-import.flux"
+```
+
+This attempts to import RPPD data to strapi, and prints the server responses.
+
 ### Run transformation to lobid data
 
 ```bash

conf/rppd-import.flux

@@ -0,0 +1,15 @@
+//API_URL = "http://test-metadaten-nrw.hbz-nrw.de:1339/api/rppds";
+API_URL = "http://localhost:1339/api/rppds";
+
+// FLUX_DIR + "output/output-rppd-strapi.ndjson"
+FLUX_DIR + "output/test-output-rppd.json"
+| open-file
+| as-lines
+| regex-decode("(?<data>.+)")
+| stream-to-triples
+| template("{\"${p}\":${o}}") // wrap into 'data' object for strapi
+| log-object("Will POST: ")
+| open-http(url=API_URL, method="POST", contentType="application/json")
+| as-lines
+| log-object("POST Response: ")
+;

conf/rppd-to-strapi.fix

@@ -0,0 +1,63 @@
+do put_macro("to_bool")
+  if exists("$[field]")
+    move_field("$[field]","$[field]~")
+    set_field("$[field]~", "true")
+  end
+end
+
+do put_macro("add")
+  if exists("$[from]")
+    set_array("_temp")
+    copy_field("$[from]", "_temp.$append.$[to]")
+    move_field("_temp.*", "$[to][].$append")
+    remove_field("$[from]")
+  end
+end
+
+do put_macro("to_array")
+  split_field("$[field]", "@")
+  set_array("$[field][]")
+  call_macro("add", to: "$[field]", from: "$[field]")
+end
+
+do put_macro("add_bio")
+  set_array("_temp")
+  move_field("$[field]", "_temp.$append") # aufgeteilt: f1?1..9
+  join_field("_temp", "\n\n") # als Absätze...
+  move_field("_temp", "f1z1[].$append.f1z1") # in 1 Feld
+end
+
+call_macro("to_bool", field: "f11_")
+call_macro("to_bool", field: "f13_")
+call_macro("to_bool", field: "f14_")
+call_macro("to_bool", field: "f15_")
+
+call_macro("to_array", field: "f1nc")
+call_macro("to_array", field: "f1nd")
+call_macro("to_array", field: "f1nn")
+call_macro("to_array", field: "f1no")
+call_macro("to_array", field: "f1nt")
+call_macro("to_array", field: "f1nu")
+call_macro("to_array", field: "f1nw")
+call_macro("to_array", field: "f1nx")
+
+set_array("f1z1[]") # Biogramme
+call_macro("add_bio", field: "f1z?")
+call_macro("add_bio", field: "f1ü?")
+call_macro("add_bio", field: "f1ä?")
+call_macro("add_bio", field: "f1ö?")
+call_macro("add_bio", field: "f1ß?")
+
+# f82b ist 'required' und 'unique'
+if all_match("f82b", "Familienmitglied|Keine GND-Ansetzung")
+  paste("f82b", "~Keine GND-Ansetzung für", "f00_")
+end
+unless exists("f82b")
+  paste("f82b", "~Keine GND-Ansetzung für", "f00_")
+end
+replace_all("f82b", "(\\d+)n(\\d)", "$1-$2")
+
+replace_all("f1nv", "w+", "www") # z.B. ww -> www
+replace_all("f1nv", "m+", "mmm") # z.B. mmmm -> mmm
+
+vacuum()

conf/rppd-to-strapi.flux

@@ -0,0 +1,13 @@
+// default outfile = "conf/output/output-rppd-strapi.ndjson";
+default outfile = "conf/output/test-output-rppd.json";
+
+// wget http://lobid.org/download/rpb-gesamtexport/2023-06-01/RPB-Export_HBZ_Bio.txt
+// FLUX_DIR + "RPB-Export_HBZ_Bio.txt"
+FLUX_DIR + "RPB-Export_HBZ_Bio_Test.txt"
+| open-file(encoding="IBM437")
+| as-lines
+| rpb.Decode
+| fix(FLUX_DIR + "rppd-to-strapi.fix")
+| encode-json(prettyPrinting="false", booleanMarker="~")
+| write(outfile)
+;