diff --git a/conf/rpb-sw-import.flux b/conf/rpb-sw-import.flux index 7178d214..b7858109 100644 --- a/conf/rpb-sw-import.flux +++ b/conf/rpb-sw-import.flux @@ -1,8 +1,8 @@ -//API_URL = "http://test-metadaten-nrw.hbz-nrw.de:1339/api/rpb-authorities"; -API_URL = "http://localhost:1339/api/rpb-authorities"; +default IN_FILE = "test-output-sw.json"; // pass e.g. IN_FILE=output-sw-strapi.ndjson +default HOST = "localhost"; // pass e.g. HOST=test-metadaten-nrw.hbz-nrw.de +API_URL = "http://" + HOST + ":1339/api/rpb-authorities"; -// FLUX_DIR + "output/output-sw-strapi.ndjson" -FLUX_DIR + "output/test-output-sw.json" +FLUX_DIR + "output/" + IN_FILE | open-file | as-lines | regex-decode("(?.+)") diff --git a/conf/rpb-sw-to-strapi.flux b/conf/rpb-sw-to-strapi.flux index 89ee43af..68fbd7d7 100644 --- a/conf/rpb-sw-to-strapi.flux +++ b/conf/rpb-sw-to-strapi.flux @@ -1,13 +1,12 @@ -// default outfile = "conf/output/output-sw-strapi.ndjson"; -default outfile = "conf/output/test-output-sw.json"; +default OUT_FILE = "test-output-sw.json"; // pass e.g. OUT_FILE=output-sw-strapi.ndjson +default IN_FILE = "RPB-Export_HBZ_SW_Test.txt"; // pass e.g. IN_FILE=RPB-Export_HBZ_SW.txt +// (wget http://lobid.org/download/rpb-gesamtexport/2023-06-01/RPB-Export_HBZ_SW.txt) -// wget http://lobid.org/download/rpb-gesamtexport/2023-06-01/RPB-Export_HBZ_SW.txt -// FLUX_DIR + "RPB-Export_HBZ_SW.txt" -FLUX_DIR + "RPB-Export_HBZ_SW_Test.txt" +FLUX_DIR + IN_FILE | open-file(encoding="IBM437") | as-lines | rpb.Decode | fix(FLUX_DIR + "rpb-sw-to-strapi.fix") | encode-json(prettyPrinting="false", booleanMarker="~") -| write(outfile) +| write(FLUX_DIR + "output/" + OUT_FILE) ; diff --git a/conf/rpb-systematik-import.flux b/conf/rpb-systematik-import.flux index db38cd14..a71f59c2 100644 --- a/conf/rpb-systematik-import.flux +++ b/conf/rpb-systematik-import.flux @@ -1,5 +1,5 @@ -// API_URL = "http://test-metadaten-nrw.hbz-nrw.de:1339/api/" + PATH; -API_URL = "http://localhost:1339/api/" + PATH; +default HOST = "localhost"; // pass e.g. HOST=test-metadaten-nrw.hbz-nrw.de +API_URL = "http://" + HOST + ":1339/api/" + PATH; "https://raw.githubusercontent.com/acka47/scripts/master/skos2json/" + INPUT | open-http diff --git a/conf/rppd-import.flux b/conf/rppd-import.flux index 14841285..51c7cffa 100644 --- a/conf/rppd-import.flux +++ b/conf/rppd-import.flux @@ -1,8 +1,8 @@ -//API_URL = "http://test-metadaten-nrw.hbz-nrw.de:1339/api/rppds"; -API_URL = "http://localhost:1339/api/rppds"; +default HOST = "localhost"; // pass e.g. HOST=test-metadaten-nrw.hbz-nrw.de +default IN_FILE = "test-output-rppd.json"; // pass e.g. IN_FILE=output-rppd-strapi.ndjson +API_URL = "http://" + HOST + ":1339/api/rppds"; -// FLUX_DIR + "output/output-rppd-strapi.ndjson" -FLUX_DIR + "output/test-output-rppd.json" +FLUX_DIR + "output/" + IN_FILE | open-file | as-lines | regex-decode("(?.+)") diff --git a/conf/rppd-to-strapi.flux b/conf/rppd-to-strapi.flux index 20162cb2..c0b6d173 100644 --- a/conf/rppd-to-strapi.flux +++ b/conf/rppd-to-strapi.flux @@ -1,13 +1,12 @@ -// default outfile = "conf/output/output-rppd-strapi.ndjson"; -default outfile = "conf/output/test-output-rppd.json"; +default OUT_FILE = "test-output-rppd.json"; // pass e.g. OUT_FILE=output-rppd-strapi.ndjson +default IN_FILE = "RPB-Export_HBZ_Bio_Test.txt"; // pass e.g. IN_FILE=RPB-Export_HBZ_Bio.txt +// (wget http://lobid.org/download/rpb-gesamtexport/2023-06-01/RPB-Export_HBZ_Bio.txt) -// wget http://lobid.org/download/rpb-gesamtexport/2023-06-01/RPB-Export_HBZ_Bio.txt -// FLUX_DIR + "RPB-Export_HBZ_Bio.txt" -FLUX_DIR + "RPB-Export_HBZ_Bio_Test.txt" +FLUX_DIR + IN_FILE | open-file(encoding="IBM437") | as-lines | rpb.Decode | fix(FLUX_DIR + "rppd-to-strapi.fix") | encode-json(prettyPrinting="false", booleanMarker="~") -| write(outfile) +| write(FLUX_DIR + "output/" + OUT_FILE) ; diff --git a/transformAndImport.sh b/transformAndImport.sh index dd67e988..bc82fb4e 100644 --- a/transformAndImport.sh +++ b/transformAndImport.sh @@ -3,16 +3,16 @@ set -eu IFS=$'\n\t' # Notationen -sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb.ndjson PATH=rpb-notations" -sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-spatial.ndjson PATH=rpb-spatials" +sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb.ndjson PATH=rpb-notations HOST=localhost" +sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-spatial.ndjson PATH=rpb-spatials HOST=localhost" # Personen -sbt "runMain rpb.ETL conf/rppd-to-strapi.flux" -sbt "runMain rpb.ETL conf/rppd-import.flux" +sbt "runMain rpb.ETL conf/rppd-to-strapi.flux IN_FILE=RPB-Export_HBZ_Bio_Test.txt OUT_FILE=test-output-rppd.json" +sbt "runMain rpb.ETL conf/rppd-import.flux IN_FILE=test-output-rppd.json HOST=localhost" # Normdaten -sbt "runMain rpb.ETL conf/rpb-sw-to-strapi.flux" -sbt "runMain rpb.ETL conf/rpb-sw-import.flux" +sbt "runMain rpb.ETL conf/rpb-sw-to-strapi.flux IN_FILE=RPB-Export_HBZ_SW_Test.txt OUT_FILE=test-output-sw.json" +sbt "runMain rpb.ETL conf/rpb-sw-import.flux IN_FILE=test-output-sw.json HOST=localhost" # Titeldaten sbt "runMain rpb.ETL conf/rpb-test-titel-to-strapi.flux"