Skip to content

Commit

Permalink
Adjust transformation and add tests for hebis RPB-225
Browse files Browse the repository at this point in the history
  • Loading branch information
TobiasNx committed Dec 3, 2024
1 parent 39d4908 commit d52a7b6
Show file tree
Hide file tree
Showing 20 changed files with 1,857 additions and 10 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ bin
.settings
/conf/maps/beacons/*.tsv
/conf/output/*
!/conf/output/test-output-*.json
!/conf/output/test*.json
!/conf/output/rpb-50*
/data/*
!/data/.empty
Expand Down
2 changes: 2 additions & 0 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ libraryDependencies ++= Seq(
"com.typesafe.play" % "play-test_2.11" % "2.4.11",
"org.metafacture" % "metafacture-elasticsearch" % "5.7.0",
"org.metafacture" % "metafacture-io" % "5.7.0",
"org.metafacture" % "metafacture-xml" % "5.7.0",
"org.metafacture" % "metafacture-biblio" % "5.7.0",
"org.metafacture" % "metafacture-strings" % "5.7.0",
"org.metafacture" % "metafacture-json" % "5.7.0",
"org.metafacture" % "metafacture-flux" % "5.7.0",
Expand Down
14 changes: 9 additions & 5 deletions conf/fetchAndTransformHebisRecord.flux
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
SRUQUERRY = "http://sru.hebis.de/sru/DB=2.1?query=pica.ort+%3D+%22Mainz%22+and+pica.ppn+%3D+%22524204101%22&version=1.1&operation=searchRetrieve&stylesheet=http%3A%2F%2Fsru.hebis.de%2Fsru%2F%3Fxsl%3DsearchRetrieveResponse&recordSchema=marc21&maximumRecords=10&startRecord=1&recordPacking=xml&sortKeys=LST_Y%2Cpica%2C0%2C%2C";
createEndTime = "1";
default IDS = FLUX_DIR + "hebisTestIds.txt";
createEndTime = "1"; //needed for lobid transformation

SRUQUERRY
IDS
| open-file
| as-lines
| match(pattern="^(.*)$", replacement="http://sru.hebis.de/sru/DB=2.1?query=pica.ppn+%3D+%22$1%22&version=1.1&operation=searchRetrieve&stylesheet=http%3A%2F%2Fsru.hebis.de%2Fsru%2F%3Fxsl%3DsearchRetrieveResponse&recordSchema=marc21&maximumRecords=10&startRecord=1&recordPacking=xml&sortKeys=LST_Y%2Cpica%2C0%2C%2C")
| open-http(accept="application/xml")
| decode-xml
| handle-marcxml
| fix(FLUX_DIR + "lobid-transformation/marcToLobid.fix",*)
| batch-reset(batchsize="1")
| encode-json(prettyPrinting="true")
| print
;
| write(FLUX_DIR + "output/test-hebis-to-lobid-output-${i}.json")
;
14 changes: 14 additions & 0 deletions conf/hebisTestIds.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
514226781
512839662
512013683
512849919
516707701
475390555
488844266
487803779
478607776
090680634
090358163
090363000
508037980
487803779
8 changes: 4 additions & 4 deletions conf/lobid-transformation/fix/maps.fix
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
put_filemap("./maps/maps-institutions.tsv","alma-iz-code-to-isil", sep_char:"\t",key_column:"2",value_column:"4",expected_columns:"-1")
put_filemap("./maps/maps-institutions.tsv","alma-institution-code-to-isil", sep_char:"\t",key_column:"3",value_column:"4",expected_columns:"-1")

put_filemap("./maps/sublibraryIsil","sublibraryIsil", sep_char:"\t")
put_filemap("./maps/suppressedLocations","suppressedLocations", sep_char:"\t")
put_filemap("./maps/sublibraryIsil.tsv","sublibraryIsil", sep_char:"\t")
put_filemap("./maps/suppressedLocations.tsv","suppressedLocations", sep_char:"\t")

# lobid Organisations id -> label
put_filemap("./maps/lobidOrganisationsMapping.tsv","lobidOrgLabels", sep_char:"\t",key_column:"0",value_column:"3",expected_columns:"-1")
Expand All @@ -19,8 +19,8 @@ put_filemap("./maps/nwbibWikidataLabelTypeCoords.tsv","nwbibWikidataTypes", sep_
put_filemap("./maps/nwbibWikidataLabelTypeCoords.tsv","nwbibWikidataGeodata", sep_char:"\t",key_column:"0",value_column:"4",expected_columns:"-1")

# DDC & DNB-Sachgruppen
put_filemap("./maps/deweyLabels","deweyLabels",sep_char:"\t")
put_filemap("./maps/dnbSachgruppen","dnbSachgruppen",sep_char:"\t")
put_filemap("./maps/deweyLabels.tsv","deweyLabels",sep_char:"\t")
put_filemap("./maps/dnbSachgruppen.tsv","dnbSachgruppen",sep_char:"\t")

# Library Classifications
put_filemap("./maps/classification.tsv","classificationCode2Label", sep_char:"\t",key_column:"0",value_column:"1",expected_columns:"-1")
Expand Down
100 changes: 100 additions & 0 deletions conf/output/test-hebis-to-lobid-output-0.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
{
"@context" : "http://lobid.org/resources/context.jsonld",
"id" : "http://lobid.org/resources/514226781#!",
"type" : [ "BibliographicResource", "Festschrift", "Book" ],
"medium" : [ {
"label" : "Print",
"id" : "http://rdaregistry.info/termList/RDAproductionMethod/1010"
} ],
"title" : "Wie wir wurden was wir sind - 10-jähriges Jubiläum des Mehrgenerationenhauses Ingelheim",
"almaMmsId" : "514226781",
"oclcNumber" : [ "1415743560" ],
"publication" : [ {
"startDate" : "2019",
"type" : [ "PublicationEvent" ],
"location" : [ "Ingelheim am Rhein" ],
"publishedBy" : [ "Stadtverwaltung Ingelheim am Rhein" ]
} ],
"describedBy" : {
"id" : "http://lobid.org/resources/514226781",
"label" : "Webseite der hbz-Ressource 514226781",
"type" : [ "BibliographicDescription" ],
"inDataset" : {
"id" : "http://lobid.org/resources/dataset#!",
"label" : "lobid-resources – Der hbz-Verbundkatalog als Linked Open Data"
},
"resultOf" : {
"type" : [ "CreateAction" ],
"endTime" : "2024-12-03T10:18:30",
"instrument" : {
"id" : "https://github.com/hbz/lobid-resources",
"type" : [ "SoftwareApplication" ],
"label" : "Software lobid-resources"
},
"object" : {
"id" : "https://lobid.org/marcxml/514226781",
"type" : [ "DataFeedItem" ],
"label" : "hbz-Ressource 514226781 im Exportformat MARC21 XML",
"inDataset" : {
"id" : "https://datahub.io/dataset/hbz_unioncatalog",
"label" : "hbz_unioncatalog"
},
"sourceOrganization" : {
"id" : "http://lobid.org/organisations/DE-603#!",
"label" : "Hessisches BibliotheksInformationsSystem hebis"
},
"provider" : {
"id" : "http://lobid.org/organisations/DE-603#!",
"label" : "Hessisches BibliotheksInformationsSystem hebis"
},
"modifiedBy" : [ {
"id" : "http://lobid.org/organisations/DE-603#!",
"label" : "Hessisches BibliotheksInformationsSystem hebis"
} ]
}
},
"license" : [ {
"id" : "http://creativecommons.org/publicdomain/zero/1.0",
"label" : "Creative Commons-Lizenz CC0 1.0 Universal"
} ]
},
"sameAs" : [ {
"id" : "http://worldcat.org/oclc/1415743560",
"label" : "OCLC Ressource"
} ],
"language" : [ {
"id" : "http://id.loc.gov/vocabulary/iso639-2/ger",
"label" : "Deutsch"
} ],
"extent" : "62 Seiten : Illustrationen",
"natureOfContent" : [ {
"label" : "Festschrift",
"id" : "https://d-nb.info/gnd/4016928-5"
} ],
"bibliographicLevel" : {
"label" : "Monograph/Item",
"id" : "https://www.loc.gov/marc/bibliographic/bdleader.html#Monograph_Item"
},
"responsibilityStatement" : [ "Redaktionsteam: Anke Caltapanides [und 3 weitere] ; Herausgeber: Stadtverwaltung Ingelheim am Rhein" ],
"contribution" : [ {
"agent" : {
"label" : "Caltapanides, Anke",
"type" : [ "Person" ]
},
"role" : {
"id" : "http://id.loc.gov/vocabulary/relators/edt"
},
"type" : [ "Contribution" ]
}, {
"agent" : {
"gndIdentifier" : "123750354X",
"id" : "https://d-nb.info/gnd/123750354X",
"label" : "Ingelheim am Rhein. Stadtverwaltung",
"type" : [ "CorporateBody" ]
},
"role" : {
"id" : "http://id.loc.gov/vocabulary/relators/isb"
},
"type" : [ "Contribution" ]
} ]
}
36 changes: 36 additions & 0 deletions conf/output/test-hebis-to-lobid-output-1.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"@context" : "http://lobid.org/resources/context.jsonld",
"id" : "http://lobid.org/resources/#!",
"type" : [ "BibliographicResource", "Miscellaneous" ],
"medium" : [ {
"label" : "Sonstige",
"id" : "http://purl.org/lobid/lv#Miscellaneous"
} ],
"describedBy" : {
"type" : [ "BibliographicDescription" ],
"inDataset" : {
"id" : "http://lobid.org/resources/dataset#!",
"label" : "lobid-resources – Der hbz-Verbundkatalog als Linked Open Data"
},
"resultOf" : {
"type" : [ "CreateAction" ],
"endTime" : "2024-12-03T10:18:31",
"instrument" : {
"id" : "https://github.com/hbz/lobid-resources",
"type" : [ "SoftwareApplication" ],
"label" : "Software lobid-resources"
},
"object" : {
"type" : [ "DataFeedItem" ],
"inDataset" : {
"id" : "https://datahub.io/dataset/hbz_unioncatalog",
"label" : "hbz_unioncatalog"
}
}
},
"license" : [ {
"id" : "http://creativecommons.org/publicdomain/zero/1.0",
"label" : "Creative Commons-Lizenz CC0 1.0 Universal"
} ]
}
}
134 changes: 134 additions & 0 deletions conf/output/test-hebis-to-lobid-output-10.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
{
"@context" : "http://lobid.org/resources/context.jsonld",
"id" : "http://lobid.org/resources/090358163#!",
"type" : [ "BibliographicResource", "Book" ],
"medium" : [ {
"label" : "Print",
"id" : "http://rdaregistry.info/termList/RDAproductionMethod/1010"
} ],
"title" : "Mainz transparent",
"almaMmsId" : "090358163",
"isbn" : [ "3874394883", "9783874394888" ],
"oclcNumber" : [ "76115483" ],
"dnbId" : "958853371",
"otherTitleInformation" : [ "2000 Jahre einer Stadt ; [Stadtbegleiter]" ],
"publication" : [ {
"startDate" : "2000",
"type" : [ "PublicationEvent" ],
"location" : [ "Mainz" ],
"publishedBy" : [ "Schmidt" ]
} ],
"describedBy" : {
"id" : "http://lobid.org/resources/090358163",
"label" : "Webseite der hbz-Ressource 090358163",
"type" : [ "BibliographicDescription" ],
"inDataset" : {
"id" : "http://lobid.org/resources/dataset#!",
"label" : "lobid-resources – Der hbz-Verbundkatalog als Linked Open Data"
},
"resultOf" : {
"type" : [ "CreateAction" ],
"endTime" : "2024-12-03T10:18:33",
"instrument" : {
"id" : "https://github.com/hbz/lobid-resources",
"type" : [ "SoftwareApplication" ],
"label" : "Software lobid-resources"
},
"object" : {
"id" : "https://lobid.org/marcxml/090358163",
"type" : [ "DataFeedItem" ],
"label" : "hbz-Ressource 090358163 im Exportformat MARC21 XML",
"inDataset" : {
"id" : "https://datahub.io/dataset/hbz_unioncatalog",
"label" : "hbz_unioncatalog"
},
"sourceOrganization" : {
"id" : "http://lobid.org/organisations/DE-603#!",
"label" : "Hessisches BibliotheksInformationsSystem hebis"
},
"provider" : {
"id" : "http://lobid.org/organisations/DE-603#!",
"label" : "Hessisches BibliotheksInformationsSystem hebis"
},
"modifiedBy" : [ {
"id" : "http://lobid.org/organisations/DE-603#!",
"label" : "Hessisches BibliotheksInformationsSystem hebis"
} ]
}
},
"license" : [ {
"id" : "http://creativecommons.org/publicdomain/zero/1.0",
"label" : "Creative Commons-Lizenz CC0 1.0 Universal"
} ]
},
"sameAs" : [ {
"id" : "http://worldcat.org/oclc/76115483",
"label" : "OCLC Ressource"
}, {
"id" : "https://d-nb.info/958853371",
"label" : "DNB-Ressource"
} ],
"language" : [ {
"id" : "http://id.loc.gov/vocabulary/iso639-2/ger",
"label" : "Deutsch"
} ],
"extent" : "127 S. : zahlr. Ill., Kt.",
"natureOfContent" : [ {
"label" : "Führer",
"id" : "https://d-nb.info/gnd/4155569-7"
} ],
"subject" : [ {
"type" : [ "Concept" ],
"source" : {
"label" : "Gemeinsame Normdatei (GND)",
"id" : "https://d-nb.info/gnd/7749153-1"
},
"label" : "Mainz",
"id" : "https://d-nb.info/gnd/4037124-4",
"gndIdentifier" : "4037124-4"
}, {
"notation" : "61",
"type" : [ "Concept" ],
"source" : {
"label" : "Systematik der DNB (bis 2003)",
"id" : "https://bartoc.org/en/node/18497"
}
}, {
"notation" : "19.1d",
"type" : [ "Concept" ]
} ],
"subjectslabels" : [ "Mainz" ],
"bibliographicLevel" : {
"label" : "Monograph/Item",
"id" : "https://www.loc.gov/marc/bibliographic/bdleader.html#Monograph_Item"
},
"responsibilityStatement" : [ "hrsg. von Georg Schuler ... [Konzept und Texte: Frank Herda ... Fotos: Philippa Pfahler ...]" ],
"contribution" : [ {
"agent" : {
"label" : "Schuler, Georg",
"type" : [ "Person" ]
},
"role" : {
"id" : "http://id.loc.gov/vocabulary/relators/edt"
},
"type" : [ "Contribution" ]
}, {
"agent" : {
"label" : "Herda, Frank",
"type" : [ "Person" ]
},
"role" : {
"id" : "http://id.loc.gov/vocabulary/relators/ctb"
},
"type" : [ "Contribution" ]
}, {
"agent" : {
"label" : "Pfahler, Philippa",
"type" : [ "Person" ]
},
"role" : {
"id" : "http://id.loc.gov/vocabulary/relators/ctb"
},
"type" : [ "Contribution" ]
} ]
}
Loading

0 comments on commit d52a7b6

Please sign in to comment.