Skip to content

Commit

Permalink
Improve intermediate ttl output and symbols generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@fkleedorfer
fkleedorfer committed Feb 14, 2024
1 parent a3b0bee commit 26b1e07
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 26 deletions.
25 changes: 20 additions & 5 deletions qudtlib-common-rdf/src/main/java/io/github/qudtlib/common/RdfOps.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,16 @@
import java.util.ArrayList;
import java.util.List;
import org.eclipse.rdf4j.model.Model;
import org.eclipse.rdf4j.model.Namespace;
import org.eclipse.rdf4j.model.Resource;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.impl.LinkedHashModel;
import org.eclipse.rdf4j.query.GraphQueryResult;
import org.eclipse.rdf4j.query.QueryResults;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.repository.RepositoryResult;
import org.eclipse.rdf4j.rio.RDFFormat;
import org.eclipse.rdf4j.rio.RDFParser;
import org.eclipse.rdf4j.rio.RDFWriter;
import org.eclipse.rdf4j.rio.Rio;
import org.eclipse.rdf4j.rio.*;
import org.eclipse.rdf4j.rio.helpers.BasicWriterSettings;
import org.eclipse.rdf4j.rio.helpers.StatementCollector;

/**
Expand All @@ -36,7 +35,15 @@ public static void writeTurtleFile(RepositoryConnection con, Path outfile) {
System.out.println("writing RDF data to file " + outfile.toFile().getAbsolutePath());
try (FileOutputStream out = new FileOutputStream(outfile.toFile())) {
RDFWriter writer = Rio.createWriter(RDFFormat.TURTLE, out);
WriterConfig config =
writer.getWriterConfig()
.set(BasicWriterSettings.PRETTY_PRINT, true)
.set(BasicWriterSettings.INLINE_BLANK_NODES, true);
writer.setWriterConfig(config);
writer.startRDF();
try (RepositoryResult<Namespace> namespaces = con.getNamespaces()) {
namespaces.forEach(n -> writer.handleNamespace(n.getPrefix(), n.getName()));
}
try (RepositoryResult<Statement> statements =
con.getStatements(null, null, null, (Resource) null)) {
for (Statement st : statements) {
Expand All @@ -53,7 +60,13 @@ public static void writeTurtleFile(Model model, Path outfile) {
System.out.println("writing RDF data to file " + outfile.toFile().getAbsolutePath());
try (FileOutputStream out = new FileOutputStream(outfile.toFile())) {
RDFWriter writer = Rio.createWriter(RDFFormat.TURTLE, out);
WriterConfig config =
writer.getWriterConfig()
.set(BasicWriterSettings.PRETTY_PRINT, true)
.set(BasicWriterSettings.INLINE_BLANK_NODES, true);
writer.setWriterConfig(config);
writer.startRDF();
model.getNamespaces().forEach(n -> writer.handleNamespace(n.getPrefix(), n.getName()));
for (Statement st : model.getStatements(null, null, null, (Resource) null)) {
writer.handleStatement(st);
}
Expand Down Expand Up @@ -108,7 +121,9 @@ public static void updateDataUsingNQueries(
}

public static void addStatementsFromFile(RepositoryConnection con, String filename) {
con.add(loadTurtleToModel(filename));
Model model = loadTurtleToModel(filename);
con.add(model);
model.getNamespaces().forEach(ns -> con.setNamespace(ns.getPrefix(), ns.getName()));
con.commit();
}

Expand Down
18 changes: 13 additions & 5 deletions qudtlib-data-gen/src/main/java/io/github/qudtlib/data/DataGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,12 @@
import java.nio.file.Path;
import java.util.Collection;
import java.util.stream.Collectors;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Model;
import org.eclipse.rdf4j.model.Resource;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.*;
import org.eclipse.rdf4j.model.impl.LinkedHashModel;
import org.eclipse.rdf4j.model.util.Models;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.RepositoryConnection;
import org.eclipse.rdf4j.repository.RepositoryResult;
import org.eclipse.rdf4j.repository.sail.SailRepository;
import org.eclipse.rdf4j.repository.util.RepositoryUtil;
import org.eclipse.rdf4j.sail.memory.MemoryStore;
Expand Down Expand Up @@ -129,7 +127,6 @@ private void processQuantityKinds() {
RdfOps.removeStatementsFromFile(outputCon, DELETE_FROM_QUANTITYKINDS);
// add missing triples
RdfOps.addStatementsFromFile(outputCon, ADD_TO_QUANTITYKINDS);

RdfOps.writeTurtleFile(outputCon, outFile(QUANTITYKINDS_OUTFILE));
}
}
Expand Down Expand Up @@ -167,6 +164,7 @@ void processUnits() {
// we generate some units in the above, add basic unit info for those, write to
// INPUT and OUTPUT repos
RdfOps.addDataUsingQuery(inputCon, MISSING_UNITS_QUERY, inputCon, outputCon);
copyNamespaces(inputCon, outputCon);
// write units file from OUTPUT repo
RdfOps.writeTurtleFile(outputCon, outFile(UNITS_OUTFILE));
// comparison with expected is useful during version upgrades, keep it commented out
Expand All @@ -176,6 +174,16 @@ void processUnits() {
}
}

private static void copyNamespaces(RepositoryConnection fromCon, RepositoryConnection toCon) {
try (RepositoryResult<Namespace> namespaces = fromCon.getNamespaces()) {
namespaces.stream()
.forEach(
n -> {
toCon.setNamespace(n.getPrefix(), n.getName());
});
}
}

private void compareWithExpected(RepositoryConnection outputCon) {
if (Thread.currentThread().getContextClassLoader().getResourceAsStream(UNITS_EXPECTED_DATA)
== null) {
Expand Down
19 changes: 11 additions & 8 deletions qudtlib-tools/src/main/java/io/github/qudtlib/tools/contributions/CheckSymbols.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.io.PrintStream;
import java.lang.reflect.Field;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand All @@ -30,18 +31,12 @@ public static void main(String[] args) {
Qudt.allUnits().stream()
.filter(u -> !u.isCurrencyUnit() && !u.isDeprecated())
.filter(u -> !u.isGenerated())
.filter(
u ->
u.getIriLocalname()
.contains("-")) // units that don't
// contain a hyphen cannot
// profit from this
// algorithm
.filter(isLikelyDerivedUnit())
.sorted(Comparator.comparing(u -> u.getIri()))
.collect(Collectors.toList());
int correctUnits = -1;
Qudt.allUnits().stream()
.filter(u -> !u.getIriLocalname().contains("-"))
.filter(isLikelyDerivedUnit().negate())
.forEach(globalData.correctUnits::add);
while (correctUnits < globalData.correctUnits.size()) {
correctUnits = globalData.correctUnits.size();
Expand Down Expand Up @@ -80,6 +75,14 @@ public static void main(String[] args) {
printDeleteQuery(globalData);
}

private static Predicate<Unit> isLikelyDerivedUnit() {
return u ->
u.getIriLocalname().contains("-")
|| (u.getFactorUnits().getLocalname().matches(".+\\d$")
&& u.getFactorUnits().getFactorUnits().size() == 1
&& u.getFactorUnits().getFactorUnits().get(0).getExponent() != 1);
}

private static void printStatements(ByteArrayOutputStream ttlOut) {
System.out.println("STATEMENTS TO ADD:\n\n");
System.out.println(ttlOut.toString());
Expand Down
19 changes: 11 additions & 8 deletions qudtlib-tools/src/main/java/io/github/qudtlib/tools/contributions/CheckUcumCode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.io.PrintStream;
import java.lang.reflect.Field;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
Expand All @@ -31,13 +32,7 @@ public static void main(String[] args) {
Qudt.allUnits().stream()
.filter(u -> !u.isCurrencyUnit() && !u.isDeprecated())
.filter(u -> !u.isGenerated())
.filter(
u ->
u.getIriLocalname()
.contains("-")) // units that don't
// contain a hyphen cannot
// profit from this
// algorithm
.filter(isLikelyDerivedUnit())
.filter(
u ->
!(u.getIriLocalname().equals("P")
Expand All @@ -49,7 +44,7 @@ public static void main(String[] args) {
.collect(Collectors.toList());
int correctUnits = -1;
Qudt.allUnits().stream()
.filter(u -> !u.getIriLocalname().contains("-"))
.filter(isLikelyDerivedUnit().negate())
.forEach(globalData.correctUnits::add);
while (correctUnits < globalData.correctUnits.size()) {
correctUnits = globalData.correctUnits.size();
Expand Down Expand Up @@ -145,6 +140,14 @@ private static void checkUnit(Unit unit, GlobalData globalData, PrintStream ttlP
}
}

private static Predicate<Unit> isLikelyDerivedUnit() {
return u ->
u.getIriLocalname().contains("-")
|| (u.getFactorUnits().getLocalname().matches(".+\\d$")
&& u.getFactorUnits().getFactorUnits().size() == 1
&& u.getFactorUnits().getFactorUnits().get(0).getExponent() != 1);
}

private static void setUcumCode(Unit unit, String calculatedUcumCode) {
try {
Field ucumCodeField = Unit.class.getDeclaredField("ucumCode");
Expand Down

0 comments on commit 26b1e07

Please sign in to comment.