diff --git a/core/src/main/scala/eu/ostrzyciel/jelly/core/ProtoTranscoderImpl.scala b/core/src/main/scala/eu/ostrzyciel/jelly/core/ProtoTranscoderImpl.scala index a461a452..2e04021b 100644 --- a/core/src/main/scala/eu/ostrzyciel/jelly/core/ProtoTranscoderImpl.scala +++ b/core/src/main/scala/eu/ostrzyciel/jelly/core/ProtoTranscoderImpl.scala @@ -183,5 +183,6 @@ private final class ProtoTranscoderImpl( if !emittedOptions then emittedOptions = true rowBuffer.append(RdfStreamRow(outputOptions.copy( - version = Constants.protoVersion + version = if inputOptions.version == Constants.protoVersionNoNsDecl then + Constants.protoVersionNoNsDecl else Constants.protoVersion, ))) diff --git a/core/src/test/scala/eu/ostrzyciel/jelly/core/ProtoTranscoderSpec.scala b/core/src/test/scala/eu/ostrzyciel/jelly/core/ProtoTranscoderSpec.scala index c35f1ae5..cf5074a0 100644 --- a/core/src/test/scala/eu/ostrzyciel/jelly/core/ProtoTranscoderSpec.scala +++ b/core/src/test/scala/eu/ostrzyciel/jelly/core/ProtoTranscoderSpec.scala @@ -202,6 +202,14 @@ class ProtoTranscoderSpec extends AnyWordSpec, Inspectors, Matchers: output(i) shouldBe expectedOutput(i) } + "maintain protocol version 1 if input uses it" in { + val options = JellyOptions.smallStrict.withVersion(Constants.protoVersionNoNsDecl) + val input = RdfStreamRow(options) + val transcoder = ProtoTranscoder.fastMergingTranscoderUnsafe(options.withVersion(Constants.protoVersion)) + val output = transcoder.ingestRow(input) + output.head shouldBe input + } + "throw an exception on a null row" in { val transcoder = ProtoTranscoder.fastMergingTranscoderUnsafe(JellyOptions.smallStrict) val ex = intercept[RdfProtoTranscodingError] { diff --git a/integration-tests/src/test/resources/backcompat/riverbench_main_v1_1_0.jelly b/integration-tests/src/test/resources/backcompat/riverbench_main_v1_1_0.jelly new file mode 100644 index 00000000..059a94e2 Binary files /dev/null and b/integration-tests/src/test/resources/backcompat/riverbench_main_v1_1_0.jelly differ diff --git a/integration-tests/src/test/resources/backcompat/riverbench_nanopubs_v1_1_0.jelly b/integration-tests/src/test/resources/backcompat/riverbench_nanopubs_v1_1_0.jelly new file mode 100644 index 00000000..18b3ebb2 Binary files /dev/null and b/integration-tests/src/test/resources/backcompat/riverbench_nanopubs_v1_1_0.jelly differ diff --git a/integration-tests/src/test/resources/backcompat/weather_quads_v1_1_0.jelly b/integration-tests/src/test/resources/backcompat/weather_quads_v1_1_0.jelly new file mode 100644 index 00000000..ab57a8db Binary files /dev/null and b/integration-tests/src/test/resources/backcompat/weather_quads_v1_1_0.jelly differ diff --git a/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/BackCompatSpec.scala b/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/BackCompatSpec.scala index f2d649b7..967de958 100644 --- a/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/BackCompatSpec.scala +++ b/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/BackCompatSpec.scala @@ -2,25 +2,54 @@ package eu.ostrzyciel.jelly.integration_tests import eu.ostrzyciel.jelly.convert.jena.riot.JellyLanguage import eu.ostrzyciel.jelly.convert.jena.traits.JenaTest +import eu.ostrzyciel.jelly.core.Constants +import eu.ostrzyciel.jelly.core.proto.v1.* import org.apache.jena.riot.{Lang, RDFDataMgr} import org.apache.jena.sparql.core.DatasetGraphFactory import org.scalatest.concurrent.ScalaFutures import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec +import java.io.File import scala.jdk.CollectionConverters.* -class BackCompatSpec extends AnyWordSpec, Matchers, ScalaFutures, JenaTest: - private val testCases = Seq( - ("riverbench_main", "RiverBench main metadata", Seq("v1_0_0")), - ("riverbench_nanopubs", "RiverBench nanopubs dataset metadata", Seq("v1_0_0")), - ("weather_quads", "Jelly 1.0.0, weather data example (QUADS stream)", Seq("v1_0_0")), +object BackCompatSpec: + val descriptions = Map( + "riverbench_main" -> "RiverBench main metadata", + "riverbench_nanopubs" -> "RiverBench nanopubs dataset metadata", + "weather_quads" -> "weather data example (QUADS stream)" + ) + + val versionToNumber = Map( + "v1_0_0" -> 1, + "v1_1_0" -> 2, ) + lazy val testCases: Seq[(String, String, Seq[String])] = + File(getClass.getResource("/backcompat").toURI).listFiles().toSeq + .filter(_.getName.endsWith(".jelly")) + .map(f => (f.getName, f.getName.split("_v").head.split('.').head)) + .groupBy(_._2) + .map { case (name, files) => + val versions = files.map(f => "v" + f._1.split("_v").last.split('.').head).sorted + (name, descriptions(name), versions) + } + .toSeq + +class BackCompatSpec extends AnyWordSpec, Matchers, ScalaFutures, JenaTest: + import BackCompatSpec.* + + val currentVersion = "v" + Constants.protoSemanticVersion.replace(".", "_") + for (fileName, description, versions) <- testCases do - for version <- versions do s"Backward compatibility with $description" should { - s"be maintained for Jelly version $version when parsing" in { + s"be tested with the current version ($currentVersion)" in { + // If this test is failing, it means that you have to update this spec :) + // Go to util/MakeBackCompatTestCases.scala and run it. This should fix it. + versions should contain (currentVersion) + } + + for version <- versions do s"be maintained for Jelly version $version when parsing" in { val jellyDg = DatasetGraphFactory.create() RDFDataMgr.read( jellyDg, @@ -35,4 +64,10 @@ class BackCompatSpec extends AnyWordSpec, Matchers, ScalaFutures, JenaTest: ) CrossStreamingSpec.compareDatasets(jellyDg, jenaDg) } + + for version <- versions do s"work for $version, reading the correct version number from file" in { + val is = getClass.getResourceAsStream(s"/backcompat/${fileName}_$version.jelly") + val opt: RdfStreamOptions = RdfStreamFrame.parseDelimitedFrom(is).get.rows.head.row.options + opt.version should be (versionToNumber(version)) + } } diff --git a/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/ForwardCompatSpec.scala b/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/ForwardCompatSpec.scala index e17a4ee6..c8d294d7 100644 --- a/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/ForwardCompatSpec.scala +++ b/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/ForwardCompatSpec.scala @@ -51,12 +51,12 @@ class ForwardCompatSpec extends AnyWordSpec, Matchers, ScalaFutures, JenaTest: private val futureFrameBytes2: Array[Byte] = futureFrame2.toByteArray "current Jelly version" should { - "be 1" in { + "be 2" in { // If this test is failing, it means that you have to update this spec :) // Go to integration-tests/src/main/protobuf and update the proto file to what you are using now. // Then, reintroduce the "future" changes that are tested here. // You can then update this test to the version number you are using. - Constants.protoVersion should be (1) + Constants.protoVersion should be (2) } } diff --git a/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/io/IoSerDesSpec.scala b/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/io/IoSerDesSpec.scala index c7934f05..811c89d7 100644 --- a/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/io/IoSerDesSpec.scala +++ b/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/io/IoSerDesSpec.scala @@ -85,7 +85,7 @@ class IoSerDesSpec extends AnyWordSpec, Matchers, ScalaFutures, JenaTest: options.maxNameTableSize should be (expOpt.maxNameTableSize) options.maxPrefixTableSize should be (expOpt.maxPrefixTableSize) options.maxDatatypeTableSize should be (expOpt.maxDatatypeTableSize) - options.version should be (Constants.protoVersion) + options.version should be (Constants.protoVersionNoNsDecl) runTest(JenaSerDes, JenaSerDes) runTest(JenaSerDes, JenaStreamSerDes) diff --git a/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/util/MakeBackCompatTestCases.scala b/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/util/MakeBackCompatTestCases.scala new file mode 100644 index 00000000..852dbcef --- /dev/null +++ b/integration-tests/src/test/scala/eu/ostrzyciel/jelly/integration_tests/util/MakeBackCompatTestCases.scala @@ -0,0 +1,39 @@ +package eu.ostrzyciel.jelly.integration_tests.util + +import eu.ostrzyciel.jelly.convert.jena.riot.{JellyFormatVariant, JellyLanguage} +import eu.ostrzyciel.jelly.core.Constants +import eu.ostrzyciel.jelly.integration_tests.BackCompatSpec.testCases +import org.apache.jena.riot.{Lang, RDFDataMgr, RDFFormat} +import org.apache.jena.sparql.core.DatasetGraphFactory + +import java.nio.file.{Files, Path} + +/** + * Utility to generate *.jelly files for later back-compat testing. + * These live in resources/backcompat. + * + * Run this utility after increasing the protocol version in the Constants class. + */ +object MakeBackCompatTestCases: + + @main + def runMakeBackCompatTestCases(): Unit = + val version = Constants.protoSemanticVersion.replace(".", "_") + for (fileName, description, versions) <- testCases do + val jenaDg = DatasetGraphFactory.create() + RDFDataMgr.read( + jenaDg, + getClass.getResourceAsStream(s"/backcompat/$fileName.trig"), + Lang.TRIG + ) + val v2Format = new RDFFormat( + JellyLanguage.JELLY, + // enable this to make this into a Jelly 1.1.0 file + JellyFormatVariant(enableNamespaceDeclarations = true) + ) + RDFDataMgr.write( + Files.newOutputStream(Path.of(s"integration-tests/src/test/resources/backcompat/${fileName}_v$version.jelly")), + jenaDg, + v2Format + ) + println(s"Generated ${fileName}_v$version.jelly") diff --git a/jena/src/main/scala/eu/ostrzyciel/jelly/convert/jena/riot/JellyWriter.scala b/jena/src/main/scala/eu/ostrzyciel/jelly/convert/jena/riot/JellyWriter.scala index 9703326f..d66631ab 100644 --- a/jena/src/main/scala/eu/ostrzyciel/jelly/convert/jena/riot/JellyWriter.scala +++ b/jena/src/main/scala/eu/ostrzyciel/jelly/convert/jena/riot/JellyWriter.scala @@ -42,7 +42,8 @@ private[riot] object Util: baseVariant.copy( opt = context.get[RdfStreamOptions](JellyLanguage.SYMBOL_STREAM_OPTIONS, preset), frameSize = context.getInt(JellyLanguage.SYMBOL_FRAME_SIZE, baseVariant.frameSize), - enableNamespaceDeclarations = context.isTrue(JellyLanguage.SYMBOL_ENABLE_NAMESPACE_DECLARATIONS), + enableNamespaceDeclarations = context.isTrue(JellyLanguage.SYMBOL_ENABLE_NAMESPACE_DECLARATIONS) || + baseVariant.enableNamespaceDeclarations, ) /**