added outYaml comparisons, discovered that our Presenter is rendering…

… yaml differently, added passing ScalarStyle to nodes so that we render nodes as they were, unborked 6H3V this way, this is like constant cramp in my gray matter, yaml is horrible
VirtusLab · Jul 22, 2024 · 3ff55a3 · 3ff55a3
1 parent 7703795
commit 3ff55a3
Show file tree

Hide file tree

Showing 13 changed files with 243 additions and 56 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -61,6 +61,8 @@ jobs:
         env:
           PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
           PGP_SECRET: ${{ secrets.PGP_SECRET }}
+          SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+          SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
 
   documentation:
     needs: test

diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/Node.scala b/core/shared/src/main/scala/org/virtuslab/yaml/Node.scala
@@ -23,11 +23,45 @@ sealed trait Node {
 }
 
 object Node {
-  final case class ScalarNode private[yaml] (value: String, tag: Tag, pos: Option[Range] = None)
-      extends Node
+
+  import org.virtuslab.yaml.internal.load.reader.token.{ScalarStyle => ParserScalarStyle}
+
+  sealed abstract class ScalarStyle(indicator: Char) {
+    def toParserStyle: ParserScalarStyle = this match {
+      case ScalarStyle.Plain        => ParserScalarStyle.Plain
+      case ScalarStyle.DoubleQuoted => ParserScalarStyle.DoubleQuoted
+      case ScalarStyle.SingleQuoted => ParserScalarStyle.SingleQuoted
+      case ScalarStyle.Folded       => ParserScalarStyle.Folded
+      case ScalarStyle.Literal      => ParserScalarStyle.Literal
+    }
+  }
+  object ScalarStyle {
+
+    def fromParserStyle(style: ParserScalarStyle): ScalarStyle = style match {
+      case ParserScalarStyle.Plain        => ScalarStyle.Plain
+      case ParserScalarStyle.DoubleQuoted => ScalarStyle.DoubleQuoted
+      case ParserScalarStyle.SingleQuoted => ScalarStyle.SingleQuoted
+      case ParserScalarStyle.Folded       => ScalarStyle.Folded
+      case ParserScalarStyle.Literal      => ScalarStyle.Literal
+    }
+
+    case object Plain        extends ScalarStyle(' ')
+    case object DoubleQuoted extends ScalarStyle('"')
+    case object SingleQuoted extends ScalarStyle('\'')
+    case object Folded       extends ScalarStyle('>')
+    case object Literal      extends ScalarStyle('|')
+  }
+
+  final case class ScalarNode private[yaml] (
+      value: String,
+      tag: Tag,
+      style: ScalarStyle,
+      pos: Option[Range] = None
+  ) extends Node
 
   object ScalarNode {
-    def apply(value: String): ScalarNode = new ScalarNode(value, Tag.resolveTag(value))
+    def apply(value: String): ScalarNode =
+      new ScalarNode(value, style = ScalarStyle.Plain, tag = Tag.resolveTag(value))
     def unapply(node: ScalarNode): Option[(String, Tag)] = Some((node.value, node.tag))
   }
 

diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala b/core/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala
@@ -48,7 +48,7 @@ object YamlEncoder extends YamlEncoderCrossCompanionCompat {
 
   implicit def forOption[T](implicit encoder: YamlEncoder[T]): YamlEncoder[Option[T]] = {
     case Some(t) => encoder.asNode(t)
-    case None    => Node.ScalarNode("", Tag.nullTag)
+    case None    => Node.ScalarNode("", Tag.nullTag, Node.ScalarStyle.Plain)
   }
 
   implicit def forSet[T](implicit encoder: YamlEncoder[T]): YamlEncoder[Set[T]] = (nodes) =>

diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/present/PresenterImpl.scala b/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/present/PresenterImpl.scala
@@ -8,6 +8,7 @@ import org.virtuslab.yaml.Tag
 import org.virtuslab.yaml.internal.load.parse.EventKind
 import org.virtuslab.yaml.internal.load.parse.EventKind._
 import org.virtuslab.yaml.internal.load.parse.NodeEventMetadata
+import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle
 
 object PresenterImpl extends Presenter {
   override def asString(events: Seq[EventKind]): String = {
@@ -30,11 +31,18 @@ object PresenterImpl extends Presenter {
               insertSequencePadding()
               pushAndIncreaseIndent(SequenceStart())
               parseSequence(tail)
-            case Scalar(value, _, NodeEventMetadata(_, tag)) =>
+            case Scalar(value, style, NodeEventMetadata(_, tag)) =>
               insertSequencePadding()
               // todo escape string using doublequotes
               if (tag.contains(Tag.nullTag)) sb.append("!!null")
-              else sb.append(value)
+              else
+                style match {
+                  case ScalarStyle.Plain        => sb.append(value)
+                  case ScalarStyle.DoubleQuoted => sb.append(s"\"$value\"")
+                  case ScalarStyle.SingleQuoted => sb.append(s"'$value'")
+                  case ScalarStyle.Folded       => sb.append(value)
+                  case ScalarStyle.Literal      => sb.append(value)
+                }
               sb.append(newline)
               tail
             case DocumentStart(_) => parseNode(tail)
@@ -50,8 +58,8 @@ object PresenterImpl extends Presenter {
         case MappingEnd :: tail =>
           popAndDecreaseIndent()
           tail
-        case Scalar(value, _, _) :: tail =>
-          appendKey(value)
+        case Scalar(value, style, _) :: tail =>
+          appendKey(value, style)
           val rest = parseNode(tail)
           parseMapping(rest)
         case _ => events
@@ -69,9 +77,15 @@ object PresenterImpl extends Presenter {
           parseSequence(rest)
       }
 
-    def appendKey(value: String) = {
+    def appendKey(value: String, style: ScalarStyle) = {
       sb.append(" " * indent)
-      sb.append(value)
+      style match {
+        case ScalarStyle.Plain        => sb.append(value)
+        case ScalarStyle.DoubleQuoted => sb.append(s"\"$value\"")
+        case ScalarStyle.SingleQuoted => sb.append(s"'$value'")
+        case ScalarStyle.Folded       => sb.append(value)
+        case ScalarStyle.Literal      => sb.append(value)
+      }
       sb.append(": ")
     }
 

diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala b/core/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala
@@ -29,6 +29,12 @@ object SerializerImpl extends Serializer {
   }
 
   private def convertScalarNode(node: Node.ScalarNode): Seq[EventKind] =
-    Seq(Scalar(node.value, metadata = NodeEventMetadata(tag = node.tag)))
+    Seq(
+      Scalar(
+        node.value,
+        style = node.style.toParserStyle,
+        metadata = NodeEventMetadata(tag = node.tag)
+      )
+    )
 
 }
diff --git a/core/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala b/core/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala
@@ -48,7 +48,8 @@ object ComposerImpl extends Composer {
           composeMappingNode(tail, anchor, aliases)
         case s: EventKind.Scalar =>
           val tag: Tag = s.metadata.tag.getOrElse(Tag.resolveTag(s.value, Some(s.style)))
-          val node     = Node.ScalarNode(s.value, tag, head.pos)
+          val node =
+            Node.ScalarNode(s.value, tag, Node.ScalarStyle.fromParserStyle(s.style), head.pos)
           s.metadata.anchor.foreach(anchor => aliases.put(anchor, node))
           Right(Result(node, tail))
         // todo #88

diff --git a/core/shared/src/test/scala/org/virtuslab/yaml/traverse/NodeVisitorSuite.scala b/core/shared/src/test/scala/org/virtuslab/yaml/traverse/NodeVisitorSuite.scala
@@ -27,17 +27,17 @@ class NodeVisitorSuite extends munit.FunSuite {
     val modifiedYaml = modifiedNode.asYaml
 
     val exptectedYaml =
-      s"""version: 3.9
+      s"""version: "3.9"
          |services: 
          |  web: 
          |    build: .
          |    volumes: 
          |      - .:/code
          |      - logvolume01:/var/log
          |    ports: 
-         |      - 6000:6000
+         |      - "6000:6000"
          |  redis: 
-         |    image: redis:alpine
+         |    image: "redis:alpine"
          |""".stripMargin
 
     assertEquals(modifiedYaml, exptectedYaml)
@@ -71,17 +71,17 @@ class NodeVisitorSuite extends munit.FunSuite {
     val modifiedYaml = modifiedNode.asYaml
 
     val exptectedYaml =
-      s"""version: 3.9
+      s"""version: "3.9"
          |services: 
          |  web: 
          |    build: .
          |    volumes: 
          |      - .:/code
          |      - logvolume01:/var/log
          |    ports: 
-         |      - 5000:5000:6000
+         |      - "5000:5000:6000"
          |  redis: 
-         |    image: redis:alpine:latest
+         |    image: "redis:alpine:latest"
          |""".stripMargin
 
     assertEquals(modifiedYaml, exptectedYaml)
@@ -115,7 +115,7 @@ class NodeVisitorSuite extends munit.FunSuite {
     val modifiedYaml = modifiedNode.asYaml
 
     val exptectedYaml =
-      s"""version: 3.9
+      s"""version: "3.9"
          |services: 
          |  web: 
          |    build: .

diff --git a/integration-tests/src/main/scala/org/virtuslab/yaml/RunnerResult.scala b/integration-tests/src/main/scala/org/virtuslab/yaml/RunnerResult.scala
@@ -20,6 +20,11 @@ object RunnerResult {
     else Error(eventsAsStr, error)
   }
 
+  case class InvalidOutYaml(eventsResult: RunnerResult, outYaml: String, expectedOutYaml: String)
+      extends RunnerResult {
+    override val isValid = false
+  }
+
   case class Success(events: String) extends RunnerResult {
     override val isValid = true
   }

diff --git a/integration-tests/src/main/scala/org/virtuslab/yaml/TestMlEntry.scala b/integration-tests/src/main/scala/org/virtuslab/yaml/TestMlEntry.scala
@@ -5,6 +5,7 @@ case class TestMlEntry(
     from: String,
     tags: String,
     inYaml: String,
+    outYaml: Option[String],
     seqEvent: String
 )
 
@@ -24,6 +25,21 @@ case object TestMlEntry {
       .head
   }
 
+  private def extractOutYaml(testMl: String): Option[String] = {
+    val patternOut =
+      raw"--- out-yaml(\(<\)|\(\+\)|\(<\+\)|)(([^\n]*\n+)+?)--- (in-json|error|emit-yaml|test-event)".r
+
+    patternOut
+      .findAllIn(testMl)
+      .matchData
+      .map { m =>
+        m.group(2)
+      }
+      .toList
+      .headOption
+      .map(_.strip())
+  }
+
   private def extractSeqEvent(testMl: String): String = {
     val patternEvent = raw"--- test-event(([^\n]*\n+)+).*".r
 
@@ -42,6 +58,7 @@ case object TestMlEntry {
       from = "",
       tags = "",
       inYaml = extractInYaml(content),
+      outYaml = extractOutYaml(content),
       seqEvent = extractSeqEvent(content)
     )
   }

diff --git a/integration-tests/src/main/scala/org/virtuslab/yaml/TestRunner.scala b/integration-tests/src/main/scala/org/virtuslab/yaml/TestRunner.scala
@@ -16,7 +16,9 @@ import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle
 import org.virtuslab.yaml.internal.load.reader.Tokenizer
 
 trait TestRunner {
+  def testOutYaml: Boolean
   def inYaml: String
+  def outYaml: Option[String] = None
   def expectedEvents: String
 
   def run(): RunnerResult = {
@@ -35,7 +37,28 @@ trait TestRunner {
           RunnerResult(acc.toList, expectedEvents, error)
       }
     }
-    loop()
+
+    val result = loop()
+
+    if (testOutYaml) {
+      // we don't compare output yaml if there is an error in event parsing
+      if (
+        outYaml.isDefined && (result match {
+          case RunnerResult.Error(_, _) => false; case _ => true
+        })
+      ) {
+        val roundtrip = inYaml.asNode match
+          case Left(error) =>
+            println(s"Error while parsing inYaml:\n$error")
+            s"!!! ERROR: $error"
+          case Right(value) => value.asYaml
+
+        if (roundtrip != outYaml.map(_.appended('\n')).get)
+          RunnerResult.InvalidOutYaml(result, roundtrip, outYaml.get)
+        else result
+      } else result
+
+    } else result
   }
 
 }
@@ -69,10 +92,11 @@ object TestRunnerUtils {
         case MappingEnd              => "-MAP"
         case Alias(alias)            => s"=ALI *$alias"
         case Scalar(value, style, data) =>
+          val escapedValue = value.replace("\\", "\\\\").replace("\n", "\\n")
           style match {
             case ScalarStyle.Plain        => s"=VAL${data.asString} :$value"
             case ScalarStyle.DoubleQuoted => s"""=VAL${data.asString} "$value"""
-            case ScalarStyle.SingleQuoted => s"=VAL${data.asString} '$value"
+            case ScalarStyle.SingleQuoted => s"=VAL${data.asString} '$escapedValue"
             case ScalarStyle.Folded       => s"=VAL${data.asString} >$value"
             case ScalarStyle.Literal      => s"=VAL${data.asString} |$value"
           }
@@ -81,7 +105,8 @@ object TestRunnerUtils {
 
 }
 
-case class K8sYamlTestRunner(yamlPath: os.Path, libYaml: os.Path) extends TestRunner {
+case class K8sYamlTestRunner(yamlPath: os.Path, libYaml: os.Path, testOutYaml: Boolean)
+    extends TestRunner {
   override val inYaml = os.read(yamlPath)
   override val expectedEvents = os
     .proc(libYaml, yamlPath)
@@ -97,9 +122,10 @@ case class K8sYamlTestRunner(yamlPath: os.Path, libYaml: os.Path) extends TestRu
 
 }
 
-case class YamlSuiteTestRunner(testYamlML: os.Path) extends TestRunner {
+case class YamlSuiteTestRunner(testYamlML: os.Path, testOutYaml: Boolean) extends TestRunner {
   private val testMl = TestMlEntry.from(testYamlML)
 
+  override val outYaml        = testMl.outYaml
   override val inYaml         = testMl.inYaml
   override val expectedEvents = testMl.seqEvent
 

diff --git a/integration-tests/src/test/scala/org/virtuslab/yaml/K8sConfigSpec.scala b/integration-tests/src/test/scala/org/virtuslab/yaml/K8sConfigSpec.scala
@@ -4,7 +4,8 @@ class K8sConfigSpec extends YamlRunnerSpec {
 
   val libYamlPath = os.Path(System.getenv("LIB_YAML_PATH"))
 
-  override def resourcePath: String                   = "/yaml/configs"
-  def createTestRunner(yamlPath: os.Path): TestRunner = K8sYamlTestRunner(yamlPath, libYamlPath)
+  override def resourcePath: String = "/yaml/configs"
+  def createTestRunner(yamlPath: os.Path): TestRunner =
+    K8sYamlTestRunner(yamlPath, libYamlPath, testOutYaml)
 
 }
diff --git a/integration-tests/src/test/scala/org/virtuslab/yaml/OfficialTestSuiteSpec.scala b/integration-tests/src/test/scala/org/virtuslab/yaml/OfficialTestSuiteSpec.scala
@@ -1,7 +1,17 @@
 package org.virtuslab.yaml
 
+import scala.util.matching.Regex
+
 class OfficialTestSuiteSpec extends YamlRunnerSpec {
 
-  override def resourcePath: String                            = "/yaml/test-suite"
-  override def createTestRunner(yamlPath: os.Path): TestRunner = YamlSuiteTestRunner(yamlPath)
+  override val verbose: Boolean           = true
+  override val testOutYaml: Boolean       = false
+  override val skipErrors: Boolean        = false
+  override val skipInvalidEvents: Boolean = false
+
+  // override def runMatching: Regex = ".*LQZ7.*".r
+
+  override def resourcePath: String = "/yaml/test-suite"
+  override def createTestRunner(yamlPath: os.Path): TestRunner =
+    YamlSuiteTestRunner(yamlPath, testOutYaml)
 }