Fix normalization and value translation of LF1.15 values in 1.17 contexts

sdk/daml-lf/engine/src/main/scala/com/digitalasset/daml/lf/engine/ValueEnricher.scala

@@ -28,7 +28,7 @@ import com.daml.lf.speedy.SValue
 
 final class ValueEnricher(
     compiledPackages: CompiledPackages,
-    translateValue: (Ast.Type, Value) => Result[SValue],
+    translateValue: (Ast.Type, Boolean, Value) => Result[SValue],
     loadPackage: (PackageId, language.Reference) => Result[Unit],
 ) {
 
@@ -39,15 +39,16 @@ final class ValueEnricher(
       engine.loadPackage,
     )
 
-  def enrichValue(typ: Ast.Type, value: Value): Result[Value] =
-    translateValue(typ, value).map(_.toUnnormalizedValue)
+  def enrichValue(typ: Ast.Type, upgradable: Boolean, value: Value): Result[Value] =
+    translateValue(typ, upgradable, value).map(_.toUnnormalizedValue)
 
   def enrichVersionedValue(
       typ: Ast.Type,
+      upgradable: Boolean,
       versionedValue: VersionedValue,
   ): Result[VersionedValue] =
     for {
-      value <- enrichValue(typ, versionedValue.unversioned)
+      value <- enrichValue(typ, upgradable, versionedValue.unversioned)
     } yield versionedValue.map(_ => value)
 
   def enrichContract(
@@ -61,17 +62,27 @@ final class ValueEnricher(
       contract: Value.VersionedContractInstance
   ): Result[Value.VersionedContractInstance] =
     for {
-      arg <- enrichValue(Ast.TTyCon(contract.unversioned.template), contract.unversioned.arg)
+      pkg <- handleLookup(
+        compiledPackages.pkgInterface.lookupPackage(contract.unversioned.template.packageId)
+      )
+      arg <- enrichValue(
+        Ast.TTyCon(contract.unversioned.template),
+        pkg.upgradable,
+        contract.unversioned.arg,
+      )
     } yield contract.map(_.copy(arg = arg))
 
   def enrichView(
       interfaceId: Identifier,
       viewValue: Value,
   ): Result[Value] = for {
+    pkg <- handleLookup(
+      compiledPackages.pkgInterface.lookupPackage(interfaceId.packageId)
+    )
     iface <- handleLookup(
       compiledPackages.pkgInterface.lookupInterface(interfaceId)
     )
-    r <- enrichValue(iface.view, viewValue)
+    r <- enrichValue(iface.view, pkg.upgradable, viewValue)
   } yield r
 
   def enrichVersionedView(
@@ -82,7 +93,12 @@ final class ValueEnricher(
   } yield viewValue.copy(unversioned = view)
 
   def enrichContract(tyCon: Identifier, value: Value): Result[Value] =
-    enrichValue(Ast.TTyCon(tyCon), value)
+    for {
+      pkg <- handleLookup(
+        compiledPackages.pkgInterface.lookupPackage(tyCon.packageId)
+      )
+      enrichedValue <- enrichValue(Ast.TTyCon(tyCon), pkg.upgradable, value)
+    } yield enrichedValue
 
   private[this] def pkgInterface = compiledPackages.pkgInterface
 
@@ -106,15 +122,19 @@ final class ValueEnricher(
       interfaceId: Option[Identifier],
       choiceName: Name,
       value: Value,
-  ): Result[Value] =
-    handleLookup(
+  ): Result[Value] = for {
+    choice <- handleLookup(
       pkgInterface.lookupChoice(
         Identifier(choicePackageId, qualifiedTemplateName),
         interfaceId,
         choiceName,
       )
     )
-      .flatMap(choice => enrichValue(choice.argBinder._2, value))
+    pkg <- handleLookup(
+      compiledPackages.pkgInterface.lookupPackage(choicePackageId)
+    )
+    enrichedValue <- enrichValue(choice.argBinder._2, pkg.upgradable, value)
+  } yield enrichedValue
 
   // Deprecated
   def enrichChoiceArgument(
@@ -132,21 +152,24 @@ final class ValueEnricher(
     )
 
   def enrichChoiceResult(
-      choicePackageId: PackageId,
+      templatePackageId: PackageId,
       qualifiedTemplateName: QualifiedName,
       interfaceId: Option[Identifier],
       choiceName: Name,
       value: Value,
-  ): Result[Value] = {
-    handleLookup(
+  ): Result[Value] = for {
+    choice <- handleLookup(
       pkgInterface.lookupChoice(
-        Identifier(choicePackageId, qualifiedTemplateName),
+        Identifier(templatePackageId, qualifiedTemplateName),
         interfaceId,
         choiceName,
       )
     )
-      .flatMap(choice => enrichValue(choice.returnType, value))
-  }
+    pkg <- handleLookup(
+      pkgInterface.lookupPackage(interfaceId.fold(templatePackageId)(_.packageId))
+    )
+    enrichedValue <- enrichValue(choice.returnType, pkg.upgradable, value)
+  } yield enrichedValue
 
   // Deprecated
   def enrichChoiceResult(
@@ -163,8 +186,11 @@ final class ValueEnricher(
   )
 
   def enrichContractKey(tyCon: Identifier, value: Value): Result[Value] =
-    handleLookup(pkgInterface.lookupTemplateKey(tyCon))
-      .flatMap(key => enrichValue(key.typ, value))
+    for {
+      key <- handleLookup(pkgInterface.lookupTemplateKey(tyCon))
+      pkg <- handleLookup(pkgInterface.lookupPackage(tyCon.packageId))
+      enrichedValue <- enrichValue(key.typ, pkg.upgradable, value)
+    } yield enrichedValue
 
   private val ResultNone = ResultDone(None)
 
@@ -196,7 +222,8 @@ final class ValueEnricher(
         ResultDone(rb)
       case create: Node.Create =>
         for {
-          arg <- enrichValue(Ast.TTyCon(create.templateId), create.arg)
+          pkg <- handleLookup(pkgInterface.lookupPackage(create.templateId.packageId))
+          arg <- enrichValue(Ast.TTyCon(create.templateId), pkg.upgradable, create.arg)
           key <- enrichContractKey(create.keyOpt, create.version, create.packageName)
         } yield create.copy(arg = arg, keyOpt = key)
       case fetch: Node.Fetch =>

sdk/daml-lf/engine/src/main/scala/com/digitalasset/daml/lf/engine/preprocessing/CommandPreprocessor.scala

@@ -30,13 +30,14 @@ private[lf] final class CommandPreprocessor(
 
   private[this] def translateArg(
       typ: Ast.Type,
+      upgradable: Boolean,
       value: Value,
   ) =
-    valueTranslator.unsafeTranslateValue(ty = typ, value = value)
+    valueTranslator.unsafeTranslateValue(ty = typ, upgradable = upgradable, value = value)
 
   // This is used by value enricher
-  def unsafeTranslateValue(typ: Ast.Type, value: Value) =
-    valueTranslator.unsafeTranslateValue(typ, value)
+  def unsafeTranslateValue(typ: Ast.Type, upgradable: Boolean, value: Value) =
+    valueTranslator.unsafeTranslateValue(typ, upgradable = upgradable, value)
 
   @throws[Error.Preprocessing.Error]
   def unsafePreprocessDisclosedContract(
@@ -59,7 +60,7 @@ private[lf] final class CommandPreprocessor(
     }
     // TODO: https://github.com/digital-asset/daml/issues/17082
     //  for now we need the package of the disclosed contract
-    val arg = translateArg(Ast.TTyCon(disc.templateId), disc.argument)
+    val arg = translateArg(Ast.TTyCon(disc.templateId), upgradable(disc.templateId), disc.argument)
     validateCid(disc.contractId)
     speedy.DisclosedContract(
       templateId = disc.templateId,
@@ -72,7 +73,7 @@ private[lf] final class CommandPreprocessor(
   @throws[Error.Preprocessing.Error]
   def unsafePreprocessCreate(templateId: Ref.Identifier, argument: Value): speedy.Command.Create = {
     discard(handleLookup(pkgInterface.lookupTemplate(templateId)))
-    val arg = translateArg(Ast.TTyCon(templateId), argument)
+    val arg = translateArg(Ast.TTyCon(templateId), upgradable(templateId), argument)
     speedy.Command.Create(templateId, arg)
   }
 
@@ -100,7 +101,7 @@ private[lf] final class CommandPreprocessor(
       templateId = templateId,
       contractId = valueTranslator.unsafeTranslateCid(contractId),
       choiceId = choiceId,
-      argument = translateArg(choice.argBinder._2, argument),
+      argument = translateArg(choice.argBinder._2, upgradable(templateId), argument),
     )
   }
 
@@ -115,7 +116,7 @@ private[lf] final class CommandPreprocessor(
       interfaceId = ifaceId,
       contractId = valueTranslator.unsafeTranslateCid(contractId),
       choiceId = choiceId,
-      argument = translateArg(choice.argBinder._2, argument),
+      argument = translateArg(choice.argBinder._2, upgradable(ifaceId), argument),
     )
   }
 
@@ -129,9 +130,10 @@ private[lf] final class CommandPreprocessor(
     val choiceArgType = handleLookup(
       pkgInterface.lookupTemplateChoice(templateId, choiceId)
     ).argBinder._2
+    val upgradable = this.upgradable(templateId)
     val ckTtype = handleLookup(pkgInterface.lookupTemplateKey(templateId)).typ
-    val arg = translateArg(choiceArgType, argument)
-    val key = translateArg(ckTtype, contractKey)
+    val arg = translateArg(choiceArgType, upgradable, argument)
+    val key = translateArg(ckTtype, upgradable, contractKey)
     speedy.Command.ExerciseByKey(templateId, key, choiceId, arg)
   }
 
@@ -142,11 +144,12 @@ private[lf] final class CommandPreprocessor(
       choiceId: Ref.ChoiceName,
       choiceArgument: Value,
   ): speedy.Command.CreateAndExercise = {
-    val createArg = translateArg(Ast.TTyCon(templateId), createArgument)
+    val upgradable = this.upgradable(templateId)
+    val createArg = translateArg(Ast.TTyCon(templateId), upgradable, createArgument)
     val choiceArgType = handleLookup(
       pkgInterface.lookupTemplateChoice(templateId, choiceId)
     ).argBinder._2
-    val choiceArg = translateArg(choiceArgType, choiceArgument)
+    val choiceArg = translateArg(choiceArgType, upgradable, choiceArgument)
     speedy.Command.CreateAndExercise(templateId, createArg, choiceId, choiceArg)
   }
 
@@ -156,7 +159,7 @@ private[lf] final class CommandPreprocessor(
       contractKey: Value,
   ): speedy.Command.LookupByKey = {
     val ckTtype = handleLookup(pkgInterface.lookupTemplateKey(templateId)).typ
-    val key = translateArg(ckTtype, contractKey)
+    val key = translateArg(ckTtype, upgradable(templateId), contractKey)
     speedy.Command.LookupByKey(templateId, key)
   }
 
@@ -277,11 +280,11 @@ private[lf] final class CommandPreprocessor(
         }
       case command.ReplayCommand.FetchByKey(templateId, key) =>
         val ckTtype = handleLookup(pkgInterface.lookupTemplateKey(templateId)).typ
-        val sKey = translateArg(ckTtype, key)
+        val sKey = translateArg(ckTtype, upgradable(templateId), key)
         speedy.Command.FetchByKey(templateId, sKey)
       case command.ReplayCommand.LookupByKey(templateId, key) =>
         val ckTtype = handleLookup(pkgInterface.lookupTemplateKey(templateId)).typ
-        val sKey = translateArg(ckTtype, key)
+        val sKey = translateArg(ckTtype, upgradable(templateId), key)
         speedy.Command.LookupByKey(templateId, sKey)
     }
 
@@ -323,7 +326,7 @@ private[lf] final class CommandPreprocessor(
     discard(handleLookup(pkgInterface.lookupInterface(interfaceId)))
     discard(handleLookup(pkgInterface.lookupInterfaceInstance(interfaceId, templateId)))
 
-    val arg = translateArg(Ast.TTyCon(templateId), argument)
+    val arg = translateArg(Ast.TTyCon(templateId), upgradable(templateId), argument)
 
     speedy.InterfaceView(templateId, arg, interfaceId)
   }
@@ -353,7 +356,7 @@ private[lf] final class CommandPreprocessor(
         key.contractKey,
       )
     val ckTtype = handleLookup(pkgInterface.lookupTemplateKey(templateId)).typ
-    val preprocessedKey = translateArg(ckTtype, key.contractKey)
+    val preprocessedKey = translateArg(ckTtype, upgradable(templateId), key.contractKey)
 
     speedy.Speedy.Machine
       .globalKey(pkgInterface, templateId, preprocessedKey)
@@ -362,4 +365,7 @@ private[lf] final class CommandPreprocessor(
       )
   }
 
+  def upgradable(tyCon: Ref.TypeConName): Boolean =
+    handleLookup(pkgInterface.lookupPackage(tyCon.packageId)).upgradable
+
 }

sdk/daml-lf/engine/src/main/scala/com/digitalasset/daml/lf/engine/preprocessing/Preprocessor.scala

@@ -158,10 +158,10 @@ private[engine] final class Preprocessor(
     * Fails if the nesting is too deep or if v0 does not match the type `ty0`.
     * Assumes ty0 is a well-formed serializable typ.
     */
-  def translateValue(ty0: Ast.Type, v0: Value): Result[SValue] =
+  def translateValue(ty0: Ast.Type, upgradable: Boolean, v0: Value): Result[SValue] =
     safelyRun(pullTypePackages(ty0)) {
       // this is used only by the value enricher
-      commandPreprocessor.unsafeTranslateValue(ty0, v0)
+      commandPreprocessor.unsafeTranslateValue(ty0, upgradable, v0)
     }
 
   private[engine] def preprocessApiCommand(

sdk/daml-lf/engine/src/main/scala/com/digitalasset/daml/lf/engine/preprocessing/ValueTranslator.scala

@@ -15,7 +15,7 @@ import com.daml.lf.value.Value._
 import scala.annotation.tailrec
 
 private[lf] final class ValueTranslator(
-    pkgInterface: language.PackageInterface,
+    val pkgInterface: language.PackageInterface,
     checkV1ContractIdSuffixes: Boolean,
 ) {
 
@@ -62,6 +62,7 @@ private[lf] final class ValueTranslator(
   @throws[Error.Preprocessing.Error]
   private[preprocessing] def unsafeTranslateValue(
       ty: Type,
+      upgradable: Boolean,
       value: Value,
   ): SValue = {
     // TODO: https://github.com/digital-asset/daml/issues/17082
@@ -178,7 +179,6 @@ private[lf] final class ValueTranslator(
                 typeError()
             }
           case TTyCon(tyCon) =>
-            val upgradable = handleLookup(pkgInterface.lookupPackage(tyCon.packageId)).upgradable
             value0 match {
               // variant
               case ValueVariant(mbId, constructorName, val0) =>
@@ -197,8 +197,11 @@ private[lf] final class ValueTranslator(
               case ValueRecord(mbId, sourceElements) =>
                 checkUserTypeId(upgradable, tyCon, mbId)
                 val lookupResult = handleLookup(pkgInterface.lookupDataRecord(tyCon))
-                val targetFieldsAndTypes = lookupResult.dataRecord.fields
                 val subst = lookupResult.subst(tyArgs)
+                val targetFieldsAndTypes =
+                  lookupResult.dataRecord.fields.map { case (lbl, typ) =>
+                    lbl -> AstUtil.substitute(typ, subst)
+                  }
 
                 def addMissingField(lbl: Ref.Name, ty: Type): (Option[Ref.Name], Value) =
                   ty match {
@@ -279,8 +282,7 @@ private[lf] final class ValueTranslator(
 
                   // Recursive substitution
                   val translatedCorrectFields = correctFields.map { case (lbl, v, typ) =>
-                    val replacedTyp = AstUtil.substitute(typ, subst)
-                    lbl -> go(replacedTyp, v, newNesting)
+                    lbl -> go(typ, v, newNesting)
                   }
 
                   extraFields.foreach {
@@ -361,9 +363,13 @@ private[lf] final class ValueTranslator(
     go(ty, value)
   }
 
-  def translateValue(ty: Type, value: Value): Either[Error.Preprocessing.Error, SValue] =
+  def translateValue(
+      ty: Type,
+      upgradable: Boolean,
+      value: Value,
+  ): Either[Error.Preprocessing.Error, SValue] =
     safelyRun(
-      unsafeTranslateValue(ty, value)
+      unsafeTranslateValue(ty, upgradable, value)
     )
 
 }