Lazy serializers

rd-kt/rd-core/src/main/kotlin/com/jetbrains/rd/util/hash/IncrementalHash.kt

@@ -4,4 +4,10 @@ class IncrementalHash64(r: Long = 0) {
     fun mix(mix: String?) = IncrementalHash64(mix?.fold(result) { acc, c -> acc*31 + c.toInt()} ?: result)
     fun<T> mix(collection : Iterable<T>, map : (T) -> String) = collection.fold(this) {acc, elt -> acc.mix(map(elt))}
     val result: Long = r
-}
+}
+
+
+//PLEASE DO NOT CHANGE IT!!! IT'S EXACTLY THE SAME ON C# SIDE
+fun String?.getPlatformIndependentHash(initial: Long = 19L) : Long = this?.fold(initial) { acc, c -> acc*31 + c.toInt()} ?:0
+fun Int.getPlatformIndependentHash(initial: Long = 19L) : Long = initial*31 + (this + 1)
+fun Long.getPlatformIndependentHash(initial: Long = 19L) : Long = initial*31 + (this + 1)

rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/FrameworkMarshallers.kt

@@ -3,6 +3,7 @@ package com.jetbrains.rd.framework
 import com.jetbrains.rd.framework.impl.RdSecureString
 import com.jetbrains.rd.util.*
 import kotlin.reflect.*
+import kotlin.time.Duration
 
 open class UniversalMarshaller<T : Any>(
         override val _type: KClass<*>,
@@ -27,13 +28,18 @@ open class DelegatedMarshaller<TFrom : Any, TTo : Any>(marshaller: IMarshaller<T
 )
 
 object FrameworkMarshallers {
-    inline fun <reified T : Any> create(crossinline reader: (AbstractBuffer) -> T, crossinline writer: (AbstractBuffer, T) -> Unit, predefinedId: Int? = null): UniversalMarshaller<T> {
-        return UniversalMarshaller(T::class, { _, stream -> reader(stream) }, { _, stream, v -> writer(stream, v) }, predefinedId)
+    @Deprecated("Use an overload without inlining", ReplaceWith("create(T::class, reader, writer, predefinedId)", "com.jetbrains.rd.framework.FrameworkMarshallers.create"))
+    inline fun <reified T : Any> create(noinline reader: (AbstractBuffer) -> T, noinline writer: (AbstractBuffer, T) -> Unit, predefinedId: Int? = null): UniversalMarshaller<T> {
+        return create(T::class, reader, writer, predefinedId)
     }
-
+
+    fun <T : Any> create(clazz: KClass<T>, reader: (AbstractBuffer) -> T, writer: (AbstractBuffer, T) -> Unit, predefinedId: Int? = null): UniversalMarshaller<T> {
+        return UniversalMarshaller(clazz, { _, stream -> reader(stream) }, { _, stream, v -> writer(stream, v) }, predefinedId)
+    }
+
 
     inline fun <reified T : Enum<T>> enum(): UniversalMarshaller<T> {
-        return create({ it.readEnum<T>() }, { stream, x -> stream.writeEnum(x) })
+        return create(T::class, { it.readEnum<T>() }, { stream, x -> stream.writeEnum(x) } )
     }
 
     inline fun <reified T : Enum<T>> enumSet(): UniversalMarshaller<EnumSet<T>> {
@@ -50,21 +56,21 @@ object FrameworkMarshallers {
     }
 
 
-    val Int8 : IMarshaller<Byte> = create(AbstractBuffer::readByte, AbstractBuffer::writeByte, 1)
-    val Int16: IMarshaller<Short> = create(AbstractBuffer::readShort, AbstractBuffer::writeShort, 2)
-    val Int32: IMarshaller<Int> = create(AbstractBuffer::readInt, AbstractBuffer::writeInt, 3)
-    val Int64: IMarshaller<Long> = create(AbstractBuffer::readLong, AbstractBuffer::writeLong, 4)
+    val Int8 : IMarshaller<Byte> = create(kotlin.Byte::class, AbstractBuffer::readByte, AbstractBuffer::writeByte, 1)
+    val Int16: IMarshaller<Short> = create(kotlin.Short::class, AbstractBuffer::readShort, AbstractBuffer::writeShort, 2)
+    val Int32: IMarshaller<Int> = create(kotlin.Int::class, AbstractBuffer::readInt, AbstractBuffer::writeInt, 3)
+    val Int64: IMarshaller<Long> = create(kotlin.Long::class, AbstractBuffer::readLong, AbstractBuffer::writeLong, 4)
 
-    val Float: IMarshaller<Float> = create(AbstractBuffer::readFloat, AbstractBuffer::writeFloat, 5)
-    val Double: IMarshaller<Double> = create(AbstractBuffer::readDouble, AbstractBuffer::writeDouble, 6)
-    val Char: IMarshaller<Char> = create(AbstractBuffer::readChar, AbstractBuffer::writeChar, 7)
-    val Bool: IMarshaller<Boolean> = create(AbstractBuffer::readBoolean, AbstractBuffer::writeBoolean, 8)
+    val Float: IMarshaller<Float> = create(kotlin.Float::class, AbstractBuffer::readFloat, AbstractBuffer::writeFloat, 5)
+    val Double: IMarshaller<Double> = create(kotlin.Double::class, AbstractBuffer::readDouble, AbstractBuffer::writeDouble, 6)
+    val Char: IMarshaller<Char> = create(kotlin.Char::class, AbstractBuffer::readChar, AbstractBuffer::writeChar, 7)
+    val Bool: IMarshaller<Boolean> = create(Boolean::class, AbstractBuffer::readBoolean, AbstractBuffer::writeBoolean, 8)
 
     //empty
-    val Void: IMarshaller<Unit> = create({ }, { _, _ -> }, 9)
+    val Void: IMarshaller<Unit> = create(Unit::class, { }, { _, _ -> }, 9)
 
     //normal string
-    val String: UniversalMarshaller<String> = create(AbstractBuffer::readString, AbstractBuffer::writeString, 10)
+    val String: UniversalMarshaller<String> = create(kotlin.String::class, AbstractBuffer::readString, AbstractBuffer::writeString, 10)
 
     //aliases
     val Byte = Int8
@@ -74,48 +80,48 @@ object FrameworkMarshallers {
 
 
     //jvm-based
-    val Guid: UniversalMarshaller<UUID> = create({ it.readUuid() }, AbstractBuffer::writeUuid, 11)
-    val DateTime: UniversalMarshaller<Date> = create(AbstractBuffer::readDateTime, AbstractBuffer::writeDateTime, 12)
-    val Uri: UniversalMarshaller<URI> = create({ it.readUri() }, AbstractBuffer::writeUri, 13)
-    var TimeSpan: UniversalMarshaller<kotlin.time.Duration> = create(AbstractBuffer::readTimeSpan, AbstractBuffer::writeTimeSpan, 14)
+    val Guid: UniversalMarshaller<UUID> = create(java.util.UUID::class, { it.readUuid() }, AbstractBuffer::writeUuid, 11)
+    val DateTime: UniversalMarshaller<Date> = create(java.util.Date::class, AbstractBuffer::readDateTime, AbstractBuffer::writeDateTime, 12)
+    val Uri: UniversalMarshaller<URI> = create(java.net.URI::class, { it.readUri() }, AbstractBuffer::writeUri, 13)
+    var TimeSpan: UniversalMarshaller<Duration> = create(Duration::class, AbstractBuffer::readTimeSpan, AbstractBuffer::writeTimeSpan, 14)
 
     //rdId
-    val RdId: IMarshaller<RdId> = create(AbstractBuffer::readRdId, AbstractBuffer::writeRdId, 15)
+    val RdId: IMarshaller<RdId> = create(com.jetbrains.rd.framework.RdId::class, AbstractBuffer::readRdId, AbstractBuffer::writeRdId, 15)
 
     //string for passwords
-    val SecureString: IMarshaller<RdSecureString> = create({ RdSecureString(it.readString()) }, { buf, str -> buf.writeString(str.contents) }, 16)
+    val SecureString: IMarshaller<RdSecureString> = create(RdSecureString::class, { RdSecureString(it.readString()) }, { buf, str -> buf.writeString(str.contents) }, 16)
 
     //arrays
-    val ByteArray: UniversalMarshaller<ByteArray> = create(AbstractBuffer::readByteArray, AbstractBuffer::writeByteArray, 31)
-    val ShortArray: UniversalMarshaller<ShortArray> = create(AbstractBuffer::readShortArray, AbstractBuffer::writeShortArray, 32)
-    val IntArray: UniversalMarshaller<IntArray> = create(AbstractBuffer::readIntArray, AbstractBuffer::writeIntArray, 33)
-    val LongArray: UniversalMarshaller<LongArray> = create(AbstractBuffer::readLongArray, AbstractBuffer::writeLongArray, 34)
+    val ByteArray: UniversalMarshaller<ByteArray> = create(kotlin.ByteArray::class, AbstractBuffer::readByteArray, AbstractBuffer::writeByteArray, 31)
+    val ShortArray: UniversalMarshaller<ShortArray> = create(kotlin.ShortArray::class, AbstractBuffer::readShortArray, AbstractBuffer::writeShortArray, 32)
+    val IntArray: UniversalMarshaller<IntArray> = create(kotlin.IntArray::class, AbstractBuffer::readIntArray, AbstractBuffer::writeIntArray, 33)
+    val LongArray: UniversalMarshaller<LongArray> = create(kotlin.LongArray::class, AbstractBuffer::readLongArray, AbstractBuffer::writeLongArray, 34)
 
-    val FloatArray: UniversalMarshaller<FloatArray> = create(AbstractBuffer::readFloatArray, AbstractBuffer::writeFloatArray, 35)
-    val DoubleArray: UniversalMarshaller<DoubleArray> = create(AbstractBuffer::readDoubleArray, AbstractBuffer::writeDoubleArray, 36)
+    val FloatArray: UniversalMarshaller<FloatArray> = create(kotlin.FloatArray::class, AbstractBuffer::readFloatArray, AbstractBuffer::writeFloatArray, 35)
+    val DoubleArray: UniversalMarshaller<DoubleArray> = create(kotlin.DoubleArray::class, AbstractBuffer::readDoubleArray, AbstractBuffer::writeDoubleArray, 36)
 
-    val CharArray: UniversalMarshaller<CharArray> = create(AbstractBuffer::readCharArray, AbstractBuffer::writeCharArray, 37)
-    val BooleanArray: UniversalMarshaller<BooleanArray> = create(AbstractBuffer::readBooleanArray, AbstractBuffer::writeBooleanArray, 38)
+    val CharArray: UniversalMarshaller<CharArray> = create(kotlin.CharArray::class, AbstractBuffer::readCharArray, AbstractBuffer::writeCharArray, 37)
+    val BooleanArray: UniversalMarshaller<BooleanArray> = create(kotlin.BooleanArray::class, AbstractBuffer::readBooleanArray, AbstractBuffer::writeBooleanArray, 38)
 
 
     //unsigned
     @ExperimentalUnsignedTypes
-    val UByte : IMarshaller<UByte> = create(AbstractBuffer::readUByte, AbstractBuffer::writeUByte, 41)
+    val UByte : IMarshaller<UByte> = create(kotlin.UByte::class, AbstractBuffer::readUByte, AbstractBuffer::writeUByte, 41)
     @ExperimentalUnsignedTypes
-    val UShort: IMarshaller<UShort> = create(AbstractBuffer::readUShort, AbstractBuffer::writeUShort, 42)
+    val UShort: IMarshaller<UShort> = create(kotlin.UShort::class, AbstractBuffer::readUShort, AbstractBuffer::writeUShort, 42)
     @ExperimentalUnsignedTypes
-    val UInt: IMarshaller<UInt> = create(AbstractBuffer::readUInt, AbstractBuffer::writeUInt, 43)
+    val UInt: IMarshaller<UInt> = create(kotlin.UInt::class, AbstractBuffer::readUInt, AbstractBuffer::writeUInt, 43)
     @ExperimentalUnsignedTypes
-    val ULong: IMarshaller<ULong> = create(AbstractBuffer::readULong, AbstractBuffer::writeULong, 44)
+    val ULong: IMarshaller<ULong> = create(kotlin.ULong::class, AbstractBuffer::readULong, AbstractBuffer::writeULong, 44)
 
     @ExperimentalUnsignedTypes
-    val UByteArray: UniversalMarshaller<UByteArray> = create(AbstractBuffer::readUByteArray, AbstractBuffer::writeUByteArray, 45)
+    val UByteArray: UniversalMarshaller<UByteArray> = create(kotlin.UByteArray::class, AbstractBuffer::readUByteArray, AbstractBuffer::writeUByteArray, 45)
     @ExperimentalUnsignedTypes
-    val UShortArray: UniversalMarshaller<UShortArray> = create(AbstractBuffer::readUShortArray, AbstractBuffer::writeUShortArray, 46)
+    val UShortArray: UniversalMarshaller<UShortArray> = create(kotlin.UShortArray::class, AbstractBuffer::readUShortArray, AbstractBuffer::writeUShortArray, 46)
     @ExperimentalUnsignedTypes
-    val UIntArray: UniversalMarshaller<UIntArray> = create(AbstractBuffer::readUIntArray, AbstractBuffer::writeUIntArray, 47)
+    val UIntArray: UniversalMarshaller<UIntArray> = create(kotlin.UIntArray::class, AbstractBuffer::readUIntArray, AbstractBuffer::writeUIntArray, 47)
     @ExperimentalUnsignedTypes
-    val ULongArray: UniversalMarshaller<ULongArray> = create(AbstractBuffer::readULongArray, AbstractBuffer::writeULongArray, 48)
+    val ULongArray: UniversalMarshaller<ULongArray> = create(kotlin.ULongArray::class, AbstractBuffer::readULongArray, AbstractBuffer::writeULongArray, 48)
 
     fun registerIn(serializers: ISerializers) {
         serializers.register(Int8)

rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Identities.kt

@@ -2,6 +2,7 @@ package com.jetbrains.rd.framework
 
 
 import com.jetbrains.rd.util.AtomicInteger
+import com.jetbrains.rd.util.hash.getPlatformIndependentHash
 import com.jetbrains.rd.util.string.condstr
 
 enum class IdKind {
@@ -10,15 +11,19 @@ enum class IdKind {
 }
 
 //PLEASE DO NOT CHANGE IT!!! IT'S EXACTLY THE SAME ON C# SIDE
-fun String?.getPlatformIndependentHash(initial: Long = 19L) : Long = this?.fold(initial) { acc, c -> acc*31 + c.toInt()} ?:0
-fun Int.getPlatformIndependentHash(initial: Long = 19L) : Long = initial*31 + (this + 1)
-fun Long.getPlatformIndependentHash(initial: Long = 19L) : Long = initial*31 + (this + 1)
+@Deprecated("Api moved to com.jetbrains.rd.util.hash", ReplaceWith("getPlatformIndependentHash(initial)","com.jetbrains.rd.util.hash.getPlatformIndependentHash"))
+fun String?.getPlatformIndependentHash(initial: Long = 19L) : Long = getPlatformIndependentHash(initial)
+@Deprecated("Api moved to com.jetbrains.rd.util.hash", ReplaceWith("getPlatformIndependentHash(initial)","com.jetbrains.rd.util.hash.getPlatformIndependentHash"))
+fun Int.getPlatformIndependentHash(initial: Long = 19L) : Long = getPlatformIndependentHash(initial)
+@Deprecated("Api moved to com.jetbrains.rd.util.hash", ReplaceWith("getPlatformIndependentHash(initial)","com.jetbrains.rd.util.hash.getPlatformIndependentHash"))
+fun Long.getPlatformIndependentHash(initial: Long = 19L) : Long = getPlatformIndependentHash(initial)
 
 
 /**
  * An identifier of the object that participates in the object graph.
  */
-data class RdId(val hash: Long) {
+@JvmInline
+value class RdId(val hash: Long) {
 
     companion object {
         val Null : RdId = RdId( 0)

rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Interfaces.kt

@@ -6,8 +6,8 @@ import com.jetbrains.rd.framework.impl.ProtocolContexts
 import com.jetbrains.rd.util.lifetime.Lifetime
 import com.jetbrains.rd.util.reactive.*
 import com.jetbrains.rd.util.string.RName
-import java.rmi.NotBoundException
 import kotlin.reflect.KClass
+import kotlin.reflect.jvm.jvmName
 
 /**
  * A node in a graph of entities that can be synchronized with its remote copy over a network or a similar connection.
@@ -97,6 +97,32 @@ interface IMarshaller<T : Any> : ISerializer<T> {
         get() = RdId(_type.simpleName.getPlatformIndependentHash())
 }
 
+val IMarshaller<*>.fqn: String get() {
+    return if (this is LazyCompanionMarshaller) this.fgn
+    else _type.qualifiedName ?: _type.jvmName
+}
+
+class LazyCompanionMarshaller<T : Any>(
+    override val id: RdId,
+    val classLoader: ClassLoader,
+    val fqn: String
+) : IMarshaller<T> {
+    private val lazy = lazy(LazyThreadSafetyMode.PUBLICATION) {
+        Class.forName(fgn, true, classLoader).getDeclaredField("Companion").get(null) as IMarshaller<T>
+    }
+
+    override val _type: KClass<*>
+        get() = lazy.value._type
+
+    override fun read(ctx: SerializationCtx, buffer: AbstractBuffer): T {
+        return lazy.value.read(ctx, buffer)
+    }
+
+    override fun write(ctx: SerializationCtx, buffer: AbstractBuffer, value: T) {
+        lazy.value.write(ctx, buffer, value)
+    }
+}
+
 interface IAbstractDeclaration<T> {
     fun readUnknownInstance(ctx: SerializationCtx, buffer: AbstractBuffer, unknownId: RdId, size: Int): T
 }

rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/MarshallerUtils.kt

@@ -1,5 +1,6 @@
 package com.jetbrains.rd.framework
 
+import com.jetbrains.rd.framework.FrameworkMarshallers.create
 import com.jetbrains.rd.framework.base.IRdWireable
 import com.jetbrains.rd.framework.impl.RdSignal
 import com.jetbrains.rd.util.string.RName
@@ -9,7 +10,7 @@ object RNameMarshaller {
         val isEmpty = buffer.readBoolean()
         if (isEmpty)
             return RName.Empty
-        
+
         val rootName = buffer.readString()
         var last = buffer.readBoolean()
         var rName = RName(rootName)
@@ -41,8 +42,7 @@ object RNameMarshaller {
 }
 
 internal fun IRdDynamic.createExtSignal(): RdSignal<ExtCreationInfo> {
-    val marshaller = FrameworkMarshallers.create(
-        { buffer ->
+    val marshaller = create(ExtCreationInfo::class, { buffer ->
             val rName = RNameMarshaller.read(buffer)
             val rdId = buffer.readNullable { buffer.readRdId() }
             val hash = buffer.readLong()

rd-kt/rd-framework/src/main/kotlin/com/jetbrains/rd/framework/Serializers.kt

@@ -3,8 +3,11 @@ package com.jetbrains.rd.framework
 import com.jetbrains.rd.framework.base.ISerializersOwner
 import com.jetbrains.rd.framework.impl.RdSecureString
 import com.jetbrains.rd.util.*
+import com.jetbrains.rd.util.hash.getPlatformIndependentHash
 import com.jetbrains.rd.util.lifetime.Lifetime
+import java.util.concurrent.TimeUnit
 import kotlin.reflect.KClass
+import kotlin.reflect.jvm.jvmName
 import kotlin.time.Duration
 import kotlin.time.DurationUnit
 import kotlin.time.toDuration
@@ -33,9 +36,8 @@ class Serializers : ISerializers {
 
 
 
-    val types = hashMapOf<RdId, KClass<*>>()
-    val writers = hashMapOf<KClass<*>, Pair<RdId, (SerializationCtx, AbstractBuffer, Any) -> Unit>>()
-    val marshallers = hashMapOf<RdId, IMarshaller<*>>()
+    private val writers = ConcurrentHashMap<KClass<*>, IMarshaller<*>>()
+    private val marshallers = ConcurrentHashMap<RdId, IMarshaller<*>>()
 
     init {
         backgroundRegistrar.invokeOrQueue {
@@ -50,19 +52,15 @@ class Serializers : ISerializers {
         }
 
         val id = serializer.id
-        val t = serializer._type
-
-        Protocol.initializationLogger.trace { "Registering type ${t.simpleName}, id = $id" }
-
-        val existing = types[id]
+        val existing = marshallers[id]
         if (existing != null) {
-            require(existing == t) { "Can't register ${t.simpleName} with id: $id, already registered: ${existing.simpleName}" }
-        } else {
-            types[id] = t
+            require(existing.fqn == serializer.fqn) { "Can't register ${serializer.fqn} with id: $id, already registered: ${serializer.fqn}" }
+        }  else {
+            Protocol.initializationLogger.trace { "Registering type ${serializer.fqn}, id = $id" }
+            marshallers[id] = serializer
+            if (serializer !is LazyCompanionMarshaller)
+                writers[serializer._type] = serializer
         }
-
-        marshallers[id] = serializer
-        writers[t] = Pair(id, serializer::write) as Pair<RdId, (SerializationCtx, AbstractBuffer, Any) -> Unit>
     }
 
     override fun get(id: RdId): IMarshaller<*>? {
@@ -102,22 +100,34 @@ class Serializers : ISerializers {
                 ?: throw IllegalStateException("Non-null object expected")
     }
 
+    private fun <T : Any> getWriter(clazz: KClass<out T>): IMarshaller<T> {
+        val marshaller = writers.getOrPut(clazz) {
+            val id = RdId(clazz.simpleName.getPlatformIndependentHash())
+            marshallers[id] ?: cantFindWriter(clazz)
+        }
+
+        return marshaller as? IMarshaller<T> ?: cantFindWriter(clazz)
+    }
+
+    private fun <T : Any> cantFindWriter(clazz: KClass<T>): Nothing {
+        throw IllegalStateException("Can't find writer by class: ${clazz}. $notRegisteredErrorMessage")
+    }
+
     override fun <T : Any> writePolymorphic(ctx: SerializationCtx, stream: AbstractBuffer, value: T) {
         backgroundRegistrar.flush()
 
-        val (id, writer) = writers[value::class]
-                ?: throw IllegalStateException("Can't find writer by class: ${value::class}. $notRegisteredErrorMessage")
+        val serializer = getWriter(value::class)
 
         if (value is IUnknownInstance) {
             value.unknownId.write(stream)
         } else {
-            id.write(stream)
+            serializer.id.write(stream)
         }
 
         val lengthTagPosition = stream.position
         stream.writeInt(0)
         val objectStartPosition = stream.position
-        writer(ctx, stream, value)
+        serializer.write(ctx, stream, value)
         val objectEndPosition = stream.position
         stream.position = lengthTagPosition
         stream.writeInt(objectEndPosition - objectStartPosition)

rd-kt/rd-gen/src/main/kotlin/com/jetbrains/rd/generator/nova/RdGen.kt

@@ -22,7 +22,7 @@ class RdGen : Kli() {
         /**
          * Moving this field forward you trigger rebuild even if inputs and output of generator hasn't changed.
          */
-        const val version = "1.12"
+        const val version = "1.13"
 
         /**
          * File to store all information for incremental work