From c4fe8184e99f5572a15ce51bce0c606119c63585 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Thu, 28 Mar 2024 08:50:20 +0100 Subject: [PATCH 1/3] Fixes a few warnings. --- .../vitrivr/cottontail/math/knn/FloatVectorDistanceTest.kt | 6 +++--- cottontaildb-ui-backend/src/main/kotlin/Main.kt | 2 +- .../vitrivr/cottontail/ui/utilities/KotlinxJsonMapper.kt | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cottontaildb-dbms/src/test/kotlin/org/vitrivr/cottontail/math/knn/FloatVectorDistanceTest.kt b/cottontaildb-dbms/src/test/kotlin/org/vitrivr/cottontail/math/knn/FloatVectorDistanceTest.kt index 23c78634c..ec56ec888 100644 --- a/cottontaildb-dbms/src/test/kotlin/org/vitrivr/cottontail/math/knn/FloatVectorDistanceTest.kt +++ b/cottontaildb-dbms/src/test/kotlin/org/vitrivr/cottontail/math/knn/FloatVectorDistanceTest.kt @@ -5,11 +5,11 @@ import org.junit.jupiter.params.provider.MethodSource import org.vitrivr.cottontail.core.queries.functions.math.distance.binary.* import org.vitrivr.cottontail.core.queries.functions.math.distance.ternary.HyperplaneDistance import org.vitrivr.cottontail.core.queries.functions.math.distance.ternary.WeightedManhattanDistance +import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.core.values.FloatValue import org.vitrivr.cottontail.core.values.FloatVectorValue import org.vitrivr.cottontail.core.values.generators.FloatValueGenerator import org.vitrivr.cottontail.core.values.generators.FloatVectorValueGenerator -import org.vitrivr.cottontail.core.types.Types import org.vitrivr.cottontail.math.isApproximatelyTheSame import org.vitrivr.cottontail.test.TestConstants import org.vitrivr.cottontail.utilities.VectorUtility @@ -53,7 +53,7 @@ class FloatVectorDistanceTest : AbstractDistanceTest() { sum1 += kernel(query, it).value.toFloat() } time2 += measureTime { - sum2 += kernelVectorised(query, it)!!.value.toFloat() + sum2 += kernelVectorised(query, it).value.toFloat() } time3 += measureTime { sum3 += (query - it).abs().sum().value @@ -142,7 +142,7 @@ class FloatVectorDistanceTest : AbstractDistanceTest() { sum1 += kernel(query, it).value.toFloat() } time2 += measureTime { - sum2 += kernelVectorised(query, it)!!.value.toFloat() + sum2 += kernelVectorised(query, it).value.toFloat() } time3 += measureTime { sum3 += (query - it).pow(2).sum().sqrt().value diff --git a/cottontaildb-ui-backend/src/main/kotlin/Main.kt b/cottontaildb-ui-backend/src/main/kotlin/Main.kt index b52974ac6..c4871c0c5 100644 --- a/cottontaildb-ui-backend/src/main/kotlin/Main.kt +++ b/cottontaildb-ui-backend/src/main/kotlin/Main.kt @@ -38,7 +38,7 @@ fun fileSessionHandler() = SessionHandler().apply { /** * */ -fun main(args: Array) { +fun main() { Javalin.create { config -> config.staticFiles.add{ it.directory = "html" diff --git a/cottontaildb-ui-backend/src/main/kotlin/org/vitrivr/cottontail/ui/utilities/KotlinxJsonMapper.kt b/cottontaildb-ui-backend/src/main/kotlin/org/vitrivr/cottontail/ui/utilities/KotlinxJsonMapper.kt index f803b6d9d..78a38dbd3 100644 --- a/cottontaildb-ui-backend/src/main/kotlin/org/vitrivr/cottontail/ui/utilities/KotlinxJsonMapper.kt +++ b/cottontaildb-ui-backend/src/main/kotlin/org/vitrivr/cottontail/ui/utilities/KotlinxJsonMapper.kt @@ -67,6 +67,7 @@ object KotlinxJsonMapper: JsonMapper { * @param targetType The target [Type] * @return Object [T] */ + @Suppress("UNCHECKED_CAST") override fun fromJsonString(json: String, targetType: Type): T { val deserializer = this.projectModule.serializer(targetType) return this.json.decodeFromString(deserializer, json) as T From 21e52299ccebc42eec0be29080ff657e4d9922a4 Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Thu, 28 Mar 2024 08:51:01 +0100 Subject: [PATCH 2/3] Fixes a naming issue between simple client and protocol buffer definition. --- .../cli/entity/CreateEntityCommand.kt | 2 +- .../cli/entity/ImportDataCommand.kt | 88 +++++++++---------- .../vitrivr/cottontail/core/DataExtensions.kt | 8 +- .../vitrivr/cottontail/core/types/Types.kt | 2 +- .../src/main/protobuf/cottontail.proto | 4 +- 5 files changed, 51 insertions(+), 53 deletions(-) diff --git a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/CreateEntityCommand.kt b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/CreateEntityCommand.kt index 65841ecfb..5430e9db3 100644 --- a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/CreateEntityCommand.kt +++ b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/CreateEntityCommand.kt @@ -148,7 +148,7 @@ class CreateEntityCommand(client: SimpleClient) : AbstractEntityCommand(client, "LONG_VECTOR", "FLOAT_VECTOR", "DOUBLE_VECTOR", - "BOOL_VECTOR", + "BOOLEAN_VECTOR", "COMPLEX32_VECTOR", "COMPLEX64_VECTOR" -> this.terminal.prompt("\rColumn lengths (i.e., number of entries for vectors)")?.toIntOrNull() ?: 1 else -> -1 diff --git a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/ImportDataCommand.kt b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/ImportDataCommand.kt index b4babc1c9..90efd3d65 100644 --- a/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/ImportDataCommand.kt +++ b/cottontaildb-cli/src/main/kotlin/org/vitrivr/cottontail/cli/entity/ImportDataCommand.kt @@ -59,61 +59,59 @@ class ImportDataCommand(client: SimpleClient) : AbstractEntityCommand(client, na override fun exec() { /* Read schema and prepare Iterator. */ val schema = this.client.readSchema(this.entityName).toTypedArray() - val data: Sequence = when(format) { - Format.CBOR -> Cbor.decodeFromByteArray(ListSerializer(schema.serializer()), Files.readAllBytes(this.input)).asSequence() - Format.JSON -> Files.newInputStream(this.input).use { - Json.decodeToSequence(it, schema.serializer(), DecodeSequenceMode.ARRAY_WRAPPED) + Files.newInputStream(this.input).use { input -> + val data: Sequence = when(format) { + Format.CBOR -> Cbor.decodeFromByteArray(ListSerializer(schema.serializer()), input.readAllBytes()).asSequence() + Format.JSON -> Json.decodeToSequence(input, schema.serializer(), DecodeSequenceMode.ARRAY_WRAPPED) + Format.CSV -> Csv.decodeFromString(ListSerializer(schema.descriptionSerializer()), input.readAllBytes().toString()).asSequence() } - Format.CSV ->Files.newInputStream(this.input).use { - Csv.decodeFromString(ListSerializer(schema.descriptionSerializer()), it.readAllBytes().toString()) - }.asSequence() - } - - /** Begin transaction (if single transaction option has been set). */ - val txId = if (this.singleTransaction) { - this.client.begin() - } else { - null - } - try { - /* Prepare batch insert message. */ - val batchedInsert = BatchInsert(this.entityName.fqn) - if (txId != null) { - batchedInsert.txId(txId) + /** Begin transaction (if single transaction option has been set). */ + val txId = if (this.singleTransaction) { + this.client.begin() + } else { + null } - batchedInsert.columns(*schema.map { it.name.simple }.toTypedArray()) - var count = 0L - val duration = measureTime { - for (t in data) { - if (!batchedInsert.values(*t.values().mapNotNull { it as? PublicValue }.toTypedArray())) { - /* Execute insert... */ - client.insert(batchedInsert) - /* ... now clear and append. */ - batchedInsert.clear() - if (!batchedInsert.any(*t.values().mapNotNull { it as? PublicValue }.toTypedArray())) { - throw IllegalArgumentException("The appended data is too large for a single message.") + try { + /* Prepare batch insert message. */ + val batchedInsert = BatchInsert(this.entityName.fqn) + if (txId != null) { + batchedInsert.txId(txId) + } + batchedInsert.columns(*schema.map { it.name.simple }.toTypedArray()) + var count = 0L + val duration = measureTime { + for (t in data) { + if (!batchedInsert.values(*t.values().mapNotNull { it as? PublicValue }.toTypedArray())) { + /* Execute insert... */ + client.insert(batchedInsert) + + /* ... now clear and append. */ + batchedInsert.clear() + if (!batchedInsert.any(*t.values().mapNotNull { it as? PublicValue }.toTypedArray())) { + throw IllegalArgumentException("The appended data is too large for a single message.") + } } + count += 1 } - count += 1 - } - /** Insert remainder. */ - if (batchedInsert.count() > 0) { - this.client.insert(batchedInsert) - } + /** Insert remainder. */ + if (batchedInsert.count() > 0) { + this.client.insert(batchedInsert) + } - /** Commit transaction, if single transaction option has been set. */ - if (txId != null) { - this.client.commit(txId) + /** Commit transaction, if single transaction option has been set. */ + if (txId != null) { + this.client.commit(txId) + } } + println("Importing $count entries into $entityName took $duration.") + } catch (e: Throwable) { + /** Rollback transaction, if single transaction option has been set. */ + if (txId != null) this.client.rollback(txId) + println("Importing entries into $entityName failed due to error: ${e.message}") } - println("Importing $count entries into $entityName took $duration.") - } catch (e: Throwable) { - /** Rollback transaction, if single transaction option has been set. */ - if (txId != null) this.client.rollback(txId) - println("Importing entries into $entityName failed due to error: ${e.message}") } } } \ No newline at end of file diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DataExtensions.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DataExtensions.kt index f37563160..c1d36285c 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DataExtensions.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/DataExtensions.kt @@ -189,8 +189,8 @@ fun CottontailGrpc.Type.toType(size: Int = 0): Types<*> = when(this) { CottontailGrpc.Type.DOUBLE_VECTOR -> Types.DoubleVector(size) CottontailGrpc.Type.FLOAT_VECTOR -> Types.FloatVector(size) CottontailGrpc.Type.LONG_VECTOR -> Types.LongVector(size) - CottontailGrpc.Type.INT_VECTOR -> Types.IntVector(size) - CottontailGrpc.Type.BOOL_VECTOR -> Types.BooleanVector(size) + CottontailGrpc.Type.INTEGER_VECTOR -> Types.IntVector(size) + CottontailGrpc.Type.BOOLEAN_VECTOR -> Types.BooleanVector(size) CottontailGrpc.Type.COMPLEX32_VECTOR -> Types.Complex32Vector(size) CottontailGrpc.Type.COMPLEX64_VECTOR -> Types.Complex64Vector(size) CottontailGrpc.Type.SHORT_VECTOR -> Types.ShortVector(size) @@ -216,8 +216,8 @@ fun Types<*>.proto(): CottontailGrpc.Type = when(this) { Types.Short -> CottontailGrpc.Type.SHORT Types.String -> CottontailGrpc.Type.STRING Types.Uuid -> CottontailGrpc.Type.UUID - is Types.BooleanVector -> CottontailGrpc.Type.BOOL_VECTOR - is Types.IntVector -> CottontailGrpc.Type.INT_VECTOR + is Types.BooleanVector -> CottontailGrpc.Type.BOOLEAN_VECTOR + is Types.IntVector -> CottontailGrpc.Type.INTEGER_VECTOR is Types.LongVector -> CottontailGrpc.Type.LONG_VECTOR is Types.FloatVector -> CottontailGrpc.Type.FLOAT_VECTOR is Types.DoubleVector -> CottontailGrpc.Type.DOUBLE_VECTOR diff --git a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/types/Types.kt b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/types/Types.kt index ed09266a0..fb72fab2a 100644 --- a/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/types/Types.kt +++ b/cottontaildb-client/src/main/kotlin/org/vitrivr/cottontail/core/types/Types.kt @@ -44,7 +44,7 @@ sealed class Types { "LONG_VECTOR" -> LongVector(logicalSize) "FLOAT_VECTOR" -> FloatVector(logicalSize) "DOUBLE_VECTOR" -> DoubleVector(logicalSize) - "BOOL_VECTOR" -> BooleanVector(logicalSize) + "BOOLEAN_VECTOR" -> BooleanVector(logicalSize) "COMPLEX32_VECTOR" -> Complex32Vector(logicalSize) "COMPLEX64_VECTOR" -> Complex64Vector(logicalSize) "BYTESTRING" -> ByteString diff --git a/cottontaildb-client/src/main/protobuf/cottontail.proto b/cottontaildb-client/src/main/protobuf/cottontail.proto index 4069fdf46..f4018ec6f 100644 --- a/cottontaildb-client/src/main/protobuf/cottontail.proto +++ b/cottontaildb-client/src/main/protobuf/cottontail.proto @@ -224,8 +224,8 @@ enum Type { DOUBLE_VECTOR = 30; FLOAT_VECTOR = 31; LONG_VECTOR = 32; - INT_VECTOR = 33; - BOOL_VECTOR = 34; + INTEGER_VECTOR = 33; + BOOLEAN_VECTOR = 34; COMPLEX32_VECTOR = 35; COMPLEX64_VECTOR = 36; SHORT_VECTOR = 37; From e57ce7dafbb3e115cddd2b521729b46319f814bd Mon Sep 17 00:00:00 2001 From: Ralph Gasser Date: Thu, 28 Mar 2024 08:58:06 +0100 Subject: [PATCH 3/3] Minor changes to build.gradle file. --- cottontaildb-dbms/build.gradle | 4 ++-- gradle.properties | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cottontaildb-dbms/build.gradle b/cottontaildb-dbms/build.gradle index 0be7181be..ab9325cff 100644 --- a/cottontaildb-dbms/build.gradle +++ b/cottontaildb-dbms/build.gradle @@ -7,7 +7,7 @@ apply plugin: 'signing' /* Cottontail DB dependencies. */ dependencies { /* The Cottontail DB core, which includes the client library. */ - api project(':cottontaildb-core') + implementation project(':cottontaildb-core') /* Xodus dependency. */ implementation group: 'org.jetbrains.xodus', name: 'xodus-openAPI', version: version_xodus @@ -16,7 +16,7 @@ dependencies { /* Snappy */ implementation group: 'org.xerial.snappy', name: 'snappy-java', version: version_snappy - implementation group: 'org.lz4', name: 'lz4-java', version: '1.8.0' + implementation group: 'org.lz4', name: 'lz4-java', version: version_lz4 /* FastUtil */ implementation group: 'it.unimi.dsi', name: 'fastutil', version: version_fastutil diff --git a/gradle.properties b/gradle.properties index 86e30202d..2e57c5ce1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -20,6 +20,7 @@ version_kotlinx_serialization=1.6.2 version_kotlinx_serialization_csv=2.0.0 version_log4j2=2.22.0 version_lucene=9.5.0 +version_lz4=1.8.0 version_picnic=0.6.0 version_protobuf=3.25.1 version_slf4j=2.0.9