From 924a8db58f6848f7d385a06dc0faa13ac31236e6 Mon Sep 17 00:00:00 2001 From: Russell Wolf Date: Sat, 12 Mar 2022 13:26:18 -0500 Subject: [PATCH] WIP LMDB implementation --- .github/workflows/build-linux.yml | 4 + .../api/multiplatform-settings.klib.api | 29 +++ multiplatform-settings/build.gradle.kts | 9 + .../com/russhwolf/settings/LmdbSettings.kt | 184 ++++++++++++++++++ .../russhwolf/settings/LmdbSettingsTest.kt | 30 +++ .../src/nativeInterop/cinterop/lmdb.def | 4 + 6 files changed, 260 insertions(+) create mode 100644 multiplatform-settings/src/linuxX64Main/kotlin/com/russhwolf/settings/LmdbSettings.kt create mode 100644 multiplatform-settings/src/linuxX64Test/kotlin/com/russhwolf/settings/LmdbSettingsTest.kt create mode 100644 multiplatform-settings/src/nativeInterop/cinterop/lmdb.def diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index f214f13f..29b264a3 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -25,6 +25,10 @@ jobs: restore-keys: | ${{ runner.os }}-konan- + - name: LMDB install + run: | + sudo apt-get install liblmdb-dev + - name: Linux build run: | ./gradlew build publishToMavenLocal --no-daemon --stacktrace diff --git a/multiplatform-settings/api/multiplatform-settings.klib.api b/multiplatform-settings/api/multiplatform-settings.klib.api index e16a837f..1c78b7b2 100644 --- a/multiplatform-settings/api/multiplatform-settings.klib.api +++ b/multiplatform-settings/api/multiplatform-settings.klib.api @@ -202,6 +202,35 @@ final class com.russhwolf.settings/StorageSettings : com.russhwolf.settings/Sett final val size // com.russhwolf.settings/StorageSettings.size|{}size[0] final fun (): kotlin/Int // com.russhwolf.settings/StorageSettings.size.|(){}[0] } +// Targets: [linuxX64] +final class com.russhwolf.settings/LmdbSettings : com.russhwolf.settings/Settings { // com.russhwolf.settings/LmdbSettings|null[0] + constructor (kotlin/String) // com.russhwolf.settings/LmdbSettings.|(kotlin.String){}[0] + final fun clear() // com.russhwolf.settings/LmdbSettings.clear|clear(){}[0] + final fun getBoolean(kotlin/String, kotlin/Boolean): kotlin/Boolean // com.russhwolf.settings/LmdbSettings.getBoolean|getBoolean(kotlin.String;kotlin.Boolean){}[0] + final fun getBooleanOrNull(kotlin/String): kotlin/Boolean? // com.russhwolf.settings/LmdbSettings.getBooleanOrNull|getBooleanOrNull(kotlin.String){}[0] + final fun getDouble(kotlin/String, kotlin/Double): kotlin/Double // com.russhwolf.settings/LmdbSettings.getDouble|getDouble(kotlin.String;kotlin.Double){}[0] + final fun getDoubleOrNull(kotlin/String): kotlin/Double? // com.russhwolf.settings/LmdbSettings.getDoubleOrNull|getDoubleOrNull(kotlin.String){}[0] + final fun getFloat(kotlin/String, kotlin/Float): kotlin/Float // com.russhwolf.settings/LmdbSettings.getFloat|getFloat(kotlin.String;kotlin.Float){}[0] + final fun getFloatOrNull(kotlin/String): kotlin/Float? // com.russhwolf.settings/LmdbSettings.getFloatOrNull|getFloatOrNull(kotlin.String){}[0] + final fun getInt(kotlin/String, kotlin/Int): kotlin/Int // com.russhwolf.settings/LmdbSettings.getInt|getInt(kotlin.String;kotlin.Int){}[0] + final fun getIntOrNull(kotlin/String): kotlin/Int? // com.russhwolf.settings/LmdbSettings.getIntOrNull|getIntOrNull(kotlin.String){}[0] + final fun getLong(kotlin/String, kotlin/Long): kotlin/Long // com.russhwolf.settings/LmdbSettings.getLong|getLong(kotlin.String;kotlin.Long){}[0] + final fun getLongOrNull(kotlin/String): kotlin/Long? // com.russhwolf.settings/LmdbSettings.getLongOrNull|getLongOrNull(kotlin.String){}[0] + final fun getString(kotlin/String, kotlin/String): kotlin/String // com.russhwolf.settings/LmdbSettings.getString|getString(kotlin.String;kotlin.String){}[0] + final fun getStringOrNull(kotlin/String): kotlin/String? // com.russhwolf.settings/LmdbSettings.getStringOrNull|getStringOrNull(kotlin.String){}[0] + final fun hasKey(kotlin/String): kotlin/Boolean // com.russhwolf.settings/LmdbSettings.hasKey|hasKey(kotlin.String){}[0] + final fun putBoolean(kotlin/String, kotlin/Boolean) // com.russhwolf.settings/LmdbSettings.putBoolean|putBoolean(kotlin.String;kotlin.Boolean){}[0] + final fun putDouble(kotlin/String, kotlin/Double) // com.russhwolf.settings/LmdbSettings.putDouble|putDouble(kotlin.String;kotlin.Double){}[0] + final fun putFloat(kotlin/String, kotlin/Float) // com.russhwolf.settings/LmdbSettings.putFloat|putFloat(kotlin.String;kotlin.Float){}[0] + final fun putInt(kotlin/String, kotlin/Int) // com.russhwolf.settings/LmdbSettings.putInt|putInt(kotlin.String;kotlin.Int){}[0] + final fun putLong(kotlin/String, kotlin/Long) // com.russhwolf.settings/LmdbSettings.putLong|putLong(kotlin.String;kotlin.Long){}[0] + final fun putString(kotlin/String, kotlin/String) // com.russhwolf.settings/LmdbSettings.putString|putString(kotlin.String;kotlin.String){}[0] + final fun remove(kotlin/String) // com.russhwolf.settings/LmdbSettings.remove|remove(kotlin.String){}[0] + final val keys // com.russhwolf.settings/LmdbSettings.keys|{}keys[0] + final fun (): kotlin.collections/Set // com.russhwolf.settings/LmdbSettings.keys.|(){}[0] + final val size // com.russhwolf.settings/LmdbSettings.size|{}size[0] + final fun (): kotlin/Int // com.russhwolf.settings/LmdbSettings.size.|(){}[0] +} // Targets: [mingwX64] final class com.russhwolf.settings/RegistrySettings : com.russhwolf.settings/Settings { // com.russhwolf.settings/RegistrySettings|null[0] constructor (kotlin/String) // com.russhwolf.settings/RegistrySettings.|(kotlin.String){}[0] diff --git a/multiplatform-settings/build.gradle.kts b/multiplatform-settings/build.gradle.kts index 8d1fc281..85e86ad9 100644 --- a/multiplatform-settings/build.gradle.kts +++ b/multiplatform-settings/build.gradle.kts @@ -17,6 +17,8 @@ import org.jetbrains.kotlin.gradle.targets.js.dsl.ExperimentalWasmDsl * limitations under the License. */ +import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget + plugins { id("standard-configuration") id("module-publication") @@ -27,6 +29,13 @@ standardConfig { } kotlin { + targets.getByName("linuxX64") { + compilations["main"].cinterops.create("lmdb") + binaries.configureEach { + // lmdb appears to be using a newer gcc than Kotlin. This lets us still work as long as host has newer gcc too + linkerOpts += "--allow-shlib-undefined" + } + } @OptIn(ExperimentalKotlinGradlePluginApi::class) compilerOptions { freeCompilerArgs.add("-Xexpect-actual-classes") diff --git a/multiplatform-settings/src/linuxX64Main/kotlin/com/russhwolf/settings/LmdbSettings.kt b/multiplatform-settings/src/linuxX64Main/kotlin/com/russhwolf/settings/LmdbSettings.kt new file mode 100644 index 00000000..f3b3bb2d --- /dev/null +++ b/multiplatform-settings/src/linuxX64Main/kotlin/com/russhwolf/settings/LmdbSettings.kt @@ -0,0 +1,184 @@ +/* + * Copyright 2022 Russell Wolf + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.russhwolf.settings + +import cnames.structs.MDB_cursor +import cnames.structs.MDB_env +import cnames.structs.MDB_txn +import com.russhwolf.settings.cinterop.lmdb.MDB_CREATE +import com.russhwolf.settings.cinterop.lmdb.MDB_NOTFOUND +import com.russhwolf.settings.cinterop.lmdb.MDB_SUCCESS +import com.russhwolf.settings.cinterop.lmdb.MDB_cursor_op +import com.russhwolf.settings.cinterop.lmdb.MDB_dbi +import com.russhwolf.settings.cinterop.lmdb.MDB_dbiVar +import com.russhwolf.settings.cinterop.lmdb.MDB_stat +import com.russhwolf.settings.cinterop.lmdb.MDB_val +import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_close +import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_del +import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_get +import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_open +import com.russhwolf.settings.cinterop.lmdb.mdb_dbi_close +import com.russhwolf.settings.cinterop.lmdb.mdb_dbi_open +import com.russhwolf.settings.cinterop.lmdb.mdb_del +import com.russhwolf.settings.cinterop.lmdb.mdb_env_close +import com.russhwolf.settings.cinterop.lmdb.mdb_env_create +import com.russhwolf.settings.cinterop.lmdb.mdb_env_open +import com.russhwolf.settings.cinterop.lmdb.mdb_get +import com.russhwolf.settings.cinterop.lmdb.mdb_put +import com.russhwolf.settings.cinterop.lmdb.mdb_stat +import com.russhwolf.settings.cinterop.lmdb.mdb_strerror +import com.russhwolf.settings.cinterop.lmdb.mdb_txn_begin +import com.russhwolf.settings.cinterop.lmdb.mdb_txn_commit +import kotlinx.cinterop.ByteVar +import kotlinx.cinterop.CPointer +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.MemScope +import kotlinx.cinterop.alloc +import kotlinx.cinterop.allocPointerTo +import kotlinx.cinterop.cstr +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr +import kotlinx.cinterop.reinterpret +import kotlinx.cinterop.toKString +import kotlinx.cinterop.value +import platform.posix.S_IRWXG +import platform.posix.S_IRWXO +import platform.posix.S_IRWXU +import platform.posix.mkdir +import kotlin.experimental.ExperimentalNativeApi + +@OptIn(ExperimentalForeignApi::class) +@ExperimentalSettingsImplementation +public class LmdbSettings(private val path: String) : Settings { + override val keys: Set + get() = buildSet { + lmdbCursorTransaction { _, key -> + add(key.mv_data!!.reinterpret().toKString()) + } + } + + override val size: Int + get() = lmdbTransaction { txn, dbi -> + val stat = alloc() + mdb_stat(txn, dbi, stat.ptr) + stat.ms_entries.toInt() + } + + public override fun clear(): Unit = lmdbCursorTransaction { cursor, _ -> + mdb_cursor_del(cursor, 0.toUInt()) + } + + public override fun remove(key: String): Unit = + lmdbTransaction { txn, dbi -> mdb_del(txn, dbi, createMdbVal(key).ptr, null).checkError(MDB_NOTFOUND) } + + public override fun hasKey(key: String): Boolean = loadString(key) != null + + public override fun putInt(key: String, value: Int): Unit = saveString(key, value.toString()) + public override fun getInt(key: String, defaultValue: Int): Int = getIntOrNull(key) ?: defaultValue + public override fun getIntOrNull(key: String): Int? = loadString(key)?.toInt() + + public override fun putLong(key: String, value: Long): Unit = saveString(key, value.toString()) + public override fun getLong(key: String, defaultValue: Long): Long = getLongOrNull(key) ?: defaultValue + public override fun getLongOrNull(key: String): Long? = loadString(key)?.toLong() + + public override fun putString(key: String, value: String): Unit = saveString(key, value) + public override fun getString(key: String, defaultValue: String): String = getStringOrNull(key) ?: defaultValue + public override fun getStringOrNull(key: String): String? = loadString(key) + + public override fun putFloat(key: String, value: Float): Unit = saveString(key, value.toString()) + public override fun getFloat(key: String, defaultValue: Float): Float = getFloatOrNull(key) ?: defaultValue + public override fun getFloatOrNull(key: String): Float? = loadString(key)?.toFloat() + + public override fun putDouble(key: String, value: Double): Unit = saveString(key, value.toString()) + public override fun getDouble(key: String, defaultValue: Double): Double = getDoubleOrNull(key) ?: defaultValue + public override fun getDoubleOrNull(key: String): Double? = loadString(key)?.toDouble() + + public override fun putBoolean(key: String, value: Boolean): Unit = saveString(key, value.toString()) + public override fun getBoolean(key: String, defaultValue: Boolean): Boolean = getBooleanOrNull(key) ?: defaultValue + public override fun getBooleanOrNull(key: String): Boolean? = loadString(key)?.toBoolean() + + private fun saveString(key: String, value: String): Unit = lmdbTransaction { txn, dbi -> + mdb_put(txn, dbi, createMdbVal(key).ptr, createMdbVal(value).ptr, 0.toUInt()).checkError() + } + + private fun loadString(key: String): String? = lmdbTransaction { txn, dbi -> + val value = alloc() + mdb_get(txn, dbi, createMdbVal(key).ptr, value.ptr).checkError(MDB_NOTFOUND) + value.mv_data?.reinterpret()?.toKString() + } + + private fun lmdbCursorTransaction(action: MemScope.(cursor: CPointer?, key: MDB_val) -> Unit) = + lmdbTransaction { txn, dbi -> + val cursor = allocPointerTo() + mdb_cursor_open(txn, dbi, cursor.ptr).checkError() + val mdbKey = alloc() + val mdbValue = alloc() + mdb_cursor_get(cursor.value, mdbKey.ptr, mdbValue.ptr, MDB_cursor_op.MDB_FIRST).checkError( + MDB_NOTFOUND + ) + while (true) { + if (mdbKey.mv_data == null) { + break + } else { + action(cursor.value, mdbKey) + } + mdbKey.mv_data = null + mdbKey.mv_size = 0u + mdb_cursor_get(cursor.value, mdbKey.ptr, mdbValue.ptr, MDB_cursor_op.MDB_NEXT).checkError( + MDB_NOTFOUND + ) + } + mdb_cursor_close(cursor.value) + + } + + private fun MemScope.createMdbVal(string: String): MDB_val { + val mdbVal = alloc() + val cstr = string.cstr + mdbVal.mv_data = cstr.ptr + mdbVal.mv_size = cstr.size.toULong() + return mdbVal + } + + private inline fun lmdbTransaction(action: MemScope.(txn: CPointer?, dbi: MDB_dbi) -> T): T = + memScoped { + val env = allocPointerTo() + mdb_env_create(env.ptr).checkError() + val mode = (S_IRWXU or S_IRWXG or S_IRWXO).toUInt() + mkdir(path, mode) + mdb_env_open(env.value, path, 0.toUInt(), mode).checkError() + + val txn = allocPointerTo() + mdb_txn_begin(env.value, null, 0.toUInt(), txn.ptr).checkError() + + val dbi = alloc() + mdb_dbi_open(txn.value, null, MDB_CREATE.toUInt(), dbi.ptr).checkError() + + val out = action(txn.value, dbi.value) + + mdb_dbi_close(env.value, dbi.value) + mdb_txn_commit(txn.value).checkError() + mdb_env_close(env.value) + + out + } + + private fun Int.checkError(vararg expectedErrors: Int) { + @OptIn(ExperimentalNativeApi::class) + assert(this == MDB_SUCCESS || this in expectedErrors) { "Error: ${mdb_strerror(this)?.toKString()}" } + } +} diff --git a/multiplatform-settings/src/linuxX64Test/kotlin/com/russhwolf/settings/LmdbSettingsTest.kt b/multiplatform-settings/src/linuxX64Test/kotlin/com/russhwolf/settings/LmdbSettingsTest.kt new file mode 100644 index 00000000..3edf4554 --- /dev/null +++ b/multiplatform-settings/src/linuxX64Test/kotlin/com/russhwolf/settings/LmdbSettingsTest.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Russell Wolf + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.russhwolf.settings + +@OptIn(ExperimentalSettingsImplementation::class) +public class LmdbSettingsTest + : BaseSettingsTest( + platformFactory = object : Settings.Factory { + override fun create(name: String?): Settings = LmdbSettings(name ?: "lmdbTest") + }, + hasListeners = false +) { + // TODO add test cases to verify that we write to the files we think we do + + // TODO add cleanup methods so we don't leave test DBs lying around +} diff --git a/multiplatform-settings/src/nativeInterop/cinterop/lmdb.def b/multiplatform-settings/src/nativeInterop/cinterop/lmdb.def new file mode 100644 index 00000000..c1343be1 --- /dev/null +++ b/multiplatform-settings/src/nativeInterop/cinterop/lmdb.def @@ -0,0 +1,4 @@ +headers = lmdb.h +package = com.russhwolf.settings.cinterop.lmdb +compilerOpts = -I/usr/include -I/usr/include/x86_64-linux-gnu/ +linkerOpts = -llmdb -L/usr/lib -L/usr/lib/x86_64-linux-gnu/