Working DBM prototype via QDBM

azure-pipelines.yml

@@ -20,6 +20,12 @@ jobs:
             mkdir -p ./jdk/binaries/
             curl https://cdn.azul.com/zulu/bin/zulu8.42.0.23-ca-fx-jdk8.0.232-linux_x64.tar.gz --output ./jdk/zulu8.42.0.23-ca-fx-jdk8.0.232-linux_x64.tar.gz
 
+      - task: Bash@3
+        inputs:
+          targetType: 'inline'
+          script: |
+            sudo apt-get install libqdbm-dev
+
       - task: JavaToolInstaller@0
         inputs:
           jdkFile: ./jdk/zulu8.42.0.23-ca-fx-jdk8.0.232-linux_x64.tar.gz

multiplatform-settings/build.gradle.kts

@@ -15,6 +15,7 @@
  */
 
 import com.russhwolf.settings.build.standardConfiguration
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
 
 plugins {
     id("com.android.library")
@@ -27,6 +28,9 @@ plugins {
 standardConfiguration()
 
 kotlin {
+    targets.getByName<KotlinNativeTarget>("linuxX64") {
+        compilations["main"].cinterops.create("qdbm")
+    }
     sourceSets {
         commonMain {
             dependencies {

multiplatform-settings/src/linuxX64Main/kotlin/DbmSettings.kt

@@ -0,0 +1,166 @@
+/*
+ * Copyright 2020 Russell Wolf
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.russhwolf.settings.Settings
+import com.russhwolf.settings.cinterop.qdbm.DBM
+import com.russhwolf.settings.cinterop.qdbm.DBM_REPLACE
+import com.russhwolf.settings.cinterop.qdbm.datum
+import com.russhwolf.settings.cinterop.qdbm.dbm_clearerr
+import com.russhwolf.settings.cinterop.qdbm.dbm_close
+import com.russhwolf.settings.cinterop.qdbm.dbm_delete
+import com.russhwolf.settings.cinterop.qdbm.dbm_error
+import com.russhwolf.settings.cinterop.qdbm.dbm_fetch
+import com.russhwolf.settings.cinterop.qdbm.dbm_firstkey
+import com.russhwolf.settings.cinterop.qdbm.dbm_nextkey
+import com.russhwolf.settings.cinterop.qdbm.dbm_open
+import com.russhwolf.settings.cinterop.qdbm.dbm_store
+import kotlinx.cinterop.ByteVar
+import kotlinx.cinterop.CPointer
+import kotlinx.cinterop.CValue
+import kotlinx.cinterop.MemScope
+import kotlinx.cinterop.cValue
+import kotlinx.cinterop.cstr
+import kotlinx.cinterop.memScoped
+import kotlinx.cinterop.plus
+import kotlinx.cinterop.pointed
+import kotlinx.cinterop.reinterpret
+import kotlinx.cinterop.toCValues
+import kotlinx.cinterop.useContents
+import kotlinx.cinterop.value
+import platform.posix.O_CREAT
+import platform.posix.O_RDWR
+import platform.posix.S_IRGRP
+import platform.posix.S_IROTH
+import platform.posix.S_IRUSR
+import platform.posix.S_IWUSR
+import platform.posix.errno
+
+// TODO clean up error checking?
+// TODO allow specifying directory
+@OptIn(ExperimentalUnsignedTypes::class)
+public class DbmSettings(private val path: String) : Settings {
+
+    override val keys: Set<String>
+        get() = dbmOperation { dbm ->
+            dbm.foldKeys(mutableListOf<String>()) { list, key -> list.apply { add(key.toKString()!!) } }.toSet()
+        }
+
+    override val size: Int get() = dbmOperation { dbm -> dbm.foldKeys(0) { size, _ -> size + 1 } }
+
+    public override fun clear(): Unit = dbmOperation { dbm -> dbm.forEachKey { dbm_delete(dbm, it) } }
+    public override fun remove(key: String): Unit = dbmOperation { dbm -> dbm_delete(dbm, datumOf(key)) }
+    public override fun hasKey(key: String): Boolean = dbmOperation { dbm ->
+        dbm.foldKeys(false) { out, thisKey -> if (key == thisKey.toKString()) true else out }
+    }
+
+    public override fun putInt(key: String, value: Int): Unit = saveBytes(key, value.toByteArray())
+    public override fun getInt(key: String, defaultValue: Int): Int = getIntOrNull(key) ?: defaultValue
+    public override fun getIntOrNull(key: String): Int? = loadBytes(key)?.toInt()
+
+    public override fun putLong(key: String, value: Long): Unit = saveBytes(key, value.toByteArray())
+    public override fun getLong(key: String, defaultValue: Long): Long = getLongOrNull(key) ?: defaultValue
+    public override fun getLongOrNull(key: String): Long? = loadBytes(key)?.toLong()
+
+    public override fun putString(key: String, value: String): Unit = saveBytes(key, value.encodeToByteArray())
+    public override fun getString(key: String, defaultValue: String): String = getStringOrNull(key) ?: defaultValue
+    public override fun getStringOrNull(key: String): String? = loadBytes(key)?.decodeToString()
+
+    public override fun putFloat(key: String, value: Float): Unit = saveBytes(key, value.toRawBits().toByteArray())
+    public override fun getFloat(key: String, defaultValue: Float): Float = getFloatOrNull(key) ?: defaultValue
+    public override fun getFloatOrNull(key: String): Float? = loadBytes(key)?.toInt()?.let { Float.fromBits(it) }
+
+    public override fun putDouble(key: String, value: Double): Unit = saveBytes(key, value.toRawBits().toByteArray())
+    public override fun getDouble(key: String, defaultValue: Double): Double = getDoubleOrNull(key) ?: defaultValue
+    public override fun getDoubleOrNull(key: String): Double? = loadBytes(key)?.toLong()?.let { Double.fromBits(it) }
+
+    public override fun putBoolean(key: String, value: Boolean): Unit = saveBytes(key, byteArrayOf(if (value) 1 else 0))
+    public override fun getBoolean(key: String, defaultValue: Boolean): Boolean = getBooleanOrNull(key) ?: defaultValue
+    public override fun getBooleanOrNull(key: String): Boolean? = loadBytes(key)?.get(0)?.equals(0)?.not()
+
+    private inline fun <T> dbmOperation(action: MemScope.(dbm: CPointer<DBM>) -> T): T = memScoped {
+        val dbm = dbm_open(path.cstr, O_RDWR or O_CREAT, S_IRUSR or S_IWUSR or S_IRGRP or S_IROTH)
+            ?: error("Error on dbm_open: $errno")
+        val out = action(dbm)
+        val error = dbm_error(dbm)
+        if (error != 0) {
+            try {
+                error("error: $error")
+            } finally {
+                dbm_clearerr(dbm)
+            }
+        }
+        dbm_close(dbm)
+        out
+    }
+
+    private inline fun saveBytes(key: String, bytes: ByteArray): Unit = dbmOperation { dbm ->
+        dbm_store(dbm, datumOf(key), datumOf(bytes), DBM_REPLACE.toInt())
+    }
+
+    private inline fun loadBytes(key: String): ByteArray? = dbmOperation { dbm ->
+        val datum = dbm_fetch(dbm, datumOf(key))
+        datum.toByteArray()
+    }
+
+    private inline fun ByteArray.toLong(): Long = foldIndexed(0) { index, total: Long, byte: Byte ->
+        ((0xff.toLong() and byte.toLong()) shl index * Byte.SIZE_BITS) or total
+    }
+
+    private inline fun ByteArray.toInt(): Int = foldIndexed(0) { index, total: Int, byte: Byte ->
+        ((0xff and byte.toInt()) shl index * Byte.SIZE_BITS) or total
+    }
+
+    private inline fun Long.toByteArray(): ByteArray = ByteArray(Long.SIZE_BYTES) { index ->
+        ((this shr (Byte.SIZE_BITS * index)) and 0xff).toByte()
+    }
+
+    private inline fun Int.toByteArray(): ByteArray = ByteArray(Int.SIZE_BYTES) { index ->
+        ((this shr (Byte.SIZE_BITS * index)) and 0xff).toByte()
+    }
+
+    private inline fun CPointer<DBM>.forEachKey(block: (key: CValue<datum>) -> Unit) {
+        val dbm = this
+        var nextKey = dbm_firstkey(dbm)
+        while (nextKey.useContents { dptr != null }) {
+            block(nextKey)
+            nextKey = dbm_nextkey(dbm)
+        }
+    }
+
+    private inline fun <A> CPointer<DBM>.foldKeys(initial: A, block: (accumulator: A, key: CValue<datum>) -> A): A {
+        var accumulator = initial
+        forEachKey { accumulator = block(accumulator, it) }
+        return accumulator
+    }
+
+    private inline fun CValue<datum>.toKString(): String? = toByteArray()?.decodeToString()
+    private inline fun CValue<datum>.toByteArray(): ByteArray? = useContents {
+        val size = dsize.toInt()
+        val firstPtr: CPointer<ByteVar> = dptr?.reinterpret()
+            ?: return null
+        return ByteArray(size) {
+            val pointedValue = firstPtr.plus(it)?.pointed?.value
+            pointedValue ?: 0
+        }
+    }
+
+    private inline fun MemScope.datumOf(string: String): CValue<datum> = datumOf(string.encodeToByteArray())
+    private inline fun MemScope.datumOf(bytes: ByteArray): CValue<datum> = cValue {
+        val cValues = bytes.toCValues()
+        dptr = cValues.ptr
+        dsize = cValues.size.toULong()
+    }
+}

multiplatform-settings/src/linuxX64Test/kotlin/DbmSettingsTest.kt

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2020 Russell Wolf
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.russhwolf.settings
+
+import DbmSettings
+import com.russhwolf.settings.cinterop.qdbm.DBM
+import com.russhwolf.settings.cinterop.qdbm.DBM_REPLACE
+import com.russhwolf.settings.cinterop.qdbm.datum
+import com.russhwolf.settings.cinterop.qdbm.dbm_close
+import com.russhwolf.settings.cinterop.qdbm.dbm_error
+import com.russhwolf.settings.cinterop.qdbm.dbm_open
+import com.russhwolf.settings.cinterop.qdbm.dbm_store
+import kotlinx.cinterop.CValuesRef
+import kotlinx.cinterop.cValue
+import kotlinx.cinterop.cstr
+import kotlinx.cinterop.memScoped
+import kotlinx.cinterop.pointed
+import kotlinx.cinterop.toCValues
+import kotlinx.cinterop.toKString
+import platform.posix.O_CREAT
+import platform.posix.O_RDWR
+import platform.posix.S_IRGRP
+import platform.posix.S_IROTH
+import platform.posix.S_IRUSR
+import platform.posix.S_IWUSR
+import platform.posix.errno
+import platform.posix.opendir
+import platform.posix.readdir
+import platform.posix.remove
+import kotlin.test.AfterTest
+import kotlin.test.BeforeTest
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+@OptIn(ExperimentalUnsignedTypes::class)
+class DbmSettingsTest : BaseSettingsTest(
+    platformFactory = object : Settings.Factory {
+        override fun create(name: String?): Settings = DbmSettings(name ?: "dbm")
+    },
+    hasListeners = false
+) {
+
+    private lateinit var initialDbFiles: List<String>
+
+    @BeforeTest
+    fun getInitialDbFiles() {
+        initialDbFiles = getDbFiles()
+    }
+
+    @AfterTest
+    fun cleanup() {
+        val newDbFiles = getDbFiles() - initialDbFiles
+        newDbFiles.forEach {
+            remove(it)
+        }
+    }
+
+    private fun getDbFiles(): List<String> {
+        val directory = opendir("./") ?: return emptyList()
+
+        val out = mutableListOf<String>()
+        while (true) {
+            val entry = readdir(directory) ?: break
+
+            val filename = entry.pointed.d_name.toKString()
+            if (filename.run { endsWith(".db") || endsWith(".dir") || endsWith(".pag") }) {
+                out.add(filename)
+            }
+        }
+        return out
+    }
+
+    @Test
+    fun constructor_filename() {
+        val filename = "test_dbm"
+        val settings = DbmSettings(filename)
+
+        memScoped {
+            val dbm = dbm_open(
+                filename.cstr,
+                O_RDWR or O_CREAT,
+                S_IRUSR or S_IWUSR or S_IRGRP or S_IROTH
+            )
+                ?: error("error: $errno")
+            dbm.checkError()
+
+            val key = cValue<datum> {
+                val cValues = "key".encodeToByteArray().toCValues()
+                dptr = cValues.ptr
+                dsize = cValues.size.toULong()
+            }
+            val value = cValue<datum> {
+                val cValues = "value".encodeToByteArray().toCValues()
+                dptr = cValues.ptr
+                dsize = cValues.size.toULong()
+            }
+
+            if (dbm_store(
+                    dbm,
+                    key,
+                    value,
+                    DBM_REPLACE.toInt()
+                ) == -1) {
+                dbm.checkError()
+            }
+
+            dbm_close(dbm)
+        }
+
+        assertEquals("value", settings["key", ""])
+    }
+}
+
+private fun CValuesRef<DBM>.checkError() {
+    assertEquals(0, dbm_error(this))
+}

multiplatform-settings/src/nativeInterop/cinterop/qdbm.def

@@ -0,0 +1,4 @@
+headers = relic.h
+package = com.russhwolf.settings.cinterop.qdbm
+compilerOpts = -I/usr/include/qdbm
+linkerOpts = -lqdbm -L/usr/lib