From de24dab3190b65bec5b54679363d74f4146dc2ba Mon Sep 17 00:00:00 2001 From: Matthew Neeley Date: Mon, 11 Apr 2016 16:56:40 -0700 Subject: [PATCH] Add ability to preserve text form (formatting, etc.) of registry data. --- core/src/main/scala/org/labrad/Proxies.scala | 7 +++ .../org/labrad/manager/RemoteConnector.scala | 2 +- .../org/labrad/registry/DelphiStore.scala | 6 +-- .../scala/org/labrad/registry/FileStore.scala | 38 ++++++++++----- .../scala/org/labrad/registry/Migrate.scala | 6 +-- .../scala/org/labrad/registry/Registry.scala | 43 ++++++++++++++--- .../org/labrad/registry/RemoteStore.scala | 43 +++++++++++++---- .../org/labrad/registry/SQLiteStore.scala | 36 +++++++++++---- .../test/scala/org/labrad/RegistryTest.scala | 46 +++++++++++++++++-- 9 files changed, 180 insertions(+), 47 deletions(-) diff --git a/core/src/main/scala/org/labrad/Proxies.scala b/core/src/main/scala/org/labrad/Proxies.scala index 94dd6ec..9f7fc99 100644 --- a/core/src/main/scala/org/labrad/Proxies.scala +++ b/core/src/main/scala/org/labrad/Proxies.scala @@ -120,7 +120,14 @@ trait RegistryServer extends Requester { case None => call("get", Str(key), Str(pat)) } } + def getAsString(key: String, pat: String = "?", default: Option[(Boolean, Data)] = None): Future[(Data, String)] = { + default match { + case Some((set, default)) => call[(Data, String)]("get as string", Str(key), Str(pat), Bool(set), default) + case None => call[(Data, String)]("get as string", Str(key), Str(pat)) + } + } def set(key: String, value: Data): Future[Unit] = callUnit("set", Str(key), value) + def setAsString(key: String, text: String): Future[Unit] = callUnit("set as string", Str(key), Str(text)) def del(key: String): Future[Unit] = callUnit("del", Str(key)) def notifyOnChange(id: Long, enable: Boolean): Future[Unit] = diff --git a/manager/src/main/scala/org/labrad/manager/RemoteConnector.scala b/manager/src/main/scala/org/labrad/manager/RemoteConnector.scala index 16b6f3b..a595879 100644 --- a/manager/src/main/scala/org/labrad/manager/RemoteConnector.scala +++ b/manager/src/main/scala/org/labrad/manager/RemoteConnector.scala @@ -72,7 +72,7 @@ class MultiheadServer(name: String, registry: RegistryStore, server: LocalServer dir = registry.child(dir, name, create = true) dir = registry.child(dir, "Multihead", create = true) val default = DataBuilder("*(sws)").array(0).result() - val result = registry.getValue(dir, "Managers", default = Some((true, default))) + val (result, textOpt) = registry.getValue(dir, "Managers", default = Some((true, default))) val configs = result.get[Seq[(String, Long, String)]].map { case (host, port, pw) => externalConfig.copy( diff --git a/manager/src/main/scala/org/labrad/registry/DelphiStore.scala b/manager/src/main/scala/org/labrad/registry/DelphiStore.scala index a8f3a73..a361dab 100644 --- a/manager/src/main/scala/org/labrad/registry/DelphiStore.scala +++ b/manager/src/main/scala/org/labrad/registry/DelphiStore.scala @@ -54,12 +54,12 @@ class DelphiFileStore(rootDir: File) extends FileStore(rootDir) { /** * Encode and decode data for storage in individual key files. */ - override def encodeData(data: Data): Array[Byte] = { + override def encodeData(data: Data, textOpt: Option[String]): Array[Byte] = { DelphiFormat.dataToString(data).getBytes(UTF_8) } - override def decodeData(bytes: Array[Byte]): Data = { - DelphiFormat.stringToData(new String(bytes, UTF_8)) + override def decodeData(bytes: Array[Byte]): (Data, Option[String]) = { + (DelphiFormat.stringToData(new String(bytes, UTF_8)), None) } } diff --git a/manager/src/main/scala/org/labrad/registry/FileStore.scala b/manager/src/main/scala/org/labrad/registry/FileStore.scala index dca7537..0e602fa 100644 --- a/manager/src/main/scala/org/labrad/registry/FileStore.scala +++ b/manager/src/main/scala/org/labrad/registry/FileStore.scala @@ -33,8 +33,8 @@ abstract class FileStore(rootDir: File) extends RegistryStore { /** * Encode and decode data for storage in individual key files. */ - def encodeData(data: Data): Array[Byte] - def decodeData(bytes: Array[Byte]): Data + def encodeData(data: Data, textOpt: Option[String]): Array[Byte] + def decodeData(bytes: Array[Byte]): (Data, Option[String]) /** * Convert the given directory into a registry path. @@ -82,7 +82,7 @@ abstract class FileStore(rootDir: File) extends RegistryStore { if (!ok) sys.error(s"failed to remove directory: $name") } - def getValue(dir: File, key: String, default: Option[(Boolean, Data)]): Data = { + def getValue(dir: File, key: String, default: Option[(Boolean, Data)]): (Data, Option[String]) = { val path = keyFile(dir, key) if (path.exists) { val bytes = readFile(path) @@ -91,15 +91,15 @@ abstract class FileStore(rootDir: File) extends RegistryStore { default match { case None => sys.error(s"key does not exist: $key") case Some((set, default)) => - if (set) setValue(dir, key, default) - default + if (set) setValue(dir, key, default, None) + (default, None) } } } - def setValueImpl(dir: File, key: String, value: Data): Unit = { + def setValueImpl(dir: File, key: String, value: Data, textOpt: Option[String]): Unit = { val path = keyFile(dir, key) - val bytes = encodeData(value) + val bytes = encodeData(value, textOpt) writeFile(path, bytes) } @@ -166,12 +166,26 @@ class BinaryFileStore(rootDir: File) extends FileStore(rootDir) { /** * Encode and decode data for storage in individual key files. */ - override def encodeData(data: Data): Array[Byte] = { - Cluster(Str(data.t.toString), Bytes(data.toBytes)).toBytes + override def encodeData(data: Data, textOpt: Option[String]): Array[Byte] = { + val b = DataBuilder() + b.clusterStart() + b.string(data.t.toString) + b.bytes(data.toBytes) + for (text <- textOpt) { + b.string(text) + } + b.clusterEnd() + b.result.toBytes } - override def decodeData(bytes: Array[Byte]): Data = { - val (typ, data) = Data.fromBytes(Type("sy"), bytes).get[(String, Array[Byte])] - Data.fromBytes(Type(typ), data) + override def decodeData(bytes: Array[Byte]): (Data, Option[String]) = { + try { + val (typ, data, text) = Data.fromBytes(Type("sys"), bytes).get[(String, Array[Byte], String)] + (Data.fromBytes(Type(typ), data), Some(text)) + } catch { + case _: Exception => + val (typ, data) = Data.fromBytes(Type("sy"), bytes).get[(String, Array[Byte])] + (Data.fromBytes(Type(typ), data), None) + } } } diff --git a/manager/src/main/scala/org/labrad/registry/Migrate.scala b/manager/src/main/scala/org/labrad/registry/Migrate.scala index 068ed5d..aa5e77e 100644 --- a/manager/src/main/scala/org/labrad/registry/Migrate.scala +++ b/manager/src/main/scala/org/labrad/registry/Migrate.scala @@ -202,7 +202,7 @@ object Migrate { } /** - * SQLite registry format on disk (source or sink). + * Registry on local disk (source or sink). */ class LocalRegistry(store: RegistryStore) extends Registry { private def find(path: Seq[String], create: Boolean = false): store.Dir = { @@ -219,7 +219,7 @@ object Migrate { val values = Map.newBuilder[String, Data] for (key <- keys) { - val data = store.getValue(loc, key, default = None) + val (data, textOpt) = store.getValue(loc, key, default = None) values += key -> data } (dirs, values.result) @@ -228,7 +228,7 @@ object Migrate { def set(path: Seq[String], keys: Map[String, Data]): Unit = { val loc = find(path, create = true) for ((key, value) <- keys) { - store.setValue(loc, key, value) + store.setValue(loc, key, value, None) } } } diff --git a/manager/src/main/scala/org/labrad/registry/Registry.scala b/manager/src/main/scala/org/labrad/registry/Registry.scala index 2bfd989..5022aa5 100644 --- a/manager/src/main/scala/org/labrad/registry/Registry.scala +++ b/manager/src/main/scala/org/labrad/registry/Registry.scala @@ -36,13 +36,13 @@ trait RegistryStore { } def rmDirImpl(dir: Dir, name: String): Unit - def getValue(dir: Dir, key: String, default: Option[(Boolean, Data)]): Data + def getValue(dir: Dir, key: String, default: Option[(Boolean, Data)]): (Data, Option[String]) - def setValue(dir: Dir, key: String, value: Data): Unit = { - setValueImpl(dir, key, value) + def setValue(dir: Dir, key: String, value: Data, textOpt: Option[String]): Unit = { + setValueImpl(dir, key, value, textOpt) notifyListener(dir, key, isDir=false, addOrChange=true) } - def setValueImpl(dir: Dir, key: String, value: Data): Unit + def setValueImpl(dir: Dir, key: String, value: Data, textOpt: Option[String]): Unit def delete(dir: Dir, key: String): Unit = { deleteImpl(dir, key) @@ -235,17 +235,48 @@ extends LocalServer with Logging { def getValue(key: String, pat: String, set: Boolean, default: Data): Data = _getValue(key, pat, Some((set, default))) def _getValue(key: String, pat: String = "?", default: Option[(Boolean, Data)] = None): Data = { - val data = store.getValue(curDir, key, default) + val (data, textOpt) = store.getValue(curDir, key, default) val pattern = Pattern(pat) data.convertTo(pattern) } + @Setting(id=21, + name="get as string", + doc="""Get the content of the given key in the current directory as a string.""") + def getAsString(key: String): (Data, String) = _getAsString(key) + def getAsString(key: String, pat: String): (Data, String) = _getAsString(key, pat) + def getAsString(key: String, set: Boolean, default: Data): (Data, String) = _getAsString(key, default = Some((set, default))) + def getAsString(key: String, pat: String, set: Boolean, default: Data): (Data, String) = _getAsString(key, pat, Some((set, default))) + + def _getAsString(key: String, pat: String = "?", default: Option[(Boolean, Data)] = None): (Data, String) = { + val (data, textOpt) = store.getValue(curDir, key, default) + val pattern = Pattern(pat) + val converted = data.convertTo(pattern) + val text = textOpt match { + case None => + converted.toString + + case Some(text) => + if (converted == data) text else converted.toString + } + (converted, text) + } + @Setting(id = 30, name = "set", doc = """Set the given key in the current directory to the given value.""") def setValue(key: String, value: Data): Unit = { require(key.nonEmpty, "Cannot create a key with empty name") - store.setValue(curDir, key, value) + store.setValue(curDir, key, value, None) + } + + @Setting(id=31, + name="set as string", + doc="""Set the content of the given key in the current directory to the given data in text form.""") + def setAsString(key: String, text: String): Unit = { + require(key.nonEmpty, "Cannot create a key with empty name") + val value = Data.parse(text) + store.setValue(curDir, key, value, Some(text)) } @Setting(id = 40, diff --git a/manager/src/main/scala/org/labrad/registry/RemoteStore.scala b/manager/src/main/scala/org/labrad/registry/RemoteStore.scala index ea08514..ae5a598 100644 --- a/manager/src/main/scala/org/labrad/registry/RemoteStore.scala +++ b/manager/src/main/scala/org/labrad/registry/RemoteStore.scala @@ -1,6 +1,7 @@ package org.labrad.registry -import org.labrad.{Client, Credential, RegistryServerPacket, RegistryServerProxy, TlsMode} +import org.labrad.{Client, Credential, ManagerServerProxy, RegistryServerPacket, + RegistryServerProxy, TlsMode} import org.labrad.data.{Data, Message} import org.labrad.types.Type import scala.concurrent.{Await, Future} @@ -26,7 +27,9 @@ extends RegistryStore { val root = Seq("") private val lock = new Object + private var cxn: Client = null private var reg: RegistryServerProxy = null + private var hasGetAsString: Boolean = false /** * Connect to remote registry if we are not currently connected, @@ -56,7 +59,11 @@ extends RegistryStore { cxn.connect() val registry = new RegistryServerProxy(cxn) Await.result(registry.streamChanges(id, true), 5.seconds) - reg = registry + val manager = new ManagerServerProxy(cxn) + val registrySettings = Await.result(manager.settings(Registry.NAME), 5.seconds) + this.cxn = cxn + this.reg = registry + this.hasGetAsString = registrySettings.map(_._2).toSet.contains("get as string") } } } @@ -99,12 +106,32 @@ extends RegistryStore { call(dir) { _.rmDir(name) } } - def getValue(dir: Dir, key: String, default: Option[(Boolean, Data)]): Data = { - call(dir) { _.get(key, default = default) } + def getValue(dir: Dir, key: String, default: Option[(Boolean, Data)]): (Data, Option[String]) = { + call(dir) { p => + // We need an execution context to map over futures; use our client's execution context. + implicit val executionContext = cxn.executionContext + if (hasGetAsString) { + p.getAsString(key, default = default).map { case (data, text) => (data, Some(text)) } + } else { + p.get(key, default = default).map { case data => (data, None) } + } + } } - def setValueImpl(dir: Dir, key: String, value: Data): Unit = { - call(dir) { _.set(key, value) } + def setValueImpl(dir: Dir, key: String, value: Data, textOpt: Option[String]): Unit = { + textOpt match { + case Some(text) => + call(dir) { p => + if (hasGetAsString) { + p.setAsString(key, text) + } else { + p.set(key, value) + } + } + + case None => + call(dir) { _.set(key, value) } + } } def deleteImpl(dir: Dir, key: String): Unit = { @@ -125,8 +152,8 @@ extends RegistryStore { rmDirImpl(dir, name) } - override def setValue(dir: Dir, key: String, value: Data): Unit = { - setValueImpl(dir, key, value) + override def setValue(dir: Dir, key: String, value: Data, textOpt: Option[String]): Unit = { + setValueImpl(dir, key, value, textOpt) } override def delete(dir: Dir, key: String): Unit = { diff --git a/manager/src/main/scala/org/labrad/registry/SQLiteStore.scala b/manager/src/main/scala/org/labrad/registry/SQLiteStore.scala index 692e815..0015894 100644 --- a/manager/src/main/scala/org/labrad/registry/SQLiteStore.scala +++ b/manager/src/main/scala/org/labrad/registry/SQLiteStore.scala @@ -60,6 +60,14 @@ object SQLiteStore { ) """.execute() + // add data_str column if needed + val keyColumns = SQL"""PRAGMA table_info(keys)""".as(str("name").*) + if (!keyColumns.contains("data_str")) { + SQL""" + ALTER TABLE keys ADD COLUMN data_str TEXT + """.execute() + } + SQL""" CREATE INDEX IF NOT EXISTS idx_list_keys ON keys(dir_id) """.execute() @@ -129,35 +137,43 @@ class SQLiteStore(cxn: Connection) extends RegistryStore { SQL" DELETE FROM dirs WHERE parent_id = ${dir.id} AND name = $name ".execute() } - def getValue(dir: SqlDir, key: String, default: Option[(Boolean, Data)]): Data = { + def getValue(dir: SqlDir, key: String, default: Option[(Boolean, Data)]): (Data, Option[String]) = { // SQLite return NULL for empty BLOBs, so we have to treat the data // column as nullable, even though we specified it as NOT NULL val tdOpt = SQL""" - SELECT type, data FROM keys WHERE dir_id = ${dir.id} AND name = $key - """.as((str("type") ~ byteArray("data").?).singleOpt) + SELECT type, data, data_str FROM keys WHERE dir_id = ${dir.id} AND name = $key + """.as((str("type") ~ byteArray("data").? ~ str("data_str").?).singleOpt) tdOpt match { - case Some(typ ~ data) => - Data.fromBytes(Type(typ), data.getOrElse(Array.empty)) + case Some(typ ~ bytes ~ textOpt) => + val data = Data.fromBytes(Type(typ), bytes.getOrElse(Array.empty)) + (data, textOpt) case None => default match { case None => sys.error(s"key does not exist: $key") case Some((set, default)) => - if (set) setValue(dir, key, default) - default + if (set) setValue(dir, key, default, None) + (default, None) } } } - def setValueImpl(dir: SqlDir, key: String, value: Data): Unit = { + def setValueImpl(dir: SqlDir, key: String, value: Data, textOpt: Option[String]): Unit = { val typ = value.t.toString val data = value.toBytes - val n = SQL" UPDATE keys SET type = $typ, data = $data WHERE dir_id = ${dir.id} AND name = $key ".executeUpdate() + val n = SQL""" + UPDATE keys + SET type = $typ, data = $data, data_str = $textOpt + WHERE dir_id = ${dir.id} AND name = $key + """.executeUpdate() if (n == 0) { - SQL" INSERT INTO keys(dir_id, name, type, data) VALUES (${dir.id}, $key, $typ, $data) ".executeInsert() + SQL""" + INSERT INTO keys(dir_id, name, type, data, data_str) + VALUES (${dir.id}, $key, $typ, $data, $textOpt) + """.executeInsert() } } diff --git a/manager/src/test/scala/org/labrad/RegistryTest.scala b/manager/src/test/scala/org/labrad/RegistryTest.scala index ca40a92..1ef50df 100644 --- a/manager/src/test/scala/org/labrad/RegistryTest.scala +++ b/manager/src/test/scala/org/labrad/RegistryTest.scala @@ -29,13 +29,23 @@ class RegistryTest extends FunSuite with Matchers with AsyncAssertions { } } + def testBinaryBackends(testName: String)(func: RegistryStore => Unit): Unit = { + test(s"BinaryFileStore: $testName") { + Files.withTempDir { dir => func(new BinaryFileStore(dir)) } + } + + test(s"SQLiteStore: $testName") { + Files.withTempFile { file => func(SQLiteStore(file)) } + } + } + testAllBackends("registry can store and retrieve arbitrary data") { (backend, exact) => val loc = backend.child(backend.root, "test", create = true) for (i <- 0 until 100) { val tpe = Hydrant.randomType val data = Hydrant.randomData(tpe) - backend.setValue(loc, "a", data) - val resp = backend.getValue(loc, "a", default = None) + backend.setValue(loc, "a", data, None) + val (resp, _) = backend.getValue(loc, "a", default = None) backend.delete(loc, "a") // Inexact matching function used for semi-lossy registry backends (e.g. delphi). @@ -81,10 +91,10 @@ class RegistryTest extends FunSuite with Matchers with AsyncAssertions { testAllBackends("registry can deal with unicode and strange characters in key names") { (backend, exact) => val key = "<\u03C0|\u03C1>??+*\\/:|" val data = Str("Hello!") - backend.setValue(backend.root, key, data) + backend.setValue(backend.root, key, data, None) val (_, keys) = backend.dir(backend.root) assert(keys contains key) - val result = backend.getValue(backend.root, key, default = None) + val (result, _) = backend.getValue(backend.root, key, default = None) assert(result == data) } @@ -297,4 +307,32 @@ class RegistryTest extends FunSuite with Matchers with AsyncAssertions { } } } + + testBinaryBackends("text form of stored data is preserved") { backend => + val text = """ + [("a", 1), + ("b", 2), + ("c", 3)] + """ + val data = Data.parse(text) + backend.setValue(backend.root, "foo", data, Some(text)) + val (resultData, resultTextOpt) = backend.getValue(backend.root, "foo", None) + assert(resultTextOpt == Some(text)) + assert(resultData == data) + } + + testBinaryBackends("text form is cleared if set without it") { backend => + val text = """ + [("a", 1), + ("b", 2), + ("c", 3)] + """ + val data = Data.parse(text) + backend.setValue(backend.root, "foo", data, Some(text)) + backend.setValue(backend.root, "foo", data, None) + val (resultData, resultTextOpt) = backend.getValue(backend.root, "foo", None) + assert(resultTextOpt == None) + assert(resultData == data) + + } }