Merge pull request #41 from cyberbit/hotfix/secure-modem-serde

📦 improve MetricCollection serde for SecureModem

LICENSE

@@ -21,8 +21,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
 Portions of this software are copyright of their respective authors and released under the MIT license:
- - lua-object-model, Copyright 2015 Pavel Batečko (Shira). For licensing see src/telem/lib/LICENSE-ObjectModel.txt
- - ECNet2, Copyright 2020 Miguel Oliveira. For licensing see src/telem/vendor/LICENSE-ECNet2.txt
  - CCryptoLib, Copyright 2023 Miguel Oliveira. For licensing see src/telem/vendor/LICENSE-CCryptoLib.txt
- - RedRun, By JackMacWindows. For licensing see src/telem/vendor/redrun.lua
- - Plotter, Copyright 2023 Daniel Marcolesco. For licensing see src/telem/vendor/LICENSE-Plotter.txt
+ - ECNet2, Copyright 2020 Miguel Oliveira. For licensing see src/telem/vendor/LICENSE-ECNet2.txt
+ - lualzw, Copyright 2016 Rochet2. For licensing see src/telem/vendor/lualzw.lua
+ - lua-object-model, Copyright 2015 Pavel Batečko (Shira). For licensing see src/telem/lib/LICENSE-ObjectModel.txt
+ - Plotter, Copyright 2023 Daniel Marcolesco. For licensing see src/telem/vendor/LICENSE-Plotter.txt
+ - RedRun, By JackMacWindows. For licensing see src/telem/vendor/redrun.lua

src/telem/init.lua

@@ -1,9 +1,9 @@
 -- Telem by cyberbit
 -- MIT License
--- Version 0.4.2
+-- Version 0.5.0
 
 local _Telem = {
-    _VERSION = '0.4.2',
+    _VERSION = '0.5.0',
     util = require 'telem.lib.util',
     input = require 'telem.lib.input',
     output = require 'telem.lib.output',

src/telem/lib/Metric.lua

@@ -39,4 +39,32 @@ function Metric:__tostring ()
     return label .. unit .. adapter .. source
 end
 
+function Metric.pack (self)
+    return {
+        n = self.name,
+        v = self.value,
+        u = self.unit,
+        s = self.source,
+        a = self.adapter,
+    }
+end
+
+function Metric.unpack (data)
+    return Metric({
+        name = data.n,
+        value = data.v,
+        unit = data.u,
+        source = data.s,
+        adapter = data.a
+    })
+end
+
+function Metric.serialize (self)
+    return textutils.serialize(self:pack(), { compact = true })
+end
+
+function Metric.unserialize (data)
+    return Metric.unpack(textutils.unserialize(data))
+end
+
 return Metric

src/telem/lib/MetricCollection.lua

@@ -31,7 +31,7 @@ function MetricCollection:insert (element)
 end
 
 function MetricCollection:setContext (ctx)
-    self.context = { table.unpack(ctx) }
+    self.context = ctx
 
     return self
 end
@@ -59,4 +59,139 @@ function MetricCollection:find (filter)
     return nil
 end
 
+function MetricCollection.pack (self)
+    local packedMetrics = {}
+    local adapterLookup = {}
+    local sourceLookup = {}
+    local unitLookup = {}
+    local nameLookup = {
+        -- first name token
+        {},
+
+        -- second name token
+        {}
+    }
+
+    for _,v in ipairs(self.metrics) do
+        local packed = v:pack()
+
+        -- create name tokens
+        local nameTokens = {}
+
+        for token in (packed.n .. ':'):gmatch('([^:]*):') do
+            table.insert(nameTokens, token)
+        end
+
+        local t1 = nameTokens[1]
+        local t2 = nameTokens[2]
+        local t3 = nameTokens[3]
+
+        if #nameTokens > 2 then
+            t3 = table.concat(nameTokens, ':', 3)
+        end
+
+        local n1, n2, nn
+
+        if t3 then
+            n1, n2, nn = t1, t2, t3
+        elseif t2 then
+            n1, n2, nn = t1, nil, t2
+        elseif t1 then
+            n1, n2, nn = nil, nil, t1
+        end
+
+        -- pull LUT
+        local ln1 = n1 and t.indexOf(nameLookup[1], n1)
+        local ln2 = n2 and t.indexOf(nameLookup[2], n2)
+        local la = t.indexOf(adapterLookup, packed.a)
+        local ls = t.indexOf(sourceLookup, packed.s)
+        local lu = t.indexOf(unitLookup, packed.u)
+
+        -- register missing LUT
+        if ln1 and ln1 < 0 then
+            table.insert(nameLookup[1], n1)
+
+            ln1 = #nameLookup[1]
+        end
+        if ln2 and ln2 < 0 then
+            table.insert(nameLookup[2], n2)
+
+            ln2 = #nameLookup[2]
+        end
+        if la < 0 then
+            table.insert(adapterLookup, packed.a)
+
+            la = #adapterLookup
+        end
+        if ls < 0 then
+            table.insert(sourceLookup, packed.s)
+
+            ls = #sourceLookup
+        end
+        if lu < 0 then
+            table.insert(unitLookup, packed.u)
+
+            lu = #unitLookup
+        end
+
+        local ln
+        if ln1 or ln2 then
+            ln = {ln1, ln2}
+        end
+
+        table.insert(packedMetrics, {
+            ln = ln,
+            n = nn,
+            v = packed.v,
+            la = la,
+            ls = ls,
+            lu = lu
+        })
+    end
+
+    return {
+        c = self.context,
+        ln = nameLookup,
+        la = adapterLookup,
+        ls = sourceLookup,
+        lu = unitLookup,
+        m = packedMetrics
+    }
+end
+
+function MetricCollection.unpack (data)
+    local undata = data
+
+    local nameLookup = undata.ln
+    local adapterLookup = undata.la
+    local sourceLookup = undata.ls
+    local unitLookup = undata.lu
+
+    local collection = MetricCollection()
+
+    for _, v in ipairs(undata.m) do
+        local nPrefix = ''
+
+        if v.ln then
+            for lni, lnv in ipairs(v.ln) do
+                nPrefix = nPrefix .. nameLookup[lni][lnv] .. ':'
+            end
+        end
+
+        local tempMetric = Metric{
+            name = nPrefix .. v.n,
+            value = v.v,
+            adapter = v.la and adapterLookup[v.la],
+            source = v.ls and sourceLookup[v.ls],
+            unit = v.lu and unitLookup[v.lu]
+        }
+
+        collection:insert(tempMetric)
+    end
+
+    collection:setContext(undata.c)
+
+    return collection
+end
+
 return MetricCollection

src/telem/lib/input/SecureModemInputAdapter.lua

@@ -3,6 +3,7 @@ local t = require 'telem.lib.util'
 local vendor
 local ecnet2
 local random
+local lualzw
 
 local InputAdapter      = require 'telem.lib.InputAdapter'
 local Metric            = require 'telem.lib.Metric'
@@ -11,8 +12,7 @@ local MetricCollection  = require 'telem.lib.MetricCollection'
 local SecureModemInputAdapter = o.class(InputAdapter)
 SecureModemInputAdapter.type = 'SecureModemInputAdapter'
 
-
-SecureModemInputAdapter.VERSION = 'v1.0.0'
+SecureModemInputAdapter.VERSION = 'v2.0.0'
 
 SecureModemInputAdapter.REQUEST_PREAMBLE = 'telem://'
 SecureModemInputAdapter.REQUESTS = {
@@ -66,6 +66,14 @@ function SecureModemInputAdapter:constructor (peripheralName, address)
             self:dlog('SecureModemInput:boot :: ECNet2 ready. Address = ' .. ecnet2.address())
         end
 
+        if not lualzw then
+            self:dlog('SecureModemInput:boot :: Loading lualzw...')
+
+            lualzw = vendor.lualzw
+
+            self:dlog('SecureModemInput:boot :: lualzw ready.')
+        end
+
         self:dlog('SecureModemInput:boot :: Opening modem...')
 
         ecnet2.open(peripheralName)
@@ -131,7 +139,7 @@ function SecureModemInputAdapter:read ()
     end
 
     self:dlog('SecureModemInput:read :: listening for response')
-    local sender, collection = self.connection:receive(self.receiveTimeout)
+    local sender, message = self.connection:receive(self.receiveTimeout)
 
     if not sender then
         t.log('SecureModemInput:read :: Receive timed out after ' .. self.receiveTimeout .. ' seconds, retrying next cycle')
@@ -141,15 +149,18 @@ function SecureModemInputAdapter:read ()
         return MetricCollection()
     end
 
-    self:dlog('SecureModemInput:read :: response received, hydrating collection')
+    local unwrapped = message
+
+    -- decompress if needed
+    if type(message) == 'string' and string.sub(message, 1, 1) == 'c' then
+        unwrapped = textutils.unserialize(lualzw.decompress(message))
+    end
 
-    local hydratedCollection = MetricCollection()
+    local unpacked = MetricCollection.unpack(unwrapped)
 
-    for _, value in ipairs(collection.metrics) do
-        hydratedCollection:insert(Metric(value))
-    end
+    self:dlog('SecureModemInput:read :: response received')
 
-    return hydratedCollection
+    return unpacked
 end
 
 return SecureModemInputAdapter

src/telem/lib/output/SecureModemOutputAdapter.lua

@@ -3,14 +3,15 @@ local t = require 'telem.lib.util'
 local vendor
 local ecnet2
 local random
+local lualzw
 
 local OutputAdapter     = require 'telem.lib.OutputAdapter'
 local MetricCollection  = require 'telem.lib.MetricCollection'
 
 local SecureModemOutputAdapter = o.class(OutputAdapter)
 SecureModemOutputAdapter.type = 'SecureModemOutputAdapter'
 
-SecureModemOutputAdapter.VERSION = 'v1.0.0'
+SecureModemOutputAdapter.VERSION = 'v2.0.0'
 
 SecureModemOutputAdapter.REQUEST_PREAMBLE = 'telem://'
 SecureModemOutputAdapter.REQUESTS = {
@@ -31,11 +32,11 @@ function SecureModemOutputAdapter:constructor (peripheralName)
         self:addComponentByPeripheralID(peripheralName)
 
         if not vendor then
-            self:dlog('SecureModemInput:boot :: Loading vendor modules...')
+            self:dlog('SecureModemOutput:boot :: Loading vendor modules...')
 
             vendor = require 'telem.vendor'
 
-            self:dlog('SecureModemInput:boot :: Vendor modules ready.')
+            self:dlog('SecureModemOutput:boot :: Vendor modules ready.')
         end
 
         if not random then
@@ -62,6 +63,14 @@ function SecureModemOutputAdapter:constructor (peripheralName)
             t.log('SecureModemOutput:boot :: ECNet2 ready. Address = ' .. ecnet2.address())
         end
 
+        if not lualzw then
+            self:dlog('SecureModemOutput:boot :: Loading lualzw...')
+
+            lualzw = vendor.lualzw
+
+            self:dlog('SecureModemOutput:boot :: lualzw ready.')
+        end
+
         self:dlog('SecureModemOutput:boot :: Opening modem...')
 
         ecnet2.open(peripheralName)
@@ -102,9 +111,18 @@ function SecureModemOutputAdapter:constructor (peripheralName)
                 self:dlog('SecureModemOutput:asyncCycleHandler :: received request from ' .. p2)
 
                 if p3 == self.REQUESTS.GET_COLLECTION then
-                    self:dlog('SecureModemOutput:asyncCacheHandler :: request = GET_COLLECTION')
+                    self:dlog('SecureModemOutput:asyncCycleHandler :: request = GET_COLLECTION')
+
+                    local payload = backplane.collection:pack()
+
+                    -- use compression for payloads with > 256 metrics
+                    if #payload.m > 256 then
+                        self:dlog('SecureModemOutput:asyncCycleHandler :: compressing large payload...')
+
+                        payload = lualzw.compress(textutils.serialize(payload, { compact = true }))
+                    end
 
-                    self.connections[id]:send(backplane.collection)
+                    self.connections[id]:send(payload)
 
                     self:dlog('SecureModemOutput:asyncCycleHandler :: response sent')
                 else

src/telem/lib/util.lua

@@ -112,6 +112,18 @@ local function constrainAppend (data, value, width)
     return removed
 end
 
+local function indexOf (tab, value)
+    if type(value) == 'nil' then return -1 end
+
+    for i,v in ipairs(tab) do
+        if v == value then
+            return i
+        end
+    end
+
+    return -1
+end
+
 return {
     log = log,
     err = err,
@@ -121,4 +133,5 @@ return {
     shortnum = shortnum,
     shortnum2 = shortnum2,
     constrainAppend = constrainAppend,
+    indexOf = indexOf
 }

src/telem/vendor/init.lua

@@ -1,18 +1,20 @@
 -- Telem Vendor Loader by cyberbit
 -- MIT License
--- Version 0.4.0
+-- Version 0.5.0
 -- Submodules are copyright of their respective authors. For licensing, see https://github.com/cyberbit/telem/blob/main/LICENSE
 
 if package.path:find('telem/vendor') == nil then package.path = package.path .. ';telem/vendor/?;telem/vendor/?.lua;telem/vendor/?/init.lua' end
 
 local ecnet2 = require 'ecnet2'
 local random = require 'ccryptolib.random'
 local plotter = require 'plotter'
+local lualzw = require 'lualzw'
 
 return {
     ccryptolib = {
         random = random
     },
     ecnet2 = ecnet2,
+    lualzw = lualzw,
     plotter = plotter
 }