Merge pull request #27 from bieli/add-crc-implementation-for-v0

add CRC16 implementation for IoText protocol + unit tests

Makefile

@@ -1,3 +1,5 @@
+test:
+	pytest tests/
 ci:
 	black src/
 	black tests/

README.md

@@ -60,7 +60,8 @@ We have a few basic item types:
 - `t` - timestamp UNIX milis item
 - `d` - device name item
 - `m` - metric item
-- `h` - helth check
+- `h` - health check
+- `c` - CRC-16 (from MODBUS control sum) for all combined IoText payload as string
 
 You can see reference implementation in `ItemTypes` class  in `src/types` package. 
 
@@ -85,16 +86,40 @@ We have a few basic data codecs:
 
 You can see reference implementation in `src/codecs` package.
 
+### How to prepare IoText message
+You need two required informations:
+- timestamp in UNIX millis
+- device name
+- all data metrics are optional, so it means, that you can use IoText protocol like ping - or health check - as well (we have dedicated `h` item type for helth check, too).
+- CRC item it's optional sum control method (using CRC-16 MODBUS implementation)
+
 ### Examples
+
+#### IoText message with generic values
+
 IoText protocol in `schema-less version` data row example:
 ```bash
 t|3900237526042,d|device_name_001,m|val_water_001=i:1234,m|val_water_002=i:15,m|bulb_state=b:1,m|connector_state=b:0,m|temp_01=d:34.4,m|temp_02=d:36.4,m|temp_03=d:10.4,m|pwr=d:12.231,m|current=d:1.429,m|current_battery=d:1.548
 ```
 and after de-serialization to Python data structures we can see:
 ```bash
-[Item(kind='t', name='3900237526042', metric=None), Item(kind='d', name='device_name_001', metric=None), Item(kind='m', name='val_water_001', metric=MetricDataItem(data_type='i', value=1234)), Item(kind='m', name='val_water_002', metric=MetricDataItem(data_type='i', value=15)), Item(kind='m', name='bulb_state', metric=MetricDataItem(data_type='b', value=True)), Item(kind='m', name='connector_state', metric=MetricDataItem(data_type='b', value=False)), Item(kind='m', name='temp_01', metric=MetricDataItem(data_type='d', value=Decimal('34.4'))), Item(kind='m', name='temp_02', metric=MetricDataItem(data_type='d', value=Decimal('36.4'))), Item(kind='m', name='temp_03', metric=MetricDataItem(data_type='d', value=Decimal('10.4'))), Item(kind='m', name='pwr', metric=MetricDataItem(data_type='d', value=Decimal('12.231'))), Item(kind='m', name='current', metric=MetricDataItem(data_type='d', value=Decimal('1.429'))), Item(kind='m', name='current_battery', metric=MetricDataItem(data_type='d', value=Decimal('1.548')))]
+[
+  Item(kind='t', name='3900237526042', metric=None),
+  Item(kind='d', name='device_name_001', metric=None),
+  Item(kind='m', name='val_water_001', metric=MetricDataItem(data_type='i', value=1234)),
+  Item(kind='m', name='val_water_002', metric=MetricDataItem(data_type='i', value=15)),
+  Item(kind='m', name='bulb_state', metric=MetricDataItem(data_type='b', value=True)),
+  Item(kind='m', name='connector_state', metric=MetricDataItem(data_type='b', value=False)),
+  Item(kind='m', name='temp_01', metric=MetricDataItem(data_type='d', value=Decimal('34.4'))),
+  Item(kind='m', name='temp_02', metric=MetricDataItem(data_type='d', value=Decimal('36.4'))),
+  Item(kind='m', name='temp_03', metric=MetricDataItem(data_type='d', value=Decimal('10.4'))),
+  Item(kind='m', name='pwr', metric=MetricDataItem(data_type='d', value=Decimal('12.231'))),
+  Item(kind='m', name='current', metric=MetricDataItem(data_type='d', value=Decimal('1.429'))),
+  Item(kind='m', name='current_battery', metric=MetricDataItem(data_type='d', value=Decimal('1.548')))]
 ```
 
+#### IoText message with lists of values
+
 IoText protocol in `schema-less version` data row example with lists values:
 ```bash
 t|3900237526142,d|device_name_002,m|ints_list=I:+1-22+333333,m|bools_list=B:0111,m|decimals_list=D:-123.456+1234567890.98765+999.8,m|texts_list=T:Wyd8JywgJzphYmMnLCAnISFAJywgJ3h5MHonLCAnMWFiYywnXQ
@@ -127,11 +152,29 @@ and after de-serialization to Python data structures with lists values we can se
 ]
 ```
 
-#### Preparing message
-You need two required informations:
-- timestamp in UNIX millis
-- device name
-- all data metrics are optional, so it means, that you can use IoText protocol like ping - or health check - as well (we have dedicated `h` item type for helth check, too).
+#### IoText message with CRC16 sum control item
+
+IoText protocol in `schema-less version` data row example:
+```bash
+t|123123123123,d|device_one,m|value=d:123.321,c|4C5A
+```
+and after de-serialization to Python data structures we can see:
+```bash
+[
+    Item(kind=ItemTypes.TIMESTAMP_MILIS, name="123123123123", metric=None),
+    Item(kind=ItemTypes.DEVICE_ID, name="device_one", metric=None),
+    Item(
+        kind=ItemTypes.METRIC_ITEM,
+        name="value",
+        metric=MetricDataItem(data_type=MetricDataTypes.DECIMAL, value=Decimal('123.321')),
+    ),
+    Item(
+        kind=ItemTypes.CRC,
+        name="4C5A",
+        metric=None
+    )
+]
+```
 
 
 ## How to use this library?

src/builders/iot_ext_item_data_builder.py

@@ -2,17 +2,21 @@
 
 from src.codecs.item_codec import ItemCodec
 from src.codecs.metric_data_item_codec import MetricDataItemCodec
+from src.tools.crc16 import Tools
 from src.types.item import Item
 from src.types.item_type import ItemTypes
 from src.types.metric_data_item import MetricValueType
 
 
 class IoTextItemDataBuilder:
-    def __init__(self, timestamp: int, device_name: str) -> None:
+    def __init__(
+        self, timestamp: int, device_name: str, add_crc16: bool = False
+    ) -> None:
         self.timestamp: int = timestamp
         self.device_name: str = device_name
         self.output: List[Item] = []
         self.metrics: List[Item] = []
+        self.add_crc16 = add_crc16
 
     def add_measure(self, metric_name: str, metric_value: MetricValueType) -> None:
         item = Item(
@@ -22,9 +26,15 @@ def add_measure(self, metric_name: str, metric_value: MetricValueType) -> None:
         )
         self.metrics.append(item)
 
+    def __prepare_msg(self, items_separator=","):
+        return items_separator.join([ItemCodec.encode(item) for item in self.output])
+
     def __str__(self, items_separator=",") -> str:
         self.output.append(Item(ItemTypes.TIMESTAMP_MILIS, str(self.timestamp)))
         self.output.append(Item(ItemTypes.DEVICE_ID, self.device_name))
         for metric_item in self.metrics:
             self.output.append(metric_item)
-        return items_separator.join([ItemCodec.encode(item) for item in self.output])
+        if self.add_crc16:
+            msg = self.__prepare_msg(items_separator=items_separator)
+            self.output.append(Item(ItemTypes.CRC, Tools.crc16(msg)))
+        return self.__prepare_msg(items_separator=items_separator)

src/types/item_type.py

@@ -9,3 +9,4 @@ class ItemTypes(str, Enum):
     DEVICE_ID = "d"
     METRIC_ITEM = "m"
     HEALTH_CHECK = "h"
+    CRC = "c"

tests/builders/test_iot_ext_item_data_builder.py

@@ -39,3 +39,36 @@ def test_should_add_a_few_measures_and_serialize_to_str(self):
         builder.add_measure("counter_01", 1234)
 
         self.assertEqual(expected, str(builder))
+
+    @parameterized.expand(
+        [
+            (12.07, "d:12.07", "3E3C"),
+            (True, "b:1", "D1F8"),
+            (False, "b:0", "1139"),
+            (42, "i:42", "2350"),
+            ("abc", "t:abc", "07A3"),
+        ]
+    )
+    def test_should_add_measure_and_serialize_to_str_with_crc(
+        self, value, expected_output_suffix, crc16_value
+    ):
+        metric_name = "example_metric_name"
+        expected = f"t|3900237526042,d|DEV_NAME_002,m|{metric_name}={expected_output_suffix},c|{crc16_value}"
+        builder = IoTextItemDataBuilder(3900237526042, "DEV_NAME_002", add_crc16=True)
+        builder.add_measure(metric_name, value)
+
+        self.assertEqual(expected, str(builder))
+
+    def test_should_add_a_few_measures_and_serialize_to_str_with_crc(self):
+        expected = (
+            "t|3900237526042,d|DEV_NAME_002,m|"
+            "battery_level=d:12.07,m|open_door=b:1,m|open_window=b:0,m|counter_01=i:1234,c|DCF7"
+        )
+
+        builder = IoTextItemDataBuilder(3900237526042, "DEV_NAME_002", add_crc16=True)
+        builder.add_measure("battery_level", 12.07)
+        builder.add_measure("open_door", True)
+        builder.add_measure("open_window", False)
+        builder.add_measure("counter_01", 1234)
+
+        self.assertEqual(expected, str(builder))

tests/codecs/test_iot_ext_codec.py

@@ -116,6 +116,21 @@
     ),
 ]
 
+MSG_EXAMPLE_WITH_CRC = """t|123123123123,d|device_one,m|value=d:123.321,c|4C5A"""
+
+MSG_EXAMPLE_WITH_CRC_AS_DATA_STRUCTS = [
+    Item(kind=ItemTypes.TIMESTAMP_MILIS, name="123123123123", metric=None),
+    Item(kind=ItemTypes.DEVICE_ID, name="device_one", metric=None),
+    Item(
+        kind=ItemTypes.METRIC_ITEM,
+        name="value",
+        metric=MetricDataItem(
+            data_type=MetricDataTypes.DECIMAL, value=Decimal("123.321")
+        ),
+    ),
+    Item(kind=ItemTypes.CRC, name="4C5A", metric=None),
+]
+
 
 class IoTextCodecTest(TestCase):
     def test_decode(self):
@@ -148,3 +163,9 @@ def test_lists_decode(self):
         result = IoTextCodec.decode(iotext_list_msg)
 
         self.assertEqual(expected, result)
+
+    def test_decode_msg_with_crc(self):
+        expected = MSG_EXAMPLE_WITH_CRC_AS_DATA_STRUCTS
+
+        result = IoTextCodec.decode(MSG_EXAMPLE_WITH_CRC)
+        self.assertEqual(expected, result)

tests/tools/test_crc16.py

@@ -7,5 +7,9 @@ class ToolsTest(unittest.TestCase):
     def test_should_crc16(self):
         self.assertEqual("5749", Tools.crc16("abc"))
 
+    def test_should_iotext_message_with_crc16(self):
+        example_iotext_msg = "t|123123123123,d|device_one,m|value=d:123.321"
+        self.assertEqual("4C5A", Tools.crc16(example_iotext_msg))
+
     def test_should_crc16_with_empty_string(self):
         self.assertEqual("FFFF", Tools.crc16(""))