From 66242a9617e8dda65c0581b50070a171d3786981 Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Sun, 7 Apr 2024 20:58:14 +0200 Subject: [PATCH 1/2] add CRC16 implementation for IoText protocol + unit tests --- Makefile | 2 + README.md | 57 ++++++++++++++++--- src/builders/iot_ext_item_data_builder.py | 14 ++++- src/types/item_type.py | 1 + .../test_iot_ext_item_data_builder.py | 33 +++++++++++ tests/codecs/test_iot_ext_codec.py | 21 +++++++ tests/tools/test_crc16.py | 4 ++ 7 files changed, 123 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index ee01306..92c3703 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +test: + pytest tests/ ci: black src/ black tests/ diff --git a/README.md b/README.md index 046713e..f80a93b 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ We have a few basic item types: - `t` - timestamp UNIX milis item - `d` - device name item - `m` - metric item -- `h` - helth check +- `h` - health check +- `c` - CRC-16 (from MODBUS control sum) for all combined IoText payload as string You can see reference implementation in `ItemTypes` class in `src/types` package. @@ -85,16 +86,40 @@ We have a few basic data codecs: You can see reference implementation in `src/codecs` package. +### How to prepare IoText message +You need two required informations: +- timestamp in UNIX millis +- device name +- all data metrics are optional, so it means, that you can use IoText protocol like ping - or health check - as well (we have dedicated `h` item type for helth check, too). +- CRC item it's optional sum control method (using CRC-16 MODBUS implementation) + ### Examples + +#### IoText message with generic values + IoText protocol in `schema-less version` data row example: ```bash t|3900237526042,d|device_name_001,m|val_water_001=i:1234,m|val_water_002=i:15,m|bulb_state=b:1,m|connector_state=b:0,m|temp_01=d:34.4,m|temp_02=d:36.4,m|temp_03=d:10.4,m|pwr=d:12.231,m|current=d:1.429,m|current_battery=d:1.548 ``` and after de-serialization to Python data structures we can see: ```bash -[Item(kind='t', name='3900237526042', metric=None), Item(kind='d', name='device_name_001', metric=None), Item(kind='m', name='val_water_001', metric=MetricDataItem(data_type='i', value=1234)), Item(kind='m', name='val_water_002', metric=MetricDataItem(data_type='i', value=15)), Item(kind='m', name='bulb_state', metric=MetricDataItem(data_type='b', value=True)), Item(kind='m', name='connector_state', metric=MetricDataItem(data_type='b', value=False)), Item(kind='m', name='temp_01', metric=MetricDataItem(data_type='d', value=Decimal('34.4'))), Item(kind='m', name='temp_02', metric=MetricDataItem(data_type='d', value=Decimal('36.4'))), Item(kind='m', name='temp_03', metric=MetricDataItem(data_type='d', value=Decimal('10.4'))), Item(kind='m', name='pwr', metric=MetricDataItem(data_type='d', value=Decimal('12.231'))), Item(kind='m', name='current', metric=MetricDataItem(data_type='d', value=Decimal('1.429'))), Item(kind='m', name='current_battery', metric=MetricDataItem(data_type='d', value=Decimal('1.548')))] +[ + Item(kind='t', name='3900237526042', metric=None), + Item(kind='d', name='device_name_001', metric=None), + Item(kind='m', name='val_water_001', metric=MetricDataItem(data_type='i', value=1234)), + Item(kind='m', name='val_water_002', metric=MetricDataItem(data_type='i', value=15)), + Item(kind='m', name='bulb_state', metric=MetricDataItem(data_type='b', value=True)), + Item(kind='m', name='connector_state', metric=MetricDataItem(data_type='b', value=False)), + Item(kind='m', name='temp_01', metric=MetricDataItem(data_type='d', value=Decimal('34.4'))), + Item(kind='m', name='temp_02', metric=MetricDataItem(data_type='d', value=Decimal('36.4'))), + Item(kind='m', name='temp_03', metric=MetricDataItem(data_type='d', value=Decimal('10.4'))), + Item(kind='m', name='pwr', metric=MetricDataItem(data_type='d', value=Decimal('12.231'))), + Item(kind='m', name='current', metric=MetricDataItem(data_type='d', value=Decimal('1.429'))), + Item(kind='m', name='current_battery', metric=MetricDataItem(data_type='d', value=Decimal('1.548')))] ``` +#### IoText message with lists of values + IoText protocol in `schema-less version` data row example with lists values: ```bash t|3900237526142,d|device_name_002,m|ints_list=I:+1-22+333333,m|bools_list=B:0111,m|decimals_list=D:-123.456+1234567890.98765+999.8,m|texts_list=T:Wyd8JywgJzphYmMnLCAnISFAJywgJ3h5MHonLCAnMWFiYywnXQ @@ -127,11 +152,29 @@ and after de-serialization to Python data structures with lists values we can se ] ``` -#### Preparing message -You need two required informations: -- timestamp in UNIX millis -- device name -- all data metrics are optional, so it means, that you can use IoText protocol like ping - or health check - as well (we have dedicated `h` item type for helth check, too). +#### IoText message with CRC16 sum control item + +IoText protocol in `schema-less version` data row example: +```bash +t|123123123123,d|device_one,m|value=d:123.321,c|4C5A +``` +and after de-serialization to Python data structures we can see: +```bash +[ + Item(kind=ItemTypes.TIMESTAMP_MILIS, name="123123123123", metric=None), + Item(kind=ItemTypes.DEVICE_ID, name="device_one", metric=None), + Item( + kind=ItemTypes.METRIC_ITEM, + name="value", + metric=MetricDataItem(data_type=MetricDataTypes.DECIMAL, value=Decimal('123.321')), + ), + Item( + kind=ItemTypes.CRC, + name="4C5A", + metric=None + ) +] +``` ## How to use this library? diff --git a/src/builders/iot_ext_item_data_builder.py b/src/builders/iot_ext_item_data_builder.py index 2c28e3f..7f4cfc4 100644 --- a/src/builders/iot_ext_item_data_builder.py +++ b/src/builders/iot_ext_item_data_builder.py @@ -2,17 +2,21 @@ from src.codecs.item_codec import ItemCodec from src.codecs.metric_data_item_codec import MetricDataItemCodec +from src.tools.crc16 import Tools from src.types.item import Item from src.types.item_type import ItemTypes from src.types.metric_data_item import MetricValueType class IoTextItemDataBuilder: - def __init__(self, timestamp: int, device_name: str) -> None: + def __init__( + self, timestamp: int, device_name: str, add_crc16: bool = False + ) -> None: self.timestamp: int = timestamp self.device_name: str = device_name self.output: List[Item] = [] self.metrics: List[Item] = [] + self.add_crc16 = add_crc16 def add_measure(self, metric_name: str, metric_value: MetricValueType) -> None: item = Item( @@ -22,9 +26,15 @@ def add_measure(self, metric_name: str, metric_value: MetricValueType) -> None: ) self.metrics.append(item) + def __prepare_msg(self, items_separator=","): + return items_separator.join([ItemCodec.encode(item) for item in self.output]) + def __str__(self, items_separator=",") -> str: self.output.append(Item(ItemTypes.TIMESTAMP_MILIS, str(self.timestamp))) self.output.append(Item(ItemTypes.DEVICE_ID, self.device_name)) for metric_item in self.metrics: self.output.append(metric_item) - return items_separator.join([ItemCodec.encode(item) for item in self.output]) + if self.add_crc16: + msg = self.__prepare_msg(items_separator=items_separator) + self.output.append(Item(ItemTypes.CRC, Tools.crc16(msg))) + return self.__prepare_msg(items_separator=items_separator) diff --git a/src/types/item_type.py b/src/types/item_type.py index 24e230c..ee6c1b1 100644 --- a/src/types/item_type.py +++ b/src/types/item_type.py @@ -9,3 +9,4 @@ class ItemTypes(str, Enum): DEVICE_ID = "d" METRIC_ITEM = "m" HEALTH_CHECK = "h" + CRC = "c" diff --git a/tests/builders/test_iot_ext_item_data_builder.py b/tests/builders/test_iot_ext_item_data_builder.py index 5d993bb..2142194 100644 --- a/tests/builders/test_iot_ext_item_data_builder.py +++ b/tests/builders/test_iot_ext_item_data_builder.py @@ -39,3 +39,36 @@ def test_should_add_a_few_measures_and_serialize_to_str(self): builder.add_measure("counter_01", 1234) self.assertEqual(expected, str(builder)) + + @parameterized.expand( + [ + (12.07, "d:12.07", "3E3C"), + (True, "b:1", "D1F8"), + (False, "b:0", "1139"), + (42, "i:42", "2350"), + ("abc", "t:abc", "07A3"), + ] + ) + def test_should_add_measure_and_serialize_to_str_with_crc( + self, value, expected_output_suffix, crc16_value + ): + metric_name = "example_metric_name" + expected = f"t|3900237526042,d|DEV_NAME_002,m|{metric_name}={expected_output_suffix},c|{crc16_value}" + builder = IoTextItemDataBuilder(3900237526042, "DEV_NAME_002", add_crc16=True) + builder.add_measure(metric_name, value) + + self.assertEqual(expected, str(builder)) + + def test_should_add_a_few_measures_and_serialize_to_str_with_crc(self): + expected = ( + "t|3900237526042,d|DEV_NAME_002,m|" + "battery_level=d:12.07,m|open_door=b:1,m|open_window=b:0,m|counter_01=i:1234,c|DCF7" + ) + + builder = IoTextItemDataBuilder(3900237526042, "DEV_NAME_002", add_crc16=True) + builder.add_measure("battery_level", 12.07) + builder.add_measure("open_door", True) + builder.add_measure("open_window", False) + builder.add_measure("counter_01", 1234) + + self.assertEqual(expected, str(builder)) diff --git a/tests/codecs/test_iot_ext_codec.py b/tests/codecs/test_iot_ext_codec.py index 9e91b12..273482a 100644 --- a/tests/codecs/test_iot_ext_codec.py +++ b/tests/codecs/test_iot_ext_codec.py @@ -116,6 +116,21 @@ ), ] +MSG_EXAMPLE_WITH_CRC = """t|123123123123,d|device_one,m|value=d:123.321,c|70AA""" + +MSG_EXAMPLE_WITH_CRC_AS_DATA_STRUCTS = [ + Item(kind=ItemTypes.TIMESTAMP_MILIS, name="123123123123", metric=None), + Item(kind=ItemTypes.DEVICE_ID, name="device_one", metric=None), + Item( + kind=ItemTypes.METRIC_ITEM, + name="value", + metric=MetricDataItem( + data_type=MetricDataTypes.DECIMAL, value=Decimal("123.321") + ), + ), + Item(kind=ItemTypes.CRC, name="70AA", metric=None), +] + class IoTextCodecTest(TestCase): def test_decode(self): @@ -148,3 +163,9 @@ def test_lists_decode(self): result = IoTextCodec.decode(iotext_list_msg) self.assertEqual(expected, result) + + def test_decode_msg_with_crc(self): + expected = MSG_EXAMPLE_WITH_CRC_AS_DATA_STRUCTS + + result = IoTextCodec.decode(MSG_EXAMPLE_WITH_CRC) + self.assertEqual(expected, result) diff --git a/tests/tools/test_crc16.py b/tests/tools/test_crc16.py index f83f218..e50462b 100644 --- a/tests/tools/test_crc16.py +++ b/tests/tools/test_crc16.py @@ -7,5 +7,9 @@ class ToolsTest(unittest.TestCase): def test_should_crc16(self): self.assertEqual("5749", Tools.crc16("abc")) + def test_should_iotext_message_with_crc16(self): + example_iotext_msg = "t|123123123123,d|device_one,m|value=d:123.321" + self.assertEqual("4C5A", Tools.crc16(example_iotext_msg)) + def test_should_crc16_with_empty_string(self): self.assertEqual("FFFF", Tools.crc16("")) From 7e784de4e2cb803b84581dba63bbc97697a3d01c Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Sun, 7 Apr 2024 21:01:18 +0200 Subject: [PATCH 2/2] update CRC value in one unit test --- tests/codecs/test_iot_ext_codec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/codecs/test_iot_ext_codec.py b/tests/codecs/test_iot_ext_codec.py index 273482a..d5bdc05 100644 --- a/tests/codecs/test_iot_ext_codec.py +++ b/tests/codecs/test_iot_ext_codec.py @@ -116,7 +116,7 @@ ), ] -MSG_EXAMPLE_WITH_CRC = """t|123123123123,d|device_one,m|value=d:123.321,c|70AA""" +MSG_EXAMPLE_WITH_CRC = """t|123123123123,d|device_one,m|value=d:123.321,c|4C5A""" MSG_EXAMPLE_WITH_CRC_AS_DATA_STRUCTS = [ Item(kind=ItemTypes.TIMESTAMP_MILIS, name="123123123123", metric=None), @@ -128,7 +128,7 @@ data_type=MetricDataTypes.DECIMAL, value=Decimal("123.321") ), ), - Item(kind=ItemTypes.CRC, name="70AA", metric=None), + Item(kind=ItemTypes.CRC, name="4C5A", metric=None), ]