Merge pull request #65 from orgua/dev

Dev

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.2
+current_version = 0.8.3
 commit = False
 tag = False
 
@@ -21,6 +21,6 @@ tag = False
 
 [bumpversion:file:software/pps-gmtimer/src/pps-gmtimer.c]
 
-[bumpversion:file:software/time_sync_analyzer/sync_analysis/__init__.py]
+[bumpversion:file:software/debug_analyze_time_sync/sync_analysis/__init__.py]
 
 [bumpversion:file:docs/dev/contributing.md]

.pre-commit-config.yaml

@@ -36,13 +36,13 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.41.0
+    rev: v0.42.0
     hooks:
       - id: markdownlint
         args: ["--disable", "MD013"] # ignore line length
 
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.19
+    rev: v0.20.2
     hooks:
       - id: validate-pyproject
         # files: "./software/python-package/pyproject.toml"
@@ -62,7 +62,7 @@ repos:
       - id: text-unicode-replacement-char
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.6.4'
+    rev: 'v0.6.8'
     hooks:
       - id: ruff-format
       - id: ruff
@@ -91,7 +91,7 @@ repos:
         exclude: \.(sch|brd|lbr)$
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v18.1.8
+    rev: v19.1.0
     hooks:
       - id: clang-format
         types_or: [c++, c]

.ruff.toml

@@ -19,8 +19,9 @@ ignore = [
     "D203", "D212", "D213", "D214", "D215",  # conflicts with PEP257-convention (A)
     "D404", "D405", "D406", "D407", "D408", "D409",  # conflicts with PEP257-convention (B)
     "D410", "D411", "D413", "D415", "D416", "D417",  # conflicts with PEP257-convention (C)
-    "EM101", "TRY003", "TRY400",  # xpt & strings
-    "D102",  # TODO: for releasing now - do add documentation later
+    "EM101", "TRY003",  # xpt & strings
+    # TODO: for releasing now - do add documentation later
+    "D",
 ]
 
 exclude= [  # external projects
@@ -30,11 +31,13 @@ exclude= [  # external projects
 ]
 
 [lint.per-file-ignores]
-"*/tests/**" = ["ARG", "S", "D", "SLF001"]
+"*/tests/**" = ["ARG", "S", "D", "SLF001", "T201"]
 "*/examples/**" = ["INP001", "ERA001"]  # no namespace + commented-out code
 "software/shepherd-calibration/**" = ["ERA001"]  # comments
-"software/time_sync_analyzer/**" = ["ERA001", "S301"]  # comments
+"software/debug_analyze_time_sync/**" = ["ERA001", "S301"]  # comments
 "software/shepherd-devicetest/**" = ["ERA001", "ARG001", "PLW0603", "F405", "F403", "ANN001"]
+"software/python-package/shepherd_sheep/shepherd_debug.py" = ["FBT001", "FBT002"]
+"software/shepherd-calibration/shepherd_cal/**" = ["FBT003"]  # rpc had trouble with named params
 "docs/**" = ["INP001"]
 
 [lint.mccabe]

CHANGELOG.md

@@ -1,5 +1,23 @@
 # History of Changes
 
+## 0.8.3
+
+- setup
+  - fix ambitious removal of ciphers for secure host
+  - test and adapt to raspberry pi OS
+  - create smaller playbook to redeploy roles (`dev_redeploy_roles.yml`)
+- herd
+  - fix StringIO-Bug
+- timesync: improve configuration for client and server
+- sync analysis: move to `software/debug_analyze_time_sync` and also add CLI
+- vsrc comparison: move to `software/debug_compare_vsrouces`
+- py
+  - fix > 100 linting error
+  - improve exception-system
+- pru
+  - extend error-system
+  - refactor and fix messaging-system
+
 ## 0.8.2
 
 - PRU now gets partially zeroed buffer-segments
@@ -93,9 +111,9 @@
 - toolchain: replace isort, black, flake8, pylint by ruff
 - debug
   - add option to generate kernel gpio edges (`trigger_loop_callback()` in `pru_sync_control.c`)
-  - allow analyzing recorded gpio sync traces (`software/time_sync_analyzer`)
+  - allow analyzing recorded gpio sync traces (`software/debug_analyze_time_sync`)
 - update floorplan of testbed
-- add tooling to allow analyzing timesync-behavior (software/time_sync_analyzer)
+- add tooling to allow analyzing timesync-behavior (software/debug_analyze_time_sync)
 - add current hardware design files
 - major overhaul documentation
 - **tested (fully)**: pytest sheep, pytest herd, ansible install

Pipfile

@@ -10,7 +10,7 @@ shepherd-sheep = {editable = true, path = "./software/python-package"}
 shepherd-herd = {editable = true, path = "./software/shepherd-herd"}
 shepherd-cal = {editable = true, path = "./software/shepherd-calibration"}
 shepherd-data = "*"
-sync-analysis = {editable = true, path = "./software/time_sync_analyzer"}
+sync-analysis = {editable = true, path = "./software/debug_analyze_time_sync"}
 # temporary
 dearpygui = "*"
 

deploy/dev_redeploy_roles.yml

@@ -0,0 +1,56 @@
+---
+# install shepherd-software, all dependencies & roles set in inventory-file
+
+- name: Install Shepherd-Software - Role-based
+  hosts: all
+  become: true
+
+  pre_tasks:
+
+    - name: Get time and date from ansible controller
+      ansible.builtin.command: 'date -u +"%Y-%m-%dT%H:%M:%SZ"'
+      delegate_to: localhost
+      changed_when: false
+      register: local_time
+      become: false
+    - name: Set correct time
+      ansible.builtin.command: "date -s {{ local_time.stdout }}"
+      changed_when: true
+
+- name: Role - PTP-Client only (not recommended! all should be server as well)
+  hosts: ptp_clients
+  become: true
+  strategy: free # noqa: run-once[play]
+  roles:
+    - ptp_client
+
+- name: Role - NTP-Client
+  hosts: ntp_clients
+  become: true
+  strategy: free # noqa: run-once[play]
+  roles:
+    - ntp_client
+
+- name: Role - GPS-Client
+  hosts: gps_clients
+  become: true
+  strategy: free # noqa: run-once[play]
+  roles:
+    - gps_client
+# TODO: how to avoid allowing gps & ptp & ntp client? trouble ahead
+#       -> could switch to components and argument_specs
+#       https://steampunk.si/blog/ansible-role-argument-specification/
+
+- name: Role - PTP-Server
+  hosts: ptp_servers
+  become: true
+  strategy: free # noqa: run-once[play]
+  roles:
+    - ptp_server
+
+- name: Role - Secured Testbed
+  hosts: secured
+  become: true
+  strategy: free # noqa: run-once[play]
+  roles:
+    - secure_testbed

deploy/roles/ptp_client/tasks/main.yml

@@ -2,9 +2,22 @@
 - name: Configure PTP for client-only mode
   ansible.builtin.lineinfile:
     dest: "/etc/linuxptp/ptp4l.conf"
-    regexp: "slaveOnly.*$"
-    line: "slaveOnly  1"
+    regexp: "{{ item.regex }}"
+    line: "{{ item.replacement }}"
     state: present
+  loop:
+    - {
+      regex: "slaveOnly.*$",
+      replacement: "slaveOnly  0",
+    }
+    - {
+      regex: "masterOnly.*$",
+      replacement: "masterOnly  0",
+    }
+    - {
+      regex: "priority1.*$",
+      replacement: "priority1  128",
+    }
   tags:
     - ptp
     - conf

deploy/roles/ptp_host/files/phc2sys@.service

@@ -7,7 +7,7 @@ Before=time-sync.target
 # -> ptp4l needs to start AFTER phc2sys, otherwise the time gets reset
 
 [Service]
-Type=simple
+Type=idle
 ExecStartPre=-/bin/sleep 5
 # ExecStart=/usr/sbin/phc2sys -w -s %I
 # ExecStart=/usr/sbin/phc2sys -rr -w -s %I

deploy/roles/ptp_host/files/ptp4l@.service

@@ -5,7 +5,7 @@ After=sys-subsystem-net-devices-%i.device
 # -> wanted: this service waits for network-connectivity
 
 [Service]
-Type=simple
+Type=idle
 ExecStartPre=/usr/sbin/ntpdate -b -s -u pool.ntp.org
 # ExecStart=/usr/sbin/ptp4l -f /etc/linuxptp/ptp4l.conf -i %I
 ExecStart=/usr/sbin/ptp4l -A -H -f /etc/linuxptp/ptp4l.conf -i %I

deploy/roles/ptp_host/tasks/main.yml

@@ -10,6 +10,26 @@
     state: present
   tags: ptp
 
+- name: Configure PTP
+  ansible.builtin.lineinfile:
+    dest: "/etc/linuxptp/ptp4l.conf"
+    regexp: "{{ item.regex }}"
+    line: "{{ item.replacement }}"
+    state: present
+  loop:
+    - {
+      regex: "clock_servo.*$",
+      replacement: "clock_servo  pi",
+      # NOTE: linreg is faster at start, BUT Pi is more precise here
+    }
+    - {
+      regex: "time_stamping.*$",
+      replacement: "time_stamping     hardware",
+    }
+  tags:
+    - ptp
+    - conf
+
 - name: Install ntpdate package
   ansible.builtin.apt:
     name: ntpdate
@@ -31,3 +51,19 @@
   # Tests
   # manual: sudo systemctl restart ptp4l@eth0
   # status: sudo journalctl -u ptp4l@eth0 -f
+
+- name: Install ethtool
+  ansible.builtin.apt:
+    name: ethtool
+    state: present
+  tags: ptp
+
+- name: Check interface capabilities
+  ansible.builtin.shell:
+    cmd: "set -o pipefail && ethtool -T eth0 | grep {{ item }}"
+  loop:
+    - hardware-raw-clock
+    - hardware-transmit
+    - hardware-receive
+  failed_when: false
+  tags: ptp

deploy/roles/ptp_server/tasks/main.yml

@@ -2,9 +2,22 @@
 - name: Configure PTP for server-mode (time-coordinator)
   ansible.builtin.lineinfile:
     dest: "/etc/linuxptp/ptp4l.conf"
-    regexp: "slaveOnly.*$"
-    line: "slaveOnly  0"
+    regexp: "{{ item.regex }}"
+    line: "{{ item.replacement }}"
     state: present
+  loop:
+    - {
+      regex: "slaveOnly.*$",
+      replacement: "slaveOnly  0",
+    }
+    - {
+      regex: "masterOnly.*$",
+      replacement: "masterOnly  1",
+    }
+    - {
+      regex: "priority1.*$",
+      replacement: "priority1  100",
+    }
   tags:
     - ptp
     - conf

deploy/roles/secure_testbed/tasks/main.yml

@@ -34,7 +34,8 @@
     # select good AND exclude weak algorithms: TODO: just forbid
     - {regex: '^.*KexAlgorithms.*$', line: 'KexAlgorithms -ecdh-sha2*,diffie-hellman-group-exchange*,diffie-hellman-group14-sha1'}
     - {regex: '^.*HostKeyAlgorithms.*$', line: 'HostKeyAlgorithms -ecda-sha2*,ecdsa-sha2*'}
-    - {regex: '^.*Ciphers.*$', line: 'Ciphers -arcfour*,[email protected],aes128-ctr,aes192-ctr,aes256-ctr'}
+    - {regex: '^.*Ciphers.*$', line: 'Ciphers -arcfour*,[email protected],aes128-ctr,aes192-ctr'}
+    # cipher aes256-ctr is used by paramico (herd) - last remaining option - so keep it for now even it is considered insecure
     - {regex: '^.*MACs.*$', line: 'MACs -umac-64*,hmac-sha1*,hmac-sha2-256,hmac-sha2-512,umac-128@open*'}
   # TODO: x11Forwarding is twice in file, one yes and one no
 
@@ -65,6 +66,7 @@
     state: stopped
     enabled: false
   changed_when: true
+  failed_when: false  # RPi does not have this timer
 
 - name: CFG - Disable Terminal over Serial -> Services
   ansible.builtin.systemd:
@@ -77,6 +79,7 @@
     - [email protected]
     - [email protected]
   changed_when: true
+  failed_when: false  # RPi does not have some services
   # change active ones with: sudo systemctl list-units
 
 - name: CFG - Disable Terminal over Serial -> Grub
@@ -91,6 +94,7 @@
     dest: /boot/uEnv.txt
     regexp: '^console='
     replace: '#console='
+  failed_when: false  # RPi does not have this file
 
 # TODO: part 3 needed, terminal still active
 # TODO: uEnv has UART enabled

docs/conf.py

@@ -18,7 +18,7 @@
 project = "SHEPHERD"
 project_copyright = "2019-2024, Networked Embedded Systems Lab, TU Dresden & TU Darmstadt"
 author = "Kai Geissdoerfer, Ingmar Splitt"
-release = "0.8.2"
+release = "0.8.3"
 
 # -- General configuration ---------------------------------------------------
 

docs/dev/contributing.md

@@ -123,7 +123,7 @@ Use `bump2version` to update the version number across the repository:
 ```shell
 pipenv shell
 pre-commit run --all-files
-bump2version --allow-dirty --new-version 0.8.2 patch
+bump2version --allow-dirty --new-version 0.8.3 patch
 # version-format: major.minor.patch
 ```
 

docs/external/readme_sync_analyzer.md

@@ -1,2 +1,2 @@
-:::{include} ../../software/time_sync_analyzer/Readme.md
+:::{include} ../../software/debug_analyze_time_sync/Readme.md
 :::

software/README.md

@@ -14,4 +14,4 @@ A collection of components and subprojects needed for the testbed. Bold items ar
 - **`shepherd-herd`**: python-software to control sheep / observers
 - `shepherd-targets`: sub-git for target hardware and default firmwares
 - `shepherd-webservice`: sub-git for the API and Website of the official testbed instance
-- `time_sync_analyzer`: python-software for measuring sync between observers on gpio-level
+- `debug_analyze_time_sync`: python-software for measuring sync between observers on gpio-level