Merge branch 'develop/features-for-comp' into 89-telemetry-implement-…

…temp-and-humidity-onboard-tpu

.gitattributes

@@ -1 +1 @@
-*.sh            text eol=lf
+* text=auto eol=lf

.github/workflows/build_image.yml

@@ -0,0 +1,63 @@
+#
+name: Create and publish a Docker image
+
+# Configures this workflow to run every time a docker artificat is edited
+on:
+  push:
+    paths:
+        - 'odysseus/Dockerfile'
+        - 'odysseus/docker_scripts/**'
+        - '.github/workflows/build_image.yml'
+  workflow_dispatch:
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      # 
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Encrypt and inject passwords
+        shell: bash
+        working-directory: ./odysseus
+        env:
+            ODY_BASE_WIFI_PASSWORD: ${{ secrets.ODY_BASE_WIFI_PASSWORD }}
+            ODY_AP_ROOT_PASSWORD: ${{ secrets.ODY_AP_ROOT_PASSWORD }}
+            ODY_TPU_ROOT_PASSWORD: ${{ secrets.ODY_TPU_ROOT_PASSWORD }}
+            ODY_IROH_ROOT_PASSWORD: ${{ secrets.ODY_IROH_ROOT_PASSWORD }}
+            MASTER_PASSWORD: ${{ secrets.ODY_MASTER_PASSWORD }}
+        run:  for i in ODY_AP_ROOT_PASSWORD  ODY_TPU_ROOT_PASSWORD ODY_IROH_ROOT_PASSWORD ODY_BASE_WIFI_PASSWORD; do echo "$i=${!i}" >> SECRETS.env; done && gpg --batch --symmetric --passphrase "$MASTER_PASSWORD" --no-symkey-cache --cipher-algo AES256 SECRETS.env
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: ./odysseus
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -3,8 +3,14 @@
 
 *.o
 *.bin
-
 */__pycache__
 
 .DS_Store
 odysseus/outputs/**
+
+# for passwords
+odysseus/SECRETS.env
+odysseus/SECRETS.env-example
+# redundant safeguard for all env files named SECRETS.env in case it is renamed wrong
+**/*SECRETS.env*
+/.idea

extra/emqx_base/README.md

@@ -5,3 +5,6 @@ Clone repo and cd to this directory.
 
 run with (note: absolute paths must be used):  
 `docker run -d --restart=always --name emqx -p 18083:18083 -p 1883:1883 -p 14567:14567/udp -v /path/to/emqx.conf:/opt/emqx/etc/emqx.conf -v /path/to/listeners.quic.conf:/opt/emqx/etc/listeners.quic.conf -v /path/to/listeners.tcp.conf:/opt/emqx/etc/listeners.tcp.conf emqx:5.5.0`
+
+Ex.  
+`docker run -d --restart=always --name emqx -p 18083:18083 -p 1883:1883 -p 14567:14567/udp -v /opt/Odysseus/extra/emqx_base/emqx.conf:/opt/emqx/etc/emqx.conf -v /opt/Odysseus/extra/emqx_base/listeners.quic.conf:/opt/emqx/etc/listeners.quic.conf -v /opt/Odysseus/extra/emqx_base/listeners.tcp.conf:/opt/emqx/etc/listeners.tcp.conf emqx:5.5.0`

extra/mosquitto_base/README.md

@@ -6,7 +6,8 @@ All of the steps should be done in the ssh console of the base station, with IP
 3. run the below command.  note that the path to the moquitto.conf in this directory **must be absolute**.
 
 `docker run --restart=always -it -p 1883:1883 -p 9001:9001 -v /path/to/mosquitto.conf:/mosquitto/config/mosquitto.conf -v /mosquitto/data -v /mosquitto/log -d eclipse-mosquitto`
-
+Ex.  
+`docker run --restart=always -it -p 1883:1883 -p 9001:9001 -v /opt/Odysseus/extra/mosquitto_base/mosquitto.conf:/mosquitto/config/mosquitto.conf -v /mosquitto/data -v /mosquitto/log -d eclipse-mosquitto`
 
 ### Tips and tricks
 

extra/mosquitto_base/mosquitto.conf

@@ -172,6 +172,7 @@ address 192.168.100.12
 # *** diff from tpu (needed as topic key required)
 topic reserved out 2 dummy dummyremote
 
+
 #bridge_bind_address
 
 #bridge_attempt_unsubscribe true

odysseus/Dockerfile

@@ -5,7 +5,7 @@ RUN apt-get update && \
     apt-get -y install tzdata
 
 
-RUN apt-get install -y \
+RUN apt-get update && apt-get install -y \
     make \
     binutils \
     build-essential \
@@ -31,18 +31,23 @@ RUN apt-get install -y \
     python3-matplotlib \
     graphviz \
     git-lfs \
-    util-linux
+    util-linux \ 
+    wpasupplicant
 
 # just for calypso require ruamel.yaml
-RUN apt-get install -y python3-pip && pip install "ruamel.yaml<0.18.0"
+RUN apt-get update && apt-get install -y python3-pip && pip install "ruamel.yaml<0.18.0"
 
 # add all buildroot files there
 WORKDIR /home/odysseus/build
 
 RUN  git clone https://gitlab.com/buildroot.org/buildroot.git && cd ./buildroot && git checkout 2024.02
 
+
 WORKDIR /home/odysseus/outputs/
 COPY ./docker_scripts /home/odysseus/scripts
 RUN echo "source /home/odysseus/scripts/setup_env.sh" >> ~/.bashrc
 
+# install password using wildcard so failures arent deadly
+COPY ./SECRETS.env.* /home/odysseus/
+
 ENTRYPOINT "/bin/bash"

odysseus/README.md

@@ -27,19 +27,34 @@ All defconfigs come with (in addition to busybox and util_linux utilities):
 - iperf3, iw, iputils, and other network configuration utilities
 
 ## Quick start
-Download and install to PATH git and docker.
+Download and install to PATH git and docker.  Docker desktop is recommended for macOS and Windows.
 ```
 git clone https://github.com/Northeastern-Electric-Racing/Siren.git
 git submodule update --init --recursive
 cd ./odysseus
+```
+
+For linux (current support includes all build defconfigs):
+```
 docker compose run --rm --build odysseus # Future launches can omit `--build` for time savings and space savings, but it should be used if the Dockerfile or docker_out_of_tree.sh files change.  
+
 ```
+
+For mac and windows: (current support includes all _debug defconfigs, on x86_64 host normal defconfigs can work (experimental)):
+```
+docker compose -f "compose-compat.yml" run  --rm --build odysseus # Future launches can omit `--build` for time savings and space savings, but it should be used if the Dockerfile or docker_out_of_tree.sh files change.  
+```
+
+
+
 Now you are in the docker container.  To build cd into the defconfig directory (either ap, or tpu), then run the make command alias:
 ```
 cd ./<defconfig>
 make-current
 ```
-You can view the `output.log` for more info.  Now, to deploy just flash the image in `images/
+
+**Note: If failure occurs on mac or windows very early in the process, run `make nanomq-source` and try again before reporting the error.**
+You can view the `output.log` for more info.
 
 ### More on docker configuration
 The container has a directory structure as so:
@@ -49,13 +64,16 @@ The container has a directory structure as so:
     - `./odysseus_tree`: The odyssues external tree, bound to the same directory in the git repository on your local machine!
 - `./shared_data`: The download and ccache cache for buildroot, should be persisted as long as space is available, there is usually no reason to enter this. A persistent docker volume with the name   `odysseus_shared_data`.
 - `./outputs/*`:
-    - **The output folders for odysseus.  `cd` into the one named for what defconfig you would like to build, and run the `make` configuration and build commands as described below.  It is recommended to save space to run `make clean` in defconfig directories rather than removing this volume all together. This is bound to the `./odysseus/outputs` directory in the repository. *Remember to use `make savedefconfig` when you are done as changes are overriden when you re-open the docker image!*
+    - **The output folders for odysseus.  `cd` into the one named for what defconfig you would like to build, and run the `make` configuration and build commands as described below.  It is recommended to save space to run `make clean` in defconfig directories rather than removing this volume all together. For Linux hosts, this is bound to the `./odysseus/outputs` directory in the repository. For Windows users, this is a docker volume.  *Remember to use `make savedefconfig` when you are done as changes are overriden when you re-open the docker image!*
 
 ### Extra docker tips
 All paths relative to Siren root.
 
+#### Note for Windows/macOS users
+The outputs are stored in a docker volume on these platforms to ensure rsync compatability.  Therefore fetching the files requires first running the image, then use `docker cp` to get them to the userspace.  Alternatively, docker desktop has a file explorer for docker volumes that may come in handy.
+
 #### Writing the sd card
-The image is present in `./odysseus/outputs/<defconfig name>/images/sdcard.img`.
+The image is present in `./odysseus/outputs/<defconfig name>/images/sdcard.img`.  One can flash this with tools like the Raspberry Pi OS Imager.
 
 #### Pulling source files for scp, etc.
 The target binaries are located in `./odysseus/outputs/<defconfig name>/target`.
@@ -66,6 +84,10 @@ The target binaries are located in `./odysseus/outputs/<defconfig name>/target`.
 **IMPORTANT**: this WILL wipe all shared cache (don't do this unless you need the space):  
 `docker volume rm odysseus_shared_data`
 
+This will wipe all outputs, not the cache so just requires a full rebuild to recover from.  On compose-compat images only:
+`docker volume rm odysseus_outputs`
+
+
 #### Open another tty
 `docker ps`
 Find the container ID of odysseus-odysseus then run:  
@@ -78,12 +100,20 @@ docker exec <container_id> -d -w /home/odysseus/outputs/<defconfig> make-current
 ```
 Be careful with this.
 
-Notes about docker:
-- Build time may be slower due to docker isolations (not dramatic, about 5-15%)
-- Since odysseus only supports amd64 hosts for non-debug defconfigs, full releases cannot be built on mac
+Docker limitations:
+- Build time may be slower due to docker isolations (not dramatic, about 5%)
 - Launch time is longer
 - Space is used up by rebuilds, prune often or omit `--build`
 
+### Passwords
+Root passwords are stored via Github secrets and an encrypted file within a ghcr docker image.  Below are the steps to load and decrypt such passwords for use by buildroot.  Requirements are you know the team's master password and are running x86_64.
+
+1. Authenticate with ghcr.  First make a [classic PAT](https://github.com/settings/tokens/new) with the permission `read:packages`.  Make sure to copy the token, then run `sudo docker login ghcr.io -u <GITHUB_USERNAME> -p <PAT>`
+2. cd into odysseus folder and `docker compose pull`
+3. Run `docker compose run odysseus` to enter the docker image.  At this point the image should be identical to a locally built one, but it is less preferable for development purposes.
+4. Run `load-secrets` and enter the master password.  Consult Odysseus lead if you need this info.  Now your passwords are loaded (can be viewed with `env | grep ODY`), and will be set when you make the sdcard.img.  Note this step must be repeated on each `docker compose run odysseus`, and if the passwords change on Github steps 2 and 3 must be rerun as well.
+
+
 
 See below to learn more about developing, and check confluence for most info.  Once in the docker image, all the normal make commands (in an out-of-tree context only) apply.
 
@@ -92,6 +122,13 @@ See below to learn more about developing, and check confluence for most info.  O
 2. Make any customizations you want in the menu
 3. Save changes after you've made them by running ```make savedefconfig```.  Ensure you are saving changes to the intended defconfig, it is saved to whatever directory you `cd`ed into!
 
+#### Adding defconfigs
+
+Checklist when adding a defconfig:
+[ ] Add a secret for SSH password in settings, and load it into the workflow env in `.github/workflows/build_image.yml`
+[ ] Add the defconfig itself, changing path names, etc.  Board folders and overlays can still be shared as needed.
+[ ] Add a pretty name to `post-build-os-release.sh`
+[ ] Add a load command to `setup_env.sh`
 
 <!--
 ## Build locally

odysseus/compose-compat.yml

@@ -0,0 +1,32 @@
+version: "3.8"
+services:
+    odysseus:
+        build: .
+        command: /bin/bash
+        privileged: true
+        tty: true
+        # for fakeroot failure
+        ulimits:
+          nofile:
+            soft: 1024
+            hard: 1048576
+        volumes:
+            - ./odysseus_tree:/home/odysseus/build/odysseus_tree
+            - shared_data:/home/odysseus/shared_data
+            - outputs:/home/odysseus/outputs
+        environment:
+            - BR2_DL_DIR=/home/odysseus/shared_data/dl 
+            - BR2_CCACHE_DIR=/home/odysseus/shared_data/ccache
+            - ODY_TPU_ROOT_PASSWORD=password
+            - ODY_IROH_ROOT_PASSWORD=password
+            - ODY_AP_ROOT_PASSWORD=password
+            - ODY_BASE_WIFI_PASSWORD=password
+
+volumes:
+    # required for mac as no hard linking:(
+    outputs:
+        labels:
+            com.northeastern_electric_racing.description: "Buildroot outputs, including disk images"
+    shared_data:
+        labels:
+            com.northeastern_electric_racing.description: "Shared output between buildroot runs"

odysseus/compose.yml

@@ -2,9 +2,11 @@ version: "3.8"
 services:
     odysseus:
         build: .
+        image: ghcr.io/northeastern-electric-racing/odysseus:develop-features-for-comp
         command: /bin/bash
         privileged: true
         tty: true
+        stdin_open: true
         # for fakeroot failure
         ulimits:
           nofile:
@@ -14,10 +16,19 @@ services:
             - ./odysseus_tree:/home/odysseus/build/odysseus_tree
             - shared_data:/home/odysseus/shared_data
             - ./outputs:/home/odysseus/outputs
+            - ../.git:/home/odysseus/build/githistory/.git:ro # read only as this is an unsafe operation
         environment:
             - BR2_DL_DIR=/home/odysseus/shared_data/dl 
             - BR2_CCACHE_DIR=/home/odysseus/shared_data/ccache
+            - ODY_TPU_ROOT_PASSWORD=password
+            - ODY_IROH_ROOT_PASSWORD=password
+            - ODY_AP_ROOT_PASSWORD=password
+            - ODY_BASE_WIFI_PASSWORD=password
+
 volumes:
     shared_data:
         labels:
             com.northeastern_electric_racing.description: "Shared output between buildroot runs"
+
+
+

odysseus/docker_scripts/load-secrets.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# must be sourced for this to work
+eval "$(gpg -d --no-symkey-cache --cipher-algo AES256 /home/odysseus/SECRETS.env.gpg)"

odysseus/docker_scripts/setup_env.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
 
 alias make-current="/home/odysseus/scripts/make-current.sh"
+alias load-secrets="source /home/odysseus/scripts/load-secrets.sh"
 
-# for each defconfig make output subdirectory
+# for each defconfig make output subdirectory 
 make -C /home/odysseus/build/buildroot O=/home/odysseus/outputs/tpu BR2_EXTERNAL=/home/odysseus/build/odysseus_tree raspberrypi4_64_tpu_defconfig
 make -C /home/odysseus/build/buildroot O=/home/odysseus/outputs/ap BR2_EXTERNAL=/home/odysseus/build/odysseus_tree raspberrypi3_64_ap_defconfig
 make -C /home/odysseus/build/buildroot O=/home/odysseus/outputs/iroh BR2_EXTERNAL=/home/odysseus/build/odysseus_tree raspberrypi3_64_iroh_defconfig

odysseus/odysseus_tree/board/ap/cmdline.txt

@@ -1 +1 @@
-root=/dev/mmcblk0p2 rootwait console=tty1 console=serial0,115200
+root=/dev/mmcblk0p2 rootwait console=tty1 console=serial0,115200 init=/sbin/ody-resize

odysseus/odysseus_tree/board/iroh/cmdline.txt

@@ -1 +1 @@
-root=/dev/mmcblk0p2 rootwait console=tty1 console=serial0,115200
+root=/dev/mmcblk0p2 rootwait console=tty1 console=serial0,115200 init=/sbin/ody-resize

odysseus/odysseus_tree/board/iroh/config.txt

@@ -39,4 +39,4 @@ arm_64bit=1
 #dtoverlay=disable-wifi
 
 # for can
-dtoverlay=mcp2515-can0,oscillator=16000000,spimaxfrequency=10000000
+dtoverlay=mcp2515-can0,oscillator=12000000,spimaxfrequency=10000000

odysseus/odysseus_tree/board/tpu/cmdline.txt

@@ -1 +1 @@
-root=/dev/mmcblk0p2 rootwait console=tty1
+root=/dev/mmcblk0p2 rootwait console=tty1 logo.nologo init=/sbin/ody-resize

odysseus/odysseus_tree/board/tpu/config.txt

@@ -16,6 +16,8 @@ kernel=Image
 # If the text shown on the screen disappears off the edge, comment this out
 disable_overscan=1
 
+disable_splash=1
+
 # How much memory in MB to assign to the GPU on Pi models having
 # 256, 512 or 1024 MB total memory
 gpu_mem_256=100
@@ -60,14 +62,17 @@ dtoverlay=mmc_spi-spi1
 # for temp sensor
 dtoverlay=i2c-sensor,sht3x
 
-# for NER 22A dashboard
+# for NER 22A dashboard (HDM1)
 #Set the Monitor mode to DMT.
-hdmi_group=2
+hdmi_group:1=2
 #Use a custom resolution.
-hdmi_mode=87
+hdmi_mode:1=87
 #Use an 800 x 480 resolution @ 60Hz
-hdmi_cvt= 800 480 60 6 0 0 0
+hdmi_cvt:1= 800 480 60 6 0 0 0
 #Deliver max current through USB
 # # Max_usb_current=1 # is this needed (external power??)
-hdmi_drive=1
-hdmi_force_hotplug=1
+hdmi_drive:1=1
+hdmi_force_hotplug:1=1
+
+# for HDMI 1
+framebuffer_priority=7