Get tests running again

.github/workflows/repo.yml

@@ -0,0 +1,28 @@
+name: repo
+
+on:
+  schedule:
+    - cron: "0 15 1 * *"
+  workflow_dispatch:
+
+jobs:
+  pre-commit-autoupdate:
+    name: pre-commit autoupdate
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/commaai/cereal:latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: pre-commit autoupdate
+      run: |
+        git config --global --add safe.directory '*'
+        pre-commit autoupdate
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5
+      with:
+        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
+        commit-message: Update pre-commit hook versions
+        title: 'pre-commit: autoupdate hooks'
+        branch: pre-commit-updates
+        base: master
+        delete-branch: true

.github/workflows/tests.yml

@@ -0,0 +1,61 @@
+name: tests
+
+on: [push, pull_request]
+
+env:
+  DOCKER_REGISTRY: ghcr.io/commaai
+  RUN: docker run -e PYTHONWARNINGS=error --shm-size 1G --name cereal cereal /bin/sh -c
+  RUN_NAMED: docker run -e PYTHONWARNINGS=error --shm-size 1G --rm cereal /bin/sh -c
+  CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm cerealci /bin/bash -c
+  BUILD: docker buildx build --pull --load --cache-to type=inline --cache-from $DOCKER_REGISTRY/cereal:latest -t cereal -f Dockerfile .
+  PYTHONWARNINGS: error
+
+jobs:
+  build:
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build docker image
+      run: eval "$BUILD"
+    - name: Push to dockerhub
+      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/cereal'
+      run: |
+        docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+        docker tag cereal $DOCKER_REGISTRY/cereal:latest
+        docker push $DOCKER_REGISTRY/cereal:latest
+
+  unit_tests:
+    name: unit tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        flags: ['', '--asan', '--ubsan']
+        backend: ['MSGQ', 'ZMQ']
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build docker image
+      run: eval "$BUILD"
+    - name: C++ tests
+      run: |
+        $RUN "export ${{ matrix.backend }}=1 && \
+              scons ${{ matrix.flags }} -j$(nproc) && \
+              messaging/test_runner && \
+              visionipc/test_runner"
+    - name: python tests
+      run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m unittest discover ."
+    - name: Upload coverage
+      run: |
+        docker commit cereal cerealci
+        $CI_RUN "cd /project/cereal && bash <(curl -s https://codecov.io/bash) -v -F unit_tests_${{ matrix.backend }}"
+
+  static_analysis:
+    name: static analysis
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build docker image
+      run: eval "$BUILD"
+    - name: Static analysis
+      # TODO: a package pre-commit installs has a warning, remove the unset once that's fixed
+      run: $RUN "git init && git add -A && unset PYTHONWARNINGS && pre-commit run --all"

Dockerfile

@@ -0,0 +1,54 @@
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    autoconf \
+    build-essential \
+    ca-certificates \
+    capnproto \
+    clang \
+    cppcheck \
+    curl \
+    git \
+    libbz2-dev \
+    libcapnp-dev \
+    libclang-rt-dev \
+    libffi-dev \
+    liblzma-dev \
+    libncurses5-dev \
+    libncursesw5-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    libtool \
+    libzmq3-dev \
+    llvm \
+    make \
+    cmake \
+    ocl-icd-opencl-dev \
+    opencl-headers  \
+    python3-dev \
+    python3-pip \
+    tk-dev \
+    wget \
+    xz-utils \
+    zlib1g-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy
+
+WORKDIR /project/
+RUN cd /tmp/ && \
+    git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \
+    cd Catch2 && \
+    mv single_include/catch2/ /project/ && \
+    cd .. \
+    rm -rf Catch2
+
+WORKDIR /project/cereal
+
+ENV PYTHONPATH=/project
+
+COPY . .
+RUN rm -rf .git && \
+    scons -c && scons -j$(nproc)

SConstruct

@@ -0,0 +1,88 @@
+import os
+import platform
+import subprocess
+import sysconfig
+import numpy as np
+
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+if platform.system() == "Darwin":
+  arch = "Darwin"
+
+common = ''
+
+cpppath = [
+  f"#/../",
+  '/usr/lib/include',
+  '/opt/homebrew/include',
+  sysconfig.get_paths()['include'],
+]
+
+libpath = [
+  '/opt/homebrew/lib',
+]
+
+AddOption('--minimal',
+          action='store_false',
+          dest='extras',
+          default=True,
+          help='the minimum build. no tests, tools, etc.')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+AddOption('--ubsan',
+          action='store_true',
+          help='turn on UBSan')
+
+ccflags = []
+ldflags = []
+if GetOption('ubsan'):
+  flags = [
+    "-fsanitize=undefined",
+    "-fno-sanitize-recover=undefined",
+  ]
+  ccflags += flags
+  ldflags += flags
+elif GetOption('asan'):
+  ccflags += ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags += ["-fsanitize=address"]
+
+env = Environment(
+  ENV=os.environ,
+  CC='clang',
+  CXX='clang++',
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Wunused",
+    "-Werror",
+    "-Wshadow",
+  ] + ccflags,
+  LDFLAGS=ldflags,
+  LINKFLAGS=ldflags,
+
+  CFLAGS="-std=gnu11",
+  CXXFLAGS="-std=c++1z",
+  CPPPATH=cpppath,
+  LIBPATH=libpath,
+  CYTHONCFILESUFFIX=".cpp",
+  tools=["default", "cython"]
+)
+
+Export('env', 'arch', 'common')
+
+envCython = env.Clone(LIBS=[])
+envCython["CPPPATH"] += [np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CCFLAGS"].remove('-Werror')
+if arch == "Darwin":
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
+else:
+  envCython["LINKFLAGS"] = ["-pthread", "-shared"]
+
+Export('envCython')
+
+
+SConscript(['SConscript'])

codecov.yml

@@ -0,0 +1,8 @@
+comment: false
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch: off
+