ci + build

Signed-off-by: Pierrick Bouvier <[email protected]>

.github/workflows/build.yml

@@ -0,0 +1,14 @@
+on: push
+
+jobs:
+  build_qemu:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: check merge is success
+        run: bash -c 'if [ -f shazam.log ]; then echo "merge failed"; cat shazam.log; exit 1; fi'
+      - name: build_container
+        run: ./run.sh amd64 true
+      - name: build opt
+        run: ./run.sh amd64 ./build.sh opt

.github/workflows/new_series.yml

@@ -0,0 +1,36 @@
+on:
+  schedule:
+    - cron: '*/10 * * * *'
+  workflow_dispatch:
+
+permissions: write-all
+
+jobs:
+  push_new_series:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          # a PAT must be generated with workflow permission, else it's not
+          # possible to push any change for those files
+          # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/triggering-a-workflow#triggering-a-workflow-from-a-workflow
+          token: ${{ secrets.WORKFLOW_COMMIT_TOKEN }}
+      - run: git fetch -a origin --unshallow || true
+      - run: git config user.name "GitHub Actions Bot"
+      - run: git config user.email "<>"
+      - run: sudo pip install b4
+      - run: ./push_new_series.sh
+
+  keepalive-job:
+    name: Keepalive Workflow
+    if: ${{ always() }}
+    needs: push_new_series
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: gautamkrishnar/keepalive-workflow@v2
+        with:
+          use_api: false

Dockerfile

@@ -0,0 +1,27 @@
+ARG arch=
+FROM docker.io/${arch}/debian:bookworm
+
+RUN apt update && apt upgrade -y
+# https://wiki.qemu.org/Hosts/Linux#Building_QEMU_for_Linux
+RUN apt update && apt install -y \
+    git libglib2.0-dev libfdt-dev libpixman-1-dev zlib1g-dev ninja-build\
+    git-email\
+    libaio-dev libbluetooth-dev libcapstone-dev libbrlapi-dev libbz2-dev\
+    libcap-ng-dev libcurl4-gnutls-dev libgtk-3-dev\
+    libibverbs-dev libjpeg-dev libncurses5-dev libnuma-dev\
+    librbd-dev librdmacm-dev\
+    libsasl2-dev libsdl2-dev libseccomp-dev libsnappy-dev libssh-dev\
+    libvde-dev libvdeplug-dev libvte-2.91-dev liblzo2-dev\
+    valgrind xfslibs-dev
+
+RUN apt update && apt install -y \
+    python3-venv meson coreutils build-essential git ccache python3-tomli
+
+RUN apt update && apt install -y xvfb
+
+RUN apt update && apt install -y flex bison
+
+RUN apt update && apt install -y libunwind-dev
+
+ARG arch=
+ENV ARCH=${arch}

build.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+type=
+if [ $# -eq 1 ]; then
+    type=$1
+fi
+
+mkdir -p build
+touch build/.build_type
+previous_type="$(cat build/.build_type)"
+
+if [ "$type" == "" ]; then
+    type=$previous_type
+fi
+if [ "$type" == "" ]; then
+    type=debug
+fi
+
+if [ "$previous_type" != "$type" ]; then
+    echo "build type is different ($previous_type -> $type)"
+    rm -rf build || true
+    mkdir -p build
+    echo "$type" > build/.build_type
+fi
+
+echo "build: $type"
+
+if [ $type == tsan ]; then
+    echo "Build with TSAN: use export LD_LIBRARY_PATH=$(pwd)/build/glib_tsan"
+fi
+
+pushd build
+if [ ! -f .configured ]; then
+    export CC="ccache cc"
+    export CXX="ccache cxx"
+    configure_flags=
+    case $type in
+    opt)
+        configure_flags=""
+        export CFLAGS="-O2 -g -fno-omit-frame-pointer"
+        ;;
+    opt-gcov)
+        configure_flags=""
+        export CFLAGS="-O2 -g -fno-omit-frame-pointer --coverage"
+        export LDFLAGS="--coverage"
+        ;;
+    debug)
+        configure_flags="--enable-debug -Dasan=true -Dubsan=true"
+        ;;
+    tsan)
+        if [ ! -d glib_tsan ]; then
+            rm -rf glib
+            git clone --depth=1 --branch=2.81.0 \
+                https://github.com/GNOME/glib.git
+            pushd glib
+            CFLAGS="-O2 -g -fsanitize=thread" meson setup \
+                --prefix=$(pwd)/out \
+                build -Dtests=false
+            ninja -C build install
+            popd
+            mkdir -p glib_tsan
+            rsync -av glib/out/lib/*/* glib_tsan/
+            rm -rf glib/
+        fi
+        configure_flags="--enable-debug -Dtsan=true"
+        ;;
+    clang)
+        export CC="ccache clang"
+        export CXX="ccache clang++"
+        ;;
+    *)
+        choices="opt, debug, tsan, clang"
+        echo "Unknown build type $type (choices: $choices)"
+        exit 1
+        ;;
+    esac
+    ../configure $configure_flags
+    touch .configured
+fi
+ninja
+make -B -C contrib/plugins/ > /dev/null
+popd

push_new_series.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+die()
+{
+    echo 1>&2 "$@"
+    exit 1
+}
+
+# return URL for mailboxes for previous, current and next month. This way, we
+# don't miss any email, even with different timezones.
+mailbox_archives()
+{
+    current_date=$1
+
+    current_month=$(date +%m --date "$current_date")
+    current_year=$(date +%Y --date "$current_date")
+
+    if [ $current_month == "01" ]; then
+        previous_month=12
+        previous_year=$((current_year - 1))
+    elif [ $current_month == "12" ]; then
+        next_month=01
+        next_year=$((current_year + 1))
+    else
+        previous_year=$current_year
+        next_year=$current_year
+        previous_month=$(printf "%02d" $((current_month - 1)))
+        next_month=$(printf "%02d" $((current_month + 1)))
+    fi
+
+    qemu_archive="https://lists.gnu.org/archive/mbox/qemu-devel/"
+    echo $qemu_archive$previous_year-$previous_month \
+         $qemu_archive$current_year-$current_month \
+         $qemu_archive$next_year-$next_month
+}
+
+# download all emails for previous, current and next month.
+fetch_mails()
+{
+    out=$1
+    rm -rf $out
+    mkdir -p $out
+    mkdir -p $out.mailbox
+    pushd $out.mailbox
+    archives=$(mailbox_archives "$(date)")
+    # we can have missing current or next mailbox depending on timezone
+    wget --no-verbose --no-clobber $archives || true
+    popd
+    git mailsplit -o$out $out.mailbox/*
+}
+
+find_series()
+{
+    mail_dir=$1
+    # find all message id, for mails without a reference (i.e. first in series).
+    grep -Lri '^References: .' $mail_dir | sort | while read m
+    do
+        # skip messages replying to thread
+        if grep -qi '^In-Reply-to: .' $m; then
+            continue
+        fi
+        # skip messages whose subject does not start with [
+        if ! grep -q 'Subject: \[' $m; then
+            continue
+        fi
+        msg_id=$(grep -i '^message-id: ' $m | head -n1 |
+                 sed -e 's/.*<//' -e 's/>$//')
+        date=$(grep -i '^date: ' $m | head -n1 | sed -e 's/^date: //I')
+        echo "$msg_id|$date"
+    done
+}
+
+pull_repositories()
+{
+    git remote remove upstream || true
+    git remote add upstream -f https://gitlab.com/qemu-project/qemu
+    git fetch -a origin
+}
+
+push_one_series()
+{
+    s="$1"
+    echo "-----------------------------------------------------------------"
+    msg_id=$(echo "$s" | cut -f 1 -d '|')
+    date=$(echo "$s" | cut -f 2 -d '|')
+    echo "$msg_id | $date"
+    if git rev-parse "remotes/origin/$msg_id" >& /dev/null; then
+        return
+    fi
+
+    # find git commit on master close to date of series
+    base_git_revision=$(git log -n1 --format=format:%H --before="$date" \
+                        --first-parent upstream/master)
+    echo "push this new series, applied from $base_git_revision"
+    git checkout "$base_git_revision" >& /dev/null
+    git branch -D new_series >& /dev/null || true
+    git checkout -b new_series
+
+    # apply CI patch
+    git cherry-pick origin/ci
+
+    # apply series
+    if ! b4 shazam --allow-unicode-control-chars $msg_id |& tee shazam.log; then
+        git am --abort
+        git add shazam.log
+        git commit -m 'b4 shazam failed'
+    fi
+
+    if ! grep -qi 'no patches found' shazam.log; then
+        git push --set-upstream origin "new_series:$msg_id"
+    else
+        echo "no patches found in series: $msg_id"
+    fi
+
+    # reset branch
+    git checkout origin/ci
+    git branch -D new_series
+}
+
+pull_repositories
+fetch_mails mails
+find_series mails | while read s; do push_one_series "$s"; done

run.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+die()
+{
+    echo "$@" >&2
+    exit 1
+}
+
+[ $# -ge 2 ] || die "usage: arch command [args...]
+arch: (amd64|arm64v8|i386|s390x)"
+arch=$1;shift
+
+script_dir=$(dirname $(readlink -f $0))
+pushd $script_dir
+
+image=qemu-$arch
+podman build -t $image -f - --build-arg arch=$arch < Dockerfile
+mkdir -p build_$arch build
+mkdir -p $HOME/.cache/ccache
+# run privileged container: kvm + all capability (ptrace needed for Lsan)
+podman run -it \
+    --privileged \
+    -e CCACHE_DIR=$HOME/.cache/ccache \
+    -v $HOME/.cache/ccache/:$HOME/.cache/ccache/ \
+    -v $(pwd):$(pwd) -v $(pwd)/build_$arch:$(pwd)/build -w $(pwd)\
+    -v /:/host \
+    $image "$@"