build + ci support

Signed-off-by: Pierrick Bouvier <[email protected]>

.github/workflows/build.yml

@@ -0,0 +1,12 @@
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  build_qemu:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: build
+        run: ./run.sh amd64 ./build.sh

.gitignore

@@ -1,5 +1,6 @@
 /GNUmakefile
 /build/
+/build_*/
 /.cache/
 /.vscode/
 *.pyc

Dockerfile

@@ -0,0 +1,27 @@
+ARG arch=
+FROM docker.io/${arch}/debian:bookworm
+
+RUN apt update && apt upgrade -y
+# https://wiki.qemu.org/Hosts/Linux#Building_QEMU_for_Linux
+RUN apt update && apt install -y \
+    git libglib2.0-dev libfdt-dev libpixman-1-dev zlib1g-dev ninja-build\
+    git-email\
+    libaio-dev libbluetooth-dev libcapstone-dev libbrlapi-dev libbz2-dev\
+    libcap-ng-dev libcurl4-gnutls-dev libgtk-3-dev\
+    libibverbs-dev libjpeg-dev libncurses5-dev libnuma-dev\
+    librbd-dev librdmacm-dev\
+    libsasl2-dev libsdl2-dev libseccomp-dev libsnappy-dev libssh-dev\
+    libvde-dev libvdeplug-dev libvte-2.91-dev liblzo2-dev\
+    valgrind xfslibs-dev
+
+RUN apt update && apt install -y \
+    python3-venv meson coreutils build-essential git ccache python3-tomli
+
+RUN apt update && apt install -y xvfb
+
+RUN apt update && apt install -y flex bison
+
+RUN apt update && apt install -y libunwind-dev
+
+ARG arch=
+ENV ARCH=${arch}

build.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+x64="x86_64-linux-user,x86_64-softmmu"
+i386="i386-linux-user,i386-softmmu"
+aarch64="aarch64-linux-user,aarch64-softmmu"
+
+type=
+if [ $# -eq 1 ]; then
+    type=$1
+fi
+
+mkdir -p build
+touch build/.build_type
+previous_type="$(cat build/.build_type)"
+
+if [ "$type" == "" ]; then
+    type=$previous_type
+fi
+if [ "$type" == "" ]; then
+    type=debug
+fi
+
+if [ "$previous_type" != "$type" ]; then
+    echo "build type is different ($previous_type -> $type)"
+    rm -rf build || true
+    mkdir -p build
+    echo "$type" > build/.build_type
+fi
+
+echo "build: $type"
+
+if [ $type == tsan ]; then
+    echo "Build with TSAN: use export LD_LIBRARY_PATH=$(pwd)/build/glib_tsan"
+fi
+
+pushd build
+if [ ! -f .configured ]; then
+    export CC="ccache cc"
+    export CXX="ccache cxx"
+    configure_flags=
+    target="--target-list=$x64,$i386,$aarch64"
+    case $type in
+    opt)
+        configure_flags=""
+        export CFLAGS="-O2 -g -fno-omit-frame-pointer"
+        ;;
+    opt-gcov)
+        configure_flags=""
+        export CFLAGS="-O2 -g -fno-omit-frame-pointer --coverage"
+        export LDFLAGS="--coverage"
+        ;;
+    debug)
+        configure_flags="--enable-debug -Dasan=true -Dubsan=true"
+        ;;
+    tsan)
+        if [ ! -d glib_tsan ]; then
+            rm -rf glib
+            git clone --depth=1 --branch=2.81.0 https://github.com/GNOME/glib.git
+            pushd glib
+            CFLAGS="-O2 -g -fsanitize=thread" meson setup \
+                --prefix=$(pwd)/out \
+                build -Dtests=false
+            ninja -C build install
+            popd
+            mkdir -p glib_tsan
+            rsync -av glib/out/lib/*/* glib_tsan/
+            rm -rf glib/
+        fi
+        configure_flags="--enable-debug -Dtsan=true"
+        ;;
+    all)
+        target=
+        ;;
+    all-clang)
+        target=
+        export CC="ccache clang"
+        export CXX="ccache clang++"
+        ;;
+    *)
+        echo "Unknown build type $type (choices: opt, debug, tsan, all, all-clang)"
+        exit 1
+        ;;
+    esac
+    ../configure $target $configure_flags
+    touch .configured
+fi
+ninja -k0
+make -B -C contrib/plugins/ > /dev/null
+popd

mt.cpp

@@ -0,0 +1,44 @@
+#include <cassert>
+#include <cstdint>
+#include <cstdlib>
+#include <atomic>
+#include <iostream>
+#include <thread>
+#include <vector>
+
+static std::atomic<uint64_t> count;
+static std::atomic<int> threads_launched;
+
+static void run(int num_threads, int num_iterations)
+{
+    threads_launched++;
+    while (threads_launched != num_threads) {
+        /* active wait */
+    }
+
+    uint64_t res = 0;
+    for (int i = 0; i < num_iterations; ++i) {
+        res += 1;
+    }
+    count += res;
+}
+
+int main(int argc, char *argv[])
+{
+    if (argc != 3) {
+        std::cout << "usage: num_threads num_iterations\n";
+        exit(EXIT_FAILURE);
+    }
+
+    const int num_threads = std::stoi(argv[1]);
+    const int num_iterations = std::stoi(argv[2]);
+    std::vector<std::thread> threads;
+
+    for (int i = 0; i < num_threads; ++i) {
+        threads.push_back(std::thread(run, num_threads, num_iterations));
+    }
+    for (auto & t : threads) {
+        t.join();
+    }
+    assert(count == num_iterations * num_threads);
+}

push_new_series.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+die()
+{
+    echo 1>&2 "$@"
+    exit 1
+}
+
+# return URL for mailboxes for previous, current and next month. This way, we
+# don't miss any email, even with different timezones.
+mailbox_archives()
+{
+    current_date=$1
+
+    current_month=$(date +%m --date "$current_date")
+    current_year=$(date +%Y --date "$current_date")
+
+    if [ $current_month == "01" ]; then
+        previous_month=12
+        previous_year=$((current_year - 1))
+    elif [ $current_month == "12" ]; then
+        next_month=01
+        next_year=$((current_year + 1))
+    else
+        previous_year=$current_year
+        next_year=$current_year
+        previous_month=$(printf "%02d" $((current_month - 1)))
+        next_month=$(printf "%02d" $((current_month + 1)))
+    fi
+
+    qemu_archive="https://lists.gnu.org/archive/mbox/qemu-devel/"
+    echo $qemu_archive$previous_year-$previous_month \
+         $qemu_archive$current_year-$current_month \
+         $qemu_archive$next_year-$next_month
+}
+
+# download all emails for previous, current and next month.
+fetch_mails()
+{
+    out=$1
+    rm -rf $out
+    mkdir -p $out
+    mkdir -p $out.mailbox
+    pushd $out.mailbox
+    archives=$(mailbox_archives "$(date)")
+    # we can have missing current or next mailbox
+    wget --no-clobber $archives || true
+    popd
+    git mailsplit -o$out $out.mailbox/*
+}
+
+find_series()
+{
+    mail_dir=$1
+    # find all message id, for mails without a reference (i.e. first in series).
+    # we reverse sort to bring newest series first.
+    grep -Lri '^References:' $mail_dir | sort -r | while read m
+    do
+        msg_id=$(grep -i '^message-id: ' $m | head -n1 | sed -e 's/.*<//' -e 's/>$//')
+        date=$(grep -i '^date: ' $m | head -n1 | sed -e 's/^date: //I')
+        echo "$msg_id|$date"
+    done
+}
+
+git remote remove upstream || true
+git remote add upstream -f https://gitlab.com/qemu-project/qemu
+
+fetch_mails mails
+find_series mails | while read s
+do
+    msg_id=$(echo "$s" | cut -f 1 -d '|')
+    date=$(echo "$s" | cut -f 2 -d '|')
+    echo $msg_id
+    echo $date
+    if ! git rev-parse "remotes/origin/$msg_id"; then
+        base_git_revision=$(git log -n1 --format=format:%H --before="$date" --first-parent upstream/master)
+        git checkout "$base_git_revision"
+        git branch -D new_series > /dev/null || true
+        git checkout -b new_series
+        git cherry-pick origin/ci # apply CI patch
+        if ! b4 shazam $msg_id |& tee shazam.log; then
+            git am --abort
+            git add shazam.log
+            git commit -m 'b4 shazam failed'
+        fi
+        git push --set-upstream origin new_series:$msg_id
+        git checkout master
+        git branch -D new_series
+    fi
+done

run.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+die()
+{
+    echo "$@" >&2
+    exit 1
+}
+
+[ $# -ge 2 ] || die "usage: arch command [args...]
+arch: (amd64|arm64v8|i386|s390x)"
+arch=$1;shift
+
+script_dir=$(dirname $(readlink -f $0))
+pushd $script_dir
+
+image=qemu-$arch
+podman build -t $image -f - --build-arg arch=$arch < Dockerfile
+mkdir -p build_$arch build
+mkdir -p $HOME/.cache/ccache
+podman run -it \
+    -e CCACHE_DIR=$HOME/.cache/ccache \
+    -v $HOME/.cache/ccache/:$HOME/.cache/ccache/ \
+    -v $(pwd):$(pwd) -v $(pwd)/build_$arch:$(pwd)/build -w $(pwd)\
+    -v /:/host \
+    $image "$@"

test.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -eu
+
+./build.sh
+
+ARCH=${ARCH:-amd64}
+qemu_system_args=
+if [ "$ARCH" == amd64 ]; then
+    qemu_suffix=x86_64
+elif [ "$ARCH" == i386 ]; then
+    qemu_suffix=i386
+elif [ "$ARCH" == arm64v8 ]; then
+    qemu_suffix=aarch64
+    qemu_system_args="-M virt"
+fi
+
+gui_run=$(which xvfb-run || true)
+
+g++ mt.cpp -o build/mt
+
+qemu_user=./build/qemu-$qemu_suffix
+qemu_system="./build/qemu-system-$qemu_suffix $qemu_system_args"
+bin="build/mt 17 1000" # 17 threads, 1000 iterations
+$qemu_user -plugin build/tests/tcg/plugins/libinline.so -d plugin "$@" $bin
+#$qemu_user -plugin build/tests/tcg/plugins/libinline.so "$@" \
+#  -d op,op_opt,in_asm,out_asm $bin |& head -n 1000 > build/plugin.on
+#$qemu_user "$@" -d op,op_opt,in_asm,out_asm $bin |&
+# head -n 1000 > build/plugin.off
+echo -----------------------------------------
+#$gui_run timeout --preserve-status 2 $qemu_system\
+#    -plugin build/tests/plugin/libinline.so -smp 8
+#vim build/plugin_sys
+#vimdiff build/plugin.*