staging.kernelci.org

#!/bin/dash

# Periodic job run on staging.kernelci.org to merge together all the pending
# PRs and update the staging.kernelci.org branches, then update Docker images
# and trigger a full build/test cycle with a kernel branch based on linux-next.

set -e

# Trap function for errors
ABSPATH=$(readlink -f "$0")
ABSDIR=$(dirname "$ABSPATH")

crash() {
    local exit_code="$?"
    # If exit code is 0, then it's not an error
    [ "$exit_code" -eq 0 ] && exit 0
    # Generate syslog message
    logger -t kernelci-deploy "Error in script $0"
    ${ABSDIR}/tools/kci-slackbot.py --message "Staging FAILED!!!"
    exit $exit_code
}

trap 'crash' EXIT

cmd_pull() {
    echo "Updating local repository"
    git pull --ff-only
}

cmd_jenkins() {
    echo "Updating Jenkins jobs"
    ./pending.py kernelci-jenkins --push

    [ -d checkout/kernelci-jenkins/data ] || {
        # To set this up:
        # git clone https://github.com/kernelci/kernelci-jenkins-data.git data
        # cd data; git-crypt unlock
        # cd ..; ln -s data/bot.staging.kernelci.org/env .env

        echo "Encrypted kernelci-jenkins-data repository not set up"
        return 1
    }

    echo "Updating encrypted data checkout"
    topdir=$(pwd)
    cd checkout/kernelci-jenkins/data
    git prune
    git remote update origin
    data_diff=$(git diff origin/main)
    #if [ -n "$data_diff" ]; then
        git checkout origin/main
        cd "$topdir/checkout/kernelci-jenkins"
        echo "Recreating Jenkins container"
        docker-compose down
        docker-compose up --build -d
        sleep 60  # magic - another way to do this would be to poll something
    #else
    #    cd "$topdir/checkout/kernelci-jenkins"
    #    # ToDo: restart the container but only when there were some changes in
    #    # kernelci-jenkins to avoid aborting running jobs unnecessarily
    #fi
    cd "$topdir"

    echo "Triggering Jenkins seed job"
    python3 job.py --no-params trigger job-dsl
}

cmd_core() {
    echo "Updating kernelci-core"
    ./pending.py kernelci-core --push
}

cmd_bootrr() {
    echo "Updating bootrr"
    ./pending.py bootrr --push
}

cmd_buildroot() {
    echo "Updating buildroot"
    ./pending.py buildroot --push
}

cmd_test_definitions() {
    echo "Updating test-definitions"
    ./pending.py test-definitions --push --main=master
    ./pending.py iec-security --push --main=master
}

cmd_kcidb() {
    echo "Updating KCIDB"
    ./pending.py kcidb --push
}

cmd_backend() {
    opt="$1"

    echo "Checking for missing indexes by finding COLLSCAN events"
    echo "db.system.profile.aggregate([{\$match: {ns: {\$in: ['kernel-ci.test_case', 'kernel-ci.test_group', 'kernel-ci.test_regression', 'kernel-ci.build']}, planSummary: 'COLLSCAN'}}, {\$group: {_id: {ns: '\$ns', planSummary: '\$planSummary'}}}])" | mongo kernel-ci

    if [ "$opt" = "indexes" ]; then
        echo "Dropping Mongo DB indexes"
        echo "db.test_case.dropIndexes(); db.test_group.dropIndexes(); db.test_regression.dropIndexes(); db.build.dropIndexes();" | mongo kernel-ci
    fi

    echo "Updating kernelci-backend"

    if ./pending.py kernelci-backend --diff-only; then
        ./pending.py kernelci-backend --push
        ./ansible kernelci-backend api.staging.kernelci.org
    else
        echo "No changes, skipping Ansible"
    fi

    echo "Resetting and enabling the Mongo DB profiler"
    echo "db.setProfilingLevel(0); db.system.profile.drop(); db.setProfilingLevel(2);" | mongo kernel-ci
}

cmd_frontend() {
    echo "Updating kernelci-frontend"

    if ./pending.py kernelci-frontend --diff-only; then
        ./pending.py kernelci-frontend --push
        ./ansible kernelci-frontend staging.kernelci.org
    else
        echo "No changes, skipping Ansible"
    fi
}

cmd_website() {
    echo "Updating static website"
    ./pending.py kernelci-project --push
    topd=$PWD
    cd checkout/kernelci-project/kernelci.org
    git prune
    git-lfs checkout
    git submodule update --init --recursive
    for d in external/*; do
        echo "Checking out staging branch for $d"
        cd "$d"
        git remote update origin
        git checkout origin/staging.kernelci.org || echo "Never mind"
        cd -
    done
    ./docker-hugo -D -b https://static.staging.kernelci.org/
    cd "$topd"
}

cmd_kernel() {
    tree="${1:-next}"

    echo "Pushing kernel test branch for ${tree}"

    case $tree in
        next)
            url=https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
            branch=master
            ;;
        mainline)
            url=https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
            branch=master
            ;;
        stable)
            url=https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
            branch=linux-5.15.y
            ;;
        cip)
            url=https://git.kernel.org/pub/scm/linux/kernel/git/cip/linux-cip.git
            branch=linux-5.10.y-cip
            ;;
        android)
            url=https://android.googlesource.com/kernel/common
            branch=android13-5.15
            ;;
        *)
            echo "Unknown tree"
            exit 1
            ;;
    esac

    ./kernel.py \
        --push \
        --ssh-key=keys/id_rsa_staging.kernelci.org \
        --from-url=$url \
        --from-branch=$branch \
        --branch=staging-"$tree" \
        --tag-prefix=staging-"$tree"-
}

cmd_docker() {
    echo "Updating Docker images"

    # Build without cache if flag file doesn't exist or older than x days
    purge_flag_file=".last_docker_purge"
    if [ -f "$purge_flag_file" ]; then
        flag_time=$(date -r "$purge_flag_file" +%s)
        week_ago=$(date -d 'now - 7 days' +%s)
    fi
    if [ ! -f "$purge_flag_file" ] || [ $flag_time -le $week_ago ]; then
        echo "Building without cache"
        cache_arg="--no-cache"
        touch "$purge_flag_file"
    else
        cache_arg=""
    fi

    # Build the images with kci docker
    cd checkout/kernelci-core
    git prune

    core_rev=$(git show --pretty=format:%H -s origin/staging.kernelci.org)
    rev_arg="--build-arg core_rev=$core_rev"
    px_arg='--prefix=kernelci/staging-'
    args="build --push $px_arg $cache_arg $rev_arg"

    # KernelCI tools
    ./kci docker $args kernelci
    ./kci docker $args k8s kernelci $rev_arg

    # Compiler toolchains

    # clang-17 for linux-next (including weekly full build)
    ./kci docker $args clang-17 kselftest kernelci
    for arch in arm arm64 mips riscv64 x86; do
	./kci docker $args clang-17 kselftest kernelci --arch $arch
    done
    # clang-14 for chromeos
    ./kci docker $args clang-14 kselftest kernelci
    for arch in arm64 x86; do
	./kci docker $args clang-14 kselftest kernelci --arch $arch
    done
    # gcc-10 for everything (including weekly linux-next full build)
    for arch in arc arm armv5 arm64 mips riscv64 x86; do
	./kci docker $args gcc-10 kselftest kernelci --arch $arch
    done
    # missing -dev packages for sparc64
    ./kci docker $args gcc-10 kernelci --arch sparc
    # only x86 is useful for KUnit (for now)
    ./kci docker $args gcc-10 kunit kernelci --arch x86
    # latest rustc for linux-next and rust-for-linux
    ./kci docker $args rustc-1.73 kselftest kernelci --arch x86

    # rootfs
    ./kci docker $args buildroot kernelci
    ./kci docker $args debos kernelci

    # QEMU
    ./kci docker $args qemu

    # Other tools
    ./kci docker $args cvehound kernelci
    ./kci docker $args dt-validation # kernelci (PyYAML conflict)
    cd -
}

cmd_monitor() {
    echo "Triggering Jenkins kernel-tree-monitor job"
    python3 \
        job.py \
        --json=data/staging-monitor.json \
        trigger \
        kernel-tree-monitor
}

cmd_rotate_tree() {
    if [ -e .tree ]; then
        last=$(cat .tree)
    else
        last=
    fi

    case $last in
        next)
            tree=mainline
            ;;
        mainline)
            tree=stable
            ;;
        stable)
            tree=next
            ;;
        *)
            tree=next
            ;;
    esac

    echo $tree > .tree
    echo $tree
}

cmd_api_pipeline() {
    compose_files="\
-f docker-compose.yaml \
-f docker-compose-kcidb.yaml \
-f docker-compose-lava.yaml \
-f docker-compose-production.yaml \
"

    echo "Stopping pipeline containers"
    cd checkout/kernelci-pipeline
    git prune
    docker-compose $compose_files down --remove-orphans
    cd -

    echo "Creating local staging branch in tmp directory while API is running"
    mkdir -p tmp
    rm -rf tmp/patches
    cp -R patches tmp/
    pending_api="../pending.py --settings=../data/staging.ini kernelci-api"
    (cd tmp; $pending_api)

    echo "Looking for diff with current staging branch"
    if (cd tmp; $pending_api --diff-only); then
        echo "Stopping API containers"
        cd checkout/kernelci-api
        docker-compose down
        cd -

        echo "Updating API branch"
        ./pending.py kernelci-api --push

        echo "Starting API containers"
        cd checkout/kernelci-api
        git prune
        REQUIREMENTS=requirements-dev.txt docker-compose build
        docker-compose up -d
        cd -
    else
        echo "No API changes, leaving it running"
        cd checkout/kernelci-api
        docker-compose up -d
        cd -
    fi

    echo "Updating pipeline branch"
    ./pending.py kernelci-pipeline --push

    echo "Starting pipeline containers"
    cd checkout/kernelci-pipeline
    REQUIREMENTS=requirements-dev.txt docker-compose $compose_files build
    SETTINGS=/home/kernelci/config/staging.kernelci.org.secrets.toml \
            docker-compose $compose_files up -d
    cd -
}

cmd_all() {
    tree="$1"

    [ -z $tree ] && tree=$(cmd_rotate_tree)
    echo "Kernel tree: $tree"
    tools/kci-slackbot.py --message "Staging starts on tree $tree"

    cmd_pull
    cmd_jenkins
    cmd_core
    cmd_bootrr
    cmd_buildroot
    cmd_test_definitions
    cmd_kcidb
    cmd_backend
    cmd_frontend
    cmd_kernel $tree
    cmd_docker
    cmd_monitor
    cmd_api_pipeline
    cmd_website
    tools/kci-slackbot.py --message "Staging completed"

}

cmd="${1}"

if [ -n "$cmd" ]; then
    shift 1
else
    cmd="all"
fi

"cmd_"$cmd $@

exit 0