Merge branch 'dev'

.github/workflows/bbm_deploy.yml

@@ -6,26 +6,28 @@ on:
     paths:
       - ".github/workflows/bbm_deploy.yml"
       - "autogen/**"
-      - "builtbot.tac"
+      - "buildbot.tac"
       - "common_factories.py"
       - "constants.py"
       - "docker-compose/**"
       - "locks.py"
       - "master-**"
       - "master.cfg"
+      - "os_info.yaml"
       - "utils.py"
       - "validate_master_cfg.sh"
   pull_request:
     paths:
       - ".github/workflows/bbm_deploy.yml"
       - "autogen/**"
-      - "builtbot.tac"
+      - "buildbot.tac"
       - "common_factories.py"
       - "constants.py"
       - "docker-compose/**"
       - "locks.py"
       - "master-**"
       - "master.cfg"
+      - "os_info.yaml"
       - "utils.py"
       - "validate_master_cfg.sh"
 

.github/workflows/bbw_build_container.yml

@@ -72,9 +72,10 @@ jobs:
             image: ubuntu:23.10
             branch: 10.11
             platforms: linux/amd64, linux/arm64/v8
-          - dockerfile: fedora.Dockerfile
-            image: fedora:37
-            platforms: linux/amd64, linux/arm64/v8
+          - dockerfile: debian.Dockerfile
+            image: ubuntu:24.04
+            branch: 10.11
+            platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x
           - dockerfile: fedora.Dockerfile
             image: fedora:38
             platforms: linux/amd64, linux/arm64/v8

ci_build_images/README.md

@@ -13,7 +13,7 @@ cat debian.Dockerfile common.Dockerfile >Dockerfile
 docker build . -t mariadb.org/buildbot/ubuntu:22.04 --build-arg mariadb_branch=10.7 --build-arg base_image=ubuntu:22.04
 # fedora
 cat fedora.Dockerfile common.Dockerfile >Dockerfile
-docker build . -t mariadb.org/buildbot/fedora:34 --build-arg base_image=fedora:37
+docker build . -t mariadb.org/buildbot/fedora:39 --build-arg base_image=fedora:39
 # rhel9
 cat rhel.Dockerfile common.Dockerfile >Dockerfile
 echo "12345_KEYNAME" >rhel_keyname

ci_build_images/debian.Dockerfile

@@ -30,8 +30,9 @@ RUN . /etc/os-release; \
     && if [ "${VERSION_ID}" = "20.04" ]; then apt-get -y install --no-install-recommends g++-10; fi \
     && if [ "$(arch)" = "x86_64" ]; then ARCH="amd64"; else ARCH=$(arch); fi \
     && curl -s "https://ci.mariadb.org/galera/mariadb-4.x-latest-gal-${ARCH}-${ID}-$(echo "$VERSION_ID" | sed 's/\.//').sources" >/etc/apt/sources.list.d/galera-4.sources \
-    # VERSION_ID=13 once released \
-    && if [ "${VERSION_CODENAME}" = trixie ] || [ "$(getconf LONG_BIT)" = 32 ]; then rm /etc/apt/sources.list.d/galera-4.sources; fi \
+    # trixie VERSION_ID=13 once released \
+    # 24.04 noble - until galera build happens based on this image, then rebuild so galera tests can occur
+    && if [ "${VERSION_ID}" = 24.04 ] || [ "${VERSION_CODENAME}" = trixie ] || [ "$(getconf LONG_BIT)" = 32 ]; then rm /etc/apt/sources.list.d/galera-4.sources; fi \
     && apt-get update \
     && curl -skO https://raw.githubusercontent.com/MariaDB/server/44e4b93316be8df130c6d87880da3500d83dbe10/debian/control \
     && mkdir debian \

constants.py

@@ -49,7 +49,7 @@
 builders_wordpress = ["amd64-rhel8-wordpress"]
 builders_galera_mtr = [
     "aarch64-debian-12",
-    "amd64-fedora-37",
+    "amd64-fedora-39",
     "amd64-ubuntu-2304",
     "s390x-ubuntu-2004",
     "s390x-ubuntu-2204",
@@ -109,7 +109,6 @@
 supportedPlatforms["10.5"] = [
     "aarch64-centos-stream9",
     "aarch64-debian-11",
-    "aarch64-fedora-37",
     "aarch64-fedora-38",
     "aarch64-fedora-39",
     "aarch64-rhel-9",
@@ -118,7 +117,6 @@
     "amd64-debian-11-msan",
     "amd64-debian-12-asan-ubsan",
     "amd64-debian-12-rocksdb",
-    "amd64-fedora-37",
     "amd64-fedora-38",
     "amd64-fedora-38-last-N-failed",
     "amd64-fedora-39",

master-nonlatent/master.cfg

@@ -478,7 +478,7 @@ f_windows_msi.addStep(
 f_windows_msi.addStep(
     steps.MTR(
         addLogs=True,
-        name="extra",
+        name="test extra",
         test_type="nm",
         command=[
             "dojob",

os_info.yaml

@@ -152,3 +152,11 @@ ubuntu-2310:
     - aarch64
   type: deb
   has_install: False
+ubuntu-2404:
+  version_name: noble
+  arch:
+    - amd64
+    - aarch64
+    - ppc64le
+    - s390x
+  type: deb

scripts/deb-install.sh

@@ -38,21 +38,23 @@ deb_setup_mariadb_mirror "$master_branch"
 # setup repository for BB artifacts
 deb_setup_bb_artifacts_mirror
 
-wget -O - "${artifactsURL}/${tarbuildnum}/${parentbuildername}/dists/${VERSION_CODENAME}/main/binary-$(deb_arch)/Packages.gz" | gunzip >Packages
+# Once repo are created with aptly, adapt below:
+# wget -O - "${artifactsURL}/${tarbuildnum}/${parentbuildername}/dists/${VERSION_CODENAME}/main/binary-$(deb_arch)/Packages.gz" | gunzip >Packages
+wget "${artifactsURL}/${tarbuildnum}/${parentbuildername}/debs/Packages"
 
 set -x
 
 # Due to MDEV-14622 and its effect on Spider installation,
 # Spider has to be installed separately after the server
-package_list=$(grep -B 1 'Source: mariadb' Packages | grep 'Package:' | grep -vE 'galera|spider|columnstore' | awk '{print $2}' | xargs)
+package_list=$(grep "^Package:" Packages | grep -vE 'galera|spider|columnstore' | awk '{print $2}' | xargs)
 if grep -qi spider Packages; then
-  spider_package_list=$(grep -B 1 'Source: mariadb' Packages | grep 'Package:' | grep 'spider' | awk '{print $2}' | xargs)
+  spider_package_list=$(grep "^Package:" Packages | grep 'spider' | awk '{print $2}' | xargs)
 fi
 if grep -qi columnstore Packages; then
   if [[ $arch != "amd64" ]] && [[ $arch != "arm64" ]]; then
     bb_log_warn "Due to MCOL-4123, Columnstore won't be installed on $arch"
   else
-    columnstore_package_list=$(grep -B 1 'Source: mariadb' Packages | grep 'Package:' | grep 'columnstore' | awk '{print $2}' | xargs)
+    columnstore_package_list=$(grep "^Package:" Packages | grep 'columnstore' | awk '{print $2}' | xargs)
   fi
 fi
 

scripts/deb-upgrade.sh

@@ -35,10 +35,10 @@ case $test_mode in
     if grep -qi columnstore Packages; then
       bb_log_warn "due to MCOL-4120 (Columnstore leaves the server shut down) and other bugs Columnstore upgrade is tested separately"
     fi
-    package_list=$(grep -B 1 'Source: mariadb' Packages | grep 'Package:' | grep -vE 'galera|spider|columnstore' | awk '{print $2}' | sort | uniq | xargs)
+    package_list=$(grep "^Package:" Packages | grep -vE 'galera|spider|columnstore' | awk '{print $2}' | xargs)
     if grep -qi spider Packages; then
       bb_log_warn "due to MDEV-14622 Spider will be installed separately after the server"
-      spider_package_list=$(grep -B 1 'Source: mariadb' Packages | grep 'Package:' | grep 'spider' | awk '{print $2}' | sort | uniq | xargs)
+      spider_package_list=$(grep "^Package:" Packages | grep 'spider' | awk '{print $2}' | xargs)
     fi
     if grep -si tokudb Packages; then
       # For the sake of installing TokuDB, disable hugepages
@@ -59,7 +59,7 @@ case $test_mode in
       bb_log_warn "Columnstore isn't necessarily built on Sid, the test will be skipped"
       exit
     fi
-    package_list="mariadb-server "$(grep -B 1 'Source: mariadb' Packages | grep 'Package:' | grep 'columnstore' | awk '{print $2}' | sort | uniq | xargs)
+    package_list="mariadb-server "$(grep "^Package:" Packages | grep 'columnstore' | awk '{print $2}' | xargs)
     ;;
   *)
     bb_log_err "unknown test mode: $test_mode"

utils.py

@@ -186,40 +186,12 @@ def createDebRepo():
             "-exc",
             util.Interpolate(
                 """
-    case $(arch) in
-      "x86_64")
-        arch="amd64"
-      ;;
-      "x86")
-        arch="i386"
-      ;;
-      "aarch64")
-        arch="arm64"
-      ;;
-      "ppc64le")
-        arch="ppc64el"
-      ;;
-      "s390x")
-        arch="s390x"
-      ;;
-    esac
-    . /etc/os-release
-    mkdir -p ../conf/
-    cat << EOF > ../conf/distributions
-Origin: MariaDB
-Label: MariaDB
-Codename: $VERSION_CODENAME
-Architectures: $arch source
-Components: main
-Description: MariaDB Repository
-EOF
-    #//TEMP save debs until reprepro is working (or aptly)
     mkdir ../debs
     find .. -maxdepth 1 -type f | xargs cp -t ../debs
-    cat ../conf/distributions
-    set +e
-    reprepro -b .. --ignore=wrongsourceversion include $VERSION_CODENAME ../*.changes
-    set -e
+    cd ../debs
+    apt-ftparchive packages . >Packages
+    apt-ftparchive sources . >Sources
+    apt-ftparchive release . >Release
 """
             ),
         ],
@@ -247,16 +219,12 @@ def uploadDebArtifacts():
       COMPONENTS="main main/debug"
     fi
     mkdir -p /packages/%(prop:tarbuildnum)s/%(prop:buildername)s
-    cd ..
-    for dir in debs conf db dists pool; do
-      [[ -d $dir ]] && cp -r $dir /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/
-    done
+    cp -r ../debs /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/
     cat << EOF > /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/mariadb.sources
 X-Repolib-Name: MariaDB
 Types: deb
-URIs: $artifacts_url/%(prop:tarbuildnum)s/%(prop:buildername)s/
-Suites: $VERSION_CODENAME
-Components: $COMPONENTS
+URIs: $artifacts_url/%(prop:tarbuildnum)s/%(prop:buildername)s/debs
+Suites: ./
 Trusted: yes
 EOF
     cat /packages/%(prop:tarbuildnum)s/%(prop:buildername)s/mariadb.sources