monitor client integration

- install ceph-mon-client-nvmeof from rpm
- update spdk - nonce patch
- gateway.proto
  * get_subsystems required by the monitor client
  * set ana_info
  * rename list_subsystems return types (breaking).
- add monitor.proto
- add new, grpc implementations, start monitor client
- HA test matrix

Signed-off-by: Alexander Indenbaum <[email protected]>

.env

@@ -59,7 +59,7 @@ SPDK_CENTOS_REPO_VER="9.0-21.el9"
 # Ceph Cluster
 CEPH_CLUSTER_VERSION="${CEPH_VERSION}"
 CEPH_BRANCH=ceph-nvmeof-mon
-CEPH_SHA=c6e74acc2cf8261563df78c881344d10a427007e
+CEPH_SHA=e7a25f291c1057fd9e71a9f4a9c21cf635c02490
 CEPH_VSTART_ARGS="--memstore"
 CEPH_DEVEL_MGR_PATH=../ceph
 

.github/workflows/build-container.yml

@@ -502,9 +502,92 @@ jobs:
           make down
           make clean
 
+  ha:
+    needs: [build, build-ceph]
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ["sanity", "state_transitions"]
+    runs-on: ubuntu-latest
+    env:
+      HUGEPAGES: 768  # 3 spdk instances
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup huge-pages
+        run: make setup HUGEPAGES=$HUGEPAGES
+
+      - name: Download container images
+        uses: actions/download-artifact@v4
+        with:
+          pattern: container_images*
+          merge-multiple: true
+
+      - name: Load container images
+        run: |
+          docker load < nvmeof.tar
+          docker load < nvmeof-cli.tar
+          docker load < ceph.tar
+          docker load < bdevperf.tar
+
+      - name: Start HA with two gateways
+        run: |
+          docker-compose up -d --scale nvmeof=2 nvmeof
+
+      - name: Wait for gateways to be listening
+        timeout-minutes: 3
+        run: |
+          source tests/ha/wait_gateways.sh
+
+      - name: List containers
+        if: success() || failure()
+        run: |
+          docker-compose ps
+
+      - name: List processes
+        if: success() || failure()
+        run: |
+          docker-compose top
+
+      - name: Set up target
+        run: |
+          source tests/ha/setup.sh
+
+      - name: Run HA ${{ matrix.test }} test
+        timeout-minutes: 5
+        run: |
+          . .env
+          source "tests/ha/${{ matrix.test }}.sh"
+
+      - name: Check coredump existence
+        if: success() || failure()
+        id: check_coredumps
+        uses: andstor/file-existence-action@20b4d2e596410855db8f9ca21e96fbe18e12930b  # v2, pinned to SHA for security reasons
+        with:
+          files: "/tmp/coredump/core.*"
+
+      - name: Upload ha core dumps
+        if: steps.check_coredumps.outputs.files_exists == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: core_demo
+          path: /tmp/coredump/core.*
+
+      - name: Display logs
+        if: success() || failure()
+        run: make logs OPTS=''
+
+      - name: Tear down
+        if: success() || failure()
+        run: |
+          make down
+          make clean
+
   push-to-registry:
     if: github.event_name == 'release'
-    needs: [pytest, demo, discovery]
+    needs: [pytest, demo, discovery, ha]
     runs-on: ubuntu-latest
 
     steps:

Dockerfile

@@ -13,11 +13,14 @@ CMD []
 #------------------------------------------------------------------------------
 # Base image for NVMEOF_TARGET=gateway (nvmeof-gateway)
 FROM quay.io/ceph/spdk:${NVMEOF_SPDK_VERSION:-NULL} AS base-gateway
+RUN rpm -vih https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
 RUN \
     --mount=type=cache,target=/var/cache/dnf \
     --mount=type=cache,target=/var/lib/dnf \
     dnf install -y python3-rados && \
-    dnf install -y python3-rbd
+    dnf install -y python3-rbd && \
+    dnf config-manager --set-enabled crb && \
+    dnf install -y ceph-mon-client-nvmeof
 ENTRYPOINT ["python3", "-m", "control"]
 CMD ["-c", "/src/ceph-nvmeof.conf"]
 

Dockerfile.spdk

@@ -13,6 +13,7 @@ ARG DNF_OPTS="\
 FROM quay.io/centos/centos:stream9 AS build
 
 ARG SPDK_CEPH_VERSION \
+    CEPH_CLUSTER_CEPH_REPO_BASEURL \
     SPDK_VERSION \
     DNF_OPTS \
     SPDK_PKGDEP_ARGS \
@@ -27,6 +28,7 @@ RUN <<EOF
     echo SPDK_CONFIGURE_ARGS=$SPDK_CONFIGURE_ARGS
     echo SPDK_TARGET_ARCH=$SPDK_TARGET_ARCH
     echo SPDK_MAKEFLAGS=$SPDK_MAKEFLAGS
+    echo CEPH_CLUSTER_CEPH_REPO_BASEURL=$CEPH_CLUSTER_CEPH_REPO_BASEURL
     echo ======================================================================
 EOF
 
@@ -49,19 +51,17 @@ EOF
 COPY <<EOF /etc/yum.repos.d/ceph.repo
 [Ceph]
 name=Ceph packages for \$basearch
-baseurl=https://download.ceph.com/rpm-${SPDK_CEPH_VERSION:?}/el\$releasever/\$basearch
+baseurl=${CEPH_CLUSTER_CEPH_REPO_BASEURL:-https://download.ceph.com/rpm-${SPDK_CEPH_VERSION}/el\$releasever}/\$basearch
 enabled=1
 priority=2
-gpgcheck=1
-gpgkey=https://download.ceph.com/keys/release.asc
+gpgcheck=0
 
 [Ceph-noarch]
 name=Ceph noarch packages
-baseurl=https://download.ceph.com/rpm-$SPDK_CEPH_VERSION/el\$releasever/noarch
+baseurl=${CEPH_CLUSTER_CEPH_REPO_BASEURL:-https://download.ceph.com/rpm-${SPDK_CEPH_VERSION}/el\$releasever}/noarch
 enabled=1
 priority=2
-gpgcheck=1
-gpgkey=https://download.ceph.com/keys/release.asc
+gpgcheck=0
 EOF
 
 RUN \

ceph-nvmeof.conf

@@ -15,14 +15,13 @@ port = 5500
 enable_auth = False
 state_update_notify = True
 state_update_interval_sec = 5
-#min_controller_id = 1
-#max_controller_id = 65519
 enable_spdk_discovery_controller = False
 #omap_file_lock_duration = 60
 #omap_file_lock_retries = 15
 #omap_file_lock_retry_sleep_interval = 5
 #omap_file_update_reloads = 10
 log_level=debug
+bdevs_per_cluster = 1
 #log_files_enabled = True
 #log_files_rotation_enabled = True
 #verbose_log_messages = True
@@ -46,7 +45,6 @@ port = 8009
 [ceph]
 pool = rbd
 config_file = /etc/ceph/ceph.conf
-id =
 
 [mtls]
 server_key = ./server.key

control/cli.py

@@ -649,7 +649,7 @@ def subsystem_list(self, args):
         try:
             subsystems = self.stub.list_subsystems(pb2.list_subsystems_req(subsystem_nqn=args.subsystem, serial_number=args.serial_number))
         except Exception as ex:
-            subsystems = pb2.subsystems_info(status = errno.EINVAL, error_message = f"Failure listing subsystems:\n{ex}")
+            subsystems = pb2.subsystems_info_cli(status = errno.EINVAL, error_message = f"Failure listing subsystems:\n{ex}")
 
         if args.format == "text" or args.format == "plain":
             if subsystems.status == 0:
@@ -1826,6 +1826,15 @@ def namespace(self, args):
             return self.ns_set_qos(args)
         assert False
 
+    @cli.cmd()
+    def get_subsystems(self, args):
+        """Gets subsystems."""
+        subsystems = json_format.MessageToJson(
+                        self.stub.get_subsystems(pb2.get_subsystems_req()),
+                        indent=4, including_default_value_fields=True,
+                        preserving_proto_field_name=True)
+        self.logger.info(f"Get subsystems:\n{subsystems}")
+
 def main_common(client, args):
     server_address = args.server_address
     server_port = args.server_port