DAOS-623 build: Resolve issues with Dockerfiles. (#6099)

Use pip3 --user and remove temp files. Do not install pandoc, curl, man or doxygen. Move leap15 to ci2. Install missing packages to run NLT in docker. Signed-off-by: Ashley Pittman <[email protected]>
daos-stack · Jul 7, 2021 · 537e2b7 · 537e2b7
1 parent 182ab7b
commit 537e2b7
Show file tree

Hide file tree

Showing 11 changed files with 58 additions and 77 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        distro: [centos.7, leap.15]
+        distro: [centos.7]
         compiler: [gcc, clang]
     steps:
     - name: Checkout code

diff --git a/.github/workflows/ci2.yml b/.github/workflows/ci2.yml
@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        distro: [ubuntu.20.04, centos.8]
+        distro: [ubuntu.20.04, centos.8, leap.15]
     steps:
     - uses: actions/checkout@v2
     - uses: satackey/[email protected]
@@ -92,7 +92,7 @@ jobs:
       fail-fast: false
       matrix:
         # Run this on distros where we don't run the ci1 Workflow.
-        distro: [ubuntu.20.04, centos.8]
+        distro: [ubuntu.20.04, centos.8, leap.15]
         compiler: [clang, gcc]
     steps:
     - name: Checkout code

diff --git a/.github/workflows/doxygen.yml b/.github/workflows/doxygen.yml
@@ -13,13 +13,5 @@ jobs:
       uses: actions/[email protected]
       with:
         fetch-depth: 100
-    - name: Install software
-      run: sudo apt-get install doxygen
-    - name: Check files
+    - name: Run check
       run: ./ci/copy_doxygen_files.sh
-    - name: Add problem matcher
-      run: echo ::add-matcher::ci/daos-doxygen-matcher.json
-    - name: Run
-      run: doxygen Doxyfile
-    - name: Remove matcher
-      run: echo "::remove-matcher owner=daos-doxygen::"
diff --git a/ci/copy_doxygen_files.sh b/ci/copy_doxygen_files.sh
@@ -1,31 +1,44 @@
-#!/bin/sh
+#!/bin/bash
 
-# Prepare the source tree for Doxygen, see .github/workflows/doxygen.yml
+# Script to run doxygen against modified headers and report results.
+# see .github/workflows/doxygen.yml
 
-set -ex
+set -e
+
+echo ::group::Setting up src
 
 # Load the list of modified files.
 git fetch
-git branch -lr
-FILES=$(git diff --name-only origin/master... src/include)
+mapfile -t FILES < <(git diff --name-only origin/master... src/include/*.h)
+
+# Ensure there is at least one file present in new dir.
+if [ ${#FILES[@]} -eq 0 ];
+then
+    echo No headers modified, exiting.
+    echo ::endgroup::
+    exit 0
+fi
 
-# Create a new temp directory.
+# Make a new directory.
 mkdir src/include2
 
-# Copy modified files into it.
-for FILE in $FILES
-do
-    cp "$FILE" ./src/include2
-done
+# Copy the files to new dir.
+cp "${FILES[@]}" ./src/include2
 
-# Move real path aside.
+# Move the real files aside
 mv src/include src/include.old
 
-# And place the modified ones back in place.
+# Move in the ones to test.
 mv src/include2 src/include
 
-# Ensure there is at least one file present.
-if [ -f src/include.old/daos.h ]
-then
-    mv src/include.old/daos.h src/include
-fi
+echo ::endgroup::
+
+echo ::group::Installing doxygen
+sudo apt-get install doxygen
+echo ::endgroup::
+
+echo ::group::Running check
+echo ::add-matcher::ci/daos-doxygen-matcher.json
+doxygen Doxyfile
+echo ::remove-matcher owner=daos-doxygen::
+echo ::endgroup::
diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py
@@ -276,9 +276,9 @@ def define_components(reqs):
 
     retriever = GitRepoRetriever("https://github.com/pmem/pmdk.git")
 
-    pmdk_build = ["make all \"BUILD_RPMEM=n\" \"NDCTL_ENABLE=n\" "
-                  "\"NDCTL_DISABLE=y\" $JOBS_OPT install "
-                  "prefix=$PMDK_PREFIX"]
+    pmdk_build = ['make all "BUILD_RPMEM=n" "NDCTL_ENABLE=n" '
+                  '"NDCTL_DISABLE=y" "DOC=n" $JOBS_OPT install '
+                  'prefix=$PMDK_PREFIX']
 
     reqs.define('pmdk',
                 retriever=retriever,

diff --git a/src/include/cart/api.h b/src/include/cart/api.h
@@ -83,7 +83,6 @@ crt_init(crt_group_id_t grpid, uint32_t flags)
 int
 crt_context_create(crt_context_t *crt_ctx);
 
-
 /**
  * Set the timeout value for all RPC requests created on the specified context.
  * Setting the timeout after crt_req_create() call will not affect already
@@ -1011,11 +1010,6 @@ crt_bulk_access(crt_bulk_t bulk_hdl, d_sg_list_t *sgl);
 int
 crt_bulk_free(crt_bulk_t bulk_hdl);
 
-/**
- * Decrease reference of the bulk handle.
- */
-#define crt_bulk_decref(bulk_hdl)	crt_bulk_free(bulk_hdl)
-
 /**
  * Start a bulk transferring (inside an RPC handler).
  *
@@ -1301,7 +1295,6 @@ crt_group_config_remove(crt_group_t *grp);
 int
 crt_group_detach(crt_group_t *attached_grp);
 
-
 /**
  * Convert a primary group rank to a local subgroup rank. Given a primary group
  * rank \p rank_in, find its rank number \p rank_out within a sub-group \p
@@ -1747,7 +1740,6 @@ crt_register_event_cb(crt_event_cb event_handler, void *arg);
 int
 crt_unregister_event_cb(crt_event_cb event_handler, void *arg);
 
-
 typedef void
 (*crt_hlc_error_cb) (void *arg);
 
@@ -1763,7 +1755,6 @@ typedef void
 int
 crt_register_hlc_error_cb(crt_hlc_error_cb event_handler, void *arg);
 
-
 /**
  * A protocol is a set of RPCs. A protocol has a base opcode and a version,
  * member RPCs have opcodes that are contiguous numbers starting from
@@ -1801,7 +1792,6 @@ crt_register_hlc_error_cb(crt_hlc_error_cb event_handler, void *arg);
  *    sending RPCs using it's member opcodes.
  */
 
-
 /**
  * 1) define crf for each member RPC. my_rpc_crf_1, my_rpc_crf_2
  *
@@ -1874,7 +1864,6 @@ int
 crt_proto_query(crt_endpoint_t *tgt_ep, crt_opcode_t base_opc,
 		uint32_t *ver, int count, crt_proto_query_cb_t cb, void *arg);
 
-
 /**
  * Set self rank.
  *
@@ -1944,7 +1933,6 @@ crt_group_rank_remove(crt_group_t *group, d_rank_t rank);
  */
 int crt_self_uri_get(int tag, char **uri);
 
-
 /**
  * Retrieve group information containing ranks and associated uris
  *
@@ -2101,7 +2089,6 @@ int crt_group_auto_rank_remove(crt_group_t *grp, bool enable);
  */
 int crt_group_secondary_destroy(crt_group_t *grp);
 
-
 /**
  * Perform a primary group modification in an atomic fashion based on the
  * operation specified. Currently supported operations are 'add', 'remove'

diff --git a/src/include/daos_types.h b/src/include/daos_types.h
@@ -53,14 +53,13 @@ typedef enum {
 	DAOS_ANCHOR_TYPE_EOF	= 3,
 } daos_anchor_type_t;
 
-/** Iteration Anchor */
 #define DAOS_ANCHOR_BUF_MAX	104
+/** Iteration Anchor */
 typedef struct {
-	uint16_t	da_type; /** daos_anchor_type_t */
+	uint16_t	da_type;	/**< daos_anchor_type_t */
 	uint16_t	da_shard;
-	uint32_t	da_flags; /** see enum daos_anchor_flags */
-	/* record the offset for each shards for EC object */
-	uint64_t	da_sub_anchors;
+	uint32_t	da_flags;	/**< see enum daos_anchor_flags */
+	uint64_t	da_sub_anchors;	/**< record the offset for each shards for EC object */
 	uint8_t		da_buf[DAOS_ANCHOR_BUF_MAX];
 } daos_anchor_t;
 

diff --git a/utils/docker/Dockerfile.centos.7 b/utils/docker/Dockerfile.centos.7
@@ -100,11 +100,11 @@ RUN dnf -y install deltarpm && \
         clang-analyzer \
         cmake \
         CUnit-devel \
-        doxygen \
         e2fsprogs \
         file \
         flex \
         fuse3-devel \
+        fuse3 \
         gcc \
         gcc-c++ \
         git \
@@ -128,17 +128,14 @@ RUN dnf -y install deltarpm && \
         Lmod \
         lz4-devel \
         make \
-        man \
         maven \
         meson \
-        nasm \
         ndctl \
         ninja-build \
         numactl \
         numactl-devel \
         openmpi3-devel \
         openssl-devel \
-        pandoc \
         patch \
         patchelf \
         pciutils \
@@ -161,11 +158,6 @@ RUN dnf -y install deltarpm && \
         yasm && \
     dnf clean all
 
-# DPDK/SPDK needs pyelftools now
-# Longer term, we should upload our RPM build of this to the public packaging
-# site and install it by RPM, but for now, we'll just pip install it
-RUN pip3 install pyelftools
-
 ARG UID=1000
 
 # Add DAOS users
@@ -181,6 +173,13 @@ RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent &&   \
     chown daos_server.daos_server /opt/daos /mnt/daos /var/run/daos_server &&  \
     chown daos_agent.daos_agent /var/run/daos_agent
 
+# DPDK/SPDK needs pyelftools now
+# Longer term, we should upload our RPM build of this to the public packaging
+# site and install it by RPM, but for now, we'll just pip install it
+USER daos_server:daos_server
+RUN pip3 --no-cache-dir --disable-pip-version-check install --user pyelftools
+USER root:root
+
 ARG QUICKBUILD=false
 ARG QUICKBUILD_DEPS
 

diff --git a/utils/docker/Dockerfile.centos.8 b/utils/docker/Dockerfile.centos.8
@@ -116,7 +116,6 @@ RUN dnf -y upgrade && \
         clang-analyzer \
         cmake \
         CUnit-devel \
-        doxygen \
         e2fsprogs \
         file \
         flex \
@@ -145,17 +144,14 @@ RUN dnf -y upgrade && \
         Lmod \
         lz4-devel \
         make \
-        man \
         maven \
         meson \
-        nasm \
         ndctl \
         ninja-build \
         numactl \
         numactl-devel \
         openmpi-devel \
         openssl-devel \
-        pandoc \
         patch \
         patchelf \
         pciutils \
@@ -178,11 +174,6 @@ RUN dnf -y upgrade && \
         yasm && \
     dnf clean all
 
-# DPDK/SPDK needs pyelftools now
-# Longer term, we should upload our RPM build of this to the public packaging
-# site and install it by RPM, but for now, we'll just pip install it
-RUN pip3 install pyelftools
-
 ARG UID=1000
 
 # Add DAOS users
@@ -198,6 +189,13 @@ RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent &&   \
     chown daos_server.daos_server /opt/daos /mnt/daos /var/run/daos_server &&  \
     chown daos_agent.daos_agent /var/run/daos_agent
 
+# DPDK/SPDK needs pyelftools now
+# Longer term, we should upload our RPM build of this to the public packaging
+# site and install it by RPM, but for now, we'll just pip install it
+USER daos_server:daos_server
+RUN pip3 --no-cache-dir --disable-pip-version-check install --user pyelftools
+USER root:root
+
 ARG QUICKBUILD=false
 ARG QUICKBUILD_DEPS
 

diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15
@@ -76,8 +76,6 @@ RUN zypper --non-interactive --gpg-auto-import-keys --no-gpg-checks ref && \
         clang \
         cmake \
         cunit-devel \
-        curl \
-        doxygen \
         flex \
         fuse3-devel \
         gcc \
@@ -105,13 +103,11 @@ RUN zypper --non-interactive --gpg-auto-import-keys --no-gpg-checks ref && \
         libyaml-devel \
         lua-lmod \
         make \
-        man \
         maven \
         meson \
-        nasm \
+        numactl \
         ninja \
         openmpi3-devel \
-        pandoc \
         patch \
         patchelf \
         pciutils \

diff --git a/utils/docker/Dockerfile.ubuntu.20.04 b/utils/docker/Dockerfile.ubuntu.20.04
@@ -23,8 +23,6 @@ RUN apt-get -y update && \
         build-essential \
         clang \
         cmake \
-        curl \
-        doxygen \
         fuse3 \
         git \
         golang-go \
@@ -52,7 +50,6 @@ RUN apt-get -y update && \
         numactl \
         ninja-build \
         openjdk-8-jdk \
-        pandoc \
         patchelf \
         pciutils \
         pkg-config \