WIP Update conda packaging

.github/workflows/conda.yml

@@ -1,32 +1,182 @@
 ---
-name: Build conda packages
+name: Conda Packages
 on:
   workflow_dispatch:
+    inputs:
+      release-channel:
+        type: choice
+        default: khiops-dev
+        options: [khiops-dev, khiops]
+        description: Anaconda channel to release
+  push:
+    tags: ['*']
+  pull_request:
+    paths:
+      - packaging/conda/**
+      - '!packaging/conda/README.md'
+      - .github/workflows/conda.yml
+defaults:
+  run:
+    shell: bash -el {0}
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref
+    }}
+  cancel-in-progress: true
 jobs:
-  conda-build:
+  build:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-2022, macos-latest]
+        #  Use the oldest supported Mac OS and Ubuntu versions for GLIBC compatibility
+        include:
+          - os: ubuntu-22.04
+            os-family: linux
+          - os: windows-2022
+            os-family: windows
+          - os: macos-13
+            os-family: macos
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
-      - name: Install miniconda
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          python-version: '3.10'
-          auto-activate-base: true
-          activate-environment: true
-      - name: Install conda-build
-        run: conda install conda-build
-      - name: Build khiops-bin conda package
+      - name: Checkout Sources
+        uses: actions/checkout@v4
+      - name: Install Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: latest
+          python-version: '3.11'
+      - name: Install Dependency Requirements for Building Conda Packages
+        run: conda install conda-build=3.27.0 conda-verify
+      - name: Put the Khiops version in the environment
+        shell: bash
+        run: |
+          if [[ "${{ github.ref_type }}" == "tag" ]]
+          then
+            echo "KHIOPS_VERSION=${{ github.ref_name }}" >> "$GITHUB_ENV"
+          else
+            echo "KHIOPS_VERSION=$(./scripts/khiops-version)" >> "$GITHUB_ENV"
+          fi
+      - name: Build conda packages (Windows)
+        if: runner.os == 'Windows'
+        run: |
+          mkdir khiops-conda
+          conda build --output-folder khiops-conda ./packaging/conda
+      # In Linux/macOS we need the conda-forge channel to install their pinned versions
+      - name: Build conda packages (Linux/macOS)
+        if: runner.os != 'Windows'
+        run: |
+          mkdir khiops-conda
+          conda build --channel conda-forge --output-folder khiops-conda ./packaging/conda
+      - name: Upload conda packages artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: khiops-conda-${{ matrix.os-family }}
+          path: ./khiops-conda
+          retention-days: 7
+  # Test Conda package on brand new environments
+  test:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        env:
+          - {os: ubuntu-20.04, os-family: linux}
+          - {os: ubuntu-22.04, os-family: linux}
+          - {os: windows-2019, os-family: windows}
+          - {os: windows-2022, os-family: windows}
+          - {os: macos-11, os-family: macos}
+          - {os: macos-12, os-family: macos}
+          - {os: macos-13, os-family: macos}
+    runs-on: ${{ matrix.env.os }}
+    env:
+      KHIOPS_SAMPLES_DIR: ./khiops-samples-repo
+    steps:
+      - name: Install Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: latest  # needed for macOS 13
+          python-version: ${{ matrix.python-version }}
+      - name: Download Conda Package Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: khiops-conda-${{ matrix.env.os-family }}
+          path: khiops-conda
+      - name: Install the Khiops Conda pagkage (Windows)
+        if: runner.os == 'Windows'
+        run: conda install -c ./khiops-conda/ khiops-core
+      # In Linux/macOS we need the conda-forge channel to install their pinned versions
+      - name: Install the Khiops Conda package (Linux/macOS)
+        if: runner.os != 'Windows'
+        run: conda install -c conda-forge -c ./khiops-conda/ khiops-core
+      - name: Test the Conda package installation
         run: |
-          mkdir build
-          conda build --output-folder build packaging/conda
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: package
-          path: build/*/khiops-bin*.tar.bz2
-          retention-days: 1
+          MODL -v
+          MODL_Coclustering -v
+  # Release is only executed on tags
+  # Note: For this job to work the secrets variables KHIOPS_ANACONDA_CHANNEL_TOKEN and
+  #       KHIOPS_DEV_ANACONDA_CHANNEL_TOKEN must be set with valid anaconda.org access tokens
+  release:
+    #if: github.ref_type == 'tag'
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Download package artifacts
+        uses: actions/download-artifact@v4
+        with:
+          # See the upload-artifact step in the build job for the explanation of this pattern
+          path: ./khiops-conda
+          pattern: khiops-conda-*
+          merge-multiple: true
+      - name: Install Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: latest
+          python-version: '3.11'
+      - name: Install requirement packages
+        run: conda install -y anaconda-client conda-build=3.27.0
+      - name: Reindex the package directory
+        run: conda-index ./khiops-conda
+      - name: Upload the packages to anaconda.org
+        run: |
+          # Set the anaconda.org channel
+          ANACONDA_CHANNEL="${{ inputs.release-channel || 'khiops-dev' }}"
+
+          # For the release channel: upload without forcing
+          if [[ "$ANACONDA_CHANNEL" == "khiops" ]]
+          then
+            anaconda --token "${{ secrets.KHIOPS_ANACONDA_CHANNEL_TOKEN }}" upload \
+              --user "$ANACONDA_CHANNEL" ./khiops-conda/*/*.tar.bz2
+          # For the dev channel: upload with forcing
+          else
+            anaconda --token "${{ secrets.KHIOPS_DEV_ANACONDA_CHANNEL_TOKEN }}" upload \
+              --user "$ANACONDA_CHANNEL" --force ./khiops-conda/*/*.tar.bz2
+          fi
+      - name: Extract package version
+        run: |
+          PKG_VERSION=$(\
+              conda search --override-channels --channel ./khiops-conda/ khiops-core \
+                | awk '!/#|channels/ {print $2}' \
+                | sort -u \
+          )
+          echo "PKG_VERSION=$PKG_VERSION" >> "$GITHUB_ENV"
+      - name: Create the release zip archive
+        uses: thedoctor0/[email protected]
+        with:
+          type: zip
+          path: ./khiops-conda/
+          filename: khiops-${{ env.PKG_VERSION }}-conda.zip
+      - name: Upload conda package artifacts for all platforms
+        uses: actions/upload-artifact@v4
+        with:
+          name: khiops-conda-all
+          path: ./khiops-${{ env.PKG_VERSION }}-conda.zip
+      #- name: Release the zip archive
+        #uses: ncipollo/release-action@v1
+        #with:
+          #allowUpdates: true
+          #artifacts: ./khiops-${{ env.PKG_VERSION }}-conda.zip
+          #draft: false
+          #makeLatest: false
+          #prerelease: true
+          #updateOnlyUnreleased: true

packaging/conda/bld.bat

@@ -1,2 +1,13 @@
-cmake --preset windows-msvc-release
+REM Echo all output
+@echo on
+
+REM Build the Khiops binaries
+cmake --preset windows-msvc-release -DBUILD_JARS=OFF -DTESTING=OFF
 cmake --build --preset windows-msvc-release --parallel --target MODL MODL_Coclustering
+
+REM Copy the MODL binaries to the Conda PREFIX path
+mkdir %PREFIX%\bin
+copy build\windows-msvc-release\bin\MODL.exe %PREFIX%\bin
+copy build\windows-msvc-release\bin\MODL_Coclustering.exe %PREFIX%\bin
+
+if errorlevel 1 exit 1

packaging/conda/build.sh

@@ -1,12 +1,109 @@
-# !/bin/bash
+#!/bin/bash
 
-# On macOS, we have to build with the compiler outside conda. With conda's clang the following error occurs:
-# ld: unsupported tapi file type '!tapi-tbd' in YAML file '/Applications/Xcode_14.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/lib/libSystem.tbd' for architecture x86_64
-if [ "$(uname)" == "Darwin" ]
+# Set-up the shell to behave more like a general-purpose programming language
+set -euo pipefail
+
+# Build MODL and MODL_Coclustering
+# Note on macOS we need the macOS SDK 10.10 for this conda build to work
+if [[ "$(uname)" == "Darwin" ]]
 then
-    cmake --preset macos-clang-release -DCMAKE_CXX_COMPILER=/usr/bin/clang++
-    cmake --build --preset macos-clang-release --parallel --target MODL MODL_Coclustering
+  CMAKE_PRESET="macos-clang-release"
 else
-    cmake --preset linux-gcc-release
-    cmake --build --preset linux-gcc-release --parallel --target MODL MODL_Coclustering
+  CMAKE_PRESET="linux-gcc-release"
+fi
+cmake --preset $CMAKE_PRESET -DBUILD_JARS=OFF -DTESTING=OFF -DCMAKE_CXX_COMPILER="$PREFIX/bin/mpicxx"
+cmake --build --preset $CMAKE_PRESET --parallel --target MODL MODL_Coclustering
+
+# Copy the MODL binaries to the Conda PREFIX path
+cp "./build/$CMAKE_PRESET/bin/MODL" "$PREFIX/bin"
+cp "./build/$CMAKE_PRESET/bin/MODL_Coclustering" "$PREFIX/bin"
+
+# Custom rpath relocation and signing executables for macOS in arm64
+#
+# In osx-arm64 executing any binary that is not signed will make appear popups appearing demanding
+# "accepting incoming connections". Since our application doesn't need any connections from the
+# outside the machine this doesn't affect the execution but since it is launched with MPI the number
+# of popups appearing is high. This is difficult to fix for the user because the if the artifact is
+# not signed it will reappear even if we click in the "Allow" button. So we sign the MODL
+# executables to solve this (only a single popup concerning mpiexec.hydra may appear but for this
+# application pressing on "Allow" works).
+#
+# However, in the default settings, `conda build` relocalizes the executable by changing rpath of
+# the library paths at $PREFIX by relative ones and in doing so it nullifies any signature. So we
+# do ourselves this procedure first and then sign the binary.
+#
+# Note that in meta.yaml for osx-arm64 we have custom build.binary_relocation and
+# build.detect_binary_files_with_prefix option
+#
+# This part must be executed in a root machine to be non-interactive (eg. GitHub runner)
+# It also needs the following environment variable:
+# - KHIOPS_APPLE_CERTIFICATE_COMMON_NAME: The second column of the `security find-identity` command
+# A base64 encoded certificate may also be provided, the following 2 variables must be set
+# - KHIOPS_APPLE_CERTIFICATE_BASE64: The identity file .p12 (certificate + private key) in base64
+# - KHIOPS_APPLE_CERTIFICATE_PASSWORD: Password for the certificate file
+# - KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD: A temporary password to decrypt the certificate
+#
+cd ..
+if [[ "$(uname)" == "Darwin" && -n "${KHIOPS_APPLE_CERTIFICATE_COMMON_NAME-}" ]]
+then
+  # Delete the rpath of each executable
+  # Delete two times for MODL because for some reason it is there 2 times
+  install_name_tool -delete_rpath "$PREFIX/lib" "$PREFIX/bin/MODL"
+  install_name_tool -delete_rpath "$PREFIX/lib" "$PREFIX/bin/MODL"
+  install_name_tool -delete_rpath "$PREFIX/lib" "$PREFIX/bin/MODL_Coclustering"
+
+  # Add the relative rpath as conda build would
+  install_name_tool -add_rpath "@loader_path/../lib" "$PREFIX/bin/MODL"
+  install_name_tool -add_rpath "@loader_path/../lib" "$PREFIX/bin/MODL_Coclustering"
+
+  if [[ -n "${KHIOPS_APPLE_CERTIFICATE_BASE64-}" ]]
+  then
+    # Keychain setup slightly modified from: https://stackoverflow.com/a/68577995
+    # Before importing identity
+    # - Set the default user login keychain
+    # - Create a temporary keychain
+    # - Append temporary keychain to the user domain
+    # - Remove relock timeout
+    # - Unlock the temporary keychain
+    sudo security list-keychains -d user -s login.keychain
+    sudo security create-keychain -p "$KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD" kh-tmp.keychain
+    sudo security list-keychains -d user -s kh-tmp.keychain \
+      "$(security list-keychains -d user | sed s/\"//g)"
+    sudo security set-keychain-settings kh-tmp.keychain
+    sudo security unlock-keychain -p "$KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD" kh-tmp.keychain
+
+    # Add identity (certificate + private key) to keychain
+    echo "$KHIOPS_APPLE_CERTIFICATE_BASE64" \
+      | base64 --decode -i - -o kh-cert.p12
+    sudo security import kh-cert.p12 \
+      -k kh-tmp.keychain \
+      -P "$KHIOPS_APPLE_CERTIFICATE_PASSWORD" \
+      -A -T "/usr/bin/codesign"
+    rm -f kh-cert.p12
+
+    # Enable codesigning from a non user interactive shell
+    sudo security set-key-partition-list -S apple-tool:,apple:, \
+      -s -k "$KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD" \
+      -D "$KHIOPS_APPLE_CERTIFICATE_COMMON_NAME" \
+      -t private kh-tmp.keychain
+  fi
+
+  # We make sure to use the default macOS/Xcode codesign tool. This is because the sigtool python
+  # package (installed by conda build as a dependency) makes an alias "codesign" which is prioritary
+  # in the build environment. The alias, however, alias doesn't support signing with a proper
+  # identity and makes the build fail!
+  CODESIGN="/usr/bin/codesign"
+
+  # Sign the MODL executable and check
+  $CODESIGN --force --sign "$KHIOPS_APPLE_CERTIFICATE_COMMON_NAME" "$PREFIX/bin/MODL"
+  $CODESIGN --force --sign "$KHIOPS_APPLE_CERTIFICATE_COMMON_NAME" "$PREFIX/bin/MODL_Coclustering"
+  $CODESIGN -d -vvv "$PREFIX/bin/MODL"
+  $CODESIGN -d -vvv "$PREFIX/bin/MODL_Coclustering"
+
+  # Remove the temporary keychain and restore the login keychain as default if created
+  if [[ -n "${KHIOPS_APPLE_CERTIFICATE_BASE64-}" ]]
+  then
+    sudo security delete-keychain kh-tmp.keychain
+    sudo security list-keychains -d user -s login.keychain
+  fi
 fi