diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index e8e0e03f1..e6e2fb379 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -1,32 +1,192 @@ --- -name: Build conda packages +name: Conda Packages on: workflow_dispatch: + inputs: + release-channel: + type: choice + default: khiops-dev + options: [khiops-dev, khiops] + description: Anaconda channel to release + push: + tags: ['*'] + pull_request: + paths: + - packaging/conda/** + - '!packaging/conda/README.md' + - .github/workflows/conda.yml +defaults: + run: + shell: bash -el {0} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref + }} + cancel-in-progress: true jobs: - conda-build: + build: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-2022, macos-latest] + include: + # We use the oldest ubuntu available to have forward compatibility of the libc + - os: ubuntu-20.04 + os-family: linux-64 + - os: windows-2022 + os-family: win-64 + - os: macos-13 + os-family: osx-64 + - os: macos-14 + os-family: osx-arm64 runs-on: ${{ matrix.os }} steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Install miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: '3.10' - auto-activate-base: true - activate-environment: true - - name: Install conda-build - run: conda install conda-build - - name: Build khiops-bin conda package + - name: Checkout Sources + uses: actions/checkout@v4 + - name: Install Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: latest + python-version: '3.12' + - name: Install Dependency Requirements for Building Conda Packages + run: conda install conda-build conda-verify + # We need MacOS SDK 10.10 to build for macOS Intel + # See: https://docs.conda.io/projects/conda-build/en/3.21.x/resources/compiler-tools.html#macos-sdk + - name: Install Mac OS SDK 10.10 + if: runner.os == 'macOS' + run: | + wget https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.10.sdk.tar.xz + sudo tar -zxvf MacOSX10.10.sdk.tar.xz -C /opt + - name: Put the Khiops Conda version in the environment + shell: bash + run: | + # Obtain the Khiops version + if [[ "${{ github.ref_type }}" == "tag" ]] + then + KHIOPS_RAW_VERSION="${{ github.ref_name }}" + else + KHIOPS_RAW_VERSION="$(./scripts/khiops-version)" + fi + + # The conda version cannot have '-' as a character so we eliminate it + echo "KHIOPS_VERSION=$(echo $KHIOPS_RAW_VERSION | sed 's/-//')" >> "$GITHUB_ENV" + - name: Build conda packages (Windows) + if: runner.os == 'Windows' + run: conda build --output-folder ./build/conda ./packaging/conda + # In Linux/macOS we need the conda-forge channel to install their pinned versions + - name: Build conda packages (Linux/macOS) + if: runner.os != 'Windows' + run: conda build --channel conda-forge --output-folder ./build/conda ./packaging/conda + - name: Upload conda packages artifact + uses: actions/upload-artifact@v4 + with: + name: khiops-conda-${{ matrix.os-family }} + path: ./build/conda + retention-days: 7 + # Test Conda package on brand new environments + test: + needs: build + strategy: + fail-fast: false + matrix: + env: + - {os: ubuntu-20.04, os-family: linux-64} + - {os: ubuntu-22.04, os-family: linux-64} + - {os: windows-2019, os-family: win-64} + - {os: windows-2022, os-family: win-64} + - {os: macos-11, os-family: osx-64} + - {os: macos-12, os-family: osx-64} + - {os: macos-13, os-family: osx-64} + - {os: macos-14, os-family: osx-arm64} + runs-on: ${{ matrix.env.os }} + steps: + - name: Install Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: latest # needed for macOS 13 + python-version: ${{ matrix.python-version }} + - name: Download Conda Package Artifact + uses: actions/download-artifact@v4 + with: + name: khiops-conda-${{ matrix.env.os-family }} + path: ./build/conda + - name: Install the Conda package (Windows) + if: runner.os == 'Windows' + run: conda install --channel ./build/conda khiops-core + # In Linux/macOS we need the conda-forge channel to install their pinned versions + - name: Install the Conda package (Linux/macOS) + if: runner.os != 'Windows' + run: conda install --channel conda-forge --channel ./build/conda khiops-core + - name: Test that the executables are installed + run: | + MODL -v + MODL_Coclustering -v + # Release is only executed on tags + # Note: For this job to work the secrets variables KHIOPS_ANACONDA_CHANNEL_TOKEN and + # KHIOPS_DEV_ANACONDA_CHANNEL_TOKEN must be set with valid anaconda.org access tokens + release: + if: github.ref_type == 'tag' + needs: test + runs-on: ubuntu-22.04 + permissions: + contents: write + steps: + - name: Download package artifacts + uses: actions/download-artifact@v4 + with: + # See the upload-artifact step in the build job for the explanation of this pattern + path: ./build/conda + pattern: khiops-conda-* + merge-multiple: true + - name: Install Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: latest + python-version: '3.12' + - name: Install requirement packages + run: conda install -y anaconda-client conda-index + - name: Reindex the package directory + run: python -m conda_index ./build/conda + - name: Upload the packages to anaconda.org run: | - mkdir build - conda build --output-folder build packaging/conda - - name: Upload Artifacts - uses: actions/upload-artifact@v3 - with: - name: package - path: build/*/khiops-bin*.tar.bz2 - retention-days: 1 + # Set the anaconda.org channel + ANACONDA_CHANNEL="${{ inputs.release-channel || 'khiops-dev' }}" + + # For the release channel: upload without forcing + if [[ "$ANACONDA_CHANNEL" == "khiops" ]] + then + anaconda --token "${{ secrets.KHIOPS_ANACONDA_CHANNEL_TOKEN }}" upload \ + --user "$ANACONDA_CHANNEL" ./build/conda/*/*.tar.bz2 + # For the dev channel: upload with forcing + else + anaconda --token "${{ secrets.KHIOPS_DEV_ANACONDA_CHANNEL_TOKEN }}" upload \ + --user "$ANACONDA_CHANNEL" --force ./build/conda/*/*.tar.bz2 + fi + - name: Extract package version + run: | + PKG_VERSION=$(\ + conda search --override-channels --channel ./build/conda/ khiops-core \ + | awk '!/#|channels/ {print $2}' \ + | sort -u \ + ) + echo "PKG_VERSION=$PKG_VERSION" >> "$GITHUB_ENV" + - name: Create the release zip archive + uses: thedoctor0/zip-release@0.7.6 + with: + type: zip + path: ./build/conda + filename: khiops-${{ env.PKG_VERSION }}-conda.zip + - name: Upload conda package artifacts for all platforms + uses: actions/upload-artifact@v4 + with: + name: khiops-conda-all + path: ./khiops-${{ env.PKG_VERSION }}-conda.zip + - name: Release the zip archive + uses: ncipollo/release-action@v1 + with: + allowUpdates: true + body: | + **This release is for testing purporses only and there is no support for it.** + **Go to https://khiops.org to install the latest supported version.** + draft: false + makeLatest: false + prerelease: true + updateOnlyUnreleased: true diff --git a/.gitignore b/.gitignore index fdf49cb06..dab4776de 100644 --- a/.gitignore +++ b/.gitignore @@ -18,9 +18,9 @@ build/ CMakeFiles/ # Test exceptions -test/UnitTests/*/results/ -test/LearningTest/*/*/*/results/ +test/**/results/ comparisonResults.log # Python -*.pyc \ No newline at end of file +*.pyc +__pycache__/ diff --git a/packaging/conda/README.md b/packaging/conda/README.md new file mode 100644 index 000000000..833b17866 --- /dev/null +++ b/packaging/conda/README.md @@ -0,0 +1,35 @@ +# Khiops Conda Packaging Scripts + +## How to Build +You'll need `conda-build` installed in your system. + +```bash +# At the root of the repo +# These commands will leave a ready to use conda channel in `./khiops-conda` + +# Windows +conda build --output-folder ./khiops-conda packaging/conda + +# Linux/macOS +# Note: We need the conda-forge channel to obtain the pinned versions of MPICH +conda build --channel conda-forge --output-folder ./khiops-conda packaging/conda +``` + +### Signing the Executables in macOS +The script can sign the Khiops binaries. This is to avoid annoying firewall pop-ups. To enable this +set the following environment variable: +- `KHIOPS_APPLE_CERTIFICATE_COMMON_NAME`: The common name of the signing certificate. + +If the certificate is available in your keychain then the `conda build` command will use it to sign +the `MODL*` binaries. You may have to input your password. + +Alternatively, a certificate file encoded in base64 may be provided by additionally setting the +following environment variables: +- `KHIOPS_APPLE_CERTIFICATE_BASE64`: The base64 encoding of the signing certificate. +-`KHIOPS_APPLE_CERTIFICATE_PASSWORD`: The password of the signing certificate. +- `KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD` : A password for the temporary keychain created in the process. + +If the process is executed as root (eg. Github Runner) then there is no need to input a password to +access the keychain. Otherwise you'll be prompt with it. + +For more details see the comments in the signing section of `build.sh`. diff --git a/packaging/conda/bld.bat b/packaging/conda/bld.bat index 0a1f27ffe..bf2639313 100644 --- a/packaging/conda/bld.bat +++ b/packaging/conda/bld.bat @@ -1,2 +1,13 @@ -cmake --preset windows-msvc-release +REM Echo all output +@echo on + +REM Build the Khiops binaries +cmake --preset windows-msvc-release -DBUILD_JARS=OFF -DTESTING=OFF cmake --build --preset windows-msvc-release --parallel --target MODL MODL_Coclustering + +REM Copy the MODL binaries to the Conda PREFIX path +mkdir %PREFIX%\bin +copy build\windows-msvc-release\bin\MODL.exe %PREFIX%\bin +copy build\windows-msvc-release\bin\MODL_Coclustering.exe %PREFIX%\bin + +if errorlevel 1 exit 1 diff --git a/packaging/conda/build.sh b/packaging/conda/build.sh index e41eb2bac..85a4d1174 100644 --- a/packaging/conda/build.sh +++ b/packaging/conda/build.sh @@ -1,12 +1,113 @@ -# !/bin/bash +#!/bin/bash -# On macOS, we have to build with the compiler outside conda. With conda's clang the following error occurs: -# ld: unsupported tapi file type '!tapi-tbd' in YAML file '/Applications/Xcode_14.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/lib/libSystem.tbd' for architecture x86_64 -if [ "$(uname)" == "Darwin" ] +# Set-up the shell to behave more like a general-purpose programming language +set -euo pipefail + +# Choose the build preset for macOS/Linux +if [[ "$(uname)" == "Darwin" ]] then - cmake --preset macos-clang-release -DCMAKE_CXX_COMPILER=/usr/bin/clang++ - cmake --build --preset macos-clang-release --parallel --target MODL MODL_Coclustering + CMAKE_PRESET="macos-clang-release" else - cmake --preset linux-gcc-release - cmake --build --preset linux-gcc-release --parallel --target MODL MODL_Coclustering + CMAKE_PRESET="linux-gcc-release" +fi + +# Build MODL and MODL_Coclustering +cmake --fresh --preset $CMAKE_PRESET -DBUILD_JARS=OFF -DTESTING=OFF +cmake --build --preset $CMAKE_PRESET --parallel --target MODL MODL_Coclustering + +# Copy the MODL binaries to the Conda PREFIX path +cp "./build/$CMAKE_PRESET/bin/MODL" "$PREFIX/bin" +cp "./build/$CMAKE_PRESET/bin/MODL_Coclustering" "$PREFIX/bin" + +# Custom rpath relocation and signing executables for macOS in arm64 +# +# In osx-arm64 executing any binary that is not signed will make appear popups appearing demanding +# "accepting incoming connections". Since our application doesn't need any connections from the +# outside the machine this doesn't affect the execution but since it is launched with MPI the number +# of popups appearing is high. This is difficult to fix for the user because the if the artifact is +# not signed it will reappear even if we click in the "Allow" button. So we sign the MODL +# executables to solve this (only a single popup concerning mpiexec.hydra may appear but for this +# application pressing on "Allow" works). +# +# However, in the default settings, `conda build` relocalizes the executable by changing rpath of +# the library paths at $PREFIX by relative ones and in doing so it nullifies any signature. So we +# do ourselves this procedure first and then sign the binary. +# +# Note that in meta.yaml for osx-arm64 we have custom build.binary_relocation and +# build.detect_binary_files_with_prefix option +# +# This part must be executed in a root machine to be non-interactive (eg. GitHub runner) +# It also needs the following environment variable: +# - KHIOPS_APPLE_CERTIFICATE_COMMON_NAME: The second column of the `security find-identity` command +# A base64 encoded certificate may also be provided, the following 2 variables must be set +# - KHIOPS_APPLE_CERTIFICATE_BASE64: The identity file .p12 (certificate + private key) in base64 +# - KHIOPS_APPLE_CERTIFICATE_PASSWORD: Password for the certificate file +# - KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD: A temporary password to decrypt the certificate +# +cd .. +if [[ "$(uname)" == "Darwin" && -n "${KHIOPS_APPLE_CERTIFICATE_COMMON_NAME-}" ]] +then + # Inform about the signature process + echo "Signing binaries with the certificate named '${KHIOPS_APPLE_CERTIFICATE_COMMON_NAME}'" + + # Delete the rpath of each executable + # Delete two times for MODL because for some reason it is there 2 times + install_name_tool -delete_rpath "$PREFIX/lib" "$PREFIX/bin/MODL" + install_name_tool -delete_rpath "$PREFIX/lib" "$PREFIX/bin/MODL" + install_name_tool -delete_rpath "$PREFIX/lib" "$PREFIX/bin/MODL_Coclustering" + + # Add the relative rpath as conda build would + install_name_tool -add_rpath "@loader_path/../lib" "$PREFIX/bin/MODL" + install_name_tool -add_rpath "@loader_path/../lib" "$PREFIX/bin/MODL_Coclustering" + + if [[ -n "${KHIOPS_APPLE_CERTIFICATE_BASE64-}" ]] + then + # Keychain setup slightly modified from: https://stackoverflow.com/a/68577995 + # Before importing identity + # - Set the default user login keychain + # - Create a temporary keychain + # - Append temporary keychain to the user domain + # - Remove relock timeout + # - Unlock the temporary keychain + sudo security list-keychains -d user -s login.keychain + sudo security create-keychain -p "$KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD" kh-tmp.keychain + sudo security list-keychains -d user -s kh-tmp.keychain \ + "$(security list-keychains -d user | sed s/\"//g)" + sudo security set-keychain-settings kh-tmp.keychain + sudo security unlock-keychain -p "$KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD" kh-tmp.keychain + + # Add identity (certificate + private key) to keychain + echo "$KHIOPS_APPLE_CERTIFICATE_BASE64" \ + | base64 --decode -i - -o kh-cert.p12 + sudo security import kh-cert.p12 \ + -k kh-tmp.keychain \ + -P "$KHIOPS_APPLE_CERTIFICATE_PASSWORD" \ + -A -T "/usr/bin/codesign" + rm -f kh-cert.p12 + + # Enable codesigning from a non user interactive shell + sudo security set-key-partition-list -S apple-tool:,apple:, \ + -s -k "$KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD" \ + -D "$KHIOPS_APPLE_CERTIFICATE_COMMON_NAME" \ + -t private kh-tmp.keychain + fi + + # We make sure to use the default macOS/Xcode codesign tool. This is because the sigtool python + # package (installed by conda build as a dependency) makes an alias "codesign" which is prioritary + # in the build environment. The alias, however, alias doesn't support signing with a proper + # identity and makes the build fail! + CODESIGN="/usr/bin/codesign" + + # Sign the MODL executable and check + $CODESIGN --force --sign "$KHIOPS_APPLE_CERTIFICATE_COMMON_NAME" "$PREFIX/bin/MODL" + $CODESIGN --force --sign "$KHIOPS_APPLE_CERTIFICATE_COMMON_NAME" "$PREFIX/bin/MODL_Coclustering" + $CODESIGN -d -vvv "$PREFIX/bin/MODL" + $CODESIGN -d -vvv "$PREFIX/bin/MODL_Coclustering" + + # Remove the temporary keychain and restore the login keychain as default if created + if [[ -n "${KHIOPS_APPLE_CERTIFICATE_BASE64-}" ]] + then + sudo security delete-keychain kh-tmp.keychain + sudo security list-keychains -d user -s login.keychain + fi fi diff --git a/packaging/conda/conda_build_config.yaml b/packaging/conda/conda_build_config.yaml new file mode 100644 index 000000000..03d8c9e6a --- /dev/null +++ b/packaging/conda/conda_build_config.yaml @@ -0,0 +1,6 @@ +--- +# We need MacOS SDK 10.10 to be able to build on macOS Intel +# Download: https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.10.sdk.tar.xz +# Decompress then to /opt: tar -zxvf MacOSX10.10.sdk.tar.xz -C /opt +CONDA_BUILD_SYSROOT: + - /opt/MacOSX10.10.sdk # [osx and not arm64] diff --git a/packaging/conda/install-khiops.bat b/packaging/conda/install-khiops.bat deleted file mode 100644 index cceb6ca7c..000000000 --- a/packaging/conda/install-khiops.bat +++ /dev/null @@ -1,7 +0,0 @@ -REM Echo all output -@echo on - -REM Copy the MODL binaries to the anaconda PREFIX path -mkdir %PREFIX%\bin -copy build\windows-msvc-release\bin\MODL.exe %PREFIX%\bin -copy build\windows-msvc-release\bin\MODL_Coclustering.exe %PREFIX%\bin diff --git a/packaging/conda/install-khiops.sh b/packaging/conda/install-khiops.sh deleted file mode 100644 index 210bce9b8..000000000 --- a/packaging/conda/install-khiops.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Echo all output -set -x - -# The binary location depends on the preset name used at the configure step (Cf. build.sh) -if [ "$(uname)" == "Darwin" ] -then - BUILD_DIR="macos-clang-release" -else - BUILD_DIR="linux-gcc-release" -fi - -# Copy the MODL binaries to the anaconda PREFIX path -mkdir -p $PREFIX/bin -cp build/$BUILD_DIR/bin/MODL $PREFIX/bin -cp build/$BUILD_DIR/bin/MODL_Coclustering $PREFIX/bin diff --git a/packaging/conda/meta.yaml b/packaging/conda/meta.yaml index 2cda03f7e..662716986 100644 --- a/packaging/conda/meta.yaml +++ b/packaging/conda/meta.yaml @@ -1,47 +1,71 @@ -{% set version_match = load_file_regex(load_file="src/Learning/KWUtils/KWKhiopsVersion.h", regex_pattern="(\\d+\\.\\d+\\.\\d+)") %} -{% set name = "khiops-bin" %} - package: - name: "{{ name }}" - version: "{{ version_match.groups()[0] }}" + name: khiops-core + version: {{ os.environ.get('KHIOPS_VERSION') }} source: path: ../../ build: number: 0 - include_test: False + script_env: + # Variables for signing the MODL executables in osx-arm64. + {% if "KHIOPS_APPLE_CERTIFICATE_COMMON_NAME" in os.environ %} + - KHIOPS_APPLE_CERTIFICATE_COMMON_NAME # [osx] + # Only available when "KHIOPS_APPLE_CERTIFICATE_BASE64" is defined in the environment. + {% if "KHIOPS_APPLE_CERTIFICATE_BASE64" in os.environ %} + - KHIOPS_APPLE_CERTIFICATE_BASE64 # [osx] + - KHIOPS_APPLE_CERTIFICATE_PASSWORD # [osx] + - KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD # [osx] + {% endif %} + {% endif %} + # Binary relocation of MODL and MODL_Coclustering is done in build.sh script + # This is to be able to sign it, see the script for more details. + # Only done when "KHIOPS_APPLE_CERTIFICATE_COMMON_NAME is defined in the environment. + {% if "KHIOPS_APPLE_CERTIFICATE_COMMON_NAME" in os.environ %} + binary_relocation: false # [osx] + detect_binary_files_with_prefix: false # [osx] + {% endif %} +# Note on version pinning: +# OSX: +# - mpich=3.4.3 because 4.* is still unstable +# - requires conda-forge +# Linux: +# - mpich=4.0.3 because of bugs of the 3.* series +# - requires conda-forge requirements: build: - - mpich # [not win] - - msmpi # [win] + - mpich 4.0.3 # [linux] + - mpich-mpicc 4.0.3 # [linux] + - mpich-mpicxx 4.0.3 # [linux] + - mpich 3.4.3 # [osx] + - mpich-mpicc 3.4.3 # [osx] + - mpich-mpicxx 3.4.3 # [osx] + - msmpi # [win] - cmake - - ninja # [win] + - ninja - {{ compiler('cxx') }} host: - - mpich # [not win] - - msmpi # [win] + - mpich 4.0.3 # [linux] + - mpich-mpicxx 4.0.3 # [linux] + - mpich 3.4.3 # [osx] + - mpich-mpicxx 3.4.3 # [osx] + - msmpi # [win] run: - - mpich # [not win] - - msmpi # [win] + - mpich 4.0.3 # [linux] + - mpich 3.4.3 # [osx] + - msmpi # [win] outputs: - - name: "{{ name }}" - build: - activate_in_script: True - script: install-khiops.sh # [not win] - script: install-khiops.bat # [win] - -test: - commands: - - MODL -v - - MODL_Coclustering -v + - name: khiops-core + test: + commands: + - MODL -v + - MODL_Coclustering -v about: - home: www.khiops.org + home: https://khiops.org license: BSD+3-clause - license_file: LICENSE - summary: "Khiops is a data preparation and scoring tool for supervised learning and unsupervised learning" - doc_url: "https://www.khiops.com" - dev_url: "https://gitlab.com/khiops/khiops" + summary: Khiops is an AutoML suite for supervised and unsupervised learning + doc_url: https://khiops.org + dev_url: https://github.com/khiopsml/khiops