Merge pull request #154 from KhiopsML/dev

Release 10.2.0

.cmake-format.py

@@ -1,6 +1,7 @@
 with section("format"):
-  line_width = 120
-  tab_size = 2
+    line_width = 120
+    tab_size = 2
+    line_ending = "auto"
 
 with section("markup"):
-  bullet_char = '-'
+    bullet_char = "-"

.github/ISSUE_TEMPLATE/generic-issue-template.md

@@ -0,0 +1,23 @@
+---
+name: Generic issue template
+about: Should be used for all issues of this repository.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+#### Description
+_What, Why_
+
+#### Questions/Ideas
+- _A potential solution_
+  - _A potential solution detail_
+- _A potential problem_
+- _Something I still don't understand_
+
+### Context
+- Khiops version
+- Log file (use `khiops -e log.txt`)
+- Scenario file (use `khiops -o scenario._kh`)
+- OS description (use `khiops -s`)

.github/actions/build-khiops/action.yml

@@ -1,35 +1,30 @@
 ---
-name: "Build Khiops"
-
-on:
-  workflow_call:
-    inputs:
-      preset-name:
-        description: "The CMake preset name"
-        required: true
-      override-flags:
-        description: "List of overriding flags for the preset defaults"
-
+name: Build Khiops
+description: Action to execute CMake configure/build presets
+inputs:
+  preset-name:
+    description: The CMake preset name
+    required: true
+  override-flags:
+    description: "List of overriding flags for the preset defaults (ex: '-DUSE_MPI=OFF\
+      \ -DBUILD_JARS=ON')"
+  targets:
+    description: "List of targets to be built (ex: 'MODL norm_test')"
 runs:
-  using: "composite"
+  using: composite
   steps:
     - name: Setup MPI
       uses: mpi4py/setup-mpi@v1
-
+    - name: Setup Ninja
+      uses: ashutoshvarma/[email protected]
     - name: Load Visual C++ Environment Variables (Windows only)
       if: ${{ runner.os == 'Windows' }}
       shell: cmd
       run: |
         call "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Auxiliary\\Build\\vcvars64.bat"
         set >> %GITHUB_ENV%
-
-    - name: Install Ninja (Linux only)
-      if: ${{ runner.os == 'Linux' }}
-      shell: bash
-      run: sudo apt-get install ninja-build
-
     - name: Compile Khiops with CMake
       shell: bash
-      run: |
+      run: |-
         cmake --preset ${{ inputs.preset-name }} ${{ inputs.override-flags }}
-        cmake --build --preset build-${{ inputs.preset-name }}
+        cmake --build --parallel --preset ${{ inputs.preset-name }} --target ${{ inputs.targets }}

.github/actions/test-khiops-install/action.yml

@@ -0,0 +1,72 @@
+---
+name: Test Khiops Installation
+description: Tests a Khiops installation on the Iris dataset
+runs:
+  using: composite
+  steps:
+    # Python install: we don't use the setup-python action because of the following error:
+    # python3: /lib64/libc.so.6: version `GLIBC_2.34' not found (required by python3)
+    - name: Python setup for Linux
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        if [ -d "/etc/apt" ]
+        then
+          apt-get install -y python3 > /dev/null
+        else
+          yum install -y python3.11
+        fi
+        echo "PYTHON=python3" >> "$GITHUB_ENV"
+    - name: Python setup for Windows or macOS
+      if: runner.os == 'Windows' || runner.os == 'macOS'
+      shell: bash
+      run: echo "PYTHON=python" >> "$GITHUB_ENV"
+    - name: Set the name for the khiops laucher script
+      shell: bash
+      run: |
+        if [[ $RUNNER_OS == "Windows" ]]
+        then
+          # Put the full path in windows since bash doesn't find ".cmd" files
+          # in the path even if they are there
+          modl_path=$(which MODL)
+          khiops_bin_dir=$(dirname "$modl_path")
+          KHIOPS_SCRIPT="$khiops_bin_dir/khiops.cmd"
+          KHIOPS_CC_SCRIPT="$khiops_bin_dir/khiops_coclustering.cmd"
+          echo "KHIOPS_SCRIPT=$KHIOPS_SCRIPT" >> "$GITHUB_ENV"
+          echo "KHIOPS_CC_SCRIPT=$KHIOPS_CC_SCRIPT" >> "$GITHUB_ENV"
+        else
+          echo "KHIOPS_SCRIPT=khiops" >> "$GITHUB_ENV"
+          echo "KHIOPS_CC_SCRIPT=khiops_coclustering" >> "$GITHUB_ENV"
+        fi
+    - name: Check Khiops installation
+      shell: bash
+      run: |
+        "$KHIOPS_SCRIPT" -v
+        "$KHIOPS_CC_SCRIPT" -v
+    - name: Run Khiops tests
+      shell: bash
+      run: |
+        cd test/LearningTest/TestKhiops/Standard/Iris/
+        "$KHIOPS_SCRIPT" -b -i test.prm -e results/err.txt
+        cd -
+    - name: Run Khiops Coclustering tests
+      shell: bash
+      run: |
+        cd test/LearningTest/TestCoclustering/Standard/Iris/
+        "$KHIOPS_CC_SCRIPT" -b -i test.prm -e results/err.txt
+        cd -
+    - name: Check the test results
+      shell: bash
+      run: |-
+        touch test/LearningTest/TestCoclustering/Standard/Iris/results/time.log
+        touch test/LearningTest/TestKhiops/Standard/Iris/results/time.log
+        echo "Check test results"
+        cd test/LearningTest/cmd/python/
+        $PYTHON test_khiops.py Khiops nul Standard
+        $PYTHON test_khiops.py Coclustering nul Standard
+        $PYTHON apply_command.py errors ../../TestKhiops/Standard/ Iris | tee /tmp/khiops-log.txt
+        $PYTHON apply_command.py errors ../../TestCoclustering/Standard/ Iris | tee /tmp/coclustering-log.txt
+        if (grep -q error /tmp/khiops-log.txt || grep -q error /tmp/coclustering-log.txt); then
+          echo "::error::Errors in Khiops run"
+          false
+        fi

.github/actions/test-kni/action.yml

@@ -0,0 +1,111 @@
+---
+name: Test Khiops Native Interface
+description: Action to test KNI package
+runs:
+  using: composite
+  steps:
+    - name: Check ldconfig
+      shell: bash
+      if: runner.os == 'Linux'
+      run: |
+        if [ $(ldconfig -p | grep -c KhiopsNativeInterface) -eq 0 ] 
+        then
+          echo "::error::KNI is not reachable with ld, check post installation"
+          false
+        fi
+    - name: Checkout sources
+      uses: actions/checkout@v3
+      with:
+        path: khiops
+    - name: Checkout KhiopsNativeInterface-tutorial
+      uses: actions/checkout@v3
+      with:
+        repository: KhiopsML/KhiopsNativeInterface-tutorial
+        path: tutorial
+    - name: Compile C examples (Linux)
+      shell: bash
+      if: runner.os == 'Linux'
+      run: |
+        cd tutorial
+        bash $GITHUB_WORKSPACE/packaging/common/KNI/build-c-linux.sh
+    - name: Compile C examples (Windows)
+      if: runner.os == 'Windows'
+      shell: cmd
+      run: |
+        call "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\VC\\Auxiliary\\Build\\vcvars64.bat"
+        set KNI_HOME=%RUNNER_TEMP%\\KNI
+        cd tutorial
+        call %GITHUB_WORKSPACE%/packaging/common/KNI/build-c--windows.cmd
+    - name: Test mono-table example
+      shell: bash
+      run: |
+        ./tutorial/KNIRecodeFile tutorial/data/ModelingIris.kdic SNB_Iris tutorial/data/Iris.txt R_Iris.txt
+        if [ ! -f R_Iris.txt ] ;
+        then
+          echo "::error::R_Iris.txt is missing"
+          false
+        fi
+        if diff -q R_Iris.txt khiops/packaging/common/KNI/results.ref/R_Iris.txt > /dev/null ; 
+        then
+          echo "Mono table recoding is OK"
+        else
+        echo "::error::Recoding issue with Iris.txt"
+          false
+        fi
+    - name: Test multi-tables example
+      shell: bash
+      run: |-
+        ./tutorial/KNIRecodeMTFiles -d tutorial/data/ModelingSpliceJunction.kdic SNB_SpliceJunction \
+            -i tutorial/data/SpliceJunction.txt 1 -s DNA  tutorial/data/SpliceJunctionDNA.txt 1 -o R_SpliceJunction.txt
+        if [ ! -f R_SpliceJunction.txt ] ;
+        then
+          echo "::error::R_SpliceJunction.txt is missing"
+          false
+        fi
+        if diff -q R_SpliceJunction.txt khiops/packaging/common/KNI/results.ref/R_SpliceJunction.txt > /dev/null ; 
+        then
+          echo "Multi table recoding is OK"
+        else
+          echo "::error::Recoding issue with SpliceJunction.txt"
+          false
+        fi
+    - name: setup jna
+      uses: nvoxland/jar-download-action@v1
+      with:
+        groupId: net.java.dev.jna
+        artifactId: jna
+        version: 5.13.0
+        outputDirectory: ./tutorial/java/
+    - name: Compile java example
+      shell: bash
+      run: |
+        cd tutorial
+        bash  $GITHUB_WORKSPACE/packaging/common/KNI/build-java.sh
+        cd  $GITHUB_WORKSPACE
+    - name: Run java example (Linux)
+      shell: bash
+      if: runner.os == 'Linux'
+      run: |
+        cd tutorial
+        java -cp java/kni.jar:java/jna-5.13.0.jar KNIRecodeFile data/ModelingIris.kdic SNB_Iris data/Iris.txt R_Iris_java.txt 2>&1
+    - name: Test java example (Windows)
+      shell: bash
+      if: runner.os == 'Windows'
+      run: |
+        cd tutorial
+        java -cp java/kni.jar;java/jna-5.13.0.jar KNIRecodeFile data/ModelingIris.kdic SNB_Iris data/Iris.txt R_Iris_java.txt 2>&1
+    - name: Check java results
+      shell: bash
+      run: |-
+        if [ ! -f tutorial/R_Iris_java.txt ] ;
+        then
+          echo "::error::R_Iris_java.txt is missing"
+          false
+        fi
+        if diff -q tutorial/R_Iris_java.txt  $GITHUB_WORKSPACE/packaging/common/KNI/results.ref/R_Iris.txt > /dev/null ; 
+        then
+          echo "Mono table recoding is OK"
+        else
+        echo "::error::Recoding issue with java"
+          false
+        fi

.github/workflows/build-linux-pack-containers.yml

@@ -0,0 +1,43 @@
+---
+name: Build Linux containers for packaging
+on:
+  workflow_dispatch:
+jobs:
+  build-packaging-container:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - debian10
+          - debian11
+          - rocky8
+          - rocky9
+          - ubuntu18.04
+          - ubuntu20.04
+          - ubuntu22.04
+    permissions:
+      packages: write  # Allows writing in the container registry
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Github Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Downcase GitHub repository
+        run: echo "GH_REPOSITORY=${GITHUB_REPOSITORY,,}" >> "${GITHUB_ENV}"
+      - name: Build the image and push it to the registry
+        id: docker-build
+        uses: docker/build-push-action@v2
+        with:
+          file: ./packaging/dockerfiles/Dockerfile.${{ matrix.os }}
+          tags: ghcr.io/${{ env.GH_REPOSITORY }}/khiopsdev-${{ matrix.os }}:latest
+          push: true
+      - name: Display the image digest
+        run: echo ${{ steps.docker-build.outputs.digest }}