Skip to content

Commit

Permalink
Use vcpkg for dependency management on all platforms (#770)
Browse files Browse the repository at this point in the history
- Change centos7 image management. Image is now
antaresrte/xpansion-centos7 and tags follow semver schema
- Migrate depedency management to vcpkg
- Use action to setup cmake as much as possible
- jsoncpp staticly linked to ease deploiment

Comments and explanations:

- _git fetch --unshallow_ is necessary because action checkout is
shallow but vcpkg requires a "full" clone.
- tbb is still manage manualy
- jsoncpp is still link dynamicaly for MSVC due to difficulty to manage
static/synamique with MSVC
  • Loading branch information
JasonMarechal25 committed Apr 23, 2024
1 parent 14f192a commit b8e0396
Show file tree
Hide file tree
Showing 24 changed files with 335 additions and 237 deletions.
84 changes: 59 additions & 25 deletions .github/workflows/build_centos7.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,13 @@ jobs:
if: steps.changed-files.outputs.any_changed == 'true'
uses: elgohr/Publish-Docker-Github-Action@main
with:
name: antaresrte/rte-antares
name: antaresrte/xpansion-centos7
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
workdir: docker
dockerfile: centos7-system-deps
cache: false
tags: centos7-system-deps
tags: latest,1.0.0

versions:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -72,13 +72,13 @@ jobs:
build:
runs-on: ubuntu-latest
needs: [ docker_publish, versions ]
container: 'antaresrte/rte-antares:centos7-system-deps'
container: 'antaresrte/xpansion-centos7'
strategy:
matrix:
xprs: [
# { value: XPRESS-ON, ref: 8.13a },
{ value: XPRESS-ON, ref: 9.2.5 },
# { value: XPRESS-OFF }
{ value: XPRESS-ON, ref: 9.2.5 },
#{ value: XPRESS-OFF }
]
env:
XPRESSDIR: ${{ github.workspace }}/xpress
Expand All @@ -87,8 +87,12 @@ jobs:
XPRESSDIR_CONTAINER: ${GITHUB_WORKSPACE}/xpress
XPRESS_CONTAINER: ${GITHUB_WORKSPACE}/xpress/bin
XPRS_LIB_Path_CONTAINER: ${GITHUB_WORKSPACE}/xpress/lib
VCPKG_ROOT: ${{ github.workspace }}/vcpkg

steps:
- run: |
source /opt/rh/devtoolset-10/enable
echo $PATH >> $GITHUB_PATH
- name: Get release
if: github.event_name == 'release' && github.event.action == 'created'
id: get_release
Expand All @@ -98,7 +102,22 @@ jobs:
with:
submodules: true

- uses: ./.github/workflows/compile-gtest
- run: |
yum install -y nodejs
- name: Setup cmake
uses: jwlawson/[email protected]
with:
cmake-version: '3.28.x'

#Lukka's action doesn't work on runner.
#jwlawson's action doesn't work in ACT
#I left this here to quicly switch between one or the other
# - uses: lukka/get-cmake@latest
# with:
# useLocalCache: false
# useCloudCache: true

- name: Checkout xpressmp linux
uses: actions/checkout@v3 #keep v3
with:
Expand All @@ -109,59 +128,74 @@ jobs:
ref: ${{matrix.xprs.ref}}
if: matrix.xprs.value == 'XPRESS-ON'

- name: Restore vcpkg and its artifacts.
uses: actions/cache@v3
with:
# The first path is the location of vcpkg (it contains the vcpkg executable and data files).
# The other paths starting with '!' are exclusions: they contain termporary files generated during the build of the installed packages.
path: |
${{ env.VCPKG_ROOT }}
!${{ env.VCPKG_ROOT }}/buildtrees
!${{ env.VCPKG_ROOT }}/packages
!${{ env.VCPKG_ROOT }}/downloads
# The key is composed in a way that it gets properly invalidated: this must happen whenever vcpkg's Git commit id changes, or the list of packages changes. In this case a cache miss must happen and a new entry with a new key with be pushed to GitHub the cache service.
# The key includes: hash of the vcpkg.json file, the hash of the vcpkg Git commit id, and the used vcpkg's triplet. The vcpkg's commit id would suffice, but computing an hash out it does not harm.
# Note: given a key, the cache content is immutable. If a cache entry has been created improperly, in order the recreate the right content the key must be changed as well, and it must be brand new (i.e. not existing already).
key: |
${{ hashFiles( 'vcpkg.json' ) }}-${{ hashFiles( '.git/modules/vcpkg/HEAD' )}}-${{ matrix.triplet }}
- name: Download pre-compiled librairies
uses: ./.github/workflows/download-extract-precompiled-libraries-tgz
with:
antares-deps-version: ${{needs.versions.outputs.antares-deps-version}}
antares-version: ${{needs.versions.outputs.antares-version}}
os: centos7
os-full-name: CentOS-7.9.2009
#variant: -ortools-xpress

- name: Compile Boost
uses: ./.github/workflows/compile-boost
with:
prefix: "../rte-antares-deps-Release/"

- name: Compile tbb
uses: ./.github/workflows/compile-tbb
with:
cmake: 'cmake3'

- name: Install dependencies
run: |
pip3 install --upgrade pip
pip3 install wheel #Does not work in requirements
pip3 install -r requirements-tests.txt
pip3 install -r requirements-ui.txt
- name: vcpkg install
run: |
pushd vcpkg
git fetch --unshallow
./bootstrap-vcpkg.sh
popd
vcpkg/vcpkg install
- name: Configure
run: |
[[ ${{ matrix.xprs.value }} == "XPRESS-ON" ]] && XPRESS_VALUE="ON" || XPRESS_VALUE="OFF"
source /opt/rh/devtoolset-10/enable
export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib:$LD_LIBRARY_PATH
export PATH=/usr/lib64/openmpi/bin:$PATH
cmake3 -B _build -S . \
cmake -B _build -S . \
-DDEPS_INSTALL_DIR=rte-antares-deps-Release \
-DBUILD_TESTING=ON \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=_install \
-DBUILD_UI=ON \
-DXPRESS=${{ env.XPRESS_VALUE }} \
-DXPRESS_ROOT=${{ env.XPRESSDIR }} \
-DALLOW_RUN_AS_ROOT=ON
-DALLOW_RUN_AS_ROOT=ON \
-DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake
- name: Build
run: |
source /opt/rh/devtoolset-10/enable
export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib:$LD_LIBRARY_PATH
export PATH=/usr/lib64/openmpi/bin:$PATH
cmake3 --build _build --config Release -j8 --target install
cmake --build _build --config Release -j$(nproc)
- name: Running unit tests
timeout-minutes: 120
shell: bash
run: |
source /etc/profile.d/modules.sh
module load mpi
export PATH=${GITHUB_WORKSPACE}/vcpkg_installed/x64-linux/tools/openmpi/bin/:$PATH
export LD_LIBRARY_PATH=LD_LIBRARY_PATH:${{ env.XPRS_LIB_Path_CONTAINER }}
export XPRESS=${{ env.XPRESS_CONTAINER }}
cd _build
ctest3 -C Release --output-on-failure -L "unit|benders|lpnamer|medium"
ctest -C Release --output-on-failure -L "unit|benders|lpnamer|medium"
67 changes: 34 additions & 33 deletions .github/workflows/build_oracle8.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,10 @@ jobs:
container: 'oraclelinux:8'
strategy:
matrix:
xprs: [ #{ value: XPRESS-ON, ref: 8.13a },
{ value: XPRESS-ON, ref: 9.2.5 },
# { value: XPRESS-OFF }
xprs: [
# { value: XPRESS-ON, ref: 8.13a },
{ value: XPRESS-ON, ref: 9.2.5 },
# { value: XPRESS-OFF }
]
needs: [ versions ]
env:
Expand All @@ -63,18 +64,26 @@ jobs:
XPRS_LIB_Path_CONTAINER: ${GITHUB_WORKSPACE}/xpress/lib

steps:

- name: Install System
run: |
dnf install -y epel-release git cmake wget rpm-build redhat-lsb-core openmpi-devel
dnf install -y unzip libuuid-devel boost-test boost-devel gcc-toolset-10-toolchain zlib-devel python3-devel
dnf install -y epel-release git wget rpm-build redhat-lsb-core openmpi-devel
dnf install -y unzip libuuid-devel gcc-toolset-10-toolchain python3-devel zlib-devel
#make gcc &co available system wide and "action wide"
source /opt/rh/gcc-toolset-10/enable
echo $PATH >> $GITHUB_PATH
- run: |
dnf module install -y nodejs:20/common
- uses: lukka/get-cmake@latest
with:
useLocalCache: false
useCloudCache: true

- uses: actions/checkout@v4
with:
submodules: true

- uses: ./.github/workflows/compile-gtest

- name: Checkout xpressmp linux
uses: actions/checkout@v4
with:
Expand All @@ -89,9 +98,8 @@ jobs:
run: |
dnf update -y
dnf install -y python3 python3-pip
- run:
echo ${{needs.versions.outputs.antares-deps-version}}
pip3 install wheel #Too late to install in requirements.txt
pip3 install -r requirements-tests.txt
- name: Download pre-compiled librairies
uses: ./.github/workflows/download-extract-precompiled-libraries-tgz
Expand All @@ -101,51 +109,44 @@ jobs:
os: Oracle8
os-full-name: OracleServer-8.9

- name: Compile Boost
uses: ./.github/workflows/compile-boost
with:
prefix: "../rte-antares-deps-Release/"
- name: Compile TBB
uses: ./.github/workflows/compile-tbb

- name: Install dependencies
- run: |
mkdir -p ${{ github.workspace }}/vcpkg_cache
- name: vcpkg install
run: |
source /opt/rh/gcc-toolset-10/enable
pip3 install wheel #Too late to install in requirements.txt
pip3 install -r requirements-tests.txt
pushd vcpkg
git fetch --unshallow
./bootstrap-vcpkg.sh
popd
vcpkg/vcpkg install
- name: Configure
run: |
[[ ${{ matrix.xprs.value }} == "XPRESS-ON" ]] && XPRESS_VALUE="ON" || XPRESS_VALUE="OFF"
source /opt/rh/gcc-toolset-10/enable
dnf install jsoncpp-devel
export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib:$LD_LIBRARY_PATH
export PATH=/usr/lib64/openmpi/bin:$PATH
cmake3 -B _build -S . \
cmake -B _build -S . \
-DDEPS_INSTALL_DIR=rte-antares-deps-Release \
-DBUILD_TESTING=ON \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=_install \
-DBUILD_UI=OFF \
-DXPRESS=${{ env.XPRESS_VALUE }} \
-DXPRESS_ROOT=${{ env.XPRESSDIR }} \
-DALLOW_RUN_AS_ROOT=ON
-DALLOW_RUN_AS_ROOT=ON \
-DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake
- name: Build
run: |
source /opt/rh/gcc-toolset-10/enable
export LD_LIBRARY_PATH=/usr/lib64/openmpi/lib:$LD_LIBRARY_PATH
export PATH=/usr/lib64/openmpi/bin:$PATH
cmake --build _build --config Release -j2
cmake --build _build --config Release -j$(nproc)
- name: Running unit tests
timeout-minutes: 120
shell: bash
run: |
source /etc/profile.d/modules.sh
module load mpi
export PATH=${GITHUB_WORKSPACE}/vcpkg_installed/x64-linux/tools/openmpi/bin/:$PATH
export LD_LIBRARY_PATH=LD_LIBRARY_PATH:${{ env.XPRS_LIB_Path_CONTAINER }}
export XPRESS=${{ env.XPRESS_CONTAINER }}
cd _build
ctest3 -C Release --output-on-failure -L "unit|benders|lpnamer|medium"
ctest -C Release --output-on-failure -L "unit|benders|lpnamer|medium"
32 changes: 22 additions & 10 deletions .github/workflows/build_ubuntu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ jobs:
os: [ ubuntu-20.04 ]
xprs: [
#{ value: XPRESS-ON, ref: 8.13a },
{ value: XPRESS-ON, ref: 9.2.5 },
# { value: XPRESS-OFF }
{ value: XPRESS-ON, ref: 9.2.5 },
# { value: XPRESS-OFF }
]
env:
XPRESSDIR: ${{ github.workspace }}/xpress
Expand Down Expand Up @@ -65,7 +65,7 @@ jobs:
- name: Install mandatory system libraries
run: |
sudo apt-get update --fix-missing
sudo apt-get install -y ccache cmake libgtest-dev libjsoncpp-dev libtbb-dev libopenmpi-dev
sudo apt-get install -y ccache libtbb-dev
sudo apt-get install -y g++-10 gcc-10
- name: Update alternatives
Expand All @@ -78,6 +78,11 @@ jobs:
sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30
sudo update-alternatives --set c++ /usr/bin/g++
- uses: lukka/get-cmake@latest
with:
useLocalCache: false
useCloudCache: true

- name: Read antares-solver version
id: antares-version
uses: ./.github/actions/read-json-value
Expand Down Expand Up @@ -107,11 +112,16 @@ jobs:
os: ${{matrix.os}}
os-full-name: Ubuntu-20.04

- name: Compile Boost
uses: ./.github/workflows/compile-boost
with:
prefix: "../rte-antares-deps-Release/"
load-toolset: 'false'
- run: |
mkdir -p ${{ github.workspace }}/vcpkg_cache
- name: vcpkg install
run: |
pushd vcpkg
git fetch --unshallow
./bootstrap-vcpkg.sh
popd
vcpkg/vcpkg install
- name: Configure
shell: bash
Expand All @@ -129,13 +139,15 @@ jobs:
-DCMAKE_INSTALL_PREFIX=_install \
-DBUILD_UI=ON \
-DXPRESS=${{ env.XPRESS_VALUE }} \
-DXPRESS_ROOT=${{ env.XPRESSDIR }}
-DXPRESS_ROOT=${{ env.XPRESSDIR }} \
-DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake
- name: Build
run: |
cmake --build _build --config Release -j8
cmake --build _build --config Release -j$(nproc)
- name: Test
run: |
export PATH=${GITHUB_WORKSPACE}/vcpkg_installed/x64-linux/tools/openmpi/bin/:$PATH
cd _build
ctest -C Release --output-on-failure -L "medium|unit|benders|lpnamer"
8 changes: 5 additions & 3 deletions .github/workflows/build_windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
os: [ windows-latest ]
triplet: [ x64-windows ]
xprs: [ #{ value: XPRESS-ON, ref: 8.13a },
{ value: XPRESS-ON, ref: 9.2.5 },
{ value: XPRESS-ON, ref: 9.2.5 },
#{ value: XPRESS-OFF }
]
env:
Expand Down Expand Up @@ -106,6 +106,7 @@ jobs:
- name: Install deps with VCPKG
run: |
cd vcpkg
git fetch --unshallow
./bootstrap-vcpkg.sh
vcpkg install --triplet ${{matrix.triplet}}
rm -rf buildtrees
Expand Down Expand Up @@ -135,10 +136,11 @@ jobs:
- name: Configure
run: |
$pwd=Get-Location
cmake -B _build -S . -DDEPS_INSTALL_DIR=rte-antares-deps-Release -DCMAKE_PREFIX_PATH="$pwd\rte-antares-${{steps.antares-version.outputs.result}}-installer-64bits" -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE="${{env.VCPKG_ROOT}}/scripts/buildsystems/vcpkg.cmake" -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} -DCMAKE_INSTALL_PREFIX=_install -DBUILD_UI=ON -DXPRESS=${{ env.XPRESS_VALUE }} -DXPRESS_ROOT="${{ env.XPRESSDIR }}"
cmake -B _build -S . -DDEPS_INSTALL_DIR=rte-antares-deps-Release -DCMAKE_PREFIX_PATH="$pwd\rte-antares-${{steps.antares-version.outputs.result}}-installer-64bits" -DBUILD_TESTING=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE="${{env.VCPKG_ROOT}}/scripts/buildsystems/vcpkg.cmake" -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} -DCMAKE_INSTALL_PREFIX=_install -DBUILD_UI=ON -DXPRESS=${{ env.XPRESS_VALUE }} -DXPRESS_ROOT="${{ env.XPRESSDIR }}"
- name: Build
run: |
cmake --build _build --config Release -j2 --target install
cmake --build _build --config Release -j4
- name: Running unit tests
timeout-minutes: 120
shell: cmd
Expand Down
Loading

0 comments on commit b8e0396

Please sign in to comment.