Skip to content

Commit

Permalink
2021-08 spack update (#839)
Browse files Browse the repository at this point in the history
* update uberenv

* update spack

* add travis config

* config loc changes

* path fix for docker test

* use older spack since all tests with old python fell of dependency cliff

* update configs

* adj, back to new spack

* update configs

* changes

* changes

* keep fighting for python 2

* use stable git fork branch
  • Loading branch information
cyrush authored Sep 13, 2021
1 parent f18da11 commit 316779f
Show file tree
Hide file tree
Showing 34 changed files with 23 additions and 223 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@
[submodule "scripts/uberenv"]
path = scripts/uberenv
url = https://github.com/LLNL/uberenv.git
[submodule "scripts/uberenv_configs/spack_configs"]
path = scripts/uberenv_configs/spack_configs
url = https://github.com/Alpine-DAV/spack_configs.git
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ script:
- while sleep 540; do echo "=====[ $SECONDS seconds still running ]====="; done &
# build deps using uberenv
# use -k to avoid suprises related to certs
- python scripts/uberenv/uberenv.py -k --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv_configs/spack_configs/travis/ --mirror=https://www.ascent-dav.org/mirror/conduit/latest/ || travis_terminate 1;
- python scripts/uberenv/uberenv.py -k --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv_configs/spack_configs/configs/travis/ || travis_terminate 1;
# todo:
#- export SPACK_PYTHON_BIN_DIR=`ls -d ${TRAVIS_BUILD_DIR}/uberenv_libs/spack/opt/spack/*/*/python*/bin`
#- pip install cpp-coveralls
Expand Down
3 changes: 1 addition & 2 deletions .uberenv_config.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
"spack_configs_path": "scripts/uberenv_configs/spack_configs",
"spack_packages_path": "scripts/uberenv_configs/packages",
"spack_url": "https://github.com/alpine-DAV/spack",
"spack_branch": "conduit/develop",
"mirror_url": "https://www.ascent-dav.org/mirror/conduit/latest/",
"spack_branch": "ascent/develop",
"spack_activate" : {"py-numpy" : ["+python"],
"py-sphinx": ["+python","+doc"],
"py-mpi4py" : ["+python", "+mpi"],
Expand Down
223 changes: 10 additions & 213 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,31 +77,15 @@ stages:
CMAKE_BUILD_TYPE: Debug
ENABLE_COVERAGE: ON
ENABLE_MPI: ON
ENABLE_DOCS: ON
ENABLE_DOCS: OFF
ENABLE_RELAY_WEBSERVER: ON
ENABLE_SILO: ON
ENABLE_ADIOS: OFF
ENABLE_PYTHON: ON
ENABLE_PARMETIS: ON
PYTHON_VERSION: 2.7.18
ENABLE_PYTHON2: ON
PYTHON2_CONSTRAINTS: " ^[email protected] ^[email protected] "
CMAKE_VERSION: 3.14.2
BLT_CXX_STD: c++11

shared_py36:
BUILD_SHARED_LIBS: ON
CMAKE_BUILD_TYPE: Debug
ENABLE_COVERAGE: OFF
ENABLE_MPI: ON
ENABLE_RELAY_WEBSERVER: ON
ENABLE_DOCS: ON
ENABLE_SILO: ON
ENABLE_ADIOS: OFF
ENABLE_PARMETIS: ON
ENABLE_PYTHON: ON
ENABLE_PYTHON2: OFF
PYTHON_VERSION: 3.6.3
PYTHON2_CONSTRAINTS: " ^[email protected] ^[email protected]"
PYTHON_VERSION: 2.7.18
CMAKE_VERSION: 3.14.2
BLT_CXX_STD: c++11

Expand All @@ -126,15 +110,15 @@ stages:
CMAKE_BUILD_TYPE: Debug
ENABLE_COVERAGE: OFF
ENABLE_MPI: ON
ENABLE_DOCS: ON
ENABLE_DOCS: OFF
ENABLE_RELAY_WEBSERVER: ON
ENABLE_SILO: ON
ENABLE_ADIOS: OFF
ENABLE_PARMETIS: ON
ENABLE_PYTHON: ON
ENABLE_PYTHON2: ON
PYTHON2_CONSTRAINTS: " ^[email protected] ^[email protected]"
PYTHON_VERSION: 2.7.18
PYTHON2_CONSTRAINTS: " ^[email protected] ^[email protected] "
CMAKE_VERSION: 3.14.2
BLT_CXX_STD: c++11

Expand Down Expand Up @@ -224,15 +208,14 @@ stages:
# package ver selections and extra tweaks
export SPACK_SPEC="${SPACK_SPEC} ^cmake@${CMAKE_VERSION}"
if [ $ENABLE_PYTHON = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC} ^python@${PYTHON_VERSION}"; fi
# fix constraint issue for python 2
if [ $ENABLE_PYTHON2 = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC} ${PYTHON2_CONSTRAINTS}"; fi
if [ $ENABLE_SILO = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC} ^silo~mpi"; fi
# turn off hdf5 mpi support
export SPACK_SPEC="${SPACK_SPEC} ^hdf5~mpi"
# show final spec
echo $SPACK_SPEC
# run uber to build tpls
python scripts/uberenv/uberenv.py -k --pull --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv_configs/spack_configs/ci/ubuntu_16/ --mirror=https://www.ascent-dav.org/mirror/conduit/latest/
python scripts/uberenv/uberenv.py -k --pull --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv_configs/spack_configs/configs/alpinedav/ubuntu_16/
displayName: 'Spack Build Tpls'
- script: |
Expand Down Expand Up @@ -404,7 +387,7 @@ stages:
###############################################################################
- job: Ubuntu_18
pool:
vmImage: 'ubuntu-16.04'
vmImage: 'ubuntu-latest'
container: ${{ variables.ubuntu18_tag }}
timeoutInMinutes: 0
variables:
Expand All @@ -423,8 +406,7 @@ stages:
ENABLE_SILO: ON
ENABLE_ADIOS: OFF
ENABLE_PYTHON: ON
PYTHON_VERSION: 3.6.3
CMAKE_VERSION: 3.14.2
CMAKE_VERSION: 3.21.1
BLT_CXX_STD: c++11
steps:
- checkout: self
Expand Down Expand Up @@ -493,16 +475,13 @@ stages:
if [ $ENABLE_DOCS = 'OFF' ]; then export SPACK_SPEC="${SPACK_SPEC}~doc"; fi
# package ver selections and extra tweaks
export SPACK_SPEC="${SPACK_SPEC} ^cmake@${CMAKE_VERSION}"
if [ $ENABLE_PYTHON = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC} ^python@${PYTHON_VERSION}"; fi
# fix constraint issue for python 2
if [ $ENABLE_PYTHON2 = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC} ${PYTHON2_CONSTRAINTS}"; fi
if [ $ENABLE_SILO = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC} ^silo~mpi"; fi
# turn off hdf5 mpi support
export SPACK_SPEC="${SPACK_SPEC} ^hdf5~mpi"
# show final spec
echo $SPACK_SPEC
# run uber to build tpls
python scripts/uberenv/uberenv.py -k --pull --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv_configs/spack_configs/ci/ubuntu_16/ --mirror=https://www.ascent-dav.org/mirror/conduit/latest/
python scripts/uberenv/uberenv.py -k --pull --spec "${SPACK_SPEC}" --spack-config-dir=scripts/uberenv_configs/spack_configs/configs/alpinedav/ubuntu_18_devel/
displayName: 'Spack Build Tpls'
- script: |
Expand Down Expand Up @@ -703,7 +682,7 @@ stages:
###############################################################################
- job: Docker_Ubuntu
pool:
vmImage: 'ubuntu-16.04'
vmImage: 'ubuntu-latest'
timeoutInMinutes: 0
steps:
- checkout: self
Expand All @@ -718,185 +697,3 @@ stages:
displayName: 'Docker Build'
# ###############################################################################
# # macOS 10.14 testing
# #
# ###############################################################################
# - job: macOS_14
# pool:
# vmImage: 'macOS-10.14'
# timeoutInMinutes: 0
# variables:
# COMPILER_CC: clang
# COMPILER_CXX: clang++
# COMPILER_SPEC: clang
# strategy:
# matrix:
# shared_py36:
# BUILD_SHARED_LIBS: ON
# CMAKE_BUILD_TYPE: Debug
# ENABLE_COVERAGE: OFF
# ENABLE_MPI: ON
# ENABLE_DOCS: ON
# ENABLE_SILO: ON
# ENABLE_ADIOS: OFF
# ENABLE_PYTHON: ON
# PYTHON_VERSION: 3.6.3
# CMAKE_VERSION: 3.9.4
# BLT_CXX_STD: c++11
# steps:
# - checkout: self
# clean: boolean
# submodules: recursive
#
# - script: |
# #################################
# # run uber to build tpls
# #################################
# #
# pwd
# # echo system python details
# which python
# python --version
# # setup spack spec
# export SPACK_SPEC="%${COMPILER_SPEC}"
# # mpi
# if [ $ENABLE_MPI = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC}+mpi"; fi
# if [ $ENABLE_MPI = 'OFF' ]; then export SPACK_SPEC="${SPACK_SPEC}~mpi"; fi
# # silo
# if [ $ENABLE_SILO = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC}+silo"; fi
# if [ $ENABLE_SILO = 'OFF' ]; then export SPACK_SPEC="${SPACK_SPEC}~silo"; fi
# # adios
# if [ $ENABLE_ADIOS = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC}+adios~zfp"; fi
# if [ $ENABLE_ADIOS = 'OFF' ]; then export SPACK_SPEC="${SPACK_SPEC}~adios"; fi
# # python
# if [ $ENABLE_PYTHON = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC}+python"; fi
# if [ $ENABLE_PYTHON = 'OFF' ]; then export SPACK_SPEC="${SPACK_SPEC}~python"; fi
# # docs
# if [ $ENABLE_DOCS = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC}+doc"; fi
# if [ $ENABLE_DOCS = 'OFF' ]; then export SPACK_SPEC="${SPACK_SPEC}~doc"; fi
# # package ver selections and extra tweaks
# export SPACK_SPEC="${SPACK_SPEC} ^cmake@${CMAKE_VERSION}"
# if [ $ENABLE_PYTHON = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC} ^python@${PYTHON_VERSION}"; fi
# if [ $ENABLE_SILO = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC} ^silo~mpi"; fi
# if [ $ENABLE_MPI = 'ON' ]; then export SPACK_SPEC="${SPACK_SPEC} ^mpich"; fi
# # show final spec
# echo $SPACK_SPEC
# # run uber to build tpls
# python scripts/uberenv/uberenv.py -k --pull --spec "${SPACK_SPEC}"
# displayName: 'Spack Build Tpls'
#
# - script: |
# #################################
# # configure
# #################################
# # setup compiler env vars
# export CC=${COMPILER_CC}
# export CXX=${COMPILER_CXX}
# ${CC} --version
# # capture current path
# export ROOT_DIR=`pwd`
# # find spack generated host config file
# export HOST_CONFIG=`ls ${ROOT_DIR}/uberenv_libs/*.cmake`
# echo $HOST_CONFIG
# # find spack installed cmake
# export CMAKE_BIN_DIR=`ls -d ${ROOT_DIR}/uberenv_libs/spack/opt/spack/*/*/cmake*/bin`
# export PATH=${CMAKE_BIN_DIR}:$PATH
# echo $PATH
# which cmake
# cmake --version
# # prepare build dir
# mkdir build
# cd build
# # setup cmake options
# export CMAKE_OPTS="-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
# export CMAKE_OPTS="-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
# export CMAKE_OPTS="${CMAKE_OPTS} -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}"
# export CMAKE_OPTS="${CMAKE_OPTS} -DENABLE_COVERAGE=${ENABLE_COVERAGE}"
# export CMAKE_OPTS="${CMAKE_OPTS} -DBLT_CXX_STD=${BLT_CXX_STD}"
# export CMAKE_OPTS="${CMAKE_OPTS} -DCMAKE_INSTALL_PREFIX=../install"
# # configure
# cmake ${CMAKE_OPTS} -C ${HOST_CONFIG} ../src
# displayName: 'Configure with CMake'
#
# - script: |
# #################################
# # build
# #################################
# # build
# cd build
# make VERBOSE=1
# displayName: 'Build'
#
# - script: |
# #################################
# # test
# #################################
# # find spack installed cmake
# export ROOT_DIR=`pwd`
# export CMAKE_BIN_DIR=`ls -d ${ROOT_DIR}/uberenv_libs/spack/opt/spack/*/*/cmake*/bin`
# export PATH=${CMAKE_BIN_DIR}:$PATH
# which ctest
# cd build
# # run ctest
# ctest -T test --output-on-failure -V
# displayName: 'Run Unit Tests'
#
# - task: PublishTestResults@2
# inputs:
# testResultsFormat: 'cTest'
# testResultsFiles: '**/Test.xml'
# displayName: 'Unit Test Results'
# condition: always()
#
# - task: CopyFiles@2
# inputs:
# contents: 'build/tests/_output/**'
# targetFolder: $(Build.ArtifactStagingDirectory)
# condition: always()
#
# - task: PublishBuildArtifacts@1
# inputs:
# pathToPublish: $(Build.ArtifactStagingDirectory)
# artifactName: Conduit_Test_Outputs
# condition: always()
#
# - script: |
# #################################
# # install
# #################################
# cd build
# make install
# displayName: 'Install'
#
# - script: |
# ###########################
# # using with cmake example
# ###########################
# pwd
# ls -l
# # find spack installed cmake
# export ROOT_DIR=`pwd`
# export CMAKE_BIN_DIR=`ls -d ${ROOT_DIR}/uberenv_libs/spack/opt/spack/*/*/cmake*/bin`
# export PATH=${CMAKE_BIN_DIR}:$PATH
# echo $PATH
# which cmake
# cd install/examples/conduit/using-with-cmake
# mkdir _test_build
# cd _test_build
# cmake ../
# make VERBOSE=1
# ./conduit_example
# displayName: 'Test vs Install (using-with-cmake)'
#
# - script: |
# ###########################
# # using with make example
# ###########################
# cat install/share/conduit/conduit_config.mk
# pwd
# ls -l
# cd install/examples/conduit/using-with-make
# make
# ./conduit_example
# displayName: 'Test vs Install (using-with-make)'
2 changes: 1 addition & 1 deletion scripts/uberenv
1 change: 1 addition & 0 deletions scripts/uberenv_configs/spack_configs
Submodule spack_configs added at 16abef
10 changes: 5 additions & 5 deletions src/docs/sphinx/building.rst
Original file line number Diff line number Diff line change
Expand Up @@ -207,18 +207,18 @@ Uberenv Options for Building Third Party Dependencies

``uberenv.py`` has a few options that allow you to control how dependencies are built:

==================== ============================================== ================================================
==================== ============================================== ==============================================================
Option Description Default
==================== ============================================== ================================================
==================== ============================================== ==============================================================
--prefix Destination directory ``uberenv_libs``
--spec Spack spec linux: **%gcc**
osx: **%clang**
--spack-config-dir Folder with Spack settings files linux: (empty)
osx: ``scripts/uberenv/spack_configs/darwin/``
osx: ``scripts/uberenv_configs/spack_configs/config/darwin/``
-k Ignore SSL Errors **False**
--install Fully install conduit, not just dependencies **False**
--run_tests Invoke tests during build and against install **False**
==================== ============================================== ================================================
==================== ============================================== ==============================================================

The ``-k`` option exists for sites where SSL certificate interception undermines fetching
from github and https hosted source tarballs. When enabled, ``uberenv.py`` clones spack using:
Expand All @@ -243,7 +243,7 @@ Default invocation on OSX:
python scripts/uberenv/uberenv.py --prefix uberenv_libs \
--spec %clang \
--spack-config-dir scripts/uberenv/spack_configs/darwin/
--spack-config-dir scripts/uberenv_configs/spack_configs/configs/darwin/
The uberenv `--install` installs conduit\@develop (not just the development dependencies):
Expand Down
2 changes: 1 addition & 1 deletion src/examples/docker/ubuntu/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ RUN tar -xf conduit.docker.src.tar
RUN cd conduit && python scripts/uberenv/uberenv.py \
-k \
--spec "%gcc+mpi~doc~silo" \
--spack-config-dir=scripts/uberenv_configs/spack_configs/docker/ubuntu/
--spack-config-dir=scripts/uberenv_configs/spack_configs/configs/docker/ubuntu/

# configure a debug build with cmake
RUN cd conduit && mkdir build-debug
Expand Down

0 comments on commit 316779f

Please sign in to comment.