main.yml

name: Test REST VOL with socket

on:
  workflow_dispatch:
  push:
  pull_request:
    branches: [ master ]
    paths-ignore:
      - '.github/CODEOWNERS'
      - 'docs/**'
      - 'COPYING'
      - '**.md'
env:
  ADMIN_PASSWORD: admin
  ADMIN_USERNAME: admin
  USER_NAME: test_user1
  USER_PASSWORD: test
  USER2_NAME: test_user2
  USER2_PASSWORD: test
  HSDS_USERNAME: test_user1
  HSDS_PASSWORD: test
  HSDS_PATH: /home/test_user1/
  HDF5_API_TEST_PATH_PREFIX: /home/test_user1/
  HDF5_VOL_CONNECTOR: REST
  ROOT_DIR: ${{github.workspace}}/hsds/data
  BUCKET_NAME: hsdstest

jobs:
  build_and_test:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-22.04, ubuntu-latest]
        build_system: ["autotools", "cmake"]
        python-version: ["3.10"]
        hdf5-branch: ["hdf5_1_14", "develop"]
        exclude:
          # hdf5 2.0.0+ does not support autotools
          - hdf5-branch: "develop"
            build_system: "autotools"

    runs-on: ${{matrix.os}}
    steps:
      - uses: actions/checkout@v4
        with: 
          repository: HDFGroup/hdf5
          ref: ${{matrix.hdf5-branch}}
          path: ${{github.workspace}}/hdf5
                   
      - name: Get REST VOL dependencies
        run: |
          sudo apt-get update
          sudo apt-get install libcurl4-openssl-dev
          sudo apt-get install libyajl-dev

      - name: Get Autotools Dependencies
        if: matrix.build_system == 'autotools'
        run: |
          sudo apt update
          sudo apt install automake autoconf libtool libtool-bin
            
      - name: Get REST VOL
        uses: actions/checkout@v4
        with:
          path: ${{github.workspace}}/vol-rest

      - name: CMake Configure + Build HDF5
        if: matrix.build_system == 'cmake'
        run: |
          mkdir ${{github.workspace}}/hdf5/build
          cd ./build
          cmake \
          -DHDF5_BUILD_HL_LIB=ON \
          -DBUILD_SHARED_LIBS=ON \
          -DHDF5_TEST_API=ON \
          -DCMAKE_BUILD_TYPE=Release \
          -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/hdf5install \
          ..
          make 
          make install
        shell: bash
        working-directory: ${{github.workspace}}/hdf5

      - name: Autotools Configure + Build HDF5  
        if: matrix.build_system == 'autotools'      
        run: |
          ./autogen.sh
          ./configure --prefix=${{github.workspace}}/hdf5install \
          --enable-hl --disable-threadsafe \
          --enable-build-mode=production --enable-shared
          make 
          make install
        shell: bash
        working-directory: ${{github.workspace}}/hdf5

      - name: CMake Configure REST VOL
        if: matrix.build_system == 'cmake'
        run: |
          mkdir ./build
          cd ./build
          CFLAGS="-D_POSIX_C_SOURCE=200809L" cmake -G "Unix Makefiles" -DHDF5_ROOT=${{github.workspace}}/hdf5install \
          -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/vol-rest/install \
          ..
        shell: bash
        working-directory: ${{github.workspace}}/vol-rest

      - name: Autotools Configure REST VOL
        if: matrix.build_system == 'autotools'
        run: |
          ./autogen.sh
          mkdir ${{github.workspace}}/vol-rest/install
          CFLAGS="-D_POSIX_C_SOURCE=200809L" ./configure \
          --prefix=${{github.workspace}}/vol-rest/install \
          --with-hdf5=${{github.workspace}}/hdf5install
        shell: bash
        working-directory: ${{github.workspace}}/vol-rest

      - name: Build + Install REST VOL (Autotools)
        if: matrix.build_system == 'autotools'
        run: |
          make -j
          make install
        shell: bash
        working-directory: ${{github.workspace}}/vol-rest/

      - name: Build + Install REST VOL (CMake)
        if: matrix.build_system == 'cmake'
        run: |
          make -j
          make install
        shell: bash
        working-directory: ${{github.workspace}}/vol-rest/build/

      - uses: actions/checkout@v4
        with:
          repository: HDFGroup/hsds
          path: ${{github.workspace}}/hsds
          
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          
      - name: Install HSDS dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install pytest
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

      - name: Install HSDS package
        shell: bash
        run: |
          cd ${{github.workspace}}/hsds
          pip install -e .
          
      - name: Run HSDS unit tests
        shell: bash
        run: |
          cd ${{github.workspace}}/hsds
          pytest

      # Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections
      - name: Fix requests version
        run: |
          pip install requests==2.31.0

      - name: Start HSDS
        run: | 
              cd ${{github.workspace}}/hsds
              cp admin/config/groups.default admin/config/groups.txt &&
              cp admin/config/passwd.default admin/config/passwd.txt &&
              cp admin/config/groups.default admin/config/groups.txt &&
              cp admin/config/passwd.default admin/config/passwd.txt
              ./runall.sh --no-docker 1 &
              sleep 10
        working-directory: ${{github.workspace}}/hsds
        
      - name: Test HSDS + Setup Folders
        run: |
              python tests/integ/setup_test.py
        working-directory: ${{github.workspace}}/hsds

      - name: Test REST VOL (Autotools)
        if: matrix.build_system == 'autotools'
        working-directory: ${{github.workspace}}/vol-rest/
        env:
          HDF5_PLUGIN_PATH: ${{github.workspace}}/vol-rest/install/lib
        run: |
               ./test/test_rest_vol

      - name: Test REST VOL (CMake)
        if: matrix.build_system == 'cmake'
        working-directory: ${{github.workspace}}/vol-rest/build/
        env:
          HDF5_PLUGIN_PATH: ${{github.workspace}}/vol-rest/install/lib
        run: |
              ctest -R "test_rest_vol" -VV

      - name: Show HSDS Logs on Fail
        if: ${{failure()}}
        working-directory: ${{github.workspace}}/hsds
        run: |
          cat hs.log

#      TODO: Attribute, dataset, link, and testhdf5 tests currently fail
#      - name: Test REST VOL with API
#        run: |
#              ctest -R "vol-rest" -VV
#        working-directory: ${{github.workspace}}/hdf5/build/

  build_and_test_windows:
    strategy:
      fail-fast: false
      matrix:
        os: [windows-latest]
        python-version: ["3.10"]
        hdf5-branch: ["hdf5_1_14", "develop"] 
    env:
      HSDS_ENDPOINT: http://127.0.0.1:5101
    runs-on: ${{matrix.os}}
    steps:
      - name: Get HDF5
        uses: actions/checkout@v4
        with:
          repository: HDFGroup/hdf5
          ref: ${{matrix.hdf5-branch}}
          path: ${{github.workspace}}/hdf5

      - name: Get REST VOL
        uses: actions/checkout@v3
        with:
          path: ${{github.workspace}}/vol-rest
      
      - name: Get HSDS
        uses: actions/checkout@v3
        with:
          repository: HDFGroup/hsds
          path: ${{github.workspace}}/hsds

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          
      - name: Install HSDS dependencies
        working-directory: ${{github.workspace}}/hsds
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install pytest
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

      - name: Install HSDS package
        working-directory: ${{github.workspace}}/hsds
        shell: bash
        run: |
          pip install -e .

      - name: Start HSDS
        shell: bash
        working-directory: ${{github.workspace}}/hsds
        run: |
          cp admin/config/groups.default admin/config/groups.txt
          cp admin/config/passwd.default admin/config/passwd.txt
          hsds --root_dir data --host localhost --port 5101 --password_file admin/config/passwd.txt --logfile hs.log --loglevel DEBUG --config_dir=admin/config --count=4 &
          sleep 10

      - name: Test HSDS + Setup Folders
        run: |
          python tests/integ/setup_test.py
        working-directory: ${{github.workspace}}/hsds

      - name: Get curl from source
        uses: actions/checkout@v4
        with:
          repository: curl/curl
          path: ${{github.workspace}}/curl
      
      - name: Install curl
        working-directory: ${{github.workspace}}/curl
        run: |
          mkdir build
          mkdir install
          cd build
          cmake -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/curl/install `
          -DCMAKE_BUILD_TYPE=Release -DBUILD_STATIC_LIBS=ON `
          -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_FLAGS="/DCURL_STATICLIB" `
          -DCURL_STATIC_CRT=ON -DBUILD_STATIC_CURL=ON ..  
          cmake --build . --config Release
          cmake --install . --config Release
      
      - name: Get yajl from source
        uses: actions/checkout@v4
        with:
          repository: lloyd/yajl
          path: ${{github.workspace}}/yajl
      
      - name: Install yajl
        working-directory: ${{github.workspace}}/yajl
        run: |
          mkdir build
          mkdir install
          cd build
          cmake -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/yajl/install `
          -DCMAKE_BUILD_TYPE=Release ..
          cmake --build . --config Release
          cmake --install . --config Release
  
      - name: CMake Configure + Build HDF5
        run: |
          mkdir build
          cd build
          cmake `
          -DHDF5_BUILD_HL_LIB=ON `
          -DBUILD_SHARED_LIBS=ON -DHDF5_ENABLE_SZIP_SUPPORT=OFF `
          -DHDF5_TEST_API=ON `
          -DHDF5_ENABLE_Z_LIB_SUPPORT=OFF `
          -DCMAKE_BUILD_TYPE=Release -DHDF5_ENABLE_THREADSAFE=OFF `
          -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/hdf5install `
          ..
          cmake --build . -j --config Release
          cmake --install . --config Release
        working-directory: ${{github.workspace}}/hdf5

      - name: CMake Configure + Build REST VOL
        working-directory: ${{github.workspace}}/vol-rest
        run: |
          mkdir build
          cd build
          cmake -DHDF5_ROOT=${{github.workspace}}/hdf5install `
          -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/vol-rest/install `
          -DCURL_ROOT=${{github.workspace}}/curl/install/ -DCMAKE_C_FLAGS="/DCURL_STATICLIB" `
          -DCURL_USE_STATIC_LIBS=ON `
          -DYAJL_ROOT=${{github.workspace}}/yajl/install/ `
          -DBUILD_STATIC_LIBS=OFF -DBUILD_SHARED_LIBS=ON `
          -DCMAKE_BUILD_TYPE=Release ..
          cmake --build . -j --config Release
          cmake --install . --config Release

      - name: Test REST VOL
        working-directory: ${{github.workspace}}\vol-rest\build
        env: 
          HDF5_PLUGIN_PATH: ${{github.workspace}}\vol-rest\install\bin
        # Manually append lib to PATH here, since env can change between steps on the runner
        run: |
          $env:PATH="$env:PATH;${{github.workspace}}\vol-rest\install\bin;${{github.workspace}}\hdf5install\bin;${{github.workspace}}\yajl\install\lib"
          ctest . -C Release -VV --output-on-failure