hqps-db-ci.yml

name: GraphScope GIE HighQPS DB CI

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  workflow_dispatch:
  push:
    branches:
      - main
    paths:
      - 'flex/**'
      - 'interactive_engine/**'
      - '.github/workflows/hqps-db-ci.yml'
  pull_request:
    branches:
      - main
    paths:
      - 'flex/**'
      - 'interactive_engine/**'
      - '.github/workflows/hqps-db-ci.yml'

concurrency:
  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

jobs:
  test-hqps-engine:
    runs-on: ubuntu-20.04
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    container:
      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.6
    steps:
    - uses: actions/checkout@v3

    - uses: actions/cache@v3
      with:
        path: ~/.m2/repository
        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
            ${{ runner.os }}-maven-

    - uses: actions/cache@v3
      with:
        path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            ~/.cache/sccache
        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

    - name: Install latest libgrape-lite
      if: false
      run: |
        git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
        cd /tmp/libgrape-lite
        mkdir -p build && cd build
        cmake ..
        make -j$(nproc)
        make install

    - name: Setup tmate session
      if: false
      uses: mxschmitt/action-tmate@v3

    - name: Build
      env:
        GIE_HOME: ${{ github.workspace }}/interactive_engine/
        HOME: /home/graphscope/
      run: |
        cd ${GITHUB_WORKSPACE}/flex
        mkdir build && cd build
        cmake .. && sudo make -j$(nproc)
        sudo make install

        # cargo
        . /home/graphscope/.cargo/env
        which cargo

        # build compiler
        cd ${GIE_HOME}/compiler
        make build

    - name: Prepare dataset and workspace
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        # download dataset
        #git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}
        git clone -b master --single-branch --depth=1 https://github.com/zhanglei1949/gstest.git ${GS_TEST_DIR}
        mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc
        GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml
        cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml
        mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies
        cp ${GS_TEST_DIR}/flex/movies/movies_schema.yaml ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml

    - name: Sample Query test
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        cd ${GITHUB_WORKSPACE}/flex/build
        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
        ./tests/hqps/query_test ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \
        ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml \
        /tmp/csr-data-dir/

    - name: Run codegen test.
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        HOME : /home/graphscope/
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        GIE_HOME=${GITHUB_WORKSPACE}/interactive_engine
        cd ${GITHUB_WORKSPACE}/flex/bin

        for i in 1 2 3 4 5 6 7 8 9 10 11 12;
        do
          cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codgen/"
          cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
          cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
          cmd=${cmd}" --gie_home=${GIE_HOME}"
          echo $cmd
          eval ${cmd}
        done

        for i in 1 2 3 4 5 6 7 8 9;
        do
          cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codgen/"
          cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
          cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
          cmd=${cmd}" --gie_home=${GIE_HOME}"
          echo $cmd
          eval ${cmd}
        done

        # test movie graph, 8,9,10 are not supported now
        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies
        sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
        for i in 1 2 3 4 5 6 7 11 12 13 14 15;
        do
          cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codgen/"
          cmd=${cmd}" -o=/tmp/plugin --ir_conf=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml "
          cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
          cmd=${cmd}" --gie_home=${GIE_HOME}"
          echo $cmd
          eval ${cmd}
        done

    - name: Run End-to-End cypher adhoc ldbc query test
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        HOME : /home/graphscope/
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
        export ENGINE_TYPE=hiactor
        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to ldbc
        sed -i 's/default_graph: movies/default_graph: ldbc/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml \
              ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml

    - name: Run End-to-End cypher adhoc movie query test
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        HOME : /home/graphscope/
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
        export FLEX_DATA_DIR=../../interactive/examples/movies/
        export ENGINE_TYPE=hiactor
        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to movies
        sed -i 's/default_graph: ldbc/default_graph: movies/g' ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml
        bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies ${GS_TEST_DIR}/flex/movies/movies_import.yaml \
          ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml