-
Notifications
You must be signed in to change notification settings - Fork 448
223 lines (198 loc) · 9.34 KB
/
hqps-db-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
name: GraphScope GIE HighQPS DB CI
on:
# Trigger the workflow on push or pull request,
# but only for the main branch
workflow_dispatch:
push:
branches:
- main
paths:
- 'flex/**'
- 'interactive_engine/**'
- '.github/workflows/hqps-db-ci.yml'
pull_request:
branches:
- main
paths:
- 'flex/**'
- 'interactive_engine/**'
- '.github/workflows/hqps-db-ci.yml'
concurrency:
group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
jobs:
test-hqps-engine:
runs-on: ubuntu-20.04
if: ${{ github.repository == 'alibaba/GraphScope' }}
container:
image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.9
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
~/.cache/sccache
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Install latest libgrape-lite
if: false
run: |
git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
cd /tmp/libgrape-lite
mkdir -p build && cd build
cmake ..
make -j$(nproc)
make install
- name: Setup tmate session
if: false
uses: mxschmitt/action-tmate@v3
- name: Build
env:
GIE_HOME: ${{ github.workspace }}/interactive_engine/
HOME: /home/graphscope/
run: |
cd ${GITHUB_WORKSPACE}/
git submodule update --init
cd ${GITHUB_WORKSPACE}/flex
mkdir build && cd build
cmake .. && sudo make -j$(nproc)
sudo make install
# cargo
. /home/graphscope/.cargo/env
which cargo
# build compiler
cd ${GIE_HOME}/
mvn clean install -Pexperimental -DskipTests
- name: Prepare dataset and workspace
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
run: |
# download dataset
git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}
mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc
GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml
BUILD_LOAD_FILE=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml
cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml
cp ${BUILD_LOAD_FILE} ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml
mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies
cp ${GS_TEST_DIR}/flex/movies/movies_schema.yaml ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml
cp ${GS_TEST_DIR}/flex/movies/movies_import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml
mkdir -p ${INTERACTIVE_WORKSPACE}/data/graph_algo
cp ${GITHUB_WORKSPACE}/flex/tests/interactive/graph_algo_test.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml
cp ${GITHUB_WORKSPACE}/flex/interactive/examples/graph_algo/import.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml
# load graph
cd ${GITHUB_WORKSPACE}/flex/build
export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/ldbc/indices/
export FLEX_DATA_DIR=../interactive/examples/movies
GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/movies/indices/
export FLEX_DATA_DIR=../interactive/examples/graph_algo
GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/graph_algo/indices/
- name: Test HQPS admin http service
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
run:
cd ${GITHUB_WORKSPACE}/flex/tests/hqps
bash hqps_admin_test.sh /tmp/temp_workspace ./engine_config_test.yaml ${GS_TEST_DIR}
- name: Sample Query test
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
run: |
cd ${GITHUB_WORKSPACE}/flex/build
export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
./bin/bulk_loader -g ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \
-l ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml -d /tmp/csr-data-dir/
rm -r /tmp/csr-data-dir/runtime/*
rm -r /tmp/csr-data-dir/wal
./tests/hqps/query_test ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \
/tmp/csr-data-dir/
- name: Run codegen test.
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
HOME : /home/graphscope/
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
run: |
cd ${GITHUB_WORKSPACE}/flex/bin
for i in 1 2 3 4 5 6 7 8 9 10 11 12;
do
cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/"
cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
echo $cmd
eval ${cmd}
done
for i in 1 2 3 4 5 6 7 8 9 10 11 12;
do
cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/simple_match_${i}.cypher -w=/tmp/codegen/"
cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml"
echo $cmd
eval ${cmd}
done
# test movie graph, 8,9,10 are not supported now
# change the default_graph config in ../tests/hqps/engine_config_test.yaml to movies
sed -i 's/default_graph: ldbc/default_graph: movies/g' ../tests/hqps/engine_config_test.yaml
for i in 1 2 3 4 5 6 7 11 12 13 14 15;
do
cmd="./load_plan_and_gen.sh -e=hqps -i=../tests/hqps/queries/movie/query${i}.cypher -w=/tmp/codegen/"
cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml "
cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml"
echo $cmd
eval ${cmd}
done
- name: Test cypher procedure generation
run: |
cd ${GITHUB_WORKSPACE}/flex/bin
./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/get_person_name.cypher -w=/tmp/codegen \
--ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml \
--graph_schema_path=../interactive/examples/modern_graph/modern_graph.yaml
./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \
--ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml \
--graph_schema_path=../interactive/examples/modern_graph/modern_graph.yaml
- name: Run End-to-End cypher adhoc ldbc query test
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
HOME : /home/graphscope/
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
run: |
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
export ENGINE_TYPE=hiactor
# change the default_graph config in ./engine_config_test.yaml to ldbc
sed -i 's/default_graph: movies/default_graph: ldbc/g' ./engine_config_test.yaml
bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
- name: Run End-to-End cypher adhoc movie query test
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
HOME : /home/graphscope/
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
run: |
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
export ENGINE_TYPE=hiactor
# change the default_graph config in ./engine_config_test.yaml to movies
sed -i 's/default_graph: ldbc/default_graph: movies/g' ./engine_config_test.yaml
bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} movies \
${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml
- name: Run End-to-End cypher adhoc graph_algo query test
env:
GS_TEST_DIR: ${{ github.workspace }}/gstest
HOME : /home/graphscope/
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
run: |
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
export ENGINE_TYPE=hiactor
# change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/engine_config.yaml to graph_algo
sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./engine_config_test.yaml
bash hqps_cypher_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \
${GITHUB_WORKSPACE}/flex/tests/hqps/engine_config_test.yaml