-
Notifications
You must be signed in to change notification settings - Fork 14
165 lines (148 loc) · 5.07 KB
/
ci.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
name: Test
on:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
milvus-cdc-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v4
with:
go-version: '1.18'
cache-dependency-path: server/go.sum
cache: true
- name: set up python
uses: actions/setup-python@v2
with:
python-version: '3.10'
cache: 'pip'
- name: Build CDC
timeout-minutes: 15
working-directory: server
shell: bash
run: |
go build -o milvus-cdc main/main.go
ls -l
- name: Creating kind cluster
uses: helm/[email protected]
- name: Print cluster information
run: |
kubectl config view
kubectl cluster-info
kubectl get nodes
kubectl get pods -o wide -n kube-system
helm version
kubectl version
- name: Deploy Source Milvus
timeout-minutes: 15
shell: bash
working-directory: tests/deployment/upstream/
run: |
docker-compose up -d
bash ../../../scripts/check_healthy.sh
docker-compose ps -a
- name: Deploy Downstream Milvus
timeout-minutes: 15
shell: bash
working-directory: tests/deployment/downstream
run: |
helm repo add milvus https://zilliztech.github.io/milvus-helm
helm repo update
helm install --wait --timeout 720s cdc-downstream milvus/milvus -f standalone-values.yaml
kubectl get pods
kubectl port-forward service/cdc-downstream-milvus 19500:19530 >/dev/null 2>&1 &
sleep 20s
nc -vz 127.0.0.1 19500
- name: Deploy Milvus CDC
timeout-minutes: 15
working-directory: server
shell: bash
run: |
./milvus-cdc > server.log 2>&1 &
sleep 20s
- name: Create CDC task
timeout-minutes: 15
run: |
curl --location '127.0.0.1:8444/cdc' \
--header 'Content-Type: application/json' \
--data '{
"request_type": "create",
"request_data": {
"milvus_connect_param": {
"host": "127.0.0.1",
"port": 19500,
"username": "",
"password": "",
"enable_tls": false,
"connect_timeout": 120
},
"collection_infos": [
{
"name": "*"
}
],
"rpc_channel_info": {
"name": "by-dev-replicate-msg"
}
}
}'
- name: Checkout Milvus Repo
uses: actions/checkout@v2
with:
repository: zhuwenxing/milvus
ref: refine_checker
path: milvus
- name: Run test
timeout-minutes: 60
shell: bash
working-directory: milvus/tests/python_client
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
cd chaos && pytest testcases/test_concurrent_operation.py --host 127.0.0.1 --log-cli-level=INFO --capture=no --request_duration 15m
- name: List CDC task
timeout-minutes: 15
working-directory: server
shell: bash
run: |
cat server.log | tail -n 100
curl --location '127.0.0.1:8444/cdc' \
--header 'Content-Type: application/json' \
--data '{
"request_type": "list"
}'
- name: Diff upstream and downstream
if: ${{ always() }}
timeout-minutes: 10
shell: bash
working-directory: milvus/tests/python_client/chaos/scripts
run: |
python diff_upstream_downstream.py --upstream_host 127.0.0.1 --upstream_port 19530 --downstream_host 127.0.0.1 --downstream_port 19500
- name: Export upstream milvus logs
if: ${{ always() }}
timeout-minutes: 5
working-directory: tests/deployment/upstream
run: |
docker-compose ps -a
docker stats --no-stream
bash ../../../scripts/export_log_docker.sh
- name: Export downstream milvus logs
if: ${{ always() }}
timeout-minutes: 5
working-directory: scripts
run: |
kubectl top pod || true
bash export_log_k8s.sh default cdc-downstream k8s_logs
- name: Upload logs
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: logs
path: |
scripts/k8s_logs
tests/deployment/upstream/logs
server.log