-
Notifications
You must be signed in to change notification settings - Fork 7
159 lines (136 loc) · 5.4 KB
/
e2e.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
name: e2e-test
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
on:
push:
branches: [main]
pull_request:
branches: [main]
repository_dispatch:
types: [ release-tag ]
branches: [ release-** ]
env:
GO_VERSION: "1.20"
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
jobs:
e2e-tests:
runs-on: ubuntu-latest
environment: e2e-test
steps:
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
- name: Shorten SHA
if: ${{ !github.event.client_payload.isRelease }}
id: vars
run: echo "pr_sha_short=$(git rev-parse --short ${{ github.event.pull_request.head.sha }})" >> $GITHUB_OUTPUT
- name: Set e2e Resource and Cluster Name
run: |
rand=${{ steps.vars.outputs.pr_sha_short }}
if [ "$rand" = "" ]; then
rand=$RANDOM
fi
echo "VERSION=${rand}" >> $GITHUB_ENV
echo "CLUSTER_NAME=gpuprov${rand}" >> $GITHUB_ENV
echo "REGISTRY=kaito${rand}.azurecr.io" >> $GITHUB_ENV
- name: Set Registry
if: ${{ github.event.client_payload.isRelease }}
run: |
echo "REGISTRY=${{ github.event.client_payload.registry }}" >> $GITHUB_ENV
echo "VERSION=$(echo ${{ github.event.client_payload.tag }} | tr -d v)" >> $GITHUB_ENV
- name: Checkout
if: ${{ !github.event.client_payload.isRelease }}
uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
- name: Checkout
uses: actions/checkout@v4
if: ${{ github.event.client_payload.isRelease }}
with:
fetch-depth: 0
submodules: true
ref: ${{ env.REPO_TAG }}
- uses: azure/[email protected]
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Install Helm
uses: azure/setup-helm@v4
with:
token: ${{ secrets.GITHUB_TOKEN }}
id: install
- name: Create ACR
shell: bash
run: |
make az-mkacr
env:
AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
AZURE_ACR_NAME: ${{ env.CLUSTER_NAME }}
- name: Create Azure Identity
uses: azure/[email protected]
with:
inlineScript: |
az identity create --name gpuIdentity --resource-group ${{ env.CLUSTER_NAME }}
- name: Build gpu-provisioner image
if: ${{ !github.event.client_payload.isRelease }}
shell: bash
run: |
make docker-build
env:
REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
VERSION: ${{ env.VERSION }}
- name: Create cluster
shell: bash
run: |
make az-mkaks
env:
AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
AZURE_ACR_NAME: ${{ env.CLUSTER_NAME }}
AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
- name: Install GPU-Provisioner helm chart
shell: bash
run: |
echo "IDENTITY_CLIENT_ID=$(az identity show --name gpuIdentity -g "${{ env.CLUSTER_NAME }}" --query 'clientId' -otsv)" >> $GITHUB_ENV
make az-patch-helm
helm install gpu-provisioner ./charts/gpu-provisioner
kubectl wait --for=condition=available deploy "gpu-provisioner" -n gpu-provisioner --timeout=300s
env:
AZURE_RESOURCE_GROUP: ${{ env.CLUSTER_NAME }}
AZURE_ACR_NAME: ${{ env.CLUSTER_NAME }}
AZURE_CLUSTER_NAME: ${{ env.CLUSTER_NAME }}
REGISTRY: ${{ env.CLUSTER_NAME }}.azurecr.io
VERSION: ${{ env.VERSION }}
- uses: azure/[email protected]
with:
client-id: ${{ secrets.AZURE_CLIENT_ID }}
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: Create Role Assignment
uses: azure/[email protected]
with:
inlineScript: |
IDENTITY_PRINCIPAL_ID="$(az identity show --name gpuIdentity --resource-group ${{ env.CLUSTER_NAME }} --query 'principalId' -otsv)"
az role assignment create --assignee ${IDENTITY_PRINCIPAL_ID} --scope "/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.CLUSTER_NAME }}/providers/Microsoft.ContainerService/managedClusters/${{ env.CLUSTER_NAME }}" --role "Contributor"
- name: Create Azure Federated Identity
uses: azure/[email protected]
with:
inlineScript: |
AKS_OIDC_ISSUER="$(az aks show -n "${{ env.CLUSTER_NAME }}" -g "${{ env.CLUSTER_NAME }}" --query 'oidcIssuerProfile.issuerUrl' -otsv)"
az identity federated-credential create --name gpu-fed-credential --identity-name gpuIdentity --resource-group "${{ env.CLUSTER_NAME }}" \
--issuer "${AKS_OIDC_ISSUER}" --subject system:serviceaccount:"gpu-provisioner:gpu-provisioner" --audience api://AzureADTokenExchange
- name: Run e2e test
run: |
make e2etests
- name: Cleanup e2e resources
if: ${{ always() }}
uses: azure/[email protected]
with:
inlineScript: |
set +e
az group delete --name "${{ env.CLUSTER_NAME }}" --yes --no-wait || true