-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbuild-drivers.ps1
424 lines (383 loc) · 13.5 KB
/
build-drivers.ps1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
param (
[Parameter(Position = 0, mandatory = $true)]
[string] $M2
)
function Verify-Driver {
param (
$Driver
)
Write-Host "Verifying $Driver driver..."
$DriverFile = "resources\modules\$Driver.metabase-driver.jar"
if (!(Test-Path -Path $DriverFile)) {
return $false;
}
Write-Host "File exits."
return $true;
# $MungedDriver = $Driver -replace "-", "_"
# $DriverMainClass = "metabase/driver/$($MungedDriver)__init.class"
# $command = "jar -tf $DriverFile"
# $Files = Invoke-Expression -Command $command -ErrorAction Stop
# $result = $false
# foreach ($File in $Files) {
# if ($File -eq $DriverMainClass) {
# $result = $true
# break;
# }
# }
# if ($result) {
# Write-Host "Main class file found."
# }
# else {
# Write-Error "Main class file missing. Driver verification failed"
# return $false
# }
# Write-Host "Checking whether driver contains plugin manifeset..."
# $result = $false
# foreach ($File in $Files) {
# if ($File -eq "metabase-plugin.yaml") {
# $result = $true
# break;
# }
# }
# if ($result) {
# Write-Host "Plugin Manifest found."
# }
# else {
# Write-Error "Plugin manifest missing. Driver verification failed."
# return $false
# }
# Write-Host "Driver verification successful."
# return $true
}
################################ DELETING OLD INCORRECTLY BUILT DRIVERS ###############################
function Verify-ExistingBuild () {
Param(
$Driver,
$ChecksumFile
)
$VerificationStatus = Verify-Driver -Driver $Driver
if (!$VerificationStatus) {
Write-Host "No existing build, or existing build is invalid. (Re)building driver."
Remove-Item -Path $ChecksumFile -Force -ErrorAction Ignore
}
return $true
}
######################################## BUILDING THE DRIVER ########################################
# Delete existing saved copies of the driver in the plugins and resources directories
function Delete-OldDrivers() {
Param(
$Driver,
$DriverJar,
$DestLocation
)
Write-Host "Deleting old versions of $DriverJar driver from $DestLocation..."
Remove-Item -Force -Path "plugins\$DriverJar" -ErrorAction Ignore
Remove-Item -Force -Path $DestLocation -ErrorAction Ignore
return $true
}
# Check if Metabase is installed locally for building drivers; install it if not
function Install-MetabaseCore() {
$UserDir = $M2
try {
$Result = Get-ChildItem -Path "$UserDir\.m2\repository\metabase-core\metabase-core" -Include '*.jar' -Recurse -ErrorAction Stop
if (!$Result -or $Result.length -eq 0) {
Write-Host "Building and installing MetabaseCore jar locally"
lein clean
lein install-for-building-drivers
return $true;
}
else {
Write-Host "metabase-core already installed to local Maven repo."
return $true
}
}
catch {
Write-Host "Building and installing MetabaseCore jar locally"
lein clean
lein install-for-building-drivers
return $true;
}
Write-Host "Failed to install metabase core."
return $false
}
# Build Metabase uberjar if needed, we'll need this for stripping duplicate classes
function Build-MetabaseJar() {
Param(
$MetabaseUberjar
)
if (!(Test-Path $MetabaseUberjar)) {
Write-Host "Building Metabase uberjar..."
lein with-profile +uberjar uberjar
return $true
}
else {
Write-Host "Metabase uberjar alrady built"
return $true
}
return $false
}
# Take a look at the `parents` file listing the parents to build, if applicable
function Build-Parents () {
Param(
$ProjectRoot,
$DriverProjectDir
)
Write-Host "Building parent drivers (if needed)..."
$ParentListFile = "$DriverProjectDir\parents"
if (! (Test-Path -Path $ParentListFile)) {
return @()
}
$FileContent = Get-Content -Path $ParentListFile
$Parents = @()
foreach ($Parent in $FileContent) {
$Parents += $Parent
$ParentJarFile = "resources\modules\$($Parent).metabase-driver.jar"
if (! (Test-Path -Path $ParentJarFile)) {
Write-Host "Building $Parent"
Build-Driver -Driver $Parent
}
# Check whether built parent driver *JAR* exists in local maven repo
$UserDir = $M2
$ParentInstallDir = "$UserDir\.m2\repository\metabase\$($Parent)-driver"
$ParentInstalledJar = $null
if (Test-Path $ParentInstallDir) {
$ParentInstalledJar = Get-ChildItem -Path $ParentInstallDir -Include "*.jar" -Recurse -ErrorAction Ignore
}
if (!$ParentInstalledJar -or $ParentInstalledJar.length -eq 0) {
$ParentProjectDir = "$ProjectRoot\modules\drivers\$Parent"
Write-Host "Installing $Parent locally..."
Set-Location -Path $ParentProjectDir
lein clean
lein install-for-building-drivers
Set-Location -Path $ProjectRoot
}
else {
Write-Host "$Parent already installed to local Maven repo."
}
}
return $Parents
}
# Build the driver uberjar itself
function Build-DriverUberJar {
param (
$ProjectRoot,
$Driver,
$DriverProjectDir,
$TargetJar`
)
Write-Host "Building $Driver"
Set-Location $DriverProjectDir
lein clean
$env:DEBUG = "1"
$env:LEIN_SNAPSHOTS_IN_RELEASE = "true"
lein with-profile +uberjar uberjar
Set-Location $ProjectRoot
Write-Host "Check if the file exists $TargetJar"
if (!(Test-Path -Path $TargetJar)) {
Write-Error "Error: could not find $TargetJar. Build failed."
return $false
}
else {
Write-Host "$TargetJar exists"
return $true
}
return $false
}
# Strip out any classes in driver JAR found in core Metabase uberjar or parent JARs; recompress with higher compression ratio
function Strip-Compress () {
Param(
$Parents,
$TargetJar
)
try {
# strip out any classes found in the core Metabase uberjar
$LeinCmd = "lein strip-and-compress $TargetJar"
Write-Host "executing $LeinCmd"
lein strip-and-compress $TargetJar
#Invoke-Expression -Command $LeinCmd -ErrorAction Stop
# Next, remove any classes found in any of the parent jar
foreach ($Parent in $Parents) {
Write-Host "Removing duplicate classes with $Parent uberjar..."
$LeinCmd = "lein strip-and-compress $TargetJr resources\modules\$($Parent).metabase-driver.jar"
Invoke-Expression $LeinCmd -ErrorAction Stop
}
return $true
}
catch {
Write-Output $_
return $false
}
}
# Copy that JAR in the resources dir
function CopyTargetTo-Dest () {
Param(
$TargetJar,
$DestLocation
)
Write-Host "Copying $Targetjar to $DestLocation"
try {
Move-Item -Path $TargetJar -Destination $DestLocation -ErrorAction Stop
return $true
}
catch {
Write-Output $_
return $false
}
}
# check that JAR in resources dir looks correct
function Verify-Build () {
param(
$Driver,
$ChecksumFile,
$TargetJar,
$DestLocation
)
$VerificationStatus = Verify-Driver -Driver $Driver
if (!$VerificationStatus) {
Write-Host "Build $Driver FAILED."
return $false
}
return $true
}
function Calculate-Checksum {
param (
$DriverProjectDir
)
$TempCombinedFile = ".\.combinedContent"
Get-ChildItem -Path $DriverProjectDir -Include "*.clj"
Get-ChildItem -Path $DriverProjectDir -Include "*.clj" -Recurse -ErrorAction Stop | Sort-Object | ForEach-Object {
$content = Get-Content $_.FullName;
$content | Out-File -FilePath $TempCombinedFile -Append
}
Get-ChildItem -Path $DriverProjectDir -Include "*.yaml" -Recurse -ErrorAction Stop | Sort-Object | ForEach-Object {
$content = Get-Content $_.FullName;
$content | Out-File -FilePath $TempCombinedFile -Append
}
$checksum = Get-FileHash -Path $TempCombinedFile -Algorithm MD5
Remove-Item -Path $TempCombinedFile -Force -ErrorAction Stop
return $checksum.hash
}
# Save the checksum for the newly built JAR
function Save-Checksum() {
param(
$DriverProjectDir,
$ChecksumFile
)
Write-Host "Saving checksum for source files to $ChecksumFile"
$Cheksum = Calculate-Checksum -DriverProjectDir $DriverProjectDir
$Cheksum | Out-File -FilePath $ChecksumFile -Encoding utf8 -ErrorAction Stop
return $true
}
function Checksum-IsSame() {
param(
$DriverProjectDir,
$ChecksumFile,
$TargetJar,
$Driver
)
if ((Test-Path $ChecksumFile)) {
$OldChecksum = Get-Content -Path $ChecksumFile
$CurrentChecksum = Calculate-Checksum -DriverProjectDir $DriverProjectDir
Write-Host "Checksum of source files for previous build: $OldChecksum"
Write-Host "Current checksum of source files: $CurrentChecksum"
if ($OldChecksum -eq $CurrentChecksum) {
if ((Test-Path -Path $TargetJar)) {
Write-Host "$Driver driver source unchanged since last build. Skipping re-build."
return $true
}
}
}
else {
Write-Host "$Driver checksum file doesn't exist"
}
return $false
}
function Clean-LocalRepo {
Write-Host "Deleting existing installed metabase-core and driver dependencies..."
$UserDir = $M2
Remove-Item -Recurse -Force -Path "$Userdir\.m2\repository\metabase-core" -ErrorAction Ignore
Remove-Item -Recurse -Force -Path "$Userdir\.m2\repository\*-driver" -ErrorAction Ignore
Get-ChildItem "$Userdir\.m2\repository\metabase" -ErrorAction Ignore | Where { $_.Name -Match ".*-driver" } | Remove-Item -Force -Recurse -ErrorAction Stop
}
function Build-DriverPipeline () {
Param(
$Driver,
$DriverProjectDir,
$DriverJar,
$DestLocation,
$MetabaseUberJar,
$TargetJar,
$ChecksumFile,
$ProjectRoot
)
try {
Delete-OldDrivers -Driver $Driver -DriverJar $DriverJar -DestLocation $DestLocation
Install-MetabaseCore
Build-MetabaseJar -MetabaseUberjar $MetabaseUberJar
$Parents = Build-Parents -ProjectRoot $ProjectRoot -DriverProjectDir $DriverProjectDir
Write-Host "$Driver depends on the following parent drivers: $Parents "
Build-DriverUberJar -ProjectRoot $ProjectRoot -Driver $Driver -DriverProjectDir $DriverProjectDir -TargetJar $TargetJar
Strip-Compress -Parents $Parents -TargetJar $TargetJar
CopyTargetTo-Dest -TargetJar $TargetJar -DestLocation $DestLocation
Verify-Build -Driver $Driver -ChecksumFile $ChecksumFile -TargetJar $TargetJar -DestLocation $DestLocation
Save-Checksum -DriverProjectDir $DriverProjectDir -ChecksumFile $ChecksumFile
}
catch {
Write-Output $_
throw $_
}
}
function Build-Driver () {
Param(
$Driver
)
$ProjectRoot = Get-Location
$DriverProjectDir = "$ProjectRoot\modules\drivers\$Driver"
$DriverJar = "$Driver.metabase-driver.jar"
$DestLocation = "$ProjectRoot\resources\modules\$DriverJar"
$MetabaseUberJar = "$ProjectRoot\target\uberjar\metabase.jar"
$TargetJar = "$DriverProjectDir\target\uberjar\$DriverJar"
$Parents = ""
$ChecksumFile = "$DriverProjectDir\target\checksum.txt"
Write-Host "building driver $Driver from $DriverProjectDir"
if (!(Checksum-IsSame -DriverProjectDir $DriverProjectDir -ChecksumFile $ChecksumFile -TargetJar $TargetJar -Driver $Driver )) {
Write-Host "Checksum has changed."
Build-DriverPipeline -DriverProjectDir $DriverProjectDir -Driver $Driver -ProjectRoot $ProjectRoot -DriverJar $DriverJar -DestLocation $DestLocation -MetabaseUberJar $MetabaseUberjar -TargetJar $TargetJar -ChecksumFile $ChecksumFile
}
else {
Write-Host "checksum is unchanged"
$result = CopyTargetTo-Dest -TargetJar $TargetJar -DestLocation $DestLocation
$result2 = Verify-Build -Driver $Driver -ChecksumFile $ChecksumFile -TargetJar $TargetJar -DestLocation $DestLocation
$result = $result -and $result2
if (!$result) {
return Retry -DriverProjectDir $DriverProjectDir -Driver $Driver -ProjectRoot $ProjectRoot -DriverJar $DriverJar -Destination $DestLocation -MetabaseUberJar $MetabaseUberjar -TargetJar $TargetJar -ChecksumFile $ChecksumFile
}
}
}
function Retry() {
param (
$Driver,
$DriverProjectDir,
$DriverJar,
$DestLocation,
$MetabaseUberJar,
$TargetJar,
$ChecksumFile,
$ProjectRoot
)
Write-Host "Building without cleaning failed. Retrying clean build..."
Clean-LocalRepo
Build-DriverPipeline -DriverProjectDir $DriverProjectDir -Driver $Driver -ProjectRoot $ProjectRoot -DriverJar $DriverJar -Destination $DestLocation -MetabaseUberJar $MetabaseUberjar -TargetJar $TargetJar -ChecksumFile $ChecksumFile
}
mkdir -Path "resources\modules" -ErrorAction Ignore
$Drivers = Get-ChildItem -Directory -Path "modules\drivers" -Name
$DriversToBuild = @{
"sqlserver" = 1;
};
Clean-LocalRepo
foreach ($Driver in $Drivers) {
if ($DriversToBuild.Contains($Driver)) {
Write-Host "Build: $Driver"
Build-Driver -Driver $Driver
}
}