Skip to content

Commit

Permalink
fix ut for spark 35
Browse files Browse the repository at this point in the history
  • Loading branch information
baibaichen committed Aug 16, 2024
1 parent 6763354 commit 48cd373
Showing 1 changed file with 17 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -188,20 +188,33 @@ class GlutenClickHouseMergeTreeWriteOnS3Suite
var metadataGlutenExist: Boolean = false
var metadataBinExist: Boolean = false
var dataBinExist: Boolean = false
var hasCommits = false
client
.listObjects(args)
.forEach(
obj => {
objectCount += 1
if (obj.get().objectName().contains("metadata.gluten")) {
val objectName = obj.get().objectName()
if (objectName.contains("metadata.gluten")) {
metadataGlutenExist = true
} else if (obj.get().objectName().contains("meta.bin")) {
} else if (objectName.contains("meta.bin")) {
metadataBinExist = true
} else if (obj.get().objectName().contains("data.bin")) {
} else if (objectName.contains("data.bin")) {
dataBinExist = true
} else if (objectName.contains("_commits")) {
// Spark 35 has _commits directory
// table/_delta_log/_commits/
hasCommits = true
}
})
assertResult(5)(objectCount)

if (isSparkVersionGE("3.5")) {
assertResult(6)(objectCount)
assert(hasCommits)
} else {
assertResult(5)(objectCount)
}

assert(metadataGlutenExist)
assert(metadataBinExist)
assert(dataBinExist)
Expand Down

0 comments on commit 48cd373

Please sign in to comment.