From e03d75a7bcc54c8021520edb59bcb2ebc3a90cb6 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 14 Aug 2024 15:36:10 +0530 Subject: [PATCH 01/43] Add support for --base-branch in backward-compatibility-check command Abstract out the common logic in backward-compatibility-check command --- .../kotlin/application/SpecmaticCommand.kt | 4 +- .../backwardCompatibility/BCCheckCommand.kt | 137 ++++++++++++ .../BackwardCompatibilityCheckBaseCommand.kt | 198 ++++++++++++++++++ .../BackwardCompatibilityCheckCommand.kt | 0 .../CompatibilityReport.kt | 16 ++ .../CompatibilityResult.kt | 5 + .../BackwardCompatibilityCheckCommandTest.kt | 7 +- .../application/CompatibleCommandKtTest.kt | 21 +- .../src/test/kotlin/application/FakeGit.kt | 12 ++ .../main/kotlin/io/specmatic/core/Feature.kt | 2 +- .../main/kotlin/io/specmatic/core/IFeature.kt | 3 + .../io/specmatic/core/git/GitCommand.kt | 8 +- .../kotlin/io/specmatic/core/git/SystemGit.kt | 37 +++- 13 files changed, 429 insertions(+), 21 deletions(-) create mode 100644 application/src/main/kotlin/application/backwardCompatibility/BCCheckCommand.kt create mode 100644 application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt rename application/src/main/kotlin/application/{ => backwardCompatibility}/BackwardCompatibilityCheckCommand.kt (100%) create mode 100644 application/src/main/kotlin/application/backwardCompatibility/CompatibilityReport.kt create mode 100644 application/src/main/kotlin/application/backwardCompatibility/CompatibilityResult.kt create mode 100644 core/src/main/kotlin/io/specmatic/core/IFeature.kt diff --git a/application/src/main/kotlin/application/SpecmaticCommand.kt b/application/src/main/kotlin/application/SpecmaticCommand.kt index f8a43a1bb..3f301577b 100644 --- a/application/src/main/kotlin/application/SpecmaticCommand.kt +++ b/application/src/main/kotlin/application/SpecmaticCommand.kt @@ -1,5 +1,7 @@ package application +import application.backwardCompatibility.BCCheckCommand +import application.backwardCompatibility.BackwardCompatibilityCheckCommand import org.springframework.stereotype.Component import picocli.AutoComplete.GenerateCompletion import picocli.CommandLine.Command @@ -10,7 +12,7 @@ import java.util.concurrent.Callable name = "specmatic", mixinStandardHelpOptions = true, versionProvider = VersionProvider::class, - subcommands = [BackwardCompatibilityCheckCommand::class, BundleCommand::class, CompareCommand::class, CompatibleCommand::class, DifferenceCommand::class, GenerateCompletion::class, GraphCommand::class, MergeCommand::class, ToOpenAPICommand::class, ImportCommand::class, InstallCommand::class, ProxyCommand::class, PushCommand::class, ReDeclaredAPICommand::class, ExamplesCommand::class, SamplesCommand::class, StubCommand::class, SubscribeCommand::class, TestCommand::class, ValidateViaLogs::class, CentralContractRepoReportCommand::class] + subcommands = [BCCheckCommand::class, BundleCommand::class, CompareCommand::class, CompatibleCommand::class, DifferenceCommand::class, GenerateCompletion::class, GraphCommand::class, MergeCommand::class, ToOpenAPICommand::class, ImportCommand::class, InstallCommand::class, ProxyCommand::class, PushCommand::class, ReDeclaredAPICommand::class, ExamplesCommand::class, SamplesCommand::class, StubCommand::class, SubscribeCommand::class, TestCommand::class, ValidateViaLogs::class, CentralContractRepoReportCommand::class] ) class SpecmaticCommand : Callable { override fun call(): Int { diff --git a/application/src/main/kotlin/application/backwardCompatibility/BCCheckCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BCCheckCommand.kt new file mode 100644 index 000000000..e3747c49e --- /dev/null +++ b/application/src/main/kotlin/application/backwardCompatibility/BCCheckCommand.kt @@ -0,0 +1,137 @@ +package application.backwardCompatibility + +import io.specmatic.conversions.OpenApiSpecification +import io.specmatic.core.CONTRACT_EXTENSION +import io.specmatic.core.CONTRACT_EXTENSIONS +import io.specmatic.core.Feature +import io.specmatic.core.IFeature +import io.specmatic.core.Results +import io.specmatic.core.WSDL +import io.specmatic.core.testBackwardCompatibility +import io.specmatic.stub.isOpenAPI +import org.springframework.stereotype.Component +import picocli.CommandLine.Command +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.regex.Pattern +import kotlin.io.path.extension +import kotlin.io.path.pathString + +@Component +@Command( + name = "backwardCompatibilityCheck", + aliases = ["backward-compatibility-check"], + mixinStandardHelpOptions = true, + description = ["Checks backward compatibility of a directory across the current HEAD and the base branch"] +) +class BCCheckCommand: BackwardCompatibilityCheckBaseCommand() { + + override fun checkBackwardCompatibility(oldFeature: IFeature, newFeature: IFeature): Results { + return testBackwardCompatibility(oldFeature as Feature, newFeature as Feature) + } + + override fun File.isValidSpec(): Boolean { + if (this.extension !in CONTRACT_EXTENSIONS) return false + return OpenApiSpecification.isParsable(this.path) + } + + override fun getFeatureFromSpecPath(path: String): Feature { + return OpenApiSpecification.fromFile(path).toFeature() + } + + override fun getSpecsReferringTo(schemaFiles: Set): Set { + if (schemaFiles.isEmpty()) return emptySet() + + val inputFileNames = schemaFiles.map { File(it).name } + val result = allOpenApiSpecFiles().filter { + it.readText().trim().let { specContent -> + inputFileNames.any { inputFileName -> + val pattern = Pattern.compile("\\b$inputFileName\\b") + val matcher = pattern.matcher(specContent) + matcher.find() + } + } + }.map { it.path }.toSet() + + return result.flatMap { + getSpecsReferringTo(setOf(it)).ifEmpty { setOf(it) } + }.toSet() + } + + override fun getSpecsOfChangedExternalisedExamples(filesChangedInCurrentBranch: Set): Set { + data class CollectedFiles( + val specifications: MutableSet = mutableSetOf(), + val examplesMissingSpecifications: MutableList = mutableListOf(), + val ignoredFiles: MutableList = mutableListOf() + ) + + val collectedFiles = filesChangedInCurrentBranch.fold(CollectedFiles()) { acc, filePath -> + val path = Paths.get(filePath) + val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } + + if (examplesDir == null) { + acc.ignoredFiles.add(filePath) + } else { + val parentPath = examplesDir.parent + val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) + val specFiles = findSpecFiles(strippedPath) + + if (specFiles.isNotEmpty()) { + acc.specifications.addAll(specFiles.map { it.toString() }) + } else { + acc.examplesMissingSpecifications.add(filePath) + } + } + acc + } + + val result = collectedFiles.specifications.toMutableSet() + + collectedFiles.examplesMissingSpecifications.forEach { filePath -> + val path = Paths.get(filePath) + val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } + if (examplesDir != null) { + val parentPath = examplesDir.parent + val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) + val specFiles = findSpecFiles(strippedPath) + if (specFiles.isNotEmpty()) { + result.addAll(specFiles.map { it.toString() }) + } else { + result.add("${strippedPath}.yaml") + } + } + } + + return result + } + + override fun areExamplesValid(feature: IFeature, which: String): Boolean { + feature as Feature + return try { + feature.validateExamplesOrException() + true + } catch (t: Throwable) { + println() + false + } + } + + override fun getUnusedExamples(feature: IFeature): Set { + feature as Feature + return feature.loadExternalisedExamplesAndListUnloadableExamples().second + } + + private fun allOpenApiSpecFiles(): List { + return File(".").walk().toList().filterNot { + ".git" in it.path + }.filter { it.isFile && it.isValidSpec() } + } + + private fun findSpecFiles(path: Path): List { + val extensions = CONTRACT_EXTENSIONS + return extensions.map { path.resolveSibling(path.fileName.toString() + it) } + .filter { Files.exists(it) && (isOpenAPI(it.pathString) || it.extension in listOf(WSDL, CONTRACT_EXTENSION)) } + } +} \ No newline at end of file diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt new file mode 100644 index 000000000..44493635e --- /dev/null +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -0,0 +1,198 @@ +package application.backwardCompatibility + +import io.specmatic.core.IFeature +import io.specmatic.core.Results +import io.specmatic.core.git.GitCommand +import io.specmatic.core.git.SystemGit +import io.specmatic.core.log.logger +import io.specmatic.core.utilities.exitWithMessage +import picocli.CommandLine.Option +import java.io.File +import java.util.concurrent.Callable +import kotlin.system.exitProcess + +abstract class BackwardCompatibilityCheckBaseCommand : Callable { + private val gitCommand: GitCommand = SystemGit() + private val newLine = System.lineSeparator() + + @Option(names = ["--base-branch"], description = ["Base branch to compare the changes against"], required = false) + var baseBranch: String? = null + + abstract fun checkBackwardCompatibility(oldFeature: IFeature, newFeature: IFeature): Results + abstract fun File.isValidSpec(): Boolean + abstract fun getFeatureFromSpecPath(path: String): IFeature + + abstract fun getSpecsReferringTo(schemaFiles: Set): Set + abstract fun getSpecsOfChangedExternalisedExamples( + filesChangedInCurrentBranch: Set + ): Set + + + open fun areExamplesValid(feature: IFeature, which: String): Boolean = true + open fun getUnusedExamples(feature: IFeature): Set = emptySet() + + override fun call() { + val filesChangedInCurrentBranch = getChangedSpecsInCurrentBranch() + val filesReferringToChangedSchemaFiles = getSpecsReferringTo(filesChangedInCurrentBranch) + val specificationsOfChangedExternalisedExamples = + getSpecsOfChangedExternalisedExamples(filesChangedInCurrentBranch) + + logFilesToBeCheckedForBackwardCompatibility( + filesChangedInCurrentBranch, + filesReferringToChangedSchemaFiles, + specificationsOfChangedExternalisedExamples + ) + + val specificationsToCheck: Set = + filesChangedInCurrentBranch + + filesReferringToChangedSchemaFiles + + specificationsOfChangedExternalisedExamples + + val result = try { + runBackwardCompatibilityCheckFor( + files = specificationsToCheck, + baseBranch = baseBranch() + ) + } catch(e: Throwable) { + logger.newLine() + logger.newLine() + logger.log(e) + exitProcess(1) + } + + println() + println(result.report) + exitProcess(result.exitCode) + } + + private fun getChangedSpecsInCurrentBranch(): Set { + return gitCommand.getFilesChangedInCurrentBranch( + baseBranch() + ).filter { + File(it).exists() && File(it).isValidSpec() + }.toSet().also { + if(it.isEmpty()) exitWithMessage("${newLine}No specs were changed, skipping the check.$newLine") + } + } + + private fun logFilesToBeCheckedForBackwardCompatibility( + changedFiles: Set, + filesReferringToChangedFiles: Set, + specificationsOfChangedExternalisedExamples: Set + ) { + + println("Checking backward compatibility of the following files: $newLine") + println("${ONE_INDENT}Files that have changed:") + changedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } + println() + + if(filesReferringToChangedFiles.isNotEmpty()) { + println("${ONE_INDENT}Files referring to the changed files - ") + filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } + println() + } + + if(specificationsOfChangedExternalisedExamples.isNotEmpty()) { + println("${ONE_INDENT}Specifications whose externalised examples were changed - ") + filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } + println() + } + + println("-".repeat(20)) + println() + } + + private fun runBackwardCompatibilityCheckFor(files: Set, baseBranch: String): CompatibilityReport { + val branchWithChanges = gitCommand.currentBranch() + val treeishWithChanges = if (branchWithChanges == HEAD) gitCommand.detachedHEAD() else branchWithChanges + + try { + val results = files.mapIndexed { index, specFilePath -> + try { + println("${index.inc()}. Running the check for $specFilePath:") + + // newer => the file with changes on the branch + val newer = getFeatureFromSpecPath(specFilePath) + val unusedExamples = getUnusedExamples(newer) + + val olderFile = gitCommand.getFileInTheBaseBranch( + specFilePath, + treeishWithChanges, + baseBranch + ) + if (olderFile == null) { + println("$specFilePath is a new file.$newLine") + return@mapIndexed CompatibilityResult.PASSED + } + + val areLocalChangesStashed = gitCommand.stash() + gitCommand.checkout(baseBranch) + // older => the same file on the default (e.g. main) branch + val older = getFeatureFromSpecPath(olderFile.path) + if (areLocalChangesStashed) gitCommand.stashPop() + + val backwardCompatibilityResult = checkBackwardCompatibility(older, newer) + + if (backwardCompatibilityResult.success()) { + println( + "$newLine The file $specFilePath is backward compatible.$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + var errorsFound = false + + if(!areExamplesValid(newer, "newer")) { + println( + "$newLine *** Examples in $specFilePath are not valid. ***$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + errorsFound = true + } + + if(unusedExamples.isNotEmpty()) { + println( + "$newLine *** Some examples for $specFilePath could not be loaded. ***$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + errorsFound = true + } + + if(errorsFound) CompatibilityResult.FAILED + else CompatibilityResult.PASSED + } else { + println("$newLine ${backwardCompatibilityResult.report().prependIndent( + MARGIN_SPACE + )}") + println( + "$newLine *** The file $specFilePath is NOT backward compatible. ***$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + CompatibilityResult.FAILED + } + } finally { + gitCommand.checkout(treeishWithChanges) + } + } + + return CompatibilityReport(results) + } finally { + gitCommand.checkout(treeishWithChanges) + } + } + + private fun baseBranch() = baseBranch ?: gitCommand.currentRemoteBranch() + + companion object { + private const val HEAD = "HEAD" + private const val MARGIN_SPACE = " " + private const val ONE_INDENT = " " + private const val TWO_INDENTS = "${ONE_INDENT}${ONE_INDENT}" + } +} \ No newline at end of file diff --git a/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommand.kt similarity index 100% rename from application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt rename to application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommand.kt diff --git a/application/src/main/kotlin/application/backwardCompatibility/CompatibilityReport.kt b/application/src/main/kotlin/application/backwardCompatibility/CompatibilityReport.kt new file mode 100644 index 000000000..07a2f4a60 --- /dev/null +++ b/application/src/main/kotlin/application/backwardCompatibility/CompatibilityReport.kt @@ -0,0 +1,16 @@ +package application.backwardCompatibility + +class CompatibilityReport(results: List) { + val report: String + val exitCode: Int + + init { + val failed: Boolean = results.any { it == CompatibilityResult.FAILED } + val failedCount = results.count { it == CompatibilityResult.FAILED } + val passedCount = results.count { it == CompatibilityResult.PASSED } + + report = "Files checked: ${results.size} (Passed: ${passedCount}, Failed: $failedCount)" + exitCode = if(failed) 1 else 0 + } + +} diff --git a/application/src/main/kotlin/application/backwardCompatibility/CompatibilityResult.kt b/application/src/main/kotlin/application/backwardCompatibility/CompatibilityResult.kt new file mode 100644 index 000000000..c53e7255f --- /dev/null +++ b/application/src/main/kotlin/application/backwardCompatibility/CompatibilityResult.kt @@ -0,0 +1,5 @@ +package application.backwardCompatibility + +enum class CompatibilityResult { + PASSED, FAILED +} diff --git a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt index b57ea19f0..a71ea7c9b 100644 --- a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt +++ b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt @@ -1,11 +1,12 @@ package application +import application.backwardCompatibility.BackwardCompatibilityCheckCommand import io.mockk.every import io.mockk.mockk import io.mockk.spyk -import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import java.io.File @@ -13,7 +14,7 @@ class BackwardCompatibilityCheckCommandTest { @Test fun `filesReferringToChangedSchemaFiles returns empty set when input is empty`() { - val command = BackwardCompatibilityCheckCommand(mockk()) + val command = BackwardCompatibilityCheckCommand() val result = command.filesReferringToChangedSchemaFiles(emptySet()) assertTrue(result.isEmpty()) } diff --git a/application/src/test/kotlin/application/CompatibleCommandKtTest.kt b/application/src/test/kotlin/application/CompatibleCommandKtTest.kt index 120a05317..1b4d9682e 100644 --- a/application/src/test/kotlin/application/CompatibleCommandKtTest.kt +++ b/application/src/test/kotlin/application/CompatibleCommandKtTest.kt @@ -6,6 +6,7 @@ import io.specmatic.core.Result import io.specmatic.core.Results import io.specmatic.core.git.GitCommand import io.mockk.every +import io.specmatic.core.git.SystemGit import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -46,8 +47,12 @@ internal class CompatibleCommandKtTest { override val workingDirectory: String get() = "" - override fun getFilesChangeInCurrentBranch() = emptyList() - override fun getFileInTheDefaultBranch(fileName: String, currentBranch: String) = null + override fun stashPop(): SystemGit { + TODO("Not yet implemented") + } + + override fun getFilesChangedInCurrentBranch(baseBranch: String) = emptyList() + override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String) = null override fun relativeGitPath(newerContractPath: String): Pair { assertThat(newerContractPath).isEqualTo("/Users/fakeuser/newer.$CONTRACT_EXTENSION") @@ -77,9 +82,9 @@ internal class CompatibleCommandKtTest { override fun fileIsInGitDir(newerContractPath: String): Boolean = true override val workingDirectory: String get() = "" - override fun getFileInTheDefaultBranch(fileName: String, currentBranch: String) = null + override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String) = null - override fun getFilesChangeInCurrentBranch() = emptyList() + override fun getFilesChangedInCurrentBranch(baseBranch: String) = emptyList() override fun relativeGitPath(newerContractPath: String): Pair { assertThat(newerContractPath).isEqualTo("/Users/fakeuser/newer.$CONTRACT_EXTENSION") @@ -114,10 +119,10 @@ internal class CompatibleCommandKtTest { val fakeGit = object : FakeGit() { override fun fileIsInGitDir(newerContractPath: String): Boolean = true - override fun getFilesChangeInCurrentBranch() = emptyList() + override fun getFilesChangedInCurrentBranch(baseBranch: String) = emptyList() override val workingDirectory: String get() = "" - override fun getFileInTheDefaultBranch(fileName: String, currentBranch: String) = null + override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String) = null override fun relativeGitPath(newerContractPath: String): Pair { assertThat(newerContractPath).isEqualTo("/Users/fakeuser/newer.$CONTRACT_EXTENSION") @@ -151,10 +156,10 @@ internal class CompatibleCommandKtTest { val fakeGit = object : FakeGit() { override fun fileIsInGitDir(newerContractPath: String): Boolean = true - override fun getFilesChangeInCurrentBranch() = emptyList() + override fun getFilesChangedInCurrentBranch(baseBranch: String) = emptyList() override val workingDirectory: String get() = "" - override fun getFileInTheDefaultBranch(fileName: String, currentBranch: String) = null + override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String) = null override fun relativeGitPath(newerContractPath: String): Pair { assertThat(newerContractPath).isEqualTo("/Users/fakeuser/newer.$CONTRACT_EXTENSION") diff --git a/application/src/test/kotlin/application/FakeGit.kt b/application/src/test/kotlin/application/FakeGit.kt index 4e9fe8022..4a67739d7 100644 --- a/application/src/test/kotlin/application/FakeGit.kt +++ b/application/src/test/kotlin/application/FakeGit.kt @@ -16,6 +16,18 @@ abstract class FakeGit: GitCommand { TODO("Not yet implemented") } + override fun stash(): Boolean { + TODO("Not yet implemented") + } + + override fun stashPop(): SystemGit { + TODO("Not yet implemented") + } + + override fun currentRemoteBranch(): String { + TODO("Not yet implemented") + } + override fun add(relativePath: String): SystemGit { TODO("Not yet implemented") } diff --git a/core/src/main/kotlin/io/specmatic/core/Feature.kt b/core/src/main/kotlin/io/specmatic/core/Feature.kt index 475cb5f9e..38d742863 100644 --- a/core/src/main/kotlin/io/specmatic/core/Feature.kt +++ b/core/src/main/kotlin/io/specmatic/core/Feature.kt @@ -106,7 +106,7 @@ data class Feature( val stubsFromExamples: Map>> = emptyMap(), val specmaticConfig: SpecmaticConfig = SpecmaticConfig(), val flagsBased: FlagsBased = strategiesFromFlags(specmaticConfig) -) { +): IFeature { fun enableGenerativeTesting(onlyPositive: Boolean = false): Feature { val updatedSpecmaticConfig = specmaticConfig.copy( test = specmaticConfig.test?.copy( diff --git a/core/src/main/kotlin/io/specmatic/core/IFeature.kt b/core/src/main/kotlin/io/specmatic/core/IFeature.kt new file mode 100644 index 000000000..9b7f4881b --- /dev/null +++ b/core/src/main/kotlin/io/specmatic/core/IFeature.kt @@ -0,0 +1,3 @@ +package io.specmatic.core + +interface IFeature \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/core/git/GitCommand.kt b/core/src/main/kotlin/io/specmatic/core/git/GitCommand.kt index 904c0c28a..48d28ca44 100644 --- a/core/src/main/kotlin/io/specmatic/core/git/GitCommand.kt +++ b/core/src/main/kotlin/io/specmatic/core/git/GitCommand.kt @@ -9,6 +9,8 @@ interface GitCommand { fun commit(): SystemGit fun push(): SystemGit fun pull(): SystemGit + fun stash(): Boolean + fun stashPop(): SystemGit fun resetHard(): SystemGit fun resetMixed(): SystemGit fun mergeAbort(): SystemGit @@ -30,8 +32,9 @@ interface GitCommand { fun revisionsBehindCount(): Int fun getRemoteUrl(name: String = "origin"): String fun checkIgnore(path: String): String - fun getFilesChangeInCurrentBranch(): List - fun getFileInTheDefaultBranch(fileName: String, currentBranch: String): File? + fun getFilesChangedInCurrentBranch(baseBranch: String): List + fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String): File? + fun currentRemoteBranch(): String fun currentBranch(): String { return "" } @@ -43,4 +46,5 @@ interface GitCommand { fun detachedHEAD(): String { return "" } + } diff --git a/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt b/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt index 48f0c5113..839a24254 100644 --- a/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt +++ b/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt @@ -58,6 +58,14 @@ class SystemGit(override val workingDirectory: String = ".", private val prefix: } } + override fun stash(): Boolean { + val stashListSizeBefore = getStashListSize() + execute(Configuration.gitCommand, "stash", "push", "-m", "tmp") + return getStashListSize() > stashListSizeBefore + } + + override fun stashPop(): SystemGit = this.also { execute(Configuration.gitCommand, "stash", "pop") } + override fun getCurrentBranch(): String { return execute(Configuration.gitCommand, "git", "diff", "--name-only", "master") } @@ -82,17 +90,21 @@ class SystemGit(override val workingDirectory: String = ".", private val prefix: } } - override fun getFilesChangeInCurrentBranch(): List { - val defaultBranch = defaultBranch() + override fun getFilesChangedInCurrentBranch(baseBranch: String): List { + val committedLocalChanges = execute(Configuration.gitCommand, "diff", baseBranch, "HEAD", "--name-only") + .split(System.lineSeparator()) + .filter { it.isNotBlank() } - val result = execute(Configuration.gitCommand, "diff", defaultBranch, "HEAD", "--name-only") + val uncommittedChanges = execute(Configuration.gitCommand, "diff", "--name-only") + .split(System.lineSeparator()) + .filter { it.isNotBlank() } - return result.split(System.lineSeparator()).filter { it.isNotBlank() } + return (committedLocalChanges + uncommittedChanges).distinct() } - override fun getFileInTheDefaultBranch(fileName: String, currentBranch: String): File? { + override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String): File? { try { - checkout(defaultBranch()) + if(baseBranch != currentBranch) checkout(baseBranch) if (!File(fileName).exists()) return null return File(fileName) @@ -156,6 +168,15 @@ class SystemGit(override val workingDirectory: String = ".", private val prefix: return execute(Configuration.gitCommand, "rev-parse", "--abbrev-ref", "HEAD").trim() } + override fun currentRemoteBranch(): String { + val branchStatus = execute(Configuration.gitCommand, "status", "-b", "--porcelain=2").trim() + val hasUpstream = branchStatus.lines().any { it.startsWith("# branch.upstream") } + if (hasUpstream) { + return execute(Configuration.gitCommand, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}").trim() + } + return currentBranch() + } + override fun defaultBranch(): String { System.getenv("LOCAL_GIT_BRANCH")?.let { return it @@ -174,6 +195,10 @@ class SystemGit(override val workingDirectory: String = ".", private val prefix: return symbolicRef.split("/")[1].trim() } + private fun getStashListSize(): Int { + return execute(Configuration.gitCommand, "stash", "list").trim().lines().size + } + override fun detachedHEAD(): String { val result = execute(Configuration.gitCommand, "show", "-s", "--pretty=%D", "HEAD") return result.trim().split(",")[1].trim() From a7bfa424e6a23a09bf82cc246fb4fe65865796b3 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 15 Aug 2024 17:07:08 +0530 Subject: [PATCH 02/43] Add support for --target-path argument in BackwardCompatibilityCheckCommand which can let us run the check on a specific folder or file in a repository --- .../kotlin/application/SpecmaticCommand.kt | 3 +- .../backwardCompatibility/BCCheckCommand.kt | 137 -------- .../BackwardCompatibilityCheckBaseCommand.kt | 11 +- .../BackwardCompatibilityCheckCommandV1.kt | 310 ++++++++++++++++++ ...ackwardCompatibilityCheckCommandV1Test.kt} | 13 +- 5 files changed, 327 insertions(+), 147 deletions(-) delete mode 100644 application/src/main/kotlin/application/backwardCompatibility/BCCheckCommand.kt create mode 100644 application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV1.kt rename application/src/test/kotlin/application/{BackwardCompatibilityCheckCommandTest.kt => BackwardCompatibilityCheckCommandV1Test.kt} (89%) diff --git a/application/src/main/kotlin/application/SpecmaticCommand.kt b/application/src/main/kotlin/application/SpecmaticCommand.kt index 3f301577b..170d188e1 100644 --- a/application/src/main/kotlin/application/SpecmaticCommand.kt +++ b/application/src/main/kotlin/application/SpecmaticCommand.kt @@ -1,6 +1,5 @@ package application -import application.backwardCompatibility.BCCheckCommand import application.backwardCompatibility.BackwardCompatibilityCheckCommand import org.springframework.stereotype.Component import picocli.AutoComplete.GenerateCompletion @@ -12,7 +11,7 @@ import java.util.concurrent.Callable name = "specmatic", mixinStandardHelpOptions = true, versionProvider = VersionProvider::class, - subcommands = [BCCheckCommand::class, BundleCommand::class, CompareCommand::class, CompatibleCommand::class, DifferenceCommand::class, GenerateCompletion::class, GraphCommand::class, MergeCommand::class, ToOpenAPICommand::class, ImportCommand::class, InstallCommand::class, ProxyCommand::class, PushCommand::class, ReDeclaredAPICommand::class, ExamplesCommand::class, SamplesCommand::class, StubCommand::class, SubscribeCommand::class, TestCommand::class, ValidateViaLogs::class, CentralContractRepoReportCommand::class] + subcommands = [BackwardCompatibilityCheckCommand::class, BundleCommand::class, CompareCommand::class, CompatibleCommand::class, DifferenceCommand::class, GenerateCompletion::class, GraphCommand::class, MergeCommand::class, ToOpenAPICommand::class, ImportCommand::class, InstallCommand::class, ProxyCommand::class, PushCommand::class, ReDeclaredAPICommand::class, ExamplesCommand::class, SamplesCommand::class, StubCommand::class, SubscribeCommand::class, TestCommand::class, ValidateViaLogs::class, CentralContractRepoReportCommand::class] ) class SpecmaticCommand : Callable { override fun call(): Int { diff --git a/application/src/main/kotlin/application/backwardCompatibility/BCCheckCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BCCheckCommand.kt deleted file mode 100644 index e3747c49e..000000000 --- a/application/src/main/kotlin/application/backwardCompatibility/BCCheckCommand.kt +++ /dev/null @@ -1,137 +0,0 @@ -package application.backwardCompatibility - -import io.specmatic.conversions.OpenApiSpecification -import io.specmatic.core.CONTRACT_EXTENSION -import io.specmatic.core.CONTRACT_EXTENSIONS -import io.specmatic.core.Feature -import io.specmatic.core.IFeature -import io.specmatic.core.Results -import io.specmatic.core.WSDL -import io.specmatic.core.testBackwardCompatibility -import io.specmatic.stub.isOpenAPI -import org.springframework.stereotype.Component -import picocli.CommandLine.Command -import java.io.File -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.regex.Pattern -import kotlin.io.path.extension -import kotlin.io.path.pathString - -@Component -@Command( - name = "backwardCompatibilityCheck", - aliases = ["backward-compatibility-check"], - mixinStandardHelpOptions = true, - description = ["Checks backward compatibility of a directory across the current HEAD and the base branch"] -) -class BCCheckCommand: BackwardCompatibilityCheckBaseCommand() { - - override fun checkBackwardCompatibility(oldFeature: IFeature, newFeature: IFeature): Results { - return testBackwardCompatibility(oldFeature as Feature, newFeature as Feature) - } - - override fun File.isValidSpec(): Boolean { - if (this.extension !in CONTRACT_EXTENSIONS) return false - return OpenApiSpecification.isParsable(this.path) - } - - override fun getFeatureFromSpecPath(path: String): Feature { - return OpenApiSpecification.fromFile(path).toFeature() - } - - override fun getSpecsReferringTo(schemaFiles: Set): Set { - if (schemaFiles.isEmpty()) return emptySet() - - val inputFileNames = schemaFiles.map { File(it).name } - val result = allOpenApiSpecFiles().filter { - it.readText().trim().let { specContent -> - inputFileNames.any { inputFileName -> - val pattern = Pattern.compile("\\b$inputFileName\\b") - val matcher = pattern.matcher(specContent) - matcher.find() - } - } - }.map { it.path }.toSet() - - return result.flatMap { - getSpecsReferringTo(setOf(it)).ifEmpty { setOf(it) } - }.toSet() - } - - override fun getSpecsOfChangedExternalisedExamples(filesChangedInCurrentBranch: Set): Set { - data class CollectedFiles( - val specifications: MutableSet = mutableSetOf(), - val examplesMissingSpecifications: MutableList = mutableListOf(), - val ignoredFiles: MutableList = mutableListOf() - ) - - val collectedFiles = filesChangedInCurrentBranch.fold(CollectedFiles()) { acc, filePath -> - val path = Paths.get(filePath) - val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } - - if (examplesDir == null) { - acc.ignoredFiles.add(filePath) - } else { - val parentPath = examplesDir.parent - val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) - val specFiles = findSpecFiles(strippedPath) - - if (specFiles.isNotEmpty()) { - acc.specifications.addAll(specFiles.map { it.toString() }) - } else { - acc.examplesMissingSpecifications.add(filePath) - } - } - acc - } - - val result = collectedFiles.specifications.toMutableSet() - - collectedFiles.examplesMissingSpecifications.forEach { filePath -> - val path = Paths.get(filePath) - val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } - if (examplesDir != null) { - val parentPath = examplesDir.parent - val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) - val specFiles = findSpecFiles(strippedPath) - if (specFiles.isNotEmpty()) { - result.addAll(specFiles.map { it.toString() }) - } else { - result.add("${strippedPath}.yaml") - } - } - } - - return result - } - - override fun areExamplesValid(feature: IFeature, which: String): Boolean { - feature as Feature - return try { - feature.validateExamplesOrException() - true - } catch (t: Throwable) { - println() - false - } - } - - override fun getUnusedExamples(feature: IFeature): Set { - feature as Feature - return feature.loadExternalisedExamplesAndListUnloadableExamples().second - } - - private fun allOpenApiSpecFiles(): List { - return File(".").walk().toList().filterNot { - ".git" in it.path - }.filter { it.isFile && it.isValidSpec() } - } - - private fun findSpecFiles(path: Path): List { - val extensions = CONTRACT_EXTENSIONS - return extensions.map { path.resolveSibling(path.fileName.toString() + it) } - .filter { Files.exists(it) && (isOpenAPI(it.pathString) || it.extension in listOf(WSDL, CONTRACT_EXTENSION)) } - } -} \ No newline at end of file diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 44493635e..d281efa5c 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -18,6 +18,9 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { @Option(names = ["--base-branch"], description = ["Base branch to compare the changes against"], required = false) var baseBranch: String? = null + @Option(names = ["--target-path"], description = ["Specification file or folder to run the check against"], required = false) + var targetPath: String? = null + abstract fun checkBackwardCompatibility(oldFeature: IFeature, newFeature: IFeature): Results abstract fun File.isValidSpec(): Boolean abstract fun getFeatureFromSpecPath(path: String): IFeature @@ -48,9 +51,15 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { filesReferringToChangedSchemaFiles + specificationsOfChangedExternalisedExamples + val filteredSpecs = if(targetPath.isNullOrBlank()) { + specificationsToCheck + } else { + specificationsToCheck.filter { it.contains(targetPath!!) }.toSet() + } + val result = try { runBackwardCompatibilityCheckFor( - files = specificationsToCheck, + files = filteredSpecs, baseBranch = baseBranch() ) } catch(e: Throwable) { diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV1.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV1.kt new file mode 100644 index 000000000..cfda2698b --- /dev/null +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV1.kt @@ -0,0 +1,310 @@ +package application.backwardCompatibility + +import io.specmatic.conversions.OpenApiSpecification +import io.specmatic.core.* +import io.specmatic.core.git.GitCommand +import io.specmatic.core.git.SystemGit +import io.specmatic.core.log.logger +import io.specmatic.core.utilities.exitWithMessage +import io.specmatic.stub.isOpenAPI +import picocli.CommandLine.Option +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.Callable +import java.util.regex.Pattern +import kotlin.io.path.extension +import kotlin.io.path.pathString +import kotlin.system.exitProcess + +//@Component +//@Command( +// name = "backwardCompatibilityCheck", +// aliases = ["backward-compatibility-check"], +// mixinStandardHelpOptions = true, +// description = ["Checks backward compatibility of a directory across the current HEAD and the base branch"] +//) +// TODO - knock this off. +class BackwardCompatibilityCheckCommandV1() : Callable { + private val gitCommand: GitCommand = SystemGit() + private val newLine = System.lineSeparator() + + @Option(names = ["--base-branch"], description = ["Base branch to compare the changes against"], required = false) + var baseBranch: String = gitCommand.currentRemoteBranch() + + companion object { + private const val HEAD = "HEAD" + private const val MARGIN_SPACE = " " + private const val ONE_INDENT = " " + private const val TWO_INDENTS = "${ONE_INDENT}${ONE_INDENT}" + } + + override fun call() { + val filesChangedInCurrentBranch = getChangedSpecsInCurrentBranch() + val filesReferringToChangedSchemaFiles = filesReferringToChangedSchemaFiles(filesChangedInCurrentBranch) + val specificationsOfChangedExternalisedExamples = + getSpecificationsOfChangedExternalisedExamples(filesChangedInCurrentBranch) + + logFilesToBeCheckedForBackwardCompatibility( + filesChangedInCurrentBranch, + filesReferringToChangedSchemaFiles, + specificationsOfChangedExternalisedExamples + ) + + val specificationsToCheck: Set = + filesChangedInCurrentBranch + + filesReferringToChangedSchemaFiles + + specificationsOfChangedExternalisedExamples + + val result = try { + runBackwardCompatibilityCheckFor( + files = specificationsToCheck, + baseBranch = baseBranch + ) + } catch(e: Throwable) { + logger.newLine() + logger.newLine() + logger.log(e) + exitProcess(1) + } + + println() + println(result.report) + exitProcess(result.exitCode) + } + + private fun getSpecificationsOfChangedExternalisedExamples(filesChangedInCurrentBranch: Set): Set { + data class CollectedFiles( + val specifications: MutableSet = mutableSetOf(), + val examplesMissingSpecifications: MutableList = mutableListOf(), + val ignoredFiles: MutableList = mutableListOf() + ) + + val collectedFiles = filesChangedInCurrentBranch.fold(CollectedFiles()) { acc, filePath -> + val path = Paths.get(filePath) + val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } + + if (examplesDir == null) { + acc.ignoredFiles.add(filePath) + } else { + val parentPath = examplesDir.parent + val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) + val specFiles = findSpecFiles(strippedPath) + + if (specFiles.isNotEmpty()) { + acc.specifications.addAll(specFiles.map { it.toString() }) + } else { + acc.examplesMissingSpecifications.add(filePath) + } + } + acc + } + + val result = collectedFiles.specifications.toMutableSet() + + collectedFiles.examplesMissingSpecifications.forEach { filePath -> + val path = Paths.get(filePath) + val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } + if (examplesDir != null) { + val parentPath = examplesDir.parent + val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) + val specFiles = findSpecFiles(strippedPath) + if (specFiles.isNotEmpty()) { + result.addAll(specFiles.map { it.toString() }) + } else { + result.add("${strippedPath}.yaml") + } + } + } + + return result + } + + private fun Path.find(predicate: (Path) -> Boolean): Path? { + var current: Path? = this + while (current != null) { + if (predicate(current)) { + return current + } + current = current.parent + } + return null + } + + private fun findSpecFiles(path: Path): List { + val extensions = CONTRACT_EXTENSIONS + return extensions.map { path.resolveSibling(path.fileName.toString() + it) } + .filter { Files.exists(it) && (isOpenAPI(it.pathString) || it.extension in listOf(WSDL, CONTRACT_EXTENSION)) } + } + + fun runBackwardCompatibilityCheckFor(files: Set, baseBranch: String): CompatibilityReport { + val branchWithChanges = gitCommand.currentBranch() + val treeishWithChanges = if (branchWithChanges == HEAD) gitCommand.detachedHEAD() else branchWithChanges + + try { + val results = files.mapIndexed { index, specFilePath -> + try { + println("${index.inc()}. Running the check for $specFilePath:") + + // newer => the file with changes on the branch + val newer = getFeatureFromSpecPath(specFilePath) + val unusedExamples = getUnusedExamples(newer) + + val olderFile = gitCommand.getFileInTheBaseBranch( + specFilePath, + treeishWithChanges, + baseBranch + ) + if (olderFile == null) { + println("$specFilePath is a new file.$newLine") + return@mapIndexed CompatibilityResult.PASSED + } + + gitCommand.stash() + gitCommand.checkout(baseBranch) + // older => the same file on the default (e.g. main) branch + val older = getFeatureFromSpecPath(olderFile.path) + gitCommand.stashPop() + + val backwardCompatibilityResult = checkBackwardCompatibility(older, newer) + + if (backwardCompatibilityResult.success()) { + println( + "$newLine The file $specFilePath is backward compatible.$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + var errorsFound = false + + if(!areExamplesValid(newer, "newer")) { + println( + "$newLine *** Examples in $specFilePath are not valid. ***$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + errorsFound = true + } + + if(unusedExamples.isNotEmpty()) { + println( + "$newLine *** Some examples for $specFilePath could not be loaded. ***$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + errorsFound = true + } + + if(errorsFound) CompatibilityResult.FAILED + else CompatibilityResult.PASSED + } else { + println("$newLine ${backwardCompatibilityResult.report().prependIndent(MARGIN_SPACE)}") + println( + "$newLine *** The file $specFilePath is NOT backward compatible. ***$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + CompatibilityResult.FAILED + } + } finally { + gitCommand.checkout(treeishWithChanges) + } + } + + return CompatibilityReport(results) + } finally { + gitCommand.checkout(treeishWithChanges) + } + } + + fun logFilesToBeCheckedForBackwardCompatibility( + changedFiles: Set, + filesReferringToChangedFiles: Set, + specificationsOfChangedExternalisedExamples: Set + ) { + + println("Checking backward compatibility of the following files: $newLine") + println("${ONE_INDENT}Files that have changed:") + changedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } + println() + + if(filesReferringToChangedFiles.isNotEmpty()) { + println("${ONE_INDENT}Files referring to the changed files - ") + filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } + println() + } + + if(specificationsOfChangedExternalisedExamples.isNotEmpty()) { + println("${ONE_INDENT}Specifications whose externalised examples were changed - ") + filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } + println() + } + + println("-".repeat(20)) + println() + } + + internal fun filesReferringToChangedSchemaFiles(inputFiles: Set): Set { + if (inputFiles.isEmpty()) return emptySet() + + val inputFileNames = inputFiles.map { File(it).name } + val result = allOpenApiSpecFiles().filter { + it.readText().trim().let { specContent -> + inputFileNames.any { inputFileName -> + val pattern = Pattern.compile("\\b$inputFileName\\b") + val matcher = pattern.matcher(specContent) + matcher.find() + } + } + }.map { it.path }.toSet() + + return result.flatMap { + filesReferringToChangedSchemaFiles(setOf(it)).ifEmpty { setOf(it) } + }.toSet() + } + + internal fun allOpenApiSpecFiles(): List { + return File(".").walk().toList().filterNot { + ".git" in it.path + }.filter { it.isFile && it.isValidSpec() } + } + + private fun getChangedSpecsInCurrentBranch(): Set { + return gitCommand.getFilesChangedInCurrentBranch(baseBranch).filter { + File(it).exists() && File(it).isValidSpec() + }.toSet().also { + if(it.isEmpty()) exitWithMessage("${newLine}No specs were changed, skipping the check.$newLine") + } + } + + private fun File.isValidSpec(): Boolean { + if (this.extension !in CONTRACT_EXTENSIONS) return false + return OpenApiSpecification.isParsable(this.path) + } + + private fun areExamplesValid(feature: Feature, which: String): Boolean { + return try { + feature.validateExamplesOrException() + true + } catch (t: Throwable) { + println() + false + } + } + + private fun getUnusedExamples(feature: Feature): Set { + return feature.loadExternalisedExamplesAndListUnloadableExamples().second + } + + private fun getFeatureFromSpecPath(path: String): Feature { + return OpenApiSpecification.fromFile(path).toFeature() + } + + private fun checkBackwardCompatibility(oldFeature: Feature, newFeature: Feature): Results { + return testBackwardCompatibility(oldFeature, newFeature) + } +} diff --git a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV1Test.kt similarity index 89% rename from application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt rename to application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV1Test.kt index a71ea7c9b..e75d6c424 100644 --- a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt +++ b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV1Test.kt @@ -1,8 +1,7 @@ package application -import application.backwardCompatibility.BackwardCompatibilityCheckCommand +import application.backwardCompatibility.BackwardCompatibilityCheckCommandV1 import io.mockk.every -import io.mockk.mockk import io.mockk.spyk import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals @@ -10,18 +9,18 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import java.io.File -class BackwardCompatibilityCheckCommandTest { +class BackwardCompatibilityCheckCommandV1Test { @Test fun `filesReferringToChangedSchemaFiles returns empty set when input is empty`() { - val command = BackwardCompatibilityCheckCommand() + val command = BackwardCompatibilityCheckCommandV1() val result = command.filesReferringToChangedSchemaFiles(emptySet()) assertTrue(result.isEmpty()) } @Test fun `filesReferringToChangedSchemaFiles returns empty set when no files refer to changed schema files`() { - val command = spyk() + val command = spyk() every { command.allOpenApiSpecFiles() } returns listOf( File("file1.yaml").apply { writeText("content1") }, File("file2.yaml").apply { writeText("content2") } @@ -32,7 +31,7 @@ class BackwardCompatibilityCheckCommandTest { @Test fun `filesReferringToChangedSchemaFiles returns set of files that refer to changed schema files`() { - val command = spyk() + val command = spyk() every { command.allOpenApiSpecFiles() } returns listOf( File("file1.yaml").apply { writeText("file3.yaml") }, File("file2.yaml").apply { writeText("file4.yaml") } @@ -43,7 +42,7 @@ class BackwardCompatibilityCheckCommandTest { @Test fun `filesReferringToChangedSchemaFiles returns set of files which are referring to a changed schema that is one level down`() { - val command = spyk() + val command = spyk() every { command.allOpenApiSpecFiles() } returns listOf( File("file1.yaml").apply { referTo("schema_file1.yaml") }, File("schema_file2.yaml").apply { referTo("schema_file1.yaml") }, // schema within a schema From 8c7c110c34f326b3c06c36adccfbd081bc5cddd3 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 15 Aug 2024 17:50:30 +0530 Subject: [PATCH 03/43] Move the function to find referred spec files within BCC base class since the logic is reusable. --- .../BackwardCompatibilityCheckBaseCommand.kt | 85 +++-- .../BackwardCompatibilityCheckCommandV1.kt | 310 ------------------ ... BackwardCompatibilityCheckCommandTest.kt} | 34 +- 3 files changed, 74 insertions(+), 355 deletions(-) delete mode 100644 application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV1.kt rename application/src/test/kotlin/application/{BackwardCompatibilityCheckCommandV1Test.kt => BackwardCompatibilityCheckCommandTest.kt} (57%) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index d281efa5c..967664c8e 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -9,6 +9,7 @@ import io.specmatic.core.utilities.exitWithMessage import picocli.CommandLine.Option import java.io.File import java.util.concurrent.Callable +import java.util.regex.Pattern import kotlin.system.exitProcess abstract class BackwardCompatibilityCheckBaseCommand : Callable { @@ -19,44 +20,22 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { var baseBranch: String? = null @Option(names = ["--target-path"], description = ["Specification file or folder to run the check against"], required = false) - var targetPath: String? = null + var targetPath: String = "" abstract fun checkBackwardCompatibility(oldFeature: IFeature, newFeature: IFeature): Results abstract fun File.isValidSpec(): Boolean abstract fun getFeatureFromSpecPath(path: String): IFeature - abstract fun getSpecsReferringTo(schemaFiles: Set): Set + abstract fun regexForMatchingReferred(schemaFileName: String): String abstract fun getSpecsOfChangedExternalisedExamples( filesChangedInCurrentBranch: Set ): Set - open fun areExamplesValid(feature: IFeature, which: String): Boolean = true open fun getUnusedExamples(feature: IFeature): Set = emptySet() - override fun call() { - val filesChangedInCurrentBranch = getChangedSpecsInCurrentBranch() - val filesReferringToChangedSchemaFiles = getSpecsReferringTo(filesChangedInCurrentBranch) - val specificationsOfChangedExternalisedExamples = - getSpecsOfChangedExternalisedExamples(filesChangedInCurrentBranch) - - logFilesToBeCheckedForBackwardCompatibility( - filesChangedInCurrentBranch, - filesReferringToChangedSchemaFiles, - specificationsOfChangedExternalisedExamples - ) - - val specificationsToCheck: Set = - filesChangedInCurrentBranch + - filesReferringToChangedSchemaFiles + - specificationsOfChangedExternalisedExamples - - val filteredSpecs = if(targetPath.isNullOrBlank()) { - specificationsToCheck - } else { - specificationsToCheck.filter { it.contains(targetPath!!) }.toSet() - } - + final override fun call() { + val filteredSpecs = getChangedSpecs(logSpecs = true) val result = try { runBackwardCompatibilityCheckFor( files = filteredSpecs, @@ -74,6 +53,30 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { exitProcess(result.exitCode) } + fun getChangedSpecs(logSpecs: Boolean = false): Set { + val filesChangedInCurrentBranch = getChangedSpecsInCurrentBranch() + val filesReferringToChangedSchemaFiles = getSpecsReferringTo(filesChangedInCurrentBranch) + val specificationsOfChangedExternalisedExamples = + getSpecsOfChangedExternalisedExamples(filesChangedInCurrentBranch) + + if(logSpecs) { + logFilesToBeCheckedForBackwardCompatibility( + filesChangedInCurrentBranch, + filesReferringToChangedSchemaFiles, + specificationsOfChangedExternalisedExamples + ) + } + + val specificationsToCheck: Set = + filesChangedInCurrentBranch + + filesReferringToChangedSchemaFiles + + specificationsOfChangedExternalisedExamples + + val filteredSpecs = specificationsToCheck.filter { it.contains(targetPath) }.toSet() + + return filteredSpecs + } + private fun getChangedSpecsInCurrentBranch(): Set { return gitCommand.getFilesChangedInCurrentBranch( baseBranch() @@ -84,6 +87,31 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { } } + internal fun getSpecsReferringTo(schemaFiles: Set): Set { + if (schemaFiles.isEmpty()) return emptySet() + + val inputFileNames = schemaFiles.map { File(it).name } + val result = allSpecFiles().filter { + it.readText().trim().let { specContent -> + inputFileNames.any { inputFileName -> + val pattern = Pattern.compile("\\b${regexForMatchingReferred(inputFileName)}\\b") + val matcher = pattern.matcher(specContent) + matcher.find() + } + } + }.map { it.path }.toSet() + + return result.flatMap { + getSpecsReferringTo(setOf(it)).ifEmpty { setOf(it) } + }.toSet() + } + + internal fun allSpecFiles(): List { + return File(".").walk().toList().filterNot { + ".git" in it.path + }.filter { it.isFile && it.isValidSpec() } + } + private fun logFilesToBeCheckedForBackwardCompatibility( changedFiles: Set, filesReferringToChangedFiles: Set, @@ -117,6 +145,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { try { val results = files.mapIndexed { index, specFilePath -> + var areLocalChangesStashed = false try { println("${index.inc()}. Running the check for $specFilePath:") @@ -134,11 +163,10 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { return@mapIndexed CompatibilityResult.PASSED } - val areLocalChangesStashed = gitCommand.stash() + areLocalChangesStashed = gitCommand.stash() gitCommand.checkout(baseBranch) // older => the same file on the default (e.g. main) branch val older = getFeatureFromSpecPath(olderFile.path) - if (areLocalChangesStashed) gitCommand.stashPop() val backwardCompatibilityResult = checkBackwardCompatibility(older, newer) @@ -187,6 +215,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { } } finally { gitCommand.checkout(treeishWithChanges) + if (areLocalChangesStashed) gitCommand.stashPop() } } diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV1.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV1.kt deleted file mode 100644 index cfda2698b..000000000 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV1.kt +++ /dev/null @@ -1,310 +0,0 @@ -package application.backwardCompatibility - -import io.specmatic.conversions.OpenApiSpecification -import io.specmatic.core.* -import io.specmatic.core.git.GitCommand -import io.specmatic.core.git.SystemGit -import io.specmatic.core.log.logger -import io.specmatic.core.utilities.exitWithMessage -import io.specmatic.stub.isOpenAPI -import picocli.CommandLine.Option -import java.io.File -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.util.concurrent.Callable -import java.util.regex.Pattern -import kotlin.io.path.extension -import kotlin.io.path.pathString -import kotlin.system.exitProcess - -//@Component -//@Command( -// name = "backwardCompatibilityCheck", -// aliases = ["backward-compatibility-check"], -// mixinStandardHelpOptions = true, -// description = ["Checks backward compatibility of a directory across the current HEAD and the base branch"] -//) -// TODO - knock this off. -class BackwardCompatibilityCheckCommandV1() : Callable { - private val gitCommand: GitCommand = SystemGit() - private val newLine = System.lineSeparator() - - @Option(names = ["--base-branch"], description = ["Base branch to compare the changes against"], required = false) - var baseBranch: String = gitCommand.currentRemoteBranch() - - companion object { - private const val HEAD = "HEAD" - private const val MARGIN_SPACE = " " - private const val ONE_INDENT = " " - private const val TWO_INDENTS = "${ONE_INDENT}${ONE_INDENT}" - } - - override fun call() { - val filesChangedInCurrentBranch = getChangedSpecsInCurrentBranch() - val filesReferringToChangedSchemaFiles = filesReferringToChangedSchemaFiles(filesChangedInCurrentBranch) - val specificationsOfChangedExternalisedExamples = - getSpecificationsOfChangedExternalisedExamples(filesChangedInCurrentBranch) - - logFilesToBeCheckedForBackwardCompatibility( - filesChangedInCurrentBranch, - filesReferringToChangedSchemaFiles, - specificationsOfChangedExternalisedExamples - ) - - val specificationsToCheck: Set = - filesChangedInCurrentBranch + - filesReferringToChangedSchemaFiles + - specificationsOfChangedExternalisedExamples - - val result = try { - runBackwardCompatibilityCheckFor( - files = specificationsToCheck, - baseBranch = baseBranch - ) - } catch(e: Throwable) { - logger.newLine() - logger.newLine() - logger.log(e) - exitProcess(1) - } - - println() - println(result.report) - exitProcess(result.exitCode) - } - - private fun getSpecificationsOfChangedExternalisedExamples(filesChangedInCurrentBranch: Set): Set { - data class CollectedFiles( - val specifications: MutableSet = mutableSetOf(), - val examplesMissingSpecifications: MutableList = mutableListOf(), - val ignoredFiles: MutableList = mutableListOf() - ) - - val collectedFiles = filesChangedInCurrentBranch.fold(CollectedFiles()) { acc, filePath -> - val path = Paths.get(filePath) - val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } - - if (examplesDir == null) { - acc.ignoredFiles.add(filePath) - } else { - val parentPath = examplesDir.parent - val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) - val specFiles = findSpecFiles(strippedPath) - - if (specFiles.isNotEmpty()) { - acc.specifications.addAll(specFiles.map { it.toString() }) - } else { - acc.examplesMissingSpecifications.add(filePath) - } - } - acc - } - - val result = collectedFiles.specifications.toMutableSet() - - collectedFiles.examplesMissingSpecifications.forEach { filePath -> - val path = Paths.get(filePath) - val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } - if (examplesDir != null) { - val parentPath = examplesDir.parent - val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) - val specFiles = findSpecFiles(strippedPath) - if (specFiles.isNotEmpty()) { - result.addAll(specFiles.map { it.toString() }) - } else { - result.add("${strippedPath}.yaml") - } - } - } - - return result - } - - private fun Path.find(predicate: (Path) -> Boolean): Path? { - var current: Path? = this - while (current != null) { - if (predicate(current)) { - return current - } - current = current.parent - } - return null - } - - private fun findSpecFiles(path: Path): List { - val extensions = CONTRACT_EXTENSIONS - return extensions.map { path.resolveSibling(path.fileName.toString() + it) } - .filter { Files.exists(it) && (isOpenAPI(it.pathString) || it.extension in listOf(WSDL, CONTRACT_EXTENSION)) } - } - - fun runBackwardCompatibilityCheckFor(files: Set, baseBranch: String): CompatibilityReport { - val branchWithChanges = gitCommand.currentBranch() - val treeishWithChanges = if (branchWithChanges == HEAD) gitCommand.detachedHEAD() else branchWithChanges - - try { - val results = files.mapIndexed { index, specFilePath -> - try { - println("${index.inc()}. Running the check for $specFilePath:") - - // newer => the file with changes on the branch - val newer = getFeatureFromSpecPath(specFilePath) - val unusedExamples = getUnusedExamples(newer) - - val olderFile = gitCommand.getFileInTheBaseBranch( - specFilePath, - treeishWithChanges, - baseBranch - ) - if (olderFile == null) { - println("$specFilePath is a new file.$newLine") - return@mapIndexed CompatibilityResult.PASSED - } - - gitCommand.stash() - gitCommand.checkout(baseBranch) - // older => the same file on the default (e.g. main) branch - val older = getFeatureFromSpecPath(olderFile.path) - gitCommand.stashPop() - - val backwardCompatibilityResult = checkBackwardCompatibility(older, newer) - - if (backwardCompatibilityResult.success()) { - println( - "$newLine The file $specFilePath is backward compatible.$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - var errorsFound = false - - if(!areExamplesValid(newer, "newer")) { - println( - "$newLine *** Examples in $specFilePath are not valid. ***$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - errorsFound = true - } - - if(unusedExamples.isNotEmpty()) { - println( - "$newLine *** Some examples for $specFilePath could not be loaded. ***$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - errorsFound = true - } - - if(errorsFound) CompatibilityResult.FAILED - else CompatibilityResult.PASSED - } else { - println("$newLine ${backwardCompatibilityResult.report().prependIndent(MARGIN_SPACE)}") - println( - "$newLine *** The file $specFilePath is NOT backward compatible. ***$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - CompatibilityResult.FAILED - } - } finally { - gitCommand.checkout(treeishWithChanges) - } - } - - return CompatibilityReport(results) - } finally { - gitCommand.checkout(treeishWithChanges) - } - } - - fun logFilesToBeCheckedForBackwardCompatibility( - changedFiles: Set, - filesReferringToChangedFiles: Set, - specificationsOfChangedExternalisedExamples: Set - ) { - - println("Checking backward compatibility of the following files: $newLine") - println("${ONE_INDENT}Files that have changed:") - changedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } - println() - - if(filesReferringToChangedFiles.isNotEmpty()) { - println("${ONE_INDENT}Files referring to the changed files - ") - filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } - println() - } - - if(specificationsOfChangedExternalisedExamples.isNotEmpty()) { - println("${ONE_INDENT}Specifications whose externalised examples were changed - ") - filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } - println() - } - - println("-".repeat(20)) - println() - } - - internal fun filesReferringToChangedSchemaFiles(inputFiles: Set): Set { - if (inputFiles.isEmpty()) return emptySet() - - val inputFileNames = inputFiles.map { File(it).name } - val result = allOpenApiSpecFiles().filter { - it.readText().trim().let { specContent -> - inputFileNames.any { inputFileName -> - val pattern = Pattern.compile("\\b$inputFileName\\b") - val matcher = pattern.matcher(specContent) - matcher.find() - } - } - }.map { it.path }.toSet() - - return result.flatMap { - filesReferringToChangedSchemaFiles(setOf(it)).ifEmpty { setOf(it) } - }.toSet() - } - - internal fun allOpenApiSpecFiles(): List { - return File(".").walk().toList().filterNot { - ".git" in it.path - }.filter { it.isFile && it.isValidSpec() } - } - - private fun getChangedSpecsInCurrentBranch(): Set { - return gitCommand.getFilesChangedInCurrentBranch(baseBranch).filter { - File(it).exists() && File(it).isValidSpec() - }.toSet().also { - if(it.isEmpty()) exitWithMessage("${newLine}No specs were changed, skipping the check.$newLine") - } - } - - private fun File.isValidSpec(): Boolean { - if (this.extension !in CONTRACT_EXTENSIONS) return false - return OpenApiSpecification.isParsable(this.path) - } - - private fun areExamplesValid(feature: Feature, which: String): Boolean { - return try { - feature.validateExamplesOrException() - true - } catch (t: Throwable) { - println() - false - } - } - - private fun getUnusedExamples(feature: Feature): Set { - return feature.loadExternalisedExamplesAndListUnloadableExamples().second - } - - private fun getFeatureFromSpecPath(path: String): Feature { - return OpenApiSpecification.fromFile(path).toFeature() - } - - private fun checkBackwardCompatibility(oldFeature: Feature, newFeature: Feature): Results { - return testBackwardCompatibility(oldFeature, newFeature) - } -} diff --git a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV1Test.kt b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt similarity index 57% rename from application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV1Test.kt rename to application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt index e75d6c424..6873da05c 100644 --- a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV1Test.kt +++ b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt @@ -1,6 +1,6 @@ package application -import application.backwardCompatibility.BackwardCompatibilityCheckCommandV1 +import application.backwardCompatibility.BackwardCompatibilityCheckCommand import io.mockk.every import io.mockk.spyk import org.junit.jupiter.api.AfterEach @@ -9,46 +9,46 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import java.io.File -class BackwardCompatibilityCheckCommandV1Test { +class BackwardCompatibilityCheckCommandTest { @Test - fun `filesReferringToChangedSchemaFiles returns empty set when input is empty`() { - val command = BackwardCompatibilityCheckCommandV1() - val result = command.filesReferringToChangedSchemaFiles(emptySet()) + fun `getSpecsReferringTo returns empty set when input is empty`() { + val command = BackwardCompatibilityCheckCommand() + val result = command.getSpecsReferringTo(emptySet()) assertTrue(result.isEmpty()) } @Test - fun `filesReferringToChangedSchemaFiles returns empty set when no files refer to changed schema files`() { - val command = spyk() - every { command.allOpenApiSpecFiles() } returns listOf( + fun `getSpecsReferringTo returns empty set when no files refer to changed schema files`() { + val command = spyk() + every { command.allSpecFiles() } returns listOf( File("file1.yaml").apply { writeText("content1") }, File("file2.yaml").apply { writeText("content2") } ) - val result = command.filesReferringToChangedSchemaFiles(setOf("file3.yaml")) + val result = command.getSpecsReferringTo(setOf("file3.yaml")) assertTrue(result.isEmpty()) } @Test - fun `filesReferringToChangedSchemaFiles returns set of files that refer to changed schema files`() { - val command = spyk() - every { command.allOpenApiSpecFiles() } returns listOf( + fun `getSpecsReferringTo returns set of files that refer to changed schema files`() { + val command = spyk() + every { command.allSpecFiles() } returns listOf( File("file1.yaml").apply { writeText("file3.yaml") }, File("file2.yaml").apply { writeText("file4.yaml") } ) - val result = command.filesReferringToChangedSchemaFiles(setOf("file3.yaml")) + val result = command.getSpecsReferringTo(setOf("file3.yaml")) assertEquals(setOf("file1.yaml"), result) } @Test - fun `filesReferringToChangedSchemaFiles returns set of files which are referring to a changed schema that is one level down`() { - val command = spyk() - every { command.allOpenApiSpecFiles() } returns listOf( + fun `getSpecsReferringTo returns set of files which are referring to a changed schema that is one level down`() { + val command = spyk() + every { command.allSpecFiles() } returns listOf( File("file1.yaml").apply { referTo("schema_file1.yaml") }, File("schema_file2.yaml").apply { referTo("schema_file1.yaml") }, // schema within a schema File("file2.yaml").apply { referTo("schema_file2.yaml") } ) - val result = command.filesReferringToChangedSchemaFiles(setOf("schema_file1.yaml")) + val result = command.getSpecsReferringTo(setOf("schema_file1.yaml")) assertEquals(setOf("file1.yaml", "file2.yaml"), result) } From 3ba68cb9d731351eb6545cfff1ce88c3f9079869 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 21 Aug 2024 17:28:50 +0530 Subject: [PATCH 04/43] Filter only the specs that have changed using targetPath in BackwardCompatibilityCheckCommand --- .../BackwardCompatibilityCheckBaseCommand.kt | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 967664c8e..19a7b24ba 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -54,7 +54,9 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { } fun getChangedSpecs(logSpecs: Boolean = false): Set { - val filesChangedInCurrentBranch = getChangedSpecsInCurrentBranch() + val filesChangedInCurrentBranch = getChangedSpecsInCurrentBranch().filter { + it.contains(targetPath) + }.toSet() val filesReferringToChangedSchemaFiles = getSpecsReferringTo(filesChangedInCurrentBranch) val specificationsOfChangedExternalisedExamples = getSpecsOfChangedExternalisedExamples(filesChangedInCurrentBranch) @@ -67,14 +69,9 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { ) } - val specificationsToCheck: Set = - filesChangedInCurrentBranch + - filesReferringToChangedSchemaFiles + - specificationsOfChangedExternalisedExamples - - val filteredSpecs = specificationsToCheck.filter { it.contains(targetPath) }.toSet() - - return filteredSpecs + return filesChangedInCurrentBranch + + filesReferringToChangedSchemaFiles + + specificationsOfChangedExternalisedExamples } private fun getChangedSpecsInCurrentBranch(): Set { From 766f9fc7dcf4def02a05485a6bcfac8bbf417968 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Wed, 21 Aug 2024 17:58:58 +0530 Subject: [PATCH 05/43] Fix the logs in backwardCompatibilityCheckCommand --- .../BackwardCompatibilityCheckBaseCommand.kt | 110 ++++++++++-------- 1 file changed, 62 insertions(+), 48 deletions(-) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 19a7b24ba..8b3c330b3 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -115,19 +115,19 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { specificationsOfChangedExternalisedExamples: Set ) { - println("Checking backward compatibility of the following files: $newLine") - println("${ONE_INDENT}Files that have changed:") + println("Checking backward compatibility of the following specs: $newLine") + println("${ONE_INDENT}Specs that have changed:") changedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } println() if(filesReferringToChangedFiles.isNotEmpty()) { - println("${ONE_INDENT}Files referring to the changed files - ") + println("${ONE_INDENT}Specs referring to the changed specs - ") filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } println() } if(specificationsOfChangedExternalisedExamples.isNotEmpty()) { - println("${ONE_INDENT}Specifications whose externalised examples were changed - ") + println("${ONE_INDENT}Specs whose externalised examples were changed - ") filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } println() } @@ -167,49 +167,12 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { val backwardCompatibilityResult = checkBackwardCompatibility(older, newer) - if (backwardCompatibilityResult.success()) { - println( - "$newLine The file $specFilePath is backward compatible.$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - var errorsFound = false - - if(!areExamplesValid(newer, "newer")) { - println( - "$newLine *** Examples in $specFilePath are not valid. ***$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - errorsFound = true - } - - if(unusedExamples.isNotEmpty()) { - println( - "$newLine *** Some examples for $specFilePath could not be loaded. ***$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - errorsFound = true - } - - if(errorsFound) CompatibilityResult.FAILED - else CompatibilityResult.PASSED - } else { - println("$newLine ${backwardCompatibilityResult.report().prependIndent( - MARGIN_SPACE - )}") - println( - "$newLine *** The file $specFilePath is NOT backward compatible. ***$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - CompatibilityResult.FAILED - } + return@mapIndexed getCompatibilityResult( + backwardCompatibilityResult, + specFilePath, + newer, + unusedExamples + ) } finally { gitCommand.checkout(treeishWithChanges) if (areLocalChangesStashed) gitCommand.stashPop() @@ -224,10 +187,61 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { private fun baseBranch() = baseBranch ?: gitCommand.currentRemoteBranch() + private fun getCompatibilityResult( + backwardCompatibilityResult: Results, + specFilePath: String, + newer: IFeature, + unusedExamples: Set + ): CompatibilityResult { + if (backwardCompatibilityResult.success()) { + println( + "$newLine The spec $specFilePath is backward compatible with the corresponding spec from ${baseBranch()}$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + var errorsFound = false + + if(!areExamplesValid(newer, "newer")) { + println( + "$newLine *** Examples in $specFilePath are not valid. ***$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + errorsFound = true + } + + if(unusedExamples.isNotEmpty()) { + println( + "$newLine *** Some examples for $specFilePath could not be loaded. ***$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + errorsFound = true + } + + return if(errorsFound) CompatibilityResult.FAILED + else CompatibilityResult.PASSED + } else { + println("$newLine ${backwardCompatibilityResult.report().prependIndent( + MARGIN_SPACE + )}") + println( + "$newLine *** The changes to the spec $specFilePath are NOT backward compatible with the corresponding spec from ${baseBranch()}***$newLine".prependIndent( + MARGIN_SPACE + ) + ) + println() + return CompatibilityResult.FAILED + } + } + companion object { private const val HEAD = "HEAD" private const val MARGIN_SPACE = " " private const val ONE_INDENT = " " private const val TWO_INDENTS = "${ONE_INDENT}${ONE_INDENT}" } -} \ No newline at end of file +} From 04c82dd43ec721a9d993c73bd535790cfb2fbae6 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Mon, 26 Aug 2024 12:25:28 +0530 Subject: [PATCH 06/43] Bring back the existing backwardCompatibilityCheck command Add deprecation notice to all the b/w compatibility related commands which will be eventually removed --- .../BackwardCompatibilityCheckCommand.kt | 22 ++-- .../main/kotlin/application/CompareCommand.kt | 7 +- .../kotlin/application/CompatibleCommand.kt | 7 +- .../kotlin/application/DifferenceCommand.kt | 8 +- .../main/kotlin/application/PushCommand.kt | 13 +- .../kotlin/application/SpecmaticCommand.kt | 27 ++++- .../BackwardCompatibilityCheckBaseCommand.kt | 9 +- .../BackwardCompatibilityCheckCommandV2.kt | 112 ++++++++++++++++++ ...ackwardCompatibilityCheckCommandV2Test.kt} | 12 +- 9 files changed, 196 insertions(+), 21 deletions(-) rename application/src/main/kotlin/application/{backwardCompatibility => }/BackwardCompatibilityCheckCommand.kt (94%) create mode 100644 application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV2.kt rename application/src/test/kotlin/application/{BackwardCompatibilityCheckCommandTest.kt => BackwardCompatibilityCheckCommandV2Test.kt} (88%) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommand.kt b/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt similarity index 94% rename from application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommand.kt rename to application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt index 2e1f64e34..fd14d55c8 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommand.kt +++ b/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt @@ -26,7 +26,12 @@ const val TWO_INDENTS = "${ONE_INDENT}${ONE_INDENT}" @Command( name = "backwardCompatibilityCheck", mixinStandardHelpOptions = true, - description = ["Checks backward compatibility of a directory across the current HEAD and the main branch"] + description = [ +""" +Checks backward compatibility of a directory across the current HEAD and the main branch. +DEPRECATED: This command will be removed in the next major release. Use 'backward-compatibility-check' command instead. +""" + ] ) class BackwardCompatibilityCheckCommand( private val gitCommand: GitCommand = SystemGit(), @@ -42,10 +47,7 @@ class BackwardCompatibilityCheckCommand( override fun call() { val filesChangedInCurrentBranch: Set = getOpenAPISpecFilesChangedInCurrentBranch() - if (filesChangedInCurrentBranch.isEmpty()) { - logger.log("${newLine}No OpenAPI spec files were changed, skipping the check.$newLine") - exitProcess(0) - } + if (filesChangedInCurrentBranch.isEmpty()) exitWithMessage("${newLine}No OpenAPI spec files were changed, skipping the check.$newLine") val filesReferringToChangedSchemaFiles = filesReferringToChangedSchemaFiles(filesChangedInCurrentBranch) @@ -149,7 +151,11 @@ class BackwardCompatibilityCheckCommand( // newer => the file with changes on the branch val (newer, unusedExamples) = OpenApiSpecification.fromFile(specFilePath).toFeature().loadExternalisedExamplesAndListUnloadableExamples() - val olderFile = gitCommand.getFileInTheDefaultBranch(specFilePath, treeishWithChanges) + val olderFile = gitCommand.getFileInTheBaseBranch( + specFilePath, + treeishWithChanges, + gitCommand.defaultBranch() + ) if (olderFile == null) { println("$specFilePath is a new file.$newLine") return@mapIndexed PASSED @@ -289,7 +295,9 @@ class BackwardCompatibilityCheckCommand( } private fun getOpenAPISpecFilesChangedInCurrentBranch(): Set { - return gitCommand.getFilesChangeInCurrentBranch().filter { + return gitCommand.getFilesChangedInCurrentBranch( + gitCommand.defaultBranch() + ).filter { File(it).exists() && File(it).isOpenApiSpec() }.toSet() } diff --git a/application/src/main/kotlin/application/CompareCommand.kt b/application/src/main/kotlin/application/CompareCommand.kt index bdb0b7dbc..61918d226 100644 --- a/application/src/main/kotlin/application/CompareCommand.kt +++ b/application/src/main/kotlin/application/CompareCommand.kt @@ -12,7 +12,12 @@ import kotlin.system.exitProcess @Command(name = "compare", mixinStandardHelpOptions = true, - description = ["Checks if two contracts are equivalent"]) + description = [ +""" +Checks if two contracts are equivalent. +DEPRECATED: This command will be removed in the next major release. Use 'backward-compatibility-check' command instead. +""" + ]) class CompareCommand : Callable { @Parameters(index = "0", description = ["Older contract file path"]) lateinit var olderContractFilePath: String diff --git a/application/src/main/kotlin/application/CompatibleCommand.kt b/application/src/main/kotlin/application/CompatibleCommand.kt index e5c32dd27..234e28546 100644 --- a/application/src/main/kotlin/application/CompatibleCommand.kt +++ b/application/src/main/kotlin/application/CompatibleCommand.kt @@ -180,7 +180,12 @@ class GitCompatibleCommand : Callable { @Command(name = "compatible", mixinStandardHelpOptions = true, - description = ["Checks if the newer contract is backward compatible with the older one"], + description = [ +""" +Checks if the newer contract is backward compatible with the older one +DEPRECATED: This command will be removed in the next major release. Use 'backward-compatibility-check' command instead. +""" + ], subcommands = [ GitCompatibleCommand::class ]) internal class CompatibleCommand : Callable { override fun call() { diff --git a/application/src/main/kotlin/application/DifferenceCommand.kt b/application/src/main/kotlin/application/DifferenceCommand.kt index 1effdcc73..e6ce1ee61 100644 --- a/application/src/main/kotlin/application/DifferenceCommand.kt +++ b/application/src/main/kotlin/application/DifferenceCommand.kt @@ -11,7 +11,13 @@ import kotlin.system.exitProcess @Command(name = "similar", mixinStandardHelpOptions = true, - description = ["Show the difference between two contracts"]) + description = [ +""" +Show the difference between two contracts. +DEPRECATED: This command will be removed in the next major release. Use 'backward-compatibility-check' command instead. +""" + ] +) class DifferenceCommand : Callable { @Parameters(index = "0", description = ["Older contract file path"]) lateinit var olderContractFilePath: String diff --git a/application/src/main/kotlin/application/PushCommand.kt b/application/src/main/kotlin/application/PushCommand.kt index 9293a631f..771882d46 100644 --- a/application/src/main/kotlin/application/PushCommand.kt +++ b/application/src/main/kotlin/application/PushCommand.kt @@ -18,7 +18,16 @@ import kotlin.system.exitProcess private const val pipelineKeyInSpecmaticConfig = "pipeline" -@CommandLine.Command(name = "push", description = ["Check the new contract for backward compatibility with the specified version, then overwrite the old one with it."], mixinStandardHelpOptions = true) +@CommandLine.Command( + name = "push", + description = [ +""" +Check the new contract for backward compatibility with the specified version, then overwrite the old one with it. +DEPRECATED: This command will be removed in the next major release. Use 'backward-compatibility-check' command instead. +""" + ], + mixinStandardHelpOptions = true +) class PushCommand: Callable { override fun call() { val userHome = File(System.getProperty("user.home")) @@ -160,4 +169,4 @@ fun registerPipelineCredentials(manifestData: JSONObjectValue, contractPath: Str sourceGit.add() } } -} \ No newline at end of file +} diff --git a/application/src/main/kotlin/application/SpecmaticCommand.kt b/application/src/main/kotlin/application/SpecmaticCommand.kt index 170d188e1..d6591fe32 100644 --- a/application/src/main/kotlin/application/SpecmaticCommand.kt +++ b/application/src/main/kotlin/application/SpecmaticCommand.kt @@ -1,6 +1,6 @@ package application -import application.backwardCompatibility.BackwardCompatibilityCheckCommand +import application.backwardCompatibility.BackwardCompatibilityCheckCommandV2 import org.springframework.stereotype.Component import picocli.AutoComplete.GenerateCompletion import picocli.CommandLine.Command @@ -11,7 +11,30 @@ import java.util.concurrent.Callable name = "specmatic", mixinStandardHelpOptions = true, versionProvider = VersionProvider::class, - subcommands = [BackwardCompatibilityCheckCommand::class, BundleCommand::class, CompareCommand::class, CompatibleCommand::class, DifferenceCommand::class, GenerateCompletion::class, GraphCommand::class, MergeCommand::class, ToOpenAPICommand::class, ImportCommand::class, InstallCommand::class, ProxyCommand::class, PushCommand::class, ReDeclaredAPICommand::class, ExamplesCommand::class, SamplesCommand::class, StubCommand::class, SubscribeCommand::class, TestCommand::class, ValidateViaLogs::class, CentralContractRepoReportCommand::class] + subcommands = [ + BackwardCompatibilityCheckCommandV2::class, + BackwardCompatibilityCheckCommand::class, + BundleCommand::class, + CompareCommand::class, + CompatibleCommand::class, + DifferenceCommand::class, + GenerateCompletion::class, + GraphCommand::class, + MergeCommand::class, + ToOpenAPICommand::class, + ImportCommand::class, + InstallCommand::class, + ProxyCommand::class, + PushCommand::class, + ReDeclaredAPICommand::class, + ExamplesCommand::class, + SamplesCommand::class, + StubCommand::class, + SubscribeCommand::class, + TestCommand::class, + ValidateViaLogs::class, + CentralContractRepoReportCommand::class + ] ) class SpecmaticCommand : Callable { override fun call(): Int { diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 8b3c330b3..11241eeb7 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -16,7 +16,14 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { private val gitCommand: GitCommand = SystemGit() private val newLine = System.lineSeparator() - @Option(names = ["--base-branch"], description = ["Base branch to compare the changes against"], required = false) + @Option( + names = ["--base-branch"], + description = [ + "Base branch to compare the changes against", + "Default value is the local origin HEAD of the current branch" + ], + required = false + ) var baseBranch: String? = null @Option(names = ["--target-path"], description = ["Specification file or folder to run the check against"], required = false) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV2.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV2.kt new file mode 100644 index 000000000..4c59f7244 --- /dev/null +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV2.kt @@ -0,0 +1,112 @@ +package application.backwardCompatibility + +import io.specmatic.conversions.OpenApiSpecification +import io.specmatic.core.CONTRACT_EXTENSION +import io.specmatic.core.CONTRACT_EXTENSIONS +import io.specmatic.core.Feature +import io.specmatic.core.IFeature +import io.specmatic.core.Results +import io.specmatic.core.WSDL +import io.specmatic.core.testBackwardCompatibility +import io.specmatic.stub.isOpenAPI +import org.springframework.stereotype.Component +import picocli.CommandLine.Command +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.io.path.extension +import kotlin.io.path.pathString + +@Component +@Command( + name = "backward-compatibility-check", + mixinStandardHelpOptions = true, + description = ["Checks backward compatibility of OpenAPI specifications"] +) +class BackwardCompatibilityCheckCommandV2: BackwardCompatibilityCheckBaseCommand() { + + override fun checkBackwardCompatibility(oldFeature: IFeature, newFeature: IFeature): Results { + return testBackwardCompatibility(oldFeature as Feature, newFeature as Feature) + } + + override fun File.isValidSpec(): Boolean { + if (this.extension !in CONTRACT_EXTENSIONS) return false + return OpenApiSpecification.isParsable(this.path) + } + + override fun getFeatureFromSpecPath(path: String): Feature { + return OpenApiSpecification.fromFile(path).toFeature() + } + + override fun regexForMatchingReferred(schemaFileName: String) = schemaFileName + + override fun getSpecsOfChangedExternalisedExamples(filesChangedInCurrentBranch: Set): Set { + data class CollectedFiles( + val specifications: MutableSet = mutableSetOf(), + val examplesMissingSpecifications: MutableList = mutableListOf(), + val ignoredFiles: MutableList = mutableListOf() + ) + + val collectedFiles = filesChangedInCurrentBranch.fold(CollectedFiles()) { acc, filePath -> + val path = Paths.get(filePath) + val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } + + if (examplesDir == null) { + acc.ignoredFiles.add(filePath) + } else { + val parentPath = examplesDir.parent + val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) + val specFiles = findSpecFiles(strippedPath) + + if (specFiles.isNotEmpty()) { + acc.specifications.addAll(specFiles.map { it.toString() }) + } else { + acc.examplesMissingSpecifications.add(filePath) + } + } + acc + } + + val result = collectedFiles.specifications.toMutableSet() + + collectedFiles.examplesMissingSpecifications.forEach { filePath -> + val path = Paths.get(filePath) + val examplesDir = path.find { it.toString().endsWith("_examples") || it.toString().endsWith("_tests") } + if (examplesDir != null) { + val parentPath = examplesDir.parent + val strippedPath = parentPath.resolve(examplesDir.fileName.toString().removeSuffix("_examples")) + val specFiles = findSpecFiles(strippedPath) + if (specFiles.isNotEmpty()) { + result.addAll(specFiles.map { it.toString() }) + } else { + result.add("${strippedPath}.yaml") + } + } + } + + return result + } + + override fun areExamplesValid(feature: IFeature, which: String): Boolean { + feature as Feature + return try { + feature.validateExamplesOrException() + true + } catch (t: Throwable) { + println() + false + } + } + + override fun getUnusedExamples(feature: IFeature): Set { + feature as Feature + return feature.loadExternalisedExamplesAndListUnloadableExamples().second + } + + private fun findSpecFiles(path: Path): List { + val extensions = CONTRACT_EXTENSIONS + return extensions.map { path.resolveSibling(path.fileName.toString() + it) } + .filter { Files.exists(it) && (isOpenAPI(it.pathString) || it.extension in listOf(WSDL, CONTRACT_EXTENSION)) } + } +} \ No newline at end of file diff --git a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt similarity index 88% rename from application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt rename to application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt index 6873da05c..18fd33bf7 100644 --- a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandTest.kt +++ b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt @@ -1,6 +1,6 @@ package application -import application.backwardCompatibility.BackwardCompatibilityCheckCommand +import application.backwardCompatibility.BackwardCompatibilityCheckCommandV2 import io.mockk.every import io.mockk.spyk import org.junit.jupiter.api.AfterEach @@ -9,18 +9,18 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import java.io.File -class BackwardCompatibilityCheckCommandTest { +class BackwardCompatibilityCheckCommandV2Test { @Test fun `getSpecsReferringTo returns empty set when input is empty`() { - val command = BackwardCompatibilityCheckCommand() + val command = BackwardCompatibilityCheckCommandV2() val result = command.getSpecsReferringTo(emptySet()) assertTrue(result.isEmpty()) } @Test fun `getSpecsReferringTo returns empty set when no files refer to changed schema files`() { - val command = spyk() + val command = spyk() every { command.allSpecFiles() } returns listOf( File("file1.yaml").apply { writeText("content1") }, File("file2.yaml").apply { writeText("content2") } @@ -31,7 +31,7 @@ class BackwardCompatibilityCheckCommandTest { @Test fun `getSpecsReferringTo returns set of files that refer to changed schema files`() { - val command = spyk() + val command = spyk() every { command.allSpecFiles() } returns listOf( File("file1.yaml").apply { writeText("file3.yaml") }, File("file2.yaml").apply { writeText("file4.yaml") } @@ -42,7 +42,7 @@ class BackwardCompatibilityCheckCommandTest { @Test fun `getSpecsReferringTo returns set of files which are referring to a changed schema that is one level down`() { - val command = spyk() + val command = spyk() every { command.allSpecFiles() } returns listOf( File("file1.yaml").apply { referTo("schema_file1.yaml") }, File("schema_file2.yaml").apply { referTo("schema_file1.yaml") }, // schema within a schema From 827a305ace24cf74e3a886e19d3c3d7f9d00e7ec Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Mon, 7 Oct 2024 13:05:25 +0530 Subject: [PATCH 07/43] Update the import statement for exitWithMessage in BackwardCompatibilityCheckCommand --- .../application/BackwardCompatibilityCheckCommand.kt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt b/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt index fd14d55c8..2ceaf79d1 100644 --- a/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt +++ b/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt @@ -1,11 +1,17 @@ package application -import application.BackwardCompatibilityCheckCommand.CompatibilityResult.* +import application.BackwardCompatibilityCheckCommand.CompatibilityResult.FAILED +import application.BackwardCompatibilityCheckCommand.CompatibilityResult.PASSED import io.specmatic.conversions.OpenApiSpecification -import io.specmatic.core.* +import io.specmatic.core.CONTRACT_EXTENSION +import io.specmatic.core.CONTRACT_EXTENSIONS +import io.specmatic.core.Feature +import io.specmatic.core.WSDL import io.specmatic.core.git.GitCommand import io.specmatic.core.git.SystemGit import io.specmatic.core.log.logger +import io.specmatic.core.testBackwardCompatibility +import io.specmatic.core.utilities.exitWithMessage import io.specmatic.stub.isOpenAPI import org.springframework.stereotype.Component import picocli.CommandLine.Command From 5d82fd6a17aa7b308b6b86e7f5752e7baf536131 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 10 Oct 2024 11:01:41 +0530 Subject: [PATCH 08/43] Refactor the logging logic of changed files --- .../BackwardCompatibilityCheckBaseCommand.kt | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 11241eeb7..c3319b00b 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -121,26 +121,23 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { filesReferringToChangedFiles: Set, specificationsOfChangedExternalisedExamples: Set ) { + logger.log("Checking backward compatibility of the following specs:$newLine") + changedFiles.printSummaryOfChangedSpecs("Specs that have changed") + filesReferringToChangedFiles.printSummaryOfChangedSpecs("Specs referring to the changed specs") + specificationsOfChangedExternalisedExamples + .printSummaryOfChangedSpecs("Specs whose externalised examples were changed") + logger.log("-".repeat(20)) + logger.log(newLine) + } - println("Checking backward compatibility of the following specs: $newLine") - println("${ONE_INDENT}Specs that have changed:") - changedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } - println() - - if(filesReferringToChangedFiles.isNotEmpty()) { - println("${ONE_INDENT}Specs referring to the changed specs - ") - filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } - println() - } - - if(specificationsOfChangedExternalisedExamples.isNotEmpty()) { - println("${ONE_INDENT}Specs whose externalised examples were changed - ") - filesReferringToChangedFiles.forEach { println(it.prependIndent(TWO_INDENTS)) } - println() + private fun Set.printSummaryOfChangedSpecs(message: String) { + if(this.isNotEmpty()) { + logger.log("${ONE_INDENT}- $message: ") + this.forEachIndexed { index, it -> + logger.log(it.prependIndent("$TWO_INDENTS${index.inc()}. ")) + } + logger.log(newLine) } - - println("-".repeat(20)) - println() } private fun runBackwardCompatibilityCheckFor(files: Set, baseBranch: String): CompatibilityReport { From 6f1da3518cd98f44e097c66212faab476df53a57 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 10 Oct 2024 11:33:48 +0530 Subject: [PATCH 09/43] Add a shutdown hook in backward compatibility check command which will bring the repository back to the original state if the command is aborted in between --- .../BackwardCompatibilityCheckBaseCommand.kt | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index c3319b00b..e9c74d69e 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -15,6 +15,7 @@ import kotlin.system.exitProcess abstract class BackwardCompatibilityCheckBaseCommand : Callable { private val gitCommand: GitCommand = SystemGit() private val newLine = System.lineSeparator() + private var areLocalChangesStashed = false @Option( names = ["--base-branch"], @@ -42,6 +43,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { open fun getUnusedExamples(feature: IFeature): Set = emptySet() final override fun call() { + addShutdownHook() val filteredSpecs = getChangedSpecs(logSpecs = true) val result = try { runBackwardCompatibilityCheckFor( @@ -140,13 +142,16 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { } } - private fun runBackwardCompatibilityCheckFor(files: Set, baseBranch: String): CompatibilityReport { + private fun getCurrentBranch(): String { val branchWithChanges = gitCommand.currentBranch() - val treeishWithChanges = if (branchWithChanges == HEAD) gitCommand.detachedHEAD() else branchWithChanges + return if (branchWithChanges == HEAD) gitCommand.detachedHEAD() else branchWithChanges + } + + private fun runBackwardCompatibilityCheckFor(files: Set, baseBranch: String): CompatibilityReport { + val treeishWithChanges = getCurrentBranch() try { val results = files.mapIndexed { index, specFilePath -> - var areLocalChangesStashed = false try { println("${index.inc()}. Running the check for $specFilePath:") @@ -179,7 +184,10 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { ) } finally { gitCommand.checkout(treeishWithChanges) - if (areLocalChangesStashed) gitCommand.stashPop() + if (areLocalChangesStashed) { + gitCommand.stashPop() + areLocalChangesStashed = false + } } } @@ -242,6 +250,15 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { } } + private fun addShutdownHook() { + Runtime.getRuntime().addShutdownHook(object: Thread() { + override fun run() { + gitCommand.checkout(getCurrentBranch()) + if(areLocalChangesStashed) gitCommand.stashPop() + } + }) + } + companion object { private const val HEAD = "HEAD" private const val MARGIN_SPACE = " " From 768ac8e0686ce12d552531826f4e36c873a68218 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 10 Oct 2024 12:05:59 +0530 Subject: [PATCH 10/43] Improve logging of backward compatibility report --- .../BackwardCompatibilityCheckBaseCommand.kt | 101 ++++++++++-------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index e9c74d69e..55520b6ec 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -57,8 +57,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { exitProcess(1) } - println() - println(result.report) + logger.log(result.report) exitProcess(result.exitCode) } @@ -153,7 +152,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { try { val results = files.mapIndexed { index, specFilePath -> try { - println("${index.inc()}. Running the check for $specFilePath:") + logger.log("${index.inc()}. Running the check for $specFilePath:$newLine") // newer => the file with changes on the branch val newer = getFeatureFromSpecPath(specFilePath) @@ -165,7 +164,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { baseBranch ) if (olderFile == null) { - println("$specFilePath is a new file.$newLine") + logger.log("$ONE_INDENT$specFilePath is a new file.$newLine") return@mapIndexed CompatibilityResult.PASSED } @@ -205,56 +204,70 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { newer: IFeature, unusedExamples: Set ): CompatibilityResult { - if (backwardCompatibilityResult.success()) { - println( - "$newLine The spec $specFilePath is backward compatible with the corresponding spec from ${baseBranch()}$newLine".prependIndent( - MARGIN_SPACE - ) + if(backwardCompatibilityResult.success().not()) { + logger.log("_".repeat(40).prependIndent(ONE_INDENT)) + logger.log("The Incompatibility Report:$newLine".prependIndent(ONE_INDENT)) + logger.log(backwardCompatibilityResult.report().prependIndent(TWO_INDENTS)) + logVerdictFor( + specFilePath, + "(INCOMPATIBLE) The changes to the spec are NOT backward compatible with the corresponding spec from ${baseBranch()}".prependIndent(ONE_INDENT) ) - println() - var errorsFound = false + return CompatibilityResult.FAILED + } - if(!areExamplesValid(newer, "newer")) { - println( - "$newLine *** Examples in $specFilePath are not valid. ***$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - errorsFound = true - } + val errorsFound = printExampleValiditySummaryAndReturnResult(newer, unusedExamples, specFilePath) - if(unusedExamples.isNotEmpty()) { - println( - "$newLine *** Some examples for $specFilePath could not be loaded. ***$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - errorsFound = true - } - - return if(errorsFound) CompatibilityResult.FAILED - else CompatibilityResult.PASSED + val message = if(errorsFound) { + "(INCOMPATIBLE) The spec is backward compatible but the examples are NOT backward compatible or are INVALID." } else { - println("$newLine ${backwardCompatibilityResult.report().prependIndent( - MARGIN_SPACE - )}") - println( - "$newLine *** The changes to the spec $specFilePath are NOT backward compatible with the corresponding spec from ${baseBranch()}***$newLine".prependIndent( - MARGIN_SPACE - ) - ) - println() - return CompatibilityResult.FAILED + "(COMPATIBLE) The spec is backward compatible with the corresponding spec from ${baseBranch()}" + } + logVerdictFor(specFilePath, message.prependIndent(ONE_INDENT)) + + return if (errorsFound) CompatibilityResult.FAILED + else CompatibilityResult.PASSED + } + + private fun logVerdictFor(specFilePath: String, message: String) { + logger.log(newLine) + logger.log("-".repeat(20).prependIndent(ONE_INDENT)) + logger.log("Verdict for spec $specFilePath:".prependIndent(ONE_INDENT)) + logger.log("$ONE_INDENT$message") + logger.log("-".repeat(20).prependIndent(ONE_INDENT)) + logger.log(newLine) + } + + private fun printExampleValiditySummaryAndReturnResult( + newer: IFeature, + unusedExamples: Set, + specFilePath: String + ): Boolean { + var errorsFound = false + val areExamplesInvalid = areExamplesValid(newer, "newer").not() + + if(areExamplesInvalid || unusedExamples.isNotEmpty()) { + logger.log("_".repeat(40).prependIndent(ONE_INDENT)) + logger.log("The Examples Validity Summary:$newLine".prependIndent(ONE_INDENT)) + } + if (areExamplesInvalid) { + logger.log("Examples in $specFilePath are not valid.$newLine".prependIndent(TWO_INDENTS)) + errorsFound = true } + + if (unusedExamples.isNotEmpty()) { + logger.log("Some examples for $specFilePath could not be loaded.$newLine".prependIndent(TWO_INDENTS)) + errorsFound = true + } + return errorsFound } private fun addShutdownHook() { Runtime.getRuntime().addShutdownHook(object: Thread() { override fun run() { - gitCommand.checkout(getCurrentBranch()) - if(areLocalChangesStashed) gitCommand.stashPop() + runCatching { + gitCommand.checkout(getCurrentBranch()) + if(areLocalChangesStashed) gitCommand.stashPop() + } } }) } From 4a4ab17c15498e6af0837bec65a14e909086ec34 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 10 Oct 2024 15:22:23 +0530 Subject: [PATCH 11/43] Add ability to disable and enable info logging in LogStrategy to avoid unnecessary logs in the backward compatibility check result logs --- .../BackwardCompatibilityCheckBaseCommand.kt | 8 ++++---- .../BackwardCompatibilityCheckCommandV2.kt | 6 +++++- core/src/main/kotlin/io/specmatic/core/Scenario.kt | 2 +- .../src/main/kotlin/io/specmatic/core/log/LogStrategy.kt | 8 +++++++- core/src/main/kotlin/io/specmatic/core/log/NonVerbose.kt | 9 ++++++--- core/src/main/kotlin/io/specmatic/core/log/Verbose.kt | 9 ++++++--- .../io/specmatic/conversions/OpenApiSpecificationTest.kt | 6 ++++++ 7 files changed, 35 insertions(+), 13 deletions(-) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 55520b6ec..92e5b355e 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -152,7 +152,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { try { val results = files.mapIndexed { index, specFilePath -> try { - logger.log("${index.inc()}. Running the check for $specFilePath:$newLine") + logger.log("${index.inc()}. Running the check for $specFilePath:") // newer => the file with changes on the branch val newer = getFeatureFromSpecPath(specFilePath) @@ -222,14 +222,14 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { } else { "(COMPATIBLE) The spec is backward compatible with the corresponding spec from ${baseBranch()}" } - logVerdictFor(specFilePath, message.prependIndent(ONE_INDENT)) + logVerdictFor(specFilePath, message.prependIndent(ONE_INDENT), startWithNewLine = errorsFound) return if (errorsFound) CompatibilityResult.FAILED else CompatibilityResult.PASSED } - private fun logVerdictFor(specFilePath: String, message: String) { - logger.log(newLine) + private fun logVerdictFor(specFilePath: String, message: String, startWithNewLine: Boolean = true) { + if (startWithNewLine) logger.log(newLine) logger.log("-".repeat(20).prependIndent(ONE_INDENT)) logger.log("Verdict for spec $specFilePath:".prependIndent(ONE_INDENT)) logger.log("$ONE_INDENT$message") diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV2.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV2.kt index 4c59f7244..c942e898b 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV2.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckCommandV2.kt @@ -7,6 +7,7 @@ import io.specmatic.core.Feature import io.specmatic.core.IFeature import io.specmatic.core.Results import io.specmatic.core.WSDL +import io.specmatic.core.log.logger import io.specmatic.core.testBackwardCompatibility import io.specmatic.stub.isOpenAPI import org.springframework.stereotype.Component @@ -36,7 +37,10 @@ class BackwardCompatibilityCheckCommandV2: BackwardCompatibilityCheckBaseCommand } override fun getFeatureFromSpecPath(path: String): Feature { - return OpenApiSpecification.fromFile(path).toFeature() + logger.disableInfoLogging() + return OpenApiSpecification.fromFile(path).toFeature().also { + logger.enableInfoLogging() + } } override fun regexForMatchingReferred(schemaFileName: String) = schemaFileName diff --git a/core/src/main/kotlin/io/specmatic/core/Scenario.kt b/core/src/main/kotlin/io/specmatic/core/Scenario.kt index 5625e2b7e..711915cba 100644 --- a/core/src/main/kotlin/io/specmatic/core/Scenario.kt +++ b/core/src/main/kotlin/io/specmatic/core/Scenario.kt @@ -360,7 +360,7 @@ data class Scenario( } listOf(title).plus(errors).joinToString("${System.lineSeparator()}${System.lineSeparator()}").also { message -> - logger.log(message) + logger.logError(Exception(message)) logger.newLine() } diff --git a/core/src/main/kotlin/io/specmatic/core/log/LogStrategy.kt b/core/src/main/kotlin/io/specmatic/core/log/LogStrategy.kt index 3d58daa2b..b6ab01f6e 100644 --- a/core/src/main/kotlin/io/specmatic/core/log/LogStrategy.kt +++ b/core/src/main/kotlin/io/specmatic/core/log/LogStrategy.kt @@ -2,9 +2,9 @@ package io.specmatic.core.log interface LogStrategy { val printer: CompositePrinter + var infoLoggingEnabled: Boolean fun keepReady(msg: LogMessage) - fun exceptionString(e: Throwable, msg: String? = null): String fun ofTheException(e: Throwable, msg: String? = null): LogMessage fun log(e: Throwable, msg: String? = null) @@ -15,4 +15,10 @@ interface LogStrategy { fun debug(msg: String): String fun debug(msg: LogMessage) fun debug(e: Throwable, msg: String? = null) + fun disableInfoLogging() { + infoLoggingEnabled = false + } + fun enableInfoLogging() { + infoLoggingEnabled = true + } } \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/core/log/NonVerbose.kt b/core/src/main/kotlin/io/specmatic/core/log/NonVerbose.kt index 7b2b4effb..94e0eec7b 100644 --- a/core/src/main/kotlin/io/specmatic/core/log/NonVerbose.kt +++ b/core/src/main/kotlin/io/specmatic/core/log/NonVerbose.kt @@ -2,7 +2,10 @@ package io.specmatic.core.log import io.specmatic.core.utilities.exceptionCauseMessage -class NonVerbose(override val printer: CompositePrinter) : LogStrategy { +class NonVerbose( + override val printer: CompositePrinter, + override var infoLoggingEnabled: Boolean = true +) : LogStrategy { private val readyMessage = ReadyMessage() override fun keepReady(msg: LogMessage) { @@ -30,11 +33,11 @@ class NonVerbose(override val printer: CompositePrinter) : LogStrategy { } override fun log(msg: String) { - log(StringLog(msg)) + if (infoLoggingEnabled) log(StringLog(msg)) } override fun log(msg: LogMessage) { - print(msg) + if (infoLoggingEnabled) print(msg) } override fun logError(e: Throwable) { diff --git a/core/src/main/kotlin/io/specmatic/core/log/Verbose.kt b/core/src/main/kotlin/io/specmatic/core/log/Verbose.kt index e2717ea95..a5394c4a4 100644 --- a/core/src/main/kotlin/io/specmatic/core/log/Verbose.kt +++ b/core/src/main/kotlin/io/specmatic/core/log/Verbose.kt @@ -2,7 +2,10 @@ package io.specmatic.core.log import io.specmatic.core.utilities.exceptionCauseMessage -class Verbose(override val printer: CompositePrinter = CompositePrinter()) : LogStrategy { +class Verbose( + override val printer: CompositePrinter = CompositePrinter(), + override var infoLoggingEnabled: Boolean = true +) : LogStrategy { private val readyMessage = ReadyMessage() override fun keepReady(msg: LogMessage) { @@ -32,11 +35,11 @@ class Verbose(override val printer: CompositePrinter = CompositePrinter()) : Log } override fun log(msg: String) { - log(StringLog(msg)) + if (infoLoggingEnabled) log(StringLog(msg)) } override fun log(msg: LogMessage) { - print(msg) + if (infoLoggingEnabled) print(msg) } override fun logError(e: Throwable) { diff --git a/core/src/test/kotlin/io/specmatic/conversions/OpenApiSpecificationTest.kt b/core/src/test/kotlin/io/specmatic/conversions/OpenApiSpecificationTest.kt index 726edc911..bb03e29d2 100644 --- a/core/src/test/kotlin/io/specmatic/conversions/OpenApiSpecificationTest.kt +++ b/core/src/test/kotlin/io/specmatic/conversions/OpenApiSpecificationTest.kt @@ -6231,6 +6231,9 @@ paths: val messages = mutableListOf() override val printer: CompositePrinter get() = TODO("Not yet implemented") + override var infoLoggingEnabled: Boolean + get() = true + set(value) {} override fun keepReady(msg: LogMessage) { TODO("Not yet implemented") @@ -6318,6 +6321,9 @@ paths: val messages = mutableListOf() override val printer: CompositePrinter get() = TODO("Not yet implemented") + override var infoLoggingEnabled: Boolean + get() = true + set(value) {} override fun keepReady(msg: LogMessage) { TODO("Not yet implemented") From eff983435191ea7df53b061166f54bd85b04177e Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Thu, 10 Oct 2024 17:27:06 +0530 Subject: [PATCH 12/43] Change scope getSpecsReferringTo method from internal to open to enable override in sub-classes of BackwardCompatibilityCheckBaseCommand --- .../BackwardCompatibilityCheckBaseCommand.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 92e5b355e..9a6495303 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -34,11 +34,11 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { abstract fun File.isValidSpec(): Boolean abstract fun getFeatureFromSpecPath(path: String): IFeature - abstract fun regexForMatchingReferred(schemaFileName: String): String abstract fun getSpecsOfChangedExternalisedExamples( filesChangedInCurrentBranch: Set ): Set + open fun regexForMatchingReferred(schemaFileName: String): String = "" open fun areExamplesValid(feature: IFeature, which: String): Boolean = true open fun getUnusedExamples(feature: IFeature): Set = emptySet() @@ -92,7 +92,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { } } - internal fun getSpecsReferringTo(schemaFiles: Set): Set { + open fun getSpecsReferringTo(schemaFiles: Set): Set { if (schemaFiles.isEmpty()) return emptySet() val inputFileNames = schemaFiles.map { File(it).name } From 67bc469675bfa84ab087d6df9139aa25b0c02e31 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Fri, 11 Oct 2024 17:58:14 +0530 Subject: [PATCH 13/43] Make exitWithMessage method append and prepend newLine to the message --- .../kotlin/application/BackwardCompatibilityCheckCommand.kt | 5 ++++- .../BackwardCompatibilityCheckBaseCommand.kt | 2 +- .../src/main/kotlin/io/specmatic/core/utilities/Utilities.kt | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt b/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt index 2ceaf79d1..00e464df3 100644 --- a/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt +++ b/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt @@ -53,7 +53,10 @@ class BackwardCompatibilityCheckCommand( override fun call() { val filesChangedInCurrentBranch: Set = getOpenAPISpecFilesChangedInCurrentBranch() - if (filesChangedInCurrentBranch.isEmpty()) exitWithMessage("${newLine}No OpenAPI spec files were changed, skipping the check.$newLine") + if (filesChangedInCurrentBranch.isEmpty()) { + logger.log("${newLine}No OpenAPI spec files were changed, skipping the check.$newLine") + exitProcess(0) + } val filesReferringToChangedSchemaFiles = filesReferringToChangedSchemaFiles(filesChangedInCurrentBranch) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 9a6495303..7adfc2def 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -88,7 +88,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { ).filter { File(it).exists() && File(it).isValidSpec() }.toSet().also { - if(it.isEmpty()) exitWithMessage("${newLine}No specs were changed, skipping the check.$newLine") + if(it.isEmpty()) exitWithMessage("No specs were changed, skipping the check.") } } diff --git a/core/src/main/kotlin/io/specmatic/core/utilities/Utilities.kt b/core/src/main/kotlin/io/specmatic/core/utilities/Utilities.kt index 7c833866b..35184b9e4 100644 --- a/core/src/main/kotlin/io/specmatic/core/utilities/Utilities.kt +++ b/core/src/main/kotlin/io/specmatic/core/utilities/Utilities.kt @@ -37,7 +37,8 @@ import javax.xml.transform.stream.StreamResult import kotlin.system.exitProcess fun exitWithMessage(message: String): Nothing { - logger.log(message) + val newLine = System.lineSeparator() + logger.log("$newLine$message$newLine") exitProcess(1) } From a90557da5c577fe6779ef081e7409baabfbf75ea Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Fri, 11 Oct 2024 18:02:23 +0530 Subject: [PATCH 14/43] Rename getFileInTheBaseBranch to getFileInBranch --- .../application/BackwardCompatibilityCheckCommand.kt | 3 +-- .../BackwardCompatibilityCheckBaseCommand.kt | 2 +- .../test/kotlin/application/CompatibleCommandKtTest.kt | 8 ++++---- core/src/main/kotlin/io/specmatic/core/git/GitCommand.kt | 2 +- core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt b/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt index 00e464df3..f6476e0ea 100644 --- a/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt +++ b/application/src/main/kotlin/application/BackwardCompatibilityCheckCommand.kt @@ -11,7 +11,6 @@ import io.specmatic.core.git.GitCommand import io.specmatic.core.git.SystemGit import io.specmatic.core.log.logger import io.specmatic.core.testBackwardCompatibility -import io.specmatic.core.utilities.exitWithMessage import io.specmatic.stub.isOpenAPI import org.springframework.stereotype.Component import picocli.CommandLine.Command @@ -160,7 +159,7 @@ class BackwardCompatibilityCheckCommand( // newer => the file with changes on the branch val (newer, unusedExamples) = OpenApiSpecification.fromFile(specFilePath).toFeature().loadExternalisedExamplesAndListUnloadableExamples() - val olderFile = gitCommand.getFileInTheBaseBranch( + val olderFile = gitCommand.getFileInBranch( specFilePath, treeishWithChanges, gitCommand.defaultBranch() diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 7adfc2def..52dc9b9e2 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -158,7 +158,7 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { val newer = getFeatureFromSpecPath(specFilePath) val unusedExamples = getUnusedExamples(newer) - val olderFile = gitCommand.getFileInTheBaseBranch( + val olderFile = gitCommand.getFileInBranch( specFilePath, treeishWithChanges, baseBranch diff --git a/application/src/test/kotlin/application/CompatibleCommandKtTest.kt b/application/src/test/kotlin/application/CompatibleCommandKtTest.kt index 1b4d9682e..99c0acc54 100644 --- a/application/src/test/kotlin/application/CompatibleCommandKtTest.kt +++ b/application/src/test/kotlin/application/CompatibleCommandKtTest.kt @@ -52,7 +52,7 @@ internal class CompatibleCommandKtTest { } override fun getFilesChangedInCurrentBranch(baseBranch: String) = emptyList() - override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String) = null + override fun getFileInBranch(fileName: String, currentBranch: String, baseBranch: String) = null override fun relativeGitPath(newerContractPath: String): Pair { assertThat(newerContractPath).isEqualTo("/Users/fakeuser/newer.$CONTRACT_EXTENSION") @@ -82,7 +82,7 @@ internal class CompatibleCommandKtTest { override fun fileIsInGitDir(newerContractPath: String): Boolean = true override val workingDirectory: String get() = "" - override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String) = null + override fun getFileInBranch(fileName: String, currentBranch: String, baseBranch: String) = null override fun getFilesChangedInCurrentBranch(baseBranch: String) = emptyList() @@ -122,7 +122,7 @@ internal class CompatibleCommandKtTest { override fun getFilesChangedInCurrentBranch(baseBranch: String) = emptyList() override val workingDirectory: String get() = "" - override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String) = null + override fun getFileInBranch(fileName: String, currentBranch: String, baseBranch: String) = null override fun relativeGitPath(newerContractPath: String): Pair { assertThat(newerContractPath).isEqualTo("/Users/fakeuser/newer.$CONTRACT_EXTENSION") @@ -159,7 +159,7 @@ internal class CompatibleCommandKtTest { override fun getFilesChangedInCurrentBranch(baseBranch: String) = emptyList() override val workingDirectory: String get() = "" - override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String) = null + override fun getFileInBranch(fileName: String, currentBranch: String, baseBranch: String) = null override fun relativeGitPath(newerContractPath: String): Pair { assertThat(newerContractPath).isEqualTo("/Users/fakeuser/newer.$CONTRACT_EXTENSION") diff --git a/core/src/main/kotlin/io/specmatic/core/git/GitCommand.kt b/core/src/main/kotlin/io/specmatic/core/git/GitCommand.kt index 48d28ca44..7ca8bc827 100644 --- a/core/src/main/kotlin/io/specmatic/core/git/GitCommand.kt +++ b/core/src/main/kotlin/io/specmatic/core/git/GitCommand.kt @@ -33,7 +33,7 @@ interface GitCommand { fun getRemoteUrl(name: String = "origin"): String fun checkIgnore(path: String): String fun getFilesChangedInCurrentBranch(baseBranch: String): List - fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String): File? + fun getFileInBranch(fileName: String, currentBranch: String, baseBranch: String): File? fun currentRemoteBranch(): String fun currentBranch(): String { return "" diff --git a/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt b/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt index 839a24254..916ae2c70 100644 --- a/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt +++ b/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt @@ -102,7 +102,7 @@ class SystemGit(override val workingDirectory: String = ".", private val prefix: return (committedLocalChanges + uncommittedChanges).distinct() } - override fun getFileInTheBaseBranch(fileName: String, currentBranch: String, baseBranch: String): File? { + override fun getFileInBranch(fileName: String, currentBranch: String, baseBranch: String): File? { try { if(baseBranch != currentBranch) checkout(baseBranch) From 3a560bf2884b930a20fd0a01a20c8e9c41732925 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Fri, 11 Oct 2024 18:12:45 +0530 Subject: [PATCH 15/43] Staged files should be considered during the backward compatibility check --- .../main/kotlin/io/specmatic/core/git/SystemGit.kt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt b/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt index 916ae2c70..b8b351b96 100644 --- a/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt +++ b/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt @@ -91,15 +91,14 @@ class SystemGit(override val workingDirectory: String = ".", private val prefix: } override fun getFilesChangedInCurrentBranch(baseBranch: String): List { - val committedLocalChanges = execute(Configuration.gitCommand, "diff", baseBranch, "HEAD", "--name-only") + val committedLocalChanges = execute(Configuration.gitCommand, "diff", baseBranch, "HEAD", "--name-status") .split(System.lineSeparator()) - .filter { it.isNotBlank() } - - val uncommittedChanges = execute(Configuration.gitCommand, "diff", "--name-only") + val uncommittedChanges = execute(Configuration.gitCommand, "diff", "HEAD", "--name-status") .split(System.lineSeparator()) - .filter { it.isNotBlank() } - return (committedLocalChanges + uncommittedChanges).distinct() + return (committedLocalChanges + uncommittedChanges).map { + it.split("\t").last() + }.distinct() } override fun getFileInBranch(fileName: String, currentBranch: String, baseBranch: String): File? { From 37c001db7f5b16731c9ce6dd5e3bb697b14bc1f5 Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Mon, 14 Oct 2024 10:27:35 +0530 Subject: [PATCH 16/43] Update the description for the argument --target-path in backward-compatibility-check command --- .../BackwardCompatibilityCheckBaseCommand.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt index 52dc9b9e2..37980d827 100644 --- a/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt +++ b/application/src/main/kotlin/application/backwardCompatibility/BackwardCompatibilityCheckBaseCommand.kt @@ -27,7 +27,11 @@ abstract class BackwardCompatibilityCheckBaseCommand : Callable { ) var baseBranch: String? = null - @Option(names = ["--target-path"], description = ["Specification file or folder to run the check against"], required = false) + @Option( + names = ["--target-path"], + description = ["Specify the file or directory to limit the backward compatibility check scope. If omitted, all changed files will be checked."], + required = false + ) var targetPath: String = "" abstract fun checkBackwardCompatibility(oldFeature: IFeature, newFeature: IFeature): Results From 0a8dd21c5f053854ae39a38943070377c740a64b Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Mon, 14 Oct 2024 14:34:17 +0530 Subject: [PATCH 17/43] Add tests for getFilesChangedInCurrentBranch function --- ...BackwardCompatibilityCheckCommandV2Test.kt | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt index 18fd33bf7..2f70baa70 100644 --- a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt +++ b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt @@ -3,13 +3,46 @@ package application import application.backwardCompatibility.BackwardCompatibilityCheckCommandV2 import io.mockk.every import io.mockk.spyk +import io.specmatic.core.git.SystemGit import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test import java.io.File +import java.nio.file.Files class BackwardCompatibilityCheckCommandV2Test { + private lateinit var tempDir: File + private lateinit var remoteDir: File + + @BeforeEach + fun setup() { + tempDir = Files.createTempDirectory("git-local").toFile() + tempDir.deleteOnExit() + + remoteDir = Files.createTempDirectory("git-remote").toFile() + remoteDir.deleteOnExit() + + ProcessBuilder("git", "init", "--bare") + .directory(remoteDir) + .inheritIO() + .start() + .waitFor() + + ProcessBuilder("git", "init") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + + ProcessBuilder("git", "remote", "add", "origin", remoteDir.absolutePath) + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + } @Test fun `getSpecsReferringTo returns empty set when input is empty`() { @@ -52,11 +85,85 @@ class BackwardCompatibilityCheckCommandV2Test { assertEquals(setOf("file1.yaml", "file2.yaml"), result) } + @Nested + inner class SystemGitTestsSpecificToBackwardCompatibility { + @Test + fun `getFilesChangedInCurrentBranch returns the uncommitted, unstaged changed file`() { + File(tempDir, "file1.txt").writeText("File 1 content") + ProcessBuilder("git", "add", "file1.txt") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + ProcessBuilder("git", "commit", "-m", "Add file1") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + // Push the committed changes to the remote repository + ProcessBuilder("git", "push", "origin", "main") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + + + val uncommittedFile = File(tempDir, "file1.txt") + uncommittedFile.writeText("File 1 changed content") + + val gitCommand = SystemGit(tempDir.absolutePath) + val result = gitCommand.getFilesChangedInCurrentBranch( + gitCommand.currentRemoteBranch() + ) + + assert(result.contains("file1.txt")) + } + + @Test + fun `getFilesChangedInCurrentBranch returns the uncommitted, staged changed file`() { + File(tempDir, "file1.txt").writeText("File 1 content") + ProcessBuilder("git", "add", "file1.txt") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + ProcessBuilder("git", "commit", "-m", "Add file1") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + // Push the committed changes to the remote repository + ProcessBuilder("git", "push", "origin", "main") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + + + val uncommittedFile = File(tempDir, "file1.txt") + uncommittedFile.writeText("File 1 changed content") + ProcessBuilder("git", "add", "file1.txt") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + + val gitCommand = SystemGit(tempDir.absolutePath) + val result = gitCommand.getFilesChangedInCurrentBranch( + gitCommand.currentRemoteBranch() + ) + + assert(result.contains("file1.txt")) + } + } + @AfterEach fun `cleanup files`() { listOf("file1.yaml", "file2.yaml", "file3.yaml", "file4.yaml", "schema_file1.yaml", "schema_file2.yaml").forEach { File(it).delete() } + tempDir.deleteRecursively() + remoteDir.deleteRecursively() } private fun File.referTo(schemaFileName: String) { From 9b37ba7953d3dbbc4e0dd0baf25f2b41cec1835b Mon Sep 17 00:00:00 2001 From: Yogesh Ananda Nikam Date: Mon, 14 Oct 2024 17:50:30 +0530 Subject: [PATCH 18/43] Fix the failing test around git command --- .../BackwardCompatibilityCheckCommandV2Test.kt | 12 ++++++++++++ .../main/kotlin/io/specmatic/core/git/SystemGit.kt | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt index 2f70baa70..401b504d4 100644 --- a/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt +++ b/application/src/test/kotlin/application/BackwardCompatibilityCheckCommandV2Test.kt @@ -37,6 +37,18 @@ class BackwardCompatibilityCheckCommandV2Test { .start() .waitFor() + ProcessBuilder("git", "config", "--local", "user.name", "developer") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + + ProcessBuilder("git", "config", "--local", "user.email", "developer@example.com") + .directory(tempDir) + .inheritIO() + .start() + .waitFor() + ProcessBuilder("git", "remote", "add", "origin", remoteDir.absolutePath) .directory(tempDir) .inheritIO() diff --git a/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt b/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt index b8b351b96..4b46ced05 100644 --- a/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt +++ b/core/src/main/kotlin/io/specmatic/core/git/SystemGit.kt @@ -92,9 +92,9 @@ class SystemGit(override val workingDirectory: String = ".", private val prefix: override fun getFilesChangedInCurrentBranch(baseBranch: String): List { val committedLocalChanges = execute(Configuration.gitCommand, "diff", baseBranch, "HEAD", "--name-status") - .split(System.lineSeparator()) + .split("\n") val uncommittedChanges = execute(Configuration.gitCommand, "diff", "HEAD", "--name-status") - .split(System.lineSeparator()) + .split("\n") return (committedLocalChanges + uncommittedChanges).map { it.split("\t").last() From c333ac62ef2c633be03803b32c24c4dbb15e9deb Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Fri, 27 Sep 2024 16:03:41 +0530 Subject: [PATCH 19/43] Create abstract class ExamplesBaseCommand. - This class will be used in example generation and parity with other protocol impls. --- .../kotlin/application/SpecmaticCommand.kt | 3 + .../exampleGeneration/ExamplesBaseCommand.kt | 114 ++++++++++++++++++ .../exampleGeneration/ExamplesCommandV2.kt | 43 +++++++ 3 files changed, 160 insertions(+) create mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesBaseCommand.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesCommandV2.kt diff --git a/application/src/main/kotlin/application/SpecmaticCommand.kt b/application/src/main/kotlin/application/SpecmaticCommand.kt index d6591fe32..1a7fc32e2 100644 --- a/application/src/main/kotlin/application/SpecmaticCommand.kt +++ b/application/src/main/kotlin/application/SpecmaticCommand.kt @@ -1,5 +1,6 @@ package application +import application.exampleGeneration.ExamplesCommandV2 import application.backwardCompatibility.BackwardCompatibilityCheckCommandV2 import org.springframework.stereotype.Component import picocli.AutoComplete.GenerateCompletion @@ -12,6 +13,7 @@ import java.util.concurrent.Callable mixinStandardHelpOptions = true, versionProvider = VersionProvider::class, subcommands = [ + ExamplesCommandV2::class, BackwardCompatibilityCheckCommandV2::class, BackwardCompatibilityCheckCommand::class, BundleCommand::class, @@ -36,6 +38,7 @@ import java.util.concurrent.Callable CentralContractRepoReportCommand::class ] ) + class SpecmaticCommand : Callable { override fun call(): Int { return 0 diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBaseCommand.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBaseCommand.kt new file mode 100644 index 000000000..3dcc4dd17 --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBaseCommand.kt @@ -0,0 +1,114 @@ +package application.exampleGeneration + +import io.specmatic.core.EXAMPLES_DIR_SUFFIX +import io.specmatic.core.log.* +import picocli.CommandLine.Option +import picocli.CommandLine.Parameters +import java.io.File +import java.util.concurrent.Callable + +abstract class ExamplesBaseCommand: Callable { + + @Parameters(index = "0", description = ["Contract file path"], arity = "0..1") + lateinit var contractFile: File + + @Option(names = ["--debug"], description = ["Debug logs"]) + var verbose = false + + override fun call(): Int { + configureLogger(this.verbose) + + if (!contractFile.exists()) { + logger.log("Could not find file ${contractFile.path}") + return 1 // Failure + } + + return try { + val result = generateExamples() + logGenerationResult(result) + 0 // Success + } catch (e: Throwable) { + logger.log("Example generation failed with error: ${e.message}") + logger.debug(e) + 1 // Failure + } + } + + abstract fun contractFileToFeature(contractFile: File): Feature + + abstract fun getScenariosFromFeature(feature: Feature): List + + abstract fun getScenarioDescription(feature: Feature, scenario: Scenario): String + + abstract fun generateExampleFromScenario(feature: Feature, scenario: Scenario): Pair + + private fun configureLogger(verbose: Boolean) { + val logPrinters = listOf(ConsolePrinter) + + logger = if (verbose) + Verbose(CompositePrinter(logPrinters)) + else + NonVerbose(CompositePrinter(logPrinters)) + } + + private fun getExamplesDirectory(): File { + val examplesDirectory = contractFile.canonicalFile.parentFile.resolve("${this.contractFile.nameWithoutExtension}$EXAMPLES_DIR_SUFFIX") + if (!examplesDirectory.exists()) { + logger.log("Creating examples directory: $examplesDirectory") + examplesDirectory.mkdirs() + } + return examplesDirectory + } + + private fun generateExamples(): List { + val feature = contractFileToFeature(contractFile) + + return getScenariosFromFeature(feature).map { scenario -> + val description = getScenarioDescription(feature, scenario) + + try { + logger.log("Generating example for $description") + val (uniqueName, exampleContent) = generateExampleFromScenario(feature, scenario) + writeExampleToFile(exampleContent, uniqueName) + } catch (e: Throwable) { + logger.log("Failed to generate example for $description") + logger.debug(e) + ExampleGenerationResult(status = ExampleGenerationStatus.ERROR) + } finally { + logger.log("----------------------------------------") + } + } + } + + private fun writeExampleToFile(exampleContent: String, exampleFileName: String): ExampleGenerationResult { + val exampleFile = getExamplesDirectory().resolve("$exampleFileName.json") + + try { + exampleFile.writeText(exampleContent) + logger.log("Successfully saved example: $exampleFile") + return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.CREATED) + } catch (e: Throwable) { + logger.log("Failed to save example: $exampleFile") + logger.debug(e) + return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) + } + } + + private fun logGenerationResult(result: List) { + val resultCounts = result.groupBy { it.status }.mapValues { it.value.size } + val createdFileCount = resultCounts[ExampleGenerationStatus.CREATED] ?: 0 + val errorCount = resultCounts[ExampleGenerationStatus.ERROR] ?: 0 + val examplesDirectory = getExamplesDirectory().canonicalFile + + logger.log("\nNOTE: All examples can be found in $examplesDirectory") + logger.log("=============== Example Generation Summary ===============") + logger.log("$createdFileCount example(s) created, $errorCount examples(s) failed") + logger.log("==========================================================") + } + + enum class ExampleGenerationStatus { + CREATED, ERROR + } + + data class ExampleGenerationResult(val exampleFile: File? = null, val status: ExampleGenerationStatus) +} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesCommandV2.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesCommandV2.kt new file mode 100644 index 000000000..02504527e --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesCommandV2.kt @@ -0,0 +1,43 @@ +package application.exampleGeneration + +import io.specmatic.core.* +import io.specmatic.core.utilities.uniqueNameForApiOperation +import io.specmatic.mock.ScenarioStub +import org.springframework.stereotype.Component +import picocli.CommandLine +import java.io.File + +@Component +@CommandLine.Command( + name = "examples-v2", + mixinStandardHelpOptions = true, + description = ["Generate examples from a OpenAPI contract file"] +) +class ExamplesCommandV2: ExamplesBaseCommand() { + override fun contractFileToFeature(contractFile: File): Feature { + return parseContractFileToFeature(contractFile) + } + + override fun getScenariosFromFeature(feature: Feature): List { + return feature.scenarios + } + + override fun getScenarioDescription(feature: Feature, scenario: Scenario): String { + return scenario.testDescription().split("Scenario: ").last() + } + + override fun generateExampleFromScenario(feature: Feature, scenario: Scenario): Pair { + val request = scenario.generateHttpRequest() + val response = feature.lookupResponse(request).cleanup() + + val scenarioStub = ScenarioStub(request, response) + val stubJSON = scenarioStub.toJSON().toStringLiteral() + val uniqueName = uniqueNameForApiOperation(request, "", response.status) + + return Pair(uniqueName, stubJSON) + } + + private fun HttpResponse.cleanup(): HttpResponse { + return this.copy(headers = this.headers.minus(SPECMATIC_RESULT_HEADER)) + } +} \ No newline at end of file From 221b8c6161993eb4f4ddc462f38c04216f63f39e Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 3 Oct 2024 15:46:49 +0530 Subject: [PATCH 20/43] WIP Commit, Major Refactor For Examples Parity - New Abstract Classes for Examples Command. - Common Interface for example generation. - Moved InteractiveServer to application. - Move example template to JunitSupport. - Other HTML, CSS, JS Fixes. --- application/build.gradle | 7 + .../kotlin/application/ExamplesCommand.kt | 289 -------- .../kotlin/application/SpecmaticCommand.kt | 5 +- .../exampleGeneration/ExamplesBase.kt | 98 +++ .../exampleGeneration/ExamplesBaseCommand.kt | 114 --- .../exampleGeneration/ExamplesCommandV2.kt | 43 -- .../exampleGeneration/ExamplesCommon.kt | 152 ++++ .../ExamplesInteractiveBase.kt | 428 ++++++++++++ .../ExamplesValidationBase.kt | 174 +++++ .../exampleGeneration/ScenarioFilter.kt | 11 + .../openApiExamples/OpenApiExamples.kt | 18 + .../openApiExamples/OpenApiExamplesCommon.kt | 146 ++++ .../OpenApiExamplesInteractive.kt | 104 +++ .../OpenApiExamplesValidate.kt | 18 + .../server/ExamplesInteractiveServer.kt | 648 ------------------ .../core/examples/server/ExamplesView.kt | 122 ---- .../io/specmatic/test/TestInteractionsLog.kt | 6 +- .../test/reports/coverage/html/HtmlReport.kt | 2 +- .../html/HtmlTemplateConfiguration.kt | 8 + .../test/reports/coverage/html/HtmlUtils.kt | 2 +- .../resources/templates/example}/index.html | 278 ++++---- .../templates/{ => report}/assets/badge.svg | 0 .../templates/{ => report}/assets/blocked.svg | 0 .../{ => report}/assets/check-badge.svg | 0 .../assets/clipboard-document-list.svg | 0 .../templates/{ => report}/assets/clock.svg | 0 .../{ => report}/assets/download.svg | 0 .../assets/exclamation-triangle.svg | 0 .../templates/{ => report}/assets/favicon.svg | 0 .../templates/{ => report}/assets/main.js | 0 .../{ => report}/assets/mark-approved.svg | 0 .../{ => report}/assets/mark-rejected.svg | 0 .../{ => report}/assets/specmatic-logo.svg | 0 .../templates/{ => report}/assets/styles.css | 0 .../{ => report}/assets/summaryUpdater.js | 0 .../{ => report}/assets/tableFilter.js | 0 .../{ => report}/assets/trend-up.svg | 0 .../templates/{ => report}/assets/utils.js | 0 .../{ => report}/assets/x-circle.svg | 0 .../{report.html => report/index.html} | 0 40 files changed, 1315 insertions(+), 1358 deletions(-) delete mode 100644 application/src/main/kotlin/application/ExamplesCommand.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt delete mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesBaseCommand.kt delete mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesCommandV2.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesCommon.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesValidationBase.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/ScenarioFilter.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamples.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt delete mode 100644 core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServer.kt delete mode 100644 core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesView.kt rename {core/src/main/resources/templates/examples => junit5-support/src/main/resources/templates/example}/index.html (88%) rename junit5-support/src/main/resources/templates/{ => report}/assets/badge.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/blocked.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/check-badge.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/clipboard-document-list.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/clock.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/download.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/exclamation-triangle.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/favicon.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/main.js (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/mark-approved.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/mark-rejected.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/specmatic-logo.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/styles.css (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/summaryUpdater.js (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/tableFilter.js (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/trend-up.svg (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/utils.js (100%) rename junit5-support/src/main/resources/templates/{ => report}/assets/x-circle.svg (100%) rename junit5-support/src/main/resources/templates/{report.html => report/index.html} (100%) diff --git a/application/build.gradle b/application/build.gradle index 9e365a76a..25946da5c 100644 --- a/application/build.gradle +++ b/application/build.gradle @@ -82,6 +82,13 @@ dependencies { implementation "org.jetbrains.kotlinx:kotlinx-serialization-json-jvm:1.6.3" + implementation "io.ktor:ktor-server-netty:$ktor_version" + implementation "io.ktor:ktor-server-core:$ktor_version" + implementation "io.ktor:ktor-server-cors:$ktor_version" + implementation("io.ktor:ktor-server-content-negotiation:$ktor_version") + implementation("io.ktor:ktor-server-status-pages:$ktor_version") + implementation "io.ktor:ktor-serialization-jackson:$ktor_version" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit_version" testImplementation('org.springframework.boot:spring-boot-starter-test:3.3.4') { diff --git a/application/src/main/kotlin/application/ExamplesCommand.kt b/application/src/main/kotlin/application/ExamplesCommand.kt deleted file mode 100644 index d42270e61..000000000 --- a/application/src/main/kotlin/application/ExamplesCommand.kt +++ /dev/null @@ -1,289 +0,0 @@ -package application - -import io.specmatic.core.Result -import io.specmatic.core.Results -import io.specmatic.core.SPECMATIC_STUB_DICTIONARY -import io.specmatic.core.examples.server.ExamplesInteractiveServer -import io.specmatic.core.examples.server.ExamplesInteractiveServer.Companion.validateSingleExample -import io.specmatic.core.examples.server.loadExternalExamples -import io.specmatic.core.log.* -import io.specmatic.core.parseContractFileToFeature -import io.specmatic.core.pattern.ContractException -import io.specmatic.core.utilities.Flags -import io.specmatic.core.utilities.capitalizeFirstChar -import io.specmatic.core.utilities.exceptionCauseMessage -import io.specmatic.core.utilities.exitWithMessage -import io.specmatic.mock.ScenarioStub -import picocli.CommandLine.* -import java.io.File -import java.lang.Thread.sleep -import java.util.concurrent.Callable - -@Command( - name = "examples", - mixinStandardHelpOptions = true, - description = ["Generate externalised JSON example files with API requests and responses"], - subcommands = [ExamplesCommand.Validate::class, ExamplesCommand.Interactive::class] -) -class ExamplesCommand : Callable { - @Option( - names = ["--filter-name"], - description = ["Use only APIs with this value in their name"], - defaultValue = "\${env:SPECMATIC_FILTER_NAME}" - ) - var filterName: String = "" - - @Option( - names = ["--filter-not-name"], - description = ["Use only APIs which do not have this value in their name"], - defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}" - ) - var filterNotName: String = "" - - @Option( - names = ["--extensive"], - description = ["Generate all examples (by default, generates one example per 2xx API)"], - defaultValue = "false" - ) - var extensive: Boolean = false - - @Parameters(index = "0", description = ["Contract file path"], arity = "0..1") - var contractFile: File? = null - - @Option(names = ["--debug"], description = ["Debug logs"]) - var verbose = false - - @Option(names = ["--dictionary"], description = ["External Dictionary File Path, defaults to dictionary.json"]) - var dictionaryFile: File? = null - - override fun call(): Int { - if (contractFile == null) { - println("No contract file provided. Use a subcommand or provide a contract file. Use --help for more details.") - return 1 - } - if (!contractFile!!.exists()) { - logger.log("Could not find file ${contractFile!!.path}") - return 1 - } - - configureLogger(this.verbose) - - try { - dictionaryFile?.also { - System.setProperty(SPECMATIC_STUB_DICTIONARY, it.path) - } - - ExamplesInteractiveServer.generate( - contractFile!!, - ExamplesInteractiveServer.ScenarioFilter(filterName, filterNotName), - extensive, - ) - } catch (e: Throwable) { - logger.log(e) - return 1 - } - - return 0 - } - - @Command( - name = "validate", - mixinStandardHelpOptions = true, - description = ["Validate the examples"] - ) - class Validate : Callable { - @Option(names = ["--contract-file"], description = ["Contract file path"], required = true) - lateinit var contractFile: File - - @Option(names = ["--example-file"], description = ["Example file path"], required = false) - val exampleFile: File? = null - - @Option(names = ["--debug"], description = ["Debug logs"]) - var verbose = false - - @Option( - names = ["--filter-name"], - description = ["Validate examples of only APIs with this value in their name"], - defaultValue = "\${env:SPECMATIC_FILTER_NAME}" - ) - var filterName: String = "" - - @Option( - names = ["--filter-not-name"], - description = ["Validate examples of only APIs which do not have this value in their name"], - defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}" - ) - var filterNotName: String = "" - - override fun call(): Int { - if (!contractFile.exists()) { - logger.log("Could not find file ${contractFile.path}") - return 1 - } - - configureLogger(this.verbose) - - if (exampleFile != null) { - try { - validateSingleExample(contractFile, exampleFile).throwOnFailure() - - logger.log("The provided example ${exampleFile.name} is valid.") - } catch (e: ContractException) { - logger.log("The provided example ${exampleFile.name} is invalid. Reason:\n") - logger.log(exceptionCauseMessage(e)) - return 1 - } - } else { - val scenarioFilter = ExamplesInteractiveServer.ScenarioFilter(filterName, filterNotName) - - val (validateInline, validateExternal) = if(!Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") && !Flags.getBooleanValue("IGNORE_INLINE_EXAMPLES")) { - true to true - } else { - Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") to Flags.getBooleanValue("IGNORE_INLINE_EXAMPLES") - } - - val feature = parseContractFileToFeature(contractFile) - - val inlineExampleValidationResults = if(validateInline) { - val inlineExamples = feature.stubsFromExamples.mapValues { - it.value.map { - ScenarioStub(it.first, it.second) - } - } - - ExamplesInteractiveServer.validateMultipleExamples(feature, examples = inlineExamples, inline = true, scenarioFilter = scenarioFilter) - } else emptyMap() - - val externalExampleValidationResults = if(validateExternal) { - val (externalExampleDir, externalExamples) = loadExternalExamples(contractFile) - - if(!externalExampleDir.exists()) { - logger.log("$externalExampleDir does not exist, did not find any files to validate") - return 1 - } - - if(externalExamples.none()) { - logger.log("No example files found in $externalExampleDir") - return 1 - } - - ExamplesInteractiveServer.validateMultipleExamples(feature, examples = externalExamples, scenarioFilter = scenarioFilter) - } else emptyMap() - - val hasFailures = inlineExampleValidationResults.any { it.value is Result.Failure } || externalExampleValidationResults.any { it.value is Result.Failure } - - printValidationResult(inlineExampleValidationResults, "Inline example") - printValidationResult(externalExampleValidationResults, "Example file") - - if(hasFailures) - return 1 - } - - return 0 - } - - private fun printValidationResult(validationResults: Map, tag: String) { - if(validationResults.isEmpty()) - return - - val hasFailures = validationResults.any { it.value is Result.Failure } - - val titleTag = tag.split(" ").joinToString(" ") { if(it.isBlank()) it else it.capitalizeFirstChar() } - - if(hasFailures) { - println() - logger.log("=============== $titleTag Validation Results ===============") - - validationResults.forEach { (exampleFileName, result) -> - if (!result.isSuccess()) { - logger.log(System.lineSeparator() + "$tag $exampleFileName has the following validation error(s):") - logger.log(result.reportString()) - } - } - } - - println() - val summaryTitle = "=============== $titleTag Validation Summary ===============" - logger.log(summaryTitle) - logger.log(Results(validationResults.values.toList()).summary()) - logger.log("=".repeat(summaryTitle.length)) - } - } - - @Command( - name = "interactive", - mixinStandardHelpOptions = true, - description = ["Run the example generation interactively"] - ) - class Interactive : Callable { - @Option(names = ["--contract-file"], description = ["Contract file path"], required = false) - var contractFile: File? = null - - @Option( - names = ["--filter-name"], - description = ["Use only APIs with this value in their name"], - defaultValue = "\${env:SPECMATIC_FILTER_NAME}" - ) - var filterName: String = "" - - @Option( - names = ["--filter-not-name"], - description = ["Use only APIs which do not have this value in their name"], - defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}" - ) - var filterNotName: String = "" - - @Option(names = ["--debug"], description = ["Debug logs"]) - var verbose = false - - @Option(names = ["--dictionary"], description = ["External Dictionary File Path"]) - var dictFile: File? = null - - @Option(names = ["--testBaseURL"], description = ["The baseURL of system to test"], required = false) - var testBaseURL: String? = null - - var server: ExamplesInteractiveServer? = null - - override fun call() { - configureLogger(verbose) - - try { - if (contractFile != null && !contractFile!!.exists()) - exitWithMessage("Could not find file ${contractFile!!.path}") - - server = ExamplesInteractiveServer("0.0.0.0", 9001, testBaseURL, contractFile, filterName, filterNotName, dictFile) - addShutdownHook() - - consoleLog(StringLog("Examples Interactive server is running on http://0.0.0.0:9001/_specmatic/examples. Ctrl + C to stop.")) - while (true) sleep(10000) - } catch (e: Exception) { - logger.log(exceptionCauseMessage(e)) - exitWithMessage(e.message.orEmpty()) - } - } - - private fun addShutdownHook() { - Runtime.getRuntime().addShutdownHook(object : Thread() { - override fun run() { - try { - println("Shutting down examples interactive server...") - server?.close() - } catch (e: InterruptedException) { - currentThread().interrupt() - } catch (e: Throwable) { - logger.log(e) - } - } - }) - } - } -} - -private fun configureLogger(verbose: Boolean) { - val logPrinters = listOf(ConsolePrinter) - - logger = if (verbose) - Verbose(CompositePrinter(logPrinters)) - else - NonVerbose(CompositePrinter(logPrinters)) -} diff --git a/application/src/main/kotlin/application/SpecmaticCommand.kt b/application/src/main/kotlin/application/SpecmaticCommand.kt index 1a7fc32e2..639fb5a47 100644 --- a/application/src/main/kotlin/application/SpecmaticCommand.kt +++ b/application/src/main/kotlin/application/SpecmaticCommand.kt @@ -1,7 +1,7 @@ package application -import application.exampleGeneration.ExamplesCommandV2 import application.backwardCompatibility.BackwardCompatibilityCheckCommandV2 +import application.exampleGeneration.openApiExamples.OpenApiExamples import org.springframework.stereotype.Component import picocli.AutoComplete.GenerateCompletion import picocli.CommandLine.Command @@ -13,7 +13,7 @@ import java.util.concurrent.Callable mixinStandardHelpOptions = true, versionProvider = VersionProvider::class, subcommands = [ - ExamplesCommandV2::class, + OpenApiExamples::class, BackwardCompatibilityCheckCommandV2::class, BackwardCompatibilityCheckCommand::class, BundleCommand::class, @@ -29,7 +29,6 @@ import java.util.concurrent.Callable ProxyCommand::class, PushCommand::class, ReDeclaredAPICommand::class, - ExamplesCommand::class, SamplesCommand::class, StubCommand::class, SubscribeCommand::class, diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt new file mode 100644 index 000000000..2ff6d1f73 --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt @@ -0,0 +1,98 @@ +package application.exampleGeneration + +import io.specmatic.core.* +import io.specmatic.core.log.* +import picocli.CommandLine.* +import java.io.File +import java.util.concurrent.Callable + +abstract class ExamplesBase(private val common: ExamplesCommon): Callable { + @Parameters(index = "0", description = ["Contract file path"], arity = "0..1") + private var contractFile: File? = null + + @Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") + var filterName: String = "" + + @Option(names = ["--filter-not-name"], description = ["Use only APIs which do not have this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}") + var filterNotName: String = "" + + @Option(names = ["--dictionary"], description = ["External Dictionary File Path, defaults to dictionary.json"]) + private var dictFile: File? = null + + @Option(names = ["--debug"], description = ["Debug logs"]) + private var verbose = false + + abstract var extensive: Boolean + + override fun call(): Int { + common.configureLogger(this.verbose) + + contractFile?.let { contract -> + if (!contract.exists()) { + logger.log("Could not find Contract file ${contract.path}") + return 1 + } + + if (contract.extension !in common.contractFileExtensions) { + logger.log("Invalid Contract file ${contract.path} - File extension must be one of ${common.contractFileExtensions.joinToString()}") + return 1 + } + + try { + val externalDictionary = common.loadExternalDictionary(dictFile, contract) + val examplesDir = common.getExamplesDirectory(contract) + val result = generateExamples(contract, externalDictionary, examplesDir) + logGenerationResult(result, examplesDir) + return 0 + } catch (e: Throwable) { + logger.log("Example generation failed with error: ${e.message}") + logger.debug(e) + return 1 + } + } ?: run { + logger.log("No contract file provided. Use a subcommand or provide a contract file. Use --help for more details.") + return 1 + } + } + + // GENERATION METHODS + private fun getFilteredScenarios(feature: Feature): List { + val scenarioFilter = ScenarioFilter(filterName, filterNotName, extensive) + return common.getFilteredScenarios(feature, scenarioFilter) + } + + private fun generateExamples(contractFile: File, externalDictionary: Dictionary, examplesDir: File): List { + val feature = common.contractFileToFeature(contractFile) + val filteredScenarios = getFilteredScenarios(feature) + + if (filteredScenarios.isEmpty()) { + return emptyList() + } + + val exampleFiles = common.getExternalExampleFiles(examplesDir) + return filteredScenarios.map { scenario -> + common.generateOrGetExistingExample(feature, scenario, externalDictionary, exampleFiles, examplesDir) + } + } + + // HELPERS + private fun logGenerationResult(generations: List, examplesDir: File) { + val generationGroup = generations.groupBy { it.status }.mapValues { it.value.size } + val createdFileCount = generationGroup[ExampleGenerationStatus.CREATED] ?: 0 + val errorCount = generationGroup[ExampleGenerationStatus.ERROR] ?: 0 + val existingCount = generationGroup[ExampleGenerationStatus.EXISTS] ?: 0 + val examplesDirectory = examplesDir.canonicalFile.absolutePath + + common.logFormattedOutput( + header = "Example Generation Summary", + summary = "$createdFileCount example(s) created, $existingCount example(s) already existed, $errorCount example(s) failed", + note = "NOTE: All examples can be found in $examplesDirectory" + ) + } +} + +enum class ExampleGenerationStatus { + CREATED, ERROR, EXISTS +} + +data class ExampleGenerationResult(val exampleFile: File? = null, val status: ExampleGenerationStatus) \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBaseCommand.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBaseCommand.kt deleted file mode 100644 index 3dcc4dd17..000000000 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesBaseCommand.kt +++ /dev/null @@ -1,114 +0,0 @@ -package application.exampleGeneration - -import io.specmatic.core.EXAMPLES_DIR_SUFFIX -import io.specmatic.core.log.* -import picocli.CommandLine.Option -import picocli.CommandLine.Parameters -import java.io.File -import java.util.concurrent.Callable - -abstract class ExamplesBaseCommand: Callable { - - @Parameters(index = "0", description = ["Contract file path"], arity = "0..1") - lateinit var contractFile: File - - @Option(names = ["--debug"], description = ["Debug logs"]) - var verbose = false - - override fun call(): Int { - configureLogger(this.verbose) - - if (!contractFile.exists()) { - logger.log("Could not find file ${contractFile.path}") - return 1 // Failure - } - - return try { - val result = generateExamples() - logGenerationResult(result) - 0 // Success - } catch (e: Throwable) { - logger.log("Example generation failed with error: ${e.message}") - logger.debug(e) - 1 // Failure - } - } - - abstract fun contractFileToFeature(contractFile: File): Feature - - abstract fun getScenariosFromFeature(feature: Feature): List - - abstract fun getScenarioDescription(feature: Feature, scenario: Scenario): String - - abstract fun generateExampleFromScenario(feature: Feature, scenario: Scenario): Pair - - private fun configureLogger(verbose: Boolean) { - val logPrinters = listOf(ConsolePrinter) - - logger = if (verbose) - Verbose(CompositePrinter(logPrinters)) - else - NonVerbose(CompositePrinter(logPrinters)) - } - - private fun getExamplesDirectory(): File { - val examplesDirectory = contractFile.canonicalFile.parentFile.resolve("${this.contractFile.nameWithoutExtension}$EXAMPLES_DIR_SUFFIX") - if (!examplesDirectory.exists()) { - logger.log("Creating examples directory: $examplesDirectory") - examplesDirectory.mkdirs() - } - return examplesDirectory - } - - private fun generateExamples(): List { - val feature = contractFileToFeature(contractFile) - - return getScenariosFromFeature(feature).map { scenario -> - val description = getScenarioDescription(feature, scenario) - - try { - logger.log("Generating example for $description") - val (uniqueName, exampleContent) = generateExampleFromScenario(feature, scenario) - writeExampleToFile(exampleContent, uniqueName) - } catch (e: Throwable) { - logger.log("Failed to generate example for $description") - logger.debug(e) - ExampleGenerationResult(status = ExampleGenerationStatus.ERROR) - } finally { - logger.log("----------------------------------------") - } - } - } - - private fun writeExampleToFile(exampleContent: String, exampleFileName: String): ExampleGenerationResult { - val exampleFile = getExamplesDirectory().resolve("$exampleFileName.json") - - try { - exampleFile.writeText(exampleContent) - logger.log("Successfully saved example: $exampleFile") - return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.CREATED) - } catch (e: Throwable) { - logger.log("Failed to save example: $exampleFile") - logger.debug(e) - return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) - } - } - - private fun logGenerationResult(result: List) { - val resultCounts = result.groupBy { it.status }.mapValues { it.value.size } - val createdFileCount = resultCounts[ExampleGenerationStatus.CREATED] ?: 0 - val errorCount = resultCounts[ExampleGenerationStatus.ERROR] ?: 0 - val examplesDirectory = getExamplesDirectory().canonicalFile - - logger.log("\nNOTE: All examples can be found in $examplesDirectory") - logger.log("=============== Example Generation Summary ===============") - logger.log("$createdFileCount example(s) created, $errorCount examples(s) failed") - logger.log("==========================================================") - } - - enum class ExampleGenerationStatus { - CREATED, ERROR - } - - data class ExampleGenerationResult(val exampleFile: File? = null, val status: ExampleGenerationStatus) -} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesCommandV2.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesCommandV2.kt deleted file mode 100644 index 02504527e..000000000 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesCommandV2.kt +++ /dev/null @@ -1,43 +0,0 @@ -package application.exampleGeneration - -import io.specmatic.core.* -import io.specmatic.core.utilities.uniqueNameForApiOperation -import io.specmatic.mock.ScenarioStub -import org.springframework.stereotype.Component -import picocli.CommandLine -import java.io.File - -@Component -@CommandLine.Command( - name = "examples-v2", - mixinStandardHelpOptions = true, - description = ["Generate examples from a OpenAPI contract file"] -) -class ExamplesCommandV2: ExamplesBaseCommand() { - override fun contractFileToFeature(contractFile: File): Feature { - return parseContractFileToFeature(contractFile) - } - - override fun getScenariosFromFeature(feature: Feature): List { - return feature.scenarios - } - - override fun getScenarioDescription(feature: Feature, scenario: Scenario): String { - return scenario.testDescription().split("Scenario: ").last() - } - - override fun generateExampleFromScenario(feature: Feature, scenario: Scenario): Pair { - val request = scenario.generateHttpRequest() - val response = feature.lookupResponse(request).cleanup() - - val scenarioStub = ScenarioStub(request, response) - val stubJSON = scenarioStub.toJSON().toStringLiteral() - val uniqueName = uniqueNameForApiOperation(request, "", response.status) - - return Pair(uniqueName, stubJSON) - } - - private fun HttpResponse.cleanup(): HttpResponse { - return this.copy(headers = this.headers.minus(SPECMATIC_RESULT_HEADER)) - } -} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesCommon.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesCommon.kt new file mode 100644 index 000000000..cbe6f4f0c --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesCommon.kt @@ -0,0 +1,152 @@ +package application.exampleGeneration + +import io.specmatic.core.* +import io.specmatic.core.log.* +import io.specmatic.mock.loadDictionary +import java.io.File + +interface ExamplesCommon { + val exampleFileExtensions: Set + val contractFileExtensions: Set + + fun contractFileToFeature(contractFile: File): Feature + + fun getScenariosFromFeature(feature: Feature, extensive: Boolean) : List + + fun getScenarioDescription(scenario: Scenario): String + + fun getExistingExampleOrNull(scenario: Scenario, exampleFiles: List): Pair? + + fun generateExample(feature: Feature, scenario: Scenario, dictionary: Dictionary): Pair + + fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature + + fun validateExternalExample(feature: Feature, exampleFile: File): Pair + + fun configureLogger(verbose: Boolean) { + val logPrinters = listOf(ConsolePrinter) + + logger = if (verbose) + Verbose(CompositePrinter(logPrinters)) + else + NonVerbose(CompositePrinter(logPrinters)) + } + + fun getFilteredScenarios(feature: Feature, scenarioFilter: ScenarioFilter): List { + val filteredScenarios = getScenariosFromFeature(feature, scenarioFilter.extensive) + .filterScenarios(scenarioFilter.filterNameTokens, shouldMatch = true) + .filterScenarios(scenarioFilter.filterNotNameTokens, shouldMatch = false) + + if (filteredScenarios.isEmpty()) { + logger.log("Note: All examples were filtered out by the filter expression") + } + + return filteredScenarios + } + + fun getExamplesDirectory(contractFile: File): File { + val examplesDirectory = contractFile.canonicalFile.parentFile.resolve("${contractFile.nameWithoutExtension}$EXAMPLES_DIR_SUFFIX") + if (!examplesDirectory.exists()) { + logger.log("Creating examples directory: $examplesDirectory") + examplesDirectory.mkdirs() + } + return examplesDirectory + } + + fun getExternalExampleFiles(examplesDirectory: File): List { + return examplesDirectory.walk().filter { it.isFile && it.extension in exampleFileExtensions }.toList() + } + + fun generateOrGetExistingExample(feature: Feature, scenario: Scenario, externalDictionary: Dictionary, exampleFiles: List, examplesDir: File): ExampleGenerationResult { + return try { + val existingExample = getExistingExampleOrNull(scenario, exampleFiles) + val description = getScenarioDescription(scenario) + + if (existingExample != null) { + logger.log("Using existing example for $description\nExample File: ${existingExample.first.absolutePath}") + return ExampleGenerationResult(existingExample.first, ExampleGenerationStatus.EXISTS) + } + + logger.log("Generating example for $description") + val (uniqueFileName, exampleContent) = generateExample(feature, scenario, externalDictionary) + return writeExampleToFile(exampleContent, uniqueFileName, examplesDir) + } catch (e: Throwable) { + logger.log("Failed to generate example: ${e.message}") + logger.debug(e) + ExampleGenerationResult(null, ExampleGenerationStatus.ERROR) + } finally { + logSeparator(50) + } + } + + fun writeExampleToFile(exampleContent: String, exampleFileName: String, examplesDir: File): ExampleGenerationResult { + val exampleFile = examplesDir.resolve(exampleFileName) + + if (exampleFile.extension !in exampleFileExtensions) { + logger.log("Invalid example file extension: ${exampleFile.extension}") + return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) + } + + try { + exampleFile.writeText(exampleContent) + logger.log("Successfully saved example: $exampleFile") + return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.CREATED) + } catch (e: Throwable) { + logger.log("Failed to save example: $exampleFile") + logger.debug(e) + return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) + } + } + + fun loadExternalDictionary(dictFile: File?, contractFile: File?): Dictionary { + val dictFilePath = when { + dictFile != null -> { + if (!dictFile.exists()) throw IllegalStateException("Dictionary file not found: ${dictFile.path}") + else dictFile.path + } + + contractFile != null -> { + val dictFileName = "${contractFile.nameWithoutExtension}$DICTIONARY_FILE_SUFFIX" + contractFile.canonicalFile.parentFile.resolve(dictFileName).takeIf { it.exists() }?.path + } + + else -> { + val currentDir = File(System.getProperty("user.dir")) + currentDir.resolve("dictionary.json").takeIf { it.exists() }?.path + } + } + + return dictFilePath?.let { + Dictionary(loadDictionary(dictFilePath)) + } ?: Dictionary(emptyMap()) + } + + fun logSeparator(length: Int, separator: String = "-") { + logger.log(separator.repeat(length)) + } + + fun logFormattedOutput(header: String, summary: String, note: String) { + val maxLength = maxOf(summary.length, note.length, 50).let { it + it % 2 } + val headerSidePadding = (maxLength - 2 - header.length) / 2 + + val paddedHeaderLine = "=".repeat(headerSidePadding) + " $header " + "=".repeat(headerSidePadding) + val paddedSummaryLine = summary.padEnd(maxLength) + val paddedNoteLine = note.padEnd(maxLength) + + logger.log("\n$paddedHeaderLine") + logger.log(paddedSummaryLine) + logger.log("=".repeat(maxLength)) + logger.log(paddedNoteLine) + } + + private fun List.filterScenarios(tokens: Set, shouldMatch: Boolean): List { + if (tokens.isEmpty()) return this + + return this.filter { + val description = getScenarioDescription(it) + tokens.any { token -> + description.contains(token) + } == shouldMatch + } + } +} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt new file mode 100644 index 000000000..defdcbf25 --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -0,0 +1,428 @@ +package application.exampleGeneration + +import io.ktor.http.* +import io.ktor.serialization.jackson.* +import io.ktor.server.application.* +import io.ktor.server.engine.* +import io.ktor.server.netty.* +import io.ktor.server.plugins.contentnegotiation.* +import io.ktor.server.plugins.cors.routing.* +import io.ktor.server.plugins.statuspages.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* +import io.ktor.util.pipeline.* +import io.specmatic.core.TestResult +import io.specmatic.core.log.logger +import io.specmatic.core.route.modules.HealthCheckModule.Companion.configureHealthCheckModule +import io.specmatic.test.reports.coverage.html.HtmlTemplateConfiguration +import picocli.CommandLine.Option +import java.io.Closeable +import java.io.File +import java.io.FileNotFoundException +import java.lang.Thread.sleep +import java.util.concurrent.Callable + +abstract class ExamplesInteractiveBase(val common: ExamplesCommon): Callable { + @Option(names = ["--testBaseURL"], description = ["BaseURL of the SUT"], required = true) + lateinit var serverHost: String + + @Option(names = ["--contract-file"], description = ["Contract file path"], required = false) + var contractFile: File? = null + + @Option(names = ["--debug"], description = ["Debug logs"]) + var verbose = false + + @Option(names = ["--dictionary"], description = ["External Dictionary File Path"]) + var dictFile: File? = null + + @Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") + var filterName: String = "" + + @Option(names = ["--filter-not-name"], description = ["Use only APIs which do not have this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}") + var filterNotName: String = "" + + abstract var extensive: Boolean + abstract val htmlTableColumns: List + private var cachedContractFileFromRequest: File? = null + + override fun call(): Int { + common.configureLogger(verbose) + + try { + contractFile?.let { contract -> + if (!contract.exists()) { + logger.log("Could not find Contract file ${contract.path}") + return 1 + } + + if (contract.extension !in common.contractFileExtensions) { + logger.log("Invalid Contract file ${contract.path} - File extension must be one of ${common.contractFileExtensions.joinToString()}") + return 1 + } + + } ?: logger.log("No contract file provided, Please provide a contract file in the HTTP request.") + + val server = InteractiveServer("0.0.0.0", 9001) + addShutdownHook(server) + logger.log("Examples Interactive server is running on http://0.0.0.0:9001/_specmatic/examples. Ctrl + C to stop.") + while (true) sleep(10000) + } catch (e: Throwable) { + logger.log("Example Interactive server failed with error: ${e.message}") + logger.debug(e) + return 1 + } + } + + // HOOKS + abstract fun createTableRows(scenarios: List, exampleFiles: List): List + + abstract suspend fun getScenarioFromRequestOrNull(call: ApplicationCall, feature: Feature): Scenario? + + abstract fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair + + // HELPER METHODS + private suspend fun generateExample(call: ApplicationCall, contractFile: File): ExampleGenerationResult { + val feature = common.contractFileToFeature(contractFile) + val dictionary = common.loadExternalDictionary(dictFile, contractFile) + val examplesDir = common.getExamplesDirectory(contractFile) + val exampleFiles = common.getExternalExampleFiles(examplesDir) + + return getScenarioFromRequestOrNull(call, feature)?.let { + common.generateOrGetExistingExample(feature, it, dictionary, exampleFiles, examplesDir) + } ?: throw IllegalArgumentException("No matching scenario found for request") + } + + private fun validateExample(contractFile: File, exampleFile: File): ExampleValidationResult { + val feature = common.contractFileToFeature(contractFile) + val result = common.validateExternalExample(feature, exampleFile) + return ExampleValidationResult(exampleFile.absolutePath, result.second, ExampleType.EXTERNAL) + } + + private fun testExample(contractFile: File, exampleFile: File): Pair { + val feature = common.contractFileToFeature(contractFile) + val result = testExternalExample(feature, exampleFile, serverHost) + return Pair(result.first, result.second) + } + + private fun getFilteredScenarios(feature: Feature): List { + val scenarioFilter = ScenarioFilter(filterName, filterNotName, extensive) + return common.getFilteredScenarios(feature, scenarioFilter) + } + + private fun getContractFileOrNull(request: ExamplePageRequest? = null): File? { + return contractFile?.takeIf { it.exists() }?.also { contract -> + logger.debug("Using Contract file ${contract.path} provided via command line") + } ?: request?.contractFile?.takeIf { it.exists() }?.also { contract -> + logger.debug("Using Contract file ${contract.path} provided via HTTP request") + } ?: cachedContractFileFromRequest?.takeIf { it.exists() }?.also { contract -> + logger.debug("Using Contract file ${contract.path} provided via cached HTTP request") + } + } + + private fun validateRows(tableRows: List): List { + tableRows.forEach { row -> + require(row.columns.size == htmlTableColumns.size) { + logger.debug("Invalid Row: $row") + throw IllegalArgumentException("Incorrect number of columns in table row. Expected: ${htmlTableColumns.size}, Actual: ${row.columns.size}") + } + + row.columns.forEachIndexed { index, it -> + require(it.columnName == htmlTableColumns[index].name) { + logger.debug("Invalid Column Row: $row") + throw IllegalArgumentException("Incorrect column name in table row. Expected: ${htmlTableColumns[index].name}, Actual: ${it.columnName}") + } + } + } + + return tableRows + } + + private fun getTableRows(contractFile: File): List { + val feature = common.contractFileToFeature(contractFile) + val scenarios = getFilteredScenarios(feature) + val examplesDir = common.getExamplesDirectory(contractFile) + val examples = common.getExternalExampleFiles(examplesDir) + val tableRows = createTableRows(scenarios, examples) + + return validateRows(tableRows) + } + + // INTERACTIVE SERVER + inner class InteractiveServer(private val serverHost: String, private val serverPort: Int) : Closeable { + private val environment = applicationEngineEnvironment { + module { + install(CORS) { + allowMethod(HttpMethod.Options) + allowMethod(HttpMethod.Post) + allowMethod(HttpMethod.Get) + allowHeader(HttpHeaders.AccessControlAllowOrigin) + allowHeader(HttpHeaders.ContentType) + anyHost() + } + + install(ContentNegotiation) { + jackson {} + } + + install(StatusPages) { + exception { call, cause -> + logger.debug(cause) + call.respondWithError(cause) + } + } + + configureHealthCheckModule() + + routing { + get("/_specmatic/examples") { + getValidatedContractFileOrNull()?.let { contract -> + val hostPort = getServerHostAndPort() + val htmlContent = getHtmlContent(contract, hostPort) + call.respondText(htmlContent, contentType = ContentType.Text.Html) + } + } + + post("/_specmatic/examples") { + val request = call.receive() + getValidatedContractFileOrNull(request)?.let { contract -> + val hostPort = getServerHostAndPort() + val htmlContent = getHtmlContent(contract, hostPort) + call.respondText(htmlContent, contentType = ContentType.Text.Html) + } + } + + post("/_specmatic/examples/generate") { + getValidatedContractFileOrNull()?.let { contractFile -> + val result = generateExample(call, contractFile) + call.respond(HttpStatusCode.OK, GenerateExampleResponse(result)) + } + } + + post("/_specmatic/examples/validate") { + val request = call.receive() + getValidatedContractFileOrNull()?.let { contract -> + getValidatedExampleOrNull(request.exampleFile)?.let { example -> + val result = validateExample(contract, example) + call.respond(HttpStatusCode.OK, ExampleValidationResponse(result)) + } + } + } + + post("/_specmatic/examples/content") { + val request = call.receive() + getValidatedExampleOrNull(request.exampleFile)?.let { example -> + call.respond(HttpStatusCode.OK, mapOf("content" to example.readText())) + } + } + + post("/_specmatic/examples/test") { + val request = call.receive() + getValidatedContractFileOrNull()?.let { contract -> + getValidatedExampleOrNull(request.exampleFile)?.let { example -> + val (result, testLog) = testExample(contract, example) + call.respond(HttpStatusCode.OK, ExampleTestResponse(result, testLog, example)) + } + } + } + } + } + + connector { + this.host = serverHost + this.port = serverPort + } + } + + private val server: ApplicationEngine = embeddedServer(Netty, environment, configure = { + this.requestQueueLimit = 1000 + this.callGroupSize = 5 + this.connectionGroupSize = 20 + this.workerGroupSize = 20 + }) + + init { + server.start() + } + + override fun close() { + server.stop(0, 0) + } + + private fun getServerHostAndPort(request: ExamplePageRequest? = null): String { + return request?.hostPort.takeIf { + !it.isNullOrEmpty() + } ?: "localhost:$serverPort" + } + + private fun getHtmlContent(contractFile: File, hostPort: String): String { + val tableRows = getTableRows(contractFile) + val htmlContent = renderTemplate(contractFile, hostPort, tableRows) + return htmlContent + } + + private suspend fun ApplicationCall.respondWithError(httpStatusCode: HttpStatusCode, errorMessage: String) { + this.respond(httpStatusCode, mapOf("error" to errorMessage)) + } + + private suspend fun ApplicationCall.respondWithError(throwable: Throwable, errorMessage: String? = throwable.message) { + val statusCode = when (throwable) { + is IllegalArgumentException, is FileNotFoundException -> HttpStatusCode.BadRequest + else -> HttpStatusCode.InternalServerError + } + this.respondWithError(statusCode, errorMessage ?: throwable.message ?: "Something went wrong") + } + + private suspend fun PipelineContext.getValidatedContractFileOrNull(request: ExamplePageRequest? = null): File? { + val contractFile = getContractFileOrNull(request) ?: run { + val errorMessage = "No Contract File Found - Please provide a contract file in the command line or in the HTTP request." + logger.log(errorMessage) + call.respondWithError(HttpStatusCode.BadRequest, errorMessage) + return null + } + + return contractFile.takeIf { it.extension in common.contractFileExtensions } ?: run { + val errorMessage = "Invalid Contract file ${contractFile.path} - File extension must be one of ${common.contractFileExtensions.joinToString()}" + logger.log(errorMessage) + call.respondWithError(HttpStatusCode.BadRequest, errorMessage) + return null + } + } + + private suspend fun PipelineContext.getValidatedExampleOrNull(exampleFile: File): File? { + return when { + !exampleFile.exists() -> { + val errorMessage = "Could not find Example file ${exampleFile.path}" + logger.log(errorMessage) + call.respondWithError(HttpStatusCode.BadRequest, errorMessage) + return null + } + + exampleFile.extension !in common.exampleFileExtensions -> { + val errorMessage = "Invalid Example file ${exampleFile.path} - File extension must be one of ${common.exampleFileExtensions.joinToString()}" + logger.log(errorMessage) + call.respondWithError(HttpStatusCode.BadRequest, errorMessage) + return null + } + + else -> exampleFile + } + } + + private fun renderTemplate(contractFile: File, hostPort: String, tableRows: List): String { + val variables = mapOf( + "tableColumns" to htmlTableColumns, + "tableRows" to tableRows, + "contractFileName" to contractFile.name, + "contractFilePath" to contractFile.absolutePath, + "hostPort" to hostPort, + "hasExamples" to tableRows.any { it.exampleFilePath != null }, + "validationDetails" to tableRows.mapIndexed { index, row -> + (index + 1) to row.exampleMismatchReason + }.toMap() + ) + + return HtmlTemplateConfiguration.process( + templateName = "example/index.html", + variables = variables + ) + } + } + + private fun addShutdownHook(server: InteractiveServer) { + Runtime.getRuntime().addShutdownHook(Thread { + logger.log("Shutting down examples interactive server...") + try { + server.close() + logger.log("Server shutdown completed successfully.") + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + logger.log("Server shutdown interrupted.") + } catch (e: Throwable) { + logger.log("Server shutdown failed with error: ${e.message}") + logger.debug(e) + } + }) + } +} + +data class ExamplePageRequest ( + val contractFile: File, + val hostPort: String? +) + +data class GenerateExampleResponse( + val exampleFilePath: String, + val status: String +) { + constructor(result: ExampleGenerationResult): this( + exampleFilePath = result.exampleFile?.absolutePath ?: throw Exception("Failed to generate example file"), + status = result.status.toString() + ) +} + +data class ExampleValidationRequest ( + val exampleFile: File +) + +data class ExampleValidationResponse( + val exampleFilePath: String, + val error: String? = null +) { + constructor(result: ExampleValidationResult): this( + exampleFilePath = result.exampleName, error = result.result.reportString().takeIf { it.isNotBlank() } + ) +} + +data class ExampleContentRequest ( + val exampleFile: File +) + +data class ExampleTestRequest ( + val exampleFile: File +) + +data class ExampleTestResponse( + val result: TestResult, + val details: String, + val testLog: String +) { + constructor(result: TestResult, testLog: String, exampleFile: File): this ( + result = result, + details = resultToDetails(result, exampleFile), + testLog = testLog.trim('-', ' ', '\n', '\r') + ) + + companion object { + fun resultToDetails(result: TestResult, exampleFile: File): String { + val postFix = when(result) { + TestResult.Success -> "has SUCCEEDED" + TestResult.Error -> "has ERROR" + else -> "has FAILED" + } + + return "Example test for ${exampleFile.nameWithoutExtension} $postFix" + } + } +} + +data class HtmlTableColumn ( + val name: String, + val colSpan: Int +) + +data class TableRowGroup ( + val columnName: String, + val value: String, + val rowSpan: Int = 1, + val showRow: Boolean = true, + val rawValue: String = value, + val extraInfo: String? = null +) + +data class TableRow ( + val columns: List, + val exampleFilePath: String? = null, + val exampleFileName: String? = null, + val exampleMismatchReason: String? = null +) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidationBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidationBase.kt new file mode 100644 index 000000000..977824d1d --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidationBase.kt @@ -0,0 +1,174 @@ +package application.exampleGeneration + +import io.specmatic.core.Result +import io.specmatic.core.log.logger +import picocli.CommandLine.Option +import java.io.File +import java.util.concurrent.Callable + +abstract class ExamplesValidationBase(private val common: ExamplesCommon): Callable { + @Option(names = ["--contract-file"], description = ["Contract file path"], required = true) + private lateinit var contractFile: File + + @Option(names = ["--example-file"], description = ["Example file path"], required = false) + private val exampleFile: File? = null + + @Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") + var filterName: String = "" + + @Option(names = ["--filter-not-name"], description = ["Use only APIs which do not have this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}") + var filterNotName: String = "" + + @Option(names = ["--debug"], description = ["Debug logs"]) + private var verbose = false + + abstract var validateExternal: Boolean + abstract var validateInline: Boolean + abstract var extensive: Boolean + + override fun call(): Int { + common.configureLogger(this.verbose) + + if (!contractFile.exists()) { + logger.log("Could not find Contract file ${contractFile.path}") + return 1 + } + + if (contractFile.extension !in common.contractFileExtensions) { + logger.log("Invalid Contract file ${contractFile.path} - File extension must be one of ${common.contractFileExtensions.joinToString()}") + return 1 + } + + try { + exampleFile?.let { exFile -> + if (!exFile.exists()) { + logger.log("Could not find Example file ${exFile.path}") + return 1 + } + + if (exFile.extension !in common.exampleFileExtensions) { + logger.log("Invalid Example file ${exFile.path} - File extension must be one of ${common.exampleFileExtensions.joinToString()}") + return 1 + } + + val validation = validateExampleFile(exFile, contractFile) + return getExitCode(validation) + } + + val inlineResults = if(validateInline) validateInlineExamples(contractFile) else emptyList() + val externalResults = if(validateExternal) validateExternalExamples(contractFile) else emptyList() + + logValidationResult(inlineResults, externalResults) + return getExitCode(inlineResults.plus(externalResults)) + } catch (e: Throwable) { + logger.log("Validation failed with error: ${e.message}") + logger.debug(e) + return 1 + } + } + + // HOOKS + open fun validateInlineExamples(feature: Feature): List> { + return emptyList() + } + + // VALIDATION METHODS + private fun validateExampleFile(exampleFile: File, contractFile: File): ExampleValidationResult { + val feature = common.contractFileToFeature(contractFile) + return validateExternalExample(exampleFile, feature) + } + + private fun validateExternalExample(exampleFile: File, feature: Feature): ExampleValidationResult { + return try { + val result = common.validateExternalExample(feature, exampleFile) + ExampleValidationResult(exampleFile.absolutePath, result.second, ExampleType.EXTERNAL) + } catch (e: Throwable) { + logger.log("Example validation failed with error: ${e.message}") + logger.debug(e) + ExampleValidationResult(exampleFile.absolutePath, Result.Failure(e.message.orEmpty()), ExampleType.EXTERNAL) + } + } + + private fun getFilteredFeature(contractFile: File): Feature { + val scenarioFilter = ScenarioFilter(filterName, filterNotName, extensive) + val feature = common.contractFileToFeature(contractFile) + val filteredScenarios = common.getFilteredScenarios(feature, scenarioFilter) + + return common.updateFeatureForValidation(feature, filteredScenarios) + } + + private fun validateExternalExamples(contractFile: File): List { + val examplesDir = common.getExamplesDirectory(contractFile) + val exampleFiles = common.getExternalExampleFiles(examplesDir) + val feature = getFilteredFeature(contractFile) + + return exampleFiles.mapIndexed { index, it -> + validateExternalExample(it, feature).also { + it.logErrors(index.inc()) + common.logSeparator(75) + } + } + } + + private fun validateInlineExamples(contractFile: File): List { + val feature = getFilteredFeature(contractFile) + return validateInlineExamples(feature).mapIndexed { index, it -> + ExampleValidationResult(it.first, it.second, ExampleType.INLINE).also { + it.logErrors(index.inc()) + common.logSeparator(75) + } + } + } + + // HELPER METHODS + private fun getExitCode(validations: List): Int { + return if (validations.any { !it.result.isSuccess() }) 1 else 0 + } + + private fun getExitCode(validation: ExampleValidationResult): Int { + return if (!validation.result.isSuccess()) 1 else 0 + } + + private fun logValidationResult(inlineResults: List, externalResults: List) { + logResultSummary(inlineResults, ExampleType.INLINE) + logResultSummary(externalResults, ExampleType.EXTERNAL) + } + + private fun logResultSummary(results: List, type: ExampleType) { + if (results.isNotEmpty()) { + val successCount = results.count { it.result.isSuccess() } + val failureCount = results.size - successCount + printSummary(type, successCount, failureCount) + } + } + + private fun printSummary(type: ExampleType, successCount: Int, failureCount: Int) { + common.logFormattedOutput( + header = "$type Examples Validation Summary", + summary = "$successCount example(s) are valid. $failureCount example(s) are invalid", + note = "" + ) + } + + private fun ExampleValidationResult.logErrors(index: Int? = null) { + val prefix = index?.let { "$it. " } ?: "" + + if (this.result.isSuccess()) { + return logger.log("$prefix${this.exampleName} is valid") + } + + logger.log("\n$prefix${this.exampleName} has the following validation error(s):") + logger.log(this.result.reportString()) + } +} + +enum class ExampleType(val value: String) { + INLINE("Inline"), + EXTERNAL("External"); + + override fun toString(): String { + return this.value + } +} + +data class ExampleValidationResult(val exampleName: String, val result: Result, val type: ExampleType) diff --git a/application/src/main/kotlin/application/exampleGeneration/ScenarioFilter.kt b/application/src/main/kotlin/application/exampleGeneration/ScenarioFilter.kt new file mode 100644 index 000000000..aa5fa47b6 --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/ScenarioFilter.kt @@ -0,0 +1,11 @@ +package application.exampleGeneration + +class ScenarioFilter(filterName: String, filterNotName: String, val extensive: Boolean) { + val filterNameTokens = filterToTokens(filterName) + val filterNotNameTokens = filterToTokens(filterNotName) + + private fun filterToTokens(filterValue: String): Set { + if (filterValue.isBlank()) return emptySet() + return filterValue.split(",").map { it.trim() }.toSet() + } +} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamples.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamples.kt new file mode 100644 index 000000000..a3b10eac0 --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamples.kt @@ -0,0 +1,18 @@ +package application.exampleGeneration.openApiExamples + +import application.exampleGeneration.ExamplesBase +import io.specmatic.core.Feature +import io.specmatic.core.Scenario +import picocli.CommandLine.Command +import picocli.CommandLine.Option + +@Command( + name = "examples", + mixinStandardHelpOptions = true, + description = ["Generate JSON Examples with Request and Response from an OpenApi Contract File"], + subcommands = [OpenApiExamplesValidate::class, OpenApiExamplesInteractive::class] +) +class OpenApiExamples: ExamplesBase(OpenApiExamplesCommon()) { + @Option(names = ["--extensive"], description = ["Generate all examples (by default, generates one example per 2xx API)"], defaultValue = "false") + override var extensive: Boolean = false +} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt new file mode 100644 index 000000000..8ab7b0a53 --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt @@ -0,0 +1,146 @@ +package application.exampleGeneration.openApiExamples + +import application.exampleGeneration.ExamplesCommon +import io.specmatic.conversions.ExampleFromFile +import io.specmatic.core.* +import io.specmatic.core.utilities.capitalizeFirstChar +import io.specmatic.core.utilities.uniqueNameForApiOperation +import io.specmatic.mock.NoMatchingScenario +import io.specmatic.mock.ScenarioStub +import io.specmatic.stub.HttpStub +import io.specmatic.stub.HttpStubData +import java.io.File + +class OpenApiExamplesCommon: ExamplesCommon { + override val exampleFileExtensions: Set = setOf("json") + override val contractFileExtensions: Set = OPENAPI_FILE_EXTENSIONS.toSet() + + override fun contractFileToFeature(contractFile: File): Feature { + return parseContractFileToFeature(contractFile) + } + + override fun getScenarioDescription(scenario: Scenario): String { + return scenario.testDescription().split("Scenario: ").last() + } + + override fun getScenariosFromFeature(feature: Feature, extensive: Boolean): List { + if (!extensive) { + return feature.scenarios.filter { it.status in 200..299 } + } + + return feature.scenarios + } + + // GENERATION METHODS + override fun generateExample(feature: Feature, scenario: Scenario, dictionary: Dictionary): Pair { + val request = scenario.generateHttpRequest() + val requestHttpPathPattern = scenario.httpRequestPattern.httpPathPattern + val updatedRequest = request.substituteDictionaryValues(dictionary, forceSubstitution = true, requestHttpPathPattern) + + val response = feature.lookupResponse(scenario).cleanup() + val updatedResponse = response.substituteDictionaryValues(dictionary, forceSubstitution = true) + + val scenarioStub = ScenarioStub(updatedRequest, updatedResponse) + val stubJSON = scenarioStub.toJSON().toStringLiteral() + val uniqueName = uniqueNameForApiOperation(request, "", scenarioStub.response.status) + + return Pair("$uniqueName.json", stubJSON) + } + + override fun getExistingExampleOrNull(scenario: Scenario, exampleFiles: List): Pair? { + val examples = exampleFiles.toExamples() + return examples.firstNotNullOfOrNull { example -> + val response = example.response + + when (val matchResult = scenario.matchesMock(example.request, response)) { + is Result.Success -> example.file to matchResult + is Result.Failure -> { + val isFailureRelatedToScenario = matchResult.getFailureBreadCrumbs("").none { breadCrumb -> + breadCrumb.contains(PATH_BREAD_CRUMB) + || breadCrumb.contains(METHOD_BREAD_CRUMB) + || breadCrumb.contains("REQUEST.HEADERS.Content-Type") + || breadCrumb.contains("STATUS") + } + if (isFailureRelatedToScenario) example.file to matchResult else null + } + } + } + } + + private fun List.toExamples(): List { + return this.map { ExampleFromFile(it) } + } + + private fun HttpResponse.cleanup(): HttpResponse { + return this.copy(headers = this.headers.minus(SPECMATIC_RESULT_HEADER)) + } + + // VALIDATION METHODS + override fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature { + return feature.copy(scenarios = filteredScenarios) + } + + override fun validateExternalExample(feature: Feature, exampleFile: File): Pair { + val examples = mapOf(exampleFile.nameWithoutExtension to listOf(ScenarioStub.readFromFile(exampleFile))) + return feature.validateMultipleExamples(examples).first() + } + + private fun getCleanedUpFailure(failureResults: Results, noMatchingScenario: NoMatchingScenario?): Results { + return failureResults.toResultIfAny().let { + if (it.reportString().isBlank()) + Results(listOf(Result.Failure(noMatchingScenario?.message ?: "", failureReason = FailureReason.ScenarioMismatch))) + else + failureResults + } + } + + private fun Feature.validateMultipleExamples(examples: Map>, inline: Boolean = false): List> { + val results = examples.map { (name, exampleList) -> + val results = exampleList.mapNotNull { example -> + try { + this.validateExample(example) + Result.Success() + } catch (e: NoMatchingScenario) { + if (inline && !e.results.withoutFluff().hasResults()) + null + else + e.results.toResultIfAny() + } + } + name to Result.fromResults(results) + } + + return results + } + + private fun Feature.validateExample(scenarioStub: ScenarioStub) { + val result: Pair>?, NoMatchingScenario?> = HttpStub.setExpectation(scenarioStub, this, InteractiveExamplesMismatchMessages) + val validationResult = result.first + val noMatchingScenario = result.second + + if (validationResult == null) { + val failures = noMatchingScenario?.results?.withoutFluff()?.results ?: emptyList() + + val failureResults = getCleanedUpFailure(Results(failures).withoutFluff(), noMatchingScenario) + throw NoMatchingScenario( + failureResults, + cachedMessage = failureResults.report(scenarioStub.request), + msg = failureResults.report(scenarioStub.request) + ) + } + } + + object InteractiveExamplesMismatchMessages : MismatchMessages { + override fun mismatchMessage(expected: String, actual: String): String { + return "Specification expected $expected but example contained $actual" + } + + override fun unexpectedKey(keyLabel: String, keyName: String): String { + return "${keyLabel.capitalizeFirstChar()} $keyName in the example is not in the specification" + } + + override fun expectedKeyWasMissing(keyLabel: String, keyName: String): String { + return "${keyLabel.capitalizeFirstChar()} $keyName in the specification is missing from the example" + } + } +} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt new file mode 100644 index 000000000..188eacc00 --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -0,0 +1,104 @@ +package application.exampleGeneration.openApiExamples + +import application.exampleGeneration.ExamplesInteractiveBase +import application.exampleGeneration.HtmlTableColumn +import application.exampleGeneration.TableRow +import application.exampleGeneration.TableRowGroup +import io.ktor.server.application.* +import io.ktor.server.request.* +import io.specmatic.conversions.convertPathParameterStyle +import io.specmatic.core.DEFAULT_TIMEOUT_IN_MILLISECONDS +import io.specmatic.core.Feature +import io.specmatic.core.Scenario +import io.specmatic.core.TestResult +import io.specmatic.test.TestInteractionsLog +import io.specmatic.test.TestInteractionsLog.combineLog +import picocli.CommandLine.Command +import picocli.CommandLine.Option +import java.io.File + +@Command( + name = "interactive", + mixinStandardHelpOptions = true, + description = ["Generate and validate examples interactively through a Web UI"], +) +class OpenApiExamplesInteractive : ExamplesInteractiveBase(OpenApiExamplesCommon()) { + @Option(names = ["--extensive"], description = ["Display all responses, not just 2xx, in the table."], defaultValue = "false") + override var extensive: Boolean = false + + override val htmlTableColumns: List = listOf( + HtmlTableColumn(name = "path", colSpan = 2), + HtmlTableColumn(name = "method", colSpan = 1), + HtmlTableColumn(name = "response", colSpan = 1) + ) + + override fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair { + val contractTests = feature.loadExternalisedExamples().generateContractTests(emptyList()) + + val test = contractTests.firstOrNull { + it.testDescription().contains(exampleFile.nameWithoutExtension) + } ?: return Pair(TestResult.Error, "Test not found for example ${exampleFile.nameWithoutExtension}") + + val testResultRecord = test.runTest(testBaseUrl, timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS).let { + test.testResultRecord(it.first, it.second) + } ?: return Pair(TestResult.Error, "TestResult record not found for example ${exampleFile.nameWithoutExtension}") + + return testResultRecord.scenarioResult?.let { scenarioResult -> + Pair(testResultRecord.result, TestInteractionsLog.testHttpLogMessages.last { it.scenario == scenarioResult.scenario }.combineLog()) + } ?: Pair(TestResult.Error, "Interaction logs not found for example ${exampleFile.nameWithoutExtension}") + } + + override suspend fun getScenarioFromRequestOrNull(call: ApplicationCall, feature: Feature): Scenario? { + val request = call.receive() + return feature.scenarios.firstOrNull { + it.method == request.method && it.status == request.response && it.path == request.path + && (request.contentType == null || it.httpRequestPattern.headersPattern.contentType == request.contentType) + } + } + + override fun createTableRows(scenarios: List, exampleFiles: List): List { + val groupedScenarios = scenarios.sortScenarios().groupScenarios() + + return groupedScenarios.flatMap { (_, methodMap) -> + val pathSpan = methodMap.values.sumOf { it.size } + val methodSet: MutableSet = mutableSetOf() + var showPath = true + + methodMap.flatMap { (method, scenarios) -> + scenarios.map { + val existingExample = common.getExistingExampleOrNull(it, exampleFiles) + + TableRow( + columns = listOf( + TableRowGroup("path", convertPathParameterStyle(it.path), rawValue = it.path, rowSpan = pathSpan, showRow = showPath), + TableRowGroup("method", it.method, showRow = !methodSet.contains(method), rowSpan = scenarios.size), + TableRowGroup("response", it.status.toString(), showRow = true, rowSpan = 1, extraInfo = it.httpRequestPattern.headersPattern.contentType) + ), + exampleFilePath = existingExample?.first?.absolutePath, + exampleFileName = existingExample?.first?.nameWithoutExtension, + exampleMismatchReason = existingExample?.second?.reportString().takeIf { reason -> reason?.isNotBlank() == true } + ).also { methodSet.add(method); showPath = false } + } + } + } + } + + private fun List.groupScenarios(): Map>> { + return this.groupBy { it.path }.mapValues { pathGroup -> + pathGroup.value.groupBy { it.method } + } + } + + private fun List.sortScenarios(): List { + return this.sortedBy { + "${it.path}_${it.method}_${it.status}" + } + } + + data class ExampleGenerationRequest ( + val method: String, + val path: String, + val response: Int, + val contentType: String? = null + ) +} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt new file mode 100644 index 000000000..223213cd1 --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt @@ -0,0 +1,18 @@ +package application.exampleGeneration.openApiExamples + +import application.exampleGeneration.ExamplesValidationBase +import io.specmatic.core.Feature +import io.specmatic.core.Scenario +import picocli.CommandLine.Command +import picocli.CommandLine.Option + +@Command(name = "validate", description = ["Validate OpenAPI inline and external examples"]) +class OpenApiExamplesValidate: ExamplesValidationBase(OpenApiExamplesCommon()) { + @Option(names = ["--validate-external"], description = ["Validate external examples, defaults to true"]) + override var validateExternal: Boolean = true + + @Option(names = ["--validate-inline"], description = ["Validate inline examples, defaults to false"]) + override var validateInline: Boolean = false + + override var extensive: Boolean = true +} \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServer.kt b/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServer.kt deleted file mode 100644 index fb8ecb444..000000000 --- a/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServer.kt +++ /dev/null @@ -1,648 +0,0 @@ -package io.specmatic.core.examples.server - -import io.ktor.http.* -import io.ktor.serialization.jackson.* -import io.ktor.server.application.* -import io.ktor.server.engine.* -import io.ktor.server.netty.* -import io.ktor.server.plugins.contentnegotiation.* -import io.ktor.server.plugins.cors.routing.* -import io.ktor.server.request.* -import io.ktor.server.response.* -import io.ktor.server.routing.* -import io.specmatic.conversions.ExampleFromFile -import io.specmatic.conversions.OpenApiSpecification -import io.specmatic.core.* -import io.specmatic.core.examples.server.ExamplesView.Companion.groupEndpoints -import io.specmatic.core.examples.server.ExamplesView.Companion.toTableRows -import io.specmatic.core.log.logger -import io.specmatic.core.pattern.ContractException -import io.specmatic.core.route.modules.HealthCheckModule.Companion.configureHealthCheckModule -import io.specmatic.core.utilities.* -import io.specmatic.mock.NoMatchingScenario -import io.specmatic.mock.ScenarioStub -import io.specmatic.stub.HttpStub -import io.specmatic.stub.HttpStubData -import io.specmatic.test.ContractTest -import io.specmatic.test.TestInteractionsLog -import io.specmatic.test.TestInteractionsLog.combineLog -import java.io.Closeable -import java.io.File -import java.io.FileNotFoundException -import kotlin.system.exitProcess - -class ExamplesInteractiveServer( - private val serverHost: String, - private val serverPort: Int, - private val testBaseUrl: String?, - private val inputContractFile: File? = null, - private val filterName: String, - private val filterNotName: String, - private val externalDictionaryFile: File? = null -) : Closeable { - private var contractFileFromRequest: File? = null - - init { - if(externalDictionaryFile != null) System.setProperty(SPECMATIC_STUB_DICTIONARY, externalDictionaryFile.path) - } - - private fun getContractFile(): File { - if(inputContractFile != null && inputContractFile.exists()) return inputContractFile - if(contractFileFromRequest != null && contractFileFromRequest!!.exists()) return contractFileFromRequest!! - throw ContractException("Invalid contract file provided to the examples interactive server") - } - - private fun getServerHostPort(request: ExamplePageRequest? = null) : String { - return request?.hostPort ?: "localhost:$serverPort" - } - - private val environment = applicationEngineEnvironment { - module { - install(CORS) { - allowMethod(HttpMethod.Options) - allowMethod(HttpMethod.Post) - allowMethod(HttpMethod.Get) - allowHeader(HttpHeaders.AccessControlAllowOrigin) - allowHeader(HttpHeaders.ContentType) - anyHost() - } - - install(ContentNegotiation) { - jackson {} - } - - configureHealthCheckModule() - routing { - get("/_specmatic/examples") { - val contractFile = getContractFileOrBadRequest(call) ?: return@get - try { - respondWithExamplePageHtmlContent(contractFile, getServerHostPort(), call) - } catch (e: Exception) { - call.respond(HttpStatusCode.InternalServerError, "An unexpected error occurred: ${e.message}") - } - } - - post("/_specmatic/examples") { - val request = call.receive() - contractFileFromRequest = File(request.contractFile) - val contractFile = getContractFileOrBadRequest(call) ?: return@post - try { - respondWithExamplePageHtmlContent(contractFile,getServerHostPort(request), call) - } catch (e: Exception) { - call.respond(HttpStatusCode.InternalServerError, "An unexpected error occurred: ${e.message}") - } - } - - post("/_specmatic/examples/generate") { - val contractFile = getContractFile() - - try { - val request = call.receive() - val generatedExample = generate( - contractFile, - request.method, - request.path, - request.responseStatusCode, - request.contentType, - ) - - call.respond(HttpStatusCode.OK, GenerateExampleResponse(generatedExample)) - } catch(e: Exception) { - call.respond(HttpStatusCode.InternalServerError, "An unexpected error occurred: ${e.message}") - } - } - - post("/_specmatic/examples/validate") { - val request = call.receive() - try { - val contractFile = getContractFile() - val validationResultResponse = try { - val result = validateSingleExample(contractFile, File(request.exampleFile)) - if(result.isSuccess()) - ValidateExampleResponse(request.exampleFile) - else - ValidateExampleResponse(request.exampleFile, result.reportString()) - } catch (e: FileNotFoundException) { - ValidateExampleResponse(request.exampleFile, e.message ?: "File not found") - } catch (e: ContractException) { - ValidateExampleResponse(request.exampleFile, exceptionCauseMessage(e)) - } catch (e: Exception) { - ValidateExampleResponse(request.exampleFile, e.message ?: "An unexpected error occurred") - } - call.respond(HttpStatusCode.OK, validationResultResponse) - } catch(e: Exception) { - call.respond(HttpStatusCode.InternalServerError, mapOf("error" to "An unexpected error occurred: ${e.message}")) - } - } - - post("/_specmatic/v2/examples/validate") { - val request = call.receive>() - try { - val contractFile = getContractFile() - - val examples = request.map { - val exampleFilePath = it.exampleFile - exampleFilePath to listOf(ScenarioStub.readFromFile(File(exampleFilePath))) - }.toMap() - - val results = validateMultipleExamples(contractFile, examples = examples) - - val validationResults = results.map { (exampleFilePath, result) -> - try { - result.throwOnFailure() - ValidateExampleResponseV2( - ValidateExampleVerdict.SUCCESS, - "The provided example is valid", - exampleFilePath - ) - } catch (e: ContractException) { - ValidateExampleResponseV2( - ValidateExampleVerdict.FAILURE, - exceptionCauseMessage(e), - exampleFilePath - ) - } catch (e: Exception) { - ValidateExampleResponseV2( - ValidateExampleVerdict.FAILURE, - e.message ?: "An unexpected error occurred", - exampleFilePath - ) - } - } - - call.respond(HttpStatusCode.OK, validationResults) - } catch(e: Exception) { - call.respond(HttpStatusCode.InternalServerError, mapOf("error" to "An unexpected error occurred: ${e.message}")) - } - } - - get("/_specmatic/examples/content") { - val fileName = call.request.queryParameters["fileName"] - if(fileName == null) { - call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Invalid request. Missing required query param named 'fileName'")) - return@get - } - val file = File(fileName) - if(file.exists().not() || file.extension != "json") { - val message = if(file.extension == "json") "The provided example file ${file.name} does not exist" - else "The provided example file ${file.name} is not a valid example file" - call.respond(HttpStatusCode.BadRequest, mapOf("error" to message)) - return@get - } - call.respond( - HttpStatusCode.OK, - mapOf("content" to File(fileName).readText()) - ) - } - - post ("/_specmatic/examples/test") { - if (testBaseUrl.isNullOrEmpty()) { - return@post call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Invalid request, No Test Base URL provided via command-line")) - } - - val request = call.receive() - try { - val feature = OpenApiSpecification.fromFile(getContractFile().path).toFeature() - - val contractTest = feature.createContractTestFromExampleFile(request.exampleFile).value - - val (result, testLog) = testExample(contractTest, testBaseUrl) - - call.respond(HttpStatusCode.OK, ExampleTestResponse(result, testLog, exampleFile = File(request.exampleFile))) - } catch (e: Throwable) { - call.respond(HttpStatusCode.InternalServerError, mapOf("error" to "An unexpected error occurred: ${e.message}")) - } - } - } - } - connector { - this.host = serverHost - this.port = serverPort - } - } - - private val server: ApplicationEngine = embeddedServer(Netty, environment, configure = { - this.requestQueueLimit = 1000 - this.callGroupSize = 5 - this.connectionGroupSize = 20 - this.workerGroupSize = 20 - }) - - init { - server.start() - } - - override fun close() { - server.stop(0, 0) - } - - private suspend fun getContractFileOrBadRequest(call: ApplicationCall): File? { - return try { - getContractFile() - } catch(e: ContractException) { - call.respond(HttpStatusCode.BadRequest, mapOf("error" to e.message)) - return null - } - } - - private suspend fun respondWithExamplePageHtmlContent(contractFile: File, hostPort: String, call: ApplicationCall) { - try { - val html = getExamplePageHtmlContent(contractFile, hostPort) - call.respondText(html, contentType = ContentType.Text.Html) - } catch (e: Exception) { - println(e) - call.respond(HttpStatusCode.InternalServerError, "An unexpected error occurred: ${e.message}") - } - } - - private fun getExamplePageHtmlContent(contractFile: File, hostPort: String): String { - val feature = ScenarioFilter(filterName, filterNotName).filter(parseContractFileToFeature(contractFile)) - - val endpoints = ExamplesView.getEndpoints(feature, getExamplesDirPath(contractFile)) - val tableRows = endpoints.groupEndpoints().toTableRows() - - return HtmlTemplateConfiguration.process( - templateName = "examples/index.html", - variables = mapOf( - "tableRows" to tableRows, - "contractFile" to contractFile.name, - "contractFilePath" to contractFile.absolutePath, - "hostPort" to hostPort, - "hasExamples" to tableRows.any {it.example != null}, - "validationDetails" to tableRows.withIndex().associate { (idx, row) -> - idx.inc() to row.exampleMismatchReason - }, - "isTestMode" to (testBaseUrl != null) - ) - ) - } - - class ScenarioFilter(filterName: String = "", filterNotName: String = "") { - private val filterNameTokens = if(filterName.isNotBlank()) { - filterName.trim().split(",").map { it.trim() } - } else emptyList() - - private val filterNotNameTokens = if(filterNotName.isNotBlank()) { - filterNotName.trim().split(",").map { it.trim() } - } else emptyList() - - fun filter(feature: Feature): Feature { - val scenarios = feature.scenarios.filter { scenario -> - if(filterNameTokens.isNotEmpty()) { - filterNameTokens.any { name -> scenario.testDescription().contains(name) } - } else true - }.filter { scenario -> - if(filterNotNameTokens.isNotEmpty()) { - filterNotNameTokens.none { name -> scenario.testDescription().contains(name) } - } else true - } - - return feature.copy(scenarios = scenarios) - } - } - - companion object { - enum class ExampleGenerationStatus { - CREATED, EXISTED, ERROR - } - - class ExampleGenerationResult private constructor (val path: String?, val status: ExampleGenerationStatus) { - constructor(path: String, created: Boolean) : this(path, if(created) ExampleGenerationStatus.CREATED else ExampleGenerationStatus.EXISTED) - constructor(): this(null, ExampleGenerationStatus.ERROR) - } - - fun testExample(test: ContractTest, testBaseUrl: String): Pair { - val testResultRecord = test.runTest(testBaseUrl, timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS).let { - test.testResultRecord(it.first, it.second) - } ?: return Pair(TestResult.Error, "No Test Result Record Found") - - return testResultRecord.scenarioResult?.let { scenarioResult -> - Pair(testResultRecord.result, TestInteractionsLog.testHttpLogMessages.last { it.scenario == scenarioResult.scenario }.combineLog()) - } ?: Pair(TestResult.Error, "No Log Message Found") - } - - fun generate(contractFile: File, scenarioFilter: ScenarioFilter, extensive: Boolean): List { - try { - val feature: Feature = parseContractFileToFeature(contractFile).let { feature -> - val filteredScenarios = if (!extensive) { - feature.scenarios.filter { - it.status.toString().startsWith("2") - } - } else { - feature.scenarios - } - - scenarioFilter.filter(feature.copy(scenarios = filteredScenarios.map { - it.copy(examples = emptyList()) - })).copy(stubsFromExamples = emptyMap()) - } - - val examplesDir = - getExamplesDirPath(contractFile) - - examplesDir.mkdirs() - - if (feature.scenarios.isEmpty()) { - logger.log("All examples were filtered out by the filter expression") - return emptyList() - } - - return feature.scenarios.map { scenario -> - try { - val generatedExampleFilePath = generateExampleFile(contractFile, feature, scenario) - - generatedExampleFilePath.also { - val loggablePath = - File(it.path).canonicalFile.relativeTo(contractFile.canonicalFile.parentFile).path - - val trimmedScenarioDescription = scenario.testDescription().trim() - - if (!it.created) { - println("Example exists for $trimmedScenarioDescription: $loggablePath") - } else { - println("Created example for $trimmedScenarioDescription: $loggablePath") - } - } - - ExampleGenerationResult(generatedExampleFilePath.path, generatedExampleFilePath.created) - } catch(e: Throwable) { - logger.log(e, "Exception generating example for ${scenario.testDescription()}") - ExampleGenerationResult() - } - }.also { exampleFiles -> - val resultCounts = exampleFiles.groupBy { it.status }.mapValues { it.value.size } - val createdFileCount = resultCounts[ExampleGenerationStatus.CREATED] ?: 0 - val errorCount = resultCounts[ExampleGenerationStatus.ERROR] ?: 0 - val existingFileCount = resultCounts[ExampleGenerationStatus.EXISTED] ?: 0 - - logger.log(System.lineSeparator() + "NOTE: All examples may be found in ${getExamplesDirPath(contractFile).canonicalFile}" + System.lineSeparator()) - - val errorsClause = if(errorCount > 0) ", $errorCount examples could not be generated due to errors" else "" - - logger.log("=============== Example Generation Summary ===============") - logger.log("$createdFileCount example(s) created, $existingFileCount examples already existed$errorsClause") - logger.log("==========================================================") - }.mapNotNull { it.path } - } catch (e: StackOverflowError) { - logger.log("Got a stack overflow error. You probably have a recursive data structure definition in the contract.") - throw e - } - } - - fun generate( - contractFile: File, - method: String, - path: String, - responseStatusCode: Int, - contentType: String? = null, - ): String? { - val feature = parseContractFileToFeature(contractFile) - val scenario: Scenario? = feature.scenarios.firstOrNull { - it.method == method && it.status == responseStatusCode && it.path == path - && (contentType == null || it.httpRequestPattern.headersPattern.contentType == contentType) - } - if(scenario == null) return null - - return generateExampleFile(contractFile, feature, scenario).also { - println("Writing to file: ${File(it.path).canonicalFile.relativeTo(contractFile.canonicalFile.parentFile).path}") - }.path - } - - data class ExamplePathInfo(val path: String, val created: Boolean) - - private fun generateExampleFile( - contractFile: File, - feature: Feature, - scenario: Scenario, - ): ExamplePathInfo { - val examplesDir = getExamplesDirPath(contractFile) - val existingExampleFile = getExistingExampleFile(scenario, examplesDir.getExamplesFromDir()) - if (existingExampleFile != null) return ExamplePathInfo(existingExampleFile.first.absolutePath, false) - else examplesDir.mkdirs() - - val request = scenario.generateHttpRequest() - - val response = feature.lookupResponse(scenario).cleanup() - - val scenarioStub = ScenarioStub(request, response) - val stubJSON = scenarioStub.toJSON() - val uniqueNameForApiOperation = - uniqueNameForApiOperation(scenarioStub.request, "", scenarioStub.response.status) - - val file = examplesDir.resolve("${uniqueNameForApiOperation}.json") - println("Writing to file: ${file.relativeTo(contractFile.canonicalFile.parentFile).path}") - file.writeText(stubJSON.toStringLiteral()) - return ExamplePathInfo(file.absolutePath, true) - } - - fun validateSingleExample(contractFile: File, exampleFile: File): Result { - val feature = parseContractFileToFeature(contractFile) - return validateSingleExample(feature, exampleFile) - } - - fun validateSingleExample(feature: Feature, exampleFile: File): Result { - val scenarioStub = ScenarioStub.readFromFile(exampleFile) - - return try { - validateExample(feature, scenarioStub) - Result.Success() - } catch(e: NoMatchingScenario) { - e.results.toResultIfAny() - } - } - - fun validateMultipleExamples(contractFile: File, examples: Map> = emptyMap(), scenarioFilter: ScenarioFilter = ScenarioFilter()): Map { - val feature = parseContractFileToFeature(contractFile) - return validateMultipleExamples(feature, examples, false, scenarioFilter) - } - - fun validateMultipleExamples(feature: Feature, examples: Map> = emptyMap(), inline: Boolean = false, scenarioFilter: ScenarioFilter = ScenarioFilter()): Map { - val updatedFeature = scenarioFilter.filter(feature) - - val results = examples.mapValues { (name, exampleList) -> - logger.log("Validating ${name}") - - exampleList.map { example -> - try { - validateExample(updatedFeature, example) - Result.Success() - } catch(e: NoMatchingScenario) { - if(inline && e.results.withoutFluff().hasResults() == false) - null - else - e.results.toResultIfAny() - } - }.filterNotNull().let { - Result.fromResults(it) - } - } - - return results - } - - private fun getCleanedUpFailure( - failureResults: Results, - noMatchingScenario: NoMatchingScenario? - ): Results { - return failureResults.toResultIfAny().let { - if (it.reportString().isBlank()) - Results(listOf(Result.Failure(noMatchingScenario?.message ?: "", failureReason = FailureReason.ScenarioMismatch))) - else - failureResults - } - } - - private fun validateExample( - feature: Feature, - scenarioStub: ScenarioStub - ) { - val result: Pair>?, NoMatchingScenario?> = - HttpStub.setExpectation(scenarioStub, feature, InteractiveExamplesMismatchMessages) - val validationResult = result.first - val noMatchingScenario = result.second - - if (validationResult == null) { - val failures = noMatchingScenario?.results?.withoutFluff()?.results ?: emptyList() - - val failureResults = Results(failures).withoutFluff().let { - getCleanedUpFailure(it, noMatchingScenario) - } - throw NoMatchingScenario( - failureResults, - cachedMessage = failureResults.report(scenarioStub.request), - msg = failureResults.report(scenarioStub.request) - ) - } - } - - private fun HttpResponse.cleanup(): HttpResponse { - return this.copy(headers = this.headers.minus(SPECMATIC_RESULT_HEADER)) - } - - fun getExistingExampleFile(scenario: Scenario, examples: List): Pair? { - return examples.firstNotNullOfOrNull { example -> - val response = example.response - - when (val matchResult = scenario.matchesMock(example.request, response)) { - is Result.Success -> example.file to "" - is Result.Failure -> { - val isFailureRelatedToScenario = matchResult.getFailureBreadCrumbs("").none { breadCrumb -> - breadCrumb.contains(PATH_BREAD_CRUMB) - || breadCrumb.contains(METHOD_BREAD_CRUMB) - || breadCrumb.contains("REQUEST.HEADERS.Content-Type") - || breadCrumb.contains("STATUS") - } - if (isFailureRelatedToScenario) example.file to matchResult.reportString() else null - } - } - } - } - - private fun getExamplesDirPath(contractFile: File): File { - return contractFile.canonicalFile - .parentFile - .resolve("""${contractFile.nameWithoutExtension}$EXAMPLES_DIR_SUFFIX""") - } - - fun File.getExamplesFromDir(): List { - return this.listFiles()?.map { ExampleFromFile(it) } ?: emptyList() - } - - } -} - -object InteractiveExamplesMismatchMessages : MismatchMessages { - override fun mismatchMessage(expected: String, actual: String): String { - return "Specification expected $expected but example contained $actual" - } - - override fun unexpectedKey(keyLabel: String, keyName: String): String { - return "${keyLabel.capitalizeFirstChar()} $keyName in the example is not in the specification" - } - - override fun expectedKeyWasMissing(keyLabel: String, keyName: String): String { - return "${keyLabel.capitalizeFirstChar()} $keyName in the specification is missing from the example" - } -} - -data class ExamplePageRequest( - val contractFile: String, - val hostPort: String -) - -data class ValidateExampleRequest( - val exampleFile: String -) - -data class ValidateExampleResponse( - val absPath: String, - val error: String? = null -) - -enum class ValidateExampleVerdict { - SUCCESS, - FAILURE -} - -data class ValidateExampleResponseV2( - val verdict: ValidateExampleVerdict, - val message: String, - val exampleFilePath: String -) - -data class GenerateExampleRequest( - val method: String, - val path: String, - val responseStatusCode: Int, - val contentType: String? = null -) - -data class GenerateExampleResponse( - val generatedExample: String? -) - -data class ExampleTestRequest( - val exampleFile: String -) - -data class ExampleTestResponse( - val result: TestResult, - val details: String, - val testLog: String -) { - constructor(result: TestResult, testLog: String, exampleFile: File): this ( - result = result, - details = resultToDetails(result, exampleFile), - testLog = testLog.trim('-', ' ', '\n', '\r') - ) - - companion object { - fun resultToDetails(result: TestResult, exampleFile: File): String { - val postFix = when(result) { - TestResult.Success -> "has SUCCEEDED" - TestResult.Error -> "has ERROR" - else -> "has FAILED" - } - - return "Example test for ${exampleFile.nameWithoutExtension} $postFix" - } - } -} - -fun loadExternalExamples(contractFile: File): Pair>> { - val examplesDir = - contractFile.absoluteFile.parentFile.resolve(contractFile.nameWithoutExtension + "_examples") - if (!examplesDir.isDirectory) { - logger.log("$examplesDir does not exist, did not find any files to validate") - exitProcess(1) - } - - return examplesDir to examplesDir.walk().mapNotNull { - if (it.isFile) - Pair(it.path, it) - else - null - }.toMap().mapValues { - listOf(ScenarioStub.readFromFile(it.value)) - } -} diff --git a/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesView.kt b/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesView.kt deleted file mode 100644 index 98380b770..000000000 --- a/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesView.kt +++ /dev/null @@ -1,122 +0,0 @@ -package io.specmatic.core.examples.server - -import io.specmatic.conversions.convertPathParameterStyle -import io.specmatic.core.Feature -import io.specmatic.core.examples.server.ExamplesInteractiveServer.Companion.getExamplesFromDir -import io.specmatic.core.examples.server.ExamplesInteractiveServer.Companion.getExistingExampleFile -import org.thymeleaf.TemplateEngine -import org.thymeleaf.context.Context -import org.thymeleaf.templatemode.TemplateMode -import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver -import java.io.File - -class ExamplesView { - companion object { - fun getEndpoints(feature: Feature, examplesDir: File): List { - val examples = examplesDir.getExamplesFromDir() - return feature.scenarios.map { - val example = getExistingExampleFile(it, examples) - Endpoint( - path = convertPathParameterStyle(it.path), - rawPath = it.path, - method = it.method, - responseStatus = it.httpResponsePattern.status, - contentType = it.httpRequestPattern.headersPattern.contentType, - exampleFile = example?.first, - exampleMismatchReason = example?.second - ) - }.filterEndpoints().sortEndpoints() - } - - private fun List.sortEndpoints(): List { - return this.sortedWith(compareBy({ it.path }, { it.method }, { it.responseStatus })) - } - - private fun List.filterEndpoints(): List { - return this.filter { it.responseStatus in 200..299 } - } - - fun List.groupEndpoints(): Map>> { - return this.groupBy { it.path }.mapValues { pathGroup -> - pathGroup.value.groupBy { it.method } - } - } - - fun Map>>.toTableRows(): List { - return this.flatMap { (_, methodGroup) -> - val pathSpan = methodGroup.values.sumOf { it.size } - val methodSet: MutableSet = mutableSetOf() - var showPath = true - - methodGroup.flatMap { (method, endpoints) -> - endpoints.map { - TableRow( - rawPath = it.rawPath, - path = it.path, - method = it.method, - responseStatus = it.responseStatus.toString(), - pathSpan = pathSpan, - methodSpan = endpoints.size, - showPath = showPath, - showMethod = !methodSet.contains(method), - contentType = it.contentType ?: "", - example = it.exampleFile?.absolutePath, - exampleName = it.exampleFile?.nameWithoutExtension, - exampleMismatchReason = if(it.exampleMismatchReason?.isBlank() == true) null else it.exampleMismatchReason - ).also { methodSet.add(method); showPath = false } - } - } - } - } - } -} - -data class TableRow( - val rawPath: String, - val path: String, - val method: String, - val responseStatus: String, - val contentType: String, - val pathSpan: Int, - val methodSpan: Int, - val showPath: Boolean, - val showMethod: Boolean, - val example: String? = null, - val exampleName: String? = null, - val exampleMismatchReason: String? = null -) - -data class Endpoint( - val path: String, - val rawPath: String, - val method: String, - val responseStatus: Int, - val contentType: String? = null, - val exampleFile: File? = null, - val exampleMismatchReason: String? = null -) - -class HtmlTemplateConfiguration { - companion object { - private fun configureTemplateEngine(): TemplateEngine { - val templateResolver = ClassLoaderTemplateResolver().apply { - prefix = "templates/" - suffix = ".html" - templateMode = TemplateMode.HTML - characterEncoding = "UTF-8" - } - - return TemplateEngine().apply { - setTemplateResolver(templateResolver) - } - } - - fun process(templateName: String, variables: Map): String { - val templateEngine = configureTemplateEngine() - return templateEngine.process(templateName, Context().apply { - setVariables(variables) - }) - } - } -} - diff --git a/core/src/main/kotlin/io/specmatic/test/TestInteractionsLog.kt b/core/src/main/kotlin/io/specmatic/test/TestInteractionsLog.kt index 0d38d05a8..f431dfec7 100644 --- a/core/src/main/kotlin/io/specmatic/test/TestInteractionsLog.kt +++ b/core/src/main/kotlin/io/specmatic/test/TestInteractionsLog.kt @@ -17,10 +17,10 @@ object TestInteractionsLog { } fun HttpLogMessage.combineLog(): String { - val request = this.request.toLogString() - val response = this.response?.toLogString() ?: "No response" + val request = this.request.toLogString().trim('\n') + val response = this.response?.toLogString()?.trim('\n') ?: "No response" - return "$request\n$response" + return "$request\n\n$response" } fun HttpLogMessage.duration() = (responseTime?.toEpochMillis() ?: requestTime.toEpochMillis()) - requestTime.toEpochMillis() diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlReport.kt b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlReport.kt index afe8a39e2..2a049f28e 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlReport.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlReport.kt @@ -71,7 +71,7 @@ class HtmlReport(private val htmlReportInformation: HtmlReportInformation) { ) return configureTemplateEngine().process( - "report", + "report/index.html", Context().apply { setVariables(templateVariables) } ) } diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlTemplateConfiguration.kt b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlTemplateConfiguration.kt index cfc35917c..55a2e499d 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlTemplateConfiguration.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlTemplateConfiguration.kt @@ -1,5 +1,6 @@ package io.specmatic.test.reports.coverage.html import org.thymeleaf.TemplateEngine +import org.thymeleaf.context.Context import org.thymeleaf.templatemode.TemplateMode import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver @@ -17,6 +18,13 @@ class HtmlTemplateConfiguration { setTemplateResolver(templateResolver) } } + + fun process(templateName: String, variables: Map): String { + val templateEngine = configureTemplateEngine() + return templateEngine.process(templateName, Context().apply { + setVariables(variables) + }) + } } } diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlUtils.kt b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlUtils.kt index 57ef3dce2..2ed267848 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlUtils.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlUtils.kt @@ -44,7 +44,7 @@ fun createAssetsDir(reportsDir: String) { fileNames.forEach { fileName -> loadFileFromClasspathAndSaveIt( - "templates/assets/$fileName", + "templates/report/assets/$fileName", reportsDir, "assets/$fileName" ) diff --git a/core/src/main/resources/templates/examples/index.html b/junit5-support/src/main/resources/templates/example/index.html similarity index 88% rename from core/src/main/resources/templates/examples/index.html rename to junit5-support/src/main/resources/templates/example/index.html index 0d39185b0..83e2078ff 100644 --- a/core/src/main/resources/templates/examples/index.html +++ b/junit5-support/src/main/resources/templates/example/index.html @@ -460,14 +460,10 @@ } } - & .btn-grp { + .btn-grp { display: flex; align-items: center; gap: 1rem; - - &[data-test-mode="false"] > #bulk-test { - display: none; - } } & button { @@ -573,6 +569,7 @@ th { white-space: nowrap; + text-transform: capitalize; } td, @@ -581,9 +578,7 @@ border: 1px solid rgba(var(--smoky-black), 0.2); padding: var(--_td-padding); - } - .response-cell { & > span { font-size: 0.75rem; } @@ -632,7 +627,6 @@ & > td:last-child { & > span { - font-size: .75rem; line-height: 1.5rem; text-decoration: underline; } @@ -651,9 +645,21 @@ & button.validate { display: inline-block; } + } - & > span { - display: block; + & > span { + display: block; + } + } + + &[data-valid="success"] { + & td:last-child { + & button.validate { + display: none; + } + + & button.test { + display: inline-block; } } } @@ -687,21 +693,6 @@ } } - - tbody { - &[data-test-mode="true"] > tr[data-valid="success"] { - & td:last-child { - & button.validate { - display: none; - } - - & button.test { - display: inline-block; - } - } - } - } - td:nth-child(3) { text-align: initial; min-width: 10rem; @@ -994,18 +985,18 @@
-

+

-
+
-
- +
+
- - - + - + + th:attr="data-examples=${row.exampleFilePath}, + data-generate=${row.exampleFilePath != null ? 'success' : ''}, + data-valid=${row.exampleFilePath != null ? (row.exampleMismatchReason == null ? 'success' : 'failed') : ''}"> - - - - + } } - blockGenValidate = false; bulkGenerateBtn.removeAttribute("data-generate"); const message = `${createdCount} out of ${rowsWithNoExamples.length} Example(s) Created\n${existedCount} out of ${rowsWithNoExamples.length} Example(s) Already Existed`; createAlert("Generation Complete", message, failedCount !== 0); @@ -1274,7 +1272,6 @@

}); async function validateAllSelected() { - blockGenValidate = true; bulkValidateBtn.setAttribute("data-valid", "processing"); let errorsCount = 0; @@ -1292,12 +1289,43 @@

} } - blockGenValidate = false; bulkValidateBtn.removeAttribute("data-generate"); createAlert("Validations Complete", `${errorsCount} out of ${rowsWithExamples.length} are invalid`, errorsCount !== 0); return cleanUpSelections(); } + bulkTestBtn.addEventListener("click", async () => { + blockGenValidate = true; + bulkTestBtn.setAttribute("data-test", "processing"); + + switch (bulkValidateBtn.getAttribute("data-panel")) { + case "table": { + await testAllSelected(); + break; + } + + case "details": { + await testRowExample(selectedTableRow); + const exampleFilePath = getExampleData(selectedTableRow); + const {example: exampleContent, error} = await getExampleContent(exampleFilePath); + + const docFragment = createExampleRows([{ + exampleFilePath: exampleFilePath, + exampleJson: exampleContent, + error: validationDetails[Number.parseInt(selectedTableRow.children[1].textContent)] || error, + hasBeenValidated: selectedTableRow.hasAttribute("data-valid"), + test: testDetails[Number.parseInt(selectedTableRow.children[1].textContent)] + }]); + + examplesOl.replaceChildren(docFragment); + break; + } + } + + bulkTestBtn.removeAttribute("data-test"); + return cleanUpSelections(); + }); + async function testAllSelected() { const selectedRows = Array.from(table.querySelectorAll("td > input[type=checkbox]:checked")).map((checkbox) => checkbox.closest("tr")); const rowsWithValidations = selectedRows.filter(row => row.getAttribute("data-valid") === "success"); @@ -1396,14 +1424,15 @@

if (error) { if (!bulkMode) createAlert("Invalid Example", `Example name: ${parseFileName(exampleFilePath)}`, true); + tableRow.removeAttribute("data-test"); return false; } if (!bulkMode) createAlert("Valid Example", `Example name: ${parseFileName(exampleFilePath)}`, false); + tableRow.removeAttribute("data-test"); return true; } - async function testRowExample(tableRow, bulkMode = false) { tableRow.setAttribute("data-test", "processing"); @@ -1440,7 +1469,7 @@

exampleJson: example, error: validationDetails[Number.parseInt(tableRow.children[1].textContent)] || error, hasBeenValidated: tableRow.hasAttribute("data-valid"), - test: testDetails[Number.parseInt(tableRow.children[1].textContent)] + test: testDetails[Number.parseInt(tableRow.children[1].textContent)] }]); } From a6200eb8c4e708b254463ea8c073b36ea489692f Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Tue, 8 Oct 2024 15:21:13 +0530 Subject: [PATCH 25/43] Add tests OpenApi Examples Generate and Validate. - Reuse tests from `ExamplesCommandTest` and `ExamplesInteractiveServerTest`. - Fix typo in ExampleValidationResult. - add few other tests. --- .../exampleGeneration/ExamplesBase.kt | 4 + .../exampleGeneration/ExamplesGenerateBase.kt | 2 +- .../exampleGeneration/ExamplesValidateBase.kt | 14 +- .../OpenApiExamplesInteractive.kt | 2 +- .../OpenApiExamplesGenerateTest.kt | 466 ++++++++++++++++++ .../OpenApiExamplesValidateTest.kt} | 174 +++---- .../server/ExamplesInteractiveServerTest.kt | 231 --------- 7 files changed, 561 insertions(+), 332 deletions(-) create mode 100644 application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerateTest.kt rename application/src/test/kotlin/application/{ExamplesCommandTest.kt => exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt} (65%) delete mode 100644 application/src/test/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServerTest.kt diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt index 1e5a8613a..3153ba7f1 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt @@ -71,6 +71,10 @@ abstract class ExamplesBase(open val featureStrategy: Example }.toList() } + fun getExternalExampleFilesFromContract(contractFile: File): List { + return getExternalExampleFiles(getExamplesDirectory(contractFile)) + } + fun logSeparator(length: Int, separator: String = "-") { logger.log(separator.repeat(length)) } diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt index d16d995b8..3c66f0766 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt @@ -10,7 +10,7 @@ abstract class ExamplesGenerateBase( private val generationStrategy: ExamplesGenerationStrategy ): ExamplesBase(featureStrategy) { @Parameters(index = "0", description = ["Contract file path"], arity = "0..1") - override var contractFile: File? = null + public override var contractFile: File? = null @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) private var dictFile: File? = null diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt index 3ac21f585..9bd5ea0b3 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt @@ -10,10 +10,10 @@ abstract class ExamplesValidateBase( private val validationStrategy: ExamplesValidationStrategy ): ExamplesBase(featureStrategy) { @Option(names = ["--contract-file"], description = ["Contract file path"], required = true) - override var contractFile: File? = null + public override var contractFile: File? = null @Option(names = ["--example-file"], description = ["Example file path"], required = false) - private val exampleFile: File? = null + var exampleFile: File? = null abstract var validateExternal: Boolean abstract var validateInline: Boolean @@ -67,7 +67,9 @@ abstract class ExamplesValidateBase( private fun validateExampleFile(exampleFile: File, contractFile: File): ExampleValidationResult { val feature = featureStrategy.contractFileToFeature(contractFile) - return validateExternalExample(exampleFile, feature) + return validateExternalExample(exampleFile, feature).also { + it.logErrors() + } } private fun validateExternalExample(exampleFile: File, feature: Feature): ExampleValidationResult { @@ -139,10 +141,10 @@ abstract class ExamplesValidateBase( val prefix = index?.let { "$it. " } ?: "" if (this.result.isSuccess()) { - return logger.log("$prefix${this.exampleName} is valid") + return logger.log("$prefix${this.exampleFile?.name ?: this.exampleName} is valid") } - logger.log("\n$prefix${this.exampleName} has the following validation error(s):") + logger.log("\n$prefix${this.exampleFile?.name ?: this.exampleName} has the following validation error(s):") logger.log(this.result.reportString()) } } @@ -156,7 +158,7 @@ enum class ExampleType(val value: String) { } } -data class ExampleValidationResult(val exampleName: String, val result: Result, val type: ExampleType, val exampleFIle: File? = null) { +data class ExampleValidationResult(val exampleName: String, val result: Result, val type: ExampleType, val exampleFile: File? = null) { constructor(exampleFile: File, result: Result) : this(exampleFile.nameWithoutExtension, result, ExampleType.EXTERNAL, exampleFile) } diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index f6b3175d5..dd72307ad 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -67,7 +67,7 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( TableRowGroup("method", scenario.method, showRow = !methodSet.contains(method), rowSpan = scenarios.size), TableRowGroup("response", scenario.status.toString(), showRow = true, rowSpan = 1, extraInfo = scenario.httpRequestPattern.headersPattern.contentType) ), - exampleFilePath = example?.exampleFIle?.absolutePath, + exampleFilePath = example?.exampleFile?.absolutePath, exampleFileName = example?.exampleName, exampleMismatchReason = example?.result?.reportString().takeIf { reason -> reason?.isNotBlank() == true } ).also { methodSet.add(method); showPath = false } diff --git a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerateTest.kt b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerateTest.kt new file mode 100644 index 000000000..5684d05f6 --- /dev/null +++ b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerateTest.kt @@ -0,0 +1,466 @@ +package application.exampleGeneration.openApiExamples + +import application.captureStandardOutput +import io.specmatic.conversions.ExampleFromFile +import io.specmatic.core.QueryParameters +import io.specmatic.core.SPECMATIC_STUB_DICTIONARY +import io.specmatic.core.pattern.parsedJSONObject +import io.specmatic.core.utilities.Flags +import io.specmatic.core.value.JSONArrayValue +import io.specmatic.core.value.JSONObjectValue +import io.specmatic.core.value.Value +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File + +class OpenApiExamplesGenerateTest { + companion object { + fun generateExamples(specFile: File, examplesDir: File): Triple> { + return OpenApiExamplesGenerate().also { it.contractFile = specFile }.let { + val (output, exitCode) = captureStandardOutput { it.execute(specFile) } + Triple(output, exitCode, examplesDir.listFiles()?.toList() ?: emptyList()) + } + } + + private val externalDictionaryWithoutHeaders = parsedJSONObject(""" + { + "QUERY-PARAMS.name": "Jane Doe", + "QUERY-PARAMS.address": "123-Main-Street", + "PATH-PARAMS.name": "Jane-Doe", + "PATH-PARAMS.address": "123-Main-Street", + "Tracker.name": "Jane Doe", + "Tracker.address": "123-Main-Street", + "Tracker.trackerId": 100, + "Tracker_FVO.name": "Jane Doe", + "Tracker_FVO.address": "123-Main-Street" + } + """.trimIndent()) + + private val externalDictionary = parsedJSONObject(""" + { + "HEADERS.Authentication": "Bearer 123", + "QUERY-PARAMS.name": "Jane Doe", + "QUERY-PARAMS.address": "123-Main-Street", + "PATH-PARAMS.name": "Jane-Doe", + "PATH-PARAMS.address": "123-Main-Street", + "Tracker.name": "Jane Doe", + "Tracker.address": "123-Main-Street", + "Tracker.trackerId": 100, + "Tracker_FVO.name": "Jane Doe", + "Tracker_FVO.address": "123-Main-Street" + } + """.trimIndent()) + + fun assertHeaders(headers: Map, apiKey: String) { + assertThat(headers["Authentication"]).isEqualTo(apiKey) + } + + fun assertPathParameters(path: String?, name: String, address: String) { + assertThat(path).contains("/generate/names/$name/address/$address") + } + + fun assertQueryParameters(queryParameters: QueryParameters, name: String, address: String) { + assertThat(queryParameters.getValues("name")).contains(name) + assertThat(queryParameters.getValues("address")).contains(address) + } + + fun assertBody(body: Value, name: String, address: String) { + body as JSONObjectValue + assertThat(body.findFirstChildByPath("name")?.toStringLiteral()).isEqualTo(name) + assertThat(body.findFirstChildByPath("address")?.toStringLiteral()).isEqualTo(address) + } + } + + @AfterEach + fun cleanUp() { + val examplesFolder = File("src/test/resources/specifications/tracker_examples") + if (examplesFolder.exists()) { + examplesFolder.listFiles()?.forEach { it.delete() } + examplesFolder.delete() + } + } + + @Test + fun `should fail with error for invalid contract file`(@TempDir tempDir: File) { + val specFile = tempDir.resolve("spec.graphqls") + val examplesDir = tempDir.resolve("spec_examples") + + specFile.createNewFile() + + val (stdOut, exitCode, _) = generateExamples(specFile, examplesDir) + println(stdOut) + + assertThat(exitCode).isEqualTo(1) + assertThat(stdOut).contains("spec.graphqls has an unsupported extension") + } + + @Test + fun `should generate an example when it is missing`(@TempDir tempDir: File) { + val specFile = tempDir.resolve("spec.yaml") + val examplesDir = tempDir.resolve("spec_examples") + + specFile.createNewFile() + val spec = """ +openapi: 3.0.0 +info: + title: Product API + version: 1.0.0 +paths: + /product/{id}: + get: + summary: Get product details + parameters: + - name: id + in: path + required: true + schema: + type: integer + responses: + '200': + description: Product details + content: + application/json: + schema: + type: object + properties: + id: + type: integer + name: + type: string + price: + type: number + format: float + """.trimIndent() + specFile.writeText(spec) + + examplesDir.deleteRecursively() + examplesDir.mkdirs() + + val (stdOut, exitCode, examples) = generateExamples(specFile, examplesDir) + println(stdOut) + + assertThat(exitCode).isEqualTo(0) + assertThat(stdOut).contains("1 example(s) created, 0 example(s) already existed, 0 example(s) failed") + + assertThat(examples).hasSize(1) + assertThat(examples.single().name).matches("product_[0-9]*_GET_200.json") + } + + @Test + fun `should retain existing examples and generate missing ones`(@TempDir tempDir: File) { + val specFile = tempDir.resolve("spec.yaml") + val examplesDir = tempDir.resolve("spec_examples") + + specFile.createNewFile() + val spec = """ +openapi: 3.0.0 +info: + title: Product API + version: 1.0.0 +paths: + /product: + get: + summary: Get product details + responses: + '200': + description: Product details + content: + application/json: + schema: + schema: + type: array + items: + type: object + properties: + id: + type: integer + name: + type: string + price: + type: number + format: float + /product/{id}: + get: + summary: Get product details + parameters: + - name: id + in: path + required: true + schema: + type: integer + responses: + '200': + description: Product details + content: + application/json: + schema: + type: object + properties: + id: + type: integer + name: + type: string + price: + type: number + format: float + """.trimIndent() + specFile.writeText(spec) + + examplesDir.deleteRecursively() + examplesDir.mkdirs() + + val example = """ +{ + "http-request": { + "method": "GET", + "path": "/product/1" + }, + "http-response": { + "status": 200, + "body": { + "id": 1, + "name": "Laptop", + "price": 1000.99 + }, + "headers": { + "Content-Type": "application/json" + } + } +} + """.trimIndent() + val exampleFile = examplesDir.resolve("example.json") + exampleFile.writeText(example) + + val (stdOut, exitCode, examples) = generateExamples(specFile, examplesDir) + println(stdOut) + + assertThat(exitCode).isEqualTo(0) + assertThat(stdOut).contains("Using existing example for GET /product/(id:number) -> 200") + .contains("1 example(s) created, 1 example(s) already existed, 0 example(s) failed") + + assertThat(examples).hasSize(2) + assertThat(examples.map { it.name }).containsExactlyInAnyOrder("example.json", "product_GET_200.json") + assertThat(examples.find { it.name == "example.json" }?.readText() ?: "") + .contains(""""name": "Laptop"""") + .contains(""""price": 1000.99""") + + val generatedExample = examples.first { it.name == "product_GET_200.json" } + assertThat(generatedExample.readText()).contains(""""path": "/product"""") + } + + @Test + fun `should generate only 2xx examples by default, when extensive is false`(@TempDir tempDir: File) { + val specFile = tempDir.resolve("spec.yaml") + val examplesDir = tempDir.resolve("spec_examples") + + specFile.createNewFile() + val spec = """ +openapi: 3.0.0 +info: + title: Product API + version: 1.0.0 +paths: + /product/{id}: + get: + summary: Get product details + parameters: + - name: id + in: path + required: true + schema: + type: integer + responses: + '200': + description: Product details + content: + application/json: + schema: + type: object + properties: + id: + type: integer + name: + type: string + price: + type: number + format: float + '404': + description: Bad Request - Invalid input + content: + application/json: + schema: + type: object + properties: + error: + type: string + example: "Invalid product ID" + '500': + description: Internal Server Error - Unexpected error occurred + content: + application/json: + schema: + type: object + properties: + error: + type: string + example: "An unexpected error occurred" + """.trimIndent() + specFile.writeText(spec) + + examplesDir.deleteRecursively() + examplesDir.mkdirs() + + val (stdOut, exitCode, examples) = generateExamples(specFile, examplesDir) + println(stdOut) + + assertThat(exitCode).isEqualTo(0) + assertThat(stdOut).doesNotContain("Using dictionary file") + assertThat(stdOut).contains("${examples.size} example(s) created, 0 example(s) already existed, 0 example(s) failed") + + assertThat(examples).allSatisfy { + val example = ExampleFromFile(it) + val response = example.response + assertThat(response.status).isGreaterThanOrEqualTo(200).isLessThan(300) + } + } + + @Test + fun `should generate with random values when no dictionary is provided`() { + val contractFile = File("src/test/resources/specifications/tracker.yaml") + val examplesDir = contractFile.parentFile.canonicalFile.resolve("tracker_examples") + + val (stdOut, exitCode, examples) = generateExamples(contractFile, examplesDir) + println(stdOut) + + assertThat(exitCode).isEqualTo(0) + assertThat(stdOut).doesNotContain("Using dictionary file") + assertThat(stdOut).contains("${examples.size} example(s) created, 0 example(s) already existed, 0 example(s) failed") + + examples.forEach { + val example = ExampleFromFile(it) + val request = example.request + val response = example.response + val responseBody = response.body as JSONArrayValue + + assertThat(request.headers["Authentication"]) + .withFailMessage("Header values should be randomly generated") + .isNotEqualTo("Bearer 123") + + when(request.method) { + "POST" -> { + val body = request.body as JSONObjectValue + assertThat(body.findFirstChildByPath("name")?.toStringLiteral()).isNotEqualTo("John-Doe") + assertThat(body.findFirstChildByPath("address")?.toStringLiteral()).isNotEqualTo("123-Main-Street") + + } + "GET" -> { + val queryParameters = request.queryParams + assertThat(queryParameters.getValues("name")).doesNotContain("John-Doe") + assertThat(queryParameters.getValues("address")).doesNotContain("123-Main-Street") + } + "DELETE" -> { + val path = request.path as String + assertThat(path).doesNotContain("/generate/names/John-Doe/address/123-Main-Street") + assertThat(path.trim('/').split('/').last()).isNotEqualTo("(string)") + } + else -> throw IllegalArgumentException("Unexpected method ${request.method}") + } + + assertThat(responseBody.list).allSatisfy {value -> + value as JSONObjectValue + assertThat(value.findFirstChildByPath("name")?.toStringLiteral()).isNotEqualTo("John Doe") + assertThat(value.findFirstChildByPath("address")?.toStringLiteral()).isNotEqualTo("123 Main Street") + } + } + } + + @Test + fun `should use values from dictionary when provided`(@TempDir tempDir: File) { + val dictionaryFileName = "dictionary.json" + val contractFile = File("src/test/resources/specifications/tracker.yaml") + val examplesDir = contractFile.parentFile.canonicalFile.resolve("tracker_examples") + + val dictionaryFile = tempDir.resolve(dictionaryFileName) + dictionaryFile.writeText(externalDictionary.toStringLiteral()) + + val (stdOut, exitCode, examples) = Flags.using(SPECMATIC_STUB_DICTIONARY to dictionaryFile.absolutePath) { + generateExamples(contractFile, examplesDir) + } + println(stdOut) + + assertThat(exitCode).isEqualTo(0) + assertThat(stdOut).contains("Using dictionary file ${dictionaryFile.absolutePath}") + assertThat(stdOut).contains("${examples.size} example(s) created, 0 example(s) already existed, 0 example(s) failed") + + examples.forEach { + val example = ExampleFromFile(it) + val request = example.request + val response = example.response + + assertHeaders(request.headers, "Bearer 123") + + when(request.method) { + "POST" -> assertBody(request.body, "Jane Doe", "123-Main-Street") + "GET" -> assertQueryParameters(request.queryParams, "Jane Doe", "123-Main-Street") + "DELETE" -> { + assertPathParameters(request.path, "Jane-Doe", "123-Main-Street") + assertThat(request.path!!.trim('/').split('/').last()).isNotEqualTo("(string)") + } + else -> throw IllegalArgumentException("Unexpected method ${request.method}") + } + + val jsonResponseBody = response.body as JSONArrayValue + assertThat(jsonResponseBody.list).allSatisfy { value -> + value as JSONObjectValue + assertBody(value, "Jane Doe", "123-Main-Street") + } + } + } + + @Test + fun `should only replace values if key is in dictionary`(@TempDir tempDir: File) { + val dictionaryFileName = "dictionary.json" + val contractFile = File("src/test/resources/specifications/tracker.yaml") + val examplesDir = contractFile.parentFile.canonicalFile.resolve("tracker_examples") + + val dictionaryFile = tempDir.resolve(dictionaryFileName) + dictionaryFile.writeText(externalDictionaryWithoutHeaders.toStringLiteral()) + + val (stdOut, exitCode, examples) = Flags.using(SPECMATIC_STUB_DICTIONARY to dictionaryFile.absolutePath) { + generateExamples(contractFile, examplesDir) + } + println(stdOut) + + assertThat(exitCode).isEqualTo(0) + assertThat(stdOut).contains("Using dictionary file ${dictionaryFile.absolutePath}") + assertThat(stdOut).contains("${examples.size} example(s) created, 0 example(s) already existed, 0 example(s) failed") + + examples.forEach { + val example = ExampleFromFile(it) + val request = example.request + val response = example.response + val responseBody = response.body as JSONArrayValue + + assertThat(request.headers["Authentication"]) + .withFailMessage("Header values should be randomly generated") + .isNotEqualTo("Bearer 123") + + when(request.method) { + "POST" -> assertBody(request.body, "Jane Doe", "123-Main-Street") + "GET" -> assertQueryParameters(request.queryParams, "Jane Doe", "123-Main-Street") + "DELETE" -> { + assertPathParameters(request.path, "Jane-Doe", "123-Main-Street") + assertThat(request.path!!.trim('/').split('/').last()).isNotEqualTo("(string)") + } + else -> throw IllegalArgumentException("Unexpected method ${request.method}") + } + + assertThat(responseBody.list).allSatisfy { value -> + value as JSONObjectValue + assertBody(value, "Jane Doe", "123-Main-Street") + } + } + } +} \ No newline at end of file diff --git a/application/src/test/kotlin/application/ExamplesCommandTest.kt b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt similarity index 65% rename from application/src/test/kotlin/application/ExamplesCommandTest.kt rename to application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt index 27e9e4bf7..2204a684a 100644 --- a/application/src/test/kotlin/application/ExamplesCommandTest.kt +++ b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt @@ -1,13 +1,30 @@ -package application +package application.exampleGeneration.openApiExamples +import application.captureStandardOutput import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.io.TempDir import java.io.File -class ExamplesCommandTest { +class OpenApiExamplesValidateTest { + companion object { + fun validateExamples(specFile: File): Pair { + return OpenApiExamplesValidate().also { it.contractFile = specFile }.let { + val (output, exitCode) = captureStandardOutput { it.execute(specFile) } + Pair(output, exitCode) + } + } + + fun validateSingleExample(specFile: File, exampleFile: File): Pair { + return OpenApiExamplesValidate().also { it.contractFile = specFile; it.exampleFile = exampleFile }.let { + val (output, exitCode) = captureStandardOutput { it.execute(specFile) } + Pair(output, exitCode) + } + } + } + @Test - fun `examples validate command should not print an empty error when it sees an inline example for a filtered-out scenario`(@TempDir tempDir: File) { + fun `should display an error message for an invalid example`(@TempDir tempDir: File) { val specFile = tempDir.resolve("spec.yaml") val examplesDir = tempDir.resolve("spec_examples") @@ -45,12 +62,14 @@ paths: """.trimIndent() specFile.writeText(spec) + examplesDir.deleteRecursively() examplesDir.mkdirs() + val example = """ { "http-request": { "method": "GET", - "path": "/products/1" + "path": "/product/abc123" }, "http-response": { "status": 200, @@ -65,26 +84,20 @@ paths: } } """.trimIndent() - val exampleFile = examplesDir.resolve("example.json") exampleFile.writeText(example) - val command = ExamplesCommand.Validate().also { - it.contractFile = specFile - } - - val (output, returnValue: Int) = captureStandardOutput { - command.call() - } - - println(output) + val (stdOut, exitCode) = validateExamples(specFile) + println(stdOut) - assertThat(returnValue).isNotEqualTo(0) - assertThat(output).contains("No matching REST stub or contract found") + assertThat(exitCode).isNotEqualTo(0) + assertThat(stdOut).contains("example.json has the following validation error(s)") + .contains("""expected number but example contained""") + assertThat(stdOut).contains("0 example(s) are valid. 1 example(s) are invalid") } @Test - fun `should display an error message for an invalid example`(@TempDir tempDir: File) { + fun `should not display an error message when all examples are valid`(@TempDir tempDir: File) { val specFile = tempDir.resolve("spec.yaml") val examplesDir = tempDir.resolve("spec_examples") @@ -122,12 +135,14 @@ paths: """.trimIndent() specFile.writeText(spec) + examplesDir.deleteRecursively() examplesDir.mkdirs() + val example = """ { "http-request": { "method": "GET", - "path": "/product/abc123" + "path": "/product/1" }, "http-response": { "status": 200, @@ -142,26 +157,19 @@ paths: } } """.trimIndent() - val exampleFile = examplesDir.resolve("example.json") exampleFile.writeText(example) - val command = ExamplesCommand.Validate().also { - it.contractFile = specFile - } + val (stdOut, exitCode) = validateExamples(specFile) + println(stdOut) - val (output, returnValue: Int) = captureStandardOutput { - command.call() - } - - println(output) - - assertThat(returnValue).isNotEqualTo(0) - assertThat(output).contains("""expected number but example contained""") + assertThat(exitCode).isEqualTo(0) + assertThat(stdOut).contains("example.json is valid") + assertThat(stdOut).contains("1 example(s) are valid. 0 example(s) are invalid") } @Test - fun `should not display an error message when all examples are valid`(@TempDir tempDir: File) { + fun `should not print an empty error when it sees an inline example for a filtered-out scenario`(@TempDir tempDir: File) { val specFile = tempDir.resolve("spec.yaml") val examplesDir = tempDir.resolve("spec_examples") @@ -199,12 +207,14 @@ paths: """.trimIndent() specFile.writeText(spec) + examplesDir.deleteRecursively() examplesDir.mkdirs() + val example = """ { "http-request": { "method": "GET", - "path": "/product/1" + "path": "/products/1" }, "http-response": { "status": 200, @@ -219,26 +229,20 @@ paths: } } """.trimIndent() - val exampleFile = examplesDir.resolve("example.json") exampleFile.writeText(example) - val command = ExamplesCommand.Validate().also { - it.contractFile = specFile - } - - val (output, returnValue: Int) = captureStandardOutput { - command.call() - } - - println(output) + val (stdOut, exitCode) = validateExamples(specFile) + println(stdOut) - assertThat(returnValue).isEqualTo(0) - assertThat(output).contains("are valid") + assertThat(exitCode).isNotEqualTo(0) + assertThat(stdOut).contains("example.json has the following validation error(s)") + .contains("No matching REST stub or contract found") + assertThat(stdOut).contains("0 example(s) are valid. 1 example(s) are invalid") } @Test - fun `should generate an example if missing`(@TempDir tempDir: File) { + fun `should only validate the specified example and ignore others`(@TempDir tempDir: File) { val specFile = tempDir.resolve("spec.yaml") val examplesDir = tempDir.resolve("spec_examples") @@ -273,25 +277,43 @@ paths: price: type: number format: float + '202': + description: Request accepted for processing + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: "Request accepted for processing" """.trimIndent() specFile.writeText(spec) examplesDir.deleteRecursively() examplesDir.mkdirs() - ExamplesCommand().also { - it.contractFile = specFile - }.call() + val (_, generationExitCode, examples) = OpenApiExamplesGenerateTest.generateExamples(specFile, examplesDir) - val examplesCreated = examplesDir.walk().filter { it.isFile }.toList() + assertThat(generationExitCode).isEqualTo(0) + assertThat(examples.size).isEqualTo(2) - assertThat(examplesCreated).hasSize(1) - assertThat(examplesCreated.single().name).matches("product_[0-9]*_GET_200.json") + assertThat(examples).allSatisfy{ + val (stdOut, exitCode) = validateSingleExample(specFile, it) + println(stdOut) + assertThat(exitCode).isEqualTo(0) + assertThat(stdOut).contains("${it.name} is valid") + + val otherExamples = examples.filter { exFile -> exFile != it } + otherExamples.forEach { otherExample -> + assertThat(stdOut).doesNotContain(otherExample.name) + } + } } @Test - fun `should generate only the missing examples and leave the existing examples as is`(@TempDir tempDir: File) { + fun `should fail validation with error on invalid file extension for specified example`(@TempDir tempDir: File) { val specFile = tempDir.resolve("spec.yaml") val examplesDir = tempDir.resolve("spec_examples") @@ -302,27 +324,6 @@ info: title: Product API version: 1.0.0 paths: - /product: - get: - summary: Get product details - responses: - '200': - description: Product details - content: - application/json: - schema: - schema: - type: array - items: - type: object - properties: - id: - type: integer - name: - type: string - price: - type: number - format: float /product/{id}: get: summary: Get product details @@ -353,7 +354,6 @@ paths: examplesDir.deleteRecursively() examplesDir.mkdirs() - examplesDir.mkdirs() val example = """ { "http-request": { @@ -373,26 +373,14 @@ paths: } } """.trimIndent() - - val exampleFile = examplesDir.resolve("example.json") + val exampleFile = examplesDir.resolve("example.txt") exampleFile.writeText(example) - ExamplesCommand().also { - it.contractFile = specFile - }.call() - - val examplesCreated = examplesDir.walk().filter { it.isFile }.toList() + val (stdOut, exitCode) = validateSingleExample(specFile, exampleFile) + println(stdOut) - assertThat(examplesCreated).hasSize(2) - println(examplesCreated.map { it.name }) - assertThat(examplesCreated.filter { it.name == "example.json" }).hasSize(1) - assertThat(examplesCreated.filter { it.name == "product_GET_200.json" }).hasSize(1) - - assertThat(examplesCreated.find { it.name == "example.json" }?.readText() ?: "") - .contains(""""name": "Laptop"""") - .contains(""""price": 1000.99""") - - val generatedExample = examplesCreated.first { it.name == "product_GET_200.json" } - assertThat(generatedExample.readText()).contains(""""path": "/product"""") + assertThat(exitCode).isNotEqualTo(0) + assertThat(stdOut).contains("Invalid Example file ${exampleFile.absolutePath}") + .contains("File extension must be one of json") } -} \ No newline at end of file +} diff --git a/application/src/test/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServerTest.kt b/application/src/test/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServerTest.kt deleted file mode 100644 index 37f108f4f..000000000 --- a/application/src/test/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServerTest.kt +++ /dev/null @@ -1,231 +0,0 @@ -package io.specmatic.core.examples.server - -import io.specmatic.conversions.ExampleFromFile -import io.specmatic.core.QueryParameters -import io.specmatic.core.SPECMATIC_STUB_DICTIONARY -import io.specmatic.core.pattern.parsedJSONObject -import io.specmatic.core.utilities.Flags -import io.specmatic.core.value.JSONArrayValue -import io.specmatic.core.value.JSONObjectValue -import io.specmatic.core.value.Value -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.io.TempDir -import java.io.File - -class ExamplesInteractiveServerTest { - companion object { - private val externalDictionaryWithoutHeaders = - parsedJSONObject(""" - { - "QUERY-PARAMS.name": "Jane Doe", - "QUERY-PARAMS.address": "123-Main-Street", - "PATH-PARAMS.name": "Jane-Doe", - "PATH-PARAMS.address": "123-Main-Street", - "Tracker.name": "Jane Doe", - "Tracker.address": "123-Main-Street", - "Tracker.trackerId": 100, - "Tracker_FVO.name": "Jane Doe", - "Tracker_FVO.address": "123-Main-Street" - } - """.trimIndent()) - - private val externalDictionary = - parsedJSONObject(""" - { - "HEADERS.Authentication": "Bearer 123", - "QUERY-PARAMS.name": "Jane Doe", - "QUERY-PARAMS.address": "123-Main-Street", - "PATH-PARAMS.name": "Jane-Doe", - "PATH-PARAMS.address": "123-Main-Street", - "Tracker.name": "Jane Doe", - "Tracker.address": "123-Main-Street", - "Tracker.trackerId": 100, - "Tracker_FVO.name": "Jane Doe", - "Tracker_FVO.address": "123-Main-Street" - } - """.trimIndent()) - - private val partialDictionary = - parsedJSONObject(""" - { - "name": "John-Doe", - "address": "123-Main-Street" - } - """.trimIndent()) - - fun assertHeaders(headers: Map, apiKey: String) { - assertThat(headers["Authentication"]).isEqualTo(apiKey) - } - - fun assertPathParameters(path: String?, name: String, address: String) { - assertThat(path).contains("/generate/names/$name/address/$address") - } - - fun assertQueryParameters(queryParameters: QueryParameters, name: String, address: String) { - assertThat(queryParameters.getValues("name")).contains(name) - assertThat(queryParameters.getValues("address")).contains(address) - } - - fun assertRequestBody(body: Value, name: String, address: String) { - body as JSONObjectValue - assertThat(body.findFirstChildByPath("name")?.toStringLiteral()).isEqualTo(name) - assertThat(body.findFirstChildByPath("address")?.toStringLiteral()).isEqualTo(address) - } - - fun assertResponseBody(body: Value, getNameAddress: (index: Int) -> Pair) { - body as JSONArrayValue - body.list.forEachIndexed { index, value -> - value as JSONObjectValue - - val (name, address) = getNameAddress(index) - assertThat(value.findFirstChildByPath("name")?.toStringLiteral()).isEqualTo(name) - assertThat(value.findFirstChildByPath("address")?.toStringLiteral()).isEqualTo(address) - } - } - } - - @AfterEach - fun cleanUp() { - val examplesFolder = File("src/test/resources/specifications/tracker_examples") - if (examplesFolder.exists()) { - examplesFolder.listFiles()?.forEach { it.delete() } - examplesFolder.delete() - } - } - - @Test - fun `should generate all random values when no dictionary is provided`() { - val examples = ExamplesInteractiveServer.generate( - contractFile = File("src/test/resources/specifications/tracker.yaml"), - scenarioFilter = ExamplesInteractiveServer.ScenarioFilter("", ""), extensive = false, - ).map { File(it) } - - examples.forEach { - val example = ExampleFromFile(it) - val request = example.request - val response = example.response - val responseBody = response.body as JSONArrayValue - - assertThat(request.headers["Authentication"]) - .withFailMessage("Header values should be randomly generated") - .isNotEqualTo("Bearer 123") - - when(request.method) { - "POST" -> { - val body = request.body as JSONObjectValue - assertThat(body.findFirstChildByPath("name")?.toStringLiteral()).isNotEqualTo("John-Doe") - assertThat(body.findFirstChildByPath("address")?.toStringLiteral()).isNotEqualTo("123-Main-Street") - - } - "GET" -> { - val queryParameters = request.queryParams - assertThat(queryParameters.getValues("name")).doesNotContain("John-Doe") - assertThat(queryParameters.getValues("address")).doesNotContain("123-Main-Street") - } - "DELETE" -> { - val path = request.path as String - assertThat(path).doesNotContain("/generate/names/John-Doe/address/123-Main-Street") - assertThat(path.trim('/').split('/').last()).isNotEqualTo("(string)") - } - else -> throw IllegalArgumentException("Unexpected method ${request.method}") - } - - responseBody.list.forEachIndexed { index, value -> - value as JSONObjectValue - val (name, address) = when(index) { - 0 -> "John Doe" to "123 Main Street" - else -> "Jane Doe" to "456 Main Street" - } - - assertThat(value.findFirstChildByPath("name")?.toStringLiteral()).isNotEqualTo(name) - assertThat(value.findFirstChildByPath("address")?.toStringLiteral()).isNotEqualTo(address) - } - } - } - - @Test - fun `should use values from dictionary when provided`(@TempDir tempDir: File) { - val dictionaryFileName = "dictionary.json" - - val dictionaryFile = tempDir.resolve(dictionaryFileName) - dictionaryFile.writeText(externalDictionary.toStringLiteral()) - - val examples = Flags.using(SPECMATIC_STUB_DICTIONARY to dictionaryFile.path) { - ExamplesInteractiveServer.generate( - contractFile = File("src/test/resources/specifications/tracker.yaml"), - scenarioFilter = ExamplesInteractiveServer.ScenarioFilter("", ""), extensive = false, - ).map { File(it) } - } - - examples.forEach { - val example = ExampleFromFile(it) - val request = example.request - val response = example.response - - assertHeaders(request.headers, "Bearer 123") - - when(request.method) { - "POST" -> assertRequestBody(request.body, "Jane Doe", "123-Main-Street") - "GET" -> assertQueryParameters(request.queryParams, "Jane Doe", "123-Main-Street") - "DELETE" -> { - assertPathParameters(request.path, "Jane-Doe", "123-Main-Street") - assertThat(request.path!!.trim('/').split('/').last()).isNotEqualTo("(string)") - } - else -> throw IllegalArgumentException("Unexpected method ${request.method}") - } - - val jsonResponseBody = response.body as JSONArrayValue - assertThat(jsonResponseBody.list).allSatisfy { - it as JSONObjectValue - - assertThat(it.findFirstChildByPath("name")?.toStringLiteral()).isEqualTo("Jane Doe") - assertThat(it.findFirstChildByPath("address")?.toStringLiteral()).isEqualTo("123-Main-Street") - } - } - } - - @Test - fun `should only replace values if key is in dictionary`(@TempDir tempDir: File) { - val dictionaryFileName = "dictionary.json" - - val dictionaryFile = tempDir.resolve(dictionaryFileName) - dictionaryFile.writeText(externalDictionaryWithoutHeaders.toStringLiteral()) - - val examples = Flags.using(SPECMATIC_STUB_DICTIONARY to dictionaryFile.path) { - ExamplesInteractiveServer.generate( - contractFile = File("src/test/resources/specifications/tracker.yaml"), - scenarioFilter = ExamplesInteractiveServer.ScenarioFilter("", ""), extensive = false, - ).map { File(it) } - } - - examples.forEach { - val example = ExampleFromFile(it) - val request = example.request - val response = example.response - val responseBody = response.body as JSONArrayValue - - assertThat(request.headers["Authentication"]) - .withFailMessage("Header values should be randomly generated") - .isNotEqualTo("Bearer 123") - - when(request.method) { - "POST" -> assertRequestBody(request.body, "Jane Doe", "123-Main-Street") - "GET" -> assertQueryParameters(request.queryParams, "Jane Doe", "123-Main-Street") - "DELETE" -> { - assertPathParameters(request.path, "Jane-Doe", "123-Main-Street") - assertThat(request.path!!.trim('/').split('/').last()).isNotEqualTo("(string)") - } - else -> throw IllegalArgumentException("Unexpected method ${request.method}") - } - - assertThat(responseBody.list).allSatisfy { value -> - value as JSONObjectValue - - assertThat(value.findFirstChildByPath("name")?.toStringLiteral()).isEqualTo("Jane Doe") - assertThat(value.findFirstChildByPath("address")?.toStringLiteral()).isEqualTo("123-Main-Street") - } - } - } -} \ No newline at end of file From eaa05af16e55f331b9b4f813055363f76b2cefc9 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Wed, 9 Oct 2024 15:33:26 +0530 Subject: [PATCH 26/43] Fix logger based on verbosity on command execution - Don't print dictionary in use, already printed by loadDictionary method. --- .../kotlin/application/exampleGeneration/ExamplesBase.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt index 3153ba7f1..8e0d3335d 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt @@ -21,6 +21,8 @@ abstract class ExamplesBase(open val featureStrategy: Example private var verbose = false override fun call(): Int { + configureLogger(verbose) + contractFile?.let { if (!it.exists()) { logger.log("Contract file does not exist: ${it.absolutePath}") @@ -106,12 +108,10 @@ abstract class ExamplesBase(open val featureStrategy: Example } } - dictFile?.let { - logger.log("Using Dictionary file: ${it.absolutePath}") + return dictFile?.let { System.setProperty(SPECMATIC_STUB_DICTIONARY, it.absolutePath) + it } - - return dictFile } private fun getFilteredScenarios(scenarios: List, scenarioFilter: ScenarioFilter): List { From c3d24483442ad0d24e7f813dcb1a6274b4662d28 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Wed, 9 Oct 2024 17:18:39 +0530 Subject: [PATCH 27/43] Minor refactor, update command descriptions - update descriptions so they're similar across protocols / implementations. - move `extensive` argument to ExamplesBase.kt --- .../kotlin/application/exampleGeneration/ExamplesBase.kt | 6 ++++-- .../application/exampleGeneration/ExamplesGenerateBase.kt | 4 +--- .../exampleGeneration/ExamplesInteractiveBase.kt | 3 +-- .../application/exampleGeneration/ExamplesValidateBase.kt | 1 - .../openApiExamples/OpenApiExamplesGenerate.kt | 2 +- .../openApiExamples/OpenApiExamplesInteractive.kt | 2 +- .../openApiExamples/OpenApiExamplesValidate.kt | 2 +- 7 files changed, 9 insertions(+), 11 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt index 8e0d3335d..d504a67ab 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt @@ -20,6 +20,8 @@ abstract class ExamplesBase(open val featureStrategy: Example @CommandLine.Option(names = ["--debug"], description = ["Debug logs"]) private var verbose = false + abstract var extensive: Boolean + override fun call(): Int { configureLogger(verbose) @@ -43,7 +45,7 @@ abstract class ExamplesBase(open val featureStrategy: Example abstract fun execute(contract: File?): Int // HELPER METHODS - fun configureLogger(verbose: Boolean) { + private fun configureLogger(verbose: Boolean) { val logPrinters = listOf(ConsolePrinter) logger = if (verbose) @@ -52,7 +54,7 @@ abstract class ExamplesBase(open val featureStrategy: Example NonVerbose(CompositePrinter(logPrinters)) } - fun getFilteredScenarios(feature: Feature, extensive: Boolean = false): List { + fun getFilteredScenarios(feature: Feature): List { val scenarioFilter = ScenarioFilter(filterName, filterNotName) val scenarios = featureStrategy.getScenariosFromFeature(feature, extensive) return getFilteredScenarios(scenarios, scenarioFilter) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt index 3c66f0766..fd0a410dc 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt @@ -15,8 +15,6 @@ abstract class ExamplesGenerateBase( @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) private var dictFile: File? = null - abstract var extensive: Boolean - override fun execute(contract: File?): Int { if (contract == null) { logger.log("No contract file provided. Use a subcommand or provide a contract file. Use --help for more details.") @@ -39,7 +37,7 @@ abstract class ExamplesGenerateBase( // GENERATOR METHODS private fun generateExamples(contractFile: File, examplesDir: File): List { val feature = featureStrategy.contractFileToFeature(contractFile) - val filteredScenarios = getFilteredScenarios(feature, extensive) + val filteredScenarios = getFilteredScenarios(feature) if (filteredScenarios.isEmpty()) { return emptyList() diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index 83757cc5a..a0338ec84 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -36,7 +36,6 @@ abstract class ExamplesInteractiveBase ( @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) var dictFile: File? = null - abstract var extensive: Boolean abstract val htmlTableColumns: List private var cachedContractFileFromRequest: File? = null @@ -365,7 +364,7 @@ data class ExampleValidationRequest ( val exampleFile: File ) -data class ExampleValidationResponse( +data class ExampleValidationResponse ( val exampleFilePath: String, val error: String? = null ) { diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt index 9bd5ea0b3..4150c8c8a 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt @@ -17,7 +17,6 @@ abstract class ExamplesValidateBase( abstract var validateExternal: Boolean abstract var validateInline: Boolean - abstract var extensive: Boolean override fun execute(contract: File?): Int { if (contract == null) { diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt index bbb41cc6d..932efd619 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt @@ -12,7 +12,7 @@ import java.io.File @Command( name = "examples", - description = ["Generate JSON Examples with Request and Response from an OpenApi Contract File"], + description = ["Generate Externalised Examples from an OpenApi Contract"], subcommands = [OpenApiExamplesValidate::class, OpenApiExamplesInteractive::class] ) class OpenApiExamplesGenerate: ExamplesGenerateBase ( diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index dd72307ad..7052e6af5 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -14,7 +14,7 @@ import picocli.CommandLine.Command import picocli.CommandLine.Option import java.io.File -@Command(name = "interactive", description = ["Generate and validate examples interactively through a Web UI"],) +@Command(name = "interactive", description = ["Generate, validate and test examples interactively through a Web UI"]) class OpenApiExamplesInteractive : ExamplesInteractiveBase( featureStrategy = OpenApiExamplesFeatureStrategy(), generationStrategy = OpenApiExamplesGenerationStrategy(), validationStrategy = OpenApiExamplesValidationStrategy() ) { diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt index fd92242ac..eb0e7cacc 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt @@ -12,7 +12,7 @@ import picocli.CommandLine.Command import picocli.CommandLine.Option import java.io.File -@Command(name = "validate", description = ["Validate OpenAPI inline and external examples"]) +@Command(name = "validate", description = ["Validate OpenAPI inline and externalised examples"]) class OpenApiExamplesValidate: ExamplesValidateBase( featureStrategy = OpenApiExamplesFeatureStrategy(), validationStrategy = OpenApiExamplesValidationStrategy() ) { From f567aa37d31c1d1b70df36ba8902bc6bec0d9e14 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 10 Oct 2024 13:35:19 +0530 Subject: [PATCH 28/43] Examples Testing cleanup, minor refactorings. - Use Feature.createContractTestFromExampleFile in example testing. --- .../openApiExamples/OpenApiExamplesInteractive.kt | 6 +----- .../openApiExamples/OpenApiExamplesGenerateTest.kt | 4 ++-- .../openApiExamples/OpenApiExamplesValidateTest.kt | 8 ++++---- core/src/main/kotlin/io/specmatic/core/Feature.kt | 12 +++++++----- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index 7052e6af5..48afb578e 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -28,11 +28,7 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( ) override fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair { - val contractTests = feature.loadExternalisedExamples().generateContractTests(emptyList()) - - val test = contractTests.firstOrNull { - it.testDescription().contains(exampleFile.nameWithoutExtension) - } ?: return Pair(TestResult.Error, "Test not found for example ${exampleFile.nameWithoutExtension}") + val test = feature.createContractTestFromExampleFile(exampleFile.absolutePath).value val testResultRecord = test.runTest(testBaseUrl, timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS).let { test.testResultRecord(it.first, it.second) diff --git a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerateTest.kt b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerateTest.kt index 5684d05f6..7d5ba57e9 100644 --- a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerateTest.kt +++ b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerateTest.kt @@ -19,7 +19,7 @@ class OpenApiExamplesGenerateTest { companion object { fun generateExamples(specFile: File, examplesDir: File): Triple> { return OpenApiExamplesGenerate().also { it.contractFile = specFile }.let { - val (output, exitCode) = captureStandardOutput { it.execute(specFile) } + val (output, exitCode) = captureStandardOutput { it.call() } Triple(output, exitCode, examplesDir.listFiles()?.toList() ?: emptyList()) } } @@ -93,7 +93,7 @@ class OpenApiExamplesGenerateTest { println(stdOut) assertThat(exitCode).isEqualTo(1) - assertThat(stdOut).contains("spec.graphqls has an unsupported extension") + assertThat(stdOut).contains("spec.graphqls - File extension must be one of yaml, yml, json") } @Test diff --git a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt index 2204a684a..407ec545a 100644 --- a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt +++ b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt @@ -10,14 +10,14 @@ class OpenApiExamplesValidateTest { companion object { fun validateExamples(specFile: File): Pair { return OpenApiExamplesValidate().also { it.contractFile = specFile }.let { - val (output, exitCode) = captureStandardOutput { it.execute(specFile) } + val (output, exitCode) = captureStandardOutput { it.call() } Pair(output, exitCode) } } fun validateSingleExample(specFile: File, exampleFile: File): Pair { return OpenApiExamplesValidate().also { it.contractFile = specFile; it.exampleFile = exampleFile }.let { - val (output, exitCode) = captureStandardOutput { it.execute(specFile) } + val (output, exitCode) = captureStandardOutput { it.call() } Pair(output, exitCode) } } @@ -69,14 +69,14 @@ paths: { "http-request": { "method": "GET", - "path": "/product/abc123" + "path": "/product/abc" }, "http-response": { "status": 200, "body": { "id": 1, "name": "Laptop", - "price": 1000.99 + "price": 1000 }, "headers": { "Content-Type": "application/json" diff --git a/core/src/main/kotlin/io/specmatic/core/Feature.kt b/core/src/main/kotlin/io/specmatic/core/Feature.kt index 38d742863..3c5a2f6a1 100644 --- a/core/src/main/kotlin/io/specmatic/core/Feature.kt +++ b/core/src/main/kotlin/io/specmatic/core/Feature.kt @@ -374,18 +374,20 @@ data class Feature( fun createContractTestFromExampleFile(filePath: String): ReturnValue { val scenarioStub = ScenarioStub.readFromFile(File(filePath)) + return createContractTestFromScenarioStub(scenarioStub, filePath) + } + @Suppress("MemberVisibilityCanBePrivate") // Used by GraphQL when testing examples interactively + fun createContractTestFromScenarioStub(scenarioStub: ScenarioStub, filePath: String): ReturnValue { val originalScenario = scenarios.firstOrNull { scenario -> scenario.matches(scenarioStub.request, scenarioStub.response) is Result.Success } ?: return HasFailure(Result.Failure("Could not find an API matching example $filePath")) - val concreteTestScenario = io.specmatic.core.Scenario( - name = "", + val concreteTestScenario = Scenario( + name = originalScenario.name, httpRequestPattern = scenarioStub.request.toPattern(), httpResponsePattern = HttpResponsePattern(scenarioStub.response) - ).let { - it.copy(name = it.apiIdentifier) - } + ) return HasValue(scenarioAsTest(concreteTestScenario, null, Workflow(), originalScenario)) } From 2c2589fbcbe4a3ec5bb585092c38973815f5a760 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 10 Oct 2024 16:54:22 +0530 Subject: [PATCH 29/43] Implement inline examples validation. - Add tests for inline examples validation. --- .../exampleGeneration/ExamplesValidateBase.kt | 9 +- .../OpenApiExamplesValidate.kt | 10 ++ .../OpenApiExamplesValidateTest.kt | 142 +++++++++++++++++- 3 files changed, 150 insertions(+), 11 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt index 4150c8c8a..ea2981ebe 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt @@ -52,11 +52,6 @@ abstract class ExamplesValidateBase( } } - // HOOKS - open fun validateInlineExamples(feature: Feature): List> { - return emptyList() - } - // VALIDATION METHODS private fun getFilteredFeature(contractFile: File): Feature { val feature = featureStrategy.contractFileToFeature(contractFile) @@ -84,7 +79,7 @@ abstract class ExamplesValidateBase( private fun validateInlineExamples(contractFile: File): List { val feature = getFilteredFeature(contractFile) - return validateInlineExamples(feature).mapIndexed { index, it -> + return validationStrategy.validateInlineExamples(feature).mapIndexed { index, it -> ExampleValidationResult(it.first, it.second, ExampleType.INLINE).also { it.logErrors(index.inc()) logSeparator(75) @@ -165,4 +160,6 @@ interface ExamplesValidationStrategy { fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature fun validateExternalExample(feature: Feature, exampleFile: File): Pair + + fun validateInlineExamples(feature: Feature): List> { return emptyList() } } diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt index eb0e7cacc..6ae6b637c 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt @@ -35,6 +35,16 @@ class OpenApiExamplesValidationStrategy: ExamplesValidationStrategy> { + val inlineExamples = feature.stubsFromExamples.mapValues { + it.value.map { stub -> + ScenarioStub(stub.first, stub.second) + } + } + + return feature.validateMultipleExamples(inlineExamples, inline = true) + } + private fun getCleanedUpFailure(failureResults: Results, noMatchingScenario: NoMatchingScenario?): Results { return failureResults.toResultIfAny().let { if (it.reportString().isBlank()) diff --git a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt index 407ec545a..99fe7a811 100644 --- a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt +++ b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt @@ -21,10 +21,137 @@ class OpenApiExamplesValidateTest { Pair(output, exitCode) } } + + fun validateOnlyInlineExamples(specFile: File): Pair { + return OpenApiExamplesValidate().also { it.contractFile = specFile; it.validateInline = true; it.validateExternal = false }.let { + val (output, exitCode) = captureStandardOutput { it.call() } + Pair(output, exitCode) + } + } + } + + @Test + fun `should display an error message for an invalid inline example`(@TempDir tempDir: File) { + val specFile = tempDir.resolve("spec.yaml") + specFile.createNewFile() + val spec = """ +openapi: 3.0.0 +info: + title: Product API + version: 1.0.0 +paths: + /product/{id}: + get: + summary: Get product details + parameters: + - name: id + in: path + required: true + schema: + type: integer + examples: + BAD_EXAMPLE: + value: 1 + responses: + '200': + description: Product details + content: + application/json: + schema: + type: object + properties: + id: + type: integer + name: + type: string + price: + type: number + format: float + required: + - id + - name + - price + examples: + BAD_EXAMPLE: + value: + id: 1 + name: "Sample Product" + """.trimIndent() + specFile.writeText(spec) + + val (stdOut, exitCode) = validateOnlyInlineExamples(specFile) + println(stdOut) + + assertThat(exitCode).isNotEqualTo(0) + assertThat(stdOut).contains("BAD_EXAMPLE has the following validation error(s)") + .contains("""Key price in the specification is missing from the example""") + assertThat(stdOut).contains("Inline Examples Validation Summary") + .contains("0 example(s) are valid. 1 example(s) are invalid") + .doesNotContain("External Examples Validation Summary") + } + + @Test + fun `should not display an error message when all inline examples are valid`(@TempDir tempDir: File) { + val specFile = tempDir.resolve("spec.yaml") + specFile.createNewFile() + val spec = """ +openapi: 3.0.0 +info: + title: Product API + version: 1.0.0 +paths: + /product/{id}: + get: + summary: Get product details + parameters: + - name: id + in: path + required: true + schema: + type: integer + examples: + GOOD_EXAMPLE: + value: 1 + responses: + '200': + description: Product details + content: + application/json: + schema: + type: object + properties: + id: + type: integer + name: + type: string + price: + type: number + format: float + required: + - id + - name + - price + examples: + GOOD_EXAMPLE: + value: + id: 1 + name: "Sample Product" + price: 100.50 + """.trimIndent() + specFile.writeText(spec) + + val (stdOut, exitCode) = validateOnlyInlineExamples(specFile) + println(stdOut) + + assertThat(exitCode).isEqualTo(0) + assertThat(stdOut).contains("GOOD_EXAMPLE is valid") + assertThat(stdOut).contains("Inline Examples Validation Summary") + .contains("1 example(s) are valid. 0 example(s) are invalid") + .doesNotContain("External Examples Validation Summary") } @Test - fun `should display an error message for an invalid example`(@TempDir tempDir: File) { + fun `should display an error message for an invalid external example`(@TempDir tempDir: File) { val specFile = tempDir.resolve("spec.yaml") val examplesDir = tempDir.resolve("spec_examples") @@ -93,11 +220,12 @@ paths: assertThat(exitCode).isNotEqualTo(0) assertThat(stdOut).contains("example.json has the following validation error(s)") .contains("""expected number but example contained""") - assertThat(stdOut).contains("0 example(s) are valid. 1 example(s) are invalid") + assertThat(stdOut).doesNotContain("Inline Examples Validation Summary") + .contains("0 example(s) are valid. 1 example(s) are invalid") } @Test - fun `should not display an error message when all examples are valid`(@TempDir tempDir: File) { + fun `should not display an error message when all external examples are valid`(@TempDir tempDir: File) { val specFile = tempDir.resolve("spec.yaml") val examplesDir = tempDir.resolve("spec_examples") @@ -165,7 +293,8 @@ paths: assertThat(exitCode).isEqualTo(0) assertThat(stdOut).contains("example.json is valid") - assertThat(stdOut).contains("1 example(s) are valid. 0 example(s) are invalid") + assertThat(stdOut).doesNotContain("Inline Examples Validation Summary") + .contains("1 example(s) are valid. 0 example(s) are invalid") } @Test @@ -238,7 +367,8 @@ paths: assertThat(exitCode).isNotEqualTo(0) assertThat(stdOut).contains("example.json has the following validation error(s)") .contains("No matching REST stub or contract found") - assertThat(stdOut).contains("0 example(s) are valid. 1 example(s) are invalid") + assertThat(stdOut).doesNotContain("Inline Examples Validation Summary") + .contains("0 example(s) are valid. 1 example(s) are invalid") } @Test @@ -304,6 +434,8 @@ paths: assertThat(exitCode).isEqualTo(0) assertThat(stdOut).contains("${it.name} is valid") + assertThat(stdOut).doesNotContain("Inline Examples Validation Summary") + .doesNotContain("External Examples Validation Summary") val otherExamples = examples.filter { exFile -> exFile != it } otherExamples.forEach { otherExample -> From 807c8b63800aea1fe67f9ca8a6a1e2d4d00586b0 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Fri, 11 Oct 2024 02:55:44 +0530 Subject: [PATCH 30/43] Refactor interactive example testing, better logs. - Use Result instead of TestResult, add result report in-case of test Failure. - [WIP] modifications to Feature test function createContractTestFromExampleFile. - Don't use ExactValue Pattern for response. --- .../ExamplesInteractiveBase.kt | 13 +++++++++++-- .../OpenApiExamplesInteractive.kt | 18 +++++++----------- .../main/kotlin/io/specmatic/core/Feature.kt | 6 ++---- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index a0338ec84..3df4b4d63 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -12,6 +12,7 @@ import io.ktor.server.request.* import io.ktor.server.response.* import io.ktor.server.routing.* import io.ktor.util.pipeline.* +import io.specmatic.core.Result import io.specmatic.core.TestResult import io.specmatic.core.log.logger import io.specmatic.core.route.modules.HealthCheckModule.Companion.configureHealthCheckModule @@ -62,7 +63,7 @@ abstract class ExamplesInteractiveBase ( abstract suspend fun getScenarioFromRequestOrNull(call: ApplicationCall, feature: Feature): Scenario? - abstract fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair + abstract fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair // HELPER METHODS private suspend fun generateExample(call: ApplicationCall, contractFile: File): ExampleGenerationResult { @@ -342,7 +343,15 @@ abstract class ExamplesInteractiveBase ( }) } - data class ExampleTestResult(val result: TestResult, val testLog: String, val exampleFile: File) + data class ExampleTestResult(val result: TestResult, val testLog: String, val exampleFile: File) { + constructor(result: Result, testLog: String, exampleFile: File): this( + result.testResult(), + result.takeIf { !it.isSuccess() }?.let { + "${it.reportString()}\n\n$testLog" + } ?: testLog, + exampleFile + ) + } } data class ExamplePageRequest ( diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index 48afb578e..61f9fe1a2 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -4,10 +4,7 @@ import application.exampleGeneration.* import io.ktor.server.application.* import io.ktor.server.request.* import io.specmatic.conversions.convertPathParameterStyle -import io.specmatic.core.DEFAULT_TIMEOUT_IN_MILLISECONDS -import io.specmatic.core.Feature -import io.specmatic.core.Scenario -import io.specmatic.core.TestResult +import io.specmatic.core.* import io.specmatic.test.TestInteractionsLog import io.specmatic.test.TestInteractionsLog.combineLog import picocli.CommandLine.Command @@ -27,16 +24,15 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( HtmlTableColumn(name = "response", colSpan = 1) ) - override fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair { + override fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair { val test = feature.createContractTestFromExampleFile(exampleFile.absolutePath).value - val testResultRecord = test.runTest(testBaseUrl, timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS).let { - test.testResultRecord(it.first, it.second) - } ?: return Pair(TestResult.Error, "TestResult record not found for example ${exampleFile.nameWithoutExtension}") + val testResult = test.runTest(testBaseUrl, timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS) + val testLogs = TestInteractionsLog.testHttpLogMessages.lastOrNull { + it.scenario == testResult.first.scenario + }?.combineLog() ?: "Test logs not found for example" - return testResultRecord.scenarioResult?.let { scenarioResult -> - Pair(testResultRecord.result, TestInteractionsLog.testHttpLogMessages.last { it.scenario == scenarioResult.scenario }.combineLog()) - } ?: Pair(TestResult.Error, "Interaction logs not found for example ${exampleFile.nameWithoutExtension}") + return testResult.first to testLogs } override suspend fun getScenarioFromRequestOrNull(call: ApplicationCall, feature: Feature): Scenario? { diff --git a/core/src/main/kotlin/io/specmatic/core/Feature.kt b/core/src/main/kotlin/io/specmatic/core/Feature.kt index 3c5a2f6a1..213e6e200 100644 --- a/core/src/main/kotlin/io/specmatic/core/Feature.kt +++ b/core/src/main/kotlin/io/specmatic/core/Feature.kt @@ -383,10 +383,8 @@ data class Feature( scenario.matches(scenarioStub.request, scenarioStub.response) is Result.Success } ?: return HasFailure(Result.Failure("Could not find an API matching example $filePath")) - val concreteTestScenario = Scenario( - name = originalScenario.name, - httpRequestPattern = scenarioStub.request.toPattern(), - httpResponsePattern = HttpResponsePattern(scenarioStub.response) + val concreteTestScenario = originalScenario.copy( + httpRequestPattern = scenarioStub.request.toPattern() ) return HasValue(scenarioAsTest(concreteTestScenario, null, Workflow(), originalScenario)) From 3d840871d206bce592f5ce3606ec2975105d5948 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Fri, 11 Oct 2024 16:14:59 +0530 Subject: [PATCH 31/43] Minor refactors, code cleanups, improvements. - use consoleLog and consoleDebug from log package - use restrictive access modifiers when possible. - add consoleDebug to log package. --- .../exampleGeneration/ExamplesBase.kt | 95 +++++++++------- .../exampleGeneration/ExamplesGenerateBase.kt | 37 ++++--- .../ExamplesInteractiveBase.kt | 75 ++++++------- .../exampleGeneration/ExamplesValidateBase.kt | 101 +++++++++--------- .../kotlin/io/specmatic/core/log/Logging.kt | 19 ++++ 5 files changed, 179 insertions(+), 148 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt index d504a67ab..81d4c57bd 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt @@ -4,45 +4,35 @@ import io.specmatic.core.DICTIONARY_FILE_SUFFIX import io.specmatic.core.EXAMPLES_DIR_SUFFIX import io.specmatic.core.SPECMATIC_STUB_DICTIONARY import io.specmatic.core.log.* -import picocli.CommandLine +import picocli.CommandLine.Option import java.io.File import java.util.concurrent.Callable -abstract class ExamplesBase(open val featureStrategy: ExamplesFeatureStrategy) : Callable { - protected abstract var contractFile: File? - - @CommandLine.Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") +abstract class ExamplesBase(protected open val featureStrategy: ExamplesFeatureStrategy) : Callable { + @Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") private var filterName: String = "" - @CommandLine.Option(names = ["--filter-not-name"], description = ["Use only APIs which do not have this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}") + @Option(names = ["--filter-not-name"], description = ["Use only APIs which do not have this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}") private var filterNotName: String = "" - @CommandLine.Option(names = ["--debug"], description = ["Debug logs"]) + @Option(names = ["--debug"], description = ["Debug logs"]) private var verbose = false - abstract var extensive: Boolean + protected abstract var contractFile: File? + protected abstract var extensive: Boolean override fun call(): Int { configureLogger(verbose) - contractFile?.let { - if (!it.exists()) { - logger.log("Contract file does not exist: ${it.absolutePath}") - return 1 - } - - if (it.extension !in featureStrategy.contractFileExtensions) { - logger.log("Invalid Contract file ${it.path} - File extension must be one of ${featureStrategy.contractFileExtensions.joinToString()}") - return 1 - } - } - - val exitCode = execute(contractFile) - return exitCode + return contractFile?.let { contract -> + getValidatedContractFileOrNull(contract)?.let { + execute(it) + } ?: 1 + } ?: execute(contractFile) } // HOOKS - abstract fun execute(contract: File?): Int + protected abstract fun execute(contract: File?): Int // HELPER METHODS private fun configureLogger(verbose: Boolean) { @@ -54,36 +44,64 @@ abstract class ExamplesBase(open val featureStrategy: Example NonVerbose(CompositePrinter(logPrinters)) } - fun getFilteredScenarios(feature: Feature): List { + protected fun getFilteredScenarios(feature: Feature): List { val scenarioFilter = ScenarioFilter(filterName, filterNotName) val scenarios = featureStrategy.getScenariosFromFeature(feature, extensive) return getFilteredScenarios(scenarios, scenarioFilter) } - fun getExamplesDirectory(contractFile: File): File { + protected fun getValidatedContractFileOrNull(contractFile: File): File? { + if (!contractFile.exists()) { + consoleLog("Contract file does not exist: ${contractFile.absolutePath}") + return null + } + + if (contractFile.extension !in featureStrategy.contractFileExtensions) { + consoleLog("Invalid Contract file ${contractFile.path} - File extension must be one of ${featureStrategy.contractFileExtensions.joinToString()}") + return null + } + + return contractFile + } + + protected fun getValidatedExampleFileOrNull(exampleFile: File): File? { + if (!exampleFile.exists()) { + consoleLog("Example file does not exist: ${exampleFile.absolutePath}") + return null + } + + if (exampleFile.extension !in featureStrategy.exampleFileExtensions) { + consoleLog("Invalid Example file ${exampleFile.path} - File extension must be one of ${featureStrategy.exampleFileExtensions.joinToString()}") + return null + } + + return exampleFile + } + + protected fun getExamplesDirectory(contractFile: File): File { val examplesDirectory = contractFile.canonicalFile.parentFile.resolve("${contractFile.nameWithoutExtension}$EXAMPLES_DIR_SUFFIX") if (!examplesDirectory.exists()) { - logger.log("Creating examples directory: $examplesDirectory") + consoleLog("Creating examples directory: $examplesDirectory") examplesDirectory.mkdirs() } return examplesDirectory } - fun getExternalExampleFiles(examplesDirectory: File): List { + protected fun getExternalExampleFiles(examplesDirectory: File): List { return examplesDirectory.walk().filter { it.isFile && it.extension in featureStrategy.exampleFileExtensions }.toList() } - fun getExternalExampleFilesFromContract(contractFile: File): List { + protected fun getExternalExampleFilesFromContract(contractFile: File): List { return getExternalExampleFiles(getExamplesDirectory(contractFile)) } - fun logSeparator(length: Int, separator: String = "-") { - logger.log(separator.repeat(length)) + protected fun logSeparator(length: Int, separator: String = "-") { + consoleLog(separator.repeat(length)) } - fun logFormattedOutput(header: String, summary: String, note: String) { + protected fun logFormattedOutput(header: String, summary: String, note: String) { val maxLength = maxOf(summary.length, note.length, 50).let { it + it % 2 } val headerSidePadding = (maxLength - 2 - header.length) / 2 @@ -91,13 +109,13 @@ abstract class ExamplesBase(open val featureStrategy: Example val paddedSummaryLine = summary.padEnd(maxLength) val paddedNoteLine = note.padEnd(maxLength) - logger.log("\n$paddedHeaderLine") - logger.log(paddedSummaryLine) - logger.log("=".repeat(maxLength)) - logger.log(paddedNoteLine) + consoleLog("\n$paddedHeaderLine") + consoleLog(paddedSummaryLine) + consoleLog("=".repeat(maxLength)) + consoleLog(paddedNoteLine) } - fun updateDictionaryFile(dictFileFromArgs: File? = null, contract: File? = contractFile): File? { + protected fun updateDictionaryFile(dictFileFromArgs: File? = null, contract: File? = contractFile) { val dictFile = when(dictFileFromArgs != null) { true -> { dictFileFromArgs.takeIf { it.exists() } ?: throw Exception("Dictionary file does not exist: ${dictFileFromArgs.absolutePath}") @@ -110,9 +128,8 @@ abstract class ExamplesBase(open val featureStrategy: Example } } - return dictFile?.let { + dictFile?.let { System.setProperty(SPECMATIC_STUB_DICTIONARY, it.absolutePath) - it } } @@ -122,7 +139,7 @@ abstract class ExamplesBase(open val featureStrategy: Example .filterScenarios(scenarioFilter.filterNotNameTokens, shouldMatch = false) if (filteredScenarios.isEmpty()) { - logger.log("Note: All examples were filtered out by the filter expression") + consoleLog("Note: All examples were filtered out by the filter expression") } return filteredScenarios diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt index fd0a410dc..83dc02262 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt @@ -9,7 +9,7 @@ abstract class ExamplesGenerateBase( override val featureStrategy: ExamplesFeatureStrategy, private val generationStrategy: ExamplesGenerationStrategy ): ExamplesBase(featureStrategy) { - @Parameters(index = "0", description = ["Contract file path"], arity = "0..1") + @Option(names = ["--contract-file"], description = ["Contract file path"], required = true) public override var contractFile: File? = null @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) @@ -17,7 +17,7 @@ abstract class ExamplesGenerateBase( override fun execute(contract: File?): Int { if (contract == null) { - logger.log("No contract file provided. Use a subcommand or provide a contract file. Use --help for more details.") + consoleLog("No contract file provided. Use a subcommand or provide a contract file. Use --help for more details.") return 1 } @@ -25,11 +25,10 @@ abstract class ExamplesGenerateBase( updateDictionaryFile(dictFile) val examplesDir = getExamplesDirectory(contract) val result = generateExamples(contract, examplesDir) - logGenerationResult(result, examplesDir) - return 0 + return result.logGenerationResult(examplesDir).getExitCode() } catch (e: Throwable) { - logger.log("Example generation failed with error: ${e.message}") - logger.debug(e) + consoleLog("Example generation failed with error: ${e.message}") + consoleDebug(e) return 1 } } @@ -57,8 +56,8 @@ abstract class ExamplesGenerateBase( } // HELPER METHODS - private fun logGenerationResult(generations: List, examplesDir: File) { - val generationGroup = generations.groupBy { it.status }.mapValues { it.value.size } + private fun List.logGenerationResult(examplesDir: File): List { + val generationGroup = this.groupBy { it.status }.mapValues { it.value.size } val createdFileCount = generationGroup[ExampleGenerationStatus.CREATED] ?: 0 val errorCount = generationGroup[ExampleGenerationStatus.ERROR] ?: 0 val existingCount = generationGroup[ExampleGenerationStatus.EXISTS] ?: 0 @@ -69,6 +68,12 @@ abstract class ExamplesGenerateBase( summary = "$createdFileCount example(s) created, $existingCount example(s) already existed, $errorCount example(s) failed", note = "NOTE: All examples can be found in $examplesDirectory" ) + + return this + } + + private fun List.getExitCode(): Int { + return if (this.any { it.status == ExampleGenerationStatus.ERROR }) 1 else 0 } } @@ -95,16 +100,16 @@ interface ExamplesGenerationStrategy { val scenarioDescription = request.scenarioDescription if (existingExample != null) { - logger.log("Using existing example for ${scenarioDescription}\nExample File: ${existingExample.first.absolutePath}") + consoleLog("Using existing example for ${scenarioDescription}\nExample File: ${existingExample.first.absolutePath}") return ExampleGenerationResult(existingExample.first, ExampleGenerationStatus.EXISTS) } - logger.log("Generating example for $scenarioDescription") + consoleLog("Generating example for $scenarioDescription") val (uniqueFileName, exampleContent) = generateExample(request.feature, request.scenario) return writeExampleToFile(exampleContent, uniqueFileName, request.examplesDir, request.validExampleExtensions) } catch (e: Throwable) { - logger.log("Failed to generate example: ${e.message}") - logger.debug(e) + consoleLog("Failed to generate example: ${e.message}") + consoleDebug(e) ExampleGenerationResult(null, ExampleGenerationStatus.ERROR) } } @@ -113,17 +118,17 @@ interface ExamplesGenerationStrategy { val exampleFile = examplesDir.resolve(exampleFileName) if (exampleFile.extension !in validExampleExtensions) { - logger.log("Invalid example file extension: ${exampleFile.extension}") + consoleLog("Invalid example file extension: ${exampleFile.extension}") return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) } try { exampleFile.writeText(exampleContent) - logger.log("Successfully saved example: $exampleFile") + consoleLog("Successfully saved example: $exampleFile") return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.CREATED) } catch (e: Throwable) { - logger.log("Failed to save example: $exampleFile") - logger.debug(e) + consoleLog("Failed to save example: $exampleFile") + consoleDebug(e) return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) } } diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index 3df4b4d63..b9324ef2c 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -14,7 +14,8 @@ import io.ktor.server.routing.* import io.ktor.util.pipeline.* import io.specmatic.core.Result import io.specmatic.core.TestResult -import io.specmatic.core.log.logger +import io.specmatic.core.log.consoleDebug +import io.specmatic.core.log.consoleLog import io.specmatic.core.route.modules.HealthCheckModule.Companion.configureHealthCheckModule import io.specmatic.test.reports.coverage.html.HtmlTemplateConfiguration import picocli.CommandLine.Option @@ -29,31 +30,31 @@ abstract class ExamplesInteractiveBase ( private val validationStrategy: ExamplesValidationStrategy ): ExamplesBase(featureStrategy) { @Option(names = ["--testBaseURL"], description = ["BaseURL of the the system to test"], required = false) - var sutBaseUrl: String? = null + protected var sutBaseUrl: String? = null @Option(names = ["--contract-file"], description = ["Contract file path"], required = false) override var contractFile: File? = null @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) - var dictFile: File? = null + protected var dictFile: File? = null - abstract val htmlTableColumns: List + protected abstract val htmlTableColumns: List private var cachedContractFileFromRequest: File? = null override fun execute(contract: File?): Int { try { if (contract == null) { - logger.log("Contract file not provided, Please provide one via HTTP request") + consoleLog("Contract file not provided, Please provide one via HTTP request") } updateDictionaryFile(dictFile) val server = InteractiveServer(contract, sutBaseUrl, "0.0.0.0", 9001) addShutdownHook(server) - logger.log("Examples Interactive server is running on http://0.0.0.0:9001/_specmatic/examples. Ctrl + C to stop.") + consoleLog("Examples Interactive server is running on http://0.0.0.0:9001/_specmatic/examples. Ctrl + C to stop.") while (true) sleep(10000) } catch (e: Throwable) { - logger.log("Example Interactive server failed with error: ${e.message}") - logger.debug(e) + consoleLog("Example Interactive server failed with error: ${e.message}") + consoleDebug(e) return 1 } } @@ -97,24 +98,24 @@ abstract class ExamplesInteractiveBase ( private fun getContractFileOrNull(contractFile: File?, request: ExamplePageRequest? = null): File? { return contractFile?.takeIf { it.exists() }?.also { contract -> - logger.debug("Using Contract file ${contract.path} provided via command line") + consoleDebug("Using Contract file ${contract.path} provided via command line") } ?: request?.contractFile?.takeIf { it.exists() }?.also { contract -> - logger.debug("Using Contract file ${contract.path} provided via HTTP request") + consoleDebug("Using Contract file ${contract.path} provided via HTTP request") } ?: cachedContractFileFromRequest?.takeIf { it.exists() }?.also { contract -> - logger.debug("Using Contract file ${contract.path} provided via cached HTTP request") + consoleDebug("Using Contract file ${contract.path} provided via cached HTTP request") } } private fun validateRows(tableRows: List): List { tableRows.forEach { row -> require(row.columns.size == htmlTableColumns.size) { - logger.debug("Invalid Row: $row") + consoleDebug("Invalid Row: $row") throw IllegalArgumentException("Incorrect number of columns in table row. Expected: ${htmlTableColumns.size}, Actual: ${row.columns.size}") } row.columns.forEachIndexed { index, it -> require(it.columnName == htmlTableColumns[index].name) { - logger.debug("Invalid Column Row: $row") + consoleDebug("Invalid Column Row: $row") throw IllegalArgumentException("Incorrect column name in table row. Expected: ${htmlTableColumns[index].name}, Actual: ${it.columnName}") } } @@ -158,7 +159,7 @@ abstract class ExamplesInteractiveBase ( install(StatusPages) { exception { call, cause -> - logger.debug(cause) + consoleDebug(cause) call.respondWithError(cause) } } @@ -245,9 +246,7 @@ abstract class ExamplesInteractiveBase ( } private fun getServerHostAndPort(request: ExamplePageRequest? = null): String { - return request?.hostPort.takeIf { - !it.isNullOrEmpty() - } ?: "localhost:$serverPort" + return request?.hostPort.takeIf { !it.isNullOrEmpty() } ?: "localhost:$serverPort" } private fun getHtmlContent(contractFile: File, hostPort: String): String { @@ -291,54 +290,48 @@ abstract class ExamplesInteractiveBase ( private suspend fun PipelineContext.getValidatedContractFileOrNull(request: ExamplePageRequest? = null): File? { val contractFile = getContractFileOrNull(contract, request) ?: run { val errorMessage = "No Contract File Found - Please provide a contract file in the command line or in the HTTP request." - logger.log(errorMessage) + consoleLog(errorMessage) call.respondWithError(HttpStatusCode.BadRequest, errorMessage) return null } - return contractFile.takeIf { it.extension in featureStrategy.contractFileExtensions }?.also { - updateDictionaryFile(dictFile, it) + return getValidatedContractFileOrNull(contractFile)?.let { contract -> + updateDictionaryFile(dictFile, contract) + return contract } ?: run { val errorMessage = "Invalid Contract file ${contractFile.path} - File extension must be one of ${featureStrategy.contractFileExtensions.joinToString()}" - logger.log(errorMessage) call.respondWithError(HttpStatusCode.BadRequest, errorMessage) return null } } private suspend fun PipelineContext.getValidatedExampleOrNull(exampleFile: File): File? { - return when { - !exampleFile.exists() -> { - val errorMessage = "Could not find Example file ${exampleFile.path}" - logger.log(errorMessage) - call.respondWithError(HttpStatusCode.BadRequest, errorMessage) - return null - } - - exampleFile.extension !in featureStrategy.exampleFileExtensions -> { - val errorMessage = "Invalid Example file ${exampleFile.path} - File extension must be one of ${featureStrategy.exampleFileExtensions.joinToString()}" - logger.log(errorMessage) - call.respondWithError(HttpStatusCode.BadRequest, errorMessage) - return null - } + if (!exampleFile.exists()) { + val errorMessage = "Example file does not exist: ${exampleFile.absolutePath}" + call.respondWithError(HttpStatusCode.BadRequest, errorMessage) + return null + } - else -> exampleFile + return getValidatedExampleFileOrNull(exampleFile) ?: run { + val errorMessage = "Invalid Example file ${exampleFile.path} - File extension must be one of ${featureStrategy.exampleFileExtensions.joinToString()}" + call.respondWithError(HttpStatusCode.BadRequest, errorMessage) + return null } } } private fun addShutdownHook(server: InteractiveServer) { Runtime.getRuntime().addShutdownHook(Thread { - logger.log("Shutting down examples interactive server...") + consoleLog("Shutting down examples interactive server...") try { server.close() - logger.log("Server shutdown completed successfully.") + consoleLog("Server shutdown completed successfully.") } catch (e: InterruptedException) { Thread.currentThread().interrupt() - logger.log("Server shutdown interrupted.") + consoleLog("Server shutdown interrupted.") } catch (e: Throwable) { - logger.log("Server shutdown failed with error: ${e.message}") - logger.debug(e) + consoleLog("Server shutdown failed with error: ${e.message}") + consoleDebug(e) } }) } diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt index ea2981ebe..502a19d8d 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt @@ -1,7 +1,8 @@ package application.exampleGeneration import io.specmatic.core.Result -import io.specmatic.core.log.logger +import io.specmatic.core.log.consoleDebug +import io.specmatic.core.log.consoleLog import picocli.CommandLine.Option import java.io.File @@ -20,34 +21,27 @@ abstract class ExamplesValidateBase( override fun execute(contract: File?): Int { if (contract == null) { - logger.log("No contract file provided, please provide a contract file. Use --help for more details.") + consoleLog("No contract file provided, please provide a contract file. Use --help for more details.") return 1 } try { exampleFile?.let { exFile -> - if (!exFile.exists()) { - logger.log("Could not find Example file ${exFile.path}") - return 1 - } - - if (exFile.extension !in featureStrategy.exampleFileExtensions) { - logger.log("Invalid Example file ${exFile.path} - File extension must be one of ${featureStrategy.exampleFileExtensions.joinToString()}") - return 1 - } - - val validation = validateExampleFile(exFile, contract) - return getExitCode(validation) + return validateSingleExampleFile(exFile, contract) } - val inlineResults = if (validateInline) validateInlineExamples(contract) else emptyList() - val externalResults = if (validateExternal) validateExternalExamples(contract) else emptyList() + val inlineResults = if (validateInline) + validateInlineExamples(contract).logValidationResult() + else emptyList() - logValidationResult(inlineResults, externalResults) - return getExitCode(inlineResults + externalResults) + val externalResults = if (validateExternal) + validateExternalExamples(contract).logValidationResult() + else emptyList() + + return externalResults.getExitCode(inlineResults) } catch (e: Throwable) { - logger.log("Validation failed with error: ${e.message}") - logger.debug(e) + consoleLog("Validation failed with error: ${e.message}") + consoleDebug(e) return 1 } } @@ -59,11 +53,14 @@ abstract class ExamplesValidateBase( return validationStrategy.updateFeatureForValidation(feature, filteredScenarios) } - private fun validateExampleFile(exampleFile: File, contractFile: File): ExampleValidationResult { - val feature = featureStrategy.contractFileToFeature(contractFile) - return validateExternalExample(exampleFile, feature).also { - it.logErrors() - } + private fun validateSingleExampleFile(exampleFile: File, contractFile: File): Int { + getValidatedExampleFileOrNull(exampleFile)?.let { + val feature = featureStrategy.contractFileToFeature(contractFile) + return validateExternalExample(exampleFile, feature).let { + it.logErrors() + it.getExitCode() + } + } ?: return 1 } private fun validateExternalExample(exampleFile: File, feature: Feature): ExampleValidationResult { @@ -71,8 +68,8 @@ abstract class ExamplesValidateBase( val result = validationStrategy.validateExternalExample(feature, exampleFile) ExampleValidationResult(exampleFile, result.second) } catch (e: Throwable) { - logger.log("Example validation failed with error: ${e.message}") - logger.debug(e) + consoleLog("Example validation failed with error: ${e.message}") + consoleDebug(e) ExampleValidationResult(exampleFile, Result.Failure(e.message.orEmpty())) } } @@ -101,28 +98,6 @@ abstract class ExamplesValidateBase( } // HELPER METHODS - private fun getExitCode(validations: List): Int { - return if (validations.any { !it.result.isSuccess() }) 1 else 0 - } - - private fun getExitCode(validation: ExampleValidationResult): Int { - return if (!validation.result.isSuccess()) 1 else 0 - } - - private fun logValidationResult(inlineResults: List, externalResults: List) { - if (inlineResults.isNotEmpty()) { - val successCount = inlineResults.count { it.result.isSuccess() } - val failureCount = inlineResults.size - successCount - logResultSummary(ExampleType.INLINE, successCount, failureCount) - } - - if (externalResults.isNotEmpty()) { - val successCount = externalResults.count { it.result.isSuccess() } - val failureCount = externalResults.size - successCount - logResultSummary(ExampleType.EXTERNAL, successCount, failureCount) - } - } - private fun logResultSummary(type: ExampleType, successCount: Int, failureCount: Int) { logFormattedOutput( header = "$type Examples Validation Summary", @@ -135,11 +110,33 @@ abstract class ExamplesValidateBase( val prefix = index?.let { "$it. " } ?: "" if (this.result.isSuccess()) { - return logger.log("$prefix${this.exampleFile?.name ?: this.exampleName} is valid") + return consoleLog("$prefix${this.exampleFile?.name ?: this.exampleName} is valid") } - logger.log("\n$prefix${this.exampleFile?.name ?: this.exampleName} has the following validation error(s):") - logger.log(this.result.reportString()) + consoleLog("\n$prefix${this.exampleFile?.name ?: this.exampleName} has the following validation error(s):") + consoleLog(this.result.reportString()) + } + + private fun List.logValidationResult(): List { + if (this.isNotEmpty()) { + require(this.all { it.type == this.first().type }) { + "All example validation results must be of the same type" + } + + val successCount = this.count { it.result.isSuccess() } + val failureCount = this.size - successCount + logResultSummary(this.first().type, successCount, failureCount) + } + + return this + } + + private fun List.getExitCode(other: List): Int { + return if (this.any { !it.result.isSuccess() } || other.any { !it.result.isSuccess() }) 1 else 0 + } + + private fun ExampleValidationResult.getExitCode(): Int { + return if (!this.result.isSuccess()) 1 else 0 } } diff --git a/core/src/main/kotlin/io/specmatic/core/log/Logging.kt b/core/src/main/kotlin/io/specmatic/core/log/Logging.kt index 86080f349..b88872340 100644 --- a/core/src/main/kotlin/io/specmatic/core/log/Logging.kt +++ b/core/src/main/kotlin/io/specmatic/core/log/Logging.kt @@ -35,6 +35,25 @@ fun consoleLog(e: Throwable, msg: String) { logger.log(e, msg) } +fun consoleDebug(event: String) { + consoleDebug(StringLog(event)) +} + +fun consoleDebug(event: LogMessage) { + LogTail.append(event) + logger.debug(event) +} + +fun consoleDebug(e: Throwable) { + LogTail.append(logger.ofTheException(e)) + logger.debug(e) +} + +fun consoleDebug(e: Throwable, msg: String) { + LogTail.append(logger.ofTheException(e, msg)) + logger.debug(e, msg) +} + val dontPrintToConsole = { event: LogMessage -> LogTail.append(event) } From 5da5e5124c19da0db1107b2483e76d8021666a84 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Fri, 11 Oct 2024 19:56:53 +0530 Subject: [PATCH 32/43] Major Refactoring, moved InteractiveSever to core - Code cleanup, moved thymeleaf to core completely, junit calls to core for html report - Move examples Dataclasses to core. - Remove unneeded Ktor deps from application. --- application/build.gradle | 8 +- .../exampleGeneration/ExamplesBase.kt | 44 ++- .../exampleGeneration/ExamplesGenerateBase.kt | 16 +- .../ExamplesInteractiveBase.kt | 343 +----------------- .../exampleGeneration/ExamplesValidateBase.kt | 19 +- .../OpenApiExamplesGenerate.kt | 2 +- .../OpenApiExamplesInteractive.kt | 20 +- core/build.gradle | 1 + .../examples/ExampleGenerationResult.kt | 15 + .../specmatic/examples/ExampleTestResult.kt | 15 + .../examples/ExampleValidationResult.kt | 17 + .../examples/ExamplesInteractiveServer.kt | 240 ++++++++++++ .../examples/InteractiveServerProvider.kt | 107 ++++++ .../templates}/HtmlTemplateConfiguration.kt | 4 +- .../resources/templates/example/index.html | 0 junit5-support/build.gradle | 1 - .../test/reports/coverage/html/HtmlReport.kt | 7 +- 17 files changed, 455 insertions(+), 404 deletions(-) create mode 100644 core/src/main/kotlin/io/specmatic/examples/ExampleGenerationResult.kt create mode 100644 core/src/main/kotlin/io/specmatic/examples/ExampleTestResult.kt create mode 100644 core/src/main/kotlin/io/specmatic/examples/ExampleValidationResult.kt create mode 100644 core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt create mode 100644 core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt rename {junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html => core/src/main/kotlin/io/specmatic/templates}/HtmlTemplateConfiguration.kt (89%) rename {junit5-support => core}/src/main/resources/templates/example/index.html (100%) diff --git a/application/build.gradle b/application/build.gradle index 25946da5c..0cc764276 100644 --- a/application/build.gradle +++ b/application/build.gradle @@ -78,17 +78,11 @@ dependencies { implementation(project(':junit5-support')) implementation "io.ktor:ktor-client-cio:$ktor_version" + implementation "io.ktor:ktor-server-core:$ktor_version" implementation 'io.swagger.parser.v3:swagger-parser:2.1.22' implementation "org.jetbrains.kotlinx:kotlinx-serialization-json-jvm:1.6.3" - implementation "io.ktor:ktor-server-netty:$ktor_version" - implementation "io.ktor:ktor-server-core:$ktor_version" - implementation "io.ktor:ktor-server-cors:$ktor_version" - implementation("io.ktor:ktor-server-content-negotiation:$ktor_version") - implementation("io.ktor:ktor-server-status-pages:$ktor_version") - implementation "io.ktor:ktor-serialization-jackson:$ktor_version" - testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit_version" testImplementation('org.springframework.boot:spring-boot-starter-test:3.3.4') { diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt index 81d4c57bd..b64c1eee2 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt @@ -24,9 +24,9 @@ abstract class ExamplesBase(protected open val featureStrateg override fun call(): Int { configureLogger(verbose) - return contractFile?.let { contract -> - getValidatedContractFileOrNull(contract)?.let { - execute(it) + return contractFile?.let { + ensureValidContractFile(it).first?.let { contract -> + execute(contract) } ?: 1 } ?: execute(contractFile) } @@ -50,32 +50,30 @@ abstract class ExamplesBase(protected open val featureStrateg return getFilteredScenarios(scenarios, scenarioFilter) } - protected fun getValidatedContractFileOrNull(contractFile: File): File? { - if (!contractFile.exists()) { - consoleLog("Contract file does not exist: ${contractFile.absolutePath}") - return null + @Suppress("MemberVisibilityCanBePrivate") // Used By InteractiveServer through ExamplesInteractiveBase + fun ensureValidContractFile(contractFile: File): Pair { + val errorMessage = when { + !contractFile.exists() -> "Contract file does not exist: ${contractFile.absolutePath}" + contractFile.extension !in featureStrategy.contractFileExtensions -> + "Invalid Contract file ${contractFile.path} - File extension must be one of ${featureStrategy.contractFileExtensions.joinToString()}" + else -> return contractFile to null } - if (contractFile.extension !in featureStrategy.contractFileExtensions) { - consoleLog("Invalid Contract file ${contractFile.path} - File extension must be one of ${featureStrategy.contractFileExtensions.joinToString()}") - return null - } - - return contractFile + consoleLog(errorMessage) + return null to errorMessage } - protected fun getValidatedExampleFileOrNull(exampleFile: File): File? { - if (!exampleFile.exists()) { - consoleLog("Example file does not exist: ${exampleFile.absolutePath}") - return null - } - - if (exampleFile.extension !in featureStrategy.exampleFileExtensions) { - consoleLog("Invalid Example file ${exampleFile.path} - File extension must be one of ${featureStrategy.exampleFileExtensions.joinToString()}") - return null + @Suppress("MemberVisibilityCanBePrivate") // Used By InteractiveServer through ExamplesInteractiveBase + fun ensureValidExampleFile(exampleFile: File): Pair { + val errorMessage = when { + !exampleFile.exists() -> "Example file does not exist: ${exampleFile.absolutePath}" + exampleFile.extension !in featureStrategy.exampleFileExtensions -> + "Invalid Example file ${exampleFile.path} - File extension must be one of ${featureStrategy.exampleFileExtensions.joinToString()}" + else -> return exampleFile to null } - return exampleFile + consoleLog(errorMessage) + return null to errorMessage } protected fun getExamplesDirectory(contractFile: File): File { diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt index 83dc02262..d44e50ba8 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt @@ -2,6 +2,8 @@ package application.exampleGeneration import io.specmatic.core.* import io.specmatic.core.log.* +import io.specmatic.examples.ExampleGenerationResult +import io.specmatic.examples.ExampleGenerationStatus import picocli.CommandLine.* import java.io.File @@ -9,7 +11,7 @@ abstract class ExamplesGenerateBase( override val featureStrategy: ExamplesFeatureStrategy, private val generationStrategy: ExamplesGenerationStrategy ): ExamplesBase(featureStrategy) { - @Option(names = ["--contract-file"], description = ["Contract file path"], required = true) + @Parameters(paramLabel = "contractFile", description = ["Path to contract file"], arity = "0..1") public override var contractFile: File? = null @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) @@ -77,18 +79,6 @@ abstract class ExamplesGenerateBase( } } -enum class ExampleGenerationStatus(val value: String) { - CREATED("Inline"), - ERROR("External"), - EXISTS("Exists"); - - override fun toString(): String { - return this.value - } -} - -data class ExampleGenerationResult(val exampleFile: File? = null, val status: ExampleGenerationStatus) - interface ExamplesGenerationStrategy { fun getExistingExampleOrNull(scenario: Scenario, exampleFiles: List): Pair? diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index b9324ef2c..a60013df4 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -1,36 +1,21 @@ package application.exampleGeneration -import io.ktor.http.* -import io.ktor.serialization.jackson.* import io.ktor.server.application.* -import io.ktor.server.engine.* -import io.ktor.server.netty.* -import io.ktor.server.plugins.contentnegotiation.* -import io.ktor.server.plugins.cors.routing.* -import io.ktor.server.plugins.statuspages.* -import io.ktor.server.request.* -import io.ktor.server.response.* -import io.ktor.server.routing.* -import io.ktor.util.pipeline.* import io.specmatic.core.Result -import io.specmatic.core.TestResult import io.specmatic.core.log.consoleDebug import io.specmatic.core.log.consoleLog -import io.specmatic.core.route.modules.HealthCheckModule.Companion.configureHealthCheckModule -import io.specmatic.test.reports.coverage.html.HtmlTemplateConfiguration +import io.specmatic.examples.* import picocli.CommandLine.Option -import java.io.Closeable import java.io.File -import java.io.FileNotFoundException import java.lang.Thread.sleep abstract class ExamplesInteractiveBase ( override val featureStrategy: ExamplesFeatureStrategy, private val generationStrategy: ExamplesGenerationStrategy, private val validationStrategy: ExamplesValidationStrategy -): ExamplesBase(featureStrategy) { +): ExamplesBase(featureStrategy), InteractiveServerProvider { @Option(names = ["--testBaseURL"], description = ["BaseURL of the the system to test"], required = false) - protected var sutBaseUrl: String? = null + override var sutBaseUrl: String? = null @Option(names = ["--contract-file"], description = ["Contract file path"], required = false) override var contractFile: File? = null @@ -38,8 +23,9 @@ abstract class ExamplesInteractiveBase ( @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) protected var dictFile: File? = null - protected abstract val htmlTableColumns: List - private var cachedContractFileFromRequest: File? = null + override val serverHost: String = "0.0.0.0" + override val serverPort: Int = 9001 + abstract val server: ExamplesInteractiveServer override fun execute(contract: File?): Int { try { @@ -48,7 +34,7 @@ abstract class ExamplesInteractiveBase ( } updateDictionaryFile(dictFile) - val server = InteractiveServer(contract, sutBaseUrl, "0.0.0.0", 9001) + server.start() addShutdownHook(server) consoleLog("Examples Interactive server is running on http://0.0.0.0:9001/_specmatic/examples. Ctrl + C to stop.") while (true) sleep(10000) @@ -60,14 +46,14 @@ abstract class ExamplesInteractiveBase ( } // HOOKS - abstract fun createTableRows(scenarioExamplePair: List>): List + abstract fun createTableRows(scenarioExamplePair: List>): List abstract suspend fun getScenarioFromRequestOrNull(call: ApplicationCall, feature: Feature): Scenario? abstract fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair // HELPER METHODS - private suspend fun generateExample(call: ApplicationCall, contractFile: File): ExampleGenerationResult { + override suspend fun generateExample(call: ApplicationCall, contractFile: File): ExampleGenerationResult { val feature = featureStrategy.contractFileToFeature(contractFile) val examplesDir = getExamplesDirectory(contractFile) val exampleFiles = getExternalExampleFiles(examplesDir) @@ -84,47 +70,19 @@ abstract class ExamplesInteractiveBase ( } ?: throw IllegalArgumentException("No matching scenario found for request") } - private fun validateExample(contractFile: File, exampleFile: File): ExampleValidationResult { + override fun validateExample(contractFile: File, exampleFile: File): ExampleValidationResult { val feature = featureStrategy.contractFileToFeature(contractFile) val result = validationStrategy.validateExternalExample(feature, exampleFile) return ExampleValidationResult(exampleFile.absolutePath, result.second, ExampleType.EXTERNAL) } - private fun testExample(contractFile: File, exampleFile: File, sutBaseUrl: String): ExampleTestResult { + override fun testExample(contractFile: File, exampleFile: File, sutBaseUrl: String): ExampleTestResult { val feature = featureStrategy.contractFileToFeature(contractFile) val result = testExternalExample(feature, exampleFile, sutBaseUrl) return ExampleTestResult(result.first, result.second, exampleFile) } - private fun getContractFileOrNull(contractFile: File?, request: ExamplePageRequest? = null): File? { - return contractFile?.takeIf { it.exists() }?.also { contract -> - consoleDebug("Using Contract file ${contract.path} provided via command line") - } ?: request?.contractFile?.takeIf { it.exists() }?.also { contract -> - consoleDebug("Using Contract file ${contract.path} provided via HTTP request") - } ?: cachedContractFileFromRequest?.takeIf { it.exists() }?.also { contract -> - consoleDebug("Using Contract file ${contract.path} provided via cached HTTP request") - } - } - - private fun validateRows(tableRows: List): List { - tableRows.forEach { row -> - require(row.columns.size == htmlTableColumns.size) { - consoleDebug("Invalid Row: $row") - throw IllegalArgumentException("Incorrect number of columns in table row. Expected: ${htmlTableColumns.size}, Actual: ${row.columns.size}") - } - - row.columns.forEachIndexed { index, it -> - require(it.columnName == htmlTableColumns[index].name) { - consoleDebug("Invalid Column Row: $row") - throw IllegalArgumentException("Incorrect column name in table row. Expected: ${htmlTableColumns[index].name}, Actual: ${it.columnName}") - } - } - } - - return tableRows - } - - private fun getTableRows(contractFile: File): List { + override fun getTableRows(contractFile: File): List { val feature = featureStrategy.contractFileToFeature(contractFile) val scenarios = getFilteredScenarios(feature) val examplesDir = getExamplesDirectory(contractFile) @@ -135,192 +93,10 @@ abstract class ExamplesInteractiveBase ( ExampleValidationResult(exRes.first, exRes.second) } } - val tableRows = createTableRows(scenarioExamplePair) - - return validateRows(tableRows) + return createTableRows(scenarioExamplePair) } - // INTERACTIVE SERVER - inner class InteractiveServer(private var contract: File?, sutBaseUrl: String?, private val serverHost: String, private val serverPort: Int) : Closeable { - private val environment = applicationEngineEnvironment { - module { - install(CORS) { - allowMethod(HttpMethod.Options) - allowMethod(HttpMethod.Post) - allowMethod(HttpMethod.Get) - allowHeader(HttpHeaders.AccessControlAllowOrigin) - allowHeader(HttpHeaders.ContentType) - anyHost() - } - - install(ContentNegotiation) { - jackson {} - } - - install(StatusPages) { - exception { call, cause -> - consoleDebug(cause) - call.respondWithError(cause) - } - } - - configureHealthCheckModule() - - routing { - get("/_specmatic/examples") { - getValidatedContractFileOrNull()?.let { contract -> - val hostPort = getServerHostAndPort() - val htmlContent = getHtmlContent(contract, hostPort) - call.respondText(htmlContent, contentType = ContentType.Text.Html) - } - } - - post("/_specmatic/examples") { - val request = call.receive() - getValidatedContractFileOrNull(request)?.let { contract -> - val hostPort = getServerHostAndPort() - val htmlContent = getHtmlContent(contract, hostPort) - call.respondText(htmlContent, contentType = ContentType.Text.Html) - } - } - - post("/_specmatic/examples/generate") { - getValidatedContractFileOrNull()?.let { contractFile -> - val result = generateExample(call, contractFile) - call.respond(HttpStatusCode.OK, GenerateExampleResponse(result)) - } - } - - post("/_specmatic/examples/validate") { - val request = call.receive() - getValidatedContractFileOrNull()?.let { contract -> - getValidatedExampleOrNull(request.exampleFile)?.let { example -> - val result = validateExample(contract, example) - call.respond(HttpStatusCode.OK, ExampleValidationResponse(result)) - } - } - } - - post("/_specmatic/examples/content") { - val request = call.receive() - getValidatedExampleOrNull(request.exampleFile)?.let { example -> - call.respond(HttpStatusCode.OK, mapOf("content" to example.readText())) - } - } - - post("/_specmatic/examples/test") { - if (sutBaseUrl.isNullOrBlank()) { - return@post call.respondWithError(HttpStatusCode.BadRequest, "No SUT URL provided") - } - - val request = call.receive() - getValidatedContractFileOrNull()?.let { contract -> - getValidatedExampleOrNull(request.exampleFile)?.let { example -> - val result = testExample(contract, example, sutBaseUrl) - call.respond(HttpStatusCode.OK, ExampleTestResponse(result)) - } - } - } - } - } - - connector { - this.host = serverHost - this.port = serverPort - } - } - - private val server: ApplicationEngine = embeddedServer(Netty, environment, configure = { - this.requestQueueLimit = 1000 - this.callGroupSize = 5 - this.connectionGroupSize = 20 - this.workerGroupSize = 20 - }) - - init { - server.start() - } - - override fun close() { - server.stop(0, 0) - } - - private fun getServerHostAndPort(request: ExamplePageRequest? = null): String { - return request?.hostPort.takeIf { !it.isNullOrEmpty() } ?: "localhost:$serverPort" - } - - private fun getHtmlContent(contractFile: File, hostPort: String): String { - val tableRows = getTableRows(contractFile) - val htmlContent = renderTemplate(contractFile, hostPort, tableRows) - return htmlContent - } - - private fun renderTemplate(contractFile: File, hostPort: String, tableRows: List): String { - val variables = mapOf( - "tableColumns" to htmlTableColumns, - "tableRows" to tableRows, - "contractFileName" to contractFile.name, - "contractFilePath" to contractFile.absolutePath, - "hostPort" to hostPort, - "hasExamples" to tableRows.any { it.exampleFilePath != null }, - "validationDetails" to tableRows.mapIndexed { index, row -> - (index + 1) to row.exampleMismatchReason - }.toMap(), - "isTestMode" to (sutBaseUrl != null) - ) - - return HtmlTemplateConfiguration.process( - templateName = "example/index.html", - variables = variables - ) - } - - private suspend fun ApplicationCall.respondWithError(httpStatusCode: HttpStatusCode, errorMessage: String) { - this.respond(httpStatusCode, mapOf("error" to errorMessage)) - } - - private suspend fun ApplicationCall.respondWithError(throwable: Throwable, errorMessage: String? = throwable.message) { - val statusCode = when (throwable) { - is IllegalArgumentException, is FileNotFoundException -> HttpStatusCode.BadRequest - else -> HttpStatusCode.InternalServerError - } - this.respondWithError(statusCode, errorMessage ?: throwable.message ?: "Something went wrong") - } - - private suspend fun PipelineContext.getValidatedContractFileOrNull(request: ExamplePageRequest? = null): File? { - val contractFile = getContractFileOrNull(contract, request) ?: run { - val errorMessage = "No Contract File Found - Please provide a contract file in the command line or in the HTTP request." - consoleLog(errorMessage) - call.respondWithError(HttpStatusCode.BadRequest, errorMessage) - return null - } - - return getValidatedContractFileOrNull(contractFile)?.let { contract -> - updateDictionaryFile(dictFile, contract) - return contract - } ?: run { - val errorMessage = "Invalid Contract file ${contractFile.path} - File extension must be one of ${featureStrategy.contractFileExtensions.joinToString()}" - call.respondWithError(HttpStatusCode.BadRequest, errorMessage) - return null - } - } - - private suspend fun PipelineContext.getValidatedExampleOrNull(exampleFile: File): File? { - if (!exampleFile.exists()) { - val errorMessage = "Example file does not exist: ${exampleFile.absolutePath}" - call.respondWithError(HttpStatusCode.BadRequest, errorMessage) - return null - } - - return getValidatedExampleFileOrNull(exampleFile) ?: run { - val errorMessage = "Invalid Example file ${exampleFile.path} - File extension must be one of ${featureStrategy.exampleFileExtensions.joinToString()}" - call.respondWithError(HttpStatusCode.BadRequest, errorMessage) - return null - } - } - } - - private fun addShutdownHook(server: InteractiveServer) { + private fun addShutdownHook(server: ExamplesInteractiveServer) { Runtime.getRuntime().addShutdownHook(Thread { consoleLog("Shutting down examples interactive server...") try { @@ -335,95 +111,4 @@ abstract class ExamplesInteractiveBase ( } }) } - - data class ExampleTestResult(val result: TestResult, val testLog: String, val exampleFile: File) { - constructor(result: Result, testLog: String, exampleFile: File): this( - result.testResult(), - result.takeIf { !it.isSuccess() }?.let { - "${it.reportString()}\n\n$testLog" - } ?: testLog, - exampleFile - ) - } -} - -data class ExamplePageRequest ( - val contractFile: File, - val hostPort: String? -) - -data class GenerateExampleResponse ( - val exampleFilePath: String, - val status: String -) { - constructor(result: ExampleGenerationResult): this( - exampleFilePath = result.exampleFile?.absolutePath ?: throw Exception("Failed to generate example file"), - status = result.status.toString() - ) -} - -data class ExampleValidationRequest ( - val exampleFile: File -) - -data class ExampleValidationResponse ( - val exampleFilePath: String, - val error: String? = null -) { - constructor(result: ExampleValidationResult): this( - exampleFilePath = result.exampleName, error = result.result.reportString().takeIf { it.isNotBlank() } - ) } - -data class ExampleContentRequest ( - val exampleFile: File -) - -data class ExampleTestRequest ( - val exampleFile: File -) - -data class ExampleTestResponse ( - val result: TestResult, - val details: String, - val testLog: String -) { - constructor(result: ExamplesInteractiveBase.ExampleTestResult): this ( - result = result.result, - details = resultToDetails(result.result, result.exampleFile), - testLog = result.testLog.trim('-', ' ', '\n', '\r') - ) - - companion object { - fun resultToDetails(result: TestResult, exampleFile: File): String { - val postFix = when(result) { - TestResult.Success -> "has SUCCEEDED" - TestResult.Error -> "has ERROR" - else -> "has FAILED" - } - - return "Example test for ${exampleFile.nameWithoutExtension} $postFix" - } - } -} - -data class HtmlTableColumn ( - val name: String, - val colSpan: Int -) - -data class TableRowGroup ( - val columnName: String, - val value: String, - val rowSpan: Int = 1, - val showRow: Boolean = true, - val rawValue: String = value, - val extraInfo: String? = null -) - -data class TableRow ( - val columns: List, - val exampleFilePath: String? = null, - val exampleFileName: String? = null, - val exampleMismatchReason: String? = null -) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt index 502a19d8d..d2ef2f55b 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt @@ -3,6 +3,8 @@ package application.exampleGeneration import io.specmatic.core.Result import io.specmatic.core.log.consoleDebug import io.specmatic.core.log.consoleLog +import io.specmatic.examples.ExampleType +import io.specmatic.examples.ExampleValidationResult import picocli.CommandLine.Option import java.io.File @@ -54,9 +56,9 @@ abstract class ExamplesValidateBase( } private fun validateSingleExampleFile(exampleFile: File, contractFile: File): Int { - getValidatedExampleFileOrNull(exampleFile)?.let { + ensureValidExampleFile(exampleFile).first?.let { exFile -> val feature = featureStrategy.contractFileToFeature(contractFile) - return validateExternalExample(exampleFile, feature).let { + return validateExternalExample(exFile, feature).let { it.logErrors() it.getExitCode() } @@ -140,19 +142,6 @@ abstract class ExamplesValidateBase( } } -enum class ExampleType(val value: String) { - INLINE("Inline"), - EXTERNAL("External"); - - override fun toString(): String { - return this.value - } -} - -data class ExampleValidationResult(val exampleName: String, val result: Result, val type: ExampleType, val exampleFile: File? = null) { - constructor(exampleFile: File, result: Result) : this(exampleFile.nameWithoutExtension, result, ExampleType.EXTERNAL, exampleFile) -} - interface ExamplesValidationStrategy { fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt index 932efd619..d3e1a30cd 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt @@ -11,7 +11,7 @@ import picocli.CommandLine.Option import java.io.File @Command( - name = "examples", + name = "examples", mixinStandardHelpOptions = true, description = ["Generate Externalised Examples from an OpenApi Contract"], subcommands = [OpenApiExamplesValidate::class, OpenApiExamplesInteractive::class] ) diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index 61f9fe1a2..b1bb90f94 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -5,6 +5,7 @@ import io.ktor.server.application.* import io.ktor.server.request.* import io.specmatic.conversions.convertPathParameterStyle import io.specmatic.core.* +import io.specmatic.examples.* import io.specmatic.test.TestInteractionsLog import io.specmatic.test.TestInteractionsLog.combineLog import picocli.CommandLine.Command @@ -18,10 +19,11 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( @Option(names = ["--extensive"], description = ["Display all responses, not just 2xx, in the table."], defaultValue = "false") override var extensive: Boolean = false - override val htmlTableColumns: List = listOf ( - HtmlTableColumn(name = "path", colSpan = 2), - HtmlTableColumn(name = "method", colSpan = 1), - HtmlTableColumn(name = "response", colSpan = 1) + override val server: ExamplesInteractiveServer = ExamplesInteractiveServer(this) + override val exampleTableColumns: List = listOf ( + ExampleTableColumn(name = "path", colSpan = 2), + ExampleTableColumn(name = "method", colSpan = 1), + ExampleTableColumn(name = "response", colSpan = 1) ) override fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair { @@ -43,7 +45,7 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( } } - override fun createTableRows(scenarioExamplePair: List>): List { + override fun createTableRows(scenarioExamplePair: List>): List { val groupedScenarios = scenarioExamplePair.sortScenarios().groupScenarios() return groupedScenarios.flatMap { (_, methodMap) -> @@ -53,11 +55,11 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( methodMap.flatMap { (method, scenarios) -> scenarios.map { (scenario, example) -> - TableRow( + ExampleTableRow( columns = listOf( - TableRowGroup("path", convertPathParameterStyle(scenario.path), rawValue = scenario.path, rowSpan = pathSpan, showRow = showPath), - TableRowGroup("method", scenario.method, showRow = !methodSet.contains(method), rowSpan = scenarios.size), - TableRowGroup("response", scenario.status.toString(), showRow = true, rowSpan = 1, extraInfo = scenario.httpRequestPattern.headersPattern.contentType) + ExampleRowGroup("path", convertPathParameterStyle(scenario.path), rawValue = scenario.path, rowSpan = pathSpan, showRow = showPath), + ExampleRowGroup("method", scenario.method, showRow = !methodSet.contains(method), rowSpan = scenarios.size), + ExampleRowGroup("response", scenario.status.toString(), showRow = true, rowSpan = 1, extraInfo = scenario.httpRequestPattern.headersPattern.contentType) ), exampleFilePath = example?.exampleFile?.absolutePath, exampleFileName = example?.exampleName, diff --git a/core/build.gradle b/core/build.gradle index bf323a901..d122291b4 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -36,6 +36,7 @@ dependencies { implementation("io.ktor:ktor-server-double-receive:$ktor_version") implementation("io.ktor:ktor-server-content-negotiation:$ktor_version") implementation "io.ktor:ktor-serialization-jackson:$ktor_version" + implementation("io.ktor:ktor-server-status-pages:$ktor_version") implementation 'com.fasterxml.jackson.module:jackson-module-kotlin:2.18.0' diff --git a/core/src/main/kotlin/io/specmatic/examples/ExampleGenerationResult.kt b/core/src/main/kotlin/io/specmatic/examples/ExampleGenerationResult.kt new file mode 100644 index 000000000..e76f793dd --- /dev/null +++ b/core/src/main/kotlin/io/specmatic/examples/ExampleGenerationResult.kt @@ -0,0 +1,15 @@ +package io.specmatic.examples + +import java.io.File + +enum class ExampleGenerationStatus(val value: String) { + CREATED("Inline"), + ERROR("External"), + EXISTS("Exists"); + + override fun toString(): String { + return this.value + } +} + +data class ExampleGenerationResult(val exampleFile: File? = null, val status: ExampleGenerationStatus) \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/examples/ExampleTestResult.kt b/core/src/main/kotlin/io/specmatic/examples/ExampleTestResult.kt new file mode 100644 index 000000000..2db7145f9 --- /dev/null +++ b/core/src/main/kotlin/io/specmatic/examples/ExampleTestResult.kt @@ -0,0 +1,15 @@ +package io.specmatic.examples + +import io.specmatic.core.Result +import io.specmatic.core.TestResult +import java.io.File + +data class ExampleTestResult(val result: TestResult, val testLog: String, val exampleFile: File) { + constructor(result: Result, testLog: String, exampleFile: File): this( + result.testResult(), + result.takeIf { !it.isSuccess() }?.let { + "${it.reportString()}\n\n$testLog" + } ?: testLog, + exampleFile + ) +} diff --git a/core/src/main/kotlin/io/specmatic/examples/ExampleValidationResult.kt b/core/src/main/kotlin/io/specmatic/examples/ExampleValidationResult.kt new file mode 100644 index 000000000..74684ebcd --- /dev/null +++ b/core/src/main/kotlin/io/specmatic/examples/ExampleValidationResult.kt @@ -0,0 +1,17 @@ +package io.specmatic.examples + +import io.specmatic.core.Result +import java.io.File + +enum class ExampleType(val value: String) { + INLINE("Inline"), + EXTERNAL("External"); + + override fun toString(): String { + return this.value + } +} + +data class ExampleValidationResult(val exampleName: String, val result: Result, val type: ExampleType, val exampleFile: File? = null) { + constructor(exampleFile: File, result: Result) : this(exampleFile.nameWithoutExtension, result, ExampleType.EXTERNAL, exampleFile) +} diff --git a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt new file mode 100644 index 000000000..7e02c1612 --- /dev/null +++ b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt @@ -0,0 +1,240 @@ +package io.specmatic.examples + +import io.ktor.http.* +import io.ktor.serialization.jackson.* +import io.ktor.server.application.* +import io.ktor.server.engine.* +import io.ktor.server.netty.* +import io.ktor.server.plugins.contentnegotiation.* +import io.ktor.server.plugins.cors.routing.* +import io.ktor.server.plugins.statuspages.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* +import io.ktor.util.pipeline.* +import io.specmatic.core.log.consoleDebug +import io.specmatic.core.log.consoleLog +import io.specmatic.core.route.modules.HealthCheckModule.Companion.configureHealthCheckModule +import io.specmatic.templates.HtmlTemplateConfiguration +import java.io.File +import java.io.FileNotFoundException + +class ExamplesInteractiveServer(provider: InteractiveServerProvider) : InteractiveServerProvider by provider { + private var cachedContractFile: File? = null + + private val environment = applicationEngineEnvironment { + module { + install(CORS) { + allowMethod(HttpMethod.Options) + allowMethod(HttpMethod.Post) + allowMethod(HttpMethod.Get) + allowHeader(HttpHeaders.AccessControlAllowOrigin) + allowHeader(HttpHeaders.ContentType) + anyHost() + } + + install(ContentNegotiation) { + jackson {} + } + + install(StatusPages) { + exception { call, cause -> + consoleDebug(cause) + call.respondWithError(cause) + } + } + + configureHealthCheckModule() + + routing { + getHtmlPageRoute() + generateExampleRoute() + validateExampleRoute() + getExampleContentRoute() + testExampleRoute() + } + } + + connector { + this.host = serverHost + this.port = serverPort + } + } + + private val server: ApplicationEngine = embeddedServer(Netty, environment) { + requestQueueLimit = 1000 + callGroupSize = 5 + connectionGroupSize = 20 + workerGroupSize = 20 + } + + fun start() { + server.start() + } + + fun close() { + server.stop(0, 0) + } + + private fun getServerHostAndPort(request: ExamplePageRequest? = null): String { + return request?.hostPort?.takeIf { it.isNotEmpty() } ?: "localhost:$serverPort" + } + + private fun getContractFileOrNull(request: ExamplePageRequest? = null): Pair { + val contractFile = contractFile?.also { + logContractFileUsage("command line", it) + } ?: request?.contractFile?.takeIf { it.exists() }?.also { + logContractFileUsage("HTTP request", it) + } ?: cachedContractFile?.takeIf { it.exists() }?.also { + logContractFileUsage("cached HTTP request", it) + } + + return contractFile?.let { it to null } ?: (null to "Contract file not provided, Please provide one via HTTP request") + } + + private fun logContractFileUsage(source: String, contract: File) { + consoleDebug("Using Contract file ${contract.path} provided via $source") + } + + private fun getHtmlContent(contractFile: File, hostPort: String): String { + val tableRows = validateRows(getTableRows(contractFile)) + return renderTemplate(contractFile, hostPort, tableRows) + } + + private fun validateRows(tableRows: List): List { + tableRows.forEach { row -> + require(row.columns.size == exampleTableColumns.size) { + consoleDebug("Invalid Row: $row") + throw IllegalArgumentException("Incorrect number of columns in table row. Expected: ${exampleTableColumns.size}, Actual: ${row.columns.size}") + } + + row.columns.forEachIndexed { index, it -> + require(it.columnName == exampleTableColumns[index].name) { + consoleDebug("Invalid Column Row: $row") + throw IllegalArgumentException("Incorrect column name in table row. Expected: ${exampleTableColumns[index].name}, Actual: ${it.columnName}") + } + } + } + + return tableRows + } + + private fun renderTemplate(contractFile: File, hostPort: String, tableRows: List): String { + val variables = mapOf( + "tableColumns" to exampleTableColumns, + "tableRows" to tableRows, + "contractFileName" to contractFile.name, + "contractFilePath" to contractFile.absolutePath, + "hostPort" to hostPort, + "hasExamples" to tableRows.any { it.exampleFilePath != null }, + "validationDetails" to tableRows.mapIndexed { index, row -> index.inc() to row.exampleMismatchReason }.toMap(), + "isTestMode" to (sutBaseUrl != null) + ) + + return HtmlTemplateConfiguration.process("example/index.html", variables) + } + + private suspend fun ApplicationCall.respondWithError(httpStatusCode: HttpStatusCode, errorMessage: String) { + respond(httpStatusCode, mapOf("error" to errorMessage)) + } + + private suspend fun ApplicationCall.respondWithError(throwable: Throwable, errorMessage: String? = throwable.message) { + val statusCode = when (throwable) { + is IllegalArgumentException, is FileNotFoundException -> HttpStatusCode.BadRequest + else -> HttpStatusCode.InternalServerError + } + respondWithError(statusCode, errorMessage ?: "Something went wrong") + } + + private suspend fun PipelineContext.getValidatedContractFileOrNull(request: ExamplePageRequest? = null): File? { + val (contractFile, nullErrorMessage) = getContractFileOrNull(request) + nullErrorMessage?.let { + consoleLog(nullErrorMessage) + call.respondWithError(HttpStatusCode.BadRequest, it) + return null + } + + requireNotNull(contractFile) { "Contract file should not be null if there's no error message" } + val (contract, errorMessage) = ensureValidContractFile(contractFile) + errorMessage?.let { + call.respondWithError(HttpStatusCode.BadRequest, errorMessage) + return null + } + + return requireNotNull(contract) { "Contract file should not be null if there's no error message" } + } + + private suspend fun PipelineContext.getValidatedExampleOrNull(exampleFile: File): File? { + val (validatedExample, errorMessage) = ensureValidExampleFile(exampleFile) + + errorMessage?.let { + call.respondWithError(HttpStatusCode.BadRequest, errorMessage) + return null + } + + return requireNotNull(validatedExample) { "Example file should not be null if there's no error message" } + } + + private fun Routing.getHtmlPageRoute() { + get("/_specmatic/examples") { + getValidatedContractFileOrNull()?.let { contract -> + val htmlContent = getHtmlContent(contract, getServerHostAndPort()) + call.respondText(htmlContent, contentType = ContentType.Text.Html) + } + } + + post("/_specmatic/examples") { + val request = call.receive() + getValidatedContractFileOrNull(request)?.let { contract -> + val htmlContent = getHtmlContent(contract, getServerHostAndPort(request)) + call.respondText(htmlContent, contentType = ContentType.Text.Html) + } + } + } + + private fun Routing.generateExampleRoute() { + post("/_specmatic/examples/generate") { + getValidatedContractFileOrNull()?.let { contractFile -> + val result = generateExample(call, contractFile) + call.respond(HttpStatusCode.OK, ExampleGenerationResponse(result)) + } + } + } + + private fun Routing.validateExampleRoute() { + post("/_specmatic/examples/validate") { + val request = call.receive() + getValidatedContractFileOrNull()?.let { contract -> + getValidatedExampleOrNull(request.exampleFile)?.let { example -> + val result = validateExample(contract, example) + call.respond(HttpStatusCode.OK, ExampleValidationResponse(result)) + } + } + } + } + + private fun Routing.getExampleContentRoute() { + post("/_specmatic/examples/content") { + val request = call.receive() + getValidatedExampleOrNull(request.exampleFile)?.let { example -> + call.respond(HttpStatusCode.OK, mapOf("content" to example.readText())) + } + } + } + + private fun Routing.testExampleRoute() { + post("/_specmatic/examples/test") { + if (sutBaseUrl.isNullOrBlank()) { + return@post call.respondWithError(HttpStatusCode.BadRequest, "No SUT URL provided") + } + + val request = call.receive() + getValidatedContractFileOrNull()?.let { contract -> + getValidatedExampleOrNull(request.exampleFile)?.let { example -> + val result = testExample(contract, example, sutBaseUrl) + call.respond(HttpStatusCode.OK, ExampleTestResponse(result)) + } + } + } + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt b/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt new file mode 100644 index 000000000..2211a7c65 --- /dev/null +++ b/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt @@ -0,0 +1,107 @@ +package io.specmatic.examples + +import io.ktor.server.application.* +import io.specmatic.core.TestResult +import java.io.File + +interface InteractiveServerProvider { + val serverHost: String + val serverPort: Int + val sutBaseUrl: String? + + val contractFile: File? + val exampleTableColumns: List + + suspend fun generateExample(call: ApplicationCall, contractFile: File): ExampleGenerationResult + + fun validateExample(contractFile: File, exampleFile: File): ExampleValidationResult + + fun testExample(contractFile: File, exampleFile: File, sutBaseUrl: String): ExampleTestResult + + fun getTableRows(contractFile: File): List + + fun ensureValidContractFile(contractFile: File): Pair + + fun ensureValidExampleFile(exampleFile: File): Pair +} + +data class ExamplePageRequest ( + val contractFile: File, + val hostPort: String? +) + +data class ExampleGenerationResponse ( + val exampleFilePath: String, + val status: String +) { + constructor(result: ExampleGenerationResult): this( + exampleFilePath = result.exampleFile?.absolutePath ?: throw Exception("Failed to generate example file"), + status = result.status.toString() + ) +} + +data class ExampleValidationRequest ( + val exampleFile: File +) + +data class ExampleValidationResponse ( + val exampleFilePath: String, + val error: String? = null +) { + constructor(result: ExampleValidationResult): this( + exampleFilePath = result.exampleName, error = result.result.reportString().takeIf { it.isNotBlank() } + ) +} + +data class ExampleContentRequest ( + val exampleFile: File +) + +data class ExampleTestRequest ( + val exampleFile: File +) + +data class ExampleTestResponse ( + val result: TestResult, + val details: String, + val testLog: String +) { + constructor(result: ExampleTestResult): this ( + result = result.result, + details = resultToDetails(result.result, result.exampleFile), + testLog = result.testLog.trim('-', ' ', '\n', '\r') + ) + + companion object { + fun resultToDetails(result: TestResult, exampleFile: File): String { + val postFix = when(result) { + TestResult.Success -> "has SUCCEEDED" + TestResult.Error -> "has ERROR" + else -> "has FAILED" + } + + return "Example test for ${exampleFile.nameWithoutExtension} $postFix" + } + } +} + +data class ExampleTableColumn ( + val name: String, + val colSpan: Int +) + +data class ExampleRowGroup ( + val columnName: String, + val value: String, + val rowSpan: Int = 1, + val showRow: Boolean = true, + val rawValue: String = value, + val extraInfo: String? = null +) + +data class ExampleTableRow ( + val columns: List, + val exampleFilePath: String? = null, + val exampleFileName: String? = null, + val exampleMismatchReason: String? = null +) \ No newline at end of file diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlTemplateConfiguration.kt b/core/src/main/kotlin/io/specmatic/templates/HtmlTemplateConfiguration.kt similarity index 89% rename from junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlTemplateConfiguration.kt rename to core/src/main/kotlin/io/specmatic/templates/HtmlTemplateConfiguration.kt index 55a2e499d..8c1a41d9e 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlTemplateConfiguration.kt +++ b/core/src/main/kotlin/io/specmatic/templates/HtmlTemplateConfiguration.kt @@ -1,4 +1,4 @@ -package io.specmatic.test.reports.coverage.html +package io.specmatic.templates import org.thymeleaf.TemplateEngine import org.thymeleaf.context.Context import org.thymeleaf.templatemode.TemplateMode @@ -6,7 +6,7 @@ import org.thymeleaf.templateresolver.ClassLoaderTemplateResolver class HtmlTemplateConfiguration { companion object { - fun configureTemplateEngine(): TemplateEngine { + private fun configureTemplateEngine(): TemplateEngine { val templateResolver = ClassLoaderTemplateResolver().apply { prefix = "templates/" suffix = ".html" diff --git a/junit5-support/src/main/resources/templates/example/index.html b/core/src/main/resources/templates/example/index.html similarity index 100% rename from junit5-support/src/main/resources/templates/example/index.html rename to core/src/main/resources/templates/example/index.html diff --git a/junit5-support/build.gradle b/junit5-support/build.gradle index 4a7f750a3..6af565456 100644 --- a/junit5-support/build.gradle +++ b/junit5-support/build.gradle @@ -25,7 +25,6 @@ dependencies { implementation 'org.assertj:assertj-core:3.26.3' implementation 'org.junit.jupiter:junit-jupiter-api:5.11.2' implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.0' - implementation 'org.thymeleaf:thymeleaf:3.1.2.RELEASE' implementation "io.ktor:ktor-client-core-jvm:${ktor_version}" implementation "io.ktor:ktor-client-cio:${ktor_version}" diff --git a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlReport.kt b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlReport.kt index 2a049f28e..84ae0b77c 100644 --- a/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlReport.kt +++ b/junit5-support/src/main/kotlin/io/specmatic/test/reports/coverage/html/HtmlReport.kt @@ -6,9 +6,8 @@ import io.specmatic.core.ReportFormatter import io.specmatic.core.SpecmaticConfig import io.specmatic.core.SuccessCriteria import io.specmatic.core.TestResult +import io.specmatic.templates.HtmlTemplateConfiguration import io.specmatic.test.reports.coverage.console.Remarks -import io.specmatic.test.reports.coverage.html.HtmlTemplateConfiguration.Companion.configureTemplateEngine -import org.thymeleaf.context.Context import java.io.File import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -70,9 +69,9 @@ class HtmlReport(private val htmlReportInformation: HtmlReportInformation) { "jsonTestData" to dumpTestData(updatedScenarios) ) - return configureTemplateEngine().process( + return HtmlTemplateConfiguration.process( "report/index.html", - Context().apply { setVariables(templateVariables) } + variables = templateVariables ) } From 08a7fe93b65734be6c53a00a70adce91c9f42318 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Fri, 11 Oct 2024 21:27:10 +0530 Subject: [PATCH 33/43] refactor use Kotlin.Result whenever possible. - Clean up code - minor fixes --- .../exampleGeneration/ExamplesBase.kt | 39 +++++------ .../ExamplesInteractiveBase.kt | 14 ++-- .../exampleGeneration/ExamplesValidateBase.kt | 21 +++--- .../examples/ExampleGenerationResult.kt | 6 +- .../examples/ExamplesInteractiveServer.kt | 70 +++++++++---------- .../examples/InteractiveServerProvider.kt | 6 +- 6 files changed, 78 insertions(+), 78 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt index b64c1eee2..695f2ef22 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt @@ -7,6 +7,8 @@ import io.specmatic.core.log.* import picocli.CommandLine.Option import java.io.File import java.util.concurrent.Callable +import kotlin.Result.Companion.success +import kotlin.Result.Companion.failure abstract class ExamplesBase(protected open val featureStrategy: ExamplesFeatureStrategy) : Callable { @Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") @@ -24,10 +26,11 @@ abstract class ExamplesBase(protected open val featureStrateg override fun call(): Int { configureLogger(verbose) - return contractFile?.let { - ensureValidContractFile(it).first?.let { contract -> - execute(contract) - } ?: 1 + return contractFile?.let { contract -> + ensureValidContractFile(contract).fold( + onSuccess = { execute(contract) }, + onFailure = { 1 } + ) } ?: execute(contractFile) } @@ -50,30 +53,24 @@ abstract class ExamplesBase(protected open val featureStrateg return getFilteredScenarios(scenarios, scenarioFilter) } - @Suppress("MemberVisibilityCanBePrivate") // Used By InteractiveServer through ExamplesInteractiveBase - fun ensureValidContractFile(contractFile: File): Pair { + private fun ensureValidFile(file: File, validExtensions: Set, fileType: String): Result { val errorMessage = when { - !contractFile.exists() -> "Contract file does not exist: ${contractFile.absolutePath}" - contractFile.extension !in featureStrategy.contractFileExtensions -> - "Invalid Contract file ${contractFile.path} - File extension must be one of ${featureStrategy.contractFileExtensions.joinToString()}" - else -> return contractFile to null + !file.exists() -> "$fileType file does not exist: ${file.absolutePath}" + file.extension !in validExtensions -> "Invalid $fileType file ${file.path} - File extension must be one of ${validExtensions.joinToString()}" + else -> return success(file) } - consoleLog(errorMessage) - return null to errorMessage + return failure(IllegalArgumentException(errorMessage)) } @Suppress("MemberVisibilityCanBePrivate") // Used By InteractiveServer through ExamplesInteractiveBase - fun ensureValidExampleFile(exampleFile: File): Pair { - val errorMessage = when { - !exampleFile.exists() -> "Example file does not exist: ${exampleFile.absolutePath}" - exampleFile.extension !in featureStrategy.exampleFileExtensions -> - "Invalid Example file ${exampleFile.path} - File extension must be one of ${featureStrategy.exampleFileExtensions.joinToString()}" - else -> return exampleFile to null - } + fun ensureValidContractFile(contractFile: File): Result { + return ensureValidFile(contractFile, featureStrategy.contractFileExtensions, "Contract") + } - consoleLog(errorMessage) - return null to errorMessage + @Suppress("MemberVisibilityCanBePrivate") // Used By InteractiveServer through ExamplesInteractiveBase + fun ensureValidExampleFile(exampleFile: File): Result { + return ensureValidFile(exampleFile, featureStrategy.exampleFileExtensions, "Example") } protected fun getExamplesDirectory(contractFile: File): File { diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index a60013df4..a4561d526 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -7,7 +7,7 @@ import io.specmatic.core.log.consoleLog import io.specmatic.examples.* import picocli.CommandLine.Option import java.io.File -import java.lang.Thread.sleep +import java.util.concurrent.CountDownLatch abstract class ExamplesInteractiveBase ( override val featureStrategy: ExamplesFeatureStrategy, @@ -34,10 +34,13 @@ abstract class ExamplesInteractiveBase ( } updateDictionaryFile(dictFile) + + val latch = CountDownLatch(1) server.start() - addShutdownHook(server) + addShutdownHook(server, latch) consoleLog("Examples Interactive server is running on http://0.0.0.0:9001/_specmatic/examples. Ctrl + C to stop.") - while (true) sleep(10000) + latch.await() + return 0 } catch (e: Throwable) { consoleLog("Example Interactive server failed with error: ${e.message}") consoleDebug(e) @@ -96,9 +99,10 @@ abstract class ExamplesInteractiveBase ( return createTableRows(scenarioExamplePair) } - private fun addShutdownHook(server: ExamplesInteractiveServer) { + private fun addShutdownHook(server: ExamplesInteractiveServer, latch: CountDownLatch) { Runtime.getRuntime().addShutdownHook(Thread { - consoleLog("Shutting down examples interactive server...") + consoleLog("Shutdown signal received (Ctrl + C).") + latch.countDown() try { server.close() consoleLog("Server shutdown completed successfully.") diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt index d2ef2f55b..c4198b96c 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt @@ -28,9 +28,7 @@ abstract class ExamplesValidateBase( } try { - exampleFile?.let { exFile -> - return validateSingleExampleFile(exFile, contract) - } + exampleFile?.let { exFile -> return validateSingleExampleFile(exFile, contract) } val inlineResults = if (validateInline) validateInlineExamples(contract).logValidationResult() @@ -56,13 +54,16 @@ abstract class ExamplesValidateBase( } private fun validateSingleExampleFile(exampleFile: File, contractFile: File): Int { - ensureValidExampleFile(exampleFile).first?.let { exFile -> - val feature = featureStrategy.contractFileToFeature(contractFile) - return validateExternalExample(exFile, feature).let { - it.logErrors() - it.getExitCode() - } - } ?: return 1 + return ensureValidExampleFile(exampleFile).fold( + onSuccess = { + val feature = featureStrategy.contractFileToFeature(contractFile) + validateExternalExample(exampleFile, feature).let { + it.logErrors() + it.getExitCode() + } + }, + onFailure = { return 1 } + ) } private fun validateExternalExample(exampleFile: File, feature: Feature): ExampleValidationResult { diff --git a/core/src/main/kotlin/io/specmatic/examples/ExampleGenerationResult.kt b/core/src/main/kotlin/io/specmatic/examples/ExampleGenerationResult.kt index e76f793dd..e5726e0e6 100644 --- a/core/src/main/kotlin/io/specmatic/examples/ExampleGenerationResult.kt +++ b/core/src/main/kotlin/io/specmatic/examples/ExampleGenerationResult.kt @@ -3,9 +3,9 @@ package io.specmatic.examples import java.io.File enum class ExampleGenerationStatus(val value: String) { - CREATED("Inline"), - ERROR("External"), - EXISTS("Exists"); + CREATED("Created"), + EXISTS("Exists"), + ERROR("Error"); override fun toString(): String { return this.value diff --git a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt index 7e02c1612..9a64dd13d 100644 --- a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt +++ b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt @@ -18,6 +18,8 @@ import io.specmatic.core.route.modules.HealthCheckModule.Companion.configureHeal import io.specmatic.templates.HtmlTemplateConfiguration import java.io.File import java.io.FileNotFoundException +import kotlin.Result.Companion.failure +import kotlin.Result.Companion.success class ExamplesInteractiveServer(provider: InteractiveServerProvider) : InteractiveServerProvider by provider { private var cachedContractFile: File? = null @@ -80,7 +82,7 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti return request?.hostPort?.takeIf { it.isNotEmpty() } ?: "localhost:$serverPort" } - private fun getContractFileOrNull(request: ExamplePageRequest? = null): Pair { + private fun getContractFileOrNull(request: ExamplePageRequest? = null): Result { val contractFile = contractFile?.also { logContractFileUsage("command line", it) } ?: request?.contractFile?.takeIf { it.exists() }?.also { @@ -89,7 +91,13 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti logContractFileUsage("cached HTTP request", it) } - return contractFile?.let { it to null } ?: (null to "Contract file not provided, Please provide one via HTTP request") + return contractFile?.let { + success(it) + } ?: run { + val errorMessage = "Contract file not provided, Please provide one via HTTP request or command line" + consoleLog(errorMessage) + failure(IllegalArgumentException(errorMessage)) + } } private fun logContractFileUsage(source: String, contract: File) { @@ -146,46 +154,36 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti respondWithError(statusCode, errorMessage ?: "Something went wrong") } - private suspend fun PipelineContext.getValidatedContractFileOrNull(request: ExamplePageRequest? = null): File? { - val (contractFile, nullErrorMessage) = getContractFileOrNull(request) - nullErrorMessage?.let { - consoleLog(nullErrorMessage) - call.respondWithError(HttpStatusCode.BadRequest, it) - return null - } - - requireNotNull(contractFile) { "Contract file should not be null if there's no error message" } - val (contract, errorMessage) = ensureValidContractFile(contractFile) - errorMessage?.let { - call.respondWithError(HttpStatusCode.BadRequest, errorMessage) - return null - } - - return requireNotNull(contract) { "Contract file should not be null if there's no error message" } + private suspend fun PipelineContext.handleContractFile(request: ExamplePageRequest? = null, block: suspend (File) -> Unit) { + getContractFileOrNull(request).fold( + onSuccess = { contract -> + ensureValidContractFile(contract).fold( + onSuccess = { block(it) }, + onFailure = { call.respondWithError(HttpStatusCode.BadRequest, it.message.orEmpty()) } + ) + }, + onFailure = { call.respondWithError(HttpStatusCode.BadRequest, it.message.orEmpty()) } + ) } - private suspend fun PipelineContext.getValidatedExampleOrNull(exampleFile: File): File? { - val (validatedExample, errorMessage) = ensureValidExampleFile(exampleFile) - - errorMessage?.let { - call.respondWithError(HttpStatusCode.BadRequest, errorMessage) - return null - } - - return requireNotNull(validatedExample) { "Example file should not be null if there's no error message" } + private suspend fun PipelineContext.handleExampleFile(exampleFile: File, block: suspend (File) -> Unit) { + ensureValidExampleFile(exampleFile).fold( + onSuccess = { block(it) }, + onFailure = { call.respondWithError(HttpStatusCode.BadRequest, it.message.orEmpty()) } + ) } private fun Routing.getHtmlPageRoute() { get("/_specmatic/examples") { - getValidatedContractFileOrNull()?.let { contract -> - val htmlContent = getHtmlContent(contract, getServerHostAndPort()) + handleContractFile { + val htmlContent = getHtmlContent(it, getServerHostAndPort()) call.respondText(htmlContent, contentType = ContentType.Text.Html) } } post("/_specmatic/examples") { val request = call.receive() - getValidatedContractFileOrNull(request)?.let { contract -> + handleContractFile(request) { contract -> val htmlContent = getHtmlContent(contract, getServerHostAndPort(request)) call.respondText(htmlContent, contentType = ContentType.Text.Html) } @@ -194,7 +192,7 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti private fun Routing.generateExampleRoute() { post("/_specmatic/examples/generate") { - getValidatedContractFileOrNull()?.let { contractFile -> + handleContractFile { contractFile -> val result = generateExample(call, contractFile) call.respond(HttpStatusCode.OK, ExampleGenerationResponse(result)) } @@ -204,8 +202,8 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti private fun Routing.validateExampleRoute() { post("/_specmatic/examples/validate") { val request = call.receive() - getValidatedContractFileOrNull()?.let { contract -> - getValidatedExampleOrNull(request.exampleFile)?.let { example -> + handleContractFile { contract -> + handleExampleFile(request.exampleFile) { example -> val result = validateExample(contract, example) call.respond(HttpStatusCode.OK, ExampleValidationResponse(result)) } @@ -216,7 +214,7 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti private fun Routing.getExampleContentRoute() { post("/_specmatic/examples/content") { val request = call.receive() - getValidatedExampleOrNull(request.exampleFile)?.let { example -> + handleExampleFile(request.exampleFile) { example -> call.respond(HttpStatusCode.OK, mapOf("content" to example.readText())) } } @@ -229,8 +227,8 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti } val request = call.receive() - getValidatedContractFileOrNull()?.let { contract -> - getValidatedExampleOrNull(request.exampleFile)?.let { example -> + handleContractFile { contract -> + handleExampleFile(request.exampleFile) { example -> val result = testExample(contract, example, sutBaseUrl) call.respond(HttpStatusCode.OK, ExampleTestResponse(result)) } diff --git a/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt b/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt index 2211a7c65..78e3a689b 100644 --- a/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt +++ b/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt @@ -20,9 +20,9 @@ interface InteractiveServerProvider { fun getTableRows(contractFile: File): List - fun ensureValidContractFile(contractFile: File): Pair + fun ensureValidContractFile(contractFile: File): Result - fun ensureValidExampleFile(exampleFile: File): Pair + fun ensureValidExampleFile(exampleFile: File): Result } data class ExamplePageRequest ( @@ -36,7 +36,7 @@ data class ExampleGenerationResponse ( ) { constructor(result: ExampleGenerationResult): this( exampleFilePath = result.exampleFile?.absolutePath ?: throw Exception("Failed to generate example file"), - status = result.status.toString() + status = result.status.name ) } From 9faf8a7f7a6613294f4ef58a7cea9641dc78f29c Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Mon, 14 Oct 2024 10:30:00 +0530 Subject: [PATCH 34/43] make contentType available in openapi interactive. - Move validation result logging into class for reusability. - Other CSS and JS Fixes. --- .../exampleGeneration/ExamplesInteractiveBase.kt | 10 +++++++++- .../exampleGeneration/ExamplesValidateBase.kt | 11 ----------- .../OpenApiExamplesInteractive.kt | 13 +++++++------ .../examples/ExampleValidationResult.kt | 12 ++++++++++++ .../main/resources/templates/example/index.html | 16 ++++++++++------ 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index a4561d526..909c49fea 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -76,7 +76,9 @@ abstract class ExamplesInteractiveBase ( override fun validateExample(contractFile: File, exampleFile: File): ExampleValidationResult { val feature = featureStrategy.contractFileToFeature(contractFile) val result = validationStrategy.validateExternalExample(feature, exampleFile) - return ExampleValidationResult(exampleFile.absolutePath, result.second, ExampleType.EXTERNAL) + return ExampleValidationResult(exampleFile.absolutePath, result.second, ExampleType.EXTERNAL).also { + it.logErrors() + } } override fun testExample(contractFile: File, exampleFile: File, sutBaseUrl: String): ExampleTestResult { @@ -115,4 +117,10 @@ abstract class ExamplesInteractiveBase ( } }) } + + data class ValueWithInfo( + val value: T, + val rawValue: String, + val extraInfo: String? + ) } diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt index c4198b96c..8df646aa8 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt @@ -109,17 +109,6 @@ abstract class ExamplesValidateBase( ) } - private fun ExampleValidationResult.logErrors(index: Int? = null) { - val prefix = index?.let { "$it. " } ?: "" - - if (this.result.isSuccess()) { - return consoleLog("$prefix${this.exampleFile?.name ?: this.exampleName} is valid") - } - - consoleLog("\n$prefix${this.exampleFile?.name ?: this.exampleName} has the following validation error(s):") - consoleLog(this.result.reportString()) - } - private fun List.logValidationResult(): List { if (this.isNotEmpty()) { require(this.all { it.type == this.first().type }) { diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index b1bb90f94..e1f0dec49 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -40,7 +40,7 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( override suspend fun getScenarioFromRequestOrNull(call: ApplicationCall, feature: Feature): Scenario? { val request = call.receive() return feature.scenarios.firstOrNull { - it.method == request.method && it.status == request.response && it.path == request.path + it.method == request.method.value && it.status == request.response.value && it.path == request.path.rawValue && (request.contentType == null || it.httpRequestPattern.headersPattern.contentType == request.contentType) } } @@ -83,9 +83,10 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( } data class ExampleGenerationRequest ( - val method: String, - val path: String, - val response: Int, - val contentType: String? = null - ) + val method: ValueWithInfo, + val path: ValueWithInfo, + val response: ValueWithInfo, + ) { + val contentType = response.extraInfo + } } \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/examples/ExampleValidationResult.kt b/core/src/main/kotlin/io/specmatic/examples/ExampleValidationResult.kt index 74684ebcd..b840cdce0 100644 --- a/core/src/main/kotlin/io/specmatic/examples/ExampleValidationResult.kt +++ b/core/src/main/kotlin/io/specmatic/examples/ExampleValidationResult.kt @@ -1,6 +1,7 @@ package io.specmatic.examples import io.specmatic.core.Result +import io.specmatic.core.log.consoleLog import java.io.File enum class ExampleType(val value: String) { @@ -14,4 +15,15 @@ enum class ExampleType(val value: String) { data class ExampleValidationResult(val exampleName: String, val result: Result, val type: ExampleType, val exampleFile: File? = null) { constructor(exampleFile: File, result: Result) : this(exampleFile.nameWithoutExtension, result, ExampleType.EXTERNAL, exampleFile) + + fun logErrors(index: Int? = null) { + val prefix = index?.let { "$it. " } ?: "" + + if (this.result.isSuccess()) { + return consoleLog("$prefix${this.exampleFile?.name ?: this.exampleName} is valid") + } + + consoleLog("\n$prefix${this.exampleFile?.name ?: this.exampleName} has the following validation error(s):") + consoleLog(this.result.reportString()) + } } diff --git a/core/src/main/resources/templates/example/index.html b/core/src/main/resources/templates/example/index.html index 202f8b98b..765174408 100644 --- a/core/src/main/resources/templates/example/index.html +++ b/core/src/main/resources/templates/example/index.html @@ -705,13 +705,14 @@ } td:nth-child(3) { + & > p { + white-space: normal; + } text-align: initial; min-width: 10rem; } td:last-child, td:nth-last-child(2) { - max-width: 10rem; - & p { white-space: normal; word-break: break-word; @@ -1352,8 +1353,11 @@

const colName = td.getAttribute("data-col-name"); const rawValue = td.getAttribute("data-raw-value"); + const value = td.querySelector("p")?.textContent; + const extraInfo = td.querySelector("span")?.textContent; + if (colName && rawValue) { - acc[colName] = rawValue; + acc[colName] = { rawValue, extraInfo, value }; } return acc; @@ -1500,7 +1504,7 @@

function createPathSummary(rowValues) { const docFragment = document.createDocumentFragment(); - for (const [key, value] of Object.entries(rowValues)) { + for (const [key, {value}] of Object.entries(rowValues)) { if (!value) continue; const li = document.createElement("li"); @@ -1597,11 +1601,11 @@

return dropDownDiv; } - async function generateExample(pathInfo) { + async function generateExample(rowValues) { try { const resp = await fetch(`http://${getHostPort()}/_specmatic/examples/generate`, { method: "POST", - body: JSON.stringify(pathInfo), + body: JSON.stringify(rowValues), headers: { "Content-Type": "application/json", } From d44775841f96c8fe4ed03227528c6d7f8b4c3f48 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Mon, 14 Oct 2024 12:15:28 +0530 Subject: [PATCH 35/43] Fix bug causing example test to run twice. --- .../resources/templates/example/index.html | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/core/src/main/resources/templates/example/index.html b/core/src/main/resources/templates/example/index.html index 765174408..1aae29741 100644 --- a/core/src/main/resources/templates/example/index.html +++ b/core/src/main/resources/templates/example/index.html @@ -1295,38 +1295,6 @@

return cleanUpSelections(); } - bulkTestBtn.addEventListener("click", async () => { - blockGenValidate = true; - bulkTestBtn.setAttribute("data-test", "processing"); - - switch (bulkValidateBtn.getAttribute("data-panel")) { - case "table": { - await testAllSelected(); - break; - } - - case "details": { - await testRowExample(selectedTableRow); - const exampleFilePath = getExampleData(selectedTableRow); - const {example: exampleContent, error} = await getExampleContent(exampleFilePath); - - const docFragment = createExampleRows([{ - exampleFilePath: exampleFilePath, - exampleJson: exampleContent, - error: validationDetails[Number.parseInt(selectedTableRow.children[1].textContent)] || error, - hasBeenValidated: selectedTableRow.hasAttribute("data-valid"), - test: testDetails[Number.parseInt(selectedTableRow.children[1].textContent)] - }]); - - examplesOl.replaceChildren(docFragment); - break; - } - } - - bulkTestBtn.removeAttribute("data-test"); - return cleanUpSelections(); - }); - async function testAllSelected() { const selectedRows = Array.from(table.querySelectorAll("td > input[type=checkbox]:checked")).map((checkbox) => checkbox.closest("tr")); const rowsWithValidations = selectedRows.filter(row => row.getAttribute("data-valid") === "success"); From 0622ab00577b8b41bb19562c7de379095f4343cd Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Wed, 16 Oct 2024 10:13:48 +0530 Subject: [PATCH 36/43] modify logic for creating test from example file. - load externalised example after parsing contract file to feature in OpenApiExamplesFeatureStrategy - Add test to ensure canonicalPath is used when matching example File to examples in scenario. - update test for example creation in Feature. --- .../OpenApiExamplesFeatureStrategy.kt | 2 +- .../OpenApiExamplesInteractive.kt | 18 ++--- .../OpenApiExamplesValidateTest.kt | 2 +- .../main/kotlin/io/specmatic/core/Feature.kt | 21 ++---- .../main/kotlin/io/specmatic/core/Scenario.kt | 5 ++ .../kotlin/io/specmatic/core/FeatureTest.kt | 70 ++++++++++--------- .../kotlin/io/specmatic/core/ScenarioTest.kt | 45 ++++++++++++ 7 files changed, 106 insertions(+), 57 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesFeatureStrategy.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesFeatureStrategy.kt index cad473d52..ae7ded6bd 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesFeatureStrategy.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesFeatureStrategy.kt @@ -9,7 +9,7 @@ class OpenApiExamplesFeatureStrategy: ExamplesFeatureStrategy override val contractFileExtensions: Set get() = OPENAPI_FILE_EXTENSIONS.toSet() override fun contractFileToFeature(contractFile: File): Feature { - return parseContractFileToFeature(contractFile) + return parseContractFileToFeature(contractFile).loadExternalisedExamples() } override fun getScenarioDescription(scenario: Scenario): String { diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index e1f0dec49..0ecf623cc 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -27,14 +27,16 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( ) override fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair { - val test = feature.createContractTestFromExampleFile(exampleFile.absolutePath).value - - val testResult = test.runTest(testBaseUrl, timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS) - val testLogs = TestInteractionsLog.testHttpLogMessages.lastOrNull { - it.scenario == testResult.first.scenario - }?.combineLog() ?: "Test logs not found for example" - - return testResult.first to testLogs + return feature.createContractTestFromExampleFile(exampleFile).realise( + orFailure = { it.toFailure() to "" }, orException = { it.toHasFailure().toFailure() to "" }, + hasValue = { test, _ -> + val testResult = test.runTest(testBaseUrl, timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS) + val testLogs = TestInteractionsLog.testHttpLogMessages.lastOrNull { + it.scenario == testResult.first.scenario + }?.combineLog() ?: "Test logs not found for example" + testResult.first to testLogs + } + ) } override suspend fun getScenarioFromRequestOrNull(call: ApplicationCall, feature: Feature): Scenario? { diff --git a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt index 99fe7a811..e25234668 100644 --- a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt +++ b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidateTest.kt @@ -439,7 +439,7 @@ paths: val otherExamples = examples.filter { exFile -> exFile != it } otherExamples.forEach { otherExample -> - assertThat(stdOut).doesNotContain(otherExample.name) + assertThat(stdOut).doesNotContain("${otherExample.name} is valid") } } } diff --git a/core/src/main/kotlin/io/specmatic/core/Feature.kt b/core/src/main/kotlin/io/specmatic/core/Feature.kt index 213e6e200..b1be136a1 100644 --- a/core/src/main/kotlin/io/specmatic/core/Feature.kt +++ b/core/src/main/kotlin/io/specmatic/core/Feature.kt @@ -372,22 +372,15 @@ data class Feature( } } - fun createContractTestFromExampleFile(filePath: String): ReturnValue { - val scenarioStub = ScenarioStub.readFromFile(File(filePath)) - return createContractTestFromScenarioStub(scenarioStub, filePath) - } - - @Suppress("MemberVisibilityCanBePrivate") // Used by GraphQL when testing examples interactively - fun createContractTestFromScenarioStub(scenarioStub: ScenarioStub, filePath: String): ReturnValue { - val originalScenario = scenarios.firstOrNull { scenario -> - scenario.matches(scenarioStub.request, scenarioStub.response) is Result.Success - } ?: return HasFailure(Result.Failure("Could not find an API matching example $filePath")) + fun createContractTestFromExampleFile(exampleFile: File): ReturnValue { + val matchingScenario = scenarios.firstOrNull { it.matchesExample(exampleFile) } + ?: return HasFailure(Result.Failure("Could not find an API matching example ${exampleFile.canonicalPath}")) - val concreteTestScenario = originalScenario.copy( - httpRequestPattern = scenarioStub.request.toPattern() - ) + val testScenario = matchingScenario.generateTestScenarios(flagsBased) + .firstOrNull { it.value.exampleName == exampleFile.nameWithoutExtension } + ?.value ?: return HasFailure(Result.Failure("Could not find an API matching example ${exampleFile.canonicalPath}")) - return HasValue(scenarioAsTest(concreteTestScenario, null, Workflow(), originalScenario)) + return HasValue(scenarioAsTest(testScenario, null, Workflow(), matchingScenario)) } private fun scenarioAsTest( diff --git a/core/src/main/kotlin/io/specmatic/core/Scenario.kt b/core/src/main/kotlin/io/specmatic/core/Scenario.kt index 711915cba..3f50fbe68 100644 --- a/core/src/main/kotlin/io/specmatic/core/Scenario.kt +++ b/core/src/main/kotlin/io/specmatic/core/Scenario.kt @@ -9,6 +9,7 @@ import io.specmatic.core.utilities.nullOrExceptionString import io.specmatic.core.value.* import io.specmatic.mock.ScenarioStub import io.specmatic.stub.RequestContext +import java.io.File object ContractAndStubMismatchMessages : MismatchMessages { override fun mismatchMessage(expected: String, actual: String): String { @@ -615,6 +616,10 @@ data class Scenario( return Result.fromResults(listOf(requestMatch, responseMatch)) } + + fun matchesExample(exampleFile: File): Boolean { + return examples.any { it.rows.any { row -> row.fileSource == exampleFile.canonicalPath } } + } } fun newExpectedServerStateBasedOn( diff --git a/core/src/test/kotlin/io/specmatic/core/FeatureTest.kt b/core/src/test/kotlin/io/specmatic/core/FeatureTest.kt index cfbd8b897..b1f23cb7b 100644 --- a/core/src/test/kotlin/io/specmatic/core/FeatureTest.kt +++ b/core/src/test/kotlin/io/specmatic/core/FeatureTest.kt @@ -4,6 +4,8 @@ import io.specmatic.conversions.OpenApiSpecification import io.specmatic.core.pattern.NumberPattern import io.specmatic.core.pattern.StringPattern import io.specmatic.core.pattern.parsedJSONObject +import io.specmatic.core.utilities.Flags +import io.specmatic.core.utilities.Flags.Companion.EXAMPLE_DIRECTORIES import io.specmatic.core.utilities.exceptionCauseMessage import io.specmatic.core.value.* import io.specmatic.stub.captureStandardOutput @@ -2470,9 +2472,37 @@ paths: } @Test - fun `shoudl be able to create a contract test based on an example`(@TempDir tempDir: File) { - val feature = OpenApiSpecification.fromYAML( + fun `should be able to create a contract test based on an example`(@TempDir tempDir: File) { + val exampleFile = tempDir.resolve("example.json") + exampleFile.writeText( """ + { + "http-request": { + "method": "POST", + "path": "/products", + "body": { + "name": "James" + }, + "headers": { + "Content-Type": "application/json" + } + }, + "http-response": { + "status": 200, + "body": { + "id": 10 + }, + "headers": { + "Content-Type": "application/json" + } + } + } + """.trimIndent() + ) + + val feature = Flags.using(EXAMPLE_DIRECTORIES to tempDir.canonicalPath) { + OpenApiSpecification.fromYAML( + """ openapi: 3.0.0 info: title: Sample Product API @@ -2509,42 +2539,16 @@ paths: id: type: integer """.trimIndent(), "" - ).toFeature() - - val exampleFile = tempDir.resolve("example.json") - exampleFile.writeText( - """ - { - "http-request": { - "method": "POST", - "path": "/products", - "body": { - "name": "James" - }, - "headers": { - "Content-Type": "application/json" - } - }, - "http-response": { - "status": 200, - "body": { - "id": 10 - }, - "headers": { - "Content-Type": "application/json" - } - } - } - """.trimIndent() - ) + ).toFeature().loadExternalisedExamples() + } - val contractTest = feature.createContractTestFromExampleFile(exampleFile.path).value + val contractTest = feature.createContractTestFromExampleFile(exampleFile) - val results = contractTest.runTest(object : TestExecutor { + val results = contractTest.value.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { val jsonRequestBody = request.body as JSONObjectValue assertThat(jsonRequestBody.findFirstChildByPath("name")?.toStringLiteral()).isEqualTo("James") - return HttpResponse.ok(parsedJSONObject("""{"id": 10}""")).also { + return HttpResponse.ok(parsedJSONObject("""{"id": 99}""")).also { println(request.toLogString()) println() println(it.toLogString()) diff --git a/core/src/test/kotlin/io/specmatic/core/ScenarioTest.kt b/core/src/test/kotlin/io/specmatic/core/ScenarioTest.kt index 3ecc98925..248e65205 100644 --- a/core/src/test/kotlin/io/specmatic/core/ScenarioTest.kt +++ b/core/src/test/kotlin/io/specmatic/core/ScenarioTest.kt @@ -1,8 +1,12 @@ package io.specmatic.core +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify import io.specmatic.core.pattern.* import org.assertj.core.api.Assertions.* import org.junit.jupiter.api.Test +import java.io.File import java.util.function.Consumer class ScenarioTest { @@ -255,4 +259,45 @@ class ScenarioTest { .hasMessageContaining("REQUEST.BODY.id") }) } + + @Test + fun `should only use canonical paths for matching example file`() { + val scenario = Scenario( + "", + HttpRequestPattern( + method = "POST", + body = JSONObjectPattern( + pattern = mapOf( + "id" to NumberPattern() + ) + ) + ), + HttpResponsePattern( + status = 200, + body = JSONObjectPattern( + pattern = mapOf( + "id" to NumberPattern() + ) + ) + ), + emptyMap(), + listOf( + Examples( + listOf("(REQUEST-BODY)"), + listOf(Row( + mapOf("(REQUEST-BODY)" to """{"id": "(number)" }""") + ).copy(fileSource = "example.json")) + ) + ), + emptyMap(), + emptyMap() + ) + + val mockExampleFile = mockk { + every { canonicalPath } returns "example.json" + } + + assertThat(scenario.matchesExample(mockExampleFile)).isTrue() + verify { mockExampleFile.canonicalPath } + } } \ No newline at end of file From 01c1e365e3e36fdee6fca85df0726f87ecdf602b Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Wed, 16 Oct 2024 15:36:50 +0530 Subject: [PATCH 37/43] Add tests for OpenApiExamplesInteractive. - minor refactors to ExamplesInteractiveServer --- .../ExamplesInteractiveBase.kt | 2 +- .../OpenApiExamplesInteractiveTest.kt | 418 ++++++++++++++++++ .../examples/ExamplesInteractiveServer.kt | 6 +- 3 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractiveTest.kt diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index 909c49fea..6f4da872b 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -18,7 +18,7 @@ abstract class ExamplesInteractiveBase ( override var sutBaseUrl: String? = null @Option(names = ["--contract-file"], description = ["Contract file path"], required = false) - override var contractFile: File? = null + public override var contractFile: File? = null @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) protected var dictFile: File? = null diff --git a/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractiveTest.kt b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractiveTest.kt new file mode 100644 index 000000000..80484649a --- /dev/null +++ b/application/src/test/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractiveTest.kt @@ -0,0 +1,418 @@ +package application.exampleGeneration.openApiExamples + +import io.ktor.http.* +import io.specmatic.conversions.ExampleFromFile +import io.specmatic.core.HttpRequest +import io.specmatic.core.HttpResponse +import io.specmatic.core.value.JSONObjectValue +import io.specmatic.core.value.StringValue +import io.specmatic.test.HttpClient +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File +import java.net.URI +import java.net.URL +import kotlin.concurrent.thread + +@Suppress("SameParameterValue") +class OpenApiExamplesInteractiveTest { + companion object { + private lateinit var serverThread: Thread + private val trackerContract = File("src/test/resources/specifications/tracker.yaml") + private val testServerHostPort = URL("http://localhost:8080").toURI() + private val exampleServerClient = HttpClient("http://localhost:9001/_specmatic/examples") + private val execCommand = OpenApiExamplesInteractive() + + private fun waitForServer(maxRetries: Int = 10, sleepDuration: Long = 200) { + repeat(maxRetries) { + try { + val resp = HttpClient("http://localhost:9001", timeoutInMilliseconds = 2000) + .execute(HttpRequest().updatePath("actuator/health").updateMethod("GET")).status + + if (resp == 200) { + return println("Server is up and healthy.") + } + } catch (_: Exception) { + Thread.sleep(sleepDuration) + } + } + + throw IllegalStateException("Server did not start after $maxRetries attempts.") + } + + private fun createHttpRequest(path: String, method: String, body: String? = null): HttpRequest { + return HttpRequest().updatePath(path).updateMethod(method).updateBody(body) + } + + private fun exampleGenerateRequest(contractFile: File?, jsonRequest: String): HttpResponse { + val generateRequest = createHttpRequest("/generate", "POST", jsonRequest) + return withContractFile(contractFile) { + exampleServerClient.execute(generateRequest) + } + } + + private fun exampleValidateRequest(contractFile: File?, jsonRequest: String): HttpResponse { + val validateRequest = createHttpRequest("/validate", "POST", jsonRequest) + return withContractFile(contractFile) { + exampleServerClient.execute(validateRequest) + } + } + + private fun exampleTestRequest(contractFile: File?, sutBaseUrl: URI, jsonRequest: String): HttpResponse { + val testRequest = createHttpRequest("/test", "POST", jsonRequest) + return withContractFile(contractFile) { + withSutBaseUrl(sutBaseUrl.toString()) { + exampleServerClient.execute(testRequest) + } + } + } + + private fun exampleContentRequest(contractFile: File?, jsonRequest: String): HttpResponse { + val exampleContentRequest = createHttpRequest("/content", "POST", jsonRequest) + return withContractFile(contractFile) { + exampleServerClient.execute(exampleContentRequest) + } + } + + private fun exampleHtmlPageRequest(contractFile: File?, jsonRequest: String): HttpResponse { + val exampleContentRequest = createHttpRequest("/", "POST", jsonRequest) + return withContractFile(contractFile) { + exampleServerClient.execute(exampleContentRequest) + } + } + + private fun exampleHtmlPageRequest(contractFile: File?): HttpResponse { + val request = HttpRequest().updatePath("/").updateMethod("GET") + return withContractFile(contractFile) { + exampleServerClient.execute(request) + } + } + + private fun createJsonGenerateRequest(path: String, method: String, response: String): String { + return """ + { + "path": { "rawValue": "$path", "value": "$path" }, + "method": { "rawValue": "$method", "value": "$method" }, + "response": { "rawValue": "$response", "value": "$response" } + } + """.trimIndent() + } + + private fun withContractFile(contractFile: File?, block: () -> HttpResponse): HttpResponse { + val originalContractFile = execCommand.contractFile + execCommand.contractFile = contractFile + try { + return block() + } finally { + execCommand.contractFile = originalContractFile + } + } + + private fun withSutBaseUrl(sutBaseUrl: String, block: () -> HttpResponse): HttpResponse { + val originalSutBaseUrl = execCommand.sutBaseUrl + execCommand.sutBaseUrl = sutBaseUrl + try { + return block() + } finally { + execCommand.sutBaseUrl = originalSutBaseUrl + } + } + + @JvmStatic + @BeforeAll + fun startServer() { + serverThread = thread(start = true) { execCommand.call() } + waitForServer() + } + + @JvmStatic + @AfterAll + fun stopServer() { + serverThread.interrupt() + serverThread.join() + } + } + + @AfterEach + fun cleanUp() { + val examplesFolder = File("src/test/resources/specifications/tracker_examples") + if (examplesFolder.exists()) { + examplesFolder.listFiles()?.forEach { it.delete() } + examplesFolder.delete() + } + } + + @Test + fun `should generate example from request`() { + val jsonRequest = createJsonGenerateRequest("/generate", "POST", "200") + + val response = exampleGenerateRequest(trackerContract, jsonRequest) + assertThat(response.status).isEqualTo(HttpStatusCode.OK.value) + assertThat(response.body).isInstanceOf(JSONObjectValue::class.java) + + val responseBody = response.body as JSONObjectValue + assertThat(responseBody.findFirstChildByPath("exampleFilePath")?.toStringLiteral()).contains("generate_POST_200.json") + assertThat(responseBody.findFirstChildByPath("status")?.toStringLiteral()).isEqualTo("CREATED") + } + + @Test + fun `should validate example from request`() { + val generateJsonRequest = createJsonGenerateRequest("/generate", "POST", "200") + + val generateResponse = exampleGenerateRequest(trackerContract, generateJsonRequest) + assertThat(generateResponse.status).isEqualTo(HttpStatusCode.OK.value) + assertThat(generateResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val generateResponseBody = generateResponse.body as JSONObjectValue + val exampleFilePath = generateResponseBody.findFirstChildByPath("exampleFilePath")!! + val validateJsonRequest = """ + { + "exampleFile": ${exampleFilePath.displayableValue()} + } + """.trimIndent() + + val validateResponse = exampleValidateRequest(trackerContract, validateJsonRequest) + assertThat(validateResponse.status).isEqualTo(HttpStatusCode.OK.value) + assertThat(validateResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val validateResponseBody = validateResponse.body as JSONObjectValue + assertThat(validateResponseBody.findFirstChildByPath("exampleFilePath")).isEqualTo(exampleFilePath) + assertThat(validateResponseBody.findFirstChildByPath("error")?.toStringLiteral()).isBlank() + } + + @Test + fun `should retrieve content of generated example file from request`() { + val generateJsonRequest = createJsonGenerateRequest("/generate", "POST", "200") + val generateResponse = exampleGenerateRequest(trackerContract, generateJsonRequest) + assertThat(generateResponse.status).isEqualTo(HttpStatusCode.OK.value) + assertThat(generateResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val generateResponseBody = generateResponse.body as JSONObjectValue + val exampleFilePath = generateResponseBody.findFirstChildByPath("exampleFilePath")!! + val getExampleContentJsonRequest = """ + { + "exampleFile": ${exampleFilePath.displayableValue()} + } + """.trimIndent() + + val getExampleContentResponse = exampleContentRequest(trackerContract, getExampleContentJsonRequest) + assertThat(getExampleContentResponse.status).isEqualTo(HttpStatusCode.OK.value) + assertThat(getExampleContentResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val getExampleContentResponseBody = getExampleContentResponse.body as JSONObjectValue + val content = getExampleContentResponseBody.findFirstChildByPath("content")!! + + assertThat(content.toStringLiteral()).isEqualTo(File(exampleFilePath.toStringLiteral()).readText()) + } + + @Test + fun `invalid or non existing example file request should return error`() { + val validateJsonRequest = """ + { + "exampleFile": "invalid_file_path.json" + } + """.trimIndent() + + val validateResponse = exampleValidateRequest(trackerContract, validateJsonRequest) + assertThat(validateResponse.status).isEqualTo(HttpStatusCode.BadRequest.value) + assertThat(validateResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val validateResponseBody = validateResponse.body as JSONObjectValue + assertThat(validateResponseBody.findFirstChildByPath("error")!!.toStringLiteral()) + .contains("Example file does not exist").contains("invalid_file_path.json") + + val exampleContentResponse = exampleContentRequest(trackerContract, validateJsonRequest) + assertThat(exampleContentResponse.status).isEqualTo(HttpStatusCode.BadRequest.value) + assertThat(exampleContentResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val exampleContentResponseBody = exampleContentResponse.body as JSONObjectValue + assertThat(exampleContentResponseBody.findFirstChildByPath("error")!!.toStringLiteral()) + .contains("Example file does not exist").contains("invalid_file_path.json") + } + + @Test + fun `should be able to test example from request`() { + val generateJsonRequest = createJsonGenerateRequest("/generate", "POST", "200") + val generateResponse = exampleGenerateRequest(trackerContract, generateJsonRequest) + assertThat(generateResponse.status).isEqualTo(HttpStatusCode.OK.value) + assertThat(generateResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val generateResponseBody = generateResponse.body as JSONObjectValue + val exampleFilePath = generateResponseBody.findFirstChildByPath("exampleFilePath")!! + val example = ExampleFromFile(File(exampleFilePath.toStringLiteral())) + + val testJsonRequest = """ + { + "exampleFile": ${exampleFilePath.displayableValue()} + } + """.trimIndent() + val testResponse = exampleTestRequest(trackerContract, testServerHostPort, testJsonRequest) + assertThat(testResponse.status).isEqualTo(HttpStatusCode.OK.value) + + val testResponseBody = testResponse.body as JSONObjectValue + assertThat(testResponseBody.findFirstChildByPath("result")?.toStringLiteral()).isEqualTo("Failed") + assertThat(testResponseBody.findFirstChildByPath("details")?.toStringLiteral()).contains("Example test for generate_POST_200 has FAILED") + assertThat(testResponseBody.findFirstChildByPath("testLog")?.toStringLiteral()) + .contains("POST /generate").contains(example.request.body.toStringLiteral()) + } + + @Test + fun `initial page request should use existing examples correctly`() { + val generateJsonRequest = createJsonGenerateRequest("/generate", "POST", "200") + val generateResponse = exampleGenerateRequest(trackerContract, generateJsonRequest) + assertThat(generateResponse.status).isEqualTo(HttpStatusCode.OK.value) + assertThat(generateResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val generateResponseBody = generateResponse.body as JSONObjectValue + val exampleFilePath = generateResponseBody.findFirstChildByPath("exampleFilePath")!! + assertThat(exampleFilePath.toStringLiteral()).isNotBlank() + + val htmlPageResponse = exampleHtmlPageRequest(trackerContract) + val htmlText = (htmlPageResponse.body as StringValue).toStringLiteral() + + assertThat(htmlText).contains(trackerContract.name).contains(trackerContract.absolutePath) + assertThat(htmlText).contains(exampleFilePath.toStringLiteral()) + } + + @Test + fun `should use contract file sent in post request when no initial contract is set`(@TempDir tempDir: File) { + val specFile = tempDir.resolve("spec.yaml") + + specFile.createNewFile() + val spec = """ +openapi: 3.0.0 +info: + title: Product API + version: 1.0.0 +paths: + /product/{id}: + get: + summary: Get product details + parameters: + - name: id + in: path + required: true + schema: + type: integer + responses: + '200': + description: Product details + content: + application/json: + schema: + type: object + properties: + id: + type: integer + name: + type: string + price: + type: number + format: float + """.trimIndent() + specFile.writeText(spec) + + val generateJsonRequest = """ + { + "contractFile": ${specFile.absolutePath.quote()} + } + """.trimIndent() + + val htmlPageResponse = exampleHtmlPageRequest(null, generateJsonRequest) + assertThat(htmlPageResponse.status).isEqualTo(HttpStatusCode.OK.value) + + val htmlText = (htmlPageResponse.body as StringValue).toStringLiteral() + assertThat(htmlText).contains(specFile.name).contains(specFile.absolutePath) + } + + @Test + fun `contract file specified in arguments should override post request`(@TempDir tempDir: File) { + val specFile = tempDir.resolve("spec.yaml") + + specFile.createNewFile() + val spec = """ +openapi: 3.0.0 +info: + title: Product API + version: 1.0.0 +paths: + /product/{id}: + get: + summary: Get product details + parameters: + - name: id + in: path + required: true + schema: + type: integer + responses: + '200': + description: Product details + content: + application/json: + schema: + type: object + properties: + id: + type: integer + name: + type: string + price: + type: number + format: float + """.trimIndent() + specFile.writeText(spec) + + val generateJsonRequest = """ + { + "contractFile": ${specFile.absolutePath.quote()} + } + """.trimIndent() + + val htmlPageResponse = exampleHtmlPageRequest(trackerContract, generateJsonRequest) + assertThat(htmlPageResponse.status).isEqualTo(HttpStatusCode.OK.value) + + val htmlText = (htmlPageResponse.body as StringValue).toStringLiteral() + assertThat(htmlText).doesNotContain(specFile.name).doesNotContain(specFile.absolutePath) + assertThat(htmlText).contains(trackerContract.name).contains(trackerContract.absolutePath) + } + + @Test + fun `invalid or non existing contract file request should return error`(@TempDir tempDir: File) { + val nonExistingSpec = """ + { + "contractFile": "invalid_file_path.yaml" + } + """.trimIndent() + + val nonExistingSpecResponse = exampleHtmlPageRequest(null, nonExistingSpec) + assertThat(nonExistingSpecResponse.status).isEqualTo(HttpStatusCode.BadRequest.value) + assertThat(nonExistingSpecResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val nonExistingSpecBody = nonExistingSpecResponse.body as JSONObjectValue + assertThat(nonExistingSpecBody.findFirstChildByPath("error")!!.toStringLiteral()) + .isEqualTo("Contract file not provided or does not exist, Please provide one via HTTP request or command line") + + val invalidSpec = tempDir.resolve("spec.graphql") + invalidSpec.createNewFile() + invalidSpec.writeText(nonExistingSpec) + + val invalidSpecRequest = """ + { + "contractFile": ${invalidSpec.absolutePath.quote()} + } + """.trimIndent() + + val invalidSpecResponse = exampleHtmlPageRequest(null, invalidSpecRequest) + assertThat(invalidSpecResponse.status).isEqualTo(HttpStatusCode.BadRequest.value) + assertThat(invalidSpecResponse.body).isInstanceOf(JSONObjectValue::class.java) + + val invalidSpecBody = invalidSpecResponse.body as JSONObjectValue + assertThat(invalidSpecBody.findFirstChildByPath("error")!!.toStringLiteral()) + .isEqualTo("Invalid Contract file ${invalidSpec.absolutePath} - File extension must be one of yaml, yml, json") + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt index 9a64dd13d..887407236 100644 --- a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt +++ b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt @@ -94,7 +94,7 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti return contractFile?.let { success(it) } ?: run { - val errorMessage = "Contract file not provided, Please provide one via HTTP request or command line" + val errorMessage = "Contract file not provided or does not exist, Please provide one via HTTP request or command line" consoleLog(errorMessage) failure(IllegalArgumentException(errorMessage)) } @@ -175,8 +175,8 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti private fun Routing.getHtmlPageRoute() { get("/_specmatic/examples") { - handleContractFile { - val htmlContent = getHtmlContent(it, getServerHostAndPort()) + handleContractFile { contract -> + val htmlContent = getHtmlContent(contract, getServerHostAndPort()) call.respondText(htmlContent, contentType = ContentType.Text.Html) } } From f72df868b620a2dbadcda803997db401932fbd98 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 17 Oct 2024 12:08:51 +0530 Subject: [PATCH 38/43] Implement Ability to load multiple existing exs. - [WIP] Example generation no longer checks for existing examples in interactive server. - Modifications to tableRow Generation, pre-calculate rowSpans for table rows. --- .../exampleGeneration/ExamplesGenerateBase.kt | 21 +++--- .../ExamplesInteractiveBase.kt | 19 ++---- .../OpenApiExamplesGenerate.kt | 4 +- .../OpenApiExamplesInteractive.kt | 65 +++++++++++++------ .../resources/templates/example/index.html | 15 ++--- 5 files changed, 71 insertions(+), 53 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt index d44e50ba8..e17ee78d0 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt @@ -45,8 +45,8 @@ abstract class ExamplesGenerateBase( } val exampleFiles = getExternalExampleFiles(examplesDir) - return filteredScenarios.map { scenario -> - generationStrategy.generateOrGetExistingExample( + return filteredScenarios.flatMap { scenario -> + generationStrategy.generateOrGetExistingExamples( ExamplesGenerationStrategy.GenerateOrGetExistingExampleArgs( feature, scenario, featureStrategy.getScenarioDescription(scenario), @@ -80,27 +80,28 @@ abstract class ExamplesGenerateBase( } interface ExamplesGenerationStrategy { - fun getExistingExampleOrNull(scenario: Scenario, exampleFiles: List): Pair? + fun getExistingExamples(scenario: Scenario, exampleFiles: List): List> fun generateExample(feature: Feature, scenario: Scenario): Pair - fun generateOrGetExistingExample(request: GenerateOrGetExistingExampleArgs): ExampleGenerationResult { + fun generateOrGetExistingExamples(request: GenerateOrGetExistingExampleArgs): List { return try { - val existingExample = getExistingExampleOrNull(request.scenario, request.exampleFiles) + val existingExamples = getExistingExamples(request.scenario, request.exampleFiles) val scenarioDescription = request.scenarioDescription - if (existingExample != null) { - consoleLog("Using existing example for ${scenarioDescription}\nExample File: ${existingExample.first.absolutePath}") - return ExampleGenerationResult(existingExample.first, ExampleGenerationStatus.EXISTS) + if (existingExamples.isNotEmpty()) { + consoleLog("Using existing example(s) for $scenarioDescription") + existingExamples.forEach { consoleLog("Example File: ${it.first.absolutePath}\"") } + return existingExamples.map { ExampleGenerationResult(it.first, ExampleGenerationStatus.EXISTS) } } consoleLog("Generating example for $scenarioDescription") val (uniqueFileName, exampleContent) = generateExample(request.feature, request.scenario) - return writeExampleToFile(exampleContent, uniqueFileName, request.examplesDir, request.validExampleExtensions) + return listOf(writeExampleToFile(exampleContent, uniqueFileName, request.examplesDir, request.validExampleExtensions)) } catch (e: Throwable) { consoleLog("Failed to generate example: ${e.message}") consoleDebug(e) - ExampleGenerationResult(null, ExampleGenerationStatus.ERROR) + listOf(ExampleGenerationResult(null, ExampleGenerationStatus.ERROR)) } } diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index 6f4da872b..b9c007ca6 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -59,17 +59,10 @@ abstract class ExamplesInteractiveBase ( override suspend fun generateExample(call: ApplicationCall, contractFile: File): ExampleGenerationResult { val feature = featureStrategy.contractFileToFeature(contractFile) val examplesDir = getExamplesDirectory(contractFile) - val exampleFiles = getExternalExampleFiles(examplesDir) return getScenarioFromRequestOrNull(call, feature)?.let { - generationStrategy.generateOrGetExistingExample( - ExamplesGenerationStrategy.GenerateOrGetExistingExampleArgs( - feature, it, - featureStrategy.getScenarioDescription(it), - exampleFiles, examplesDir, - featureStrategy.exampleFileExtensions - ) - ) + val (exampleFileName, exampleContent) = generationStrategy.generateExample(feature, scenario = it) + generationStrategy.writeExampleToFile(exampleContent, exampleFileName, examplesDir, featureStrategy.exampleFileExtensions) } ?: throw IllegalArgumentException("No matching scenario found for request") } @@ -93,10 +86,10 @@ abstract class ExamplesInteractiveBase ( val examplesDir = getExamplesDirectory(contractFile) val examples = getExternalExampleFiles(examplesDir) - val scenarioExamplePair = scenarios.map { - it to generationStrategy.getExistingExampleOrNull(it, examples)?.let { exRes -> - ExampleValidationResult(exRes.first, exRes.second) - } + val scenarioExamplePair = scenarios.flatMap { + generationStrategy.getExistingExamples(it, examples).map { exRes -> + it to ExampleValidationResult(exRes.first, exRes.second) + }.ifEmpty { listOf(it to null) } } return createTableRows(scenarioExamplePair) } diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt index d3e1a30cd..2a0473323 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt @@ -34,9 +34,9 @@ class OpenApiExamplesGenerationStrategy: ExamplesGenerationStrategy): Pair? { + override fun getExistingExamples(scenario: Scenario, exampleFiles: List): List> { val examples = exampleFiles.toExamples() - return examples.firstNotNullOfOrNull { example -> + return examples.mapNotNull { example -> val response = example.response when (val matchResult = scenario.matchesMock(example.request, response)) { diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index 0ecf623cc..d81c7d80a 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -50,31 +50,42 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( override fun createTableRows(scenarioExamplePair: List>): List { val groupedScenarios = scenarioExamplePair.sortScenarios().groupScenarios() - return groupedScenarios.flatMap { (_, methodMap) -> - val pathSpan = methodMap.values.sumOf { it.size } - val methodSet: MutableSet = mutableSetOf() + return groupedScenarios.flatMap { (path, pathGroup) -> var showPath = true - - methodMap.flatMap { (method, scenarios) -> - scenarios.map { (scenario, example) -> - ExampleTableRow( - columns = listOf( - ExampleRowGroup("path", convertPathParameterStyle(scenario.path), rawValue = scenario.path, rowSpan = pathSpan, showRow = showPath), - ExampleRowGroup("method", scenario.method, showRow = !methodSet.contains(method), rowSpan = scenarios.size), - ExampleRowGroup("response", scenario.status.toString(), showRow = true, rowSpan = 1, extraInfo = scenario.httpRequestPattern.headersPattern.contentType) - ), - exampleFilePath = example?.exampleFile?.absolutePath, - exampleFileName = example?.exampleName, - exampleMismatchReason = example?.result?.reportString().takeIf { reason -> reason?.isNotBlank() == true } - ).also { methodSet.add(method); showPath = false } + pathGroup.methods.flatMap { (method, methodGroup) -> + var showMethod = true + methodGroup.statuses.flatMap { (status, statusGroup) -> + var showStatus = true + statusGroup.examples.map { (scenario, example) -> + ExampleTableRow( + columns = listOf( + ExampleRowGroup("path", convertPathParameterStyle(path), rawValue = path, rowSpan = pathGroup.count, showRow = showPath), + ExampleRowGroup("method", method, rowSpan = methodGroup.count, showRow = showMethod), + ExampleRowGroup("response", status, rowSpan = statusGroup.count, showRow = showStatus, extraInfo = scenario.httpRequestPattern.headersPattern.contentType) + ), + exampleFilePath = example?.exampleFile?.absolutePath, + exampleFileName = example?.exampleName, + exampleMismatchReason = example?.result?.reportString().takeIf { reason -> reason?.isNotBlank() == true } + ).also { showPath = false; showMethod = false; showStatus = false } + } } } } } - private fun List>.groupScenarios(): Map>>> { - return this.groupBy { it.first.path }.mapValues { pathGroup -> - pathGroup.value.groupBy { it.first.method } + private fun List>.groupScenarios(): Map { + return this.groupBy { it.first.path }.mapValues { (_, pathGroup) -> + PathGroup( + count = pathGroup.size, + methods = pathGroup.groupBy { it.first.method }.mapValues { (_, methodGroup) -> + MethodGroup( + count = methodGroup.size, + statuses = methodGroup.groupBy { it.first.status.toString() }.mapValues { (_, statusGroup) -> + StatusGroup(count = statusGroup.size, examples = statusGroup) + } + ) + } + ) } } @@ -91,4 +102,20 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( ) { val contentType = response.extraInfo } + + data class StatusGroup ( + val count: Int, + val examples: List> + ) + + data class MethodGroup ( + val count: Int, + val statuses: Map + ) + + data class PathGroup ( + val count: Int, + val methods: Map + ) + } \ No newline at end of file diff --git a/core/src/main/resources/templates/example/index.html b/core/src/main/resources/templates/example/index.html index 1aae29741..3eb983b5b 100644 --- a/core/src/main/resources/templates/example/index.html +++ b/core/src/main/resources/templates/example/index.html @@ -602,7 +602,7 @@ background-color: rgb(var(--slate)); } - & > td:nth-child(n+3):nth-last-child(n+4) { + &[data-generate="success"] > td:nth-child(n+3):nth-last-child(n+3), & > td:nth-child(n+3):nth-last-child(n+4) { background-color: rgb(var(--white)); word-break: break-word; pointer-events: none; @@ -1019,10 +1019,8 @@

- - From f6affbdcbe7cf8cc0b0ac0d1755ac3339429daf7 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Fri, 18 Oct 2024 16:05:27 +0530 Subject: [PATCH 39/43] [WIP] Major JS refactor, ability to gen multi exs. - Rewrite Interactive JS to support multi example - Refactor to template and CSS styling - Generate Button needs to be moved - [WIP] Generate Button needs to be moved --- .../ExamplesInteractiveBase.kt | 32 +- .../examples/ExamplesInteractiveServer.kt | 15 +- .../examples/InteractiveServerProvider.kt | 26 +- .../resources/templates/example/index.html | 848 +++++++++--------- 4 files changed, 484 insertions(+), 437 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index b9c007ca6..bed9336a1 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -23,6 +23,7 @@ abstract class ExamplesInteractiveBase ( @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) protected var dictFile: File? = null + override val multiGenerate: Boolean = false override val serverHost: String = "0.0.0.0" override val serverPort: Int = 9001 abstract val server: ExamplesInteractiveServer @@ -56,14 +57,24 @@ abstract class ExamplesInteractiveBase ( abstract fun testExternalExample(feature: Feature, exampleFile: File, testBaseUrl: String): Pair // HELPER METHODS - override suspend fun generateExample(call: ApplicationCall, contractFile: File): ExampleGenerationResult { + override suspend fun generateExample(call: ApplicationCall, contractFile: File): List { val feature = featureStrategy.contractFileToFeature(contractFile) val examplesDir = getExamplesDirectory(contractFile) - - return getScenarioFromRequestOrNull(call, feature)?.let { - val (exampleFileName, exampleContent) = generationStrategy.generateExample(feature, scenario = it) - generationStrategy.writeExampleToFile(exampleContent, exampleFileName, examplesDir, featureStrategy.exampleFileExtensions) - } ?: throw IllegalArgumentException("No matching scenario found for request") + val exampleFiles = getExternalExampleFiles(examplesDir) + val scenario = getScenarioFromRequestOrNull(call, feature) + ?: throw IllegalArgumentException("No scenario found for request") + + val examples = generationStrategy.generateOrGetExistingExamples( + ExamplesGenerationStrategy.GenerateOrGetExistingExampleArgs( + feature = feature, scenario = scenario, + scenarioDescription = featureStrategy.getScenarioDescription(scenario), + exampleFiles = exampleFiles, examplesDir = examplesDir, + validExampleExtensions = featureStrategy.exampleFileExtensions + ) + ) + + val anyCreatedOrFailed = examples.any { it.status != ExampleGenerationStatus.EXISTS } + return examples.takeIf { anyCreatedOrFailed } ?: examples.plus(generateExampleSkipChecking(feature, scenario, examplesDir)) } override fun validateExample(contractFile: File, exampleFile: File): ExampleValidationResult { @@ -87,13 +98,18 @@ abstract class ExamplesInteractiveBase ( val examples = getExternalExampleFiles(examplesDir) val scenarioExamplePair = scenarios.flatMap { - generationStrategy.getExistingExamples(it, examples).map { exRes -> + listOf(it to null) + generationStrategy.getExistingExamples(it, examples).map { exRes -> it to ExampleValidationResult(exRes.first, exRes.second) - }.ifEmpty { listOf(it to null) } + } } return createTableRows(scenarioExamplePair) } + private fun generateExampleSkipChecking(feature: Feature, scenario: Scenario, examplesDir: File): ExampleGenerationResult { + val (fileName, exampleContent) = generationStrategy.generateExample(feature, scenario) + return generationStrategy.writeExampleToFile(exampleContent, fileName, examplesDir, featureStrategy.exampleFileExtensions) + } + private fun addShutdownHook(server: ExamplesInteractiveServer, latch: CountDownLatch) { Runtime.getRuntime().addShutdownHook(Thread { consoleLog("Shutdown signal received (Ctrl + C).") diff --git a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt index 887407236..2588966fa 100644 --- a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt +++ b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt @@ -134,9 +134,10 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti "contractFileName" to contractFile.name, "contractFilePath" to contractFile.absolutePath, "hostPort" to hostPort, - "hasExamples" to tableRows.any { it.exampleFilePath != null }, - "validationDetails" to tableRows.mapIndexed { index, row -> index.inc() to row.exampleMismatchReason }.toMap(), - "isTestMode" to (sutBaseUrl != null) + "hasExamples" to tableRows.any { it.isGenerated }, + "exampleDetails" to tableRows.transform(), + "isTestMode" to (sutBaseUrl != null), + "multiGenerate" to multiGenerate ) return HtmlTemplateConfiguration.process("example/index.html", variables) @@ -194,7 +195,7 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti post("/_specmatic/examples/generate") { handleContractFile { contractFile -> val result = generateExample(call, contractFile) - call.respond(HttpStatusCode.OK, ExampleGenerationResponse(result)) + call.respond(HttpStatusCode.OK, ExampleGenerationResponseList.from(result)) } } } @@ -235,4 +236,10 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti } } } + + private fun List.transform(): Map> { + return this.groupBy { it.uniqueKey }.mapValues { (_, keyGroup) -> + keyGroup.associateBy({ it.exampleFilePath ?: "null" }, { it.exampleMismatchReason }) + } + } } \ No newline at end of file diff --git a/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt b/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt index 78e3a689b..14d5b3e05 100644 --- a/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt +++ b/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt @@ -8,11 +8,12 @@ interface InteractiveServerProvider { val serverHost: String val serverPort: Int val sutBaseUrl: String? + val multiGenerate: Boolean val contractFile: File? val exampleTableColumns: List - suspend fun generateExample(call: ApplicationCall, contractFile: File): ExampleGenerationResult + suspend fun generateExample(call: ApplicationCall, contractFile: File): List fun validateExample(contractFile: File, exampleFile: File): ExampleValidationResult @@ -31,25 +32,35 @@ data class ExamplePageRequest ( ) data class ExampleGenerationResponse ( - val exampleFilePath: String, + val exampleFile: String, val status: String ) { constructor(result: ExampleGenerationResult): this( - exampleFilePath = result.exampleFile?.absolutePath ?: throw Exception("Failed to generate example file"), + exampleFile = result.exampleFile?.absolutePath ?: throw Exception("Failed to generate example file"), status = result.status.name ) } +data class ExampleGenerationResponseList ( + val examples: List +) { + companion object { + fun from(results: List): ExampleGenerationResponseList { + return ExampleGenerationResponseList(results.map { ExampleGenerationResponse(it) }) + } + } +} + data class ExampleValidationRequest ( val exampleFile: File ) data class ExampleValidationResponse ( - val exampleFilePath: String, + val exampleFile: String, val error: String? = null ) { constructor(result: ExampleValidationResult): this( - exampleFilePath = result.exampleName, error = result.result.reportString().takeIf { it.isNotBlank() } + exampleFile = result.exampleName, error = result.result.reportString().takeIf { it.isNotBlank() } ) } @@ -103,5 +114,8 @@ data class ExampleTableRow ( val columns: List, val exampleFilePath: String? = null, val exampleFileName: String? = null, - val exampleMismatchReason: String? = null + val exampleMismatchReason: String? = null, + val isGenerated: Boolean = exampleFileName != null, + val isValid: Boolean = isGenerated && exampleMismatchReason == null, + val uniqueKey: String = columns.joinToString("-") { it.rawValue } ) \ No newline at end of file diff --git a/core/src/main/resources/templates/example/index.html b/core/src/main/resources/templates/example/index.html index 3eb983b5b..4149ce6ee 100644 --- a/core/src/main/resources/templates/example/index.html +++ b/core/src/main/resources/templates/example/index.html @@ -493,7 +493,7 @@ --_content: "Validate Selected"; &[data-panel="details"] { - --_content: "Validate Examples"; + --_content: "Validate Example"; } } @@ -528,7 +528,7 @@ --_content: "Test Selected"; &[data-panel="details"] { - --_content: "Test Examples"; + --_content: "Test Example"; } } @@ -562,6 +562,7 @@ table-layout: auto; text-align: center; font-size: 0.99rem; + counter-reset: Serial; } thead { @@ -602,7 +603,7 @@ background-color: rgb(var(--slate)); } - &[data-generate="success"] > td:nth-child(n+3):nth-last-child(n+3), & > td:nth-child(n+3):nth-last-child(n+4) { + & > td:nth-child(n+3):nth-last-child(n+3) { background-color: rgb(var(--white)); word-break: break-word; pointer-events: none; @@ -633,6 +634,11 @@ } } + & > td:nth-child(2)::before { + content: counter(Serial); + counter-increment: Serial; + } + & > td:last-child { & > span { line-height: 1.5rem; @@ -702,6 +708,12 @@ } } } + + &[data-multi-gen="false"] > tr > td:nth-last-child(3) { + & > button { + display: none; + } + } } td:nth-child(3) { @@ -1018,25 +1030,25 @@

- - + + - + - + data-col-name=${group.columnName}, data-raw-value=${group.rawValue}" >

[[${group.value}]]

[[${group.extraInfo}]] - +
+
@@ -1014,39 +1005,33 @@

S. NoPathMethodResponse Examples Action
[[${iter.index + 1}]] - [[${row.path}]] - -

[[${row.method}]]

- 		-

[[${row.responseStatus}]]

- [[${row.contentType}]] + 		+

[[${group.value}]]

+ [[${group.extraInfo}]] 		- - + +

@@ -1078,7 +1063,6 @@

const testDetails = {}; - + \ No newline at end of file diff --git a/junit5-support/src/main/resources/templates/assets/badge.svg b/junit5-support/src/main/resources/templates/report/assets/badge.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/badge.svg rename to junit5-support/src/main/resources/templates/report/assets/badge.svg diff --git a/junit5-support/src/main/resources/templates/assets/blocked.svg b/junit5-support/src/main/resources/templates/report/assets/blocked.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/blocked.svg rename to junit5-support/src/main/resources/templates/report/assets/blocked.svg diff --git a/junit5-support/src/main/resources/templates/assets/check-badge.svg b/junit5-support/src/main/resources/templates/report/assets/check-badge.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/check-badge.svg rename to junit5-support/src/main/resources/templates/report/assets/check-badge.svg diff --git a/junit5-support/src/main/resources/templates/assets/clipboard-document-list.svg b/junit5-support/src/main/resources/templates/report/assets/clipboard-document-list.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/clipboard-document-list.svg rename to junit5-support/src/main/resources/templates/report/assets/clipboard-document-list.svg diff --git a/junit5-support/src/main/resources/templates/assets/clock.svg b/junit5-support/src/main/resources/templates/report/assets/clock.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/clock.svg rename to junit5-support/src/main/resources/templates/report/assets/clock.svg diff --git a/junit5-support/src/main/resources/templates/assets/download.svg b/junit5-support/src/main/resources/templates/report/assets/download.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/download.svg rename to junit5-support/src/main/resources/templates/report/assets/download.svg diff --git a/junit5-support/src/main/resources/templates/assets/exclamation-triangle.svg b/junit5-support/src/main/resources/templates/report/assets/exclamation-triangle.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/exclamation-triangle.svg rename to junit5-support/src/main/resources/templates/report/assets/exclamation-triangle.svg diff --git a/junit5-support/src/main/resources/templates/assets/favicon.svg b/junit5-support/src/main/resources/templates/report/assets/favicon.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/favicon.svg rename to junit5-support/src/main/resources/templates/report/assets/favicon.svg diff --git a/junit5-support/src/main/resources/templates/assets/main.js b/junit5-support/src/main/resources/templates/report/assets/main.js similarity index 100% rename from junit5-support/src/main/resources/templates/assets/main.js rename to junit5-support/src/main/resources/templates/report/assets/main.js diff --git a/junit5-support/src/main/resources/templates/assets/mark-approved.svg b/junit5-support/src/main/resources/templates/report/assets/mark-approved.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/mark-approved.svg rename to junit5-support/src/main/resources/templates/report/assets/mark-approved.svg diff --git a/junit5-support/src/main/resources/templates/assets/mark-rejected.svg b/junit5-support/src/main/resources/templates/report/assets/mark-rejected.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/mark-rejected.svg rename to junit5-support/src/main/resources/templates/report/assets/mark-rejected.svg diff --git a/junit5-support/src/main/resources/templates/assets/specmatic-logo.svg b/junit5-support/src/main/resources/templates/report/assets/specmatic-logo.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/specmatic-logo.svg rename to junit5-support/src/main/resources/templates/report/assets/specmatic-logo.svg diff --git a/junit5-support/src/main/resources/templates/assets/styles.css b/junit5-support/src/main/resources/templates/report/assets/styles.css similarity index 100% rename from junit5-support/src/main/resources/templates/assets/styles.css rename to junit5-support/src/main/resources/templates/report/assets/styles.css diff --git a/junit5-support/src/main/resources/templates/assets/summaryUpdater.js b/junit5-support/src/main/resources/templates/report/assets/summaryUpdater.js similarity index 100% rename from junit5-support/src/main/resources/templates/assets/summaryUpdater.js rename to junit5-support/src/main/resources/templates/report/assets/summaryUpdater.js diff --git a/junit5-support/src/main/resources/templates/assets/tableFilter.js b/junit5-support/src/main/resources/templates/report/assets/tableFilter.js similarity index 100% rename from junit5-support/src/main/resources/templates/assets/tableFilter.js rename to junit5-support/src/main/resources/templates/report/assets/tableFilter.js diff --git a/junit5-support/src/main/resources/templates/assets/trend-up.svg b/junit5-support/src/main/resources/templates/report/assets/trend-up.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/trend-up.svg rename to junit5-support/src/main/resources/templates/report/assets/trend-up.svg diff --git a/junit5-support/src/main/resources/templates/assets/utils.js b/junit5-support/src/main/resources/templates/report/assets/utils.js similarity index 100% rename from junit5-support/src/main/resources/templates/assets/utils.js rename to junit5-support/src/main/resources/templates/report/assets/utils.js diff --git a/junit5-support/src/main/resources/templates/assets/x-circle.svg b/junit5-support/src/main/resources/templates/report/assets/x-circle.svg similarity index 100% rename from junit5-support/src/main/resources/templates/assets/x-circle.svg rename to junit5-support/src/main/resources/templates/report/assets/x-circle.svg diff --git a/junit5-support/src/main/resources/templates/report.html b/junit5-support/src/main/resources/templates/report/index.html similarity index 100% rename from junit5-support/src/main/resources/templates/report.html rename to junit5-support/src/main/resources/templates/report/index.html From c1178dce7f925f3d27b3e7494ff8b46551494d3e Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 3 Oct 2024 16:37:06 +0530 Subject: [PATCH 21/43] Fix testRowExample method in interactive html js --- .../src/main/resources/templates/example/index.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/junit5-support/src/main/resources/templates/example/index.html b/junit5-support/src/main/resources/templates/example/index.html index 83e2078ff..0b64930dd 100644 --- a/junit5-support/src/main/resources/templates/example/index.html +++ b/junit5-support/src/main/resources/templates/example/index.html @@ -1405,9 +1405,7 @@

} const exampleFilePath = getExampleData(tableRow); - const {data, error} = await testExample({ - exampleFile: exampleFilePath - }); + const {data, error} = await testExample(exampleFilePath); tableRow.setAttribute("data-test", (error || data?.result !== "Success") ? "failed": "success"); testDetails[Number.parseInt(tableRow.children[1].textContent)] = data; From dc7d1c7c44e2fca5b97174891e5078945644c300 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Thu, 3 Oct 2024 17:47:46 +0530 Subject: [PATCH 22/43] CSS and JS alert fixes and improvements. - never show validation alert when testing. - fix main tag size and example name break. --- .../main/resources/templates/example/index.html | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/junit5-support/src/main/resources/templates/example/index.html b/junit5-support/src/main/resources/templates/example/index.html index 0b64930dd..edc6e1b3b 100644 --- a/junit5-support/src/main/resources/templates/example/index.html +++ b/junit5-support/src/main/resources/templates/example/index.html @@ -410,6 +410,10 @@ body { font-family: var(--roboto); + display: flex; + flex-direction: column; + min-height: 100vh; + min-width: 100vw; & > .chevron-down { display: none; @@ -708,8 +712,12 @@ td:nth-last-child(2) { max-width: 15rem; - word-break: break-word; - font-size: .85rem; + + & p { + white-space: normal; + word-break: break-word; + font-size: .85rem; + } } td p, td button { @@ -1397,7 +1405,7 @@

async function testRowExample(tableRow, bulkMode = false) { tableRow.setAttribute("data-test", "processing"); - const isExampleValid = await validateRowExamples(tableRow, bulkMode); + const isExampleValid = await validateRowExamples(tableRow, true); tableRow.setAttribute("data-valid", isExampleValid ? "success": "failed"); if (!isExampleValid) { From c0eb051931048b84337ee453517fbed1d48547c3 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Mon, 7 Oct 2024 14:14:24 +0530 Subject: [PATCH 23/43] Revamp Example Parity architecture. - Use multi-level inheritance with interfaces. - Introduced a common base command with shared functions and cmd line options. - UI, UX fixes on frontend, etc. --- .../kotlin/application/SpecmaticCommand.kt | 4 +- .../exampleGeneration/ExamplesBase.kt | 161 +++++++++++------- .../exampleGeneration/ExamplesCommon.kt | 152 ----------------- .../exampleGeneration/ExamplesGenerateBase.kt | 150 ++++++++++++++++ .../ExamplesInteractiveBase.kt | 150 +++++++--------- ...idationBase.kt => ExamplesValidateBase.kt} | 114 ++++++------- .../exampleGeneration/ScenarioFilter.kt | 11 -- .../openApiExamples/OpenApiExamples.kt | 18 -- .../openApiExamples/OpenApiExamplesCommon.kt | 126 +------------- .../OpenApiExamplesGenerate.kt | 66 +++++++ .../OpenApiExamplesInteractive.kt | 45 ++--- .../OpenApiExamplesValidate.kt | 84 ++++++++- .../resources/templates/example/index.html | 6 +- 13 files changed, 538 insertions(+), 549 deletions(-) delete mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesCommon.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt rename application/src/main/kotlin/application/exampleGeneration/{ExamplesValidationBase.kt => ExamplesValidateBase.kt} (55%) delete mode 100644 application/src/main/kotlin/application/exampleGeneration/ScenarioFilter.kt delete mode 100644 application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamples.kt create mode 100644 application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt diff --git a/application/src/main/kotlin/application/SpecmaticCommand.kt b/application/src/main/kotlin/application/SpecmaticCommand.kt index 639fb5a47..e3931bb3d 100644 --- a/application/src/main/kotlin/application/SpecmaticCommand.kt +++ b/application/src/main/kotlin/application/SpecmaticCommand.kt @@ -1,7 +1,7 @@ package application import application.backwardCompatibility.BackwardCompatibilityCheckCommandV2 -import application.exampleGeneration.openApiExamples.OpenApiExamples +import application.exampleGeneration.openApiExamples.OpenApiExamplesGenerate import org.springframework.stereotype.Component import picocli.AutoComplete.GenerateCompletion import picocli.CommandLine.Command @@ -13,7 +13,7 @@ import java.util.concurrent.Callable mixinStandardHelpOptions = true, versionProvider = VersionProvider::class, subcommands = [ - OpenApiExamples::class, + OpenApiExamplesGenerate::class, BackwardCompatibilityCheckCommandV2::class, BackwardCompatibilityCheckCommand::class, BundleCommand::class, diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt index 2ff6d1f73..ebd628712 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt @@ -1,98 +1,131 @@ package application.exampleGeneration -import io.specmatic.core.* +import io.specmatic.core.EXAMPLES_DIR_SUFFIX import io.specmatic.core.log.* -import picocli.CommandLine.* +import picocli.CommandLine import java.io.File import java.util.concurrent.Callable -abstract class ExamplesBase(private val common: ExamplesCommon): Callable { - @Parameters(index = "0", description = ["Contract file path"], arity = "0..1") - private var contractFile: File? = null +abstract class ExamplesBase : Callable, ExamplesCommon { + protected abstract var contractFile: File? - @Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") - var filterName: String = "" + @CommandLine.Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") + private var filterName: String = "" - @Option(names = ["--filter-not-name"], description = ["Use only APIs which do not have this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}") - var filterNotName: String = "" + @CommandLine.Option(names = ["--filter-not-name"], description = ["Use only APIs which do not have this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}") + private var filterNotName: String = "" - @Option(names = ["--dictionary"], description = ["External Dictionary File Path, defaults to dictionary.json"]) - private var dictFile: File? = null - - @Option(names = ["--debug"], description = ["Debug logs"]) + @CommandLine.Option(names = ["--debug"], description = ["Debug logs"]) private var verbose = false - abstract var extensive: Boolean - override fun call(): Int { - common.configureLogger(this.verbose) - - contractFile?.let { contract -> - if (!contract.exists()) { - logger.log("Could not find Contract file ${contract.path}") + contractFile?.let { + if (!it.exists()) { + logger.log("Contract file does not exist: ${it.absolutePath}") return 1 } - if (contract.extension !in common.contractFileExtensions) { - logger.log("Invalid Contract file ${contract.path} - File extension must be one of ${common.contractFileExtensions.joinToString()}") + if (it.extension !in contractFileExtensions) { + logger.log("Invalid Contract file ${it.path} - File extension must be one of ${contractFileExtensions.joinToString()}") return 1 } + } - try { - val externalDictionary = common.loadExternalDictionary(dictFile, contract) - val examplesDir = common.getExamplesDirectory(contract) - val result = generateExamples(contract, externalDictionary, examplesDir) - logGenerationResult(result, examplesDir) - return 0 - } catch (e: Throwable) { - logger.log("Example generation failed with error: ${e.message}") - logger.debug(e) - return 1 - } - } ?: run { - logger.log("No contract file provided. Use a subcommand or provide a contract file. Use --help for more details.") - return 1 + val exitCode = execute(contractFile) + return exitCode + } + + // HOOKS + abstract fun execute(contract: File?): Int + + // HELPER METHODS + fun configureLogger(verbose: Boolean) { + val logPrinters = listOf(ConsolePrinter) + + logger = if (verbose) + Verbose(CompositePrinter(logPrinters)) + else + NonVerbose(CompositePrinter(logPrinters)) + } + + fun getFilteredScenarios(feature: Feature, extensive: Boolean = false): List { + val scenarioFilter = ScenarioFilter(filterName, filterNotName) + val scenarios = getScenariosFromFeature(feature, extensive) + return getFilteredScenarios(scenarios, scenarioFilter) + } + + fun getExamplesDirectory(contractFile: File): File { + val examplesDirectory = contractFile.canonicalFile.parentFile.resolve("${contractFile.nameWithoutExtension}$EXAMPLES_DIR_SUFFIX") + if (!examplesDirectory.exists()) { + logger.log("Creating examples directory: $examplesDirectory") + examplesDirectory.mkdirs() } + return examplesDirectory } - // GENERATION METHODS - private fun getFilteredScenarios(feature: Feature): List { - val scenarioFilter = ScenarioFilter(filterName, filterNotName, extensive) - return common.getFilteredScenarios(feature, scenarioFilter) + fun getExternalExampleFiles(examplesDirectory: File): List { + return examplesDirectory.walk().filter { it.isFile && it.extension in exampleFileExtensions }.toList() } - private fun generateExamples(contractFile: File, externalDictionary: Dictionary, examplesDir: File): List { - val feature = common.contractFileToFeature(contractFile) - val filteredScenarios = getFilteredScenarios(feature) + fun logSeparator(length: Int, separator: String = "-") { + logger.log(separator.repeat(length)) + } + + fun logFormattedOutput(header: String, summary: String, note: String) { + val maxLength = maxOf(summary.length, note.length, 50).let { it + it % 2 } + val headerSidePadding = (maxLength - 2 - header.length) / 2 + + val paddedHeaderLine = "=".repeat(headerSidePadding) + " $header " + "=".repeat(headerSidePadding) + val paddedSummaryLine = summary.padEnd(maxLength) + val paddedNoteLine = note.padEnd(maxLength) + + logger.log("\n$paddedHeaderLine") + logger.log(paddedSummaryLine) + logger.log("=".repeat(maxLength)) + logger.log(paddedNoteLine) + } + + private fun getFilteredScenarios(scenarios: List, scenarioFilter: ScenarioFilter): List { + val filteredScenarios = scenarios + .filterScenarios(scenarioFilter.filterNameTokens, shouldMatch = true) + .filterScenarios(scenarioFilter.filterNotNameTokens, shouldMatch = false) if (filteredScenarios.isEmpty()) { - return emptyList() + logger.log("Note: All examples were filtered out by the filter expression") } - val exampleFiles = common.getExternalExampleFiles(examplesDir) - return filteredScenarios.map { scenario -> - common.generateOrGetExistingExample(feature, scenario, externalDictionary, exampleFiles, examplesDir) - } + return filteredScenarios } - // HELPERS - private fun logGenerationResult(generations: List, examplesDir: File) { - val generationGroup = generations.groupBy { it.status }.mapValues { it.value.size } - val createdFileCount = generationGroup[ExampleGenerationStatus.CREATED] ?: 0 - val errorCount = generationGroup[ExampleGenerationStatus.ERROR] ?: 0 - val existingCount = generationGroup[ExampleGenerationStatus.EXISTS] ?: 0 - val examplesDirectory = examplesDir.canonicalFile.absolutePath - - common.logFormattedOutput( - header = "Example Generation Summary", - summary = "$createdFileCount example(s) created, $existingCount example(s) already existed, $errorCount example(s) failed", - note = "NOTE: All examples can be found in $examplesDirectory" - ) + private fun List.filterScenarios(tokens: Set, shouldMatch: Boolean): List { + if (tokens.isEmpty()) return this + + return this.filter { + val description = getScenarioDescription(it) + tokens.any { token -> + description.contains(token) + } == shouldMatch + } } } -enum class ExampleGenerationStatus { - CREATED, ERROR, EXISTS +interface ExamplesCommon { + val exampleFileExtensions: Set + val contractFileExtensions: Set + + fun contractFileToFeature(contractFile: File): Feature + + fun getScenariosFromFeature(feature: Feature, extensive: Boolean) : List + + fun getScenarioDescription(scenario: Scenario): String } -data class ExampleGenerationResult(val exampleFile: File? = null, val status: ExampleGenerationStatus) \ No newline at end of file +class ScenarioFilter(filterName: String, filterNotName: String) { + val filterNameTokens = filterToTokens(filterName) + val filterNotNameTokens = filterToTokens(filterNotName) + + private fun filterToTokens(filterValue: String): Set { + if (filterValue.isBlank()) return emptySet() + return filterValue.split(",").map { it.trim() }.toSet() + } +} diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesCommon.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesCommon.kt deleted file mode 100644 index cbe6f4f0c..000000000 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesCommon.kt +++ /dev/null @@ -1,152 +0,0 @@ -package application.exampleGeneration - -import io.specmatic.core.* -import io.specmatic.core.log.* -import io.specmatic.mock.loadDictionary -import java.io.File - -interface ExamplesCommon { - val exampleFileExtensions: Set - val contractFileExtensions: Set - - fun contractFileToFeature(contractFile: File): Feature - - fun getScenariosFromFeature(feature: Feature, extensive: Boolean) : List - - fun getScenarioDescription(scenario: Scenario): String - - fun getExistingExampleOrNull(scenario: Scenario, exampleFiles: List): Pair? - - fun generateExample(feature: Feature, scenario: Scenario, dictionary: Dictionary): Pair - - fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature - - fun validateExternalExample(feature: Feature, exampleFile: File): Pair - - fun configureLogger(verbose: Boolean) { - val logPrinters = listOf(ConsolePrinter) - - logger = if (verbose) - Verbose(CompositePrinter(logPrinters)) - else - NonVerbose(CompositePrinter(logPrinters)) - } - - fun getFilteredScenarios(feature: Feature, scenarioFilter: ScenarioFilter): List { - val filteredScenarios = getScenariosFromFeature(feature, scenarioFilter.extensive) - .filterScenarios(scenarioFilter.filterNameTokens, shouldMatch = true) - .filterScenarios(scenarioFilter.filterNotNameTokens, shouldMatch = false) - - if (filteredScenarios.isEmpty()) { - logger.log("Note: All examples were filtered out by the filter expression") - } - - return filteredScenarios - } - - fun getExamplesDirectory(contractFile: File): File { - val examplesDirectory = contractFile.canonicalFile.parentFile.resolve("${contractFile.nameWithoutExtension}$EXAMPLES_DIR_SUFFIX") - if (!examplesDirectory.exists()) { - logger.log("Creating examples directory: $examplesDirectory") - examplesDirectory.mkdirs() - } - return examplesDirectory - } - - fun getExternalExampleFiles(examplesDirectory: File): List { - return examplesDirectory.walk().filter { it.isFile && it.extension in exampleFileExtensions }.toList() - } - - fun generateOrGetExistingExample(feature: Feature, scenario: Scenario, externalDictionary: Dictionary, exampleFiles: List, examplesDir: File): ExampleGenerationResult { - return try { - val existingExample = getExistingExampleOrNull(scenario, exampleFiles) - val description = getScenarioDescription(scenario) - - if (existingExample != null) { - logger.log("Using existing example for $description\nExample File: ${existingExample.first.absolutePath}") - return ExampleGenerationResult(existingExample.first, ExampleGenerationStatus.EXISTS) - } - - logger.log("Generating example for $description") - val (uniqueFileName, exampleContent) = generateExample(feature, scenario, externalDictionary) - return writeExampleToFile(exampleContent, uniqueFileName, examplesDir) - } catch (e: Throwable) { - logger.log("Failed to generate example: ${e.message}") - logger.debug(e) - ExampleGenerationResult(null, ExampleGenerationStatus.ERROR) - } finally { - logSeparator(50) - } - } - - fun writeExampleToFile(exampleContent: String, exampleFileName: String, examplesDir: File): ExampleGenerationResult { - val exampleFile = examplesDir.resolve(exampleFileName) - - if (exampleFile.extension !in exampleFileExtensions) { - logger.log("Invalid example file extension: ${exampleFile.extension}") - return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) - } - - try { - exampleFile.writeText(exampleContent) - logger.log("Successfully saved example: $exampleFile") - return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.CREATED) - } catch (e: Throwable) { - logger.log("Failed to save example: $exampleFile") - logger.debug(e) - return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) - } - } - - fun loadExternalDictionary(dictFile: File?, contractFile: File?): Dictionary { - val dictFilePath = when { - dictFile != null -> { - if (!dictFile.exists()) throw IllegalStateException("Dictionary file not found: ${dictFile.path}") - else dictFile.path - } - - contractFile != null -> { - val dictFileName = "${contractFile.nameWithoutExtension}$DICTIONARY_FILE_SUFFIX" - contractFile.canonicalFile.parentFile.resolve(dictFileName).takeIf { it.exists() }?.path - } - - else -> { - val currentDir = File(System.getProperty("user.dir")) - currentDir.resolve("dictionary.json").takeIf { it.exists() }?.path - } - } - - return dictFilePath?.let { - Dictionary(loadDictionary(dictFilePath)) - } ?: Dictionary(emptyMap()) - } - - fun logSeparator(length: Int, separator: String = "-") { - logger.log(separator.repeat(length)) - } - - fun logFormattedOutput(header: String, summary: String, note: String) { - val maxLength = maxOf(summary.length, note.length, 50).let { it + it % 2 } - val headerSidePadding = (maxLength - 2 - header.length) / 2 - - val paddedHeaderLine = "=".repeat(headerSidePadding) + " $header " + "=".repeat(headerSidePadding) - val paddedSummaryLine = summary.padEnd(maxLength) - val paddedNoteLine = note.padEnd(maxLength) - - logger.log("\n$paddedHeaderLine") - logger.log(paddedSummaryLine) - logger.log("=".repeat(maxLength)) - logger.log(paddedNoteLine) - } - - private fun List.filterScenarios(tokens: Set, shouldMatch: Boolean): List { - if (tokens.isEmpty()) return this - - return this.filter { - val description = getScenarioDescription(it) - tokens.any { token -> - description.contains(token) - } == shouldMatch - } - } -} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt new file mode 100644 index 000000000..77fb6b14d --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt @@ -0,0 +1,150 @@ +package application.exampleGeneration + +import io.specmatic.core.* +import io.specmatic.core.log.* +import io.specmatic.mock.loadDictionary +import picocli.CommandLine.* +import java.io.File + +abstract class ExamplesGenerateBase: ExamplesBase(), ExamplesGenerateCommon { + @Parameters(index = "0", description = ["Contract file path"], arity = "0..1") + override var contractFile: File? = null + + @Option(names = ["--dictionary"], description = ["External Dictionary File Path, defaults to dictionary.json"]) + private var dictFile: File? = null + + abstract var extensive: Boolean + + override fun execute(contract: File?): Int { + if (contract == null) { + logger.log("No contract file provided. Use a subcommand or provide a contract file. Use --help for more details.") + return 1 + } + + try { + val externalDictionary = loadExternalDictionary(dictFile, contract) + val examplesDir = getExamplesDirectory(contract) + val result = generateExamples(contract, externalDictionary, examplesDir) + logGenerationResult(result, examplesDir) + return 0 + } catch (e: Throwable) { + logger.log("Example generation failed with error: ${e.message}") + logger.debug(e) + return 1 + } + } + + // GENERATOR METHODS + private fun generateExamples(contractFile: File, externalDictionary: Dictionary, examplesDir: File): List { + val feature = contractFileToFeature(contractFile) + val filteredScenarios = getFilteredScenarios(feature, extensive) + + if (filteredScenarios.isEmpty()) { + return emptyList() + } + + val exampleFiles = getExternalExampleFiles(examplesDir) + return filteredScenarios.map { scenario -> + generateOrGetExistingExample(feature, scenario, externalDictionary, exampleFiles, examplesDir).also { + logSeparator(75) + } + } + } + + // HELPER METHODS + private fun logGenerationResult(generations: List, examplesDir: File) { + val generationGroup = generations.groupBy { it.status }.mapValues { it.value.size } + val createdFileCount = generationGroup[ExampleGenerationStatus.CREATED] ?: 0 + val errorCount = generationGroup[ExampleGenerationStatus.ERROR] ?: 0 + val existingCount = generationGroup[ExampleGenerationStatus.EXISTS] ?: 0 + val examplesDirectory = examplesDir.canonicalFile.absolutePath + + logFormattedOutput( + header = "Example Generation Summary", + summary = "$createdFileCount example(s) created, $existingCount example(s) already existed, $errorCount example(s) failed", + note = "NOTE: All examples can be found in $examplesDirectory" + ) + } +} + +enum class ExampleGenerationStatus(val value: String) { + CREATED("Inline"), + ERROR("External"), + EXISTS("Exists"); + + override fun toString(): String { + return this.value + } +} + +data class ExampleGenerationResult(val exampleFile: File? = null, val status: ExampleGenerationStatus) + +interface ExamplesGenerateCommon : ExamplesCommon { + fun getExistingExampleOrNull(scenario: Scenario, exampleFiles: List): Pair? + + fun generateExample(feature: Feature, scenario: Scenario, dictionary: Dictionary): Pair + + fun loadExternalDictionary(dictFile: File?, contractFile: File?): Dictionary { + val dictFilePath = when(dictFile != null) { + true -> { + dictFile.takeIf { it.exists() }?.path ?: throw IllegalStateException("Dictionary file not found: ${dictFile.path}") + } + + false -> { + val contractDictFile = contractFile?.let { contract -> + val contractDictFile = "${contract.nameWithoutExtension}$DICTIONARY_FILE_SUFFIX" + contract.canonicalFile.parentFile.resolve(contractDictFile).takeIf { it.exists() }?.path + } + + val currentDirDictFile = File(System.getProperty("user.dir")).resolve("dictionary.json").takeIf { + it.exists() + }?.path + + contractDictFile ?: currentDirDictFile + } + } + + return dictFilePath?.let { + Dictionary(loadDictionary(dictFilePath)) + } ?: Dictionary(emptyMap()) + } + + fun generateOrGetExistingExample(feature: Feature, scenario: Scenario, externalDictionary: Dictionary, exampleFiles: List, examplesDir: File): ExampleGenerationResult { + return try { + val existingExample = getExistingExampleOrNull(scenario, exampleFiles) + val description = getScenarioDescription(scenario) + + if (existingExample != null) { + logger.log("Using existing example for $description\nExample File: ${existingExample.first.absolutePath}") + return ExampleGenerationResult(existingExample.first, ExampleGenerationStatus.EXISTS) + } + + logger.log("Generating example for $description") + val (uniqueFileName, exampleContent) = generateExample(feature, scenario, externalDictionary) + return writeExampleToFile(exampleContent, uniqueFileName, examplesDir) + } catch (e: Throwable) { + logger.log("Failed to generate example: ${e.message}") + logger.debug(e) + ExampleGenerationResult(null, ExampleGenerationStatus.ERROR) + } + } + + fun writeExampleToFile(exampleContent: String, exampleFileName: String, examplesDir: File): ExampleGenerationResult { + val exampleFile = examplesDir.resolve(exampleFileName) + + if (exampleFile.extension !in exampleFileExtensions) { + logger.log("Invalid example file extension: ${exampleFile.extension}") + return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) + } + + try { + exampleFile.writeText(exampleContent) + logger.log("Successfully saved example: $exampleFile") + return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.CREATED) + } catch (e: Throwable) { + logger.log("Failed to save example: $exampleFile") + logger.debug(e) + return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) + } + } +} diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index defdcbf25..81ad4c662 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -21,49 +21,28 @@ import java.io.Closeable import java.io.File import java.io.FileNotFoundException import java.lang.Thread.sleep -import java.util.concurrent.Callable -abstract class ExamplesInteractiveBase(val common: ExamplesCommon): Callable { +abstract class ExamplesInteractiveBase: ExamplesBase(), ExamplesGenerateCommon, ExamplesValidateCommon { @Option(names = ["--testBaseURL"], description = ["BaseURL of the SUT"], required = true) - lateinit var serverHost: String + lateinit var sutBaseUrl: String @Option(names = ["--contract-file"], description = ["Contract file path"], required = false) - var contractFile: File? = null - - @Option(names = ["--debug"], description = ["Debug logs"]) - var verbose = false + override var contractFile: File? = null @Option(names = ["--dictionary"], description = ["External Dictionary File Path"]) var dictFile: File? = null - @Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") - var filterName: String = "" - - @Option(names = ["--filter-not-name"], description = ["Use only APIs which do not have this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}") - var filterNotName: String = "" - abstract var extensive: Boolean abstract val htmlTableColumns: List private var cachedContractFileFromRequest: File? = null - override fun call(): Int { - common.configureLogger(verbose) - + override fun execute(contract: File?): Int { try { - contractFile?.let { contract -> - if (!contract.exists()) { - logger.log("Could not find Contract file ${contract.path}") - return 1 - } - - if (contract.extension !in common.contractFileExtensions) { - logger.log("Invalid Contract file ${contract.path} - File extension must be one of ${common.contractFileExtensions.joinToString()}") - return 1 - } - - } ?: logger.log("No contract file provided, Please provide a contract file in the HTTP request.") + if (contract == null) { + logger.log("Contract file not provided, Please provide one via HTTP request") + } - val server = InteractiveServer("0.0.0.0", 9001) + val server = InteractiveServer(contract, "0.0.0.0", 9001) addShutdownHook(server) logger.log("Examples Interactive server is running on http://0.0.0.0:9001/_specmatic/examples. Ctrl + C to stop.") while (true) sleep(10000) @@ -75,7 +54,7 @@ abstract class ExamplesInteractiveBase(val common: ExamplesCo } // HOOKS - abstract fun createTableRows(scenarios: List, exampleFiles: List): List + abstract fun createTableRows(scenarioExamplePair: List>): List abstract suspend fun getScenarioFromRequestOrNull(call: ApplicationCall, feature: Feature): Scenario? @@ -83,34 +62,29 @@ abstract class ExamplesInteractiveBase(val common: ExamplesCo // HELPER METHODS private suspend fun generateExample(call: ApplicationCall, contractFile: File): ExampleGenerationResult { - val feature = common.contractFileToFeature(contractFile) - val dictionary = common.loadExternalDictionary(dictFile, contractFile) - val examplesDir = common.getExamplesDirectory(contractFile) - val exampleFiles = common.getExternalExampleFiles(examplesDir) + val feature = contractFileToFeature(contractFile) + val dictionary = loadExternalDictionary(dictFile, contractFile) + val examplesDir = getExamplesDirectory(contractFile) + val exampleFiles = getExternalExampleFiles(examplesDir) return getScenarioFromRequestOrNull(call, feature)?.let { - common.generateOrGetExistingExample(feature, it, dictionary, exampleFiles, examplesDir) + generateOrGetExistingExample(feature, it, dictionary, exampleFiles, examplesDir) } ?: throw IllegalArgumentException("No matching scenario found for request") } private fun validateExample(contractFile: File, exampleFile: File): ExampleValidationResult { - val feature = common.contractFileToFeature(contractFile) - val result = common.validateExternalExample(feature, exampleFile) + val feature = contractFileToFeature(contractFile) + val result = validateExternalExample(feature, exampleFile) return ExampleValidationResult(exampleFile.absolutePath, result.second, ExampleType.EXTERNAL) } - private fun testExample(contractFile: File, exampleFile: File): Pair { - val feature = common.contractFileToFeature(contractFile) - val result = testExternalExample(feature, exampleFile, serverHost) - return Pair(result.first, result.second) - } - - private fun getFilteredScenarios(feature: Feature): List { - val scenarioFilter = ScenarioFilter(filterName, filterNotName, extensive) - return common.getFilteredScenarios(feature, scenarioFilter) + private fun testExample(contractFile: File, exampleFile: File): ExampleTestResult { + val feature = contractFileToFeature(contractFile) + val result = testExternalExample(feature, exampleFile, sutBaseUrl) + return ExampleTestResult(result.first, result.second, exampleFile) } - private fun getContractFileOrNull(request: ExamplePageRequest? = null): File? { + private fun getContractFileOrNull(contractFile: File?, request: ExamplePageRequest? = null): File? { return contractFile?.takeIf { it.exists() }?.also { contract -> logger.debug("Using Contract file ${contract.path} provided via command line") } ?: request?.contractFile?.takeIf { it.exists() }?.also { contract -> @@ -139,17 +113,23 @@ abstract class ExamplesInteractiveBase(val common: ExamplesCo } private fun getTableRows(contractFile: File): List { - val feature = common.contractFileToFeature(contractFile) + val feature = contractFileToFeature(contractFile) val scenarios = getFilteredScenarios(feature) - val examplesDir = common.getExamplesDirectory(contractFile) - val examples = common.getExternalExampleFiles(examplesDir) - val tableRows = createTableRows(scenarios, examples) + val examplesDir = getExamplesDirectory(contractFile) + val examples = getExternalExampleFiles(examplesDir) + + val scenarioExamplePair = scenarios.map { + it to getExistingExampleOrNull(it, examples)?.let { exRes -> + ExampleValidationResult(exRes.first, exRes.second) + } + } + val tableRows = createTableRows(scenarioExamplePair) return validateRows(tableRows) } // INTERACTIVE SERVER - inner class InteractiveServer(private val serverHost: String, private val serverPort: Int) : Closeable { + inner class InteractiveServer(private var contract: File?, private val serverHost: String, private val serverPort: Int) : Closeable { private val environment = applicationEngineEnvironment { module { install(CORS) { @@ -220,8 +200,8 @@ abstract class ExamplesInteractiveBase(val common: ExamplesCo val request = call.receive() getValidatedContractFileOrNull()?.let { contract -> getValidatedExampleOrNull(request.exampleFile)?.let { example -> - val (result, testLog) = testExample(contract, example) - call.respond(HttpStatusCode.OK, ExampleTestResponse(result, testLog, example)) + val result = testExample(contract, example) + call.respond(HttpStatusCode.OK, ExampleTestResponse(result)) } } } @@ -261,6 +241,25 @@ abstract class ExamplesInteractiveBase(val common: ExamplesCo return htmlContent } + private fun renderTemplate(contractFile: File, hostPort: String, tableRows: List): String { + val variables = mapOf( + "tableColumns" to htmlTableColumns, + "tableRows" to tableRows, + "contractFileName" to contractFile.name, + "contractFilePath" to contractFile.absolutePath, + "hostPort" to hostPort, + "hasExamples" to tableRows.any { it.exampleFilePath != null }, + "validationDetails" to tableRows.mapIndexed { index, row -> + (index + 1) to row.exampleMismatchReason + }.toMap() + ) + + return HtmlTemplateConfiguration.process( + templateName = "example/index.html", + variables = variables + ) + } + private suspend fun ApplicationCall.respondWithError(httpStatusCode: HttpStatusCode, errorMessage: String) { this.respond(httpStatusCode, mapOf("error" to errorMessage)) } @@ -274,15 +273,15 @@ abstract class ExamplesInteractiveBase(val common: ExamplesCo } private suspend fun PipelineContext.getValidatedContractFileOrNull(request: ExamplePageRequest? = null): File? { - val contractFile = getContractFileOrNull(request) ?: run { + val contractFile = getContractFileOrNull(contract, request) ?: run { val errorMessage = "No Contract File Found - Please provide a contract file in the command line or in the HTTP request." logger.log(errorMessage) call.respondWithError(HttpStatusCode.BadRequest, errorMessage) return null } - return contractFile.takeIf { it.extension in common.contractFileExtensions } ?: run { - val errorMessage = "Invalid Contract file ${contractFile.path} - File extension must be one of ${common.contractFileExtensions.joinToString()}" + return contractFile.takeIf { it.extension in contractFileExtensions } ?: run { + val errorMessage = "Invalid Contract file ${contractFile.path} - File extension must be one of ${contractFileExtensions.joinToString()}" logger.log(errorMessage) call.respondWithError(HttpStatusCode.BadRequest, errorMessage) return null @@ -298,8 +297,8 @@ abstract class ExamplesInteractiveBase(val common: ExamplesCo return null } - exampleFile.extension !in common.exampleFileExtensions -> { - val errorMessage = "Invalid Example file ${exampleFile.path} - File extension must be one of ${common.exampleFileExtensions.joinToString()}" + exampleFile.extension !in exampleFileExtensions -> { + val errorMessage = "Invalid Example file ${exampleFile.path} - File extension must be one of ${exampleFileExtensions.joinToString()}" logger.log(errorMessage) call.respondWithError(HttpStatusCode.BadRequest, errorMessage) return null @@ -308,25 +307,6 @@ abstract class ExamplesInteractiveBase(val common: ExamplesCo else -> exampleFile } } - - private fun renderTemplate(contractFile: File, hostPort: String, tableRows: List): String { - val variables = mapOf( - "tableColumns" to htmlTableColumns, - "tableRows" to tableRows, - "contractFileName" to contractFile.name, - "contractFilePath" to contractFile.absolutePath, - "hostPort" to hostPort, - "hasExamples" to tableRows.any { it.exampleFilePath != null }, - "validationDetails" to tableRows.mapIndexed { index, row -> - (index + 1) to row.exampleMismatchReason - }.toMap() - ) - - return HtmlTemplateConfiguration.process( - templateName = "example/index.html", - variables = variables - ) - } } private fun addShutdownHook(server: InteractiveServer) { @@ -344,6 +324,8 @@ abstract class ExamplesInteractiveBase(val common: ExamplesCo } }) } + + data class ExampleTestResult(val result: TestResult, val testLog: String, val exampleFile: File) } data class ExamplePageRequest ( @@ -351,7 +333,7 @@ data class ExamplePageRequest ( val hostPort: String? ) -data class GenerateExampleResponse( +data class GenerateExampleResponse ( val exampleFilePath: String, val status: String ) { @@ -382,15 +364,15 @@ data class ExampleTestRequest ( val exampleFile: File ) -data class ExampleTestResponse( +data class ExampleTestResponse ( val result: TestResult, val details: String, val testLog: String ) { - constructor(result: TestResult, testLog: String, exampleFile: File): this ( - result = result, - details = resultToDetails(result, exampleFile), - testLog = testLog.trim('-', ' ', '\n', '\r') + constructor(result: ExamplesInteractiveBase.ExampleTestResult): this ( + result = result.result, + details = resultToDetails(result.result, result.exampleFile), + testLog = result.testLog.trim('-', ' ', '\n', '\r') ) companion object { diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidationBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt similarity index 55% rename from application/src/main/kotlin/application/exampleGeneration/ExamplesValidationBase.kt rename to application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt index 977824d1d..0dd8fdb64 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidationBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt @@ -4,38 +4,21 @@ import io.specmatic.core.Result import io.specmatic.core.log.logger import picocli.CommandLine.Option import java.io.File -import java.util.concurrent.Callable -abstract class ExamplesValidationBase(private val common: ExamplesCommon): Callable { +abstract class ExamplesValidateBase: ExamplesBase(), ExamplesValidateCommon { @Option(names = ["--contract-file"], description = ["Contract file path"], required = true) - private lateinit var contractFile: File + override var contractFile: File? = null @Option(names = ["--example-file"], description = ["Example file path"], required = false) private val exampleFile: File? = null - @Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") - var filterName: String = "" - - @Option(names = ["--filter-not-name"], description = ["Use only APIs which do not have this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NOT_NAME}") - var filterNotName: String = "" - - @Option(names = ["--debug"], description = ["Debug logs"]) - private var verbose = false - abstract var validateExternal: Boolean abstract var validateInline: Boolean abstract var extensive: Boolean - override fun call(): Int { - common.configureLogger(this.verbose) - - if (!contractFile.exists()) { - logger.log("Could not find Contract file ${contractFile.path}") - return 1 - } - - if (contractFile.extension !in common.contractFileExtensions) { - logger.log("Invalid Contract file ${contractFile.path} - File extension must be one of ${common.contractFileExtensions.joinToString()}") + override fun execute(contract: File?): Int { + if (contract == null) { + logger.log("No contract file provided, please provide a contract file. Use --help for more details.") return 1 } @@ -46,20 +29,20 @@ abstract class ExamplesValidationBase(private val common: Exa return 1 } - if (exFile.extension !in common.exampleFileExtensions) { - logger.log("Invalid Example file ${exFile.path} - File extension must be one of ${common.exampleFileExtensions.joinToString()}") + if (exFile.extension !in exampleFileExtensions) { + logger.log("Invalid Example file ${exFile.path} - File extension must be one of ${exampleFileExtensions.joinToString()}") return 1 } - val validation = validateExampleFile(exFile, contractFile) + val validation = validateExampleFile(exFile, contract) return getExitCode(validation) } - val inlineResults = if(validateInline) validateInlineExamples(contractFile) else emptyList() - val externalResults = if(validateExternal) validateExternalExamples(contractFile) else emptyList() + val inlineResults = if (validateInline) validateInlineExamples(contract) else emptyList() + val externalResults = if (validateExternal) validateExternalExamples(contract) else emptyList() logValidationResult(inlineResults, externalResults) - return getExitCode(inlineResults.plus(externalResults)) + return getExitCode(inlineResults + externalResults) } catch (e: Throwable) { logger.log("Validation failed with error: ${e.message}") logger.debug(e) @@ -73,49 +56,47 @@ abstract class ExamplesValidationBase(private val common: Exa } // VALIDATION METHODS + private fun getFilteredFeature(contractFile: File): Feature { + val feature = contractFileToFeature(contractFile) + val filteredScenarios = getFilteredScenarios(feature) + return updateFeatureForValidation(feature, filteredScenarios) + } + private fun validateExampleFile(exampleFile: File, contractFile: File): ExampleValidationResult { - val feature = common.contractFileToFeature(contractFile) + val feature = contractFileToFeature(contractFile) return validateExternalExample(exampleFile, feature) } private fun validateExternalExample(exampleFile: File, feature: Feature): ExampleValidationResult { return try { - val result = common.validateExternalExample(feature, exampleFile) - ExampleValidationResult(exampleFile.absolutePath, result.second, ExampleType.EXTERNAL) + val result = validateExternalExample(feature, exampleFile) + ExampleValidationResult(exampleFile, result.second) } catch (e: Throwable) { logger.log("Example validation failed with error: ${e.message}") logger.debug(e) - ExampleValidationResult(exampleFile.absolutePath, Result.Failure(e.message.orEmpty()), ExampleType.EXTERNAL) + ExampleValidationResult(exampleFile, Result.Failure(e.message.orEmpty())) } } - private fun getFilteredFeature(contractFile: File): Feature { - val scenarioFilter = ScenarioFilter(filterName, filterNotName, extensive) - val feature = common.contractFileToFeature(contractFile) - val filteredScenarios = common.getFilteredScenarios(feature, scenarioFilter) - - return common.updateFeatureForValidation(feature, filteredScenarios) - } - - private fun validateExternalExamples(contractFile: File): List { - val examplesDir = common.getExamplesDirectory(contractFile) - val exampleFiles = common.getExternalExampleFiles(examplesDir) + private fun validateInlineExamples(contractFile: File): List { val feature = getFilteredFeature(contractFile) - - return exampleFiles.mapIndexed { index, it -> - validateExternalExample(it, feature).also { + return validateInlineExamples(feature).mapIndexed { index, it -> + ExampleValidationResult(it.first, it.second, ExampleType.INLINE).also { it.logErrors(index.inc()) - common.logSeparator(75) + logSeparator(75) } } } - private fun validateInlineExamples(contractFile: File): List { + private fun validateExternalExamples(contractFile: File): List { + val examplesDir = getExamplesDirectory(contractFile) + val exampleFiles = getExternalExampleFiles(examplesDir) val feature = getFilteredFeature(contractFile) - return validateInlineExamples(feature).mapIndexed { index, it -> - ExampleValidationResult(it.first, it.second, ExampleType.INLINE).also { + + return exampleFiles.mapIndexed { index, it -> + validateExternalExample(it, feature).also { it.logErrors(index.inc()) - common.logSeparator(75) + logSeparator(75) } } } @@ -130,20 +111,21 @@ abstract class ExamplesValidationBase(private val common: Exa } private fun logValidationResult(inlineResults: List, externalResults: List) { - logResultSummary(inlineResults, ExampleType.INLINE) - logResultSummary(externalResults, ExampleType.EXTERNAL) - } + if (inlineResults.isNotEmpty()) { + val successCount = inlineResults.count { it.result.isSuccess() } + val failureCount = inlineResults.size - successCount + logResultSummary(ExampleType.INLINE, successCount, failureCount) + } - private fun logResultSummary(results: List, type: ExampleType) { - if (results.isNotEmpty()) { - val successCount = results.count { it.result.isSuccess() } - val failureCount = results.size - successCount - printSummary(type, successCount, failureCount) + if (externalResults.isNotEmpty()) { + val successCount = externalResults.count { it.result.isSuccess() } + val failureCount = externalResults.size - successCount + logResultSummary(ExampleType.EXTERNAL, successCount, failureCount) } } - private fun printSummary(type: ExampleType, successCount: Int, failureCount: Int) { - common.logFormattedOutput( + private fun logResultSummary(type: ExampleType, successCount: Int, failureCount: Int) { + logFormattedOutput( header = "$type Examples Validation Summary", summary = "$successCount example(s) are valid. $failureCount example(s) are invalid", note = "" @@ -171,4 +153,12 @@ enum class ExampleType(val value: String) { } } -data class ExampleValidationResult(val exampleName: String, val result: Result, val type: ExampleType) +data class ExampleValidationResult(val exampleName: String, val result: Result, val type: ExampleType, val exampleFIle: File? = null) { + constructor(exampleFile: File, result: Result) : this(exampleFile.nameWithoutExtension, result, ExampleType.EXTERNAL, exampleFile) +} + +interface ExamplesValidateCommon : ExamplesCommon { + fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature + + fun validateExternalExample(feature: Feature, exampleFile: File): Pair +} diff --git a/application/src/main/kotlin/application/exampleGeneration/ScenarioFilter.kt b/application/src/main/kotlin/application/exampleGeneration/ScenarioFilter.kt deleted file mode 100644 index aa5fa47b6..000000000 --- a/application/src/main/kotlin/application/exampleGeneration/ScenarioFilter.kt +++ /dev/null @@ -1,11 +0,0 @@ -package application.exampleGeneration - -class ScenarioFilter(filterName: String, filterNotName: String, val extensive: Boolean) { - val filterNameTokens = filterToTokens(filterName) - val filterNotNameTokens = filterToTokens(filterNotName) - - private fun filterToTokens(filterValue: String): Set { - if (filterValue.isBlank()) return emptySet() - return filterValue.split(",").map { it.trim() }.toSet() - } -} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamples.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamples.kt deleted file mode 100644 index a3b10eac0..000000000 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamples.kt +++ /dev/null @@ -1,18 +0,0 @@ -package application.exampleGeneration.openApiExamples - -import application.exampleGeneration.ExamplesBase -import io.specmatic.core.Feature -import io.specmatic.core.Scenario -import picocli.CommandLine.Command -import picocli.CommandLine.Option - -@Command( - name = "examples", - mixinStandardHelpOptions = true, - description = ["Generate JSON Examples with Request and Response from an OpenApi Contract File"], - subcommands = [OpenApiExamplesValidate::class, OpenApiExamplesInteractive::class] -) -class OpenApiExamples: ExamplesBase(OpenApiExamplesCommon()) { - @Option(names = ["--extensive"], description = ["Generate all examples (by default, generates one example per 2xx API)"], defaultValue = "false") - override var extensive: Boolean = false -} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt index 8ab7b0a53..9a32b12ba 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt @@ -1,19 +1,12 @@ package application.exampleGeneration.openApiExamples import application.exampleGeneration.ExamplesCommon -import io.specmatic.conversions.ExampleFromFile import io.specmatic.core.* -import io.specmatic.core.utilities.capitalizeFirstChar -import io.specmatic.core.utilities.uniqueNameForApiOperation -import io.specmatic.mock.NoMatchingScenario -import io.specmatic.mock.ScenarioStub -import io.specmatic.stub.HttpStub -import io.specmatic.stub.HttpStubData import java.io.File -class OpenApiExamplesCommon: ExamplesCommon { - override val exampleFileExtensions: Set = setOf("json") - override val contractFileExtensions: Set = OPENAPI_FILE_EXTENSIONS.toSet() +interface OpenApiExamplesCommon: ExamplesCommon { + override val exampleFileExtensions: Set get() = setOf(JSON) + override val contractFileExtensions: Set get() = OPENAPI_FILE_EXTENSIONS.toSet() override fun contractFileToFeature(contractFile: File): Feature { return parseContractFileToFeature(contractFile) @@ -30,117 +23,4 @@ class OpenApiExamplesCommon: ExamplesCommon { return feature.scenarios } - - // GENERATION METHODS - override fun generateExample(feature: Feature, scenario: Scenario, dictionary: Dictionary): Pair { - val request = scenario.generateHttpRequest() - val requestHttpPathPattern = scenario.httpRequestPattern.httpPathPattern - val updatedRequest = request.substituteDictionaryValues(dictionary, forceSubstitution = true, requestHttpPathPattern) - - val response = feature.lookupResponse(scenario).cleanup() - val updatedResponse = response.substituteDictionaryValues(dictionary, forceSubstitution = true) - - val scenarioStub = ScenarioStub(updatedRequest, updatedResponse) - val stubJSON = scenarioStub.toJSON().toStringLiteral() - val uniqueName = uniqueNameForApiOperation(request, "", scenarioStub.response.status) - - return Pair("$uniqueName.json", stubJSON) - } - - override fun getExistingExampleOrNull(scenario: Scenario, exampleFiles: List): Pair? { - val examples = exampleFiles.toExamples() - return examples.firstNotNullOfOrNull { example -> - val response = example.response - - when (val matchResult = scenario.matchesMock(example.request, response)) { - is Result.Success -> example.file to matchResult - is Result.Failure -> { - val isFailureRelatedToScenario = matchResult.getFailureBreadCrumbs("").none { breadCrumb -> - breadCrumb.contains(PATH_BREAD_CRUMB) - || breadCrumb.contains(METHOD_BREAD_CRUMB) - || breadCrumb.contains("REQUEST.HEADERS.Content-Type") - || breadCrumb.contains("STATUS") - } - if (isFailureRelatedToScenario) example.file to matchResult else null - } - } - } - } - - private fun List.toExamples(): List { - return this.map { ExampleFromFile(it) } - } - - private fun HttpResponse.cleanup(): HttpResponse { - return this.copy(headers = this.headers.minus(SPECMATIC_RESULT_HEADER)) - } - - // VALIDATION METHODS - override fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature { - return feature.copy(scenarios = filteredScenarios) - } - - override fun validateExternalExample(feature: Feature, exampleFile: File): Pair { - val examples = mapOf(exampleFile.nameWithoutExtension to listOf(ScenarioStub.readFromFile(exampleFile))) - return feature.validateMultipleExamples(examples).first() - } - - private fun getCleanedUpFailure(failureResults: Results, noMatchingScenario: NoMatchingScenario?): Results { - return failureResults.toResultIfAny().let { - if (it.reportString().isBlank()) - Results(listOf(Result.Failure(noMatchingScenario?.message ?: "", failureReason = FailureReason.ScenarioMismatch))) - else - failureResults - } - } - - private fun Feature.validateMultipleExamples(examples: Map>, inline: Boolean = false): List> { - val results = examples.map { (name, exampleList) -> - val results = exampleList.mapNotNull { example -> - try { - this.validateExample(example) - Result.Success() - } catch (e: NoMatchingScenario) { - if (inline && !e.results.withoutFluff().hasResults()) - null - else - e.results.toResultIfAny() - } - } - name to Result.fromResults(results) - } - - return results - } - - private fun Feature.validateExample(scenarioStub: ScenarioStub) { - val result: Pair>?, NoMatchingScenario?> = HttpStub.setExpectation(scenarioStub, this, InteractiveExamplesMismatchMessages) - val validationResult = result.first - val noMatchingScenario = result.second - - if (validationResult == null) { - val failures = noMatchingScenario?.results?.withoutFluff()?.results ?: emptyList() - - val failureResults = getCleanedUpFailure(Results(failures).withoutFluff(), noMatchingScenario) - throw NoMatchingScenario( - failureResults, - cachedMessage = failureResults.report(scenarioStub.request), - msg = failureResults.report(scenarioStub.request) - ) - } - } - - object InteractiveExamplesMismatchMessages : MismatchMessages { - override fun mismatchMessage(expected: String, actual: String): String { - return "Specification expected $expected but example contained $actual" - } - - override fun unexpectedKey(keyLabel: String, keyName: String): String { - return "${keyLabel.capitalizeFirstChar()} $keyName in the example is not in the specification" - } - - override fun expectedKeyWasMissing(keyLabel: String, keyName: String): String { - return "${keyLabel.capitalizeFirstChar()} $keyName in the specification is missing from the example" - } - } } \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt new file mode 100644 index 000000000..a7558181e --- /dev/null +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt @@ -0,0 +1,66 @@ +package application.exampleGeneration.openApiExamples + +import application.exampleGeneration.ExamplesGenerateBase +import application.exampleGeneration.ExamplesGenerateCommon +import io.specmatic.conversions.ExampleFromFile +import io.specmatic.core.* +import io.specmatic.core.utilities.uniqueNameForApiOperation +import io.specmatic.mock.ScenarioStub +import picocli.CommandLine.Command +import picocli.CommandLine.Option +import java.io.File + +@Command( + name = "examples", + description = ["Generate JSON Examples with Request and Response from an OpenApi Contract File"], + subcommands = [OpenApiExamplesValidate::class, OpenApiExamplesInteractive::class] +) +class OpenApiExamplesGenerate: ExamplesGenerateBase(), OpenApiExamplesGenerateCommon { + @Option(names = ["--extensive"], description = ["Generate all examples (by default, generates one example per 2xx API)"], defaultValue = "false") + override var extensive: Boolean = false +} + +interface OpenApiExamplesGenerateCommon: ExamplesGenerateCommon, OpenApiExamplesCommon { + override fun generateExample(feature: Feature, scenario: Scenario, dictionary: Dictionary): Pair { + val request = scenario.generateHttpRequest() + val requestHttpPathPattern = scenario.httpRequestPattern.httpPathPattern + val updatedRequest = request.substituteDictionaryValues(dictionary, forceSubstitution = true, requestHttpPathPattern) + + val response = feature.lookupResponse(scenario).cleanup() + val updatedResponse = response.substituteDictionaryValues(dictionary, forceSubstitution = true) + + val scenarioStub = ScenarioStub(updatedRequest, updatedResponse) + val stubJSON = scenarioStub.toJSON().toStringLiteral() + val uniqueName = uniqueNameForApiOperation(request, "", scenarioStub.response.status) + + return Pair("$uniqueName.json", stubJSON) + } + + override fun getExistingExampleOrNull(scenario: Scenario, exampleFiles: List): Pair? { + val examples = exampleFiles.toExamples() + return examples.firstNotNullOfOrNull { example -> + val response = example.response + + when (val matchResult = scenario.matchesMock(example.request, response)) { + is Result.Success -> example.file to matchResult + is Result.Failure -> { + val isFailureRelatedToScenario = matchResult.getFailureBreadCrumbs("").none { breadCrumb -> + breadCrumb.contains(PATH_BREAD_CRUMB) + || breadCrumb.contains(METHOD_BREAD_CRUMB) + || breadCrumb.contains("REQUEST.HEADERS.Content-Type") + || breadCrumb.contains("STATUS") + } + if (isFailureRelatedToScenario) example.file to matchResult else null + } + } + } + } + + private fun List.toExamples(): List { + return this.map { ExampleFromFile(it) } + } + + private fun HttpResponse.cleanup(): HttpResponse { + return this.copy(headers = this.headers.minus(SPECMATIC_RESULT_HEADER)) + } +} \ No newline at end of file diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index 188eacc00..bb1db457e 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -1,9 +1,6 @@ package application.exampleGeneration.openApiExamples -import application.exampleGeneration.ExamplesInteractiveBase -import application.exampleGeneration.HtmlTableColumn -import application.exampleGeneration.TableRow -import application.exampleGeneration.TableRowGroup +import application.exampleGeneration.* import io.ktor.server.application.* import io.ktor.server.request.* import io.specmatic.conversions.convertPathParameterStyle @@ -17,16 +14,12 @@ import picocli.CommandLine.Command import picocli.CommandLine.Option import java.io.File -@Command( - name = "interactive", - mixinStandardHelpOptions = true, - description = ["Generate and validate examples interactively through a Web UI"], -) -class OpenApiExamplesInteractive : ExamplesInteractiveBase(OpenApiExamplesCommon()) { +@Command(name = "interactive", description = ["Generate and validate examples interactively through a Web UI"],) +class OpenApiExamplesInteractive : ExamplesInteractiveBase(), OpenApiExamplesGenerateCommon, OpenApiExamplesValidateCommon { @Option(names = ["--extensive"], description = ["Display all responses, not just 2xx, in the table."], defaultValue = "false") override var extensive: Boolean = false - override val htmlTableColumns: List = listOf( + override val htmlTableColumns: List = listOf ( HtmlTableColumn(name = "path", colSpan = 2), HtmlTableColumn(name = "method", colSpan = 1), HtmlTableColumn(name = "response", colSpan = 1) @@ -56,8 +49,8 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase(Op } } - override fun createTableRows(scenarios: List, exampleFiles: List): List { - val groupedScenarios = scenarios.sortScenarios().groupScenarios() + override fun createTableRows(scenarioExamplePair: List>): List { + val groupedScenarios = scenarioExamplePair.sortScenarios().groupScenarios() return groupedScenarios.flatMap { (_, methodMap) -> val pathSpan = methodMap.values.sumOf { it.size } @@ -65,33 +58,31 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase(Op var showPath = true methodMap.flatMap { (method, scenarios) -> - scenarios.map { - val existingExample = common.getExistingExampleOrNull(it, exampleFiles) - + scenarios.map { (scenario, example) -> TableRow( columns = listOf( - TableRowGroup("path", convertPathParameterStyle(it.path), rawValue = it.path, rowSpan = pathSpan, showRow = showPath), - TableRowGroup("method", it.method, showRow = !methodSet.contains(method), rowSpan = scenarios.size), - TableRowGroup("response", it.status.toString(), showRow = true, rowSpan = 1, extraInfo = it.httpRequestPattern.headersPattern.contentType) + TableRowGroup("path", convertPathParameterStyle(scenario.path), rawValue = scenario.path, rowSpan = pathSpan, showRow = showPath), + TableRowGroup("method", scenario.method, showRow = !methodSet.contains(method), rowSpan = scenarios.size), + TableRowGroup("response", scenario.status.toString(), showRow = true, rowSpan = 1, extraInfo = scenario.httpRequestPattern.headersPattern.contentType) ), - exampleFilePath = existingExample?.first?.absolutePath, - exampleFileName = existingExample?.first?.nameWithoutExtension, - exampleMismatchReason = existingExample?.second?.reportString().takeIf { reason -> reason?.isNotBlank() == true } + exampleFilePath = example?.exampleFIle?.absolutePath, + exampleFileName = example?.exampleName, + exampleMismatchReason = example?.result?.reportString().takeIf { reason -> reason?.isNotBlank() == true } ).also { methodSet.add(method); showPath = false } } } } } - private fun List.groupScenarios(): Map>> { - return this.groupBy { it.path }.mapValues { pathGroup -> - pathGroup.value.groupBy { it.method } + private fun List>.groupScenarios(): Map>>> { + return this.groupBy { it.first.path }.mapValues { pathGroup -> + pathGroup.value.groupBy { it.first.method } } } - private fun List.sortScenarios(): List { + private fun List>.sortScenarios(): List> { return this.sortedBy { - "${it.path}_${it.method}_${it.status}" + "${it.first.path}_${it.first.method}_${it.first.status}" } } diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt index 223213cd1..ab9c2659c 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt @@ -1,13 +1,19 @@ package application.exampleGeneration.openApiExamples -import application.exampleGeneration.ExamplesValidationBase -import io.specmatic.core.Feature -import io.specmatic.core.Scenario +import application.exampleGeneration.ExamplesValidateCommon +import application.exampleGeneration.ExamplesValidateBase +import io.specmatic.core.* +import io.specmatic.core.utilities.capitalizeFirstChar +import io.specmatic.mock.NoMatchingScenario +import io.specmatic.mock.ScenarioStub +import io.specmatic.stub.HttpStub +import io.specmatic.stub.HttpStubData import picocli.CommandLine.Command import picocli.CommandLine.Option +import java.io.File @Command(name = "validate", description = ["Validate OpenAPI inline and external examples"]) -class OpenApiExamplesValidate: ExamplesValidationBase(OpenApiExamplesCommon()) { +class OpenApiExamplesValidate: ExamplesValidateBase(), OpenApiExamplesValidateCommon { @Option(names = ["--validate-external"], description = ["Validate external examples, defaults to true"]) override var validateExternal: Boolean = true @@ -15,4 +21,74 @@ class OpenApiExamplesValidate: ExamplesValidationBase(OpenApi override var validateInline: Boolean = false override var extensive: Boolean = true +} + +interface OpenApiExamplesValidateCommon: ExamplesValidateCommon, OpenApiExamplesCommon { + override fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature { + return feature.copy(scenarios = filteredScenarios) + } + + override fun validateExternalExample(feature: Feature, exampleFile: File): Pair { + val examples = mapOf(exampleFile.nameWithoutExtension to listOf(ScenarioStub.readFromFile(exampleFile))) + return feature.validateMultipleExamples(examples).first() + } + + private fun getCleanedUpFailure(failureResults: Results, noMatchingScenario: NoMatchingScenario?): Results { + return failureResults.toResultIfAny().let { + if (it.reportString().isBlank()) + Results(listOf(Result.Failure(noMatchingScenario?.message ?: "", failureReason = FailureReason.ScenarioMismatch))) + else + failureResults + } + } + + private fun Feature.validateMultipleExamples(examples: Map>, inline: Boolean = false): List> { + val results = examples.map { (name, exampleList) -> + val results = exampleList.mapNotNull { example -> + try { + this.validateExample(example) + Result.Success() + } catch (e: NoMatchingScenario) { + if (inline && !e.results.withoutFluff().hasResults()) + null + else + e.results.toResultIfAny() + } + } + name to Result.fromResults(results) + } + + return results + } + + private fun Feature.validateExample(scenarioStub: ScenarioStub) { + val result: Pair>?, NoMatchingScenario?> = HttpStub.setExpectation(scenarioStub, this, InteractiveExamplesMismatchMessages) + val validationResult = result.first + val noMatchingScenario = result.second + + if (validationResult == null) { + val failures = noMatchingScenario?.results?.withoutFluff()?.results ?: emptyList() + + val failureResults = getCleanedUpFailure(Results(failures).withoutFluff(), noMatchingScenario) + throw NoMatchingScenario( + failureResults, + cachedMessage = failureResults.report(scenarioStub.request), + msg = failureResults.report(scenarioStub.request) + ) + } + } + + object InteractiveExamplesMismatchMessages : MismatchMessages { + override fun mismatchMessage(expected: String, actual: String): String { + return "Specification expected $expected but example contained $actual" + } + + override fun unexpectedKey(keyLabel: String, keyName: String): String { + return "${keyLabel.capitalizeFirstChar()} $keyName in the example is not in the specification" + } + + override fun expectedKeyWasMissing(keyLabel: String, keyName: String): String { + return "${keyLabel.capitalizeFirstChar()} $keyName in the specification is missing from the example" + } + } } \ No newline at end of file diff --git a/junit5-support/src/main/resources/templates/example/index.html b/junit5-support/src/main/resources/templates/example/index.html index edc6e1b3b..233512f19 100644 --- a/junit5-support/src/main/resources/templates/example/index.html +++ b/junit5-support/src/main/resources/templates/example/index.html @@ -1028,7 +1028,7 @@

- [[${iter.index + 1}]] +

[[${iter.index + 1}]]

 storeExampleData(tableRow, exampleFilePath, status); enableValidateBtn(tableRow); const generateColumn = tableRow.querySelector("td:nth-last-child(2)") - generateColumn.textContent = parseFileName(exampleFilePath); + const examplePara = document.createElement("p"); + examplePara.textContent = parseFileName(exampleFilePath); + generateColumn.replaceChildren(examplePara); const message = status === "EXISTS" ? "Example Already Exists" : "Example Generated"; if (!bulkMode) createAlert(message, `Example name: ${parseFileName(exampleFilePath)}`, false); From bc4e73e0da692ec3a58e87ecf6f40b7c1983268c Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Tue, 8 Oct 2024 12:27:02 +0530 Subject: [PATCH 24/43] Architecture Refactor and merge fixes. - Use composition / delegation inplace of multi- inheritance. - Update dictionary usage in example_parity. - Fix errors caused by main merge. --- .../exampleGeneration/ExamplesBase.kt | 39 ++++++-- .../exampleGeneration/ExamplesGenerateBase.kt | 83 ++++++++--------- .../ExamplesInteractiveBase.kt | 60 +++++++----- .../exampleGeneration/ExamplesValidateBase.kt | 19 ++-- ...n.kt => OpenApiExamplesFeatureStrategy.kt} | 4 +- .../OpenApiExamplesGenerate.kt | 16 ++-- .../OpenApiExamplesInteractive.kt | 4 +- .../OpenApiExamplesValidate.kt | 8 +- .../resources/templates/example/index.html | 93 ++++++++++++------- 9 files changed, 197 insertions(+), 129 deletions(-) rename application/src/main/kotlin/application/exampleGeneration/openApiExamples/{OpenApiExamplesCommon.kt => OpenApiExamplesFeatureStrategy.kt} (84%) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt index ebd628712..1e5a8613a 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesBase.kt @@ -1,12 +1,14 @@ package application.exampleGeneration +import io.specmatic.core.DICTIONARY_FILE_SUFFIX import io.specmatic.core.EXAMPLES_DIR_SUFFIX +import io.specmatic.core.SPECMATIC_STUB_DICTIONARY import io.specmatic.core.log.* import picocli.CommandLine import java.io.File import java.util.concurrent.Callable -abstract class ExamplesBase : Callable, ExamplesCommon { +abstract class ExamplesBase(open val featureStrategy: ExamplesFeatureStrategy) : Callable { protected abstract var contractFile: File? @CommandLine.Option(names = ["--filter-name"], description = ["Use only APIs with this value in their name, Case sensitive"], defaultValue = "\${env:SPECMATIC_FILTER_NAME}") @@ -25,8 +27,8 @@ abstract class ExamplesBase : Callable, ExamplesCommon : Callable, ExamplesCommon { val scenarioFilter = ScenarioFilter(filterName, filterNotName) - val scenarios = getScenariosFromFeature(feature, extensive) + val scenarios = featureStrategy.getScenariosFromFeature(feature, extensive) return getFilteredScenarios(scenarios, scenarioFilter) } @@ -64,7 +66,9 @@ abstract class ExamplesBase : Callable, ExamplesCommon { - return examplesDirectory.walk().filter { it.isFile && it.extension in exampleFileExtensions }.toList() + return examplesDirectory.walk().filter { + it.isFile && it.extension in featureStrategy.exampleFileExtensions + }.toList() } fun logSeparator(length: Int, separator: String = "-") { @@ -85,6 +89,27 @@ abstract class ExamplesBase : Callable, ExamplesCommon { + dictFileFromArgs.takeIf { it.exists() } ?: throw Exception("Dictionary file does not exist: ${dictFileFromArgs.absolutePath}") + } + false -> { + val dictInContractFolder = contract?.parentFile?.resolve("${contract.nameWithoutExtension}$DICTIONARY_FILE_SUFFIX") + val dictFileInCurDir = File(".").resolve("${contractFile?.nameWithoutExtension}$DICTIONARY_FILE_SUFFIX") + + dictInContractFolder?.takeIf { it.exists() } ?: dictFileInCurDir.takeIf { it.exists() } + } + } + + dictFile?.let { + logger.log("Using Dictionary file: ${it.absolutePath}") + System.setProperty(SPECMATIC_STUB_DICTIONARY, it.absolutePath) + } + + return dictFile + } + private fun getFilteredScenarios(scenarios: List, scenarioFilter: ScenarioFilter): List { val filteredScenarios = scenarios .filterScenarios(scenarioFilter.filterNameTokens, shouldMatch = true) @@ -101,7 +126,7 @@ abstract class ExamplesBase : Callable, ExamplesCommon description.contains(token) } == shouldMatch @@ -109,7 +134,7 @@ abstract class ExamplesBase : Callable, ExamplesCommon { +interface ExamplesFeatureStrategy { val exampleFileExtensions: Set val contractFileExtensions: Set diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt index 77fb6b14d..d16d995b8 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt @@ -2,15 +2,17 @@ package application.exampleGeneration import io.specmatic.core.* import io.specmatic.core.log.* -import io.specmatic.mock.loadDictionary import picocli.CommandLine.* import java.io.File -abstract class ExamplesGenerateBase: ExamplesBase(), ExamplesGenerateCommon { +abstract class ExamplesGenerateBase( + override val featureStrategy: ExamplesFeatureStrategy, + private val generationStrategy: ExamplesGenerationStrategy +): ExamplesBase(featureStrategy) { @Parameters(index = "0", description = ["Contract file path"], arity = "0..1") override var contractFile: File? = null - @Option(names = ["--dictionary"], description = ["External Dictionary File Path, defaults to dictionary.json"]) + @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) private var dictFile: File? = null abstract var extensive: Boolean @@ -22,9 +24,9 @@ abstract class ExamplesGenerateBase: ExamplesBase: ExamplesBase { - val feature = contractFileToFeature(contractFile) + private fun generateExamples(contractFile: File, examplesDir: File): List { + val feature = featureStrategy.contractFileToFeature(contractFile) val filteredScenarios = getFilteredScenarios(feature, extensive) if (filteredScenarios.isEmpty()) { @@ -45,9 +47,14 @@ abstract class ExamplesGenerateBase: ExamplesBase - generateOrGetExistingExample(feature, scenario, externalDictionary, exampleFiles, examplesDir).also { - logSeparator(75) - } + generationStrategy.generateOrGetExistingExample( + ExamplesGenerationStrategy.GenerateOrGetExistingExampleArgs( + feature, scenario, + featureStrategy.getScenarioDescription(scenario), + exampleFiles, examplesDir, + featureStrategy.exampleFileExtensions + ) + ).also { logSeparator(75) } } } @@ -79,49 +86,24 @@ enum class ExampleGenerationStatus(val value: String) { data class ExampleGenerationResult(val exampleFile: File? = null, val status: ExampleGenerationStatus) -interface ExamplesGenerateCommon : ExamplesCommon { +interface ExamplesGenerationStrategy { fun getExistingExampleOrNull(scenario: Scenario, exampleFiles: List): Pair? - fun generateExample(feature: Feature, scenario: Scenario, dictionary: Dictionary): Pair - - fun loadExternalDictionary(dictFile: File?, contractFile: File?): Dictionary { - val dictFilePath = when(dictFile != null) { - true -> { - dictFile.takeIf { it.exists() }?.path ?: throw IllegalStateException("Dictionary file not found: ${dictFile.path}") - } - - false -> { - val contractDictFile = contractFile?.let { contract -> - val contractDictFile = "${contract.nameWithoutExtension}$DICTIONARY_FILE_SUFFIX" - contract.canonicalFile.parentFile.resolve(contractDictFile).takeIf { it.exists() }?.path - } - - val currentDirDictFile = File(System.getProperty("user.dir")).resolve("dictionary.json").takeIf { - it.exists() - }?.path + fun generateExample(feature: Feature, scenario: Scenario): Pair - contractDictFile ?: currentDirDictFile - } - } - - return dictFilePath?.let { - Dictionary(loadDictionary(dictFilePath)) - } ?: Dictionary(emptyMap()) - } - - fun generateOrGetExistingExample(feature: Feature, scenario: Scenario, externalDictionary: Dictionary, exampleFiles: List, examplesDir: File): ExampleGenerationResult { + fun generateOrGetExistingExample(request: GenerateOrGetExistingExampleArgs): ExampleGenerationResult { return try { - val existingExample = getExistingExampleOrNull(scenario, exampleFiles) - val description = getScenarioDescription(scenario) + val existingExample = getExistingExampleOrNull(request.scenario, request.exampleFiles) + val scenarioDescription = request.scenarioDescription if (existingExample != null) { - logger.log("Using existing example for $description\nExample File: ${existingExample.first.absolutePath}") + logger.log("Using existing example for ${scenarioDescription}\nExample File: ${existingExample.first.absolutePath}") return ExampleGenerationResult(existingExample.first, ExampleGenerationStatus.EXISTS) } - logger.log("Generating example for $description") - val (uniqueFileName, exampleContent) = generateExample(feature, scenario, externalDictionary) - return writeExampleToFile(exampleContent, uniqueFileName, examplesDir) + logger.log("Generating example for $scenarioDescription") + val (uniqueFileName, exampleContent) = generateExample(request.feature, request.scenario) + return writeExampleToFile(exampleContent, uniqueFileName, request.examplesDir, request.validExampleExtensions) } catch (e: Throwable) { logger.log("Failed to generate example: ${e.message}") logger.debug(e) @@ -129,10 +111,10 @@ interface ExamplesGenerateCommon : ExamplesCommon): ExampleGenerationResult { val exampleFile = examplesDir.resolve(exampleFileName) - if (exampleFile.extension !in exampleFileExtensions) { + if (exampleFile.extension !in validExampleExtensions) { logger.log("Invalid example file extension: ${exampleFile.extension}") return ExampleGenerationResult(exampleFile, ExampleGenerationStatus.ERROR) } @@ -147,4 +129,13 @@ interface ExamplesGenerateCommon : ExamplesCommon ( + val feature: Feature, + val scenario: Scenario, + val scenarioDescription: String, + val exampleFiles: List, + val examplesDir: File, + val validExampleExtensions: Set + ) } diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index 81ad4c662..83757cc5a 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -22,14 +22,18 @@ import java.io.File import java.io.FileNotFoundException import java.lang.Thread.sleep -abstract class ExamplesInteractiveBase: ExamplesBase(), ExamplesGenerateCommon, ExamplesValidateCommon { - @Option(names = ["--testBaseURL"], description = ["BaseURL of the SUT"], required = true) - lateinit var sutBaseUrl: String +abstract class ExamplesInteractiveBase ( + override val featureStrategy: ExamplesFeatureStrategy, + private val generationStrategy: ExamplesGenerationStrategy, + private val validationStrategy: ExamplesValidationStrategy +): ExamplesBase(featureStrategy) { + @Option(names = ["--testBaseURL"], description = ["BaseURL of the the system to test"], required = false) + var sutBaseUrl: String? = null @Option(names = ["--contract-file"], description = ["Contract file path"], required = false) override var contractFile: File? = null - @Option(names = ["--dictionary"], description = ["External Dictionary File Path"]) + @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) var dictFile: File? = null abstract var extensive: Boolean @@ -42,7 +46,8 @@ abstract class ExamplesInteractiveBase: ExamplesBase: ExamplesBase: ExamplesBase { - val feature = contractFileToFeature(contractFile) + val feature = featureStrategy.contractFileToFeature(contractFile) val scenarios = getFilteredScenarios(feature) val examplesDir = getExamplesDirectory(contractFile) val examples = getExternalExampleFiles(examplesDir) val scenarioExamplePair = scenarios.map { - it to getExistingExampleOrNull(it, examples)?.let { exRes -> + it to generationStrategy.getExistingExampleOrNull(it, examples)?.let { exRes -> ExampleValidationResult(exRes.first, exRes.second) } } @@ -129,7 +140,7 @@ abstract class ExamplesInteractiveBase: ExamplesBase: ExamplesBase() getValidatedContractFileOrNull()?.let { contract -> getValidatedExampleOrNull(request.exampleFile)?.let { example -> - val result = testExample(contract, example) + val result = testExample(contract, example, sutBaseUrl) call.respond(HttpStatusCode.OK, ExampleTestResponse(result)) } } @@ -251,7 +266,8 @@ abstract class ExamplesInteractiveBase: ExamplesBase (index + 1) to row.exampleMismatchReason - }.toMap() + }.toMap(), + "isTestMode" to (sutBaseUrl != null) ) return HtmlTemplateConfiguration.process( @@ -280,8 +296,10 @@ abstract class ExamplesInteractiveBase: ExamplesBase: ExamplesBase { - val errorMessage = "Invalid Example file ${exampleFile.path} - File extension must be one of ${exampleFileExtensions.joinToString()}" + exampleFile.extension !in featureStrategy.exampleFileExtensions -> { + val errorMessage = "Invalid Example file ${exampleFile.path} - File extension must be one of ${featureStrategy.exampleFileExtensions.joinToString()}" logger.log(errorMessage) call.respondWithError(HttpStatusCode.BadRequest, errorMessage) return null diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt index 0dd8fdb64..3ac21f585 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesValidateBase.kt @@ -5,7 +5,10 @@ import io.specmatic.core.log.logger import picocli.CommandLine.Option import java.io.File -abstract class ExamplesValidateBase: ExamplesBase(), ExamplesValidateCommon { +abstract class ExamplesValidateBase( + override val featureStrategy: ExamplesFeatureStrategy, + private val validationStrategy: ExamplesValidationStrategy +): ExamplesBase(featureStrategy) { @Option(names = ["--contract-file"], description = ["Contract file path"], required = true) override var contractFile: File? = null @@ -29,8 +32,8 @@ abstract class ExamplesValidateBase: ExamplesBase: ExamplesBase : ExamplesCommon { +interface ExamplesValidationStrategy { fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature fun validateExternalExample(feature: Feature, exampleFile: File): Pair diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesFeatureStrategy.kt similarity index 84% rename from application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt rename to application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesFeatureStrategy.kt index 9a32b12ba..cad473d52 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesCommon.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesFeatureStrategy.kt @@ -1,10 +1,10 @@ package application.exampleGeneration.openApiExamples -import application.exampleGeneration.ExamplesCommon +import application.exampleGeneration.ExamplesFeatureStrategy import io.specmatic.core.* import java.io.File -interface OpenApiExamplesCommon: ExamplesCommon { +class OpenApiExamplesFeatureStrategy: ExamplesFeatureStrategy { override val exampleFileExtensions: Set get() = setOf(JSON) override val contractFileExtensions: Set get() = OPENAPI_FILE_EXTENSIONS.toSet() diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt index a7558181e..bbb41cc6d 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt @@ -1,7 +1,7 @@ package application.exampleGeneration.openApiExamples +import application.exampleGeneration.ExamplesGenerationStrategy import application.exampleGeneration.ExamplesGenerateBase -import application.exampleGeneration.ExamplesGenerateCommon import io.specmatic.conversions.ExampleFromFile import io.specmatic.core.* import io.specmatic.core.utilities.uniqueNameForApiOperation @@ -15,21 +15,19 @@ import java.io.File description = ["Generate JSON Examples with Request and Response from an OpenApi Contract File"], subcommands = [OpenApiExamplesValidate::class, OpenApiExamplesInteractive::class] ) -class OpenApiExamplesGenerate: ExamplesGenerateBase(), OpenApiExamplesGenerateCommon { +class OpenApiExamplesGenerate: ExamplesGenerateBase ( + featureStrategy = OpenApiExamplesFeatureStrategy(), generationStrategy = OpenApiExamplesGenerationStrategy() +) { @Option(names = ["--extensive"], description = ["Generate all examples (by default, generates one example per 2xx API)"], defaultValue = "false") override var extensive: Boolean = false } -interface OpenApiExamplesGenerateCommon: ExamplesGenerateCommon, OpenApiExamplesCommon { - override fun generateExample(feature: Feature, scenario: Scenario, dictionary: Dictionary): Pair { +class OpenApiExamplesGenerationStrategy: ExamplesGenerationStrategy { + override fun generateExample(feature: Feature, scenario: Scenario): Pair { val request = scenario.generateHttpRequest() - val requestHttpPathPattern = scenario.httpRequestPattern.httpPathPattern - val updatedRequest = request.substituteDictionaryValues(dictionary, forceSubstitution = true, requestHttpPathPattern) - val response = feature.lookupResponse(scenario).cleanup() - val updatedResponse = response.substituteDictionaryValues(dictionary, forceSubstitution = true) - val scenarioStub = ScenarioStub(updatedRequest, updatedResponse) + val scenarioStub = ScenarioStub(request, response) val stubJSON = scenarioStub.toJSON().toStringLiteral() val uniqueName = uniqueNameForApiOperation(request, "", scenarioStub.response.status) diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index bb1db457e..f6b3175d5 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -15,7 +15,9 @@ import picocli.CommandLine.Option import java.io.File @Command(name = "interactive", description = ["Generate and validate examples interactively through a Web UI"],) -class OpenApiExamplesInteractive : ExamplesInteractiveBase(), OpenApiExamplesGenerateCommon, OpenApiExamplesValidateCommon { +class OpenApiExamplesInteractive : ExamplesInteractiveBase( + featureStrategy = OpenApiExamplesFeatureStrategy(), generationStrategy = OpenApiExamplesGenerationStrategy(), validationStrategy = OpenApiExamplesValidationStrategy() +) { @Option(names = ["--extensive"], description = ["Display all responses, not just 2xx, in the table."], defaultValue = "false") override var extensive: Boolean = false diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt index ab9c2659c..fd92242ac 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesValidate.kt @@ -1,6 +1,6 @@ package application.exampleGeneration.openApiExamples -import application.exampleGeneration.ExamplesValidateCommon +import application.exampleGeneration.ExamplesValidationStrategy import application.exampleGeneration.ExamplesValidateBase import io.specmatic.core.* import io.specmatic.core.utilities.capitalizeFirstChar @@ -13,7 +13,9 @@ import picocli.CommandLine.Option import java.io.File @Command(name = "validate", description = ["Validate OpenAPI inline and external examples"]) -class OpenApiExamplesValidate: ExamplesValidateBase(), OpenApiExamplesValidateCommon { +class OpenApiExamplesValidate: ExamplesValidateBase( + featureStrategy = OpenApiExamplesFeatureStrategy(), validationStrategy = OpenApiExamplesValidationStrategy() +) { @Option(names = ["--validate-external"], description = ["Validate external examples, defaults to true"]) override var validateExternal: Boolean = true @@ -23,7 +25,7 @@ class OpenApiExamplesValidate: ExamplesValidateBase(), OpenAp override var extensive: Boolean = true } -interface OpenApiExamplesValidateCommon: ExamplesValidateCommon, OpenApiExamplesCommon { +class OpenApiExamplesValidationStrategy: ExamplesValidationStrategy { override fun updateFeatureForValidation(feature: Feature, filteredScenarios: List): Feature { return feature.copy(scenarios = filteredScenarios) } diff --git a/junit5-support/src/main/resources/templates/example/index.html b/junit5-support/src/main/resources/templates/example/index.html index 233512f19..202f8b98b 100644 --- a/junit5-support/src/main/resources/templates/example/index.html +++ b/junit5-support/src/main/resources/templates/example/index.html @@ -464,10 +464,14 @@ } } - .btn-grp { + & .btn-grp { display: flex; align-items: center; gap: 1rem; + + &[data-test-mode="false"] > #bulk-test { + display: none; + } } & button { @@ -649,21 +653,9 @@ & button.validate { display: inline-block; } - } - - & > span { - display: block; - } - } - - &[data-valid="success"] { - & td:last-child { - & button.validate { - display: none; - } - & button.test { - display: inline-block; + & > span { + display: block; } } } @@ -697,21 +689,28 @@ } } - td:nth-child(3) { - text-align: initial; - min-width: 10rem; - } - td:nth-child(4) { - min-width: 2rem; + tbody { + &[data-test-mode="true"] > tr[data-valid="success"] { + & td:last-child { + & button.validate { + display: none; + } + + & button.test { + display: inline-block; + } + } + } } - td:last-child { - max-width: 9rem; + td:nth-child(3) { + text-align: initial; + min-width: 10rem; } - td:nth-last-child(2) { - max-width: 15rem; + td:last-child, td:nth-last-child(2) { + max-width: 10rem; & p { white-space: normal; @@ -996,7 +995,7 @@

-
+
@@ -1018,7 +1017,7 @@

Action
@@ -1030,10 +1028,9 @@

[[${iter.index + 1}]]

 +

[[${group.value}]]

[[${group.extraInfo}]] 		Action
-

[[${iter.index + 1}]]

-

[[${group.value}]]

[[${group.extraInfo}]] + 		- -

+ +

@@ -1064,93 +1076,86 @@

\ No newline at end of file From 2aa4f75286f6a2e47b6eb492671081853215021c Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Sat, 19 Oct 2024 08:39:47 +0530 Subject: [PATCH 40/43] Remove multi generate flag, ui ux changes. - Generate More Button should appear on last group column of the row. - use viewTransitions if available, fallback to requestAnimationFrame for smooth layout shifts. - Minor refactorings. --- .../ExamplesInteractiveBase.kt | 5 +- .../examples/ExamplesInteractiveServer.kt | 3 +- .../examples/InteractiveServerProvider.kt | 1 - .../resources/templates/example/index.html | 77 ++++++++++++++----- 4 files changed, 61 insertions(+), 25 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt index bed9336a1..549ef7a50 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesInteractiveBase.kt @@ -23,7 +23,6 @@ abstract class ExamplesInteractiveBase ( @Option(names = ["--dictionary"], description = ["Path to external dictionary file (default: contract_file_name_dictionary.json or dictionary.json)"]) protected var dictFile: File? = null - override val multiGenerate: Boolean = false override val serverHost: String = "0.0.0.0" override val serverPort: Int = 9001 abstract val server: ExamplesInteractiveServer @@ -98,9 +97,9 @@ abstract class ExamplesInteractiveBase ( val examples = getExternalExampleFiles(examplesDir) val scenarioExamplePair = scenarios.flatMap { - listOf(it to null) + generationStrategy.getExistingExamples(it, examples).map { exRes -> + generationStrategy.getExistingExamples(it, examples).map { exRes -> it to ExampleValidationResult(exRes.first, exRes.second) - } + }.ifEmpty { listOf(it to null) } } return createTableRows(scenarioExamplePair) } diff --git a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt index 2588966fa..523e08c68 100644 --- a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt +++ b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt @@ -136,8 +136,7 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti "hostPort" to hostPort, "hasExamples" to tableRows.any { it.isGenerated }, "exampleDetails" to tableRows.transform(), - "isTestMode" to (sutBaseUrl != null), - "multiGenerate" to multiGenerate + "isTestMode" to (sutBaseUrl != null) ) return HtmlTemplateConfiguration.process("example/index.html", variables) diff --git a/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt b/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt index 14d5b3e05..87ce9fb60 100644 --- a/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt +++ b/core/src/main/kotlin/io/specmatic/examples/InteractiveServerProvider.kt @@ -8,7 +8,6 @@ interface InteractiveServerProvider { val serverHost: String val serverPort: Int val sutBaseUrl: String? - val multiGenerate: Boolean val contractFile: File? val exampleTableColumns: List diff --git a/core/src/main/resources/templates/example/index.html b/core/src/main/resources/templates/example/index.html index 4149ce6ee..ff405a5ca 100644 --- a/core/src/main/resources/templates/example/index.html +++ b/core/src/main/resources/templates/example/index.html @@ -620,14 +620,17 @@ --_padding: 0.5rem 0rem; --_background-color: var(--blue); --_text-color: var(--white); + --_width: 7rem; + --_font-size: 1rem; + --_radius: 0.5rem; padding: var(--_padding); display: inline-block; - width: 7rem; - border-radius: 0.5rem; + width: var(--_width); + border-radius: var(--_radius); color: rgb(var(--_text-color)); background-color: rgba(var(--_background-color)); - font-size: 1rem; + font-size: var(--_font-size); &:hover { --_background-color: var(--blue-dark); @@ -693,6 +696,21 @@ & button.test::after { content: var(--_content, "Test"); } + + &[data-generate="success"] > td:nth-last-child(3) { + pointer-events: all; + + & > button.multigen { + --_padding: 0.35rem 0.35rem; + --_font-size: 0.75rem; + --_radius: 0.35rem; + display: inline-block; + + &::after { + content: var(--_content, "Generate More"); + } + } + } } @@ -708,12 +726,6 @@ } } } - - &[data-multi-gen="false"] > tr > td:nth-last-child(3) { - & > button { - display: none; - } - } } td:nth-child(3) { @@ -1030,7 +1042,7 @@

Action
@@ -1179,7 +1192,8 @@

// LISTENERS HELPERS async function handleTableButtons(target, tableRow, rowValues) { switch (target.getAttribute("aria-label")) { - case "Generate": { + case "Generate": + case "Generate More": { return await generateRowExamples(tableRow, rowValues); } case "Validate": { @@ -1270,10 +1284,12 @@

// ROW ACTIONS async function generateRowExamples(tableRow, rowValues, bulkMode = false) { + const originalState = tableRow.getAttribute("data-generate"); + tableRow.setAttribute("data-generate", "processing"); const { examples, error } = await generateExample(rowValues); const {createdCount, failedCount, existedCount, totalCount} = getExamplesCount(examples); - tableRow.removeAttribute("data-generate"); + tableRow.setAttribute("data-generate", originalState); if (error) { if (!bulkMode) createAlert("Example Generation Failed", error, true); @@ -1281,9 +1297,17 @@

} const newExamples = getOnlyNewExamples(tableRow, examples); - const newRows = newExamples.map((ex) => exampleToRow(tableRow, ex)); - updateSpans(tableRow, rowValues, newRows.length); - newRows.forEach(row => tableRow.parentElement.insertBefore(row, tableRow.nextElementSibling)); + const thisRowIsGenerated = tableRow.getAttribute("data-generate") === "success"; + const newRows = newExamples.map((ex, idx) => updateRowWithExample(tableRow, thisRowIsGenerated || idx > 0, ex)); + + const rowsToBeAdded = newRows.filter((row, idx) => idx > 0 || thisRowIsGenerated); + const exampleFragment = document.createDocumentFragment(); + rowsToBeAdded.forEach(row => exampleFragment.appendChild(row)) + + withViewTransitionOrAnimationFrame(() => { + updateSpans(tableRow, rowValues, thisRowIsGenerated ? newExamples.length : newExamples.length - 1); + tableRow.parentElement.insertBefore(exampleFragment, tableRow.nextElementSibling); + }) if (!bulkMode) { const allExist = newExamples.every(example => example.status === "EXISTS"); @@ -1368,11 +1392,16 @@

return examples.filter(example => !existingExamples.has(example.exampleFile)); } - function exampleToRow(tableRow, example) { - const newRow = tableRow.cloneNode(true); + function updateRowWithExample(tableRow, shouldClone, example) { + const newRow = shouldClone ? tableRow.cloneNode(true) : tableRow; storeExampleData(newRow, example.exampleFile, example.status); insertExampleIntoRow(example.exampleFile, newRow); - Array.from(newRow.children).slice(2, -2).forEach(cell => cell.classList.add("hidden")); + + if (shouldClone) { + newRow.removeAttribute("data-valid"); newRow.removeAttribute("data-test"); + Array.from(newRow.children).slice(2, -2).forEach(cell => cell.classList.add("hidden")); + } + newRow.setAttribute("data-generate", "success"); enableValidateBtn(newRow); return newRow; @@ -1647,6 +1676,16 @@

} // HELPERS + function withViewTransitionOrAnimationFrame(fn) { + if (!document.startViewTransition) { + return requestAnimationFrame(fn); + } + + document.startViewTransition(() => { + return fn(); + }) + } + function enableValidateBtn(tableRow) { examplesTable.setAttribute("data-generated", "true"); } From a739125a866d64c13c48749a62457fd5b6e1b2e8 Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Sat, 19 Oct 2024 09:11:15 +0530 Subject: [PATCH 41/43] Remove http prefix from fetch calls in JS. - ExamplesInteractiveServer should send the URL with http prefix, - POST requests being made to render HTML should also include protocol prefix with host and port. - Include iframe in core/resources to test server with a post request mimicking TMF and codespace --- .../examples/ExamplesInteractiveServer.kt | 3 +- .../resources/templates/example/index.html | 2 +- .../resources/interactive_iframe_test.html | 50 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 core/src/test/resources/interactive_iframe_test.html diff --git a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt index 523e08c68..4a04cc9b7 100644 --- a/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt +++ b/core/src/main/kotlin/io/specmatic/examples/ExamplesInteractiveServer.kt @@ -79,7 +79,7 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti } private fun getServerHostAndPort(request: ExamplePageRequest? = null): String { - return request?.hostPort?.takeIf { it.isNotEmpty() } ?: "localhost:$serverPort" + return request?.hostPort ?: "http://localhost:$serverPort" } private fun getContractFileOrNull(request: ExamplePageRequest? = null): Result { @@ -184,6 +184,7 @@ class ExamplesInteractiveServer(provider: InteractiveServerProvider) : Interacti post("/_specmatic/examples") { val request = call.receive() handleContractFile(request) { contract -> + cachedContractFile = contract val htmlContent = getHtmlContent(contract, getServerHostAndPort(request)) call.respondText(htmlContent, contentType = ContentType.Text.Html) } diff --git a/core/src/main/resources/templates/example/index.html b/core/src/main/resources/templates/example/index.html index ff405a5ca..a451b3465 100644 --- a/core/src/main/resources/templates/example/index.html +++ b/core/src/main/resources/templates/example/index.html @@ -1542,7 +1542,7 @@

// FETCH CALLS async function fetchFromBackend(endpoint, body) { try { - const resp = await fetch(`http://${getHostPort()}/_specmatic/examples/${endpoint}`, { + const resp = await fetch(`${getHostPort()}/_specmatic/examples/${endpoint}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) diff --git a/core/src/test/resources/interactive_iframe_test.html b/core/src/test/resources/interactive_iframe_test.html new file mode 100644 index 000000000..d09589631 --- /dev/null +++ b/core/src/test/resources/interactive_iframe_test.html @@ -0,0 +1,50 @@ + + + + + Interactive Test + + + +

Interactive Server POST Request Test

+

Ensure that you delete the generated example files in the resource folder if the already specified contract file is utilized.

+
+ +
+ + + + \ No newline at end of file From 411f12276902ae13ef7c544bdf234599a538a0df Mon Sep 17 00:00:00 2001 From: Sufiyan Date: Mon, 21 Oct 2024 11:56:04 +0530 Subject: [PATCH 42/43] Fix - Unique file names issue on generations. - Use a global counter to append a unique number to dile name. - Fix - OpenApi Interactive tableRow creation for cases with same path method and status but differing content-type. - Remove dropdown in drill-downs and chevron-down icon. - Modify updateSpans to take extraInfo into account. - Other CSS and JS fixes. --- .../exampleGeneration/ExamplesGenerateBase.kt | 3 + .../OpenApiExamplesGenerate.kt | 5 +- .../OpenApiExamplesInteractive.kt | 32 ++--- .../resources/templates/example/index.html | 115 ++++++------------ 4 files changed, 59 insertions(+), 96 deletions(-) diff --git a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt index e17ee78d0..47b83b3e2 100644 --- a/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt +++ b/application/src/main/kotlin/application/exampleGeneration/ExamplesGenerateBase.kt @@ -6,6 +6,7 @@ import io.specmatic.examples.ExampleGenerationResult import io.specmatic.examples.ExampleGenerationStatus import picocli.CommandLine.* import java.io.File +import java.util.concurrent.atomic.AtomicInteger abstract class ExamplesGenerateBase( override val featureStrategy: ExamplesFeatureStrategy, @@ -80,6 +81,8 @@ abstract class ExamplesGenerateBase( } interface ExamplesGenerationStrategy { + val atomicCounter: AtomicInteger + fun getExistingExamples(scenario: Scenario, exampleFiles: List): List> fun generateExample(feature: Feature, scenario: Scenario): Pair diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt index 2a0473323..af95fff30 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesGenerate.kt @@ -9,6 +9,7 @@ import io.specmatic.mock.ScenarioStub import picocli.CommandLine.Command import picocli.CommandLine.Option import java.io.File +import java.util.concurrent.atomic.AtomicInteger @Command( name = "examples", mixinStandardHelpOptions = true, @@ -23,6 +24,8 @@ class OpenApiExamplesGenerate: ExamplesGenerateBase ( } class OpenApiExamplesGenerationStrategy: ExamplesGenerationStrategy { + override val atomicCounter: AtomicInteger = AtomicInteger(1) + override fun generateExample(feature: Feature, scenario: Scenario): Pair { val request = scenario.generateHttpRequest() val response = feature.lookupResponse(scenario).cleanup() @@ -31,7 +34,7 @@ class OpenApiExamplesGenerationStrategy: ExamplesGenerationStrategy): List> { diff --git a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt index d81c7d80a..656acc27f 100644 --- a/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt +++ b/application/src/main/kotlin/application/exampleGeneration/openApiExamples/OpenApiExamplesInteractive.kt @@ -55,18 +55,20 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( pathGroup.methods.flatMap { (method, methodGroup) -> var showMethod = true methodGroup.statuses.flatMap { (status, statusGroup) -> - var showStatus = true - statusGroup.examples.map { (scenario, example) -> - ExampleTableRow( - columns = listOf( - ExampleRowGroup("path", convertPathParameterStyle(path), rawValue = path, rowSpan = pathGroup.count, showRow = showPath), - ExampleRowGroup("method", method, rowSpan = methodGroup.count, showRow = showMethod), - ExampleRowGroup("response", status, rowSpan = statusGroup.count, showRow = showStatus, extraInfo = scenario.httpRequestPattern.headersPattern.contentType) - ), - exampleFilePath = example?.exampleFile?.absolutePath, - exampleFileName = example?.exampleName, - exampleMismatchReason = example?.result?.reportString().takeIf { reason -> reason?.isNotBlank() == true } - ).also { showPath = false; showMethod = false; showStatus = false } + statusGroup.examples.flatMap { (_, scenarioExamplePair) -> + var showStatus = true + scenarioExamplePair.map { (scenario, example) -> + ExampleTableRow( + columns = listOf( + ExampleRowGroup("path", convertPathParameterStyle(path), rawValue = path, rowSpan = pathGroup.count, showRow = showPath), + ExampleRowGroup("method", method, rowSpan = methodGroup.count, showRow = showMethod), + ExampleRowGroup("response", status, rowSpan = scenarioExamplePair.size, showRow = showStatus, extraInfo = scenario.httpRequestPattern.headersPattern.contentType) + ), + exampleFilePath = example?.exampleFile?.absolutePath, + exampleFileName = example?.exampleName, + exampleMismatchReason = example?.result?.reportString().takeIf { reason -> reason?.isNotBlank() == true } + ).also { showPath = false; showMethod = false; showStatus = false } + } } } } @@ -81,7 +83,7 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( MethodGroup( count = methodGroup.size, statuses = methodGroup.groupBy { it.first.status.toString() }.mapValues { (_, statusGroup) -> - StatusGroup(count = statusGroup.size, examples = statusGroup) + StatusGroup(count = statusGroup.size, examples = statusGroup.groupBy { it.first.httpRequestPattern.headersPattern.contentType }) } ) } @@ -103,9 +105,9 @@ class OpenApiExamplesInteractive : ExamplesInteractiveBase( val contentType = response.extraInfo } - data class StatusGroup ( + data class StatusGroup( val count: Int, - val examples: List> + val examples: Map>> ) data class MethodGroup ( diff --git a/core/src/main/resources/templates/example/index.html b/core/src/main/resources/templates/example/index.html index a451b3465..b7d36e6e6 100644 --- a/core/src/main/resources/templates/example/index.html +++ b/core/src/main/resources/templates/example/index.html @@ -405,7 +405,6 @@ html { scroll-behavior: smooth; scrollbar-gutter: stable; - text-wrap: balance; } body { @@ -414,17 +413,6 @@ flex-direction: column; min-height: 100vh; min-width: 100vw; - - & > .chevron-down { - display: none; - } - } - - .chevron-down { - width: 2rem; - height: 2rem; - transition: all 0.25s ease-in-out; - flex-shrink: 0; } main { @@ -847,21 +835,7 @@ border-radius: 0.25rem; border: 1px solid rgba(var(--smoky-black), 0.25); box-shadow: var(--shadow-md); - - &[data-expand="false"] .dropdown { - height: 0px; - } - - &[data-expand="true"] { - .dropdown { - height: auto; - padding: 0rem 1rem 1rem 1rem; - } - - & .chevron-down { - transform: rotate(180deg); - } - } + padding: 0rem 1rem 1rem 1rem; } } @@ -869,11 +843,11 @@ display: flex; align-items: center; justify-content: space-between; - padding: 0.5rem 0.5rem 0.5rem 1rem; - cursor: pointer; + padding: 0.5rem; & > p { word-break: break-all; + margin-right: 4rem; flex: 1; } @@ -885,11 +859,6 @@ } div.dropdown { - --_anim-duration: 0.25s; - - transition: all var(--_anim-duration) ease-in-out; - overflow: hidden; - & > div:nth-child(2) > p { margin: 1rem 0 0.5rem 0; } @@ -1084,9 +1053,6 @@

- - -