From 894634901c684eccc2a9d01ba83fb33d07613b8c Mon Sep 17 00:00:00 2001 From: VladislavFetisov <1> Date: Sun, 22 Jan 2023 10:42:47 +0300 Subject: [PATCH 1/6] Put hashes in table ready --- .../research/kotoed/api/CourseVerticle.kt | 60 ++++- .../kotoed/api/SubmissionCodeVerticle.kt | 56 ++++- .../kotoed/db/FunctionPartHashVerticle.kt | 10 + .../research/kotoed/db/FunctionVerticle.kt | 10 + .../processors/SubmissionProcessorVerticle.kt | 220 +++++++++++++++++- .../research/kotoed/eventbus/Address.kt | 1 + .../jetbrains/research/kotoed/util/Util.kt | 34 +++ .../V93__Create_FunctionHash_table.sql | 19 ++ .../kotoed/klones/HashComputingTest.kt | 41 ++++ 9 files changed, 443 insertions(+), 8 deletions(-) create mode 100644 kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt create mode 100644 kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionVerticle.kt create mode 100644 kotoed-server/src/main/resources/db/migration/V93__Create_FunctionHash_table.sql create mode 100644 kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/HashComputingTest.kt diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt index 781d1993..d9b96fc2 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt @@ -2,10 +2,10 @@ package org.jetbrains.research.kotoed.api import io.vertx.core.json.JsonArray import io.vertx.core.json.JsonObject -import org.jetbrains.research.kotoed.data.api.CountResponse -import org.jetbrains.research.kotoed.data.api.DbRecordWrapper -import org.jetbrains.research.kotoed.data.api.SearchQuery -import org.jetbrains.research.kotoed.data.api.VerificationData +import kotlinx.coroutines.withContext +import org.jetbrains.kotlin.psi.KtNamedFunction +import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType +import org.jetbrains.research.kotoed.data.api.* import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery import org.jetbrains.research.kotoed.data.db.setPageForQuery import org.jetbrains.research.kotoed.data.db.textSearch @@ -14,13 +14,26 @@ import org.jetbrains.research.kotoed.database.Tables.COURSE_TEXT_SEARCH import org.jetbrains.research.kotoed.database.tables.records.BuildTemplateRecord import org.jetbrains.research.kotoed.database.tables.records.CourseRecord import org.jetbrains.research.kotoed.database.tables.records.CourseStatusRecord -import org.jetbrains.research.kotoed.db.condition.lang.formatToQuery +import org.jetbrains.research.kotoed.database.tables.records.FunctionRecord import org.jetbrains.research.kotoed.eventbus.Address import org.jetbrains.research.kotoed.util.* +import org.jetbrains.research.kotoed.util.code.getPsi +import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv import org.jetbrains.research.kotoed.util.database.toRecord +import java.lang.StringBuilder + +fun KtNamedFunction.getFullName(): String { + val resultName = StringBuilder(this.containingFile.name) + resultName.append(this.name) + for (valueParameter in this.valueParameters) { + resultName.append(valueParameter.typeReference!!.text) + } + return resultName.toString() +} @AutoDeployable class CourseVerticle : AbstractKotoedVerticle(), Loggable { + private val ee by lazy { betterSingleThreadContext("courseVerticle.executor") } @JsonableEventBusConsumerFor(Address.Api.Course.Create) suspend fun handleCreate(course: CourseRecord): DbRecordWrapper { @@ -51,6 +64,43 @@ class CourseVerticle : AbstractKotoedVerticle(), Loggable { suspend fun handleUpdate(course: CourseRecord): DbRecordWrapper { val res: CourseRecord = dbUpdateAsync(course) val status: VerificationData = dbProcessAsync(res) + + val files: Code.ListResponse = sendJsonableAsync( + Address.Api.Course.Code.List, + Code.Course.ListRequest(course.id) + ) + temporaryKotlinEnv {//FIXME COPYPASTE + withContext(ee) { + val ktFiles = + files.root?.toFileSeq() + .orEmpty() + .filter { it.endsWith(".kt") } + .toList() //FIXME + .map { filename -> + val resp: Code.Submission.ReadResponse = sendJsonableAsync( + Address.Api.Submission.Code.Read, + Code.Submission.ReadRequest( + submissionId = res.id, path = filename + ) + ) + getPsi(resp.contents, filename) + } + val functionsList = ktFiles.asSequence() + .flatMap { file -> + file.collectDescendantsOfType().asSequence() + } + .filter { method -> + method.annotationEntries.all { anno -> "@Test" != anno.text } + } + .forEach { + val resultName = it.getFullName() + val functionRecord = FunctionRecord().apply { name = resultName } + if (dbFindAsync(functionRecord).isEmpty()) { + dbCreateAsync(functionRecord) + } + } + } + } return DbRecordWrapper(res, status) } diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt index 6a1a9920..fa85e7f4 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt @@ -1,10 +1,12 @@ package org.jetbrains.research.kotoed.api +import org.jetbrains.research.kotoed.data.api.Code import org.jetbrains.research.kotoed.data.api.Code.FileRecord import org.jetbrains.research.kotoed.data.api.Code.FileType.directory import org.jetbrains.research.kotoed.data.api.Code.FileType.file import org.jetbrains.research.kotoed.data.api.Code.ListResponse import org.jetbrains.research.kotoed.data.api.Code.Submission.RemoteRequest +import org.jetbrains.research.kotoed.data.api.DiffType import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery import org.jetbrains.research.kotoed.data.vcs.* import org.jetbrains.research.kotoed.database.enums.SubmissionState @@ -132,6 +134,18 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() { @JsonableEventBusConsumerFor(Address.Api.Submission.Code.Diff) suspend fun handleSubmissionCodeDiff(message: SubDiffRequest): SubDiffResponse { + return diffResponse(message, DiffType.DIFF_WITH_CLOSED) + } + + @JsonableEventBusConsumerFor(Address.Api.Submission.Code.DiffWithPrevious) + suspend fun handleSubmissionCodeDiffWithPrevious(message: SubDiffRequest): SubDiffResponse { + return diffResponse(message, DiffType.DIFF_WITH_PREVIOUS) + } + + private suspend fun diffResponse( + message: Code.Submission.DiffRequest, + diffType: DiffType + ): Code.Submission.DiffResponse { val submission: SubmissionRecord = dbFetchAsync(SubmissionRecord().apply { id = message.submissionId }) val repoInfo = getCommitInfo(submission) when (repoInfo.cloneStatus) { @@ -140,7 +154,7 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() { else -> { } } - val diff = submissionCodeDiff(submission, repoInfo) + val diff = getDiff(submission, repoInfo, diffType) return SubDiffResponse(diff = diff.contents, status = repoInfo.cloneStatus) } @@ -262,7 +276,21 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() { ) } - private suspend fun submissionCodeDiff(submission: SubmissionRecord, repoInfo: CommitInfo): DiffResponse { + private suspend fun getDiff( + submission: SubmissionRecord, + repoInfo: CommitInfo, + diffType: DiffType + ): DiffResponse { + return when (diffType) { + DiffType.DIFF_WITH_CLOSED -> submissionCodeDiff(submission, repoInfo) + DiffType.DIFF_WITH_PREVIOUS -> submissionCodeDiffWithPrevious(submission, repoInfo) + } + } + + private suspend fun submissionCodeDiff( + submission: SubmissionRecord, + repoInfo: CommitInfo + ): DiffResponse { val closedSubs = dbFindAsync(SubmissionRecord().apply { projectId = submission.projectId state = SubmissionState.closed @@ -272,6 +300,30 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() { it.datetime < submission.datetime }.sortedByDescending { it.datetime }.firstOrNull() + return getDiffBetweenSubmission(foundationSub, submission, repoInfo) + } + + private suspend fun submissionCodeDiffWithPrevious( + submission: SubmissionRecord, + repoInfo: CommitInfo + ): DiffResponse { + val prevSubs = dbFindAsync(SubmissionRecord().apply { + projectId = submission.projectId + state != SubmissionState.invalid + }) + + val newestPrevSub = prevSubs.filter { + it.datetime < submission.datetime + }.sortedByDescending { it.datetime }.firstOrNull() + + return getDiffBetweenSubmission(newestPrevSub, submission, repoInfo) + } + + private suspend fun getDiffBetweenSubmission( + foundationSub: SubmissionRecord?, + submission: SubmissionRecord, + repoInfo: CommitInfo + ): DiffResponse { var baseRev = foundationSub?.revision if (baseRev == null) { diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt new file mode 100644 index 00000000..fe928480 --- /dev/null +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt @@ -0,0 +1,10 @@ +package org.jetbrains.research.kotoed.db + +import org.jetbrains.research.kotoed.database.Tables +import org.jetbrains.research.kotoed.database.tables.records.FunctionPartHashRecord +import org.jetbrains.research.kotoed.util.AutoDeployable + +@AutoDeployable +class FunctionPartHashVerticle : CrudDatabaseVerticleWithReferences(Tables.FUNCTION_PART_HASH){ + +} \ No newline at end of file diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionVerticle.kt new file mode 100644 index 00000000..db40115c --- /dev/null +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionVerticle.kt @@ -0,0 +1,10 @@ +package org.jetbrains.research.kotoed.db + +import org.jetbrains.research.kotoed.database.Tables +import org.jetbrains.research.kotoed.database.tables.records.FunctionRecord +import org.jetbrains.research.kotoed.util.AutoDeployable + +@AutoDeployable +class FunctionVerticle : CrudDatabaseVerticleWithReferences(Tables.FUNCTION) { + +} \ No newline at end of file diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/processors/SubmissionProcessorVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/processors/SubmissionProcessorVerticle.kt index 75fc22ef..2a5e0bf1 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/processors/SubmissionProcessorVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/processors/SubmissionProcessorVerticle.kt @@ -1,8 +1,19 @@ package org.jetbrains.research.kotoed.db.processors +import com.intellij.psi.PsiElement import io.vertx.core.json.JsonObject +import kotlinx.coroutines.withContext +import kotlinx.warnings.Warnings +import org.jetbrains.kotlin.psi.KtNamedFunction +import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType +import org.jetbrains.kotlin.psi.psiUtil.endOffset +import org.jetbrains.kotlin.psi.psiUtil.startOffsetSkippingComments +import org.jetbrains.research.kotoed.api.getFullName import org.jetbrains.research.kotoed.code.Filename import org.jetbrains.research.kotoed.code.Location +import org.jetbrains.research.kotoed.code.diff.HunkJsonable +import org.jetbrains.research.kotoed.code.diff.RangeJsonable +import org.jetbrains.research.kotoed.data.api.Code import org.jetbrains.research.kotoed.data.api.VerificationData import org.jetbrains.research.kotoed.data.api.VerificationStatus import org.jetbrains.research.kotoed.data.buildSystem.BuildAck @@ -13,9 +24,14 @@ import org.jetbrains.research.kotoed.database.enums.SubmissionState import org.jetbrains.research.kotoed.database.tables.records.* import org.jetbrains.research.kotoed.eventbus.Address import org.jetbrains.research.kotoed.util.* +import org.jetbrains.research.kotoed.util.code.getPsi +import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv import org.jetbrains.research.kotoed.util.database.executeKAsync import org.jetbrains.research.kotoed.util.database.toRecord import org.jooq.ForeignKey +import java.io.File +import java.util.function.BiConsumer +import java.util.function.BiFunction data class BuildTriggerResult( val result: String, @@ -24,6 +40,7 @@ data class BuildTriggerResult( @AutoDeployable class SubmissionProcessorVerticle : ProcessorVerticle(Tables.SUBMISSION) { + private val ee by lazy { betterSingleThreadContext("submissionProcessorVerticle.executor") } // parent submission id can be invalid, filter it out override val checkedReferences: List> @@ -193,7 +210,7 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S buildRequestId = ack.buildId } ) - + computeHashesFromSub(sub) //FIXME call only when create submission VerificationData.Processed } 1 -> { @@ -337,5 +354,206 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S return VerificationData.Unknown } + private suspend fun computeHashesFromSub(res: SubmissionRecord) { + log.info("Start computing hashing for submission=[${res.id}]") + val diffResponse: Code.Submission.DiffResponse = sendJsonableAsync( + Address.Api.Submission.Code.DiffWithPrevious, + Code.Submission.DiffRequest(submissionId = res.id) + ) + val files: Code.ListResponse = sendJsonableAsync( + Address.Api.Submission.Code.List, + Code.Submission.ListRequest(res.id) + ) + + temporaryKotlinEnv { + withContext(ee) { + val ktFiles = + files.root?.toFileSeq() + .orEmpty() + .filter { it.endsWith(".kt") } + .toList() //FIXME + .map { filename -> + val resp: Code.Submission.ReadResponse = sendJsonableAsync( + Address.Api.Submission.Code.Read, + Code.Submission.ReadRequest( + submissionId = res.id, path = filename + ) + ) + getPsi(resp.contents, filename) + } + val functionsList = ktFiles.asSequence() + .flatMap { file -> + file.collectDescendantsOfType().asSequence() + } + .filter { method -> + method.annotationEntries.all { anno -> "@Test" != anno.text } + } + .map { + @Suppress(Warnings.USELESS_CAST) + it as PsiElement + } + .toList() + + val changesInFiles = diffResponse.diff.associate { + if (it.toFile != it.fromFile) { + log.warn("File [${it.fromFile}] is renamed to [${it.toFile}]") + } + it.toFile to it.changes + } + val project = dbFindAsync(ProjectRecord().apply { id = res.projectId }).first() + + for (function in functionsList) { + processFunction(function, res, changesInFiles, project) + } + + } + } + + } + + private suspend fun processFunction( + function: PsiElement, + res: SubmissionRecord, + changesInFiles: Map>, + project: ProjectRecord + ) { + val functionFullName = (function as KtNamedFunction).getFullName() + val functionRecord = dbFindAsync(FunctionRecord().apply { name = functionFullName }) + val functionFromDb: FunctionRecord + when (functionRecord.size) { + 0 -> { + log.info("Add new function=[${functionFullName}] in submission=[${res.id}]") + functionFromDb = dbCreateAsync(FunctionRecord().apply { name = functionFullName }) + } + + 1 -> { + functionFromDb = functionRecord.first()!! + } + + else -> { + throw IllegalStateException( + "Amount of function [${functionFullName}] in table is ${functionRecord.size}" + ) + } + } + val document = function.containingFile.viewProvider.document + ?: throw IllegalStateException("Function's=[${function.containingFile.name}] document is null") + val fileChanges = changesInFiles[function.containingFile.name] ?: return + val funStartLine = document.getLineNumber(function.startOffsetSkippingComments) + 1 + val funFinishLine = document.getLineNumber(function.endOffset) + 1 + for (change in fileChanges) { + val range = change.to + if (isNeedToRecomputeHash(funStartLine, funFinishLine, range)) { + val hashesForLevels: List> = computeHashesForElement(function.bodyExpression as PsiElement) + putHashesInTable(hashesForLevels, functionFromDb, res, project) + break + } + } + } + + private suspend fun putHashesInTable( + hashesForLevels: List>, + functionFromDb: FunctionRecord, + res: SubmissionRecord, + project: ProjectRecord + ) { + hashesForLevels.forEachIndexed { index, hashes -> + dbBatchCreateAsync(hashes.map { + FunctionPartHashRecord().apply { + functionid = functionFromDb.id + submissionid = res.id + denizenId = project.denizenId + hash = it + level = index + } + }) + } + } + + fun computeHashesForElement(root: PsiElement): List> { + val associativeArray = mutableListOf>(mutableListOf()) + val treeVisitor = object : TreeVisitor { + override fun getLeaf(root: PsiElement): Int { + return getSelf(root) + } + + override fun getSelf(element: PsiElement): Int { + return element.javaClass.name.hashCode() + } + + override fun getAccumulator(): BiFunction { + return BiFunction { a, b -> a + b } + } + + override fun getConsumers(): List> { + return listOf(BiConsumer { hash, level -> + if (level > associativeArray.size) { + throw IllegalStateException( + "Level=[${level}] more than associativeArray size=[${associativeArray.size}]" + ) + } + if (level == associativeArray.size) { + associativeArray.add(mutableListOf()) + } + associativeArray[level].add(hash) + }) + } + } + treeVisitor.dfs(root) + return associativeArray + } + + private fun isNeedToRecomputeHash(funStartLine: Int, funFinishLine: Int, changesRange: RangeJsonable): Boolean { + val start = changesRange.start + val finish = start + changesRange.count + val out = start > funFinishLine || finish < funStartLine + return !out + } + + + private fun computeStatistics(list: List) { + val treeVisitor = object : TreeVisitor { + override fun getLeaf(root: PsiElement): Int { + return 0 + } + override fun getSelf(element: PsiElement): Int { + return 1 + } + + override fun getAccumulator(): BiFunction { + return BiFunction { a, b -> a + b } + } + } + val otherList = list.map { + val levelsCount = treeVisitor.dfs(it.children.last()).first + val linesCount = it.children[it.children.size - 1].text.count { chr -> chr == '\n' } + 1 + (it as KtNamedFunction).name to Pair(levelsCount, linesCount) + }.toList() + val sortedRatio = otherList + .map { el -> el.second.first * 1.0 / el.second.second } + .sorted() + val median: Double = + if (sortedRatio.size % 2 == 0) (sortedRatio[sortedRatio.size / 2 - 1] + sortedRatio[sortedRatio.size / 2]) / 2.0 + else sortedRatio[sortedRatio.size / 2] * 1.0 + val percentile95: Double = sortedRatio[(sortedRatio.size * 0.95).toInt()] + + File("InformationAboutCommit") + .bufferedWriter() + .use { out -> + var treeLevelsSum = 0 + var linesSum = 0 + for (pair in otherList) { + treeLevelsSum += pair.second.first + val lineInFunctions = pair.second.second + linesSum += lineInFunctions + out.write("Name:${pair.first} ${pair.second.first} $lineInFunctions") + out.newLine() + } + out.write("All statistics: TreeLevels:${treeLevelsSum} FunctionsLines:${linesSum}") + out.newLine() + out.write("Median:${median} 95_Percentile:${percentile95}") + out.flush() + } + } } diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt index f1fa430f..54c8bcdd 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt @@ -110,6 +110,7 @@ object Address { const val List = "kotoed.api.submission.code.list" const val Date = "kotoed.api.submission.code.date" const val Diff = "kotoed.api.submission.code.diff" + const val DiffWithPrevious = "kotoed.api.submission.code.diffWithPrevious" } object Result { diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt index c1d021fe..77dbb66a 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt @@ -6,6 +6,7 @@ import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader import com.google.common.cache.LoadingCache import com.hazelcast.util.Base64 +import com.intellij.psi.PsiElement import io.vertx.core.MultiMap import io.vertx.core.logging.Logger import io.vertx.core.logging.LoggerFactory @@ -20,6 +21,8 @@ import java.net.URI import java.net.URLEncoder import java.util.* import java.util.concurrent.TimeUnit +import java.util.function.BiConsumer +import java.util.function.BiFunction import kotlin.reflect.KClass import kotlin.reflect.KType import kotlin.reflect.KTypeProjection @@ -295,3 +298,34 @@ fun GenericKType(kClass: KClass) = GenericKType(kClass.starPr @OptIn(ExperimentalStdlibApi::class) inline fun genericTypeOf(): GenericKType = GenericKType(typeOf()) + +interface TreeVisitor { + fun getLeaf(root: PsiElement): T + + fun getSelf(element: PsiElement): T + + fun getAccumulator(): BiFunction + + fun getStartLevelNum(): Int { + return 0 + } + + fun getConsumers(): List> { + return emptyList() + } + + fun dfs(root: PsiElement): Pair { + if (root.children.isEmpty()) { + return Pair(getLeaf(root), getStartLevelNum()) + } + var maxLevel = 0 + var accumulation = getSelf(root) + for (child in root.children) { + val nextLevel = dfs(child) + maxLevel = Math.max(maxLevel, nextLevel.second) + accumulation = getAccumulator().apply(accumulation, nextLevel.first) + } + getConsumers().forEach { it.accept(accumulation, maxLevel + 1) } + return Pair(accumulation, maxLevel + 1) + } +} \ No newline at end of file diff --git a/kotoed-server/src/main/resources/db/migration/V93__Create_FunctionHash_table.sql b/kotoed-server/src/main/resources/db/migration/V93__Create_FunctionHash_table.sql new file mode 100644 index 00000000..ccf830a1 --- /dev/null +++ b/kotoed-server/src/main/resources/db/migration/V93__Create_FunctionHash_table.sql @@ -0,0 +1,19 @@ +CREATE TABLE function +( + id SERIAL NOT NULL PRIMARY KEY, + name varchar(100) UNIQUE NOT NULL +); + +CREATE TABLE function_part_hash +( + id SERIAL NOT NULL PRIMARY KEY, + functionId INT REFERENCES function NOT NULL, + submissionId INT REFERENCES submission NOT NULL, + denizen_id INT REFERENCES denizen_unsafe NOT NULL, + hash INT NOT NULL, + level INT NOT NULL +); + +CREATE INDEX IF NOT EXISTS function_part_hash_key ON function_part_hash(hash); +CREATE INDEX IF NOT EXISTS function_part_denizenId_key ON function_part_hash(denizen_id); + diff --git a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/HashComputingTest.kt b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/HashComputingTest.kt new file mode 100644 index 00000000..b9860255 --- /dev/null +++ b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/HashComputingTest.kt @@ -0,0 +1,41 @@ +package org.jetbrains.research.kotoed.klones + +import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.psi.KtNamedFunction +import org.jetbrains.research.kotoed.db.processors.SubmissionProcessorVerticle +import org.jetbrains.research.kotoed.util.Loggable +import org.jetbrains.research.kotoed.util.code.getPsi +import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv +import org.junit.Test +import kotlin.test.assertEquals + +class HashComputingTest: Loggable { + val subProcessorVarticle = SubmissionProcessorVerticle() + + @Test + fun differentExpressionEqualsHashesTest() { + val firstFunction = + //language=Kotlin + """ + fun foo(a:Int, b:Int): Int = a + b + """ + val secondFunction = + //language=Kotlin + """ + fun bar(cap:Int, barbaris:Int): Int = cap / barbaris + """ + + val firstFun = getPsiElementFromString(firstFunction) + val hashesForFirstFunction = subProcessorVarticle.computeHashesForElement(firstFun) + val secondFun = getPsiElementFromString(secondFunction) + val hashesForSecondFunction = subProcessorVarticle.computeHashesForElement(secondFun) + assertEquals(hashesForFirstFunction, hashesForSecondFunction) + } + + private fun getPsiElementFromString(expression: String): PsiElement { + val firstFunctionFile = temporaryKotlinEnv { + getPsi(expression) + } + return firstFunctionFile + } +} \ No newline at end of file From 7eaa5005329a0fd5fceb68a07ff704bf10e78e10 Mon Sep 17 00:00:00 2001 From: VladislavFetisov <1> Date: Sun, 22 Jan 2023 10:44:28 +0300 Subject: [PATCH 2/6] Put hashes in table ready --- .../main/kotlin/org/jetbrains/research/kotoed/data/api/Data.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/data/api/Data.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/data/api/Data.kt index 0ea927fa..711aa4bc 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/data/api/Data.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/data/api/Data.kt @@ -23,6 +23,9 @@ enum class VerificationStatus { Processed, Invalid } +enum class DiffType { + DIFF_WITH_PREVIOUS, DIFF_WITH_CLOSED +} fun VerificationData?.bang() = this ?: VerificationData.Unknown From e81c24b99de886d4626fbf1e4220bf2620efe964 Mon Sep 17 00:00:00 2001 From: VladislavFetisov <1> Date: Wed, 29 Mar 2023 21:08:41 +0300 Subject: [PATCH 3/6] Put hashes in table ready --- kotoed-server/db.properties | 2 +- .../research/kotoed/api/CourseVerticle.kt | 45 --- .../kotoed/api/SubmissionCodeVerticle.kt | 29 +- .../kotoed/code/klones/KloneVerticle.kt | 16 + .../kotoed/db/FunctionLeavesVerticle.kt | 8 + .../kotoed/db/FunctionPartHashVerticle.kt | 369 +++++++++++++++++- .../research/kotoed/db/HashClonesVerticle.kt | 9 + .../db/ProcessedProjectSubmissionVerticle.kt | 11 + .../processors/SubmissionProcessorVerticle.kt | 350 +++++++++-------- .../research/kotoed/eventbus/Address.kt | 2 + .../research/kotoed/util/TreeVisitor.kt | 42 ++ .../jetbrains/research/kotoed/util/Util.kt | 36 +- .../V93__Add_Incremental_Clone_Detecting.sql | 45 +++ .../V93__Create_FunctionHash_table.sql | 19 - .../klones/KlonesAlgorithmComparingTest.kt | 101 +++++ .../kotoed/klones/SegmentAbsorbingTest.kt | 59 +++ .../research/kotoed/klones/VisitorTest.kt | 302 ++++++++++++++ 17 files changed, 1168 insertions(+), 277 deletions(-) create mode 100644 kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionLeavesVerticle.kt create mode 100644 kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/HashClonesVerticle.kt create mode 100644 kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/ProcessedProjectSubmissionVerticle.kt create mode 100644 kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeVisitor.kt create mode 100644 kotoed-server/src/main/resources/db/migration/V93__Add_Incremental_Clone_Detecting.sql delete mode 100644 kotoed-server/src/main/resources/db/migration/V93__Create_FunctionHash_table.sql create mode 100644 kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/KlonesAlgorithmComparingTest.kt create mode 100644 kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/SegmentAbsorbingTest.kt create mode 100644 kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/VisitorTest.kt diff --git a/kotoed-server/db.properties b/kotoed-server/db.properties index ef650101..43ff4c11 100644 --- a/kotoed-server/db.properties +++ b/kotoed-server/db.properties @@ -1,4 +1,4 @@ -db.url=jdbc:postgresql://localhost/kotoed +db.url=jdbc:postgresql://localhost:5432/kotoedFinal db.user=kotoed db.password=kotoed testdb.url=jdbc:postgresql://localhost/kotoed-test diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt index d9b96fc2..a38e2abd 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt @@ -22,14 +22,6 @@ import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv import org.jetbrains.research.kotoed.util.database.toRecord import java.lang.StringBuilder -fun KtNamedFunction.getFullName(): String { - val resultName = StringBuilder(this.containingFile.name) - resultName.append(this.name) - for (valueParameter in this.valueParameters) { - resultName.append(valueParameter.typeReference!!.text) - } - return resultName.toString() -} @AutoDeployable class CourseVerticle : AbstractKotoedVerticle(), Loggable { @@ -64,43 +56,6 @@ class CourseVerticle : AbstractKotoedVerticle(), Loggable { suspend fun handleUpdate(course: CourseRecord): DbRecordWrapper { val res: CourseRecord = dbUpdateAsync(course) val status: VerificationData = dbProcessAsync(res) - - val files: Code.ListResponse = sendJsonableAsync( - Address.Api.Course.Code.List, - Code.Course.ListRequest(course.id) - ) - temporaryKotlinEnv {//FIXME COPYPASTE - withContext(ee) { - val ktFiles = - files.root?.toFileSeq() - .orEmpty() - .filter { it.endsWith(".kt") } - .toList() //FIXME - .map { filename -> - val resp: Code.Submission.ReadResponse = sendJsonableAsync( - Address.Api.Submission.Code.Read, - Code.Submission.ReadRequest( - submissionId = res.id, path = filename - ) - ) - getPsi(resp.contents, filename) - } - val functionsList = ktFiles.asSequence() - .flatMap { file -> - file.collectDescendantsOfType().asSequence() - } - .filter { method -> - method.annotationEntries.all { anno -> "@Test" != anno.text } - } - .forEach { - val resultName = it.getFullName() - val functionRecord = FunctionRecord().apply { name = resultName } - if (dbFindAsync(functionRecord).isEmpty()) { - dbCreateAsync(functionRecord) - } - } - } - } return DbRecordWrapper(res, status) } diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt index fa85e7f4..2ec4a62b 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt @@ -8,14 +8,21 @@ import org.jetbrains.research.kotoed.data.api.Code.ListResponse import org.jetbrains.research.kotoed.data.api.Code.Submission.RemoteRequest import org.jetbrains.research.kotoed.data.api.DiffType import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery +import org.jetbrains.research.kotoed.data.db.setPageForQuery import org.jetbrains.research.kotoed.data.vcs.* +import org.jetbrains.research.kotoed.database.Tables import org.jetbrains.research.kotoed.database.enums.SubmissionState +import org.jetbrains.research.kotoed.database.tables.records.CommentTemplateRecord import org.jetbrains.research.kotoed.database.tables.records.CourseRecord import org.jetbrains.research.kotoed.database.tables.records.ProjectRecord import org.jetbrains.research.kotoed.database.tables.records.SubmissionRecord +import org.jetbrains.research.kotoed.db.condition.lang.formatToQuery import org.jetbrains.research.kotoed.eventbus.Address import org.jetbrains.research.kotoed.util.* +import org.jetbrains.research.kotoed.util.AnyAsJson.get import org.jetbrains.research.kotoed.util.database.toRecord +import java.sql.Timestamp +import java.time.OffsetDateTime import org.jetbrains.research.kotoed.data.api.Code.Course.ListRequest as CrsListRequest import org.jetbrains.research.kotoed.data.api.Code.Course.ReadRequest as CrsReadRequest import org.jetbrains.research.kotoed.data.api.Code.Course.ReadResponse as CrsReadResponse @@ -307,14 +314,22 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() { submission: SubmissionRecord, repoInfo: CommitInfo ): DiffResponse { - val prevSubs = dbFindAsync(SubmissionRecord().apply { - projectId = submission.projectId - state != SubmissionState.invalid - }) - val newestPrevSub = prevSubs.filter { - it.datetime < submission.datetime - }.sortedByDescending { it.datetime }.firstOrNull() + val newestPrevSub: SubmissionRecord? = dbQueryAsync( + ComplexDatabaseQuery(Tables.SUBMISSION) + .filter( + ("${Tables.SUBMISSION.PROJECT_ID.name} == %s and " + + "${Tables.SUBMISSION.STATE.name} != %s and " + + "${Tables.SUBMISSION.STATE.name} != %s and " + + "${Tables.SUBMISSION.DATETIME.name} < %s").formatToQuery( + submission.projectId, + SubmissionState.invalid, + SubmissionState.pending, + submission.datetime + ) + ) + .sortBy(Tables.SUBMISSION.DATETIME.name) + ).lastOrNull()?.toRecord() return getDiffBetweenSubmission(newestPrevSub, submission, repoInfo) } diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt index 13be5645..25587d5b 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt @@ -14,9 +14,13 @@ import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType import org.jetbrains.research.kotoed.code.Filename import org.jetbrains.research.kotoed.data.api.Code import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery +import org.jetbrains.research.kotoed.data.db.setPageForQuery import org.jetbrains.research.kotoed.data.vcs.CloneStatus +import org.jetbrains.research.kotoed.database.Tables import org.jetbrains.research.kotoed.database.enums.SubmissionState import org.jetbrains.research.kotoed.database.tables.records.CourseRecord +import org.jetbrains.research.kotoed.database.tables.records.FunctionPartHashRecord +import org.jetbrains.research.kotoed.database.tables.records.ProcessedProjectSubRecord import org.jetbrains.research.kotoed.database.tables.records.ProjectRecord import org.jetbrains.research.kotoed.database.tables.records.SubmissionResultRecord import org.jetbrains.research.kotoed.db.condition.lang.formatToQuery @@ -25,6 +29,7 @@ import org.jetbrains.research.kotoed.parsers.HaskellLexer import org.jetbrains.research.kotoed.util.* import org.jetbrains.research.kotoed.util.code.getPsi import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv +import org.jooq.impl.DSL import org.kohsuke.randname.RandomNameGenerator import ru.spbstu.ktuples.placeholders._0 import ru.spbstu.ktuples.placeholders.bind @@ -75,6 +80,17 @@ class KloneVerticle : AbstractKotoedVerticle(), Loggable { return sendJsonableCollectAsync(Address.DB.query("submission"), q) } + @JsonableEventBusConsumerFor(Address.Code.ProjectKloneCheck) + suspend fun handleSimilarHashesForProject(projectRecord: ProjectRecord) { + dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH).filter("${projectRecord.id}")) + //TODO remember lastProcessedSubId + } + + @JsonableEventBusConsumerFor(Address.Code.DifferenceBetweenKlones) + suspend fun handleDifference(projectRecord: ProjectRecord) { + dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH)) + } + @JsonableEventBusConsumerFor(Address.Code.KloneCheck) suspend fun handleCheck(course: CourseRecord) { diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionLeavesVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionLeavesVerticle.kt new file mode 100644 index 00000000..d1e8ee2f --- /dev/null +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionLeavesVerticle.kt @@ -0,0 +1,8 @@ +package org.jetbrains.research.kotoed.db + +import org.jetbrains.research.kotoed.database.Tables +import org.jetbrains.research.kotoed.database.tables.records.FunctionLeavesRecord +import org.jetbrains.research.kotoed.util.AutoDeployable + +@AutoDeployable +class FunctionLeavesVerticle : CrudDatabaseVerticleWithReferences(Tables.FUNCTION_LEAVES) \ No newline at end of file diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt index fe928480..736fd2cb 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt @@ -1,10 +1,375 @@ package org.jetbrains.research.kotoed.db +import io.vertx.core.json.JsonArray +import io.vertx.core.json.JsonObject +import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery import org.jetbrains.research.kotoed.database.Tables import org.jetbrains.research.kotoed.database.tables.records.FunctionPartHashRecord +import org.jetbrains.research.kotoed.database.tables.records.FunctionRecord +import org.jetbrains.research.kotoed.database.tables.records.HashClonesRecord +import org.jetbrains.research.kotoed.database.tables.records.ProcessedProjectSubRecord +import org.jetbrains.research.kotoed.database.tables.records.SubmissionResultRecord import org.jetbrains.research.kotoed.util.AutoDeployable +import org.jetbrains.research.kotoed.util.dbBatchCreateAsync +import org.jetbrains.research.kotoed.util.dbFetchAsync +import org.jetbrains.research.kotoed.util.dbFindAsync +import org.jooq.Record10 +import java.io.BufferedWriter +import java.io.File +import java.util.Comparator +import java.util.function.Function +import kotlin.math.abs @AutoDeployable -class FunctionPartHashVerticle : CrudDatabaseVerticleWithReferences(Tables.FUNCTION_PART_HASH){ +class FunctionPartHashVerticle : CrudDatabaseVerticleWithReferences(Tables.FUNCTION_PART_HASH) { + private val fph1 = Tables.FUNCTION_PART_HASH.`as`("fph1") + private val fph2 = Tables.FUNCTION_PART_HASH.`as`("fph2") -} \ No newline at end of file + override suspend fun handleQuery(message_: ComplexDatabaseQuery): JsonArray { + if (message_.filter == null) { + compareKlones() + return JsonArray() + } + val projectId = Integer.parseInt(message_.filter) + + val lastProcessedSub = dbFindAsync(ProcessedProjectSubRecord().apply { + this.projectid = projectId + }) + val lastProcessedSubId: Int = if (lastProcessedSub.isEmpty()) -1 else lastProcessedSub.first().submissionid + val records = db { + select( + fph1.FUNCTIONID, + fph1.SUBMISSIONID, + fph1.PROJECTID, + fph1.LEFTBOUND, + fph1.RIGHTBOUND, + fph2.FUNCTIONID, + fph2.SUBMISSIONID, + fph2.PROJECTID, + fph2.LEFTBOUND, + fph2.RIGHTBOUND + ) + .from(fph1) + .join(fph2) + .on(fph1.HASH.eq(fph2.HASH)) + .where( + fph1.PROJECTID.eq(projectId) + .and(fph1.SUBMISSIONID.greaterThan(lastProcessedSubId)) + .and(fph1.PROJECTID.notEqual(fph2.PROJECTID)) + ) + .orderBy(fph1.FUNCTIONID, fph1.SUBMISSIONID, fph2.FUNCTIONID, fph2.SUBMISSIONID) + .fetch() + .map { record -> intoHashCloneRecord(record) } + } + if (records.isEmpty()) { + return JsonArray() + } + val comparingList: MutableList = mutableListOf(records[0]) + val segmentsMap: MutableMap, MutableList>> = hashMapOf() + putClonesRecordsSegmentsIntoMap(segmentsMap, records[0]) + + for (i in 1 until records.size) { + if (isCloneRecordsFromDifferentFunctions(records[i], records[i - 1])) { + val fFun = records[i - 1].fFunctionid + val fSub = records[i - 1].fSubmissionid + val fProj = records[i - 1].fProjectid + val sFun = records[i - 1].sFunctionid + val sSub = records[i - 1].sSubmissionid + val sProj = records[i - 1].sProjectid + + val segments = comparingList.map { record -> + record.fLeftbound to record.fRightbound + } + val nonAbsorbedSegments = absorbingSegments(segments) + nonAbsorbedSegments.forEach { nonAbsorbedSegment -> + val otherSegments = segmentsMap[nonAbsorbedSegment] ?: throw IllegalStateException( + "After absorbing segments=${segments} for firstFunId=${fFun}, firstSubId=${fSub}, secondFunId=${sFun}," + + " secondSubId=${sSub} for nonAbsorbedSegment=${nonAbsorbedSegment} second functions segments are null" + ) + dbBatchCreateAsync(otherSegments.map { otherSegment -> + HashClonesRecord().apply { + fFunctionid = fFun + fSubmissionid = fSub + fProjectid = fProj + fLeftbound = nonAbsorbedSegment.first + fRightbound = nonAbsorbedSegment.second + sFunctionid = sFun + sSubmissionid = sSub + sProjectid = sProj + sLeftbound = otherSegment.first + sRightbound = otherSegment.second + } + }) + } + comparingList.clear() + segmentsMap.clear() + } + comparingList.add(records[i]) + putClonesRecordsSegmentsIntoMap(segmentsMap, records[i]) + } + //TODO remember lastProcessedSub + return JsonArray() + } + + private suspend fun compareKlones() { + val oldAlgoClones = JsonArray(dbFindAsync(SubmissionResultRecord().apply { + id = 287 + }).first().body.toString()) + + val newAlgoClones = dbFindAsync( + HashClonesRecord().apply { + fSubmissionid = 261 + }) + compareKlones(oldAlgoClones, newAlgoClones, null, "prodTest", setOf(261)) + } + + suspend fun compareKlones( + oldAlgoClones: JsonArray, + newAlgoClones: List, + funIdProducer: (Function)?, + fileName: String, + subNums: (Set)?, + ) { + + + val oldAlgoMap = fillMapForOldAlgo(oldAlgoClones, funIdProducer, subNums) + val newAlgoMap = fillNewAlgoMap(newAlgoClones) + val sameRecords = mutableMapOf, MutableSet>>() + File(fileName) + .bufferedWriter() + .use { + it.write("Functions in Both maps") + it.newLine() + for (entry in oldAlgoMap.first.entries) { + val firstFun = entry.key + val sameOtherFunctionForFirstFun = mutableSetOf>() + sameRecords[firstFun] = sameOtherFunctionForFirstFun + val newOtherFuns = newAlgoMap[firstFun] ?: continue + for (otherEntry in entry.value) { + val otherFun = otherEntry.key + val newCount = newOtherFuns[otherFun] ?: continue + sameOtherFunctionForFirstFun.add(otherFun) + val oldCount = otherEntry.value + val diff = abs(newCount - oldCount) + it.write( + "{funId=${firstFun.first} subId=${firstFun.second}} and {funId=${otherFun.first} subId=${otherFun.second}}:" + + "newAlgo=$newCount oldAlgo=$oldCount diff=${diff}" + ) + it.newLine() + } + } + it.write("Skipped functions count=${oldAlgoMap.second}") + it.newLine() + processFunctionsThatNotInBothMaps(oldAlgoMap.first, sameRecords, it, "OLD ALGO") + processFunctionsThatNotInBothMaps(newAlgoMap, sameRecords, it, "NEW ALGO") + } + } + + private fun processFunctionsThatNotInBothMaps( + map: MutableMap, MutableMap, Int>>, + sameRecordsMap: MutableMap, MutableSet>>, + writer: BufferedWriter, + algoName: String + ) { + writer.write("For algo $algoName") + writer.newLine() + for (entry in map.entries) { + val func = entry.key + val sameOtherFunctions = sameRecordsMap[func] + if (sameOtherFunctions == null) { + for (notSameFunction in entry.value.entries) { + notInBothMapFunctionWrite(writer, func, notSameFunction.key, notSameFunction.value) + } + continue + } + for (otherEntry in entry.value.entries) { + val otherFun = otherEntry.key + if (!sameOtherFunctions.contains(otherFun)) { + notInBothMapFunctionWrite(writer, func, otherFun, otherEntry.value) + } + } + } + writer.write("END----------") + writer.newLine() + } + + private fun notInBothMapFunctionWrite( + writer: BufferedWriter, + func: Pair, + otherFun: Pair, + klonesCount: Int + ) { + writer.write( + "Correlation between {funId=${func.first} subId=${func.second}} and {funId=${otherFun.first} subId=${otherFun.second}} " + + "with num=${klonesCount}" + ) + writer.newLine() + } + + private fun fillNewAlgoMap(allRecords: List): MutableMap, MutableMap, Int>> { + val map = allRecords.groupBy { + with(it) { + listOf(fFunctionid, fSubmissionid, sFunctionid, sSubmissionid) + } + } + val newAlgoMap = mutableMapOf, MutableMap, Int>>() + for (entry in map.entries) { + with(entry) { + val fFunId = key[0] to key[1] + val sFunId = key[2] to key[3] + var fFunClones = newAlgoMap[fFunId] + if (fFunClones == null) { + fFunClones = mutableMapOf() + newAlgoMap[fFunId] = fFunClones + } + if (fFunClones[sFunId] != null) { + throw IllegalStateException("fFunId=${fFunId} and sFunId=${sFunId} found again in groupBy") + } + fFunClones[sFunId] = value.size + } + } + return newAlgoMap + } + + private suspend fun fillMapForOldAlgo( + clonesArray: JsonArray, + funIdProducer: Function?, + subNums: Set?, + ): + Pair, MutableMap, Int>>, Int> { + var skippedFunctions = 0 + val countMap = mutableMapOf, MutableMap, Int>>() + for (i in 0 until clonesArray.size()) { + val sameTypeClones = clonesArray.getJsonArray(i) + for (j in 0 until sameTypeClones.size()) { + val firstFun = sameTypeClones.getJsonObject(j) + if (subNums != null && !subNums.contains(firstFun.getInteger("submission_id"))) { + continue + } + val fFunUniqueId = getFunUniqueId(firstFun, funIdProducer) + if (fFunUniqueId == null) { + skippedFunctions++ + continue + } + if (countMap[fFunUniqueId] == null) { + countMap[fFunUniqueId] = mutableMapOf() + } + for (k in 0 until sameTypeClones.size()) { + val secondFun = sameTypeClones.getJsonObject(k) + if (secondFun.getInteger("submission_id") == firstFun.getInteger("submission_id")) { + continue + } + val sFunUniqueId = getFunUniqueId(secondFun, funIdProducer) + if (sFunUniqueId == null) { + skippedFunctions++ + continue + } + val counter = countMap[fFunUniqueId]!!.getOrDefault(sFunUniqueId, 0) + countMap[fFunUniqueId]!![sFunUniqueId] = counter + 1 + } + } + } + return countMap to skippedFunctions + } + + private suspend fun getFunUniqueId( + funObj: JsonObject, + funIdProducer: (Function)? + ): Pair? { + val subId = funObj.getInteger("submission_id") + val funId: Int = funIdProducer?.apply(funObj) ?: getFunId(funObj) ?: return null + return funId to subId + } + + private suspend fun getFunId(firstFun: JsonObject): Int? { + val funName = firstFun.getString("function_name") + if (funName.isEmpty()) { + return null + } + val functionFromDb = dbFetchAsync(FunctionRecord().apply { + name = funName + }) + return functionFromDb.id + } + + private fun intoHashCloneRecord(record: Record10): HashClonesRecord { + val hashClonesRecord = HashClonesRecord() + hashClonesRecord.fFunctionid = record.value1() + hashClonesRecord.fSubmissionid = record.value2() + hashClonesRecord.fProjectid = record.value3() + hashClonesRecord.fLeftbound = record.value4() + hashClonesRecord.fRightbound = record.value5() + hashClonesRecord.sFunctionid = record.value6() + hashClonesRecord.sSubmissionid = record.value7() + hashClonesRecord.sProjectid = record.value8() + hashClonesRecord.sLeftbound = record.value9() + hashClonesRecord.sRightbound = record.value10() + return hashClonesRecord + } + + private fun putClonesRecordsSegmentsIntoMap( + segmentsMap: MutableMap, MutableList>>, + record: HashClonesRecord + ) { + val recordFunctionSegment = record.fLeftbound to record.fRightbound + val otherFunctionSegment = record.sLeftbound to record.sRightbound + var otherSegments = segmentsMap[recordFunctionSegment] + if (otherSegments == null) { + otherSegments = mutableListOf() + segmentsMap[recordFunctionSegment] = otherSegments + } + otherSegments.add(otherFunctionSegment) + } + + private fun isCloneRecordsFromDifferentFunctions( + firstRecord: HashClonesRecord, + secondRecord: HashClonesRecord + ): Boolean { + return secondRecord.sSubmissionid != firstRecord.sSubmissionid || + secondRecord.sFunctionid != firstRecord.sFunctionid || + secondRecord.fSubmissionid != firstRecord.fSubmissionid || + secondRecord.fFunctionid != firstRecord.fFunctionid + + } + + /** + * This method is intended to absorption by large ranges of smaller ranges + * + * @param segments segments where bigger ranges absorb smaller ranges, it must not contain pairs like 'pairA' + * and 'pairB' where pairA.second==pairB.first==true, because there is no guarantee what result it will produce. + * @return + */ + fun absorbingSegments(segments: List>): List> { + val res: MutableList> = mutableListOf() + val segmentPoints: MutableList = mutableListOf() + for (segment in segments) { + segmentPoints.add(SegmentPoint(segment.first, false)) + segmentPoints.add(SegmentPoint(segment.second, true)) + } + segmentPoints.sortWith(Comparator.comparingInt(SegmentPoint::value)) + + var segmentLength = 0 + var counter = 0 + for (i in segmentPoints.indices) { + if (counter != 0) { + segmentLength += (segmentPoints[i].value - segmentPoints[i - 1].value) + } + if (segmentPoints[i].isRightBound) { + counter-- + if (counter < 0) { + throw IllegalStateException("Counter is $counter for segments = $segments") + } + if (counter == 0) { + val x = segmentPoints[i].value + res.add(Pair(x - segmentLength, x)) + segmentLength = 0 + } + continue + } + counter++ + } + return res + } +} + +private data class SegmentPoint(val value: Int, val isRightBound: Boolean) \ No newline at end of file diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/HashClonesVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/HashClonesVerticle.kt new file mode 100644 index 00000000..7a14acee --- /dev/null +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/HashClonesVerticle.kt @@ -0,0 +1,9 @@ +package org.jetbrains.research.kotoed.db + +import org.jetbrains.research.kotoed.database.Tables +import org.jetbrains.research.kotoed.database.tables.records.FunctionLeavesRecord +import org.jetbrains.research.kotoed.database.tables.records.HashClonesRecord +import org.jetbrains.research.kotoed.util.AutoDeployable + +@AutoDeployable +class HashClonesVerticle : CrudDatabaseVerticleWithReferences(Tables.HASH_CLONES) \ No newline at end of file diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/ProcessedProjectSubmissionVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/ProcessedProjectSubmissionVerticle.kt new file mode 100644 index 00000000..cf466ea1 --- /dev/null +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/ProcessedProjectSubmissionVerticle.kt @@ -0,0 +1,11 @@ +package org.jetbrains.research.kotoed.db + +import io.vertx.core.json.JsonArray +import org.jetbrains.research.kotoed.database.Tables +import org.jetbrains.research.kotoed.database.tables.records.ProcessedProjectSubRecord +import org.jetbrains.research.kotoed.util.AutoDeployable + +@AutoDeployable +class ProcessedProjectSubVerticle : CrudDatabaseVerticleWithReferences(Tables.PROCESSED_PROJECT_SUB){ + +} diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/processors/SubmissionProcessorVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/processors/SubmissionProcessorVerticle.kt index 2a5e0bf1..9e142ad8 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/processors/SubmissionProcessorVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/processors/SubmissionProcessorVerticle.kt @@ -3,12 +3,12 @@ package org.jetbrains.research.kotoed.db.processors import com.intellij.psi.PsiElement import io.vertx.core.json.JsonObject import kotlinx.coroutines.withContext -import kotlinx.warnings.Warnings +import org.jetbrains.kotlin.psi.KtClass +import org.jetbrains.kotlin.psi.KtClassBody import org.jetbrains.kotlin.psi.KtNamedFunction import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType import org.jetbrains.kotlin.psi.psiUtil.endOffset import org.jetbrains.kotlin.psi.psiUtil.startOffsetSkippingComments -import org.jetbrains.research.kotoed.api.getFullName import org.jetbrains.research.kotoed.code.Filename import org.jetbrains.research.kotoed.code.Location import org.jetbrains.research.kotoed.code.diff.HunkJsonable @@ -29,18 +29,37 @@ import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv import org.jetbrains.research.kotoed.util.database.executeKAsync import org.jetbrains.research.kotoed.util.database.toRecord import org.jooq.ForeignKey -import java.io.File -import java.util.function.BiConsumer -import java.util.function.BiFunction +import java.util.function.Consumer data class BuildTriggerResult( val result: String, val buildRequestId: Int ) : Jsonable +fun KtNamedFunction.getFullName(): String { + val resultName = StringBuilder(this.containingFile.name) + if (!this.isTopLevel) { + if (this.parent is KtClassBody) { + val classParent = this.parent.parent as KtClass + resultName.append(classParent.name) + } else { + throw IllegalArgumentException("Function =${this.name} is not topLevel or class function") + } + } + if (this.receiverTypeReference != null) { + resultName.append(this.receiverTypeReference!!.text) + } + resultName.append(this.name) + for (valueParameter in this.valueParameters) { + resultName.append(valueParameter.typeReference?.text) + } + return resultName.toString() +} + @AutoDeployable -class SubmissionProcessorVerticle : ProcessorVerticle(Tables.SUBMISSION) { +class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSION) { private val ee by lazy { betterSingleThreadContext("submissionProcessorVerticle.executor") } + private val treeVisitor = TreeVisitor() // parent submission id can be invalid, filter it out override val checkedReferences: List> @@ -51,35 +70,35 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S get() = Location(Filename(path = sourcefile), sourceline) private suspend fun recreateCommentsAsync(vcsUid: String, parent: SubmissionRecord, child: SubmissionRecord) { - val submissionCacheAsync = AsyncCache { id: Int -> fetchByIdAsync(Tables.SUBMISSION, id) } - val commentCacheAsync = AsyncCache { id: Int -> fetchByIdAsync(Tables.SUBMISSION_COMMENT, id) } + val submissionCacheAsync = AsyncCache { id: Int -> fetchByIdAsync(SUBMISSION, id) } + val commentCacheAsync = AsyncCache { id: Int -> fetchByIdAsync(SUBMISSION_COMMENT, id) } val ancestorCommentCacheAsync = AsyncCache { comment: SubmissionCommentRecord -> dbFindAsync(SubmissionCommentRecord().apply { submissionId = comment.originalSubmissionId persistentCommentId = comment.persistentCommentId }).expecting( - message = "Duplicate or missing comment in chain detected: " + - "submission.id = ${comment.originalSubmissionId} " + - "comment.id = ${comment.persistentCommentId}" + message = "Duplicate or missing comment in chain detected: " + + "submission.id = ${comment.originalSubmissionId} " + + "comment.id = ${comment.persistentCommentId}" ) { 1 == it.size } - .first() + .first() } val parentComments = - dbFindAsync(SubmissionCommentRecord().apply { submissionId = parent.id }) + dbFindAsync(SubmissionCommentRecord().apply { submissionId = parent.id }) val alreadyMappedPersistentIds = - dbFindAsync(SubmissionCommentRecord().apply { submissionId = child.id }).map { it.persistentCommentId } + dbFindAsync(SubmissionCommentRecord().apply { submissionId = child.id }).map { it.persistentCommentId } // first, we create all the missing comments val childComments: List = - parentComments - .asSequence() - .filter { it.persistentCommentId !in alreadyMappedPersistentIds } - .mapTo(mutableListOf()) { comment -> - dbCreateAsync(comment.copy().apply { submissionId = child.id }) - } + parentComments + .asSequence() + .filter { it.persistentCommentId !in alreadyMappedPersistentIds } + .mapTo(mutableListOf()) { comment -> + dbCreateAsync(comment.copy().apply { submissionId = child.id }) + } // second, we remap all the locations and reply-chains @@ -88,15 +107,15 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S val ancestorSubmission = submissionCacheAsync(ancestorComment.submissionId) val adjustedLocation: LocationResponse = - sendJsonableAsync( - Address.Code.LocationDiff, - LocationRequest( - vcsUid, - ancestorComment.location, - ancestorSubmission.revision, - child.revision - ) + sendJsonableAsync( + Address.Code.LocationDiff, + LocationRequest( + vcsUid, + ancestorComment.location, + ancestorSubmission.revision, + child.revision ) + ) comment.sourcefile = adjustedLocation.location.filename.path comment.sourceline = adjustedLocation.location.line @@ -113,7 +132,7 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S val head: DialoguePoint get() = prev?.head?.also { prev = it } ?: this } - val dialogues = childComments.map { it.id to DialoguePoint(value = it) }.toMap() + val dialogues = childComments.map { it.id to DialoguePoint(value = it) }.toMap() dialogues.forEach { (_, v) -> v.prev = dialogues[v.value.previousCommentId] } @@ -126,7 +145,7 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S private suspend fun copyTagsFrom(parent: SubmissionRecord, child: SubmissionRecord) { val parentTags = dbFindAsync( - SubmissionTagRecord().apply { submissionId = parent.id }) + SubmissionTagRecord().apply { submissionId = parent.id }) try { dbBatchCreateAsync(parentTags.map { it.apply { submissionId = child.id } }) @@ -137,37 +156,38 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S private suspend fun getVcsInfo(project: ProjectRecord): RepositoryInfo { return sendJsonableAsync( - Address.Code.Download, - RemoteRequest(VCS.valueOf(project.repoType), project.repoUrl).toJson() + Address.Code.Download, + RemoteRequest(VCS.valueOf(project.repoType), project.repoUrl).toJson() ) } private suspend fun getVcsStatus( - vcsInfo: RepositoryInfo, - submission: SubmissionRecord): VerificationData { + vcsInfo: RepositoryInfo, + submission: SubmissionRecord + ): VerificationData { return when (vcsInfo.status) { CloneStatus.pending -> VerificationData.Unknown CloneStatus.done -> VerificationData.Processed CloneStatus.failed -> dbCreateAsync( - SubmissionStatusRecord().apply { - this.submissionId = submission.id - this.data = JsonObject( - "failure" to "Fetching remote repository failed", - "details" to vcsInfo.toJson() - ) - } + SubmissionStatusRecord().apply { + this.submissionId = submission.id + this.data = JsonObject( + "failure" to "Fetching remote repository failed", + "details" to vcsInfo.toJson() + ) + } ).id.let { VerificationData.Invalid(it) } } } - suspend override fun doProcess(data: JsonObject): VerificationData = run { + override suspend fun doProcess(data: JsonObject): VerificationData = run { val sub: SubmissionRecord = data.toRecord() - val project: ProjectRecord = fetchByIdAsync(Tables.PROJECT, sub.projectId) + val project: ProjectRecord = fetchByIdAsync(PROJECT, sub.projectId) val parentSub: SubmissionRecord? = sub.parentSubmissionId?.let { - fetchByIdAsync(Tables.SUBMISSION, sub.parentSubmissionId) + fetchByIdAsync(SUBMISSION, sub.parentSubmissionId) } parentSub?.let { @@ -200,8 +220,8 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S when (buildInfos.size) { 0 -> { val ack: BuildAck = sendJsonableAsync( - Address.BuildSystem.Build.Submission.Request, - SubmissionRecord().apply { id = sub.id } + Address.BuildSystem.Build.Submission.Request, + SubmissionRecord().apply { id = sub.id } ) dbCreateAsync( @@ -259,13 +279,13 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S } } - suspend override fun verify(data: JsonObject?): VerificationData { + override suspend fun verify(data: JsonObject?): VerificationData { data ?: throw IllegalArgumentException("Cannot verify null submission") val sub: SubmissionRecord = data.toRecord() - val project: ProjectRecord = fetchByIdAsync(Tables.PROJECT, sub.projectId) + val project: ProjectRecord = fetchByIdAsync(PROJECT, sub.projectId) val parentSub: SubmissionRecord? = sub.parentSubmissionId?.let { - fetchByIdAsync(Tables.SUBMISSION, sub.parentSubmissionId) + fetchByIdAsync(SUBMISSION, sub.parentSubmissionId) } val vcsReq = getVcsInfo(project) @@ -276,21 +296,21 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S try { val list: ListResponse = sendJsonableAsync( - Address.Code.List, - ListRequest(vcsReq.uid, sub.revision) + Address.Code.List, + ListRequest(vcsReq.uid, sub.revision) ) list.ignore() } catch (ex: Exception) { val errorId = dbCreateAsync( - SubmissionStatusRecord().apply { - this.submissionId = sub.id - this.data = JsonObject( - "failure" to "Fetching revision ${sub.revision} for repository ${project.repoUrl} failed", - "details" to ex.message - ) - } + SubmissionStatusRecord().apply { + this.submissionId = sub.id + this.data = JsonObject( + "failure" to "Fetching revision ${sub.revision} for repository ${project.repoUrl} failed", + "details" to ex.message + ) + } ).id return VerificationData.Invalid(errorId) @@ -300,9 +320,9 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S val parentComments = dbFindAsync(SubmissionCommentRecord().apply { submissionId = parentSub.id }) val ourComments = dbFindAsync(SubmissionCommentRecord().apply { submissionId = sub.id }) - .asSequence() - .map { it.persistentCommentId } - .toSet() + .asSequence() + .map { it.persistentCommentId } + .toSet() if (parentSub.state != SubmissionState.obsolete) { return VerificationData.Unknown @@ -320,19 +340,19 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S 0 -> VerificationData.Unknown else -> { dbCreateAsync( - SubmissionStatusRecord().apply { - this.submissionId = sub.id - this.data = JsonObject( - "failure" to "Several builds found for submission ${sub.id}", - "details" to buildInfos.tryToJson() - ) - } + SubmissionStatusRecord().apply { + this.submissionId = sub.id + this.data = JsonObject( + "failure" to "Several builds found for submission ${sub.id}", + "details" to buildInfos.tryToJson() + ) + } ).id.let { VerificationData.Invalid(it) } } } } - suspend override fun doClean(data: JsonObject): VerificationData { + override suspend fun doClean(data: JsonObject): VerificationData { val sub: SubmissionRecord = data.toRecord() async { @@ -342,14 +362,14 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S dbWithTransactionAsync { deleteFrom(SUBMISSION_STATUS) - .where(SUBMISSION_STATUS.SUBMISSION_ID.eq(sub.id)) - .executeKAsync() + .where(SUBMISSION_STATUS.SUBMISSION_ID.eq(sub.id)) + .executeKAsync() deleteFrom(SUBMISSION_RESULT) - .where(SUBMISSION_RESULT.SUBMISSION_ID.eq(sub.id)) - .executeKAsync() + .where(SUBMISSION_RESULT.SUBMISSION_ID.eq(sub.id)) + .executeKAsync() deleteFrom(BUILD) - .where(BUILD.SUBMISSION_ID.eq(sub.id)) - .executeKAsync() + .where(BUILD.SUBMISSION_ID.eq(sub.id)) + .executeKAsync() } return VerificationData.Unknown @@ -386,11 +406,8 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S file.collectDescendantsOfType().asSequence() } .filter { method -> - method.annotationEntries.all { anno -> "@Test" != anno.text } - } - .map { - @Suppress(Warnings.USELESS_CAST) - it as PsiElement + method.annotationEntries.all { anno -> "@Test" != anno.text } && + !method.containingFile.name.startsWith("test") } .toList() @@ -401,8 +418,11 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S it.toFile to it.changes } val project = dbFindAsync(ProjectRecord().apply { id = res.projectId }).first() - for (function in functionsList) { + val needProcess = function.isTopLevel || function.parent is KtClassBody //FIXME + if (!needProcess) { + continue + } processFunction(function, res, changesInFiles, project) } @@ -412,17 +432,22 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S } private suspend fun processFunction( - function: PsiElement, + function: KtNamedFunction, res: SubmissionRecord, changesInFiles: Map>, project: ProjectRecord ) { - val functionFullName = (function as KtNamedFunction).getFullName() + val functionFullName = function.getFullName() + if (function.bodyExpression == null) { + log.info("BodyExpression is null in function=${functionFullName}, submission=${res.id}") + return + } val functionRecord = dbFindAsync(FunctionRecord().apply { name = functionFullName }) val functionFromDb: FunctionRecord when (functionRecord.size) { 0 -> { log.info("Add new function=[${functionFullName}] in submission=[${res.id}]") + //TODO add try catch functionFromDb = dbCreateAsync(FunctionRecord().apply { name = functionFullName }) } @@ -444,63 +469,48 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S for (change in fileChanges) { val range = change.to if (isNeedToRecomputeHash(funStartLine, funFinishLine, range)) { - val hashesForLevels: List> = computeHashesForElement(function.bodyExpression as PsiElement) + val hashesForLevels: MutableList = computeHashesForElement(function.bodyExpression!!) putHashesInTable(hashesForLevels, functionFromDb, res, project) - break + return } } } private suspend fun putHashesInTable( - hashesForLevels: List>, + hashes: MutableList, functionFromDb: FunctionRecord, res: SubmissionRecord, project: ProjectRecord ) { - hashesForLevels.forEachIndexed { index, hashes -> - dbBatchCreateAsync(hashes.map { + if (hashes.isEmpty()) { + log.info("Hashes for funId=${functionFromDb.id}, subId=${res.id} is empty") + return + } + dbBatchCreateAsync(hashes.map { FunctionPartHashRecord().apply { functionid = functionFromDb.id submissionid = res.id - denizenId = project.denizenId - hash = it - level = index + projectid = project.id + leftbound = it.leftBound + rightbound = it.rightBound + hash = it.levelHash } }) - } - } - - fun computeHashesForElement(root: PsiElement): List> { - val associativeArray = mutableListOf>(mutableListOf()) - val treeVisitor = object : TreeVisitor { - override fun getLeaf(root: PsiElement): Int { - return getSelf(root) - } - - override fun getSelf(element: PsiElement): Int { - return element.javaClass.name.hashCode() - } - - override fun getAccumulator(): BiFunction { - return BiFunction { a, b -> a + b } - } - - override fun getConsumers(): List> { - return listOf(BiConsumer { hash, level -> - if (level > associativeArray.size) { - throw IllegalStateException( - "Level=[${level}] more than associativeArray size=[${associativeArray.size}]" - ) - } - if (level == associativeArray.size) { - associativeArray.add(mutableListOf()) - } - associativeArray[level].add(hash) - }) - } - } - treeVisitor.dfs(root) - return associativeArray + log.info("functionid = ${functionFromDb.id}, submissionid = ${res.id}, leavescount = ${hashes.last().leafNum}") + dbCreateAsync(FunctionLeavesRecord().apply { + functionid = functionFromDb.id + submissionid = res.id + leavescount = hashes.last().leafNum + }) + } + + fun computeHashesForElement(root: PsiElement): MutableList { + val visitResults = mutableListOf() + val consumers = listOf(Consumer{ + visitResults.add(it) + }) + treeVisitor.visitTree(root, consumers) + return visitResults } private fun isNeedToRecomputeHash(funStartLine: Int, funFinishLine: Int, changesRange: RangeJsonable): Boolean { @@ -511,49 +521,53 @@ class SubmissionProcessorVerticle : ProcessorVerticle(Tables.S } - private fun computeStatistics(list: List) { - val treeVisitor = object : TreeVisitor { - override fun getLeaf(root: PsiElement): Int { - return 0 - } - override fun getSelf(element: PsiElement): Int { - return 1 - } +// +// private fun computeStatistics(list: List) { +// val treeVisitor = object : TreeVisitor( +// accumulator = BiFunction { a, b -> a + b }, +// startLevelNum = 0, +// consumers = emptyList() +// ) { +// +// override fun processLeaf(root: PsiElement): Int { +// return 0 +// } +// +// override fun processNode(element: PsiElement): Int { +// return 1 +// } +// } +// val otherList = list.map { +// val levelsCount = treeVisitor.dfs(it.children.last()).first +// val linesCount = it.children[it.children.size - 1].text.count { chr -> chr == '\n' } + 1 +// (it as KtNamedFunction).name to Pair(levelsCount, linesCount) +// }.toList() +// val sortedRatio = otherList +// .map { el -> el.second.first * 1.0 / el.second.second } +// .sorted() +// val median: Double = +// if (sortedRatio.size % 2 == 0) (sortedRatio[sortedRatio.size / 2 - 1] + sortedRatio[sortedRatio.size / 2]) / 2.0 +// else sortedRatio[sortedRatio.size / 2] * 1.0 +// val percentile95: Double = sortedRatio[(sortedRatio.size * 0.95).toInt()] +// +// File("InformationAboutCommit") +// .bufferedWriter() +// .use { out -> +// var treeLevelsSum = 0 +// var linesSum = 0 +// for (pair in otherList) { +// treeLevelsSum += pair.second.first +// val lineInFunctions = pair.second.second +// linesSum += lineInFunctions +// out.write("Name:${pair.first} ${pair.second.first} $lineInFunctions") +// out.newLine() +// } +// out.write("All statistics: TreeLevels:${treeLevelsSum} FunctionsLines:${linesSum}") +// out.newLine() +// out.write("Median:${median} 95_Percentile:${percentile95}") +// out.flush() +// } +// } - override fun getAccumulator(): BiFunction { - return BiFunction { a, b -> a + b } - } - } - val otherList = list.map { - val levelsCount = treeVisitor.dfs(it.children.last()).first - val linesCount = it.children[it.children.size - 1].text.count { chr -> chr == '\n' } + 1 - (it as KtNamedFunction).name to Pair(levelsCount, linesCount) - }.toList() - val sortedRatio = otherList - .map { el -> el.second.first * 1.0 / el.second.second } - .sorted() - val median: Double = - if (sortedRatio.size % 2 == 0) (sortedRatio[sortedRatio.size / 2 - 1] + sortedRatio[sortedRatio.size / 2]) / 2.0 - else sortedRatio[sortedRatio.size / 2] * 1.0 - val percentile95: Double = sortedRatio[(sortedRatio.size * 0.95).toInt()] - - File("InformationAboutCommit") - .bufferedWriter() - .use { out -> - var treeLevelsSum = 0 - var linesSum = 0 - for (pair in otherList) { - treeLevelsSum += pair.second.first - val lineInFunctions = pair.second.second - linesSum += lineInFunctions - out.write("Name:${pair.first} ${pair.second.first} $lineInFunctions") - out.newLine() - } - out.write("All statistics: TreeLevels:${treeLevelsSum} FunctionsLines:${linesSum}") - out.newLine() - out.write("Median:${median} 95_Percentile:${percentile95}") - out.flush() - } - } } diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt index 54c8bcdd..f0158fc8 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt @@ -190,6 +190,8 @@ object Address { const val PurgeCache = "kotoed.code.purgecache" const val KloneCheck = "kotoed.code.klonecheck" + const val ProjectKloneCheck = "kotoed.code.project.klonecheck" + const val DifferenceBetweenKlones = "kotoed.code.difference" } object User { diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeVisitor.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeVisitor.kt new file mode 100644 index 00000000..9ea3f854 --- /dev/null +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeVisitor.kt @@ -0,0 +1,42 @@ +package org.jetbrains.research.kotoed.util + +import com.intellij.psi.PsiElement +import java.util.function.Consumer + +class TreeVisitor { + fun visitTree(root: PsiElement, consumers: List>) { + dfs(root, consumers, 0) + } + + private fun dfs(root: PsiElement, consumers: List>, leafCount: Int): VisitResult { + if (root.children.isEmpty()) { + return VisitResult(computeHash(root).toLong(), leafCount, leafCount, leafCount + 1) + } + var levelLeafCount = leafCount + var leftBound = -1 + var levelHash: Long = computeHash(root).toLong() + for (child in root.children) { + val nextLevelResult = dfs(child, consumers, levelLeafCount) + levelLeafCount = nextLevelResult.leafNum + if (leftBound == -1) { + leftBound = nextLevelResult.leftBound + } + levelHash += nextLevelResult.levelHash + } + val currentResult = VisitResult(levelHash, leftBound, levelLeafCount - 1, levelLeafCount) + if (currentResult.leftBound != currentResult.rightBound) { + consumers.forEach { it.accept(currentResult) } + } + return currentResult + } + private fun computeHash(element: PsiElement): Int { + return element.javaClass.name.hashCode() + } +} + +data class VisitResult( + val levelHash: Long, + val leftBound: Int, + val rightBound: Int, + val leafNum: Int +) \ No newline at end of file diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt index 77dbb66a..f5ea7927 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt @@ -21,14 +21,11 @@ import java.net.URI import java.net.URLEncoder import java.util.* import java.util.concurrent.TimeUnit -import java.util.function.BiConsumer -import java.util.function.BiFunction import kotlin.reflect.KClass import kotlin.reflect.KType import kotlin.reflect.KTypeProjection import kotlin.reflect.full.createType import kotlin.reflect.full.starProjectedType -import kotlin.reflect.jvm.jvmErasure import kotlin.reflect.typeOf /******************************************************************************/ @@ -297,35 +294,4 @@ infix fun GenericKType>.of(argument: GenericKType): GenericKType< fun GenericKType(kClass: KClass) = GenericKType(kClass.starProjectedType) @OptIn(ExperimentalStdlibApi::class) -inline fun genericTypeOf(): GenericKType = GenericKType(typeOf()) - -interface TreeVisitor { - fun getLeaf(root: PsiElement): T - - fun getSelf(element: PsiElement): T - - fun getAccumulator(): BiFunction - - fun getStartLevelNum(): Int { - return 0 - } - - fun getConsumers(): List> { - return emptyList() - } - - fun dfs(root: PsiElement): Pair { - if (root.children.isEmpty()) { - return Pair(getLeaf(root), getStartLevelNum()) - } - var maxLevel = 0 - var accumulation = getSelf(root) - for (child in root.children) { - val nextLevel = dfs(child) - maxLevel = Math.max(maxLevel, nextLevel.second) - accumulation = getAccumulator().apply(accumulation, nextLevel.first) - } - getConsumers().forEach { it.accept(accumulation, maxLevel + 1) } - return Pair(accumulation, maxLevel + 1) - } -} \ No newline at end of file +inline fun genericTypeOf(): GenericKType = GenericKType(typeOf()) \ No newline at end of file diff --git a/kotoed-server/src/main/resources/db/migration/V93__Add_Incremental_Clone_Detecting.sql b/kotoed-server/src/main/resources/db/migration/V93__Add_Incremental_Clone_Detecting.sql new file mode 100644 index 00000000..9e016487 --- /dev/null +++ b/kotoed-server/src/main/resources/db/migration/V93__Add_Incremental_Clone_Detecting.sql @@ -0,0 +1,45 @@ +CREATE TABLE function +( + id SERIAL NOT NULL PRIMARY KEY, + name varchar(100) UNIQUE NOT NULL +); +CREATE TABLE function_part_hash +( + id SERIAL NOT NULL PRIMARY KEY, + functionId INT REFERENCES function NOT NULL, + submissionId INT REFERENCES submission NOT NULL, + projectId INT REFERENCES project NOT NULL, + leftBound integer not null, + rightBound integer not null, + hash int8 NOT NULL +); + +CREATE INDEX IF NOT EXISTS function_part_hash_key ON function_part_hash(hash); +CREATE INDEX IF NOT EXISTS function_part_denizenId_key ON function_part_hash(projectId); + +CREATE TABLE processed_project_sub +( + id SERIAL NOT NULL PRIMARY KEY, + projectId INT REFERENCES project NOT NULL, + submissionId INT REFERENCES submission NOT NULL +); +create table function_leaves +( + id SERIAL NOT NULL PRIMARY KEY, + functionId INT REFERENCES function NOT NULL, + submissionId INT REFERENCES submission NOT NULL, + leavesCount INT NOT NULL +); +CREATE TABLE hash_clones( + id bigserial primary key, + f_functionId INT REFERENCES function NOT NULL, + f_submissionId INT REFERENCES submission NOT NULL, + f_projectId INT REFERENCES project NOT NULL, + f_leftBound INT NOT NULL, + f_rightBound INT NOT NULL, + s_functionId INT REFERENCES function NOT NULL, + s_submissionId INT REFERENCES submission NOT NULL, + s_projectId INT REFERENCES project NOT NULL, + s_leftBound INT NOT NULL, + s_rightBound INT NOT NULL +) diff --git a/kotoed-server/src/main/resources/db/migration/V93__Create_FunctionHash_table.sql b/kotoed-server/src/main/resources/db/migration/V93__Create_FunctionHash_table.sql deleted file mode 100644 index ccf830a1..00000000 --- a/kotoed-server/src/main/resources/db/migration/V93__Create_FunctionHash_table.sql +++ /dev/null @@ -1,19 +0,0 @@ -CREATE TABLE function -( - id SERIAL NOT NULL PRIMARY KEY, - name varchar(100) UNIQUE NOT NULL -); - -CREATE TABLE function_part_hash -( - id SERIAL NOT NULL PRIMARY KEY, - functionId INT REFERENCES function NOT NULL, - submissionId INT REFERENCES submission NOT NULL, - denizen_id INT REFERENCES denizen_unsafe NOT NULL, - hash INT NOT NULL, - level INT NOT NULL -); - -CREATE INDEX IF NOT EXISTS function_part_hash_key ON function_part_hash(hash); -CREATE INDEX IF NOT EXISTS function_part_denizenId_key ON function_part_hash(denizen_id); - diff --git a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/KlonesAlgorithmComparingTest.kt b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/KlonesAlgorithmComparingTest.kt new file mode 100644 index 00000000..7c5a4798 --- /dev/null +++ b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/KlonesAlgorithmComparingTest.kt @@ -0,0 +1,101 @@ +package org.jetbrains.research.kotoed.klones + +import io.vertx.core.json.JsonArray +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.jetbrains.research.kotoed.database.tables.records.HashClonesRecord +//import org.jetbrains.research.kotoed.db.FunctionPartHashVerticle +import org.jetbrains.research.kotoed.util.Jsonable +import org.jetbrains.research.kotoed.util.tryToJson +import java.io.FileReader + +class KlonesAlgorithmComparingTest { + +// private val service = FunctionPartHashVerticle() + + private val id = 1L + private val projId = 1 + private val FIRST_FUN_ID = 1 + private val SECOND_FUN_ID = 2 + private val THIRD_FUN_ID = 3 + private val FIRST_SUB_ID = 1 + private val SECOND_SUB_ID = 2 + + + suspend fun comparingTest() { + val newAlgoRecords = mutableListOf() + addClonesBetween(newAlgoRecords, FIRST_FUN_ID, FIRST_SUB_ID, SECOND_FUN_ID, SECOND_SUB_ID, 2) + addClonesBetween(newAlgoRecords, FIRST_FUN_ID, FIRST_SUB_ID, THIRD_FUN_ID, SECOND_SUB_ID, 2) + + addClonesBetween(newAlgoRecords, SECOND_FUN_ID, FIRST_SUB_ID, THIRD_FUN_ID, SECOND_SUB_ID, 5) + addClonesBetween(newAlgoRecords, SECOND_FUN_ID, FIRST_SUB_ID, FIRST_FUN_ID, SECOND_SUB_ID, 1) + + val oldAlgoClones = JsonArray() + createOldCloneBetween(oldAlgoClones, FIRST_FUN_ID, FIRST_SUB_ID, SECOND_FUN_ID, SECOND_SUB_ID, 1) + createOldCloneBetween(oldAlgoClones, FIRST_FUN_ID, FIRST_SUB_ID, THIRD_FUN_ID, SECOND_SUB_ID, 3) + + createOldCloneBetween(oldAlgoClones, SECOND_FUN_ID, FIRST_SUB_ID, THIRD_FUN_ID, SECOND_SUB_ID, 5) + + createOldCloneBetween(oldAlgoClones, THIRD_FUN_ID, FIRST_SUB_ID, THIRD_FUN_ID, SECOND_SUB_ID, 5) + +// service.compareKlones(oldAlgoClones, newAlgoRecords, { t -> t.getInteger("fun_id") },"testFile", setOf(FIRST_SUB_ID)) + } + + private fun addClonesBetween( + newAlgoRecords: MutableList, + fFunId: Int, + fSubId: Int, + sFunId: Int, + sSubId: Int, + count: Int + ) { + var start = 0 + val length = 2 + for (i in 0 until count) { + newAlgoRecords.add( + HashClonesRecord( + id, + fFunId, + fSubId, + projId, + start, + start + length - 1, + sFunId, + sSubId, + projId, + start, + start + length - 1 + ) + ) + start += length + } + } + + private fun createOldCloneBetween( + oldAlgoClones: JsonArray, + fFunId: Int, + fSubId: Int, + sFunId: Int, + sSubId: Int, + count: Int + ) { + val res = JsonArray() + res.add(CloneInfo(fSubId, fFunId).toJson()) + for (i in 0 until count) { + res.add(CloneInfo(sSubId, sFunId).toJson()) + } + oldAlgoClones.add(res) + } + + data class CloneInfo( + val submissionId: Int, + val funId: Int + ) : Jsonable + + +} + +suspend fun main() { + KlonesAlgorithmComparingTest().comparingTest() + +} \ No newline at end of file diff --git a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/SegmentAbsorbingTest.kt b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/SegmentAbsorbingTest.kt new file mode 100644 index 00000000..4b2dacd9 --- /dev/null +++ b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/SegmentAbsorbingTest.kt @@ -0,0 +1,59 @@ +package org.jetbrains.research.kotoed.klones + +import io.vertx.core.json.JsonArray +import io.vertx.core.json.JsonObject +import org.jetbrains.research.kotoed.db.FunctionPartHashVerticle +import org.jetbrains.research.kotoed.util.tryToJson +import org.junit.Test +import kotlin.test.assertEquals + +class SegmentAbsorbingTest { + private val service = FunctionPartHashVerticle() + + @Test + fun simpleTest() { + val segment = 1 to 3 + val segments = listOf(segment) + val unionSegments = service.absorbingSegments(segments) + assertEquals(segments, unionSegments) + } + + @Test + fun nonIntersectingSegmentsTest() { + val segment = 1 to 3 + val segment1 = 4 to 5 + val segment2 = 6 to 6 + val segment3 = 7 to 10 + val segment4 = 11 to 11 + val segments = listOf(segment, segment1, segment2, segment3, segment4).shuffled() + val unionSegments = service.absorbingSegments(segments) + assertEquals(segments.sortedBy(Pair::first), unionSegments) + } + + @Test + fun absorbingTest() { + val segment = 5 to 7 + val segment1 = 8 to 11 + val segment2 = 12 to 13 + val segment3 = 5 to 13 + + val segments = listOf(segment, segment1, segment2, segment3).shuffled() + val unionSegments = service.absorbingSegments(segments) + assertEquals(listOf(segment3), unionSegments) + } + + @Test + fun intersectingWithAbsorbingTest() { + val segment = 1 to 3 + val segment1 = 1 to 3 + val segment3 = 4 to 8 + val segment4 = 4 to 9 + val segment5 = 10 to 11 + val segment6 = 12 to 13 + val segment7 = 10 to 13 + val segments = listOf(segment, segment1, segment3, + segment4, segment5, segment6, segment7) + val unionSegments = service.absorbingSegments(segments) + assertEquals(listOf(1 to 3, 4 to 9, 10 to 13), unionSegments) + } +} \ No newline at end of file diff --git a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/VisitorTest.kt b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/VisitorTest.kt new file mode 100644 index 00000000..799d1ac8 --- /dev/null +++ b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/VisitorTest.kt @@ -0,0 +1,302 @@ +package org.jetbrains.research.kotoed.klones + +import com.intellij.lang.ASTNode +import com.intellij.lang.Language +import com.intellij.openapi.project.Project +import com.intellij.openapi.util.Key +import com.intellij.openapi.util.TextRange +import com.intellij.psi.* +import com.intellij.psi.scope.PsiScopeProcessor +import com.intellij.psi.search.GlobalSearchScope +import com.intellij.psi.search.SearchScope +import org.jetbrains.research.kotoed.util.TreeVisitor +import org.jetbrains.research.kotoed.util.VisitResult +import org.junit.Test +import java.util.function.Consumer +import javax.swing.Icon +import kotlin.test.assertEquals + +class VisitorTest { + private val treeVisitor = TreeVisitor() + // root + // 0-5 6-7 + // 0-2 3-4 5; 6-6; 7; + // 0,1,2; 3,4; 6; + @Test + fun visitTest() { + val node0 = newPsiElement("node0") + val node1 = newPsiElement("node1") + val node2 = newPsiElement("node2") + val node0_2 = newPsiElement(arrayOf(node0, node1, node2),"node0_2") + val node3 = newPsiElement("node3") + val node4 = newPsiElement("node4") + val node3_4 = newPsiElement(arrayOf(node3, node4),"node3_4") + val node5 = newPsiElement("node5") + val node0_5 = newPsiElement(arrayOf(node0_2, node3_4, node5),"node0_5") + val node6 = newPsiElement("node6") + val node6_6 = newPsiElement(arrayOf(node6),"node6_6") + val node7 = newPsiElement("node7") + val node6_7 = newPsiElement(arrayOf(node6_6, node7), "node6_7") + val root = newPsiElement(arrayOf(node0_5, node6_7), "root") + val results = arrayListOf() + val consumers = listOf>(Consumer { + results.add(it) + }) + treeVisitor.visitTree(root, consumers) + var result = results[0] + assertEquals(0, result.leftBound) + assertEquals(2, result.rightBound) + assertEquals(3, result.leafNum) + result = results[1] + assertEquals(3, result.leftBound) + assertEquals(4, result.rightBound) + assertEquals(5, result.leafNum) + result = results[2] + assertEquals(0, result.leftBound) + assertEquals(5, result.rightBound) + assertEquals(6, result.leafNum) + result = results[3] + assertEquals(6, result.leftBound) + assertEquals(7, result.rightBound) + assertEquals(8, result.leafNum) + result = results[4] + assertEquals(0, result.leftBound) + assertEquals(7, result.rightBound) + assertEquals(8, result.leafNum) + } +} + +private fun newPsiElement(children: Array, name: String): PsiElement { + return psiElement(children, name) +} + +private fun newPsiElement(name: String): PsiElement { + return psiElement(emptyArray(), name) +} + +private fun psiElement(children: Array, name: String): PsiElement { + + return object : PsiElement { + + override fun getChildren(): Array { + return children + } + + override fun getIcon(p0: Int): Icon { + TODO("Not yet implemented") + } + + override fun getUserData(p0: Key): T? { + TODO("Not yet implemented") + } + + override fun putUserData(p0: Key, p1: T?) { + TODO("Not yet implemented") + } + + override fun getProject(): Project { + TODO("Not yet implemented") + } + + override fun getLanguage(): Language { + TODO("Not yet implemented") + } + + override fun getManager(): PsiManager { + TODO("Not yet implemented") + } + + override fun getParent(): PsiElement { + TODO("Not yet implemented") + } + + override fun getFirstChild(): PsiElement { + TODO("Not yet implemented") + } + + override fun getLastChild(): PsiElement { + TODO("Not yet implemented") + } + + override fun getNextSibling(): PsiElement { + TODO("Not yet implemented") + } + + override fun getPrevSibling(): PsiElement { + TODO("Not yet implemented") + } + + override fun getContainingFile(): PsiFile { + TODO("Not yet implemented") + } + + override fun getTextRange(): TextRange { + TODO("Not yet implemented") + } + + override fun getStartOffsetInParent(): Int { + TODO("Not yet implemented") + } + + override fun getTextLength(): Int { + TODO("Not yet implemented") + } + + override fun findElementAt(p0: Int): PsiElement? { + TODO("Not yet implemented") + } + + override fun findReferenceAt(p0: Int): PsiReference? { + TODO("Not yet implemented") + } + + override fun getTextOffset(): Int { + TODO("Not yet implemented") + } + + override fun getText(): String { + TODO("Not yet implemented") + } + + override fun textToCharArray(): CharArray { + TODO("Not yet implemented") + } + + override fun getNavigationElement(): PsiElement { + TODO("Not yet implemented") + } + + override fun getOriginalElement(): PsiElement { + TODO("Not yet implemented") + } + + override fun textMatches(p0: CharSequence): Boolean { + TODO("Not yet implemented") + } + + override fun textMatches(p0: PsiElement): Boolean { + TODO("Not yet implemented") + } + + override fun textContains(p0: Char): Boolean { + TODO("Not yet implemented") + } + + override fun accept(p0: PsiElementVisitor) { + TODO("Not yet implemented") + } + + override fun acceptChildren(p0: PsiElementVisitor) { + TODO("Not yet implemented") + } + + override fun copy(): PsiElement { + TODO("Not yet implemented") + } + + override fun add(p0: PsiElement): PsiElement { + TODO("Not yet implemented") + } + + override fun addBefore(p0: PsiElement, p1: PsiElement?): PsiElement { + TODO("Not yet implemented") + } + + override fun addAfter(p0: PsiElement, p1: PsiElement?): PsiElement { + TODO("Not yet implemented") + } + + override fun checkAdd(p0: PsiElement) { + TODO("Not yet implemented") + } + + override fun addRange(p0: PsiElement?, p1: PsiElement?): PsiElement { + TODO("Not yet implemented") + } + + override fun addRangeBefore(p0: PsiElement, p1: PsiElement, p2: PsiElement?): PsiElement { + TODO("Not yet implemented") + } + + override fun addRangeAfter(p0: PsiElement?, p1: PsiElement?, p2: PsiElement?): PsiElement { + TODO("Not yet implemented") + } + + override fun delete() { + TODO("Not yet implemented") + } + + override fun checkDelete() { + TODO("Not yet implemented") + } + + override fun deleteChildRange(p0: PsiElement?, p1: PsiElement?) { + TODO("Not yet implemented") + } + + override fun replace(p0: PsiElement): PsiElement { + TODO("Not yet implemented") + } + + override fun isValid(): Boolean { + TODO("Not yet implemented") + } + + override fun isWritable(): Boolean { + TODO("Not yet implemented") + } + + override fun getReference(): PsiReference? { + TODO("Not yet implemented") + } + + override fun getReferences(): Array { + TODO("Not yet implemented") + } + + override fun getCopyableUserData(p0: Key?): T? { + TODO("Not yet implemented") + } + + override fun putCopyableUserData(p0: Key?, p1: T?) { + TODO("Not yet implemented") + } + + override fun processDeclarations( + p0: PsiScopeProcessor, + p1: ResolveState, + p2: PsiElement?, + p3: PsiElement + ): Boolean { + TODO("Not yet implemented") + } + + override fun getContext(): PsiElement? { + TODO("Not yet implemented") + } + + override fun isPhysical(): Boolean { + TODO("Not yet implemented") + } + + override fun getResolveScope(): GlobalSearchScope { + TODO("Not yet implemented") + } + + override fun getUseScope(): SearchScope { + TODO("Not yet implemented") + } + + override fun getNode(): ASTNode { + TODO("Not yet implemented") + } + + override fun isEquivalentTo(p0: PsiElement?): Boolean { + TODO("Not yet implemented") + } + + override fun toString(): String { + return name + } + } +} \ No newline at end of file From 82056efe74a7a07c5fbeef8dd739aeeec8a4cde9 Mon Sep 17 00:00:00 2001 From: VladislavFetisov <1> Date: Tue, 11 Apr 2023 19:27:51 +0300 Subject: [PATCH 4/6] tests is good --- .../kotoed/db/FunctionPartHashVerticle.kt | 97 +++++++++++-------- .../klones/KlonesAlgorithmComparingTest.kt | 11 +-- 2 files changed, 58 insertions(+), 50 deletions(-) diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt index 736fd2cb..fb4f28c6 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt @@ -5,13 +5,11 @@ import io.vertx.core.json.JsonObject import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery import org.jetbrains.research.kotoed.database.Tables import org.jetbrains.research.kotoed.database.tables.records.FunctionPartHashRecord -import org.jetbrains.research.kotoed.database.tables.records.FunctionRecord import org.jetbrains.research.kotoed.database.tables.records.HashClonesRecord import org.jetbrains.research.kotoed.database.tables.records.ProcessedProjectSubRecord import org.jetbrains.research.kotoed.database.tables.records.SubmissionResultRecord import org.jetbrains.research.kotoed.util.AutoDeployable import org.jetbrains.research.kotoed.util.dbBatchCreateAsync -import org.jetbrains.research.kotoed.util.dbFetchAsync import org.jetbrains.research.kotoed.util.dbFindAsync import org.jooq.Record10 import java.io.BufferedWriter @@ -70,57 +68,66 @@ class FunctionPartHashVerticle : CrudDatabaseVerticleWithReferences - record.fLeftbound to record.fRightbound - } - val nonAbsorbedSegments = absorbingSegments(segments) - nonAbsorbedSegments.forEach { nonAbsorbedSegment -> - val otherSegments = segmentsMap[nonAbsorbedSegment] ?: throw IllegalStateException( - "After absorbing segments=${segments} for firstFunId=${fFun}, firstSubId=${fSub}, secondFunId=${sFun}," + - " secondSubId=${sSub} for nonAbsorbedSegment=${nonAbsorbedSegment} second functions segments are null" - ) - dbBatchCreateAsync(otherSegments.map { otherSegment -> - HashClonesRecord().apply { - fFunctionid = fFun - fSubmissionid = fSub - fProjectid = fProj - fLeftbound = nonAbsorbedSegment.first - fRightbound = nonAbsorbedSegment.second - sFunctionid = sFun - sSubmissionid = sSub - sProjectid = sProj - sLeftbound = otherSegment.first - sRightbound = otherSegment.second - } - }) - } - comparingList.clear() - segmentsMap.clear() + processFunctionClones(comparingList, segmentsMap, records[i - 1]) } comparingList.add(records[i]) putClonesRecordsSegmentsIntoMap(segmentsMap, records[i]) } + processFunctionClones(comparingList, segmentsMap, records[records.size - 1]) //TODO remember lastProcessedSub return JsonArray() } + private suspend fun processFunctionClones( + comparingList: MutableList, + segmentsMap: MutableMap, MutableList>>, + prevRecord: HashClonesRecord + ) { + val fFun = prevRecord.fFunctionid + val fSub = prevRecord.fSubmissionid + val fProj = prevRecord.fProjectid + val sFun = prevRecord.sFunctionid + val sSub = prevRecord.sSubmissionid + val sProj = prevRecord.sProjectid + + val segments = comparingList.map { record -> + record.fLeftbound to record.fRightbound + } + val nonAbsorbedSegments = absorbingSegments(segments) + nonAbsorbedSegments.forEach { nonAbsorbedSegment -> + val otherSegments = segmentsMap[nonAbsorbedSegment] ?: throw IllegalStateException( + "After absorbing segments=${segments} for firstFunId=${fFun}, firstSubId=${fSub}, secondFunId=${sFun}," + + " secondSubId=${sSub} for nonAbsorbedSegment=${nonAbsorbedSegment} second functions segments are null" + ) + dbBatchCreateAsync(otherSegments.map { otherSegment -> + HashClonesRecord().apply { + fFunctionid = fFun + fSubmissionid = fSub + fProjectid = fProj + fLeftbound = nonAbsorbedSegment.first + fRightbound = nonAbsorbedSegment.second + sFunctionid = sFun + sSubmissionid = sSub + sProjectid = sProj + sLeftbound = otherSegment.first + sRightbound = otherSegment.second + } + }) + } + comparingList.clear() + segmentsMap.clear() + } + private suspend fun compareKlones() { val oldAlgoClones = JsonArray(dbFindAsync(SubmissionResultRecord().apply { - id = 287 + id = 13 }).first().body.toString()) val newAlgoClones = dbFindAsync( HashClonesRecord().apply { - fSubmissionid = 261 + fSubmissionid = 5 }) - compareKlones(oldAlgoClones, newAlgoClones, null, "prodTest", setOf(261)) + compareKlones(oldAlgoClones, newAlgoClones, null, "prodTest", setOf(5)) } suspend fun compareKlones( @@ -283,13 +290,17 @@ class FunctionPartHashVerticle : CrudDatabaseVerticleWithReferences): HashClonesRecord { diff --git a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/KlonesAlgorithmComparingTest.kt b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/KlonesAlgorithmComparingTest.kt index 7c5a4798..0cb75b0f 100644 --- a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/KlonesAlgorithmComparingTest.kt +++ b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/KlonesAlgorithmComparingTest.kt @@ -1,17 +1,14 @@ package org.jetbrains.research.kotoed.klones import io.vertx.core.json.JsonArray -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext import org.jetbrains.research.kotoed.database.tables.records.HashClonesRecord -//import org.jetbrains.research.kotoed.db.FunctionPartHashVerticle +import org.jetbrains.research.kotoed.db.FunctionPartHashVerticle import org.jetbrains.research.kotoed.util.Jsonable -import org.jetbrains.research.kotoed.util.tryToJson -import java.io.FileReader + class KlonesAlgorithmComparingTest { -// private val service = FunctionPartHashVerticle() + private val service = FunctionPartHashVerticle() private val id = 1L private val projId = 1 @@ -38,7 +35,7 @@ class KlonesAlgorithmComparingTest { createOldCloneBetween(oldAlgoClones, THIRD_FUN_ID, FIRST_SUB_ID, THIRD_FUN_ID, SECOND_SUB_ID, 5) -// service.compareKlones(oldAlgoClones, newAlgoRecords, { t -> t.getInteger("fun_id") },"testFile", setOf(FIRST_SUB_ID)) + service.compareKlones(oldAlgoClones, newAlgoRecords, { t -> t.getInteger("fun_id") },"testFile", setOf(FIRST_SUB_ID)) } private fun addClonesBetween( From acc6be8cef83dbbbc1c9271def848bcc0332c991 Mon Sep 17 00:00:00 2001 From: VladislavFetisov <1> Date: Tue, 11 Apr 2023 20:40:03 +0300 Subject: [PATCH 5/6] refactor --- .../research/kotoed/api/CourseVerticle.kt | 15 +- .../kotoed/code/klones/KloneVerticle.kt | 1 - .../kotoed/db/FunctionPartHashVerticle.kt | 9 +- .../processors/SubmissionProcessorVerticle.kt | 227 +++++++----------- .../{TreeVisitor.kt => TreeHashVisitor.kt} | 2 +- .../jetbrains/research/kotoed/util/Util.kt | 4 +- .../research/kotoed/klones/VisitorTest.kt | 6 +- 7 files changed, 103 insertions(+), 161 deletions(-) rename kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/{TreeVisitor.kt => TreeHashVisitor.kt} (98%) diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt index a38e2abd..781d1993 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/CourseVerticle.kt @@ -2,10 +2,10 @@ package org.jetbrains.research.kotoed.api import io.vertx.core.json.JsonArray import io.vertx.core.json.JsonObject -import kotlinx.coroutines.withContext -import org.jetbrains.kotlin.psi.KtNamedFunction -import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType -import org.jetbrains.research.kotoed.data.api.* +import org.jetbrains.research.kotoed.data.api.CountResponse +import org.jetbrains.research.kotoed.data.api.DbRecordWrapper +import org.jetbrains.research.kotoed.data.api.SearchQuery +import org.jetbrains.research.kotoed.data.api.VerificationData import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery import org.jetbrains.research.kotoed.data.db.setPageForQuery import org.jetbrains.research.kotoed.data.db.textSearch @@ -14,18 +14,13 @@ import org.jetbrains.research.kotoed.database.Tables.COURSE_TEXT_SEARCH import org.jetbrains.research.kotoed.database.tables.records.BuildTemplateRecord import org.jetbrains.research.kotoed.database.tables.records.CourseRecord import org.jetbrains.research.kotoed.database.tables.records.CourseStatusRecord -import org.jetbrains.research.kotoed.database.tables.records.FunctionRecord +import org.jetbrains.research.kotoed.db.condition.lang.formatToQuery import org.jetbrains.research.kotoed.eventbus.Address import org.jetbrains.research.kotoed.util.* -import org.jetbrains.research.kotoed.util.code.getPsi -import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv import org.jetbrains.research.kotoed.util.database.toRecord -import java.lang.StringBuilder - @AutoDeployable class CourseVerticle : AbstractKotoedVerticle(), Loggable { - private val ee by lazy { betterSingleThreadContext("courseVerticle.executor") } @JsonableEventBusConsumerFor(Address.Api.Course.Create) suspend fun handleCreate(course: CourseRecord): DbRecordWrapper { diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt index 25587d5b..9057b61c 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt @@ -83,7 +83,6 @@ class KloneVerticle : AbstractKotoedVerticle(), Loggable { @JsonableEventBusConsumerFor(Address.Code.ProjectKloneCheck) suspend fun handleSimilarHashesForProject(projectRecord: ProjectRecord) { dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH).filter("${projectRecord.id}")) - //TODO remember lastProcessedSubId } @JsonableEventBusConsumerFor(Address.Code.DifferenceBetweenKlones) diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt index fb4f28c6..07e3b404 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt @@ -138,7 +138,6 @@ class FunctionPartHashVerticle : CrudDatabaseVerticleWithReferences)?, ) { - val oldAlgoMap = fillMapForOldAlgo(oldAlgoClones, funIdProducer, subNums) val newAlgoMap = fillNewAlgoMap(newAlgoClones) val sameRecords = mutableMapOf, MutableSet>>() @@ -324,12 +323,8 @@ class FunctionPartHashVerticle : CrudDatabaseVerticleWithReferences(SUBMISSION) { private val ee by lazy { betterSingleThreadContext("submissionProcessorVerticle.executor") } - private val treeVisitor = TreeVisitor() + private val treeHashVisitor = TreeHashVisitor() // parent submission id can be invalid, filter it out override val checkedReferences: List> @@ -70,35 +70,35 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI get() = Location(Filename(path = sourcefile), sourceline) private suspend fun recreateCommentsAsync(vcsUid: String, parent: SubmissionRecord, child: SubmissionRecord) { - val submissionCacheAsync = AsyncCache { id: Int -> fetchByIdAsync(SUBMISSION, id) } - val commentCacheAsync = AsyncCache { id: Int -> fetchByIdAsync(SUBMISSION_COMMENT, id) } + val submissionCacheAsync = AsyncCache { id: Int -> fetchByIdAsync(Tables.SUBMISSION, id) } + val commentCacheAsync = AsyncCache { id: Int -> fetchByIdAsync(Tables.SUBMISSION_COMMENT, id) } val ancestorCommentCacheAsync = AsyncCache { comment: SubmissionCommentRecord -> dbFindAsync(SubmissionCommentRecord().apply { submissionId = comment.originalSubmissionId persistentCommentId = comment.persistentCommentId }).expecting( - message = "Duplicate or missing comment in chain detected: " + - "submission.id = ${comment.originalSubmissionId} " + - "comment.id = ${comment.persistentCommentId}" + message = "Duplicate or missing comment in chain detected: " + + "submission.id = ${comment.originalSubmissionId} " + + "comment.id = ${comment.persistentCommentId}" ) { 1 == it.size } - .first() + .first() } val parentComments = - dbFindAsync(SubmissionCommentRecord().apply { submissionId = parent.id }) + dbFindAsync(SubmissionCommentRecord().apply { submissionId = parent.id }) val alreadyMappedPersistentIds = - dbFindAsync(SubmissionCommentRecord().apply { submissionId = child.id }).map { it.persistentCommentId } + dbFindAsync(SubmissionCommentRecord().apply { submissionId = child.id }).map { it.persistentCommentId } // first, we create all the missing comments val childComments: List = - parentComments - .asSequence() - .filter { it.persistentCommentId !in alreadyMappedPersistentIds } - .mapTo(mutableListOf()) { comment -> - dbCreateAsync(comment.copy().apply { submissionId = child.id }) - } + parentComments + .asSequence() + .filter { it.persistentCommentId !in alreadyMappedPersistentIds } + .mapTo(mutableListOf()) { comment -> + dbCreateAsync(comment.copy().apply { submissionId = child.id }) + } // second, we remap all the locations and reply-chains @@ -107,15 +107,15 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI val ancestorSubmission = submissionCacheAsync(ancestorComment.submissionId) val adjustedLocation: LocationResponse = - sendJsonableAsync( - Address.Code.LocationDiff, - LocationRequest( - vcsUid, - ancestorComment.location, - ancestorSubmission.revision, - child.revision + sendJsonableAsync( + Address.Code.LocationDiff, + LocationRequest( + vcsUid, + ancestorComment.location, + ancestorSubmission.revision, + child.revision + ) ) - ) comment.sourcefile = adjustedLocation.location.filename.path comment.sourceline = adjustedLocation.location.line @@ -132,7 +132,7 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI val head: DialoguePoint get() = prev?.head?.also { prev = it } ?: this } - val dialogues = childComments.map { it.id to DialoguePoint(value = it) }.toMap() + val dialogues = childComments.map { it.id to DialoguePoint(value = it) }.toMap() dialogues.forEach { (_, v) -> v.prev = dialogues[v.value.previousCommentId] } @@ -145,7 +145,7 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI private suspend fun copyTagsFrom(parent: SubmissionRecord, child: SubmissionRecord) { val parentTags = dbFindAsync( - SubmissionTagRecord().apply { submissionId = parent.id }) + SubmissionTagRecord().apply { submissionId = parent.id }) try { dbBatchCreateAsync(parentTags.map { it.apply { submissionId = child.id } }) @@ -156,38 +156,37 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI private suspend fun getVcsInfo(project: ProjectRecord): RepositoryInfo { return sendJsonableAsync( - Address.Code.Download, - RemoteRequest(VCS.valueOf(project.repoType), project.repoUrl).toJson() + Address.Code.Download, + RemoteRequest(VCS.valueOf(project.repoType), project.repoUrl).toJson() ) } private suspend fun getVcsStatus( - vcsInfo: RepositoryInfo, - submission: SubmissionRecord - ): VerificationData { + vcsInfo: RepositoryInfo, + submission: SubmissionRecord): VerificationData { return when (vcsInfo.status) { CloneStatus.pending -> VerificationData.Unknown CloneStatus.done -> VerificationData.Processed CloneStatus.failed -> dbCreateAsync( - SubmissionStatusRecord().apply { - this.submissionId = submission.id - this.data = JsonObject( - "failure" to "Fetching remote repository failed", - "details" to vcsInfo.toJson() - ) - } + SubmissionStatusRecord().apply { + this.submissionId = submission.id + this.data = JsonObject( + "failure" to "Fetching remote repository failed", + "details" to vcsInfo.toJson() + ) + } ).id.let { VerificationData.Invalid(it) } } } - override suspend fun doProcess(data: JsonObject): VerificationData = run { + suspend override fun doProcess(data: JsonObject): VerificationData = run { val sub: SubmissionRecord = data.toRecord() - val project: ProjectRecord = fetchByIdAsync(PROJECT, sub.projectId) + val project: ProjectRecord = fetchByIdAsync(Tables.PROJECT, sub.projectId) val parentSub: SubmissionRecord? = sub.parentSubmissionId?.let { - fetchByIdAsync(SUBMISSION, sub.parentSubmissionId) + fetchByIdAsync(Tables.SUBMISSION, sub.parentSubmissionId) } parentSub?.let { @@ -220,8 +219,8 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI when (buildInfos.size) { 0 -> { val ack: BuildAck = sendJsonableAsync( - Address.BuildSystem.Build.Submission.Request, - SubmissionRecord().apply { id = sub.id } + Address.BuildSystem.Build.Submission.Request, + SubmissionRecord().apply { id = sub.id } ) dbCreateAsync( @@ -279,13 +278,13 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI } } - override suspend fun verify(data: JsonObject?): VerificationData { + suspend override fun verify(data: JsonObject?): VerificationData { data ?: throw IllegalArgumentException("Cannot verify null submission") val sub: SubmissionRecord = data.toRecord() - val project: ProjectRecord = fetchByIdAsync(PROJECT, sub.projectId) + val project: ProjectRecord = fetchByIdAsync(Tables.PROJECT, sub.projectId) val parentSub: SubmissionRecord? = sub.parentSubmissionId?.let { - fetchByIdAsync(SUBMISSION, sub.parentSubmissionId) + fetchByIdAsync(Tables.SUBMISSION, sub.parentSubmissionId) } val vcsReq = getVcsInfo(project) @@ -296,21 +295,21 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI try { val list: ListResponse = sendJsonableAsync( - Address.Code.List, - ListRequest(vcsReq.uid, sub.revision) + Address.Code.List, + ListRequest(vcsReq.uid, sub.revision) ) list.ignore() } catch (ex: Exception) { val errorId = dbCreateAsync( - SubmissionStatusRecord().apply { - this.submissionId = sub.id - this.data = JsonObject( - "failure" to "Fetching revision ${sub.revision} for repository ${project.repoUrl} failed", - "details" to ex.message - ) - } + SubmissionStatusRecord().apply { + this.submissionId = sub.id + this.data = JsonObject( + "failure" to "Fetching revision ${sub.revision} for repository ${project.repoUrl} failed", + "details" to ex.message + ) + } ).id return VerificationData.Invalid(errorId) @@ -320,9 +319,9 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI val parentComments = dbFindAsync(SubmissionCommentRecord().apply { submissionId = parentSub.id }) val ourComments = dbFindAsync(SubmissionCommentRecord().apply { submissionId = sub.id }) - .asSequence() - .map { it.persistentCommentId } - .toSet() + .asSequence() + .map { it.persistentCommentId } + .toSet() if (parentSub.state != SubmissionState.obsolete) { return VerificationData.Unknown @@ -340,19 +339,19 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI 0 -> VerificationData.Unknown else -> { dbCreateAsync( - SubmissionStatusRecord().apply { - this.submissionId = sub.id - this.data = JsonObject( - "failure" to "Several builds found for submission ${sub.id}", - "details" to buildInfos.tryToJson() - ) - } + SubmissionStatusRecord().apply { + this.submissionId = sub.id + this.data = JsonObject( + "failure" to "Several builds found for submission ${sub.id}", + "details" to buildInfos.tryToJson() + ) + } ).id.let { VerificationData.Invalid(it) } } } } - override suspend fun doClean(data: JsonObject): VerificationData { + suspend override fun doClean(data: JsonObject): VerificationData { val sub: SubmissionRecord = data.toRecord() async { @@ -362,14 +361,14 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI dbWithTransactionAsync { deleteFrom(SUBMISSION_STATUS) - .where(SUBMISSION_STATUS.SUBMISSION_ID.eq(sub.id)) - .executeKAsync() + .where(SUBMISSION_STATUS.SUBMISSION_ID.eq(sub.id)) + .executeKAsync() deleteFrom(SUBMISSION_RESULT) - .where(SUBMISSION_RESULT.SUBMISSION_ID.eq(sub.id)) - .executeKAsync() + .where(SUBMISSION_RESULT.SUBMISSION_ID.eq(sub.id)) + .executeKAsync() deleteFrom(BUILD) - .where(BUILD.SUBMISSION_ID.eq(sub.id)) - .executeKAsync() + .where(BUILD.SUBMISSION_ID.eq(sub.id)) + .executeKAsync() } return VerificationData.Unknown @@ -419,7 +418,7 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI } val project = dbFindAsync(ProjectRecord().apply { id = res.projectId }).first() for (function in functionsList) { - val needProcess = function.isTopLevel || function.parent is KtClassBody //FIXME + val needProcess = function.isTopLevel || function.parent is KtClassBody if (!needProcess) { continue } @@ -432,13 +431,13 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI } private suspend fun processFunction( - function: KtNamedFunction, + psiFunction: KtNamedFunction, res: SubmissionRecord, changesInFiles: Map>, project: ProjectRecord ) { - val functionFullName = function.getFullName() - if (function.bodyExpression == null) { + val functionFullName = psiFunction.getFullName() + if (psiFunction.bodyExpression == null) { log.info("BodyExpression is null in function=${functionFullName}, submission=${res.id}") return } @@ -447,8 +446,12 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI when (functionRecord.size) { 0 -> { log.info("Add new function=[${functionFullName}] in submission=[${res.id}]") - //TODO add try catch - functionFromDb = dbCreateAsync(FunctionRecord().apply { name = functionFullName }) + try { + functionFromDb = dbCreateAsync(FunctionRecord().apply { name = functionFullName }) + } catch (e: Exception) { + log.error("Cant add function $functionFullName to functions table", e) + return + } } 1 -> { @@ -461,15 +464,15 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI ) } } - val document = function.containingFile.viewProvider.document - ?: throw IllegalStateException("Function's=[${function.containingFile.name}] document is null") - val fileChanges = changesInFiles[function.containingFile.name] ?: return - val funStartLine = document.getLineNumber(function.startOffsetSkippingComments) + 1 - val funFinishLine = document.getLineNumber(function.endOffset) + 1 + val document = psiFunction.containingFile.viewProvider.document + ?: throw IllegalStateException("Function's=[${psiFunction.containingFile.name}] document is null") + val fileChanges = changesInFiles[psiFunction.containingFile.name] ?: return //no changes in file at all + val funStartLine = document.getLineNumber(psiFunction.startOffsetSkippingComments) + 1 + val funFinishLine = document.getLineNumber(psiFunction.endOffset) + 1 for (change in fileChanges) { - val range = change.to - if (isNeedToRecomputeHash(funStartLine, funFinishLine, range)) { - val hashesForLevels: MutableList = computeHashesForElement(function.bodyExpression!!) + val fileRange = change.to + if (isNeedToRecomputeHash(funStartLine, funFinishLine, fileRange)) { + val hashesForLevels: MutableList = computeHashesForElement(psiFunction.bodyExpression!!) putHashesInTable(hashesForLevels, functionFromDb, res, project) return } @@ -496,7 +499,7 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI hash = it.levelHash } }) - log.info("functionid = ${functionFromDb.id}, submissionid = ${res.id}, leavescount = ${hashes.last().leafNum}") + log.info("functionid = ${functionFromDb.id}, submissionid = ${res.id}, leavesСount = ${hashes.last().leafNum}") dbCreateAsync(FunctionLeavesRecord().apply { functionid = functionFromDb.id submissionid = res.id @@ -509,7 +512,7 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI val consumers = listOf(Consumer{ visitResults.add(it) }) - treeVisitor.visitTree(root, consumers) + treeHashVisitor.visitTree(root, consumers) return visitResults } @@ -520,54 +523,4 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI return !out } - - -// -// private fun computeStatistics(list: List) { -// val treeVisitor = object : TreeVisitor( -// accumulator = BiFunction { a, b -> a + b }, -// startLevelNum = 0, -// consumers = emptyList() -// ) { -// -// override fun processLeaf(root: PsiElement): Int { -// return 0 -// } -// -// override fun processNode(element: PsiElement): Int { -// return 1 -// } -// } -// val otherList = list.map { -// val levelsCount = treeVisitor.dfs(it.children.last()).first -// val linesCount = it.children[it.children.size - 1].text.count { chr -> chr == '\n' } + 1 -// (it as KtNamedFunction).name to Pair(levelsCount, linesCount) -// }.toList() -// val sortedRatio = otherList -// .map { el -> el.second.first * 1.0 / el.second.second } -// .sorted() -// val median: Double = -// if (sortedRatio.size % 2 == 0) (sortedRatio[sortedRatio.size / 2 - 1] + sortedRatio[sortedRatio.size / 2]) / 2.0 -// else sortedRatio[sortedRatio.size / 2] * 1.0 -// val percentile95: Double = sortedRatio[(sortedRatio.size * 0.95).toInt()] -// -// File("InformationAboutCommit") -// .bufferedWriter() -// .use { out -> -// var treeLevelsSum = 0 -// var linesSum = 0 -// for (pair in otherList) { -// treeLevelsSum += pair.second.first -// val lineInFunctions = pair.second.second -// linesSum += lineInFunctions -// out.write("Name:${pair.first} ${pair.second.first} $lineInFunctions") -// out.newLine() -// } -// out.write("All statistics: TreeLevels:${treeLevelsSum} FunctionsLines:${linesSum}") -// out.newLine() -// out.write("Median:${median} 95_Percentile:${percentile95}") -// out.flush() -// } -// } - } diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeVisitor.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeHashVisitor.kt similarity index 98% rename from kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeVisitor.kt rename to kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeHashVisitor.kt index 9ea3f854..6aceac01 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeVisitor.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/TreeHashVisitor.kt @@ -3,7 +3,7 @@ package org.jetbrains.research.kotoed.util import com.intellij.psi.PsiElement import java.util.function.Consumer -class TreeVisitor { +class TreeHashVisitor { fun visitTree(root: PsiElement, consumers: List>) { dfs(root, consumers, 0) } diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt index f5ea7927..c1d021fe 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/util/Util.kt @@ -6,7 +6,6 @@ import com.google.common.cache.CacheBuilder import com.google.common.cache.CacheLoader import com.google.common.cache.LoadingCache import com.hazelcast.util.Base64 -import com.intellij.psi.PsiElement import io.vertx.core.MultiMap import io.vertx.core.logging.Logger import io.vertx.core.logging.LoggerFactory @@ -26,6 +25,7 @@ import kotlin.reflect.KType import kotlin.reflect.KTypeProjection import kotlin.reflect.full.createType import kotlin.reflect.full.starProjectedType +import kotlin.reflect.jvm.jvmErasure import kotlin.reflect.typeOf /******************************************************************************/ @@ -294,4 +294,4 @@ infix fun GenericKType>.of(argument: GenericKType): GenericKType< fun GenericKType(kClass: KClass) = GenericKType(kClass.starProjectedType) @OptIn(ExperimentalStdlibApi::class) -inline fun genericTypeOf(): GenericKType = GenericKType(typeOf()) \ No newline at end of file +inline fun genericTypeOf(): GenericKType = GenericKType(typeOf()) diff --git a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/VisitorTest.kt b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/VisitorTest.kt index 799d1ac8..f960293f 100644 --- a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/VisitorTest.kt +++ b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/VisitorTest.kt @@ -9,7 +9,7 @@ import com.intellij.psi.* import com.intellij.psi.scope.PsiScopeProcessor import com.intellij.psi.search.GlobalSearchScope import com.intellij.psi.search.SearchScope -import org.jetbrains.research.kotoed.util.TreeVisitor +import org.jetbrains.research.kotoed.util.TreeHashVisitor import org.jetbrains.research.kotoed.util.VisitResult import org.junit.Test import java.util.function.Consumer @@ -17,7 +17,7 @@ import javax.swing.Icon import kotlin.test.assertEquals class VisitorTest { - private val treeVisitor = TreeVisitor() + private val treeHashVisitor = TreeHashVisitor() // root // 0-5 6-7 // 0-2 3-4 5; 6-6; 7; @@ -42,7 +42,7 @@ class VisitorTest { val consumers = listOf>(Consumer { results.add(it) }) - treeVisitor.visitTree(root, consumers) + treeHashVisitor.visitTree(root, consumers) var result = results[0] assertEquals(0, result.leftBound) assertEquals(2, result.rightBound) From 31d71906926d43d178d67303c4fe6ce921395835 Mon Sep 17 00:00:00 2001 From: VladislavFetisov <1> Date: Sat, 20 May 2023 14:24:48 +0300 Subject: [PATCH 6/6] tests --- kotoed-server/db.properties | 2 +- .../kotoed/api/HashComputingVerticle.kt | 185 ++++++++++++++++++ .../kotoed/code/klones/KloneVerticle.kt | 34 +++- .../kotoed/db/FunctionPartHashVerticle.kt | 30 ++- .../processors/SubmissionProcessorVerticle.kt | 167 +--------------- .../research/kotoed/eventbus/Address.kt | 2 + .../kotoed/klones/HashComputingTest.kt | 7 +- 7 files changed, 250 insertions(+), 177 deletions(-) create mode 100644 kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/HashComputingVerticle.kt diff --git a/kotoed-server/db.properties b/kotoed-server/db.properties index 43ff4c11..bb9f06e9 100644 --- a/kotoed-server/db.properties +++ b/kotoed-server/db.properties @@ -1,4 +1,4 @@ -db.url=jdbc:postgresql://localhost:5432/kotoedFinal +db.url=jdbc:postgresql://localhost:5432/kotoedProd db.user=kotoed db.password=kotoed testdb.url=jdbc:postgresql://localhost/kotoed-test diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/HashComputingVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/HashComputingVerticle.kt new file mode 100644 index 00000000..834978aa --- /dev/null +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/HashComputingVerticle.kt @@ -0,0 +1,185 @@ +package org.jetbrains.research.kotoed.api + +import com.intellij.psi.PsiElement +import kotlinx.coroutines.withContext +import org.jetbrains.kotlin.psi.KtClassBody +import org.jetbrains.kotlin.psi.KtNamedFunction +import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType +import org.jetbrains.kotlin.psi.psiUtil.endOffset +import org.jetbrains.kotlin.psi.psiUtil.startOffsetSkippingComments +import org.jetbrains.research.kotoed.code.diff.HunkJsonable +import org.jetbrains.research.kotoed.code.diff.RangeJsonable +import org.jetbrains.research.kotoed.data.api.Code +import org.jetbrains.research.kotoed.data.api.VerificationData +import org.jetbrains.research.kotoed.data.api.VerificationStatus +import org.jetbrains.research.kotoed.data.vcs.CloneStatus +import org.jetbrains.research.kotoed.database.tables.records.* +import org.jetbrains.research.kotoed.db.processors.getFullName +import org.jetbrains.research.kotoed.eventbus.Address +import org.jetbrains.research.kotoed.util.* +import org.jetbrains.research.kotoed.util.code.getPsi +import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv +import java.util.function.Consumer + +@AutoDeployable +class HashComputingVerticle : AbstractKotoedVerticle(), Loggable { + private val ee by lazy { betterSingleThreadContext("hashComputingVerticle.executor") } + private val treeHashVisitor = TreeHashVisitor() + + @JsonableEventBusConsumerFor(Address.Code.Hashes) + suspend fun computeHashesFromSub(res: SubmissionRecord): VerificationData { + log.info("Start computing hashing for submission=[${res.id}]") + try { + val diffResponse: Code.Submission.DiffResponse = sendJsonableAsync( + Address.Api.Submission.Code.DiffWithPrevious, + Code.Submission.DiffRequest(submissionId = res.id) + ) + + val files: Code.ListResponse = sendJsonableAsync( + Address.Api.Submission.Code.List, + Code.Submission.ListRequest(res.id) + ) + + temporaryKotlinEnv { + withContext(ee) { + val ktFiles = + files.root?.toFileSeq() + .orEmpty() + .filter { it.endsWith(".kt") } + .toList() //FIXME + .map { filename -> + val resp: Code.Submission.ReadResponse = sendJsonableAsync( + Address.Api.Submission.Code.Read, + Code.Submission.ReadRequest( + submissionId = res.id, path = filename + ) + ) + getPsi(resp.contents, filename) + } + val functionsList = ktFiles.asSequence() + .flatMap { file -> + file.collectDescendantsOfType().asSequence() + } + .filter { method -> + method.annotationEntries.all { anno -> "@Test" != anno.text } && + !method.containingFile.name.startsWith("test") + } + .toList() + + val changesInFiles = diffResponse.diff.associate { + if (it.toFile != it.fromFile) { + log.warn("File [${it.fromFile}] is renamed to [${it.toFile}]") + } + it.toFile to it.changes + } + val project = dbFindAsync(ProjectRecord().apply { id = res.projectId }).first() + for (function in functionsList) { + val needProcess = function.isTopLevel || function.parent is KtClassBody + if (!needProcess) { + continue + } + processFunction(function, res, changesInFiles, project) + } + + } + } + } catch (ex: Throwable) { + log.error(ex) + return VerificationData(VerificationStatus.Invalid, emptyList()) + } + return VerificationData(VerificationStatus.Processed, emptyList()) + } + + private suspend fun processFunction( + psiFunction: KtNamedFunction, + res: SubmissionRecord, + changesInFiles: Map>, + project: ProjectRecord + ) { + val functionFullName = psiFunction.getFullName() + if (psiFunction.bodyExpression == null) { + log.info("BodyExpression is null in function=${functionFullName}, submission=${res.id}") + return + } + val functionRecord = dbFindAsync(FunctionRecord().apply { name = functionFullName }) + val functionFromDb: FunctionRecord + when (functionRecord.size) { + 0 -> { + log.info("Add new function=[${functionFullName}] in submission=[${res.id}]") + try { + functionFromDb = dbCreateAsync(FunctionRecord().apply { name = functionFullName }) + } catch (e: Exception) { + log.error("Cant add function $functionFullName to functions table", e) + return + } + } + + 1 -> { + functionFromDb = functionRecord.first()!! + } + + else -> { + throw IllegalStateException( + "Amount of function [${functionFullName}] in table is ${functionRecord.size}" + ) + } + } + val document = psiFunction.containingFile.viewProvider.document + ?: throw IllegalStateException("Function's=[${psiFunction.containingFile.name}] document is null") + val fileChanges = changesInFiles[psiFunction.containingFile.name] ?: return //no changes in file at all + val funStartLine = document.getLineNumber(psiFunction.startOffsetSkippingComments) + 1 + val funFinishLine = document.getLineNumber(psiFunction.endOffset) + 1 + for (change in fileChanges) { + val fileRange = change.to + if (isNeedToRecomputeHash(funStartLine, funFinishLine, fileRange)) { + val hashesForLevels: MutableList = computeHashesForElement(psiFunction.bodyExpression!!) + putHashesInTable(hashesForLevels, functionFromDb, res, project) + return + } + } + } + + private suspend fun putHashesInTable( + hashes: MutableList, + functionFromDb: FunctionRecord, + res: SubmissionRecord, + project: ProjectRecord + ) { + if (hashes.isEmpty()) { + log.info("Hashes for funId=${functionFromDb.id}, subId=${res.id} is empty") + return + } + dbBatchCreateAsync(hashes.map { + FunctionPartHashRecord().apply { + functionid = functionFromDb.id + submissionid = res.id + projectid = project.id + leftbound = it.leftBound + rightbound = it.rightBound + hash = it.levelHash + } + }) + log.info("functionid = ${functionFromDb.id}, submissionid = ${res.id}, leavesСount = ${hashes.last().leafNum}") + dbCreateAsync(FunctionLeavesRecord().apply { + functionid = functionFromDb.id + submissionid = res.id + leavescount = hashes.last().leafNum + }) + } + + fun computeHashesForElement(root: PsiElement): MutableList { + val visitResults = mutableListOf() + val consumers = listOf(Consumer{ + visitResults.add(it) + }) + treeHashVisitor.visitTree(root, consumers) + return visitResults + } + + private fun isNeedToRecomputeHash(funStartLine: Int, funFinishLine: Int, changesRange: RangeJsonable): Boolean { + val start = changesRange.start + val finish = start + changesRange.count + val out = start > funFinishLine || finish < funStartLine + return !out + } +} \ No newline at end of file diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt index 9057b61c..e4173bfa 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt @@ -13,26 +13,26 @@ import org.jetbrains.kotlin.psi.KtNamedFunction import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType import org.jetbrains.research.kotoed.code.Filename import org.jetbrains.research.kotoed.data.api.Code +import org.jetbrains.research.kotoed.data.api.VerificationData +import org.jetbrains.research.kotoed.data.api.VerificationStatus import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery import org.jetbrains.research.kotoed.data.db.setPageForQuery import org.jetbrains.research.kotoed.data.vcs.CloneStatus import org.jetbrains.research.kotoed.database.Tables import org.jetbrains.research.kotoed.database.enums.SubmissionState -import org.jetbrains.research.kotoed.database.tables.records.CourseRecord -import org.jetbrains.research.kotoed.database.tables.records.FunctionPartHashRecord -import org.jetbrains.research.kotoed.database.tables.records.ProcessedProjectSubRecord -import org.jetbrains.research.kotoed.database.tables.records.ProjectRecord -import org.jetbrains.research.kotoed.database.tables.records.SubmissionResultRecord +import org.jetbrains.research.kotoed.database.tables.records.* import org.jetbrains.research.kotoed.db.condition.lang.formatToQuery import org.jetbrains.research.kotoed.eventbus.Address import org.jetbrains.research.kotoed.parsers.HaskellLexer import org.jetbrains.research.kotoed.util.* import org.jetbrains.research.kotoed.util.code.getPsi import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv +import org.jetbrains.research.kotoed.util.database.toRecord import org.jooq.impl.DSL import org.kohsuke.randname.RandomNameGenerator import ru.spbstu.ktuples.placeholders._0 import ru.spbstu.ktuples.placeholders.bind +import java.util.concurrent.atomic.AtomicInteger sealed class KloneRequest(val priority: Int) : Jsonable, Comparable { override fun compareTo(other: KloneRequest): Int = priority - other.priority @@ -82,7 +82,7 @@ class KloneVerticle : AbstractKotoedVerticle(), Loggable { @JsonableEventBusConsumerFor(Address.Code.ProjectKloneCheck) suspend fun handleSimilarHashesForProject(projectRecord: ProjectRecord) { - dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH).filter("${projectRecord.id}")) + dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH).filter("${projectRecord.id}")) } @JsonableEventBusConsumerFor(Address.Code.DifferenceBetweenKlones) @@ -90,6 +90,28 @@ class KloneVerticle : AbstractKotoedVerticle(), Loggable { dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH)) } + @JsonableEventBusConsumerFor(Address.Code.AllHashes) + suspend fun computeHashesForAllSubs(projectRecord: ProjectRecord) { + val startTime = System.currentTimeMillis() + val count = AtomicInteger() + val allSubs: List = dbQueryAsync( + ComplexDatabaseQuery(Tables.SUBMISSION) + .join(ComplexDatabaseQuery(Tables.PROJECT).join(Tables.COURSE)) + .filter("state != %s and state != %s and project.course.name == %s" + .formatToQuery(SubmissionState.invalid, SubmissionState.pending, "KotlinAsFirst-2022")) + .limit(1000) + ) + for (sub in allSubs) { + val submissionRecord = sub.toRecord() + val data: VerificationData = sendJsonableAsync(Address.Code.Hashes, submissionRecord) + if (data.status != VerificationStatus.Invalid) { + log.info("Count hashes for ${count.incrementAndGet()} submission") + } + } + log.info("All time in millis: ${System.currentTimeMillis() - startTime}") + log.info("Count hashes for ${count.get()} submission") + } + @JsonableEventBusConsumerFor(Address.Code.KloneCheck) suspend fun handleCheck(course: CourseRecord) { diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt index 07e3b404..8e1d11f7 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionPartHashVerticle.kt @@ -11,10 +11,12 @@ import org.jetbrains.research.kotoed.database.tables.records.SubmissionResultRec import org.jetbrains.research.kotoed.util.AutoDeployable import org.jetbrains.research.kotoed.util.dbBatchCreateAsync import org.jetbrains.research.kotoed.util.dbFindAsync +import org.jooq.Record1 import org.jooq.Record10 import java.io.BufferedWriter import java.io.File import java.util.Comparator +import java.util.concurrent.atomic.AtomicInteger import java.util.function.Function import kotlin.math.abs @@ -29,7 +31,29 @@ class FunctionPartHashVerticle : CrudDatabaseVerticleWithReferences intoHashCloneRecord(record) } } if (records.isEmpty()) { - return JsonArray() + return } val comparingList: MutableList = mutableListOf(records[0]) val segmentsMap: MutableMap, MutableList>> = hashMapOf() @@ -74,8 +98,6 @@ class FunctionPartHashVerticle : CrudDatabaseVerticleWithReferences(SUBMISSION) { - private val ee by lazy { betterSingleThreadContext("submissionProcessorVerticle.executor") } - private val treeHashVisitor = TreeHashVisitor() // parent submission id can be invalid, filter it out override val checkedReferences: List> @@ -229,7 +216,10 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI buildRequestId = ack.buildId } ) - computeHashesFromSub(sub) //FIXME call only when create submission + val hashesVerificationData: VerificationData = sendJsonableAsync( + Address.Code.Hashes, + sub + ) //FIXME call only when create submission VerificationData.Processed } 1 -> { @@ -373,154 +363,5 @@ class SubmissionProcessorVerticle : ProcessorVerticle(SUBMISSI return VerificationData.Unknown } - private suspend fun computeHashesFromSub(res: SubmissionRecord) { - log.info("Start computing hashing for submission=[${res.id}]") - val diffResponse: Code.Submission.DiffResponse = sendJsonableAsync( - Address.Api.Submission.Code.DiffWithPrevious, - Code.Submission.DiffRequest(submissionId = res.id) - ) - val files: Code.ListResponse = sendJsonableAsync( - Address.Api.Submission.Code.List, - Code.Submission.ListRequest(res.id) - ) - - temporaryKotlinEnv { - withContext(ee) { - val ktFiles = - files.root?.toFileSeq() - .orEmpty() - .filter { it.endsWith(".kt") } - .toList() //FIXME - .map { filename -> - val resp: Code.Submission.ReadResponse = sendJsonableAsync( - Address.Api.Submission.Code.Read, - Code.Submission.ReadRequest( - submissionId = res.id, path = filename - ) - ) - getPsi(resp.contents, filename) - } - val functionsList = ktFiles.asSequence() - .flatMap { file -> - file.collectDescendantsOfType().asSequence() - } - .filter { method -> - method.annotationEntries.all { anno -> "@Test" != anno.text } && - !method.containingFile.name.startsWith("test") - } - .toList() - - val changesInFiles = diffResponse.diff.associate { - if (it.toFile != it.fromFile) { - log.warn("File [${it.fromFile}] is renamed to [${it.toFile}]") - } - it.toFile to it.changes - } - val project = dbFindAsync(ProjectRecord().apply { id = res.projectId }).first() - for (function in functionsList) { - val needProcess = function.isTopLevel || function.parent is KtClassBody - if (!needProcess) { - continue - } - processFunction(function, res, changesInFiles, project) - } - - } - } - - } - - private suspend fun processFunction( - psiFunction: KtNamedFunction, - res: SubmissionRecord, - changesInFiles: Map>, - project: ProjectRecord - ) { - val functionFullName = psiFunction.getFullName() - if (psiFunction.bodyExpression == null) { - log.info("BodyExpression is null in function=${functionFullName}, submission=${res.id}") - return - } - val functionRecord = dbFindAsync(FunctionRecord().apply { name = functionFullName }) - val functionFromDb: FunctionRecord - when (functionRecord.size) { - 0 -> { - log.info("Add new function=[${functionFullName}] in submission=[${res.id}]") - try { - functionFromDb = dbCreateAsync(FunctionRecord().apply { name = functionFullName }) - } catch (e: Exception) { - log.error("Cant add function $functionFullName to functions table", e) - return - } - } - - 1 -> { - functionFromDb = functionRecord.first()!! - } - - else -> { - throw IllegalStateException( - "Amount of function [${functionFullName}] in table is ${functionRecord.size}" - ) - } - } - val document = psiFunction.containingFile.viewProvider.document - ?: throw IllegalStateException("Function's=[${psiFunction.containingFile.name}] document is null") - val fileChanges = changesInFiles[psiFunction.containingFile.name] ?: return //no changes in file at all - val funStartLine = document.getLineNumber(psiFunction.startOffsetSkippingComments) + 1 - val funFinishLine = document.getLineNumber(psiFunction.endOffset) + 1 - for (change in fileChanges) { - val fileRange = change.to - if (isNeedToRecomputeHash(funStartLine, funFinishLine, fileRange)) { - val hashesForLevels: MutableList = computeHashesForElement(psiFunction.bodyExpression!!) - putHashesInTable(hashesForLevels, functionFromDb, res, project) - return - } - } - } - - private suspend fun putHashesInTable( - hashes: MutableList, - functionFromDb: FunctionRecord, - res: SubmissionRecord, - project: ProjectRecord - ) { - if (hashes.isEmpty()) { - log.info("Hashes for funId=${functionFromDb.id}, subId=${res.id} is empty") - return - } - dbBatchCreateAsync(hashes.map { - FunctionPartHashRecord().apply { - functionid = functionFromDb.id - submissionid = res.id - projectid = project.id - leftbound = it.leftBound - rightbound = it.rightBound - hash = it.levelHash - } - }) - log.info("functionid = ${functionFromDb.id}, submissionid = ${res.id}, leavesСount = ${hashes.last().leafNum}") - dbCreateAsync(FunctionLeavesRecord().apply { - functionid = functionFromDb.id - submissionid = res.id - leavescount = hashes.last().leafNum - }) - } - - fun computeHashesForElement(root: PsiElement): MutableList { - val visitResults = mutableListOf() - val consumers = listOf(Consumer{ - visitResults.add(it) - }) - treeHashVisitor.visitTree(root, consumers) - return visitResults - } - - private fun isNeedToRecomputeHash(funStartLine: Int, funFinishLine: Int, changesRange: RangeJsonable): Boolean { - val start = changesRange.start - val finish = start + changesRange.count - val out = start > funFinishLine || finish < funStartLine - return !out - } } diff --git a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt index f0158fc8..aa1925a5 100644 --- a/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt +++ b/kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/eventbus/Address.kt @@ -192,6 +192,8 @@ object Address { const val KloneCheck = "kotoed.code.klonecheck" const val ProjectKloneCheck = "kotoed.code.project.klonecheck" const val DifferenceBetweenKlones = "kotoed.code.difference" + const val Hashes = "kotoed.code.hashes" + const val AllHashes = "kotoed.code.all.hashes" } object User { diff --git a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/HashComputingTest.kt b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/HashComputingTest.kt index b9860255..41f8a774 100644 --- a/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/HashComputingTest.kt +++ b/kotoed-server/test/main/kotlin/org/jetbrains/research/kotoed/klones/HashComputingTest.kt @@ -2,6 +2,7 @@ package org.jetbrains.research.kotoed.klones import com.intellij.psi.PsiElement import org.jetbrains.kotlin.psi.KtNamedFunction +import org.jetbrains.research.kotoed.api.HashComputingVerticle import org.jetbrains.research.kotoed.db.processors.SubmissionProcessorVerticle import org.jetbrains.research.kotoed.util.Loggable import org.jetbrains.research.kotoed.util.code.getPsi @@ -10,7 +11,7 @@ import org.junit.Test import kotlin.test.assertEquals class HashComputingTest: Loggable { - val subProcessorVarticle = SubmissionProcessorVerticle() + val hashComputingVerticle = HashComputingVerticle() @Test fun differentExpressionEqualsHashesTest() { @@ -26,9 +27,9 @@ class HashComputingTest: Loggable { """ val firstFun = getPsiElementFromString(firstFunction) - val hashesForFirstFunction = subProcessorVarticle.computeHashesForElement(firstFun) + val hashesForFirstFunction = hashComputingVerticle.computeHashesForElement(firstFun) val secondFun = getPsiElementFromString(secondFunction) - val hashesForSecondFunction = subProcessorVarticle.computeHashesForElement(secondFun) + val hashesForSecondFunction = hashComputingVerticle.computeHashesForElement(secondFun) assertEquals(hashesForFirstFunction, hashesForSecondFunction) }