From e742d410d6aaa2da59289a2eec3334b3c6362233 Mon Sep 17 00:00:00 2001 From: Andrey Tarbeev Date: Tue, 10 Jan 2023 09:46:03 +0300 Subject: [PATCH] Support symbolic executions in ContestEstimator --- .../kotlin/org/utbot/framework/UtSettings.kt | 25 +++++- .../generator/UtTestsDialogProcessor.kt | 2 +- .../plugin/models/GenerateTestsModel.kt | 3 +- .../intellij/plugin/settings/Settings.kt | 13 ++-- .../plugin/settings/SettingsWindow.kt | 9 ++- .../plugin/ui/GenerateTestsDialogWindow.kt | 3 +- .../main/kotlin/org/utbot/contest/Contest.kt | 10 ++- .../kotlin/org/utbot/summary/Summarization.kt | 77 +++++++++++++++++-- .../utbot/testing/UtValueTestCaseChecker.kt | 3 +- 9 files changed, 121 insertions(+), 24 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 5dda9af68b..02bf1bc3f3 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -134,9 +134,10 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS /** * Enable the Summarization module to generate summaries for methods under test. * - * Note: if it is false, all the execution for a particular method will be stored at the same nameless region. + * Note: if it is [SummariesGenerationType.NONE], + * all the execution for a particular method will be stored at the same nameless region. */ - var enableSummariesGeneration by getBooleanProperty(true) + var summaryGenerationType by getEnumProperty(SummariesGenerationType.FULL) /** * If True test comments will be generated. @@ -632,3 +633,23 @@ enum class MLPredictorType { */ LINREG } + +/** + * Enum to describe how we analyze code to obtain summaries. + */ +enum class SummariesGenerationType { + /** + * All possible analysis actions are taken + */ + FULL, + + /** + * Analysis actions based on sources are NOT taken + */ + LIGHT, + + /** + * No summaries are generated + */ + NONE, +} diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt index 4405702541..d6012247fe 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt @@ -141,7 +141,7 @@ object UtTestsDialogProcessor { UtSettings.concreteExecutionTimeoutInInstrumentedProcess = model.hangingTestsTimeout.timeoutMs UtSettings.useCustomJavaDocTags = model.commentStyle == JavaDocCommentStyle.CUSTOM_JAVADOC_TAGS - UtSettings.enableSummariesGeneration = model.enableSummariesGeneration + UtSettings.summaryGenerationType = model.summariesGenerationType fun now() = LocalDateTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss.SSS")) diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/models/GenerateTestsModel.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/models/GenerateTestsModel.kt index 9519674eaa..da2ff9338d 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/models/GenerateTestsModel.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/models/GenerateTestsModel.kt @@ -16,6 +16,7 @@ import com.intellij.psi.PsiClass import com.intellij.psi.PsiJavaFile import com.intellij.refactoring.util.classMembers.MemberInfo import org.jetbrains.kotlin.psi.KtFile +import org.utbot.framework.SummariesGenerationType import org.utbot.framework.UtSettings import org.utbot.framework.plugin.api.JavaDocCommentStyle import org.utbot.framework.util.ConflictTriggers @@ -55,7 +56,7 @@ class GenerateTestsModel( val conflictTriggers: ConflictTriggers = ConflictTriggers() var runGeneratedTestsWithCoverage : Boolean = false - var enableSummariesGeneration : Boolean = UtSettings.enableSummariesGeneration + var summariesGenerationType : SummariesGenerationType = UtSettings.summaryGenerationType } val PsiClass.packageName: String diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Settings.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Settings.kt index 661fcd3471..f1e028ed5e 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Settings.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Settings.kt @@ -36,6 +36,7 @@ import org.utbot.intellij.plugin.models.GenerateTestsModel import java.util.concurrent.CompletableFuture import kotlin.reflect.KClass import org.utbot.common.isWindows +import org.utbot.framework.SummariesGenerationType import org.utbot.framework.plugin.api.isSummarizationCompatible @State( @@ -63,7 +64,7 @@ class Settings(val project: Project) : PersistentStateComponent var fuzzingValue: Double = 0.05, var runGeneratedTestsWithCoverage: Boolean = false, var commentStyle: JavaDocCommentStyle = JavaDocCommentStyle.defaultItem, - var enableSummariesGeneration: Boolean = UtSettings.enableSummariesGeneration, + var summariesGenerationType: SummariesGenerationType = UtSettings.summaryGenerationType, var enableExperimentalLanguagesSupport: Boolean = false, ) { constructor(model: GenerateTestsModel) : this( @@ -82,7 +83,7 @@ class Settings(val project: Project) : PersistentStateComponent fuzzingValue = model.fuzzingValue, runGeneratedTestsWithCoverage = model.runGeneratedTestsWithCoverage, commentStyle = model.commentStyle, - enableSummariesGeneration = model.enableSummariesGeneration + summariesGenerationType = model.summariesGenerationType ) override fun equals(other: Any?): Boolean { @@ -107,7 +108,7 @@ class Settings(val project: Project) : PersistentStateComponent if (fuzzingValue != other.fuzzingValue) return false if (runGeneratedTestsWithCoverage != other.runGeneratedTestsWithCoverage) return false if (commentStyle != other.commentStyle) return false - if (enableSummariesGeneration != other.enableSummariesGeneration) return false + if (summariesGenerationType != other.summariesGenerationType) return false return true } @@ -127,7 +128,7 @@ class Settings(val project: Project) : PersistentStateComponent result = 31 * result + classesToMockAlways.contentHashCode() result = 31 * result + fuzzingValue.hashCode() result = 31 * result + if (runGeneratedTestsWithCoverage) 1 else 0 - result = 31 * result + if (enableSummariesGeneration) 1 else 0 + result = 31 * result + summariesGenerationType.hashCode() return result } @@ -173,7 +174,7 @@ class Settings(val project: Project) : PersistentStateComponent } var runGeneratedTestsWithCoverage = state.runGeneratedTestsWithCoverage - var enableSummariesGeneration = state.enableSummariesGeneration + var enableSummariesGeneration = state.summariesGenerationType fun setClassesToMockAlways(classesToMockAlways: List) { state.classesToMockAlways = classesToMockAlways.distinct().toTypedArray() @@ -209,7 +210,7 @@ class Settings(val project: Project) : PersistentStateComponent override fun loadState(state: State) { this.state = state if (!state.codegenLanguage.isSummarizationCompatible()) { - this.state.enableSummariesGeneration = false + this.state.summariesGenerationType = SummariesGenerationType.NONE } } diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt index 14661839ce..5d45c6cc25 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt @@ -18,6 +18,7 @@ import com.intellij.ui.layout.withValueBinding import com.intellij.util.castSafelyTo import com.intellij.util.ui.UIUtil import com.intellij.util.ui.components.BorderLayoutPanel +import org.utbot.framework.SummariesGenerationType import org.utbot.framework.UtSettings import org.utbot.framework.codegen.domain.ForceStaticMocking import org.utbot.framework.codegen.domain.HangingTestsTimeout @@ -138,13 +139,15 @@ class SettingsWindow(val project: Project) { cell { enableSummarizationGenerationCheckBox = checkBox("Enable Summaries Generation") .onApply { - settings.state.enableSummariesGeneration = enableSummarizationGenerationCheckBox.isSelected + settings.state.summariesGenerationType = + if (enableSummarizationGenerationCheckBox.isSelected) SummariesGenerationType.FULL else SummariesGenerationType.NONE } .onReset { - enableSummarizationGenerationCheckBox.isSelected = settings.state.enableSummariesGeneration + enableSummarizationGenerationCheckBox.isSelected = + settings.state.summariesGenerationType != SummariesGenerationType.NONE } .onIsModified { - enableSummarizationGenerationCheckBox.isSelected xor settings.state.enableSummariesGeneration + enableSummarizationGenerationCheckBox.isSelected xor (settings.state.summariesGenerationType != SummariesGenerationType.NONE) } .enableIf(codegenLanguageCombo.selectedValueMatches(CodegenLanguage?::isSummarizationCompatible)) .component diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/GenerateTestsDialogWindow.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/GenerateTestsDialogWindow.kt index 0cc2fde656..8f031ce53e 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/GenerateTestsDialogWindow.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/GenerateTestsDialogWindow.kt @@ -116,7 +116,6 @@ import org.utbot.framework.plugin.api.utils.MOCKITO_EXTENSIONS_FILE_CONTENT import org.utbot.framework.plugin.api.utils.MOCKITO_EXTENSIONS_FOLDER import org.utbot.framework.plugin.api.utils.MOCKITO_MOCKMAKER_FILE_NAME import org.utbot.framework.util.Conflict -import org.utbot.intellij.plugin.generator.UtTestsDialogProcessor import org.utbot.intellij.plugin.models.GenerateTestsModel import org.utbot.intellij.plugin.models.id import org.utbot.intellij.plugin.models.jUnit4LibraryDescriptor @@ -530,7 +529,7 @@ class GenerateTestsDialogWindow(val model: GenerateTestsModel) : DialogWrapper(m model.chosenClassesToMockAlways = chosenClassesToMockAlways() model.fuzzingValue = fuzzingValue model.commentStyle = javaDocCommentStyle - model.enableSummariesGeneration = state.enableSummariesGeneration + model.summariesGenerationType = state.summariesGenerationType UtSettings.treatOverflowAsError = treatOverflowAsError == TreatOverflowAsError.AS_ERROR } diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt index fddccaac70..2b913dd9d5 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt @@ -60,9 +60,12 @@ import kotlinx.coroutines.newSingleThreadContext import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeoutOrNull import kotlinx.coroutines.yield +import org.utbot.framework.SummariesGenerationType import org.utbot.framework.codegen.services.language.CgLanguageAssistant import org.utbot.framework.minimization.minimizeExecutions import org.utbot.framework.plugin.api.util.isSynthetic +import org.utbot.framework.util.jimpleBody +import org.utbot.summary.summarize internal const val junitVersion = 4 private val logger = KotlinLogging.logger {} @@ -161,7 +164,7 @@ fun setOptions() { UtSettings.classfilesCanChange = false // We need to use assemble model generator to increase readability UtSettings.useAssembleModelGenerator = true - UtSettings.enableSummariesGeneration = false + UtSettings.summaryGenerationType = SummariesGenerationType.LIGHT UtSettings.preferredCexOption = false UtSettings.warmupConcreteExecution = true UtSettings.testMinimizationStrategyType = TestSelectionStrategyType.COVERAGE_STRATEGY @@ -393,7 +396,10 @@ fun runGeneration( } cancellator.cancel() - val testSets = testsByMethod.map { (method, executions) -> UtMethodTestSet(method, minimizeExecutions(executions)) } + val testSets = testsByMethod.map { (method, executions) -> + UtMethodTestSet(method, minimizeExecutions(executions), jimpleBody(method)) + .summarize(cut.classfileDir.toPath()) + } logger.info().bracket("Flushing tests for [${cut.simpleName}] on disk") { writeTestClass(cut, codeGenerator.generateAsString(testSets)) diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt index f9973346fb..a80ccf5f19 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt @@ -1,7 +1,6 @@ package org.utbot.summary import com.github.javaparser.ast.body.MethodDeclaration -import org.utbot.framework.UtSettings import org.utbot.framework.plugin.api.UtClusterInfo import org.utbot.framework.plugin.api.UtSymbolicExecution import org.utbot.framework.plugin.api.UtExecutionCluster @@ -18,12 +17,15 @@ import java.io.File import java.nio.file.Path import java.nio.file.Paths import mu.KotlinLogging +import org.utbot.framework.SummariesGenerationType import org.utbot.framework.UtSettings.enableClusterCommentsGeneration import org.utbot.framework.UtSettings.enableJavaDocGeneration import org.utbot.framework.UtSettings.useDisplayNameArrowStyle import org.utbot.framework.UtSettings.enableDisplayNameGeneration import org.utbot.framework.UtSettings.enableTestNamesGeneration +import org.utbot.framework.UtSettings.summaryGenerationType import org.utbot.framework.UtSettings.useCustomJavaDocTags +import org.utbot.framework.plugin.api.util.isConstructor import org.utbot.framework.plugin.api.util.jClass import org.utbot.fuzzer.FuzzedMethodDescription import org.utbot.fuzzer.FuzzedValue @@ -36,11 +38,18 @@ import soot.SootMethod private val logger = KotlinLogging.logger {} fun UtMethodTestSet.summarize(sourceFile: File?, searchDirectory: Path = Paths.get("")): UtMethodTestSet { - if (!UtSettings.enableSummariesGeneration) return this + if (summaryGenerationType == SummariesGenerationType.NONE) return this return try { makeDiverseExecutions(this) - val invokeDescriptions = invokeDescriptions(this, searchDirectory) + + // HACK: we avoid calling [invokeDescriptions] method to save time, it is useless in Contest + val invokeDescriptions = when (summaryGenerationType) { + SummariesGenerationType.FULL -> invokeDescriptions(this, searchDirectory) + SummariesGenerationType.LIGHT, + SummariesGenerationType.NONE -> emptyList() + } + // every cluster has summary and list of executions val executionClusters = Summarization(sourceFile, invokeDescriptions).fillSummaries(this) val updatedExecutions = executionClusters.flatMap { it.executions } @@ -83,9 +92,18 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List() - executionClusters += generateSummariesForTestsWithNonEmptyPathsProducedBySymbolicExecutor(testSet) - executionClusters += generateSummariesForTestsProducedByFuzzer(testSet) - executionClusters += generateSummariesForTestsWithEmptyPathsProducedBySymbolicExecutor(testSet) + when (summaryGenerationType) { + SummariesGenerationType.FULL -> { + executionClusters += generateSummariesForTestsWithNonEmptyPathsProducedBySymbolicExecutor(testSet) + executionClusters += generateSummariesForTestsProducedByFuzzer(testSet) + executionClusters += generateSummariesForTestsWithEmptyPathsProducedBySymbolicExecutor(testSet) + } + SummariesGenerationType.LIGHT -> { + executionClusters += generateSummariesForTestsProducedBySymbolicExecutorWithoutSources(testSet) + executionClusters += generateSummariesForTestsProducedByFuzzer(testSet) + } + SummariesGenerationType.NONE -> error("We must not fill summaries if SummariesGenerationType is NONE") + } return if (enableClusterCommentsGeneration && executionClusters.size > 0) executionClusters @@ -210,6 +228,53 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List { + val clustersToReturn: MutableList = mutableListOf() + + val testSetWithFuzzedExecutions = prepareTestSetForByteCodeAnalysis(testSet) + val executions = testSetWithFuzzedExecutions.executions as List + + if (executions.isNotEmpty()) { + executions.forEach { utExecution -> + + val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester()) + val testMethodName = try { + nameSuggester.flatMap { + val executableId = testSet.method + val description = FuzzedMethodDescription(executableId).apply { + compilableName = if (!executableId.isConstructor) executableId.name else null + } + it.suggest( + description, + utExecution.stateBefore.parameters.map { value -> FuzzedValue(value) }, + utExecution.result + ) + }.firstOrNull() + } catch (t: Throwable) { + logger.error(t) { "Cannot create suggested test name for $utExecution" } + null + } + utExecution.testMethodName = testMethodName?.testName + utExecution.displayName = testMethodName?.displayName + utExecution.summary = testMethodName?.javaDoc + } + + val clusteredExecutions = groupFuzzedExecutions(testSetWithFuzzedExecutions) + clusteredExecutions.forEach { + clustersToReturn.add( + UtExecutionCluster( + UtClusterInfo(it.header), + it.executions + ) + ) + } + } + + return clustersToReturn.toList() + } + private fun generateSummariesForTestsProducedByFuzzer( testSet: UtMethodTestSet ): List { diff --git a/utbot-testing/src/main/kotlin/org/utbot/testing/UtValueTestCaseChecker.kt b/utbot-testing/src/main/kotlin/org/utbot/testing/UtValueTestCaseChecker.kt index eb3812898b..b2c2393a05 100644 --- a/utbot-testing/src/main/kotlin/org/utbot/testing/UtValueTestCaseChecker.kt +++ b/utbot-testing/src/main/kotlin/org/utbot/testing/UtValueTestCaseChecker.kt @@ -8,6 +8,7 @@ import org.utbot.common.FileUtil.clearTempDirectory import org.utbot.common.FileUtil.findPathToClassFiles import org.utbot.common.FileUtil.locateClass import org.utbot.engine.prettify +import org.utbot.framework.SummariesGenerationType import org.utbot.framework.UtSettings import org.utbot.framework.UtSettings.daysLimitForTempFiles import org.utbot.framework.UtSettings.testDisplayName @@ -87,7 +88,7 @@ abstract class UtValueTestCaseChecker( UtSettings.saveRemainingStatesForConcreteExecution = false UtSettings.useFuzzing = false UtSettings.useCustomJavaDocTags = false - UtSettings.enableSummariesGeneration = true + UtSettings.summaryGenerationType = SummariesGenerationType.FULL } // checks paramsBefore and result