Use summaries in ContestEstimator (#1661)

EgorkaKulikov · web-flow · commit f0095ab83b54 · 2023-01-13T14:18:40.000+03:00
Support symbolic executions in ContestEstimator

Co-authored-by: Andrey Tarbeev
diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt
@@ -134,9 +134,10 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS
     /**
      * Enable the Summarization module to generate summaries for methods under test.
      *
-     * Note: if it is false, all the execution for a particular method will be stored at the same nameless region.
+     * Note: if it is [SummariesGenerationType.NONE],
+     * all the execution for a particular method will be stored at the same nameless region.
      */
-    var enableSummariesGeneration by getBooleanProperty(true)
+    var summaryGenerationType by getEnumProperty(SummariesGenerationType.FULL)
 
     /**
      * If True test comments will be generated.
@@ -632,3 +633,23 @@ enum class MLPredictorType {
      */
     LINREG
 }
+
+/**
+ * Enum to describe how we analyze code to obtain summaries.
+ */
+enum class SummariesGenerationType {
+    /**
+     * All possible analysis actions are taken
+     */
+    FULL,
+
+    /**
+     * Analysis actions based on sources are NOT taken
+     */
+    LIGHT,
+
+    /**
+     * No summaries are generated
+     */
+    NONE,
+}
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt
@@ -141,7 +141,7 @@ object UtTestsDialogProcessor {
 
                     UtSettings.concreteExecutionTimeoutInInstrumentedProcess = model.hangingTestsTimeout.timeoutMs
                     UtSettings.useCustomJavaDocTags = model.commentStyle == JavaDocCommentStyle.CUSTOM_JAVADOC_TAGS
-                    UtSettings.enableSummariesGeneration = model.enableSummariesGeneration
+                    UtSettings.summaryGenerationType = model.summariesGenerationType
 
                     fun now() = LocalDateTime.now().format(DateTimeFormatter.ofPattern("HH:mm:ss.SSS"))
 
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/models/GenerateTestsModel.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/models/GenerateTestsModel.kt
@@ -16,6 +16,7 @@ import com.intellij.psi.PsiClass
 import com.intellij.psi.PsiJavaFile
 import com.intellij.refactoring.util.classMembers.MemberInfo
 import org.jetbrains.kotlin.psi.KtFile
+import org.utbot.framework.SummariesGenerationType
 import org.utbot.framework.UtSettings
 import org.utbot.framework.plugin.api.JavaDocCommentStyle
 import org.utbot.framework.util.ConflictTriggers
@@ -55,7 +56,7 @@ class GenerateTestsModel(
     val conflictTriggers: ConflictTriggers = ConflictTriggers()
 
     var runGeneratedTestsWithCoverage : Boolean = false
-    var enableSummariesGeneration : Boolean = UtSettings.enableSummariesGeneration
+    var summariesGenerationType : SummariesGenerationType = UtSettings.summaryGenerationType
 }
 
 val PsiClass.packageName: String
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Settings.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Settings.kt
@@ -36,6 +36,7 @@ import org.utbot.intellij.plugin.models.GenerateTestsModel
 import java.util.concurrent.CompletableFuture
 import kotlin.reflect.KClass
 import org.utbot.common.isWindows
+import org.utbot.framework.SummariesGenerationType
 import org.utbot.framework.plugin.api.isSummarizationCompatible
 
 @State(
@@ -63,7 +64,7 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
         var fuzzingValue: Double = 0.05,
         var runGeneratedTestsWithCoverage: Boolean = false,
         var commentStyle: JavaDocCommentStyle = JavaDocCommentStyle.defaultItem,
-        var enableSummariesGeneration: Boolean = UtSettings.enableSummariesGeneration,
+        var summariesGenerationType: SummariesGenerationType = UtSettings.summaryGenerationType,
         var enableExperimentalLanguagesSupport: Boolean = false,
     ) {
         constructor(model: GenerateTestsModel) : this(
@@ -82,7 +83,7 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
             fuzzingValue = model.fuzzingValue,
             runGeneratedTestsWithCoverage = model.runGeneratedTestsWithCoverage,
             commentStyle = model.commentStyle,
-            enableSummariesGeneration = model.enableSummariesGeneration
+            summariesGenerationType = model.summariesGenerationType
         )
 
         override fun equals(other: Any?): Boolean {
@@ -107,7 +108,7 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
             if (fuzzingValue != other.fuzzingValue) return false
             if (runGeneratedTestsWithCoverage != other.runGeneratedTestsWithCoverage) return false
             if (commentStyle != other.commentStyle) return false
-            if (enableSummariesGeneration != other.enableSummariesGeneration) return false
+            if (summariesGenerationType != other.summariesGenerationType) return false
 
             return true
         }
@@ -127,7 +128,7 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
             result = 31 * result + classesToMockAlways.contentHashCode()
             result = 31 * result + fuzzingValue.hashCode()
             result = 31 * result + if (runGeneratedTestsWithCoverage) 1 else 0
-            result = 31 * result + if (enableSummariesGeneration) 1 else 0
+            result = 31 * result + summariesGenerationType.hashCode()
 
             return result
         }
@@ -173,7 +174,7 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
         }
     var runGeneratedTestsWithCoverage = state.runGeneratedTestsWithCoverage
 
-    var enableSummariesGeneration = state.enableSummariesGeneration
+    var enableSummariesGeneration = state.summariesGenerationType
 
     fun setClassesToMockAlways(classesToMockAlways: List<String>) {
         state.classesToMockAlways = classesToMockAlways.distinct().toTypedArray()
@@ -209,7 +210,7 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
     override fun loadState(state: State) {
         this.state = state
         if (!state.codegenLanguage.isSummarizationCompatible()) {
-            this.state.enableSummariesGeneration = false
+            this.state.summariesGenerationType = SummariesGenerationType.NONE
         }
     }
 
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt
@@ -18,6 +18,7 @@ import com.intellij.ui.layout.withValueBinding
 import com.intellij.util.castSafelyTo
 import com.intellij.util.ui.UIUtil
 import com.intellij.util.ui.components.BorderLayoutPanel
+import org.utbot.framework.SummariesGenerationType
 import org.utbot.framework.UtSettings
 import org.utbot.framework.codegen.domain.ForceStaticMocking
 import org.utbot.framework.codegen.domain.HangingTestsTimeout
@@ -138,13 +139,15 @@ class SettingsWindow(val project: Project) {
             cell {
                 enableSummarizationGenerationCheckBox = checkBox("Enable Summaries Generation")
                     .onApply {
-                        settings.state.enableSummariesGeneration = enableSummarizationGenerationCheckBox.isSelected
+                        settings.state.summariesGenerationType =
+                            if (enableSummarizationGenerationCheckBox.isSelected) SummariesGenerationType.FULL else SummariesGenerationType.NONE
                     }
                     .onReset {
-                        enableSummarizationGenerationCheckBox.isSelected = settings.state.enableSummariesGeneration
+                        enableSummarizationGenerationCheckBox.isSelected =
+                            settings.state.summariesGenerationType != SummariesGenerationType.NONE
                     }
                     .onIsModified {
-                        enableSummarizationGenerationCheckBox.isSelected xor settings.state.enableSummariesGeneration
+                        enableSummarizationGenerationCheckBox.isSelected xor (settings.state.summariesGenerationType != SummariesGenerationType.NONE)
                     }
                     .enableIf(codegenLanguageCombo.selectedValueMatches(CodegenLanguage?::isSummarizationCompatible))
                     .component
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/GenerateTestsDialogWindow.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/GenerateTestsDialogWindow.kt
@@ -116,7 +116,6 @@ import org.utbot.framework.plugin.api.utils.MOCKITO_EXTENSIONS_FILE_CONTENT
 import org.utbot.framework.plugin.api.utils.MOCKITO_EXTENSIONS_FOLDER
 import org.utbot.framework.plugin.api.utils.MOCKITO_MOCKMAKER_FILE_NAME
 import org.utbot.framework.util.Conflict
-import org.utbot.intellij.plugin.generator.UtTestsDialogProcessor
 import org.utbot.intellij.plugin.models.GenerateTestsModel
 import org.utbot.intellij.plugin.models.id
 import org.utbot.intellij.plugin.models.jUnit4LibraryDescriptor
@@ -530,7 +529,7 @@ class GenerateTestsDialogWindow(val model: GenerateTestsModel) : DialogWrapper(m
             model.chosenClassesToMockAlways = chosenClassesToMockAlways()
             model.fuzzingValue = fuzzingValue
             model.commentStyle = javaDocCommentStyle
-            model.enableSummariesGeneration = state.enableSummariesGeneration
+            model.summariesGenerationType = state.summariesGenerationType
             UtSettings.treatOverflowAsError = treatOverflowAsError == TreatOverflowAsError.AS_ERROR
         }
 
diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt
@@ -60,9 +60,12 @@ import kotlinx.coroutines.newSingleThreadContext
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withTimeoutOrNull
 import kotlinx.coroutines.yield
+import org.utbot.framework.SummariesGenerationType
 import org.utbot.framework.codegen.services.language.CgLanguageAssistant
 import org.utbot.framework.minimization.minimizeExecutions
 import org.utbot.framework.plugin.api.util.isSynthetic
+import org.utbot.framework.util.jimpleBody
+import org.utbot.summary.summarize
 
 internal const val junitVersion = 4
 private val logger = KotlinLogging.logger {}
@@ -161,7 +164,7 @@ fun setOptions() {
     UtSettings.classfilesCanChange = false
     // We need to use assemble model generator to increase readability
     UtSettings.useAssembleModelGenerator = true
-    UtSettings.enableSummariesGeneration = false
+    UtSettings.summaryGenerationType = SummariesGenerationType.LIGHT
     UtSettings.preferredCexOption = false
     UtSettings.warmupConcreteExecution = true
     UtSettings.testMinimizationStrategyType = TestSelectionStrategyType.COVERAGE_STRATEGY
@@ -393,7 +396,10 @@ fun runGeneration(
         }
         cancellator.cancel()
 
-        val testSets = testsByMethod.map { (method, executions) -> UtMethodTestSet(method, minimizeExecutions(executions)) }
+        val testSets = testsByMethod.map { (method, executions) ->
+            UtMethodTestSet(method, minimizeExecutions(executions), jimpleBody(method))
+                .summarize(cut.classfileDir.toPath())
+        }
 
         logger.info().bracket("Flushing tests for [${cut.simpleName}] on disk") {
             writeTestClass(cut, codeGenerator.generateAsString(testSets))
diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt
@@ -1,7 +1,6 @@
 package org.utbot.summary
 
 import com.github.javaparser.ast.body.MethodDeclaration
-import org.utbot.framework.UtSettings
 import org.utbot.framework.plugin.api.UtClusterInfo
 import org.utbot.framework.plugin.api.UtSymbolicExecution
 import org.utbot.framework.plugin.api.UtExecutionCluster
@@ -18,12 +17,15 @@ import java.io.File
 import java.nio.file.Path
 import java.nio.file.Paths
 import mu.KotlinLogging
+import org.utbot.framework.SummariesGenerationType
 import org.utbot.framework.UtSettings.enableClusterCommentsGeneration
 import org.utbot.framework.UtSettings.enableJavaDocGeneration
 import org.utbot.framework.UtSettings.useDisplayNameArrowStyle
 import org.utbot.framework.UtSettings.enableDisplayNameGeneration
 import org.utbot.framework.UtSettings.enableTestNamesGeneration
+import org.utbot.framework.UtSettings.summaryGenerationType
 import org.utbot.framework.UtSettings.useCustomJavaDocTags
+import org.utbot.framework.plugin.api.util.isConstructor
 import org.utbot.framework.plugin.api.util.jClass
 import org.utbot.fuzzer.FuzzedMethodDescription
 import org.utbot.fuzzer.FuzzedValue
@@ -36,11 +38,18 @@ import soot.SootMethod
 private val logger = KotlinLogging.logger {}
 
 fun UtMethodTestSet.summarize(sourceFile: File?, searchDirectory: Path = Paths.get("")): UtMethodTestSet {
-    if (!UtSettings.enableSummariesGeneration) return this
+    if (summaryGenerationType == SummariesGenerationType.NONE) return this
 
     return try {
         makeDiverseExecutions(this)
-        val invokeDescriptions = invokeDescriptions(this, searchDirectory)
+
+        // HACK: we avoid calling [invokeDescriptions] method to save time, it is useless in Contest
+        val invokeDescriptions = when (summaryGenerationType) {
+            SummariesGenerationType.FULL -> invokeDescriptions(this, searchDirectory)
+            SummariesGenerationType.LIGHT,
+            SummariesGenerationType.NONE -> emptyList()
+        }
+
         // every cluster has summary and list of executions
         val executionClusters = Summarization(sourceFile, invokeDescriptions).fillSummaries(this)
         val updatedExecutions = executionClusters.flatMap { it.executions }
@@ -83,9 +92,18 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
 
         val executionClusters = mutableListOf<UtExecutionCluster>()
 
-        executionClusters += generateSummariesForTestsWithNonEmptyPathsProducedBySymbolicExecutor(testSet)
-        executionClusters += generateSummariesForTestsProducedByFuzzer(testSet)
-        executionClusters += generateSummariesForTestsWithEmptyPathsProducedBySymbolicExecutor(testSet)
+        when (summaryGenerationType) {
+            SummariesGenerationType.FULL -> {
+                executionClusters += generateSummariesForTestsWithNonEmptyPathsProducedBySymbolicExecutor(testSet)
+                executionClusters += generateSummariesForTestsProducedByFuzzer(testSet)
+                executionClusters += generateSummariesForTestsWithEmptyPathsProducedBySymbolicExecutor(testSet)
+            }
+            SummariesGenerationType.LIGHT -> {
+                executionClusters += generateSummariesForTestsProducedBySymbolicExecutorWithoutSources(testSet)
+                executionClusters += generateSummariesForTestsProducedByFuzzer(testSet)
+            }
+            SummariesGenerationType.NONE -> error("We must not fill summaries if SummariesGenerationType is NONE")
+        }
 
         return if (enableClusterCommentsGeneration && executionClusters.size > 0)
             executionClusters
@@ -210,6 +228,53 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
         return clustersToReturn.toList()
     }
 
+    private fun generateSummariesForTestsProducedBySymbolicExecutorWithoutSources(
+        testSet: UtMethodTestSet
+    ) : List<UtExecutionCluster> {
+        val clustersToReturn: MutableList<UtExecutionCluster> = mutableListOf()
+
+        val testSetWithFuzzedExecutions = prepareTestSetForByteCodeAnalysis(testSet)
+        val executions = testSetWithFuzzedExecutions.executions as List<UtSymbolicExecution>
+
+        if (executions.isNotEmpty()) {
+            executions.forEach { utExecution ->
+
+                val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester())
+                val testMethodName = try {
+                    nameSuggester.flatMap {
+                        val executableId = testSet.method
+                        val description = FuzzedMethodDescription(executableId).apply {
+                            compilableName = if (!executableId.isConstructor) executableId.name else null
+                        }
+                        it.suggest(
+                            description,
+                            utExecution.stateBefore.parameters.map { value -> FuzzedValue(value) },
+                            utExecution.result
+                        )
+                    }.firstOrNull()
+                } catch (t: Throwable) {
+                    logger.error(t) { "Cannot create suggested test name for $utExecution" }
+                    null
+                }
+                utExecution.testMethodName = testMethodName?.testName
+                utExecution.displayName = testMethodName?.displayName
+                utExecution.summary = testMethodName?.javaDoc
+            }
+
+            val clusteredExecutions = groupFuzzedExecutions(testSetWithFuzzedExecutions)
+            clusteredExecutions.forEach {
+                clustersToReturn.add(
+                    UtExecutionCluster(
+                        UtClusterInfo(it.header),
+                        it.executions
+                    )
+                )
+            }
+        }
+
+        return clustersToReturn.toList()
+    }
+
     private fun generateSummariesForTestsProducedByFuzzer(
         testSet: UtMethodTestSet
     ): List<UtExecutionCluster> {
diff --git a/utbot-testing/src/main/kotlin/org/utbot/testing/UtValueTestCaseChecker.kt b/utbot-testing/src/main/kotlin/org/utbot/testing/UtValueTestCaseChecker.kt
@@ -8,6 +8,7 @@ import org.utbot.common.FileUtil.clearTempDirectory
 import org.utbot.common.FileUtil.findPathToClassFiles
 import org.utbot.common.FileUtil.locateClass
 import org.utbot.engine.prettify
+import org.utbot.framework.SummariesGenerationType
 import org.utbot.framework.UtSettings
 import org.utbot.framework.UtSettings.daysLimitForTempFiles
 import org.utbot.framework.UtSettings.testDisplayName
@@ -87,7 +88,7 @@ abstract class UtValueTestCaseChecker(
         UtSettings.saveRemainingStatesForConcreteExecution = false
         UtSettings.useFuzzing = false
         UtSettings.useCustomJavaDocTags = false
-        UtSettings.enableSummariesGeneration = true
+        UtSettings.summaryGenerationType = SummariesGenerationType.FULL
     }
 
     // checks paramsBefore and result
