Add an option to control fuzzing #451 (#552)

Markoutte · web-flow · commit b7ea573aa9e5 · 2022-07-21T12:16:38.000+03:00
diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt
@@ -270,7 +270,7 @@ object UtSettings {
     /**
      * Fuzzer tries to generate and run tests during this time.
      */
-    var fuzzingTimeoutInMillis: Int by getIntProperty(3_000)
+    var fuzzingTimeoutInMillis: Long by getLongProperty(3_000L)
 
     /**
      * Generate tests that treat possible overflows in arithmetic operations as errors
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestCaseGenerator.kt
@@ -5,8 +5,6 @@ import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.cancel
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.collect
-import kotlinx.coroutines.flow.flattenConcat
-import kotlinx.coroutines.flow.flowOf
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
@@ -119,15 +117,15 @@ open class TestCaseGenerator(
         executionTimeEstimator: ExecutionTimeEstimator = ExecutionTimeEstimator(utBotGenerationTimeoutInMillis, 1)
     ): Flow<UtResult> {
         val engine = createSymbolicEngine(controller, method, mockStrategy, chosenClassesToMockAlways, executionTimeEstimator)
-        return createDefaultFlow(engine)
+        return defaultTestFlow(engine, executionTimeEstimator.userTimeout)
     }
 
     fun generate(
         methods: List<UtMethod<*>>,
         mockStrategy: MockStrategyApi,
         chosenClassesToMockAlways: Set<ClassId> = Mocker.javaDefaultClasses.mapTo(mutableSetOf()) { it.id },
         methodsGenerationTimeout: Long = utBotGenerationTimeoutInMillis,
-        generate: (engine: UtBotSymbolicEngine) -> Flow<UtResult> = ::createDefaultFlow
+        generate: (engine: UtBotSymbolicEngine) -> Flow<UtResult> = defaultTestFlow(methodsGenerationTimeout)
     ): List<UtMethodTestSet> {
         if (isCanceled()) return methods.map { UtMethodTestSet(it) }
 
@@ -260,17 +258,6 @@ open class TestCaseGenerator(
         )
     }
 
-    private fun createDefaultFlow(engine: UtBotSymbolicEngine): Flow<UtResult> {
-        var flow = engine.traverse()
-        if (UtSettings.useFuzzing) {
-            flow = flowOf(
-                engine.fuzzing(System.currentTimeMillis() + UtSettings.fuzzingTimeoutInMillis),
-                flow,
-            ).flattenConcat()
-        }
-        return flow
-    }
-
     // CONFLUENCE:The+UtBot+Java+timeouts
 
     class ExecutionTimeEstimator(val userTimeout: Long, methodsUnderTestNumber: Int) {
diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestFlow.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/plugin/api/TestFlow.kt
@@ -0,0 +1,76 @@
+package org.utbot.framework.plugin.api
+
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.emptyFlow
+import kotlinx.coroutines.flow.flattenConcat
+import kotlinx.coroutines.flow.flowOf
+import org.utbot.engine.UtBotSymbolicEngine
+import org.utbot.framework.UtSettings
+
+/**
+ * Constructs [TestFlow] for customization and creates flow producer.
+ */
+fun testFlow(block: TestFlow.() -> Unit): UtBotSymbolicEngine.() -> Flow<UtResult> =  { TestFlow(block).build(this) }
+
+/**
+ * Creates default flow that uses [UtSettings] for customization.
+ */
+fun defaultTestFlow(timeout: Long) = testFlow {
+    isSymbolicEngineEnabled = true
+    generationTimeout = timeout
+    isFuzzingEnabled = UtSettings.useFuzzing
+    if (generationTimeout > 0) {
+        fuzzingValue = UtSettings.fuzzingTimeoutInMillis.coerceIn(0, generationTimeout) / generationTimeout.toDouble()
+    }
+}
+
+/**
+ * Creates default flow that uses [UtSettings] for customization.
+ */
+fun defaultTestFlow(engine: UtBotSymbolicEngine, timeout: Long) = defaultTestFlow(timeout).invoke(engine)
+
+/**
+ * TestFlow helps construct flows with different settings but with common sequence of test generation process.
+ *
+ * Use [testFlow] to set a custom test flow or [defaultTestFlow] to create flow based on [UtSettings].
+ */
+class TestFlow internal constructor(block: TestFlow.() -> Unit) {
+    var generationTimeout = 0L
+        set(value) {
+            field = maxOf(0, value)
+        }
+    var isSymbolicEngineEnabled = true
+    var isFuzzingEnabled = false
+    var fuzzingValue: Double = 0.1
+        set(value) {
+            field = value.coerceIn(0.0, 1.0)
+        }
+
+    init {
+        apply(block)
+    }
+
+    /*
+        Constructs default flow that is having the following steps at the moment:
+        1. If fuzzer is enabled then run it before symbolic execution for [fuzzingValue] * [generationTimeout] ms.
+        2. Run symbolic execution for the rest time.
+        3. If both (fuzzer and symbolic execution) are off then do nothing.
+     */
+    fun build(engine: UtBotSymbolicEngine): Flow<UtResult>  {
+        return when {
+            generationTimeout == 0L -> emptyFlow()
+            isFuzzingEnabled -> {
+                when (val value = if (isSymbolicEngineEnabled) (fuzzingValue * generationTimeout).toLong() else generationTimeout) {
+                    0L -> engine.traverse()
+                    generationTimeout -> engine.fuzzing()
+                    else -> flowOf(
+                        engine.fuzzing(System.currentTimeMillis() + value),
+                        engine.traverse()
+                    ).flattenConcat()
+                }
+            }
+            isSymbolicEngineEnabled -> engine.traverse()
+            else -> emptyFlow()
+        }
+    }
+}
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt
@@ -8,6 +8,7 @@ import com.intellij.openapi.application.invokeLater
 import com.intellij.openapi.compiler.CompileContext
 import com.intellij.openapi.compiler.CompilerManager
 import com.intellij.openapi.compiler.CompilerPaths
+import com.intellij.openapi.components.service
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.progress.ProgressIndicator
 import com.intellij.openapi.progress.Task
@@ -49,6 +50,8 @@ import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
 import org.utbot.common.filterWhen
 import org.utbot.engine.util.mockListeners.ForceStaticMockListener
+import org.utbot.framework.plugin.api.testFlow
+import org.utbot.intellij.plugin.settings.Settings
 import kotlin.reflect.KClass
 import kotlin.reflect.full.functions
 
@@ -188,7 +191,18 @@ object UtTestsDialogProcessor {
 
                                     val notEmptyCases = withUtContext(context) {
                                         testCaseGenerator
-                                            .generate(methods, model.mockStrategy, model.chosenClassesToMockAlways, model.timeout)
+                                            .generate(
+                                                methods,
+                                                model.mockStrategy,
+                                                model.chosenClassesToMockAlways,
+                                                model.timeout,
+                                                generate = testFlow {
+                                                    generationTimeout = model.timeout
+                                                    isSymbolicEngineEnabled = true
+                                                    isFuzzingEnabled = UtSettings.useFuzzing
+                                                    fuzzingValue = project.service<Settings>().fuzzingValue
+                                                }
+                                            )
                                             .map { it.summarize(searchDirectory) }
                                             .filterNot { it.executions.isEmpty() && it.errors.isEmpty() }
                                     }
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/models/GenerateTestsModel.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/models/GenerateTestsModel.kt
@@ -28,6 +28,7 @@ data class GenerateTestsModel(
     var selectedMethods: Set<MemberInfo>?,
     var timeout:Long,
     var generateWarningsForStaticMocking: Boolean = false,
+    var fuzzingValue: Double = 0.05
 ) {
     var testSourceRoot: VirtualFile? = null
     var testPackageName: String? = null
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Configurable.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Configurable.kt
@@ -11,7 +11,7 @@ class Configurable(val project: Project) : SearchableConfigurable {
 
     override fun createComponent(): JComponent = settingsWindow.panel
 
-    override fun isModified(): Boolean = settingsWindow.isModified
+    override fun isModified(): Boolean = settingsWindow.isModified()
 
     override fun apply() = settingsWindow.apply()
 
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Settings.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/Settings.kt
@@ -51,7 +51,8 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
         var forceStaticMocking: ForceStaticMocking = ForceStaticMocking.defaultItem,
         var treatOverflowAsError: TreatOverflowAsError = TreatOverflowAsError.defaultItem,
         var parametrizedTestSource: ParametrizedTestSource = ParametrizedTestSource.defaultItem,
-        var classesToMockAlways: Array<String> = Mocker.defaultSuperClassesToMockAlwaysNames.toTypedArray()
+        var classesToMockAlways: Array<String> = Mocker.defaultSuperClassesToMockAlwaysNames.toTypedArray(),
+        var fuzzingValue: Double = 0.05,
     ) {
         constructor(model: GenerateTestsModel) : this(
             codegenLanguage = model.codegenLanguage,
@@ -63,7 +64,8 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
             hangingTestsTimeout = model.hangingTestsTimeout,
             forceStaticMocking = model.forceStaticMocking,
             parametrizedTestSource = model.parametrizedTestSource,
-            classesToMockAlways = model.chosenClassesToMockAlways.mapTo(mutableSetOf()) { it.name }.toTypedArray()
+            classesToMockAlways = model.chosenClassesToMockAlways.mapTo(mutableSetOf()) { it.name }.toTypedArray(),
+            fuzzingValue = model.fuzzingValue
         )
 
         override fun equals(other: Any?): Boolean {
@@ -83,6 +85,7 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
             if (treatOverflowAsError != other.treatOverflowAsError) return false
             if (parametrizedTestSource != other.parametrizedTestSource) return false
             if (!classesToMockAlways.contentEquals(other.classesToMockAlways)) return false
+            if (fuzzingValue != other.fuzzingValue) return false
 
             return true
         }
@@ -98,6 +101,7 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
             result = 31 * result + treatOverflowAsError.hashCode()
             result = 31 * result + parametrizedTestSource.hashCode()
             result = 31 * result + classesToMockAlways.contentHashCode()
+            result = 31 * result + fuzzingValue.hashCode()
 
             return result
         }
@@ -129,6 +133,12 @@ class Settings(val project: Project) : PersistentStateComponent<Settings.State>
 
     val classesToMockAlways: Set<String> get() = state.classesToMockAlways.toSet()
 
+    var fuzzingValue: Double
+        get() = state.fuzzingValue
+        set(value) {
+            state.fuzzingValue = value.coerceIn(0.0, 1.0)
+        }
+
     fun setClassesToMockAlways(classesToMockAlways: List<String>) {
         state.classesToMockAlways = classesToMockAlways.distinct().toTypedArray()
     }
@@ -217,8 +227,7 @@ private class HangingTestsTimeoutConverter : Converter<HangingTestsTimeout>() {
 
     override fun fromString(value: String): HangingTestsTimeout {
         val arguments = value.substringAfter("HangingTestsTimeout:")
-        val timeoutMs = arguments.first().toLong()
-
+        val timeoutMs = arguments.toLong()
         return HangingTestsTimeout(timeoutMs)
     }
 }
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/GenerateTestsDialogWindow.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/GenerateTestsDialogWindow.kt
diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/utils/UiUtils.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/utils/UiUtils.kt
