From 82483461e1e54fde572277b09c40076a7050a0fa Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 14:48:55 +0300 Subject: [PATCH 01/16] Added an initial solution --- settings.gradle | 1 + utbot-analytics-torch/build.gradle | 63 +++++++++++++++++++ .../org/utbot/TorchAnalyticsConfiguration.kt | 21 +++++++ .../predictors/StateRewardPredictorFactory.kt | 16 +++++ .../predictors/StateRewardPredictorTorch.kt | 2 +- .../predictors/NNStateRewardPredictorTest.kt | 51 +++++++++++++++ .../src/test/resources/log4j2.xml | 40 ++++++++++++ utbot-analytics/build.gradle | 20 +++--- .../predictors/LinearStateRewardPredictor.kt | 2 +- .../kotlin/org/utbot/predictors/NNJson.kt | 2 +- .../predictors/StateRewardPredictorFactory.kt | 2 +- .../kotlin/org/utbot/predictors/scalers.kt | 2 +- .../LinearStateRewardPredictorTest.kt | 2 +- .../predictors/NNStateRewardPredictorTest.kt | 14 ++--- .../kotlin/org/utbot/framework/UtSettings.kt | 22 +++---- .../utbot/analytics/EngineAnalyticsContext.kt | 8 +-- .../org/utbot/engine/UtBotSymbolicEngine.kt | 2 +- .../utbot/examples/UtValueTestCaseChecker.kt | 6 +- .../generator/UtTestsDialogProcessor.kt | 5 +- .../org/utbot/contest/ContestEstimator.kt | 6 +- 20 files changed, 233 insertions(+), 54 deletions(-) create mode 100644 utbot-analytics-torch/build.gradle create mode 100644 utbot-analytics-torch/src/main/kotlin/org/utbot/TorchAnalyticsConfiguration.kt create mode 100644 utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt rename {utbot-analytics => utbot-analytics-torch}/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt (94%) create mode 100644 utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt create mode 100644 utbot-analytics-torch/src/test/resources/log4j2.xml diff --git a/settings.gradle b/settings.gradle index 78fb689b9d..5fd5795863 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,6 +18,7 @@ include 'utbot-sample' include 'utbot-fuzzers' include 'utbot-junit-contest' include 'utbot-analytics' +include 'utbot-analytics-torch' include 'utbot-cli' include 'utbot-api' include 'utbot-instrumentation' diff --git a/utbot-analytics-torch/build.gradle b/utbot-analytics-torch/build.gradle new file mode 100644 index 0000000000..c4f1ffeee6 --- /dev/null +++ b/utbot-analytics-torch/build.gradle @@ -0,0 +1,63 @@ +apply from: "${parent.projectDir}/gradle/include/jvm-project.gradle" + +configurations { + torchmodels +} + +def osName = System.getProperty('os.name').toLowerCase().split()[0] +if (osName == "mac") osName = "macosx" +String classifier = osName + "-x86_64" + +evaluationDependsOn(':utbot-framework') +compileTestJava.dependsOn tasks.getByPath(':utbot-framework:testClasses') + +dependencies { + api project(':utbot-analytics') + testImplementation project(':utbot-sample') + testImplementation group: 'junit', name: 'junit', version: junit4_version + + implementation group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" + implementation group: 'org.jsoup', name: 'jsoup', version: jsoup_version + + implementation "ai.djl:api:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-engine:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" + + testImplementation project(':utbot-framework').sourceSets.test.output +} + +test { + minHeapSize = "128m" + maxHeapSize = "3072m" + + jvmArgs '-XX:MaxHeapSize=3072m' + + useJUnitPlatform() { + excludeTags 'slow', 'IntegrationTest' + } +} + +processResources { + configurations.torchmodels.resolvedConfiguration.resolvedArtifacts.each { artifact -> + from(zipTree(artifact.getFile())) { + into "models" + } + } +} + +jar { + dependsOn classes + manifest { + attributes 'Main-Class': 'org.utbot.QualityAnalysisKt' + } + + dependsOn configurations.runtimeClasspath + from { + configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } + } + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + zip64 = true +} \ No newline at end of file diff --git a/utbot-analytics-torch/src/main/kotlin/org/utbot/TorchAnalyticsConfiguration.kt b/utbot-analytics-torch/src/main/kotlin/org/utbot/TorchAnalyticsConfiguration.kt new file mode 100644 index 0000000000..386329ab81 --- /dev/null +++ b/utbot-analytics-torch/src/main/kotlin/org/utbot/TorchAnalyticsConfiguration.kt @@ -0,0 +1,21 @@ +package org.utbot + +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.features.FeatureExtractorFactoryImpl +import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory +import org.utbot.predictors.StateRewardPredictorWithTorchModelsSupportFactoryImpl + +/** + * The basic configuration of the utbot-analytics-torch module used in utbot-intellij and (as planned) in utbot-cli + * to implement the hidden configuration initialization to avoid direct calls of this configuration and usage of utbot-analytics-torch imports. + * + * @see + * Issue: Enable utbot-analytics module in utbot-intellij module + */ +object AnalyticsWithTorchModelsSupportConfiguration { + init { + EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() + EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() + EngineAnalyticsContext.stateRewardPredictorFactory = StateRewardPredictorWithTorchModelsSupportFactoryImpl() + } +} \ No newline at end of file diff --git a/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt new file mode 100644 index 0000000000..71cc408515 --- /dev/null +++ b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt @@ -0,0 +1,16 @@ +package org.utbot.predictors + +import org.utbot.analytics.StateRewardPredictorFactory +import org.utbot.framework.StateRewardPredictorType +import org.utbot.framework.UtSettings + +/** + * Creates [StateRewardPredictor], by checking the [UtSettings] configuration. + */ +class StateRewardPredictorWithTorchModelsSupportFactoryImpl : StateRewardPredictorFactory { + override operator fun invoke() = when (UtSettings.stateRewardPredictorType) { + StateRewardPredictorType.BASE -> NNStateRewardPredictorBase() + StateRewardPredictorType.TORCH -> StateRewardPredictorTorch() + StateRewardPredictorType.LINEAR -> LinearStateRewardPredictor() + } +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt similarity index 94% rename from utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt rename to utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt index f0cfe17571..2935b9f555 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt +++ b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt @@ -15,7 +15,7 @@ class StateRewardPredictorTorch : StateRewardPredictor, Closeable { val model: Model = Model.newInstance("model") init { - model.load(Paths.get(UtSettings.rewardModelPath, "model.pt1")) + model.load(Paths.get(UtSettings.modelPath, "model.pt1")) } private val predictor: Predictor, Float> = model.newPredictor(object : Translator, Float> { diff --git a/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt new file mode 100644 index 0000000000..af649db823 --- /dev/null +++ b/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -0,0 +1,51 @@ +package org.utbot.predictors + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test +import org.utbot.examples.withPathSelectorType +import org.utbot.analytics.StateRewardPredictor +import org.utbot.examples.withRewardModelPath +import org.utbot.framework.PathSelectorType +import org.utbot.framework.UtSettings +import kotlin.system.measureNanoTime + +class NNStateRewardPredictorTest { + @Test + @Disabled("Just to see the performance of predictors") + fun simpleTest() { + withRewardModelPath("src/test/resources") { + val pred = StateRewardPredictorTorch() + + val features = listOf(0.0, 0.0) + + assertEquals(5.0, pred.predict(features)) + } + } + + @Disabled("Just to see the performance of predictors") + @Test + fun performanceTest() { + val features = (1..13).map { 1.0 }.toList() + withRewardModelPath("models") { + val averageTime = calcAverageTimeForModelPredict(::StateRewardPredictorTorch, 100, features) + println(averageTime) + } + } + + private fun calcAverageTimeForModelPredict( + model: () -> StateRewardPredictor, + iterations: Int, + features: List + ): Double { + val pred = model() + + (1..iterations).map { + pred.predict(features) + } + + return (1..iterations) + .map { measureNanoTime { pred.predict(features) } } + .average() + } +} \ No newline at end of file diff --git a/utbot-analytics-torch/src/test/resources/log4j2.xml b/utbot-analytics-torch/src/test/resources/log4j2.xml new file mode 100644 index 0000000000..7dde3c2fea --- /dev/null +++ b/utbot-analytics-torch/src/test/resources/log4j2.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/utbot-analytics/build.gradle b/utbot-analytics/build.gradle index ba34f8081b..d4a51683e8 100644 --- a/utbot-analytics/build.gradle +++ b/utbot-analytics/build.gradle @@ -12,13 +12,13 @@ evaluationDependsOn(':utbot-framework') compileTestJava.dependsOn tasks.getByPath(':utbot-framework:testClasses') dependencies { - implementation(project(":utbot-api")) - implementation(project(":utbot-core")) - implementation(project(":utbot-summary")) - implementation(project(":utbot-framework-api")) - implementation(project(":utbot-fuzzers")) - implementation(project(":utbot-instrumentation")) - implementation(project(":utbot-framework")) + api(project(":utbot-api")) + api(project(":utbot-core")) + api(project(":utbot-summary")) + api(project(":utbot-framework-api")) + api(project(":utbot-fuzzers")) + api(project(":utbot-instrumentation")) + api(project(":utbot-framework")) testImplementation project(':utbot-sample') testImplementation group: 'junit', name: 'junit', version: junit4_version @@ -41,12 +41,6 @@ dependencies { implementation group: 'com.github.javaparser', name: 'javaparser-core', version: '3.22.1' - implementation group: 'org.jsoup', name: 'jsoup', version: jsoup_version - - implementation "ai.djl:api:$djl_api_version" - implementation "ai.djl.pytorch:pytorch-engine:$djl_api_version" - implementation "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" - testImplementation project(':utbot-framework').sourceSets.test.output } diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt index 2d3dc434de..7c6cee6b25 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -19,7 +19,7 @@ private val logger = KotlinLogging.logger {} * Last weight is bias */ private fun loadWeights(path: String): Matrix { - val weightsFile = File("${UtSettings.rewardModelPath}/${path}") + val weightsFile = File("${UtSettings.modelPath}/${path}") lateinit var weightsArray: DoubleArray try { diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt index d14034e64d..ed0066ec06 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt @@ -33,7 +33,7 @@ data class NNJson( } internal fun loadModel(path: String): NNJson { - val modelFile = Paths.get(UtSettings.rewardModelPath, path).toFile() + val modelFile = Paths.get(UtSettings.modelPath, path).toFile() lateinit var nnJson: NNJson try { diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt index c0b7dffe7c..43504fee28 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt @@ -10,7 +10,7 @@ import org.utbot.framework.UtSettings class StateRewardPredictorFactoryImpl : StateRewardPredictorFactory { override operator fun invoke() = when (UtSettings.stateRewardPredictorType) { StateRewardPredictorType.BASE -> NNStateRewardPredictorBase() - StateRewardPredictorType.TORCH -> StateRewardPredictorTorch() + StateRewardPredictorType.TORCH -> error("The torch model adapter is bundled with the utbot-analytics-torch module!") StateRewardPredictorType.LINEAR -> LinearStateRewardPredictor() } } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt index b65e78d478..db38bc7743 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt @@ -13,7 +13,7 @@ data class StandardScaler(val mean: Matrix?, val variance: Matrix?) internal fun loadScaler(path: String): StandardScaler = try { - Paths.get(UtSettings.rewardModelPath, path).toFile().bufferedReader().use { + Paths.get(UtSettings.modelPath, path).toFile().bufferedReader().use { val mean = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not mean in $path") val variance = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not variance in $path") StandardScaler(Matrix(mean), Matrix(variance)) diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt index 6a68e83212..47d873d099 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt @@ -25,7 +25,7 @@ class LinearStateRewardPredictorTest { @Test fun wrongFormatTest() { withRewardModelPath("src/test/resources") { - withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + withPathSelectorType(PathSelectorType.ML_SELECTOR) { LinearStateRewardPredictor("wrong_format_linear.txt") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt index 42dd3bac40..795793ab8e 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -30,15 +30,9 @@ class NNStateRewardPredictorTest { val averageTime = calcAverageTimeForModelPredict(::NNStateRewardPredictorBase, 100, features) println(averageTime) } - - - withRewardModelPath("models") { - val averageTime = calcAverageTimeForModelPredict(::StateRewardPredictorTorch, 100, features) - println(averageTime) - } } - private fun calcAverageTimeForModelPredict( + internal fun calcAverageTimeForModelPredict( model: () -> StateRewardPredictor, iterations: Int, features: List @@ -57,7 +51,7 @@ class NNStateRewardPredictorTest { @Test fun corruptedModelFileTest() { withRewardModelPath("src/test/resources") { - withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + withPathSelectorType(PathSelectorType.ML_SELECTOR) { NNStateRewardPredictorBase(modelPath = "corrupted_nn.json") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } @@ -67,7 +61,7 @@ class NNStateRewardPredictorTest { @Test fun emptyModelFileTest() { withRewardModelPath("src/test/resources") { - withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + withPathSelectorType(PathSelectorType.ML_SELECTOR) { NNStateRewardPredictorBase(modelPath = "empty_nn.json") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } @@ -77,7 +71,7 @@ class NNStateRewardPredictorTest { @Test fun corruptedScalerTest() { withRewardModelPath("src/test/resources") { - withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + withPathSelectorType(PathSelectorType.ML_SELECTOR) { NNStateRewardPredictorBase(scalerPath = "corrupted_scaler.txt") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index df9e6e3a58..2e085ddddf 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -88,9 +88,9 @@ object UtSettings : AbstractSettings( var pathSelectorType: PathSelectorType by getEnumProperty(PathSelectorType.INHERITORS_SELECTOR) /** - * Type of nnRewardGuidedSelector + * Type of MLSelector */ - var nnRewardGuidedSelectorType: NNRewardGuidedSelectorType by getEnumProperty(NNRewardGuidedSelectorType.WITHOUT_RECALCULATION) + var mlSelectorType: MLSelectorType by getEnumProperty(MLSelectorType.WITHOUT_RECALCULATION) /** * Type of [StateRewardPredictor] @@ -327,9 +327,9 @@ object UtSettings : AbstractSettings( var enableFeatureProcess by getBooleanProperty(false) /** - * Path to deserialized reward models + * Path to deserialized ML models */ - var rewardModelPath by getStringProperty("../models/0") + var modelPath by getStringProperty("../models/0") /** * Full class name of the class containing the configuration for the ML models to solve path selection task. @@ -405,9 +405,9 @@ enum class PathSelectorType { FORK_DEPTH_SELECTOR, /** - * [NNRewardGuidedSelector] + * [MLSelector] */ - NN_REWARD_GUIDED_SELECTOR, + ML_SELECTOR, /** * [RandomSelector] @@ -428,14 +428,14 @@ enum class TestSelectionStrategyType { /** * Enum to specify [NNRewardGuidedSelector], see implementations for more details */ -enum class NNRewardGuidedSelectorType { +enum class MLSelectorType { /** - * [NNRewardGuidedSelectorWithRecalculation] + * [MLSelectorWithRecalculation] */ WITH_RECALCULATION, /** - * [NNRewardGuidedSelectorWithoutRecalculation] + * [MLSelectorWithoutRecalculation] */ WITHOUT_RECALCULATION } @@ -445,7 +445,7 @@ enum class NNRewardGuidedSelectorType { */ enum class StateRewardPredictorType { /** - * [NNStateRewardPredictorBase] + * [MLStateRewardPredictorBase] */ BASE, @@ -455,7 +455,7 @@ enum class StateRewardPredictorType { TORCH, /** - * [NNStateRewardPredictorBase] + * [LinearStateRewardPredictorBase] */ LINEAR } diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt index b5624e8874..42febfcda7 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt @@ -4,7 +4,7 @@ import org.utbot.engine.InterProceduralUnitGraph import org.utbot.engine.selectors.NNRewardGuidedSelectorFactory import org.utbot.engine.selectors.NNRewardGuidedSelectorWithRecalculationFactory import org.utbot.engine.selectors.NNRewardGuidedSelectorWithoutRecalculationFactory -import org.utbot.framework.NNRewardGuidedSelectorType +import org.utbot.framework.MLSelectorType import org.utbot.framework.UtSettings /** @@ -23,9 +23,9 @@ object EngineAnalyticsContext { } } - val nnRewardGuidedSelectorFactory: NNRewardGuidedSelectorFactory = when (UtSettings.nnRewardGuidedSelectorType) { - NNRewardGuidedSelectorType.WITHOUT_RECALCULATION -> NNRewardGuidedSelectorWithoutRecalculationFactory() - NNRewardGuidedSelectorType.WITH_RECALCULATION -> NNRewardGuidedSelectorWithRecalculationFactory() + val nnRewardGuidedSelectorFactory: NNRewardGuidedSelectorFactory = when (UtSettings.mlSelectorType) { + MLSelectorType.WITHOUT_RECALCULATION -> NNRewardGuidedSelectorWithoutRecalculationFactory() + MLSelectorType.WITH_RECALCULATION -> NNRewardGuidedSelectorWithRecalculationFactory() } var stateRewardPredictorFactory: StateRewardPredictorFactory = object : StateRewardPredictorFactory { diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 4eb56f7ba2..6a63d884fe 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -137,7 +137,7 @@ private fun pathSelector(graph: InterProceduralUnitGraph, typeRegistry: TypeRegi PathSelectorType.FORK_DEPTH_SELECTOR -> forkDepthSelector(graph, StrategyOption.DISTANCE) { withStepsLimit(pathSelectorStepsLimit) } - PathSelectorType.NN_REWARD_GUIDED_SELECTOR -> nnRewardGuidedSelector(graph, StrategyOption.DISTANCE) { + PathSelectorType.ML_SELECTOR -> nnRewardGuidedSelector(graph, StrategyOption.DISTANCE) { withStepsLimit(pathSelectorStepsLimit) } PathSelectorType.RANDOM_SELECTOR -> randomSelector(graph, StrategyOption.DISTANCE) { diff --git a/utbot-framework/src/test/kotlin/org/utbot/examples/UtValueTestCaseChecker.kt b/utbot-framework/src/test/kotlin/org/utbot/examples/UtValueTestCaseChecker.kt index 6e31fd4d83..d14d1a5229 100644 --- a/utbot-framework/src/test/kotlin/org/utbot/examples/UtValueTestCaseChecker.kt +++ b/utbot-framework/src/test/kotlin/org/utbot/examples/UtValueTestCaseChecker.kt @@ -2847,12 +2847,12 @@ inline fun withTreatingOverflowAsError(block: () -> T): T { } inline fun withRewardModelPath(rewardModelPath: String, block: () -> T): T { - val prev = UtSettings.rewardModelPath - UtSettings.rewardModelPath = rewardModelPath + val prev = UtSettings.modelPath + UtSettings.modelPath = rewardModelPath try { return block() } finally { - UtSettings.rewardModelPath = prev + UtSettings.modelPath = prev } } diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt index 8311f8d20a..72a4b6e073 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt @@ -58,7 +58,6 @@ import org.utbot.framework.PathSelectorType import org.utbot.framework.plugin.services.WorkingDirService import org.utbot.intellij.plugin.settings.Settings import org.utbot.intellij.plugin.ui.utils.isBuildWithGradle -import org.utbot.intellij.plugin.ui.utils.suitableTestSourceRoots import org.utbot.intellij.plugin.util.PluginWorkingDirProvider import org.utbot.intellij.plugin.util.isAbstract import kotlin.reflect.KClass @@ -282,13 +281,13 @@ object UtTestsDialogProcessor { private fun configureML() { logger.info { "PathSelectorType: ${UtSettings.pathSelectorType}" } - if (UtSettings.pathSelectorType == PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR) { val analyticsConfigurationClassPath = UtSettings.analyticsConfigurationClassPath try { Class.forName(analyticsConfigurationClassPath) Predictors.stateRewardPredictor = EngineAnalyticsContext.stateRewardPredictorFactory() - logger.info { "RewardModelPath: ${UtSettings.rewardModelPath}" } + logger.info { "RewardModelPath: ${UtSettings.modelPath}" } } catch (e: ClassNotFoundException) { logger.error { "Configuration of the predictors from the utbot-analytics module described in the class: " + diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index eceac3d30f..b6bbf3e9a3 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -329,13 +329,13 @@ fun runEstimator( EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() EngineAnalyticsContext.stateRewardPredictorFactory = StateRewardPredictorFactoryImpl() - if (UtSettings.pathSelectorType == PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR) { Predictors.stateRewardPredictor = EngineAnalyticsContext.stateRewardPredictorFactory() } logger.info { "PathSelectorType: ${UtSettings.pathSelectorType}" } - if (UtSettings.pathSelectorType == PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { - logger.info { "RewardModelPath: ${UtSettings.rewardModelPath}" } + if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR) { + logger.info { "RewardModelPath: ${UtSettings.modelPath}" } } // fix for CTRL-ALT-SHIFT-C from IDEA, which copies in class#method form From ceda18a8f7f49bec3b7fe8290105e3a7cb82ef88 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 14:52:44 +0300 Subject: [PATCH 02/16] Just renamed some weird names --- .../utbot/analytics/EngineAnalyticsContext.kt | 12 ++++++------ ...NNRewardGuidedSelector.kt => MLSelector.kt} | 12 ++++++------ ...SelectorFactory.kt => MLSelectorFactory.kt} | 18 +++++++++--------- .../engine/selectors/PathSelectorBuilder.kt | 8 ++++---- 4 files changed, 25 insertions(+), 25 deletions(-) rename utbot-framework/src/main/kotlin/org/utbot/engine/selectors/{NNRewardGuidedSelector.kt => MLSelector.kt} (85%) rename utbot-framework/src/main/kotlin/org/utbot/engine/selectors/{NNRewardGuidedSelectorFactory.kt => MLSelectorFactory.kt} (68%) diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt index 42febfcda7..6ae1b8571d 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt @@ -1,9 +1,9 @@ package org.utbot.analytics import org.utbot.engine.InterProceduralUnitGraph -import org.utbot.engine.selectors.NNRewardGuidedSelectorFactory -import org.utbot.engine.selectors.NNRewardGuidedSelectorWithRecalculationFactory -import org.utbot.engine.selectors.NNRewardGuidedSelectorWithoutRecalculationFactory +import org.utbot.engine.selectors.MLSelectorFactory +import org.utbot.engine.selectors.MLSelectorWithRecalculationFactory +import org.utbot.engine.selectors.MLSelectorWithoutRecalculationFactory import org.utbot.framework.MLSelectorType import org.utbot.framework.UtSettings @@ -23,9 +23,9 @@ object EngineAnalyticsContext { } } - val nnRewardGuidedSelectorFactory: NNRewardGuidedSelectorFactory = when (UtSettings.mlSelectorType) { - MLSelectorType.WITHOUT_RECALCULATION -> NNRewardGuidedSelectorWithoutRecalculationFactory() - MLSelectorType.WITH_RECALCULATION -> NNRewardGuidedSelectorWithRecalculationFactory() + val mlSelectorFactory: MLSelectorFactory = when (UtSettings.mlSelectorType) { + MLSelectorType.WITHOUT_RECALCULATION -> MLSelectorWithoutRecalculationFactory() + MLSelectorType.WITH_RECALCULATION -> MLSelectorWithRecalculationFactory() } var stateRewardPredictorFactory: StateRewardPredictorFactory = object : StateRewardPredictorFactory { diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelector.kt similarity index 85% rename from utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt rename to utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelector.kt index c4f37c2924..1e4b17de0a 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelector.kt @@ -14,11 +14,11 @@ import org.utbot.engine.selectors.strategies.StoppingStrategy * * Calculates reward using neural network, when state is offered, and then peeks state with maximum reward * - * @see choosingStrategy [ChossingStrategy] for [GreedySearch] + * @see choosingStrategy [ChoosingStrategy] for [GreedySearch] * * [GreedySearch] */ -abstract class NNRewardGuidedSelector( +abstract class MLSelector( protected val generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, @@ -35,13 +35,13 @@ abstract class NNRewardGuidedSelector( * Calculate weight of execution state only when it is offered. It has advantage, because it works faster, * than with recalculation but disadvantage is that some features of execution state can change. */ -class NNRewardGuidedSelectorWithoutWeightsRecalculation( +class MLSelectorWithoutWeightsRecalculation( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int = 42, graph: InterProceduralUnitGraph -) : NNRewardGuidedSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { +) : MLSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { override fun offerImpl(state: ExecutionState) { super.offerImpl(state) featureExtractor.extractFeatures(state, generatedTestCountingStatistics.generatedTestsCount) @@ -58,13 +58,13 @@ class NNRewardGuidedSelectorWithoutWeightsRecalculation( * Calculate weight of execution state every time when it needed. It works slower, * than without recalculation but features are always relevant */ -class NNRewardGuidedSelectorWithWeightsRecalculation( +class MLSelectorWithWeightsRecalculation( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int = 42, graph: InterProceduralUnitGraph -) : NNRewardGuidedSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { +) : MLSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { override val ExecutionState.weight: Double get() { featureExtractor.extractFeatures(this, generatedTestCountingStatistics.generatedTestsCount) diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelectorFactory.kt similarity index 68% rename from utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt rename to utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelectorFactory.kt index cad9632381..881fab3f9c 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelectorFactory.kt @@ -6,44 +6,44 @@ import org.utbot.engine.selectors.strategies.GeneratedTestCountingStatistics import org.utbot.engine.selectors.strategies.StoppingStrategy /** - * Creates [NNRewardGuidedSelector] + * Creates [MLSelector] */ -interface NNRewardGuidedSelectorFactory { +interface MLSelectorFactory { operator fun invoke( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int = 42, graph: InterProceduralUnitGraph - ): NNRewardGuidedSelector + ): MLSelector } /** - * Creates [NNRewardGuidedSelectorWithWeightsRecalculation] + * Creates [MLSelectorWithWeightsRecalculation] */ -class NNRewardGuidedSelectorWithRecalculationFactory : NNRewardGuidedSelectorFactory { +class MLSelectorWithRecalculationFactory : MLSelectorFactory { override fun invoke( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int, graph: InterProceduralUnitGraph - ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithWeightsRecalculation( + ): MLSelector = MLSelectorWithWeightsRecalculation( generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph ) } /** - * Creates [NNRewardGuidedSelectorWithoutWeightsRecalculation] + * Creates [MLSelectorWithoutWeightsRecalculation] */ -class NNRewardGuidedSelectorWithoutRecalculationFactory : NNRewardGuidedSelectorFactory { +class MLSelectorWithoutRecalculationFactory : MLSelectorFactory { override fun invoke( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int, graph: InterProceduralUnitGraph - ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithoutWeightsRecalculation( + ): MLSelector = MLSelectorWithoutWeightsRecalculation( generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph ) } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt index 1d5fb681d8..f27b41a4a6 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt @@ -169,7 +169,7 @@ fun interleavedSelector(graph: InterProceduralUnitGraph, builder: InterleavedSel InterleavedSelectorBuilder(graph).apply(builder).build() /** - * build [NNRewardGuidedSelector] using [NNRewardGuidedSelectorBuilder] + * build [MLSelector] using [NNRewardGuidedSelectorBuilder] */ fun nnRewardGuidedSelector( graph: InterProceduralUnitGraph, @@ -525,15 +525,15 @@ class InterleavedSelectorBuilder internal constructor( } /** - * Builder for [NNRewardGuidedSelector]. Used in [] + * Builder for [MLSelector]. Used in [] */ class NNRewardGuidedSelectorBuilder internal constructor( graph: InterProceduralUnitGraph, private val strategy: StrategyOption, context: PathSelectorContext = PathSelectorContext(graph), -) : PathSelectorBuilder(graph, context) { +) : PathSelectorBuilder(graph, context) { private val seed = seedInPathSelector - override fun build() = EngineAnalyticsContext.nnRewardGuidedSelectorFactory( + override fun build() = EngineAnalyticsContext.mlSelectorFactory( withGeneratedTestCountingStatistics(), withChoosingStrategy(strategy), requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, From d41389239fdbbd47ce8aab7ccecf97aeaf9099dd Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 15:10:17 +0300 Subject: [PATCH 03/16] Removed unused settings --- .../predictors/StateRewardPredictorFactory.kt | 7 +--- .../predictors/StateRewardPredictorFactory.kt | 1 - .../kotlin/org/utbot/framework/UtSettings.kt | 15 +++++--- .../generator/UtTestsDialogProcessor.kt | 37 ++++++++++++------- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt index 71cc408515..46699f30ba 100644 --- a/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt +++ b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt @@ -1,16 +1,11 @@ package org.utbot.predictors import org.utbot.analytics.StateRewardPredictorFactory -import org.utbot.framework.StateRewardPredictorType import org.utbot.framework.UtSettings /** * Creates [StateRewardPredictor], by checking the [UtSettings] configuration. */ class StateRewardPredictorWithTorchModelsSupportFactoryImpl : StateRewardPredictorFactory { - override operator fun invoke() = when (UtSettings.stateRewardPredictorType) { - StateRewardPredictorType.BASE -> NNStateRewardPredictorBase() - StateRewardPredictorType.TORCH -> StateRewardPredictorTorch() - StateRewardPredictorType.LINEAR -> LinearStateRewardPredictor() - } + override operator fun invoke() = StateRewardPredictorTorch() } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt index 43504fee28..b84879f7d8 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt @@ -10,7 +10,6 @@ import org.utbot.framework.UtSettings class StateRewardPredictorFactoryImpl : StateRewardPredictorFactory { override operator fun invoke() = when (UtSettings.stateRewardPredictorType) { StateRewardPredictorType.BASE -> NNStateRewardPredictorBase() - StateRewardPredictorType.TORCH -> error("The torch model adapter is bundled with the utbot-analytics-torch module!") StateRewardPredictorType.LINEAR -> LinearStateRewardPredictor() } } \ No newline at end of file diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 2e085ddddf..26c85ccfe2 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -336,6 +336,11 @@ object UtSettings : AbstractSettings( */ var analyticsConfigurationClassPath by getStringProperty("org.utbot.AnalyticsConfiguration") + /** + * Full class name of the class containing the configuration for the ML models exported from the PyTorch to solve path selection task. + */ + var analyticsTorchConfigurationClassPath by getStringProperty("org.utbot.AnalyticsTorchConfiguration") + /** * Number of model iterations that will be used during ContestEstimator */ @@ -409,6 +414,11 @@ enum class PathSelectorType { */ ML_SELECTOR, + /** + * [TorchSelector] + */ + TORCH_SELECTOR, + /** * [RandomSelector] */ @@ -449,11 +459,6 @@ enum class StateRewardPredictorType { */ BASE, - /** - * [StateRewardPredictorTorch] - */ - TORCH, - /** * [LinearStateRewardPredictorBase] */ diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt index 72a4b6e073..f8ba40a425 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt @@ -283,22 +283,31 @@ object UtTestsDialogProcessor { if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR) { val analyticsConfigurationClassPath = UtSettings.analyticsConfigurationClassPath - try { - Class.forName(analyticsConfigurationClassPath) - Predictors.stateRewardPredictor = EngineAnalyticsContext.stateRewardPredictorFactory() - - logger.info { "RewardModelPath: ${UtSettings.modelPath}" } - } catch (e: ClassNotFoundException) { - logger.error { - "Configuration of the predictors from the utbot-analytics module described in the class: " + - "$analyticsConfigurationClassPath is not found!" - } + tryToSetUpMLSelector(analyticsConfigurationClassPath) + } - logger.info(e) { - "Error while initialization of ${UtSettings.pathSelectorType}. Changing pathSelectorType on INHERITORS_SELECTOR" - } - UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR + if (UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) { + val analyticsConfigurationClassPath = UtSettings.analyticsTorchConfigurationClassPath + tryToSetUpMLSelector(analyticsConfigurationClassPath) + } + } + + private fun tryToSetUpMLSelector(analyticsConfigurationClassPath: String) { + try { + Class.forName(analyticsConfigurationClassPath) + Predictors.stateRewardPredictor = EngineAnalyticsContext.stateRewardPredictorFactory() + + logger.info { "RewardModelPath: ${UtSettings.modelPath}" } + } catch (e: ClassNotFoundException) { + logger.error { + "Configuration of the predictors from the utbot-analytics module described in the class: " + + "$analyticsConfigurationClassPath is not found!" + } + + logger.info(e) { + "Error while initialization of ${UtSettings.pathSelectorType}. Changing pathSelectorType on INHERITORS_SELECTOR" } + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR } } From f2c89df4da7a24342eff85866d20abf22f293b89 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 15:31:23 +0300 Subject: [PATCH 04/16] Added minimal readme.md and fix some typos --- utbot-analytics-torch/readme.md | 5 +++++ .../src/main/kotlin/org/utbot/framework/UtSettings.kt | 2 +- .../kotlin/org/utbot/engine/UtBotSymbolicEngine.kt | 7 +++++-- .../org/utbot/engine/selectors/PathSelectorBuilder.kt | 10 +++++----- utbot-intellij/build.gradle | 2 +- .../main/kotlin/org/utbot/contest/ContestEstimator.kt | 4 ++-- 6 files changed, 19 insertions(+), 11 deletions(-) create mode 100644 utbot-analytics-torch/readme.md diff --git a/utbot-analytics-torch/readme.md b/utbot-analytics-torch/readme.md new file mode 100644 index 0000000000..9890bd4ff5 --- /dev/null +++ b/utbot-analytics-torch/readme.md @@ -0,0 +1,5 @@ +To enable support of the `utbot-analytics-torch` models in `utbot-intellij` module the following steps should be made: + +- change the row `api project(':utbot-analytics-torch')` to the `api project(':utbot-analytics-torch')` in the `build.gradle` file in the `utbot-intellij` module +- change the `pathSelectorType` in the `UtSettings.kt` to the `PathSelectorType.TORCH_SELECTOR` +- don't forget the put the Torch model in the path ruled by the setting `modelPath` in the `UtSettings.kt` \ No newline at end of file diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 26c85ccfe2..806a32ec6e 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -85,7 +85,7 @@ object UtSettings : AbstractSettings( /** * Type of path selector */ - var pathSelectorType: PathSelectorType by getEnumProperty(PathSelectorType.INHERITORS_SELECTOR) + var pathSelectorType: PathSelectorType by getEnumProperty(PathSelectorType.TORCH_SELECTOR) /** * Type of MLSelector diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index 6a63d884fe..5d10263a0e 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -36,7 +36,7 @@ import org.utbot.engine.selectors.coveredNewSelector import org.utbot.engine.selectors.cpInstSelector import org.utbot.engine.selectors.forkDepthSelector import org.utbot.engine.selectors.inheritorsSelector -import org.utbot.engine.selectors.nnRewardGuidedSelector +import org.utbot.engine.selectors.mlSelector import org.utbot.engine.selectors.nurs.NonUniformRandomSearch import org.utbot.engine.selectors.pollUntilFastSAT import org.utbot.engine.selectors.randomPathSelector @@ -137,7 +137,10 @@ private fun pathSelector(graph: InterProceduralUnitGraph, typeRegistry: TypeRegi PathSelectorType.FORK_DEPTH_SELECTOR -> forkDepthSelector(graph, StrategyOption.DISTANCE) { withStepsLimit(pathSelectorStepsLimit) } - PathSelectorType.ML_SELECTOR -> nnRewardGuidedSelector(graph, StrategyOption.DISTANCE) { + PathSelectorType.ML_SELECTOR -> mlSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.TORCH_SELECTOR -> mlSelector(graph, StrategyOption.DISTANCE) { withStepsLimit(pathSelectorStepsLimit) } PathSelectorType.RANDOM_SELECTOR -> randomSelector(graph, StrategyOption.DISTANCE) { diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt index f27b41a4a6..b529e6d456 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt @@ -169,13 +169,13 @@ fun interleavedSelector(graph: InterProceduralUnitGraph, builder: InterleavedSel InterleavedSelectorBuilder(graph).apply(builder).build() /** - * build [MLSelector] using [NNRewardGuidedSelectorBuilder] + * build [MLSelector] using [MLSelectorBuilder] */ -fun nnRewardGuidedSelector( +fun mlSelector( graph: InterProceduralUnitGraph, strategy: StrategyOption, - builder: NNRewardGuidedSelectorBuilder.() -> Unit -) = NNRewardGuidedSelectorBuilder(graph, strategy).apply(builder).build() + builder: MLSelectorBuilder.() -> Unit +) = MLSelectorBuilder(graph, strategy).apply(builder).build() data class PathSelectorContext( val graph: InterProceduralUnitGraph, @@ -527,7 +527,7 @@ class InterleavedSelectorBuilder internal constructor( /** * Builder for [MLSelector]. Used in [] */ -class NNRewardGuidedSelectorBuilder internal constructor( +class MLSelectorBuilder internal constructor( graph: InterProceduralUnitGraph, private val strategy: StrategyOption, context: PathSelectorContext = PathSelectorContext(graph), diff --git a/utbot-intellij/build.gradle b/utbot-intellij/build.gradle index 1c6cb973c2..dfe02043ed 100644 --- a/utbot-intellij/build.gradle +++ b/utbot-intellij/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation(project(":utbot-framework")) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation(project(":utbot-fuzzers")) - api project(':utbot-analytics') + api project(':utbot-analytics-torch') testImplementation 'org.mock-server:mockserver-netty:5.4.1' testImplementation(project(":utbot-sample")) testApi(project(":utbot-framework")) diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index b6bbf3e9a3..b0ba271e1c 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -329,12 +329,12 @@ fun runEstimator( EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() EngineAnalyticsContext.stateRewardPredictorFactory = StateRewardPredictorFactoryImpl() - if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR) { + if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR || UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) { Predictors.stateRewardPredictor = EngineAnalyticsContext.stateRewardPredictorFactory() } logger.info { "PathSelectorType: ${UtSettings.pathSelectorType}" } - if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR) { + if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR || UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) { logger.info { "RewardModelPath: ${UtSettings.modelPath}" } } From ef4d4be6c2fe102117c1612637bb7ede52730dde Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 16:47:19 +0300 Subject: [PATCH 05/16] Fixed some names --- .../main/kotlin/org/utbot/framework/UtSettings.kt | 12 ++++++------ .../org/utbot/analytics/EngineAnalyticsContext.kt | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 806a32ec6e..e716f9daaa 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -83,17 +83,17 @@ object UtSettings : AbstractSettings( var seedInPathSelector: Int? by getProperty(42, String::toInt) /** - * Type of path selector + * Type of path selector. */ var pathSelectorType: PathSelectorType by getEnumProperty(PathSelectorType.TORCH_SELECTOR) /** - * Type of MLSelector + * Type of MLSelector recalculation. */ - var mlSelectorType: MLSelectorType by getEnumProperty(MLSelectorType.WITHOUT_RECALCULATION) + var mlSelectorRecalculationType: MLSelectorRecalculationType by getEnumProperty(MLSelectorRecalculationType.WITHOUT_RECALCULATION) /** - * Type of [StateRewardPredictor] + * Type of [StateRewardPredictor]. */ var stateRewardPredictorType: StateRewardPredictorType by getEnumProperty(StateRewardPredictorType.BASE) @@ -436,9 +436,9 @@ enum class TestSelectionStrategyType { } /** - * Enum to specify [NNRewardGuidedSelector], see implementations for more details + * Enum to specify [MLSelector], see implementations for more details */ -enum class MLSelectorType { +enum class MLSelectorRecalculationType { /** * [MLSelectorWithRecalculation] */ diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt index 6ae1b8571d..1830f50328 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt @@ -4,7 +4,7 @@ import org.utbot.engine.InterProceduralUnitGraph import org.utbot.engine.selectors.MLSelectorFactory import org.utbot.engine.selectors.MLSelectorWithRecalculationFactory import org.utbot.engine.selectors.MLSelectorWithoutRecalculationFactory -import org.utbot.framework.MLSelectorType +import org.utbot.framework.MLSelectorRecalculationType import org.utbot.framework.UtSettings /** @@ -23,9 +23,9 @@ object EngineAnalyticsContext { } } - val mlSelectorFactory: MLSelectorFactory = when (UtSettings.mlSelectorType) { - MLSelectorType.WITHOUT_RECALCULATION -> MLSelectorWithoutRecalculationFactory() - MLSelectorType.WITH_RECALCULATION -> MLSelectorWithRecalculationFactory() + val mlSelectorFactory: MLSelectorFactory = when (UtSettings.mlSelectorRecalculationType) { + MLSelectorRecalculationType.WITHOUT_RECALCULATION -> MLSelectorWithoutRecalculationFactory() + MLSelectorRecalculationType.WITH_RECALCULATION -> MLSelectorWithRecalculationFactory() } var stateRewardPredictorFactory: StateRewardPredictorFactory = object : StateRewardPredictorFactory { From 2b0a2b89549e6612cebdcc20aeb32fb8889a56cf Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 17:34:37 +0300 Subject: [PATCH 06/16] Fixed some names --- ...nalyticsConfiguration.kt => AnalyticsTorchConfiguration.kt} | 2 +- .../utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) rename utbot-analytics-torch/src/main/kotlin/org/utbot/{TorchAnalyticsConfiguration.kt => AnalyticsTorchConfiguration.kt} (94%) diff --git a/utbot-analytics-torch/src/main/kotlin/org/utbot/TorchAnalyticsConfiguration.kt b/utbot-analytics-torch/src/main/kotlin/org/utbot/AnalyticsTorchConfiguration.kt similarity index 94% rename from utbot-analytics-torch/src/main/kotlin/org/utbot/TorchAnalyticsConfiguration.kt rename to utbot-analytics-torch/src/main/kotlin/org/utbot/AnalyticsTorchConfiguration.kt index 386329ab81..4e44875f36 100644 --- a/utbot-analytics-torch/src/main/kotlin/org/utbot/TorchAnalyticsConfiguration.kt +++ b/utbot-analytics-torch/src/main/kotlin/org/utbot/AnalyticsTorchConfiguration.kt @@ -12,7 +12,7 @@ import org.utbot.predictors.StateRewardPredictorWithTorchModelsSupportFactoryImp * @see * Issue: Enable utbot-analytics module in utbot-intellij module */ -object AnalyticsWithTorchModelsSupportConfiguration { +object AnalyticsTorchConfiguration { init { EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt index f8ba40a425..fce03415cf 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt @@ -309,6 +309,9 @@ object UtTestsDialogProcessor { } UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR } + catch (e: Exception) { + logger.error { e.message } + } } private fun errorMessage(className: String?, timeout: Long) = buildString { From 5af2f41d2cc9675c2c6afaf02c18cbdbb705db6d Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 18:24:43 +0300 Subject: [PATCH 07/16] Fixed gradle --- utbot-analytics-torch/build.gradle | 16 ++++++++-------- utbot-analytics/build.gradle | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/utbot-analytics-torch/build.gradle b/utbot-analytics-torch/build.gradle index c4f1ffeee6..32fe8d5da5 100644 --- a/utbot-analytics-torch/build.gradle +++ b/utbot-analytics-torch/build.gradle @@ -16,14 +16,14 @@ dependencies { testImplementation project(':utbot-sample') testImplementation group: 'junit', name: 'junit', version: junit4_version - implementation group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" - implementation group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" - implementation group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" - implementation group: 'org.jsoup', name: 'jsoup', version: jsoup_version - - implementation "ai.djl:api:$djl_api_version" - implementation "ai.djl.pytorch:pytorch-engine:$djl_api_version" - implementation "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" + api group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" + api group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" + api group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" + api group: 'org.jsoup', name: 'jsoup', version: jsoup_version + + api "ai.djl:api:$djl_api_version" + api "ai.djl.pytorch:pytorch-engine:$djl_api_version" + api "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" testImplementation project(':utbot-framework').sourceSets.test.output } diff --git a/utbot-analytics/build.gradle b/utbot-analytics/build.gradle index d4a51683e8..56d317e8d3 100644 --- a/utbot-analytics/build.gradle +++ b/utbot-analytics/build.gradle @@ -30,9 +30,9 @@ dependencies { implementation group: 'io.github.microutils', name: 'kotlin-logging', version: kotlin_logging_version implementation group: 'com.google.code.gson', name: 'gson', version: '2.8.6' - implementation group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" - implementation group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" - implementation group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" + api group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" + api group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" + api group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" implementation group: 'tech.tablesaw', name: 'tablesaw-core', version: '0.38.2' implementation group: 'tech.tablesaw', name: 'tablesaw-jsplot', version: '0.38.2' From 22b490453a0eb66b799b40b93a8fc4443e1ffcdc Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 19:18:49 +0300 Subject: [PATCH 08/16] Handle engine not found exception from DJL --- .../kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt | 3 ++- .../utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt index 2935b9f555..d0ec41a3da 100644 --- a/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt +++ b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt @@ -12,9 +12,10 @@ import java.io.Closeable import java.nio.file.Paths class StateRewardPredictorTorch : StateRewardPredictor, Closeable { - val model: Model = Model.newInstance("model") + val model: Model init { + model = Model.newInstance("model") model.load(Paths.get(UtSettings.modelPath, "model.pt1")) } diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt index fce03415cf..736f0bc754 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt @@ -311,6 +311,7 @@ object UtTestsDialogProcessor { } catch (e: Exception) { logger.error { e.message } + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR } } From 979b9f6972b9f0989750c23434c44b8817f60b11 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 19:22:57 +0300 Subject: [PATCH 09/16] Handle engine not found exception from DJL --- .../utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt index 736f0bc754..7f43f13f51 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt @@ -309,7 +309,7 @@ object UtTestsDialogProcessor { } UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR } - catch (e: Exception) { + catch (e: Exception) { // engine not found, for example logger.error { e.message } UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR } From ebfe04e17bdcbcc896701653b76e28b42602b783 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Tue, 30 Aug 2022 19:33:06 +0300 Subject: [PATCH 10/16] Handle engine not found exception from DJL --- .../utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt index 7f43f13f51..443b61451e 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt @@ -311,7 +311,7 @@ object UtTestsDialogProcessor { } catch (e: Exception) { // engine not found, for example logger.error { e.message } - UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR // TODO: it's not enough just switch on alternative selector, probably we need to reconfigure all the stuff EngineAnalyticsContext } } From e53b367067e7a91502475e62ceb2b6b28abe49dc Mon Sep 17 00:00:00 2001 From: amandelpie Date: Thu, 1 Sep 2022 10:48:33 +0300 Subject: [PATCH 11/16] Handle engine not found exception from DJL --- .../src/main/kotlin/org/utbot/framework/UtSettings.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index e716f9daaa..bb926aa785 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -85,7 +85,7 @@ object UtSettings : AbstractSettings( /** * Type of path selector. */ - var pathSelectorType: PathSelectorType by getEnumProperty(PathSelectorType.TORCH_SELECTOR) + var pathSelectorType: PathSelectorType by getEnumProperty(PathSelectorType.INHERITORS_SELECTOR) /** * Type of MLSelector recalculation. From 5e9e145492224b2578ae51581456071d108b15ae Mon Sep 17 00:00:00 2001 From: amandelpie Date: Thu, 1 Sep 2022 10:54:45 +0300 Subject: [PATCH 12/16] Changed the setter --- .../predictors/LinearStateRewardPredictorTest.kt | 8 ++++---- .../utbot/predictors/NNStateRewardPredictorTest.kt | 12 ++++++------ .../org/utbot/testcheckers/SettingsModificators.kt | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt index 4c32994b1e..f78c749322 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt @@ -5,12 +5,12 @@ import org.junit.jupiter.api.Test import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings import org.utbot.testcheckers.withPathSelectorType -import org.utbot.testcheckers.withRewardModelPath +import org.utbot.testcheckers.withModelPath class LinearStateRewardPredictorTest { @Test fun simpleTest() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { val pred = LinearStateRewardPredictor() val features = listOf( @@ -24,7 +24,7 @@ class LinearStateRewardPredictorTest { @Test fun wrongFormatTest() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { withPathSelectorType(PathSelectorType.ML_SELECTOR) { LinearStateRewardPredictor("wrong_format_linear.txt") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) @@ -34,7 +34,7 @@ class LinearStateRewardPredictorTest { @Test fun simpleTestNotBatch() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { val pred = LinearStateRewardPredictor() val features = listOf(2.0, 3.0) diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt index 72f0e63db6..eb2144bad0 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -7,13 +7,13 @@ import org.utbot.analytics.StateRewardPredictor import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings import org.utbot.testcheckers.withPathSelectorType -import org.utbot.testcheckers.withRewardModelPath +import org.utbot.testcheckers.withModelPath import kotlin.system.measureNanoTime class NNStateRewardPredictorTest { @Test fun simpleTest() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { val pred = NNStateRewardPredictorBase() val features = listOf(0.0, 0.0) @@ -26,7 +26,7 @@ class NNStateRewardPredictorTest { @Test fun performanceTest() { val features = (1..13).map { 1.0 }.toList() - withRewardModelPath("models\\test\\0") { + withModelPath("models\\test\\0") { val averageTime = calcAverageTimeForModelPredict(::NNStateRewardPredictorBase, 100, features) println(averageTime) } @@ -50,7 +50,7 @@ class NNStateRewardPredictorTest { @Test fun corruptedModelFileTest() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { withPathSelectorType(PathSelectorType.ML_SELECTOR) { NNStateRewardPredictorBase(modelPath = "corrupted_nn.json") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) @@ -60,7 +60,7 @@ class NNStateRewardPredictorTest { @Test fun emptyModelFileTest() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { withPathSelectorType(PathSelectorType.ML_SELECTOR) { NNStateRewardPredictorBase(modelPath = "empty_nn.json") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) @@ -70,7 +70,7 @@ class NNStateRewardPredictorTest { @Test fun corruptedScalerTest() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { withPathSelectorType(PathSelectorType.ML_SELECTOR) { NNStateRewardPredictorBase(scalerPath = "corrupted_scaler.txt") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/SettingsModificators.kt b/utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/SettingsModificators.kt index 8f44bfc96f..7bcd3bfce9 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/SettingsModificators.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/SettingsModificators.kt @@ -39,13 +39,13 @@ inline fun withPathSelectorType(pathSelectorType: PathSelectorType, } } -inline fun withRewardModelPath(rewardModelPath: String, block: () -> T): T { - val prev = UtSettings.rewardModelPath - UtSettings.rewardModelPath = rewardModelPath +inline fun withModelPath(modelPath: String, block: () -> T): T { + val prev = UtSettings.modelPath + UtSettings.modelPath = modelPath try { return block() } finally { - UtSettings.rewardModelPath = prev + UtSettings.modelPath = prev } } From c2c1d5bc4d8201d68f8582ce93d1d381b8dafbe7 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Thu, 1 Sep 2022 12:11:58 +0300 Subject: [PATCH 13/16] Removed the comment --- .../utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt index c59f9c1735..635620067d 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt @@ -333,7 +333,7 @@ object UtTestsDialogProcessor { } catch (e: Exception) { // engine not found, for example logger.error { e.message } - UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR // TODO: it's not enough just switch on alternative selector, probably we need to reconfigure all the stuff EngineAnalyticsContext + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR } } From ffd578b3560e74520bde01a85700360cc972a603 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Thu, 1 Sep 2022 12:15:39 +0300 Subject: [PATCH 14/16] Changed the visibility level for some dependencies --- utbot-analytics-torch/build.gradle | 16 ++++++++-------- utbot-analytics/build.gradle | 8 +++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/utbot-analytics-torch/build.gradle b/utbot-analytics-torch/build.gradle index 32fe8d5da5..c4f1ffeee6 100644 --- a/utbot-analytics-torch/build.gradle +++ b/utbot-analytics-torch/build.gradle @@ -16,14 +16,14 @@ dependencies { testImplementation project(':utbot-sample') testImplementation group: 'junit', name: 'junit', version: junit4_version - api group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" - api group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" - api group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" - api group: 'org.jsoup', name: 'jsoup', version: jsoup_version - - api "ai.djl:api:$djl_api_version" - api "ai.djl.pytorch:pytorch-engine:$djl_api_version" - api "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" + implementation group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" + implementation group: 'org.jsoup', name: 'jsoup', version: jsoup_version + + implementation "ai.djl:api:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-engine:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" testImplementation project(':utbot-framework').sourceSets.test.output } diff --git a/utbot-analytics/build.gradle b/utbot-analytics/build.gradle index 56d317e8d3..2dda46ebe1 100644 --- a/utbot-analytics/build.gradle +++ b/utbot-analytics/build.gradle @@ -30,22 +30,20 @@ dependencies { implementation group: 'io.github.microutils', name: 'kotlin-logging', version: kotlin_logging_version implementation group: 'com.google.code.gson', name: 'gson', version: '2.8.6' - api group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" - api group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" - api group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" implementation group: 'tech.tablesaw', name: 'tablesaw-core', version: '0.38.2' implementation group: 'tech.tablesaw', name: 'tablesaw-jsplot', version: '0.38.2' implementation group: 'org.apache.commons', name: 'commons-text', version: '1.9' - implementation group: 'com.github.javaparser', name: 'javaparser-core', version: '3.22.1' testImplementation project(':utbot-framework').sourceSets.test.output } test { - minHeapSize = "128m" maxHeapSize = "3072m" From a86191601f37918dda123cee05f83c812c24e76f Mon Sep 17 00:00:00 2001 From: amandelpie Date: Thu, 1 Sep 2022 12:44:58 +0300 Subject: [PATCH 15/16] Turn off utbot-analytics-torch module --- utbot-analytics-torch/readme.md | 7 ++++++- utbot-intellij/build.gradle | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/utbot-analytics-torch/readme.md b/utbot-analytics-torch/readme.md index 9890bd4ff5..8a43998d0e 100644 --- a/utbot-analytics-torch/readme.md +++ b/utbot-analytics-torch/readme.md @@ -2,4 +2,9 @@ To enable support of the `utbot-analytics-torch` models in `utbot-intellij` modu - change the row `api project(':utbot-analytics-torch')` to the `api project(':utbot-analytics-torch')` in the `build.gradle` file in the `utbot-intellij` module - change the `pathSelectorType` in the `UtSettings.kt` to the `PathSelectorType.TORCH_SELECTOR` -- don't forget the put the Torch model in the path ruled by the setting `modelPath` in the `UtSettings.kt` \ No newline at end of file +- don't forget the put the Torch model in the path ruled by the setting `modelPath` in the `UtSettings.kt` + +NOTE: for Windows you could obtain the error message related to the "engine not found problem" from DJL library during the Torch model initialization. +The proposed solution from DJL authors includes the installation of the [Microsoft Visual C++ Redistributable.](https://docs.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170) + +But at this moment it doesn't work on Windows at all. \ No newline at end of file diff --git a/utbot-intellij/build.gradle b/utbot-intellij/build.gradle index b28d103a60..df4cba5f9d 100644 --- a/utbot-intellij/build.gradle +++ b/utbot-intellij/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation(project(":utbot-framework")) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation(project(":utbot-fuzzers")) - api project(':utbot-analytics-torch') + implementation project(':utbot-analytics') testImplementation 'org.mock-server:mockserver-netty:5.4.1' testImplementation(project(":utbot-sample")) testApi(project(":utbot-framework")) From 44dc9d7b1fedffc83eecb1438ed5621dffe3c0e2 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Thu, 1 Sep 2022 13:12:20 +0300 Subject: [PATCH 16/16] Turn off utbot-analytics-torch module --- .../org/utbot/predictors/NNStateRewardPredictorTest.kt | 9 +++------ utbot-intellij/build.gradle | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt index af649db823..e1465aa9a6 100644 --- a/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt +++ b/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -3,18 +3,15 @@ package org.utbot.predictors import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test -import org.utbot.examples.withPathSelectorType import org.utbot.analytics.StateRewardPredictor -import org.utbot.examples.withRewardModelPath -import org.utbot.framework.PathSelectorType -import org.utbot.framework.UtSettings +import org.utbot.testcheckers.withModelPath import kotlin.system.measureNanoTime class NNStateRewardPredictorTest { @Test @Disabled("Just to see the performance of predictors") fun simpleTest() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { val pred = StateRewardPredictorTorch() val features = listOf(0.0, 0.0) @@ -27,7 +24,7 @@ class NNStateRewardPredictorTest { @Test fun performanceTest() { val features = (1..13).map { 1.0 }.toList() - withRewardModelPath("models") { + withModelPath("models") { val averageTime = calcAverageTimeForModelPredict(::StateRewardPredictorTorch, 100, features) println(averageTime) } diff --git a/utbot-intellij/build.gradle b/utbot-intellij/build.gradle index df4cba5f9d..1de4df3def 100644 --- a/utbot-intellij/build.gradle +++ b/utbot-intellij/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation(project(":utbot-framework")) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation(project(":utbot-fuzzers")) - implementation project(':utbot-analytics') + // implementation project(':utbot-analytics') // to reduce the plugin size testImplementation 'org.mock-server:mockserver-netty:5.4.1' testImplementation(project(":utbot-sample")) testApi(project(":utbot-framework"))