diff --git a/settings.gradle b/settings.gradle index c6ea552e41..a9919175f2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -18,6 +18,7 @@ include 'utbot-sample' include 'utbot-fuzzers' include 'utbot-junit-contest' include 'utbot-analytics' +include 'utbot-analytics-torch' include 'utbot-cli' include 'utbot-api' include 'utbot-instrumentation' diff --git a/utbot-analytics-torch/build.gradle b/utbot-analytics-torch/build.gradle new file mode 100644 index 0000000000..c4f1ffeee6 --- /dev/null +++ b/utbot-analytics-torch/build.gradle @@ -0,0 +1,63 @@ +apply from: "${parent.projectDir}/gradle/include/jvm-project.gradle" + +configurations { + torchmodels +} + +def osName = System.getProperty('os.name').toLowerCase().split()[0] +if (osName == "mac") osName = "macosx" +String classifier = osName + "-x86_64" + +evaluationDependsOn(':utbot-framework') +compileTestJava.dependsOn tasks.getByPath(':utbot-framework:testClasses') + +dependencies { + api project(':utbot-analytics') + testImplementation project(':utbot-sample') + testImplementation group: 'junit', name: 'junit', version: junit4_version + + implementation group: 'org.bytedeco', name: 'arpack-ng', version: "3.7.0-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'openblas', version: "0.3.10-1.5.4", classifier: "$classifier" + implementation group: 'org.bytedeco', name: 'javacpp', version: javacpp_version, classifier: "$classifier" + implementation group: 'org.jsoup', name: 'jsoup', version: jsoup_version + + implementation "ai.djl:api:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-engine:$djl_api_version" + implementation "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" + + testImplementation project(':utbot-framework').sourceSets.test.output +} + +test { + minHeapSize = "128m" + maxHeapSize = "3072m" + + jvmArgs '-XX:MaxHeapSize=3072m' + + useJUnitPlatform() { + excludeTags 'slow', 'IntegrationTest' + } +} + +processResources { + configurations.torchmodels.resolvedConfiguration.resolvedArtifacts.each { artifact -> + from(zipTree(artifact.getFile())) { + into "models" + } + } +} + +jar { + dependsOn classes + manifest { + attributes 'Main-Class': 'org.utbot.QualityAnalysisKt' + } + + dependsOn configurations.runtimeClasspath + from { + configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } + } + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + zip64 = true +} \ No newline at end of file diff --git a/utbot-analytics-torch/readme.md b/utbot-analytics-torch/readme.md new file mode 100644 index 0000000000..8a43998d0e --- /dev/null +++ b/utbot-analytics-torch/readme.md @@ -0,0 +1,10 @@ +To enable support of the `utbot-analytics-torch` models in `utbot-intellij` module the following steps should be made: + +- change the row `api project(':utbot-analytics-torch')` to the `api project(':utbot-analytics-torch')` in the `build.gradle` file in the `utbot-intellij` module +- change the `pathSelectorType` in the `UtSettings.kt` to the `PathSelectorType.TORCH_SELECTOR` +- don't forget the put the Torch model in the path ruled by the setting `modelPath` in the `UtSettings.kt` + +NOTE: for Windows you could obtain the error message related to the "engine not found problem" from DJL library during the Torch model initialization. +The proposed solution from DJL authors includes the installation of the [Microsoft Visual C++ Redistributable.](https://docs.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170) + +But at this moment it doesn't work on Windows at all. \ No newline at end of file diff --git a/utbot-analytics-torch/src/main/kotlin/org/utbot/AnalyticsTorchConfiguration.kt b/utbot-analytics-torch/src/main/kotlin/org/utbot/AnalyticsTorchConfiguration.kt new file mode 100644 index 0000000000..4e44875f36 --- /dev/null +++ b/utbot-analytics-torch/src/main/kotlin/org/utbot/AnalyticsTorchConfiguration.kt @@ -0,0 +1,21 @@ +package org.utbot + +import org.utbot.analytics.EngineAnalyticsContext +import org.utbot.features.FeatureExtractorFactoryImpl +import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory +import org.utbot.predictors.StateRewardPredictorWithTorchModelsSupportFactoryImpl + +/** + * The basic configuration of the utbot-analytics-torch module used in utbot-intellij and (as planned) in utbot-cli + * to implement the hidden configuration initialization to avoid direct calls of this configuration and usage of utbot-analytics-torch imports. + * + * @see + * Issue: Enable utbot-analytics module in utbot-intellij module + */ +object AnalyticsTorchConfiguration { + init { + EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() + EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() + EngineAnalyticsContext.stateRewardPredictorFactory = StateRewardPredictorWithTorchModelsSupportFactoryImpl() + } +} \ No newline at end of file diff --git a/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt new file mode 100644 index 0000000000..46699f30ba --- /dev/null +++ b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt @@ -0,0 +1,11 @@ +package org.utbot.predictors + +import org.utbot.analytics.StateRewardPredictorFactory +import org.utbot.framework.UtSettings + +/** + * Creates [StateRewardPredictor], by checking the [UtSettings] configuration. + */ +class StateRewardPredictorWithTorchModelsSupportFactoryImpl : StateRewardPredictorFactory { + override operator fun invoke() = StateRewardPredictorTorch() +} \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt similarity index 89% rename from utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt rename to utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt index f0cfe17571..d0ec41a3da 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt +++ b/utbot-analytics-torch/src/main/kotlin/org/utbot/predictors/StateRewardPredictorTorch.kt @@ -12,10 +12,11 @@ import java.io.Closeable import java.nio.file.Paths class StateRewardPredictorTorch : StateRewardPredictor, Closeable { - val model: Model = Model.newInstance("model") + val model: Model init { - model.load(Paths.get(UtSettings.rewardModelPath, "model.pt1")) + model = Model.newInstance("model") + model.load(Paths.get(UtSettings.modelPath, "model.pt1")) } private val predictor: Predictor, Float> = model.newPredictor(object : Translator, Float> { diff --git a/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt new file mode 100644 index 0000000000..e1465aa9a6 --- /dev/null +++ b/utbot-analytics-torch/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -0,0 +1,48 @@ +package org.utbot.predictors + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test +import org.utbot.analytics.StateRewardPredictor +import org.utbot.testcheckers.withModelPath +import kotlin.system.measureNanoTime + +class NNStateRewardPredictorTest { + @Test + @Disabled("Just to see the performance of predictors") + fun simpleTest() { + withModelPath("src/test/resources") { + val pred = StateRewardPredictorTorch() + + val features = listOf(0.0, 0.0) + + assertEquals(5.0, pred.predict(features)) + } + } + + @Disabled("Just to see the performance of predictors") + @Test + fun performanceTest() { + val features = (1..13).map { 1.0 }.toList() + withModelPath("models") { + val averageTime = calcAverageTimeForModelPredict(::StateRewardPredictorTorch, 100, features) + println(averageTime) + } + } + + private fun calcAverageTimeForModelPredict( + model: () -> StateRewardPredictor, + iterations: Int, + features: List + ): Double { + val pred = model() + + (1..iterations).map { + pred.predict(features) + } + + return (1..iterations) + .map { measureNanoTime { pred.predict(features) } } + .average() + } +} \ No newline at end of file diff --git a/utbot-analytics-torch/src/test/resources/log4j2.xml b/utbot-analytics-torch/src/test/resources/log4j2.xml new file mode 100644 index 0000000000..7dde3c2fea --- /dev/null +++ b/utbot-analytics-torch/src/test/resources/log4j2.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/utbot-analytics/build.gradle b/utbot-analytics/build.gradle index ba34f8081b..2dda46ebe1 100644 --- a/utbot-analytics/build.gradle +++ b/utbot-analytics/build.gradle @@ -12,13 +12,13 @@ evaluationDependsOn(':utbot-framework') compileTestJava.dependsOn tasks.getByPath(':utbot-framework:testClasses') dependencies { - implementation(project(":utbot-api")) - implementation(project(":utbot-core")) - implementation(project(":utbot-summary")) - implementation(project(":utbot-framework-api")) - implementation(project(":utbot-fuzzers")) - implementation(project(":utbot-instrumentation")) - implementation(project(":utbot-framework")) + api(project(":utbot-api")) + api(project(":utbot-core")) + api(project(":utbot-summary")) + api(project(":utbot-framework-api")) + api(project(":utbot-fuzzers")) + api(project(":utbot-instrumentation")) + api(project(":utbot-framework")) testImplementation project(':utbot-sample') testImplementation group: 'junit', name: 'junit', version: junit4_version @@ -38,20 +38,12 @@ dependencies { implementation group: 'tech.tablesaw', name: 'tablesaw-jsplot', version: '0.38.2' implementation group: 'org.apache.commons', name: 'commons-text', version: '1.9' - implementation group: 'com.github.javaparser', name: 'javaparser-core', version: '3.22.1' - implementation group: 'org.jsoup', name: 'jsoup', version: jsoup_version - - implementation "ai.djl:api:$djl_api_version" - implementation "ai.djl.pytorch:pytorch-engine:$djl_api_version" - implementation "ai.djl.pytorch:pytorch-native-auto:$pytorch_native_version" - testImplementation project(':utbot-framework').sourceSets.test.output } test { - minHeapSize = "128m" maxHeapSize = "3072m" diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt index 2d3dc434de..7c6cee6b25 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/LinearStateRewardPredictor.kt @@ -19,7 +19,7 @@ private val logger = KotlinLogging.logger {} * Last weight is bias */ private fun loadWeights(path: String): Matrix { - val weightsFile = File("${UtSettings.rewardModelPath}/${path}") + val weightsFile = File("${UtSettings.modelPath}/${path}") lateinit var weightsArray: DoubleArray try { diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt index d14034e64d..ed0066ec06 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/NNJson.kt @@ -33,7 +33,7 @@ data class NNJson( } internal fun loadModel(path: String): NNJson { - val modelFile = Paths.get(UtSettings.rewardModelPath, path).toFile() + val modelFile = Paths.get(UtSettings.modelPath, path).toFile() lateinit var nnJson: NNJson try { diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt index c0b7dffe7c..b84879f7d8 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/StateRewardPredictorFactory.kt @@ -10,7 +10,6 @@ import org.utbot.framework.UtSettings class StateRewardPredictorFactoryImpl : StateRewardPredictorFactory { override operator fun invoke() = when (UtSettings.stateRewardPredictorType) { StateRewardPredictorType.BASE -> NNStateRewardPredictorBase() - StateRewardPredictorType.TORCH -> StateRewardPredictorTorch() StateRewardPredictorType.LINEAR -> LinearStateRewardPredictor() } } \ No newline at end of file diff --git a/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt b/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt index b65e78d478..db38bc7743 100644 --- a/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt +++ b/utbot-analytics/src/main/kotlin/org/utbot/predictors/scalers.kt @@ -13,7 +13,7 @@ data class StandardScaler(val mean: Matrix?, val variance: Matrix?) internal fun loadScaler(path: String): StandardScaler = try { - Paths.get(UtSettings.rewardModelPath, path).toFile().bufferedReader().use { + Paths.get(UtSettings.modelPath, path).toFile().bufferedReader().use { val mean = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not mean in $path") val variance = it.readLine()?.splitByCommaIntoDoubleArray() ?: error("There is not variance in $path") StandardScaler(Matrix(mean), Matrix(variance)) diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt index e02f7e6c12..f78c749322 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/LinearStateRewardPredictorTest.kt @@ -5,12 +5,12 @@ import org.junit.jupiter.api.Test import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings import org.utbot.testcheckers.withPathSelectorType -import org.utbot.testcheckers.withRewardModelPath +import org.utbot.testcheckers.withModelPath class LinearStateRewardPredictorTest { @Test fun simpleTest() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { val pred = LinearStateRewardPredictor() val features = listOf( @@ -24,8 +24,8 @@ class LinearStateRewardPredictorTest { @Test fun wrongFormatTest() { - withRewardModelPath("src/test/resources") { - withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + withModelPath("src/test/resources") { + withPathSelectorType(PathSelectorType.ML_SELECTOR) { LinearStateRewardPredictor("wrong_format_linear.txt") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } @@ -34,7 +34,7 @@ class LinearStateRewardPredictorTest { @Test fun simpleTestNotBatch() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { val pred = LinearStateRewardPredictor() val features = listOf(2.0, 3.0) diff --git a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt index 3bb7a50d48..eb2144bad0 100644 --- a/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt +++ b/utbot-analytics/src/test/kotlin/org/utbot/predictors/NNStateRewardPredictorTest.kt @@ -7,13 +7,13 @@ import org.utbot.analytics.StateRewardPredictor import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings import org.utbot.testcheckers.withPathSelectorType -import org.utbot.testcheckers.withRewardModelPath +import org.utbot.testcheckers.withModelPath import kotlin.system.measureNanoTime class NNStateRewardPredictorTest { @Test fun simpleTest() { - withRewardModelPath("src/test/resources") { + withModelPath("src/test/resources") { val pred = NNStateRewardPredictorBase() val features = listOf(0.0, 0.0) @@ -26,19 +26,13 @@ class NNStateRewardPredictorTest { @Test fun performanceTest() { val features = (1..13).map { 1.0 }.toList() - withRewardModelPath("models\\test\\0") { + withModelPath("models\\test\\0") { val averageTime = calcAverageTimeForModelPredict(::NNStateRewardPredictorBase, 100, features) println(averageTime) } - - - withRewardModelPath("models") { - val averageTime = calcAverageTimeForModelPredict(::StateRewardPredictorTorch, 100, features) - println(averageTime) - } } - private fun calcAverageTimeForModelPredict( + internal fun calcAverageTimeForModelPredict( model: () -> StateRewardPredictor, iterations: Int, features: List @@ -56,8 +50,8 @@ class NNStateRewardPredictorTest { @Test fun corruptedModelFileTest() { - withRewardModelPath("src/test/resources") { - withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + withModelPath("src/test/resources") { + withPathSelectorType(PathSelectorType.ML_SELECTOR) { NNStateRewardPredictorBase(modelPath = "corrupted_nn.json") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } @@ -66,8 +60,8 @@ class NNStateRewardPredictorTest { @Test fun emptyModelFileTest() { - withRewardModelPath("src/test/resources") { - withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + withModelPath("src/test/resources") { + withPathSelectorType(PathSelectorType.ML_SELECTOR) { NNStateRewardPredictorBase(modelPath = "empty_nn.json") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } @@ -76,8 +70,8 @@ class NNStateRewardPredictorTest { @Test fun corruptedScalerTest() { - withRewardModelPath("src/test/resources") { - withPathSelectorType(PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + withModelPath("src/test/resources") { + withPathSelectorType(PathSelectorType.ML_SELECTOR) { NNStateRewardPredictorBase(scalerPath = "corrupted_scaler.txt") assertEquals(PathSelectorType.INHERITORS_SELECTOR, UtSettings.pathSelectorType) } diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index df9e6e3a58..bb926aa785 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -83,17 +83,17 @@ object UtSettings : AbstractSettings( var seedInPathSelector: Int? by getProperty(42, String::toInt) /** - * Type of path selector + * Type of path selector. */ var pathSelectorType: PathSelectorType by getEnumProperty(PathSelectorType.INHERITORS_SELECTOR) /** - * Type of nnRewardGuidedSelector + * Type of MLSelector recalculation. */ - var nnRewardGuidedSelectorType: NNRewardGuidedSelectorType by getEnumProperty(NNRewardGuidedSelectorType.WITHOUT_RECALCULATION) + var mlSelectorRecalculationType: MLSelectorRecalculationType by getEnumProperty(MLSelectorRecalculationType.WITHOUT_RECALCULATION) /** - * Type of [StateRewardPredictor] + * Type of [StateRewardPredictor]. */ var stateRewardPredictorType: StateRewardPredictorType by getEnumProperty(StateRewardPredictorType.BASE) @@ -327,15 +327,20 @@ object UtSettings : AbstractSettings( var enableFeatureProcess by getBooleanProperty(false) /** - * Path to deserialized reward models + * Path to deserialized ML models */ - var rewardModelPath by getStringProperty("../models/0") + var modelPath by getStringProperty("../models/0") /** * Full class name of the class containing the configuration for the ML models to solve path selection task. */ var analyticsConfigurationClassPath by getStringProperty("org.utbot.AnalyticsConfiguration") + /** + * Full class name of the class containing the configuration for the ML models exported from the PyTorch to solve path selection task. + */ + var analyticsTorchConfigurationClassPath by getStringProperty("org.utbot.AnalyticsTorchConfiguration") + /** * Number of model iterations that will be used during ContestEstimator */ @@ -405,9 +410,14 @@ enum class PathSelectorType { FORK_DEPTH_SELECTOR, /** - * [NNRewardGuidedSelector] + * [MLSelector] */ - NN_REWARD_GUIDED_SELECTOR, + ML_SELECTOR, + + /** + * [TorchSelector] + */ + TORCH_SELECTOR, /** * [RandomSelector] @@ -426,16 +436,16 @@ enum class TestSelectionStrategyType { } /** - * Enum to specify [NNRewardGuidedSelector], see implementations for more details + * Enum to specify [MLSelector], see implementations for more details */ -enum class NNRewardGuidedSelectorType { +enum class MLSelectorRecalculationType { /** - * [NNRewardGuidedSelectorWithRecalculation] + * [MLSelectorWithRecalculation] */ WITH_RECALCULATION, /** - * [NNRewardGuidedSelectorWithoutRecalculation] + * [MLSelectorWithoutRecalculation] */ WITHOUT_RECALCULATION } @@ -445,17 +455,12 @@ enum class NNRewardGuidedSelectorType { */ enum class StateRewardPredictorType { /** - * [NNStateRewardPredictorBase] + * [MLStateRewardPredictorBase] */ BASE, /** - * [StateRewardPredictorTorch] - */ - TORCH, - - /** - * [NNStateRewardPredictorBase] + * [LinearStateRewardPredictorBase] */ LINEAR } diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/SettingsModificators.kt b/utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/SettingsModificators.kt index 8f44bfc96f..7bcd3bfce9 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/SettingsModificators.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/testcheckers/SettingsModificators.kt @@ -39,13 +39,13 @@ inline fun withPathSelectorType(pathSelectorType: PathSelectorType, } } -inline fun withRewardModelPath(rewardModelPath: String, block: () -> T): T { - val prev = UtSettings.rewardModelPath - UtSettings.rewardModelPath = rewardModelPath +inline fun withModelPath(modelPath: String, block: () -> T): T { + val prev = UtSettings.modelPath + UtSettings.modelPath = modelPath try { return block() } finally { - UtSettings.rewardModelPath = prev + UtSettings.modelPath = prev } } diff --git a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt index b5624e8874..1830f50328 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/analytics/EngineAnalyticsContext.kt @@ -1,10 +1,10 @@ package org.utbot.analytics import org.utbot.engine.InterProceduralUnitGraph -import org.utbot.engine.selectors.NNRewardGuidedSelectorFactory -import org.utbot.engine.selectors.NNRewardGuidedSelectorWithRecalculationFactory -import org.utbot.engine.selectors.NNRewardGuidedSelectorWithoutRecalculationFactory -import org.utbot.framework.NNRewardGuidedSelectorType +import org.utbot.engine.selectors.MLSelectorFactory +import org.utbot.engine.selectors.MLSelectorWithRecalculationFactory +import org.utbot.engine.selectors.MLSelectorWithoutRecalculationFactory +import org.utbot.framework.MLSelectorRecalculationType import org.utbot.framework.UtSettings /** @@ -23,9 +23,9 @@ object EngineAnalyticsContext { } } - val nnRewardGuidedSelectorFactory: NNRewardGuidedSelectorFactory = when (UtSettings.nnRewardGuidedSelectorType) { - NNRewardGuidedSelectorType.WITHOUT_RECALCULATION -> NNRewardGuidedSelectorWithoutRecalculationFactory() - NNRewardGuidedSelectorType.WITH_RECALCULATION -> NNRewardGuidedSelectorWithRecalculationFactory() + val mlSelectorFactory: MLSelectorFactory = when (UtSettings.mlSelectorRecalculationType) { + MLSelectorRecalculationType.WITHOUT_RECALCULATION -> MLSelectorWithoutRecalculationFactory() + MLSelectorRecalculationType.WITH_RECALCULATION -> MLSelectorWithRecalculationFactory() } var stateRewardPredictorFactory: StateRewardPredictorFactory = object : StateRewardPredictorFactory { diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt index d344ed6c72..9a55432678 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt @@ -36,7 +36,7 @@ import org.utbot.engine.selectors.coveredNewSelector import org.utbot.engine.selectors.cpInstSelector import org.utbot.engine.selectors.forkDepthSelector import org.utbot.engine.selectors.inheritorsSelector -import org.utbot.engine.selectors.nnRewardGuidedSelector +import org.utbot.engine.selectors.mlSelector import org.utbot.engine.selectors.nurs.NonUniformRandomSearch import org.utbot.engine.selectors.pollUntilFastSAT import org.utbot.engine.selectors.randomPathSelector @@ -137,7 +137,10 @@ private fun pathSelector(graph: InterProceduralUnitGraph, typeRegistry: TypeRegi PathSelectorType.FORK_DEPTH_SELECTOR -> forkDepthSelector(graph, StrategyOption.DISTANCE) { withStepsLimit(pathSelectorStepsLimit) } - PathSelectorType.NN_REWARD_GUIDED_SELECTOR -> nnRewardGuidedSelector(graph, StrategyOption.DISTANCE) { + PathSelectorType.ML_SELECTOR -> mlSelector(graph, StrategyOption.DISTANCE) { + withStepsLimit(pathSelectorStepsLimit) + } + PathSelectorType.TORCH_SELECTOR -> mlSelector(graph, StrategyOption.DISTANCE) { withStepsLimit(pathSelectorStepsLimit) } PathSelectorType.RANDOM_SELECTOR -> randomSelector(graph, StrategyOption.DISTANCE) { diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelector.kt similarity index 85% rename from utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt rename to utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelector.kt index c4f37c2924..1e4b17de0a 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelector.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelector.kt @@ -14,11 +14,11 @@ import org.utbot.engine.selectors.strategies.StoppingStrategy * * Calculates reward using neural network, when state is offered, and then peeks state with maximum reward * - * @see choosingStrategy [ChossingStrategy] for [GreedySearch] + * @see choosingStrategy [ChoosingStrategy] for [GreedySearch] * * [GreedySearch] */ -abstract class NNRewardGuidedSelector( +abstract class MLSelector( protected val generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, @@ -35,13 +35,13 @@ abstract class NNRewardGuidedSelector( * Calculate weight of execution state only when it is offered. It has advantage, because it works faster, * than with recalculation but disadvantage is that some features of execution state can change. */ -class NNRewardGuidedSelectorWithoutWeightsRecalculation( +class MLSelectorWithoutWeightsRecalculation( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int = 42, graph: InterProceduralUnitGraph -) : NNRewardGuidedSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { +) : MLSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { override fun offerImpl(state: ExecutionState) { super.offerImpl(state) featureExtractor.extractFeatures(state, generatedTestCountingStatistics.generatedTestsCount) @@ -58,13 +58,13 @@ class NNRewardGuidedSelectorWithoutWeightsRecalculation( * Calculate weight of execution state every time when it needed. It works slower, * than without recalculation but features are always relevant */ -class NNRewardGuidedSelectorWithWeightsRecalculation( +class MLSelectorWithWeightsRecalculation( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int = 42, graph: InterProceduralUnitGraph -) : NNRewardGuidedSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { +) : MLSelector(generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph) { override val ExecutionState.weight: Double get() { featureExtractor.extractFeatures(this, generatedTestCountingStatistics.generatedTestsCount) diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelectorFactory.kt similarity index 68% rename from utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt rename to utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelectorFactory.kt index cad9632381..881fab3f9c 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/NNRewardGuidedSelectorFactory.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/MLSelectorFactory.kt @@ -6,44 +6,44 @@ import org.utbot.engine.selectors.strategies.GeneratedTestCountingStatistics import org.utbot.engine.selectors.strategies.StoppingStrategy /** - * Creates [NNRewardGuidedSelector] + * Creates [MLSelector] */ -interface NNRewardGuidedSelectorFactory { +interface MLSelectorFactory { operator fun invoke( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int = 42, graph: InterProceduralUnitGraph - ): NNRewardGuidedSelector + ): MLSelector } /** - * Creates [NNRewardGuidedSelectorWithWeightsRecalculation] + * Creates [MLSelectorWithWeightsRecalculation] */ -class NNRewardGuidedSelectorWithRecalculationFactory : NNRewardGuidedSelectorFactory { +class MLSelectorWithRecalculationFactory : MLSelectorFactory { override fun invoke( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int, graph: InterProceduralUnitGraph - ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithWeightsRecalculation( + ): MLSelector = MLSelectorWithWeightsRecalculation( generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph ) } /** - * Creates [NNRewardGuidedSelectorWithoutWeightsRecalculation] + * Creates [MLSelectorWithoutWeightsRecalculation] */ -class NNRewardGuidedSelectorWithoutRecalculationFactory : NNRewardGuidedSelectorFactory { +class MLSelectorWithoutRecalculationFactory : MLSelectorFactory { override fun invoke( generatedTestCountingStatistics: GeneratedTestCountingStatistics, choosingStrategy: ChoosingStrategy, stoppingStrategy: StoppingStrategy, seed: Int, graph: InterProceduralUnitGraph - ): NNRewardGuidedSelector = NNRewardGuidedSelectorWithoutWeightsRecalculation( + ): MLSelector = MLSelectorWithoutWeightsRecalculation( generatedTestCountingStatistics, choosingStrategy, stoppingStrategy, seed, graph ) } \ No newline at end of file diff --git a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt index 1d5fb681d8..b529e6d456 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/engine/selectors/PathSelectorBuilder.kt @@ -169,13 +169,13 @@ fun interleavedSelector(graph: InterProceduralUnitGraph, builder: InterleavedSel InterleavedSelectorBuilder(graph).apply(builder).build() /** - * build [NNRewardGuidedSelector] using [NNRewardGuidedSelectorBuilder] + * build [MLSelector] using [MLSelectorBuilder] */ -fun nnRewardGuidedSelector( +fun mlSelector( graph: InterProceduralUnitGraph, strategy: StrategyOption, - builder: NNRewardGuidedSelectorBuilder.() -> Unit -) = NNRewardGuidedSelectorBuilder(graph, strategy).apply(builder).build() + builder: MLSelectorBuilder.() -> Unit +) = MLSelectorBuilder(graph, strategy).apply(builder).build() data class PathSelectorContext( val graph: InterProceduralUnitGraph, @@ -525,15 +525,15 @@ class InterleavedSelectorBuilder internal constructor( } /** - * Builder for [NNRewardGuidedSelector]. Used in [] + * Builder for [MLSelector]. Used in [] */ -class NNRewardGuidedSelectorBuilder internal constructor( +class MLSelectorBuilder internal constructor( graph: InterProceduralUnitGraph, private val strategy: StrategyOption, context: PathSelectorContext = PathSelectorContext(graph), -) : PathSelectorBuilder(graph, context) { +) : PathSelectorBuilder(graph, context) { private val seed = seedInPathSelector - override fun build() = EngineAnalyticsContext.nnRewardGuidedSelectorFactory( + override fun build() = EngineAnalyticsContext.mlSelectorFactory( withGeneratedTestCountingStatistics(), withChoosingStrategy(strategy), requireNotNull(context.stoppingStrategy) { "StoppingStrategy isn't specified" }, diff --git a/utbot-intellij/build.gradle b/utbot-intellij/build.gradle index 1da449731b..1de4df3def 100644 --- a/utbot-intellij/build.gradle +++ b/utbot-intellij/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation(project(":utbot-framework")) { exclude group: 'org.slf4j', module: 'slf4j-api' } implementation(project(":utbot-fuzzers")) - api project(':utbot-analytics') + // implementation project(':utbot-analytics') // to reduce the plugin size testImplementation 'org.mock-server:mockserver-netty:5.4.1' testImplementation(project(":utbot-sample")) testApi(project(":utbot-framework")) diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt index eec8a7cbf8..635620067d 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt @@ -303,24 +303,37 @@ object UtTestsDialogProcessor { private fun configureML() { logger.info { "PathSelectorType: ${UtSettings.pathSelectorType}" } - if (UtSettings.pathSelectorType == PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR) { val analyticsConfigurationClassPath = UtSettings.analyticsConfigurationClassPath - try { - Class.forName(analyticsConfigurationClassPath) - Predictors.stateRewardPredictor = EngineAnalyticsContext.stateRewardPredictorFactory() - - logger.info { "RewardModelPath: ${UtSettings.rewardModelPath}" } - } catch (e: ClassNotFoundException) { - logger.error { - "Configuration of the predictors from the utbot-analytics module described in the class: " + - "$analyticsConfigurationClassPath is not found!" - } + tryToSetUpMLSelector(analyticsConfigurationClassPath) + } - logger.info(e) { - "Error while initialization of ${UtSettings.pathSelectorType}. Changing pathSelectorType on INHERITORS_SELECTOR" - } - UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR + if (UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) { + val analyticsConfigurationClassPath = UtSettings.analyticsTorchConfigurationClassPath + tryToSetUpMLSelector(analyticsConfigurationClassPath) + } + } + + private fun tryToSetUpMLSelector(analyticsConfigurationClassPath: String) { + try { + Class.forName(analyticsConfigurationClassPath) + Predictors.stateRewardPredictor = EngineAnalyticsContext.stateRewardPredictorFactory() + + logger.info { "RewardModelPath: ${UtSettings.modelPath}" } + } catch (e: ClassNotFoundException) { + logger.error { + "Configuration of the predictors from the utbot-analytics module described in the class: " + + "$analyticsConfigurationClassPath is not found!" } + + logger.info(e) { + "Error while initialization of ${UtSettings.pathSelectorType}. Changing pathSelectorType on INHERITORS_SELECTOR" + } + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR + } + catch (e: Exception) { // engine not found, for example + logger.error { e.message } + UtSettings.pathSelectorType = PathSelectorType.INHERITORS_SELECTOR } } diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index eceac3d30f..b0ba271e1c 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -329,13 +329,13 @@ fun runEstimator( EngineAnalyticsContext.featureProcessorFactory = FeatureProcessorWithStatesRepetitionFactory() EngineAnalyticsContext.featureExtractorFactory = FeatureExtractorFactoryImpl() EngineAnalyticsContext.stateRewardPredictorFactory = StateRewardPredictorFactoryImpl() - if (UtSettings.pathSelectorType == PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { + if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR || UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) { Predictors.stateRewardPredictor = EngineAnalyticsContext.stateRewardPredictorFactory() } logger.info { "PathSelectorType: ${UtSettings.pathSelectorType}" } - if (UtSettings.pathSelectorType == PathSelectorType.NN_REWARD_GUIDED_SELECTOR) { - logger.info { "RewardModelPath: ${UtSettings.rewardModelPath}" } + if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR || UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) { + logger.info { "RewardModelPath: ${UtSettings.modelPath}" } } // fix for CTRL-ALT-SHIFT-C from IDEA, which copies in class#method form