Introduce inter-process communication between utbot and utbot-spring-analyzer #2038 #2039 (#2085)

Author: IlyaMuravjov

.run/Debug All.run.xml

@@ -3,6 +3,7 @@
     <toRun name="Run IDEA" type="GradleRunConfiguration" />
     <toRun name="Listen for Engine Process" type="Remote" />
     <toRun name="Listen for Instrumented Process" type="Remote" />
+    <toRun name="Listen for Spring Analyzer Process" type="Remote" />
     <method v="2" />
   </configuration>
 </component>

.run/Debug Spring Analyzer Process.run.xml

@@ -0,0 +1,7 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Debug Spring Analyzer Process" type="CompoundRunConfigurationType">
+    <toRun name="Run IDEA" type="GradleRunConfiguration" />
+    <toRun name="Listen for Spring Analyzer Process" type="Remote" />
+    <method v="2" />
+  </configuration>
+</component>

.run/Listen for Spring Analyzer Process.run.xml

@@ -0,0 +1,15 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Listen for Spring Analyzer Process" type="Remote" folderName="Utility Configurations">
+    <option name="USE_SOCKET_TRANSPORT" value="true" />
+    <option name="SERVER_MODE" value="true" />
+    <option name="SHMEM_ADDRESS" />
+    <option name="HOST" value="localhost" />
+    <option name="PORT" value="5007" />
+    <option name="AUTO_RESTART" value="true" />
+    <RunnerSettings RunnerId="Debug">
+      <option name="DEBUG_PORT" value="5007" />
+      <option name="LOCAL" value="false" />
+    </RunnerSettings>
+    <method v="2" />
+  </configuration>
+</component>

build.gradle.kts

@@ -19,11 +19,6 @@ plugins {
     `maven-publish`
 }
 
-configure<JavaPluginExtension> {
-    sourceCompatibility = VERSION_11
-    targetCompatibility = VERSION_17
-}
-
 allprojects {
 
     apply {

gradle.properties

@@ -77,6 +77,9 @@ pytorchNativeVersion=1.9.1
 shadowJarVersion=7.1.2
 openblasVersion=0.3.10-1.5.4
 arpackNgVersion=3.7.0-1.5.4
+commonsLoggingVersion=1.2
+commonsIOVersion=2.11.0
+springBootVersion=2.7.8
 
 # configuration for build server
 #

utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt

@@ -283,7 +283,7 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS
     var runEngineProcessWithDebug by getBooleanProperty(false)
 
     /**
-     * The engine process JDWP agent's port of the instrumented process.
+     * The engine process JDWP agent's port of the engine process.
      * A debugger attaches to the port in order to debug the process.
      */
     var engineProcessDebugPort by getIntProperty(5005)
@@ -296,6 +296,29 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS
 
 // endregion
 
+// region spring analyzer process debug
+    /**
+     * The property is useful only for the IntelliJ IDEs.
+     * If the property is set in true the spring analyzer process opens a debug port.
+     * @see runInstrumentedProcessWithDebug
+     * @see org.utbot.framework.process.SpringAnalyzerProcess
+     */
+    var runSpringAnalyzerProcessWithDebug by getBooleanProperty(false)
+
+    /**
+     * The spring analyzer process JDWP agent's port.
+     * A debugger attaches to the port in order to debug the process.
+     */
+    var springAnalyzerProcessDebugPort by getIntProperty(5007)
+
+    /**
+     * Value of the suspend mode for the JDWP agent of the spring analyzer process.
+     * If the value is true, the spring analyzer process will suspend until a debugger attaches to it.
+     */
+    var suspendSpringAnalyzerProcessExecutionInDebugMode by getBooleanProperty(true)
+
+// endregion
+
 // region instrumented process debug
     /**
      * The instrumented process JDWP agent's port of the instrumented process.
@@ -312,15 +335,16 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS
     /**
      * If true, runs the instrumented process with the ability to attach a debugger.
      *
-     * To debug the instrumented process, set the breakpoint in the instrumentedProcessRunner.start() line
+     * To debug the instrumented process, set the breakpoint in the
+     * [org.utbot.instrumentation.rd.InstrumentedProcess.Companion.invoke]
      * and in the instrumented process's main function and run the main process.
      * Then run the remote JVM debug configuration in IDEA.
      * If you see the message in console about successful connection, then
      * the debugger is attached successfully.
      * Now you can put the breakpoints in the instrumented process and debug
      * both processes simultaneously.
      *
-     * @see [org.utbot.instrumentation.process.InstrumentedProcessRunner.cmds]
+     * @see [org.utbot.instrumentation.rd.InstrumentedProcess.Companion.invoke]
      */
     var runInstrumentedProcessWithDebug by getBooleanProperty(false)
 // endregion

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/services/JdkInfoService.kt

@@ -11,7 +11,7 @@ data class JdkInfo(
 /**
  * Singleton to enable abstract access to path to JDK.
 
- * Used in [org.utbot.instrumentation.process.InstrumentedProcessRunner].
+ * Used in [org.utbot.framework.process.AbstractRDProcessCompanion].
  * The purpose is to use the same JDK in [org.utbot.instrumentation.ConcreteExecutor] and in the test runs.
  * This is necessary because the engine can be run from the various starting points, like IDEA plugin, CLI, etc.
  */

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/services/WorkingDirService.kt

@@ -6,7 +6,7 @@ import java.nio.file.Paths
 /**
  * Singleton to enable abstract access to the working directory.
  *
- * Used in [org.utbot.instrumentation.process.InstrumentedProcessRunner].
+ * Used in [org.utbot.instrumentation.rd.InstrumentedProcess].
  * The purpose is to use the same working directory in [org.utbot.instrumentation.ConcreteExecutor]
  * and in the test runs.
  */

utbot-framework-api/src/main/kotlin/org/utbot/framework/process/AbstractRDProcessCompanion.kt

@@ -0,0 +1,38 @@
+package org.utbot.framework.process
+
+import org.utbot.common.osSpecificJavaExecutable
+import org.utbot.framework.plugin.services.JdkInfoService
+import org.utbot.rd.rdPortArgument
+import java.io.File
+import kotlin.io.path.pathString
+
+abstract class AbstractRDProcessCompanion(
+    private val debugPort: Int,
+    private val runWithDebug: Boolean,
+    private val suspendExecutionInDebugMode: Boolean
+) {
+    private val javaExecutablePathString get() =
+        JdkInfoService.provide().path.resolve("bin${File.separatorChar}${osSpecificJavaExecutable()}")
+
+    protected abstract fun obtainProcessSpecificCommandLineArgs(): List<String>
+
+    protected fun obtainProcessCommandLine(port: Int): List<String> = buildList {
+        addAll(obtainCommonProcessCommandLineArgs())
+        addAll(obtainProcessSpecificCommandLineArgs())
+        add(rdPortArgument(port))
+    }
+
+    private fun obtainCommonProcessCommandLineArgs(): List<String> = buildList {
+        val suspendValue = if (suspendExecutionInDebugMode) "y" else "n"
+        val debugArgument =
+            "-agentlib:jdwp=transport=dt_socket,server=n,suspend=${suspendValue},quiet=y,address=$debugPort"
+                .takeIf { runWithDebug }
+
+        add(javaExecutablePathString.pathString)
+        val javaVersionSpecificArgs = OpenModulesContainer.javaVersionSpecificArguments
+        if (javaVersionSpecificArgs.isNotEmpty()) {
+            addAll(javaVersionSpecificArgs)
+        }
+        debugArgument?.let { add(it) }
+    }
+}

utbot-framework-api/src/main/kotlin/org/utbot/framework/process/OpenModulesContainer.kt

@@ -11,6 +11,7 @@ object OpenModulesContainer {
     init {
         modulesContainer = buildList {
             openPackage("java.base", "sun.security.util")
+            openPackage("java.base", "sun.reflect.annotation")
             openPackage("java.base", "java.text")
             openPackage("java.base", "java.lang.invoke")
             openPackage("java.base", "jdk.internal.misc")

utbot-framework/build.gradle

@@ -1,3 +1,7 @@
+configurations {
+    fetchSpringAnalyzerJar
+}
+
 dependencies {
 
     api project(':utbot-fuzzers')
@@ -34,4 +38,13 @@ dependencies {
 
     implementation group: 'com.github.UnitTestBot.ksmt', name: 'ksmt-core', version: ksmtVersion
     implementation group: 'com.github.UnitTestBot.ksmt', name: 'ksmt-z3', version: ksmtVersion
+    implementation project(':utbot-spring-analyzer')
+
+    fetchSpringAnalyzerJar project(path: ':utbot-spring-analyzer', configuration: 'springAnalyzerJar')
+}
+
+processResources {
+    from(configurations.fetchSpringAnalyzerJar) {
+        into "lib"
+    }
 }

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/Domain.kt

@@ -3,7 +3,6 @@ package org.utbot.framework.codegen.domain
 import org.utbot.framework.DEFAULT_EXECUTION_TIMEOUT_IN_INSTRUMENTED_PROCESS_MS
 import org.utbot.framework.codegen.domain.builtin.mockitoClassId
 import org.utbot.framework.codegen.domain.builtin.ongoingStubbingClassId
-import org.utbot.framework.codegen.domain.context.CgContext
 import org.utbot.framework.codegen.domain.models.CgClassId
 import org.utbot.framework.codegen.tree.argumentsClassId
 import org.utbot.framework.plugin.api.BuiltinClassId
@@ -751,7 +750,7 @@ sealed class TypeReplacementApproach {
      *
      * Currently used in Spring applications only.
      */
-    class ReplaceIfPossible(val configFqn: String) : TypeReplacementApproach()
+    class ReplaceIfPossible(val config: String) : TypeReplacementApproach()
 }
 
 abstract class DependencyInjectionFramework(

utbot-framework/src/main/kotlin/org/utbot/framework/process/EngineProcessMain.kt

@@ -34,8 +34,10 @@ import org.utbot.rd.RdSettingsContainerFactory
 import org.utbot.rd.findRdPort
 import org.utbot.rd.generated.settingsModel
 import org.utbot.rd.loggers.UtRdKLoggerFactory
+import org.utbot.rd.terminateOnException
 import org.utbot.sarif.RdSourceFindingStrategyFacade
 import org.utbot.sarif.SarifReport
+import org.utbot.spring.process.SpringAnalyzerProcess
 import org.utbot.summary.summarizeAll
 import java.io.File
 import java.net.URLClassLoader
@@ -51,16 +53,11 @@ object EngineProcessMain
 
 // use log4j2.configurationFile property to set log4j configuration
 suspend fun main(args: Array<String>) = runBlocking {
-    Logger.set(Lifetime.Eternal, UtRdKLoggerFactory(logger))
-
     logger.info("-----------------------------------------------------------------------")
     logger.info("-------------------NEW ENGINE PROCESS STARTED--------------------------")
     logger.info("-----------------------------------------------------------------------")
-    // 0 - auto port for server, should not be used here
-    val port = findRdPort(args)
-
 
-    ClientProtocolBuilder().withProtocolTimeout(messageFromMainTimeoutMillis).start(port) {
+    ClientProtocolBuilder().withProtocolTimeout(messageFromMainTimeoutMillis).start(args) {
         AbstractSettings.setupFactory(RdSettingsContainerFactory(protocol.settingsModel))
         val kryoHelper = KryoHelper(lifetime)
         engineProcessModel.setup(kryoHelper, it, protocol)
@@ -79,6 +76,21 @@ private fun EngineProcessModel.setup(kryoHelper: KryoHelper, watchdog: IdleWatch
             File(it).toURI().toURL()
         }.toTypedArray())))
     }
+    watchdog.measureTimeForActiveCall(getSpringBeanQualifiedNames, "Getting Spring bean definitions") { params ->
+        val springAnalyzerProcess = SpringAnalyzerProcess.createBlocking()
+        val beans = springAnalyzerProcess.terminateOnException { _ ->
+            springAnalyzerProcess.getBeanQualifiedNames(
+                params.classpath.toList(),
+                params.config,
+                // TODO remove once spring-analyzer learns to find resources on its own, temporarily leaving it here for testing with hardcoded absolute paths
+                propertyFilesPaths = emptyList(),
+                xmlConfigurationPaths = emptyList(),
+                params.useSpringAnalyzer
+            ).toTypedArray()
+        }
+        springAnalyzerProcess.terminate()
+        beans
+    }
     watchdog.measureTimeForActiveCall(createTestGenerator, "Creating Test Generator") { params ->
         AnalyticsConfigureUtil.configureML()
         Instrumenter.adapter = RdInstrumenter(realProtocol.rdInstrumenterAdapter)