Skip to content

Commit cac84c1

Browse files
nikitavlaevnikitavlaev
committed
Test report reworked (#522)
Removed tsv. Sarif and detailed statistics appears only in event log. Bubble with warnings is now of warning type.
1 parent 213555c commit cac84c1

File tree

6 files changed

+132
-138
lines changed

6 files changed

+132
-138
lines changed

‎utbot-cli/src/main/kotlin/org/utbot/cli/GenerateTestsCommand.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ class GenerateTestsCommand :
145145
SourceFindingStrategyDefault(classFqn, sourceCodeFile, testsFilePath, projectRootPath)
146146
val report = SarifReport(testCases, testClassBody, sourceFinding).createReport()
147147
saveToFile(report, sarifReport)
148-
println("The report was saved to \"$sarifReport\". You can open it using the VS Code extension \"Sarif Viewer\".")
148+
println("The report was saved to \"$sarifReport\".")
149149
}
150150
}
151151
} catch (t: Throwable) {

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt

Lines changed: 33 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -189,9 +189,29 @@ data class TestsGenerationReport(
189189
get() = executables.firstOrNull()?.clazz
190190
?: error("No executables found in test report")
191191

192-
// Summary message is generated lazily to avoid evaluation of classUnderTest
193-
var summaryMessage: () -> String = { "Unit tests for $classUnderTest were generated successfully." }
194192
val initialWarnings: MutableList<() -> String> = mutableListOf()
193+
val hasWarnings: Boolean
194+
get() = initialWarnings.isNotEmpty()
195+
196+
val detailedStatistics: String
197+
get() = buildString {
198+
appendHtmlLine("Class: ${classUnderTest.qualifiedName}")
199+
val testMethodsStatistic = executables.map { it.countTestMethods() }
200+
val errors = executables.map { it.countErrors() }
201+
val overallErrors = errors.sum()
202+
203+
appendHtmlLine("Successful test methods: ${testMethodsStatistic.sumBy { it.successful }}")
204+
appendHtmlLine(
205+
"Failing because of unexpected exception test methods: ${testMethodsStatistic.sumBy { it.failing }}"
206+
)
207+
appendHtmlLine(
208+
"Failing because of exceeding timeout test methods: ${testMethodsStatistic.sumBy { it.timeout }}"
209+
)
210+
appendHtmlLine(
211+
"Failing because of possible JVM crash test methods: ${testMethodsStatistic.sumBy { it.crashes }}"
212+
)
213+
appendHtmlLine("Not generated because of internal errors test methods: $overallErrors")
214+
}
195215

196216
fun addMethodErrors(testCase: UtTestCase, errors: Map<String, Int>) {
197217
this.errors[testCase.method] = errors
@@ -216,61 +236,24 @@ data class TestsGenerationReport(
216236
}
217237
}
218238

219-
override fun toString(): String = buildString {
220-
appendHtmlLine(summaryMessage())
221-
appendHtmlLine()
222-
initialWarnings.forEach { appendHtmlLine(it()) }
223-
appendHtmlLine()
239+
fun toString(isShort: Boolean): String = buildString {
240+
appendHtmlLine("Target: ${classUnderTest.qualifiedName}")
241+
if (initialWarnings.isNotEmpty()) {
242+
initialWarnings.forEach { appendHtmlLine(it()) }
243+
appendHtmlLine()
244+
}
224245

225246
val testMethodsStatistic = executables.map { it.countTestMethods() }
226-
val errors = executables.map { it.countErrors() }
227247
val overallTestMethods = testMethodsStatistic.sumBy { it.count }
228-
val overallErrors = errors.sum()
248+
229249
appendHtmlLine("Overall test methods: $overallTestMethods")
230-
appendHtmlLine("Successful test methods: ${testMethodsStatistic.sumBy { it.successful }}")
231-
appendHtmlLine(
232-
"Failing because of unexpected exception test methods: ${testMethodsStatistic.sumBy { it.failing }}"
233-
)
234-
appendHtmlLine(
235-
"Failing because of exceeding timeout test methods: ${testMethodsStatistic.sumBy { it.timeout }}"
236-
)
237-
appendHtmlLine(
238-
"Failing because of possible JVM crash test methods: ${testMethodsStatistic.sumBy { it.crashes }}"
239-
)
240-
appendHtmlLine("Not generated because of internal errors test methods: $overallErrors")
241-
}
242250

243-
// TODO: should we use TsvWriter from univocity instead of this manual implementation?
244-
fun getFileContent(): String =
245-
(listOf(getHeader()) + getLines()).joinToString(System.lineSeparator())
246-
247-
private fun getHeader(): String {
248-
val columnNames = listOf(
249-
"Executable/Number of test methods",
250-
SUCCESSFUL,
251-
FAILING,
252-
TIMEOUT,
253-
CRASH,
254-
"Errors tests"
255-
)
256-
257-
return columnNames.joinToString(TAB_SEPARATOR)
251+
if (!isShort) {
252+
appendHtmlLine(detailedStatistics)
253+
}
258254
}
259255

260-
private fun getLines(): List<String> =
261-
executables.map { executable ->
262-
val testMethodStatistic = executable.countTestMethods()
263-
with(testMethodStatistic) {
264-
listOf(
265-
executable,
266-
successful,
267-
failing,
268-
timeout,
269-
crashes,
270-
executable.countErrors()
271-
).joinToString(TAB_SEPARATOR)
272-
}
273-
}
256+
override fun toString(): String = toString(false)
274257

275258
private fun UtMethod<*>.countTestMethods(): TestMethodStatistic = TestMethodStatistic(
276259
testMethodsNumber(successfulExecutions),
@@ -291,9 +274,4 @@ data class TestsGenerationReport(
291274
private data class TestMethodStatistic(val successful: Int, val failing: Int, val timeout: Int, val crashes: Int) {
292275
val count: Int = successful + failing + timeout + crashes
293276
}
294-
295-
companion object {
296-
private const val TAB_SEPARATOR: String = "\t"
297-
const val EXTENSION: String = ".tsv"
298-
}
299277
}

‎utbot-framework/src/main/kotlin/org/utbot/framework/plugin/sarif/GenerateTestsAndSarifReportFacade.kt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@ class GenerateTestsAndSarifReportFacade(
5353
mergedSarifReportFile.writeText(mergedReport)
5454
if (verbose) {
5555
println("SARIF report was saved to \"${mergedSarifReportFile.path}\"")
56-
println("You can open it using the VS Code extension \"Sarif Viewer\"")
5756
}
5857
}
5958
}

‎utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/CodeGenerationController.kt

Lines changed: 14 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import com.intellij.openapi.editor.Editor
1515
import com.intellij.openapi.project.DumbService
1616
import com.intellij.openapi.project.Project
1717
import com.intellij.openapi.util.Computable
18-
import com.intellij.openapi.vfs.VfsUtil
1918
import com.intellij.psi.JavaDirectoryService
2019
import com.intellij.psi.PsiClass
2120
import com.intellij.psi.PsiClassOwner
@@ -47,7 +46,6 @@ import org.jetbrains.kotlin.psi.psiUtil.endOffset
4746
import org.jetbrains.kotlin.psi.psiUtil.startOffset
4847
import org.jetbrains.kotlin.scripting.resolve.classId
4948
import org.utbot.common.HTML_LINE_SEPARATOR
50-
import org.utbot.common.PathUtil.classFqnToPath
5149
import org.utbot.common.PathUtil.toHtmlLinkTag
5250
import org.utbot.common.appendHtmlLine
5351
import org.utbot.framework.codegen.Import
@@ -56,7 +54,6 @@ import org.utbot.framework.codegen.RegularImport
5654
import org.utbot.framework.codegen.StaticImport
5755
import org.utbot.framework.codegen.model.CodeGenerator
5856
import org.utbot.framework.codegen.model.TestsCodeWithTestReport
59-
import org.utbot.framework.codegen.model.constructor.tree.TestsGenerationReport
6057
import org.utbot.framework.plugin.api.CodegenLanguage
6158
import org.utbot.framework.plugin.api.UtMethod
6259
import org.utbot.framework.plugin.api.UtTestCase
@@ -70,16 +67,17 @@ import org.utbot.intellij.plugin.models.GenerateTestsModel
7067
import org.utbot.intellij.plugin.models.packageName
7168
import org.utbot.intellij.plugin.sarif.SarifReportIdea
7269
import org.utbot.intellij.plugin.sarif.SourceFindingStrategyIdea
70+
import org.utbot.intellij.plugin.ui.DetailsTestsReportNotifier
7371
import org.utbot.intellij.plugin.ui.SarifReportNotifier
7472
import org.utbot.intellij.plugin.ui.TestReportUrlOpeningListener
7573
import org.utbot.intellij.plugin.ui.TestsReportNotifier
74+
import org.utbot.intellij.plugin.ui.WarningTestsReportNotifier
7675
import org.utbot.intellij.plugin.ui.utils.getOrCreateSarifReportsPath
7776
import org.utbot.intellij.plugin.ui.utils.getOrCreateTestResourcesPath
7877
import org.utbot.intellij.plugin.ui.utils.showErrorDialogLater
7978
import org.utbot.intellij.plugin.util.signature
8079
import org.utbot.sarif.SarifReport
8180
import java.nio.file.Path
82-
import java.nio.file.Paths
8381
import java.util.concurrent.CountDownLatch
8482
import java.util.concurrent.TimeUnit
8583
import kotlin.reflect.KClass
@@ -167,7 +165,6 @@ object CodeGenerationController {
167165
SarifReportNotifier.notify(
168166
info = """
169167
SARIF report was saved to ${toHtmlLinkTag(mergedReportFile.path)}$HTML_LINE_SEPARATOR
170-
You can open it using the VS Code extension "Sarif Viewer"
171168
""".trimIndent()
172169
)
173170
}
@@ -394,32 +391,10 @@ object CodeGenerationController {
394391
"Test resources directory $testResourcesDirPath does not exist"
395392
}
396393

397-
val testReportSubDir = "utbot-tests-report"
398-
val classFqn = with(testsCodeWithTestReport.testsGenerationReport.classUnderTest) {
399-
qualifiedName ?: error("Could not save tests report for anonymous or local class $this")
400-
}
401-
val fileReportPath = classFqnToPath(classFqn)
402-
403-
val resultedReportedPath =
404-
Paths.get(
405-
testResourcesDirPath.toString(),
406-
testReportSubDir,
407-
fileReportPath + "TestReport" + TestsGenerationReport.EXTENSION
408-
)
409-
410-
val parent = resultedReportedPath.parent
411-
requireNotNull(parent) {
412-
"Expected from parent of $resultedReportedPath to be not null but it is null"
413-
}
414-
415-
VfsUtil.createDirectories(parent.toString())
416-
resultedReportedPath.toFile().writeText(testsCodeWithTestReport.testsGenerationReport.getFileContent())
417-
418394
processInitialWarnings(testsCodeWithTestReport, model)
419395

420396
val notifyMessage = buildString {
421-
appendHtmlLine(testsCodeWithTestReport.testsGenerationReport.toString())
422-
appendHtmlLine()
397+
appendHtmlLine(testsCodeWithTestReport.testsGenerationReport.toString(isShort = true))
423398
val classUnderTestPackageName =
424399
testsCodeWithTestReport.testsGenerationReport.classUnderTest.classId.packageFqName.toString()
425400
if (classUnderTestPackageName != model.testPackageName) {
@@ -430,12 +405,19 @@ object CodeGenerationController {
430405
appendHtmlLine(warningMessage)
431406
appendHtmlLine()
432407
}
433-
val savedFileMessage = """
434-
Tests report was saved to ${toHtmlLinkTag(resultedReportedPath.toString())} in TSV format
408+
val eventLogMessage = """
409+
<a href="${TestReportUrlOpeningListener.prefix}${TestReportUrlOpeningListener.eventLogSuffix}">See details in Event Log</a>.
435410
""".trimIndent()
436-
appendHtmlLine(savedFileMessage)
411+
appendHtmlLine(eventLogMessage)
412+
}
413+
414+
if (testsCodeWithTestReport.testsGenerationReport.hasWarnings) {
415+
WarningTestsReportNotifier.notify(notifyMessage)
416+
} else {
417+
TestsReportNotifier.notify(notifyMessage)
437418
}
438-
TestsReportNotifier.notify(notifyMessage)
419+
420+
DetailsTestsReportNotifier.notify(testsCodeWithTestReport.testsGenerationReport.detailedStatistics)
439421
}
440422

441423
private fun processInitialWarnings(testsCodeWithTestReport: TestsCodeWithTestReport, model: GenerateTestsModel) {
@@ -445,8 +427,6 @@ object CodeGenerationController {
445427
}
446428

447429
testsCodeWithTestReport.testsGenerationReport.apply {
448-
summaryMessage = { "Unit tests for $classUnderTest were generated with warnings.<br>" }
449-
450430
if (model.forceMockHappened) {
451431
initialWarnings.add {
452432
"""

‎utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/ui/GenerateTestsDialogWindow.kt

Lines changed: 43 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4,35 +4,6 @@ package org.utbot.intellij.plugin.ui
44

55
import com.intellij.codeInsight.hint.HintUtil
66
import com.intellij.icons.AllIcons
7-
import org.utbot.common.PathUtil.toPath
8-
import org.utbot.framework.UtSettings
9-
import org.utbot.framework.codegen.ForceStaticMocking
10-
import org.utbot.framework.codegen.Junit4
11-
import org.utbot.framework.codegen.Junit5
12-
import org.utbot.framework.codegen.NoStaticMocking
13-
import org.utbot.framework.codegen.ParametrizedTestSource
14-
import org.utbot.framework.codegen.StaticsMocking
15-
import org.utbot.framework.codegen.TestFramework
16-
import org.utbot.framework.codegen.TestNg
17-
import org.utbot.framework.codegen.model.util.MOCKITO_EXTENSIONS_FILE_CONTENT
18-
import org.utbot.framework.codegen.model.util.MOCKITO_EXTENSIONS_STORAGE
19-
import org.utbot.framework.codegen.model.util.MOCKITO_MOCKMAKER_FILE_NAME
20-
import org.utbot.framework.plugin.api.CodeGenerationSettingItem
21-
import org.utbot.framework.plugin.api.CodegenLanguage
22-
import org.utbot.framework.plugin.api.MockFramework
23-
import org.utbot.framework.plugin.api.MockFramework.MOCKITO
24-
import org.utbot.framework.plugin.api.MockStrategyApi
25-
import org.utbot.framework.plugin.api.TreatOverflowAsError
26-
import org.utbot.intellij.plugin.settings.Settings
27-
import org.utbot.intellij.plugin.ui.components.TestFolderComboWithBrowseButton
28-
import org.utbot.intellij.plugin.ui.utils.LibrarySearchScope
29-
import org.utbot.intellij.plugin.ui.utils.findFrameworkLibrary
30-
import org.utbot.intellij.plugin.ui.utils.getOrCreateTestResourcesPath
31-
import org.utbot.intellij.plugin.ui.utils.kotlinTargetPlatform
32-
import org.utbot.intellij.plugin.ui.utils.parseVersion
33-
import org.utbot.intellij.plugin.ui.utils.testResourceRootTypes
34-
import org.utbot.intellij.plugin.ui.utils.addSourceRootIfAbsent
35-
import org.utbot.intellij.plugin.ui.utils.testRootType
367
import com.intellij.ide.impl.ProjectNewWindowDoNotAskOption
378
import com.intellij.openapi.application.runWriteAction
389
import com.intellij.openapi.command.WriteCommandAction
@@ -64,6 +35,7 @@ import com.intellij.openapi.vfs.VfsUtil
6435
import com.intellij.openapi.vfs.VfsUtilCore.urlToPath
6536
import com.intellij.openapi.vfs.VirtualFile
6637
import com.intellij.openapi.vfs.newvfs.impl.FakeVirtualFile
38+
import com.intellij.openapi.wm.ToolWindowManager
6739
import com.intellij.psi.PsiClass
6840
import com.intellij.psi.PsiManager
6941
import com.intellij.psi.PsiMethod
@@ -100,30 +72,59 @@ import com.intellij.util.ui.JBUI.Borders.merge
10072
import com.intellij.util.ui.JBUI.scale
10173
import com.intellij.util.ui.JBUI.size
10274
import com.intellij.util.ui.UIUtil
75+
import com.intellij.util.ui.components.BorderLayoutPanel
76+
import org.jetbrains.concurrency.Promise
77+
import org.jetbrains.concurrency.thenRun
78+
import org.utbot.common.PathUtil.toPath
79+
import org.utbot.framework.UtSettings
80+
import org.utbot.framework.codegen.ForceStaticMocking
81+
import org.utbot.framework.codegen.Junit4
82+
import org.utbot.framework.codegen.Junit5
83+
import org.utbot.framework.codegen.NoStaticMocking
84+
import org.utbot.framework.codegen.ParametrizedTestSource
85+
import org.utbot.framework.codegen.StaticsMocking
86+
import org.utbot.framework.codegen.TestFramework
87+
import org.utbot.framework.codegen.TestNg
88+
import org.utbot.framework.codegen.model.util.MOCKITO_EXTENSIONS_FILE_CONTENT
89+
import org.utbot.framework.codegen.model.util.MOCKITO_EXTENSIONS_STORAGE
90+
import org.utbot.framework.codegen.model.util.MOCKITO_MOCKMAKER_FILE_NAME
91+
import org.utbot.framework.plugin.api.CodeGenerationSettingItem
92+
import org.utbot.framework.plugin.api.CodegenLanguage
93+
import org.utbot.framework.plugin.api.MockFramework
94+
import org.utbot.framework.plugin.api.MockFramework.MOCKITO
95+
import org.utbot.framework.plugin.api.MockStrategyApi
96+
import org.utbot.framework.plugin.api.TreatOverflowAsError
10397
import org.utbot.intellij.plugin.models.GenerateTestsModel
10498
import org.utbot.intellij.plugin.models.jUnit4LibraryDescriptor
10599
import org.utbot.intellij.plugin.models.jUnit5LibraryDescriptor
100+
import org.utbot.intellij.plugin.models.mockitoCoreLibraryDescriptor
106101
import org.utbot.intellij.plugin.models.packageName
107102
import org.utbot.intellij.plugin.models.testNgLibraryDescriptor
108-
import com.intellij.util.ui.components.BorderLayoutPanel
109-
import org.jetbrains.concurrency.Promise
110-
import org.utbot.intellij.plugin.models.mockitoCoreLibraryDescriptor
103+
import org.utbot.intellij.plugin.settings.Settings
104+
import org.utbot.intellij.plugin.ui.components.TestFolderComboWithBrowseButton
105+
import org.utbot.intellij.plugin.ui.utils.LibrarySearchScope
106+
import org.utbot.intellij.plugin.ui.utils.addSourceRootIfAbsent
107+
import org.utbot.intellij.plugin.ui.utils.allLibraries
108+
import org.utbot.intellij.plugin.ui.utils.findFrameworkLibrary
109+
import org.utbot.intellij.plugin.ui.utils.getOrCreateTestResourcesPath
110+
import org.utbot.intellij.plugin.ui.utils.kotlinTargetPlatform
111+
import org.utbot.intellij.plugin.ui.utils.parseVersion
112+
import org.utbot.intellij.plugin.ui.utils.testResourceRootTypes
113+
import org.utbot.intellij.plugin.ui.utils.testRootType
111114
import org.utbot.intellij.plugin.util.AndroidApiHelper
112115
import java.awt.BorderLayout
113116
import java.awt.Color
114117
import java.nio.file.Files
115118
import java.nio.file.Path
116119
import java.nio.file.Paths
117-
import java.util.Objects
120+
import java.util.*
118121
import java.util.concurrent.TimeUnit
119122
import javax.swing.DefaultComboBoxModel
120123
import javax.swing.JComboBox
121124
import javax.swing.JComponent
122125
import javax.swing.JList
123126
import javax.swing.JPanel
124127
import kotlin.streams.toList
125-
import org.jetbrains.concurrency.thenRun
126-
import org.utbot.intellij.plugin.ui.utils.allLibraries
127128

128129
private const val RECENTS_KEY = "org.utbot.recents"
129130

@@ -183,18 +184,23 @@ class GenerateTestsDialogWindow(val model: GenerateTestsModel) : DialogWrapper(m
183184
setResizable(false)
184185

185186
// Configure notification urls callbacks
186-
TestsReportNotifier.urlOpeningListener.callbacks[TestReportUrlOpeningListener.mockitoSuffix]?.plusAssign {
187+
TestReportUrlOpeningListener.callbacks[TestReportUrlOpeningListener.mockitoSuffix]?.plusAssign {
187188
if (createMockFrameworkNotificationDialog() == Messages.YES) {
188189
configureMockFramework()
189190
}
190191
}
191192

192-
TestsReportNotifier.urlOpeningListener.callbacks[TestReportUrlOpeningListener.mockitoInlineSuffix]?.plusAssign {
193+
TestReportUrlOpeningListener.callbacks[TestReportUrlOpeningListener.mockitoInlineSuffix]?.plusAssign {
193194
if (createStaticsMockingNotificationDialog() == Messages.YES) {
194195
configureStaticMocking()
195196
}
196197
}
197198

199+
TestReportUrlOpeningListener.callbacks[TestReportUrlOpeningListener.eventLogSuffix]?.plusAssign {
200+
val twm = ToolWindowManager.getInstance(model.project)
201+
twm.getToolWindow("Event Log")?.activate(null)
202+
}
203+
198204
init()
199205
}
200206

0 commit comments

Comments
 (0)