Skip to content

Fix coverage calculation #856

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 5, 2022
Merged

Fix coverage calculation #856

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,7 @@ fun runGeneration(
logger.error("Seems like classloader for cut not valid (maybe it was backported to system): ${cut.classLoader}")
}

val statsForClass = StatsForClass(Type.getInternalName(cut.classId.jClass))

val testedClasses = ConcurrentSkipListSet<String>()
val statsForClass = StatsForClass()

val codeGenerator = CodeGenerator(
cut.classId,
Expand Down Expand Up @@ -321,17 +319,17 @@ fun runGeneration(
is UtExecution -> {
try {
val testMethodName = testMethodName(method.toString(), ++testsCounter)
val className = method.clazz.qualifiedName
val className = Type.getInternalName(method.clazz.java)
logger.debug { "--new testCase collected, to generate: $testMethodName" }
statsForMethod.testsGeneratedCount++
result.coverage?.let {
statsForClass.updateCoverage(
newCoverage = it,
isNewClass = !testedClasses.contains(className),
isNewClass = !statsForClass.testedClassNames.contains(className),
fromFuzzing = result is UtFuzzedExecution
)
}
testedClasses.add(className)
statsForClass.testedClassNames.add(className)

//TODO: it is a strange hack to create fake test case for one [UtResult]
testSets.add(UtMethodTestSet(method, listOf(result)))
Expand Down
28 changes: 19 additions & 9 deletions utbot-junit-contest/src/main/kotlin/org/utbot/contest/Statistics.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.utbot.contest

import java.io.File
import java.util.concurrent.ConcurrentSkipListSet
import org.utbot.common.MutableMultiset
import org.utbot.common.mutableMultisetOf
import org.utbot.framework.plugin.api.Instruction
Expand Down Expand Up @@ -48,21 +49,21 @@ class GlobalStats {
get() = statsForClasses.count { it.failedToCompile }

val coveredInstructions: Int
get() = statsForClasses.sumOf { it.coverage.getCoverageInfo(it.className).covered }
get() = statsForClasses.sumOf { it.coverage.getCoverageInfo(it.testedClassNames).covered }

val coveredInstructionsByFuzzing: Int
get() = statsForClasses.sumOf { it.fuzzedCoverage.getCoverageInfo(it.className).covered }
get() = statsForClasses.sumOf { it.fuzzedCoverage.getCoverageInfo(it.testedClassNames).covered }

val coveredInstructionsByConcolic: Int
get() = statsForClasses.sumOf { it.concolicCoverage.getCoverageInfo(it.className).covered }
get() = statsForClasses.sumOf { it.concolicCoverage.getCoverageInfo(it.testedClassNames).covered }

val totalInstructions: Int
get() = statsForClasses.sumOf { it.coverage.totalInstructions.toInt() }

val avgCoverage: Double
get() = statsForClasses
.filter { it.coverage.totalInstructions != 0L }
.map { it.coverage.getCoverageInfo(it.className).run { 100.0 * covered / total } }
.map { it.coverage.getCoverageInfo(it.testedClassNames).run { 100.0 * covered / total } }
.average().run {
if (isNaN()) 0.0
else this
Expand Down Expand Up @@ -108,7 +109,9 @@ class GlobalStats {
avgCoverage.format(PRECISION) + " %"
}

class StatsForClass(val className: String) {
class StatsForClass {
val testedClassNames: MutableSet<String> = ConcurrentSkipListSet()

var methodsCount: Int = -1
val statsForMethods = mutableListOf<StatsForMethod>()

Expand All @@ -123,8 +126,15 @@ class StatsForClass(val className: String) {
var fuzzedCoverage = CoverageInstructionsSet()
var concolicCoverage = CoverageInstructionsSet()

/**
* Add class [className] to respect coverage from this class.
*/
fun addTestedClass(className: String) {
testedClassNames.add(className)
}

private fun CoverageInstructionsSet.prettyInfo(): String =
getCoverageInfo(className).run { "$covered/$total" }
getCoverageInfo(testedClassNames).run { "$covered/$total" }

override fun toString(): String = "\n<StatsForClass> :" +
"\n\tcanceled by timeout = $canceledByTimeout" +
Expand Down Expand Up @@ -191,12 +201,12 @@ data class CoverageInstructionsSet(
data class CoverageStatistic(val covered: Int, val total: Int)

/**
* Compute coverage of class named [className] with its anonymous, nested and inner classes.
* Compute coverage of classes with names in [classNames].
*/
private fun CoverageInstructionsSet?.getCoverageInfo(className: String): CoverageStatistic = this?.run {
private fun CoverageInstructionsSet?.getCoverageInfo(classNames: Set<String>): CoverageStatistic = this?.run {
CoverageStatistic(
coveredInstructions.filter {
instr -> instr.className.startsWith(className)
instr -> classNames.contains(instr.className)
}.toSet().size,
totalInstructions.toInt()
)
Expand Down