Skip to content

Sort tests by estimated size and exception type #2590

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 15, 2023
Merged

Sort tests by estimated size and exception type #2590

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
860d34f
Sort tests by estimated size and exception type
IlyaMuravjov Sep 11, 2023
72033a3
Cache `calculateSize()` for rendered executions
IlyaMuravjov Sep 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 26 additions & 5 deletions ...mework/src/main/kotlin/org/utbot/framework/codegen/tree/CgAbstractTestClassConstructor.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,19 @@ import org.utbot.framework.codegen.domain.models.CgTestMethodCluster
import org.utbot.framework.codegen.domain.models.CgTripleSlashMultilineComment
import org.utbot.framework.codegen.domain.models.CgUtilEntity
import org.utbot.framework.codegen.domain.models.CgUtilMethod
import org.utbot.framework.codegen.domain.models.SimpleTestClassModel
import org.utbot.framework.codegen.domain.models.TestClassModel
import org.utbot.framework.codegen.renderer.importUtilMethodDependencies
import org.utbot.framework.codegen.reports.TestsGenerationReport
import org.utbot.framework.codegen.services.CgNameGenerator
import org.utbot.framework.codegen.services.framework.TestFrameworkManager
import org.utbot.framework.plugin.api.ClassId
import org.utbot.framework.plugin.api.MethodId
import org.utbot.framework.plugin.api.UtExecution
import org.utbot.framework.plugin.api.UtExecutionFailure
import org.utbot.framework.plugin.api.UtExecutionSuccess
import org.utbot.framework.plugin.api.UtMethodTestSet
import org.utbot.framework.plugin.api.util.description
import org.utbot.framework.util.calculateSize

abstract class CgAbstractTestClassConstructor<T : TestClassModel>(val context: CgContext):
CgContextOwner by context,
Expand Down Expand Up @@ -91,6 +94,12 @@ abstract class CgAbstractTestClassConstructor<T : TestClassModel>(val context: C
) {
val (_, _, clustersInfo) = testSet

// `stateAfter` is not accounted here, because usually most of it is not rendered
val executionToSizeCache = testSet.executions.associateWith { execution ->
execution.stateBefore.calculateSize() +
((execution.result as? UtExecutionSuccess)?.model?.calculateSize() ?: 1)
}

for ((clusterSummary, executionIndices) in clustersInfo) {
val currentTestCaseTestMethods = mutableListOf<CgTestMethod>()
emptyLineIfNeeded()
Expand All @@ -100,11 +109,23 @@ abstract class CgAbstractTestClassConstructor<T : TestClassModel>(val context: C
executionIndices to false
}

for (i in checkedRange) {
withExecutionIdScope(i) {
currentTestCaseTestMethods += methodConstructor.createTestMethod(testSet, testSet.executions[i])
testSet.executions
.withIndex()
.toList()
.slice(checkedRange)
.sortedWith(
// NPE tests are rendered last, because oftentimes they are meaningless in a sense
// that they pass `null` somewhere where `null` is never passed in production
compareBy<IndexedValue<UtExecution>> { (_, execution) ->
if ((execution.result as? UtExecutionFailure)?.exception is NullPointerException) 1 else -1
}
// we place "smaller" tests earlier, since they are easier to read
.thenComparingInt { (_, execution) -> executionToSizeCache.getValue(execution) }
).forEach { (i, execution) ->
withExecutionIdScope(i) {
currentTestCaseTestMethods += methodConstructor.createTestMethod(testSet, execution)
}
}
}

val comments = listOf("Actual number of generated tests (${executionIndices.last - executionIndices.first}) exceeds per-method limit (${UtSettings.maxTestsPerMethodInRegion})",
"The limit can be configured in '{HOME_DIR}/.utbot/settings.properties' with 'maxTestsPerMethod' property")
Expand Down
2 changes: 1 addition & 1 deletion utbot-framework/src/main/kotlin/org/utbot/framework/util/SizeUtils.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ fun EnvironmentModels.calculateSize(): Int {
* calculated the size for [this] model and [this] is [UtReferenceModel], then in codegen we would have already
* created variable for [this] model and do not need to create it again, so size should be equal to 0.
*/
private fun UtModel.calculateSize(used: MutableSet<UtReferenceModel> = mutableSetOf()): Int {
fun UtModel.calculateSize(used: MutableSet<UtReferenceModel> = mutableSetOf()): Int {
if (this in used) return 0

if (this is UtReferenceModel)
Expand Down