From 14184b1392817f7bcf898e41857c3b3c59c20c03 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Sun, 23 Feb 2025 22:45:01 +0100 Subject: [PATCH 01/14] [943] minor change: - show number of args passed to at the end of a test --- .../Event.HumanReadableOutputRecorder.swift | 34 ++++++++++++++++-- Tests/TestingTests/EventRecorderTests.swift | 36 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index be92fe932..66483e056 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -146,6 +146,36 @@ extension Event.HumanReadableOutputRecorder { return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) } + + /// Calculates the total number of arguments across all test cases in a sequence. + /// + /// - Parameters: + /// - testCases: A sequence of `Test.Case` objects containing the test cases.. + /// + /// - Returns: The total count of arguments across all test cases. If `testCases` is `nil` or empty, returns `0`. + /// + private func _numberOfArguments(for testCases: (any Sequence)?) -> Int { + testCases?.reduce(into: 0) { result, testCase in + result += testCase.arguments.count + } ?? 0 + } + + /// Returns a formatted string describing the number of arguments in a test, based on verbosity level. + /// + /// - Parameters: + /// - test: The `Test` object that contains a sequence of `testCases`. + /// - verbose: If the level is very verbose, a detailed description is returned. + /// + /// - Returns: A string describing the number of arguments in the test cases, or an empty string if it's not very verbose level. + /// + private func _includeNumberOfArguments(_ test: Test, verbose: Int) -> String { + if verbose == 2 { // very verbose + let numberOfArguments = _numberOfArguments(for: test.testCases) + return " with \(numberOfArguments) " + (numberOfArguments > 1 ? "arguments" : "argument") + } + return "" + } + } /// Generate a title for the specified test (either "Test" or "Suite"), @@ -370,14 +400,14 @@ extension Event.HumanReadableOutputRecorder { CollectionOfOne( Message( symbol: .fail, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) failed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfArguments(test, verbose: verbosity)) failed after \(duration)\(issues.description)." ) ) + _formattedComments(for: test) } else { [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfArguments(test, verbose: verbosity)) passed after \(duration)\(issues.description)." ) ] } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 6b5b9bd81..f780393ff 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -178,6 +178,42 @@ struct EventRecorderTests { .first != nil ) } + + @available(_regexAPI, *) + @Test( + "number of arguments based on verbosity level at the end of test run", + arguments: [ + ("f()", #".*"# , 0), + ("g()", #".* with .+ argument.*"# , 2), + ("PredictablyFailingTests", #".*"# , 1), + ] + ) + func numberOfArgumentsAtTheEndOfTests(testName: String, expectedPattern: String, verbosity: Int) async throws { + let stream = Stream() + + var configuration = Configuration() + let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) + configuration.eventHandler = { event, context in + eventRecorder.record(event, in: context) + } + configuration.verbosity = verbosity + + await runTest(for: PredictablyFailingTests.self, configuration: configuration) + + let buffer = stream.buffer.rawValue + if testsWithSignificantIOAreEnabled { + print(buffer, terminator: "") + } + + let aurgmentRegex = try Regex(expectedPattern) + #expect( + (try? buffer + .split(whereSeparator: \.isNewline) + .compactMap(aurgmentRegex.wholeMatch(in:)) + .first) != nil + ) + } + @available(_regexAPI, *) @Test( From a1f7b4b9e3be4be6da8f44045de39a721bc2e3e1 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Mon, 24 Feb 2025 11:35:11 +0100 Subject: [PATCH 02/14] minor change: - update test cases & excluding suites --- .../Event.HumanReadableOutputRecorder.swift | 29 +++++-------------- Tests/TestingTests/EventRecorderTests.swift | 17 ++++++++--- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 66483e056..27beba141 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -147,31 +147,18 @@ extension Event.HumanReadableOutputRecorder { return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) } - /// Calculates the total number of arguments across all test cases in a sequence. - /// - /// - Parameters: - /// - testCases: A sequence of `Test.Case` objects containing the test cases.. - /// - /// - Returns: The total count of arguments across all test cases. If `testCases` is `nil` or empty, returns `0`. - /// - private func _numberOfArguments(for testCases: (any Sequence)?) -> Int { - testCases?.reduce(into: 0) { result, testCase in - result += testCase.arguments.count - } ?? 0 - } - /// Returns a formatted string describing the number of arguments in a test, based on verbosity level. /// /// - Parameters: - /// - test: The `Test` object that contains a sequence of `testCases`. + /// - test: to get the number of `testCases` out of a ``Test``. /// - verbose: If the level is very verbose, a detailed description is returned. /// - /// - Returns: A string describing the number of arguments in the test cases, or an empty string if it's not very verbose level. + /// - Returns: A string describing the number of argument(s) in the test cases, or an empty string if it's not very verbose level. /// - private func _includeNumberOfArguments(_ test: Test, verbose: Int) -> String { - if verbose == 2 { // very verbose - let numberOfArguments = _numberOfArguments(for: test.testCases) - return " with \(numberOfArguments) " + (numberOfArguments > 1 ? "arguments" : "argument") + private func _includeNumberOfTestCases(for test: Test, verbose: Int) -> String { + if verbose == 2 && !test.isSuite { // very verbose + let testCasesCount = test.testCases?.count(where: { _ in true }) ?? 0 + return " with \(testCasesCount) " + (testCasesCount > 1 ? "test cases" : "test case") } return "" } @@ -400,14 +387,14 @@ extension Event.HumanReadableOutputRecorder { CollectionOfOne( Message( symbol: .fail, - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfArguments(test, verbose: verbosity)) failed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCases(for: test, verbose: verbosity)) failed after \(duration)\(issues.description)." ) ) + _formattedComments(for: test) } else { [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfArguments(test, verbose: verbosity)) passed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCases(for: test, verbose: verbosity)) passed after \(duration)\(issues.description)." ) ] } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index f780393ff..554ac573b 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -183,9 +183,11 @@ struct EventRecorderTests { @Test( "number of arguments based on verbosity level at the end of test run", arguments: [ - ("f()", #".*"# , 0), - ("g()", #".* with .+ argument.*"# , 2), - ("PredictablyFailingTests", #".*"# , 1), + ("f()", #".* Test f\(\) failed after .*"# , 0), + ("f()", #".* Test f\(\) with 1 test case failed after .*"# , 2), + ("d(_:)", #".* Test d\(_:\) with .+ test cases passed after.*"# , 2), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 1), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 2), ] ) func numberOfArgumentsAtTheEndOfTests(testName: String, expectedPattern: String, verbosity: Int) async throws { @@ -206,8 +208,9 @@ struct EventRecorderTests { } let aurgmentRegex = try Regex(expectedPattern) + #expect( - (try? buffer + (try buffer .split(whereSeparator: \.isNewline) .compactMap(aurgmentRegex.wholeMatch(in:)) .first) != nil @@ -572,6 +575,11 @@ struct EventRecorderTests { #expect(Bool(false)) } } + + @Test(.hidden, arguments: [1, 2, 3]) + func d(_ arg: Int) { + #expect(arg > 0) + } @Test(.hidden) func g() { #expect(Bool(false)) @@ -602,3 +610,4 @@ struct EventRecorderTests { } } } + From 928f67bd98b35e6e151f2d05d86c79204fb40a29 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Mon, 24 Feb 2025 11:37:58 +0100 Subject: [PATCH 03/14] Rename _includeNumberOfTestCases to _includeNumberOfTestCasesIfNeeded for clarity --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 27beba141..700ff1ab6 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -155,7 +155,7 @@ extension Event.HumanReadableOutputRecorder { /// /// - Returns: A string describing the number of argument(s) in the test cases, or an empty string if it's not very verbose level. /// - private func _includeNumberOfTestCases(for test: Test, verbose: Int) -> String { + private func _includeNumberOfTestCasesIfNedded(for test: Test, verbose: Int) -> String { if verbose == 2 && !test.isSuite { // very verbose let testCasesCount = test.testCases?.count(where: { _ in true }) ?? 0 return " with \(testCasesCount) " + (testCasesCount > 1 ? "test cases" : "test case") @@ -387,14 +387,14 @@ extension Event.HumanReadableOutputRecorder { CollectionOfOne( Message( symbol: .fail, - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCases(for: test, verbose: verbosity)) failed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCasesIfNedded(for: test, verbose: verbosity)) failed after \(duration)\(issues.description)." ) ) + _formattedComments(for: test) } else { [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCases(for: test, verbose: verbosity)) passed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCasesIfNedded(for: test, verbose: verbosity)) passed after \(duration)\(issues.description)." ) ] } From 2a796ecca930152f1b99cd5bb0e725fe6bc1f0ae Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Mon, 24 Feb 2025 11:53:57 +0100 Subject: [PATCH 04/14] update the documentation --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 700ff1ab6..84e2c77dc 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -153,7 +153,7 @@ extension Event.HumanReadableOutputRecorder { /// - test: to get the number of `testCases` out of a ``Test``. /// - verbose: If the level is very verbose, a detailed description is returned. /// - /// - Returns: A string describing the number of argument(s) in the test cases, or an empty string if it's not very verbose level. + /// - Returns: A string describing the number of test cases in the test, or an empty string if it's not very verbose level. /// private func _includeNumberOfTestCasesIfNedded(for test: Test, verbose: Int) -> String { if verbose == 2 && !test.isSuite { // very verbose From 16a997e631c5f32df8115a20ba26e81a9acfe89e Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Mon, 24 Feb 2025 12:05:01 +0100 Subject: [PATCH 05/14] fix typo in the method name `_includeNumberOfTestCasesIfNeeded` --- .../Recorder/Event.HumanReadableOutputRecorder.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 84e2c77dc..840a2dcf3 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -155,10 +155,10 @@ extension Event.HumanReadableOutputRecorder { /// /// - Returns: A string describing the number of test cases in the test, or an empty string if it's not very verbose level. /// - private func _includeNumberOfTestCasesIfNedded(for test: Test, verbose: Int) -> String { + private func _includeNumberOfTestCasesIfNeeded(for test: Test, verbose: Int) -> String { if verbose == 2 && !test.isSuite { // very verbose let testCasesCount = test.testCases?.count(where: { _ in true }) ?? 0 - return " with \(testCasesCount) " + (testCasesCount > 1 ? "test cases" : "test case") + return" with \(testCasesCount) \(testCasesCount > 1 ? "test cases" : "test case")" } return "" } @@ -387,14 +387,14 @@ extension Event.HumanReadableOutputRecorder { CollectionOfOne( Message( symbol: .fail, - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCasesIfNedded(for: test, verbose: verbosity)) failed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCasesIfNeeded(for: test, verbose: verbosity)) failed after \(duration)\(issues.description)." ) ) + _formattedComments(for: test) } else { [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCasesIfNedded(for: test, verbose: verbosity)) passed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCasesIfNeeded(for: test, verbose: verbosity)) passed after \(duration)\(issues.description)." ) ] } From ea309da69c56f7016d35f07a9bc9363799adef80 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Mon, 24 Feb 2025 15:53:34 +0100 Subject: [PATCH 06/14] Refactor code to use existing methods and align with document style feedback - use counting instead of ternary operator - fixing char limit per row - fixing verbosity condition check - check if test is parameterized --- .../Event.HumanReadableOutputRecorder.swift | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 840a2dcf3..80b045c32 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -147,18 +147,21 @@ extension Event.HumanReadableOutputRecorder { return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) } - /// Returns a formatted string describing the number of arguments in a test, based on verbosity level. + /// Returns a formatted string describing the number of arguments in a test, + /// based on verbosity level. /// /// - Parameters: /// - test: to get the number of `testCases` out of a ``Test``. - /// - verbose: If the level is very verbose, a detailed description is returned. + /// - verbose: If the level is very verbose, a detailed description + /// is returned. /// - /// - Returns: A string describing the number of test cases in the test, or an empty string if it's not very verbose level. + /// - Returns: A string describing the number of test cases in the test, + /// or an empty string if it's not very verbose level. /// private func _includeNumberOfTestCasesIfNeeded(for test: Test, verbose: Int) -> String { - if verbose == 2 && !test.isSuite { // very verbose + if verbose >= 2 && test.isParameterized { // very verbose let testCasesCount = test.testCases?.count(where: { _ in true }) ?? 0 - return" with \(testCasesCount) \(testCasesCount > 1 ? "test cases" : "test case")" + return" with \(testCasesCount.counting("test case"))" } return "" } From 4592e0653478e2a4b52ef2390144d8c09bf0b900 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Mon, 24 Feb 2025 16:53:17 +0100 Subject: [PATCH 07/14] Update test case count increment to occur during the test run instead of after test ends --- .../Event.HumanReadableOutputRecorder.swift | 43 ++++++++++++++----- Tests/TestingTests/EventRecorderTests.swift | 2 +- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 80b045c32..35f972f08 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -62,6 +62,9 @@ extension Event { /// The number of known issues recorded for the test. var knownIssueCount = 0 + + /// The number of test cases for the test. + var testCasesCount: Int = 0 } /// Data tracked on a per-test basis. @@ -151,19 +154,22 @@ extension Event.HumanReadableOutputRecorder { /// based on verbosity level. /// /// - Parameters: - /// - test: to get the number of `testCases` out of a ``Test``. - /// - verbose: If the level is very verbose, a detailed description - /// is returned. + /// - isParameterized: To check test cases count should be included. + /// - count: The number of test cases in the test. + /// - verbosity: The verbosity level. If it's very verbose or higher, + /// the test cases count might get included. + /// /// /// - Returns: A string describing the number of test cases in the test, /// or an empty string if it's not very verbose level. /// - private func _includeNumberOfTestCasesIfNeeded(for test: Test, verbose: Int) -> String { - if verbose >= 2 && test.isParameterized { // very verbose - let testCasesCount = test.testCases?.count(where: { _ in true }) ?? 0 - return" with \(testCasesCount.counting("test case"))" - } - return "" + private func _includeNumberOfTestCasesIfNeeded( + _ isParameterized: Bool, + count testCasesCount: Int, + verbosity verbose: Int + ) -> String { + guard verbose >= 2, isParameterized else { return "" } + return " with \(testCasesCount.counting("test case"))" } } @@ -301,6 +307,18 @@ extension Event.HumanReadableOutputRecorder { testData.issueCount[issue.severity] = issueCount + 1 } context.testData[id] = testData + + case .testCaseStarted: + guard verbosity >= 2 else { break } + let id: [String] = if let test { + test.id.keyPathRepresentation + } else { + [] + } + var testData = context.testData[id] ?? .init(startInstant: instant) + testData.testCasesCount += 1 + context.testData[id] = testData + default: // These events do not manipulate the context structure. @@ -386,18 +404,21 @@ extension Event.HumanReadableOutputRecorder { let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) + let testCasesCountMessage = _includeNumberOfTestCasesIfNeeded(test.isParameterized, + count: testData.testCasesCount, + verbosity: verbosity) return if issues.errorIssueCount > 0 { CollectionOfOne( Message( symbol: .fail, - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCasesIfNeeded(for: test, verbose: verbosity)) failed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCountMessage) failed after \(duration)\(issues.description)." ) ) + _formattedComments(for: test) } else { [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(_includeNumberOfTestCasesIfNeeded(for: test, verbose: verbosity)) passed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCountMessage) passed after \(duration)\(issues.description)." ) ] } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 554ac573b..23708ffce 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -184,7 +184,7 @@ struct EventRecorderTests { "number of arguments based on verbosity level at the end of test run", arguments: [ ("f()", #".* Test f\(\) failed after .*"# , 0), - ("f()", #".* Test f\(\) with 1 test case failed after .*"# , 2), + ("f()", #".* Test f\(\) failed after .*"# , 2), ("d(_:)", #".* Test d\(_:\) with .+ test cases passed after.*"# , 2), ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 1), ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 2), From 4cf3306c08ccd8084ba0e954048ec3a7605d19eb Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Mon, 24 Feb 2025 21:45:04 +0100 Subject: [PATCH 08/14] refactor: remove redundant helper methods and unnecessary test object creation --- .../Event.HumanReadableOutputRecorder.swift | 46 ++++--------------- 1 file changed, 9 insertions(+), 37 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 35f972f08..7f6abcff0 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -149,28 +149,6 @@ extension Event.HumanReadableOutputRecorder { return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) } - - /// Returns a formatted string describing the number of arguments in a test, - /// based on verbosity level. - /// - /// - Parameters: - /// - isParameterized: To check test cases count should be included. - /// - count: The number of test cases in the test. - /// - verbosity: The verbosity level. If it's very verbose or higher, - /// the test cases count might get included. - /// - /// - /// - Returns: A string describing the number of test cases in the test, - /// or an empty string if it's not very verbose level. - /// - private func _includeNumberOfTestCasesIfNeeded( - _ isParameterized: Bool, - count testCasesCount: Int, - verbosity verbose: Int - ) -> String { - guard verbose >= 2, isParameterized else { return "" } - return " with \(testCasesCount.counting("test case"))" - } } @@ -309,16 +287,8 @@ extension Event.HumanReadableOutputRecorder { context.testData[id] = testData case .testCaseStarted: - guard verbosity >= 2 else { break } - let id: [String] = if let test { - test.id.keyPathRepresentation - } else { - [] - } - var testData = context.testData[id] ?? .init(startInstant: instant) - testData.testCasesCount += 1 - context.testData[id] = testData - + let test = test! + context.testData[test.id.keyPathRepresentation]?.testCasesCount += 1 default: // These events do not manipulate the context structure. @@ -404,21 +374,23 @@ extension Event.HumanReadableOutputRecorder { let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - let testCasesCountMessage = _includeNumberOfTestCasesIfNeeded(test.isParameterized, - count: testData.testCasesCount, - verbosity: verbosity) + let testCasesCount = if verbosity >= 2 && test.isParameterized { + " with \(testData.testCasesCount.counting("test case"))" + } else { + "" + } return if issues.errorIssueCount > 0 { CollectionOfOne( Message( symbol: .fail, - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCountMessage) failed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) failed after \(duration)\(issues.description)." ) ) + _formattedComments(for: test) } else { [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), - stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCountMessage) passed after \(duration)\(issues.description)." + stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) passed after \(duration)\(issues.description)." ) ] } From 46d4910bafc6d23e6fc67b135f1ee0b09c3edaa2 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Mon, 24 Feb 2025 22:59:57 +0100 Subject: [PATCH 09/14] Cleanup: Remove unnecessary newline --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 1 - Tests/TestingTests/EventRecorderTests.swift | 2 -- 2 files changed, 3 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 7f6abcff0..069b026c7 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -149,7 +149,6 @@ extension Event.HumanReadableOutputRecorder { return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) } - } /// Generate a title for the specified test (either "Test" or "Suite"), diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 23708ffce..18f63bdae 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -217,7 +217,6 @@ struct EventRecorderTests { ) } - @available(_regexAPI, *) @Test( "Issue counts are summed correctly on test end", @@ -610,4 +609,3 @@ struct EventRecorderTests { } } } - From 701506c9e1a013be744062efd1de0cb8958b44a4 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Tue, 25 Feb 2025 18:03:31 +0100 Subject: [PATCH 10/14] nitpick: remove redundant whitespace in `numberOfTestsCasesAtTestEnd` --- Tests/TestingTests/EventRecorderTests.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 18f63bdae..932167c7f 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -183,14 +183,14 @@ struct EventRecorderTests { @Test( "number of arguments based on verbosity level at the end of test run", arguments: [ - ("f()", #".* Test f\(\) failed after .*"# , 0), - ("f()", #".* Test f\(\) failed after .*"# , 2), - ("d(_:)", #".* Test d\(_:\) with .+ test cases passed after.*"# , 2), - ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 1), - ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"# , 2), + ("f()", #".* Test f\(\) failed after .*"#, 0), + ("f()", #".* Test f\(\) failed after .*"#, 2), + ("d(_:)", #".* Test d\(_:\) with .+ test cases passed after.*"#, 2), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#, 1), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#, 2), ] ) - func numberOfArgumentsAtTheEndOfTests(testName: String, expectedPattern: String, verbosity: Int) async throws { + func numberOfTestsCasesAtTestEnd(testName: String, expectedPattern: String, verbosity: Int) async throws { let stream = Stream() var configuration = Configuration() From 9616b98ceb03592a0b7bcb0ec3da4542108fe128 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Tue, 25 Feb 2025 18:57:32 +0100 Subject: [PATCH 11/14] Remove verbosity limit on `testCasesCount` and update related unit tests --- .../Event.HumanReadableOutputRecorder.swift | 2 +- Tests/TestingTests/EventRecorderTests.swift | 37 ++++++++++++------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 069b026c7..847dd4389 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -373,7 +373,7 @@ extension Event.HumanReadableOutputRecorder { let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - let testCasesCount = if verbosity >= 2 && test.isParameterized { + let testCasesCount = if test.isParameterized { " with \(testData.testCasesCount.counting("test case"))" } else { "" diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 932167c7f..4a73a3560 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -183,14 +183,14 @@ struct EventRecorderTests { @Test( "number of arguments based on verbosity level at the end of test run", arguments: [ - ("f()", #".* Test f\(\) failed after .*"#, 0), - ("f()", #".* Test f\(\) failed after .*"#, 2), - ("d(_:)", #".* Test d\(_:\) with .+ test cases passed after.*"#, 2), - ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#, 1), - ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#, 2), + ("f()", #".* Test f\(\) failed after .*"#), + ("l(_:)", #".* Test l\(_:\) with .+ test cases passed after.*"#), + ("m(_:)", #".* Test m\(_:\) with .+ test cases failed after.*"#), + ("n(_:)", #".* Test n\(_:\) with .+ test case passed after.*"#), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#), ] ) - func numberOfTestsCasesAtTestEnd(testName: String, expectedPattern: String, verbosity: Int) async throws { + func numberOfTestsCasesAtTestEnd(testName: String, expectedPattern: String) async throws { let stream = Stream() var configuration = Configuration() @@ -198,7 +198,6 @@ struct EventRecorderTests { configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) } - configuration.verbosity = verbosity await runTest(for: PredictablyFailingTests.self, configuration: configuration) @@ -227,7 +226,7 @@ struct EventRecorderTests { ("i()", #".* Test i\(\) failed after .+ seconds with 2 issues \(including 1 warning\)\."#), ("j()", #".* Test j\(\) passed after .+ seconds with 1 warning and 1 known issue\."#), ("k()", #".* Test k\(\) passed after .+ seconds with 1 known issue\."#), - ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 13 issues \(including 3 warnings and 6 known issues\)\."#), + ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 16 issues \(including 3 warnings and 6 known issues\)\."#), ] ) func issueCountSummingAtTestEnd(testName: String, expectedPattern: String) async throws { @@ -322,7 +321,7 @@ struct EventRecorderTests { .compactMap(runFailureRegex.wholeMatch(in:)) .first ) - #expect(match.output.1 == 9) + #expect(match.output.1 == 12) #expect(match.output.2 == 5) } @@ -574,11 +573,6 @@ struct EventRecorderTests { #expect(Bool(false)) } } - - @Test(.hidden, arguments: [1, 2, 3]) - func d(_ arg: Int) { - #expect(arg > 0) - } @Test(.hidden) func g() { #expect(Bool(false)) @@ -608,4 +602,19 @@ struct EventRecorderTests { Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() } } + + @Test(.hidden, arguments: [1, 2, 3]) + func l(_ arg: Int) { + #expect(arg > 0) + } + + @Test(.hidden, arguments: [1, 2, 3]) + func m(_ arg: Int) { + #expect(arg < 0) + } + + @Test(.hidden, arguments: [1]) + func n(_ arg: Int) { + #expect(arg > 0) + } } From 8f61da7c4df6d3adbcd48473e6fd52605487b71d Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Tue, 25 Feb 2025 19:00:18 +0100 Subject: [PATCH 12/14] add a minor arguments to `numberOfTestsCasesAtTestEnd` test --- Tests/TestingTests/EventRecorderTests.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 4a73a3560..863418d01 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -184,6 +184,7 @@ struct EventRecorderTests { "number of arguments based on verbosity level at the end of test run", arguments: [ ("f()", #".* Test f\(\) failed after .*"#), + ("h()", #".* Test h\(\) passed after .+"#), ("l(_:)", #".* Test l\(_:\) with .+ test cases passed after.*"#), ("m(_:)", #".* Test m\(_:\) with .+ test cases failed after.*"#), ("n(_:)", #".* Test n\(_:\) with .+ test case passed after.*"#), From 5c52756cfbd9590eeaca62627b701dbf56ccc6f3 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Tue, 25 Feb 2025 20:37:55 +0100 Subject: [PATCH 13/14] Remove `testCasesCount` type definition --- .../Events/Recorder/Event.HumanReadableOutputRecorder.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 847dd4389..0e856facf 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -64,7 +64,7 @@ extension Event { var knownIssueCount = 0 /// The number of test cases for the test. - var testCasesCount: Int = 0 + var testCasesCount = 0 } /// Data tracked on a per-test basis. From a5bac9719327187d99162706e0749f2f1d427569 Mon Sep 17 00:00:00 2001 From: Sam Rayatnia Date: Tue, 25 Feb 2025 21:18:07 +0100 Subject: [PATCH 14/14] Refactor the test title for `numberOfTestCasesAtTestEnd` test for better clarity --- Tests/TestingTests/EventRecorderTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 863418d01..7712cd973 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -181,7 +181,7 @@ struct EventRecorderTests { @available(_regexAPI, *) @Test( - "number of arguments based on verbosity level at the end of test run", + "Log the total number of test cases in parameterized tests at the end of the test run", arguments: [ ("f()", #".* Test f\(\) failed after .*"#), ("h()", #".* Test h\(\) passed after .+"#), @@ -191,7 +191,7 @@ struct EventRecorderTests { ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .*"#), ] ) - func numberOfTestsCasesAtTestEnd(testName: String, expectedPattern: String) async throws { + func numberOfTestCasesAtTestEnd(testName: String, expectedPattern: String) async throws { let stream = Stream() var configuration = Configuration()