diff --git a/Benchmarks/.gitignore b/Benchmarks/.gitignore new file mode 100644 index 0000000..30bcfa4 --- /dev/null +++ b/Benchmarks/.gitignore @@ -0,0 +1 @@ +.build/ diff --git a/Benchmarks/.gitkeep b/Benchmarks/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/Benchmarks/Benchmarks/TracingBenchmarks/AtrributeDSLBenchmark.swift b/Benchmarks/Benchmarks/TracingBenchmarks/AtrributeDSLBenchmark.swift new file mode 100644 index 0000000..ac4d38b --- /dev/null +++ b/Benchmarks/Benchmarks/TracingBenchmarks/AtrributeDSLBenchmark.swift @@ -0,0 +1,87 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Distributed Tracing open source project +// +// Copyright (c) 2020-2023 Apple Inc. and the Swift Distributed Tracing project +// authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Tracing + +let benchmarks = { + let defaultMetrics: [BenchmarkMetric] = [ + .mallocCountTotal, + ] + + Benchmark( + "NoopTracing.startSpan/endSpan", + configuration: .init( + metrics: defaultMetrics, + timeUnits: .nanoseconds, + scalingFactor: .mega + ) + ) { benchmark in + let span = startSpan("name") + defer { span.end() } + } + + Benchmark( + "NoopTracing.attribute: set, span.attributes['http.status_code'] = 200", + configuration: .init( + metrics: defaultMetrics, + timeUnits: .nanoseconds, + scalingFactor: .mega + ) + ) { benchmark in + let span = startSpan("name") + span.attributes["http.status_code"] = 200 + defer { span.end() } + } + + Benchmark( + "NoopTracing.attribute: set, span.attributes.http.status_code = 200", + configuration: .init( + metrics: defaultMetrics, + timeUnits: .nanoseconds, + scalingFactor: .mega + ) + ) { benchmark in + let span = startSpan("name") + span.attributes.http.statusCode = 200 + defer { span.end() } + } +} + +@dynamicMemberLookup +struct HTTPAttributes: SpanAttributeNamespace { + var attributes: SpanAttributes + + init(attributes: SpanAttributes) { + self.attributes = attributes + } + + struct NestedSpanAttributes: NestedSpanAttributesProtocol { + init() {} + + var method: Key { "http.method" } + var statusCode: Key { "http.status_code" } + } +} + +extension SpanAttributes { + var http: HTTPAttributes { + get { + .init(attributes: self) + } + set { + self = newValue.attributes + } + } +} diff --git a/Benchmarks/Package.resolved b/Benchmarks/Package.resolved new file mode 100644 index 0000000..f606143 --- /dev/null +++ b/Benchmarks/Package.resolved @@ -0,0 +1,86 @@ +{ + "pins" : [ + { + "identity" : "hdrhistogram-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/HdrHistogram/hdrhistogram-swift", + "state" : { + "revision" : "a69fa24d7b70421870cafa86340ece900489e17e", + "version" : "0.1.2" + } + }, + { + "identity" : "package-benchmark", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/package-benchmark.git", + "state" : { + "revision" : "ddf6c1ae01e139120bcdb917ece52819ee69d47a", + "version" : "1.22.1" + } + }, + { + "identity" : "package-jemalloc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/package-jemalloc", + "state" : { + "revision" : "e8a5db026963f5bfeac842d9d3f2cc8cde323b49", + "version" : "1.0.0" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "c8ed701b513cf5177118a175d85fbbbcd707ab41", + "version" : "1.3.0" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics", + "state" : { + "revision" : "cd142fd2f64be2100422d658e7411e39489da985", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", + "version" : "1.0.2" + } + }, + { + "identity" : "swift-service-context", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-service-context.git", + "state" : { + "revision" : "ce0141c8f123132dbd02fd45fea448018762df1b", + "version" : "1.0.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system", + "state" : { + "revision" : "025bcb1165deab2e20d4eaba79967ce73013f496", + "version" : "1.2.1" + } + }, + { + "identity" : "texttable", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/TextTable", + "state" : { + "revision" : "a27a07300cf4ae322e0079ca0a475c5583dd575f", + "version" : "0.0.2" + } + } + ], + "version" : 2 +} diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift new file mode 100644 index 0000000..db61de8 --- /dev/null +++ b/Benchmarks/Package.swift @@ -0,0 +1,40 @@ +// swift-tools-version: 5.9 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftCertificates open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftCertificates project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import PackageDescription + +let package = Package( + name: "benchmarks", + platforms: [ + .macOS("14"), + ], + dependencies: [ + .package(path: "../"), + .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.22.0"), + ], + targets: [ + .executableTarget( + name: "TracingBenchmarks", + dependencies: [ + .product(name: "Benchmark", package: "package-benchmark"), + .product(name: "Tracing", package: "swift-distributed-tracing"), + ], + path: "Benchmarks/TracingBenchmarks", + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark") + ] + ), + ] +) \ No newline at end of file diff --git a/Package.swift b/Package.swift index c7e6776..285915c 100644 --- a/Package.swift +++ b/Package.swift @@ -43,24 +43,5 @@ let package = Package( .target(name: "Tracing"), ] ), - - // ==== -------------------------------------------------------------------------------------------------------- - // MARK: Performance / Benchmarks - - .executableTarget( - name: "_TracingBenchmarks", - dependencies: [ - .product(name: "ServiceContextModule", package: "swift-service-context"), - .target(name: "Tracing"), - .target(name: "_TracingBenchmarkTools"), - ] - ), - .target( - name: "_TracingBenchmarkTools", - dependencies: [ - .target(name: "Instrumentation"), - ], - exclude: ["README_SWIFT.md"] - ), ] ) diff --git a/Sources/_TracingBenchmarkTools/ArgParser.swift b/Sources/_TracingBenchmarkTools/ArgParser.swift deleted file mode 100644 index c49e103..0000000 --- a/Sources/_TracingBenchmarkTools/ArgParser.swift +++ /dev/null @@ -1,263 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift Distributed Tracing open source project -// -// Copyright (c) 2020-2023 Apple Inc. and the Swift Distributed Tracing project -// authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -//===--- ArgParse.swift ---------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import Foundation -@_spi(Locking) import Instrumentation - -enum ArgumentError: Error { - case missingValue(String) - case invalidType(value: String, type: String, argument: String?) - case unsupportedArgument(String) -} - -extension ArgumentError: CustomStringConvertible { - public var description: String { - switch self { - case .missingValue(let key): - return "missing value for '\(key)'" - case .invalidType(let value, let type, let argument): - return (argument == nil) - ? "'\(value)' is not a valid '\(type)'" - : "'\(value)' is not a valid '\(type)' for '\(argument!)'" - case .unsupportedArgument(let argument): - return "unsupported argument '\(argument)'" - } - } -} - -/// Type-checked parsing of the argument value. -/// -/// - Returns: Typed value of the argument converted using the `parse` function. -/// -/// - Throws: `ArgumentError.invalidType` when the conversion fails. -func checked( - _ parse: (String) throws -> T?, - _ value: String, - argument: String? = nil -) throws -> T { - if let t = try parse(value) { return t } - var type = "\(T.self)" - if type.starts(with: "Optional<") { - let s = type.index(after: type.firstIndex(of: "<")!) - let e = type.index(before: type.endIndex) // ">" - type = String(type[s ..< e]) // strip Optional< > - } - throw ArgumentError.invalidType( - value: value, type: type, argument: argument - ) -} - -/// Parser that converts the program's command line arguments to typed values -/// according to the parser's configuration, storing them in the provided -/// instance of a value-holding type. -class ArgumentParser { - private var result: U - private var validOptions: [String] { - self.arguments.compactMap(\.name) - } - - private var arguments: [Argument] = [] - private let programName: String = { - // Strip full path from the program name. - let r = ProcessInfo.processInfo.arguments[0].reversed() - let ss = r[r.startIndex ..< (r.firstIndex(of: "/") ?? r.endIndex)] - return String(ss.reversed()) - }() - - private var positionalArgs = [String]() - private var optionalArgsMap = [String: String]() - - /// Argument holds the name of the command line parameter, its help - /// desciption and a rule that's applied to process it. - /// - /// The the rule is typically a value processing closure used to convert it - /// into given type and storing it in the parsing result. - /// - /// See also: addArgument, parseArgument - struct Argument { - let name: String? - let help: String? - let apply: () throws -> Void - } - - /// ArgumentParser is initialized with an instance of a type that holds - /// the results of the parsing of the individual command line arguments. - init(into result: U) { - self.result = result - self.arguments += [ - Argument( - name: "--help", help: "show this help message and exit", - apply: self.printUsage - ), - ] - } - - private func printUsage() { - guard let _ = optionalArgsMap["--help"] else { return } - let space = " " - let maxLength = self.arguments.compactMap { $0.name?.count }.max()! - let padded = { (s: String) in - " \(s)\(String(repeating: space, count: maxLength - s.count)) " - } - let f: (String, String) -> String = { - "\(padded($0))\($1)" - .split(separator: "\n") - .joined(separator: "\n" + padded("")) - } - let positional = f("TEST", "name or number of the benchmark to measure") - let optional = self.arguments.filter { $0.name != nil } - .map { f($0.name!, $0.help ?? "") } - .joined(separator: "\n") - print( - """ - usage: \(self.programName) [--argument=VALUE] [TEST [TEST ...]] - positional arguments: - \(positional) - optional arguments: - \(optional) - """) - exit(0) - } - - /// Parses the command line arguments, returning the result filled with - /// specified argument values or report errors and exit the program if - /// the parsing fails. - public func parse() -> U { - do { - try self.parseArgs() // parse the argument syntax - try self.arguments.forEach { try $0.apply() } // type-check and store values - return self.result - } catch let error as ArgumentError { - lockedStderr.withValue { stderr in - _ = fputs("error: \(error)\n", stderr) - } - exit(1) - } catch { - lockedStdout.withValue { stdout in - _ = fflush(stdout) - } - fatalError("\(error)") - } - } - - /// Using CommandLine.arguments, parses the structure of optional and - /// positional arguments of this program. - /// - /// We assume that optional switch args are of the form: - /// - /// --opt-name[=opt-value] - /// -opt-name[=opt-value] - /// - /// with `opt-name` and `opt-value` not containing any '=' signs. Any - /// other option passed in is assumed to be a positional argument. - /// - /// - Throws: `ArgumentError.unsupportedArgument` on failure to parse - /// the supported argument syntax. - private func parseArgs() throws { - // For each argument we are passed... - let arguments = ProcessInfo.processInfo.arguments - for arg in arguments[1 ..< arguments.count] { - // If the argument doesn't match the optional argument pattern. Add - // it to the positional argument list and continue... - if !arg.starts(with: "-") { - self.positionalArgs.append(arg) - continue - } - // Attempt to split it into two components separated by an equals sign. - let components = arg.split(separator: "=") - let optionName = String(components[0]) - guard self.validOptions.contains(optionName) else { - throw ArgumentError.unsupportedArgument(arg) - } - var optionVal: String - switch components.count { - case 1: optionVal = "" - case 2: optionVal = String(components[1]) - default: - // If we do not have two components at this point, we can not have - // an option switch. This is an invalid argument. Bail! - throw ArgumentError.unsupportedArgument(arg) - } - self.optionalArgsMap[optionName] = optionVal - } - } - - /// Add a rule for parsing the specified argument. - /// - /// Stores the type-erased invocation of the `parseArgument` in `Argument`. - /// - /// Parameters: - /// - name: Name of the command line argument. E.g.: `--opt-arg`. - /// `nil` denotes positional arguments. - /// - property: Property on the `result`, to store the value into. - /// - defaultValue: Value used when the command line argument doesn't - /// provide one. - /// - help: Argument's description used when printing usage with `--help`. - /// - parser: Function that converts the argument value to given type `T`. - public func addArgument( - _ name: String?, - _ property: WritableKeyPath, - defaultValue: T? = nil, - help: String? = nil, - parser: @escaping (String) throws -> T? = { _ in nil } - ) { - self.arguments.append( - Argument(name: name, help: help) - { try self.parseArgument(name, property, defaultValue, parser) } - ) - } - - /// Process the specified command line argument. - /// - /// For optional arguments that have a value we attempt to convert it into - /// given type using the supplied parser, performing the type-checking with - /// the `checked` function. - /// If the value is empty the `defaultValue` is used instead. - /// The typed value is finally stored in the `result` into the specified - /// `property`. - /// - /// For the optional positional arguments, the [String] is simply assigned - /// to the specified property without any conversion. - /// - /// See `addArgument` for detailed parameter descriptions. - private func parseArgument( - _ name: String?, - _ property: WritableKeyPath, - _ defaultValue: T?, - _ parse: (String) throws -> T? - ) throws { - if let name = name, let value = optionalArgsMap[name] { - guard !value.isEmpty || defaultValue != nil - else { throw ArgumentError.missingValue(name) } - - self.result[keyPath: property] = value.isEmpty - ? defaultValue! - : try checked(parse, value, argument: name) - } else if name == nil { - self.result[keyPath: property] = self.positionalArgs as! T - } - } -} diff --git a/Sources/_TracingBenchmarkTools/BenchmarkCategory.swift b/Sources/_TracingBenchmarkTools/BenchmarkCategory.swift deleted file mode 100644 index 4acdaf3..0000000 --- a/Sources/_TracingBenchmarkTools/BenchmarkCategory.swift +++ /dev/null @@ -1,29 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift Distributed Tracing open source project -// -// Copyright (c) 2020-2023 Apple Inc. and the Swift Distributed Tracing project -// authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -//===----------------------------------------------------------------------===// -// -// Based on: https://github.com/apple/swift/tree/cf53143a47278c2a465409a67376642515956777/benchmark/utils -// -//===----------------------------------------------------------------------===// - -public enum BenchmarkCategory: String, Sendable { - // Most benchmarks are assumed to be "stable" and will be regularly tracked at - // each commit. A handful may be marked unstable if continually tracking them is - // counterproductive. - case unstable - - // Explicit skip marker - case skip -} diff --git a/Sources/_TracingBenchmarkTools/BenchmarkTools.swift b/Sources/_TracingBenchmarkTools/BenchmarkTools.swift deleted file mode 100644 index 003d6b0..0000000 --- a/Sources/_TracingBenchmarkTools/BenchmarkTools.swift +++ /dev/null @@ -1,244 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift Distributed Tracing open source project -// -// Copyright (c) 2020-2023 Apple Inc. and the Swift Distributed Tracing project -// authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -//===----------------------------------------------------------------------===// -// -// Based on: https://github.com/apple/swift/tree/cf53143a47278c2a465409a67376642515956777/benchmark/utils -// -//===----------------------------------------------------------------------===// - -#if os(Linux) -import Glibc -#else -import Darwin -#endif -import Foundation -@_spi(Locking) import Instrumentation - -extension BenchmarkCategory: CustomStringConvertible { - public var description: String { - self.rawValue - } -} - -extension BenchmarkCategory: Comparable { - public static func < (lhs: BenchmarkCategory, rhs: BenchmarkCategory) -> Bool { - lhs.rawValue < rhs.rawValue - } -} - -public struct BenchmarkPlatformSet: OptionSet, Sendable { - public let rawValue: Int - - public init(rawValue: Int) { - self.rawValue = rawValue - } - - public static let darwin = BenchmarkPlatformSet(rawValue: 1 << 0) - public static let linux = BenchmarkPlatformSet(rawValue: 1 << 1) - - public static var currentPlatform: BenchmarkPlatformSet { - #if os(Linux) - return .linux - #else - return .darwin - #endif - } - - public static var allPlatforms: BenchmarkPlatformSet { - [.darwin, .linux] - } -} - -public struct BenchmarkInfo: Sendable { - /// The name of the benchmark that should be displayed by the harness. - public var name: String - - /// Shadow static variable for runFunction. - private var _runFunction: @Sendable (Int) -> Void - - /// A function that invokes the specific benchmark routine. - public var runFunction: ((Int) -> Void)? { - if !self.shouldRun { - return nil - } - return self._runFunction - } - - /// A set of category tags that describe this benchmark. This is used by the - /// harness to allow for easy slicing of the set of benchmarks along tag - /// boundaries, e.x.: run all string benchmarks or ref count benchmarks, etc. - public var tags: Set - - /// The platforms that this benchmark supports. This is an OptionSet. - private var unsupportedPlatforms: BenchmarkPlatformSet - - /// Shadow variable for setUpFunction. - private var _setUpFunction: (@Sendable () -> Void)? - - /// An optional function that if non-null is run before benchmark samples - /// are timed. - public var setUpFunction: (() -> Void)? { - if !self.shouldRun { - return nil - } - return self._setUpFunction - } - - /// Shadow static variable for computed property tearDownFunction. - private var _tearDownFunction: (@Sendable () -> Void)? - - /// An optional function that if non-null is run after samples are taken. - public var tearDownFunction: (() -> Void)? { - if !self.shouldRun { - return nil - } - return self._tearDownFunction - } - - public var legacyFactor: Int? - - public init( - name: String, runFunction: @escaping @Sendable (Int) -> Void, tags: [BenchmarkCategory], - setUpFunction: (@Sendable () -> Void)? = nil, - tearDownFunction: (@Sendable () -> Void)? = nil, - unsupportedPlatforms: BenchmarkPlatformSet = [], - legacyFactor: Int? = nil - ) { - self.name = name - self._runFunction = runFunction - self.tags = Set(tags) - self._setUpFunction = setUpFunction - self._tearDownFunction = tearDownFunction - self.unsupportedPlatforms = unsupportedPlatforms - self.legacyFactor = legacyFactor - } - - /// Returns true if this benchmark should be run on the current platform. - var shouldRun: Bool { - !self.unsupportedPlatforms.contains(.currentPlatform) - } -} - -extension BenchmarkInfo: Comparable { - public static func < (lhs: BenchmarkInfo, rhs: BenchmarkInfo) -> Bool { - lhs.name < rhs.name - } - - public static func == (lhs: BenchmarkInfo, rhs: BenchmarkInfo) -> Bool { - lhs.name == rhs.name - } -} - -extension BenchmarkInfo: Hashable { - public func hash(into hasher: inout Hasher) { - hasher.combine(self.name) - } -} - -// Linear function shift register. -// -// This is just to drive benchmarks. I don't make any claim about its -// strength. According to Wikipedia, it has the maximal period for a -// 32-bit register. -struct LFSR { - // Set the register to some seed that I pulled out of a hat. - var lfsr: UInt32 = 0xB789_78E7 - - mutating func shift() { - self.lfsr = (self.lfsr >> 1) ^ (UInt32(bitPattern: -Int32(self.lfsr & 1)) & 0xD000_0001) - } - - mutating func randInt() -> Int64 { - var result: UInt32 = 0 - for _ in 0 ..< 32 { - result = (result << 1) | (self.lfsr & 1) - self.shift() - } - return Int64(bitPattern: UInt64(result)) - } -} - -let lfsrRandomGenerator: LockedValueBox = .init(LFSR()) - -// Start the generator from the beginning -public func SRand() { - lfsrRandomGenerator.withValue { lfsr in - lfsr = LFSR() - } -} - -public func Random() -> Int64 { - lfsrRandomGenerator.withValue { lfsr in - lfsr.randInt() - } -} - -// Can't access stdout/stderr directly in strict concurrency checking mode. -// let lockedStdout = LockedValueBox(stdout) -// let lockedStderr = LockedValueBox(stderr) - -let lockedStdout = LockedValueBox(fdopen(STDOUT_FILENO, "w")) -let lockedStderr = LockedValueBox(fdopen(STDERR_FILENO, "w")) - -@inlinable -@inline(__always) -public func CheckResults( - _ resultsMatch: Bool, - file: StaticString = #file, - function: StaticString = #function, - line: Int = #line -) { - guard _fastPath(resultsMatch) else { - print("Incorrect result in \(function), \(file):\(line)") - abort() - } -} - -public func False() -> Bool { false } - -/// This is a dummy protocol to test the speed of our protocol dispatch. -public protocol SomeProtocol { func getValue() -> Int } -struct MyStruct: SomeProtocol { - init() {} - func getValue() -> Int { 1 } -} - -public func someProtocolFactory() -> SomeProtocol { MyStruct() } - -// Just consume the argument. -// It's important that this function is in another module than the tests -// which are using it. -@inline(never) -public func blackHole(_: T) {} - -// Return the passed argument without letting the optimizer know that. -@inline(never) -public func identity(_ x: T) -> T { - x -} - -// Return the passed argument without letting the optimizer know that. -// It's important that this function is in another module than the tests -// which are using it. -@inline(never) -public func getInt(_ x: Int) -> Int { x } - -// The same for String. -@inline(never) -public func getString(_ s: String) -> String { s } - -// The same for Substring. -@inline(never) -public func getSubstring(_ s: Substring) -> Substring { s } diff --git a/Sources/_TracingBenchmarkTools/DriverUtils.swift b/Sources/_TracingBenchmarkTools/DriverUtils.swift deleted file mode 100644 index e45b30f..0000000 --- a/Sources/_TracingBenchmarkTools/DriverUtils.swift +++ /dev/null @@ -1,731 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the Swift Distributed Tracing open source project -// -// Copyright (c) 2020-2023 Apple Inc. and the Swift Distributed Tracing project -// authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -//===--- DriverUtils.swift ------------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -#if os(Linux) -import Glibc -#else -import Darwin -// import LibProc -#endif -@_spi(Locking) import Instrumentation - -public struct BenchResults { - typealias T = Int - private let samples: [T] - let maxRSS: Int? - let stats: Stats - - init(_ samples: [T], maxRSS: Int?) { - self.samples = samples.sorted() - self.maxRSS = maxRSS - self.stats = self.samples.reduce(into: Stats(), Stats.collect) - } - - /// Return measured value for given `quantile`. - /// - /// Equivalent to quantile estimate type R-1, SAS-3. See: - /// https://en.wikipedia.org/wiki/Quantile#Estimating_quantiles_from_a_sample - subscript(_ quantile: Double) -> T { - let index = Swift.max( - 0, - Int((Double(self.samples.count) * quantile).rounded(.up)) - 1 - ) - return self.samples[index] - } - - var sampleCount: T { self.samples.count } - var min: T { self.samples.first! } - var max: T { self.samples.last! } - var mean: T { Int(self.stats.mean.rounded()) } - var sd: T { Int(self.stats.standardDeviation.rounded()) } - var median: T { self[0.5] } -} - -let registeredBenchmarks: LockedValueBox<[BenchmarkInfo]> = .init([]) -public func internalRegisterBenchmark(_ benchmark: BenchmarkInfo) { - registeredBenchmarks.withValue { benchmarks in - benchmarks.append(benchmark) - } -} - -enum TestAction { - case run - case listTests -} - -struct TestConfig { - /// The delimiter to use when printing output. - let delim: String - - /// Duration of the test measurement in seconds. - /// - /// Used to compute the number of iterations, if no fixed amount is specified. - /// This is useful when one wishes for a test to run for a - /// longer amount of time to perform performance analysis on the test in - /// instruments. - let sampleTime: Double - - /// Number of iterations averaged in the sample. - /// When not specified, we'll compute the number of iterations to be averaged - /// in the sample from the actual runtime and the desired `sampleTime`. - let numIters: Int? - - /// The number of samples we should take of each test. - let numSamples: Int? - - /// Quantiles to report in results. - let quantile: Int? - - /// Time unit in which to report results (nanos, micros, millis) (default: nanoseconds) - let timeUnit: TimeUnit - - /// Report quantiles with delta encoding. - let delta: Bool - - /// Is verbose output enabled? - let verbose: Bool - - // Should we log the test's memory usage? - let logMemory: Bool - - /// After we run the tests, should the harness sleep to allow for utilities - /// like leaks that require a PID to run on the test harness. - let afterRunSleep: UInt32? - - /// The list of tests to run. - let tests: [(index: String, info: BenchmarkInfo)] - - let action: TestAction - - init(_ registeredBenchmarks: [BenchmarkInfo]) { - struct PartialTestConfig { - var delim: String? - var tags, skipTags: Set? - var numSamples: UInt? - var numIters: UInt? - var quantile: UInt? - var timeUnit: String? - var delta: Bool? - var afterRunSleep: UInt32? - var sampleTime: Double? - var verbose: Bool? - var logMemory: Bool? - var action: TestAction? - var tests: [String]? - } - - // Custom value type parsers - func tags(tags: String) throws -> Set { - // We support specifying multiple tags by splitting on comma, i.e.: - // --tags=Array,Dictionary - // --skip-tags=Array,Set,unstable,skip - try Set( - tags.split(separator: ",").map(String.init).map { - try checked({ BenchmarkCategory(rawValue: $0) }, $0) - } - ) - } - func finiteDouble(value: String) -> Double? { - Double(value).flatMap { $0.isFinite ? $0 : nil } - } - - // Configure the command line argument parser - let p = ArgumentParser(into: PartialTestConfig()) - p.addArgument( - "--num-samples", \.numSamples, - help: "number of samples to take per benchmark;\n" + - "default: 1 or auto-scaled to measure for\n" + - "`sample-time` if num-iters is also specified\n", - parser: { UInt($0) } - ) - p.addArgument( - "--num-iters", \.numIters, - help: "number of iterations averaged in the sample;\n" + - "default: auto-scaled to measure for `sample-time`", - parser: { UInt($0) } - ) - p.addArgument( - "--quantile", \.quantile, - help: "report quantiles instead of normal dist. stats;\n" + - "use 4 to get a five-number summary with quartiles,\n" + - "10 (deciles), 20 (ventiles), 100 (percentiles), etc.", - parser: { UInt($0) } - ) - p.addArgument( - "--time-unit", \.timeUnit, - help: "time unit to be used for reported measurements;\n" + - "supported values: ns, us, ms; default: ns", - parser: { $0 } - ) - p.addArgument( - "--delta", \.delta, defaultValue: true, - help: "report quantiles with delta encoding" - ) - p.addArgument( - "--sample-time", \.sampleTime, - help: "duration of test measurement in seconds\ndefault: 1", - parser: finiteDouble - ) - p.addArgument( - "--verbose", \.verbose, defaultValue: true, - help: "increase output verbosity" - ) - p.addArgument( - "--memory", \.logMemory, defaultValue: true, - help: "log the change in maximum resident set size (MAX_RSS)" - ) - p.addArgument( - "--delim", \.delim, - help: "value delimiter used for log output; default: ,", - parser: { $0 } - ) - p.addArgument( - "--tags", \PartialTestConfig.tags, - help: "run tests matching all the specified categories", - parser: tags - ) - p.addArgument( - "--skip-tags", \PartialTestConfig.skipTags, defaultValue: [], - help: "don't run tests matching any of the specified\n" + - "categories; default: unstable,skip", - parser: tags - ) - p.addArgument( - "--sleep", \.afterRunSleep, - help: "number of seconds to sleep after benchmarking", - parser: { UInt32($0) } - ) - p.addArgument( - "--list", \.action, defaultValue: .listTests, - help: "don't run the tests, just log the list of test \n" + - "numbers, names and tags (respects specified filters)" - ) - p.addArgument(nil, \.tests) // positional arguments - let c = p.parse() - - // Configure from the command line arguments, filling in the defaults. - self.delim = c.delim ?? "," - self.sampleTime = c.sampleTime ?? 1.0 - self.numIters = c.numIters.map { Int($0) } - self.numSamples = c.numSamples.map { Int($0) } - self.quantile = c.quantile.map { Int($0) } - self.timeUnit = c.timeUnit.map { TimeUnit($0) } ?? TimeUnit.nanoseconds - self.delta = c.delta ?? false - self.verbose = c.verbose ?? false - self.logMemory = c.logMemory ?? false - self.afterRunSleep = c.afterRunSleep - self.action = c.action ?? .run - self.tests = TestConfig.filterTests( - registeredBenchmarks, - specifiedTests: Set(c.tests ?? []), - tags: c.tags ?? [], - skipTags: c.skipTags ?? [.unstable, .skip] - ) - - if self.logMemory, self.tests.count > 1 { - print( - """ - warning: The memory usage of a test, reported as the change in MAX_RSS, - is based on measuring the peak memory used by the whole process. - These results are meaningful only when running a single test, - not in the batch mode! - """) - } - - // We always prepare the configuration string and call the print to have - // the same memory usage baseline between verbose and normal mode. - let testList = self.tests.map(\.1.name).joined(separator: ", ") - let configuration = """ - --- CONFIG --- - NumSamples: \(numSamples ?? 0) - Verbose: \(verbose) - LogMemory: \(logMemory) - SampleTime: \(sampleTime) - NumIters: \(numIters ?? 0) - Quantile: \(quantile ?? 0) - TimeUnit: \(timeUnit) - Delimiter: \(String(reflecting: delim)) - Tests Filter: \(c.tests ?? []) - Tests to run: \(testList) - --- DATA ---\n - """ - print(self.verbose ? configuration : "", terminator: "") - } - - /// Returns the list of tests to run. - /// - /// - Parameters: - /// - registeredBenchmarks: List of all performance tests to be filtered. - /// - specifiedTests: List of explicitly specified tests to run. These can - /// be specified either by a test name or a test number. - /// - tags: Run tests tagged with all of these categories. - /// - skipTags: Don't run tests tagged with any of these categories. - /// - Returns: An array of test number and benchmark info tuples satisfying - /// specified filtering conditions. - static func filterTests( - _ registeredBenchmarks: [BenchmarkInfo], - specifiedTests: Set, - tags: Set, - skipTags: Set - ) -> [(index: String, info: BenchmarkInfo)] { - let allTests = registeredBenchmarks.sorted() - let indices = Dictionary( - uniqueKeysWithValues: - zip( - allTests.map(\.name), - (1...).lazy.map { String($0) } - ) - ) - - func byTags(b: BenchmarkInfo) -> Bool { - b.tags.isSuperset(of: tags) && - b.tags.isDisjoint(with: skipTags) - } - func byNamesOrIndices(b: BenchmarkInfo) -> Bool { - specifiedTests.contains(b.name) || - specifiedTests.contains(indices[b.name]!) - } // !! "`allTests` have been assigned an index" - return allTests - .filter(specifiedTests.isEmpty ? byTags : byNamesOrIndices) - .map { (index: indices[$0.name]!, info: $0) } - } -} - -struct Stats { - var n: Int = 0 - var S: Double = 0.0 - var mean: Double = 0.0 - var variance: Double { self.n < 2 ? 0.0 : self.S / Double(self.n - 1) } - var standardDeviation: Double { self.variance.squareRoot() } - - static func collect(_ s: inout Stats, _ x: Int) { - Stats.runningMeanVariance(&s, Double(x)) - } - - /// Compute running mean and variance using B. P. Welford's method. - /// - /// See Knuth TAOCP vol 2, 3rd edition, page 232, or - /// https://www.johndcook.com/blog/standard_deviation/ - static func runningMeanVariance(_ s: inout Stats, _ x: Double) { - let n = s.n + 1 - let (k, M_, S_) = (Double(n), s.mean, s.S) - let M = M_ + (x - M_) / k - let S = S_ + (x - M_) * (x - M) - (s.n, s.mean, s.S) = (n, M, S) - } -} - -#if SWIFT_RUNTIME_ENABLE_LEAK_CHECKER - -@_silgen_name("_swift_leaks_startTrackingObjects") -func startTrackingObjects(_: UnsafePointer) -> Void -@_silgen_name("_swift_leaks_stopTrackingObjects") -func stopTrackingObjects(_: UnsafePointer) -> Int - -#endif - -public final class Timer { - #if os(Linux) - public typealias TimeT = timespec - - public init() {} - - public func getTime() -> TimeT { - var ts = timespec(tv_sec: 0, tv_nsec: 0) - clock_gettime(CLOCK_REALTIME, &ts) - return ts - } - - public func getTimeAsInt() -> UInt64 { - UInt64(getTime().tv_nsec) - } - - public func diffTimeInNanoSeconds(from start: TimeT, to end: TimeT) -> UInt64 { - let oneSecond = 1_000_000_000 // ns - var elapsed = timespec(tv_sec: 0, tv_nsec: 0) - if end.tv_nsec - start.tv_nsec < 0 { - elapsed.tv_sec = end.tv_sec - start.tv_sec - 1 - elapsed.tv_nsec = end.tv_nsec - start.tv_nsec + oneSecond - } else { - elapsed.tv_sec = end.tv_sec - start.tv_sec - elapsed.tv_nsec = end.tv_nsec - start.tv_nsec - } - return UInt64(elapsed.tv_sec) * UInt64(oneSecond) + UInt64(elapsed.tv_nsec) - } - - #else - public typealias TimeT = UInt64 - var info = mach_timebase_info_data_t(numer: 0, denom: 0) - - public init() { - mach_timebase_info(&info) - } - - public func getTime() -> TimeT { - mach_absolute_time() - } - - public func getTimeAsInt() -> UInt64 { - UInt64(getTime()) - } - - public func diffTimeInNanoSeconds(from start: TimeT, to end: TimeT) -> UInt64 { - let elapsed = end - start - return elapsed * UInt64(info.numer) / UInt64(info.denom) - } - #endif -} - -extension UInt64 { - public var nanoseconds: Int { Int(self) } - public var microseconds: Int { Int(self / 1000) } - public var milliseconds: Int { Int(self / 1000 / 1000) } - public var seconds: Int { Int(self / 1000 / 1000 / 1000) } -} - -enum TimeUnit: String { - case nanoseconds = "ns" - case microseconds = "μs" - case milliseconds = "ms" - case seconds = "s" - - init(_ from: String) { - switch from { - case "ns": self = .nanoseconds - case "us", "μs": self = .microseconds - case "ms": self = .milliseconds - case "s": self = .seconds - default: fatalError("Only the following time units are supported: ns, us, ms, s") - } - } - - static var `default` = TimeUnit.nanoseconds -} - -extension TimeUnit: CustomStringConvertible { - public var description: String { - self.rawValue - } -} - -/// Performance test runner that measures benchmarks and reports the results. -final class TestRunner { - let c: TestConfig - let timer = Timer() - var start, end, lastYield: Timer.TimeT - let baseline = TestRunner.getResourceUtilization() - let schedulerQuantum = UInt64(10_000_000) // nanoseconds (== 10ms, macos) - init(_ config: TestConfig) { - self.c = config - let now = timer.getTime() - (start, end, lastYield) = (now, now, now) - } - - /// Offer to yield CPU to other processes and return current time on resume. - func yield() -> Timer.TimeT { - sched_yield() - return timer.getTime() - } - - #if os(Linux) - private static func getExecutedInstructions() -> UInt64 { - // FIXME: there is a Linux PMC API you can use to get this, but it's - // not quite so straightforward. - 0 - } - - #else - private static func getExecutedInstructions() -> UInt64 { -// if #available(OSX 10.9, iOS 7.0, *) { -// var u = rusage_info_v4() -// let p = UnsafeMutablePointer(&u) -// p.withMemoryRebound(to: Optional.self, capacity: 1) { up in -// let _ = proc_pid_rusage(getpid(), RUSAGE_INFO_V4, up) -// } -// return u.ri_instructions -// } else { - 0 -// } - } - #endif - - private static func getResourceUtilization() -> rusage { - #if canImport(Darwin) - let rusageSelf = RUSAGE_SELF - #else - let rusageSelf = RUSAGE_SELF.rawValue - #endif - var u = rusage(); getrusage(rusageSelf, &u); return u - } - - /// Returns maximum resident set size (MAX_RSS) delta in bytes. - /// - /// This method of estimating memory usage is valid only for executing single - /// benchmark. That's why we don't worry about reseting the `baseline` in - /// `resetMeasurements`. - /// - // FIXME: This current implementation doesn't work on Linux. It is disabled - /// permanently to avoid linker errors. Feel free to fix. - func measureMemoryUsage() -> Int? { - #if os(Linux) - return nil - #else - guard c.logMemory else { return nil } - let current = TestRunner.getResourceUtilization() - let maxRSS = current.ru_maxrss - baseline.ru_maxrss - #if canImport(Darwin) - let pageSize = _SC_PAGESIZE - #else - let pageSize = Int32(_SC_PAGESIZE) - #endif - let pages = { maxRSS / sysconf(pageSize) } - func deltaEquation(_ stat: KeyPath) -> String { - let b = baseline[keyPath: stat], c = current[keyPath: stat] - return "\(c) - \(b) = \(c - b)" - } - logVerbose( - """ - MAX_RSS \(deltaEquation(\rusage.ru_maxrss)) (\(pages()) pages) - ICS \(deltaEquation(\rusage.ru_nivcsw)) - VCS \(deltaEquation(\rusage.ru_nvcsw)) - """) - return maxRSS - #endif - } - - private func startMeasurement() { - let spent = timer.diffTimeInNanoSeconds(from: lastYield, to: end) - let nextSampleEstimate = UInt64(Double(lastSampleTime) * 1.5) - - if spent + nextSampleEstimate < schedulerQuantum { - start = timer.getTime() - } else { - logVerbose(" Yielding after ~\(spent.nanoseconds) ns") - let now = yield() - (start, lastYield) = (now, now) - } - } - - private func stopMeasurement() { - end = timer.getTime() - } - - private func resetMeasurements() { - let now = yield() - (start, end, lastYield) = (now, now, now) - } - - /// Time in nanoseconds spent running the last function - var lastSampleTime: UInt64 { - timer.diffTimeInNanoSeconds(from: start, to: end) - } - - /// Measure the `fn` and return the average sample time per iteration (in c.timeUnit). - func measure(_ name: String, fn: (Int) -> Void, numIters: Int) -> Int { - #if SWIFT_RUNTIME_ENABLE_LEAK_CHECKER - name.withCString { p in startTrackingObjects(p) } - #endif - - startMeasurement() - fn(numIters) - stopMeasurement() - - #if SWIFT_RUNTIME_ENABLE_LEAK_CHECKER - name.withCString { p in stopTrackingObjects(p) } - #endif - - switch c.timeUnit { - case .nanoseconds: return lastSampleTime.nanoseconds / numIters - case .microseconds: return lastSampleTime.microseconds / numIters - case .milliseconds: return lastSampleTime.milliseconds / numIters - case .seconds: return lastSampleTime.seconds / numIters - } - } - - func logVerbose(_ msg: @autoclosure () -> String) { - if c.verbose { print(msg()) } - } - - /// Run the benchmark and return the measured results. - func run(_ test: BenchmarkInfo) -> BenchResults? { - // Before we do anything, check that we actually have a function to - // run. If we don't it is because the benchmark is not supported on - // the platform and we should skip it. - guard let testFn = test.runFunction else { - logVerbose("Skipping unsupported benchmark \(test.name)!") - return nil - } - logVerbose("Running \(test.name)") - - var samples: [Int] = [] - - func addSample(_ time: Int) { - logVerbose(" Sample \(samples.count),\(time)") - samples.append(time) - } - - resetMeasurements() - if let setUp = test.setUpFunction { - setUp() - stopMeasurement() - logVerbose(" SetUp \(lastSampleTime.microseconds)") - resetMeasurements() - } - - // Determine number of iterations for testFn to run for desired time. - func iterationsPerSampleTime() -> (numIters: Int, oneIter: Int) { - let oneIter = measure(test.name, fn: testFn, numIters: 1) - if oneIter > 0 { - let timePerSample = Int(c.sampleTime * 1_000_000.0) // microseconds (μs) - return (max(timePerSample / oneIter, 1), oneIter) - } else { - return (1, oneIter) - } - } - - // Determine the scale of measurements. Re-use the calibration result if - // it is just one measurement. - func calibrateMeasurements() -> Int { - let (numIters, oneIter) = iterationsPerSampleTime() - if numIters == 1 { addSample(oneIter) } - else { resetMeasurements() } // for accurate yielding reports - return numIters - } - - let numIters = min( // Cap to prevent overflow on 32-bit systems when scaled - Int.max / 10000, // by the inner loop multiplier inside the `testFn`. - c.numIters ?? calibrateMeasurements() - ) - - let numSamples = c.numSamples ?? min( - 200, // Cap the number of samples - c.numIters == nil ? 1 : calibrateMeasurements() - ) - - samples.reserveCapacity(numSamples) - logVerbose(" Collecting \(numSamples) samples.") - logVerbose(" Measuring with scale \(numIters).") - for _ in samples.count ..< numSamples { - addSample(measure(test.name, fn: testFn, numIters: numIters)) - } - - test.tearDownFunction?() - if let lf = test.legacyFactor { - logVerbose(" Applying legacy factor: \(lf)") - samples = samples.map { $0 * lf } - } - - return BenchResults(samples, maxRSS: measureMemoryUsage()) - } - - var header: String { - let withUnit = { $0 + "(\(self.c.timeUnit))" } - let withDelta = { "𝚫" + $0 } - func quantiles(q: Int) -> [String] { - // See https://en.wikipedia.org/wiki/Quantile#Specialized_quantiles - let prefix = [ - 2: "MEDIAN", 3: "T", 4: "Q", 5: "QU", 6: "S", 7: "O", 10: "D", - 12: "Dd", 16: "H", 20: "V", 33: "TT", 100: "P", 1000: "Pr", - ][q, default: "\(q)-q"] - let base20 = "0123456789ABCDEFGHIJ".map { String($0) } - let index: (Int) -> String = - { q == 2 ? "" : q <= 20 ? base20[$0] : String($0) } - let tail = (1 ..< q).map { prefix + index($0) } + ["MAX"] - return [withUnit("MIN")] + tail.map(c.delta ? withDelta : withUnit) - } - return ( - ["#", "TEST", "SAMPLES"] + - ( - c.quantile.map(quantiles) - ?? ["MIN", "MAX", "MEAN", "SD", "MEDIAN"].map(withUnit) - ) + - (c.logMemory ? ["MAX_RSS(B)"] : []) - ).joined(separator: c.delim) - } - - /// Execute benchmarks and continuously report the measurement results. - func runBenchmarks() { - var testCount = 0 - - func report(_ index: String, _ t: BenchmarkInfo, results: BenchResults?) { - func values(r: BenchResults) -> [String] { - func quantiles(q: Int) -> [Int] { - let qs = (0 ... q).map { i in r[Double(i) / Double(q)] } - return c.delta ? - qs.reduce(into: (encoded: [], last: 0)) { - $0.encoded.append($1 - $0.last); $0.last = $1 - }.encoded : qs - } - return ( - [r.sampleCount] + - ( - c.quantile.map(quantiles) - ?? [r.min, r.max, r.mean, r.sd, r.median] - ) + - [r.maxRSS].compactMap { $0 } - ).map { (c.delta && $0 == 0) ? "" : String($0) } // drop 0s in deltas - } - let benchmarkStats = ( - [index, t.name] + (results.map(values) ?? ["Unsupported"]) - ).joined(separator: c.delim) - - print(benchmarkStats) - lockedStdout.withValue { stdout in - _ = fflush(stdout) - } - - if results != nil { - testCount += 1 - } - } - - print(header) - - for (index, test) in c.tests { - report(index, test, results: run(test)) - } - - print("\nTotal performance tests executed: \(testCount)") - } -} - -public func main() { - let config = TestConfig(registeredBenchmarks.withValue { $0 }) - switch config.action { - case .listTests: - print("#\(config.delim)Test\(config.delim)[Tags]") - for (index, t) in config.tests { - let testDescription = [index, t.name, t.tags.sorted().description] - .joined(separator: config.delim) - print(testDescription) - } - case .run: - TestRunner(config).runBenchmarks() - if let x = config.afterRunSleep { - sleep(x) - } - } -} diff --git a/Sources/_TracingBenchmarkTools/README_SWIFT.md b/Sources/_TracingBenchmarkTools/README_SWIFT.md deleted file mode 100644 index a7be5c3..0000000 --- a/Sources/_TracingBenchmarkTools/README_SWIFT.md +++ /dev/null @@ -1,6 +0,0 @@ -## Swift Benchmark Utils - -> This benchmarking infrastructure is copied from -https://github.com/apple/swift/tree/cf53143a47278c2a465409a67376642515956777/benchmark/utils -with the intent of producing similar look and feel, as well as because we need some benchmark infra. -> When feasible we will aim to collaborate and contribute improvements back to the mainline Swift project.