diff --git a/Package.swift b/Package.swift index a82241527..f770d603d 100644 --- a/Package.swift +++ b/Package.swift @@ -389,6 +389,7 @@ extension Array where Element == PackageDescription.SwiftSetting { // (via CMake). Enabling it is dependent on acceptance of the @section // proposal via Swift Evolution. .enableExperimentalFeature("SymbolLinkageMarkers"), + .enableExperimentalFeature("CompileTimeValuesPreview"), .enableUpcomingFeature("InferIsolatedConformances"), diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index 1aa1362ec..4f703dea8 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -24,7 +24,13 @@ extension ABI.Version { let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder() return { [eventHandler = eventHandlerCopy] event, context in - if case .testDiscovered = event.kind, let test = context.test { + if case let .libraryDiscovered(library) = event.kind { + if let libraryRecord = ABI.Record(encoding: library) { + try? JSON.withEncoding(of: libraryRecord) { libraryJSON in + eventHandler(libraryJSON) + } + } + } else if case .testDiscovered = event.kind, let test = context.test { try? JSON.withEncoding(of: ABI.Record(encoding: test)) { testJSON in eventHandler(testJSON) } @@ -47,24 +53,25 @@ extension ABI.Xcode16 { forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void ) -> Event.Handler { return { event, context in - if case .testDiscovered = event.kind { + switch event.kind { + case .libraryDiscovered, .testDiscovered: // Discard events of this kind rather than forwarding them to avoid a // crash in Xcode 16 (which does not expect any events to occur before // .runStarted.) return - } - - struct EventAndContextSnapshot: Codable { - var event: Event.Snapshot - var eventContext: Event.Context.Snapshot - } - let snapshot = EventAndContextSnapshot( - event: Event.Snapshot(snapshotting: event), - eventContext: Event.Context.Snapshot(snapshotting: context) - ) - try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in - eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in - eventHandler(eventAndContextJSON) + default: + struct EventAndContextSnapshot: Codable { + var event: Event.Snapshot + var eventContext: Event.Context.Snapshot + } + let snapshot = EventAndContextSnapshot( + event: Event.Snapshot(snapshotting: event), + eventContext: Event.Context.Snapshot(snapshotting: context) + ) + try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in + eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in + eventHandler(eventAndContextJSON) + } } } } diff --git a/Sources/Testing/ABI/ABI.Record.swift b/Sources/Testing/ABI/ABI.Record.swift index 40a8d4bc3..b1dbd1bb2 100644 --- a/Sources/Testing/ABI/ABI.Record.swift +++ b/Sources/Testing/ABI/ABI.Record.swift @@ -18,6 +18,11 @@ extension ABI { struct Record: Sendable where V: ABI.Version { /// An enumeration describing the various kinds of record. enum Kind: Sendable { + /// A testing library. + /// + /// - Warning: Testing libraries are not yet part of the JSON schema. + case library(EncodedLibrary) + /// A test record. case test(EncodedTest) @@ -28,6 +33,13 @@ extension ABI { /// The kind of record. var kind: Kind + init?(encoding library: borrowing Library) { + guard V.includesExperimentalFields else { + return nil + } + kind = .library(EncodedLibrary(encoding: library)) + } + init(encoding test: borrowing Test) { kind = .test(EncodedTest(encoding: test)) } @@ -58,6 +70,9 @@ extension ABI.Record: Codable { var container = encoder.container(keyedBy: CodingKeys.self) try container.encode(V.versionNumber, forKey: .version) switch kind { + case let .library(library): + try container.encode("_library", forKey: .kind) + try container.encode(library, forKey: .payload) case let .test(test): try container.encode("test", forKey: .kind) try container.encode(test, forKey: .payload) @@ -81,6 +96,9 @@ extension ABI.Record: Codable { } switch try container.decode(String.self, forKey: .kind) { + case "_library": + let library = try container.decode(ABI.EncodedLibrary.self, forKey: .payload) + kind = .library(library) case "test": let test = try container.decode(ABI.EncodedTest.self, forKey: .payload) kind = .test(test) diff --git a/Sources/Testing/ABI/ABI.swift b/Sources/Testing/ABI/ABI.swift index 7a33970fc..58545577e 100644 --- a/Sources/Testing/ABI/ABI.swift +++ b/Sources/Testing/ABI/ABI.swift @@ -59,6 +59,12 @@ extension ABI { /// - Returns: A type conforming to ``ABI/Version`` that represents the given /// ABI version, or `nil` if no such type exists. static func version(forVersionNumber versionNumber: VersionNumber = ABI.CurrentVersion.versionNumber) -> (any Version.Type)? { + // Special-case the experimental ABI version number (which is intentionally + // higher than any Swift release's version number). + if versionNumber == ExperimentalVersion.versionNumber { + return ExperimentalVersion.self + } + if versionNumber > ABI.HighestVersion.versionNumber { // If the caller requested an ABI version higher than the current Swift // compiler version and it's not an ABI version we've explicitly defined, diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedLibrary.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedLibrary.swift new file mode 100644 index 000000000..4cdd275f6 --- /dev/null +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedLibrary.swift @@ -0,0 +1,39 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension ABI { + /// A type implementing the JSON encoding of ``Library`` for the ABI entry + /// point and event stream output. + /// + /// The properties and members of this type are documented in ABI/JSON.md. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + /// + /// - Warning: Testing libraries are not yet part of the JSON schema. + struct EncodedLibrary: Sendable where V: ABI.Version { + /// The human-readable name of the library. + var name: String + + /// The canonical form of the "hint" to run the testing library's tests at + /// runtime. + var canonicalHint: String + + init(encoding library: borrowing Library) { + name = library.name + canonicalHint = library.canonicalHint + } + } +} + +// MARK: - Codable + +extension ABI.EncodedLibrary: Codable {} diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 6163e7bd1..80e020270 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -30,13 +30,25 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha do { #if !SWT_NO_EXIT_TESTS - // If an exit test was specified, run it. `exitTest` returns `Never`. - if let exitTest = ExitTest.findInEnvironmentForEntryPoint() { - await exitTest() - } + // If an exit test was specified, run it. `exitTest` returns `Never`. + if let exitTest = ExitTest.findInEnvironmentForEntryPoint() { + await exitTest() + } #endif let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments) + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY + // If the user requested a different testing library, run it instead of + // Swift Testing. (If they requested Swift Testing, we're already here so + // there's no real need to recurse). + if args.experimentalListLibraries != true, + let library = args.testingLibrary.flatMap(Library.init(withHint:)), + library.canonicalHint != "swift-testing" { + return await library.callEntryPoint(passing: args) + } +#endif + // Configure the test runner. var configuration = try configurationForEntryPoint(from: args) @@ -91,9 +103,10 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // The set of matching tests (or, in the case of `swift test list`, the set // of all tests.) - let tests: [Test] + var tests = [Test]() + var libraries = [Library]() - if args.listTests ?? false { + if args.listTests == true { tests = await Array(Test.all) if args.verbosity > .min { @@ -112,6 +125,29 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha for test in tests { Event.post(.testDiscovered, for: (test, nil), configuration: configuration) } + } else if args.experimentalListLibraries == true { +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY + libraries = Array(Library.all) +#else + libraries = [Library.swiftTesting] +#endif + + if args.verbosity > .min { + for library in libraries { + // Print the test ID to stdout (classical CLI behavior.) + let libraryDescription = "\(library.name) (swift test --experimental-testing-library \(library.canonicalHint))" +#if SWT_TARGET_OS_APPLE && !SWT_NO_FILE_IO + try? FileHandle.stdout.write("\(libraryDescription)\n") +#else + print(libraryDescription) +#endif + } + } + + // Post an event for every discovered library (as with tests above). + for library in libraries { + Event.post(.libraryDiscovered(library), for: (nil, nil), configuration: configuration) + } } else { // Run the tests. let runner = await Runner(configuration: configuration) @@ -122,7 +158,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // If there were no matching tests, exit with a dedicated exit code so that // the caller (assumed to be Swift Package Manager) can implement special // handling. - if tests.isEmpty { + if tests.isEmpty && libraries.isEmpty { exitCode.withLock { exitCode in if exitCode == EXIT_SUCCESS { exitCode = EXIT_NO_TESTS_FOUND @@ -207,6 +243,9 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--list-tests` argument. public var listTests: Bool? + /// The value of the `--experimental-list-libraries` argument. + public var experimentalListLibraries: Bool? + /// The value of the `--parallel` or `--no-parallel` argument. public var parallel: Bool? @@ -331,6 +370,9 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--attachments-path` argument. public var attachmentsPath: String? + + /// The value of the `--testing-library` argument. + public var testingLibrary: String? } extension __CommandLineArguments_v0: Codable { @@ -338,6 +380,7 @@ extension __CommandLineArguments_v0: Codable { // do not end up with leading underscores when encoded. enum CodingKeys: String, CodingKey { case listTests + case experimentalListLibraries case parallel case experimentalMaximumParallelizationWidth case symbolicateBacktraces @@ -353,6 +396,7 @@ extension __CommandLineArguments_v0: Codable { case repetitions case repeatUntil case attachmentsPath + case testingLibrary } } @@ -466,6 +510,11 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum } #endif + // Testing library + if let testingLibrary = Environment.variable(named: "SWT_EXPERIMENTAL_LIBRARY") ?? args.argumentValue(forLabel: "--testing-library") { + result.testingLibrary = testingLibrary + } + // XML output if let xunitOutputPath = args.argumentValue(forLabel: "--xunit-output") { result.xunitOutput = xunitOutputPath @@ -484,6 +533,9 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum // makes invocation from e.g. Wasmtime a bit more intuitive/idiomatic. result.listTests = true } + if Environment.flag(named: "SWT_EXPERIMENTAL_LIST_LIBRARIES") == true || args.contains("--experimental-list-libraries") { + result.experimentalListLibraries = true + } // Parallelization (on by default) if args.contains("--no-parallel") { diff --git a/Sources/Testing/ABI/EntryPoints/Library.swift b/Sources/Testing/ABI/EntryPoints/Library.swift new file mode 100644 index 000000000..4f12f4d54 --- /dev/null +++ b/Sources/Testing/ABI/EntryPoints/Library.swift @@ -0,0 +1,339 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) private import _TestDiscovery +internal import _TestingInternals + +/// A type representing a testing library such as Swift Testing or XCTest. +@_spi(Experimental) +public struct Library: Sendable { + /// The underlying instance of ``SWTLibrary``. + /// + /// - Important: The in-memory layout of ``Library`` must _exactly_ match the + /// layout of this type. As such, it must not contain any other stored + /// properties. + nonisolated(unsafe) var rawValue: SWTLibrary + + /// The human-readable name of this library. + /// + /// For example, the value of this property for an instance of this type that + /// represents the Swift Testing library is `"Swift Testing"`. + public var name: String { + String(validatingCString: rawValue.name) ?? "" + } + + /// The canonical form of the "hint" to run the testing library's tests at + /// runtime. + /// + /// For example, the value of this property for an instance of this type that + /// represents the Swift Testing library is `"swift-testing"`. + @_spi(ForToolsIntegrationOnly) + public var canonicalHint: String { + String(validatingCString: rawValue.canonicalHint) ?? "" + } + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY + /// Call the entry point function of this library. + /// + /// - Parameters: + /// - args: The arguments to pass to the testing library as its + /// configuration JSON. + /// - recordHandler: A callback to invoke once per record. + /// + /// - Returns: A process exit code such as `EXIT_SUCCESS`. + /// + /// - Warning: The signature of this function is subject to change as + /// `__CommandLineArguments_v0` is not a stable interface. + @_spi(ForToolsIntegrationOnly) + public func callEntryPoint( + passing args: __CommandLineArguments_v0? = nil, + recordHandler: (@Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void)? = nil + ) async -> CInt { + do { + return try await _callEntryPoint(passing: args, recordHandler: recordHandler) + } catch { + // TODO: more advanced error recovery? + return EXIT_FAILURE + } + } + + /// The implementation of ``callEntryPoint(passing:recordHandler:)``. + /// + /// - Parameters: + /// - args: The arguments to pass to the testing library as its + /// configuration JSON. + /// - recordHandler: A callback to invoke once per record. + /// + /// - Returns: A process exit code such as `EXIT_SUCCESS`. + private func _callEntryPoint( + passing args: __CommandLineArguments_v0?, + recordHandler: (@Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void)? + ) async throws -> CInt { + var recordHandler = recordHandler + + let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments) + + // Event stream output + // Automatically write record JSON as JSON lines to the event stream if + // specified by the user. + if let eventStreamOutputPath = args.eventStreamOutputPath { + let file = try FileHandle(forWritingAtPath: eventStreamOutputPath) + recordHandler = { [oldRecordHandler = recordHandler] recordJSON in + JSON.asJSONLine(recordJSON) { recordJSON in + _ = try? file.withLock { + try file.write(recordJSON) + try file.write("\n") + } + } + oldRecordHandler?(recordJSON) + } + } + + let configurationJSON = try JSON.withEncoding(of: args) { configurationJSON in + configurationJSON.withMemoryRebound(to: UInt8.self) { Array($0) } + } + + let resultJSON: [UInt8] = await withCheckedContinuation { continuation in + struct Context { + var continuation: CheckedContinuation<[UInt8], Never> + var recordHandler: (@Sendable (UnsafeRawBufferPointer) -> Void)? + } + let context = Unmanaged.passRetained( + Context( + continuation: continuation, + recordHandler: recordHandler + ) as AnyObject + ).toOpaque() + configurationJSON.withUnsafeBytes { configurationJSON in + rawValue.entryPoint( + configurationJSON.baseAddress!, + configurationJSON.count, + 0, + context, + /* recordJSONHandler: */ { recordJSON, recordJSONByteCount, _, context in + guard let context = Unmanaged.fromOpaque(context!).takeUnretainedValue() as? Context else { + return + } + let recordJSON = UnsafeRawBufferPointer(start: recordJSON, count: recordJSONByteCount) + context.recordHandler?(recordJSON) + }, + /* completionHandler: */ { resultJSON, resultJSONByteCount, _, context in + guard let context = Unmanaged.fromOpaque(context!).takeRetainedValue() as? Context else { + return + } + // TODO: interpret more complex results than a process exit code + let resultJSON = [UInt8](UnsafeRawBufferPointer(start: resultJSON, count: resultJSONByteCount)) + context.continuation.resume(returning: resultJSON) + } + ) + } + } + + do { + return try resultJSON.withUnsafeBytes { resultJSON in + try JSON.decode(CInt.self, from: resultJSON) + } + } catch { + // TODO: more advanced error recovery? + return EXIT_FAILURE + } + } +#endif +} + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY +// MARK: - Discovery + +/// A helper protocol that prevents the conformance of ``Library`` to +/// ``DiscoverableAsTestContent`` from being emitted into the testing library's +/// Swift module or interface files. +private protocol _DiscoverableAsTestContent: DiscoverableAsTestContent {} + +extension Library: _DiscoverableAsTestContent { + fileprivate static var testContentKind: TestContentKind { + "main" + } + + fileprivate typealias TestContentAccessorHint = UnsafePointer +} + +@_spi(Experimental) +extension Library { + /// Perform a one-time check that the in-memory layout of ``Library`` matches + /// that of ``SWTLibrary``. + private static let _validateMemoryLayout: Void = { + assert(MemoryLayout.size == MemoryLayout.size, "Library.size (\(MemoryLayout.size)) != SWTLibrary.size (\(MemoryLayout.size)). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + assert(MemoryLayout.stride == MemoryLayout.stride, "Library.stride (\(MemoryLayout.stride)) != SWTLibrary.stride (\(MemoryLayout.stride)). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + assert(MemoryLayout.alignment == MemoryLayout.alignment, "Library.alignment (\(MemoryLayout.alignment)) != SWTLibrary.alignment (\(MemoryLayout.alignment)). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + }() + + /// Create an instance of this type representing the testing library with the + /// given hint. + /// + /// - Parameters: + /// - hint: The hint to match against such as `"swift-testing"`. + /// + /// If no matching testing library is found, this initializer returns `nil`. + @_spi(ForToolsIntegrationOnly) + public init?(withHint hint: String) { + Self._validateMemoryLayout + let result = hint.withCString { hint in + Library.allTestContentRecords().lazy + .compactMap { $0.load(withHint: hint) } + .first + } + if let result { + self = result + } else { + return nil + } + } + + /// All testing libraries known to the system including Swift Testing. + @_spi(ForToolsIntegrationOnly) + public static var all: some Sequence { + Self._validateMemoryLayout + return Library.allTestContentRecords().lazy.compactMap { $0.load() } + } +} +#endif + +// MARK: - Referring to Swift Testing directly + +extension Library { + /// The ABI entry point function for the testing library, thunked so that it + /// is compatible with the ``Library`` ABI. + private static let _libraryRecordEntryPoint: SWTLibraryEntryPoint = { configurationJSON, configurationJSONByteCount, _, context, recordJSONHandler, completionHandler in +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY + // Capture appropriate state from the arguments to forward into the canonical + // entry point function. + let contextBitPattern = UInt(bitPattern: context) + let configurationJSON = UnsafeRawBufferPointer(start: configurationJSON, count: configurationJSONByteCount) + var args: __CommandLineArguments_v0 + let eventHandler: Event.Handler + do { + args = try JSON.decode(__CommandLineArguments_v0.self, from: configurationJSON) + eventHandler = try eventHandlerForStreamingEvents(withVersionNumber: args.eventStreamVersionNumber, encodeAsJSONLines: false) { recordJSON in + let context = UnsafeRawPointer(bitPattern: contextBitPattern)! + recordJSONHandler(recordJSON.baseAddress!, recordJSON.count, 0, context) + } + } catch { + // TODO: interpret more complex results than a process exit code + var resultJSON = "\(EXIT_FAILURE)" + return resultJSON.withUTF8 { resultJSON in + completionHandler(resultJSON.baseAddress!, resultJSON.count, 0, context) + } + } + + // Avoid infinite recursion and double JSON output. (Other libraries don't + // need to clear these fields.) + args.testingLibrary = nil + args.eventStreamOutputPath = nil + + // Create an async context and run tests within it. + let run = { [args] in + let context = UnsafeRawPointer(bitPattern: contextBitPattern)! + let exitCode = await Testing.entryPoint(passing: args, eventHandler: eventHandler) + var resultJSON = "\(exitCode)" + resultJSON.withUTF8 { resultJSON in + completionHandler(resultJSON.baseAddress!, resultJSON.count, 0, context) + } + } +#if !SWT_NO_UNSTRUCTURED_TASKS + Task.detached { await run() } +#else + Task.runInline { await run() } +#endif +#else + // There is no way to call this function without pointer shenanigans because + // we are not exposing `callEntryPoint()` nor are we emitting a record into + // the test content section. + swt_unreachable() +#endif + } + + /// An instance of this type representing Swift Testing itself. + public static let swiftTesting: Self = { + Self( + rawValue: .init( + name: StaticString("Swift Testing").constUTF8CString, + canonicalHint: StaticString("swift-testing").constUTF8CString, + entryPoint: _libraryRecordEntryPoint, + reserved: (0, 0, 0, 0, 0) + ) + ) + }() +} + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY +// MARK: - Our very own library record + +private func _libraryRecordAccessor(_ outValue: UnsafeMutableRawPointer, _ type: UnsafeRawPointer, _ hint: UnsafeRawPointer?, _ reserved: UInt) -> CBool { +#if !hasFeature(Embedded) + // Make sure that the caller supplied the right Swift type. If a testing + // library is implemented in a language other than Swift, they can either: + // ignore this argument; or ask the Swift runtime for the type metadata + // pointer and compare it against the value `type.pointee` (`*type` in C). + guard type.load(as: Any.Type.self) == Library.self else { + return false + } +#endif + + // Check if the name of the testing library the caller wants is equivalent to + // "Swift Testing", ignoring case and punctuation. (If the caller did not + // specify a library name, the caller wants records for all libraries.) + let hint = hint.map { $0.load(as: UnsafePointer.self) } + if let hint { + guard let hint = String(validatingCString: hint), + String(hint.filter(\.isLetter)).lowercased() == "swifttesting" else { + return false + } + } + + // Initialize the provided memory to the (ABI-stable) library structure. + _ = outValue.initializeMemory( + as: SWTLibrary.self, + to: Library.swiftTesting.rawValue + ) + + return true +} + +#if compiler(>=6.3) && hasFeature(CompileTimeValuesPreview) +#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) +@section("__DATA_CONST,__swift5_tests") +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) +@section("swift5_tests") +#elseif os(Windows) +@section(".sw5test$B") +#else +//@__testing(warning: "Platform-specific implementation missing: test content section name unavailable") +#endif +@used +#elseif hasFeature(SymbolLinkageMarkers) +#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) +@_section("__DATA_CONST,__swift5_tests") +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) +@_section("swift5_tests") +#elseif os(Windows) +@_section(".sw5test$B") +#else +//@__testing(warning: "Platform-specific implementation missing: test content section name unavailable") +#endif +@_used +#endif +private let _libraryRecord: __TestContentRecord = ( + kind: 0x6D61696E, /* 'main' */ + reserved1: 0, + accessor: _libraryRecordAccessor, + context: 0, + reserved2: 0 +) +#endif diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index c6285f4f4..6e5d8ec29 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -13,6 +13,21 @@ public struct Event: Sendable { /// An enumeration describing the various kinds of event that can be observed. public enum Kind: Sendable { + /// A library was discovered during test run planning. + /// + /// - Parameters: + /// - library: A description of the discovered library. + /// + /// This event is recorded once, before events of kind ``testDiscovered``, + /// and indicates which testing library's tests will run. + /// + /// This event is also posted once per library when + /// `swift test --experimental-list-libraries` is called. In that case, + /// events are posted for all discovered libraries regardless of whether or + /// not they would run. + @_spi(Experimental) + indirect case libraryDiscovered(_ library: Library) + /// A test was discovered during test run planning. /// /// This event is recorded once per discovered test when ``Runner/run()`` is @@ -378,6 +393,18 @@ extension Event { extension Event.Kind { /// A serializable enumeration describing the various kinds of event that can be observed. public enum Snapshot: Sendable, Codable { + /// A library was discovered during test run planning. + /// + /// This event is recorded once, before events of kind ``testDiscovered``, + /// and indicates which testing library's tests will run. + /// + /// This event is also posted once per library when + /// `swift test --experimental-list-libraries` is called. In that case, + /// events are posted for all discovered libraries regardless of whether or + /// not they would run. + @_spi(Experimental) + case libraryDiscovered + /// A test was discovered during test run planning. /// /// This event is recorded once per discovered test when ``Runner/run()`` is @@ -523,6 +550,8 @@ extension Event.Kind { /// - kind: The original event kind to snapshot. public init(snapshotting kind: Event.Kind) { switch kind { + case .libraryDiscovered: + self = .libraryDiscovered case .testDiscovered: self = .testDiscovered case .runStarted: diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 9f0ac21b1..152e7f8c2 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -349,7 +349,7 @@ extension Event.HumanReadableOutputRecorder { // Finally, produce any messages for the event. switch event.kind { - case .testDiscovered: + case .libraryDiscovered, .testDiscovered: // Suppress events of this kind from output as they are not generally // interesting in human-readable output. break diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index a6a76189e..03a6d8cec 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -475,6 +475,7 @@ extension Runner { await Configuration.withCurrent(runner.configuration) { // Post an event for every test in the test plan being run. These events // are turned into JSON objects if JSON output is enabled. + Event.post(.libraryDiscovered(.swiftTesting), for: (nil, nil), configuration: runner.configuration) for test in runner.plan.steps.lazy.map(\.test) { Event.post(.testDiscovered, for: (test, nil), configuration: runner.configuration) } diff --git a/Sources/Testing/Support/Additions/StringAdditions.swift b/Sources/Testing/Support/Additions/StringAdditions.swift new file mode 100644 index 000000000..ef7c59fe8 --- /dev/null +++ b/Sources/Testing/Support/Additions/StringAdditions.swift @@ -0,0 +1,20 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension StaticString { + /// This string as a compile-time constant C string. + /// + /// - Precondition: This instance of `StaticString` must have been constructed + /// from a string literal, not a Unicode scalar value. + var constUTF8CString: UnsafePointer { + precondition(hasPointerRepresentation, "Cannot construct a compile-time constant C string from a StaticString without pointer representation.") + return UnsafeRawPointer(utf8Start).bindMemory(to: CChar.self, capacity: utf8CodeUnitCount) + } +} diff --git a/Sources/_TestingInternals/include/Library.h b/Sources/_TestingInternals/include/Library.h new file mode 100644 index 000000000..14edf3350 --- /dev/null +++ b/Sources/_TestingInternals/include/Library.h @@ -0,0 +1,52 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if !defined(SWT_LIBRARY_H) +#define SWT_LIBRARY_H + +#include "Defines.h" +#include "Includes.h" + +SWT_ASSUME_NONNULL_BEGIN + +typedef void (* SWTLibraryEntryPointRecordJSONHandler)( + const void *recordJSON, + size_t recordJSONByteCount, + uintptr_t reserved, + const void *_Null_unspecified context +); + +typedef void (* SWTLibraryEntryPointCompletionHandler)( + const void *resultJSON, + size_t resultJSONByteCount, + uintptr_t reserved, + const void *_Null_unspecified context +); + +typedef void (* SWTLibraryEntryPoint)( + const void *configurationJSON, + size_t configurationJSONByteCount, + uintptr_t reserved, + const void *_Null_unspecified context, + SWTLibraryEntryPointRecordJSONHandler SWT_SENDABLE recordJSONHandler, + SWTLibraryEntryPointCompletionHandler SWT_SENDABLE completionHandler +); + +/// A C type that provides the in-memory layout of the ``Library`` Swift type. +typedef struct SWTLibrary { + const char *name; + const char *canonicalHint; // TODO: better name + SWTLibraryEntryPoint entryPoint; + uintptr_t reserved[5]; +} SWTLibrary; + +SWT_ASSUME_NONNULL_END + +#endif diff --git a/Tests/TestingTests/LibraryTests.swift b/Tests/TestingTests/LibraryTests.swift new file mode 100644 index 000000000..ec591c1e3 --- /dev/null +++ b/Tests/TestingTests/LibraryTests.swift @@ -0,0 +1,206 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +private import _TestingInternals + +private import Foundation // for JSONSerialization + +struct `Library tests` { + @Test func `Find Swift Testing library`() throws { + let library = try #require(Library(withHint: "SwIfTtEsTiNg")) + #expect(library.name == "Swift Testing") + #expect(library.canonicalHint == "swift-testing") + } + + @Test func `Run mock library`() async throws { + try await confirmation("(Mock) issue recorded") { issueRecorded in + let library = try #require(Library(withHint: "mock")) + + var args = __CommandLineArguments_v0() + args.eventStreamVersionNumber = ABI.v0.versionNumber + let exitCode = await library.callEntryPoint(passing: args) { recordJSON in + do { + let recordJSON = Data(recordJSON) + let jsonObject = try JSONSerialization.jsonObject(with: recordJSON) + let record = try #require(jsonObject as? [String: Any]) + if let kind = record["kind"] as? String, let payload = record["payload"] as? [String: Any] { + if kind == "event", let eventKind = payload["kind"] as? String { + if eventKind == "issueRecorded" { + issueRecorded() + } + } + } + } catch { + Issue.record(error) + } + } + #expect(exitCode == EXIT_SUCCESS) + } + } +} + +// MARK: - Fixtures + +extension Library { + private static let _mockRecordEntryPoint: SWTLibraryEntryPoint = { configurationJSON, configurationJSONByteCount, _, context, recordJSONHandler, completionHandler in + let tests: [[String: Any]] = [ + [ + "kind": "function", + "name": "mock_test_1", + "sourceLocation": [ + "fileID": "__C/mock_file.pascal", + "filePath": "/tmp/mock_file.pascal", + "_filePath": "/tmp/mock_file.pascal", + "line": 1, + "column": 1, + ], + "id": "mock_test_1_id", + "isParameterized": false + ] + ] + + let events: [[String: Any]] = [ + [ + "kind": "runStarted", + ], + [ + "kind": "testStarted", + "testID": "mock_test_1_id" + ], + [ + "kind": "issueRecorded", + "testID": "mock_test_1_id", + "issue": [ + "isKnown": false, + "sourceLocation": [ + "fileID": "__C/mock_file.pascal", + "filePath": "/tmp/mock_file.pascal", + "_filePath": "/tmp/mock_file.pascal", + "line": 20, + "column": 1, + ], + ] + ], + [ + "kind": "testEnded", + "testID": "mock_test_1_id" + ], + [ + "kind": "runEnded", + ], + ] + + for var test in tests { + test = [ + "version": 0, + "kind": "test", + "payload": test + ] + let json = try! JSONSerialization.data(withJSONObject: test) + json.withUnsafeBytes { json in + recordJSONHandler(json.baseAddress!, json.count, 0, context) + } + } + let now1970 = Date().timeIntervalSince1970 + for var (i, event) in events.enumerated() { + event["instant"] = [ + "absolute": i, + "since1970": now1970 + Double(i), + ] + event = [ + "version": 0, + "kind": "event", + "payload": event + ] + let json = try! JSONSerialization.data(withJSONObject: event) + json.withUnsafeBytes { json in + recordJSONHandler(json.baseAddress!, json.count, 0, context) + } + } + + var resultJSON = "0" + resultJSON.withUTF8 { resultJSON in + completionHandler(resultJSON.baseAddress!, resultJSON.count, 0, context) + } + } + + static let mock: Self = { + Self( + rawValue: .init( + name: StaticString("Mock Testing Library").constUTF8CString, + canonicalHint: StaticString("mock").constUTF8CString, + entryPoint: _mockRecordEntryPoint, + reserved: (0, 0, 0, 0, 0) + ) + ) + }() +} + +#if compiler(>=6.3) && hasFeature(CompileTimeValuesPreview) +#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) +@section("__DATA_CONST,__swift5_tests") +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) +@section("swift5_tests") +#elseif os(Windows) +@section(".sw5test$B") +#else +//@__testing(warning: "Platform-specific implementation missing: test content section name unavailable") +#endif +@used +#elseif hasFeature(SymbolLinkageMarkers) +#if os(macOS) || os(iOS) || os(watchOS) || os(tvOS) || os(visionOS) +@_section("__DATA_CONST,__swift5_tests") +#elseif os(Linux) || os(FreeBSD) || os(OpenBSD) || os(Android) || os(WASI) +@_section("swift5_tests") +#elseif os(Windows) +@_section(".sw5test$B") +#else +//@__testing(warning: "Platform-specific implementation missing: test content section name unavailable") +#endif +@_used +#endif +private let _mockLibraryRecord: __TestContentRecord = ( + kind: 0x6D61696E, /* 'main' */ + reserved1: 0, + accessor: _mockLibraryRecordAccessor, + context: 0, + reserved2: 0 +) + +private func _mockLibraryRecordAccessor(_ outValue: UnsafeMutableRawPointer, _ type: UnsafeRawPointer, _ hint: UnsafeRawPointer?, _ reserved: UInt) -> CBool { +#if !hasFeature(Embedded) + // Make sure that the caller supplied the right Swift type. If a testing + // library is implemented in a language other than Swift, they can either: + // ignore this argument; or ask the Swift runtime for the type metadata + // pointer and compare it against the value `type.pointee` (`*type` in C). + guard type.load(as: Any.Type.self) == Library.self else { + return false + } +#endif + + // Check if the name of the testing library the caller wants is equivalent to + // "Swift Testing", ignoring case and punctuation. (If the caller did not + // specify a library name, the caller wants records for all libraries.) + let hint = hint.map { $0.load(as: UnsafePointer.self) } + if let hint, 0 != strcasecmp(hint, "mock") { + return false + } + + // Initialize the provided memory to the (ABI-stable) library structure. + _ = outValue.initializeMemory( + as: SWTLibrary.self, + to: Library.mock.rawValue + ) + + return true +} +#endif diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 2d7001e8f..43547a76b 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -363,6 +363,7 @@ struct SwiftPMTests { ("--event-stream-output-path", "--event-stream-version", ABI.v0.versionNumber), ("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.v0.versionNumber), ("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.v6_3.versionNumber), + ("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.ExperimentalVersion.versionNumber), ]) func eventStreamOutput(outputArgumentName: String, versionArgumentName: String, version: VersionNumber) async throws { let version = try #require(ABI.version(forVersionNumber: version)) @@ -382,6 +383,9 @@ struct SwiftPMTests { let test = Test(.tags(.blue)) {} let eventContext = Event.Context(test: test, testCase: nil, configuration: nil) + if V.includesExperimentalFields { + configuration.handleEvent(Event(.libraryDiscovered(.swiftTesting), testID: test.id, testCaseID: nil), in: eventContext) + } configuration.handleEvent(Event(.testDiscovered, testID: test.id, testCaseID: nil), in: eventContext) configuration.handleEvent(Event(.runStarted, testID: nil, testCaseID: nil), in: eventContext) do {