diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index e8ae5120..4c7f7b1b 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "paket": { - "version": "8.0.3", + "version": "9.0.2", "commands": [ "paket" ] diff --git a/Expecto.Tests/Expecto.Tests.fsproj b/Expecto.Tests/Expecto.Tests.fsproj index f9595e24..33e53e9c 100644 --- a/Expecto.Tests/Expecto.Tests.fsproj +++ b/Expecto.Tests/Expecto.Tests.fsproj @@ -1,12 +1,14 @@ - + Expecto.Tests Exe net6.0 + false + @@ -20,4 +22,4 @@ - + \ No newline at end of file diff --git a/Expecto.Tests/Main.fs b/Expecto.Tests/Main.fs index fae29267..fb043e7b 100644 --- a/Expecto.Tests/Main.fs +++ b/Expecto.Tests/Main.fs @@ -2,13 +2,26 @@ module Main open Expecto open Expecto.Logging +open OpenTelemetry.Resources +open OpenTelemetry +open OpenTelemetry.Trace +open System.Threading +open System.Diagnostics +open System + +let serviceName = "Expecto.Tests" + +let logger = Log.create serviceName -let logger = Log.create "Expecto.Tests" [] let main args = + let test = Impl.testFromThisAssembly() |> Option.orDefault (TestList ([], Normal)) |> Test.shuffle "." - runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml"] args test + runTestsWithCLIArgs [NUnit_Summary "bin/Expecto.Tests.TestResults.xml";] args test + + + diff --git a/Expecto.Tests/OpenTelemetry.fs b/Expecto.Tests/OpenTelemetry.fs new file mode 100644 index 00000000..9267fe1e --- /dev/null +++ b/Expecto.Tests/OpenTelemetry.fs @@ -0,0 +1,204 @@ +namespace Expecto + +module OpenTelemetry = + open System + open System.Diagnostics + open System.Collections.Generic + open Impl + open System.Runtime.CompilerServices + type Activity with + /// Sets code semantic conventions for code.function.name, code.filepath, and code.lineno + /// Optional: The current namespace. Will default to using Reflection.MethodBase.GetCurrentMethod().DeclaringType + /// Optional: The current function Don't set this. This uses CallerMemberName. + /// Optional: The current filepath. Don't set this. This uses CallerFilePath. + /// Optional: The current line number. Don't set this. This uses CallerLineNumber. + member inline x.SetSource( + ?nameSpace : string, + [] ?memberName: string, + [] ?path: string, + [] ?line: int) = + if not (isNull x) then + if x.GetTagItem "code.function.name" = null then + let nameSpace = + nameSpace + |> Option.defaultWith (fun () -> + Reflection.MethodBase.GetCurrentMethod().DeclaringType.FullName.Split("+") // F# has + in type names that refer to anonymous functions, we typically want the first named type + |> Seq.tryHead + |> Option.defaultValue "") + let memberName = defaultArg memberName "" + x.SetTag("code.function.name", $"{nameSpace}.{memberName}" ) |> ignore + if x.GetTagItem "code.filepath" = null then x.SetTag("code.filepath", defaultArg path "") |> ignore + if x.GetTagItem "code.lineno" = null then x.SetTag("code.lineno", defaultArg line 0) |> ignore + + module internal Activity = + let inline isNotNull x = isNull x |> not + + let inline setStatus (status : ActivityStatusCode) (span : Activity) = + if isNotNull span then + span.SetStatus status |> ignore + + let inline setExn (e : exn) (span : Activity) = + if isNotNull span|> not then + let tags = + ActivityTagsCollection( + seq { + KeyValuePair("exception.type", box (e.GetType().Name)) + KeyValuePair("exception.stacktrace", box (e.ToString())) + if not <| String.IsNullOrEmpty(e.Message) then + KeyValuePair("exception.message", box e.Message) + } + ) + + ActivityEvent("exception", tags = tags) + |> span.AddEvent + |> ignore + + let inline setExnMarkFailed (e : exn) (span : Activity) = + if isNotNull span then + setExn e span + span |> setStatus ActivityStatusCode.Error + + let setSourceLocation (sourceLoc : SourceLocation) (span : Activity) = + if isNotNull span && sourceLoc <> SourceLocation.empty then + span.SetTag("code.lineno", sourceLoc.lineNumber) |> ignore + span.SetTag("code.filepath", sourceLoc.sourcePath) |> ignore + + let inline addOutcome (result : TestResult) (span : Activity) = + if isNotNull span then + let status = match result with + | Passed -> "Passed" + | Ignored _ -> "Ignored" + | Failed _ -> "Failed" + | Error _ -> "Error" + span.SetTag("test.result.status", status) |> ignore + span.SetTag("test.result.message", result) |> ignore + + let inline start (span : Activity) = + if isNotNull span then + span.Start() |> ignore + span + + let inline stop (span : Activity) = + if isNotNull span then + span.Stop() |> ignore + + let inline setEndTimeNow (span : Activity) = + if isNotNull span then + span.SetEndTime DateTime.UtcNow |> ignore + + let inline createActivity (name : string) (source : ActivitySource) = + if isNotNull source then + source.CreateActivity(name, ActivityKind.Internal) + else + null + + open Activity + open System.Runtime.ExceptionServices + + let inline internal reraiseAnywhere<'a> (e: exn) : 'a = + ExceptionDispatchInfo.Capture(e).Throw() + Unchecked.defaultof<'a> + + module TestResult = + let ofException (e:Exception) : TestResult = + match e with + | :? AssertException as e -> + let msg = + "\n" + e.Message + "\n" + + (e.StackTrace.Split('\n') + |> Seq.skipWhile (fun l -> l.StartsWith(" at Expecto.Expect.")) + |> Seq.truncate 5 + |> String.concat "\n") + Failed msg + + | :? FailedException as e -> + Failed ("\n"+e.Message) + | :? IgnoreException as e -> + Ignored e.Message + | :? AggregateException as e when e.InnerExceptions.Count = 1 -> + if e.InnerException :? IgnoreException then + Ignored e.InnerException.Message + else + Error e.InnerException + | e -> + Error e + + + let addExceptionOutcomeToSpan (span: Activity) (e: Exception) = + let testResult = TestResult.ofException e + + addOutcome testResult span + match testResult with + | Ignored _ -> + setExn e span + | _ -> + setExnMarkFailed e span + + let wrapCodeWithSpan (span: Activity) (test: TestCode) = + let inline handleSuccess span = + setEndTimeNow span + addOutcome Passed span + setStatus ActivityStatusCode.Ok span + let inline handleFailure span e = + setEndTimeNow span + addExceptionOutcomeToSpan span e + reraiseAnywhere e + + match test with + | Sync test -> + TestCode.Sync (fun () -> + use span = start span + try + test () + handleSuccess span + with + | e -> + handleFailure span e + ) + + | Async test -> + TestCode.Async (async { + use span = start span + try + do! test + handleSuccess span + with + | e -> + handleFailure span e + }) + | AsyncFsCheck (testConfig, stressConfig, test) -> + TestCode.AsyncFsCheck (testConfig, stressConfig, fun fsCheckConfig -> async { + use span = start span + try + do! test fsCheckConfig + handleSuccess span + with + | e -> + handleFailure span e + }) + | SyncWithCancel test-> + TestCode.SyncWithCancel (fun ct -> + use span = start span + try + test ct + handleSuccess span + with + | e -> + handleFailure span e + ) + + /// Wraps each test with an OpenTelemetry Span/System.Diagnostics.Activity. + /// ExpectoConfig + /// Provides APIs start OpenTelemetry Span/System.Diagnostics.Activity + /// The tests to wrap in span/activity. + /// Tests wrapped in a Span/Activity + let addOpenTelemetry_SpanPerTest (config: ExpectoConfig) (activitySource: ActivitySource) (rootTest: Test) : Test = + rootTest + |> Test.toTestCodeList + |> List.map (fun test -> + let span = activitySource |> createActivity (config.joinWith.format test.name) + span |> setSourceLocation (config.locate test.test) + {test with test = wrapCodeWithSpan span test.test} + ) + |> Test.fromFlatTests config.joinWith.asString + diff --git a/Expecto.Tests/Tests.fs b/Expecto.Tests/Tests.fs index 48ea2336..6a6dd266 100644 --- a/Expecto.Tests/Tests.fs +++ b/Expecto.Tests/Tests.fs @@ -10,6 +10,32 @@ open Expecto open Expecto.Impl open Expecto.Logging open System.Globalization +open OpenTelemetry.Resources +open OpenTelemetry.Trace +open System.Diagnostics +open OpenTelemetry +open OpenTelemetry.Exporter + +let serviceName = "Expecto.Tests" + +let source = new ActivitySource(serviceName) + +let resourceBuilder () = + ResourceBuilder + .CreateDefault() + .AddService(serviceName = serviceName) + +let traceProvider () = + Sdk + .CreateTracerProviderBuilder() + .AddSource(serviceName) + .SetResourceBuilder(resourceBuilder ()) + .AddOtlpExporter() + .Build() +do + let provider = traceProvider() + AppDomain.CurrentDomain.ProcessExit.Add(fun _ -> provider.Dispose()) + module Dummy = @@ -1400,6 +1426,8 @@ let asyncTests = ] open System.Threading.Tasks +open OpenTelemetry +open System.Diagnostics [] let taskTests = @@ -1848,6 +1876,7 @@ let cancel = ) ] + [] let theory = testList "theory testing" [ @@ -1875,3 +1904,4 @@ let theory = } ] ] + |> addOpenTelemetry_SpanPerTest ExpectoConfig.defaultConfig source diff --git a/Expecto.Tests/paket.references b/Expecto.Tests/paket.references index 36cbbdb4..4c4484e0 100644 --- a/Expecto.Tests/paket.references +++ b/Expecto.Tests/paket.references @@ -1 +1,4 @@ -FsCheck \ No newline at end of file +FsCheck +OpenTelemetry.Exporter.OpenTelemetryProtocol +YoloDev.Expecto.TestSdk +Microsoft.NET.Test.Sdk \ No newline at end of file diff --git a/Expecto/Expecto.Impl.fs b/Expecto/Expecto.Impl.fs index 9381fd6e..bb14b15a 100644 --- a/Expecto/Expecto.Impl.fs +++ b/Expecto/Expecto.Impl.fs @@ -1,6 +1,7 @@ namespace Expecto open System +open System.Collections.Generic open System.Diagnostics open System.Reflection open System.Threading diff --git a/Expecto/Expecto.fs b/Expecto/Expecto.fs index e8d6be2d..7312f08f 100644 --- a/Expecto/Expecto.fs +++ b/Expecto/Expecto.fs @@ -11,6 +11,7 @@ module Tests = open Impl open Helpers open Expecto.Logging + open System.Diagnostics let mutable private afterRunTestsList = [] let private afterRunTestsListLock = obj() @@ -456,6 +457,7 @@ module Tests = /// Specify test names join character. | JoinWith of split: string + let options = [ "--sequenced", "Don't run the tests in parallel.", Args.none Sequenced "--parallel", "Run all tests in parallel (default).", Args.none Parallel diff --git a/paket.dependencies b/paket.dependencies index 2ded04dd..e124122c 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -9,6 +9,9 @@ nuget Hopac ~> 0.4 nuget DiffPlex ~> 1.5 nuget Mono.Cecil ~> 0.11 nuget BenchmarkDotNet ~> 0.14.0 +nuget OpenTelemetry.Exporter.OpenTelemetryProtocol +nuget YoloDev.Expecto.TestSdk +nuget Microsoft.NET.Test.Sdk group FsCheck3 source https://api.nuget.org/v3/index.json diff --git a/paket.lock b/paket.lock index e1287fd6..66a1cbe2 100644 --- a/paket.lock +++ b/paket.lock @@ -21,6 +21,9 @@ NUGET BenchmarkDotNet.Annotations (0.14) CommandLineParser (2.9.1) DiffPlex (1.7.1) + Expecto (10.2.1) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + FSharp.Core (>= 7.0.200) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + Mono.Cecil (>= 0.11.4 < 1.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) FsCheck (2.16.5) FSharp.Core (>= 4.2.3) FSharp.Core (7.0.200) @@ -51,6 +54,7 @@ NUGET System.Runtime.CompilerServices.Unsafe (>= 6.0) System.Text.Encoding.CodePages (>= 7.0) System.Threading.Tasks.Extensions (>= 4.5.4) + Microsoft.CodeCoverage (17.13) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net462)) (&& (== netstandard2.1) (>= netcoreapp3.1)) Microsoft.Diagnostics.NETCore.Client (0.2.410101) Microsoft.Bcl.AsyncInterfaces (>= 1.1) Microsoft.Extensions.Logging (>= 2.1.1) @@ -65,38 +69,88 @@ NUGET System.Reflection.TypeExtensions (>= 4.7) System.Runtime.CompilerServices.Unsafe (>= 6.0) Microsoft.DotNet.PlatformAbstractions (3.1.6) - Microsoft.Extensions.DependencyInjection (7.0) - Microsoft.Extensions.DependencyInjection.Abstractions (>= 7.0) - Microsoft.Extensions.DependencyInjection.Abstractions (7.0) - Microsoft.Extensions.Logging (7.0) - Microsoft.Extensions.DependencyInjection (>= 7.0) - Microsoft.Extensions.DependencyInjection.Abstractions (>= 7.0) - Microsoft.Extensions.Logging.Abstractions (>= 7.0) - Microsoft.Extensions.Options (>= 7.0) - System.Diagnostics.DiagnosticSource (>= 7.0) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) - Microsoft.Extensions.Logging.Abstractions (7.0) - System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) - System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) - Microsoft.Extensions.Options (7.0.1) - Microsoft.Extensions.DependencyInjection.Abstractions (>= 7.0) - Microsoft.Extensions.Primitives (>= 7.0) - Microsoft.Extensions.Primitives (7.0) - System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) + Microsoft.Extensions.Configuration (9.0.2) + Microsoft.Extensions.Configuration.Abstractions (>= 9.0.2) + Microsoft.Extensions.Primitives (>= 9.0.2) + Microsoft.Extensions.Configuration.Abstractions (9.0.2) + Microsoft.Extensions.Primitives (>= 9.0.2) + Microsoft.Extensions.Configuration.Binder (9.0.2) + Microsoft.Extensions.Configuration.Abstractions (>= 9.0.2) + Microsoft.Extensions.DependencyInjection (9.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (9.0.2) + Microsoft.Extensions.Diagnostics.Abstractions (9.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0.2) + Microsoft.Extensions.Options (>= 9.0.2) + System.Buffers (>= 4.5.1) + System.Diagnostics.DiagnosticSource (>= 9.0.2) + System.Memory (>= 4.5.5) + Microsoft.Extensions.Logging (9.0.2) + Microsoft.Extensions.DependencyInjection (>= 9.0.2) + Microsoft.Extensions.Logging.Abstractions (>= 9.0.2) + Microsoft.Extensions.Options (>= 9.0.2) + System.Diagnostics.DiagnosticSource (>= 9.0.2) + Microsoft.Extensions.Logging.Abstractions (9.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0.2) + System.Buffers (>= 4.5.1) + System.Diagnostics.DiagnosticSource (>= 9.0.2) + System.Memory (>= 4.5.5) + Microsoft.Extensions.Logging.Configuration (9.0.2) + Microsoft.Extensions.Configuration (>= 9.0.2) + Microsoft.Extensions.Configuration.Abstractions (>= 9.0.2) + Microsoft.Extensions.Configuration.Binder (>= 9.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0.2) + Microsoft.Extensions.Logging (>= 9.0.2) + Microsoft.Extensions.Logging.Abstractions (>= 9.0.2) + Microsoft.Extensions.Options (>= 9.0.2) + Microsoft.Extensions.Options.ConfigurationExtensions (>= 9.0.2) + Microsoft.Extensions.Options (9.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0.2) + Microsoft.Extensions.Primitives (>= 9.0.2) + System.ComponentModel.Annotations (>= 5.0) + Microsoft.Extensions.Options.ConfigurationExtensions (9.0.2) + Microsoft.Extensions.Configuration.Abstractions (>= 9.0.2) + Microsoft.Extensions.Configuration.Binder (>= 9.0.2) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0.2) + Microsoft.Extensions.Options (>= 9.0.2) + Microsoft.Extensions.Primitives (>= 9.0.2) + Microsoft.Extensions.Primitives (9.0.2) + System.Memory (>= 4.5.5) System.Runtime.CompilerServices.Unsafe (>= 6.0) + Microsoft.NET.Test.Sdk (17.13) + Microsoft.CodeCoverage (>= 17.13) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net462)) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Microsoft.TestPlatform.TestHost (>= 17.13) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Microsoft.TestPlatform.ObjectModel (17.13) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + System.Reflection.Metadata (>= 1.6) + Microsoft.TestPlatform.TestHost (17.13) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Microsoft.TestPlatform.ObjectModel (>= 17.13) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + Newtonsoft.Json (>= 13.0.1) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) Microsoft.Win32.Registry (5.0) System.Buffers (>= 4.5.1) - restriction: || (&& (== net6.0) (>= monoandroid) (< netstandard1.3)) (&& (== net6.0) (>= monotouch)) (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (>= xamarinios)) (&& (== net6.0) (>= xamarinmac)) (&& (== net6.0) (>= xamarintvos)) (&& (== net6.0) (>= xamarinwatchos)) (== netstandard2.1) System.Memory (>= 4.5.4) - restriction: || (&& (== net6.0) (< netcoreapp2.0)) (&& (== net6.0) (< netcoreapp2.1)) (&& (== net6.0) (>= uap10.1)) (== netstandard2.1) System.Security.AccessControl (>= 5.0) System.Security.Principal.Windows (>= 5.0) Mono.Cecil (0.11.4) + Newtonsoft.Json (13.0.3) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= netcoreapp3.1)) + OpenTelemetry (1.11.1) + Microsoft.Extensions.Diagnostics.Abstractions (>= 9.0) + Microsoft.Extensions.Logging.Configuration (>= 9.0) + OpenTelemetry.Api.ProviderBuilderExtensions (>= 1.11.1) + OpenTelemetry.Api (1.11.1) + System.Diagnostics.DiagnosticSource (>= 9.0) + OpenTelemetry.Api.ProviderBuilderExtensions (1.11.1) + Microsoft.Extensions.DependencyInjection.Abstractions (>= 9.0) + OpenTelemetry.Api (>= 1.11.1) + OpenTelemetry.Exporter.OpenTelemetryProtocol (1.11.1) + OpenTelemetry (>= 1.11.1) Perfolizer (0.3.17) System.Buffers (4.5.1) System.CodeDom (7.0) System.Collections.Immutable (8.0) - System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) System.Runtime.CompilerServices.Unsafe (>= 6.0) - System.Diagnostics.DiagnosticSource (7.0.2) - restriction: || (&& (== net6.0) (>= net462)) (&& (== net6.0) (< netstandard2.1)) (== netstandard2.1) - System.Memory (>= 4.5.5) - restriction: || (&& (== net6.0) (>= net462)) (== netstandard2.1) + System.ComponentModel.Annotations (5.0) + System.Diagnostics.DiagnosticSource (9.0.2) + System.Memory (>= 4.5.5) System.Runtime.CompilerServices.Unsafe (>= 6.0) System.Management (7.0) System.CodeDom (>= 7.0) @@ -120,6 +174,10 @@ NUGET System.Runtime.CompilerServices.Unsafe (>= 6.0) System.Threading.Tasks.Extensions (4.5.4) System.Runtime.CompilerServices.Unsafe (>= 4.5.3) - restriction: || (&& (== net6.0) (>= net461)) (&& (== net6.0) (< netcoreapp2.1)) (&& (== net6.0) (< netstandard1.0)) (&& (== net6.0) (< netstandard2.0)) (&& (== net6.0) (>= wp8)) (== netstandard2.1) + YoloDev.Expecto.TestSdk (0.14.3) + Expecto (>= 10.0 < 11.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + FSharp.Core (>= 7.0.200) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) + System.Collections.Immutable (>= 6.0) - restriction: || (== net6.0) (&& (== netstandard2.1) (>= net6.0)) GROUP Build STORAGE: NONE