Added retry facts for ci only (#402)

david-driscoll · web-flow · commit 69ff46bcc842 · 2020-10-28T06:07:17.000Z
diff --git a/test/Lsp.Tests/Integration/DynamicRegistrationTests.cs b/test/Lsp.Tests/Integration/DynamicRegistrationTests.cs
@@ -98,7 +98,7 @@ await TestHelper.DelayUntil(
                 client.RegistrationManager.CurrentRegistrations.Should().Contain(x => x.Method == "@/" + TextDocumentNames.SemanticTokensFull);
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Unregister_Dynamically_While_Server_Is_Running()
             {
                 var (client, server) = await Initialize(new ConfigureClient().Configure, new ConfigureServer().Configure);
diff --git a/test/Lsp.Tests/Integration/PartialItemTests.cs b/test/Lsp.Tests/Integration/PartialItemTests.cs
@@ -24,7 +24,7 @@ public Delegates(ITestOutputHelper testOutputHelper, LanguageProtocolFixture<Def
             {
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Behave_Like_A_Task()
             {
                 var result = await Client.TextDocument.RequestSemanticTokens(
@@ -34,7 +34,7 @@ public async Task Should_Behave_Like_A_Task()
                 result!.Data.Should().HaveCount(3);
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Behave_Like_An_Observable()
             {
                 var items = await Client.TextDocument
diff --git a/test/Lsp.Tests/Integration/PartialItemsTests.cs b/test/Lsp.Tests/Integration/PartialItemsTests.cs
@@ -28,7 +28,7 @@ public Delegates(ITestOutputHelper testOutputHelper, LanguageProtocolFixture<Def
             {
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Behave_Like_A_Task()
             {
                 var result = await Client.TextDocument.RequestCodeLens(
@@ -41,7 +41,7 @@ public async Task Should_Behave_Like_A_Task()
                 result.Select(z => z.Command!.Name).Should().ContainInOrder("CodeLens 1", "CodeLens 2", "CodeLens 3");
             }
 
-            [FactWithSkipOn(SkipOnPlatform.All)]
+            [RetryFact]
             public async Task Should_Behave_Like_An_Observable()
             {
                 var items = await Client.TextDocument
diff --git a/test/Lsp.Tests/Integration/TypedCodeActionTests.cs b/test/Lsp.Tests/Integration/TypedCodeActionTests.cs
@@ -204,7 +204,7 @@ public async Task Should_Resolve_With_Data_Capability()
             item.CodeAction!.Command!.Name.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_Capability()
         {
             var (client, _) = await Initialize(
@@ -297,7 +297,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()
             item.CodeAction!.Command!.Name.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_CancellationToken()
         {
             var (client, _) = await Initialize(
diff --git a/test/Lsp.Tests/Integration/TypedCodeLensTests.cs b/test/Lsp.Tests/Integration/TypedCodeLensTests.cs
@@ -195,7 +195,7 @@ public async Task Should_Resolve_With_Data_Capability()
             item.Command!.Name.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_Capability()
         {
             var (client, _) = await Initialize(
@@ -284,7 +284,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()
             item.Command!.Name.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_CancellationToken()
         {
             var (client, _) = await Initialize(
diff --git a/test/Lsp.Tests/Integration/TypedCompletionTests.cs b/test/Lsp.Tests/Integration/TypedCompletionTests.cs
@@ -195,7 +195,7 @@ public async Task Should_Resolve_With_Data_Capability()
             item.Detail.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_Capability()
         {
             var (client, _) = await Initialize(
@@ -284,7 +284,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()
             item.Detail.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_CancellationToken()
         {
             var (client, _) = await Initialize(
diff --git a/test/Lsp.Tests/Integration/TypedDocumentLinkTests.cs b/test/Lsp.Tests/Integration/TypedDocumentLinkTests.cs
@@ -178,7 +178,7 @@ public async Task Should_Resolve_With_Data_Capability()
             item.Tooltip.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_Capability()
         {
             var (client, _) = await Initialize(
@@ -261,7 +261,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()
             item.Tooltip.Should().Be("resolved");
         }
 
-        [FactWithSkipOn(SkipOnPlatform.Mac)]
+        [RetryFact]
         public async Task Should_Resolve_With_Partial_Data_CancellationToken()
         {
             var (client, _) = await Initialize(
diff --git a/test/TestingUtils/AutoNSubstitute/TestLoggerFactory.cs b/test/TestingUtils/AutoNSubstitute/TestLoggerFactory.cs
@@ -46,7 +46,7 @@ public void Swap(ITestOutputHelper testOutputHelper)
             _testOutputHelper.Swap(testOutputHelper);
         }
 
-        class InnerTestOutputHelper : ITestOutputHelper
+        private class InnerTestOutputHelper : ITestOutputHelper
         {
             private ITestOutputHelper? _testOutputHelper;
 
diff --git a/test/TestingUtils/BlockingMessageBus.cs b/test/TestingUtils/BlockingMessageBus.cs
@@ -0,0 +1,52 @@
+// See https://github.com/JoshKeegan/xRetry
+
+using System.Collections.Concurrent;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    /// <summary>
+    /// An XUnit message bus that can block messages from being passed until we want them to be.
+    /// </summary>
+    public class BlockingMessageBus : IMessageBus
+    {
+        private readonly IMessageBus _underlyingMessageBus;
+        private ConcurrentQueue<IMessageSinkMessage> _messageQueue = new ConcurrentQueue<IMessageSinkMessage>();
+
+        public BlockingMessageBus(IMessageBus underlyingMessageBus)
+        {
+            _underlyingMessageBus = underlyingMessageBus;
+        }
+
+        public bool QueueMessage(IMessageSinkMessage message)
+        {
+            _messageQueue.Enqueue(message);
+
+            // Returns if execution should continue. Since we are intercepting the message, we
+            //  have no way of checking this so always continue...
+            return true;
+        }
+
+        public void Clear()
+        {
+            _messageQueue = new ConcurrentQueue<IMessageSinkMessage>();
+        }
+
+        /// <summary>
+        /// Write the cached messages to the underlying message bus
+        /// </summary>
+        public void Flush()
+        {
+            while (_messageQueue.TryDequeue(out IMessageSinkMessage message))
+            {
+                _underlyingMessageBus.QueueMessage(message);
+            }
+        }
+
+        public void Dispose()
+        {
+            // Do not dispose of the underlying message bus - it is an externally owned resource
+        }
+    }
+}
diff --git a/test/TestingUtils/IRetryableTestCase.cs b/test/TestingUtils/IRetryableTestCase.cs
@@ -0,0 +1,12 @@
+// See https://github.com/JoshKeegan/xRetry
+
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    public interface IRetryableTestCase : IXunitTestCase
+    {
+        int MaxRetries { get; }
+        int DelayBetweenRetriesMs { get; }
+    }
+}
diff --git a/test/TestingUtils/RetryFactAttribute.cs b/test/TestingUtils/RetryFactAttribute.cs
@@ -0,0 +1,40 @@
+// See https://github.com/JoshKeegan/xRetry
+
+using System;
+using Xunit;
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    /// <summary>
+    /// Attribute that is applied to a method to indicate that it is a fact that should be run
+    /// by the test runner up to MaxRetries times, until it succeeds.
+    /// </summary>
+    [XunitTestCaseDiscoverer("xRetry.RetryFactDiscoverer", "xRetry")]
+    [AttributeUsage(AttributeTargets.Method)]
+    public class RetryFactAttribute : FactAttribute
+    {
+        public readonly int MaxRetries;
+        public readonly int DelayBetweenRetriesMs;
+
+        /// <summary>
+        /// Ctor
+        /// </summary>
+        /// <param name="maxRetries">The number of times to run a test for until it succeeds</param>
+        /// <param name="delayBetweenRetriesMs">The amount of time (in ms) to wait between each test run attempt</param>
+        public RetryFactAttribute(int maxRetries = 3, int delayBetweenRetriesMs = 0)
+        {
+            if (maxRetries < 1)
+            {
+                throw new ArgumentOutOfRangeException(nameof(maxRetries) + " must be >= 1");
+            }
+            if (delayBetweenRetriesMs < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(delayBetweenRetriesMs) + " must be >= 0");
+            }
+
+            MaxRetries = UnitTestDetector.IsCI() ? maxRetries : 1;
+            DelayBetweenRetriesMs = delayBetweenRetriesMs;
+        }
+    }
+}
diff --git a/test/TestingUtils/RetryFactDiscoverer.cs b/test/TestingUtils/RetryFactDiscoverer.cs
@@ -0,0 +1,46 @@
+using System.Collections.Generic;
+using System.Linq;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    public class RetryFactDiscoverer : IXunitTestCaseDiscoverer
+    {
+        private readonly IMessageSink _messageSink;
+
+        public RetryFactDiscoverer(IMessageSink messageSink)
+        {
+            _messageSink = messageSink;
+        }
+
+        public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod,
+            IAttributeInfo factAttribute)
+        {
+            IXunitTestCase testCase;
+
+            if (testMethod.Method.GetParameters().Any())
+            {
+                testCase = new ExecutionErrorTestCase(_messageSink, discoveryOptions.MethodDisplayOrDefault(),
+                    discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod,
+                    "[RetryFact] methods are not allowed to have parameters. Did you mean to use [RetryTheory]?");
+            }
+            else if (testMethod.Method.IsGenericMethodDefinition)
+            {
+                testCase = new ExecutionErrorTestCase(_messageSink, discoveryOptions.MethodDisplayOrDefault(),
+                    discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod,
+                    "[RetryFact] methods are not allowed to be generic.");
+            }
+            else
+            {
+                var maxRetries = factAttribute.GetNamedArgument<int>(nameof(RetryFactAttribute.MaxRetries));
+                var delayBetweenRetriesMs =
+                    factAttribute.GetNamedArgument<int>(nameof(RetryFactAttribute.DelayBetweenRetriesMs));
+                testCase = new RetryTestCase(_messageSink, discoveryOptions.MethodDisplayOrDefault(),
+                    discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs);
+            }
+
+            return new[] { testCase };
+        }
+    }
+}
diff --git a/test/TestingUtils/RetryTestCase.cs b/test/TestingUtils/RetryTestCase.cs
@@ -0,0 +1,60 @@
+using System;
+using System.ComponentModel;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestingUtils
+{
+    [Serializable]
+    public class RetryTestCase : XunitTestCase, IRetryableTestCase
+    {
+        public int MaxRetries { get; private set; }
+        public int DelayBetweenRetriesMs { get; private set; }
+
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete(
+            "Called by the de-serializer; should only be called by deriving classes for de-serialization purposes", true)]
+        public RetryTestCase() { }
+
+        public RetryTestCase(
+            IMessageSink diagnosticMessageSink,
+            TestMethodDisplay defaultMethodDisplay,
+            TestMethodDisplayOptions defaultMethodDisplayOptions,
+            ITestMethod testMethod,
+            int maxRetries,
+            int delayBetweenRetriesMs,
+            object[]? testMethodArguments = null)
+            : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod,
+                testMethodArguments)
+        {
+            MaxRetries = maxRetries;
+            DelayBetweenRetriesMs = delayBetweenRetriesMs;
+        }
+
+        public override Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus,
+            object[] constructorArguments, ExceptionAggregator aggregator,
+            CancellationTokenSource cancellationTokenSource) =>
+            RetryTestCaseRunner.RunAsync(this, diagnosticMessageSink, messageBus, cancellationTokenSource,
+                blockingMessageBus => new XunitTestCaseRunner(this, DisplayName, SkipReason, constructorArguments,
+                        TestMethodArguments, blockingMessageBus, aggregator, cancellationTokenSource)
+                    .RunAsync());
+
+        public override void Serialize(IXunitSerializationInfo data)
+        {
+            base.Serialize(data);
+
+            data.AddValue("MaxRetries", MaxRetries);
+            data.AddValue("DelayBetweenRetriesMs", DelayBetweenRetriesMs);
+        }
+
+        public override void Deserialize(IXunitSerializationInfo data)
+        {
+            base.Deserialize(data);
+
+            MaxRetries = data.GetValue<int>("MaxRetries");
+            DelayBetweenRetriesMs = data.GetValue<int>("DelayBetweenRetriesMs");
+        }
+    }
+}
diff --git a/test/TestingUtils/RetryTestCaseRunner.cs b/test/TestingUtils/RetryTestCaseRunner.cs
diff --git a/test/TestingUtils/RetryTheoryAttribute.cs b/test/TestingUtils/RetryTheoryAttribute.cs
diff --git a/test/TestingUtils/RetryTheoryDiscoverer.cs b/test/TestingUtils/RetryTheoryDiscoverer.cs
diff --git a/test/TestingUtils/RetryTheoryDiscoveryAtRuntimeCase.cs b/test/TestingUtils/RetryTheoryDiscoveryAtRuntimeCase.cs
diff --git a/test/TestingUtils/TheoryWithSkipOnAttribute.cs b/test/TestingUtils/TheoryWithSkipOnAttribute.cs
diff --git a/test/TestingUtils/UnitTestDetector.cs b/test/TestingUtils/UnitTestDetector.cs

Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,7 @@ await TestHelper.DelayUntil(`
`98`	`98`	`client.RegistrationManager.CurrentRegistrations.Should().Contain(x => x.Method == "@/" + TextDocumentNames.SemanticTokensFull);`
`99`	`99`	`}`
`100`	`100`
`101`		`- [FactWithSkipOn(SkipOnPlatform.All)]`
	`101`	`+ [RetryFact]`
`102`	`102`	`public async Task Should_Unregister_Dynamically_While_Server_Is_Running()`
`103`	`103`	`{`
`104`	`104`	`var (client, server) = await Initialize(new ConfigureClient().Configure, new ConfigureServer().Configure);`
Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ public Delegates(ITestOutputHelper testOutputHelper, LanguageProtocolFixture<Def`
`24`	`24`	`{`
`25`	`25`	`}`
`26`	`26`
`27`		`- [FactWithSkipOn(SkipOnPlatform.All)]`
	`27`	`+ [RetryFact]`
`28`	`28`	`public async Task Should_Behave_Like_A_Task()`
`29`	`29`	`{`
`30`	`30`	`var result = await Client.TextDocument.RequestSemanticTokens(`
`@@ -34,7 +34,7 @@ public async Task Should_Behave_Like_A_Task()`
`34`	`34`	`result!.Data.Should().HaveCount(3);`
`35`	`35`	`}`
`36`	`36`
`37`		`- [FactWithSkipOn(SkipOnPlatform.All)]`
	`37`	`+ [RetryFact]`
`38`	`38`	`public async Task Should_Behave_Like_An_Observable()`
`39`	`39`	`{`
`40`	`40`	`var items = await Client.TextDocument`
Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ public Delegates(ITestOutputHelper testOutputHelper, LanguageProtocolFixture<Def`
`28`	`28`	`{`
`29`	`29`	`}`
`30`	`30`
`31`		`- [FactWithSkipOn(SkipOnPlatform.All)]`
	`31`	`+ [RetryFact]`
`32`	`32`	`public async Task Should_Behave_Like_A_Task()`
`33`	`33`	`{`
`34`	`34`	`var result = await Client.TextDocument.RequestCodeLens(`
`@@ -41,7 +41,7 @@ public async Task Should_Behave_Like_A_Task()`
`41`	`41`	`result.Select(z => z.Command!.Name).Should().ContainInOrder("CodeLens 1", "CodeLens 2", "CodeLens 3");`
`42`	`42`	`}`
`43`	`43`
`44`		`- [FactWithSkipOn(SkipOnPlatform.All)]`
	`44`	`+ [RetryFact]`
`45`	`45`	`public async Task Should_Behave_Like_An_Observable()`
`46`	`46`	`{`
`47`	`47`	`var items = await Client.TextDocument`
Original file line number	Diff line number	Diff line change
`@@ -204,7 +204,7 @@ public async Task Should_Resolve_With_Data_Capability()`
`204`	`204`	`item.CodeAction!.Command!.Name.Should().Be("resolved");`
`205`	`205`	`}`
`206`	`206`
`207`		`- [FactWithSkipOn(SkipOnPlatform.Mac)]`
	`207`	`+ [RetryFact]`
`208`	`208`	`public async Task Should_Resolve_With_Partial_Data_Capability()`
`209`	`209`	`{`
`210`	`210`	`var (client, _) = await Initialize(`
`@@ -297,7 +297,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()`
`297`	`297`	`item.CodeAction!.Command!.Name.Should().Be("resolved");`
`298`	`298`	`}`
`299`	`299`
`300`		`- [FactWithSkipOn(SkipOnPlatform.Mac)]`
	`300`	`+ [RetryFact]`
`301`	`301`	`public async Task Should_Resolve_With_Partial_Data_CancellationToken()`
`302`	`302`	`{`
`303`	`303`	`var (client, _) = await Initialize(`
Original file line number	Diff line number	Diff line change
`@@ -195,7 +195,7 @@ public async Task Should_Resolve_With_Data_Capability()`
`195`	`195`	`item.Command!.Name.Should().Be("resolved");`
`196`	`196`	`}`
`197`	`197`
`198`		`- [FactWithSkipOn(SkipOnPlatform.Mac)]`
	`198`	`+ [RetryFact]`
`199`	`199`	`public async Task Should_Resolve_With_Partial_Data_Capability()`
`200`	`200`	`{`
`201`	`201`	`var (client, _) = await Initialize(`
`@@ -284,7 +284,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()`
`284`	`284`	`item.Command!.Name.Should().Be("resolved");`
`285`	`285`	`}`
`286`	`286`
`287`		`- [FactWithSkipOn(SkipOnPlatform.Mac)]`
	`287`	`+ [RetryFact]`
`288`	`288`	`public async Task Should_Resolve_With_Partial_Data_CancellationToken()`
`289`	`289`	`{`
`290`	`290`	`var (client, _) = await Initialize(`
Original file line number	Diff line number	Diff line change
`@@ -178,7 +178,7 @@ public async Task Should_Resolve_With_Data_Capability()`
`178`	`178`	`item.Tooltip.Should().Be("resolved");`
`179`	`179`	`}`
`180`	`180`
`181`		`- [FactWithSkipOn(SkipOnPlatform.Mac)]`
	`181`	`+ [RetryFact]`
`182`	`182`	`public async Task Should_Resolve_With_Partial_Data_Capability()`
`183`	`183`	`{`
`184`	`184`	`var (client, _) = await Initialize(`
`@@ -261,7 +261,7 @@ public async Task Should_Resolve_With_Data_CancellationToken()`
`261`	`261`	`item.Tooltip.Should().Be("resolved");`
`262`	`262`	`}`
`263`	`263`
`264`		`- [FactWithSkipOn(SkipOnPlatform.Mac)]`
	`264`	`+ [RetryFact]`
`265`	`265`	`public async Task Should_Resolve_With_Partial_Data_CancellationToken()`
`266`	`266`	`{`
`267`	`267`	`var (client, _) = await Initialize(`
Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,7 @@ public void Swap(ITestOutputHelper testOutputHelper)`
`46`	`46`	`_testOutputHelper.Swap(testOutputHelper);`
`47`	`47`	`}`
`48`	`48`
`49`		`- class InnerTestOutputHelper : ITestOutputHelper`
	`49`	`+ private class InnerTestOutputHelper : ITestOutputHelper`
`50`	`50`	`{`
`51`	`51`	`private ITestOutputHelper? _testOutputHelper;`
`52`	`52`