Ignore more tests that rely on timing. Enable tests that usually fail due to disk space issues

samdanaei · samdanaei · commit 1af2f9458104 · 2024-09-30T14:50:53.000+13:00
diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransferRetriesTimeout.cs
@@ -11,6 +11,7 @@
 using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Support;
 using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods;
+using Octopus.Tentacle.Tests.Integration.Support.TestAttributes;
 using Octopus.Tentacle.Tests.Integration.Util;
 using Octopus.Tentacle.Tests.Integration.Util.Builders;
 using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators;
@@ -24,6 +25,7 @@ namespace Octopus.Tentacle.Tests.Integration
     /// from RPC calls when they are being retried and the rpc timeout period elapses.
     /// </summary>
     [IntegrationTestTimeout]
+    [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
     public class ClientFileTransferRetriesTimeout : IntegrationTest
     {
         readonly TimeSpan retryIfRemainingDurationAtLeastBuffer = TimeSpan.FromSeconds(1);
diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled.cs
@@ -7,13 +7,15 @@
 using Octopus.Tentacle.Contracts.ClientServices;
 using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Support;
+using Octopus.Tentacle.Tests.Integration.Support.TestAttributes;
 using Octopus.Tentacle.Tests.Integration.Util.Builders;
 using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers;
 
 namespace Octopus.Tentacle.Tests.Integration
 {
     [IntegrationTestTimeout]
+    [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
     public class ClientFileTransfersAreNotRetriedWhenRetriesAreDisabled : IntegrationTest
     {
         [Test]
diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs b/source/Octopus.Tentacle.Tests.Integration/ClientFileTransfersAreRetriedWhenRetriesAreEnabled.cs
@@ -10,6 +10,7 @@
 using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Support;
 using Octopus.Tentacle.Tests.Integration.Support.ExtensionMethods;
+using Octopus.Tentacle.Tests.Integration.Support.TestAttributes;
 using Octopus.Tentacle.Tests.Integration.Util.Builders;
 using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Util.TcpTentacleHelpers;
@@ -20,6 +21,7 @@ public class ClientFileTransfersAreRetriedWhenRetriesAreEnabled : IntegrationTes
     {
         [Test]
         [TentacleConfigurations(testCommonVersions: true, scriptServiceToTest: ScriptServiceVersionToTest.None)]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public async Task FailedUploadsAreRetriedAndIsEventuallySuccessful(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()
diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutionRetriesTimeout.cs
@@ -218,7 +218,7 @@ public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled(
 
         [Test]
         [TentacleConfigurations]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             var retryDuration = TimeSpan.FromSeconds(15);
diff --git a/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs b/source/Octopus.Tentacle.Tests.Integration/ClientScriptExecutorObservesScriptObserverBackoffStrategy.cs
@@ -6,13 +6,15 @@
 using Octopus.Tentacle.Contracts.ClientServices;
 using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
 using Octopus.Tentacle.Tests.Integration.Support;
+using Octopus.Tentacle.Tests.Integration.Support.TestAttributes;
 using Octopus.Tentacle.Tests.Integration.Util;
 using Octopus.Tentacle.Tests.Integration.Util.Builders;
 using Octopus.Tentacle.Tests.Integration.Util.Builders.Decorators;
 
 namespace Octopus.Tentacle.Tests.Integration
 {
     [IntegrationTestTimeout]
+    [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
     public class ClientScriptExecutorObservesScriptObserverBackoffStrategy : IntegrationTest
     {
         [Test]
diff --git a/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs b/source/Octopus.Tentacle.Tests.Integration/FileTransferServiceTests.cs
@@ -16,7 +16,6 @@ public class FileTransferServiceTests : IntegrationTest
     {
         [Test]
         [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
         public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             using var fileToUpload = new RandomTemporaryFileBuilder().Build();
@@ -44,7 +43,6 @@ public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleC
 
         [Test]
         [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
         public async Task DownloadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             using var fileToDownload = new RandomTemporaryFileBuilder().Build();
diff --git a/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs b/source/Octopus.Tentacle.Tests.Integration/ScriptServiceV2IntegrationTest.cs
@@ -133,7 +133,7 @@ await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, Cancellati
 
         [Test]
         [TentacleConfigurations]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase)
         {
             await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()
diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs b/source/Octopus.Tentacle.Tests.Integration/Support/TestAttributes/SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute.cs
@@ -10,6 +10,13 @@ namespace Octopus.Tentacle.Tests.Integration.Support.TestAttributes
     [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)]
     public class SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute : NUnitAttribute, IApplyToTest
     {
+        string Reason { get; }
+
+        public SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute(string reason)
+        {
+            Reason = reason;
+        }
+        
         public void ApplyToTest(Test test)
         {
             if (test.RunState == RunState.NotRunnable || test.RunState == RunState.Ignored)
@@ -18,7 +25,7 @@ public void ApplyToTest(Test test)
             if (bool.TryParse(Environment.GetEnvironmentVariable("Has_Known_Performance_Issues"), out _))
             {
                 test.RunState = RunState.Skipped;
-                test.Properties.Add("_SKIPREASON", "This test only runs on environments without performance issues");
+                test.Properties.Add("_SKIPREASON", $"This test only runs on environments without performance issues because {Reason}");
             }
         }
     }
diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs
@@ -28,6 +28,7 @@ namespace Octopus.Tentacle.Tests.Integration
     /// Please review any changes to the assertions made by these tests carefully.
     /// </summary>
     [IntegrationTestTimeout]
+    [SkipOnEnvironmentsWithKnownPerformanceIssues("we don't expect customers to execute it in this environment in a situation where it needs to behave within a reasonable timeframe.")]
     public class TentacleCommandLineTests : IntegrationTest
     {
         [Test]
@@ -272,7 +273,6 @@ public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigu
 
         [Test]
         [TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
         public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc)
         {
             var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json");
diff --git a/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs b/source/Octopus.Tentacle.Tests.Integration/Util/SilentProcessRunnerFixture.cs
@@ -33,6 +33,7 @@ public void SetUpLocal()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void ExitCode_ShouldBeReturned()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -56,6 +57,7 @@ public void ExitCode_ShouldBeReturned()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -83,7 +85,7 @@ public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault()
 
         [Test]
         [Retry(3)]
-        [SkipOnEnvironmentsWithKnownPerformanceIssues]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void CancellationToken_ShouldForceKillTheProcess()
         {
             // Terminate the process after a very short time so the test doesn't run forever
@@ -116,6 +118,7 @@ public void CancellationToken_ShouldForceKillTheProcess()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void EchoHello_ShouldWriteToStdOut()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -138,6 +141,7 @@ public void EchoHello_ShouldWriteToStdOut()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void EchoError_ShouldWriteToStdErr()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -160,6 +164,7 @@ public void EchoError_ShouldWriteToStdErr()
         }
 
         [Test]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void RunAsCurrentUser_ShouldWork()
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
@@ -186,6 +191,7 @@ public void RunAsCurrentUser_ShouldWork()
         [Test]
         [WindowsTest]
         [TestCase("powershell.exe", "-command \"Write-Host $env:userdomain\\$env:username\"")]
+        [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]
         public void RunAsCurrentUser_PowerShell_ShouldWork(string command, string arguments)
         {
             using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))

Original file line number	Diff line number	Diff line change
`@@ -218,7 +218,7 @@ public async Task WhenRpcRetriesTimeOut_DuringStartScript_TheRpcCallIsCancelled(`
`218`	`218`
`219`	`219`	`[Test]`
`220`	`220`	`[TentacleConfigurations]`
`221`		`- [SkipOnEnvironmentsWithKnownPerformanceIssues]`
	`221`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]`
`222`	`222`	`public async Task WhenStartScriptFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase)`
`223`	`223`	`{`
`224`	`224`	`var retryDuration = TimeSpan.FromSeconds(15);`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,6 @@ public class FileTransferServiceTests : IntegrationTest`
`16`	`16`	`{`
`17`	`17`	`[Test]`
`18`	`18`	`[TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]`
`19`		`- [SkipOnEnvironmentsWithKnownPerformanceIssues]`
`20`	`19`	`public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase)`
`21`	`20`	`{`
`22`	`21`	`using var fileToUpload = new RandomTemporaryFileBuilder().Build();`
`@@ -44,7 +43,6 @@ public async Task UploadFileSuccessfully(TentacleConfigurationTestCase tentacleC`
`44`	`43`
`45`	`44`	`[Test]`
`46`	`45`	`[TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]`
`47`		`- [SkipOnEnvironmentsWithKnownPerformanceIssues]`
`48`	`46`	`public async Task DownloadFileSuccessfully(TentacleConfigurationTestCase tentacleConfigurationTestCase)`
`49`	`47`	`{`
`50`	`48`	`using var fileToDownload = new RandomTemporaryFileBuilder().Build();`
Original file line number	Diff line number	Diff line change
`@@ -133,7 +133,7 @@ await clientTentacle.TentacleClient.ExecuteScript(startScriptCommand, Cancellati`
`133`	`133`
`134`	`134`	`[Test]`
`135`	`135`	`[TentacleConfigurations]`
`136`		`- [SkipOnEnvironmentsWithKnownPerformanceIssues]`
	`136`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]`
`137`	`137`	`public async Task WhenALongRunningScriptIsCancelled_TheScriptShouldStop(TentacleConfigurationTestCase tentacleConfigurationTestCase)`
`138`	`138`	`{`
`139`	`139`	`await using var clientTentacle = await tentacleConfigurationTestCase.CreateBuilder()`
Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,13 @@ namespace Octopus.Tentacle.Tests.Integration.Support.TestAttributes`
`10`	`10`	`[AttributeUsage(AttributeTargets.Class \| AttributeTargets.Method)]`
`11`	`11`	`public class SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute : NUnitAttribute, IApplyToTest`
`12`	`12`	`{`
	`13`	`+ string Reason { get; }`
	`14`	`+`
	`15`	`+ public SkipOnEnvironmentsWithKnownPerformanceIssuesAttribute(string reason)`
	`16`	`+ {`
	`17`	`+ Reason = reason;`
	`18`	`+ }`
	`19`	`+`
`13`	`20`	`public void ApplyToTest(Test test)`
`14`	`21`	`{`
`15`	`22`	`if (test.RunState == RunState.NotRunnable \|\| test.RunState == RunState.Ignored)`
`@@ -18,7 +25,7 @@ public void ApplyToTest(Test test)`
`18`	`25`	`if (bool.TryParse(Environment.GetEnvironmentVariable("Has_Known_Performance_Issues"), out _))`
`19`	`26`	`{`
`20`	`27`	`test.RunState = RunState.Skipped;`
`21`		`- test.Properties.Add("_SKIPREASON", "This test only runs on environments without performance issues");`
	`28`	`+ test.Properties.Add("_SKIPREASON", $"This test only runs on environments without performance issues because {Reason}");`
`22`	`29`	`}`
`23`	`30`	`}`
`24`	`31`	`}`
Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,7 @@ namespace Octopus.Tentacle.Tests.Integration`
`28`	`28`	`/// Please review any changes to the assertions made by these tests carefully.`
`29`	`29`	`/// </summary>`
`30`	`30`	`[IntegrationTestTimeout]`
	`31`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("we don't expect customers to execute it in this environment in a situation where it needs to behave within a reasonable timeframe.")]`
`31`	`32`	`public class TentacleCommandLineTests : IntegrationTest`
`32`	`33`	`{`
`33`	`34`	`[Test]`
`@@ -272,7 +273,6 @@ public async Task CommandSpecificHelpAsJsonLooksSensibleToHumans(TentacleConfigu`
`272`	`273`
`273`	`274`	`[Test]`
`274`	`275`	`[TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]`
`275`		`- [SkipOnEnvironmentsWithKnownPerformanceIssues]`
`276`	`276`	`public async Task HelpForInstanceSpecificCommandsAlwaysWorks(TentacleConfigurationTestCase tc)`
`277`	`277`	`{`
`278`	`278`	`var (_, stdout, stderr) = await RunCommand(tc, null, "help", "--format=json");`
Original file line number	Diff line number	Diff line change
`@@ -33,6 +33,7 @@ public void SetUpLocal()`
`33`	`33`	`}`
`34`	`34`
`35`	`35`	`[Test]`
	`36`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]`
`36`	`37`	`public void ExitCode_ShouldBeReturned()`
`37`	`38`	`{`
`38`	`39`	`using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))`
`@@ -56,6 +57,7 @@ public void ExitCode_ShouldBeReturned()`
`56`	`57`	`}`
`57`	`58`
`58`	`59`	`[Test]`
	`60`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]`
`59`	`61`	`public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault()`
`60`	`62`	`{`
`61`	`63`	`using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))`
`@@ -83,7 +85,7 @@ public void DebugLogging_ShouldContainDiagnosticsInfo_ForDefault()`
`83`	`85`
`84`	`86`	`[Test]`
`85`	`87`	`[Retry(3)]`
`86`		`- [SkipOnEnvironmentsWithKnownPerformanceIssues]`
	`88`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]`
`87`	`89`	`public void CancellationToken_ShouldForceKillTheProcess()`
`88`	`90`	`{`
`89`	`91`	`// Terminate the process after a very short time so the test doesn't run forever`
`@@ -116,6 +118,7 @@ public void CancellationToken_ShouldForceKillTheProcess()`
`116`	`118`	`}`
`117`	`119`
`118`	`120`	`[Test]`
	`121`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]`
`119`	`122`	`public void EchoHello_ShouldWriteToStdOut()`
`120`	`123`	`{`
`121`	`124`	`using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))`
`@@ -138,6 +141,7 @@ public void EchoHello_ShouldWriteToStdOut()`
`138`	`141`	`}`
`139`	`142`
`140`	`143`	`[Test]`
	`144`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]`
`141`	`145`	`public void EchoError_ShouldWriteToStdErr()`
`142`	`146`	`{`
`143`	`147`	`using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))`
`@@ -160,6 +164,7 @@ public void EchoError_ShouldWriteToStdErr()`
`160`	`164`	`}`
`161`	`165`
`162`	`166`	`[Test]`
	`167`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]`
`163`	`168`	`public void RunAsCurrentUser_ShouldWork()`
`164`	`169`	`{`
`165`	`170`	`using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))`
`@@ -186,6 +191,7 @@ public void RunAsCurrentUser_ShouldWork()`
`186`	`191`	`[Test]`
`187`	`192`	`[WindowsTest]`
`188`	`193`	`[TestCase("powershell.exe", "-command \"Write-Host $env:userdomain\\$env:username\"")]`
	`194`	`+ [SkipOnEnvironmentsWithKnownPerformanceIssues("it relies on timing, which may be inconsistent within the environment")]`
`189`	`195`	`public void RunAsCurrentUser_PowerShell_ShouldWork(string command, string arguments)`
`190`	`196`	`{`
`191`	`197`	`using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))`