Skip to content

Commit 9b05bef

Browse files
JustinGroteJustinGrote
committed
Improve E2E Test Fixtures to be less flaky
1 parent 9d56407 commit 9b05bef

18 files changed

+587
-784
lines changed

‎src/PowerShellEditorServices/Server/PsesDebugServer.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ public async Task StartAsync()
120120
response.SupportsDelayedStackTraceLoading = true;
121121

122122
return Task.CompletedTask;
123-
});
123+
})
124+
;
124125
}).ConfigureAwait(false);
125126
}
126127

‎test/PowerShellEditorServices.Test.E2E/DebugAdapterProtocolMessageTests.cs

+229-192
Large diffs are not rendered by default.

‎test/PowerShellEditorServices.Test.E2E/Hosts/DebugOutputStream.cs

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
#nullable enable
4+
5+
using System.Diagnostics;
6+
using System.IO;
7+
using System.Text;
8+
using Nerdbank.Streams;
9+
10+
namespace PowerShellEditorServices.Test.E2E;
11+
12+
/// <summary>
13+
/// A stream that logs all data read and written to the debug stream which is visible in the debug console when a
14+
/// debugger is attached.
15+
/// </summary>
16+
internal class DebugOutputStream : MonitoringStream
17+
{
18+
public DebugOutputStream(Stream? underlyingStream)
19+
: base(underlyingStream ?? new MemoryStream())
20+
{
21+
DidRead += (_, segment) =>
22+
{
23+
if (segment.Array is null) { return; }
24+
LogData("⬅️", segment.Array, segment.Offset, segment.Count);
25+
};
26+
27+
DidWrite += (_, segment) =>
28+
{
29+
if (segment.Array is null) { return; }
30+
LogData("➡️", segment.Array, segment.Offset, segment.Count);
31+
};
32+
}
33+
34+
private static void LogData(string header, byte[] buffer, int offset, int count)
35+
{
36+
// If debugging, the raw traffic will be visible in the debug console
37+
if (Debugger.IsAttached)
38+
{
39+
string data = Encoding.UTF8.GetString(buffer, offset, count);
40+
Debug.WriteLine($"{header} {data}");
41+
}
42+
}
43+
}

‎test/PowerShellEditorServices.Test.E2E/Hosts/IAsyncLanguageServerHost.cs

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
#nullable enable
4+
5+
using System;
6+
using System.IO;
7+
using System.Threading;
8+
using System.Threading.Tasks;
9+
10+
namespace PowerShellEditorServices.Test.E2E;
11+
12+
/// <summary>
13+
/// Represents a debug adapter server host that can be started and stopped and provides streams for communication.
14+
/// </summary>
15+
public interface IAsyncLanguageServerHost : IAsyncDisposable
16+
{
17+
// Start the host and return when the host is ready to communicate. It should return a tuple of a stream Reader and stream Writer for communication with the LSP. The underlying streams can be retrieved via baseStream propertyif needed.
18+
Task<(StreamReader, StreamWriter)> Start(CancellationToken token = default);
19+
// Stops the host and returns when the host has fully stopped. It should be idempotent, such that if called while the host is already stopping/stopped, it will have the same result
20+
Task<bool> Stop(CancellationToken token = default);
21+
22+
// Optional to implement if more is required than a simple stop
23+
async ValueTask IAsyncDisposable.DisposeAsync() => await Stop();
24+
}

‎test/PowerShellEditorServices.Test.E2E/DebugAdapterClientExtensions.cs ‎test/PowerShellEditorServices.Test.E2E/Hosts/IDebugAdapterClientExtensions.cs

+3-6
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@
44
using System;
55
using System.Threading.Tasks;
66
using Microsoft.PowerShell.EditorServices.Handlers;
7-
using OmniSharp.Extensions.DebugAdapter.Client;
7+
using OmniSharp.Extensions.DebugAdapter.Protocol.Client;
88
using OmniSharp.Extensions.DebugAdapter.Protocol.Requests;
99

1010
namespace PowerShellEditorServices.Test.E2E
1111
{
12-
public static class DebugAdapterClientExtensions
12+
public static class IDebugAdapterClientExtensions
1313
{
14-
public static async Task LaunchScript(this DebugAdapterClient debugAdapterClient, string script, TaskCompletionSource<object> started, string executeMode = "DotSource")
14+
public static async Task LaunchScript(this IDebugAdapterClient debugAdapterClient, string script, string executeMode = "DotSource")
1515
{
1616
_ = await debugAdapterClient.Launch(
1717
new PsesLaunchRequestArguments
@@ -22,9 +22,6 @@ public static async Task LaunchScript(this DebugAdapterClient debugAdapterClient
2222
CreateTemporaryIntegratedConsole = false,
2323
ExecuteMode = executeMode,
2424
}) ?? throw new Exception("Launch response was null.");
25-
26-
// This will check to see if we received the Initialized event from the server.
27-
await started.Task;
2825
}
2926
}
3027
}

‎test/PowerShellEditorServices.Test.E2E/Hosts/PsesStdioLanguageServerProcessHost.cs

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Reflection;
8+
9+
namespace PowerShellEditorServices.Test.E2E;
10+
11+
/// <summary>
12+
/// A <see cref="ServerManager"/> is responsible for launching or attaching to a language server, providing access to its input and output streams, and tracking its lifetime.
13+
/// </summary>
14+
internal class PsesStdioLanguageServerProcessHost(bool isDebugAdapter)
15+
: StdioLanguageServerProcessHost(PwshExe, GeneratePsesArguments(isDebugAdapter))
16+
{
17+
protected static readonly string s_binDir =
18+
Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
19+
20+
private static readonly string s_bundledModulePath = new FileInfo(Path.Combine(
21+
s_binDir, "..", "..", "..", "..", "..", "module")).FullName;
22+
23+
private static readonly string s_sessionDetailsPath = Path.Combine(
24+
s_binDir, $"pses_test_sessiondetails_{Path.GetRandomFileName()}");
25+
26+
private static readonly string s_logPath = Path.Combine(
27+
s_binDir, $"pses_test_logs_{Path.GetRandomFileName()}");
28+
29+
private const string s_logLevel = "Diagnostic";
30+
private static readonly string[] s_featureFlags = { "PSReadLine" };
31+
private const string s_hostName = "TestHost";
32+
private const string s_hostProfileId = "TestHost";
33+
private const string s_hostVersion = "1.0.0";
34+
35+
// Adjust the environment variable if wanting to test with 5.1 or a specific pwsh path
36+
public static string PwshExe { get; } = Environment.GetEnvironmentVariable("PWSH_EXE_NAME") ?? "pwsh";
37+
public static bool IsWindowsPowerShell { get; } = PwshExe.EndsWith("powershell");
38+
public static bool RunningInConstrainedLanguageMode { get; } =
39+
Environment.GetEnvironmentVariable("__PSLockdownPolicy", EnvironmentVariableTarget.Machine) != null;
40+
41+
private static string[] GeneratePsesArguments(bool isDebugAdapter)
42+
{
43+
List<string> args = new()
44+
{
45+
"&",
46+
SingleQuoteEscape(Path.Combine(s_bundledModulePath, "PowerShellEditorServices", "Start-EditorServices.ps1")),
47+
"-LogPath",
48+
SingleQuoteEscape(s_logPath),
49+
"-LogLevel",
50+
s_logLevel,
51+
"-SessionDetailsPath",
52+
SingleQuoteEscape(s_sessionDetailsPath),
53+
"-FeatureFlags",
54+
string.Join(',', s_featureFlags),
55+
"-HostName",
56+
s_hostName,
57+
"-HostProfileId",
58+
s_hostProfileId,
59+
"-HostVersion",
60+
s_hostVersion,
61+
"-BundledModulesPath",
62+
SingleQuoteEscape(s_bundledModulePath),
63+
"-Stdio"
64+
};
65+
66+
if (isDebugAdapter)
67+
{
68+
args.Add("-DebugServiceOnly");
69+
}
70+
71+
string base64Str = Convert.ToBase64String(
72+
System.Text.Encoding.Unicode.GetBytes(string.Join(' ', args)));
73+
74+
return
75+
[
76+
"-NoLogo",
77+
"-NoProfile",
78+
"-EncodedCommand",
79+
base64Str
80+
];
81+
}
82+
83+
private static string SingleQuoteEscape(string str) => $"'{str.Replace("'", "''")}'";
84+
}

‎test/PowerShellEditorServices.Test.E2E/Hosts/StdioLanguageServerProcessHost.cs

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
#nullable enable
4+
5+
using System;
6+
using System.Collections.Generic;
7+
using System.Diagnostics;
8+
using System.IO;
9+
using System.Threading;
10+
using System.Threading.Tasks;
11+
12+
namespace PowerShellEditorServices.Test.E2E;
13+
14+
/// <summary>
15+
/// Hosts a language server process that communicates over stdio
16+
/// </summary>
17+
internal class StdioLanguageServerProcessHost(string fileName, IEnumerable<string> argumentList) : IAsyncLanguageServerHost
18+
{
19+
// The PSES process that will be started and managed
20+
private readonly Process process = new()
21+
{
22+
EnableRaisingEvents = true,
23+
StartInfo = new ProcessStartInfo(fileName, argumentList)
24+
{
25+
RedirectStandardInput = true,
26+
RedirectStandardOutput = true,
27+
RedirectStandardError = true,
28+
UseShellExecute = false,
29+
CreateNoWindow = true
30+
}
31+
};
32+
33+
// Track the state of the startup
34+
private TaskCompletionSource<(StreamReader, StreamWriter)>? startTcs;
35+
private TaskCompletionSource<bool>? stopTcs;
36+
37+
// Starts the process. Returns when the process has started and streams are available.
38+
public async Task<(StreamReader, StreamWriter)> Start(CancellationToken token = default)
39+
{
40+
// Runs this once upon process exit to clean up the state.
41+
EventHandler? exitHandler = null;
42+
exitHandler = (sender, e) =>
43+
{
44+
// Complete the stopTcs task when the process finally exits, allowing stop to complete
45+
stopTcs?.TrySetResult(true);
46+
stopTcs = null;
47+
startTcs = null;
48+
process.Exited -= exitHandler;
49+
};
50+
process.Exited += exitHandler;
51+
52+
if (stopTcs is not null)
53+
{
54+
throw new InvalidOperationException("The process is currently stopping and cannot be started.");
55+
}
56+
57+
// Await the existing task if we have already started, making this operation idempotent
58+
if (startTcs is not null)
59+
{
60+
return await startTcs.Task;
61+
}
62+
63+
// Initiate a new startTcs to track the startup
64+
startTcs = new();
65+
66+
token.ThrowIfCancellationRequested();
67+
68+
// Should throw if there are any startup problems such as invalid path, etc.
69+
process.Start();
70+
71+
// According to the source the streams should be allocated synchronously after the process has started, however it's not super clear so we will put this here in case there is an explicit race condition.
72+
if (process.StandardInput.BaseStream is null || process.StandardOutput.BaseStream is null)
73+
{
74+
throw new InvalidOperationException("The process has started but the StandardInput or StandardOutput streams are not available. This should never happen and is probably a race condition, please report it to PowerShellEditorServices.");
75+
}
76+
77+
startTcs.SetResult((
78+
process.StandardOutput,
79+
process.StandardInput
80+
));
81+
82+
// Return the result of the completion task
83+
return await startTcs.Task;
84+
}
85+
86+
public async Task WaitForExit(CancellationToken token = default)
87+
{
88+
AssertStarting();
89+
await process.WaitForExitAsync(token);
90+
}
91+
92+
/// <summary>
93+
/// Determines if the process is in the starting state and throws if not.
94+
/// </summary>
95+
private void AssertStarting()
96+
{
97+
if (startTcs is null)
98+
{
99+
throw new InvalidOperationException("The process is not starting/started, use Start() first.");
100+
}
101+
}
102+
103+
public async Task<bool> Stop(CancellationToken token = default)
104+
{
105+
AssertStarting();
106+
if (stopTcs is not null)
107+
{
108+
return await stopTcs.Task;
109+
}
110+
stopTcs = new();
111+
token.ThrowIfCancellationRequested();
112+
process.Kill();
113+
await process.WaitForExitAsync(token);
114+
return true;
115+
}
116+
}

‎test/PowerShellEditorServices.Test.E2E/LSPTestsFixures.cs ‎test/PowerShellEditorServices.Test.E2E/LSPTestsFixtures.cs

+13-10
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@
88
using System.Reflection;
99
using System.Threading;
1010
using System.Threading.Tasks;
11-
using Microsoft.Extensions.Logging;
1211
using Microsoft.PowerShell.EditorServices.Logging;
1312
using Microsoft.PowerShell.EditorServices.Services.Configuration;
13+
using Nerdbank.Streams;
1414
using Newtonsoft.Json.Linq;
1515
using OmniSharp.Extensions.LanguageServer.Client;
1616
using OmniSharp.Extensions.LanguageServer.Protocol;
@@ -38,27 +38,30 @@ public class LSPTestsFixture : IAsyncLifetime
3838
internal List<PsesTelemetryEvent> TelemetryEvents = new();
3939
public ITestOutputHelper Output { get; set; }
4040

41-
protected PsesStdioProcess _psesProcess;
42-
public int ProcessId => _psesProcess.Id;
41+
internal PsesStdioLanguageServerProcessHost _psesHost = new(IsDebugAdapterTests);
4342

4443
public async Task InitializeAsync()
4544
{
46-
LoggerFactory factory = new();
47-
_psesProcess = new PsesStdioProcess(factory, IsDebugAdapterTests);
48-
await _psesProcess.Start();
45+
(StreamReader stdout, StreamWriter stdin) = await _psesHost.Start();
46+
47+
// Splice the streams together and enable debug logging of all messages sent and received
48+
DebugOutputStream psesStream = new(
49+
FullDuplexStream.Splice(stdout.BaseStream, stdin.BaseStream)
50+
);
4951

5052
DirectoryInfo testDir =
5153
Directory.CreateDirectory(Path.Combine(s_binDir, Path.GetRandomFileName()));
5254

5355
PsesLanguageClient = LanguageClient.PreInit(options =>
5456
{
5557
options
56-
.WithInput(_psesProcess.OutputStream)
57-
.WithOutput(_psesProcess.InputStream)
58+
.WithInput(psesStream)
59+
.WithOutput(psesStream)
5860
.WithWorkspaceFolder(DocumentUri.FromFileSystemPath(testDir.FullName), "testdir")
5961
.WithInitializationOptions(new { EnableProfileLoading = false })
6062
.OnPublishDiagnostics(diagnosticParams => Diagnostics.AddRange(diagnosticParams.Diagnostics.Where(d => d != null)))
61-
.OnLogMessage(logMessageParams => {
63+
.OnLogMessage(logMessageParams =>
64+
{
6265
Output?.WriteLine($"{logMessageParams.Type}: {logMessageParams.Message}");
6366
Messages.Add(logMessageParams);
6467
})
@@ -98,7 +101,7 @@ public async Task InitializeAsync()
98101
public async Task DisposeAsync()
99102
{
100103
await PsesLanguageClient.Shutdown();
101-
await _psesProcess.Stop();
104+
await _psesHost.Stop();
102105
PsesLanguageClient?.Dispose();
103106
}
104107
}

0 commit comments

Comments
 (0)