Update integration tests to run against C# Dev Kit as well

Fixup tasks

Separate integration tests into more CI jobs

Author: dibarbet

.vscode/launch.json

@@ -19,10 +19,10 @@
             "args": [
                 // Launch VSCode using a specific profile to ensure that user settings are not used.
                 // This profile must be imported into vscode before running this launch configuration.
-                // The profile can be found under /test/csharp-standalone-profile.
+                // The profile can be found under /test/csharp-test-profile.
                 "--profile",
-                "csharp-standalone-profile",
-                "${workspaceRoot}/test/lsptoolshost/integrationTests/testAssets/slnWithCsproj/.vscode/lsp_tools_host_slnWithCsproj.code-workspace",
+                "csharp-test-profile",
+                "${workspaceRoot}/test/lsptoolshost/integrationTests/testAssets/slnWithCsproj/.vscode/slnWithCsproj.code-workspace",
                 "--extensionDevelopmentPath=${workspaceRoot}",
                 "--extensionTestsPath=${workspaceRoot}/out/test/lsptoolshost/integrationTests"
             ],
@@ -36,6 +36,31 @@
             "preLaunchTask": "buildDev",
             "internalConsoleOptions": "openOnSessionStart"
         },
+        {
+            "name": "[DevKit] Launch Current File slnWithCsproj Integration Tests",
+            "type": "extensionHost",
+            "request": "launch",
+            "runtimeExecutable": "${execPath}",
+            "args": [
+                // Launch VSCode using a specific profile to ensure that user settings are not used.
+                // This profile must be imported into vscode before running this launch configuration.
+                // The profile can be found under /test/csharp-test-profile.
+                "--profile",
+                "csharp-test-profile",
+                "${workspaceRoot}/test/lsptoolshost/integrationTests/testAssets/slnWithCsproj/.vscode/devkit_slnWithCsproj.code-workspace",
+                "--extensionDevelopmentPath=${workspaceRoot}",
+                "--extensionTestsPath=${workspaceRoot}/out/test/lsptoolshost/integrationTests"
+            ],
+            "env": {
+                "CODE_EXTENSIONS_PATH": "${workspaceRoot}",
+                "TEST_FILE_FILTER": "${file}",
+            },
+            "sourceMaps": true,
+            "outFiles": ["${workspaceRoot}/dist/*.js", "${workspaceRoot}/out/test/**/*.js"],
+            "resolveSourceMapLocations": ["${workspaceFolder}/**", "!**/node_modules/**"],
+            "preLaunchTask": "buildDev",
+            "internalConsoleOptions": "openOnSessionStart"
+        },
         {
             "name": "Launch Current File BasicRazorApp2_1 Integration Tests",
             "type": "extensionHost",
@@ -44,9 +69,9 @@
             "args": [
                 // Launch VSCode using a specific profile to ensure that user settings are not used.
                 // This profile must be imported into vscode before running this launch configuration.
-                // The profile can be found under /test/csharp-standalone-profile.
+                // The profile can be found under /test/csharp-test-profile.
                 "--profile",
-                "csharp-standalone-profile",
+                "csharp-test-profile",
                 "${workspaceRoot}/test/razor/razorIntegrationTests/testAssets/BasicRazorApp2_1/.vscode/lsp_tools_host_BasicRazorApp2_1.code-workspace",
                 "--extensionDevelopmentPath=${workspaceRoot}",
                 "--extensionTestsPath=${workspaceRoot}/out/test/razor/razorIntegrationTests"
@@ -69,9 +94,9 @@
             "args": [
                 // Launch VSCode using a specific profile to ensure that user settings are not used.
                 // This profile must be imported into vscode before running this launch configuration.
-                // The profile can be found under /test/csharp-standalone-profile.
+                // The profile can be found under /test/csharp-test-profile.
                 "--profile",
-                "csharp-standalone-profile",
+                "csharp-test-profile",
                 "${workspaceRoot}/test/omnisharp/omnisharpIntegrationTests/testAssets/${input:omnisharpAssetName}/.vscode/omnisharp_${input:omnisharpAssetName}.code-workspace",
                 "--extensionDevelopmentPath=${workspaceRoot}",
                 "--extensionTestsPath=${workspaceRoot}/out/test/omnisharp/omnisharpIntegrationTests"
@@ -94,9 +119,9 @@
             "args": [
                 // Launch VSCode using a specific profile to ensure that user settings are not used.
                 // This profile must be imported into vscode before running this launch configuration.
-                // The profile can be found under /test/csharp-standalone-profile.
+                // The profile can be found under /test/csharp-test-profile.
                 "--profile",
-                "csharp-standalone-profile",
+                "csharp-test-profile",
                 "${workspaceRoot}/test/omnisharp/omnisharpIntegrationTests/testAssets/${input:omnisharpAssetName}/.vscode/omnisharp_lsp_${input:omnisharpAssetName}.code-workspace",
                 "--extensionDevelopmentPath=${workspaceRoot}",
                 "--extensionTestsPath=${workspaceRoot}/out/test/omnisharp/omnisharpIntegrationTests"

CONTRIBUTING.md

@@ -56,9 +56,9 @@ To run all tests, execute `npm run test`.
 To debug unit tests locally, press <kbd>F5</kbd> in VS Code with the "Launch Tests" debug configuration selected.
 
 To debug integration tests
-1.  Import the `csharp-standalone-profile.code-profile` in VSCode to setup a clean profile in which to run integration tests.  This must be imported at least once to use the launch configurations.
+1.  Import the `csharp-test-profile.code-profile` in VSCode to setup a clean profile in which to run integration tests.  This must be imported at least once to use the launch configurations (ensure the extensions are updated in the profile).
 2.  Open any integration test file and <kbd>F5</kbd> launch with the correct launch configuration selected.
-    - For integration tests inside `test/lsptoolshost`, use `Launch Current File slnWithCsproj Integration Tests`
+    - For integration tests inside `test/lsptoolshost`, use either `Launch Current File slnWithCsproj Integration Tests` or `[DevKit] Launch Current File slnWithCsproj Integration Tests` (to run tests using C# + C# Dev Kit)
     - For integration tests inside `test/razor`, use `Launch Current File BasicRazorApp2_1 Integration Tests`
     - For integration tests inside `test/omnisharp`, use one of the `Omnisharp:` current file profiles
 

azure-pipelines.yml

@@ -24,58 +24,55 @@ stages:
     isOfficial: false
     signType: test
 
-- stage: Test_Linux_Stage
-  displayName: Test Linux
+- stage:
+  displayName: Test Linux (.NET 6)
   dependsOn: []
   jobs:
-  - job: Test_Linux_Job
-    displayName:  Test Linux
-    strategy:
-      matrix:
-        DotNet6:
-          containerName: mcr.microsoft.com/dotnet/sdk:6.0
-        DotNet7:
-          containerName: mcr.microsoft.com/dotnet/sdk:7.0
-        DotNet8:
-          containerName: mcr.microsoft.com/dotnet/sdk:8.0
-    pool:
-      name: NetCore-Public
-      demands: ImageOverride -equals 1es-ubuntu-2004-open
-    container: $[ variables['containerName'] ]
-    steps:
-    - template: azure-pipelines/test.yml
-      parameters:
-        # Prefer the dotnet from the container.
-        installDotNet: false
-        installAdditionalLinuxDependencies: true
+  - template: azure-pipelines/test-matrix.yml
+    parameters:
+      # Prefer the dotnet from the container.
+      installDotNet: false
+      installAdditionalLinuxDependencies: true
+      pool:
+        name: NetCore-Public
+        demands: ImageOverride -equals 1es-ubuntu-2004-open
+      containerName: mcr.microsoft.com/dotnet/sdk:6.0
+
+- stage:
+  displayName: Test Linux (.NET 8)
+  dependsOn: []
+  jobs:
+  - template: azure-pipelines/test-matrix.yml
+    parameters:
+      # Prefer the dotnet from the container.
+      installDotNet: false
+      installAdditionalLinuxDependencies: true
+      pool:
+        name: NetCore-Public
+        demands: ImageOverride -equals 1es-ubuntu-2004-open
+      containerName: mcr.microsoft.com/dotnet/sdk:8.0
 
 - stage: Test_Windows_Stage
   displayName: Test Windows
   dependsOn: []
   jobs:
-  - job: Test_Windows_Job
-    displayName: Test Windows
-    pool:
-      name: NetCore-Public
-      demands: ImageOverride -equals 1es-windows-2022-open
-    steps:
-    - template: azure-pipelines/test.yml
-      parameters:
-        installDotNet: true
+  - template: azure-pipelines/test-matrix.yml
+    parameters:
+      installDotNet: true
+      pool:
+        name: NetCore-Public
+        demands: ImageOverride -equals 1es-windows-2022-open
 
 - stage: Test_MacOS_Stage
   displayName: Test MacOS
   dependsOn: []
   jobs:
-  - job: Test_MacOS_Job
-    displayName: Test MacOS
-    pool:
-      name: Azure Pipelines
-      vmImage: macOS-13
-    steps:
-    - template: azure-pipelines/test.yml
-      parameters:
-        installDotNet: true
+  - template: azure-pipelines/test-matrix.yml
+    parameters:
+      installDotNet: true
+      pool:
+        name: Azure Pipelines
+        vmImage: macOS-13
 
 - stage: Test_OmniSharp
   displayName: Test OmniSharp

azure-pipelines/test-matrix.yml

@@ -0,0 +1,30 @@
+parameters:
+  - name: pool
+    type: object
+  - name: containerName
+    type: string
+    default: ''
+  - name: installDotNet
+    type: boolean
+  - name: installAdditionalLinuxDependencies
+    type: boolean
+    default: false
+
+jobs:
+- job:
+  strategy:
+    matrix:
+      CSharpTests:
+        npmCommand: 'test:withoutDevKit'
+      DevKitTests:
+        npmCommand: test:integration:devkit
+  pool: ${{ parameters.pool }}
+  ${{ if parameters.containerName }}:
+    container: ${{ parameters.containerName }}
+  steps:
+  - template: /azure-pipelines/test.yml@self
+    parameters:
+      # Prefer the dotnet from the container.
+      installDotNet: false
+      installAdditionalLinuxDependencies: true
+      npmCommand: $(npmCommand)

azure-pipelines/test.yml

@@ -4,6 +4,8 @@ parameters:
   - name: installAdditionalLinuxDependencies
     type: boolean
     default: false
+  - name: npmCommand
+    type: string
 
 steps:
 - checkout: self
@@ -23,8 +25,8 @@ steps:
   parameters:
     installAdditionalLinuxDependencies: ${{ parameters.installAdditionalLinuxDependencies }}
 
-- script: npm run test
-  displayName: 🧪 Run unit and integration tests
+- script: npm run ${{ parameters.npmCommand }}
+  displayName: 🧪 Run $(Agent.JobName)
   env:
     DISPLAY: :99.0
 
@@ -37,11 +39,11 @@ steps:
     searchFolder: '$(Build.SourcesDirectory)/out'
     publishRunAttachments: true
     mergeTestResults: true
-    testRunTitle: $(Agent.JobName) (Attempt $(System.JobAttempt))
+    testRunTitle: $(System.StageDisplayName) $(Agent.JobName) (Attempt $(System.JobAttempt))
 
 - task: PublishPipelineArtifact@1
   condition: failed()
   displayName: 'Upload integration test logs'
   inputs:
-    targetPath: '$(Build.SourcesDirectory)/.vscode-test/user-data/logs'
-    artifactName: 'VSCode Test Logs ($(Agent.JobName)-$(System.JobAttempt))'
+    targetPath: '$(Build.SourcesDirectory)/out/logs'
+    artifactName: 'Test Logs ($(System.StageDisplayName)-$(Agent.JobName)-$(System.JobAttempt))'

jest.config.ts

@@ -27,7 +27,8 @@ const config: Config = {
                 // Azure DevOps does not display test suites (it ignores them entirely).
                 // So we have to put all the info in the test case name so the UI shows anything relatively useful.
                 // See https://github.com/microsoft/azure-pipelines-tasks/issues/7659
-                titleTemplate: '{filename} / {suitename} / {title}',
+                classNameTemplate: '{suitename}',
+                titleTemplate: `${process.env.JEST_SUITE_NAME} {suitename} / {title}`,
             },
         ],
     ],

package-lock.json

@@ -67,7 +67,8 @@
     "compileDev": "tsc -p ./ && npx eslint ./ && webpack --mode development && npm run l10nDevGenerateLocalizationBundle",
     "watch": "tsc -watch -p ./",
     "test": "tsc -p ./ && gulp test",
-    "test:integration": "tsc -p ./ && gulp test:integration",
+    "test:withoutDevKit": "tsc -p ./ && gulp test:withoutDevKit",
+    "test:integration:devkit": "tsc -p ./ && gulp test:integration:devkit",
     "test:razor": "tsc -p ./ && npm run compile:razorTextMate && gulp test:razor",
     "test:razorintegration": "tsc -p ./ && gulp test:razorintegration",
     "test:artifacts": "tsc -p ./ && gulp test:artifacts",
@@ -126,7 +127,7 @@
     "@types/fs-extra": "5.0.4",
     "@types/gulp": "4.0.5",
     "@types/minimist": "1.2.1",
-    "@types/node": "16.11.38",
+    "@types/node": "20.14.8",
     "@types/semver": "7.3.13",
     "@types/tmp": "0.0.33",
     "@types/unzipper": "^0.9.1",

package.json

@@ -41,7 +41,7 @@ const TelemetryReportingDelay = 2 * 60 * 1000; // two minutes
 export default class TestManager extends AbstractProvider {
     private _runCounts?: { [testFrameworkName: string]: number };
     private _debugCounts?: { [testFrameworkName: string]: number };
-    private _telemetryIntervalId?: NodeJS.Timer = undefined;
+    private _telemetryIntervalId?: NodeJS.Timeout = undefined;
     private _eventStream: EventStream;
 
     constructor(

src/omnisharp/features/dotnetTest.ts

@@ -43,10 +43,7 @@ export async function InstallZip(
                     } catch (err) {
                         const error = err as NodeJS.ErrnoException; // Hack for TypeScript to type err correctly
                         return reject(
-                            new NestedError(
-                                'Error creating directory for zip directory entry:' + error.code ?? '',
-                                error
-                            )
+                            new NestedError('Error creating directory for zip directory entry:' + error.code, error)
                         );
                     }
                 } else {

src/packageManager/zipInstaller.ts

@@ -256,7 +256,7 @@ interface SynchronizationContext {
     readonly projectedDocument: IProjectedDocument;
     readonly logIdentifier: number;
     readonly toHostDocumentVersion: number;
-    readonly timeoutId: NodeJS.Timer;
+    readonly timeoutId: NodeJS.Timeout;
     readonly projectedDocumentSynchronized: () => void;
     readonly onProjectedDocumentSynchronized: Promise<void>;
     readonly projectedTextDocumentSynchronized: () => void;

src/razor/src/document/razorDocumentSynchronizer.ts

@@ -15,7 +15,7 @@ import { RazorLanguageServiceClient } from '../razorLanguageServiceClient';
 import { LanguageKind } from '../rpc/languageKind';
 
 export class HtmlTagCompletionProvider {
-    private timeout: NodeJS.Timer | undefined = void 0;
+    private timeout: NodeJS.Timeout | undefined = void 0;
     private enabled = false;
     private htmlLanguageService: HtmlLanguageService | undefined;