diff --git a/.vscode/launch.json b/.vscode/launch.json index f05d54c..2916fab 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,15 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "name": "Attach by Process ID", + "processId": "${command:PickProcess}", + "request": "attach", + "skipFiles": [ + "/**" + ], + "type": "node" + }, { "type": "node", "request": "launch", @@ -115,6 +124,32 @@ "artifacts", "false" ] - } + }, + { + "type": "node", + "request": "launch", + "name": "Launch LSP checkGithubRepos", + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder}/dist/checkGithubRepos.js", + "sourceMaps": true, + "outFiles": [ + "${workspaceFolder}/dist/**/*.js" + ], + "args": [ + "lsp", + "0", + "latest", + "./artifacts/repos.json", + "1", + "1", + "RepoResults", + "true", + "n/a", + "false" + ], + "preLaunchTask": "npm: build" + }, ] } \ No newline at end of file diff --git a/azure-pipelines-gitTests-lsp-ts.yml b/azure-pipelines-gitTests-lsp-ts.yml new file mode 100644 index 0000000..5b6f3c4 --- /dev/null +++ b/azure-pipelines-gitTests-lsp-ts.yml @@ -0,0 +1,23 @@ +# schedules: +# - cron: "0 19 * * Sun" # time is in UTC +# displayName: Sunday overnight run +# always: true +# branches: +# include: +# - main + +pr: none +trigger: none + +pool: + name: TypeScript-1ES-Large + demands: + - ImageOverride -equals azure-linux-3 + +extends: + template: azure-pipelines-gitTests-template.yml + parameters: + ENTRYPOINT: lsp + OLD_VERSION: '0' + NEW_VERSION: latest + LANGUAGE: TypeScript diff --git a/azure-pipelines-gitTests-template.yml b/azure-pipelines-gitTests-template.yml index f7c7124..31e5608 100644 --- a/azure-pipelines-gitTests-template.yml +++ b/azure-pipelines-gitTests-template.yml @@ -54,7 +54,7 @@ jobs: retryCountOnTaskFailure: 3 - task: UseNode@1 inputs: - version: '20.x' + version: '22.x' displayName: 'Install Node.js' - script: | npm ci @@ -77,7 +77,7 @@ jobs: artifact: RepoList - task: UseNode@1 inputs: - version: '20.x' + version: '22.x' displayName: 'Install Node.js' - script: | df -h @@ -118,7 +118,7 @@ jobs: - download: current - task: UseNode@1 inputs: - version: '20.x' + version: '22.x' displayName: 'Install Node.js' - script: | npm ci diff --git a/package-lock.json b/package-lock.json index 041e70f..9f10640 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,9 @@ "json5": "^2.2.3", "markdown-escape": "^2.0.0", "random-seed": "^0.3.0", - "simple-git": "^3.22.0" + "simple-git": "^3.22.0", + "vscode-jsonrpc": "^8.2.1", + "vscode-languageserver-protocol": "^3.17.5" }, "devDependencies": { "@types/jest": "^29.5.12", @@ -145,6 +147,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.0.tgz", "integrity": "sha512-fQfkg0Gjkza3nf0c7/w6Xf34BW4YvzNfACRLmmb7XRLa6XHdR+K9AlJlxneFfWYf6uhOzuzZVTjF/8KfndZANw==", "dev": true, + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.23.5", @@ -1167,6 +1170,7 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.1.0.tgz", "integrity": "sha512-BDa2VAMLSh3otEiaMJ/3Y36GU4qf6GI+VivQ/P41NC6GHcdxpKlqV0ikSZ5gdQsmS3ojXeRx5vasgNTinF0Q4g==", + "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.0.0", @@ -1702,6 +1706,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001587", "electron-to-chromium": "^1.4.668", @@ -2565,6 +2570,7 @@ "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, + "peer": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -4367,6 +4373,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.2.tgz", "integrity": "sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ==", "dev": true, + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -4430,6 +4437,36 @@ "node": ">=10.12.0" } }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vscode-languageserver-protocol": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "dependencies": { + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" + } + }, + "node_modules/vscode-languageserver-protocol/node_modules/vscode-jsonrpc": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==" + }, "node_modules/walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", @@ -4746,6 +4783,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.0.tgz", "integrity": "sha512-fQfkg0Gjkza3nf0c7/w6Xf34BW4YvzNfACRLmmb7XRLa6XHdR+K9AlJlxneFfWYf6uhOzuzZVTjF/8KfndZANw==", "dev": true, + "peer": true, "requires": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.23.5", @@ -5537,6 +5575,7 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.1.0.tgz", "integrity": "sha512-BDa2VAMLSh3otEiaMJ/3Y36GU4qf6GI+VivQ/P41NC6GHcdxpKlqV0ikSZ5gdQsmS3ojXeRx5vasgNTinF0Q4g==", + "peer": true, "requires": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.0.0", @@ -5968,6 +6007,7 @@ "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz", "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==", "dev": true, + "peer": true, "requires": { "caniuse-lite": "^1.0.30001587", "electron-to-chromium": "^1.4.668", @@ -6590,6 +6630,7 @@ "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, + "peer": true, "requires": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -7930,7 +7971,8 @@ "version": "5.4.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.2.tgz", "integrity": "sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ==", - "dev": true + "dev": true, + "peer": true }, "undici-types": { "version": "5.26.5", @@ -7964,6 +8006,32 @@ "convert-source-map": "^2.0.0" } }, + "vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==" + }, + "vscode-languageserver-protocol": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "requires": { + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" + }, + "dependencies": { + "vscode-jsonrpc": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==" + } + } + }, + "vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==" + }, "walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", diff --git a/package.json b/package.json index 368da8b..aac15c4 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,9 @@ "json5": "^2.2.3", "markdown-escape": "^2.0.0", "random-seed": "^0.3.0", - "simple-git": "^3.22.0" + "simple-git": "^3.22.0", + "vscode-jsonrpc": "^8.2.1", + "vscode-languageserver-protocol": "^3.17.5" }, "devDependencies": { "@types/jest": "^29.5.12", diff --git a/src/checkGithubRepos.ts b/src/checkGithubRepos.ts index 1c32205..acd8ad6 100644 --- a/src/checkGithubRepos.ts +++ b/src/checkGithubRepos.ts @@ -3,16 +3,16 @@ import { mainAsync, reportError, TsEntrypoint } from "./main"; const { argv } = process; -if (argv.length !== 11) { - console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); +if (argv.length < 11) { + console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} ?`); process.exit(-1); } -const [,, entrypoint, oldTsNpmVersion, newTsNpmVersion, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput, prngSeed] = argv; +const [,, entrypoint, oldTsNpmVersion, newTsNpmVersion, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput, prngSeed, tmpfs] = argv; mainAsync({ testType: "github", - tmpfs: true, + tmpfs: tmpfs && tmpfs.toLowerCase() === "false" ? false : true, entrypoint: entrypoint as TsEntrypoint, diagnosticOutput: diagnosticOutput.toLowerCase() === "true", buildWithNewWhenOldFails: false, diff --git a/src/main.ts b/src/main.ts index 27de6de..fdc3991 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,7 +9,7 @@ import fs = require("fs"); import path = require("path"); import mdEscape = require("markdown-escape"); import randomSeed = require("random-seed"); -import { getErrorMessageFromStack, getHash, getHashForStack } from "./utils/hashStackTrace"; +import { getErrorMessageFromStack, getHash, getHashForStack, getHashForGoStack } from "./utils/hashStackTrace"; import { createCopyingOverlayFS, createTempOverlayFS, OverlayBaseFS } from "./utils/overlayFS"; import { asMarkdownInlineCode } from "./utils/markdownUtils"; @@ -68,7 +68,7 @@ export interface UserParams extends Params { prNumber: number; } -export type TsEntrypoint = "tsc" | "tsserver"; +export type TsEntrypoint = "tsc" | "tsserver" | "lsp"; const processCwd = process.cwd(); const packageTimeout = 10 * 60 * 1000; @@ -106,6 +106,7 @@ interface Summary { replayScriptArtifactPath: string; replayScriptName: string; resultDirName: string; + entrypoint: TsEntrypoint; commit?: string; } @@ -338,14 +339,107 @@ async function getTsServerRepoResult( installCommands, }; - if (!oldServerFailed && !newServerFailed) { - return { status: "Detected no interesting changes" }; + return { status: "Detected interesting changes", tsServerResult: tsServerResult, replayScriptPath, rawErrorPath }; + } + catch (err) { + reportError(err, `Error running tsserver on ${repo.url ?? repo.name}`); + return { status: "Unknown failure" }; + } + finally { + console.log(`Done ${repo.url ?? repo.name}`); + logStepTime(diagnosticOutput, repo, "language service", lsStart); + } +} + +/** + * Tests the LSP server against a single TypeScript native version and reports all crashes found. + */ +export async function getLSPResult( + repo: git.Repo, + userTestsDir: string, + lspServerPath: string, + downloadDir: OverlayBaseFS, + replayScriptArtifactPath: string, + rawErrorArtifactPath: string, + diagnosticOutput: boolean, +): Promise { + + if (!await cloneRepo(repo, userTestsDir, downloadDir.path, diagnosticOutput)) { + return { status: "Git clone failed" }; + } + + const repoDir = path.join(downloadDir.path, repo.name); + const monorepoPackages = await getMonorepoPackages(repoDir); + + // Presumably, people occasionally browse repos without installing the packages first + const installCommands = (prng.random() > 0.2) && monorepoPackages + ? (await installPackagesAndGetCommands(repo, downloadDir.path, repoDir, monorepoPackages, /*cleanOnFailure*/ true, diagnosticOutput))! + : []; + + const replayScriptName = path.basename(replayScriptArtifactPath); + const replayScriptPath = path.join(downloadDir.path, replayScriptName); + + const rawErrorName = path.basename(rawErrorArtifactPath); + const rawErrorPath = path.join(downloadDir.path, rawErrorName); + + const lsStart = performance.now(); + try { + console.log(`Testing LSP server with ${lspServerPath}`); + const spawnResult = await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "utils", "exerciseLspServer.js"), repoDir, replayScriptPath, lspServerPath, diagnosticOutput.toString(), prng.string(10)], executionTimeout); + + if (!spawnResult) { + console.log(`LSP server timed out after ${executionTimeout} ms`); + return { status: "Timeout" }; + } + + if (diagnosticOutput) { + console.log("Raw spawn results:"); + dumpSpawnResult(spawnResult); + } + + switch (spawnResult.code) { + case 0: + case null: + if (spawnResult.signal !== null) { + console.log(`Exited with signal ${spawnResult.signal}`); + return { status: "Unknown failure" }; + } + + console.log("No crashes found"); + return { status: "Detected no interesting changes" }; + case exercise.EXIT_SERVER_CRASH: + case exercise.EXIT_SERVER_ERROR: + case exercise.EXIT_SERVER_EXIT_FAILED: + // These are the crashes we want to report + break; + case exercise.EXIT_BAD_ARGS: + case exercise.EXIT_UNHANDLED_EXCEPTION: + default: + console.log(`Exited with code ${spawnResult.code}`); + if (!diagnosticOutput) { + dumpSpawnResult(spawnResult); + } + return { status: "Unknown failure" }; } + // Server crashed - report the error + console.log(`Crash found in ${lspServerPath}:`); + console.log(insetLines(prettyPrintLspHarnessOutput(spawnResult.stdout, /*filter*/ false))); + await fs.promises.writeFile(rawErrorPath, prettyPrintLspHarnessOutput(spawnResult.stdout, /*filter*/ false)); + + const tsServerResult: TSServerResult = { + oldServerFailed: false, + oldSpawnResult: undefined, + newServerFailed: true, + newSpawnResult: spawnResult, + replayScriptPath, + installCommands, + }; + return { status: "Detected interesting changes", tsServerResult, replayScriptPath, rawErrorPath }; } catch (err) { - reportError(err, `Error running tsserver on ${repo.url ?? repo.name}`); + reportError(err, `Error running LSP server on ${repo.url ?? repo.name}`); return { status: "Unknown failure" }; } finally { @@ -358,18 +452,21 @@ function groupErrors(summaries: Summary[]) { const groupedOldErrors = new Map(); const groupedNewErrors = new Map(); let group: Map; - let error: ServerHarnessOutput | string; + let error: ServerHarnessOutput | LspHarnessOutput | string; for (const summary of summaries) { + const isLsp = summary.entrypoint === "lsp"; if (summary.tsServerResult.newServerFailed) { // Group new errors - error = parseServerHarnessOutput(summary.tsServerResult.newSpawnResult!.stdout); + error = isLsp + ? parseLspHarnessOutput(summary.tsServerResult.newSpawnResult!.stdout) + : parseServerHarnessOutput(summary.tsServerResult.newSpawnResult!.stdout); group = groupedNewErrors; } else if (summary.tsServerResult.oldServerFailed) { // Group old errors const { oldSpawnResult } = summary.tsServerResult; error = oldSpawnResult?.stdout - ? parseServerHarnessOutput(oldSpawnResult.stdout) + ? (isLsp ? parseLspHarnessOutput(oldSpawnResult.stdout) : parseServerHarnessOutput(oldSpawnResult.stdout)) : `Timed out after ${executionTimeout} ms`; group = groupedOldErrors; @@ -378,7 +475,11 @@ function groupErrors(summaries: Summary[]) { continue; } - const key = typeof error === "string" ? getHash([error]) : getHashForStack(error.message); + const key = typeof error === "string" + ? getHash([error]) + : summary.entrypoint === "lsp" + ? getHashForGoStack(error.message) + : getHashForStack(error.message); const value = group.get(key) ?? []; value.push(summary); group.set(key, value); @@ -393,14 +494,23 @@ function getErrorMessage(output: string): string { return typeof error === "string" ? error : getErrorMessageFromStack(error.message); } +function getErrorMessageForEntrypoint(output: string, entrypoint: TsEntrypoint): string { + return entrypoint === "lsp" ? getLspErrorMessage(output) : getErrorMessage(output); +} + +function prettyPrintForEntrypoint(output: string, filter: boolean, entrypoint: TsEntrypoint): string { + return entrypoint === "lsp" ? prettyPrintLspHarnessOutput(output, filter) : prettyPrintServerHarnessOutput(output, filter); +} + function createOldErrorSummary(summaries: Summary[]): string { const { oldSpawnResult } = summaries[0].tsServerResult; + const entrypoint = summaries[0].entrypoint; const oldServerError = oldSpawnResult?.stdout - ? prettyPrintServerHarnessOutput(oldSpawnResult.stdout, /*filter*/ true) + ? prettyPrintForEntrypoint(oldSpawnResult.stdout, /*filter*/ true, entrypoint) : `Timed out after ${executionTimeout} ms`; - const errorMessage = oldSpawnResult?.stdout ? getErrorMessage(oldSpawnResult.stdout) : oldServerError; + const errorMessage = oldSpawnResult?.stdout ? getErrorMessageForEntrypoint(oldSpawnResult.stdout, entrypoint) : oldServerError; let text = `
@@ -482,10 +592,13 @@ npx tsreplay ./${summary.repo.name} ./${summary.replayScriptName} { - let text = `

${getErrorMessage(summaries[0].tsServerResult.newSpawnResult.stdout)}

+ const entrypoint = summaries[0].entrypoint; + const stdout = summaries[0].tsServerResult.newSpawnResult.stdout; + + let text = `

${getErrorMessageForEntrypoint(stdout, entrypoint)}

\`\`\` -${prettyPrintServerHarnessOutput(summaries[0].tsServerResult.newSpawnResult.stdout, /*filter*/ true)} +${prettyPrintForEntrypoint(stdout, /*filter*/ true, entrypoint)} \`\`\`

Affected repos

`; @@ -803,7 +916,7 @@ export async function mainAsync(params: GitParams | UserParams): Promise { const { oldTsEntrypointPath, oldTsResolvedVersion, newTsEntrypointPath, newTsResolvedVersion } = await downloadTsAsync(processCwd, params); // Get the name of the typescript folder. - const oldTscDirPath = path.resolve(oldTsEntrypointPath, "../../"); + const oldTscDirPath = oldTsEntrypointPath && path.resolve(oldTsEntrypointPath, "../../"); const newTscDirPath = path.resolve(newTsEntrypointPath, "../../"); console.log("Old version = " + oldTsResolvedVersion); @@ -838,9 +951,21 @@ export async function mainAsync(params: GitParams | UserParams): Promise { const rawErrorArtifactPath = path.join(params.resultDirName, rawErrorFileName); const replayScriptArtifactPath = path.join(params.resultDirName, replayScriptFileName); - const { status, summary, tsServerResult, replayScriptPath, rawErrorPath } = params.entrypoint === "tsc" - ? await getTscRepoResult(repo, userTestsDir, oldTsEntrypointPath, newTsEntrypointPath, params.buildWithNewWhenOldFails, downloadDir, diagnosticOutput) - : await getTsServerRepoResult(repo, userTestsDir, oldTsEntrypointPath, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput, isPr); + let repoResult: RepoResult; + switch (params.entrypoint) { + case "tsc": + repoResult = await getTscRepoResult(repo, userTestsDir, oldTsEntrypointPath!, newTsEntrypointPath, params.buildWithNewWhenOldFails, downloadDir, diagnosticOutput); + break; + case "tsserver": + repoResult = await getTsServerRepoResult(repo, userTestsDir, oldTsEntrypointPath!, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput, isPr); + break; + case "lsp": + repoResult = await getLSPResult(repo, userTestsDir, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput); + break; + default: + throw new Error(`Unknown entrypoint: ${params.entrypoint}`); + } + const { status, summary, tsServerResult: tsServerResult, replayScriptPath, rawErrorPath } = repoResult; console.log(`Repo ${repo.url ?? repo.name} had status "${status}"`); statusCounts[status] = (statusCounts[status] ?? 0) + 1; @@ -865,12 +990,13 @@ export async function mainAsync(params: GitParams | UserParams): Promise { summaries.push({ tsServerResult, repo, - oldTsEntrypointPath, + oldTsEntrypointPath: oldTsEntrypointPath || "", rawErrorArtifactPath, replayScript: fs.readFileSync(replayScriptPath, { encoding: "utf-8" }).split(/\r?\n/).slice(-5).join("\n"), replayScriptArtifactPath, replayScriptName: path.basename(replayScriptArtifactPath), resultDirName: params.resultDirName, + entrypoint: params.entrypoint, commit }); } @@ -906,7 +1032,9 @@ export async function mainAsync(params: GitParams | UserParams): Promise { } } - await execAsync(processCwd, "rm -rf " + oldTscDirPath); + if (oldTscDirPath) { + await execAsync(processCwd, "rm -rf " + oldTscDirPath); + } await execAsync(processCwd, "rm -rf " + newTscDirPath); console.log("Statuses"); @@ -916,7 +1044,7 @@ export async function mainAsync(params: GitParams | UserParams): Promise { const metadata: Metadata = { newTsResolvedVersion: newTsResolvedVersion, - oldTsResolvedVersion: oldTsResolvedVersion, + oldTsResolvedVersion: oldTsResolvedVersion || "", statusCounts, }; await fs.promises.writeFile(path.join(resultDirPath, metadataFileName), JSON.stringify(metadata), { encoding: "utf-8" }); @@ -1056,6 +1184,58 @@ function filterToTsserverLines(stackLines: string): string { return tsserverLines.trimEnd(); } +// LSP harness output helpers + +export interface LspHarnessOutput { + method: string; + message: string; +} + +function parseLspHarnessOutput(output: string): LspHarnessOutput | string { + try { + const parsed = JSON.parse(output); + if (parsed.method !== undefined && parsed.message !== undefined) { + return parsed as LspHarnessOutput; + } + return output; + } + catch { + return output; + } +} + +function prettyPrintLspHarnessOutput(error: string, filter: boolean): string { + const errorObj = parseLspHarnessOutput(error); + if (typeof errorObj === "string") { + return errorObj; + } + + if (errorObj.message) { + return `${errorObj.method}\n${filter ? filterToGoLines(errorObj.message) : errorObj.message}`; + } + + return JSON.stringify(errorObj, undefined, 2); +} + +function getLspErrorMessage(output: string): string { + const error = parseLspHarnessOutput(output); + if (typeof error === "string") return error; + + // The first line of the message is typically "panic handling request : " + const firstLine = error.message.split(/\r?\n/)[0]; + return firstLine; +} + +function filterToGoLines(stackLines: string): string { + const goRegex = /^.*typescript-go.*$/mg; + let goLines = ""; + let match; + while (match = goRegex.exec(stackLines)) { + goLines += match[0] + "\n"; + } + return goLines.trimEnd() || stackLines; +} + function insetLines(text: string): string { return text.trimEnd().replace(/(^|\n)/g, "$1> "); } @@ -1073,9 +1253,13 @@ function makeMarkdownLink(url: string) { : `[${mdEscape(match[1])}](${url})`; } -async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Promise<{ oldTsEntrypointPath: string, oldTsResolvedVersion: string, newTsEntrypointPath: string, newTsResolvedVersion: string }> { +async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Promise<{ oldTsEntrypointPath: string | undefined, oldTsResolvedVersion: string | undefined, newTsEntrypointPath: string, newTsResolvedVersion: string }> { const entrypoint = params.entrypoint; if (params.testType === "user") { + // TODO user tests for lsp + if (params.entrypoint === "lsp") { + throw new Error("Not implemented"); + } const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = await downloadTsRepoAsync(cwd, params.oldTsRepoUrl, params.oldHeadRef, entrypoint); // We need to handle the ref/pull/*/merge differently as it is not a branch and cannot be pulled during clone. const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = await downloadTsPrAsync(cwd, params.oldTsRepoUrl, params.prNumber, entrypoint); @@ -1088,8 +1272,12 @@ async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Pro }; } else if (params.testType === "github") { - const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = await downloadTsNpmAsync(cwd, params.oldTsNpmVersion, entrypoint); - const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = await downloadTsNpmAsync(cwd, params.newTsNpmVersion, entrypoint); + const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = params.entrypoint === "lsp" ? + { tsEntrypointPath: undefined, resolvedVersion: undefined } : + await downloadTsNpmAsync(cwd, params.oldTsNpmVersion, entrypoint); + const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = params.entrypoint === "lsp" ? + await downloadTsNativePreviewNpmAsync(cwd, params.newTsNpmVersion) : + await downloadTsNpmAsync(cwd, params.newTsNpmVersion, entrypoint); return { oldTsEntrypointPath, @@ -1165,3 +1353,27 @@ async function downloadTsNpmAsync(cwd: string, version: string, entrypoint: TsEn return { tsEntrypointPath, resolvedVersion }; } + +async function downloadTsNativePreviewNpmAsync(cwd: string, version: string): Promise<{ tsEntrypointPath: string, resolvedVersion: string }> { + const packageName = `native-preview-${process.platform}-${process.arch}` + const tarName = (await execAsync(cwd, `npm pack @typescript/${packageName}@${version} --quiet`)).trim(); + + const tarMatch = /^(typescript-native-preview-(.+))\..+$/.exec(tarName); + if (!tarMatch) { + throw new Error("Unexpected tarball name format: " + tarName); + } + + const resolvedVersion = tarMatch[2]; + const dirName = tarMatch[1]; + const dirPath = path.join(processCwd, dirName); + + await execAsync(cwd, `tar xf ${tarName} && rm ${tarName}`); + await fs.promises.rename(path.join(processCwd, "package"), dirPath); + + const tsEntrypointPath = path.join(dirPath, "lib", "tsgo"); + if (!await pu.exists(tsEntrypointPath)) { + throw new Error("Cannot find file " + tsEntrypointPath); + } + + return { tsEntrypointPath, resolvedVersion }; +} diff --git a/src/postGithubComments.ts b/src/postGithubComments.ts index 42939fa..c42d481 100644 --- a/src/postGithubComments.ts +++ b/src/postGithubComments.ts @@ -1,6 +1,6 @@ import fs = require("fs"); import path = require("path"); -import { artifactFolderUrlPlaceholder, getArtifactsApiUrlPlaceholder, Metadata, metadataFileName, RepoStatus, resultFileNameSuffix } from "./main"; +import { artifactFolderUrlPlaceholder, getArtifactsApiUrlPlaceholder, Metadata, metadataFileName, RepoStatus, resultFileNameSuffix, TsEntrypoint } from "./main"; import git = require("./utils/gitUtils"); import pu = require("./utils/packageUtils"); import { asMarkdownInlineCode } from "./utils/markdownUtils"; @@ -82,7 +82,7 @@ let header = `@${userToTag} Here are the results of running the ${suiteDescripti ${summary.join("\n")}`; if (!outputs.length) { - git.createComment(+prNumber, +commentNumber, distinctId, postResult, [header], somethingChanged); + git.createComment(entrypoint as TsEntrypoint, +prNumber, +commentNumber, distinctId, postResult, [header], somethingChanged); } else { const oldErrorHeader = `

:warning: Old server errors :warning:

`; @@ -137,5 +137,5 @@ else { console.log(`Chunk of size ${chunk.length}`); } - git.createComment(+prNumber, +commentNumber, distinctId, postResult, bodyChunks, somethingChanged); + git.createComment(entrypoint as TsEntrypoint, +prNumber, +commentNumber, distinctId, postResult, bodyChunks, somethingChanged); } diff --git a/src/postGithubIssue.ts b/src/postGithubIssue.ts index 9e25f77..4021a16 100644 --- a/src/postGithubIssue.ts +++ b/src/postGithubIssue.ts @@ -45,11 +45,14 @@ for (const path of metadataFilePaths) { } -const title = `${entrypoint === "tsserver" ? `[ServerErrors][${language}]` : `[NewErrors]`} ${newTscResolvedVersion} vs ${oldTscResolvedVersion}`; +const title = `${entrypoint === "tsserver" || entrypoint === "lsp" ? `[ServerErrors][${language}]` : `[NewErrors]`} ${newTscResolvedVersion} vs ${oldTscResolvedVersion}`; const description = entrypoint === "tsserver" ? `The following errors were reported by ${newTscResolvedVersion} vs ${oldTscResolvedVersion}` - : `The following errors were reported by ${newTscResolvedVersion}, but not by ${oldTscResolvedVersion}`; + : entrypoint == "lsp" + ? `The following errors were reported by ${newTscResolvedVersion}` + : `The following errors were reported by ${newTscResolvedVersion}, but not by ${oldTscResolvedVersion}`; +// TODO: modify for tsgo let header = `${description} [Pipeline that generated this bug](https://typescript.visualstudio.com/TypeScript/_build?definitionId=48) [Logs for the pipeline run](${logUri}) @@ -102,4 +105,4 @@ if (entrypoint !== "tsserver") { const bodyChunks = [header, ...outputs]; -git.createIssue(postResult, title, bodyChunks, /*sawNewErrors*/ !!outputs.length); \ No newline at end of file +git.createIssue(entrypoint, postResult, title, bodyChunks, /*sawNewErrors*/ !!outputs.length); \ No newline at end of file diff --git a/src/utils/exerciseLspServer.ts b/src/utils/exerciseLspServer.ts new file mode 100644 index 0000000..fedf167 --- /dev/null +++ b/src/utils/exerciseLspServer.ts @@ -0,0 +1,510 @@ +import * as lsp from "./lspHarness"; +import fs from "fs"; +import process from "process"; +import path from "path"; +import * as glob from "glob"; +import { performance } from "perf_hooks"; +import randomSeed from "random-seed"; +import * as protocol from "vscode-languageserver-protocol"; +import { EXIT_BAD_ARGS, EXIT_SERVER_COMMUNICATION_ERROR, EXIT_SERVER_CRASH, EXIT_UNHANDLED_EXCEPTION, EXIT_SERVER_ERROR } from "./exerciseServerConstants"; +import { pathToFileURL } from "url"; + +const testDirPlaceholder = "@PROJECT_ROOT@"; + +const argv = process.argv; + +if (argv.length !== 7) { + console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); + process.exit(EXIT_BAD_ARGS); +} + +// CONVENTION: stderr is for output to the log; stdout is for output to the user + +const [, , testDir, replayScriptPath, lspServerPath, diag, seed] = argv; +const diagnosticOutput = diag.toLocaleLowerCase() === "true"; +const prng = randomSeed.create(seed); + +exerciseLspServer(testDir, replayScriptPath, lspServerPath).catch(e => { + console.error(e); + process.exit(EXIT_UNHANDLED_EXCEPTION); +}); + +export async function exerciseLspServer(testDir: string, replayScriptPath: string, lspServerPath: string): Promise { + const requestTimes: Record = {}; + const requestCounts: Record = {}; + const start = performance.now(); + + const oldCwd = process.cwd(); + const replayScriptHandle = await fs.promises.open(replayScriptPath, "w"); + try { + await exerciseLspServerWorker(testDir, lspServerPath, replayScriptHandle, requestTimes, requestCounts); + } + finally { + await replayScriptHandle.close(); + + process.chdir(oldCwd); + + const end = performance.now(); + if (diagnosticOutput) { + console.error(`Elapsed time ${Math.round(end - start)} ms`); + for (const method in requestTimes) { + console.error(`${method}:\t${Math.round(requestTimes[method])} ms (${requestCounts[method]} calls)`); + } + } + } +} + +function filePathToUri(filePath: string): string { + return pathToFileURL(filePath).toString(); +} + +function getLanguageId(filePath: string): string { + const ext = path.extname(filePath).toLowerCase(); + switch (ext) { + case ".ts": return "typescript"; + case ".mts": return "typescript"; + case ".cts": return "typescript"; + + case ".js": return "javascript"; + case ".mjs": return "javascript"; + case ".cjs": return "javascript"; + + case ".tsx": return "typescriptreact"; + case ".jsx": return "javascriptreact"; + + default: return "typescript"; + } +} + +async function exerciseLspServerWorker(testDir: string, lspServerPath: string, replayScriptHandle: fs.promises.FileHandle, requestTimes: Record, requestCounts: Record): Promise { + const files = await glob.glob("**/*.@(ts|tsx|mts|cts|js|jsx|mjs|cjs)", { cwd: testDir, absolute: true, ignore: ["**/node_modules/**", "**/*.min.js"], nodir: true, follow: false }); + + const serverArgs: string[] = ["--lsp", "--stdio"]; + + replayScriptHandle.write(JSON.stringify({ + rootDirPlaceholder: testDirPlaceholder, + serverArgs, + }) + "\n"); + + // TODO: would be nice if we could make this work with node_modules/.bin/*.CMD files on Windows + if (path.extname(lspServerPath).toLowerCase().endsWith("js")) { + // Use Node.js or Bun or whatever we ran under. + serverArgs.unshift(lspServerPath); + lspServerPath = process.execPath; + } + + const server = lsp.startServer(lspServerPath, { + args: serverArgs, + }, { traceOutput: diagnosticOutput }); + + server.handleAnyRequest(async (...args) => { + console.error("Server sent request:", ...args); + }); + + // Capture the last error-level log message from the server (e.g. Go panic stack traces) + let lastErrorLogMessage = ""; + + server.handleAnyNotification(async (...args: any[]) => { + console.error("Server sent notification:", ...args); + const [method, params] = args; + if (method === "window/logMessage" && params?.type === 1) { + lastErrorLogMessage = params.message; + } + }); + + let exitExpected = false; + server.onError(async ([error, message, count]) => { + console.error(`Server connection error: ${error} ${message} ${count}`); + exitExpected = true; + await server.kill(); + process.exit(EXIT_SERVER_COMMUNICATION_ERROR); + }); + + server.onClose((e) => { + if (!exitExpected) { + const errorMessage = lastErrorLogMessage || `Server connection closed prematurely: ${e}`; + console.log(JSON.stringify({ method: "unknown", message: errorMessage })); + console.error("Server connection closed prematurely:", e); + process.exit(EXIT_SERVER_CRASH); + } + }); + + let documentVersion = 0; + + const testDirUrl = filePathToUri(testDir); + + // Initialize the server + const initializeParams: protocol.InitializeParams = { + processId: process.pid, + capabilities: { + textDocument: { + completion: { + completionItem: { + snippetSupport: true, + insertReplaceSupport: true, + resolveSupport: { + properties: ["documentation", "detail", "additionalTextEdits"], + }, + commitCharactersSupport: true, + deprecatedSupport: true, + preselectSupport: true, + labelDetailsSupport: true, + documentationFormat: ["markdown", "plaintext"], + insertTextModeSupport: { + valueSet: [ + protocol.InsertTextMode.asIs, + protocol.InsertTextMode.adjustIndentation, + ], + }, + // TODO: ... + }, + contextSupport: true, + }, + definition: { + linkSupport: true, + }, + references: {}, + documentSymbol: { + hierarchicalDocumentSymbolSupport: true, + labelSupport: true, + // TODO: ... + }, + foldingRange: { + foldingRange: { collapsedText: true }, + // TODO: ... + }, + codeAction: { + disabledSupport: true, + dataSupport: true, + // TODO: ... + codeActionLiteralSupport: { + codeActionKind: { + valueSet: [ + protocol.CodeActionKind.QuickFix, + protocol.CodeActionKind.Refactor, + protocol.CodeActionKind.RefactorExtract, + protocol.CodeActionKind.RefactorInline, + protocol.CodeActionKind.RefactorRewrite, + protocol.CodeActionKind.Source, + protocol.CodeActionKind.SourceOrganizeImports, + ], + }, + }, + }, + hover: { contentFormat: ["markdown", "plaintext"] }, + diagnostic: { relatedDocumentSupport: true }, + declaration: { linkSupport: true }, + implementation: { linkSupport: true }, + typeDefinition: { linkSupport: true }, + rename: { + // TODO: ... + } + // TODO: ... + + }, + workspace: { + symbol: { + // TODO: ... + }, + // TODO + // codeLens: { refreshSupport: true }, + // inlayHint: { refreshSupport: true, }, + // foldingRange: { refreshSupport: true }, + // semanticTokens: { refreshSupport: true }, + // diagnostics: { refreshSupport: true }, + // configuration: true, + // TODO: ... + }, + }, + rootUri: testDirUrl, + // workspaceFolders: [ + // { + // uri: testDirUrl, + // name: path.basename(testDir), + // }, + // ], + }; + + try { + await request("initialize", initializeParams); + await notify("initialized", {}); + + const openFileUris: string[] = []; + + // NB: greater than 1 behaves the same as 1 + // const skipFileProb = 1000 / files.length; + const skipFileProb = 1; + for (const openFileAbsolutePath of files) { + if (prng.random() > skipFileProb) continue; + + const openFileUri = filePathToUri(openFileAbsolutePath); + + if (openFileUris.length === 5) { + const closedFileUri = openFileUris.shift()!; + // Close the document + await notify("textDocument/didClose", { + textDocument: { + uri: closedFileUri, + }, + }); + } + + openFileUris.push(openFileUri); + + const openFileContents = await fs.promises.readFile(openFileAbsolutePath, { encoding: "utf-8" }); + const languageId = getLanguageId(openFileAbsolutePath); + documentVersion++; + + // Open the document + await notify("textDocument/didOpen", { + textDocument: { + uri: openFileUri, + languageId, + version: documentVersion, + text: openFileContents, + }, + }); + + const triggerChars = [".", '"', "'", "`", "/", "@", "<", "#", " "]; + const signatureHelpTriggerChars = ["(", ",", "<"]; + + let line = 0; // LSP uses 0-based lines + let character = 0; // LSP uses 0-based characters + + let prev = ""; + + // Organize imports (source.organizeImports code action) + await request("textDocument/codeAction", { + textDocument: { uri: openFileUri }, + range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } }, + context: { + diagnostics: [], + only: [protocol.CodeActionKind.SourceOrganizeImports], + }, + }, 0.5); + + if (openFileContents.length < 1e6) { + // Folding ranges (equivalent to getOutliningSpans) + await request("textDocument/foldingRange", { + textDocument: { uri: openFileUri }, + }); + + // Document symbols (equivalent to navtree/navbar) + await request("textDocument/documentSymbol", { + textDocument: { uri: openFileUri }, + }); + } + + // Workspace symbol search (equivalent to navto) + const workspaceSymbolResponse = await request("workspace/symbol", { + query: "a", + }, 0.5); + + if (workspaceSymbolResponse && Array.isArray(workspaceSymbolResponse) && workspaceSymbolResponse.length > 0) { + const symbolEntry = workspaceSymbolResponse.find((x: protocol.SymbolInformation | protocol.WorkspaceSymbol) => x.name.length > 4); + if (symbolEntry) { + await request("workspace/symbol", { + query: symbolEntry.name.slice(0, 3), + }); + } + } + + // Diagnostics (equivalent to geterr) + const diagnosticsPromise = request("textDocument/diagnostic", { + textDocument: { uri: openFileUri }, + }); + + const codeLensesPromise = request("textDocument/codeLens", { + textDocument: { uri: openFileUri }, + }); + + const inlayHintsPromise = request("textDocument/inlayHint", { + textDocument: { uri: openFileUri }, + range: { start: { line: 0, character: 0 }, end: { line: 0, character: openFileContents.length } }, + }); + + await Promise.all([diagnosticsPromise, codeLensesPromise, inlayHintsPromise]); + + for (let i = 0; i < openFileContents.length; i++) { + const curr = openFileContents[i]; + const next = openFileContents[i + 1]; + + // Increase probabilities around things that look like jsdoc, where we've had problems in the past + const isAt = curr === "@"; + + // Note that this only catches Latin letters - we'll test within tokens of non-Latin characters + if (!(/\w/.test(prev) && /\w/.test(curr)) && !(/[ \t]/.test(prev) && /[ \t]/.test(curr))) { + // Definition (equivalent to definitionAndBoundSpan) + await request("textDocument/definition", { + textDocument: { uri: openFileUri }, + position: { line, character }, + }, isAt ? 0.5 : 0.001); + + // References + await request("textDocument/references", { + textDocument: { uri: openFileUri }, + position: { line, character }, + context: { includeDeclaration: true }, + }, isAt ? 0.5 : 0.00005); + + const completionsProb = 0.1; + + // Completions (equivalent to completionInfo) + const completionResponse = await request("textDocument/completion", { + textDocument: { uri: openFileUri }, + position: { line, character }, + context: { + triggerKind: protocol.CompletionTriggerKind.Invoked, + }, + }, completionsProb); + + // Completion resolve (equivalent to completionEntryDetails) + if (completionResponse) { + const items = "items" in completionResponse ? completionResponse.items : completionResponse; + if (Array.isArray(items) && items.length > 0) { + await request("completionItem/resolve", items.find(item => item.preselect) ?? items[0]); + } + } + + // Triggered completions + const triggerCharIndex = triggerChars.indexOf(curr); + if (triggerCharIndex >= 0 && /\w/.test(prev)) { + await request("textDocument/completion", { + textDocument: { uri: openFileUri }, + position: { line, character }, + context: { + triggerKind: protocol.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: triggerChars[triggerCharIndex], + }, + }, completionsProb /*previously 0.005*/); + } + } + + let currisSignatureHelpTrigger = false; + if ((currisSignatureHelpTrigger = signatureHelpTriggerChars.includes(curr)) || signatureHelpTriggerChars.includes(next)) { + // Signature help (equivalent to signatureHelp) + await request("textDocument/signatureHelp", { + textDocument: { uri: openFileUri }, + position: { line, character }, + context: { + triggerCharacter: currisSignatureHelpTrigger ? curr : undefined, + triggerKind: currisSignatureHelpTrigger ? protocol.SignatureHelpTriggerKind.TriggerCharacter : protocol.SignatureHelpTriggerKind.Invoked, + isRetrigger: signatureHelpTriggerChars.includes(prev), + } + }, 0); + } + + if (curr === "\r" || curr === "\n") { + if (line === 0) { + // Apply a text change (equivalent to updateOpen with changedFiles) + documentVersion++; + await notify("textDocument/didChange", { + textDocument: { + uri: openFileUri, + version: documentVersion, + }, + contentChanges: [ + { + range: { + start: { line, character }, + end: { line, character }, + }, + text: " //comment", + }, + ], + }); + } + + line++; + character = 0; + if (curr === "\r" && next === "\n") { + i++; + } + } + else { + character++; + } + + prev = curr; + } + } + + console.error("\nShutting down server"); + exitExpected = true; + // Send shutdown request and exit notification + void request(protocol.ShutdownRequest.method, undefined); + void notify("exit", undefined); + } catch (e) { + console.error("Killing server after unhandled exception"); + console.error(e); + + exitExpected = true; + await server.kill(); + process.exit(EXIT_UNHANDLED_EXCEPTION); + } + + await server.kill(); + + async function request( + method: K, + params: lsp.RequestToParams[K], + prob = 1, + ): Promise { + if (prng.random() > prob) return undefined as any; + + const replayEntry = { kind: "request", method, params }; + const replayStr = JSON.stringify(replayEntry).replaceAll(testDirUrl, testDirPlaceholder); + await replayScriptHandle.write(replayStr + "\n"); + + const start = performance.now(); + try { + const response = await server.sendRequest(method, params); + const end = performance.now(); + requestTimes[method] = (requestTimes[method] ?? 0) + (end - start); + requestCounts[method] = (requestCounts[method] ?? 0) + 1; + return response; + } catch (e: any) { + const end = performance.now(); + requestTimes[method] = (requestTimes[method] ?? 0) + (end - start); + requestCounts[method] = (requestCounts[method] ?? 0) + 1; + + const errorMessage = lastErrorLogMessage || e.message || "Unknown error"; + if (diagnosticOutput) { + console.error(`Request failed:\n${JSON.stringify(replayEntry, undefined, 2)}\n${e}`); + } + else { + console.error(errorMessage); + } + console.log(JSON.stringify({ method, message: errorMessage })); + + void server.kill(); + process.exit(EXIT_SERVER_ERROR); + } + } + + async function notify( + method: K, + params: lsp.NotificationToParams[K], + ): Promise { + const replayEntry = { kind: "notification", method, params }; + const replayStr = JSON.stringify(replayEntry).replaceAll(testDirUrl, testDirPlaceholder); + await replayScriptHandle.write(replayStr + "\n"); + + try { + await server.sendNotification(method, params); + } + catch (e: any) { + const errorMessage = lastErrorLogMessage || e.message || "Unknown error"; + if (diagnosticOutput) { + console.error(`Notification failed:\n${JSON.stringify(replayEntry, undefined, 2)}\n${e}`); + } + else { + console.error(errorMessage); + } + console.log(JSON.stringify({ method, message: errorMessage })); + + await server.kill(); + process.exit(EXIT_SERVER_ERROR); + } + } +} \ No newline at end of file diff --git a/src/utils/gitUtils.ts b/src/utils/gitUtils.ts index 02ad4b1..f307e8c 100644 --- a/src/utils/gitUtils.ts +++ b/src/utils/gitUtils.ts @@ -6,6 +6,7 @@ import path = require("path"); // The bundled types don't work with CJS imports import { simpleGit as git } from "simple-git"; +import { TsEntrypoint } from "../main"; export interface Repo { name: string; @@ -15,10 +16,21 @@ export interface Repo { branch?: string; } -const repoProperties = { - owner: "microsoft", - repo: "typescript", -}; +function getRepoProperties(entrypoint: TsEntrypoint) { + switch (entrypoint) { + case "tsserver": + case "tsc": + return { + owner: "microsoft", + repo: "typescript", + }; + case "lsp": + return { + owner: "microsoft", + repo: "typescript-go", + }; + } +} export async function getPopularRepos(language = "TypeScript", count = 100, repoStartIndex = 0, skipRepos?: string[], cachePath?: string): Promise { const cacheEncoding = { encoding: "utf-8" } as const; @@ -105,7 +117,8 @@ type Result = { export type GitResult = Result & { kind: 'git', title: string } export type UserResult = Result & { kind: 'user', issue_number: number } -export async function createIssue(postResult: boolean, title: string, bodyChunks: readonly string[], sawNewErrors: boolean): Promise { +export async function createIssue(entrypoint: TsEntrypoint, postResult: boolean, title: string, bodyChunks: readonly string[], sawNewErrors: boolean): Promise { + const repoProperties = getRepoProperties(entrypoint); const issue = { ...repoProperties, title, @@ -158,7 +171,8 @@ export async function createIssue(postResult: boolean, title: string, bodyChunks } } -export async function createComment(prNumber: number, statusComment: number, distinctId: string, postResult: boolean, bodyChunks: readonly string[], somethingChanged: boolean): Promise { +export async function createComment(entrypoint: TsEntrypoint, prNumber: number, statusComment: number, distinctId: string, postResult: boolean, bodyChunks: readonly string[], somethingChanged: boolean): Promise { + const repoProperties = getRepoProperties(entrypoint); const newComments = bodyChunks.map(body => ({ ...repoProperties, issue_number: prNumber, diff --git a/src/utils/hashStackTrace.ts b/src/utils/hashStackTrace.ts index c2deb6a..7ca6363 100644 --- a/src/utils/hashStackTrace.ts +++ b/src/utils/hashStackTrace.ts @@ -11,6 +11,39 @@ export function getHashForStack(stack: string): string { return getHash(stackLines); } +/** + * Produces a stable hash for Go stack traces (e.g. from the LSP server). + * Strips volatile parts that change between runs: + * - The first line (panic message with specific runtime values) + * - Memory addresses like 0xc004bfed20 + * - Goroutine IDs like "goroutine 554" + * Keeps function names and source file locations for meaningful grouping. + */ +export function getHashForGoStack(stack: string): string { + const stackLines = stack.split(/\r?\n/); + // Skip the first line (panic message with variable runtime values) + const normalized = stackLines.slice(1).map(line => { + line = line.trim(); + let ignoreIdx; + if ((ignoreIdx = line.indexOf(" +0x")) !== -1) { + // Ignore trailing offsets e.g. " +0x58" in "github.com/microsoft/typescript-go/internal/lsp/server.go:872 +0x58" + line = line.slice(0, ignoreIdx); + } else if ((ignoreIdx = line.lastIndexOf(" in goroutine ")) !== -1) { + // e.g. "created by github.com/microsoft/typescript-go/internal/lsp.(*Server).dispatchLoop in goroutine 10" -> + // "created by github.com/microsoft/typescript-go/internal/lsp.(*Server).dispatchLoop" + line = line.slice(0, ignoreIdx); + } + // Strip goroutine IDs: "goroutine 554 [running]:" -> "goroutine [running]:" + line = line.replace(/goroutine \d+/, "goroutine "); + // Strip function arguments + line = line.replace(/^(.+)\(.+$/g, "$1()"); + return line; + }); + console.log("Normalized Go stack trace:"); + console.log(normalized.join("\n")); + return getHash(normalized); +} + export function getErrorMessageFromStack(stack: string): string { const stackLines = stack.split(/\r?\n/, 2); diff --git a/src/utils/lspHarness.ts b/src/utils/lspHarness.ts new file mode 100644 index 0000000..9dca8db --- /dev/null +++ b/src/utils/lspHarness.ts @@ -0,0 +1,186 @@ +import * as cp from "child_process"; +import * as rpc from "vscode-jsonrpc/node"; +import * as protocol from "vscode-languageserver-protocol"; + +export interface ServerOptions { + args?: string[]; + execArgv?: string[]; + env?: NodeJS.ProcessEnv; +} + +export interface LanguageServer { + sendRequest: (method: K, params: RequestToParams[K]) => Promise; + sendRequestUntyped: (method: string, params: object) => Promise; + sendNotification: (method: K, params: NotificationToParams[K]) => Promise; + + handleRequest: (method: K, handler: (params: RequestToParams[K]) => Promise) => void; + handleAnyRequest: (handler: (...args: any[]) => Promise) => void; + handleNotification: (method: K, handler: (params: NotificationToParams[K]) => void) => void; + handleAnyNotification: (handler: (...args: any[]) => Promise) => void; + + kill: () => Promise; + + onError: rpc.Event<[Error, rpc.Message | undefined, number | undefined]>; + onClose: rpc.Event; +} + +export function startServer(serverPath: string, options: ServerOptions = {}, otherOptions?: { traceOutput?: boolean; }): LanguageServer { + const serverProc = cp.spawn(serverPath, options.args ?? [], { + env: options.env, + // execArgv: options.execArgv, // options.execArgv ?? process.execArgv?.map(arg => bumpDebugPort(arg)), + stdio: [ + "pipe", // stdin + "pipe", // stdout + otherOptions?.traceOutput ? "inherit" : "ignore" // stderr + ], + }); + + const connection = rpc.createMessageConnection( + new rpc.StreamMessageReader(serverProc.stdout), + new rpc.StreamMessageWriter(serverProc.stdin!) + ); + + connection.listen(); + + return { + sendRequest, + sendRequestUntyped, + sendNotification, + handleRequest, + handleAnyRequest, + handleNotification, + handleAnyNotification, + kill, + onError: connection.onError, + onClose: connection.onClose, + }; + + function sendRequest(method: K, params: RequestToParams[K]): Promise { + return connection.sendRequest(method, params); + } + + function sendRequestUntyped(method: string, params: object): Promise { + return connection.sendRequest(method, params); + } + + function sendNotification(method: K, params: NotificationToParams[K]): Promise { + return connection.sendNotification(method, params); + } + + function handleRequest(method: K, handler: (params: RequestToParams[K]) => Promise): rpc.Disposable { + return connection.onRequest(method, handler); + } + + function handleAnyRequest(handler: (...args: any[]) => Promise): rpc.Disposable { + return connection.onRequest(handler); + } + + function handleNotification(method: K, handler: (params: NotificationToParams[K]) => void): rpc.Disposable { + return connection.onNotification(method, handler); + } + + function handleAnyNotification(handler: (...args: any[]) => Promise): rpc.Disposable { + return connection.onNotification(handler); + } + + function kill(): Promise { + return new Promise((resolve, reject) => { + serverProc.once("close", () => { + resolve(); + }); + // If the server has already exited, there won't be a close event + if (serverProc.exitCode !== null || serverProc.signalCode !== null) { + resolve(); + } + if (!serverProc.kill("SIGKILL")) { + reject(new Error("Failed to send kill signal to server")); + } + }); + } +} + +export interface RequestToParams { + [protocol.ShutdownRequest.method]: undefined; + [protocol.RegistrationRequest.method]: protocol.RegistrationParams; + [protocol.UnregistrationRequest.method]: protocol.UnregistrationParams; + [protocol.InitializeRequest.method]: protocol.InitializeParams; + [protocol.ConfigurationRequest.method]: protocol.DidChangeConfigurationParams; + [protocol.ShowMessageRequest.method]: protocol.ShowMessageParams; + [protocol.CompletionRequest.method]: protocol.CompletionParams; + [protocol.CompletionResolveRequest.method]: protocol.CompletionItem; + [protocol.HoverRequest.method]: protocol.HoverParams; + [protocol.SignatureHelpRequest.method]: protocol.SignatureHelpParams; + [protocol.DefinitionRequest.method]: protocol.DefinitionParams; + [protocol.ReferencesRequest.method]: protocol.ReferenceParams; + [protocol.DocumentDiagnosticRequest.method]: protocol.DocumentDiagnosticParams; + [protocol.DocumentHighlightRequest.method]: protocol.DocumentHighlightParams; + [protocol.DocumentSymbolRequest.method]: protocol.DocumentSymbolParams; + [protocol.CodeActionRequest.method]: protocol.CodeActionParams; + [protocol.CodeActionResolveRequest.method]: protocol.CodeAction; + [protocol.WorkspaceSymbolRequest.method]: protocol.WorkspaceSymbolParams; + [protocol.WorkspaceSymbolResolveRequest.method]: protocol.WorkspaceSymbol; + [protocol.CodeLensRequest.method]: protocol.CodeLensParams; + [protocol.CodeLensResolveRequest.method]: protocol.CodeLens; + [protocol.DocumentLinkRequest.method]: protocol.DocumentLinkParams; + [protocol.DocumentLinkResolveRequest.method]: protocol.DocumentLink; + [protocol.DocumentFormattingRequest.method]: protocol.DocumentFormattingParams; + [protocol.DocumentRangeFormattingRequest.method]: protocol.DocumentRangeFormattingParams; + [protocol.DocumentRangesFormattingRequest.method]: protocol.DocumentRangesFormattingParams; + [protocol.DocumentOnTypeFormattingRequest.method]: protocol.DocumentOnTypeFormattingParams; + [protocol.RenameRequest.method]: protocol.RenameParams; + [protocol.FoldingRangeRequest.method]: protocol.FoldingRangeParams; + [protocol.PrepareRenameRequest.method]: protocol.PrepareRenameParams; + [protocol.ExecuteCommandRequest.method]: protocol.ExecuteCommandParams; + [protocol.ApplyWorkspaceEditRequest.method]: protocol.ApplyWorkspaceEditParams; + [protocol.InlayHintRequest.method]: protocol.InlayHintParams; + [protocol.InlayHintResolveRequest.method]: protocol.InlayHint; +} + +export interface MessageResponseType { + [protocol.ShutdownRequest.method]: never; + [protocol.RegistrationRequest.method]: void; + [protocol.UnregistrationRequest.method]: void; + [protocol.InitializeRequest.method]: protocol.InitializeResult; + [protocol.ConfigurationRequest.method]: protocol.LSPAny[] | null; + [protocol.ShowMessageRequest.method]: protocol.MessageActionItem | null; + [protocol.CompletionRequest.method]: protocol.CompletionList | protocol.CompletionItem[] | null; + [protocol.CompletionResolveRequest.method]: protocol.CompletionItem; + [protocol.HoverRequest.method]: protocol.Hover | null; + [protocol.SignatureHelpRequest.method]: protocol.SignatureHelp | null; + [protocol.DefinitionRequest.method]: protocol.Definition | protocol.LocationLink[] | null; + [protocol.ReferencesRequest.method]: protocol.Location[] | null; + [protocol.DocumentDiagnosticRequest.method]: protocol.DocumentDiagnosticReport; + [protocol.DocumentHighlightRequest.method]: protocol.DocumentHighlight[] | null; + [protocol.DocumentSymbolRequest.method]: protocol.DocumentSymbol[] | protocol.SymbolInformation[] | null; + [protocol.CodeActionRequest.method]: (protocol.Command | protocol.CodeAction)[] | null; + [protocol.CodeActionResolveRequest.method]: protocol.CodeAction; + [protocol.WorkspaceSymbolRequest.method]: protocol.SymbolInformation[] | protocol.WorkspaceSymbol[] | null; + [protocol.WorkspaceSymbolResolveRequest.method]: protocol.WorkspaceSymbol; + [protocol.CodeLensRequest.method]: protocol.CodeLens[] | null; + [protocol.CodeLensResolveRequest.method]: protocol.CodeLens; + [protocol.DocumentLinkRequest.method]: protocol.DocumentLink[] | null; + [protocol.DocumentLinkResolveRequest.method]: protocol.DocumentLink; + [protocol.DocumentFormattingRequest.method]: protocol.TextEdit[] | null; + [protocol.DocumentRangeFormattingRequest.method]: protocol.TextEdit[] | null; + [protocol.DocumentRangesFormattingRequest.method]: protocol.TextEdit[] | null; + [protocol.DocumentOnTypeFormattingRequest.method]: protocol.TextEdit[] | null; + [protocol.FoldingRangeRequest.method]: protocol.FoldingRange[] | null; + [protocol.RenameRequest.method]: protocol.WorkspaceEdit | null; + [protocol.PrepareRenameRequest.method]: protocol.PrepareRenameResult | null; + [protocol.ExecuteCommandRequest.method]: any; + [protocol.ApplyWorkspaceEditRequest.method]: protocol.ApplyWorkspaceEditResult; + [protocol.InlayHintRequest.method]: protocol.InlayHint[] | null; + [protocol.InlayHintResolveRequest.method]: protocol.InlayHint; +} + +export interface NotificationToParams { + [protocol.InitializedNotification.method]: protocol.InitializedParams; + [protocol.ExitNotification.method]: undefined; + [protocol.DidOpenTextDocumentNotification.method]: protocol.DidOpenTextDocumentParams; + [protocol.DidChangeTextDocumentNotification.method]: protocol.DidChangeTextDocumentParams; + [protocol.DidCloseTextDocumentNotification.method]: protocol.DidCloseTextDocumentParams; + [protocol.DidSaveTextDocumentNotification.method]: protocol.DidSaveTextDocumentParams; + [protocol.WillSaveTextDocumentNotification.method]: protocol.WillSaveTextDocumentParams; + [protocol.DidChangeWatchedFilesNotification.method]: protocol.DidChangeWatchedFilesParams; + [protocol.PublishDiagnosticsNotification.method]: protocol.PublishDiagnosticsParams; +} \ No newline at end of file