From 8867f8996c06745d7621de415a708e2e92cb30bf Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 9 Dec 2025 23:04:27 +0000 Subject: [PATCH 01/11] Initial port of `exerciseServer` to `exerciseLspServer`. --- replay.mts | 82 +++ src/utils/exerciseLspServer.mts | 492 ++++++++++++++++++ ...nstants.ts => exerciseServerConstants.mts} | 0 src/utils/lspHarness.mts | 181 +++++++ 4 files changed, 755 insertions(+) create mode 100644 replay.mts create mode 100644 src/utils/exerciseLspServer.mts rename src/utils/{exerciseServerConstants.ts => exerciseServerConstants.mts} (100%) create mode 100644 src/utils/lspHarness.mts diff --git a/replay.mts b/replay.mts new file mode 100644 index 0000000..1327713 --- /dev/null +++ b/replay.mts @@ -0,0 +1,82 @@ +import * as yadda from "./src/utils/lspHarness.mts"; +import { readFileSync } from "fs"; +import { pathToFileURL } from "url"; + +const [, , scriptPath, testDir] = process.argv; + +const testUrl = testDir ? + pathToFileURL(testDir).toString() : + "file:///workspaces/typescript-error-deltas"; + +const server = yadda.startServer("./tsgo/tsgo-noracedetection", { + args: ["--lsp", "--stdio"], +}, { + traceOutput: true, +}); + +server.handleAnyNotification(async (...args) => { + console.log("Notification received:", ...args); +}); + +server.handleAnyRequest(async (...args) => { + console.log("Request received:", ...args); + return {}; +}); + +const script = readFileSync(scriptPath, "utf-8"); +const lines = script.split(/\r?\n/g); +let rootDirPlaceholder = "@PROJECT_ROOT@"; +let initialize: object | undefined; +let initialized: object | undefined; +if (true) { + for (let line of lines) { + line = line.trim(); + if (line.length === 0) continue; + + let obj = JSON.parse(line) + if (obj.rootDirPlaceholder) { + rootDirPlaceholder = obj.rootDirPlaceholder; + continue; + } + + obj = JSON.parse(line.replaceAll(rootDirPlaceholder, testUrl)); + + console.log(obj) + try { + if (obj.kind === "request") { + if (obj.method === "initialize") { + if (initialize) { + continue; + } + initialize = obj.params; + } + + const responsePromise = server.sendRequest(obj.method, obj.params);; + if (obj.method !== "shutdown") { + await responsePromise; + continue; + } + } + else if (obj.kind === "notification") { + if (obj.method === "initialized") { + if (initialized) { + continue; + } + initialized = obj.params; + } + await server.sendNotification(obj.method, obj.params); + } + else { + throw new Error(`Unknown replay entry kind: ${JSON.stringify(obj)}`); + } + } + catch (e) { + console.log(e); + } + + // Slow down requests - sometimes helpful for ATA races. + // await new Promise(resolve => setTimeout(resolve, 2000)); + } +} + +server.kill(); diff --git a/src/utils/exerciseLspServer.mts b/src/utils/exerciseLspServer.mts new file mode 100644 index 0000000..c991058 --- /dev/null +++ b/src/utils/exerciseLspServer.mts @@ -0,0 +1,492 @@ +// @ts-check + +import * as lsp from "./lspHarness.mts"; +import fs from "fs"; +import process from "process"; +import path from "path"; +import * as glob from "glob"; +import { performance } from "perf_hooks"; +import randomSeed from "random-seed"; +import * as protocol from "vscode-languageserver-protocol"; +import { EXIT_BAD_ARGS, EXIT_UNHANDLED_EXCEPTION, EXIT_SERVER_EXIT_FAILED, EXIT_SERVER_CRASH, EXIT_SERVER_ERROR } from "./exerciseServerConstants.mts"; +import { pathToFileURL } from "url"; +import { isWhiteSpaceLike } from "typescript"; + +const testDirPlaceholder = "@PROJECT_ROOT@"; + +const argv = process.argv; + +if (argv.length !== 7) { + console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); + process.exit(EXIT_BAD_ARGS); +} + +// CONVENTION: stderr is for output to the log; stdout is for output to the user + +const [, , testDir, replayScriptPath, lspServerPath, diag, seed] = argv; +const diagnosticOutput = diag.toLocaleLowerCase() === "true"; +const prng = randomSeed.create(seed); + +await exerciseLspServer(testDir, replayScriptPath, lspServerPath).catch(e => { + console.error(e); + process.exit(EXIT_UNHANDLED_EXCEPTION); +}); +console.log("Allll done") + +export async function exerciseLspServer(testDir: string, replayScriptPath: string, lspServerPath: string): Promise { + const requestTimes: Record = {}; + const requestCounts: Record = {}; + const start = performance.now(); + + const oldCwd = process.cwd(); + const replayScriptHandle = await fs.promises.open(replayScriptPath, "w"); + try { + await exerciseLspServerWorker(testDir, lspServerPath, replayScriptHandle, requestTimes, requestCounts); + } + finally { + await replayScriptHandle.close(); + + process.chdir(oldCwd); + + const end = performance.now(); + if (diagnosticOutput) { + console.error(`Elapsed time ${Math.round(end - start)} ms`); + for (const method in requestTimes) { + console.error(`${method}:\t${Math.round(requestTimes[method])} ms (${requestCounts[method]} calls)`); + } + } + } +} + +function filePathToUri(filePath: string): string { + return pathToFileURL(filePath).toString(); +} + +function getLanguageId(filePath: string): string { + const ext = path.extname(filePath).toLowerCase(); + switch (ext) { + case ".ts": return "typescript"; + case ".mts": return "typescript"; + case ".cts": return "typescript"; + + case ".js": return "javascript"; + case ".mjs": return "javascript"; + case ".cjs": return "javascript"; + + case ".tsx": return "typescriptreact"; + case ".jsx": return "javascriptreact"; + + default: return "typescript"; + } +} + +async function exerciseLspServerWorker(testDir: string, lspServerPath: string, replayScriptHandle: fs.promises.FileHandle, requestTimes: Record, requestCounts: Record): Promise { + // TODO: re-enable for JS files + const files = await glob.glob("**/*.@(ts|tsx|mts|cts)", { cwd: testDir, absolute: true, ignore: ["**/node_modules/**", "**/*.min.js"], nodir: true, follow: false }); + + const serverArgs: string[] = ["--lsp", "--stdio"]; + + replayScriptHandle.write(JSON.stringify({ + rootDirPlaceholder: testDirPlaceholder, + serverArgs, + }) + "\n"); + + const server = lsp.startServer(lspServerPath, { + args: serverArgs, + }, { traceOutput: diagnosticOutput }); + + server.handleAnyRequest(async (...args) => { + console.log("Server sent request:", ...args); + }); + + server.handleAnyNotification(async (...args) => { + console.log("Server sent notification:", ...args); + }); + + let documentVersion = 0; + + const testDirUrl = filePathToUri(testDir); + + // Initialize the server + const initializeParams: protocol.InitializeParams = { + processId: process.pid, + capabilities: { + textDocument: { + completion: { + completionItem: { + snippetSupport: true, + insertReplaceSupport: true, + resolveSupport: { + properties: ["documentation", "detail", "additionalTextEdits"], + }, + commitCharactersSupport: true, + deprecatedSupport: true, + preselectSupport: true, + labelDetailsSupport: true, + documentationFormat: ["markdown", "plaintext"], + insertTextModeSupport: { + valueSet: [ + protocol.InsertTextMode.asIs, + protocol.InsertTextMode.adjustIndentation, + ], + }, + // TODO: ... + }, + contextSupport: true, + }, + definition: { + linkSupport: true, + }, + references: {}, + documentSymbol: { + hierarchicalDocumentSymbolSupport: true, + labelSupport: true, + // TODO: ... + }, + foldingRange: { + foldingRange: { collapsedText: true }, + // TODO: ... + }, + codeAction: { + disabledSupport: true, + dataSupport: true, + // TODO: ... + codeActionLiteralSupport: { + codeActionKind: { + valueSet: [ + protocol.CodeActionKind.QuickFix, + protocol.CodeActionKind.Refactor, + protocol.CodeActionKind.RefactorExtract, + protocol.CodeActionKind.RefactorInline, + protocol.CodeActionKind.RefactorRewrite, + protocol.CodeActionKind.Source, + protocol.CodeActionKind.SourceOrganizeImports, + ], + }, + }, + }, + hover: {contentFormat: ["markdown", "plaintext"]}, + diagnostic: {relatedDocumentSupport: true}, + declaration: {linkSupport: true}, + implementation: {linkSupport: true}, + typeDefinition: {linkSupport: true}, + rename: { + // TODO: ... + } + // TODO: ... + + }, + workspace: { + symbol: { + // TODO: ... + }, + // TODO + // codeLens: { refreshSupport: true }, + // inlayHint: { refreshSupport: true, }, + // foldingRange: { refreshSupport: true }, + // semanticTokens: { refreshSupport: true }, + // diagnostics: { refreshSupport: true }, + // configuration: true, + // TODO: ... + }, + }, + rootUri: testDirUrl, + // workspaceFolders: [ + // { + // uri: testDirUrl, + // name: path.basename(testDir), + // }, + // ], + }; + + try { + await request("initialize", initializeParams); + await notify("initialized", {}); + + const openFileUris: string[] = []; + + // NB: greater than 1 behaves the same as 1 + // const skipFileProb = 1000 / files.length; + const skipFileProb = 1; + for (const openFileAbsolutePath of files) { + if (prng.random() > skipFileProb) continue; + + const openFileUri = filePathToUri(openFileAbsolutePath); + + if (openFileUris.length === 5) { + const closedFileUri = openFileUris.shift()!; + // Close the document + await notify("textDocument/didClose", { + textDocument: { + uri: closedFileUri, + }, + }); + } + + openFileUris.push(openFileUri); + + const openFileContents = await fs.promises.readFile(openFileAbsolutePath, { encoding: "utf-8" }); + const languageId = getLanguageId(openFileAbsolutePath); + documentVersion++; + + // Open the document + await notify("textDocument/didOpen", { + textDocument: { + uri: openFileUri, + languageId, + version: documentVersion, + text: openFileContents, + }, + }); + + const triggerChars = [".", '"', "'", "`", "/", "@", "<", "#", " "]; + const signatureHelpTriggerChars = ["(", ",", "<"]; + + let line = 0; // LSP uses 0-based lines + let character = 0; // LSP uses 0-based characters + + let prev = ""; + + // Organize imports (source.organizeImports code action) + // await request("textDocument/codeAction", { + // textDocument: { uri: openFileUri }, + // range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } }, + // context: { + // diagnostics: [], + // only: [protocol.CodeActionKind.SourceOrganizeImports], + // }, + // }, 0.5); + + if (openFileContents.length < 1e6) { + // Folding ranges (equivalent to getOutliningSpans) + // TODO: folding ranges are broken for JS files + await request("textDocument/foldingRange", { + textDocument: { uri: openFileUri }, + }, +(languageId.startsWith("typescript"))); + + // Document symbols (equivalent to navtree/navbar) + await request("textDocument/documentSymbol", { + textDocument: { uri: openFileUri }, + }); + } + + // Workspace symbol search (equivalent to navto) + const workspaceSymbolResponse = await request("workspace/symbol", { + query: "a", + }, 0.5); + + if (workspaceSymbolResponse && Array.isArray(workspaceSymbolResponse) && workspaceSymbolResponse.length > 0) { + const symbolEntry = workspaceSymbolResponse.find((x: protocol.SymbolInformation | protocol.WorkspaceSymbol) => x.name.length > 4); + if (symbolEntry) { + await request("workspace/symbol", { + query: symbolEntry.name.slice(0, 3), + }); + } + } + + // Diagnostics (equivalent to geterr) + const diagnosticsPromise = request("textDocument/diagnostic", { + textDocument: { uri: openFileUri }, + }); + + const codeLensesPromise = request("textDocument/codeLens", { + textDocument: { uri: openFileUri }, + }); + + const inlayHintsPromise = request("textDocument/inlayHint", { + textDocument: { uri: openFileUri }, + range: { start: { line: 0, character: 0 }, end: { line: 0, character: openFileContents.length } }, + }); + + await Promise.all([diagnosticsPromise, codeLensesPromise, inlayHintsPromise]); + + for (let i = 0; i < openFileContents.length; i++) { + const curr = openFileContents[i]; + const next = openFileContents[i + 1]; + + // Increase probabilities around things that look like jsdoc, where we've had problems in the past + const isAt = curr === "@"; + + // Note that this only catches Latin letters - we'll test within tokens of non-Latin characters + if (!(/\w/.test(prev) && /\w/.test(curr)) && !(/[ \t]/.test(prev) && /[ \t]/.test(curr))) { + // Definition (equivalent to definitionAndBoundSpan) + await request("textDocument/definition", { + textDocument: { uri: openFileUri }, + position: { line, character }, + }, isAt ? 0.5 : 0.001); + + // // References + await request("textDocument/references", { + textDocument: { uri: openFileUri }, + position: { line, character }, + context: { includeDeclaration: true }, + }, isAt ? 0.5 : 0.00005); + + // TODO: + // - https://github.com/microsoft/typescript-go/issues/2253 + const completionsProb = isWhiteSpaceLike(prev.charCodeAt(0)) ? 0 : (isAt ? 0.5 : 0.01); + + // Completions (equivalent to completionInfo) + const completionResponse = await request("textDocument/completion", { + textDocument: { uri: openFileUri }, + position: { line, character }, + context: { + triggerKind: protocol.CompletionTriggerKind.Invoked, + }, + }, completionsProb); + + // Completion resolve (equivalent to completionEntryDetails) + if (completionResponse) { + const items = "items" in completionResponse ? completionResponse.items : completionResponse; + if (Array.isArray(items) && items.length > 0) { + await request("completionItem/resolve", items.find(item => item.preselect) ?? items[0]); + } + } + + // Triggered completions + const triggerCharIndex = triggerChars.indexOf(curr); + if (triggerCharIndex >= 0 && /\w/.test(prev)) { + await request("textDocument/completion", { + textDocument: { uri: openFileUri }, + position: { line, character }, + context: { + triggerKind: protocol.CompletionTriggerKind.TriggerCharacter, + triggerCharacter: triggerChars[triggerCharIndex], + }, + }, completionsProb /*previously 0.005*/); + } + } + + let currisSignatureHelpTrigger = false; + if ((currisSignatureHelpTrigger = signatureHelpTriggerChars.includes(curr)) || signatureHelpTriggerChars.includes(next)) { + // Signature help (equivalent to signatureHelp) + await request("textDocument/signatureHelp", { + textDocument: { uri: openFileUri }, + position: { line, character }, + context: { + triggerCharacter: currisSignatureHelpTrigger ? curr : undefined, + triggerKind: currisSignatureHelpTrigger ? protocol.SignatureHelpTriggerKind.TriggerCharacter : protocol.SignatureHelpTriggerKind.Invoked, + isRetrigger: signatureHelpTriggerChars.includes(prev), + } + }, 0.25); + } + + if (curr === "\r" || curr === "\n") { + if (line === 0) { + // Apply a text change (equivalent to updateOpen with changedFiles) + documentVersion++; + await notify("textDocument/didChange", { + textDocument: { + uri: openFileUri, + version: documentVersion, + }, + contentChanges: [ + { + range: { + start: { line, character }, + end: { line, character }, + }, + text: " //comment", + }, + ], + }); + } + + line++; + character = 0; + if (curr === "\r" && next === "\n") { + i++; + } + } + else { + character++; + } + + prev = curr; + } + } + + console.log("\nShutting down server"); + // Send shutdown request and exit notification + void request(protocol.ShutdownRequest.method, undefined); + void notify("exit", undefined); + + } catch (e) { + console.error("Killing server after unhandled exception"); + console.error(e); + + await server.kill(); + process.exit(EXIT_UNHANDLED_EXCEPTION); + } + + await server.kill(); + + async function request( + method: K, + params: lsp.RequestToParams[K], + prob = 1, + ): Promise { + // await new Promise(resolve => setTimeout(resolve, 500)); + if (prng.random() > prob) return undefined as any; + + const replayEntry = { kind: "request", method, params }; + const replayStr = JSON.stringify(replayEntry).replaceAll(testDirUrl, testDirPlaceholder); + await replayScriptHandle.write(replayStr + "\n"); + + const start = performance.now(); + try { + const response = await server.sendRequest(method, params); + const end = performance.now(); + requestTimes[method] = (requestTimes[method] ?? 0) + (end - start); + requestCounts[method] = (requestCounts[method] ?? 0) + 1; + return response; + } catch (e: any) { + const end = performance.now(); + requestTimes[method] = (requestTimes[method] ?? 0) + (end - start); + requestCounts[method] = (requestCounts[method] ?? 0) + 1; + + const errorMessage = e.message ?? "Unknown error"; + if (diagnosticOutput) { + console.error(`Request failed: +${JSON.stringify(replayEntry, undefined, 2)} +${e}`); + } + else { + console.error(errorMessage); + } + console.error(JSON.stringify({ error: e.message })); + + void server.kill(); + process.exit(EXIT_SERVER_ERROR); + } + } + + async function notify( + method: K, + params: lsp.NotificationToParams[K], + ): Promise { + // await new Promise(resolve => setTimeout(resolve, 500)); + const replayEntry = { kind: "notification", method, params }; + const replayStr = JSON.stringify(replayEntry).replaceAll(testDirUrl, testDirPlaceholder); + await replayScriptHandle.write(replayStr + "\n"); + + try { + await server.sendNotification(method, params); + } + catch (e: any) { + const errorMessage = e.message ?? "Unknown error"; + if (diagnosticOutput) { + console.error(`Notification failed: +${JSON.stringify(replayEntry, undefined, 2)} +${e}`); + } + else { + console.error(errorMessage); + } + console.log(JSON.stringify({ error: e.message })); + + await server.kill(); + process.exit(EXIT_SERVER_ERROR); + } + } +} diff --git a/src/utils/exerciseServerConstants.ts b/src/utils/exerciseServerConstants.mts similarity index 100% rename from src/utils/exerciseServerConstants.ts rename to src/utils/exerciseServerConstants.mts diff --git a/src/utils/lspHarness.mts b/src/utils/lspHarness.mts new file mode 100644 index 0000000..793232f --- /dev/null +++ b/src/utils/lspHarness.mts @@ -0,0 +1,181 @@ +import * as cp from "child_process"; +import * as rpc from "vscode-jsonrpc/node.js"; +import * as protocol from "vscode-languageserver-protocol"; + +export interface ServerOptions { + args?: string[]; + execArgv?: string[]; + env?: NodeJS.ProcessEnv; +} + +export interface LanguageServer { + sendRequest: (method: K, params: RequestToParams[K]) => Promise; + sendRequestUntyped: (method: string, params: object) => Promise; + sendNotification: (method: K, params: NotificationToParams[K]) => Promise; + + handleRequest: (method: K, handler: (params: RequestToParams[K]) => Promise) => void; + handleAnyRequest: (handler: (...args: any[]) => Promise) => void; + handleNotification: (method: K, handler: (params: NotificationToParams[K]) => void) => void; + handleAnyNotification: (handler: (...args: any[]) => Promise) => void; + + kill: () => Promise; +} + +export function startServer(serverPath: string, options: ServerOptions = {}, otherOptions?: { traceOutput?: boolean; }): LanguageServer { + const serverProc = cp.spawn(serverPath, options.args ?? [], { + env: options.env, + // execArgv: options.execArgv, // options.execArgv ?? process.execArgv?.map(arg => bumpDebugPort(arg)), + stdio: [ + "pipe", // stdin + "pipe", // stdout + otherOptions?.traceOutput ? "inherit" : "ignore" // stderr + ], + }); + + const connection = rpc.createMessageConnection( + new rpc.StreamMessageReader(serverProc.stdout!), + new rpc.StreamMessageWriter(serverProc.stdin!) + ); + + connection.listen(); + + return { + sendRequest, + sendRequestUntyped, + sendNotification, + handleRequest, + handleAnyRequest, + handleNotification, + handleAnyNotification, + kill, + }; + + function sendRequest(method: K, params: RequestToParams[K]): Promise { + return connection.sendRequest(method, params); + } + + function sendRequestUntyped(method: string, params: object): Promise { + return connection.sendRequest(method, params); + } + + function sendNotification(method: K, params: NotificationToParams[K]): Promise { + return connection.sendNotification(method, params); + } + + function handleRequest(method: K, handler: (params: RequestToParams[K]) => Promise): rpc.Disposable { + return connection.onRequest(method, handler); + } + + function handleAnyRequest(handler: (...args: any[]) => Promise): rpc.Disposable { + return connection.onRequest(handler); + } + + function handleNotification(method: K, handler: (params: NotificationToParams[K]) => void): rpc.Disposable { + return connection.onNotification(method, handler); + } + + function handleAnyNotification(handler: (...args: any[]) => Promise): rpc.Disposable { + return connection.onNotification(handler); + } + + function kill(): Promise { + return new Promise((resolve, reject) => { + serverProc.once("close", () => { + resolve(); + }); + // If the server has already exited, there won't be a close event + if (serverProc.exitCode !== null || serverProc.signalCode !== null) { + resolve(); + } + if (!serverProc.kill("SIGKILL")) { + reject(new Error("Failed to send kill signal to server")); + } + }); + } +} + +export interface RequestToParams { + [protocol.ShutdownRequest.method]: undefined; + [protocol.RegistrationRequest.method]: protocol.RegistrationParams; + [protocol.UnregistrationRequest.method]: protocol.UnregistrationParams; + [protocol.InitializeRequest.method]: protocol.InitializeParams; + [protocol.ConfigurationRequest.method]: protocol.DidChangeConfigurationParams; + [protocol.ShowMessageRequest.method]: protocol.ShowMessageParams; + [protocol.CompletionRequest.method]: protocol.CompletionParams; + [protocol.CompletionResolveRequest.method]: protocol.CompletionItem; + [protocol.HoverRequest.method]: protocol.HoverParams; + [protocol.SignatureHelpRequest.method]: protocol.SignatureHelpParams; + [protocol.DefinitionRequest.method]: protocol.DefinitionParams; + [protocol.ReferencesRequest.method]: protocol.ReferenceParams; + [protocol.DocumentDiagnosticRequest.method]: protocol.DocumentDiagnosticParams; + [protocol.DocumentHighlightRequest.method]: protocol.DocumentHighlightParams; + [protocol.DocumentSymbolRequest.method]: protocol.DocumentSymbolParams; + [protocol.CodeActionRequest.method]: protocol.CodeActionParams; + [protocol.CodeActionResolveRequest.method]: protocol.CodeAction; + [protocol.WorkspaceSymbolRequest.method]: protocol.WorkspaceSymbolParams; + [protocol.WorkspaceSymbolResolveRequest.method]: protocol.WorkspaceSymbol; + [protocol.CodeLensRequest.method]: protocol.CodeLensParams; + [protocol.CodeLensResolveRequest.method]: protocol.CodeLens; + [protocol.DocumentLinkRequest.method]: protocol.DocumentLinkParams; + [protocol.DocumentLinkResolveRequest.method]: protocol.DocumentLink; + [protocol.DocumentFormattingRequest.method]: protocol.DocumentFormattingParams; + [protocol.DocumentRangeFormattingRequest.method]: protocol.DocumentRangeFormattingParams; + [protocol.DocumentRangesFormattingRequest.method]: protocol.DocumentRangesFormattingParams; + [protocol.DocumentOnTypeFormattingRequest.method]: protocol.DocumentOnTypeFormattingParams; + [protocol.RenameRequest.method]: protocol.RenameParams; + [protocol.FoldingRangeRequest.method]: protocol.FoldingRangeParams; + [protocol.PrepareRenameRequest.method]: protocol.PrepareRenameParams; + [protocol.ExecuteCommandRequest.method]: protocol.ExecuteCommandParams; + [protocol.ApplyWorkspaceEditRequest.method]: protocol.ApplyWorkspaceEditParams; + [protocol.InlayHintRequest.method]: protocol.InlayHintParams; + [protocol.InlayHintResolveRequest.method]: protocol.InlayHint; +} + +export interface MessageResponseType { + [protocol.ShutdownRequest.method]: never; + [protocol.RegistrationRequest.method]: void; + [protocol.UnregistrationRequest.method]: void; + [protocol.InitializeRequest.method]: protocol.InitializeResult; + [protocol.ConfigurationRequest.method]: protocol.LSPAny[] | null; + [protocol.ShowMessageRequest.method]: protocol.MessageActionItem | null; + [protocol.CompletionRequest.method]: protocol.CompletionList | protocol.CompletionItem[] | null; + [protocol.CompletionResolveRequest.method]: protocol.CompletionItem; + [protocol.HoverRequest.method]: protocol.Hover | null; + [protocol.SignatureHelpRequest.method]: protocol.SignatureHelp | null; + [protocol.DefinitionRequest.method]: protocol.Definition | protocol.LocationLink[] | null; + [protocol.ReferencesRequest.method]: protocol.Location[] | null; + [protocol.DocumentDiagnosticRequest.method]: protocol.DocumentDiagnosticReport; + [protocol.DocumentHighlightRequest.method]: protocol.DocumentHighlight[] | null; + [protocol.DocumentSymbolRequest.method]: protocol.DocumentSymbol[] | protocol.SymbolInformation[] | null; + [protocol.CodeActionRequest.method]: (protocol.Command | protocol.CodeAction)[] | null; + [protocol.CodeActionResolveRequest.method]: protocol.CodeAction; + [protocol.WorkspaceSymbolRequest.method]: protocol.SymbolInformation[] | protocol.WorkspaceSymbol[] | null; + [protocol.WorkspaceSymbolResolveRequest.method]: protocol.WorkspaceSymbol; + [protocol.CodeLensRequest.method]: protocol.CodeLens[] | null; + [protocol.CodeLensResolveRequest.method]: protocol.CodeLens; + [protocol.DocumentLinkRequest.method]: protocol.DocumentLink[] | null; + [protocol.DocumentLinkResolveRequest.method]: protocol.DocumentLink; + [protocol.DocumentFormattingRequest.method]: protocol.TextEdit[] | null; + [protocol.DocumentRangeFormattingRequest.method]: protocol.TextEdit[] | null; + [protocol.DocumentRangesFormattingRequest.method]: protocol.TextEdit[] | null; + [protocol.DocumentOnTypeFormattingRequest.method]: protocol.TextEdit[] | null; + [protocol.FoldingRangeRequest.method]: protocol.FoldingRange[] | null; + [protocol.RenameRequest.method]: protocol.WorkspaceEdit | null; + [protocol.PrepareRenameRequest.method]: protocol.PrepareRenameResult | null; + [protocol.ExecuteCommandRequest.method]: any; + [protocol.ApplyWorkspaceEditRequest.method]: protocol.ApplyWorkspaceEditResult; + [protocol.InlayHintRequest.method]: protocol.InlayHint[] | null; + [protocol.InlayHintResolveRequest.method]: protocol.InlayHint; +} + +export interface NotificationToParams { + [protocol.InitializedNotification.method]: protocol.InitializedParams; + [protocol.ExitNotification.method]: undefined; + [protocol.DidOpenTextDocumentNotification.method]: protocol.DidOpenTextDocumentParams; + [protocol.DidChangeTextDocumentNotification.method]: protocol.DidChangeTextDocumentParams; + [protocol.DidCloseTextDocumentNotification.method]: protocol.DidCloseTextDocumentParams; + [protocol.DidSaveTextDocumentNotification.method]: protocol.DidSaveTextDocumentParams; + [protocol.WillSaveTextDocumentNotification.method]: protocol.WillSaveTextDocumentParams; + [protocol.DidChangeWatchedFilesNotification.method]: protocol.DidChangeWatchedFilesParams; + [protocol.PublishDiagnosticsNotification.method]: protocol.PublishDiagnosticsParams; +} \ No newline at end of file From c7cccd279b7ecd7fd127c6e809171368dd475752 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 9 Dec 2025 23:05:00 +0000 Subject: [PATCH 02/11] Dependencies. --- package-lock.json | 72 +++++++++++++++++++++++++++++++++++++++++++++-- package.json | 4 ++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 041e70f..9f10640 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,9 @@ "json5": "^2.2.3", "markdown-escape": "^2.0.0", "random-seed": "^0.3.0", - "simple-git": "^3.22.0" + "simple-git": "^3.22.0", + "vscode-jsonrpc": "^8.2.1", + "vscode-languageserver-protocol": "^3.17.5" }, "devDependencies": { "@types/jest": "^29.5.12", @@ -145,6 +147,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.0.tgz", "integrity": "sha512-fQfkg0Gjkza3nf0c7/w6Xf34BW4YvzNfACRLmmb7XRLa6XHdR+K9AlJlxneFfWYf6uhOzuzZVTjF/8KfndZANw==", "dev": true, + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.23.5", @@ -1167,6 +1170,7 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.1.0.tgz", "integrity": "sha512-BDa2VAMLSh3otEiaMJ/3Y36GU4qf6GI+VivQ/P41NC6GHcdxpKlqV0ikSZ5gdQsmS3ojXeRx5vasgNTinF0Q4g==", + "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.0.0", @@ -1702,6 +1706,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001587", "electron-to-chromium": "^1.4.668", @@ -2565,6 +2570,7 @@ "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, + "peer": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -4367,6 +4373,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.2.tgz", "integrity": "sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ==", "dev": true, + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -4430,6 +4437,36 @@ "node": ">=10.12.0" } }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vscode-languageserver-protocol": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "dependencies": { + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" + } + }, + "node_modules/vscode-languageserver-protocol/node_modules/vscode-jsonrpc": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==" + }, "node_modules/walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", @@ -4746,6 +4783,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.0.tgz", "integrity": "sha512-fQfkg0Gjkza3nf0c7/w6Xf34BW4YvzNfACRLmmb7XRLa6XHdR+K9AlJlxneFfWYf6uhOzuzZVTjF/8KfndZANw==", "dev": true, + "peer": true, "requires": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.23.5", @@ -5537,6 +5575,7 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.1.0.tgz", "integrity": "sha512-BDa2VAMLSh3otEiaMJ/3Y36GU4qf6GI+VivQ/P41NC6GHcdxpKlqV0ikSZ5gdQsmS3ojXeRx5vasgNTinF0Q4g==", + "peer": true, "requires": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.0.0", @@ -5968,6 +6007,7 @@ "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz", "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==", "dev": true, + "peer": true, "requires": { "caniuse-lite": "^1.0.30001587", "electron-to-chromium": "^1.4.668", @@ -6590,6 +6630,7 @@ "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, + "peer": true, "requires": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -7930,7 +7971,8 @@ "version": "5.4.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.2.tgz", "integrity": "sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ==", - "dev": true + "dev": true, + "peer": true }, "undici-types": { "version": "5.26.5", @@ -7964,6 +8006,32 @@ "convert-source-map": "^2.0.0" } }, + "vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==" + }, + "vscode-languageserver-protocol": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "requires": { + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" + }, + "dependencies": { + "vscode-jsonrpc": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==" + } + } + }, + "vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==" + }, "walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", diff --git a/package.json b/package.json index 368da8b..aac15c4 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,9 @@ "json5": "^2.2.3", "markdown-escape": "^2.0.0", "random-seed": "^0.3.0", - "simple-git": "^3.22.0" + "simple-git": "^3.22.0", + "vscode-jsonrpc": "^8.2.1", + "vscode-languageserver-protocol": "^3.17.5" }, "devDependencies": { "@types/jest": "^29.5.12", From aedbcac3aed7e983de8adbb9fe9ab369143bda8c Mon Sep 17 00:00:00 2001 From: Gabriela Araujo Britto Date: Wed, 4 Feb 2026 17:47:42 +0000 Subject: [PATCH 03/11] plumb lsp server through main --- src/main.ts | 154 +++++++++++++++++- ...ciseLspServer.mts => exerciseLspServer.ts} | 110 ++++++------- ...nstants.mts => exerciseServerConstants.ts} | 0 src/utils/{lspHarness.mts => lspHarness.ts} | 0 4 files changed, 199 insertions(+), 65 deletions(-) rename src/utils/{exerciseLspServer.mts => exerciseLspServer.ts} (85%) rename src/utils/{exerciseServerConstants.mts => exerciseServerConstants.ts} (100%) rename src/utils/{lspHarness.mts => lspHarness.ts} (100%) diff --git a/src/main.ts b/src/main.ts index 27de6de..0557757 100644 --- a/src/main.ts +++ b/src/main.ts @@ -68,7 +68,7 @@ export interface UserParams extends Params { prNumber: number; } -export type TsEntrypoint = "tsc" | "tsserver"; +export type TsEntrypoint = "tsc" | "tsserver" | "lsp"; const processCwd = process.cwd(); const packageTimeout = 10 * 60 * 1000; @@ -338,14 +338,107 @@ async function getTsServerRepoResult( installCommands, }; - if (!oldServerFailed && !newServerFailed) { - return { status: "Detected no interesting changes" }; + return { status: "Detected interesting changes", tsServerResult: tsServerResult, replayScriptPath, rawErrorPath }; + } + catch (err) { + reportError(err, `Error running tsserver on ${repo.url ?? repo.name}`); + return { status: "Unknown failure" }; + } + finally { + console.log(`Done ${repo.url ?? repo.name}`); + logStepTime(diagnosticOutput, repo, "language service", lsStart); + } +} + +/** + * Tests the LSP server against a single TypeScript native version and reports all crashes found. + */ +export async function getLSPResult( + repo: git.Repo, + userTestsDir: string, + lspServerPath: string, + downloadDir: OverlayBaseFS, + replayScriptArtifactPath: string, + rawErrorArtifactPath: string, + diagnosticOutput: boolean, +): Promise { + + if (!await cloneRepo(repo, userTestsDir, downloadDir.path, diagnosticOutput)) { + return { status: "Git clone failed" }; + } + + const repoDir = path.join(downloadDir.path, repo.name); + const monorepoPackages = await getMonorepoPackages(repoDir); + + // Presumably, people occasionally browse repos without installing the packages first + const installCommands = (prng.random() > 0.2) && monorepoPackages + ? (await installPackagesAndGetCommands(repo, downloadDir.path, repoDir, monorepoPackages, /*cleanOnFailure*/ true, diagnosticOutput))! + : []; + + const replayScriptName = path.basename(replayScriptArtifactPath); + const replayScriptPath = path.join(downloadDir.path, replayScriptName); + + const rawErrorName = path.basename(rawErrorArtifactPath); + const rawErrorPath = path.join(downloadDir.path, rawErrorName); + + const lsStart = performance.now(); + try { + console.log(`Testing LSP server with ${lspServerPath}`); + const spawnResult = await spawnWithTimeoutAsync(repoDir, process.argv[0], [path.join(__dirname, "utils", "exerciseLspServer.js"), repoDir, replayScriptPath, lspServerPath, diagnosticOutput.toString(), prng.string(10)], executionTimeout); + + if (!spawnResult) { + console.log(`LSP server timed out after ${executionTimeout} ms`); + return { status: "Timeout" }; } + if (diagnosticOutput) { + console.log("Raw spawn results:"); + dumpSpawnResult(spawnResult); + } + + switch (spawnResult.code) { + case 0: + case null: + if (spawnResult.signal !== null) { + console.log(`Exited with signal ${spawnResult.signal}`); + return { status: "Unknown failure" }; + } + + console.log("No crashes found"); + return { status: "Detected no interesting changes" }; + case exercise.EXIT_SERVER_CRASH: + case exercise.EXIT_SERVER_ERROR: + case exercise.EXIT_SERVER_EXIT_FAILED: + // These are the crashes we want to report + break; + case exercise.EXIT_BAD_ARGS: + case exercise.EXIT_UNHANDLED_EXCEPTION: + default: + console.log(`Exited with code ${spawnResult.code}`); + if (!diagnosticOutput) { + dumpSpawnResult(spawnResult); + } + return { status: "Unknown failure" }; + } + + // Server crashed - report the error + console.log(`Crash found in ${lspServerPath}:`); + console.log(insetLines(prettyPrintServerHarnessOutput(spawnResult.stdout, /*filter*/ false))); + await fs.promises.writeFile(rawErrorPath, prettyPrintServerHarnessOutput(spawnResult.stdout, /*filter*/ false)); + + const tsServerResult: TSServerResult = { + oldServerFailed: false, + oldSpawnResult: undefined, + newServerFailed: true, + newSpawnResult: spawnResult, + replayScriptPath, + installCommands, + }; + return { status: "Detected interesting changes", tsServerResult, replayScriptPath, rawErrorPath }; } catch (err) { - reportError(err, `Error running tsserver on ${repo.url ?? repo.name}`); + reportError(err, `Error running LSP server on ${repo.url ?? repo.name}`); return { status: "Unknown failure" }; } finally { @@ -838,9 +931,21 @@ export async function mainAsync(params: GitParams | UserParams): Promise { const rawErrorArtifactPath = path.join(params.resultDirName, rawErrorFileName); const replayScriptArtifactPath = path.join(params.resultDirName, replayScriptFileName); - const { status, summary, tsServerResult, replayScriptPath, rawErrorPath } = params.entrypoint === "tsc" - ? await getTscRepoResult(repo, userTestsDir, oldTsEntrypointPath, newTsEntrypointPath, params.buildWithNewWhenOldFails, downloadDir, diagnosticOutput) - : await getTsServerRepoResult(repo, userTestsDir, oldTsEntrypointPath, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput, isPr); + let repoResult: RepoResult; + switch (params.entrypoint) { + case "tsc": + repoResult = await getTscRepoResult(repo, userTestsDir, oldTsEntrypointPath, newTsEntrypointPath, params.buildWithNewWhenOldFails, downloadDir, diagnosticOutput); + break; + case "tsserver": + repoResult = await getTsServerRepoResult(repo, userTestsDir, oldTsEntrypointPath, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput, isPr); + break; + case "lsp": + repoResult = await getLSPResult(repo, userTestsDir, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput); + break; + default: + throw new Error(`Unknown entrypoint: ${params.entrypoint}`); + } + const { status, summary, tsServerResult: tsServerResult, replayScriptPath, rawErrorPath } = repoResult; console.log(`Repo ${repo.url ?? repo.name} had status "${status}"`); statusCounts[status] = (statusCounts[status] ?? 0) + 1; @@ -1076,6 +1181,10 @@ function makeMarkdownLink(url: string) { async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Promise<{ oldTsEntrypointPath: string, oldTsResolvedVersion: string, newTsEntrypointPath: string, newTsResolvedVersion: string }> { const entrypoint = params.entrypoint; if (params.testType === "user") { + // TODO user tests for lsp + if (params.entrypoint === "lsp") { + throw new Error("Not implemented"); + } const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = await downloadTsRepoAsync(cwd, params.oldTsRepoUrl, params.oldHeadRef, entrypoint); // We need to handle the ref/pull/*/merge differently as it is not a branch and cannot be pulled during clone. const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = await downloadTsPrAsync(cwd, params.oldTsRepoUrl, params.prNumber, entrypoint); @@ -1088,8 +1197,12 @@ async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Pro }; } else if (params.testType === "github") { - const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = await downloadTsNpmAsync(cwd, params.oldTsNpmVersion, entrypoint); - const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = await downloadTsNpmAsync(cwd, params.newTsNpmVersion, entrypoint); + const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = params.entrypoint === "lsp" ? + { tsEntrypointPath: "", resolvedVersion: "" } : + await downloadTsNpmAsync(cwd, params.oldTsNpmVersion, entrypoint); + const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = params.entrypoint === "lsp" ? + await downloadTsNativePreviewNpmAsync(cwd, params.newTsNpmVersion) : + await downloadTsNpmAsync(cwd, params.newTsNpmVersion, entrypoint); return { oldTsEntrypointPath, @@ -1165,3 +1278,26 @@ async function downloadTsNpmAsync(cwd: string, version: string, entrypoint: TsEn return { tsEntrypointPath, resolvedVersion }; } + +async function downloadTsNativePreviewNpmAsync(cwd: string, version: string): Promise<{ tsEntrypointPath: string, resolvedVersion: string }> { + const tarName = (await execAsync(cwd, `npm pack @typescript/native-preview@${version} --quiet`)).trim(); + + const tarMatch = /^(typescript-native-preview-(.+))\..+$/.exec(tarName); + if (!tarMatch) { + throw new Error("Unexpected tarball name format: " + tarName); + } + + const resolvedVersion = tarMatch[2]; + const dirName = tarMatch[1]; + const dirPath = path.join(processCwd, dirName); + + await execAsync(cwd, `tar xf ${tarName} && rm ${tarName}`); + await fs.promises.rename(path.join(processCwd, "package"), dirPath); + + const tsEntrypointPath = path.join(dirPath, "bin", "tsgo.js"); + if (!await pu.exists(tsEntrypointPath)) { + throw new Error("Cannot find file " + tsEntrypointPath); + } + + return { tsEntrypointPath, resolvedVersion }; +} diff --git a/src/utils/exerciseLspServer.mts b/src/utils/exerciseLspServer.ts similarity index 85% rename from src/utils/exerciseLspServer.mts rename to src/utils/exerciseLspServer.ts index c991058..2f0d0a7 100644 --- a/src/utils/exerciseLspServer.mts +++ b/src/utils/exerciseLspServer.ts @@ -1,6 +1,5 @@ -// @ts-check - -import * as lsp from "./lspHarness.mts"; +#!/usr/bin/env node +import * as lsp from "./lspHarness"; import fs from "fs"; import process from "process"; import path from "path"; @@ -8,9 +7,8 @@ import * as glob from "glob"; import { performance } from "perf_hooks"; import randomSeed from "random-seed"; import * as protocol from "vscode-languageserver-protocol"; -import { EXIT_BAD_ARGS, EXIT_UNHANDLED_EXCEPTION, EXIT_SERVER_EXIT_FAILED, EXIT_SERVER_CRASH, EXIT_SERVER_ERROR } from "./exerciseServerConstants.mts"; +import { EXIT_BAD_ARGS, EXIT_UNHANDLED_EXCEPTION, EXIT_SERVER_ERROR } from "./exerciseServerConstants"; import { pathToFileURL } from "url"; -import { isWhiteSpaceLike } from "typescript"; const testDirPlaceholder = "@PROJECT_ROOT@"; @@ -27,11 +25,10 @@ const [, , testDir, replayScriptPath, lspServerPath, diag, seed] = argv; const diagnosticOutput = diag.toLocaleLowerCase() === "true"; const prng = randomSeed.create(seed); -await exerciseLspServer(testDir, replayScriptPath, lspServerPath).catch(e => { +exerciseLspServer(testDir, replayScriptPath, lspServerPath).catch(e => { console.error(e); process.exit(EXIT_UNHANDLED_EXCEPTION); }); -console.log("Allll done") export async function exerciseLspServer(testDir: string, replayScriptPath: string, lspServerPath: string): Promise { const requestTimes: Record = {}; @@ -81,8 +78,7 @@ function getLanguageId(filePath: string): string { } async function exerciseLspServerWorker(testDir: string, lspServerPath: string, replayScriptHandle: fs.promises.FileHandle, requestTimes: Record, requestCounts: Record): Promise { - // TODO: re-enable for JS files - const files = await glob.glob("**/*.@(ts|tsx|mts|cts)", { cwd: testDir, absolute: true, ignore: ["**/node_modules/**", "**/*.min.js"], nodir: true, follow: false }); + const files = await glob.glob("**/*.@(ts|tsx|mts|cts|js|jsx|mjs|cjs)", { cwd: testDir, absolute: true, ignore: ["**/node_modules/**", "**/*.min.js"], nodir: true, follow: false }); const serverArgs: string[] = ["--lsp", "--stdio"]; @@ -91,10 +87,17 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r serverArgs, }) + "\n"); + // TODO: would be nice if we could make this work with node_modules/.bin/*.CMD files on Windows + if (path.extname(lspServerPath).toLowerCase().endsWith("js")) { + // Use Node.js or Bun or whatever we ran under. + serverArgs.unshift(lspServerPath); + lspServerPath = process.execPath; + } + const server = lsp.startServer(lspServerPath, { args: serverArgs, }, { traceOutput: diagnosticOutput }); - + server.handleAnyRequest(async (...args) => { console.log("Server sent request:", ...args); }); @@ -165,11 +168,11 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r }, }, }, - hover: {contentFormat: ["markdown", "plaintext"]}, - diagnostic: {relatedDocumentSupport: true}, - declaration: {linkSupport: true}, - implementation: {linkSupport: true}, - typeDefinition: {linkSupport: true}, + hover: { contentFormat: ["markdown", "plaintext"] }, + diagnostic: { relatedDocumentSupport: true }, + declaration: { linkSupport: true }, + implementation: { linkSupport: true }, + typeDefinition: { linkSupport: true }, rename: { // TODO: ... } @@ -260,21 +263,21 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r if (openFileContents.length < 1e6) { // Folding ranges (equivalent to getOutliningSpans) // TODO: folding ranges are broken for JS files - await request("textDocument/foldingRange", { - textDocument: { uri: openFileUri }, - }, +(languageId.startsWith("typescript"))); - - // Document symbols (equivalent to navtree/navbar) - await request("textDocument/documentSymbol", { - textDocument: { uri: openFileUri }, - }); + // await request("textDocument/foldingRange", { + // textDocument: { uri: openFileUri }, + // }, +(languageId.startsWith("typescript"))); + + // // Document symbols (equivalent to navtree/navbar) + // await request("textDocument/documentSymbol", { + // textDocument: { uri: openFileUri }, + // }); } // Workspace symbol search (equivalent to navto) const workspaceSymbolResponse = await request("workspace/symbol", { query: "a", }, 0.5); - + if (workspaceSymbolResponse && Array.isArray(workspaceSymbolResponse) && workspaceSymbolResponse.length > 0) { const symbolEntry = workspaceSymbolResponse.find((x: protocol.SymbolInformation | protocol.WorkspaceSymbol) => x.name.length > 4); if (symbolEntry) { @@ -284,22 +287,22 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r } } - // Diagnostics (equivalent to geterr) - const diagnosticsPromise = request("textDocument/diagnostic", { - textDocument: { uri: openFileUri }, - }); + // // Diagnostics (equivalent to geterr) + // const diagnosticsPromise = request("textDocument/diagnostic", { + // textDocument: { uri: openFileUri }, + // }); - const codeLensesPromise = request("textDocument/codeLens", { - textDocument: { uri: openFileUri }, - }); + // const codeLensesPromise = request("textDocument/codeLens", { + // textDocument: { uri: openFileUri }, + // }); - const inlayHintsPromise = request("textDocument/inlayHint", { - textDocument: { uri: openFileUri }, - range: { start: { line: 0, character: 0 }, end: { line: 0, character: openFileContents.length } }, - }); + // const inlayHintsPromise = request("textDocument/inlayHint", { + // textDocument: { uri: openFileUri }, + // range: { start: { line: 0, character: 0 }, end: { line: 0, character: openFileContents.length } }, + // }); + + // await Promise.all([diagnosticsPromise, codeLensesPromise, inlayHintsPromise]); - await Promise.all([diagnosticsPromise, codeLensesPromise, inlayHintsPromise]); - for (let i = 0; i < openFileContents.length; i++) { const curr = openFileContents[i]; const next = openFileContents[i + 1]; @@ -310,21 +313,21 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r // Note that this only catches Latin letters - we'll test within tokens of non-Latin characters if (!(/\w/.test(prev) && /\w/.test(curr)) && !(/[ \t]/.test(prev) && /[ \t]/.test(curr))) { // Definition (equivalent to definitionAndBoundSpan) - await request("textDocument/definition", { - textDocument: { uri: openFileUri }, - position: { line, character }, - }, isAt ? 0.5 : 0.001); + // await request("textDocument/definition", { + // textDocument: { uri: openFileUri }, + // position: { line, character }, + // }, isAt ? 0.5 : 0.001); // // References - await request("textDocument/references", { - textDocument: { uri: openFileUri }, - position: { line, character }, - context: { includeDeclaration: true }, - }, isAt ? 0.5 : 0.00005); + // await request("textDocument/references", { + // textDocument: { uri: openFileUri }, + // position: { line, character }, + // context: { includeDeclaration: true }, + // }, isAt ? 0.5 : 0.00005); // TODO: // - https://github.com/microsoft/typescript-go/issues/2253 - const completionsProb = isWhiteSpaceLike(prev.charCodeAt(0)) ? 0 : (isAt ? 0.5 : 0.01); + const completionsProb = 1; // Completions (equivalent to completionInfo) const completionResponse = await request("textDocument/completion", { @@ -368,7 +371,7 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r triggerKind: currisSignatureHelpTrigger ? protocol.SignatureHelpTriggerKind.TriggerCharacter : protocol.SignatureHelpTriggerKind.Invoked, isRetrigger: signatureHelpTriggerChars.includes(prev), } - }, 0.25); + }, 0); } if (curr === "\r" || curr === "\n") { @@ -410,7 +413,6 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r // Send shutdown request and exit notification void request(protocol.ShutdownRequest.method, undefined); void notify("exit", undefined); - } catch (e) { console.error("Killing server after unhandled exception"); console.error(e); @@ -447,9 +449,7 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r const errorMessage = e.message ?? "Unknown error"; if (diagnosticOutput) { - console.error(`Request failed: -${JSON.stringify(replayEntry, undefined, 2)} -${e}`); + console.error(`Request failed:\n${JSON.stringify(replayEntry, undefined, 2)}\n${e}`); } else { console.error(errorMessage); @@ -476,9 +476,7 @@ ${e}`); catch (e: any) { const errorMessage = e.message ?? "Unknown error"; if (diagnosticOutput) { - console.error(`Notification failed: -${JSON.stringify(replayEntry, undefined, 2)} -${e}`); + console.error(`Notification failed:\n${JSON.stringify(replayEntry, undefined, 2)}\n${e}`); } else { console.error(errorMessage); @@ -489,4 +487,4 @@ ${e}`); process.exit(EXIT_SERVER_ERROR); } } -} +} \ No newline at end of file diff --git a/src/utils/exerciseServerConstants.mts b/src/utils/exerciseServerConstants.ts similarity index 100% rename from src/utils/exerciseServerConstants.mts rename to src/utils/exerciseServerConstants.ts diff --git a/src/utils/lspHarness.mts b/src/utils/lspHarness.ts similarity index 100% rename from src/utils/lspHarness.mts rename to src/utils/lspHarness.ts From f5973ce311b2ab5a5f35b9d46b0dd96004f2fb9c Mon Sep 17 00:00:00 2001 From: Gabriela Araujo Britto Date: Wed, 4 Feb 2026 22:24:05 +0000 Subject: [PATCH 04/11] handle no old version case --- src/main.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main.ts b/src/main.ts index 0557757..ea27758 100644 --- a/src/main.ts +++ b/src/main.ts @@ -896,7 +896,7 @@ export async function mainAsync(params: GitParams | UserParams): Promise { const { oldTsEntrypointPath, oldTsResolvedVersion, newTsEntrypointPath, newTsResolvedVersion } = await downloadTsAsync(processCwd, params); // Get the name of the typescript folder. - const oldTscDirPath = path.resolve(oldTsEntrypointPath, "../../"); + const oldTscDirPath = oldTsEntrypointPath && path.resolve(oldTsEntrypointPath, "../../"); const newTscDirPath = path.resolve(newTsEntrypointPath, "../../"); console.log("Old version = " + oldTsResolvedVersion); @@ -934,10 +934,10 @@ export async function mainAsync(params: GitParams | UserParams): Promise { let repoResult: RepoResult; switch (params.entrypoint) { case "tsc": - repoResult = await getTscRepoResult(repo, userTestsDir, oldTsEntrypointPath, newTsEntrypointPath, params.buildWithNewWhenOldFails, downloadDir, diagnosticOutput); + repoResult = await getTscRepoResult(repo, userTestsDir, oldTsEntrypointPath!, newTsEntrypointPath, params.buildWithNewWhenOldFails, downloadDir, diagnosticOutput); break; case "tsserver": - repoResult = await getTsServerRepoResult(repo, userTestsDir, oldTsEntrypointPath, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput, isPr); + repoResult = await getTsServerRepoResult(repo, userTestsDir, oldTsEntrypointPath!, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput, isPr); break; case "lsp": repoResult = await getLSPResult(repo, userTestsDir, newTsEntrypointPath, downloadDir, replayScriptArtifactPath, rawErrorArtifactPath, diagnosticOutput); @@ -970,7 +970,7 @@ export async function mainAsync(params: GitParams | UserParams): Promise { summaries.push({ tsServerResult, repo, - oldTsEntrypointPath, + oldTsEntrypointPath: oldTsEntrypointPath || "", rawErrorArtifactPath, replayScript: fs.readFileSync(replayScriptPath, { encoding: "utf-8" }).split(/\r?\n/).slice(-5).join("\n"), replayScriptArtifactPath, @@ -1011,7 +1011,9 @@ export async function mainAsync(params: GitParams | UserParams): Promise { } } - await execAsync(processCwd, "rm -rf " + oldTscDirPath); + if (oldTscDirPath) { + await execAsync(processCwd, "rm -rf " + oldTscDirPath); + } await execAsync(processCwd, "rm -rf " + newTscDirPath); console.log("Statuses"); @@ -1021,7 +1023,7 @@ export async function mainAsync(params: GitParams | UserParams): Promise { const metadata: Metadata = { newTsResolvedVersion: newTsResolvedVersion, - oldTsResolvedVersion: oldTsResolvedVersion, + oldTsResolvedVersion: oldTsResolvedVersion || "", statusCounts, }; await fs.promises.writeFile(path.join(resultDirPath, metadataFileName), JSON.stringify(metadata), { encoding: "utf-8" }); @@ -1178,7 +1180,7 @@ function makeMarkdownLink(url: string) { : `[${mdEscape(match[1])}](${url})`; } -async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Promise<{ oldTsEntrypointPath: string, oldTsResolvedVersion: string, newTsEntrypointPath: string, newTsResolvedVersion: string }> { +async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Promise<{ oldTsEntrypointPath: string | undefined, oldTsResolvedVersion: string | undefined, newTsEntrypointPath: string, newTsResolvedVersion: string }> { const entrypoint = params.entrypoint; if (params.testType === "user") { // TODO user tests for lsp @@ -1198,7 +1200,7 @@ async function downloadTsAsync(cwd: string, params: GitParams | UserParams): Pro } else if (params.testType === "github") { const { tsEntrypointPath: oldTsEntrypointPath, resolvedVersion: oldTsResolvedVersion } = params.entrypoint === "lsp" ? - { tsEntrypointPath: "", resolvedVersion: "" } : + { tsEntrypointPath: undefined, resolvedVersion: undefined } : await downloadTsNpmAsync(cwd, params.oldTsNpmVersion, entrypoint); const { tsEntrypointPath: newTsEntrypointPath, resolvedVersion: newTsResolvedVersion } = params.entrypoint === "lsp" ? await downloadTsNativePreviewNpmAsync(cwd, params.newTsNpmVersion) : From 9f5b3e29d63d92012343fc739989a3d4eec07fb3 Mon Sep 17 00:00:00 2001 From: Gabriela Araujo Britto Date: Wed, 4 Feb 2026 23:24:32 +0000 Subject: [PATCH 05/11] fixes --- src/checkGithubRepos.ts | 6 +++--- src/main.ts | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/checkGithubRepos.ts b/src/checkGithubRepos.ts index 1c32205..cd87bda 100644 --- a/src/checkGithubRepos.ts +++ b/src/checkGithubRepos.ts @@ -3,16 +3,16 @@ import { mainAsync, reportError, TsEntrypoint } from "./main"; const { argv } = process; -if (argv.length !== 11) { +if (argv.length < 11) { console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); process.exit(-1); } -const [,, entrypoint, oldTsNpmVersion, newTsNpmVersion, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput, prngSeed] = argv; +const [,, entrypoint, oldTsNpmVersion, newTsNpmVersion, repoListPath, workerCount, workerNumber, resultDirName, diagnosticOutput, prngSeed, tmpfs] = argv; mainAsync({ testType: "github", - tmpfs: true, + tmpfs: tmpfs && tmpfs.toLowerCase() === "false" ? false : true, entrypoint: entrypoint as TsEntrypoint, diagnosticOutput: diagnosticOutput.toLowerCase() === "true", buildWithNewWhenOldFails: false, diff --git a/src/main.ts b/src/main.ts index ea27758..b48b377 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1282,7 +1282,8 @@ async function downloadTsNpmAsync(cwd: string, version: string, entrypoint: TsEn } async function downloadTsNativePreviewNpmAsync(cwd: string, version: string): Promise<{ tsEntrypointPath: string, resolvedVersion: string }> { - const tarName = (await execAsync(cwd, `npm pack @typescript/native-preview@${version} --quiet`)).trim(); + const packageName = `native-preview-${process.platform}-${process.arch}` + const tarName = (await execAsync(cwd, `npm pack @typescript/${packageName}@${version} --quiet`)).trim(); const tarMatch = /^(typescript-native-preview-(.+))\..+$/.exec(tarName); if (!tarMatch) { @@ -1296,7 +1297,7 @@ async function downloadTsNativePreviewNpmAsync(cwd: string, version: string): Pr await execAsync(cwd, `tar xf ${tarName} && rm ${tarName}`); await fs.promises.rename(path.join(processCwd, "package"), dirPath); - const tsEntrypointPath = path.join(dirPath, "bin", "tsgo.js"); + const tsEntrypointPath = path.join(dirPath, "lib", "tsgo"); if (!await pu.exists(tsEntrypointPath)) { throw new Error("Cannot find file " + tsEntrypointPath); } From 85b147d0372e76817bbf89e6a6c39dd8514e2521 Mon Sep 17 00:00:00 2001 From: Gabriela Araujo Britto Date: Tue, 10 Feb 2026 19:21:31 +0000 Subject: [PATCH 06/11] set up tsgo fuzzer --- .vscode/launch.json | 37 ++++++++++++- src/main.ts | 95 ++++++++++++++++++++++++++++++---- src/postGithubIssue.ts | 7 ++- src/utils/exerciseLspServer.ts | 46 ++++++++++++---- src/utils/exerciseServer.ts | 2 +- src/utils/hashStackTrace.ts | 25 +++++++++ src/utils/lspHarness.ts | 9 +++- 7 files changed, 194 insertions(+), 27 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index f05d54c..2916fab 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,15 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "name": "Attach by Process ID", + "processId": "${command:PickProcess}", + "request": "attach", + "skipFiles": [ + "/**" + ], + "type": "node" + }, { "type": "node", "request": "launch", @@ -115,6 +124,32 @@ "artifacts", "false" ] - } + }, + { + "type": "node", + "request": "launch", + "name": "Launch LSP checkGithubRepos", + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder}/dist/checkGithubRepos.js", + "sourceMaps": true, + "outFiles": [ + "${workspaceFolder}/dist/**/*.js" + ], + "args": [ + "lsp", + "0", + "latest", + "./artifacts/repos.json", + "1", + "1", + "RepoResults", + "true", + "n/a", + "false" + ], + "preLaunchTask": "npm: build" + }, ] } \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index b48b377..fdc3991 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,7 +9,7 @@ import fs = require("fs"); import path = require("path"); import mdEscape = require("markdown-escape"); import randomSeed = require("random-seed"); -import { getErrorMessageFromStack, getHash, getHashForStack } from "./utils/hashStackTrace"; +import { getErrorMessageFromStack, getHash, getHashForStack, getHashForGoStack } from "./utils/hashStackTrace"; import { createCopyingOverlayFS, createTempOverlayFS, OverlayBaseFS } from "./utils/overlayFS"; import { asMarkdownInlineCode } from "./utils/markdownUtils"; @@ -106,6 +106,7 @@ interface Summary { replayScriptArtifactPath: string; replayScriptName: string; resultDirName: string; + entrypoint: TsEntrypoint; commit?: string; } @@ -423,8 +424,8 @@ export async function getLSPResult( // Server crashed - report the error console.log(`Crash found in ${lspServerPath}:`); - console.log(insetLines(prettyPrintServerHarnessOutput(spawnResult.stdout, /*filter*/ false))); - await fs.promises.writeFile(rawErrorPath, prettyPrintServerHarnessOutput(spawnResult.stdout, /*filter*/ false)); + console.log(insetLines(prettyPrintLspHarnessOutput(spawnResult.stdout, /*filter*/ false))); + await fs.promises.writeFile(rawErrorPath, prettyPrintLspHarnessOutput(spawnResult.stdout, /*filter*/ false)); const tsServerResult: TSServerResult = { oldServerFailed: false, @@ -451,18 +452,21 @@ function groupErrors(summaries: Summary[]) { const groupedOldErrors = new Map(); const groupedNewErrors = new Map(); let group: Map; - let error: ServerHarnessOutput | string; + let error: ServerHarnessOutput | LspHarnessOutput | string; for (const summary of summaries) { + const isLsp = summary.entrypoint === "lsp"; if (summary.tsServerResult.newServerFailed) { // Group new errors - error = parseServerHarnessOutput(summary.tsServerResult.newSpawnResult!.stdout); + error = isLsp + ? parseLspHarnessOutput(summary.tsServerResult.newSpawnResult!.stdout) + : parseServerHarnessOutput(summary.tsServerResult.newSpawnResult!.stdout); group = groupedNewErrors; } else if (summary.tsServerResult.oldServerFailed) { // Group old errors const { oldSpawnResult } = summary.tsServerResult; error = oldSpawnResult?.stdout - ? parseServerHarnessOutput(oldSpawnResult.stdout) + ? (isLsp ? parseLspHarnessOutput(oldSpawnResult.stdout) : parseServerHarnessOutput(oldSpawnResult.stdout)) : `Timed out after ${executionTimeout} ms`; group = groupedOldErrors; @@ -471,7 +475,11 @@ function groupErrors(summaries: Summary[]) { continue; } - const key = typeof error === "string" ? getHash([error]) : getHashForStack(error.message); + const key = typeof error === "string" + ? getHash([error]) + : summary.entrypoint === "lsp" + ? getHashForGoStack(error.message) + : getHashForStack(error.message); const value = group.get(key) ?? []; value.push(summary); group.set(key, value); @@ -486,14 +494,23 @@ function getErrorMessage(output: string): string { return typeof error === "string" ? error : getErrorMessageFromStack(error.message); } +function getErrorMessageForEntrypoint(output: string, entrypoint: TsEntrypoint): string { + return entrypoint === "lsp" ? getLspErrorMessage(output) : getErrorMessage(output); +} + +function prettyPrintForEntrypoint(output: string, filter: boolean, entrypoint: TsEntrypoint): string { + return entrypoint === "lsp" ? prettyPrintLspHarnessOutput(output, filter) : prettyPrintServerHarnessOutput(output, filter); +} + function createOldErrorSummary(summaries: Summary[]): string { const { oldSpawnResult } = summaries[0].tsServerResult; + const entrypoint = summaries[0].entrypoint; const oldServerError = oldSpawnResult?.stdout - ? prettyPrintServerHarnessOutput(oldSpawnResult.stdout, /*filter*/ true) + ? prettyPrintForEntrypoint(oldSpawnResult.stdout, /*filter*/ true, entrypoint) : `Timed out after ${executionTimeout} ms`; - const errorMessage = oldSpawnResult?.stdout ? getErrorMessage(oldSpawnResult.stdout) : oldServerError; + const errorMessage = oldSpawnResult?.stdout ? getErrorMessageForEntrypoint(oldSpawnResult.stdout, entrypoint) : oldServerError; let text = `
@@ -575,10 +592,13 @@ npx tsreplay ./${summary.repo.name} ./${summary.replayScriptName} { - let text = `

${getErrorMessage(summaries[0].tsServerResult.newSpawnResult.stdout)}

+ const entrypoint = summaries[0].entrypoint; + const stdout = summaries[0].tsServerResult.newSpawnResult.stdout; + + let text = `

${getErrorMessageForEntrypoint(stdout, entrypoint)}

\`\`\` -${prettyPrintServerHarnessOutput(summaries[0].tsServerResult.newSpawnResult.stdout, /*filter*/ true)} +${prettyPrintForEntrypoint(stdout, /*filter*/ true, entrypoint)} \`\`\`

Affected repos

`; @@ -976,6 +996,7 @@ export async function mainAsync(params: GitParams | UserParams): Promise { replayScriptArtifactPath, replayScriptName: path.basename(replayScriptArtifactPath), resultDirName: params.resultDirName, + entrypoint: params.entrypoint, commit }); } @@ -1163,6 +1184,58 @@ function filterToTsserverLines(stackLines: string): string { return tsserverLines.trimEnd(); } +// LSP harness output helpers + +export interface LspHarnessOutput { + method: string; + message: string; +} + +function parseLspHarnessOutput(output: string): LspHarnessOutput | string { + try { + const parsed = JSON.parse(output); + if (parsed.method !== undefined && parsed.message !== undefined) { + return parsed as LspHarnessOutput; + } + return output; + } + catch { + return output; + } +} + +function prettyPrintLspHarnessOutput(error: string, filter: boolean): string { + const errorObj = parseLspHarnessOutput(error); + if (typeof errorObj === "string") { + return errorObj; + } + + if (errorObj.message) { + return `${errorObj.method}\n${filter ? filterToGoLines(errorObj.message) : errorObj.message}`; + } + + return JSON.stringify(errorObj, undefined, 2); +} + +function getLspErrorMessage(output: string): string { + const error = parseLspHarnessOutput(output); + if (typeof error === "string") return error; + + // The first line of the message is typically "panic handling request : " + const firstLine = error.message.split(/\r?\n/)[0]; + return firstLine; +} + +function filterToGoLines(stackLines: string): string { + const goRegex = /^.*typescript-go.*$/mg; + let goLines = ""; + let match; + while (match = goRegex.exec(stackLines)) { + goLines += match[0] + "\n"; + } + return goLines.trimEnd() || stackLines; +} + function insetLines(text: string): string { return text.trimEnd().replace(/(^|\n)/g, "$1> "); } diff --git a/src/postGithubIssue.ts b/src/postGithubIssue.ts index 9e25f77..af6fc3b 100644 --- a/src/postGithubIssue.ts +++ b/src/postGithubIssue.ts @@ -45,11 +45,14 @@ for (const path of metadataFilePaths) { } -const title = `${entrypoint === "tsserver" ? `[ServerErrors][${language}]` : `[NewErrors]`} ${newTscResolvedVersion} vs ${oldTscResolvedVersion}`; +const title = `${entrypoint === "tsserver" || entrypoint === "lsp" ? `[ServerErrors][${language}]` : `[NewErrors]`} ${newTscResolvedVersion} vs ${oldTscResolvedVersion}`; const description = entrypoint === "tsserver" ? `The following errors were reported by ${newTscResolvedVersion} vs ${oldTscResolvedVersion}` - : `The following errors were reported by ${newTscResolvedVersion}, but not by ${oldTscResolvedVersion}`; + : entrypoint == "lsp" + ? `The following errors were reported by ${newTscResolvedVersion}` + : `The following errors were reported by ${newTscResolvedVersion}, but not by ${oldTscResolvedVersion}`; +// TODO: modify for tsgo let header = `${description} [Pipeline that generated this bug](https://typescript.visualstudio.com/TypeScript/_build?definitionId=48) [Logs for the pipeline run](${logUri}) diff --git a/src/utils/exerciseLspServer.ts b/src/utils/exerciseLspServer.ts index 2f0d0a7..8ff1e2b 100644 --- a/src/utils/exerciseLspServer.ts +++ b/src/utils/exerciseLspServer.ts @@ -7,7 +7,7 @@ import * as glob from "glob"; import { performance } from "perf_hooks"; import randomSeed from "random-seed"; import * as protocol from "vscode-languageserver-protocol"; -import { EXIT_BAD_ARGS, EXIT_UNHANDLED_EXCEPTION, EXIT_SERVER_ERROR } from "./exerciseServerConstants"; +import { EXIT_BAD_ARGS, EXIT_SERVER_COMMUNICATION_ERROR, EXIT_SERVER_CRASH, EXIT_UNHANDLED_EXCEPTION, EXIT_SERVER_ERROR } from "./exerciseServerConstants"; import { pathToFileURL } from "url"; const testDirPlaceholder = "@PROJECT_ROOT@"; @@ -99,13 +99,37 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r }, { traceOutput: diagnosticOutput }); server.handleAnyRequest(async (...args) => { - console.log("Server sent request:", ...args); + console.error("Server sent request:", ...args); }); - server.handleAnyNotification(async (...args) => { - console.log("Server sent notification:", ...args); + // Capture the last error-level log message from the server (e.g. Go panic stack traces) + let lastErrorLogMessage = ""; + + server.handleAnyNotification(async (...args: any[]) => { + console.error("Server sent notification:", ...args); + const [method, params] = args; + if (method === "window/logMessage" && params?.type === 1) { + lastErrorLogMessage = params.message; + } }); + let exitExpected = false; + server.onError(async ([error, message, count]) => { + console.error(`Server connection error: ${error} ${message} ${count}`); + exitExpected = true; + await server.kill(); + process.exit(EXIT_SERVER_COMMUNICATION_ERROR); + }); + + server.onClose((e) => { + if (!exitExpected) { + const errorMessage = lastErrorLogMessage || `Server connection closed prematurely: ${e}`; + console.log(JSON.stringify({ method: "unknown", message: errorMessage })); + console.error("Server connection closed prematurely:", e); + process.exit(EXIT_SERVER_CRASH); + } + }); + let documentVersion = 0; const testDirUrl = filePathToUri(testDir); @@ -409,14 +433,16 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r } } - console.log("\nShutting down server"); + console.error("\nShutting down server"); + exitExpected = true; // Send shutdown request and exit notification void request(protocol.ShutdownRequest.method, undefined); void notify("exit", undefined); } catch (e) { console.error("Killing server after unhandled exception"); console.error(e); - + + exitExpected = true; await server.kill(); process.exit(EXIT_UNHANDLED_EXCEPTION); } @@ -447,14 +473,14 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r requestTimes[method] = (requestTimes[method] ?? 0) + (end - start); requestCounts[method] = (requestCounts[method] ?? 0) + 1; - const errorMessage = e.message ?? "Unknown error"; + const errorMessage = lastErrorLogMessage || e.message || "Unknown error"; if (diagnosticOutput) { console.error(`Request failed:\n${JSON.stringify(replayEntry, undefined, 2)}\n${e}`); } else { console.error(errorMessage); } - console.error(JSON.stringify({ error: e.message })); + console.log(JSON.stringify({ method, message: errorMessage })); void server.kill(); process.exit(EXIT_SERVER_ERROR); @@ -474,14 +500,14 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r await server.sendNotification(method, params); } catch (e: any) { - const errorMessage = e.message ?? "Unknown error"; + const errorMessage = lastErrorLogMessage || e.message || "Unknown error"; if (diagnosticOutput) { console.error(`Notification failed:\n${JSON.stringify(replayEntry, undefined, 2)}\n${e}`); } else { console.error(errorMessage); } - console.log(JSON.stringify({ error: e.message })); + console.log(JSON.stringify({ method, message: errorMessage })); await server.kill(); process.exit(EXIT_SERVER_ERROR); diff --git a/src/utils/exerciseServer.ts b/src/utils/exerciseServer.ts index 5cacdd0..dbf0f73 100644 --- a/src/utils/exerciseServer.ts +++ b/src/utils/exerciseServer.ts @@ -36,8 +36,8 @@ exerciseServer(testDir, replayScriptPath, tsserverPath).catch(e => { process.exit(EXIT_UNHANDLED_EXCEPTION); }); +const requestTimes: Record = {}; export async function exerciseServer(testDir: string, replayScriptPath: string, tsserverPath: string): Promise { - const requestTimes: Record = {}; const requestCounts: Record = {}; const start = performance.now(); diff --git a/src/utils/hashStackTrace.ts b/src/utils/hashStackTrace.ts index c2deb6a..a618725 100644 --- a/src/utils/hashStackTrace.ts +++ b/src/utils/hashStackTrace.ts @@ -11,6 +11,31 @@ export function getHashForStack(stack: string): string { return getHash(stackLines); } +/** + * Produces a stable hash for Go stack traces (e.g. from the LSP server). + * Strips volatile parts that change between runs: + * - The first line (panic message with specific runtime values) + * - Memory addresses like 0xc004bfed20 + * - Goroutine IDs like "goroutine 554" + * Keeps function names and source file locations for meaningful grouping. + */ +export function getHashForGoStack(stack: string): string { + const stackLines = stack.split(/\r?\n/); + // Skip the first line (panic message with variable runtime values) + const normalized = stackLines.slice(1).map(line => { + // Strip goroutine IDs: "goroutine 554 [running]:" -> "goroutine [running]:" + line = line.replace(/goroutine \d+/, "goroutine "); + // Strip hex memory addresses: 0xc004bfed20, 0x441ea5? + line = line.replace(/0x[0-9a-fA-F]+\??/g, "0x?"); + // Strip Go function argument lists (all hex pointers in parens) + line = line.replace(/\((?:0x\?[,} ]*)+\)/g, "(...)"); + // Strip +0x... offsets at end of lines + line = line.replace(/\+0x\?$/g, ""); + return line; + }); + return getHash(normalized); +} + export function getErrorMessageFromStack(stack: string): string { const stackLines = stack.split(/\r?\n/, 2); diff --git a/src/utils/lspHarness.ts b/src/utils/lspHarness.ts index 793232f..9dca8db 100644 --- a/src/utils/lspHarness.ts +++ b/src/utils/lspHarness.ts @@ -1,5 +1,5 @@ import * as cp from "child_process"; -import * as rpc from "vscode-jsonrpc/node.js"; +import * as rpc from "vscode-jsonrpc/node"; import * as protocol from "vscode-languageserver-protocol"; export interface ServerOptions { @@ -19,6 +19,9 @@ export interface LanguageServer { handleAnyNotification: (handler: (...args: any[]) => Promise) => void; kill: () => Promise; + + onError: rpc.Event<[Error, rpc.Message | undefined, number | undefined]>; + onClose: rpc.Event; } export function startServer(serverPath: string, options: ServerOptions = {}, otherOptions?: { traceOutput?: boolean; }): LanguageServer { @@ -33,7 +36,7 @@ export function startServer(serverPath: string, options: ServerOptions = {}, oth }); const connection = rpc.createMessageConnection( - new rpc.StreamMessageReader(serverProc.stdout!), + new rpc.StreamMessageReader(serverProc.stdout), new rpc.StreamMessageWriter(serverProc.stdin!) ); @@ -48,6 +51,8 @@ export function startServer(serverPath: string, options: ServerOptions = {}, oth handleNotification, handleAnyNotification, kill, + onError: connection.onError, + onClose: connection.onClose, }; function sendRequest(method: K, params: RequestToParams[K]): Promise { From a34194e884b3c86827b22fd20d22dd33cfa0084c Mon Sep 17 00:00:00 2001 From: Gabriela Araujo Britto Date: Tue, 10 Feb 2026 19:32:02 +0000 Subject: [PATCH 07/11] update repo --- src/postGithubComments.ts | 6 +++--- src/postGithubIssue.ts | 2 +- src/utils/gitUtils.ts | 26 ++++++++++++++++++++------ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/postGithubComments.ts b/src/postGithubComments.ts index 42939fa..6dface4 100644 --- a/src/postGithubComments.ts +++ b/src/postGithubComments.ts @@ -1,6 +1,6 @@ import fs = require("fs"); import path = require("path"); -import { artifactFolderUrlPlaceholder, getArtifactsApiUrlPlaceholder, Metadata, metadataFileName, RepoStatus, resultFileNameSuffix } from "./main"; +import { artifactFolderUrlPlaceholder, getArtifactsApiUrlPlaceholder, Metadata, metadataFileName, RepoStatus, resultFileNameSuffix,TsEntrypoint } from "./main"; import git = require("./utils/gitUtils"); import pu = require("./utils/packageUtils"); import { asMarkdownInlineCode } from "./utils/markdownUtils"; @@ -82,7 +82,7 @@ let header = `@${userToTag} Here are the results of running the ${suiteDescripti ${summary.join("\n")}`; if (!outputs.length) { - git.createComment(+prNumber, +commentNumber, distinctId, postResult, [header], somethingChanged); + git.createComment(entrypoint as TsEntrypoint, +prNumber, +commentNumber, distinctId, postResult, [header], somethingChanged); } else { const oldErrorHeader = `

:warning: Old server errors :warning:

`; @@ -137,5 +137,5 @@ else { console.log(`Chunk of size ${chunk.length}`); } - git.createComment(+prNumber, +commentNumber, distinctId, postResult, bodyChunks, somethingChanged); + git.createComment(entrypoint as TsEntrypoint, +prNumber, +commentNumber, distinctId, postResult, bodyChunks, somethingChanged); } diff --git a/src/postGithubIssue.ts b/src/postGithubIssue.ts index af6fc3b..4021a16 100644 --- a/src/postGithubIssue.ts +++ b/src/postGithubIssue.ts @@ -105,4 +105,4 @@ if (entrypoint !== "tsserver") { const bodyChunks = [header, ...outputs]; -git.createIssue(postResult, title, bodyChunks, /*sawNewErrors*/ !!outputs.length); \ No newline at end of file +git.createIssue(entrypoint, postResult, title, bodyChunks, /*sawNewErrors*/ !!outputs.length); \ No newline at end of file diff --git a/src/utils/gitUtils.ts b/src/utils/gitUtils.ts index 02ad4b1..f307e8c 100644 --- a/src/utils/gitUtils.ts +++ b/src/utils/gitUtils.ts @@ -6,6 +6,7 @@ import path = require("path"); // The bundled types don't work with CJS imports import { simpleGit as git } from "simple-git"; +import { TsEntrypoint } from "../main"; export interface Repo { name: string; @@ -15,10 +16,21 @@ export interface Repo { branch?: string; } -const repoProperties = { - owner: "microsoft", - repo: "typescript", -}; +function getRepoProperties(entrypoint: TsEntrypoint) { + switch (entrypoint) { + case "tsserver": + case "tsc": + return { + owner: "microsoft", + repo: "typescript", + }; + case "lsp": + return { + owner: "microsoft", + repo: "typescript-go", + }; + } +} export async function getPopularRepos(language = "TypeScript", count = 100, repoStartIndex = 0, skipRepos?: string[], cachePath?: string): Promise { const cacheEncoding = { encoding: "utf-8" } as const; @@ -105,7 +117,8 @@ type Result = { export type GitResult = Result & { kind: 'git', title: string } export type UserResult = Result & { kind: 'user', issue_number: number } -export async function createIssue(postResult: boolean, title: string, bodyChunks: readonly string[], sawNewErrors: boolean): Promise { +export async function createIssue(entrypoint: TsEntrypoint, postResult: boolean, title: string, bodyChunks: readonly string[], sawNewErrors: boolean): Promise { + const repoProperties = getRepoProperties(entrypoint); const issue = { ...repoProperties, title, @@ -158,7 +171,8 @@ export async function createIssue(postResult: boolean, title: string, bodyChunks } } -export async function createComment(prNumber: number, statusComment: number, distinctId: string, postResult: boolean, bodyChunks: readonly string[], somethingChanged: boolean): Promise { +export async function createComment(entrypoint: TsEntrypoint, prNumber: number, statusComment: number, distinctId: string, postResult: boolean, bodyChunks: readonly string[], somethingChanged: boolean): Promise { + const repoProperties = getRepoProperties(entrypoint); const newComments = bodyChunks.map(body => ({ ...repoProperties, issue_number: prNumber, From 641578ef6c87027d7094b5bf1ac01b4bd3334265 Mon Sep 17 00:00:00 2001 From: Gabriela Araujo Britto Date: Tue, 10 Feb 2026 19:46:14 +0000 Subject: [PATCH 08/11] fix bad edits --- replay.mts | 82 ---------------------------------- src/utils/exerciseLspServer.ts | 1 - src/utils/exerciseServer.ts | 2 +- 3 files changed, 1 insertion(+), 84 deletions(-) delete mode 100644 replay.mts diff --git a/replay.mts b/replay.mts deleted file mode 100644 index 1327713..0000000 --- a/replay.mts +++ /dev/null @@ -1,82 +0,0 @@ -import * as yadda from "./src/utils/lspHarness.mts"; -import { readFileSync } from "fs"; -import { pathToFileURL } from "url"; - -const [, , scriptPath, testDir] = process.argv; - -const testUrl = testDir ? - pathToFileURL(testDir).toString() : - "file:///workspaces/typescript-error-deltas"; - -const server = yadda.startServer("./tsgo/tsgo-noracedetection", { - args: ["--lsp", "--stdio"], -}, { - traceOutput: true, -}); - -server.handleAnyNotification(async (...args) => { - console.log("Notification received:", ...args); -}); - -server.handleAnyRequest(async (...args) => { - console.log("Request received:", ...args); - return {}; -}); - -const script = readFileSync(scriptPath, "utf-8"); -const lines = script.split(/\r?\n/g); -let rootDirPlaceholder = "@PROJECT_ROOT@"; -let initialize: object | undefined; -let initialized: object | undefined; -if (true) { - for (let line of lines) { - line = line.trim(); - if (line.length === 0) continue; - - let obj = JSON.parse(line) - if (obj.rootDirPlaceholder) { - rootDirPlaceholder = obj.rootDirPlaceholder; - continue; - } - - obj = JSON.parse(line.replaceAll(rootDirPlaceholder, testUrl)); - - console.log(obj) - try { - if (obj.kind === "request") { - if (obj.method === "initialize") { - if (initialize) { - continue; - } - initialize = obj.params; - } - - const responsePromise = server.sendRequest(obj.method, obj.params);; - if (obj.method !== "shutdown") { - await responsePromise; - continue; - } - } - else if (obj.kind === "notification") { - if (obj.method === "initialized") { - if (initialized) { - continue; - } - initialized = obj.params; - } - await server.sendNotification(obj.method, obj.params); - } - else { - throw new Error(`Unknown replay entry kind: ${JSON.stringify(obj)}`); - } - } - catch (e) { - console.log(e); - } - - // Slow down requests - sometimes helpful for ATA races. - // await new Promise(resolve => setTimeout(resolve, 2000)); - } -} - -server.kill(); diff --git a/src/utils/exerciseLspServer.ts b/src/utils/exerciseLspServer.ts index 8ff1e2b..12877d4 100644 --- a/src/utils/exerciseLspServer.ts +++ b/src/utils/exerciseLspServer.ts @@ -1,4 +1,3 @@ -#!/usr/bin/env node import * as lsp from "./lspHarness"; import fs from "fs"; import process from "process"; diff --git a/src/utils/exerciseServer.ts b/src/utils/exerciseServer.ts index dbf0f73..5cacdd0 100644 --- a/src/utils/exerciseServer.ts +++ b/src/utils/exerciseServer.ts @@ -36,8 +36,8 @@ exerciseServer(testDir, replayScriptPath, tsserverPath).catch(e => { process.exit(EXIT_UNHANDLED_EXCEPTION); }); -const requestTimes: Record = {}; export async function exerciseServer(testDir: string, replayScriptPath: string, tsserverPath: string): Promise { + const requestTimes: Record = {}; const requestCounts: Record = {}; const start = performance.now(); From 337ea073e85abad818a3ce456257813a5b4886e7 Mon Sep 17 00:00:00 2001 From: Gabriela Araujo Britto Date: Tue, 10 Feb 2026 22:56:17 +0000 Subject: [PATCH 09/11] code review: uncomment, normalize stack like in tsgo --- src/checkGithubRepos.ts | 2 +- src/utils/exerciseLspServer.ts | 79 +++++++++++++++++----------------- src/utils/hashStackTrace.ts | 20 ++++++--- 3 files changed, 54 insertions(+), 47 deletions(-) diff --git a/src/checkGithubRepos.ts b/src/checkGithubRepos.ts index cd87bda..acd8ad6 100644 --- a/src/checkGithubRepos.ts +++ b/src/checkGithubRepos.ts @@ -4,7 +4,7 @@ import { mainAsync, reportError, TsEntrypoint } from "./main"; const { argv } = process; if (argv.length < 11) { - console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} `); + console.error(`Usage: ${path.basename(argv[0])} ${path.basename(argv[1])} ?`); process.exit(-1); } diff --git a/src/utils/exerciseLspServer.ts b/src/utils/exerciseLspServer.ts index 12877d4..01d717c 100644 --- a/src/utils/exerciseLspServer.ts +++ b/src/utils/exerciseLspServer.ts @@ -274,26 +274,25 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r let prev = ""; // Organize imports (source.organizeImports code action) - // await request("textDocument/codeAction", { - // textDocument: { uri: openFileUri }, - // range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } }, - // context: { - // diagnostics: [], - // only: [protocol.CodeActionKind.SourceOrganizeImports], - // }, - // }, 0.5); + await request("textDocument/codeAction", { + textDocument: { uri: openFileUri }, + range: { start: { line: 0, character: 0 }, end: { line: 0, character: 0 } }, + context: { + diagnostics: [], + only: [protocol.CodeActionKind.SourceOrganizeImports], + }, + }, 0.5); if (openFileContents.length < 1e6) { // Folding ranges (equivalent to getOutliningSpans) - // TODO: folding ranges are broken for JS files - // await request("textDocument/foldingRange", { - // textDocument: { uri: openFileUri }, - // }, +(languageId.startsWith("typescript"))); - - // // Document symbols (equivalent to navtree/navbar) - // await request("textDocument/documentSymbol", { - // textDocument: { uri: openFileUri }, - // }); + await request("textDocument/foldingRange", { + textDocument: { uri: openFileUri }, + }); + + // Document symbols (equivalent to navtree/navbar) + await request("textDocument/documentSymbol", { + textDocument: { uri: openFileUri }, + }); } // Workspace symbol search (equivalent to navto) @@ -310,21 +309,21 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r } } - // // Diagnostics (equivalent to geterr) - // const diagnosticsPromise = request("textDocument/diagnostic", { - // textDocument: { uri: openFileUri }, - // }); + // Diagnostics (equivalent to geterr) + const diagnosticsPromise = request("textDocument/diagnostic", { + textDocument: { uri: openFileUri }, + }); - // const codeLensesPromise = request("textDocument/codeLens", { - // textDocument: { uri: openFileUri }, - // }); + const codeLensesPromise = request("textDocument/codeLens", { + textDocument: { uri: openFileUri }, + }); - // const inlayHintsPromise = request("textDocument/inlayHint", { - // textDocument: { uri: openFileUri }, - // range: { start: { line: 0, character: 0 }, end: { line: 0, character: openFileContents.length } }, - // }); + const inlayHintsPromise = request("textDocument/inlayHint", { + textDocument: { uri: openFileUri }, + range: { start: { line: 0, character: 0 }, end: { line: 0, character: openFileContents.length } }, + }); - // await Promise.all([diagnosticsPromise, codeLensesPromise, inlayHintsPromise]); + await Promise.all([diagnosticsPromise, codeLensesPromise, inlayHintsPromise]); for (let i = 0; i < openFileContents.length; i++) { const curr = openFileContents[i]; @@ -336,17 +335,17 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r // Note that this only catches Latin letters - we'll test within tokens of non-Latin characters if (!(/\w/.test(prev) && /\w/.test(curr)) && !(/[ \t]/.test(prev) && /[ \t]/.test(curr))) { // Definition (equivalent to definitionAndBoundSpan) - // await request("textDocument/definition", { - // textDocument: { uri: openFileUri }, - // position: { line, character }, - // }, isAt ? 0.5 : 0.001); - - // // References - // await request("textDocument/references", { - // textDocument: { uri: openFileUri }, - // position: { line, character }, - // context: { includeDeclaration: true }, - // }, isAt ? 0.5 : 0.00005); + await request("textDocument/definition", { + textDocument: { uri: openFileUri }, + position: { line, character }, + }, isAt ? 0.5 : 0.001); + + // References + await request("textDocument/references", { + textDocument: { uri: openFileUri }, + position: { line, character }, + context: { includeDeclaration: true }, + }, isAt ? 0.5 : 0.00005); // TODO: // - https://github.com/microsoft/typescript-go/issues/2253 diff --git a/src/utils/hashStackTrace.ts b/src/utils/hashStackTrace.ts index a618725..7ca6363 100644 --- a/src/utils/hashStackTrace.ts +++ b/src/utils/hashStackTrace.ts @@ -23,16 +23,24 @@ export function getHashForGoStack(stack: string): string { const stackLines = stack.split(/\r?\n/); // Skip the first line (panic message with variable runtime values) const normalized = stackLines.slice(1).map(line => { + line = line.trim(); + let ignoreIdx; + if ((ignoreIdx = line.indexOf(" +0x")) !== -1) { + // Ignore trailing offsets e.g. " +0x58" in "github.com/microsoft/typescript-go/internal/lsp/server.go:872 +0x58" + line = line.slice(0, ignoreIdx); + } else if ((ignoreIdx = line.lastIndexOf(" in goroutine ")) !== -1) { + // e.g. "created by github.com/microsoft/typescript-go/internal/lsp.(*Server).dispatchLoop in goroutine 10" -> + // "created by github.com/microsoft/typescript-go/internal/lsp.(*Server).dispatchLoop" + line = line.slice(0, ignoreIdx); + } // Strip goroutine IDs: "goroutine 554 [running]:" -> "goroutine [running]:" line = line.replace(/goroutine \d+/, "goroutine "); - // Strip hex memory addresses: 0xc004bfed20, 0x441ea5? - line = line.replace(/0x[0-9a-fA-F]+\??/g, "0x?"); - // Strip Go function argument lists (all hex pointers in parens) - line = line.replace(/\((?:0x\?[,} ]*)+\)/g, "(...)"); - // Strip +0x... offsets at end of lines - line = line.replace(/\+0x\?$/g, ""); + // Strip function arguments + line = line.replace(/^(.+)\(.+$/g, "$1()"); return line; }); + console.log("Normalized Go stack trace:"); + console.log(normalized.join("\n")); return getHash(normalized); } From 71cb10e35feb7688a9b4238f7f15efe87b3e6803 Mon Sep 17 00:00:00 2001 From: Gabriela Araujo Britto Date: Tue, 10 Feb 2026 23:41:01 +0000 Subject: [PATCH 10/11] WIP: pipeline config --- azure-pipelines-gitTests-lsp-ts.yml | 135 ++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 azure-pipelines-gitTests-lsp-ts.yml diff --git a/azure-pipelines-gitTests-lsp-ts.yml b/azure-pipelines-gitTests-lsp-ts.yml new file mode 100644 index 0000000..af9026c --- /dev/null +++ b/azure-pipelines-gitTests-lsp-ts.yml @@ -0,0 +1,135 @@ +schedules: + - cron: "0 19 * * Sun" # time is in UTC + displayName: Sunday overnight run + always: true + branches: + include: + - main + +pr: none +trigger: none + +parameters: + - name: POST_RESULT + displayName: Post GitHub issue with results + type: boolean + default: true + - name: DIAGNOSTIC_OUTPUT + displayName: Log diagnostic data + type: boolean + default: false + - name: REPO_COUNT + displayName: Repo Count + type: number + default: 300 + - name: REPO_START_INDEX + displayName: Repo Start Index + type: number + default: 0 + - name: NEW_VERSION + displayName: Candidate TypeScript package version + type: string + default: next + - name: MACHINE_COUNT + displayName: Machine Count + type: number + default: 8 + - name: PRNG_SEED + displayName: Pseudo-random number generator seed + type: string + default: 'n/a' + +pool: + name: TypeScript-1ES-Large + demands: + - ImageOverride -equals azure-linux-3 + +jobs: +- job: ListRepos + pool: + name: TypeScript-1ES-Deploys + demands: + - ImageOverride -equals azure-linux-3 + steps: + - task: AzureKeyVault@2 + inputs: + azureSubscription: 'TypeScript Public CI' + KeyVaultName: 'jststeam-passwords' + SecretsFilter: 'typescript-bot-github-PAT-error-deltas' + displayName: Get secrets + retryCountOnTaskFailure: 3 + - task: UseNode@1 + inputs: + version: '22.x' + displayName: 'Install Node.js' + - script: | + npm ci + npm run build + mkdir artifacts + node dist/listTopRepos TypeScript ${{ parameters.REPO_COUNT }} ${{ parameters.REPO_START_INDEX }} artifacts/repos.json + displayName: 'List top TS repos' + env: + GITHUB_PAT: $(typescript-bot-github-PAT-error-deltas) + - publish: artifacts + artifact: RepoList +- job: DetectNewErrors + dependsOn: ListRepos + continueOnError: true + timeoutInMinutes: 360 + strategy: + parallel: ${{ parameters.MACHINE_COUNT }} + steps: + - download: current + artifact: RepoList + - task: UseNode@1 + inputs: + version: '22.x' + displayName: 'Install Node.js' + - script: | + df -h + df -h -i + displayName: Debugging + continueOnError: true + - script: | + npm ci + npm run build + npm install -g yarn + npm install -g pnpm + npm install -g corepack@latest + export COREPACK_ENABLE_AUTO_PIN=0 + export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 + export COREPACK_ENABLE_STRICT=0 + corepack enable + corepack enable npm + mkdir 'RepoResults$(System.JobPositionInPhase)' + node dist/checkGithubRepos lsp 0 ${{ parameters.NEW_VERSION }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }} ${{ parameters.PRNG_SEED }} + displayName: 'Run TypeScript on repos' + continueOnError: true + - publish: 'RepoResults$(System.JobPositionInPhase)' + artifact: 'RepoResults$(System.JobPositionInPhase)' +- job: ReportNewErrors + dependsOn: DetectNewErrors + pool: + name: TypeScript-1ES-Deploys + demands: + - ImageOverride -equals azure-linux-3 + steps: + - task: AzureKeyVault@2 + inputs: + azureSubscription: 'TypeScript Public CI' + KeyVaultName: 'jststeam-passwords' + SecretsFilter: 'typescript-bot-github-PAT-error-deltas' + displayName: Get secrets + retryCountOnTaskFailure: 3 + - download: current + - task: UseNode@1 + inputs: + version: '22.x' + displayName: 'Install Node.js' + - script: | + npm ci + npm run build + node dist/postGithubIssue lsp TypeScript ${{ parameters.REPO_COUNT }} ${{ parameters.REPO_START_INDEX }} '$(Pipeline.Workspace)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)&view=artifacts&type=publishedArtifacts' ${{ parameters.POST_RESULT }} '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_apis/build/builds/$(Build.BuildId)/artifacts' + displayName: 'Create issue from new errors' + env: + GITHUB_PAT: $(typescript-bot-github-PAT-error-deltas) From 9835b2a7a64d95f886a34ff4ec8975298e9252c7 Mon Sep 17 00:00:00 2001 From: Gabriela Araujo Britto Date: Wed, 11 Feb 2026 20:54:15 +0000 Subject: [PATCH 11/11] refactor --- azure-pipelines-gitTests-lsp-ts.yml | 140 +++----------------------- azure-pipelines-gitTests-template.yml | 6 +- src/postGithubComments.ts | 2 +- src/utils/exerciseLspServer.ts | 6 +- 4 files changed, 19 insertions(+), 135 deletions(-) diff --git a/azure-pipelines-gitTests-lsp-ts.yml b/azure-pipelines-gitTests-lsp-ts.yml index af9026c..5b6f3c4 100644 --- a/azure-pipelines-gitTests-lsp-ts.yml +++ b/azure-pipelines-gitTests-lsp-ts.yml @@ -1,135 +1,23 @@ -schedules: - - cron: "0 19 * * Sun" # time is in UTC - displayName: Sunday overnight run - always: true - branches: - include: - - main +# schedules: +# - cron: "0 19 * * Sun" # time is in UTC +# displayName: Sunday overnight run +# always: true +# branches: +# include: +# - main pr: none trigger: none -parameters: - - name: POST_RESULT - displayName: Post GitHub issue with results - type: boolean - default: true - - name: DIAGNOSTIC_OUTPUT - displayName: Log diagnostic data - type: boolean - default: false - - name: REPO_COUNT - displayName: Repo Count - type: number - default: 300 - - name: REPO_START_INDEX - displayName: Repo Start Index - type: number - default: 0 - - name: NEW_VERSION - displayName: Candidate TypeScript package version - type: string - default: next - - name: MACHINE_COUNT - displayName: Machine Count - type: number - default: 8 - - name: PRNG_SEED - displayName: Pseudo-random number generator seed - type: string - default: 'n/a' - pool: name: TypeScript-1ES-Large demands: - ImageOverride -equals azure-linux-3 -jobs: -- job: ListRepos - pool: - name: TypeScript-1ES-Deploys - demands: - - ImageOverride -equals azure-linux-3 - steps: - - task: AzureKeyVault@2 - inputs: - azureSubscription: 'TypeScript Public CI' - KeyVaultName: 'jststeam-passwords' - SecretsFilter: 'typescript-bot-github-PAT-error-deltas' - displayName: Get secrets - retryCountOnTaskFailure: 3 - - task: UseNode@1 - inputs: - version: '22.x' - displayName: 'Install Node.js' - - script: | - npm ci - npm run build - mkdir artifacts - node dist/listTopRepos TypeScript ${{ parameters.REPO_COUNT }} ${{ parameters.REPO_START_INDEX }} artifacts/repos.json - displayName: 'List top TS repos' - env: - GITHUB_PAT: $(typescript-bot-github-PAT-error-deltas) - - publish: artifacts - artifact: RepoList -- job: DetectNewErrors - dependsOn: ListRepos - continueOnError: true - timeoutInMinutes: 360 - strategy: - parallel: ${{ parameters.MACHINE_COUNT }} - steps: - - download: current - artifact: RepoList - - task: UseNode@1 - inputs: - version: '22.x' - displayName: 'Install Node.js' - - script: | - df -h - df -h -i - displayName: Debugging - continueOnError: true - - script: | - npm ci - npm run build - npm install -g yarn - npm install -g pnpm - npm install -g corepack@latest - export COREPACK_ENABLE_AUTO_PIN=0 - export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 - export COREPACK_ENABLE_STRICT=0 - corepack enable - corepack enable npm - mkdir 'RepoResults$(System.JobPositionInPhase)' - node dist/checkGithubRepos lsp 0 ${{ parameters.NEW_VERSION }} '$(Pipeline.Workspace)/RepoList/repos.json' $(System.TotalJobsInPhase) $(System.JobPositionInPhase) 'RepoResults$(System.JobPositionInPhase)' ${{ parameters.DIAGNOSTIC_OUTPUT }} ${{ parameters.PRNG_SEED }} - displayName: 'Run TypeScript on repos' - continueOnError: true - - publish: 'RepoResults$(System.JobPositionInPhase)' - artifact: 'RepoResults$(System.JobPositionInPhase)' -- job: ReportNewErrors - dependsOn: DetectNewErrors - pool: - name: TypeScript-1ES-Deploys - demands: - - ImageOverride -equals azure-linux-3 - steps: - - task: AzureKeyVault@2 - inputs: - azureSubscription: 'TypeScript Public CI' - KeyVaultName: 'jststeam-passwords' - SecretsFilter: 'typescript-bot-github-PAT-error-deltas' - displayName: Get secrets - retryCountOnTaskFailure: 3 - - download: current - - task: UseNode@1 - inputs: - version: '22.x' - displayName: 'Install Node.js' - - script: | - npm ci - npm run build - node dist/postGithubIssue lsp TypeScript ${{ parameters.REPO_COUNT }} ${{ parameters.REPO_START_INDEX }} '$(Pipeline.Workspace)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)' '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)&view=artifacts&type=publishedArtifacts' ${{ parameters.POST_RESULT }} '$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_apis/build/builds/$(Build.BuildId)/artifacts' - displayName: 'Create issue from new errors' - env: - GITHUB_PAT: $(typescript-bot-github-PAT-error-deltas) +extends: + template: azure-pipelines-gitTests-template.yml + parameters: + ENTRYPOINT: lsp + OLD_VERSION: '0' + NEW_VERSION: latest + LANGUAGE: TypeScript diff --git a/azure-pipelines-gitTests-template.yml b/azure-pipelines-gitTests-template.yml index f7c7124..31e5608 100644 --- a/azure-pipelines-gitTests-template.yml +++ b/azure-pipelines-gitTests-template.yml @@ -54,7 +54,7 @@ jobs: retryCountOnTaskFailure: 3 - task: UseNode@1 inputs: - version: '20.x' + version: '22.x' displayName: 'Install Node.js' - script: | npm ci @@ -77,7 +77,7 @@ jobs: artifact: RepoList - task: UseNode@1 inputs: - version: '20.x' + version: '22.x' displayName: 'Install Node.js' - script: | df -h @@ -118,7 +118,7 @@ jobs: - download: current - task: UseNode@1 inputs: - version: '20.x' + version: '22.x' displayName: 'Install Node.js' - script: | npm ci diff --git a/src/postGithubComments.ts b/src/postGithubComments.ts index 6dface4..c42d481 100644 --- a/src/postGithubComments.ts +++ b/src/postGithubComments.ts @@ -1,6 +1,6 @@ import fs = require("fs"); import path = require("path"); -import { artifactFolderUrlPlaceholder, getArtifactsApiUrlPlaceholder, Metadata, metadataFileName, RepoStatus, resultFileNameSuffix,TsEntrypoint } from "./main"; +import { artifactFolderUrlPlaceholder, getArtifactsApiUrlPlaceholder, Metadata, metadataFileName, RepoStatus, resultFileNameSuffix, TsEntrypoint } from "./main"; import git = require("./utils/gitUtils"); import pu = require("./utils/packageUtils"); import { asMarkdownInlineCode } from "./utils/markdownUtils"; diff --git a/src/utils/exerciseLspServer.ts b/src/utils/exerciseLspServer.ts index 01d717c..fedf167 100644 --- a/src/utils/exerciseLspServer.ts +++ b/src/utils/exerciseLspServer.ts @@ -347,9 +347,7 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r context: { includeDeclaration: true }, }, isAt ? 0.5 : 0.00005); - // TODO: - // - https://github.com/microsoft/typescript-go/issues/2253 - const completionsProb = 1; + const completionsProb = 0.1; // Completions (equivalent to completionInfo) const completionResponse = await request("textDocument/completion", { @@ -452,7 +450,6 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r params: lsp.RequestToParams[K], prob = 1, ): Promise { - // await new Promise(resolve => setTimeout(resolve, 500)); if (prng.random() > prob) return undefined as any; const replayEntry = { kind: "request", method, params }; @@ -489,7 +486,6 @@ async function exerciseLspServerWorker(testDir: string, lspServerPath: string, r method: K, params: lsp.NotificationToParams[K], ): Promise { - // await new Promise(resolve => setTimeout(resolve, 500)); const replayEntry = { kind: "notification", method, params }; const replayStr = JSON.stringify(replayEntry).replaceAll(testDirUrl, testDirPlaceholder); await replayScriptHandle.write(replayStr + "\n");