diff --git a/packages/typescript/scripts/generateSync.ts b/packages/typescript/scripts/generateSync.ts index d44b999d25b2c..3988cf5f8f17e 100755 --- a/packages/typescript/scripts/generateSync.ts +++ b/packages/typescript/scripts/generateSync.ts @@ -11,6 +11,7 @@ * // @sync-skip-block-start/end — omit all lines between (inclusive) * // @sync-only-start/end — uncomment lines between (strip "// " prefix) * // @sync: — replace this line with (preserving indent) + * // @generators: - replace this line only in generator output * * AST-based transforms (applied after directives using the TypeScript compiler API): * - Generate synchronous implementations from async methods @@ -60,6 +61,7 @@ function generateSyncFile( transform: SourceTransform, force: boolean, variant: SyncVariant = "sync", + preserveGeneratorOverrides = false, ): GeneratedFile | undefined { const generated = new GeneratedFile(destPath, [import.meta.filename, srcPath]); if (generated.isCurrent(force)) return; @@ -69,7 +71,7 @@ function generateSyncFile( const normalized = source.replace(/\r/g, ""); // Phase 1: Process sync directives (text-based, operates on comments/lines) - const afterDirectives = processDirectives(normalized.split("\n"), variant).join("\n"); + const afterDirectives = processDirectives(normalized.split("\n"), variant, preserveGeneratorOverrides).join("\n"); // Phase 2: AST-based async→sync transforms const fileName = destPath.split("/").pop()!; @@ -88,7 +90,7 @@ function generateSyncFile( // ── Directive processing ───────────────────────────────────────── -function processDirectives(lines: string[], variant: SyncVariant): string[] { +function processDirectives(lines: string[], variant: SyncVariant, preserveGeneratorOverrides = false): string[] { const output: string[] = []; let skipBlock = false; let syncOnlyBlock = false; @@ -163,7 +165,7 @@ function processDirectives(lines: string[], variant: SyncVariant): string[] { continue; } - const lineWithoutGeneratorsDirective = generatorsReplaceMatch + const lineWithoutGeneratorsDirective = generatorsReplaceMatch && !preserveGeneratorOverrides ? line.slice(0, generatorsReplaceMatch.index).trimEnd() : line; const syncReplaceMatch = lineWithoutGeneratorsDirective.match(/\/\/ @sync: (.+)$/); @@ -211,14 +213,35 @@ function transformAsyncSource(source: string, fileName: string, attachGenerators const edits: Edit[] = []; const asyncMethodNames = new Set(); const synchronousMethodNames = new Set(); + const batchedMethodNames = new Set(); + const asyncMethods: ts.MethodDeclaration[] = []; for (const classDeclaration of getNamedClasses(sourceFile).values()) { for (const group of getMethodGroups(classDeclaration).values()) { const method = group.implementation; - if (isGeneratorMethod(method)) asyncMethodNames.add((method.name as ts.Identifier).text); + if (isGeneratorMethod(method)) { + asyncMethodNames.add((method.name as ts.Identifier).text); + asyncMethods.push(method); + } else synchronousMethodNames.add((method.name as ts.Identifier).text); } } + let changed = true; + while (changed) { + changed = false; + for (const method of asyncMethods) { + const name = (method.name as ts.Identifier).text; + if (!batchedMethodNames.has(name) && containsBatchedCall(method.body!)) { + batchedMethodNames.add(name); + changed = true; + } + } + } + + function containsBatchedCall(node: ts.Node): boolean { + if (isPromiseAllCall(node) || ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression) && batchedMethodNames.has(node.expression.name.text)) return true; + return ts.forEachChild(node, containsBatchedCall) ?? false; + } visit(sourceFile); return applyEdits(source, edits); @@ -321,6 +344,9 @@ function transformAsyncSource(source: string, fileName: string, attachGenerators if (!node.type) return "void"; if (mode === "generator" && isPromiseType(node.type)) { const returnType = getUnwrappedPromiseText(node.type, source, sourceFile); + if (node.name && ts.isIdentifier(node.name) && batchedMethodNames.has(node.name.text)) { + return `APIRequestGenerator<${returnType}>`; + } return `Generator`; } return transformNodeText(node.type, mode, false); @@ -347,7 +373,10 @@ function transformAsyncSource(source: string, fileName: string, attachGenerators } visitNode(node); - return applyEdits(source.slice(start, node.end), localEdits); + const result = applyEdits(source.slice(start, node.end), localEdits); + return ts.isBlock(node) + ? processDirectives(result.split("\n"), mode === "generator" ? "generators" : "sync").join("\n") + : result; } function addGeneratorEdit(node: ts.Node, offset: number, targetEdits: Edit[]): boolean { @@ -387,6 +416,24 @@ function transformAsyncSource(source: string, fileName: string, attachGenerators } function getGeneratorAwaitText(node: ts.AwaitExpression): string { + if (isPromiseAllCall(node.expression)) { + const argument = node.expression.arguments[0]; + if ( + !ts.isCallExpression(argument) || + !ts.isPropertyAccessExpression(argument.expression) || + argument.expression.name.text !== "map" || + argument.arguments.length !== 1 || + !ts.isArrowFunction(argument.arguments[0]) || + !ts.isCallExpression(argument.arguments[0].body) || + !isGeneratorCall(argument.arguments[0].body) + ) { + throw new Error("Promise.all requires a map callback returning an API method call"); + } + const callback = argument.arguments[0]; + const receiver = getTextWithOwner(argument.expression.expression); + const parameters = callback.parameters.map(parameter => getTextWithOwner(parameter)).join(", "); + return `(yield* all(...${receiver}.map((${parameters}) => ${getGeneratorCallText(argument.arguments[0].body)})))`; + } if (isAPIRequestCall(node.expression)) { const receiver = node.expression.expression.expression; if (ts.isPropertyAccessExpression(receiver) && receiver.name.text === "client") return getRequestYieldText(node.expression); @@ -487,6 +534,11 @@ function transformAsyncSource(source: string, fileName: string, attachGenerators } function addSyncEdit(node: ts.Node, source: string, sourceFile: ts.SourceFile, edits: Edit[], offset: number = 0): boolean { + if (isPromiseAllCall(node)) { + const argument = node.arguments[0]; + edits.push({ start: node.getStart(sourceFile) - offset, end: argument.getStart(sourceFile) - offset, newText: "" }); + edits.push({ start: argument.end - offset, end: node.end - offset, newText: "" }); + } if ( (ts.isFunctionDeclaration(node) || ts.isMethodDeclaration(node) || ts.isArrowFunction(node) || ts.isFunctionExpression(node)) && node.modifiers @@ -540,6 +592,15 @@ function addSyncEdit(node: ts.Node, source: string, sourceFile: ts.SourceFile, e return false; } +function isPromiseAllCall(node: ts.Node): node is ts.CallExpression { + return ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + ts.isIdentifier(node.expression.expression) && + node.expression.expression.text === "Promise" && + node.expression.name.text === "all" && + node.arguments.length === 1; +} + function isPromiseTypeReference(node: ts.Node | undefined): node is ts.TypeReferenceNode { return !!node && ts.isTypeReferenceNode(node) && @@ -641,7 +702,7 @@ export function generateSync(force = false): void { ? (source, fileName) => [ 'import type { APIRequest as ProtocolRequest, APIResponse as ProtocolResponse } from "../proto.ts";', - 'import { apiRequest, cacheGeneratorMethod } from "./generatorSupport.ts";', + 'import { all, apiRequest, cacheGeneratorMethod, type APIRequestGenerator } from "./generatorSupport.ts";', "", transformAsyncSource(source, fileName, true), ].join("\n") @@ -652,6 +713,8 @@ export function generateSync(force = false): void { transformAsyncSource(source, fileName, true), ].join("\n"), force, + "sync", + relPath === "api.ts", )); } diff --git a/packages/typescript/src/api/async/api.ts b/packages/typescript/src/api/async/api.ts index bf3fe53bbce5d..7798b13517326 100644 --- a/packages/typescript/src/api/async/api.ts +++ b/packages/typescript/src/api/async/api.ts @@ -42,6 +42,7 @@ import { import { assertNever } from "../../internal/utils.ts"; import { encodeNode, + sourceFileResponseToUint8Array, uint8ArrayToBase64, } from "../node/encoder.ts"; import { @@ -95,6 +96,7 @@ import type { ModuleResolutionSpec, PackageId, ParsedCommandLine, + ProgramStatisticsResponse, ProjectId, ProjectReference, ProjectResponse, @@ -102,6 +104,7 @@ import type { RawCompilerOptions, ReadConfigFileResponse, ReconfigureSnapshotProgramParams as ProtocolReconfigureSnapshotProgramParams, + RelationCacheSizes, ResolutionMode, ResolvedModule, ResolvedTypeReferenceDirective, @@ -238,10 +241,12 @@ export type { ObjectType, PackageId, ParsedCommandLine, + ProgramStatisticsResponse, ProjectId, ProjectReference, RawCompilerOptions, ReadConfigFileResponse, + RelationCacheSizes, RequestTiming, ResolutionMode, ResolvedModule, @@ -344,7 +349,6 @@ export interface TranspileOutput { // @sync-only-start // export { all, defer, type APIRequestGenerator, type AnyAPIRequestGenerator, type AllAPIRequestGenerator, type DeferredAPIRequestGenerator, type ExecutedGeneratorsResults } from "./generatorSupport.ts"; // import {executeRequestGenerators, type ExecutedGeneratorsResults, type AnyAPIRequestGenerator} from "./generatorSupport.ts"; -// import { sourceFileResponseToUint8Array } from "../node/encoder.ts"; // @sync-only-end export class API implements FormatDiagnosticsHost { @@ -1796,6 +1800,21 @@ export class Program implements FormatDiagnost return this.project.parsedCommandLine.options; } + getRootFileNames(): readonly RootedFilePath[] { + return this.project.parsedCommandLine.fileNames; + } + + getProjectReferences(): readonly ProjectReference[] | undefined { + return this.project.parsedCommandLine.projectReferences; + } + + getStatistics(): Promise { + return this.client.apiRequest("getProgramStatistics", { + snapshot: this.snapshotId, + project: this.project.handle, + }); + } + async getSourceFile(file: DocumentIdentifier): Promise { const fileName = resolveFileName(file); const path = this.pathKeyForFileName(fileName); diff --git a/packages/typescript/src/api/proto.generated.ts b/packages/typescript/src/api/proto.generated.ts index 35502aade2ac5..1b1c34c4fdd74 100644 --- a/packages/typescript/src/api/proto.generated.ts +++ b/packages/typescript/src/api/proto.generated.ts @@ -80,6 +80,7 @@ export interface APIMethodInfo { getNonMissingTypeOfSymbol: APIMethod; getSourceFile: APIMethod; getSourceFileNames: APIMethod; + getProgramStatistics: APIMethod; getSourceFileMetadata: APIMethod; getModeForUsageLocation: APIMethod; getModeForResolutionAtIndex: APIMethod; @@ -602,6 +603,21 @@ export interface GetSourceFileNamesParams { project: number; } +/** GetProjectDiagnosticsParams are parameters for project-wide diagnostic methods. */ +export interface GetProjectDiagnosticsParams { + snapshot: number; + project: number; +} + +export interface ProgramStatisticsResponse { + nodeCount: number; + identifierCount: number; + symbolCount: number; + typeCount: number; + instantiationCount: number; + relationCacheSizes: RelationCacheSizes; +} + /** SourceFileMetadata carries program-stored metadata about a single source file. */ export interface SourceFileMetadata { isDefaultLibrary: boolean; @@ -675,12 +691,6 @@ export interface GetResolvedTypeReferenceDirectiveFromReferenceParams { resolutionMode: ModuleKind; } -/** GetProjectDiagnosticsParams are parameters for project-wide diagnostic methods. */ -export interface GetProjectDiagnosticsParams { - snapshot: number; - project: number; -} - export interface ResolveNameParams { snapshot: number; project: number; @@ -1290,6 +1300,7 @@ export interface BatchRequest { | "getParentOfSymbolForChecker" | "getPrivateIdentifierPropertyOfType" | "getProgramDiagnostics" + | "getProgramStatistics" | "getPropertiesOfType" | "getPropertyOfType" | "getPropertySymbolOfDestructuringAssignment" @@ -1483,6 +1494,7 @@ export interface BatchResponse { | "getParentOfSymbolForChecker" | "getPrivateIdentifierPropertyOfType" | "getProgramDiagnostics" + | "getProgramStatistics" | "getPropertiesOfType" | "getPropertyOfType" | "getPropertySymbolOfDestructuringAssignment" @@ -1737,6 +1749,13 @@ export interface TranspileOptions { reportDiagnostics?: boolean | undefined; } +export interface RelationCacheSizes { + assignable: number; + identity: number; + subtype: number; + strictSubtype: number; +} + export interface PackageId { name: string; subModuleName: string; diff --git a/packages/typescript/src/api/sync/api.ts b/packages/typescript/src/api/sync/api.ts index c8db7d186170c..4c8b08f036901 100644 --- a/packages/typescript/src/api/sync/api.ts +++ b/packages/typescript/src/api/sync/api.ts @@ -11,7 +11,9 @@ import type { APIResponse as ProtocolResponse, } from "../proto.ts"; import { + all, apiRequest, + type APIRequestGenerator, cacheGeneratorMethod, } from "./generatorSupport.ts"; @@ -59,6 +61,7 @@ import { import { assertNever } from "../../internal/utils.ts"; import { encodeNode, + sourceFileResponseToUint8Array, uint8ArrayToBase64, } from "../node/encoder.ts"; import { @@ -112,6 +115,7 @@ import type { ModuleResolutionSpec, PackageId, ParsedCommandLine, + ProgramStatisticsResponse, ProjectId, ProjectReference, ProjectResponse, @@ -119,6 +123,7 @@ import type { RawCompilerOptions, ReadConfigFileResponse, ReconfigureSnapshotProgramParams as ProtocolReconfigureSnapshotProgramParams, + RelationCacheSizes, ResolutionMode, ResolvedModule, ResolvedTypeReferenceDirective, @@ -254,10 +259,12 @@ export type { ObjectType, PackageId, ParsedCommandLine, + ProgramStatisticsResponse, ProjectId, ProjectReference, RawCompilerOptions, ReadConfigFileResponse, + RelationCacheSizes, RequestTiming, ResolutionMode, ResolvedModule, @@ -358,7 +365,6 @@ export interface TranspileOutput { } export { all, type AllAPIRequestGenerator, type AnyAPIRequestGenerator, type APIRequestGenerator, defer, type DeferredAPIRequestGenerator, type ExecutedGeneratorsResults } from "./generatorSupport.ts"; -import { sourceFileResponseToUint8Array } from "../node/encoder.ts"; import { type AnyAPIRequestGenerator, type ExecutedGeneratorsResults, @@ -3001,6 +3007,37 @@ export class Program implements FormatDiagnost return this.project.parsedCommandLine.options; } + getRootFileNames(): readonly RootedFilePath[] { + return this.project.parsedCommandLine.fileNames; + } + + getProjectReferences(): readonly ProjectReference[] | undefined { + return this.project.parsedCommandLine.projectReferences; + } + + get getStatistics(): { + (): ProgramStatisticsResponse; + gen(): Generator; + } { + const owner = this; + return cacheGeneratorMethod( + owner, + "getStatistics", + function (): ProgramStatisticsResponse { + return owner.client.apiRequest("getProgramStatistics", { + snapshot: owner.snapshotId, + project: owner.project.handle, + }); + }, + function* (): Generator { + return yield* apiRequest("getProgramStatistics", { + snapshot: owner.snapshotId, + project: owner.project.handle, + }); + }, + ); + } + get getSourceFile(): { (file: DocumentIdentifier): SourceFile | undefined; gen(file: DocumentIdentifier): Generator; diff --git a/packages/typescript/test/async/api.test.ts b/packages/typescript/test/async/api.test.ts index 3fc2126b968bd..303077ef09504 100644 --- a/packages/typescript/test/async/api.test.ts +++ b/packages/typescript/test/async/api.test.ts @@ -131,8 +131,19 @@ import { toRootedFilePath, } from "@typescript/typescript/path"; import assert from "node:assert"; -import { globSync } from "node:fs"; -import { resolve } from "node:path"; +import { + existsSync, + globSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { + join, + resolve, +} from "node:path"; import { describe, test, @@ -1520,6 +1531,48 @@ declare module "augmentation" {}`, // @sync-skip-block-start describe("API - automatic batching", { concurrency }, () => { + test("fetches Program statistics for every request", async context => { + await using api = spawnAPI({ "/src/main.ts": "export const value = 1;" }); + await using program = await api.createProgram(["/src/main.ts"], { noLib: true }); + await using empty = await api.createProgram([], { noLib: true }); + // @ts-ignore -- Internal client requests are exercised by source tests. + const client = api["client"]; + const request = context.mock.method(client, "apiRequest"); + + const [statistics, repeated, emptyStatistics] = await Promise.all([ + program.getStatistics(), + program.getStatistics(), + empty.getStatistics(), + ]); + assert.equal(request.mock.callCount(), 3); + assert.deepEqual(repeated, statistics); + assert.ok(statistics.identifierCount > 0); + assert.ok(statistics.nodeCount > statistics.identifierCount); + assert.ok(statistics.symbolCount > 0); + assert.ok(statistics.typeCount > 0); + assert.equal(statistics.instantiationCount, 0); + assert.deepEqual(statistics.relationCacheSizes, { assignable: 0, identity: 0, subtype: 0, strictSubtype: 0 }); + assert.equal(emptyStatistics.nodeCount, 0); + + assert.deepEqual(await program.getStatistics(), statistics); + assert.equal(request.mock.callCount(), 4); + }); + + test("retries Program statistics requests after failure", async context => { + await using api = spawnAPI({ "/src/main.ts": "export const value = 1;" }); + await using program = await api.createProgram(["/src/main.ts"], { noLib: true }); + // @ts-ignore -- Internal client requests are exercised by source tests. + const client = api["client"]; + const request = context.mock.method(client, "apiRequest"); + const error = new Error("Statistics request failed"); + request.mock.mockImplementationOnce(() => Promise.reject(error)); + + await assert.rejects(program.getStatistics(), error); + assert.equal(request.mock.callCount(), 1); + assert.ok((await program.getStatistics()).nodeCount > 0); + assert.equal(request.mock.callCount(), 2); + }); + test("initializes only once for concurrent first requests", async () => { await using api = spawnAPI(); const client = (api as unknown as { @@ -2701,6 +2754,153 @@ describe("SourceFile", { concurrency }, () => { ); }); + test("Program accessors distinguish root files from source files", async () => { + await using api = spawnAPI({ + "/src/main.ts": `import { value } from "./dependency"; export const result = value;`, + "/src/dependency.ts": `export const value = 1;`, + }); + await using program = await api.createProgram(["/src/main.ts"], { noLib: true }); + assert.ok(!("getTypeChecker" in program)); + assert.ok(!("getResolvedProjectReferences" in program)); + assert.ok(!("getSourceFiles" in program)); + assert.deepEqual(await program.getRootFileNames(), ["/src/main.ts"]); + assert.deepEqual(await program.getSourceFileNames(), ["/src/dependency.ts", "/src/main.ts"]); + assert.equal(await program.getProjectReferences(), undefined); + }); + + test("Program statistics reflect checker work without forcing diagnostics", async () => { + await using api = spawnAPI({ + "/src/main.ts": `type Box = { value: T }; +export const box: Box = { value: "hello" }; +export const other: { value: string } = box;`, + }, { collectTiming: true }); + const snapshot = await api.createSnapshot({ + createPrograms: [{ rootFiles: ["/src/main.ts"], compilerOptions: { noLib: true } }], + }); + const project = snapshot.getProjects()[0]; + const program = project.program; + await api.resetTimingInfo(); + const statistics = await program.getStatistics(); + assert.deepEqual(await program.getStatistics(), statistics); + assert.equal((await api.getTimingInfo()).totals.requestCount, 2); + assert.ok(statistics.identifierCount > 0); + assert.ok(statistics.nodeCount > statistics.identifierCount); + assert.ok(statistics.symbolCount > 0); + assert.ok(statistics.typeCount > 0); + assert.equal(statistics.instantiationCount, 0); + assert.deepEqual(statistics.relationCacheSizes, { assignable: 0, identity: 0, subtype: 0, strictSubtype: 0 }); + + const file = (await program.getSourceFile("/src/main.ts"))!; + const checker = project.checker; + const declaration = cast(file.statements[1], isVariableStatement).declarationList.declarations[0]; + const type = await checker.getTypeAtLocation(declaration.name); + await type.getProperties(); + const other = cast(file.statements[2], isVariableStatement).declarationList.declarations[0]; + await checker.isTypeAssignableTo(type, await checker.getTypeAtLocation(other.name)); + const updated = await program.getStatistics(); + assert.ok(updated.instantiationCount > statistics.instantiationCount); + assert.ok(updated.typeCount > statistics.typeCount); + assert.ok(updated.symbolCount >= statistics.symbolCount); + assert.ok(updated.relationCacheSizes.assignable > statistics.relationCacheSizes.assignable); + assert.equal(updated.identifierCount, statistics.identifierCount); + assert.equal(updated.nodeCount, statistics.nodeCount); + }); + + test("Program accessors handle empty programs and missing root files", async () => { + await using api = spawnAPI({}); + await using empty = await api.createProgram([], { noLib: true }); + assert.deepEqual(await empty.getRootFileNames(), []); + assert.deepEqual(await empty.getSourceFileNames(), []); + const statistics = await empty.getStatistics(); + assert.equal(statistics.nodeCount, 0); + assert.equal(statistics.identifierCount, 0); + assert.equal(await empty.getProjectReferences(), undefined); + assert.deepEqual(await empty.getProgramDiagnostics(), []); + + await using noReferences = await api.createProgram([], { noLib: true }, { projectReferences: [] }); + assert.deepEqual(await noReferences.getProjectReferences(), []); + + await using missing = await api.createProgram(["/missing.ts"], { noLib: true }); + assert.deepEqual(await missing.getRootFileNames(), ["/missing.ts"]); + assert.deepEqual(await missing.getSourceFileNames(), []); + assert.ok((await missing.getProgramDiagnostics()).some(diagnostic => diagnostic.code === 6053)); + }); + + test("Program project references are available for configured and synthetic programs", async () => { + await using api = spawnAPI({ + "/tsconfig.json": JSON.stringify({ files: [], references: [{ path: "./a" }, { path: "./missing" }, { path: "./b" }, { path: "./a" }] }), + "/a/tsconfig.json": JSON.stringify({ compilerOptions: { composite: true }, files: ["index.ts"], references: [{ path: "../b" }] }), + "/a/index.ts": "export const a = 1;", + "/b/tsconfig.json": JSON.stringify({ compilerOptions: { composite: true }, files: ["index.ts"], references: [{ path: "../a", circular: true }, { path: "../missing" }] }), + "/b/index.ts": "export const b = 1;", + }); + const snapshot = await api.createSnapshot({ openProjects: ["/tsconfig.json"] }); + const program = snapshot.getConfiguredProject("/tsconfig.json")!.program; + const references = await program.getProjectReferences(); + assert.equal(references?.length, 4); + + await using synthetic = await api.createProgram([], { noLib: true }, { + projectReferences: [{ path: "/a/tsconfig.json" }, { path: "/missing/tsconfig.json" }], + }); + assert.equal((await synthetic.getProjectReferences())!.length, 2); + }); + + test("Project does not expose membership and file-list methods", async () => { + await using api = spawnAPI(); + const snapshot = await api.createSnapshot({ openProjects: ["/tsconfig.json"] }); + const project = snapshot.getConfiguredProject("/tsconfig.json")!; + for (const method of ["containsFile", "getFileNames", "getRootFiles", "hasRoots", "isRoot", "hasConfigFile", "getSourceFile"]) { + assert.ok(!(method in project), `Project should not expose ${method}`); + } + }); + + test("getSourceFile includes supplemental files without physical files", { + skip: process.env.TS_API_TEST_WASM ? "WASM cannot spawn external content mapper processes" : false, + }, async t => { + const directory = mkdtempSync(join(tmpdir(), "ts-project-membership-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const mapperDirectory = join(directory, "node_modules", "mapper"); + mkdirSync(mapperDirectory, { recursive: true }); + const mapperScript = ` + import { createMessageConnection } from ${JSON.stringify(new URL("../../vendor/vscode-jsonrpc/lib/node/main.js", import.meta.url).href)}; + const connection = createMessageConnection(process.stdin, process.stdout); + connection.onRequest("initialize", () => ({ positionEncoding: "utf-8", diagnosticSource: "mapper" })); + connection.onRequest("openProject", () => ({})); + connection.onRequest("closeProject", () => null); + connection.onRequest("transform", ({ content }) => ({ + text: "export {};", + extension: ".ts", + supplemental: [{ text: content, extension: ".ts" }], + })); + connection.listen(); + `; + writeFileSync( + join(mapperDirectory, "package.json"), + JSON.stringify({ + name: "mapper", + version: "1.0.0", + typescript: { contentMapper: { exec: [process.execPath, "--input-type=module", "--eval", mapperScript] } }, + }), + ); + const configFileName = toRootedFilePath(join(directory, "tsconfig.json"), undefined); + writeFileSync( + configFileName, + JSON.stringify({ + compilerOptions: { noLib: true }, + files: ["app.box", "main.ts", "missing.ts"], + contentMappers: [{ package: "mapper", extensions: [".box"] }], + }), + ); + writeFileSync(join(directory, "app.box"), "declare const supplementalValue: number;"); + writeFileSync(join(directory, "main.ts"), "const value: number = supplementalValue;"); + await using api = new API({ cwd: directory, runExternalCode: true }); + const snapshot = await api.createSnapshot({ openProjects: [configFileName] }); + const program = snapshot.getConfiguredProject(configFileName)!.program; + const supplementalFileName = toRootedFilePath(join(directory, "app.box.0.ts"), undefined); + assert.equal(existsSync(supplementalFileName), false); + assert.ok(await program.getSourceFile(supplementalFileName)); + }); + test("getSourceFileNames returns all program files, not just root files", async () => { await using api = spawnAPI({ "/tsconfig.json": JSON.stringify({ diff --git a/packages/typescript/test/sync/api-generators.test.ts b/packages/typescript/test/sync/api-generators.test.ts index 355cc3302118f..b6d9a6ff1cd0b 100644 --- a/packages/typescript/test/sync/api-generators.test.ts +++ b/packages/typescript/test/sync/api-generators.test.ts @@ -142,7 +142,7 @@ export const missingValue = absent; }; type GeneratorMethod = ((...args: Args) => Result) & { - gen(...args: Args): Generator; + gen(...args: Args): APIRequestGenerator; }; type EquivalenceAssertion = (actual: Result, expected: Result, message?: string) => void; @@ -150,7 +150,7 @@ type EquivalenceAssertion = (actual: Result, expected: Result, message?: interface ParityCase { readonly ownerName: string; readonly methodName: string; - readonly runGenerator: () => Generator; + readonly runGenerator: () => APIRequestGenerator; readonly assertEquivalent: (message?: string) => void; } @@ -446,6 +446,24 @@ function assertPublicGeneratorCoverage(owners: readonly { readonly name: string; } describe("API - generator batching", { concurrency: areTestsFiltered() }, () => { + test("Program statistics are fetched for every sync and generator call", () => { + using api = spawnAPI({ "/src/main.ts": "export const value = 1;" }, { collectTiming: true }); + for (const generatorFirst of [false, true]) { + using program = api.createProgram(["/src/main.ts"], { noLib: true }); + api.resetTimingInfo(); + const statistics = generatorFirst ? api.batch(program.getStatistics.gen())[0] : program.getStatistics(); + assert.ok(statistics.nodeCount > 0); + const [next, repeated] = api.batch( + program.getStatistics.gen(), + program.getStatistics.gen(), + ); + assert.deepEqual(next, statistics); + assert.deepEqual(repeated, statistics); + assert.deepEqual(program.getStatistics(), statistics); + assert.equal(api.getTimingInfo().totals.requestCount, 3); + } + }); + test("looks up binder symbols through standalone and API generators", () => { using api = spawnAPI(); using lease = api.createSourceFile("/symbols.ts", "function present() {}"); @@ -1800,6 +1818,9 @@ describe("API - generator batching", { concurrency: areTestsFiltered() }, () => ]; runParityBatch(api, cases); + runParityBatch(api, [ + parityCase("Program", "getStatistics", program.getStatistics, assertDeepEquivalent), + ]); assert.deepEqual(temporaryProjects, ["/tsconfig.json", "/tsconfig.json"]); const snapshotFiles = { diff --git a/packages/typescript/test/sync/api.test.ts b/packages/typescript/test/sync/api.test.ts index bb4caee65856e..9c82750df0984 100644 --- a/packages/typescript/test/sync/api.test.ts +++ b/packages/typescript/test/sync/api.test.ts @@ -139,8 +139,19 @@ import { type UserPreferences, } from "@typescript/typescript/sync"; import assert from "node:assert"; -import { globSync } from "node:fs"; -import { resolve } from "node:path"; +import { + existsSync, + globSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { + join, + resolve, +} from "node:path"; import { describe, test, @@ -2559,6 +2570,153 @@ describe("SourceFile", { concurrency }, () => { ); }); + test("Program accessors distinguish root files from source files", () => { + using api = spawnAPI({ + "/src/main.ts": `import { value } from "./dependency"; export const result = value;`, + "/src/dependency.ts": `export const value = 1;`, + }); + using program = api.createProgram(["/src/main.ts"], { noLib: true }); + assert.ok(!("getTypeChecker" in program)); + assert.ok(!("getResolvedProjectReferences" in program)); + assert.ok(!("getSourceFiles" in program)); + assert.deepEqual(program.getRootFileNames(), ["/src/main.ts"]); + assert.deepEqual(program.getSourceFileNames(), ["/src/dependency.ts", "/src/main.ts"]); + assert.equal(program.getProjectReferences(), undefined); + }); + + test("Program statistics reflect checker work without forcing diagnostics", () => { + using api = spawnAPI({ + "/src/main.ts": `type Box = { value: T }; +export const box: Box = { value: "hello" }; +export const other: { value: string } = box;`, + }, { collectTiming: true }); + const snapshot = api.createSnapshot({ + createPrograms: [{ rootFiles: ["/src/main.ts"], compilerOptions: { noLib: true } }], + }); + const project = snapshot.getProjects()[0]; + const program = project.program; + api.resetTimingInfo(); + const statistics = program.getStatistics(); + assert.deepEqual(program.getStatistics(), statistics); + assert.equal((api.getTimingInfo()).totals.requestCount, 2); + assert.ok(statistics.identifierCount > 0); + assert.ok(statistics.nodeCount > statistics.identifierCount); + assert.ok(statistics.symbolCount > 0); + assert.ok(statistics.typeCount > 0); + assert.equal(statistics.instantiationCount, 0); + assert.deepEqual(statistics.relationCacheSizes, { assignable: 0, identity: 0, subtype: 0, strictSubtype: 0 }); + + const file = (program.getSourceFile("/src/main.ts"))!; + const checker = project.checker; + const declaration = cast(file.statements[1], isVariableStatement).declarationList.declarations[0]; + const type = checker.getTypeAtLocation(declaration.name); + type.getProperties(); + const other = cast(file.statements[2], isVariableStatement).declarationList.declarations[0]; + checker.isTypeAssignableTo(type, checker.getTypeAtLocation(other.name)); + const updated = program.getStatistics(); + assert.ok(updated.instantiationCount > statistics.instantiationCount); + assert.ok(updated.typeCount > statistics.typeCount); + assert.ok(updated.symbolCount >= statistics.symbolCount); + assert.ok(updated.relationCacheSizes.assignable > statistics.relationCacheSizes.assignable); + assert.equal(updated.identifierCount, statistics.identifierCount); + assert.equal(updated.nodeCount, statistics.nodeCount); + }); + + test("Program accessors handle empty programs and missing root files", () => { + using api = spawnAPI({}); + using empty = api.createProgram([], { noLib: true }); + assert.deepEqual(empty.getRootFileNames(), []); + assert.deepEqual(empty.getSourceFileNames(), []); + const statistics = empty.getStatistics(); + assert.equal(statistics.nodeCount, 0); + assert.equal(statistics.identifierCount, 0); + assert.equal(empty.getProjectReferences(), undefined); + assert.deepEqual(empty.getProgramDiagnostics(), []); + + using noReferences = api.createProgram([], { noLib: true }, { projectReferences: [] }); + assert.deepEqual(noReferences.getProjectReferences(), []); + + using missing = api.createProgram(["/missing.ts"], { noLib: true }); + assert.deepEqual(missing.getRootFileNames(), ["/missing.ts"]); + assert.deepEqual(missing.getSourceFileNames(), []); + assert.ok((missing.getProgramDiagnostics()).some(diagnostic => diagnostic.code === 6053)); + }); + + test("Program project references are available for configured and synthetic programs", () => { + using api = spawnAPI({ + "/tsconfig.json": JSON.stringify({ files: [], references: [{ path: "./a" }, { path: "./missing" }, { path: "./b" }, { path: "./a" }] }), + "/a/tsconfig.json": JSON.stringify({ compilerOptions: { composite: true }, files: ["index.ts"], references: [{ path: "../b" }] }), + "/a/index.ts": "export const a = 1;", + "/b/tsconfig.json": JSON.stringify({ compilerOptions: { composite: true }, files: ["index.ts"], references: [{ path: "../a", circular: true }, { path: "../missing" }] }), + "/b/index.ts": "export const b = 1;", + }); + const snapshot = api.createSnapshot({ openProjects: ["/tsconfig.json"] }); + const program = snapshot.getConfiguredProject("/tsconfig.json")!.program; + const references = program.getProjectReferences(); + assert.equal(references?.length, 4); + + using synthetic = api.createProgram([], { noLib: true }, { + projectReferences: [{ path: "/a/tsconfig.json" }, { path: "/missing/tsconfig.json" }], + }); + assert.equal((synthetic.getProjectReferences())!.length, 2); + }); + + test("Project does not expose membership and file-list methods", () => { + using api = spawnAPI(); + const snapshot = api.createSnapshot({ openProjects: ["/tsconfig.json"] }); + const project = snapshot.getConfiguredProject("/tsconfig.json")!; + for (const method of ["containsFile", "getFileNames", "getRootFiles", "hasRoots", "isRoot", "hasConfigFile", "getSourceFile"]) { + assert.ok(!(method in project), `Project should not expose ${method}`); + } + }); + + test("getSourceFile includes supplemental files without physical files", { + skip: process.env.TS_API_TEST_WASM ? "WASM cannot spawn external content mapper processes" : false, + }, t => { + const directory = mkdtempSync(join(tmpdir(), "ts-project-membership-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const mapperDirectory = join(directory, "node_modules", "mapper"); + mkdirSync(mapperDirectory, { recursive: true }); + const mapperScript = ` + import { createMessageConnection } from ${JSON.stringify(new URL("../../vendor/vscode-jsonrpc/lib/node/main.js", import.meta.url).href)}; + const connection = createMessageConnection(process.stdin, process.stdout); + connection.onRequest("initialize", () => ({ positionEncoding: "utf-8", diagnosticSource: "mapper" })); + connection.onRequest("openProject", () => ({})); + connection.onRequest("closeProject", () => null); + connection.onRequest("transform", ({ content }) => ({ + text: "export {};", + extension: ".ts", + supplemental: [{ text: content, extension: ".ts" }], + })); + connection.listen(); + `; + writeFileSync( + join(mapperDirectory, "package.json"), + JSON.stringify({ + name: "mapper", + version: "1.0.0", + typescript: { contentMapper: { exec: [process.execPath, "--input-type=module", "--eval", mapperScript] } }, + }), + ); + const configFileName = toRootedFilePath(join(directory, "tsconfig.json"), undefined); + writeFileSync( + configFileName, + JSON.stringify({ + compilerOptions: { noLib: true }, + files: ["app.box", "main.ts", "missing.ts"], + contentMappers: [{ package: "mapper", extensions: [".box"] }], + }), + ); + writeFileSync(join(directory, "app.box"), "declare const supplementalValue: number;"); + writeFileSync(join(directory, "main.ts"), "const value: number = supplementalValue;"); + using api = new API({ cwd: directory, runExternalCode: true }); + const snapshot = api.createSnapshot({ openProjects: [configFileName] }); + const program = snapshot.getConfiguredProject(configFileName)!.program; + const supplementalFileName = toRootedFilePath(join(directory, "app.box.0.ts"), undefined); + assert.equal(existsSync(supplementalFileName), false); + assert.ok(program.getSourceFile(supplementalFileName)); + }); + test("getSourceFileNames returns all program files, not just root files", () => { using api = spawnAPI({ "/tsconfig.json": JSON.stringify({ diff --git a/tsc/internal/api/proto.go b/tsc/internal/api/proto.go index c3e55e188956d..23b381abc4a8b 100644 --- a/tsc/internal/api/proto.go +++ b/tsc/internal/api/proto.go @@ -111,6 +111,7 @@ const ( MethodGetNonMissingTypeOfSymbol Method = "getNonMissingTypeOfSymbol" MethodGetSourceFile Method = "getSourceFile" MethodGetSourceFileNames Method = "getSourceFileNames" + MethodGetProgramStatistics Method = "getProgramStatistics" MethodGetSourceFileMetadata Method = "getSourceFileMetadata" MethodGetModeForUsageLocation Method = "getModeForUsageLocation" MethodGetModeForResolutionAtIndex Method = "getModeForResolutionAtIndex" @@ -794,6 +795,7 @@ var unmarshalers = map[Method]func([]byte) (any, error){ MethodGetSuggestionDiagnostics: unmarshallerFor[GetDiagnosticsParams], MethodGetDeclarationDiagnostics: unmarshallerFor[GetDiagnosticsParams], MethodGetProgramDiagnostics: unmarshallerFor[GetProjectDiagnosticsParams], + MethodGetProgramStatistics: unmarshallerFor[GetProjectDiagnosticsParams], MethodGetGlobalDiagnostics: unmarshallerFor[GetProjectDiagnosticsParams], MethodGetConfigFileParsingDiagnostics: unmarshallerFor[GetProjectDiagnosticsParams], MethodStartCPUProfile: unmarshallerFor[ProfileParams], @@ -1038,13 +1040,22 @@ type ConfigFileResponse struct { FileNames []tspath.RootedFilePath `json:"fileNames" nonnil:"true"` Options *core.CompilerOptions `json:"options" nonnil:"true"` BuildOptions *core.BuildOptions `json:"buildOptions,omitempty"` - ProjectReferences []*core.ProjectReference `json:"projectReferences,omitempty"` + ProjectReferences []*core.ProjectReference `json:"projectReferences,omitzero"` TypeAcquisition *core.TypeAcquisition `json:"typeAcquisition,omitempty"` CompileOnSave *bool `json:"compileOnSave,omitempty"` Raw any `json:"raw,omitempty"` Errors []*DiagnosticResponse `json:"errors" nonnil:"true"` } +type ProgramStatisticsResponse struct { + NodeCount int `json:"nodeCount"` + IdentifierCount int `json:"identifierCount"` + SymbolCount int `json:"symbolCount"` + TypeCount uint32 `json:"typeCount"` + InstantiationCount uint32 `json:"instantiationCount"` + RelationCacheSizes checker.RelationCacheSizes `json:"relationCacheSizes"` +} + type ReadConfigFileResponse struct { Config any `json:"config"` Error *DiagnosticResponse `json:"error,omitempty"` diff --git a/tsc/internal/api/session.go b/tsc/internal/api/session.go index 23c0c38170cd7..cfc8b51a112a2 100644 --- a/tsc/internal/api/session.go +++ b/tsc/internal/api/session.go @@ -951,6 +951,8 @@ func (s *Session) HandleRequest(ctx context.Context, method string, params json. return s.handleGetSourceFile(ctx, parsed.(*GetSourceFileParams)) case string(MethodGetSourceFileNames): return s.handleGetSourceFileNames(ctx, parsed.(*GetSourceFileNamesParams)) + case string(MethodGetProgramStatistics): + return s.handleGetProgramStatistics(ctx, parsed.(*GetProjectDiagnosticsParams)) case string(MethodGetSourceFileMetadata): return s.handleGetSourceFileMetadata(ctx, parsed.(*GetSourceFileParams)) case string(MethodGetModeForUsageLocation): @@ -2396,6 +2398,26 @@ func transpileOutput(ctx context.Context, input string, options TranspileOptions }, nil } +func (s *Session) handleGetProgramStatistics(ctx context.Context, params *GetProjectDiagnosticsParams) (*ProgramStatisticsResponse, error) { + setup, err := s.setupChecker(ctx, params.Snapshot, params.Project) + if err != nil { + return nil, err + } + defer setup.done() + result := &ProgramStatisticsResponse{ + SymbolCount: int(setup.checker.SymbolCount), + TypeCount: setup.checker.TypeCount, + InstantiationCount: setup.checker.TotalInstantiationCount, + RelationCacheSizes: setup.checker.GetRelationCacheSizes(), + } + for _, file := range setup.program.GetSourceFiles() { + result.NodeCount += file.NodeCount + result.IdentifierCount += file.IdentifierCount + result.SymbolCount += file.SymbolCount + } + return result, nil +} + // handleGetSourceFile returns a source file from a project within a snapshot. // @gen-proto-result: SourceFileResponse // @gen-proto-nullable diff --git a/tsc/internal/api/session_program_test.go b/tsc/internal/api/session_program_test.go new file mode 100644 index 0000000000000..6cdd99530fcd4 --- /dev/null +++ b/tsc/internal/api/session_program_test.go @@ -0,0 +1,89 @@ +package api + +import ( + "context" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/lsp/lsproto" + "github.com/microsoft/TypeScript/tsc/internal/project" + "github.com/microsoft/TypeScript/tsc/internal/testutil/contentmappertest" + "github.com/microsoft/TypeScript/tsc/internal/testutil/projecttestutil" + "gotest.tools/v3/assert" +) + +func TestProgramSourceFileIncludesSupplementalFile(t *testing.T) { + t.Parallel() + + const supplementalFileName = "/home/project/app.box.0.ts" + files := map[string]any{ + "/home/project/tsconfig.json": `{"compilerOptions":{"noLib":true},"files":["app.box","main.ts","missing.ts"],"contentMappers":[{"package":"mapper","extensions":[".box"]}]}`, + "/home/project/node_modules/mapper/package.json": contentmappertest.PackageJSON(contentmappertest.SupplementalMapper), + "/home/project/app.box": "declare const supplementalValue: number;", + "/home/project/main.ts": "const value: number = supplementalValue;", + } + init, _ := projecttestutil.GetSessionInitOptions(files, &project.SessionOptions{ + CurrentDirectory: "/home/project", + RunExternalCode: true, + }, nil) + init.Spawner = contentmappertest.NewSpawner() + projectSession := project.NewSession(init) + defer projectSession.Close() + projectSession.DidOpenFile(context.Background(), "file:///home/project/main.ts", 1, files["/home/project/main.ts"].(string), lsproto.LanguageKindTypeScript) + session := NewLSPSession(projectSession, nil) + defer session.Close() + response, err := session.handleGetCurrentLanguageServerSnapshot(context.Background(), &GetCurrentLanguageServerSnapshotParams{}) + assert.NilError(t, err) + params := &GetSourceFileParams{ + Snapshot: response.Snapshot, + Project: response.Projects[0].Handle, + File: DocumentIdentifier{FileName: supplementalFileName}, + } + sd, err := session.getSnapshotData(response.Snapshot) + assert.NilError(t, err) + assert.Assert(t, sd.snapshot.GetFile(supplementalFileName) == nil) + sourceFile, err := session.handleGetSourceFile(context.Background(), params) + assert.NilError(t, err) + assert.Assert(t, sourceFile != nil) +} + +func TestProgramStatisticsUseAPIChecker(t *testing.T) { + t.Parallel() + + const fileName = "/home/projects/p/index.ts" + projectSession, _ := projecttestutil.Setup(map[string]any{ + fileName: "export const value = 1;", + }) + defer projectSession.Close() + session := NewLSPSession(projectSession, nil) + defer session.Close() + response, err := session.handleCreateSnapshot(context.Background(), &CreateSnapshotParams{ + CreatePrograms: []*CreateSnapshotProgramParams{{ + RootFiles: []DocumentIdentifier{{FileName: fileName}}, + CompilerOptions: core.CompilerOptions{NoLib: core.TSTrue}, + }}, + }) + assert.NilError(t, err) + params := &GetProjectDiagnosticsParams{Snapshot: response.Snapshot, Project: response.Projects[0].Handle} + setup, err := session.setupChecker(context.Background(), params.Snapshot, params.Project) + assert.NilError(t, err) + file := setup.program.GetSourceFile(fileName) + setup.checker.GetTypeAtLocation(file.Statements.Nodes[0]) + expected := ProgramStatisticsResponse{ + NodeCount: file.NodeCount, + IdentifierCount: file.IdentifierCount, + SymbolCount: file.SymbolCount + int(setup.checker.SymbolCount), + TypeCount: setup.checker.TypeCount, + InstantiationCount: setup.checker.TotalInstantiationCount, + RelationCacheSizes: setup.checker.GetRelationCacheSizes(), + } + setup.done() + + actual, err := session.handleGetProgramStatistics(context.Background(), params) + assert.NilError(t, err) + assert.DeepEqual(t, *actual, expected) + _, err = session.handleGetProgramStatistics(context.Background(), &GetProjectDiagnosticsParams{ + Snapshot: response.Snapshot, Project: project.Handle(999999), + }) + assert.ErrorContains(t, err, "not found") +} diff --git a/tsc/internal/checker/exports.go b/tsc/internal/checker/exports.go index 6f39e7897e172..26c6e8180454d 100644 --- a/tsc/internal/checker/exports.go +++ b/tsc/internal/checker/exports.go @@ -8,6 +8,15 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/jsnum" ) +func (c *Checker) GetRelationCacheSizes() RelationCacheSizes { + return RelationCacheSizes{ + Assignable: c.assignableRelation.size(), + Identity: c.identityRelation.size(), + Subtype: c.subtypeRelation.size(), + StrictSubtype: c.strictSubtypeRelation.size(), + } +} + func (c *Checker) GetStringType() *Type { return c.stringType } diff --git a/tsc/internal/checker/types.go b/tsc/internal/checker/types.go index e2110ef695959..bfa846b876822 100644 --- a/tsc/internal/checker/types.go +++ b/tsc/internal/checker/types.go @@ -14,6 +14,13 @@ import ( //go:generate npx hereby generate:checker +type RelationCacheSizes struct { + Assignable int `json:"assignable"` + Identity int `json:"identity"` + Subtype int `json:"subtype"` + StrictSubtype int `json:"strictSubtype"` +} + // ParseFlags type ParseFlags uint32 diff --git a/tsc/internal/compiler/program.go b/tsc/internal/compiler/program.go index 92fc11e59372a..62b9de7ce981c 100644 --- a/tsc/internal/compiler/program.go +++ b/tsc/internal/compiler/program.go @@ -893,6 +893,27 @@ func (p *Program) GetProgramDiagnostics() []*ast.Diagnostic { )) } +func (p *Program) GetOptionsDiagnostics() []*ast.Diagnostic { + configFiles := map[tspath.PathKey]*ast.SourceFile{} + addConfig := func(config *tsoptions.ParsedCommandLine) { + if config != nil && config.ConfigFile != nil { + configFiles[config.ConfigFile.SourceFile.PathKey()] = config.ConfigFile.SourceFile + } + } + addConfig(p.CommandLine()) + p.RangeResolvedProjectReference(func(_ tspath.PathKey, config *tsoptions.ParsedCommandLine, _ *tsoptions.ParsedCommandLine, _ int) bool { + addConfig(config) + return true + }) + result := core.Filter(p.GetProgramDiagnostics(), func(d *ast.Diagnostic) bool { + return d.File() == nil || configFiles[d.File().PathKey()] != nil + }) + for _, file := range configFiles { + result = append(result, p.includeProcessor.getDiagnostics(p).GetDiagnosticsForFile(file)...) + } + return SortAndDeduplicateDiagnostics(result) +} + func (p *Program) collectContentMapperOptionDiagnostics() { project := p.ContentMapperProject() if project == nil {