From 4e803ae293eee7d15be9f263f1ce90adeb60f25f Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Fri, 23 Aug 2024 11:43:18 +0000 Subject: [PATCH] refactor(migrations): add support for simpler variant of tsurge migration (#57484) Introduces a simpler, smaller variant of the current `Tsurge` migration class. The difference is simply that for the migration phase (the third stage), some migrations do not need a full set of workers re-analyzing every compilation unit again to compute the "final migration replacements". This can be the case, for example, if a migration eagerly computes all replacements in the analyze stage, visiting every unit, and then after deriving the global metadata, problematic replacements are simply filtered (e.g. via some unique IDs again). PR Close #57484 --- .../signal-migration/src/migration.ts | 4 +- .../schematics/utils/tsurge/base_migration.ts | 58 +++++++ .../core/schematics/utils/tsurge/index.ts | 13 ++ .../core/schematics/utils/tsurge/migration.ts | 143 +++++++++++------- .../utils/tsurge/test/output_migration.ts | 4 +- .../schematics/utils/tsurge/testing/index.ts | 9 +- 6 files changed, 167 insertions(+), 64 deletions(-) create mode 100644 packages/core/schematics/utils/tsurge/base_migration.ts create mode 100644 packages/core/schematics/utils/tsurge/index.ts diff --git a/packages/core/schematics/migrations/signal-migration/src/migration.ts b/packages/core/schematics/migrations/signal-migration/src/migration.ts index 5babf0ae942..98ca7165e7f 100644 --- a/packages/core/schematics/migrations/signal-migration/src/migration.ts +++ b/packages/core/schematics/migrations/signal-migration/src/migration.ts @@ -10,7 +10,7 @@ import {NgtscProgram} from '../../../../../compiler-cli/src/ngtsc/program'; import {FileSystem} from '../../../../../compiler-cli/src/ngtsc/file_system'; import {confirmAsSerializable, Serializable} from '../../../utils/tsurge/helpers/serializable'; import {BaseProgramInfo, ProgramInfo} from '../../../utils/tsurge/program_info'; -import {TsurgeMigration} from '../../../utils/tsurge/migration'; +import {TsurgeComplexMigration} from '../../../utils/tsurge/migration'; import {CompilationUnitData} from './batch/unit_data'; import {KnownInputs} from './input_detection/known_inputs'; import {AnalysisProgramInfo, prepareAnalysisInfo} from './analysis_deps'; @@ -31,7 +31,7 @@ import {createNgtscProgram} from '../../../utils/tsurge/helpers/ngtsc_program'; * Tsurge migration for migrating Angular `@Input()` declarations to * signal inputs, with support for batch execution. */ -export class SignalInputMigration extends TsurgeMigration< +export class SignalInputMigration extends TsurgeComplexMigration< CompilationUnitData, CompilationUnitData, NgtscProgram, diff --git a/packages/core/schematics/utils/tsurge/base_migration.ts b/packages/core/schematics/utils/tsurge/base_migration.ts new file mode 100644 index 00000000000..73eb6f92f63 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/base_migration.ts @@ -0,0 +1,58 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import assert from 'assert'; +import path from 'path'; +import ts from 'typescript'; +import {FileSystem} from '../../../../compiler-cli/src/ngtsc/file_system'; +import {NgtscProgram} from '../../../../compiler-cli/src/ngtsc/program'; +import {isShim} from '../../../../compiler-cli/src/ngtsc/shims'; +import {createNgtscProgram} from './helpers/ngtsc_program'; +import {BaseProgramInfo, ProgramInfo} from './program_info'; + +/** + * Base class for the possible Tsurge migration variants. + * + * For example, this class exposes methods to conveniently create + * TypeScript programs, while also allowing migration authors to override. + */ +export abstract class TsurgeBaseMigration< + TsProgramType extends ts.Program | NgtscProgram = NgtscProgram, + PreparationInfo = ProgramInfo, +> { + // By default, ngtsc programs are being created. + createProgram(tsconfigAbsPath: string, fs?: FileSystem): BaseProgramInfo { + return createNgtscProgram(tsconfigAbsPath, fs) as BaseProgramInfo; + } + + // Optional function to prepare the base `ProgramInfo` even further, + // for the analyze and migrate phases. E.g. determining source files. + prepareProgram(info: BaseProgramInfo): PreparationInfo { + assert(info.program instanceof NgtscProgram); + + const userProgram = info.program.getTsProgram(); + const fullProgramSourceFiles = userProgram.getSourceFiles(); + const sourceFiles = fullProgramSourceFiles.filter( + (f) => + !f.isDeclarationFile && + // Note `isShim` will work for the initial program, but for TCB programs, the shims are no longer annotated. + !isShim(f) && + !f.fileName.endsWith('.ngtypecheck.ts'), + ); + + const basePath = path.dirname(info.tsconfigAbsolutePath); + const projectDirAbsPath = info.userOptions.rootDir ?? basePath; + + return { + ...info, + sourceFiles, + fullProgramSourceFiles, + projectDirAbsPath, + } as PreparationInfo; + } +} diff --git a/packages/core/schematics/utils/tsurge/index.ts b/packages/core/schematics/utils/tsurge/index.ts new file mode 100644 index 00000000000..39a35112f90 --- /dev/null +++ b/packages/core/schematics/utils/tsurge/index.ts @@ -0,0 +1,13 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +export * from './migration'; +export * from './program_info'; +export * from './replacement'; +export * from './helpers/unique_id'; +export * from './helpers/serializable'; diff --git a/packages/core/schematics/utils/tsurge/migration.ts b/packages/core/schematics/utils/tsurge/migration.ts index aaea31a6356..d5a34578839 100644 --- a/packages/core/schematics/utils/tsurge/migration.ts +++ b/packages/core/schematics/utils/tsurge/migration.ts @@ -6,21 +6,15 @@ * found in the LICENSE file at https://angular.io/license */ -import {FileSystem} from '../../../../compiler-cli/src/ngtsc/file_system'; -import {NgtscProgram} from '../../../../compiler-cli/src/ngtsc/program'; -import assert from 'assert'; -import path from 'path'; import ts from 'typescript'; -import {isShim} from '../../../../compiler-cli/src/ngtsc/shims'; -import {createNgtscProgram} from './helpers/ngtsc_program'; +import {NgtscProgram} from '../../../../compiler-cli/src/ngtsc/program'; +import {TsurgeBaseMigration} from './base_migration'; import {Serializable} from './helpers/serializable'; +import {ProgramInfo} from './program_info'; import {Replacement} from './replacement'; -import {BaseProgramInfo, ProgramInfo} from './program_info'; /** - * Class defining a `Tsurge` migration. - * - * A tsurge migration is split into three stages: + * A Tsurge migration is split into three stages: * - analyze phase * - merge phase * - migrate phase @@ -30,69 +24,104 @@ import {BaseProgramInfo, ProgramInfo} from './program_info'; * individual workers are never seeing the full project, e.g. Google3. * * The analysis phases can operate on smaller TS project units, and later - * the expect the isolated unit data to be merged into some sort of global - * metadata via the `merge` phase. For example, every analyze worker may - * contribute to a list of TS references that are later combined. + * then expect the isolated unit data to be merged into some sort of global + * metadata via the `merge` phase. As a final step then, the migration + * replacements will be computed. * - * The migrate phase can then compute actual file updates for all individual - * compilation units, leveraging the global metadata to e.g. see if there are - * any references from other compilation units that may be problematic and prevent - * migration of a given file. + * There are subtle differences in how the final stage can compute migration + * replacements. Some migrations need program access again, while other's don't. + * For this reason, there are two possible variants of Tsurge migrations: * - * More details can be found in the design doc for signal input migration, - * or in the testing examples. + * - {@link TsurgeFunnelMigration} + * - {@link TsurgeComplexMigration} * - * TODO: Link design doc. + * TODO: Link design doc */ -export abstract class TsurgeMigration< +export type TsurgeMigration< UnitAnalysisMetadata, CombinedGlobalMetadata, TsProgramType extends ts.Program | NgtscProgram = NgtscProgram, PreparationInfo = ProgramInfo, -> { - // By default, ngtsc programs are being created. - createProgram(tsconfigAbsPath: string, fs?: FileSystem): BaseProgramInfo { - return createNgtscProgram(tsconfigAbsPath, fs) as BaseProgramInfo; - } - - // Optional function to prepare the base `ProgramInfo` even further, - // for the analyze and migrate phases. E.g. determining source files. - prepareProgram(info: BaseProgramInfo): PreparationInfo { - assert(info.program instanceof NgtscProgram); - - const userProgram = info.program.getTsProgram(); - const fullProgramSourceFiles = userProgram.getSourceFiles(); - const sourceFiles = fullProgramSourceFiles.filter( - (f) => - !f.isDeclarationFile && - // Note `isShim` will work for the initial program, but for TCB programs, the shims are no longer annotated. - !isShim(f) && - !f.fileName.endsWith('.ngtypecheck.ts'), - ); - - const basePath = path.dirname(info.tsconfigAbsolutePath); - const projectDirAbsPath = info.userOptions.rootDir ?? basePath; - - return { - ...info, - sourceFiles, - fullProgramSourceFiles, - projectDirAbsPath, - } as PreparationInfo; - } +> = + | TsurgeComplexMigration< + UnitAnalysisMetadata, + CombinedGlobalMetadata, + TsProgramType, + PreparationInfo + > + | TsurgeFunnelMigration< + UnitAnalysisMetadata, + CombinedGlobalMetadata, + TsProgramType, + PreparationInfo + >; +/** + * A simpler variant of a {@link TsurgeComplexMigration} that does not + * fan-out into multiple workers per compilation unit to compute + * the final migration replacements. + * + * This is faster and less resource intensive as workers and TS programs + * are only ever created once. + * + * This is commonly the case when migrations are refactored to eagerly + * compute replacements in the analyze stage, and then leverage the + * global unit data to filter replacements that turned out to be "invalid". + */ +export abstract class TsurgeFunnelMigration< + UnitAnalysisMetadata, + CombinedGlobalMetadata, + TsProgramType extends ts.Program | NgtscProgram = NgtscProgram, + PreparationInfo = ProgramInfo, +> extends TsurgeBaseMigration { /** Analyzes the given TypeScript project and returns serializable compilation unit data. */ - abstract analyze(program: PreparationInfo): Promise>; + abstract analyze(info: PreparationInfo): Promise>; - /** Merges all compilation unit data from previous analysis phases into a global metadata. */ + /** Merges all compilation unit data from previous analysis phases into a global result. */ abstract merge(units: UnitAnalysisMetadata[]): Promise>; /** - * Computes migration updates for the given TypeScript project, leveraging the global - * metadata built up from all analyzed projects and their merged "unit data". + * Finalizes the migration result. + * + * This stage can be used to filter replacements, leveraging global combined analysis + * data to compute the overall migration result, without needing new "migrate" workers + * for every unit, as with a standard {@link TsurgeMigration}. + * + * @returns All replacements for the whole project. + */ + abstract migrate(globalData: CombinedGlobalMetadata): Promise>; +} + +/** + * Complex variant of a `Tsurge` migration. + * + * For example, every analyze worker may contribute to a list of TS + * references that are later combined. The migrate phase can then compute actual + * file updates for all individual compilation units, leveraging the global metadata + * to e.g. see if there are any references from other compilation units that may be + * problematic and prevent migration of a given file. + */ +export abstract class TsurgeComplexMigration< + UnitAnalysisMetadata, + CombinedGlobalMetadata, + TsProgramType extends ts.Program | NgtscProgram = NgtscProgram, + PreparationInfo = ProgramInfo, +> extends TsurgeBaseMigration { + /** Analyzes the given TypeScript project and returns serializable compilation unit data. */ + abstract analyze(info: PreparationInfo): Promise>; + + /** Merges all compilation unit data from previous analysis phases into a global result. */ + abstract merge(units: UnitAnalysisMetadata[]): Promise>; + + /** + * Migration phase. Workers will be started for every compilation unit again, + * instantiating a new program for every unit to compute the final migration + * replacements, leveraging combined global metadata. + * + * @returns Replacements for the given compilation unit (**not** the whole project!) */ abstract migrate( globalMetadata: CombinedGlobalMetadata, - program: PreparationInfo, + info: PreparationInfo, ): Promise; } diff --git a/packages/core/schematics/utils/tsurge/test/output_migration.ts b/packages/core/schematics/utils/tsurge/test/output_migration.ts index 38c70ff8a0d..af6907cfffe 100644 --- a/packages/core/schematics/utils/tsurge/test/output_migration.ts +++ b/packages/core/schematics/utils/tsurge/test/output_migration.ts @@ -11,7 +11,7 @@ import {absoluteFromSourceFile} from '../../../../../compiler-cli/src/ngtsc/file import {TypeScriptReflectionHost} from '../../../../../compiler-cli/src/ngtsc/reflection'; import {DtsMetadataReader} from '../../../../../compiler-cli/src/ngtsc/metadata'; import {confirmAsSerializable} from '../helpers/serializable'; -import {TsurgeMigration} from '../migration'; +import {TsurgeComplexMigration} from '../migration'; import {Replacement, TextUpdate} from '../replacement'; import {findOutputDeclarationsAndReferences, OutputID} from './output_helpers'; import {ProgramInfo} from '../program_info'; @@ -26,7 +26,7 @@ type GlobalMetadata = {[id: OutputID]: {canBeMigrated: boolean}}; * Note that this is simply a testing construct for now, to verify the migration * framework works as expected. This is **not a full migration**, but rather an example. */ -export class OutputMigration extends TsurgeMigration { +export class OutputMigration extends TsurgeComplexMigration { override async analyze(info: ProgramInfo) { const program = info.program.getTsProgram(); const typeChecker = program.getTypeChecker(); diff --git a/packages/core/schematics/utils/tsurge/testing/index.ts b/packages/core/schematics/utils/tsurge/testing/index.ts index 1e1c8337e09..0dee3c3cc7b 100644 --- a/packages/core/schematics/utils/tsurge/testing/index.ts +++ b/packages/core/schematics/utils/tsurge/testing/index.ts @@ -6,7 +6,7 @@ * found in the LICENSE file at https://angular.io/license */ -import {TsurgeMigration} from '../migration'; +import {TsurgeFunnelMigration, TsurgeMigration} from '../migration'; import { initMockFileSystem, MockFileSystem, @@ -61,9 +61,12 @@ export async function runTsurgeMigration( const unitData = await migration.analyze(info); const merged = await migration.merge([unitData]); - const replacements = await migration.migrate(merged, info); - const updates = groupReplacementsByFile(replacements); + const replacements = + migration instanceof TsurgeFunnelMigration + ? await migration.migrate(merged) + : await migration.migrate(merged, info); + const updates = groupReplacementsByFile(replacements); for (const [filePath, changes] of updates.entries()) { mockFs.writeFile(filePath, applyTextUpdates(mockFs.readFile(filePath), changes)); }