chore: add --compress flag to cdk migrate (#27184)

This also updates some of the aliases used because they clashed with other flags. We'll just remove these.

The archive file is shamelessly ripped off from the cdk-assets package.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: TheRealAmazonKendra

packages/aws-cdk/lib/cdk-toolkit.ts

@@ -713,7 +713,7 @@ export class CdkToolkit {
         await readFromStack(options.stackName, this.props.sdkProvider, setEnvironment(options.account, options.region));
       const stack = generateStack(template!, options.stackName, language);
       success(' ⏳  Generating CDK app for %s...', chalk.blue(options.stackName));
-      await generateCdkApp(options.stackName, stack!, language, options.outputPath);
+      await generateCdkApp(options.stackName, stack!, language, options.outputPath, options.compress);
     } catch (e) {
       error(' ❌  Migrate failed for `%s`: %s', chalk.blue(options.stackName), (e as Error).message);
       throw e;
@@ -1244,6 +1244,12 @@ export interface MigrateOptions {
    */
   readonly region?: string;
 
+  /**
+   * Whether to zip the generated cdk app folder.
+   *
+   * @default false
+   */
+  readonly compress?: boolean;
 }
 
 /**

packages/aws-cdk/lib/cli.ts

@@ -274,11 +274,12 @@ async function parseCommandLineArguments(args: string[]) {
     .command('migrate', false /* hidden from "cdk --help" */, (yargs: Argv) => yargs
       .option('stack-name', { type: 'string', alias: 'n', desc: 'The name assigned to the stack created in the new project. The name of the app will be based off this name as well.', requiresArg: true })
       .option('language', { type: 'string', default: 'typescript', alias: 'l', desc: 'The language to be used for the new project', choices: MIGRATE_SUPPORTED_LANGUAGES })
-      .option('account', { type: 'string', alias: 'a' })
-      .option('region', { type: 'string' })
-      .option('from-path', { type: 'string', alias: 'p', desc: 'The path to the CloudFormation template to migrate. Use this for locally stored templates' })
-      .option('from-stack', { type: 'boolean', alias: 's', desc: 'USe this flag to retrieve the template for an existing CloudFormation stack' })
-      .option('output-path', { type: 'string', alias: 'o', desc: 'The output path for the migrated cdk app' }),
+      .option('account', { type: 'string', desc: 'The account to retrieve the CloudFormation stack template from' })
+      .option('region', { type: 'string', desc: 'The region to retrieve the CloudFormation stack template from' })
+      .option('from-path', { type: 'string', desc: 'The path to the CloudFormation template to migrate. Use this for locally stored templates' })
+      .option('from-stack', { type: 'boolean', desc: 'Use this flag to retrieve the template for an existing CloudFormation stack' })
+      .option('output-path', { type: 'string', desc: 'The output path for the migrated CDK app' })
+      .option('compress', { type: 'boolean', desc: 'Use this flag to zip the generated CDK app' }),
     )
     .command('context', 'Manage cached context values', (yargs: Argv) => yargs
       .option('reset', { alias: 'e', desc: 'The context key (or its index) to reset', type: 'string', requiresArg: true })
@@ -659,7 +660,12 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise<n
         if (args.list) {
           return printAvailableTemplates(language);
         } else {
-          return cliInit(args.TEMPLATE, language, undefined, args.generateOnly);
+          return cliInit({
+            type: args.TEMPLATE,
+            language,
+            canUseNetwork: undefined,
+            generateOnly: args.generateOnly,
+          });
         }
       case 'migrate':
         return cli.migrate({
@@ -670,6 +676,7 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise<n
           outputPath: args['output-path'],
           account: args.account,
           region: args.region,
+          compress: args.compress,
         });
       case 'version':
         return data(version.DISPLAY_VERSION);

packages/aws-cdk/lib/commands/migrate.ts

@@ -1,14 +1,14 @@
+/* eslint-disable @typescript-eslint/no-require-imports */
+/* eslint-disable @typescript-eslint/no-var-requires */
 import * as fs from 'fs';
 import * as path from 'path';
 import { Environment, UNKNOWN_ACCOUNT, UNKNOWN_REGION } from '@aws-cdk/cx-api';
 import * as cdk_from_cfn from 'cdk-from-cfn';
 import { cliInit } from '../../lib/init';
 import { Mode, SdkProvider } from '../api';
+import { zipDirectory } from '../util/archive';
 
-/* eslint-disable @typescript-eslint/no-var-requires */ // Packages don't have @types module
-// eslint-disable-next-line @typescript-eslint/no-require-imports
 const camelCase = require('camelcase');
-// eslint-disable-next-line @typescript-eslint/no-require-imports
 const decamelize = require('decamelize');
 
 /** The list of languages supported by the built-in noctilucent binary. */
@@ -22,14 +22,22 @@ export const MIGRATE_SUPPORTED_LANGUAGES: readonly string[] = cdk_from_cfn.suppo
  * @param language The language to generate the CDK app in
  * @param outputPath The path at which to generate the CDK app
  */
-export async function generateCdkApp(stackName: string, stack: string, language: string, outputPath?: string) {
+export async function generateCdkApp(stackName: string, stack: string, language: string, outputPath?: string, compress?: boolean) {
   const resolvedOutputPath = path.join(outputPath ?? process.cwd(), stackName);
   const formattedStackName = decamelize(stackName);
 
   try {
     fs.rmSync(resolvedOutputPath, { recursive: true, force: true });
     fs.mkdirSync(resolvedOutputPath, { recursive: true });
-    await cliInit('app', language, true, false, resolvedOutputPath, stackName);
+    const generateOnly = compress;
+    await cliInit({
+      type: 'app',
+      language,
+      canUseNetwork: true,
+      generateOnly,
+      workDir: resolvedOutputPath,
+      stackName,
+    });
 
     let stackFileName: string;
     switch (language) {
@@ -50,6 +58,10 @@ export async function generateCdkApp(stackName: string, stack: string, language:
         throw new Error(`${language} is not supported by CDK Migrate. Please choose from: ${MIGRATE_SUPPORTED_LANGUAGES.join(', ')}`);
     }
     fs.writeFileSync(stackFileName, stack);
+    if (compress) {
+      await zipDirectory(resolvedOutputPath, `${resolvedOutputPath}.zip`);
+      fs.rmSync(resolvedOutputPath, { recursive: true, force: true });
+    }
   } catch (error) {
     fs.rmSync(resolvedOutputPath, { recursive: true, force: true });
     throw error;

packages/aws-cdk/lib/init.ts

@@ -14,39 +14,44 @@ const camelCase = require('camelcase');
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 const decamelize = require('decamelize');
 
+export interface CliInitOptions {
+  readonly type?: string;
+  readonly language?: string;
+  readonly canUseNetwork?: boolean;
+  readonly generateOnly?: boolean;
+  readonly workDir?: string;
+  readonly stackName?: string;
+}
+
 /**
  * Initialize a CDK package in the current directory
  */
-export async function cliInit(
-  type?: string,
-  language?: string,
-  canUseNetwork = true,
-  generateOnly = false,
-  workDir = process.cwd(),
-  stackName?: string,
-) {
-  if (!type && !language) {
+export async function cliInit(options: CliInitOptions) {
+  const canUseNetwork = options.canUseNetwork ?? true;
+  const generateOnly = options.generateOnly ?? false;
+  const workDir = options.workDir ?? process.cwd();
+  if (!options.type && !options.language) {
     await printAvailableTemplates();
     return;
   }
 
-  type = type || 'default'; // "default" is the default type (and maps to "app")
+  const type = options.type || 'default'; // "default" is the default type (and maps to "app")
 
   const template = (await availableInitTemplates()).find(t => t.hasName(type!));
   if (!template) {
-    await printAvailableTemplates(language);
+    await printAvailableTemplates(options.language);
     throw new Error(`Unknown init template: ${type}`);
   }
-  if (!language && template.languages.length === 1) {
-    language = template.languages[0];
+  if (!options.language && template.languages.length === 1) {
+    const language = template.languages[0];
     warning(`No --language was provided, but '${type}' supports only '${language}', so defaulting to --language=${language}`);
   }
-  if (!language) {
+  if (!options.language) {
     print(`Available languages for ${chalk.green(type)}: ${template.languages.map(l => chalk.blue(l)).join(', ')}`);
     throw new Error('No language was selected');
   }
 
-  await initializeProject(template, language, canUseNetwork, generateOnly, workDir, stackName);
+  await initializeProject(template, options.language, canUseNetwork, generateOnly, workDir, options.stackName);
 }
 
 /**

packages/aws-cdk/lib/util/archive.ts

@@ -0,0 +1,90 @@
+import { error } from 'console';
+import { createWriteStream, promises as fs } from 'fs';
+import * as path from 'path';
+import * as glob from 'glob';
+
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const archiver = require('archiver');
+
+// Adapted from cdk-assets
+export async function zipDirectory(directory: string, outputFile: string): Promise<void> {
+  // We write to a temporary file and rename at the last moment. This is so that if we are
+  // interrupted during this process, we don't leave a half-finished file in the target location.
+  const temporaryOutputFile = `${outputFile}.${randomString()}._tmp`;
+  await writeZipFile(directory, temporaryOutputFile);
+  await moveIntoPlace(temporaryOutputFile, outputFile);
+}
+
+function writeZipFile(directory: string, outputFile: string): Promise<void> {
+  return new Promise(async (ok, fail) => {
+    // The below options are needed to support following symlinks when building zip files:
+    // - nodir: This will prevent symlinks themselves from being copied into the zip.
+    // - follow: This will follow symlinks and copy the files within.
+    const globOptions = {
+      dot: true,
+      nodir: true,
+      follow: true,
+      cwd: directory,
+    };
+    const files = glob.sync('**', globOptions); // The output here is already sorted
+
+    const output = createWriteStream(outputFile);
+
+    const archive = archiver('zip');
+    archive.on('warning', fail);
+    archive.on('error', fail);
+
+    // archive has been finalized and the output file descriptor has closed, resolve promise
+    // this has to be done before calling `finalize` since the events may fire immediately after.
+    // see https://www.npmjs.com/package/archiver
+    output.once('close', ok);
+
+    archive.pipe(output);
+
+    // Append files serially to ensure file order
+    for (const file of files) {
+      const fullPath = path.resolve(directory, file);
+      const [data, stat] = await Promise.all([fs.readFile(fullPath), fs.stat(fullPath)]);
+      archive.append(data, {
+        name: file,
+        mode: stat.mode,
+      });
+    }
+
+    await archive.finalize();
+  });
+}
+
+/**
+ * Rename the file to the target location, taking into account:
+ *
+ * - That we may see EPERM on Windows while an Antivirus scanner still has the
+ *   file open, so retry a couple of times.
+ * - This same function may be called in parallel and be interrupted at any point.
+ */
+async function moveIntoPlace(source: string, target: string) {
+  let delay = 100;
+  let attempts = 5;
+  while (true) {
+    try {
+      // 'rename' is guaranteed to overwrite an existing target, as long as it is a file (not a directory)
+      await fs.rename(source, target);
+      return;
+    } catch (e: any) {
+      if (e.code !== 'EPERM' || attempts-- <= 0) {
+        throw e;
+      }
+      error(e.message);
+      await sleep(Math.floor(Math.random() * delay));
+      delay *= 2;
+    }
+  }
+}
+
+function sleep(ms: number) {
+  return new Promise(ok => setTimeout(ok, ms));
+}
+
+function randomString() {
+  return Math.random().toString(36).replace(/[^a-z0-9]+/g, '');
+}

packages/aws-cdk/test/commands/migrate.test.ts

@@ -1,9 +1,13 @@
+import { exec as _exec } from 'child_process';
 import * as os from 'os';
 import * as path from 'path';
+import { promisify } from 'util';
 import * as fs from 'fs-extra';
 import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, validateSourceOptions } from '../../lib/commands/migrate';
 import { MockSdkProvider, MockedObject, SyncHandlerSubsetOf } from '../util/mock-sdk';
 
+const exec = promisify(_exec);
+
 describe('Migrate Function Tests', () => {
   let sdkProvider: MockSdkProvider;
   let getTemplateMock: jest.Mock;
@@ -173,6 +177,27 @@ describe('Migrate Function Tests', () => {
     const replacedStack = fs.readFileSync(path.join(workDir, 'GoodCSharp', 'src', 'GoodCSharp', 'GoodCSharpStack.cs'));
     expect(replacedStack).toEqual(fs.readFileSync(path.join(...stackPath, 'S3Stack.cs')));
   });
+
+  cliTest('generatedCdkApp generates a zip file when --compress is used', async (workDir) => {
+    const stack = generateStack(validTemplate, 'GoodTypeScript', 'typescript');
+    await generateCdkApp('GoodTypeScript', stack, 'typescript', workDir, true);
+
+    // Packages not in outDir
+    expect(fs.pathExistsSync(path.join(workDir, 'GoodTypeScript', 'package.json'))).toBeFalsy();
+    expect(fs.pathExistsSync(path.join(workDir, 'GoodTypeScript', 'bin', 'good_type_script.ts'))).toBeFalsy();
+    expect(fs.pathExistsSync(path.join(workDir, 'GoodTypeScript', 'lib', 'good_type_script-stack.ts'))).toBeFalsy();
+
+    // Zip file exists
+    expect(fs.pathExistsSync(path.join(workDir, 'GoodTypeScript.zip'))).toBeTruthy();
+
+    // Unzip it
+    await exec(`unzip ${path.join(workDir, 'GoodTypeScript.zip')}`, { cwd: workDir });
+
+    // Now the files should be there
+    expect(fs.pathExistsSync(path.join(workDir, 'package.json'))).toBeTruthy();
+    expect(fs.pathExistsSync(path.join(workDir, 'bin', 'good_type_script.ts'))).toBeTruthy();
+    expect(fs.pathExistsSync(path.join(workDir, 'lib', 'good_type_script-stack.ts'))).toBeTruthy();
+  });
 });
 
 function cliTest(name: string, handler: (dir: string) => void | Promise<any>): void {