feat(cli): allow change set imports resources that already exist (#447)

Allow `cdk diff` command to create a change set that imports existing
resources.

The current `cdk diff` command implicitly calls
CloudFormation change set creation, providing high-level details such as
"add", "delete", "modify", "import",
and etc. like the following:

```s
$ cdk diff
[-] AWS::DynamoDB::Table MyTable orphan
[+] AWS::DynamoDB::GlobalTable MyGlobalTable add
```

However, when the resource is meant to be imported, the `cdk diff`
command still shows this as add. Adding `cdk diff
--import-existing-resources` flag to show the new resource being
imported instead of `add`.

```s
$ cdk diff --import-existing-resources
[-] AWS::DynamoDB::Table MyTable orphan
[←] AWS::DynamoDB::GlobalTable MyGlobalTable import
```

Here is the underlying CFN change set JSON output
```json
[
  {
    "type": "Resource",
    "resourceChange": {
      "action": "Import", # NOTE THAT THIS SHOWS "Import"
      "logicalResourceId": "MyTable794EDED1",
      "physicalResourceId": "DemoStack-MyTable794EDED1-11W4MR8VZ0UPE",
      "resourceType": "AWS::DynamoDB::GlobalTable",
      "replacement": "True",
      "scope": [],
      "details": [],
      "afterContext": "..."
    }
  },
  {
    "type": "Resource",
    "resourceChange": {
      "policyAction": "Retain", # Note that this is "Retain"
      "action": "Remove",
      "logicalResourceId": "MyTable794EDED1",
      "physicalResourceId": "DemoStack-MyTable794EDED1-11W4MR8VZ0UPE",
      "resourceType": "AWS::DynamoDB::Table",
      "scope": [],
      "details": [],
      "beforeContext": "..."
    }
  }
]
```

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

---------

Signed-off-by: github-actions <github-actions@github.com>
Co-authored-by: Momo Kornher <kornherm@amazon.co.uk>
Co-authored-by: Rico Huijbers <rix0rrr@gmail.com>

Author: 3 people

packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/import-app/app.js

@@ -0,0 +1,38 @@
+const cdk = require('aws-cdk-lib/core');
+const dynamodb = require('aws-cdk-lib/aws-dynamodb');
+
+const stackPrefix = process.env.STACK_NAME_PREFIX;
+if (!stackPrefix) {
+  throw new Error(`the STACK_NAME_PREFIX environment variable is required`);
+}
+
+class BaseStack extends cdk.Stack {
+  constructor(scope, id, props) {
+    super(scope, id, props);
+
+    // Create a random table name with prefix
+    if (process.env.VERSION == 'v2') {
+      new dynamodb.TableV2(this, 'MyGlobalTable', {
+        partitionKey: {
+          name: 'PK',
+          type: dynamodb.AttributeType.STRING,
+        },
+        tableName: 'integ-test-import-app-base-table-1',
+      });
+    } else {
+      new dynamodb.Table(this, 'MyTable', {
+        partitionKey: {
+          name: 'PK',
+          type: dynamodb.AttributeType.STRING,
+        },
+        tableName: 'integ-test-import-app-base-table-1',
+        removalPolicy: process.env.VERSION == 'v1' ? cdk.RemovalPolicy.RETAIN : cdk.RemovalPolicy.DESTROY,
+      });
+    }
+  }
+}
+
+const app = new cdk.App();
+new BaseStack(app, `${stackPrefix}-base-1`);
+
+app.synth();

packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/import-app/cdk.json

@@ -0,0 +1,7 @@
+{
+  "app": "node app.js",
+  "versionReporting": false,
+  "context": {
+    "aws-cdk:enableDiffNoFail": "true"
+  }
+}

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-cdk-diff--import-existing-resources-shows-import.integtest.ts

@@ -0,0 +1,45 @@
+import { integTest, withSpecificFixture } from '../../lib';
+
+jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
+
+integTest(
+  'cdk diff --import-existing-resources show resource being imported',
+  withSpecificFixture('import-app', async (fixture) => {
+    // GIVEN
+    await fixture.cdkDeploy('base-1', {
+      modEnv: {
+        VERSION: 'v1',
+      },
+    });
+
+    // THEN
+    let diff = await fixture.cdk(['diff', '--import-existing-resources', fixture.fullStackName('base-1')], {
+      modEnv: {
+        VERSION: 'v2',
+      },
+    });
+
+    // Assert there are no changes and diff shows import
+    expect(diff).not.toContain('There were no differences');
+    expect(diff).toContain('[←]');
+    expect(diff).toContain('import');
+
+    // THEN
+    diff = await fixture.cdk(['diff', fixture.fullStackName('base-1')], {
+      modEnv: {
+        VERSION: 'v2',
+      },
+    });
+
+    // Assert there are no changes and diff shows add
+    expect(diff).not.toContain('There were no differences');
+    expect(diff).toContain('[+]');
+
+    // Deploy the stack with v3 to set table removal policy as destroy
+    await fixture.cdkDeploy('base-1', {
+      modEnv: {
+        VERSION: 'v3',
+      },
+    });
+  }),
+);

packages/@aws-cdk/toolkit-lib/lib/actions/diff/index.ts

@@ -26,6 +26,13 @@ export interface ChangeSetDiffOptions extends CloudFormationDiffOptions {
    * @default - no parameters
    */
   readonly parameters?: { [name: string]: string | undefined };
+
+  /**
+   * Whether or not the change set imports resources that already exist
+   *
+   * @default false
+   */
+  readonly importExistingResources?: boolean;
 }
 
 export interface LocalFileDiffOptions {

packages/@aws-cdk/toolkit-lib/lib/actions/diff/private/helpers.ts

@@ -89,6 +89,7 @@ async function cfnDiff(
       resourcesToImport,
       methodOptions.parameters,
       methodOptions.fallbackToTemplate,
+      methodOptions.importExistingResources,
     ) : undefined;
 
     templateInfos.push({
@@ -111,6 +112,7 @@ async function changeSetDiff(
   resourcesToImport?: ResourcesToImport,
   parameters: { [name: string]: string | undefined } = {},
   fallBackToTemplate: boolean = true,
+  importExistingResources: boolean = false,
 ): Promise<any | undefined> {
   let stackExists = false;
   try {
@@ -139,6 +141,7 @@ async function changeSetDiff(
       parameters: parameters,
       resourcesToImport,
       failOnError: !fallBackToTemplate,
+      importExistingResources,
     });
   } else {
     if (!fallBackToTemplate) {

packages/@aws-cdk/toolkit-lib/lib/api/deployments/cfn-api.ts

@@ -142,6 +142,7 @@ export type PrepareChangeSetOptions = {
   sdkProvider: SdkProvider;
   parameters: { [name: string]: string | undefined };
   resourcesToImport?: ResourcesToImport;
+  importExistingResources?: boolean;
   /**
    * Default behavior is to log AWS CloudFormation errors and move on. Set this property to true to instead
    * fail on errors received by AWS CloudFormation.
@@ -161,6 +162,7 @@ export type CreateChangeSetOptions = {
   bodyParameter: TemplateBodyParameter;
   parameters: { [name: string]: string | undefined };
   resourcesToImport?: ResourceToImport[];
+  importExistingResources?: boolean;
   role?: string;
 };
 
@@ -244,6 +246,7 @@ async function uploadBodyParameterAndCreateChangeSet(
       bodyParameter,
       parameters: options.parameters,
       resourcesToImport: options.resourcesToImport,
+      importExistingResources: options.importExistingResources,
       role: executionRoleArn,
     });
   } catch (e: any) {
@@ -309,6 +312,7 @@ export async function createChangeSet(
     TemplateBody: options.bodyParameter.TemplateBody,
     Parameters: stackParams.apiParameters,
     ResourcesToImport: options.resourcesToImport,
+    ImportExistingResources: options.importExistingResources,
     RoleARN: options.role,
     Tags: toCfnTags(options.stack.tags),
     Capabilities: ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND'],

packages/@aws-cdk/toolkit-lib/test/actions/diff.test.ts

@@ -5,6 +5,7 @@ import * as awsauth from '../../lib/api/aws-auth/private';
 import { StackSelectionStrategy } from '../../lib/api/cloud-assembly';
 import * as deployments from '../../lib/api/deployments';
 import { RequireApproval } from '../../lib/api/require-approval';
+import { cfnApi } from '../../lib/api/shared-private';
 import { Toolkit } from '../../lib/toolkit';
 import { builderFixture, disposableCloudAssemblySource, TestIoHost } from '../_helpers';
 import { MockSdk, restoreSdkMocksToDefault, setDefaultSTSMocks } from '../_helpers/mock-sdk';
@@ -242,6 +243,35 @@ describe('diff', () => {
       }));
     });
 
+    test('ChangeSet diff method with import existing resources option enabled', async () => {
+      // Setup mock BEFORE calling the function
+      const createDiffChangeSetMock = jest.spyOn(cfnApi, 'createDiffChangeSet').mockImplementationOnce(async () => {
+        return {
+          $metadata: {},
+          Changes: [
+            {
+              ResourceChange: {
+                Action: 'Import',
+                LogicalResourceId: 'MyBucketF68F3FF0',
+              },
+            },
+          ],
+        };
+      });
+
+      // WHEN
+      ioHost.level = 'debug';
+      const cx = await builderFixture(toolkit, 'stack-with-bucket');
+      const result = await toolkit.diff(cx, {
+        stacks: { strategy: StackSelectionStrategy.ALL_STACKS },
+        method: DiffMethod.ChangeSet({ importExistingResources: true }),
+      });
+
+      // THEN
+      expect(createDiffChangeSetMock).toHaveBeenCalled();
+      expect(result.Stack1.resources.get('MyBucketF68F3FF0').isImport).toBe(true);
+    });
+
     test('ChangeSet diff method throws if changeSet fails and fallBackToTemplate = false', async () => {
       // WHEN
       const cx = await builderFixture(toolkit, 'stack-with-bucket');

packages/aws-cdk/README.md

@@ -189,6 +189,23 @@ The `change-set` flag will make `diff` create a change set and extract resource
 The `--no-change-set` mode will consider any change to a property that requires replacement to be a resource replacement,
 even if the change is purely cosmetic (like replacing a resource reference with a hardcoded arn).
 
+The `--import-existing-resources` option will make `diff` create a change set and compare it using
+the CloudFormation resource import mechanism. This allows CDK to detect changes and show report of resources that
+will be imported rather added. Use this flag when preparing to import existing resources into a CDK stack to
+ensure and validate the changes are correctly reflected by showing 'import'.
+
+```console
+$ cdk diff
+[+] AWS::DynamoDB::GlobalTable MyGlobalTable MyGlobalTable5DC12DB4
+
+$ cdk diff --import-existing-resources
+[←] AWS::DynamoDB::GlobalTable MyGlobalTable MyGlobalTable5DC12DB4 import
+```
+
+In the output above:
+[+] indicates a new resource that would be created.
+[←] indicates a resource that would be imported into the stack instead.
+
 ### `cdk deploy`
 
 Deploys a stack of your CDK app to its environment. During the deployment, the toolkit will output progress

packages/aws-cdk/lib/cli/cdk-toolkit.ts

@@ -213,6 +213,12 @@ export class CdkToolkit {
         throw new ToolkitError(`There is no file at ${options.templatePath}`);
       }
 
+      if (options.importExistingResources) {
+        throw new ToolkitError(
+          'Can only use --import-existing-resources flag when comparing against deployed stacks.',
+        );
+      }
+
       const template = deserializeStructure(await fs.readFile(options.templatePath, { encoding: 'UTF-8' }));
       const formatter = new DiffFormatter({
         ioHelper: asIoHelper(this.ioHost, 'diff'),
@@ -287,6 +293,7 @@ export class CdkToolkit {
               sdkProvider: this.props.sdkProvider,
               parameters: Object.assign({}, parameterMap['*'], parameterMap[stack.stackName]),
               resourcesToImport,
+              importExistingResources: options.importExistingResources,
             });
           } else {
             debug(
@@ -1514,6 +1521,13 @@ export interface DiffOptions {
    * @default true
    */
   readonly changeSet?: boolean;
+
+  /**
+   * Whether or not the change set imports resources that already exist.
+   *
+   * @default false
+   */
+  readonly importExistingResources?: boolean;
 }
 
 interface CfnDeployOptions {

packages/aws-cdk/lib/cli/cli-config.ts

@@ -336,6 +336,7 @@ export async function makeConfig(): Promise<CliConfig> {
           'processed': { type: 'boolean', desc: 'Whether to compare against the template with Transforms already processed', default: false },
           'quiet': { type: 'boolean', alias: 'q', desc: 'Do not print stack name and default message when there is no diff to stdout', default: false },
           'change-set': { type: 'boolean', alias: 'changeset', desc: 'Whether to create a changeset to analyze resource replacements. In this mode, diff will use the deploy role instead of the lookup role.', default: true },
+          'import-existing-resources': { type: 'boolean', desc: 'Whether or not the change set imports resources that already exist', default: false },
         },
       },
       metadata: {

packages/aws-cdk/lib/cli/cli.ts

@@ -258,6 +258,7 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise<n
           quiet: args.quiet,
           changeSet: args['change-set'],
           toolkitStackName: toolkitStackName,
+          importExistingResources: args.importExistingResources,
         });
 
       case 'refactor':

packages/aws-cdk/lib/cli/convert-to-user-input.ts

@@ -191,6 +191,7 @@ export function convertYargsToUserInput(args: any): UserInput {
         processed: args.processed,
         quiet: args.quiet,
         changeSet: args.changeSet,
+        importExistingResources: args.importExistingResources,
         STACKS: args.STACKS,
       };
       break;
@@ -421,6 +422,7 @@ export function convertConfigToUserInput(config: any): UserInput {
     processed: config.diff?.processed,
     quiet: config.diff?.quiet,
     changeSet: config.diff?.changeSet,
+    importExistingResources: config.diff?.importExistingResources,
   };
   const metadataOptions = {};
   const acknowledgeOptions = {};

packages/aws-cdk/lib/cli/parse-command-line-arguments.ts

@@ -744,6 +744,11 @@ export function parseCommandLineArguments(args: Array<string>): any {
             type: 'boolean',
             alias: 'changeset',
             desc: 'Whether to create a changeset to analyze resource replacements. In this mode, diff will use the deploy role instead of the lookup role.',
+          })
+          .option('import-existing-resources', {
+            default: false,
+            type: 'boolean',
+            desc: 'Whether or not the change set imports resources that already exist',
           }),
     )
     .command('metadata [STACK]', 'Returns all metadata associated with this stack')

packages/aws-cdk/lib/cli/user-input.ts

@@ -1152,6 +1152,13 @@ export interface DiffOptions {
    */
   readonly changeSet?: boolean;
 
+  /**
+   * Whether or not the change set imports resources that already exist
+   *
+   * @default - false
+   */
+  readonly importExistingResources?: boolean;
+
   /**
    * Positional argument for diff
    */