feat(pipelines): Expose stack output namespaces in custom pipelines.Steps (#23110)

Implements #23000 as per request from #23000 (comment).

In order for custom steps to include `CfnOutput` in their action configurations, there needed to be a way to access and/or generate the output variable namespaces. `ShellStep` already had this capability.

This change generalizes the `StackOutputReference` usages by letting implementors of `Step` define the `StackOutputReference`s their step consumes, instead of hardwiring it to `ShellStep`.

To actually consume the references, the `ICodePipelineActionFactory` provides a `StackOutputsMap` that exposes a method to 
 render `StackOutputReference`s into their assigned CodePipeline variable names.

Author: tobni

allowed-breaking-changes.txt

@@ -149,4 +149,8 @@ incompatible-argument:@aws-cdk/aws-route53-targets.InterfaceVpcEndpointTarget.<i
 changed-type:@aws-cdk/cx-api.AssetManifestArtifact.requiresBootstrapStackVersion
 
 # removed mistyped ec2 instance class
-removed:aws-cdk-lib.aws_ec2.InstanceClass.COMPUTE6_GRAVITON2_HIGH_NETWORK_BANDWITH
+removed:aws-cdk-lib.aws_ec2.InstanceClass.COMPUTE6_GRAVITON2_HIGH_NETWORK_BANDWITH
+
+# added new required property StackOutputsMap
+strengthened:@aws-cdk/pipelines.ProduceActionOptions
+strengthened:aws-cdk-lib.pipelines.ProduceActionOptions

packages/@aws-cdk/pipelines/README.md

@@ -830,7 +830,7 @@ class MyJenkinsStep extends pipelines.Step implements pipelines.ICodePipelineAct
   ) {
     super('MyJenkinsStep');
 
-    // This is necessary if your step accepts parametres, like environment variables,
+    // This is necessary if your step accepts parameters, like environment variables,
     // that may contain outputs from other steps. It doesn't matter what the
     // structure is, as long as it contains the values that may contain outputs.
     this.discoverReferencedOutputs({
@@ -861,6 +861,43 @@ class MyJenkinsStep extends pipelines.Step implements pipelines.ICodePipelineAct
 }
 ```
 
+Another example, adding a lambda step referencing outputs from a stack:
+
+```ts
+class MyLambdaStep extends pipelines.Step implements pipelines.ICodePipelineActionFactory {
+  private stackOutputReference: pipelines.StackOutputReference
+
+  constructor(
+    private readonly function: lambda.Function,
+    stackOutput: CfnOutput,
+  ) {
+    super('MyLambdaStep');
+    this.stackOutputReference = pipelines.StackOutputReference.fromCfnOutput(stackOutput);
+  }
+
+  public produceAction(stage: codepipeline.IStage, options: pipelines.ProduceActionOptions): pipelines.CodePipelineActionFactoryResult {
+
+    stage.addAction(new cpactions.LambdaInvokeAction({
+      actionName: options.actionName,
+      runOrder: options.runOrder,
+      // Map the reference to the variable name the CDK has generated for you.
+      userParameters: {stackOutput: options.stackOutputsMap.toCodePipeline(this.stackOutputReference)},
+      lambda: this.function,
+    }));
+
+    return { runOrdersConsumed: 1 };
+  }
+
+  /**
+   * Expose stack output references, letting the CDK know
+   * we want these variables accessible for this step.
+   */
+  public get consumedStackOutputs(): pipelines.StackOutputReference[] {
+    return [this.stackOutputReference];
+  }
+}
+```
+
 ### Using an existing AWS Codepipeline
 
 If you wish to use an existing `CodePipeline.Pipeline` while using the modern API's

packages/@aws-cdk/pipelines/lib/blueprint/shell-step.ts

@@ -233,6 +233,10 @@ export class ShellStep extends Step {
     }
     return fileSet;
   }
+
+  public get consumedStackOutputs(): StackOutputReference[] {
+    return Object.values(this.envFromCfnOutputs);
+  }
 }
 
 /**

packages/@aws-cdk/pipelines/lib/blueprint/step.ts

@@ -1,6 +1,7 @@
 import { Stack, Token } from '@aws-cdk/core';
 import { StepOutput } from '../helpers-internal/step-output';
 import { FileSet, IFileSetProducer } from './file-set';
+import { StackOutputReference } from './shell-step';
 
 /**
  * A generic Step which can be added to a Pipeline
@@ -116,6 +117,13 @@ export abstract class Step implements IFileSetProducer {
       StepOutput.recordProducer(output);
     }
   }
+
+  /**
+   * StackOutputReferences this step consumes.
+   */
+  public get consumedStackOutputs(): StackOutputReference[] {
+    return [];
+  }
 }
 
 /**

packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline-action-factory.ts

@@ -3,6 +3,7 @@ import * as cp from '@aws-cdk/aws-codepipeline';
 import { Construct } from 'constructs';
 import { ArtifactMap } from './artifact-map';
 import { CodeBuildOptions, CodePipeline } from './codepipeline';
+import { StackOutputsMap } from './stack-outputs-map';
 
 /**
  * Options for the `CodePipelineActionFactory.produce()` method.
@@ -70,6 +71,18 @@ export interface ProduceActionOptions {
    * @default false
    */
   readonly beforeSelfMutation?: boolean;
+
+  /**
+   * Helper object to produce variables exported from stack deployments.
+   *
+   * If your step references outputs from a stack deployment, use
+   * this to map the output references to Codepipeline variable names.
+   *
+   * Note - Codepipeline variables can only be referenced in action
+   * configurations.
+   *
+   */
+  readonly stackOutputsMap: StackOutputsMap;
 }
 
 /**

packages/@aws-cdk/pipelines/lib/codepipeline/codepipeline.ts

@@ -24,6 +24,7 @@ import { CodeBuildStep } from './codebuild-step';
 import { CodePipelineActionFactoryResult, ICodePipelineActionFactory } from './codepipeline-action-factory';
 import { CodeBuildFactory, mergeCodeBuildOptions } from './private/codebuild-factory';
 import { namespaceStepOutputs } from './private/outputs';
+import { StackOutputsMap } from './stack-outputs-map';
 
 
 /**
@@ -313,6 +314,7 @@ export class CodePipeline extends PipelineBase {
   private _myCxAsmRoot?: string;
   private readonly dockerCredentials: DockerCredential[];
   private readonly cachedFnSub = new CachedFnSub();
+  private stackOutputs: StackOutputsMap;
 
   /**
    * Asset roles shared for publishing
@@ -337,6 +339,7 @@ export class CodePipeline extends PipelineBase {
     this.singlePublisherPerAssetType = !(props.publishAssetsInParallel ?? true);
     this.cliVersion = props.cliVersion ?? preferredCliVersion();
     this.useChangeSets = props.useChangeSets ?? true;
+    this.stackOutputs = new StackOutputsMap(this);
   }
 
   /**
@@ -467,6 +470,7 @@ export class CodePipeline extends PipelineBase {
               codeBuildDefaults: nodeType ? this.codeBuildDefaultsFor(nodeType) : undefined,
               beforeSelfMutation,
               variablesNamespace,
+              stackOutputsMap: this.stackOutputs,
             });
 
             if (node.data?.type === 'self-update') {

packages/@aws-cdk/pipelines/lib/codepipeline/index.ts

@@ -3,4 +3,5 @@ export * from './codebuild-step';
 export * from './confirm-permissions-broadening';
 export * from './codepipeline';
 export * from './codepipeline-action-factory';
-export * from './codepipeline-source';
+export * from './codepipeline-source';
+export * from './stack-outputs-map';

packages/@aws-cdk/pipelines/lib/codepipeline/private/codebuild-factory.ts

@@ -8,10 +8,9 @@ import * as iam from '@aws-cdk/aws-iam';
 import { Stack, Token } from '@aws-cdk/core';
 import { Construct, IDependable, Node } from 'constructs';
 import { FileSetLocation, ShellStep, StackOutputReference } from '../../blueprint';
-import { PipelineQueries } from '../../helpers-internal/pipeline-queries';
 import { StepOutput } from '../../helpers-internal/step-output';
 import { cloudAssemblyBuildSpecDir, obtainScope } from '../../private/construct-internals';
-import { hash, stackVariableNamespace } from '../../private/identifiers';
+import { hash } from '../../private/identifiers';
 import { mapValues, mkdict, noEmptyObject, noUndefined, partition } from '../../private/javascript';
 import { ArtifactMap } from '../artifact-map';
 import { CodeBuildStep } from '../codebuild-step';
@@ -315,10 +314,8 @@ export class CodeBuildFactory implements ICodePipelineActionFactory {
       });
     }
 
-    const queries = new PipelineQueries(options.pipeline);
-
     const stackOutputEnv = mapValues(this.props.envFromCfnOutputs ?? {}, outputRef =>
-      `#{${stackVariableNamespace(queries.producingStack(outputRef))}.${outputRef.outputName}}`,
+      options.stackOutputsMap.toCodePipeline(outputRef),
     );
 
     const configHashEnv = options.beforeSelfMutation

packages/@aws-cdk/pipelines/lib/codepipeline/stack-outputs-map.ts

@@ -0,0 +1,22 @@
+import { StackOutputReference } from '../blueprint';
+import { PipelineQueries } from '../helpers-internal/pipeline-queries';
+import { PipelineBase } from '../main';
+import { stackVariableNamespace } from '../private/identifiers';
+
+/**
+ * Translate stack outputs to Codepipline variable references
+ */
+export class StackOutputsMap {
+  private queries: PipelineQueries
+
+  constructor(pipeline: PipelineBase) {
+    this.queries = new PipelineQueries(pipeline);
+  }
+
+  /**
+   * Return the matching variable reference string for a StackOutputReference
+   */
+  public toCodePipeline(x: StackOutputReference): string {
+    return `#{${stackVariableNamespace(this.queries.producingStack(x))}.${x.outputName}}`;
+  }
+}

packages/@aws-cdk/pipelines/lib/helpers-internal/pipeline-graph.ts

@@ -1,4 +1,4 @@
-import { AssetType, FileSet, ShellStep, StackAsset, StackDeployment, StageDeployment, Step, Wave } from '../blueprint';
+import { AssetType, FileSet, StackAsset, StackDeployment, StageDeployment, Step, Wave } from '../blueprint';
 import { PipelineBase } from '../main/pipeline-base';
 import { DependencyBuilders, Graph, GraphNode, GraphNodeCollection } from './graph';
 import { PipelineQueries } from './pipeline-queries';
@@ -274,11 +274,9 @@ export class PipelineGraph {
 
     // Add stack dependencies (by use of the dependency builder this also works
     // if we encounter the Step before the Stack has been properly added yet)
-    if (step instanceof ShellStep) {
-      for (const output of Object.values(step.envFromCfnOutputs)) {
-        const stack = this.queries.producingStack(output);
-        this.stackOutputDependencies.get(stack).dependBy(node);
-      }
+    for (const output of step.consumedStackOutputs) {
+      const stack = this.queries.producingStack(output);
+      this.stackOutputDependencies.get(stack).dependBy(node);
     }
 
     return node;

packages/@aws-cdk/pipelines/lib/helpers-internal/pipeline-queries.ts

@@ -1,4 +1,4 @@
-import { Step, ShellStep, StackOutputReference, StackDeployment, StackAsset, StageDeployment } from '../blueprint';
+import { Step, StackOutputReference, StackDeployment, StackAsset, StageDeployment } from '../blueprint';
 import { PipelineBase } from '../main/pipeline-base';
 
 /**
@@ -25,9 +25,7 @@ export class PipelineQueries {
 
     const ret = new Array<string>();
     for (const step of steps) {
-      if (!(step instanceof ShellStep)) { continue; }
-
-      for (const outputRef of Object.values(step.envFromCfnOutputs)) {
+      for (const outputRef of step.consumedStackOutputs) {
         if (outputRef.isProducedBy(stack)) {
           ret.push(outputRef.outputName);
         }

packages/@aws-cdk/pipelines/rosetta/default.ts-fixture

@@ -10,6 +10,7 @@ import dynamodb = require('@aws-cdk/aws-dynamodb');
 import ecr = require('@aws-cdk/aws-ecr');
 import ec2 = require('@aws-cdk/aws-ec2');
 import iam = require('@aws-cdk/aws-iam');
+import lambda = require('@aws-cdk/lambda');
 import pipelines = require('@aws-cdk/pipelines');
 import secretsmanager = require('@aws-cdk/aws-secretsmanager');
 import sns = require('@aws-cdk/aws-sns');

packages/@aws-cdk/pipelines/test/codepipeline/codebuild-step.test.ts

@@ -4,6 +4,7 @@ import * as iam from '@aws-cdk/aws-iam';
 import * as s3 from '@aws-cdk/aws-s3';
 import { Duration, Stack } from '@aws-cdk/core';
 import * as cdkp from '../../lib';
+import { StackOutputReference } from '../../lib';
 import { PIPELINE_ENV, TestApp, ModernTestGitHubNpmPipeline, AppWithOutput } from '../testhelpers';
 
 let app: TestApp;
@@ -296,4 +297,20 @@ test('step has caching set', () => {
       },
     },
   });
+});
+
+test('step exposes consumed stack output reference', () => {
+  // WHEN
+  const myApp = new AppWithOutput(app, 'AppWithOutput', {
+    stackId: 'Stack',
+  });
+  const step = new cdkp.ShellStep('AStep', {
+    commands: ['/bin/true'],
+    envFromCfnOutputs: {
+      THE_OUTPUT: myApp.theOutput,
+    },
+  });
+
+  // THEN
+  expect(step.consumedStackOutputs).toContainEqual(StackOutputReference.fromCfnOutput(myApp.theOutput));
 });

packages/@aws-cdk/pipelines/test/integ.pipeline-with-stack-outputs-in-custom-step.js.snapshot/PipelineWithCustomStepStackOutputTestDefaultTestDeployAssert6C17E8C5.assets.json

@@ -0,0 +1,19 @@
+{
+  "version": "22.0.0",
+  "files": {
+    "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
+      "source": {
+        "path": "PipelineWithCustomStepStackOutputTestDefaultTestDeployAssert6C17E8C5.template.json",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    }
+  },
+  "dockerImages": {}
+}

packages/@aws-cdk/pipelines/test/integ.pipeline-with-stack-outputs-in-custom-step.js.snapshot/PipelineWithCustomStepStackOutputTestDefaultTestDeployAssert6C17E8C5.template.json

@@ -0,0 +1,36 @@
+{
+ "Parameters": {
+  "BootstrapVersion": {
+   "Type": "AWS::SSM::Parameter::Value<String>",
+   "Default": "/cdk-bootstrap/hnb659fds/version",
+   "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
+  }
+ },
+ "Rules": {
+  "CheckBootstrapVersion": {
+   "Assertions": [
+    {
+     "Assert": {
+      "Fn::Not": [
+       {
+        "Fn::Contains": [
+         [
+          "1",
+          "2",
+          "3",
+          "4",
+          "5"
+         ],
+         {
+          "Ref": "BootstrapVersion"
+         }
+        ]
+       }
+      ]
+     },
+     "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
+    }
+   ]
+  }
+ }
+}

packages/@aws-cdk/pipelines/test/integ.pipeline-with-stack-outputs-in-custom-step.js.snapshot/StackOutputPipelineStack.assets.json

@@ -0,0 +1,19 @@
+{
+  "version": "22.0.0",
+  "files": {
+    "1ceb4a1b5ab571218f34d0b4a62df8830a54d1ea4f95e4f115b3b4202b5fef3d": {
+      "source": {
+        "path": "StackOutputPipelineStack.template.json",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "1ceb4a1b5ab571218f34d0b4a62df8830a54d1ea4f95e4f115b3b4202b5fef3d.json",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    }
+  },
+  "dockerImages": {}
+}