feat(toolkit-lib): make the plugin host configurable (#375)

This makes the plugin host configurable for toolkit users:

## Design decisions

- Every `Toolkit` instance gets a fresh `PluginHost` by default (not a
global singleton `PluginHost`)
- ...in fact, the concept of a global singleton `PluginHost` gets fully
moved to the CLI. That concept disappears from the toolkit and the
helpers, and all functions that used to implicitly assume a global
plugin host now get an explicit parameter, that defaults to a fresh
plugin host if not supplied.

These choices lead to a lot of downstream test changes.

- Plugin module resolution, module deduplication and logging was moved
from the CLI into the plugin host.

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

Author: rix0rrr

.projenrc.ts

@@ -756,6 +756,12 @@ const tmpToolkitHelpers = configureProject(
         target: 'es2022',
         lib: ['es2022', 'esnext.disposable', 'dom'],
         module: 'NodeNext',
+
+        // Temporarily, because it makes it impossible for us to use @internal functions
+        // from other packages. Annoyingly, VSCode won't show an error if you use @internal
+        // because it looks at the .ts, but the compilation will fail because it will use
+        // the .d.ts.
+        stripInternal: false,
       },
     },
 

packages/@aws-cdk/tmp-toolkit-helpers/src/api/aws-auth/credential-plugins.ts

@@ -4,8 +4,8 @@ import type { AwsCredentialIdentity, AwsCredentialIdentityProvider } from '@smit
 import { credentialsAboutToExpire, makeCachingProvider } from './provider-caching';
 import { formatErrorMessage } from '../../util';
 import { IO, type IoHelper } from '../io/private';
+import type { PluginHost } from '../plugin';
 import type { Mode } from '../plugin/mode';
-import type { PluginHost } from '../plugin/plugin';
 import { AuthenticationError } from '../toolkit-error';
 
 /**
@@ -22,12 +22,8 @@ import { AuthenticationError } from '../toolkit-error';
  */
 export class CredentialPlugins {
   private readonly cache: { [key: string]: PluginCredentialsFetchResult | undefined } = {};
-  private readonly host: PluginHost;
-  private readonly ioHelper: IoHelper;
 
-  constructor(host: PluginHost, ioHelper: IoHelper) {
-    this.host = host;
-    this.ioHelper = ioHelper;
+  constructor(private readonly host: PluginHost, private readonly ioHelper: IoHelper) {
   }
 
   public async fetchCredentialsFor(awsAccountId: string, mode: Mode): Promise<PluginCredentialsFetchResult | undefined> {

packages/@aws-cdk/tmp-toolkit-helpers/src/api/aws-auth/sdk-provider.ts

@@ -14,7 +14,7 @@ import { SDK } from './sdk';
 import { callTrace, traceMemberMethods } from './tracing';
 import { formatErrorMessage } from '../../util';
 import { IO, type IoHelper } from '../io/private';
-import { Mode, PluginHost } from '../plugin';
+import { PluginHost, Mode } from '../plugin';
 import { AuthenticationError } from '../toolkit-error';
 
 export type AssumeRoleAdditionalOptions = Partial<Omit<AssumeRoleCommandInput, 'ExternalId' | 'RoleArn'>>;
@@ -44,6 +44,13 @@ export interface SdkProviderOptions {
    * The logger for sdk calls.
    */
   readonly logger?: Logger;
+
+  /**
+   * The plugin host to use
+   *
+   * @default - an empty plugin host
+   */
+  readonly pluginHost?: PluginHost;
 }
 
 /**
@@ -136,7 +143,7 @@ export class SdkProvider {
 
     const region = await builder.region(options.profile);
     const requestHandler = await builder.requestHandlerBuilder(options.httpOptions);
-    return new SdkProvider(credentialProvider, region, requestHandler, options.ioHelper, options.logger);
+    return new SdkProvider(credentialProvider, region, requestHandler, options.pluginHost ?? new PluginHost(), options.ioHelper, options.logger);
   }
 
   private readonly plugins;
@@ -148,10 +155,11 @@ export class SdkProvider {
      */
     public readonly defaultRegion: string,
     private readonly requestHandler: NodeHttpHandlerOptions = {},
+    pluginHost: PluginHost,
     private readonly ioHelper: IoHelper,
     private readonly logger?: Logger,
   ) {
-    this.plugins = new CredentialPlugins(PluginHost.instance, ioHelper);
+    this.plugins = new CredentialPlugins(pluginHost, ioHelper);
   }
 
   /**
@@ -183,7 +191,7 @@ export class SdkProvider {
 
       // Our current credentials must be valid and not expired. Confirm that before we get into doing
       // actual CloudFormation calls, which might take a long time to hang.
-      const sdk = new SDK(baseCreds.credentials, env.region, this.requestHandler, this.ioHelper, this.logger);
+      const sdk = this._makeSdk(baseCreds.credentials, env.region);
       await sdk.validateCredentials();
       return { sdk, didAssumeRole: false };
     }
@@ -216,7 +224,7 @@ export class SdkProvider {
           `${fmtObtainedCredentials(baseCreds)} could not be used to assume '${options.assumeRoleArn}', but are for the right account. Proceeding anyway.`,
         ));
         return {
-          sdk: new SDK(baseCreds.credentials, env.region, this.requestHandler, this.ioHelper, this.logger),
+          sdk: this._makeSdk(baseCreds.credentials, env.region),
           didAssumeRole: false,
         };
       }
@@ -236,7 +244,7 @@ export class SdkProvider {
     if (baseCreds.source === 'none') {
       return undefined;
     }
-    return (await new SDK(baseCreds.credentials, env.region, this.requestHandler, this.ioHelper, this.logger).currentAccount()).partition;
+    return (await this._makeSdk(baseCreds.credentials, env.region).currentAccount()).partition;
   }
 
   /**
@@ -282,7 +290,7 @@ export class SdkProvider {
   public async defaultAccount(): Promise<Account | undefined> {
     return cached(this, CACHED_ACCOUNT, async () => {
       try {
-        return await new SDK(this.defaultCredentialProvider, this.defaultRegion, this.requestHandler, this.ioHelper, this.logger).currentAccount();
+        return await this._makeSdk(this.defaultCredentialProvider, this.defaultRegion).currentAccount();
       } catch (e: any) {
         // Treat 'ExpiredToken' specially. This is a common situation that people may find themselves in, and
         // they are complaining about if we fail 'cdk synth' on them. We loudly complain in order to show that
@@ -384,7 +392,7 @@ export class SdkProvider {
       // Call the provider at least once here, to catch an error if it occurs
       await credentials();
 
-      return new SDK(credentials, region, this.requestHandler, this.ioHelper, this.logger);
+      return this._makeSdk(credentials, region);
     } catch (err: any) {
       if (err.name === 'ExpiredToken') {
         throw err;
@@ -402,6 +410,29 @@ export class SdkProvider {
       );
     }
   }
+
+  /**
+   * Factory function that creates a new SDK instance
+   *
+   * This is a function here, instead of all the places where this is used creating a `new SDK`
+   * instance, so that it is trivial to mock from tests.
+   *
+   * Use like this:
+   *
+   * ```ts
+   * const mockSdk = jest.spyOn(SdkProvider.prototype, '_makeSdk').mockReturnValue(new MockSdk());
+   * // ...
+   * mockSdk.mockRestore();
+   * ```
+   *
+   * @internal
+   */
+  public _makeSdk(
+    credProvider: AwsCredentialIdentityProvider,
+    region: string,
+  ) {
+    return new SDK(credProvider, region, this.requestHandler, this.ioHelper, this.logger);
+  }
 }
 
 /**

packages/@aws-cdk/tmp-toolkit-helpers/src/api/plugin/plugin.ts

@@ -1,6 +1,8 @@
 import { inspect } from 'util';
 import type { CredentialProviderSource, IPluginHost, Plugin } from '@aws-cdk/cli-plugin-contract';
 import { type ContextProviderPlugin, isContextProviderPlugin } from './context-provider-plugin';
+import type { IIoHost } from '../io';
+import { IoDefaultMessages, IoHelper } from '../private';
 import { ToolkitError } from '../toolkit-error';
 
 export let TESTING = false;
@@ -10,12 +12,13 @@ export function markTesting() {
 }
 
 /**
- * A utility to manage plug-ins.
+ * Class to manage a plugin collection
  *
+ * It provides a `load()` function that loads a JavaScript
+ * module from disk, and gives it access to the `IPluginHost` interface
+ * to register itself.
  */
 export class PluginHost implements IPluginHost {
-  public static instance = new PluginHost();
-
   /**
    * Access the currently registered CredentialProviderSources. New sources can
    * be registered using the +registerCredentialProviderSource+ method.
@@ -24,30 +27,60 @@ export class PluginHost implements IPluginHost {
 
   public readonly contextProviderPlugins: Record<string, ContextProviderPlugin> = {};
 
-  constructor() {
-    if (!TESTING && PluginHost.instance && PluginHost.instance !== this) {
-      throw new ToolkitError('New instances of PluginHost must not be built. Use PluginHost.instance instead!');
-    }
-  }
+  public ioHost?: IIoHost;
+
+  private readonly alreadyLoaded = new Set<string>();
 
   /**
    * Loads a plug-in into this PluginHost.
    *
+   * Will use `require.resolve()` to get the most accurate representation of what
+   * code will get loaded in error messages. As such, it will not work in
+   * unit tests with Jest virtual modules becauase of <https://github.com/jestjs/jest/issues/9543>.
+   *
    * @param moduleSpec the specification (path or name) of the plug-in module to be loaded.
+   * @param ioHost the I/O host to use for printing progress information
    */
-  public load(moduleSpec: string) {
+  public load(moduleSpec: string, ioHost?: IIoHost) {
     try {
+      const resolved = require.resolve(moduleSpec);
+      if (ioHost) {
+        new IoDefaultMessages(IoHelper.fromIoHost(ioHost, 'init')).debug(`Loading plug-in: ${resolved} from ${moduleSpec}`);
+      }
+      return this._doLoad(resolved);
+    } catch (e: any) {
+      // according to Node.js docs `MODULE_NOT_FOUND` is the only possible error here
+      // @see https://nodejs.org/api/modules.html#requireresolverequest-options
+      // Not using `withCause()` here, since the node error contains a "Require Stack"
+      // as part of the error message that is inherently useless to our users.
+      throw new ToolkitError(`Unable to resolve plug-in: Cannot find module '${moduleSpec}': ${e}`);
+    }
+  }
+
+  /**
+   * Do the loading given an already-resolved module name
+   *
+   * @internal
+   */
+  public _doLoad(resolved: string) {
+    try {
+      if (this.alreadyLoaded.has(resolved)) {
+        return;
+      }
+
       /* eslint-disable @typescript-eslint/no-require-imports */
-      const plugin = require(moduleSpec);
+      const plugin = require(resolved);
       /* eslint-enable */
       if (!isPlugin(plugin)) {
-        throw new ToolkitError(`Module ${moduleSpec} is not a valid plug-in, or has an unsupported version.`);
+        throw new ToolkitError(`Module ${resolved} is not a valid plug-in, or has an unsupported version.`);
       }
       if (plugin.init) {
         plugin.init(this);
       }
+
+      this.alreadyLoaded.add(resolved);
     } catch (e: any) {
-      throw ToolkitError.withCause(`Unable to load plug-in '${moduleSpec}'`, e);
+      throw ToolkitError.withCause(`Unable to load plug-in '${resolved}'`, e);
     }
 
     function isPlugin(x: any): x is Plugin {

packages/@aws-cdk/tmp-toolkit-helpers/src/context-providers/index.ts

@@ -16,8 +16,7 @@ import { TRANSIENT_CONTEXT_KEY } from '../api/context';
 import { replaceEnvPlaceholders } from '../api/environment';
 import { IO } from '../api/io/private';
 import type { IoHelper } from '../api/io/private';
-import { PluginHost } from '../api/plugin';
-import type { ContextProviderPlugin } from '../api/plugin';
+import type { PluginHost, ContextProviderPlugin } from '../api/plugin';
 import { ContextProviderError } from '../api/toolkit-error';
 import { formatErrorMessage } from '../util';
 
@@ -72,6 +71,7 @@ export async function provideContextValues(
   missingValues: cxschema.MissingContext[],
   context: Context,
   sdk: SdkProvider,
+  pluginHost: PluginHost,
   ioHelper: IoHelper,
 ) {
   for (const missingContext of missingValues) {
@@ -83,7 +83,7 @@ export async function provideContextValues(
 
     let factory;
     if (providerName.startsWith(`${PLUGIN_PROVIDER_PREFIX}:`)) {
-      const plugin = PluginHost.instance.contextProviderPlugins[providerName.substring(PLUGIN_PROVIDER_PREFIX.length + 1)];
+      const plugin = pluginHost.contextProviderPlugins[providerName.substring(PLUGIN_PROVIDER_PREFIX.length + 1)];
       if (!plugin) {
         // eslint-disable-next-line max-len
         throw new ContextProviderError(`Unrecognized plugin context provider name: ${missingContext.provider}.`);

packages/@aws-cdk/tmp-toolkit-helpers/src/context-providers/security-groups.ts

@@ -60,8 +60,7 @@ export class SecurityGroupContextProviderPlugin implements ContextProviderPlugin
 }
 
 /**
- * TODO: We intend this to be @*internal but a test in aws-cdk depends on it.
- * Put the tag back later.
+ * @internal
  */
 export function hasAllTrafficEgress(securityGroup: SecurityGroup) {
   let hasAllTrafficCidrV4 = false;

packages/@aws-cdk/tmp-toolkit-helpers/tsconfig.dev.json

@@ -103,6 +103,7 @@ export class ContextAwareCloudAssemblySource implements ICloudAssemblySource {
             assembly.manifest.missing,
             this.context,
             this.props.services.sdkProvider,
+            this.props.services.pluginHost,
             this.ioHelper,
           );
 

packages/@aws-cdk/tmp-toolkit-helpers/tsconfig.json

@@ -20,5 +20,6 @@ export type {
 } from '../../../tmp-toolkit-helpers/src/api/io/io-message';
 export type { IIoHost } from '../../../tmp-toolkit-helpers/src/api/io/io-host';
 export type { ToolkitAction } from '../../../tmp-toolkit-helpers/src/api/io/toolkit-action';
+export { PluginHost } from '../../../tmp-toolkit-helpers/src/api/plugin/plugin';
 
 export * from '../../../tmp-toolkit-helpers/src/payloads';

packages/@aws-cdk/toolkit-lib/lib/api/cloud-assembly/private/context-aware-source.ts

@@ -1,14 +1,15 @@
 
 import type { ICloudAssemblySource } from '../../api/cloud-assembly';
 import { StackAssembly } from '../../api/cloud-assembly/private';
-import type { SdkProvider, IoHelper } from '../../api/shared-private';
+import type { SdkProvider, IoHelper, PluginHost } from '../../api/shared-private';
 
 /**
  * Helper struct to pass internal services around.
  */
 export interface ToolkitServices {
   sdkProvider: SdkProvider;
   ioHelper: IoHelper;
+  pluginHost: PluginHost;
 }
 
 /**