feat(core): enable additional metadata collection (under feature flag)

packages/aws-cdk-lib/core/lib/metadata-resource.ts

@@ -0,0 +1,22 @@
+/**
+ * Enumeration of metadata types used for tracking analytics in AWS CDK.
+ */
+export enum MetadataType {
+  /**
+   * Metadata type for construct properties.
+   * This is used to represent properties of CDK constructs.
+   */
+  CONSTRUCT = 'aws:cdk:analytics:construct',
+
+  /**
+   * Metadata type for method properties.
+   * This is used to track parameters and details of CDK method calls.
+   */
+  METHOD = 'aws:cdk:analytics:method',
+
+  /**
+   * Metadata type for feature flags.
+   * This is used to track analytics related to feature flags in the CDK.
+   */
+  FEATURE_FLAG = 'aws:cdk:analytics:featureflag',
+}

packages/aws-cdk-lib/core/lib/private/metadata-resource.ts

@@ -1,11 +1,13 @@
 import * as zlib from 'zlib';
 import { Construct } from 'constructs';
 import { ConstructInfo, constructInfoFromStack } from './runtime-info';
+import * as cxapi from '../../../cx-api';
 import { RegionInfo } from '../../../region-info';
 import { CfnCondition } from '../cfn-condition';
 import { Fn } from '../cfn-fn';
 import { Aws } from '../cfn-pseudo';
 import { CfnResource } from '../cfn-resource';
+import { FeatureFlags } from '../feature-flags';
 import { Lazy } from '../lazy';
 import { Stack } from '../stack';
 import { Token } from '../token';
@@ -17,11 +19,13 @@ export class MetadataResource extends Construct {
   constructor(scope: Stack, id: string) {
     super(scope, id);
     const metadataServiceExists = Token.isUnresolved(scope.region) || RegionInfo.get(scope.region).cdkMetadataResourceAvailable;
+    const enableAdditionalTelemtry = FeatureFlags.of(scope).isEnabled(cxapi.ENABLE_ADDITIONAL_METADATA_COLLECTION) ?? false;
     if (metadataServiceExists) {
+      const constructInfo = constructInfoFromStack(scope);
       const resource = new CfnResource(this, 'Default', {
         type: 'AWS::CDK::Metadata',
         properties: {
-          Analytics: Lazy.string({ produce: () => formatAnalytics(constructInfoFromStack(scope)) }),
+          Analytics: Lazy.string({ produce: () => formatAnalytics(constructInfo, enableAdditionalTelemtry) }),
         },
       });
 
@@ -76,9 +80,16 @@ class Trie extends Map<string, Trie> { }
  *
  * Exported/visible for ease of testing.
  */
-export function formatAnalytics(infos: ConstructInfo[]) {
+export function formatAnalytics(infos: ConstructInfo[], enableAdditionalTelemtry: boolean = false) {
   const trie = new Trie();
-  infos.forEach(info => insertFqnInTrie(`${info.version}!${info.fqn}`, trie));
+
+  // only append additional telemetry information to prefix encoding and gzip compress
+  // if feature flag is enabled; otherwise keep the old behaviour.
+  if (enableAdditionalTelemtry) {
+    infos.forEach(info => insertFqnInTrie(`${info.version}!${info.fqn}`, trie, info.metadata));
+  } else {
+    infos.forEach(info => insertFqnInTrie(`${info.version}!${info.fqn}`, trie));
+  }
 
   const plaintextEncodedConstructs = prefixEncodeTrie(trie);
   const compressedConstructsBuffer = zlib.gzipSync(Buffer.from(plaintextEncodedConstructs));
@@ -103,12 +114,17 @@ export function formatAnalytics(infos: ConstructInfo[]) {
  * Splits after non-alphanumeric characters (e.g., '.', '/') in the FQN
  * and insert each piece of the FQN in nested map (i.e., simple trie).
  */
-function insertFqnInTrie(fqn: string, trie: Trie) {
+function insertFqnInTrie(fqn: string, trie: Trie, metadata?: Record<string, any>[]) {
   for (const fqnPart of fqn.replace(/[^a-z0-9]/gi, '$& ').split(' ')) {
     const nextLevelTreeRef = trie.get(fqnPart) ?? new Trie();
     trie.set(fqnPart, nextLevelTreeRef);
     trie = nextLevelTreeRef;
   }
+
+  // if 'metadata' is defined, add it to end of Trie
+  if (metadata) {
+    trie.set(JSON.stringify(metadata), new Trie());
+  }
   return trie;
 }
 

packages/aws-cdk-lib/core/lib/private/runtime-info.ts

@@ -1,5 +1,7 @@
-import { IConstruct } from 'constructs';
+import { IConstruct, MetadataEntry } from 'constructs';
 import { App } from '../app';
+import { MetadataType } from '../metadata-resource';
+import { Resource } from '../resource';
 import { Stack } from '../stack';
 import { Stage } from '../stage';
 import { IPolicyValidationPluginBeta1 } from '../validation';
@@ -24,6 +26,7 @@ const JSII_RUNTIME_SYMBOL = Symbol.for('jsii.rtti');
 export interface ConstructInfo {
   readonly fqn: string;
   readonly version: string;
+  readonly metadata?: Record<string, any>[];
 }
 
 export function constructInfoFromConstruct(construct: IConstruct): ConstructInfo | undefined {
@@ -32,14 +35,74 @@ export function constructInfoFromConstruct(construct: IConstruct): ConstructInfo
     && jsiiRuntimeInfo !== null
     && typeof jsiiRuntimeInfo.fqn === 'string'
     && typeof jsiiRuntimeInfo.version === 'string') {
-    return { fqn: jsiiRuntimeInfo.fqn, version: jsiiRuntimeInfo.version };
+    return {
+      fqn: jsiiRuntimeInfo.fqn,
+      version: jsiiRuntimeInfo.version,
+      metadata: isResource(construct) ? redactTelemetryData(construct.node.metadata) : undefined,
+    };
   } else if (jsiiRuntimeInfo) {
     // There is something defined, but doesn't match our expectations. Fail fast and hard.
     throw new Error(`malformed jsii runtime info for construct: '${construct.node.path}'`);
   }
   return undefined;
 }
 
+/**
+ * Filter for Construct, Method, and Feature flag metadata. Redact values from it.
+ *
+ * @param metadata a list of metadata entries
+ */
+export function redactTelemetryData(metadata: MetadataEntry[]): Record<string, any>[] {
+  const validTypes = new Set([
+    MetadataType.CONSTRUCT,
+    MetadataType.METHOD,
+    MetadataType.FEATURE_FLAG,
+  ]);
+
+  return metadata
+    .filter((entry) => validTypes.has(entry.type as MetadataType))
+    .map((entry) => ({
+      type: entry.type,
+      data: redactTelemetryDataHelper(entry.data),
+    }));
+}
+
+/**
+ * Redact values from dictionary values other than Boolean and ENUM-type values.
+ * @TODO we will build a JSON blueprint of ENUM-type values in the codebase and
+ * do not redact the ENUM-type values if it match any key in the blueprint.
+ */
+function redactTelemetryDataHelper(data: any): any {
+  if (typeof data === 'boolean') {
+    return data; // Return booleans as-is
+  }
+
+  if (Array.isArray(data)) {
+    // Handle arrays by recursively redacting each element
+    return data.map((item) => redactTelemetryDataHelper(item));
+  }
+
+  if (data && typeof data === 'object') {
+    // Handle objects by iterating over their key-value pairs
+    if (isResource(data)) {
+      return '*';
+    }
+
+    /**
+     * @TODO we need to build a JSON blueprint of class and props. If 'data' matches
+     * any leaf node in the blueprint, then redact the value to avoid logging customer
+     * data.
+     */
+    const result: Record<string, any> = {};
+    for (const [key, value] of Object.entries(data)) {
+      result[key] = redactTelemetryDataHelper(value);
+    }
+    return result;
+  }
+
+  return '*';
+}
+
 /**
  * Add analytics data for any validation plugins that are used.
  * Since validation plugins are not constructs we have to handle them
@@ -106,14 +169,31 @@ export function constructInfoFromStack(stack: Stack): ConstructInfo[] {
 
   addValidationPluginInfo(stack, allConstructInfos);
 
-  // Filter out duplicate values
-  const uniqKeys = new Set();
-  return allConstructInfos.filter(construct => {
-    const constructKey = `${construct.fqn}@${construct.version}`;
-    const isDuplicate = uniqKeys.has(constructKey);
-    uniqKeys.add(constructKey);
-    return !isDuplicate;
+  // Filter out duplicate values and append the metadata information to the array
+  const uniqueMap = new Map<string, ConstructInfo>();
+  allConstructInfos.forEach(info => {
+    const key = `${info.fqn}@${info.version}`;
+    if (uniqueMap.has(key)) {
+      const existingInfo = uniqueMap.get(key);
+      if (existingInfo && existingInfo.metadata && info.metadata) {
+        existingInfo.metadata.push(...info.metadata);
+      }
+    } else {
+      uniqueMap.set(key, info);
+    }
   });
+
+  return Array.from(uniqueMap.values());
+}
+
+/**
+ * Check whether the given construct is a Resource. Note that this is
+ * duplicated function from 'core/lib/resource.ts' to avoid circular
+ * dependencies in imports.
+ */
+function isResource(construct: IConstruct): construct is Resource {
+  const RESOURCE_SYMBOL = Symbol.for('@aws-cdk/core.Resource');
+  return construct !== null && typeof(construct) === 'object' && RESOURCE_SYMBOL in construct;
 }
 
 /**

packages/aws-cdk-lib/core/test/metadata-resource.test.ts

@@ -1,6 +1,10 @@
 import * as zlib from 'zlib';
 import { Construct } from 'constructs';
-import { App, Stack, IPolicyValidationPluginBeta1, IPolicyValidationContextBeta1, Stage, PolicyValidationPluginReportBeta1 } from '../lib';
+import { Code, Function, Runtime } from '../../aws-lambda';
+import { Queue } from '../../aws-sqs';
+import { ENABLE_ADDITIONAL_METADATA_COLLECTION } from '../../cx-api';
+import { App, Stack, IPolicyValidationPluginBeta1, IPolicyValidationContextBeta1, Stage, PolicyValidationPluginReportBeta1, FeatureFlags, Duration } from '../lib';
+import { MetadataType } from '../lib/metadata-resource';
 import { formatAnalytics } from '../lib/private/metadata-resource';
 import { ConstructInfo } from '../lib/private/runtime-info';
 
@@ -49,6 +53,74 @@ describe('MetadataResource', () => {
     expect(stackTemplate.Resources?.CDKMetadata?.Condition).toBeDefined();
   });
 
+  test('when no metadata is added by default, CDKMetadata should be the same', () => {
+    const myApps = [
+      new App({
+        analyticsReporting: true,
+        postCliContext: {
+          [ENABLE_ADDITIONAL_METADATA_COLLECTION]: true,
+        },
+      }),
+      new App({
+        analyticsReporting: true,
+        postCliContext: {
+          [ENABLE_ADDITIONAL_METADATA_COLLECTION]: false,
+        },
+      }),
+      new App({
+        analyticsReporting: true,
+        postCliContext: {
+          [ENABLE_ADDITIONAL_METADATA_COLLECTION]: undefined,
+        },
+      }),
+    ];
+
+    for (const myApp of myApps) {
+      const myStack = new Stack(myApp, 'MyStack');
+      new Function(myStack, 'MyFunction', {
+        runtime: Runtime.PYTHON_3_9,
+        handler: 'index.handler',
+        code: Code.fromInline(
+          "def handler(event, context):\n\tprint('The function has been invoked.')",
+        ),
+      });
+    }
+
+    const stackTemplate1 = myApps[0].synth().getStackByName('MyStack').template;
+    const stackTemplate2 = myApps[1].synth().getStackByName('MyStack').template;
+    const stackTemplate3 = myApps[2].synth().getStackByName('MyStack').template;
+    expect(stackTemplate1.Resources?.CDKMetadata).toEqual(stackTemplate2.Resources?.CDKMetadata);
+    expect(stackTemplate1.Resources?.CDKMetadata).toEqual(stackTemplate3.Resources?.CDKMetadata);
+  });
+
+  test('enable additional metadata with metadata', () => {
+    const myApp = new App({
+      analyticsReporting: true,
+      postCliContext: {
+        [ENABLE_ADDITIONAL_METADATA_COLLECTION]: true,
+      },
+    });
+
+    const myStack = new Stack(myApp, 'EnableTelemtryStack');
+    const queueProp = {
+      visibilityTimeout: Duration.seconds(300),
+    };
+    const queue = new Queue(myStack, '01234test', queueProp);
+    queue.node.addMetadata(MetadataType.CONSTRUCT, queueProp);
+
+    const funcProp = {
+      runtime: Runtime.PYTHON_3_9,
+      handler: 'index.handler',
+      code: Code.fromInline('def handler(event, context):\n\tprint(\'The function has been invoked.\')'),
+    };
+    const func = new Function(myStack, 'MyFunction', funcProp);
+    func.node.addMetadata(MetadataType.CONSTRUCT, funcProp);
+
+    const template = myApp.synth().getStackByName('EnableTelemtryStack').template;
+    expect(template.Resources?.CDKMetadata).toBeDefined();
+    expect(template.Resources?.CDKMetadata?.Properties?.Analytics).toEqual('v2:deflate64:H4sIAAAAAAAA/8vLT0nVyyrWLzO00DMy0DNQzCrOzNQtKs0rycxN1QuC0ADZIqxKJQAAAA==');
+  });
+
   test('includes the formatted Analytics property', () => {
     // A very simple check that the jsii runtime psuedo-construct is present.
     // This check works whether we're running locally or on CodeBuild, on v1 or v2.
@@ -162,6 +234,18 @@ describe('formatAnalytics', () => {
     expectAnalytics(constructInfo, '1.2.3!aws-cdk-lib.{Construct,CfnResource,Stack},0.1.2!aws-cdk-lib.{CoolResource,OtherResource}');
   });
 
+  it.each([
+    [true, '1.2.3!aws-cdk-lib.Construct[{\"custom\":{\"foo\":\"bar\"}}]'],
+    [false, '1.2.3!aws-cdk-lib.Construct'],
+    [undefined, '1.2.3!aws-cdk-lib.Construct'],
+  ])('format analytics with metadata and enabled additional telemetry', (enableAdditionalTelemtry, output) => {
+    const constructInfo = [
+      { fqn: 'aws-cdk-lib.Construct', version: '1.2.3', metadata: [{ custom: { foo: 'bar' } }] },
+    ];
+
+    expect(plaintextConstructsFromAnalytics(formatAnalytics(constructInfo, enableAdditionalTelemtry))).toMatch(output);
+  });
+
   test('ensure gzip is encoded with "unknown" operating system to maintain consistent output across systems', () => {
     const constructInfo = [{ fqn: 'aws-cdk-lib.Construct', version: '1.2.3' }];
     const analytics = formatAnalytics(constructInfo);