perf(graphql-model-transformer): minimal provider framework and inlin…

…e policies (#2490)

packages/amplify-graphql-model-transformer/API.md

@@ -201,7 +201,7 @@ export class ModelTransformer extends TransformerModelBase implements Transforme
     // (undocumented)
     before: (ctx: TransformerBeforeStepContextProvider) => void;
     // (undocumented)
-    createIAMRole: (context: TransformerContextProvider, def: ObjectTypeDefinitionNode, stack: cdk.Stack, tableName: string) => iam.Role;
+    createIAMRole: (context: TransformerContextProvider, def: ObjectTypeDefinitionNode, stack: cdk.Stack, tableName: string) => iam.IRole;
     // (undocumented)
     ensureModelSortDirectionEnum: (ctx: TransformerValidationStepContextProvider) => void;
     // (undocumented)

packages/amplify-graphql-model-transformer/package.json

@@ -46,6 +46,7 @@
   "devDependencies": {
     "@aws-amplify/graphql-transformer-test-utils": "0.5.1",
     "@aws-sdk/client-dynamodb": "^3.431.0",
+    "@aws-sdk/client-sfn": "^3.431.0",
     "@types/aws-lambda": "8.10.119",
     "@types/node": "^12.12.6"
   },

packages/amplify-graphql-model-transformer/src/__tests__/__snapshots__/amplify-dynamodb-table-generator.test.ts.snap

@@ -2,48 +2,37 @@
 
 exports[`ModelTransformer: should successfully transform simple valid schema 1`] = `
 Object {
-  "Properties": Object {
-    "PolicyDocument": Object {
-      "Statement": Array [
-        Object {
-          "Action": Array [
-            "dynamodb:CreateTable",
-            "dynamodb:UpdateTable",
-            "dynamodb:DeleteTable",
-            "dynamodb:DescribeTable",
-            "dynamodb:DescribeContinuousBackups",
-            "dynamodb:DescribeTimeToLive",
-            "dynamodb:UpdateContinuousBackups",
-            "dynamodb:UpdateTimeToLive",
-          ],
-          "Effect": "Allow",
-          "Resource": Object {
-            "Fn::Sub": Array [
-              "arn:aws:dynamodb:\${AWS::Region}:\${AWS::AccountId}:table/*-\${apiId}-\${envName}",
-              Object {
-                "apiId": Object {
-                  "Ref": "referencetotransformerrootstackGraphQLAPI20497F53ApiId",
-                },
-                "envName": Object {
-                  "Ref": "referencetotransformerrootstackenv10C5A902Ref",
-                },
+  "PolicyDocument": Object {
+    "Statement": Array [
+      Object {
+        "Action": Array [
+          "dynamodb:CreateTable",
+          "dynamodb:UpdateTable",
+          "dynamodb:DeleteTable",
+          "dynamodb:DescribeTable",
+          "dynamodb:DescribeContinuousBackups",
+          "dynamodb:DescribeTimeToLive",
+          "dynamodb:UpdateContinuousBackups",
+          "dynamodb:UpdateTimeToLive",
+        ],
+        "Effect": "Allow",
+        "Resource": Object {
+          "Fn::Sub": Array [
+            "arn:aws:dynamodb:\${AWS::Region}:\${AWS::AccountId}:table/*-\${apiId}-\${envName}",
+            Object {
+              "apiId": Object {
+                "Ref": "referencetotransformerrootstackGraphQLAPI20497F53ApiId",
               },
-            ],
-          },
+              "envName": Object {
+                "Ref": "referencetotransformerrootstackenv10C5A902Ref",
+              },
+            },
+          ],
         },
-      ],
-      "Version": "2012-10-17",
-    },
-    "PolicyName": "CreateUpdateDeleteTablesPolicyB7B6ADB5",
-    "Roles": Array [
-      Object {
-        "Ref": "TableManagerOnEventHandlerServiceRoleD69E8A0C",
-      },
-      Object {
-        "Ref": "TableManagerIsCompleteHandlerServiceRole73EE73E4",
       },
     ],
+    "Version": "2012-10-17",
   },
-  "Type": "AWS::IAM::Policy",
+  "PolicyName": "CreateUpdateDeleteTablesPolicy",
 }
 `;

packages/amplify-graphql-model-transformer/src/__tests__/amplify-dynamodb-table-generator.test.ts

@@ -33,8 +33,12 @@ describe('ModelTransformer:', () => {
     const amplifyTableManagerStack = out.stacks[ITERATIVE_TABLE_STACK_NAME];
     expect(amplifyTableManagerStack).toBeDefined();
     // DynamoDB manager policy should be generated correctly
-    const policyKey = Object.keys(amplifyTableManagerStack.Resources!).find((r) => r.includes('CreateUpdateDeleteTablesPolicy'));
-    const ddbManagerPolicy = amplifyTableManagerStack.Resources![`${policyKey}`];
+    const ddbManagerPolicy = Object.values(amplifyTableManagerStack.Resources!)
+      .filter((resource) => resource.Type === 'AWS::IAM::Role')
+      .flatMap((role: any) => role.Properties.Policies)
+      .filter((policies: any) => policies !== undefined)
+      .reduce((acc, value) => acc.concat(value), [])
+      .find((policy: any) => policy.PolicyName === 'CreateUpdateDeleteTablesPolicy');
     expect(ddbManagerPolicy).toBeDefined();
     expect(ddbManagerPolicy).toMatchSnapshot();
     // Post table resource should be generated within the custom table type
@@ -50,5 +54,37 @@ describe('ModelTransformer:', () => {
     expect(commentTable).toBeDefined();
     expect(commentTable.Type).toBe('AWS::DynamoDB::Table');
     validateModelSchema(parse(out.schema));
+
+    // Outputs should contain a reference to the Arn to the entry point (onEventHandler)
+    // of the provider for the AmplifyTableManager custom resource.
+    // If any of these assertions should fail, it is likely caused by a change in the custom resource provider
+    /** {@link Provider} */ // that caused the entry point ARN to change.
+    // !! This will result in broken redeployments !!
+    // Friends don't let friends mutate custom resource entry point ARNs.
+    const outputs = amplifyTableManagerStack.Outputs!;
+    expect(outputs).toBeDefined();
+    const rootStackName = 'transformerrootstack';
+    const tableManagerStackName = 'AmplifyTableManager';
+    const onEventHandlerName = 'TableManagerCustomProviderframeworkonEvent';
+    // See https://github.com/aws/aws-cdk/blob/6fdc4582f659549021a64a4d676fce12fc241715/packages/aws-cdk-lib/core/lib/stack.ts#L1288-L1333 for more information
+    const entryPointKeyStableLogicalIdHash = 'F1C8BD67';
+    const onEventHandlerStableLogicalIdHash = '1DFC2ECC';
+
+    const entrypointArnOutputsKey = `${rootStackName}${tableManagerStackName}${onEventHandlerName}${entryPointKeyStableLogicalIdHash}Arn`;
+    const entryPointOutput = outputs[entrypointArnOutputsKey];
+
+    const onEventHandlerResourceName = `${onEventHandlerName}${onEventHandlerStableLogicalIdHash}`;
+    expect(entryPointOutput).toBeDefined();
+    expect(entryPointOutput['Value']['Fn::GetAtt']).toEqual([
+      onEventHandlerResourceName /* TableManagerCustomProviderframeworkonEvent1DFC2ECC */,
+      'Arn',
+    ]);
+
+    // Since we verified above that the ARN for this resource is included in the stack outputs,
+    // we know that the resource itself exists in the stack. But better safe than sorry.
+    const amplifyTableManagerResources = amplifyTableManagerStack.Resources;
+    expect(amplifyTableManagerResources).toBeDefined();
+    const onEventHandlerLambda = amplifyTableManagerResources![onEventHandlerResourceName];
+    expect(onEventHandlerLambda).toBeDefined();
   });
 });

packages/amplify-graphql-model-transformer/src/__tests__/provider.test.ts

@@ -0,0 +1,45 @@
+import { Stack, aws_iam, aws_lambda } from 'aws-cdk-lib';
+import { Template } from 'aws-cdk-lib/assertions';
+import path from 'path';
+import { Provider } from '../resources/amplify-dynamodb-table/provider';
+
+test('if isComplete is specified, the isComplete framework handler is also included', () => {
+  // GIVEN
+  const stack = new Stack();
+  const lambdaCode = aws_lambda.Code.fromAsset(
+    path.join(__dirname, '..', '..', 'lib', 'resources', 'amplify-dynamodb-table', 'amplify-table-manager-lambda'),
+    { exclude: ['*.ts'] },
+  );
+
+  const onEventRole = new aws_iam.Role(stack, 'OnEventRole', {
+    assumedBy: new aws_iam.ServicePrincipal('lambda.amazonaws.com'),
+    managedPolicies: [aws_iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')],
+  });
+
+  const isCompleteRole = new aws_iam.Role(stack, 'IsCompleteRole', {
+    assumedBy: new aws_iam.ServicePrincipal('lambda.amazonaws.com'),
+    managedPolicies: [aws_iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')],
+  });
+
+  // WHEN
+  new Provider(stack, 'MyProvider', {
+    lambdaCode,
+    onEventHandlerName: 'amplify-table-manager-handler.onEvent',
+    onEventRole,
+    isCompleteHandlerName: 'amplify-table-manager-handler.isComplete',
+    isCompleteRole,
+  });
+
+  // THEN
+  Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Function', {
+    Handler: 'amplify-table-manager-handler.onEvent',
+    Timeout: 840,
+    Role: { 'Fn::GetAtt': ['OnEventRole56094035', 'Arn'] },
+  });
+
+  Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Function', {
+    Handler: 'amplify-table-manager-handler.isComplete',
+    Timeout: 840,
+    Role: { 'Fn::GetAtt': ['IsCompleteRole3501BB5A', 'Arn'] },
+  });
+});

packages/amplify-graphql-model-transformer/src/__tests__/util.test.ts

@@ -0,0 +1,30 @@
+import { getEnv, log, withRetries } from '../resources/amplify-dynamodb-table/amplify-table-manager-lambda/util';
+
+test('withRetries() will invoke a throwing function multiple times', async () => {
+  let invocations = 0;
+  const retryOptions = {
+    attempts: 3,
+    sleep: 0,
+  };
+
+  await expect(() =>
+    withRetries(retryOptions, async () => {
+      invocations += 1;
+      throw new Error('Ruh roh!');
+    })(),
+  ).rejects.toThrow(/Ruh roh!/);
+
+  expect(invocations).toBeGreaterThan(1);
+});
+
+test('getEnv succeeds with existing / fails with non-existing', () => {
+  process.env['FOO'] = 'BAR';
+  const fooValue = getEnv('FOO');
+  expect(fooValue).toEqual('BAR');
+  expect(() => getEnv('')).toThrowError();
+});
+
+test('log helper coverage', () => {
+  log('foo', 'bar');
+  log('foo', { bar: 'baz' });
+});

packages/amplify-graphql-model-transformer/src/graphql-model-transformer.ts

@@ -746,7 +746,7 @@ export class ModelTransformer extends TransformerModelBase implements Transforme
   /**
    * createIAMRole
    */
-  createIAMRole = (context: TransformerContextProvider, def: ObjectTypeDefinitionNode, stack: cdk.Stack, tableName: string): iam.Role => {
+  createIAMRole = (context: TransformerContextProvider, def: ObjectTypeDefinitionNode, stack: cdk.Stack, tableName: string): iam.IRole => {
     const ddbGenerator = this.resourceGeneratorMap.get(DDB_DB_TYPE) as DynamoModelResourceGenerator;
     return ddbGenerator.createIAMRole(context, def, stack, tableName);
   };

packages/amplify-graphql-model-transformer/src/resources/amplify-dynamodb-table/amplify-dynamo-model-resource-generator.ts

@@ -5,11 +5,12 @@ import { ObjectTypeDefinitionNode } from 'graphql';
 import { setResourceName } from '@aws-amplify/graphql-transformer-core';
 import { AttributeType, StreamViewType, TableEncryption } from 'aws-cdk-lib/aws-dynamodb';
 import { Construct } from 'constructs';
-
-import { Duration, aws_iam, aws_lambda, custom_resources, aws_logs } from 'aws-cdk-lib';
+import { Duration, aws_iam, aws_lambda } from 'aws-cdk-lib';
 import { DynamoModelResourceGenerator } from '../dynamo-model-resource-generator';
 import * as path from 'path';
 import { AmplifyDynamoDBTable } from './amplify-dynamodb-table-construct';
+import { WaiterStateMachine } from './waiter-state-machine';
+import { Provider } from './provider';
 
 /**
  * AmplifyDynamoModelResourceGenerator is a subclass of DynamoModelResourceGenerator,
@@ -18,8 +19,7 @@ import { AmplifyDynamoDBTable } from './amplify-dynamodb-table-construct';
 
 export const ITERATIVE_TABLE_STACK_NAME = 'AmplifyTableManager';
 export class AmplifyDynamoModelResourceGenerator extends DynamoModelResourceGenerator {
-  private customResourceServiceToken: string = '';
-  private ddbManagerPolicy?: aws_iam.Policy;
+  private customResourceServiceToken = '';
 
   generateResources(ctx: TransformerContextProvider): void {
     if (!this.isEnabled()) {
@@ -44,8 +44,18 @@ export class AmplifyDynamoModelResourceGenerator extends DynamoModelResourceGene
       this.createModelTable(scope, model, ctx);
     });
 
-    if (this.ddbManagerPolicy) {
-      this.ddbManagerPolicy?.addStatements(
+    this.generateResolvers(ctx);
+  }
+
+  protected createCustomProviderResource(scope: Construct, context: TransformerContextProvider): void {
+    const lambdaCode = aws_lambda.Code.fromAsset(
+      path.join(__dirname, '..', '..', '..', 'lib', 'resources', 'amplify-dynamodb-table', 'amplify-table-manager-lambda'),
+      { exclude: ['*.ts'] },
+    );
+
+    // PolicyDocument that grants access to Create/Update/Delete relevant DynamoDB tables
+    const lambdaPolicyDocument = new aws_iam.PolicyDocument({
+      statements: [
         new aws_iam.PolicyStatement({
           actions: [
             'dynamodb:CreateTable',
@@ -58,52 +68,82 @@ export class AmplifyDynamoModelResourceGenerator extends DynamoModelResourceGene
             'dynamodb:UpdateTimeToLive',
           ],
           resources: [
+            // eslint-disable-next-line no-template-curly-in-string
             cdk.Fn.sub('arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/*-${apiId}-${envName}', {
-              apiId: ctx.api.apiId,
-              envName: ctx.synthParameters.amplifyEnvironmentName,
+              apiId: context.api.apiId,
+              envName: context.synthParameters.amplifyEnvironmentName,
             }),
           ],
         }),
-      );
-    }
-
-    this.generateResolvers(ctx);
-  }
-
-  protected createCustomProviderResource(scope: Construct, context: TransformerContextProvider): void {
-    // Policy that grants access to Create/Update/Delete DynamoDB tables
-    this.ddbManagerPolicy = new aws_iam.Policy(scope, 'CreateUpdateDeleteTablesPolicy');
-
-    const lambdaCode = aws_lambda.Code.fromAsset(
-      path.join(__dirname, '..', '..', '..', 'lib', 'resources', 'amplify-dynamodb-table', 'amplify-table-manager-lambda'),
-    );
-
-    // lambda that will handle DDB CFN events
-    const gsiOnEventHandler = new aws_lambda.Function(scope, ResourceConstants.RESOURCES.TableManagerOnEventHandlerLogicalID, {
-      runtime: aws_lambda.Runtime.NODEJS_18_X,
-      code: lambdaCode,
-      handler: 'amplify-table-manager-handler.onEvent',
-      timeout: Duration.minutes(14),
+      ],
     });
 
-    // lambda that will poll for provisioning to complete
-    const gsiIsCompleteHandler = new aws_lambda.Function(scope, ResourceConstants.RESOURCES.TableManagerIsCompleteHandlerLogicalID, {
-      runtime: aws_lambda.Runtime.NODEJS_18_X,
-      code: lambdaCode,
-      handler: 'amplify-table-manager-handler.isComplete',
-      timeout: Duration.minutes(14),
+    // Note: The isCompleteRole and onEventRole are similar enough that you might ask "why not just use a single role?"
+    // 1. Doing so creates a circular dependency between someCombinedRole <-> waiterStateMachine
+    // 2. The isCompleteHandler doesn't need permissions to invoke the waiterStateMachine.
+
+    // Role assumed by the isCompleteHandler.
+    // We want to avoid the auto-generated default policy for this to avoid unnecessary deployment time
+    // slowdowns, hence the `withoutPolicyUpdates()`
+    const isCompleteRole = new aws_iam.Role(scope, 'AmplifyManagedTableIsCompleteRole', {
+      assumedBy: new aws_iam.ServicePrincipal('lambda.amazonaws.com'),
+      managedPolicies: [aws_iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')],
+      inlinePolicies: {
+        CreateUpdateDeleteTablesPolicy: lambdaPolicyDocument,
+      },
+    }).withoutPolicyUpdates();
+
+    // Role assumed by the onEventHandler (custom resource entry point).
+    // We need to keep this open to modification so that waiter state machine can grant it
+    // invocation permissions below, hence no `withoutPolicyUpdates()`
+    const onEventRole = new aws_iam.Role(scope, 'AmplifyManagedTableOnEventRole', {
+      assumedBy: new aws_iam.ServicePrincipal('lambda.amazonaws.com'),
+      managedPolicies: [aws_iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicExecutionRole')],
+      inlinePolicies: {
+        CreateUpdateDeleteTablesPolicy: lambdaPolicyDocument,
+      },
     });
 
-    this.ddbManagerPolicy.attachToRole(gsiOnEventHandler.role!);
-    this.ddbManagerPolicy.attachToRole(gsiIsCompleteHandler.role!);
-    const customResourceProvider = new custom_resources.Provider(scope, ResourceConstants.RESOURCES.TableManagerCustomProviderLogicalID, {
-      onEventHandler: gsiOnEventHandler,
-      isCompleteHandler: gsiIsCompleteHandler,
-      logRetention: aws_logs.RetentionDays.ONE_MONTH,
-      queryInterval: Duration.seconds(30),
-      totalTimeout: Duration.hours(2),
+    // Create the custom resource provider with the infrastructure to handle resource modifications.
+    /** !! Be extra cautious about any modifications to this code -- see inline note in {@link Provider} !! */
+    const customResourceProvider = new Provider(scope, ResourceConstants.RESOURCES.TableManagerCustomProviderLogicalID, {
+      lambdaCode,
+      onEventHandlerName: 'amplify-table-manager-handler.onEvent',
+      onEventRole,
+      isCompleteHandlerName: 'amplify-table-manager-handler.isComplete',
+      isCompleteRole,
     });
-    this.customResourceServiceToken = customResourceProvider.serviceToken;
+
+    const { onEventHandler, isCompleteHandler, serviceToken } = customResourceProvider;
+
+    // --- Waiter state machine configuration
+    // Invoke isCompleteHandler every 10 seconds to query completion status.
+    // 10 seconds is the current value because it showed deployment time improvements
+    // over higher values. < 10 seconds showed diminishing returns of those improvements
+    // at the cost of more lambda invocations.
+    const queryInterval = Duration.seconds(10);
+    // CloudFormation times out custom resource requests at 1 hour.
+    // https://github.com/aws/aws-cdk/blob/11621e78c8f8188fcdd528d01cd2aa8bd97db58f/packages/aws-cdk-lib/custom-resources/lib/provider-framework/provider.ts#L59-L66
+    // Once that happens, there's no use continuing to invoke the isComplete handler.
+    const totalTimeout = Duration.hours(1);
+    const stateMachineProps = {
+      isCompleteHandler,
+      queryInterval,
+      totalTimeout,
+      maxAttempts: totalTimeout.toSeconds() / queryInterval.toSeconds(),
+      backoffRate: 1,
+    };
+
+    const waiterStateMachine = new WaiterStateMachine(scope, 'AmplifyTableWaiterStateMachine', stateMachineProps);
+
+    // The onEventHandler needs to know the state machine ARN to start it, so that it can query completion status
+    // when invoking the isCompleteHandler.
+    onEventHandler.addEnvironment('WAITER_STATE_MACHINE_ARN', waiterStateMachine.stateMachineArn);
+    // It also needs permissions to invoke it.
+    waiterStateMachine.grantStartExecution(onEventHandler);
+    // This is the entry point of the custom resource -- make sure this value never changes!
+    /** See inline note in {@link Provider} for more details */
+    this.customResourceServiceToken = serviceToken;
   }
 
   protected createModelTable(scope: Construct, def: ObjectTypeDefinitionNode, context: TransformerContextProvider): void {