Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(generation-transformer): add generation transformer #2820

Merged
merged 36 commits into from
Aug 30, 2024
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
a3c265e
add graphqlUrl field to GraphQLAPIProvider
atierian Aug 27, 2024
65bf4b8
add generation-transformer
atierian Aug 27, 2024
df59807
add generation transformer to transformer chain
atierian Aug 27, 2024
26165a9
add generation e2e tests
atierian Aug 27, 2024
b73ae3e
add generation transformer as dep for api and data construct
atierian Aug 27, 2024
4255f75
update data and api construct jsii
atierian Aug 27, 2024
666dd56
some cleanup and inline docs
atierian Aug 27, 2024
8d1524a
update readme
atierian Aug 28, 2024
b540c60
update number of transformers for test assertion
atierian Aug 28, 2024
3e3b867
update api extract for generation transformer
atierian Aug 28, 2024
d4e5ab5
update api extract in transformer-interfaces
atierian Aug 28, 2024
a8281de
lint readme
atierian Aug 28, 2024
c8052fd
cleanup, inline docs, cascading types
atierian Aug 28, 2024
296c497
split out req / res resolverfn and add some more inline docs
atierian Aug 28, 2024
ed314e9
update inline comment for disabled model generation e2e test
atierian Aug 28, 2024
9b7acb7
alphabetize package dep order
atierian Aug 28, 2024
7839a2c
fix typo in test comment
atierian Aug 30, 2024
75a0972
remove unnecessary noise from snapshots
atierian Aug 30, 2024
118f4d2
remove graphqlurl change that snuck in again
atierian Aug 30, 2024
8ab7bec
move query type validation to validate function
atierian Aug 30, 2024
d85ea0c
update generation API.md
atierian Aug 30, 2024
27c2f37
update split-e2e to force us-west-2 for generation e2es
atierian Aug 30, 2024
51fe60a
use parent account for generation e2e
atierian Aug 30, 2024
61ff76a
split e2e
atierian Aug 30, 2024
54db06d
fix dep versions in generation transformer
atierian Aug 30, 2024
a3775a7
fix typo in e2e schema definition
atierian Aug 30, 2024
97018fa
fix threshold numbers and add a few more test cases
atierian Aug 30, 2024
0704d88
toUpper
atierian Aug 30, 2024
a941491
add comment clarifying disallowed fields
atierian Aug 30, 2024
48ff9f0
stringify system prompt
atierian Aug 30, 2024
0aaacc5
remove ueber complex ip address pattern prop
atierian Aug 30, 2024
4de6c72
magic-stringectomy
atierian Aug 30, 2024
7919c94
add over / under validation tests
atierian Aug 30, 2024
3c4fc18
add doc comment clarifying intent of scalar types for json schema
atierian Aug 30, 2024
c0c662b
lint
atierian Aug 30, 2024
174470b
fix api extract
atierian Aug 30, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@
"@aws-amplify/graphql-api-construct/@aws-amplify/graphql-default-value-transformer",
"@aws-amplify/graphql-api-construct/@aws-amplify/graphql-directives",
"@aws-amplify/graphql-api-construct/@aws-amplify/graphql-function-transformer",
"@aws-amplify/graphql-api-construct/@aws-amplify/graphql-generation-transformer",
"@aws-amplify/graphql-api-construct/@aws-amplify/graphql-http-transformer",
"@aws-amplify/graphql-api-construct/@aws-amplify/graphql-index-transformer",
"@aws-amplify/graphql-api-construct/@aws-amplify/graphql-maps-to-transformer",
Expand Down Expand Up @@ -149,6 +150,7 @@
"@aws-amplify/data-construct/@aws-amplify/graphql-auth-transformer",
"@aws-amplify/data-construct/@aws-amplify/graphql-default-value-transformer",
"@aws-amplify/data-construct/@aws-amplify/graphql-directives",
"@aws-amplify/data-construct/@aws-amplify/graphql-generation-transformer",
"@aws-amplify/data-construct/@aws-amplify/graphql-function-transformer",
"@aws-amplify/data-construct/@aws-amplify/graphql-http-transformer",
"@aws-amplify/data-construct/@aws-amplify/graphql-index-transformer",
Expand Down
3 changes: 2 additions & 1 deletion packages/amplify-data-construct/.jsii
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"@aws-amplify/graphql-default-value-transformer": "3.0.0",
"@aws-amplify/graphql-directives": "2.0.0",
"@aws-amplify/graphql-function-transformer": "3.0.0",
"@aws-amplify/graphql-generation-transformer": "0.1.0",
"@aws-amplify/graphql-http-transformer": "3.0.0",
"@aws-amplify/graphql-index-transformer": "3.0.0",
"@aws-amplify/graphql-maps-to-transformer": "4.0.0",
Expand Down Expand Up @@ -3886,5 +3887,5 @@
},
"types": {},
"version": "1.9.6",
"fingerprint": "SdJZCjnS8qNDxzovyPRKBjmX8BKQnFxJLzNMF5SN3Zs="
"fingerprint": "4GG4kJbY8BkoSdR5tyKQKBR+c7lqlaZWxFJJjeWILUA="
}
2 changes: 2 additions & 0 deletions packages/amplify-data-construct/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
"@aws-amplify/graphql-relational-transformer",
"@aws-amplify/graphql-searchable-transformer",
"@aws-amplify/graphql-sql-transformer",
"@aws-amplify/graphql-generation-transformer",
"@aws-amplify/platform-core",
"@aws-amplify/plugin-types",
"fs-extra",
Expand Down Expand Up @@ -91,6 +92,7 @@
"@aws-amplify/graphql-transformer": "2.0.0",
"@aws-amplify/graphql-transformer-core": "3.0.0",
"@aws-amplify/graphql-transformer-interfaces": "4.0.0",
"@aws-amplify/graphql-generation-transformer": "0.1.0",
"@aws-amplify/platform-core": "^0.2.0",
"@aws-amplify/plugin-types": "^0.4.1",
"charenc": "^0.0.2",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
/* tslint:disable */
/* eslint-disable */
// This file was automatically generated and should not be edited.

export type Recipe = {
__typename: 'Recipe';
ingredients?: Array<string | null> | null;
instructions?: string | null;
name?: string | null;
};

export type Todo = {
__typename: 'Todo';
content?: string | null;
createdAt: string;
id: string;
isDone?: boolean | null;
updatedAt: string;
};

export type GenerateRecipeQueryVariables = {
description?: string | null;
};

export type GenerateRecipeQuery = {
generateRecipe?: {
__typename: 'Recipe';
ingredients?: Array<string | null> | null;
instructions?: string | null;
name?: string | null;
} | null;
};

export type GetTodoQueryVariables = {
id: string;
};

export type GetTodoQuery = {
getTodo?: {
__typename: 'Todo';
content?: string | null;
createdAt: string;
id: string;
isDone?: boolean | null;
updatedAt: string;
} | null;
};

export type MakeTodoQueryVariables = {
description: string;
};

export type MakeTodoQuery = {
makeTodo?: {
__typename: 'Todo';
content?: string | null;
createdAt: string;
id: string;
isDone?: boolean | null;
updatedAt: string;
} | null;
};

export type SummarizeQueryVariables = {
input?: string | null;
};

export type SummarizeQuery = {
summarize?: string | null;
};

export type SolveEquationQueryVariables = {
equation?: string | null;
};

export type SolveEquation = {
solveEquation?: number | null;
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
/* eslint-disable import/namespace */
import * as path from 'path';
import * as fs from 'fs-extra';
import { DURATION_20_MINUTES } from '../../utils/duration-constants';
import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core';
import { cdkDeploy, cdkDestroy, initCDKProject } from '../../commands';
import { DDB_AMPLIFY_MANAGED_DATASOURCE_STRATEGY } from '@aws-amplify/graphql-transformer-core';
import { doAppSyncGraphqlQuery, TestDefinition, writeStackConfig, writeTestDefinitions } from '../../utils';
import { generateRecipe, makeTodo, solveEquation, summarize } from './graphql/queries';

jest.setTimeout(DURATION_20_MINUTES);

describe('generation', () => {
const baseProjFolderName = path.basename(__filename, '.test.ts');

describe('Generation Model', () => {
const projFolderName = `${baseProjFolderName}-model`;
let apiEndpoint: string;
let apiKey: string;
let projRoot: string;

beforeAll(async () => {
projRoot = await createNewProjectDir(projFolderName);
const templatePath = path.resolve(path.join(__dirname, '..', 'backends', 'configurable-stack'));
const name = await initCDKProject(projRoot, templatePath);

const generationSchemaPath = path.resolve(path.join(__dirname, 'graphql', 'schema-generation.graphql'));
const generationSchema = fs.readFileSync(generationSchemaPath).toString();

const testDefinitions: Record<string, TestDefinition> = {
generation: {
schema: [generationSchema].join('\n'),
strategy: DDB_AMPLIFY_MANAGED_DATASOURCE_STRATEGY,
},
};

writeStackConfig(projRoot, { prefix: 'Gen', useSandbox: true });
writeTestDefinitions(testDefinitions, projRoot);

const outputs = await cdkDeploy(projRoot, '--all');
apiEndpoint = outputs[name].awsAppsyncApiEndpoint;
apiKey = outputs[name].awsAppsyncApiKey;
});

afterAll(async () => {
try {
await cdkDestroy(projRoot, '--all');
} catch (err) {
console.log(`Error invoking 'cdk destroy': ${err}`);
}
deleteProjectDir(projRoot);
});

describe('Generation type', () => {
test('should generate a type', async () => {
const args = {
apiEndpoint,
auth: { apiKey },
};

const variables = {
description: `I'd like to make a gluten-free chocolate cake.`,
};

const generateRecipeResult = await doAppSyncGraphqlQuery({ ...args, query: generateRecipe, variables });
const recipe = generateRecipeResult.body.data.generateRecipe;
expect(recipe.name).toBeDefined();
palpatim marked this conversation as resolved.
Show resolved Hide resolved
});
});

describe('Generation scalar', () => {
test('should generate a string scalar type', async () => {
const args = {
apiEndpoint,
auth: { apiKey },
};

const variables = {
input: `
Two roads diverged in a yellow wood,
And sorry I could not travel both
And be one traveler, long I stood
And looked down one as far as I could
To where it bent in the undergrowth;

Then took the other, as just as fair,
And having perhaps the better claim,
Because it was grassy and wanted wear;
Though as for that the passing there
Had worn them really about the same,

And both that morning equally lay
In leaves no step had trodden black.
Oh, I kept the first for another day!
Yet knowing how way leads on to way,
I doubted if I should ever come back.

I shall be telling this with a sigh
Somewhere ages and ages hence:
Two roads diverged in a wood, and I—
I took the one less traveled by,
And that has made all the difference.`,
};
const summarizeResult = await doAppSyncGraphqlQuery({ ...args, query: summarize, variables });
const summary = summarizeResult.body.data.summarize;
expect(summary).toBeDefined();
});
});

test('should generate an int scalar type', async () => {
const args = {
apiEndpoint,
auth: { apiKey },
};

const variables = {
equation: `
There is a three-digit number. The second digit is four times as big as the third digit,
while the first digit is three less than the second digit. What is the number?
`,
};
const solveEquationResult = await doAppSyncGraphqlQuery({ ...args, query: solveEquation, variables });
const solution = solveEquationResult.body.data.solveEquation;
expect(solution).toBeDefined();
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're asserting that we get an answer here, not the correct answer because LLMs are not good (being generous) at math.

});

describe('Generation model', () => {
// TODO: This currently doesn't work because LLMs are not great at following regex pattern requirements, they'll sometimes return "<UNKNOWN>"
// which fails GraphQL type validation for implicitly generated required model values like id, createdAt, updatedAt.
xtest('should generate a model', async () => {
Comment on lines +128 to +130
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test is currently disabled because we throw in the transformer when the return type contains required fields typed as certain AppSync scalars.

We're exploring options, including prompt improvements, better regex pattern in JSON Schema tool definitions, and special case handling for models (omitting createdAt, updatedAt, and id in tool definition, and populating them in the resolver).

For now, this is an accepted current limitation.

const args = {
apiEndpoint,
auth: { apiKey },
};
const variables = {
description:
'I have to pick up the kids from school. One goes to soccer practice at 3:30pm and the other to swim practice at 4:30pm.',
};
const makeTodoResult = await doAppSyncGraphqlQuery({ ...args, query: makeTodo, variables });
const todo = makeTodoResult.body.data.makeTodo;
expect(todo.content).toBeDefined();
palpatim marked this conversation as resolved.
Show resolved Hide resolved
});
});
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/* tslint:disable */
/* eslint-disable */
// this is an auto generated file. This will be overwritten

import * as APITypes from '../API';
type GeneratedQuery<InputType, OutputType> = string & {
__generatedQueryInput: InputType;
__generatedQueryOutput: OutputType;
};

export const generateRecipe = /* GraphQL */ `query GenerateRecipe($description: String) {
generateRecipe(description: $description) {
ingredients
instructions
name
__typename
}
}
` as GeneratedQuery<APITypes.GenerateRecipeQueryVariables, APITypes.GenerateRecipeQuery>;
export const getTodo = /* GraphQL */ `query GetTodo($id: ID!) {
getTodo(id: $id) {
content
createdAt
id
isDone
updatedAt
__typename
}
}
` as GeneratedQuery<APITypes.GetTodoQueryVariables, APITypes.GetTodoQuery>;
export const makeTodo = /* GraphQL */ `query MakeTodo($description: String!) {
makeTodo(description: $description) {
content
createdAt
id
isDone
updatedAt
__typename
}
}
` as GeneratedQuery<APITypes.MakeTodoQueryVariables, APITypes.MakeTodoQuery>;
export const summarize = /* GraphQL */ `query Summarize($input: String) {
summarize(input: $input)
}
` as GeneratedQuery<APITypes.SummarizeQueryVariables, APITypes.SummarizeQuery>;

export const solveEquation = /* GraphQL */ `query SolveEquation($equation: String) {
solveEquation(equation: $equation)
}
` as GeneratedQuery<APITypes.SolveEquationQueryVariables, APITypes.SolveEquation>;
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
type Todo @model(queries: { list: null }, mutations: null, subscriptions: null) {
content: String
isDone: Boolean
}

type Recipe {
name: String
ingredients: [String]
instructions: String
}

type Query {
makeTodo(description: String!): Todo
@generation(aiModel: "anthropic.claude-3-haiku-20240307-v1:0", systemPrompt: "Make a list of todo items based on the description.")

summarize(input: String): String @generation(aiModel: "anthropic.claude-3-haiku-20240307-v1:0", systemPrompt: "summarize the input.")

generateRecipe(description: String): Recipe
@generation(aiModel: "anthropic.claude-3-haiku-20240307-v1:0", systemPrompt: "You are a 3 start michelin chef that generates recipes.")
atierian marked this conversation as resolved.
Show resolved Hide resolved

solveEquation(equation: String): Int
@generation(aiModel: "anthropic.claude-3-haiku-20240307-v1:0", systemPrompt: "Solve the equation and return the result.")
}
3 changes: 2 additions & 1 deletion packages/amplify-graphql-api-construct/.jsii
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"@aws-amplify/graphql-default-value-transformer": "3.0.0",
"@aws-amplify/graphql-directives": "2.0.0",
"@aws-amplify/graphql-function-transformer": "3.0.0",
"@aws-amplify/graphql-generation-transformer": "0.1.0",
"@aws-amplify/graphql-http-transformer": "3.0.0",
"@aws-amplify/graphql-index-transformer": "3.0.0",
"@aws-amplify/graphql-maps-to-transformer": "4.0.0",
Expand Down Expand Up @@ -8803,5 +8804,5 @@
}
},
"version": "1.11.6",
"fingerprint": "jJBbP9Aj3KaF2GpI7kUkS7tzKs7wX3M5l9q2wtnSSjs="
"fingerprint": "y1xQmViCyD5+ag1HJuKzy9fnaVPN3Wtfz+iDk01/mwQ="
}
2 changes: 2 additions & 0 deletions packages/amplify-graphql-api-construct/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
"@aws-amplify/graphql-default-value-transformer",
"@aws-amplify/graphql-directives",
"@aws-amplify/graphql-function-transformer",
"@aws-amplify/graphql-generation-transformer",
"@aws-amplify/graphql-http-transformer",
"@aws-amplify/graphql-index-transformer",
"@aws-amplify/graphql-maps-to-transformer",
Expand Down Expand Up @@ -80,6 +81,7 @@
"@aws-amplify/graphql-default-value-transformer": "3.0.0",
"@aws-amplify/graphql-directives": "2.0.0",
"@aws-amplify/graphql-function-transformer": "3.0.0",
"@aws-amplify/graphql-generation-transformer": "0.1.0",
"@aws-amplify/graphql-http-transformer": "3.0.0",
"@aws-amplify/graphql-index-transformer": "3.0.0",
"@aws-amplify/graphql-maps-to-transformer": "4.0.0",
Expand Down
5 changes: 5 additions & 0 deletions packages/amplify-graphql-generation-transformer/.npmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
**/__mocks__/**
**/__tests__/**
src
tsconfig.json
tsconfig.tsbuildinfo
Loading
Loading