diff --git a/pom.xml b/pom.xml index 1687785..3e95b08 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,7 @@ 2.43.0 3.1.1 3.5.0 + 1.10.4 @@ -44,6 +45,11 @@ guava ${guava.version} + + com.google.auto.value + auto-value-annotations + ${auto-value.version} + org.slf4j slf4j-api @@ -101,6 +107,19 @@ + + maven-compiler-plugin + + + + com.google.auto.value + auto-value + ${auto-value.version} + + + + + org.codehaus.mojo exec-maven-plugin diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java b/src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java index d694412..2c9dd21 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/diff/DdlDiff.java @@ -18,6 +18,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.auto.value.AutoValue; import com.google.cloud.solutions.spannerddl.parser.ASTadd_row_deletion_policy; import com.google.cloud.solutions.spannerddl.parser.ASTalter_database_statement; import com.google.cloud.solutions.spannerddl.parser.ASTalter_table_statement; @@ -25,6 +26,7 @@ import com.google.cloud.solutions.spannerddl.parser.ASTcolumn_def; import com.google.cloud.solutions.spannerddl.parser.ASTcolumn_default_clause; import com.google.cloud.solutions.spannerddl.parser.ASTcolumn_type; +import com.google.cloud.solutions.spannerddl.parser.ASTcreate_change_stream_statement; import com.google.cloud.solutions.spannerddl.parser.ASTcreate_index_statement; import com.google.cloud.solutions.spannerddl.parser.ASTcreate_table_statement; import com.google.cloud.solutions.spannerddl.parser.ASTddl_statement; @@ -59,7 +61,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; @@ -98,72 +99,38 @@ public class DdlDiff { public static final String ALLOW_DROP_STATEMENTS_OPT = "allowDropStatements"; public static final String HELP_OPT = "help"; + private final DatbaseDefinition originalDb; + private final DatbaseDefinition newDb; private final MapDifference indexDifferences; private final MapDifference tableDifferences; private final MapDifference constraintDifferences; - private final Map newTablesCreationOrder; - private final Map originalTablesCreationOrder; private final MapDifference ttlDifferences; private final MapDifference alterDatabaseOptionsDifferences; + private final MapDifference changeStreamDifferences; private final String databaseName; // for alter Database - /** - * Wrapper class for Check and Foreign Key constraints to include the table name for when they are - * separated from their create table/alter table statements in - * separateTablesIndexesConstraintsTtls(). - */ - private static class ConstraintWrapper { - - private final String tableName; - private final SimpleNode constraint; - - private ConstraintWrapper(String tableName, SimpleNode constraint) { - this.tableName = tableName; - this.constraint = constraint; - if (!(constraint instanceof ASTforeign_key) && !(constraint instanceof ASTcheck_constraint)) { - throw new IllegalArgumentException( - "not a valid constraint type : " + constraint.toString()); - } - } - - private String getName() { - if (constraint instanceof ASTcheck_constraint) { - return ((ASTcheck_constraint) constraint).getName(); - } - if (constraint instanceof ASTforeign_key) { - return ((ASTforeign_key) constraint).getName(); - } - throw new IllegalArgumentException("not a valid constraint type : " + constraint.toString()); - } + private DdlDiff(DatbaseDefinition originalDb, DatbaseDefinition newDb, String databaseName) + throws DdlDiffException { + this.originalDb = originalDb; + this.newDb = newDb; + this.databaseName = databaseName; - @Override - public boolean equals(Object other) { - if (other instanceof ConstraintWrapper) { - return this.constraint.equals(((ConstraintWrapper) other).constraint); - } - return false; + this.tableDifferences = + Maps.difference(originalDb.tablesInCreationOrder(), newDb.tablesInCreationOrder()); + this.indexDifferences = Maps.difference(originalDb.indexes(), newDb.indexes()); + this.constraintDifferences = Maps.difference(originalDb.constraints(), newDb.constraints()); + this.ttlDifferences = Maps.difference(originalDb.ttls(), newDb.ttls()); + this.alterDatabaseOptionsDifferences = + Maps.difference(originalDb.alterDatabaseOptions(), newDb.alterDatabaseOptions()); + this.changeStreamDifferences = + Maps.difference(originalDb.changeStreams(), newDb.changeStreams()); + + if (!alterDatabaseOptionsDifferences.areEqual() && Strings.isNullOrEmpty(databaseName)) { + // should never happen, but... + throw new DdlDiffException("No database ID defined - required for Alter Database statements"); } } - private DdlDiff( - MapDifference tableDifferences, - Map originalTablesCreationOrder, - Map newTablesCreationOrder, - MapDifference indexDifferences, - MapDifference constraintDifferences, - MapDifference ttlDifferences, - MapDifference alterDatabaseOptionsDifferences, - String databaseName) { - this.tableDifferences = tableDifferences; - this.originalTablesCreationOrder = originalTablesCreationOrder; - this.newTablesCreationOrder = newTablesCreationOrder; - this.indexDifferences = indexDifferences; - this.constraintDifferences = constraintDifferences; - this.ttlDifferences = ttlDifferences; - this.alterDatabaseOptionsDifferences = alterDatabaseOptionsDifferences; - this.databaseName = databaseName; - } - public List generateDifferenceStatements(Map options) throws DdlDiffException { ImmutableList.Builder output = ImmutableList.builder(); @@ -173,7 +140,9 @@ public List generateDifferenceStatements(Map options) if (!indexDifferences.entriesDiffering().isEmpty() && !options.get(ALLOW_RECREATE_INDEXES_OPT)) { throw new DdlDiffException( - "At least one Index differs, and allowRecreateIndexes is not set.\n" + "At least one Index differs, and " + + ALLOW_RECREATE_INDEXES_OPT + + " is not set.\n" + "Indexes: " + Joiner.on(", ").join(indexDifferences.entriesDiffering().keySet())); } @@ -204,6 +173,15 @@ public List generateDifferenceStatements(Map options) } } + // Drop deleted change streams. + if (allowDropStatements) { + // Drop deleted indexes. + for (String changeStreamName : changeStreamDifferences.entriesOnlyOnLeft().keySet()) { + LOG.info("Dropping deleted change stream: {}", changeStreamName); + output.add("DROP CHANGE STREAM " + changeStreamName); + } + } + // Drop modified indexes that need to be re-created... for (String indexName : indexDifferences.entriesDiffering().keySet()) { LOG.info("Dropping changed index for re-creation: {}", indexName); @@ -212,7 +190,7 @@ public List generateDifferenceStatements(Map options) // Drop deleted constraints for (ConstraintWrapper fk : constraintDifferences.entriesOnlyOnLeft().values()) { - output.add("ALTER TABLE " + fk.tableName + " DROP CONSTRAINT " + fk.getName()); + output.add("ALTER TABLE " + fk.tableName() + " DROP CONSTRAINT " + fk.getName()); } // Drop modified constraints that need to be re-created... @@ -220,7 +198,7 @@ public List generateDifferenceStatements(Map options) constraintDifferences.entriesDiffering().values()) { output.add( "ALTER TABLE " - + fkDiff.leftValue().tableName + + fkDiff.leftValue().tableName() + " DROP CONSTRAINT " + fkDiff.leftValue().getName()); } @@ -232,7 +210,8 @@ public List generateDifferenceStatements(Map options) if (allowDropStatements) { // Drop tables that have been deleted -- need to do it in reverse creation order. - List reverseOrderedTableNames = new ArrayList<>(originalTablesCreationOrder.keySet()); + List reverseOrderedTableNames = + new ArrayList<>(originalDb.tablesInCreationOrder().keySet()); Collections.reverse(reverseOrderedTableNames); for (String tableName : reverseOrderedTableNames) { if (tableDifferences.entriesOnlyOnLeft().containsKey(tableName)) { @@ -252,7 +231,7 @@ public List generateDifferenceStatements(Map options) // Create new tables. Must be done in the order of creation in the new DDL. for (Map.Entry newTableEntry : - newTablesCreationOrder.entrySet()) { + newDb.tablesInCreationOrder().entrySet()) { if (tableDifferences.entriesOnlyOnRight().containsKey(newTableEntry.getKey())) { LOG.info("Creating new table: {}", newTableEntry.getKey()); output.add(newTableEntry.getValue().toStringOptionalExistClause(false)); @@ -290,7 +269,7 @@ public List generateDifferenceStatements(Map options) // Create new constraints. for (ConstraintWrapper fk : constraintDifferences.entriesOnlyOnRight().values()) { - output.add("ALTER TABLE " + fk.tableName + " ADD " + fk.constraint); + output.add("ALTER TABLE " + fk.tableName() + " ADD " + fk.constraint()); } // Re-create modified constraints. @@ -298,10 +277,43 @@ public List generateDifferenceStatements(Map options) constraintDifferences.entriesDiffering().values()) { output.add( "ALTER TABLE " - + constraintDiff.rightValue().tableName + + constraintDiff.rightValue().tableName() + " ADD " - + constraintDiff.rightValue().constraint.toString()); + + constraintDiff.rightValue().constraint().toString()); } + + // Create new change streams + for (ASTcreate_change_stream_statement newChangeStream : + changeStreamDifferences.entriesOnlyOnRight().values()) { + LOG.info("Creating new change stream: {}", newChangeStream.getName()); + output.add(newChangeStream.toString()); + } + + // Alter existing change streams + for (ValueDifference changedChangeStream : + changeStreamDifferences.entriesDiffering().values()) { + String oldForClause = changedChangeStream.leftValue().getForClause().toString(); + String newForClause = changedChangeStream.rightValue().getForClause().toString(); + + String oldOptions = changedChangeStream.leftValue().getOptionsClause().toString(); + String newOptions = changedChangeStream.rightValue().getOptionsClause().toString(); + + if (!oldForClause.equals(newForClause)) { + output.add( + "ALTER CHANGE STREAM " + + changedChangeStream.rightValue().getName() + + " SET " + + newForClause); + } + if (!oldOptions.equals(newOptions)) { + output.add( + "ALTER CHANGE STREAM " + + changedChangeStream.rightValue().getName() + + " SET " + + newOptions); + } + } + return output.build(); } @@ -527,57 +539,24 @@ private static String generateOptionsUpdates(MapDifference optio } static DdlDiff build(String originalDDL, String newDDL) throws DdlDiffException { - List originalStatements = parseDDL(Strings.nullToEmpty(originalDDL)); - List newStatements = parseDDL(Strings.nullToEmpty(newDDL)); - - Map originalTablesInCreationOrder = new LinkedHashMap<>(); - Map originalIndexes = new TreeMap<>(); - Map originalConstraints = new TreeMap<>(); - Map originalTtls = new TreeMap<>(); - - separateTablesIndexesConstraintsTtls( - originalStatements, - originalTablesInCreationOrder, - originalIndexes, - originalConstraints, - originalTtls); - - Map newTablesInCreationOrder = new LinkedHashMap<>(); - Map newIndexes = new TreeMap<>(); - Map newConstraints = new TreeMap<>(); - Map newTtls = new TreeMap<>(); - - separateTablesIndexesConstraintsTtls( - newStatements, newTablesInCreationOrder, newIndexes, newConstraints, newTtls); - - // Get DatabaseOptions diffs - Map originalOptions = getOptionsFromAlterDatabase(originalStatements); - Map newOptions = getOptionsFromAlterDatabase(newStatements); - MapDifference dbOptionsDiff = Maps.difference(originalOptions, newOptions); - String dbName = getDatabaseNameFromAlterDatabase(originalStatements, newStatements); - if (!dbOptionsDiff.areEqual() && Strings.isNullOrEmpty(dbName)) { - // should never happen, but... - throw new DdlDiffException("No database ID defined - required for Alter Database statements"); + List originalStatements; + List newStatements; + try { + originalStatements = parseDDL(Strings.nullToEmpty(originalDDL)); + } catch (DdlDiffException e) { + throw new DdlDiffException("Failed parsing ORIGINAL DDL: " + e.getMessage(), e); + } + try { + newStatements = parseDDL(Strings.nullToEmpty(newDDL)); + } catch (DdlDiffException e) { + throw new DdlDiffException("Failed parsing NEW DDL: " + e.getMessage(), e); } - return new DdlDiff( - Maps.difference(originalTablesInCreationOrder, newTablesInCreationOrder), - originalTablesInCreationOrder, - newTablesInCreationOrder, - Maps.difference(originalIndexes, newIndexes), - Maps.difference(originalConstraints, newConstraints), - Maps.difference(originalTtls, newTtls), - dbOptionsDiff, - dbName); - } + DatbaseDefinition originalDb = DatbaseDefinition.create(originalStatements); + DatbaseDefinition newDb = DatbaseDefinition.create(newStatements); - private static Map getOptionsFromAlterDatabase( - List originalStatements) { - Map originalOptions = new TreeMap<>(); - getAlterDatabaseStatementStream(originalStatements) - .map(s -> s.getOptionsClause().getKeyValueMap()) - .forEach(originalOptions::putAll); - return originalOptions; + return new DdlDiff( + originalDb, newDb, getDatabaseNameFromAlterDatabase(originalStatements, newStatements)); } private static String getDatabaseNameFromAlterDatabase( @@ -601,7 +580,9 @@ private static String getDatabaseNameFromAlterDatabase( private static String getDatabaseNameFromAlterDatabase(List statements) throws DdlDiffException { Set names = - getAlterDatabaseStatementStream(statements) + statements.stream() + .filter(s -> s.jjtGetChild(0) instanceof ASTalter_database_statement) + .map(s -> ((ASTalter_database_statement) s.jjtGetChild(0))) .map(ASTalter_database_statement::getDbName) .collect(Collectors.toSet()); if (names.size() > 1) { @@ -614,90 +595,6 @@ private static String getDatabaseNameFromAlterDatabase(List st } } - private static Stream getAlterDatabaseStatementStream( - List statements) { - return statements.stream() - .filter(s -> s.jjtGetChild(0) instanceof ASTalter_database_statement) - .map(s -> ((ASTalter_database_statement) s.jjtGetChild(0))); - } - - /** - * Separarates the index, constraints, and Row Deletion policy creation statements from the Table - * creation statement, and put them - along with any Alter statements that create these same - * objects - into a separate maps. - * - *

This allows the diff tool to handle these objects which are created inline with the table in - * the same way as if they were created separately with ALTER statements. - */ - private static void separateTablesIndexesConstraintsTtls( - List statements, - Map tables, - Map indexes, - Map constraints, - Map ttls) { - for (ASTddl_statement ddlStatement : statements) { - final SimpleNode statement = (SimpleNode) ddlStatement.jjtGetChild(0); - - switch (statement.getId()) { - case DdlParserTreeConstants.JJTCREATE_TABLE_STATEMENT: - { - ASTcreate_table_statement createTable = (ASTcreate_table_statement) statement; - // Remove embedded constraint statements from the CreateTable node - // as they are taken into account via `constraints` - tables.put(createTable.getTableName(), createTable.clearConstraints()); - - // convert embedded constraint statements into wrapper object with table name - // use a single map for all foreign keys, constraints and row deletion polcies whether - // created in table or externally - createTable.getConstraints().values().stream() - .map(c -> new ConstraintWrapper(createTable.getTableName(), c)) - .forEach(c -> constraints.put(c.getName(), c)); - - // Move embedded Row Deletion Policies - final Optional rowDeletionPolicyClause = - createTable.getRowDeletionPolicyClause(); - rowDeletionPolicyClause.ifPresent(rdp -> ttls.put(createTable.getTableName(), rdp)); - } - break; - case DdlParserTreeConstants.JJTCREATE_INDEX_STATEMENT: - { - ASTcreate_index_statement createIndex = (ASTcreate_index_statement) statement; - indexes.put(createIndex.getIndexName(), createIndex); - } - break; - case DdlParserTreeConstants.JJTALTER_TABLE_STATEMENT: - { - // Alter table can be adding Index, Constraint or Row Deletion Policy - ASTalter_table_statement alterTable = (ASTalter_table_statement) statement; - final String tableName = alterTable.jjtGetChild(0).toString(); - - if (alterTable.jjtGetChild(1) instanceof ASTforeign_key - || alterTable.jjtGetChild(1) instanceof ASTcheck_constraint) { - ConstraintWrapper constraint = - new ConstraintWrapper(tableName, (SimpleNode) alterTable.jjtGetChild(1)); - constraints.put(constraint.getName(), constraint); - - } else if (statement.jjtGetChild(1) instanceof ASTadd_row_deletion_policy) { - ttls.put( - tableName, - (ASTrow_deletion_policy_clause) alterTable.jjtGetChild(1).jjtGetChild(0)); - } else { - // other ALTER statements are not supported. - throw new IllegalArgumentException( - "Unsupported ALTER TABLE statement: " - + ASTTreeUtils.tokensToString(ddlStatement)); - } - } - break; - case DdlParserTreeConstants.JJTALTER_DATABASE_STATEMENT: - break; - default: - throw new IllegalArgumentException( - "Unsupported statement: " + ASTTreeUtils.tokensToString(ddlStatement)); - } - } - } - @VisibleForTesting static List parseDDL(String original) throws DdlDiffException { // Remove "--" comments and split by ";" @@ -726,8 +623,9 @@ static List parseDDL(String original) throws DdlDiffException throw new IllegalArgumentException( "Unsupported statement:\n" + statement - + "\nCan only create diffs from 'CREATE TABLE, CREATE INDEX and " - + "'ALTER TABLE table_name ADD ' DDL statements"); + + "\n" + + "Can only create diffs from 'CREATE TABLE, CREATE INDEX and 'ALTER TABLE" + + " table_name ADD [constraint|row deletion policy]' DDL statements"); } if (alterTableStatement.jjtGetChild(1) instanceof ASTforeign_key && ((ASTforeign_key) alterTableStatement.jjtGetChild(1)) @@ -761,6 +659,7 @@ static List parseDDL(String original) throws DdlDiffException break; case DdlParserTreeConstants.JJTCREATE_INDEX_STATEMENT: case DdlParserTreeConstants.JJTALTER_DATABASE_STATEMENT: + case DdlParserTreeConstants.JJTCREATE_CHANGE_STREAM_STATEMENT: // no-op break; default: @@ -773,7 +672,9 @@ static List parseDDL(String original) throws DdlDiffException ddlStatements.add(ddlStatement); } catch (ParseException e) { throw new DdlDiffException( - String.format("Unable to parse statement:\n%s\n%s", statement, e.getMessage()), e); + String.format( + "Unable to parse statement:\n'%s'\nFailure: %s", statement, e.getMessage()), + e); } } return ddlStatements; @@ -920,3 +821,143 @@ private static void printHelpAndExit(int exitStatus) { System.exit(exitStatus); } } + +/** + * Wrapper class for Check and Foreign Key constraints to include the table name for when they are + * separated from their create table/alter table statements in + * separateTablesIndexesConstraintsTtls(). + */ +@AutoValue +abstract class ConstraintWrapper { + + static ConstraintWrapper create(String tableName, SimpleNode constraint) { + if (!(constraint instanceof ASTforeign_key) && !(constraint instanceof ASTcheck_constraint)) { + throw new IllegalArgumentException("not a valid constraint type : " + constraint.toString()); + } + return new AutoValue_ConstraintWrapper(tableName, constraint); + } + + abstract String tableName(); + + abstract SimpleNode constraint(); + + String getName() { + if (constraint() instanceof ASTcheck_constraint) { + return ((ASTcheck_constraint) constraint()).getName(); + } + if (constraint() instanceof ASTforeign_key) { + return ((ASTforeign_key) constraint()).getName(); + } + throw new IllegalArgumentException("not a valid constraint type : " + constraint().toString()); + } +} + +/** + * Separarates the different DDL creation statements into separate maps. + * + *

Constraints which were created inline with their table are separated into a map with any other + * ALTER statements which adds constraints. + * + *

This allows the diff tool to handle these objects which are created inline with the table in + * the same way as if they were created separately with ALTER statements. + */ +@AutoValue +abstract class DatbaseDefinition { + static DatbaseDefinition create(List statements) { + // Use LinkedHashMap to preserve creation order in original DDL. + LinkedHashMap tablesInCreationOrder = new LinkedHashMap<>(); + LinkedHashMap indexes = new LinkedHashMap<>(); + LinkedHashMap constraints = new LinkedHashMap<>(); + LinkedHashMap ttls = new LinkedHashMap<>(); + LinkedHashMap changeStreams = new LinkedHashMap<>(); + LinkedHashMap alterDatabaseOptions = new LinkedHashMap<>(); + + for (ASTddl_statement ddlStatement : statements) { + final SimpleNode statement = (SimpleNode) ddlStatement.jjtGetChild(0); + + switch (statement.getId()) { + case DdlParserTreeConstants.JJTCREATE_TABLE_STATEMENT: + { + ASTcreate_table_statement createTable = (ASTcreate_table_statement) statement; + // Remove embedded constraint statements from the CreateTable node + // as they are taken into account via `constraints` + tablesInCreationOrder.put(createTable.getTableName(), createTable.clearConstraints()); + + // convert embedded constraint statements into wrapper object with table name + // use a single map for all foreign keys, constraints and row deletion polcies whether + // created in table or externally + createTable.getConstraints().values().stream() + .map(c -> ConstraintWrapper.create(createTable.getTableName(), c)) + .forEach(c -> constraints.put(c.getName(), c)); + + // Move embedded Row Deletion Policies + final Optional rowDeletionPolicyClause = + createTable.getRowDeletionPolicyClause(); + rowDeletionPolicyClause.ifPresent(rdp -> ttls.put(createTable.getTableName(), rdp)); + } + break; + case DdlParserTreeConstants.JJTCREATE_INDEX_STATEMENT: + indexes.put( + ((ASTcreate_index_statement) statement).getIndexName(), + (ASTcreate_index_statement) statement); + break; + case DdlParserTreeConstants.JJTALTER_TABLE_STATEMENT: + { + // Alter table can be adding Index, Constraint or Row Deletion Policy + ASTalter_table_statement alterTable = (ASTalter_table_statement) statement; + final String tableName = alterTable.jjtGetChild(0).toString(); + + if (alterTable.jjtGetChild(1) instanceof ASTforeign_key + || alterTable.jjtGetChild(1) instanceof ASTcheck_constraint) { + ConstraintWrapper constraint = + ConstraintWrapper.create(tableName, (SimpleNode) alterTable.jjtGetChild(1)); + constraints.put(constraint.getName(), constraint); + + } else if (statement.jjtGetChild(1) instanceof ASTadd_row_deletion_policy) { + ttls.put( + tableName, + (ASTrow_deletion_policy_clause) alterTable.jjtGetChild(1).jjtGetChild(0)); + } else { + // other ALTER statements are not supported. + throw new IllegalArgumentException( + "Unsupported ALTER TABLE statement: " + + ASTTreeUtils.tokensToString(ddlStatement)); + } + } + break; + case DdlParserTreeConstants.JJTALTER_DATABASE_STATEMENT: + alterDatabaseOptions.putAll( + ((ASTalter_database_statement) statement).getOptionsClause().getKeyValueMap()); + break; + case DdlParserTreeConstants.JJTCREATE_CHANGE_STREAM_STATEMENT: + changeStreams.put( + ((ASTcreate_change_stream_statement) statement).getName(), + (ASTcreate_change_stream_statement) statement); + break; + default: + throw new IllegalArgumentException( + "Unsupported statement: " + ASTTreeUtils.tokensToString(ddlStatement)); + } + } + System.out.println(constraints.keySet()); + return new AutoValue_DatbaseDefinition( + ImmutableMap.copyOf(tablesInCreationOrder), + ImmutableMap.copyOf(indexes), + ImmutableMap.copyOf(constraints), + ImmutableMap.copyOf(ttls), + ImmutableMap.copyOf(changeStreams), + ImmutableMap.copyOf(alterDatabaseOptions)); + } + + abstract Map tablesInCreationOrder(); + + abstract Map indexes(); + + abstract Map constraints(); + + abstract Map ttls(); + + abstract Map changeStreams(); + + abstract Map alterDatabaseOptions(); +} diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTchange_stream_for_clause.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTchange_stream_for_clause.java new file mode 100644 index 0000000..664aba3 --- /dev/null +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTchange_stream_for_clause.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.solutions.spannerddl.parser; + +import com.google.cloud.solutions.spannerddl.diff.ASTTreeUtils; + +public class ASTchange_stream_for_clause extends SimpleNode { + public ASTchange_stream_for_clause(int id) { + super(id); + } + + public ASTchange_stream_for_clause(DdlParser p, int id) { + super(p, id); + } + + @Override + public String toString() { + ASTchange_stream_tracked_tables tables = + ASTTreeUtils.getOptionalChildByType(children, ASTchange_stream_tracked_tables.class); + if (tables != null) { + return "FOR " + ASTTreeUtils.tokensToString(tables, false); + } + return "FOR ALL"; + } +} diff --git a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_change_stream_statement.java b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_change_stream_statement.java index d8b927a..1b026d8 100644 --- a/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_change_stream_statement.java +++ b/src/main/java/com/google/cloud/solutions/spannerddl/parser/ASTcreate_change_stream_statement.java @@ -15,15 +15,44 @@ */ package com.google.cloud.solutions.spannerddl.parser; +import com.google.cloud.solutions.spannerddl.diff.ASTTreeUtils; +import com.google.common.base.Joiner; + +/** + * @link + * https://cloud.google.com/spanner/docs/reference/standard-sql/data-definition-language#create-change-stream + */ public class ASTcreate_change_stream_statement extends SimpleNode { public ASTcreate_change_stream_statement(int id) { - super(id); - throw new UnsupportedOperationException("Not Implemented"); } public ASTcreate_change_stream_statement(DdlParser p, int id) { super(p, id); - throw new UnsupportedOperationException("Not Implemented"); + } + + public String getName() { + return ASTTreeUtils.getChildByType(children, ASTname.class).toString(); + } + + public ASTchange_stream_for_clause getForClause() { + return ASTTreeUtils.getOptionalChildByType(children, ASTchange_stream_for_clause.class); + } + + public ASToptions_clause getOptionsClause() { + return ASTTreeUtils.getOptionalChildByType(children, ASToptions_clause.class); + } + + @Override + public String toString() { + return Joiner.on(" ") + .skipNulls() + .join("CREATE CHANGE STREAM", getName(), getForClause(), getOptionsClause()); + } + + @Override + public boolean equals(Object other) { + return (other instanceof ASTcreate_change_stream_statement) + && this.toString().equals(other.toString()); } } diff --git a/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java b/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java index 6c36b0a..7790e99 100644 --- a/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java +++ b/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffFromFilesTest.java @@ -77,7 +77,7 @@ public void compareDddTextFiles() throws IOException { // build an expectedResults without any column or table drops. List expectedDiffNoDrops = expectedDiff.stream() - .filter(statement -> !statement.matches(".*DROP (TABLE|COLUMN).*")) + .filter(statement -> !statement.matches(".*DROP (TABLE|COLUMN|CHANGE STREAM).*")) .collect(Collectors.toCollection(LinkedList::new)); // remove any drop indexes from the expectedResults if they do not have an equivalent @@ -109,9 +109,11 @@ public void compareDddTextFiles() throws IOException { .isEqualTo(expectedDiffNoDrops); } } catch (DdlDiffException e) { - fail("DdlDiffException when processing segment " + segmentName + ": " + e); + fail("DdlDiffException when processing segment:\n'" + segmentName + "''\n" + e.getMessage()); } catch (Exception e) { - throw new Error("Unexpected exception when processing segment " + segmentName + ": " + e, e); + throw new Error( + "Unexpected exception when processing segment \n'" + segmentName + "'\n" + e.getMessage(), + e); } if (originalSegmentIt.hasNext()) { diff --git a/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffTest.java b/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffTest.java index 3278032..cd1bb3c 100644 --- a/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffTest.java +++ b/src/test/java/com/google/cloud/solutions/spannerddl/diff/DdlDiffTest.java @@ -407,7 +407,7 @@ public void parseAlterDatabaseDifferentDbNames() throws ParseException { getDiffCheckDdlDiffException( "ALTER DATABASE dbname SET OPTIONS(hello='world');" - + "ALTER DATABASE otherdbname SET OPTIONS(hello='world');", + + "ALTER DATABASE otherdbname SET OPTIONS(goodbye='world');", "", true, "Multiple database IDs defined"); diff --git a/src/test/resources/ddlParserUnsupported.txt b/src/test/resources/ddlParserUnsupported.txt index ad392e0..4ea0ca5 100644 --- a/src/test/resources/ddlParserUnsupported.txt +++ b/src/test/resources/ddlParserUnsupported.txt @@ -32,10 +32,6 @@ CREATE OR REPLACE VIEW test1 SQL SECURITY INVOKER AS SELECT * from test2 CREATE VIEW test1 SQL SECURITY INVOKER AS SELECT * from test2 -== Test 5 - -Create change stream test1 for test2 - == Test 6 drop change stream test1 @@ -48,15 +44,11 @@ drop index test1 drop table test1 -== Test 9 // TODO Change streams - -CREATE CHANGE STREAM change_stream_name FOR ALL - -== Test 9a - alter change stream not supported +== Test 9 - alter change stream not supported ALTER CHANGE STREAM change_stream_name DROP FOR ALL -== Test 9b - drop change stream not supported +== Test 9a - drop change stream not supported DROP CHANGE STREAM change_stream_name diff --git a/src/test/resources/ddlParserValidation.txt b/src/test/resources/ddlParserValidation.txt index c4a5fb8..8b43d57 100644 --- a/src/test/resources/ddlParserValidation.txt +++ b/src/test/resources/ddlParserValidation.txt @@ -124,4 +124,12 @@ CREATE TABLE test_table ( CREATE TABLE test_table ( intcol INT64 NOT NULL HIDDEN ) PRIMARY KEY (intcol ASC) +== Test 12 change stream for all + +CREATE CHANGE STREAM change_stream_name FOR ALL OPTIONS (retention_period='1d',value_capture_type='OLD_AND_NEW_VALUES') + +== Test 12b change stream for certain cols + +CREATE CHANGE STREAM change_stream_name FOR table1, table2 ( ), table3 ( col1, col2 ) OPTIONS (retention_period='7d',value_capture_type='NEW_ROW') + == diff --git a/src/test/resources/expectedDdlDiff.txt b/src/test/resources/expectedDdlDiff.txt index 26af2f7..4857a01 100644 --- a/src/test/resources/expectedDdlDiff.txt +++ b/src/test/resources/expectedDdlDiff.txt @@ -97,18 +97,18 @@ ALTER TABLE test1 DROP CONSTRAINT fk_in_alter DROP INDEX index2 DROP INDEX index1 -ALTER TABLE test2 DROP CONSTRAINT ch_in_test2 ALTER TABLE test2 DROP CONSTRAINT fk_in_test2 -ALTER TABLE test1 DROP CONSTRAINT ch_in_test1 +ALTER TABLE test2 DROP CONSTRAINT ch_in_test2 ALTER TABLE test1 DROP CONSTRAINT fk_in_test1 +ALTER TABLE test1 DROP CONSTRAINT ch_in_test1 DROP TABLE test2 ALTER TABLE test1 ADD COLUMN col3 INT64 CREATE TABLE test3 ( col1 INT64 ) PRIMARY KEY (col1 ASC) CREATE INDEX index1 ON test1 ( col3 ASC ) -ALTER TABLE test3 ADD CONSTRAINT ch_in_test3 CHECK (col1 = col3 and col1 > 100 and col2 < -50) ALTER TABLE test3 ADD CONSTRAINT fk_in_test3 FOREIGN KEY ( col3 ) REFERENCES othertable ( othercol ) ON DELETE NO ACTION -ALTER TABLE test1 ADD CONSTRAINT ch_in_test1 CHECK (col1 = col3 and col1 > 100 and col2 < -50) +ALTER TABLE test3 ADD CONSTRAINT ch_in_test3 CHECK (col1 = col3 and col1 > 100 and col2 < -50) ALTER TABLE test1 ADD CONSTRAINT fk_in_test1 FOREIGN KEY ( col3 ) REFERENCES othertable ( othercol ) ON DELETE NO ACTION +ALTER TABLE test1 ADD CONSTRAINT ch_in_test1 CHECK (col1 = col3 and col1 > 100 and col2 < -50) == TEST 16 add check constraint via alter statement @@ -252,5 +252,17 @@ ALTER TABLE test1 ADD CONSTRAINT fk_in_table FOREIGN KEY ( col2 ) REFERENCES oth DROP INDEX test4 CREATE INDEX test4 ON test1 ( col1 ASC ) STORING ( col2 ) +== TEST 48 change streams create modify delete in correct order wrt tables + +DROP CHANGE STREAM toBeDeleted +DROP TABLE myToBeDeletedTable +CREATE TABLE myCreatedTable ( mycol INT64 ) PRIMARY KEY (mycol ASC) +CREATE CHANGE STREAM toCreate FOR mytable4 OPTIONS (retention_period='36h') +CREATE CHANGE STREAM toCreateAll FOR ALL +ALTER CHANGE STREAM toBeChanged SET FOR myTable2 ( col1, col3, col4 ), mytable3 ( ) +ALTER CHANGE STREAM toBeChanged SET OPTIONS (retention_period='48h') +ALTER CHANGE STREAM toBeChangedOnlyTable SET FOR myTable1, myTable2 ( col1 ) +ALTER CHANGE STREAM toBeChangedOnlyOptions SET OPTIONS (retention_period='48h') + == diff --git a/src/test/resources/newDdl.txt b/src/test/resources/newDdl.txt index 0802c73..1322b08 100644 --- a/src/test/resources/newDdl.txt +++ b/src/test/resources/newDdl.txt @@ -427,5 +427,14 @@ primary key (col1); Create index IF NOT EXISTS test4 on test1 ( col1 ) STORING ( col2 ) -== +== TEST 48 change streams create modify delete in correct order wrt tables + +Create table myCreatedTable (mycol int64) primary key (mycol); +create change stream toremain for all options (retention_period = '36h'); +create change stream toBeChanged for myTable2 ( col1, col3, col4), mytable3 () options (retention_period = '48h'); +create change stream toCreate for mytable4 options (retention_period = '36h'); +create change stream toCreateAll for all; +create change stream toBeChangedOnlyTable for myTable1, myTable2 ( col1) options (retention_period = '36h'); +create change stream toBeChangedOnlyOptions for myTable1, myTable2 ( col1, col3) options (retention_period = '48h'); +== diff --git a/src/test/resources/originalDdl.txt b/src/test/resources/originalDdl.txt index c603ee5..bc54e01 100644 --- a/src/test/resources/originalDdl.txt +++ b/src/test/resources/originalDdl.txt @@ -149,6 +149,7 @@ create table test1 ( constraint ch_in_test1 check (col1=col2 and col1 > 100 and col2 < -50) ) primary key (col1); + create table test2 ( col1 int64, col2 int64 NOT NULL, @@ -156,7 +157,9 @@ create table test2 ( constraint ch_in_test2 check (col1=col2 and col1 > 100 and col2 < -50) ) primary key (col1); + create index index1 on test1 (col1); + create index index2 on test2 (col1); @@ -423,5 +426,15 @@ primary key (col1); Create index IF NOT EXISTS test4 on test1 ( col1 ) +== TEST 48 change streams create modify delete in correct order wrt tables + +Create table myToBeDeletedTable (mycol int64) primary key (mycol); +create change stream toremain for all options (retention_period = '36h'); +create change stream toBeDeleted for myTable options (retention_period = '36h'); +create change stream toBeChanged for myTable1, myTable2 ( col1, col3) options (retention_period = '36h'); +create change stream toBeChangedOnlyTable for myTable1, myTable2 ( col1, col3) options (retention_period = '36h'); +create change stream toBeChangedOnlyOptions for myTable1, myTable2 ( col1, col3) options (retention_period = '36h'); + == +