From 6c6d945a6f9f920431265bd133061c946744296a Mon Sep 17 00:00:00 2001 From: Willam2004 Date: Sat, 8 Jun 2024 12:47:30 +0800 Subject: [PATCH] merge: https://github.com/alibaba/fast-modeling-language/issues/4 --- .gitignore | 2 + CHANGELOG | 20 + docs/zh-cn/layer/business_process.md | 3 + fastmodel-bom/pom.xml | 49 +- .../impl/CreateTableCompareNodeTest.java | 5 + .../compare/table/ColumnCompareTest.java | 34 + .../core/formatter/ExpressionVisitor.java | 3 +- .../core/formatter/FastModelVisitor.java | 32 +- .../aliyun/fastmodel/core/tree/Comment.java | 11 +- .../aliyun/fastmodel/core/tree/ListNode.java | 5 + .../fastmodel/core/tree/datatype/Field.java | 15 + .../core/tree/datatype/GenericDataType.java | 18 +- .../core/tree/datatype/NumericParameter.java | 10 +- .../core/tree/datatype/RowDataType.java | 14 + .../core/tree/datatype/TypeParameter.java | 9 + .../core/tree/expr/BaseExpression.java | 13 + .../statement/constants/ConstraintScope.java | 2 +- .../statement/constants/ConstraintType.java | 28 +- .../tree/statement/table/CreateTable.java | 9 +- .../table/constraint/CustomConstraint.java | 2 +- .../core/tree/util/PropertyUtil.java | 16 +- .../core/formatter/FastModelVisitorTest.java | 2 +- fastmodel-dependencies-bom/pom.xml | 53 +- fastmodel-jacoco/pom.xml | 6 + .../src/main/antlr4/imports/TableParser.g4 | 2 +- .../parser/visitor/TableVisitor.java | 12 +- .../fastmodel/parser/NodeParserTest.java | 11 +- .../resources/lsp/FastModelGrammarParser.g4 | 2 +- .../client/converter/BaseClientConverter.java | 65 +- .../api/client/dto/table/Column.java | 8 + .../api/context/TransformContext.java | 17 + .../transform/api/dialect/DialectName.java | 10 +- .../api/builder/model/TemplateDefineTest.java | 8 +- .../fastmodel-transform-example/pom.xml | 6 + .../fml/format/FmlFormatterTest.java | 3 +- .../transform/hive/HiveTransformer.java | 6 + .../client/converter/HiveClientConverter.java | 30 +- .../converter/HivePropertyConverter.java | 4 - .../hive/context/HiveTransformContext.java | 2 +- .../transform/hive/format/HiveHelper.java | 67 +- .../hive/format/HivePropertyKey.java | 23 +- .../transform/hive/format/HiveVisitor.java | 2 +- .../transform/hive/parser/HiveAstBuilder.java | 114 +- .../client/HiveCodeGeneratorExternalTest.java | 214 +- .../client/HiveCodeGeneratorModelTest.java | 3 +- .../converter/HiveClientConverterTest.java | 4 +- .../hive/parser/HiveAstBuilderTest.java | 68 + .../tree/datatype/HologresArrayDataType.java | 7 + .../datatype/HologresGenericDataType.java | 8 +- .../HologresClientConverterTest.java | 39 +- .../transform/spark/format/SparkVisitor.java | 2 +- .../spark/parser/SparkLanguageParserTest.java | 2 +- .../spark/parser/SparkTransformerTest.java | 2 +- .../fastmodel-transform-sqlite/pom.xml | 28 + .../transform/sqlite/parser/SQLiteLexer.g4 | 243 ++ .../transform/sqlite/parser/SQLiteParser.g4 | 915 ++++++ .../transform/sqlite/SqliteTransformer.java | 11 + .../format/SqliteExpressionVisitor.java | 27 + .../sqlite/format/SqliteVisitor.java | 5 + .../sqlite/parser/SqliteLanguageParser.java | 58 + .../parser/visitor/SqliteAstBuilder.java | 255 ++ .../sqlite/SqliteTransformerTest.java | 5 + .../parser/SqliteLanguageParserTest.java | 124 + .../fastmodel-transform-starrocks/pom.xml | 40 + .../transform/starrocks/parser/StarRocks.g4 | 2621 +++++++++++++++++ .../src/main/antlr4/imports/StarRocksLex.g4 | 553 ++++ .../starrocks/StarRocksTransformer.java | 64 + .../starrocks/builder/DefaultBuilder.java | 36 + .../converter/StarRocksClientConverter.java | 408 +++ .../converter/StarRocksPropertyConverter.java | 53 + .../property/column/AggrColumnProperty.java | 30 + .../property/table/DistributeBucketsNum.java | 30 + .../client/property/table/DistributeHash.java | 34 + .../table/MultiRangePartitionProperty.java | 34 + .../client/property/table/ReplicationNum.java | 30 + .../table/SingleRangePartitionProperty.java | 34 + .../property/table/TablePartitionRaw.java | 27 + .../partition/ArrayClientPartitionKey.java | 25 + .../partition/BaseClientPartitionKey.java | 10 + .../partition/LessThanClientPartitionKey.java | 27 + .../partition/MultiRangeClientPartition.java | 35 + .../table/partition/PartitionClientValue.java | 27 + .../partition/SingleRangeClientPartition.java | 39 + .../starrocks/context/StarRocksContext.java | 31 + .../format/StarRocksExpressionVisitor.java | 65 + .../starrocks/format/StarRocksOutVisitor.java | 514 ++++ .../starrocks/format/StarRocksProperty.java | 105 + .../parser/StarRocksLanguageParser.java | 64 + .../starrocks/parser/tree/AggDesc.java | 42 + .../parser/tree/AggregateConstraint.java | 44 + .../parser/tree/DuplicateConstraint.java | 44 + .../tree/datatype/StarRocksDataTypeName.java | 233 ++ .../datatype/StarRocksGenericDataType.java | 60 + .../tree/partition/ArrayPartitionKey.java | 33 + .../tree/partition/LessThanPartitionKey.java | 49 + .../tree/partition/ListPartitionValue.java | 41 + .../tree/partition/ListPartitionedBy.java | 36 + .../partition/MultiItemListPartition.java | 59 + .../tree/partition/MultiRangePartition.java | 58 + .../parser/tree/partition/PartitionDesc.java | 20 + .../parser/tree/partition/PartitionKey.java | 20 + .../parser/tree/partition/PartitionValue.java | 46 + .../tree/partition/RangePartitionedBy.java | 36 + .../partition/SingleItemListPartition.java | 59 + .../tree/partition/SingleRangePartition.java | 53 + .../util/StarRocksReservedWordUtil.java | 38 + .../parser/visitor/StarRocksAstBuilder.java | 609 ++++ .../parser/visitor/StarRocksAstVisitor.java | 192 ++ .../starrocks/StarRocksTransformerTest.java | 198 ++ .../StarRocksClientConverterTest.java | 195 ++ .../StarRocksPropertyConverterTest.java | 25 + .../StarRocksGeneratorAlterTest.java | 256 ++ .../generator/StarRocksGeneratorTest.java | 694 +++++ .../format/StarRocksOutVisitorTest.java | 298 ++ .../parser/StarRocksLanguageParserTest.java | 157 + .../datatype/StarRocksDataTypeNameTest.java | 26 + .../util/StarRocksReservedWordUtilTest.java | 20 + .../zen/compare/ZenNodeCompareTest.java | 4 +- fastmodel-transform/pom.xml | 1 + pom.xml | 130 +- 120 files changed, 11309 insertions(+), 176 deletions(-) create mode 100644 fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/parser/HiveAstBuilderTest.java create mode 100644 fastmodel-transform/fastmodel-transform-sqlite/src/main/antlr4/com/aliyun/fastmodel/transform/sqlite/parser/SQLiteLexer.g4 create mode 100644 fastmodel-transform/fastmodel-transform-sqlite/src/main/antlr4/com/aliyun/fastmodel/transform/sqlite/parser/SQLiteParser.g4 create mode 100644 fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/format/SqliteExpressionVisitor.java create mode 100644 fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/parser/SqliteLanguageParser.java create mode 100644 fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/parser/visitor/SqliteAstBuilder.java create mode 100644 fastmodel-transform/fastmodel-transform-sqlite/src/test/java/com/aliyun/fastmodel/transform/sqlite/parser/SqliteLanguageParserTest.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/pom.xml create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/antlr4/com/aliyun/fastmodel/transform/starrocks/parser/StarRocks.g4 create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/antlr4/imports/StarRocksLex.g4 create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/StarRocksTransformer.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/builder/DefaultBuilder.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksClientConverter.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksPropertyConverter.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/column/AggrColumnProperty.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/DistributeBucketsNum.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/DistributeHash.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/MultiRangePartitionProperty.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/ReplicationNum.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/SingleRangePartitionProperty.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/TablePartitionRaw.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/ArrayClientPartitionKey.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/BaseClientPartitionKey.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/LessThanClientPartitionKey.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/MultiRangeClientPartition.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/PartitionClientValue.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/SingleRangeClientPartition.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/context/StarRocksContext.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksExpressionVisitor.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksOutVisitor.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksProperty.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/StarRocksLanguageParser.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/AggDesc.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/AggregateConstraint.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/DuplicateConstraint.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksDataTypeName.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksGenericDataType.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ArrayPartitionKey.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/LessThanPartitionKey.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ListPartitionValue.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ListPartitionedBy.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/MultiItemListPartition.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/MultiRangePartition.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionDesc.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionKey.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionValue.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/RangePartitionedBy.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/SingleItemListPartition.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/SingleRangePartition.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/util/StarRocksReservedWordUtil.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/visitor/StarRocksAstBuilder.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/visitor/StarRocksAstVisitor.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/StarRocksTransformerTest.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksClientConverterTest.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksPropertyConverterTest.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/generator/StarRocksGeneratorAlterTest.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/generator/StarRocksGeneratorTest.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksOutVisitorTest.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/StarRocksLanguageParserTest.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksDataTypeNameTest.java create mode 100644 fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/util/StarRocksReservedWordUtilTest.java diff --git a/.gitignore b/.gitignore index ea58a97..69ed872 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ deploy.sh /fastmodel-parser/src/test/resources/lsp/FastModelLexer.g4 /fastmodel-parser/gen/lsp/FastModelLexer.java /fastmodel-parser/src/test/resources/lsp/keyword.txt +/fastmodel-transform/fastmodel-transform-adbmysql/src/main/antlr4/com/aliyun/fastmodel/transform/adbmysql/parser/gen/AdbMysqlLexer.java +/fastmodel-transform/fastmodel-transform-mysql/src/main/antlr4/com/aliyun/fastmodel/transform/mysql/parser/gen/MySqlLexer.java diff --git a/CHANGELOG b/CHANGELOG index 735ddcb..e82b70d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -34,6 +34,26 @@ Core • 新增Show Statics语句 +Transformer +• 新增FML到Sqlite的转换 +• 新增FML到Adb Mysql的转换 + +向下兼容（compatibility） +• 暂无 + +过期功能（deprecations） +无 +安全漏洞修复（vulnerability） +无 + + +## 0.5.8 +新功能（new features） +Core +• 新增Create ODS TABLE +• 新增Show Statics语句 + + Transformer • 新增FML到Sqlite的转换 • 新增FML到Adb Mysql的转换 diff --git a/docs/zh-cn/layer/business_process.md b/docs/zh-cn/layer/business_process.md index 303fa03..7f1b41e 100644 --- a/docs/zh-cn/layer/business_process.md +++ b/docs/zh-cn/layer/business_process.md @@ -1,4 +1,7 @@ # 业务过程 +业务过程是企业在指定的数据域中所执行的业务活动，是数据建模所需要分析的逻辑主体。例如，交易域中可以有加入购物车、下单、支付等业务过程。本文为您介绍如何创建并使用业务过程。 +具体描述：[链接](https://help.aliyun.com/document_detail/276956.html?spm=a2c4g.397548.4.1.35e72121xqDgoR&scm=20140722.H_276956._.ID_276956-OR_rec-V_1) + 业务过程是企业在指定的数据域中所执行的业务活动，是数据建模所需要分析的逻辑主体。例如，交易域中可以有加入购物车、下单、支付等业务过程。本文为您介绍如何创建并使用业务过程。 具体描述：[链接](https://help.aliyun.com/document_detail/276956.html?spm=a2c4g.397548.4.1.35e72121xqDgoR&scm=20140722.H_276956._.ID_276956-OR_rec-V_1) \ No newline at end of file diff --git a/fastmodel-bom/pom.xml b/fastmodel-bom/pom.xml index 8bca3ac..8671048 100644 --- a/fastmodel-bom/pom.xml +++ b/fastmodel-bom/pom.xml @@ -47,129 +47,134 @@ fastmodel-compare ${project.version} + com.aliyun.fastmodel fastmodel-parser ${project.version} + com.aliyun.fastmodel - fastmodel-transform-api + fastmodel-driver-model ${project.version} + com.aliyun.fastmodel - fastmodel-transform-template + fastmodel-driver-client ${project.version} + com.aliyun.fastmodel - fastmodel-transform-hive + fastmodel-ide-spi ${project.version} + com.aliyun.fastmodel - fastmodel-transform-hologres + fastmodel-converter-spi ${project.version} + com.aliyun.fastmodel fastmodel-transform-fml ${project.version} + com.aliyun.fastmodel fastmodel-transform-plantuml ${project.version} + com.aliyun.fastmodel - fastmodel-driver-model + fastmodel-transform-zen ${project.version} com.aliyun.fastmodel - fastmodel-driver-client + fastmodel-transform-mysql ${project.version} com.aliyun.fastmodel - fastmodel-processor + fastmodel-transform-oracle ${project.version} com.aliyun.fastmodel - fastmodel-transform-excel + fastmodel-transform-graph ${project.version} com.aliyun.fastmodel - fastmodel-converter-spi + fastmodel-transform-clickhouse ${project.version} com.aliyun.fastmodel - fastmodel-converter-dqc + fastmodel-transform-adbmysql ${project.version} com.aliyun.fastmodel - fastmodel-transform-zen + fastmodel-transform-spark ${project.version} com.aliyun.fastmodel - fastmodel-transform-mysql + fastmodel-transform-sqlite ${project.version} com.aliyun.fastmodel - fastmodel-transform-oracle + fastmodel-transform-starrocks ${project.version} com.aliyun.fastmodel - fastmodel-transform-graph + fastmodel-transform-api ${project.version} - com.aliyun.fastmodel - fastmodel-transform-clickhouse + fastmodel-transform-template ${project.version} com.aliyun.fastmodel - fastmodel-transform-adbmysql + fastmodel-transform-hive ${project.version} com.aliyun.fastmodel - fastmodel-transform-spark + fastmodel-transform-hologres ${project.version} - com.aliyun.fastmodel - fastmodel-transform-sqlite + fastmodel-transform-fml ${project.version} - com.aliyun.fastmodel - fastmodel-ide-spi + fastmodel-transform-plantuml ${project.version} diff --git a/fastmodel-compare/src/test/java/com/aliyun/fastmodel/compare/impl/CreateTableCompareNodeTest.java b/fastmodel-compare/src/test/java/com/aliyun/fastmodel/compare/impl/CreateTableCompareNodeTest.java index 400b951..15b8a12 100644 --- a/fastmodel-compare/src/test/java/com/aliyun/fastmodel/compare/impl/CreateTableCompareNodeTest.java +++ b/fastmodel-compare/src/test/java/com/aliyun/fastmodel/compare/impl/CreateTableCompareNodeTest.java @@ -28,7 +28,10 @@ import com.aliyun.fastmodel.core.tree.QualifiedName; import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; import com.aliyun.fastmodel.core.tree.datatype.DataTypeEnums; +import com.aliyun.fastmodel.core.tree.datatype.DataTypeParameter; +import com.aliyun.fastmodel.core.tree.datatype.GenericDataType; import com.aliyun.fastmodel.core.tree.datatype.NumericParameter; +import com.aliyun.fastmodel.core.tree.datatype.TypeParameter; import com.aliyun.fastmodel.core.tree.expr.Identifier; import com.aliyun.fastmodel.core.tree.statement.constants.ColumnPropertyDefaultKey; import com.aliyun.fastmodel.core.tree.statement.constants.TableDetailType; @@ -43,6 +46,7 @@ import com.aliyun.fastmodel.core.tree.statement.table.constraint.PrimaryConstraint; import com.aliyun.fastmodel.core.tree.util.DataTypeUtil; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -513,6 +517,7 @@ public void testCommentIsNull() { assertEquals(1, baseStatements.size()); } + private CreateDimTable getCreateDimTable(Comment comment) { CreateDimTable before = CreateDimTable.builder().tableName(QualifiedName.of("dim_shop")).columns( ImmutableList.of(ColumnDefinition.builder().dataType(DataTypeUtil.simpleType(DataTypeEnums.BIGINT)) diff --git a/fastmodel-compare/src/test/java/com/aliyun/fastmodel/compare/table/ColumnCompareTest.java b/fastmodel-compare/src/test/java/com/aliyun/fastmodel/compare/table/ColumnCompareTest.java index 7b52218..55f0dfb 100644 --- a/fastmodel-compare/src/test/java/com/aliyun/fastmodel/compare/table/ColumnCompareTest.java +++ b/fastmodel-compare/src/test/java/com/aliyun/fastmodel/compare/table/ColumnCompareTest.java @@ -16,6 +16,7 @@ package com.aliyun.fastmodel.compare.table; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -23,7 +24,11 @@ import com.aliyun.fastmodel.core.tree.BaseStatement; import com.aliyun.fastmodel.core.tree.Property; import com.aliyun.fastmodel.core.tree.QualifiedName; +import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; import com.aliyun.fastmodel.core.tree.datatype.DataTypeEnums; +import com.aliyun.fastmodel.core.tree.datatype.DataTypeParameter; +import com.aliyun.fastmodel.core.tree.datatype.GenericDataType; +import com.aliyun.fastmodel.core.tree.datatype.TypeParameter; import com.aliyun.fastmodel.core.tree.expr.Identifier; import com.aliyun.fastmodel.core.tree.statement.constants.ColumnCategory; import com.aliyun.fastmodel.core.tree.statement.constants.ColumnPropertyDefaultKey; @@ -287,6 +292,35 @@ public void testComparePropertiesAfterNull() { assertEquals(join, "ALTER TABLE dim_shop CHANGE COLUMN c1 c1 BIGINT WITH ('code_table'='')"); } + @Test + public void testCompareWithMultiDataType() { + ArrayList anInt = Lists.newArrayList(new TypeParameter(DataTypeUtil.simpleType("int", null))); + BaseDataType build = new GenericDataType("array", anInt); + List leftList = ImmutableList.of( + ColumnDefinition.builder().colName(new Identifier("col1")) + .dataType(build) + .build() + ); + BaseDataType build2 = new GenericDataType("ARRAY", anInt); + List rightList = ImmutableList.of( + ColumnDefinition.builder().colName(new Identifier("col1")) + .dataType(build2) + .build() + ); + + CreateTable before = CreateTable.builder() + .columns(leftList) + .tableName(QualifiedName.of("dim_shop")).build(); + + CreateTable after = CreateTable.builder() + .columns(rightList) + .tableName(QualifiedName.of("dim_shop")).build(); + + List baseStatementList = columnCompare.compareTableElement(before, after); + String join = Joiner.on("\n").join(baseStatementList); + assertEquals("", join); + } + @Test public void testComparePropertiesBeforeNull() { ColumnDefinition columnDefinition = ColumnDefinition.builder() diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/formatter/ExpressionVisitor.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/formatter/ExpressionVisitor.java index ad14a1b..823d854 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/formatter/ExpressionVisitor.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/formatter/ExpressionVisitor.java @@ -351,8 +351,7 @@ public String visitLongLiteral(LongLiteral node, Void context) { @Override public String visitIntervalLiteral(IntervalLiteral node, Void context) { StringBuilder builder = new StringBuilder() - .append("INTERVAL ") - .append(" '").append(node.getValue()).append("' ") + .append("INTERVAL").append(" ").append(process(node.getValue())).append(" ") .append(node.getFromDateTime()); if (node.getToDateTime() != null) { diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/formatter/FastModelVisitor.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/formatter/FastModelVisitor.java index 96efb90..3bcf15b 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/formatter/FastModelVisitor.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/formatter/FastModelVisitor.java @@ -177,7 +177,7 @@ */ public class FastModelVisitor extends AstVisitor { - private static final String INDENT = " "; + protected static final String INDENT = " "; public static final String CONSTRAINT = "CONSTRAINT "; public static final String SUFFIX = ";"; public static final String NEW_LINE = "\n"; @@ -414,18 +414,20 @@ public Boolean visitRenameTable(RenameTable renameTable, Integer context) { @Override public Boolean visitCreateTable(CreateTable node, Integer indent) { - builder.append("CREATE "); + builder.append("CREATE"); if (node.getCreateOrReplace() != null && node.getCreateOrReplace()) { - builder.append("OR REPLACE "); - } - ITableDetailType tableDetailType = node.getTableDetailType() == null ? TableDetailType.NORMAL_DIM - : node.getTableDetailType(); - boolean isNormalOrTransaction = tableDetailType == TableDetailType.NORMAL_DIM - || tableDetailType == TableDetailType.TRANSACTION_FACT; - if (!isNormalOrTransaction && !tableDetailType.isSingle()) { - builder.append(tableDetailType.getCode()).append(" "); + builder.append(" OR REPLACE"); + } + ITableDetailType tableDetailType = null; + if (node.getTableDetailType() != null) { + tableDetailType = node.getTableDetailType(); + boolean isNormalOrTransaction = tableDetailType == TableDetailType.NORMAL_DIM + || tableDetailType == TableDetailType.TRANSACTION_FACT; + if (!isNormalOrTransaction && !tableDetailType.isSingle()) { + builder.append(" ").append(tableDetailType.getCode()); + } + builder.append(" ").append(tableDetailType.getParent()); } - builder.append(tableDetailType.getParent()); builder.append(" TABLE "); if (node.isNotExists()) { builder.append("IF NOT EXISTS "); @@ -606,7 +608,7 @@ public Boolean visitPrimaryConstraint(PrimaryConstraint primaryConstraint, Integ builder.append(indentString(ident)).append("PRIMARY KEY("); } builder.append( - primaryConstraint.getColNames().stream().map(ExpressionFormatter::formatExpression).collect(joining(","))); + primaryConstraint.getColNames().stream().map(this::formatExpression).collect(joining(","))); builder.append(")"); if (primaryConstraint.getEnable() != null) { if (!primaryConstraint.getEnable()) { @@ -1214,7 +1216,7 @@ public Boolean visitSetTableProperties(SetTableProperties setTableProperties, In @Override public Boolean visitDropCol(DropCol dropCol, Integer context) { builder.append("ALTER TABLE ").append(getCode(dropCol.getQualifiedName())); - builder.append(" DROP COLUMN ").append(ExpressionFormatter.formatExpression(dropCol.getColumnName())); + builder.append(" DROP COLUMN ").append(formatExpression(dropCol.getColumnName())); return true; } @@ -1947,7 +1949,7 @@ private String formatWith(List properties, boolean isNewLine) { return stringBuilder.toString(); } - public String indentString(int indent) { + protected String indentString(int indent) { return Strings.repeat(INDENT, indent); } @@ -1980,7 +1982,7 @@ protected String formatGroupBy(List groupingElements) { return Joiner.on(", ").join(resultStrings.build()); } - public String formatStringLiteral(String s) { + protected String formatStringLiteral(String s) { if (s == null) { return null; } diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/Comment.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/Comment.java index 6bca1d4..7be1a11 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/Comment.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/Comment.java @@ -19,8 +19,8 @@ import java.util.List; import com.google.common.collect.ImmutableList; -import lombok.EqualsAndHashCode; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; /** * 备注描述信息 @@ -29,7 +29,6 @@ * @date 2020/11/6 */ @Getter -@EqualsAndHashCode(callSuper = false) public class Comment extends AbstractNode { private final String comment; @@ -52,4 +51,12 @@ public List getChildren() { public R accept(IAstVisitor visitor, C context) { return visitor.visitComment(this, context); } + + @Override + public boolean equals(Object o) { + if (this == o) {return true;} + if (o == null || getClass() != o.getClass()) {return false;} + Comment c = (Comment)o; + return StringUtils.equals(comment, c.getComment()); + } } diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/ListNode.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/ListNode.java index 441ebee..affe088 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/ListNode.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/ListNode.java @@ -35,6 +35,11 @@ public class ListNode extends AbstractNode { public ListNode(List children) {this.children = children;} + public ListNode(NodeLocation location, List children) { + super(location); + this.children = children; + } + @Override public List getChildren() { return children; diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/Field.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/Field.java index 3bc8ad4..3a882c0 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/Field.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/Field.java @@ -17,6 +17,7 @@ package com.aliyun.fastmodel.core.tree.datatype; import java.util.List; +import java.util.Objects; import com.aliyun.fastmodel.core.tree.AbstractNode; import com.aliyun.fastmodel.core.tree.Comment; @@ -71,4 +72,18 @@ public List getChildren() { builder.add(dataType); return builder.build(); } + + @Override + public int hashCode() { + return Objects.hash(name, dataType, comment); + } + + @Override + public boolean equals(Object o) { + if (this == o) {return true;} + if (o == null || getClass() != o.getClass()) {return false;} + Field field = (Field)o; + return Objects.equals(name, field.name) && Objects.equals(dataType, field.dataType) + && Objects.equals(comment, field.comment); + } } diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/GenericDataType.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/GenericDataType.java index 88574bc..84e716e 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/GenericDataType.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/GenericDataType.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Locale; +import java.util.Objects; import com.aliyun.fastmodel.core.tree.IAstVisitor; import com.aliyun.fastmodel.core.tree.Node; @@ -25,8 +26,8 @@ import com.aliyun.fastmodel.core.tree.expr.Identifier; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import lombok.EqualsAndHashCode; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; /** * 通用数据类型 @@ -35,7 +36,6 @@ * @date 2020/10/30 */ @Getter -@EqualsAndHashCode(callSuper = false) public class GenericDataType extends BaseDataType { private final String name; @@ -79,6 +79,20 @@ public R accept(IAstVisitor visitor, C context) { return visitor.visitGenericDataType(this, context); } + @Override + public boolean equals(Object o) { + if (this == o) {return true;} + if (o == null || getClass() != o.getClass()) {return false;} + GenericDataType that = (GenericDataType)o; + boolean e = StringUtils.equalsIgnoreCase(this.getTypeName().getValue(), that.getTypeName().getValue()); + if (!e) { + return false; + } + List arguments = this.getArguments(); + List arguments1 = that.getArguments(); + return Objects.equals(arguments, arguments1); + } + @Override public IDataTypeName getTypeName() { DataTypeEnums dataType = DataTypeEnums.getDataType(name); diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/NumericParameter.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/NumericParameter.java index bb96ae0..de933e1 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/NumericParameter.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/NumericParameter.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import lombok.EqualsAndHashCode; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; /** * Desc: @@ -31,7 +32,6 @@ * @date 2020/10/30 */ @Getter -@EqualsAndHashCode(callSuper = false) public class NumericParameter extends DataTypeParameter { private final String value; @@ -49,4 +49,12 @@ public R accept(IAstVisitor visitor, C context) { public List getChildren() { return ImmutableList.of(); } + + @Override + public boolean equals(Object o) { + if (this == o) {return true;} + if (o == null || getClass() != o.getClass()) {return false;} + NumericParameter numericParameter = (NumericParameter)o; + return StringUtils.equalsIgnoreCase(value, numericParameter.getValue()); + } } diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/RowDataType.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/RowDataType.java index 93bd30d..4e0fe8e 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/RowDataType.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/RowDataType.java @@ -17,11 +17,13 @@ package com.aliyun.fastmodel.core.tree.datatype; import java.util.List; +import java.util.Objects; import com.aliyun.fastmodel.core.tree.IAstVisitor; import com.aliyun.fastmodel.core.tree.Node; import com.aliyun.fastmodel.core.tree.NodeLocation; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; /** * RowDataType : 一行的类型 @@ -54,6 +56,18 @@ public List getChildren() { return fields; } + @Override + public boolean equals(Object o) { + if (this == o) {return true;} + if (o == null || getClass() != o.getClass()) {return false;} + RowDataType rowDataType = (RowDataType)o; + boolean e = StringUtils.equalsIgnoreCase(this.getTypeName().getValue(), rowDataType.getTypeName().getValue()); + if (!e) { + return false; + } + return Objects.equals(fields, rowDataType.getFields()); + } + @Override public IDataTypeName getTypeName() { return DataTypeEnums.STRUCT; diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/TypeParameter.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/TypeParameter.java index 19afd81..2ec1685 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/TypeParameter.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/datatype/TypeParameter.java @@ -17,6 +17,7 @@ package com.aliyun.fastmodel.core.tree.datatype; import java.util.List; +import java.util.Objects; import com.aliyun.fastmodel.core.tree.IAstVisitor; import com.aliyun.fastmodel.core.tree.Node; @@ -47,4 +48,12 @@ public List getChildren() { public R accept(IAstVisitor visitor, C context) { return visitor.visitTypeParameter(this, context); } + + @Override + public boolean equals(Object o) { + if (this == o) {return true;} + if (o == null || getClass() != o.getClass()) {return false;} + TypeParameter typeParameter = (TypeParameter)o; + return Objects.equals(type, typeParameter.getType()); + } } diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/expr/BaseExpression.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/expr/BaseExpression.java index b11beb5..b41c7ee 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/expr/BaseExpression.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/expr/BaseExpression.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableList; import lombok.Getter; import lombok.Setter; +import org.apache.commons.lang3.StringUtils; /** * 表达式对象: @@ -66,6 +67,18 @@ public R accept(IAstVisitor visitor, C context) { return visitor.visitExpression(this, context); } + /** + * 优先从origin里获取，如果没有，那么读取toString内容 + * + * @return + */ + public String getOrigin() { + if (StringUtils.isNotBlank(origin)) { + return origin; + } + return toString(); + } + @Override public String toString() { return ExpressionFormatter.formatExpression(this); diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/constants/ConstraintScope.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/constants/ConstraintScope.java index 3dc54e6..8d7f2c0 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/constants/ConstraintScope.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/constants/ConstraintScope.java @@ -35,5 +35,5 @@ public enum ConstraintScope { /** * 未定义 */ - UNDEFIND; + UNDEFINED; } diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/constants/ConstraintType.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/constants/ConstraintType.java index 953a252..80a40e3 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/constants/ConstraintType.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/constants/ConstraintType.java @@ -42,41 +42,41 @@ public enum ConstraintType { /** * 不为空 */ - NOT_NULL(Constants.NOT_NULL1, ConstraintScope.COLUMN, "notNull constraint"), + NOT_NULL(Constants.NOT_NULL, ConstraintScope.COLUMN, "notNull constraint"), /** * 列组的约束 */ - COLUMN_GROUP(Constants.COLUMN_GROUP1, ConstraintScope.COLUMN, "column Group constraint"), + COLUMN_GROUP(Constants.COLUMN_GROUP, ConstraintScope.COLUMN, "column Group constraint"), /** * 时间周期类型 */ - TIME_PERIOD(Constants.TIME_PERIOD1, ConstraintScope.TABLE, "time period"), + TIME_PERIOD(Constants.TIME_PERIOD, ConstraintScope.TABLE, "time period"), /** * UNIQUE */ - UNIQUE(Constants.UNIQUE1, ConstraintScope.COLUMN, "唯一值约束"), + UNIQUE(Constants.UNIQUE, ConstraintScope.COLUMN, "唯一值约束"), /** * Redundant冗余约束 */ - REDUNDANT(Constants.REDUNDANT1, ConstraintScope.COLUMN, "冗余约束"), + REDUNDANT(Constants.REDUNDANT, ConstraintScope.COLUMN, "冗余约束"), /** * 默认值约束 */ - DEFAULT_VALUE(Constants.DEFAULT_VALUE1, ConstraintScope.COLUMN, "默认值"), + DEFAULT_VALUE(Constants.DEFAULT_VALUE, ConstraintScope.COLUMN, "默认值"), /** * 自定义约束信息 */ - CHECK(Constants.CHECK, ConstraintScope.UNDEFIND, "自定义约束"), + CHECK(Constants.CHECK, ConstraintScope.UNDEFINED, "自定义约束"), /** * index */ - INDEX(Constants.INDEX, ConstraintScope.UNDEFIND, "index"), + INDEX(Constants.INDEX, ConstraintScope.UNDEFINED, "索引"), ; @@ -100,12 +100,12 @@ public static class Constants { public static final String PRIMARY = "primary"; public static final String DIM = "dim"; public static final String LEVEL = "level"; - public static final String NOT_NULL1 = "notNull"; - public static final String COLUMN_GROUP1 = "column_group"; - public static final String TIME_PERIOD1 = "time_period"; - public static final String UNIQUE1 = "unique"; - public static final String REDUNDANT1 = "redundant"; - public static final String DEFAULT_VALUE1 = "default_value"; + public static final String NOT_NULL = "notNull"; + public static final String COLUMN_GROUP = "column_group"; + public static final String TIME_PERIOD = "time_period"; + public static final String UNIQUE = "unique"; + public static final String REDUNDANT = "redundant"; + public static final String DEFAULT_VALUE = "default_value"; public static final String CHECK = "check"; public static final String INDEX = "index"; } diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/table/CreateTable.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/table/CreateTable.java index f88c8ce..bdffc00 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/table/CreateTable.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/table/CreateTable.java @@ -16,10 +16,6 @@ package com.aliyun.fastmodel.core.tree.statement.table; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - import com.aliyun.fastmodel.core.semantic.table.CreateTableSemanticCheck; import com.aliyun.fastmodel.core.tree.AliasedName; import com.aliyun.fastmodel.core.tree.AstVisitor; @@ -30,7 +26,6 @@ import com.aliyun.fastmodel.core.tree.expr.Identifier; import com.aliyun.fastmodel.core.tree.statement.BaseCreate; import com.aliyun.fastmodel.core.tree.statement.constants.StatementType; -import com.aliyun.fastmodel.core.tree.statement.constants.TableDetailType; import com.aliyun.fastmodel.core.tree.statement.element.CreateElement; import com.aliyun.fastmodel.core.tree.statement.element.MultiComment; import com.aliyun.fastmodel.core.tree.statement.table.constraint.BaseConstraint; @@ -40,6 +35,10 @@ import lombok.EqualsAndHashCode; import lombok.Getter; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + /** * 创建表的语句 * diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/table/constraint/CustomConstraint.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/table/constraint/CustomConstraint.java index a4e5558..716a662 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/table/constraint/CustomConstraint.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/statement/table/constraint/CustomConstraint.java @@ -32,7 +32,7 @@ * @date 2021/8/29 */ @Getter -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = false) public class CustomConstraint extends BaseConstraint { private final String customType; diff --git a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/util/PropertyUtil.java b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/util/PropertyUtil.java index 1609237..ba78826 100644 --- a/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/util/PropertyUtil.java +++ b/fastmodel-core/src/main/java/com/aliyun/fastmodel/core/tree/util/PropertyUtil.java @@ -22,7 +22,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.Arrays; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -69,6 +68,21 @@ public static List toProperty(Map map) { return list; } + /** + * get properties + * + * @param properties + * @param propertyKey + * @return + */ + public static String getPropertyValue(List properties, String propertyKey) { + if (properties == null || properties.isEmpty()) { + return null; + } + Map stringStringMap = toMap(properties); + return stringStringMap.get(propertyKey); + } + /** * 转换为对象处理 * diff --git a/fastmodel-core/src/test/java/com/aliyun/fastmodel/core/formatter/FastModelVisitorTest.java b/fastmodel-core/src/test/java/com/aliyun/fastmodel/core/formatter/FastModelVisitorTest.java index 5cb8078..4afe72c 100644 --- a/fastmodel-core/src/test/java/com/aliyun/fastmodel/core/formatter/FastModelVisitorTest.java +++ b/fastmodel-core/src/test/java/com/aliyun/fastmodel/core/formatter/FastModelVisitorTest.java @@ -585,7 +585,7 @@ public void testColumnWithEmpty() { .build(); fastModelVisitor.visitCreateTable(node, 0); String s = fastModelVisitor.getBuilder().toString(); - assertEquals(s, "CREATE DIM TABLE dim_shop\n" + assertEquals(s, "CREATE TABLE dim_shop\n" + "/*(\n" + " c BIGINT\n" + ")*/"); diff --git a/fastmodel-dependencies-bom/pom.xml b/fastmodel-dependencies-bom/pom.xml index 4a61f12..1951cf9 100644 --- a/fastmodel-dependencies-bom/pom.xml +++ b/fastmodel-dependencies-bom/pom.xml @@ -26,10 +26,10 @@ ${revision} - 0.5.8-jdk8 + 0.5.9 1.1.0 3.17.1 - 4.9.3 + 4.13.1 @@ -40,20 +40,50 @@ + + + jdk11 + + 0.5.9 + + + true + + + + + org.antlr + antlr4-runtime + 4.11.1 + + + + + + jdk8 + + 0.5.9-jdk8 + + + + + org.antlr + antlr4-runtime + 4.9.3 + + + + + org.projectlombok lombok - 1.18.8 + 1.18.20 - - org.antlr - antlr4-runtime - ${dep.antlr.version} - org.apache.commons @@ -254,6 +284,12 @@ provided + + org.antlr + antlr4-runtime + 4.13.1 + + @@ -286,4 +322,5 @@ + \ No newline at end of file diff --git a/fastmodel-jacoco/pom.xml b/fastmodel-jacoco/pom.xml index 639905d..df4e6b7 100644 --- a/fastmodel-jacoco/pom.xml +++ b/fastmodel-jacoco/pom.xml @@ -175,6 +175,12 @@ fastmodel-transform-adbmysql ${project.parent.version} + + + com.aliyun.fastmodel + fastmodel-transform-sqlite + ${project.parent.version} + diff --git a/fastmodel-parser/src/main/antlr4/imports/TableParser.g4 b/fastmodel-parser/src/main/antlr4/imports/TableParser.g4 index 4ba2986..b6f35ab 100644 --- a/fastmodel-parser/src/main/antlr4/imports/TableParser.g4 +++ b/fastmodel-parser/src/main/antlr4/imports/TableParser.g4 @@ -23,7 +23,7 @@ tableStatements: ; createTableStatement - : KW_CREATE (replace)? tableType KW_TABLE + : KW_CREATE (replace)? tableType? KW_TABLE ifNotExists? tableName alias? diff --git a/fastmodel-parser/src/main/java/com/aliyun/fastmodel/parser/visitor/TableVisitor.java b/fastmodel-parser/src/main/java/com/aliyun/fastmodel/parser/visitor/TableVisitor.java index da4ae18..413b8a8 100644 --- a/fastmodel-parser/src/main/java/com/aliyun/fastmodel/parser/visitor/TableVisitor.java +++ b/fastmodel-parser/src/main/java/com/aliyun/fastmodel/parser/visitor/TableVisitor.java @@ -129,12 +129,12 @@ public Node visitCreateTableStatement(CreateTableStatementContext ctx) { tableParamObject.builderColumns(ctx, this); QualifiedName tableName = getQualifiedName(ctx.tableName()); List properties = getProperties(ctx.setProperties()); - TableDetailType tableDetailType = getDetailType(ctx.tableType()); Comment comment = visitIfPresent(ctx.comment(), Comment.class).orElse(null); AliasedName aliasedName = visitIfPresent(ctx.alias(), AliasedName.class).orElse(null); boolean notExists = ctx.ifNotExists() != null; - TableType parent = tableDetailType.getParent(); + TableDetailType tableDetailType = getDetailType(ctx.tableType()); + TableType parent = tableDetailType != null ? tableDetailType.getParent() : null; boolean replace = ctx.replace() != null; //if like define use @@ -170,6 +170,9 @@ public Node visitCreateTableStatement(CreateTableStatementContext ctx) { } private TableBuilder getTableBuilder(TableType parent) { + if (parent == null) { + return CreateTable.builder(); + } if (parent == TableType.DIM) { return CreateDimTable.builder(); } else if (parent == TableType.ADS) { @@ -262,7 +265,7 @@ public Node visitIndexColumnName(IndexColumnNameContext ctx) { @Override public Node visitUniqueConstraint(UniqueConstraintContext ctx) { - Identifier constraintName = null; + Identifier constraintName; if (ctx.identifier() != null) { constraintName = (Identifier)visit(ctx.identifier()); } else { @@ -408,6 +411,9 @@ public Node visitDropIndex(DropIndexContext ctx) { } private TableDetailType getDetailType(TableTypeContext typeContext) { + if (typeContext == null) { + return null; + } Token type = typeContext.type; switch (type.getType()) { case FastModelLexer.KW_FACT: diff --git a/fastmodel-parser/src/test/java/com/aliyun/fastmodel/parser/NodeParserTest.java b/fastmodel-parser/src/test/java/com/aliyun/fastmodel/parser/NodeParserTest.java index 01fa952..81262aa 100644 --- a/fastmodel-parser/src/test/java/com/aliyun/fastmodel/parser/NodeParserTest.java +++ b/fastmodel-parser/src/test/java/com/aliyun/fastmodel/parser/NodeParserTest.java @@ -44,7 +44,7 @@ public class NodeParserTest { NodeParser nodeParser = new NodeParser(); - @Test(expected = ParseException.class) + @Test public void testMultiParse() { String sql = "create table a (a bigint) comment 'comment'"; List statements = nodeParser.multiParse(new DomainLanguage(sql)); @@ -108,7 +108,7 @@ public void testParseWithSingleQuotation() { .build(); String c = createTable.toString(); BaseStatement baseStatement = nodeParser.parseStatement(c); - assertEquals("CREATE DIM TABLE abc COMMENT 'abc ''type'''", baseStatement.toString()); + assertEquals("CREATE TABLE abc COMMENT 'abc ''type'''", baseStatement.toString()); } @Test @@ -122,4 +122,11 @@ public void testParseWithJson() { assertEquals("JSON", columnDefinition1.getDataType().toString()); } + + @Test + public void testParseOnlyTable() { + String fml = "create table a (a bigint) comment 'abc';"; + CreateTable createTable = nodeParser.parseStatement(fml); + assertEquals("a", createTable.getQualifiedName().toString()); + } } \ No newline at end of file diff --git a/fastmodel-parser/src/test/resources/lsp/FastModelGrammarParser.g4 b/fastmodel-parser/src/test/resources/lsp/FastModelGrammarParser.g4 index 488c62d..200ec0d 100644 --- a/fastmodel-parser/src/test/resources/lsp/FastModelGrammarParser.g4 +++ b/fastmodel-parser/src/test/resources/lsp/FastModelGrammarParser.g4 @@ -729,7 +729,7 @@ tableStatements: ; createTableStatement - : KW_CREATE (replace)? tableType KW_TABLE + : KW_CREATE (replace)? tableType? KW_TABLE ifNotExists? tableName alias? diff --git a/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/converter/BaseClientConverter.java b/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/converter/BaseClientConverter.java index e11aa85..89b0ace 100644 --- a/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/converter/BaseClientConverter.java +++ b/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/converter/BaseClientConverter.java @@ -14,7 +14,9 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.OptionalInt; import java.util.stream.Collectors; +import java.util.stream.IntStream; import com.aliyun.fastmodel.core.tree.Comment; import com.aliyun.fastmodel.core.tree.Node; @@ -23,6 +25,7 @@ import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; import com.aliyun.fastmodel.core.tree.datatype.DataTypeParameter; import com.aliyun.fastmodel.core.tree.datatype.GenericDataType; +import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName; import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName.Dimension; import com.aliyun.fastmodel.core.tree.datatype.NumericParameter; import com.aliyun.fastmodel.core.tree.expr.Identifier; @@ -47,7 +50,6 @@ import com.aliyun.fastmodel.transform.api.context.TransformContext; import com.aliyun.fastmodel.transform.api.util.StringJoinUtil; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; @@ -79,9 +81,9 @@ public Node covertToNode(Table table, TableConfig tableConfig) { comment = new Comment(table.getComment()); } List constraints = toConstraint(table.getColumns(), table.getConstraints()); - PartitionedBy partitionedBy = toPartitionedBy(table.getColumns()); + PartitionedBy partitionedBy = toPartitionedBy(table, table.getColumns()); List properties = toProperty(table, table.getProperties()); - List columnDefines = toColumnDefinition(table.getColumns()); + List columnDefines = toColumnDefinition(table, table.getColumns()); return CreateTable.builder() .ifNotExist(table.isIfNotExist()) .tableName(of) @@ -112,6 +114,7 @@ public Table convertToTable(Node table, T context) { List properties = toBaseClientProperty(createTable); return Table.builder() .ifNotExist(createTable.isNotExists()) + .external(isExternal(createTable)) .database(database) .schema(schema).name(suffix) .comment(createTable.getCommentValue()) @@ -140,7 +143,7 @@ protected String toSchema(CreateTable createTable, T transformContext) { return transformContext.getSchema(); } - private String toDatabase(CreateTable createTable, String database) { + protected String toDatabase(CreateTable createTable, String database) { QualifiedName qualifiedName = createTable.getQualifiedName(); boolean isThirdSchema = qualifiedName.isJoinPath() && qualifiedName.getOriginalParts().size() == THIRD_INDEX; if (isThirdSchema) { @@ -184,6 +187,16 @@ protected Long toLifeCycleSeconds(CreateTable createTable) { return 0L; } + /** + * is external + * + * @param createTable + * @return + */ + protected Boolean isExternal(CreateTable createTable) { + return false; + } + /** * toTableColumns * @@ -209,8 +222,10 @@ public List toTableColumns(CreateTable createTable) { //原有的列中是否含有分区列，如果含有，那么设置下分区信息 for (Column c : list) { if (contains(partitionColumns, c)) { + Integer partitionKeyIndex = getPartitionKeyIndex(partitionColumns, c); c.setPartitionKey(true); - c.setPartitionKeyIndex(index++); + c.setPartitionKeyIndex(partitionKeyIndex); + index++; } } //如果list不包含分区列，那么将分区列加入到原有的列中 @@ -225,6 +240,17 @@ public List toTableColumns(CreateTable createTable) { return list; } + private Integer getPartitionKeyIndex(List partitionColumns, Column c) { + int i = -1; + OptionalInt first = IntStream.range(0, partitionColumns.size()).filter( + index -> { + ColumnDefinition columnDefinition = partitionColumns.get(index); + return (Objects.equals(new Identifier(c.getName()), columnDefinition.getColName())); + } + ).findFirst(); + return first.isPresent() ? first.getAsInt() : -1; + } + /** * to outline constraint * @@ -264,28 +290,36 @@ protected List toOutlineConstraint(CreateTable createTable) { /** * to column definition * + * @param table * @param columns * @return */ - protected List toColumnDefinition(List columns) { + protected List toColumnDefinition(Table table, List columns) { if (columns == null) { return Lists.newArrayList(); } - return columns.stream().map(this::toColumnDefinition).collect(Collectors.toList()); + return columns.stream().map(c -> toColumnDefinition(table, c)).collect(Collectors.toList()); } - protected ColumnDefinition toColumnDefinition(Column c) { + protected ColumnDefinition toColumnDefinition(Table table, Column c) { String id = c.getId(); + List all = Lists.newArrayList(); Property property = null; if (StringUtils.isNotBlank(id)) { property = new Property(ColumnPropertyDefaultKey.uuid.name(), id); + all.add(property); + } + List properties = c.getProperties(); + if (properties != null) { + List columnProperty = toProperty(table, properties); + all.addAll(columnProperty); } return ColumnDefinition.builder() .colName(new Identifier(c.getName())) .comment(new Comment(c.getComment())).dataType(getDataType(c)) .notNull(BooleanUtils.isFalse(c.isNullable())) .primary(c.isPrimaryKey()) - .properties(property != null ? Lists.newArrayList(property) : ImmutableList.of()).build(); + .properties(all).build(); } /** @@ -305,7 +339,7 @@ protected ColumnDefinition toColumnDefinition(Column c) { */ protected List toProperty(Table table, List properties) { if (properties == null) { - return Collections.emptyList(); + return new ArrayList<>(); } return properties.stream().map(p -> new Property(p.getKey(), p.valueString())).collect(Collectors.toList()); } @@ -313,10 +347,11 @@ protected List toProperty(Table table, List proper /** * to partition by * + * @param table * @param columns * @return */ - protected PartitionedBy toPartitionedBy(List columns) { + protected PartitionedBy toPartitionedBy(Table table, List columns) { if (CollectionUtils.isEmpty(columns)) { return null; } @@ -395,10 +430,16 @@ protected Column getColumn(ColumnDefinition c, boolean partitionKey, Integer par .primaryKey(BooleanUtils.isTrue(c.getPrimary())) .partitionKey(partitionKey) .partitionKeyIndex(partitionKeyIndex).build(); + IDataTypeName typeName = dataType.getTypeName(); + Dimension dimension = typeName.getDimension(); + if (dimension == Dimension.MULTIPLE) { + //如果是多纬度的类型，直接设置类型文本 + column.setDataType(dataType.getOrigin()); + return column; + } if (!(dataType instanceof GenericDataType)) {return column;} GenericDataType genericDataType = (GenericDataType)dataType; List arguments = genericDataType.getArguments(); - Dimension dimension = genericDataType.getTypeName().getDimension(); //if only one if (dimension == Dimension.TWO) { //because is decimal, so must type parameter is numeric diff --git a/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/dto/table/Column.java b/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/dto/table/Column.java index eec6d60..0a8d8a5 100644 --- a/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/dto/table/Column.java +++ b/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/client/dto/table/Column.java @@ -8,6 +8,9 @@ package com.aliyun.fastmodel.transform.api.client.dto.table; +import java.util.List; + +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -70,4 +73,9 @@ public class Column { * partition key index, zero-base */ private Integer partitionKeyIndex; + + /** + * column properties + */ + private List properties; } diff --git a/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/context/TransformContext.java b/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/context/TransformContext.java index 165c718..11fd26b 100644 --- a/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/context/TransformContext.java +++ b/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/context/TransformContext.java @@ -52,6 +52,11 @@ public class TransformContext { */ private String schema; + /** + * pretty Format + */ + private boolean prettyFormat; + /** * dataTypeTransformer, 类型转换器处理 */ @@ -82,6 +87,7 @@ public TransformContext(TransformContext context) { this.database = context.getDatabase(); this.schema = context.getSchema(); this.querySetting = context.getQuerySetting(); + this.prettyFormat = context.isPrettyFormat(); } /** @@ -98,6 +104,7 @@ protected > TransformContext(Builder tBuilder) { this.querySetting = tBuilder.querySetting; this.database = tBuilder.database; this.schema = tBuilder.schema; + this.prettyFormat = tBuilder.prettyFormat; } /** @@ -138,6 +145,11 @@ public static class Builder> { */ private QuerySetting querySetting = new QuerySetting(); + /** + * 是否格式化输出 + */ + private boolean prettyFormat = true; + public T dataTypeTransformer(DataTypeConverter dataTypeTransformer) { this.dataTypeTransformer = dataTypeTransformer; return (T)this; @@ -168,6 +180,11 @@ public T schema(String schema) { return (T)this; } + public T prettyFormat(boolean prettyFormat) { + this.prettyFormat = prettyFormat; + return (T)this; + } + public TransformContext build() { return new TransformContext(this); } diff --git a/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/dialect/DialectName.java b/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/dialect/DialectName.java index 590cc36..6d0bd2b 100644 --- a/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/dialect/DialectName.java +++ b/fastmodel-transform/fastmodel-transform-api/src/main/java/com/aliyun/fastmodel/transform/api/dialect/DialectName.java @@ -112,7 +112,14 @@ public enum DialectName implements IDialectName { /** * Sqlite */ - SQLITE(Constants.SQLITE); + SQLITE(Constants.SQLITE), + + /** + * StarRocks + */ + STARROCKS(Constants.STARROCKS) + + ; @Getter private final String value; @@ -139,6 +146,7 @@ public static class Constants { public static final String POSTGRESQL = "POSTGRESQL"; public static final String SQLITE = "SQLITE"; public static final String JSON = "JSON"; + public static final String STARROCKS = "STAR_ROCKS"; } public static DialectName getByCode(String name) { diff --git a/fastmodel-transform/fastmodel-transform-api/src/test/java/com/aliyun/fastmodel/transform/api/builder/model/TemplateDefineTest.java b/fastmodel-transform/fastmodel-transform-api/src/test/java/com/aliyun/fastmodel/transform/api/builder/model/TemplateDefineTest.java index e371fd9..1747a3a 100644 --- a/fastmodel-transform/fastmodel-transform-api/src/test/java/com/aliyun/fastmodel/transform/api/builder/model/TemplateDefineTest.java +++ b/fastmodel-transform/fastmodel-transform-api/src/test/java/com/aliyun/fastmodel/transform/api/builder/model/TemplateDefineTest.java @@ -48,12 +48,12 @@ public void getTemplateIdByFunction() { TemplateDefine templateIdByFunction = TemplateDefine.getTemplateIdByFunction( new ColumnFunction(BaseFunctionName.UNIQUE_COUNT, new TableOrColumn( QualifiedName.of("a.b")), baseDataType), CheckerType.FIX_STRATEGY_CHECK); - assertEquals(templateIdByFunction.getTemplateId(), new Integer(5)); + assertEquals(templateIdByFunction.getTemplateId(), Integer.valueOf(5)); templateIdByFunction = TemplateDefine.getTemplateIdByFunction( new ColumnFunction(BaseFunctionName.UNIQUE_COUNT, new TableOrColumn( QualifiedName.of("a.b")), baseDataType), CheckerType.DYNAMIC_STRATEGY_CHECK); - assertEquals(templateIdByFunction.getTemplateId(), new Integer(306)); + assertEquals(templateIdByFunction.getTemplateId(), Integer.valueOf(306)); } @Test @@ -62,7 +62,7 @@ public void getTemplateIdByFunction2() { new ColumnFunction(BaseFunctionName.NULL_COUNT, new TableOrColumn( QualifiedName.of("a.b")), DataTypeUtil.simpleType(DataTypeEnums.BIGINT)), CheckerType.FIX_STRATEGY_CHECK); - assertEquals(templateIdByFunction.getTemplateId(), new Integer(11)); + assertEquals(templateIdByFunction.getTemplateId(), Integer.valueOf(11)); } @Test @@ -71,6 +71,6 @@ public void testGetTemplateIdByVol() { new VolFunction(new TableFunction(BaseFunctionName.TABLE_SIZE, ImmutableList.of()), ImmutableList.of(new LongLiteral("1"), new LongLiteral("7"), new LongLiteral("30"))), CheckerType.VOL_STRATEGY_CHECK); - assertEquals(new Integer(33), templateIdByFunction.getTemplateId()); + assertEquals(Integer.valueOf(33), templateIdByFunction.getTemplateId()); } } \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-example/pom.xml b/fastmodel-transform/fastmodel-transform-example/pom.xml index f296e9b..47891db 100644 --- a/fastmodel-transform/fastmodel-transform-example/pom.xml +++ b/fastmodel-transform/fastmodel-transform-example/pom.xml @@ -121,6 +121,12 @@ ${project.parent.version} + + com.aliyun.fastmodel + fastmodel-transform-sqlite + ${project.parent.version} + + com.alibaba fastjson diff --git a/fastmodel-transform/fastmodel-transform-fml/src/test/java/com/aliyun/fastmodel/transform/fml/format/FmlFormatterTest.java b/fastmodel-transform/fastmodel-transform-fml/src/test/java/com/aliyun/fastmodel/transform/fml/format/FmlFormatterTest.java index ef74a23..3015454 100644 --- a/fastmodel-transform/fastmodel-transform-fml/src/test/java/com/aliyun/fastmodel/transform/fml/format/FmlFormatterTest.java +++ b/fastmodel-transform/fastmodel-transform-fml/src/test/java/com/aliyun/fastmodel/transform/fml/format/FmlFormatterTest.java @@ -28,7 +28,6 @@ import com.aliyun.fastmodel.core.tree.statement.CompositeStatement; import com.aliyun.fastmodel.core.tree.statement.element.CreateElement; import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; -import com.aliyun.fastmodel.core.tree.statement.table.CreateDimTable; import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; import com.aliyun.fastmodel.core.tree.statement.table.SetTableComment; import com.aliyun.fastmodel.core.tree.statement.table.constraint.BaseConstraint; @@ -119,7 +118,7 @@ public void testRedunct() { .constraints(constraints) .build(); DialectNode dialectNode = FmlFormatter.formatNode(createTable,FmlTransformContext.builder().appendSemicolon(true).build()); - assertEquals(dialectNode.getNode(), "CREATE DIM TABLE bc \n" + assertEquals(dialectNode.getNode(), "CREATE TABLE bc \n" + "(\n" + " c1 BIGINT,\n" + " CONSTRAINT c2 REDUNDANT c1 REFERENCES p.t.c2\n" diff --git a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/HiveTransformer.java b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/HiveTransformer.java index 1cfd78c..b811c97 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/HiveTransformer.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/HiveTransformer.java @@ -21,6 +21,7 @@ import com.aliyun.fastmodel.transform.api.Transformer; import com.aliyun.fastmodel.transform.api.builder.BuilderFactory; import com.aliyun.fastmodel.transform.api.builder.StatementBuilder; +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; import com.aliyun.fastmodel.transform.api.client.dto.table.Table; import com.aliyun.fastmodel.transform.api.client.dto.table.TableConfig; import com.aliyun.fastmodel.transform.api.context.ReverseContext; @@ -77,4 +78,9 @@ public Table transformTable(Node table, TransformContext context) { return hiveClientConverter.convertToTable(table, new HiveTransformContext(context)); } + @Override + public BaseClientProperty create(String name, String value) { + return hiveClientConverter.getPropertyConverter().create(name, value); + } + } diff --git a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/client/converter/HiveClientConverter.java b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/client/converter/HiveClientConverter.java index c3a374d..d264604 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/client/converter/HiveClientConverter.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/client/converter/HiveClientConverter.java @@ -16,6 +16,7 @@ import com.aliyun.fastmodel.core.exception.ParseException; import com.aliyun.fastmodel.core.tree.Node; import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.QualifiedName; import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName; import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName.Dimension; @@ -44,11 +45,11 @@ * @date 2022/8/5 */ public class HiveClientConverter extends BaseClientConverter { - private final HivePropertyConverter maxComputePropertyConverter; + private final HivePropertyConverter hivePropertyConverter; private final HiveLanguageParser hiveLanguageParser; public HiveClientConverter() { - maxComputePropertyConverter = new HivePropertyConverter(); + hivePropertyConverter = new HivePropertyConverter(); hiveLanguageParser = new HiveLanguageParser(); } @@ -70,13 +71,17 @@ public Table convertToTable(Node table, HiveTransformContext context) { @Override public PropertyConverter getPropertyConverter() { - return maxComputePropertyConverter; + return hivePropertyConverter; } @Override - protected List toColumnDefinition(List columns) { + protected List toColumnDefinition(Table table, List columns) { //mc是columns分开的处理 - return columns.stream().filter(column -> !column.isPartitionKey()).map(this::toColumnDefinition).collect(Collectors.toList()); + return columns.stream().filter(column -> !column.isPartitionKey()).map( + c -> { + return toColumnDefinition(table, c); + } + ).collect(Collectors.toList()); } @Override @@ -126,4 +131,19 @@ protected BaseDataType getDataType(Column column) { throw new IllegalArgumentException("not support dataTypeName with:" + dataTypeName); } } + + @Override + protected String toSchema(CreateTable createTable, HiveTransformContext transformContext) { + return null; + } + + @Override + protected String toDatabase(CreateTable createTable, String database) { + QualifiedName qualifiedName = createTable.getQualifiedName(); + boolean isSecondSchema = qualifiedName.isJoinPath() && qualifiedName.getOriginalParts().size() == SECOND_INDEX; + if (isSecondSchema) { + return qualifiedName.getFirst(); + } + return database; + } } diff --git a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/client/converter/HivePropertyConverter.java b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/client/converter/HivePropertyConverter.java index 1aeb707..175142d 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/client/converter/HivePropertyConverter.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/client/converter/HivePropertyConverter.java @@ -56,8 +56,4 @@ protected Map> getFunctionMap() { return map; } - @Override - protected boolean returnDefaultWhenNotExist() { - return false; - } } diff --git a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/context/HiveTransformContext.java b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/context/HiveTransformContext.java index e8dd6de..2939b66 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/context/HiveTransformContext.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/context/HiveTransformContext.java @@ -38,10 +38,10 @@ public class HiveTransformContext extends TransformContext { public HiveTransformContext(TransformContext parent) { super(parent); + this.printProperty = true; if (parent instanceof HiveTransformContext) { HiveTransformContext transformContext = (HiveTransformContext)parent; enableConstraint = transformContext.isEnableConstraint(); - this.printProperty = true; } } diff --git a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HiveHelper.java b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HiveHelper.java index 457c1b1..f6d6326 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HiveHelper.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HiveHelper.java @@ -2,14 +2,18 @@ import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import com.aliyun.fastmodel.common.utils.StripUtils; import com.aliyun.fastmodel.core.tree.Property; import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; import com.aliyun.fastmodel.transform.api.util.PropertyKeyUtil; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; +import static java.util.stream.Collectors.joining; + /** * HiveHelper * @@ -32,30 +36,49 @@ public static boolean isExternal(CreateTable node) { return StringUtils.equalsIgnoreCase(property.getValue(), BooleanUtils.toStringTrueFalse(true)); } - public static String appendRowFormat(CreateTable node) { + public static String appendRowFormat(CreateTable node, String elementIndent) { StringBuilder builder = new StringBuilder(); List properties = node.getProperties(); if (node.isPropertyEmpty()) { return StringUtils.EMPTY; } + Optional serde = properties.stream().filter( + p -> StringUtils.equalsIgnoreCase(p.getName(), HivePropertyKey.ROW_FORMAT_SERDE.getValue())).findFirst(); + if (serde.isPresent()) { + builder.append("ROW FORMAT SERDE"); + String value = serde.get().getValue(); + builder.append(StringUtils.LF).append(String.format("'%s'", value)); + } + Optional first = properties.stream().filter( p -> StringUtils.equalsIgnoreCase(p.getName(), HivePropertyKey.FIELDS_TERMINATED.getValue())).findFirst(); - Optional second = properties.stream().filter( p -> StringUtils.equalsIgnoreCase(p.getName(), HivePropertyKey.LINES_TERMINATED.getValue())).findFirst(); if (first.isPresent() || second.isPresent()) { builder.append("ROW FORMAT DELIMITED"); - } else { - return builder.toString(); - } - if (first.isPresent()) { - String value = first.get().getValue(); - builder.append(StringUtils.LF).append("FIELDS TERMINATED BY ").append(String.format("'%s'", value)); + if (first.isPresent()) { + String value = first.get().getValue(); + builder.append(StringUtils.LF).append("FIELDS TERMINATED BY ").append(String.format("'%s'", value)); + } + if (second.isPresent()) { + String value = second.get().getValue(); + builder.append(StringUtils.LF).append("LINES TERMINATED BY ").append(String.format("'%s'", value)); + } } - if (second.isPresent()) { - String value = second.get().getValue(); - builder.append(StringUtils.LF).append("LINES TERMINATED BY ").append(String.format("'%s'", value)); + + // with serdeproperties + String prefix = HivePropertyKey.SERDE_PROPS.getValue() + "."; + List serdeProps = properties.stream() + .filter(property -> property.getName().toLowerCase().startsWith(prefix)) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(serdeProps)) { + builder.append("\nWITH SERDEPROPERTIES (\n"); + String props = serdeProps.stream().map(property -> elementIndent + + StripUtils.addStrip(property.getName().substring(prefix.length())) + + " = " + + StripUtils.addStrip(property.getValue())).collect(joining(",\n")); + builder.append(props).append("\n)\n"); } return builder.toString(); @@ -69,11 +92,25 @@ public static String appendStoredFormat(CreateTable node) { } Optional first = properties.stream().filter( p -> StringUtils.equalsIgnoreCase(p.getName(), HivePropertyKey.STORAGE_FORMAT.getValue())).findFirst(); - if (!first.isPresent()) { - return stringBuilder.toString(); + Optional input = properties.stream().filter( + p -> StringUtils.equalsIgnoreCase(p.getName(), HivePropertyKey.STORED_INPUT_FORMAT.getValue())).findFirst(); + Optional output = properties.stream().filter( + p -> StringUtils.equalsIgnoreCase(p.getName(), HivePropertyKey.STORED_OUTPUT_FORMAT.getValue())).findFirst(); + if (first.isPresent()) { + Property property = first.get(); + stringBuilder.append("STORED AS ").append(property.getValue()); + } else if (input.isPresent() || output.isPresent()) { + stringBuilder.append("STORED AS "); + if (input.isPresent()) { + String value = input.get().getValue(); + stringBuilder.append(StringUtils.LF).append("INPUTFORMAT ").append(String.format("'%s'", value)); + } + if (output.isPresent()) { + String value = output.get().getValue(); + stringBuilder.append(StringUtils.LF).append("OUTPUTFORMAT ").append(String.format("'%s'", value)); + } } - Property property = first.get(); - stringBuilder.append("STORED AS ").append(property.getValue()); + return stringBuilder.toString(); } diff --git a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HivePropertyKey.java b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HivePropertyKey.java index 24a8cdd..293364c 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HivePropertyKey.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HivePropertyKey.java @@ -35,14 +35,33 @@ public enum HivePropertyKey implements PropertyKey { /** * location */ - LOCATION("hive.location") + LOCATION("hive.location"), + + /** + * row format serde + */ + ROW_FORMAT_SERDE("hive.row_format_serde"), + + /** + * STORED AS INPUTFORMAT + */ + STORED_INPUT_FORMAT("hive.stored_input_format"), + + /** + * STORED AS OUTPUTFORMAT + */ + STORED_OUTPUT_FORMAT("hive.stored_output_format"), + + /** + * WITH SERDEPROPERTIES + */ + SERDE_PROPS("hive.serde_props") ; /** * 是否需要print */ - @Getter private final String value; /** diff --git a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HiveVisitor.java b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HiveVisitor.java index 325e384..ea9616c 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HiveVisitor.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/format/HiveVisitor.java @@ -136,7 +136,7 @@ public Boolean visitCreateTable(CreateTable node, Integer indent) { } //append row format - String rowFormat = HiveHelper.appendRowFormat(node); + String rowFormat = HiveHelper.appendRowFormat(node, elementIndent); if (StringUtils.isNotBlank(rowFormat)) { appendLineIfNecessary(); builder.append(rowFormat); diff --git a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/parser/HiveAstBuilder.java b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/parser/HiveAstBuilder.java index 7873658..1fd97b5 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/parser/HiveAstBuilder.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/main/java/com/aliyun/fastmodel/transform/hive/parser/HiveAstBuilder.java @@ -16,6 +16,8 @@ package com.aliyun.fastmodel.transform.hive.parser; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -34,6 +36,7 @@ import com.aliyun.fastmodel.core.tree.datatype.RowDataType; import com.aliyun.fastmodel.core.tree.datatype.TypeParameter; import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.expr.literal.BooleanLiteral; import com.aliyun.fastmodel.core.tree.expr.literal.StringLiteral; import com.aliyun.fastmodel.core.tree.statement.CompositeStatement; import com.aliyun.fastmodel.core.tree.statement.element.CreateElement; @@ -46,6 +49,7 @@ import com.aliyun.fastmodel.core.tree.statement.table.constraint.PrimaryConstraint; import com.aliyun.fastmodel.core.tree.util.IdentifierUtil; import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.hive.format.HivePropertyKey; import com.aliyun.fastmodel.transform.hive.parser.HiveParser.ColumnNameColonTypeContext; import com.aliyun.fastmodel.transform.hive.parser.HiveParser.ColumnNameContext; import com.aliyun.fastmodel.transform.hive.parser.HiveParser.ColumnNameTypeConstraintContext; @@ -65,7 +69,10 @@ import com.aliyun.fastmodel.transform.hive.parser.HiveParser.TypeParameterContext; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.TerminalNode; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import static com.aliyun.fastmodel.common.parser.ParserHelper.getLocation; import static com.aliyun.fastmodel.common.parser.ParserHelper.getOrigin; @@ -115,11 +122,14 @@ public Node visitCreateTableStatement(CreateTableStatementContext ctx) { (QualifiedName)visit(ctx.tableName(1)) ); } else { + // 表名 QualifiedName tableName = (QualifiedName)visit(ctx.tableName(0)); + // 描述 Comment comment = null; if (ctx.tableComment() != null) { comment = (Comment)visit(ctx.tableComment()); } + // properties List all = Lists.newArrayList(reverseContext.getProperties() == null ? Lists.newArrayList() : reverseContext.getProperties()); List properties = ImmutableList.of(); if (ctx.tablePropertiesPrefixed() != null) { @@ -128,6 +138,9 @@ public Node visitCreateTableStatement(CreateTableStatementContext ctx) { Property.class); all.addAll(properties); } + // 外表属性 + buildExternalProperties(ctx, all); + // constraint ColumnNameTypeOrConstraintListContext columnNameTypeOrConstraintListContext = ctx.columnNameTypeOrConstraintList(); List nodes = Lists.newArrayListWithCapacity(64); @@ -138,6 +151,7 @@ public Node visitCreateTableStatement(CreateTableStatementContext ctx) { Node.class ); } + //all columns List list = nodes.stream().filter( node -> { return node instanceof ColumnDefinition; @@ -285,7 +299,7 @@ public Node visitMapType(HiveParser.MapTypeContext ctx) { @Override public Node visitStructType(HiveParser.StructTypeContext ctx) { List list = ParserHelper.visit(this, ctx.columnNameColonTypeList().columnNameColonType(), Field.class); - return new RowDataType(list); + return new RowDataType(getLocation(ctx), getOrigin(ctx), list); } @@ -319,4 +333,102 @@ private StringLiteral getStringLiteral(TerminalNode terminalNode) { public Node visitIdentifier(IdentifierContext ctx) { return ParserHelper.getIdentifier(ctx); } + + + private void buildExternalProperties(CreateTableStatementContext ctx, List properties) { + // external + if (ctx.KW_EXTERNAL() != null) { + properties.add(new Property(HivePropertyKey.EXTERNAL_TABLE.getValue(), new BooleanLiteral(BooleanLiteral.TRUE))); + } + // row format serde + if (ctx.tableRowFormat() != null) { + List rowFormatProperties = analyzeRowFormat(ctx.tableRowFormat()); + properties.addAll(rowFormatProperties); + } + // stored as/by + if (ctx.tableFileFormat() != null) { + List fileFormatProperties = analyzeFileFormat(ctx.tableFileFormat()); + properties.addAll(fileFormatProperties); + } + // location + if (ctx.tableLocation() != null) { + String value = StripUtils.strip(ctx.tableLocation().stop.getText()); + properties.add(new Property(HivePropertyKey.LOCATION.getValue(), new StringLiteral(value))); + } + } + + private List analyzeFileFormat(HiveParser.TableFileFormatContext ctx) { + if (ctx == null) { + return Collections.emptyList(); + } + List properties = new ArrayList<>(); + List children = ctx.children.stream().map(ParseTree::getText).collect(Collectors.toList()); +// if (ctx.KW_BY() != null) { +// // stored by +// int byIndex = children.indexOf(ctx.KW_BY().getText()); +// String byValue = StripUtils.strip(children.get(byIndex + 1)); +// properties.add(new Property(HivePropertyKey.STORED_BY.getValue(), new StringLiteral(byValue))); +// } + if (ctx.KW_AS() != null && ctx.KW_INPUTFORMAT() == null && ctx.KW_OUTPUTFORMAT() == null) { + // stored as + int asIndex = children.indexOf(ctx.KW_AS().getText()); + String asValue = StripUtils.strip(children.get(asIndex + 1)); + properties.add(new Property(HivePropertyKey.STORAGE_FORMAT.getValue(), new StringLiteral(asValue))); + } + if (ctx.KW_INPUTFORMAT() != null) { + // stored as INPUTFORMAT + int inputFormatIndex = children.indexOf(ctx.KW_INPUTFORMAT().getText()); + String inputFormatValue = StripUtils.strip(children.get(inputFormatIndex + 1)); + properties.add(new Property(HivePropertyKey.STORED_INPUT_FORMAT.getValue(), new StringLiteral(inputFormatValue))); + } + if (ctx.KW_OUTPUTFORMAT() != null) { + // stored as OUTPUTFORMAT + int outputFormatIndex = children.indexOf(ctx.KW_OUTPUTFORMAT().getText()); + String outputFormatValue = StripUtils.strip(children.get(outputFormatIndex + 1)); + properties.add(new Property(HivePropertyKey.STORED_OUTPUT_FORMAT.getValue(), new StringLiteral(outputFormatValue))); + } + return properties; + } + + private List analyzeRowFormat(HiveParser.TableRowFormatContext ctx) { + if (ctx == null) { + return Collections.emptyList(); + } + List properties = new ArrayList<>(); + if (ctx.rowFormatSerde() != null) { + HiveParser.RowFormatSerdeContext rowFormatSerdeContext = ctx.rowFormatSerde(); + List children = rowFormatSerdeContext.children.stream() + .map(ParseTree::getText).collect(Collectors.toList()); + if (rowFormatSerdeContext.KW_SERDE() != null) { + // stored as + int index = children.indexOf(rowFormatSerdeContext.KW_SERDE().getText()); + String value = StripUtils.strip(children.get(index + 1)); + properties.add(new Property(HivePropertyKey.ROW_FORMAT_SERDE.getValue(), new StringLiteral(value))); + } + if (rowFormatSerdeContext.tableProperties() != null + && rowFormatSerdeContext.tableProperties().tablePropertiesList() != null + && CollectionUtils.isNotEmpty(rowFormatSerdeContext.tableProperties().tablePropertiesList().keyValueProperty())) { + List keyValuePropertyContexts = + rowFormatSerdeContext.tableProperties().tablePropertiesList().keyValueProperty(); + keyValuePropertyContexts.forEach(context -> { + String propKey = StripUtils.strip(context.getChild(0).getText()); + String propValue = StripUtils.strip(context.getChild(2).getText()); + properties.add(new Property(StringUtils.join(Lists.newArrayList(HivePropertyKey.SERDE_PROPS.getValue(), propKey), + "."), new StringLiteral(propValue))); + }); + } + } + if (ctx.rowFormatDelimited() != null) { + HiveParser.RowFormatDelimitedContext rowFormatDelimitedContext = ctx.rowFormatDelimited(); + if (rowFormatDelimitedContext.tableRowFormatFieldIdentifier() != null) { + String value = StripUtils.strip(rowFormatDelimitedContext.tableRowFormatFieldIdentifier().stop.getText()); + properties.add(new Property(HivePropertyKey.FIELDS_TERMINATED.getValue(), new StringLiteral(value))); + } + if (rowFormatDelimitedContext.tableRowFormatLinesIdentifier() != null) { + String value = StripUtils.strip(rowFormatDelimitedContext.tableRowFormatLinesIdentifier().stop.getText()); + properties.add(new Property(HivePropertyKey.LINES_TERMINATED.getValue(), new StringLiteral(value))); + } + } + return properties; + } } diff --git a/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/HiveCodeGeneratorExternalTest.java b/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/HiveCodeGeneratorExternalTest.java index 65209b9..2b76f56 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/HiveCodeGeneratorExternalTest.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/HiveCodeGeneratorExternalTest.java @@ -10,6 +10,9 @@ import java.util.List; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.transform.api.Transformer; +import com.aliyun.fastmodel.transform.api.TransformerFactory; import com.aliyun.fastmodel.transform.api.client.CodeGenerator; import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; import com.aliyun.fastmodel.transform.api.client.dto.request.DdlGeneratorModelRequest; @@ -20,6 +23,8 @@ import com.aliyun.fastmodel.transform.api.client.dto.table.Table; import com.aliyun.fastmodel.transform.api.client.dto.table.TableConfig; import com.aliyun.fastmodel.transform.api.client.generator.DefaultCodeGenerator; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.api.context.TransformContext; import com.aliyun.fastmodel.transform.api.dialect.DialectMeta; import com.aliyun.fastmodel.transform.api.dialect.DialectNode; import com.aliyun.fastmodel.transform.hive.client.property.FieldsTerminated; @@ -93,6 +98,213 @@ public void testGeneratorNewTable() { + "STORED AS ORC"); } + @Test + public void testGeneratorHiveTable() { + CodeGenerator codeGenerator = new DefaultCodeGenerator(); + String createTableDdl = "CREATE TABLE `default.test_ziliang_v1`(\n" + + " `a` string COMMENT 'aa1')\n" + + "COMMENT '123'\n" + + "PARTITIONED BY ( \n" + + " `ds` string COMMENT '')\n" + + "ROW FORMAT SERDE \n" + + " 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' \n" + + "STORED AS INPUTFORMAT \n" + + " 'org.apache.hadoop.mapred.TextInputFormat' \n" + + "OUTPUTFORMAT \n" + + " 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" + + "LOCATION\n" + + " 'hdfs://master-1-1.c-aac6b952af7280c8.cn-shanghai.emr.aliyuncs.com:9000/user/hive/warehouse/test_ziliang_v1'\n" + + "TBLPROPERTIES (\n" + + " 'bucketing_version'='2', \n" + + " 'transient_lastDdlTime'='1698806349')"; + DialectNode dialectNode = new DialectNode(createTableDdl); + ReverseContext build = ReverseContext.builder().build(); + Transformer transformer = TransformerFactory.getInstance().get(DialectMeta.getHive()); + Node reverse = transformer.reverse(dialectNode, build); + Table table = transformer.transformTable(reverse, TransformContext.builder().build()); + + + TableConfig config = TableConfig.builder() + .dialectMeta(DialectMeta.DEFAULT_HIVE) + .caseSensitive(false) + .build(); + DdlGeneratorModelRequest request = DdlGeneratorModelRequest.builder() + .after(table) + .config(config) + .build(); + DdlGeneratorResult generate = codeGenerator.generate(request); + List dialectNodes = generate.getDialectNodes(); + DialectNode dialectNode1 = dialectNodes.get(0); + assertEquals(dialectNode1.getNode(), "CREATE TABLE `default.test_ziliang_v1`\n" + + "(\n" + + " a STRING COMMENT 'aa1'\n" + + ")\n" + + "COMMENT '123'\n" + + "PARTITIONED BY\n" + + "(\n" + + " ds STRING COMMENT ''\n" + + ")\n" + + "ROW FORMAT SERDE\n" + + "'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'\n" + + "STORED AS \n" + + "INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'\n" + + "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" + + "LOCATION 'hdfs://master-1-1.c-aac6b952af7280c8.cn-shanghai.emr.aliyuncs.com:9000/user/hive/warehouse/test_ziliang_v1'\n" + + "TBLPROPERTIES ('bucketing_version'='2','transient_lastDdlTime'='1698806349','hive.row_format_serde'='org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe','hive.stored_input_format'='org.apache.hadoop.mapred.TextInputFormat','hive.stored_output_format'='org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat','hive.location'='hdfs://master-1-1.c-aac6b952af7280c8.cn-shanghai.emr.aliyuncs.com:9000/user/hive/warehouse/test_ziliang_v1');"); + } + + @Test + public void testGenerateHiveExternalTable() { + CodeGenerator codeGenerator = new DefaultCodeGenerator(); + String createTableDdl = "CREATE EXTERNAL TABLE `oss_share_feedback`(\n" + + " `uid` string, \n" + + " `os` string, \n" + + " `source_id` string, \n" + + " `type` string, \n" + + " `target_key` string, \n" + + " `created_time` string, \n" + + " `updated_time` string)\n" + + "PARTITIONED BY ( \n" + + " `pt_month` string, \n" + + " `pt_day` string)\n" + + "ROW FORMAT SERDE \n" + + " 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' \n" + + "WITH SERDEPROPERTIES ( \n" + + " 'field.delim'='\\t', \n" + + " 'line.delim'='\\n', \n" + + " 'serialization.format'='\\t') \n" + + "STORED AS INPUTFORMAT \n" + + " 'org.apache.hadoop.mapred.TextInputFormat' \n" + + "OUTPUTFORMAT \n" + + " 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" + + "LOCATION\n" + + " 'oss://{AccessKeyId}:{AccessKeySecret}@{bucket}.{endpoint}/hive/oss_share_feedback'\n" + + "TBLPROPERTIES (\n" + + " 'transient_lastDdlTime'='1495603307');"; + DialectNode dialectNode = new DialectNode(createTableDdl); + ReverseContext build = ReverseContext.builder().build(); + Transformer transformer = TransformerFactory.getInstance().get(DialectMeta.getHive()); + Node reverse = transformer.reverse(dialectNode, build); + Table table = transformer.transformTable(reverse, TransformContext.builder().build()); + + + TableConfig config = TableConfig.builder() + .dialectMeta(DialectMeta.DEFAULT_HIVE) + .caseSensitive(false) + .build(); + DdlGeneratorModelRequest request = DdlGeneratorModelRequest.builder() + .after(table) + .config(config) + .build(); + DdlGeneratorResult generate = codeGenerator.generate(request); + List dialectNodes = generate.getDialectNodes(); + DialectNode dialectNode1 = dialectNodes.get(0); + assertEquals(dialectNode1.getNode(), "CREATE EXTERNAL TABLE oss_share_feedback\n" + + "(\n" + + " uid STRING,\n" + + " os STRING,\n" + + " source_id STRING,\n" + + " type STRING,\n" + + " target_key STRING,\n" + + " created_time STRING,\n" + + " updated_time STRING\n" + + ")\n" + + "PARTITIONED BY\n" + + "(\n" + + " pt_month STRING,\n" + + " pt_day STRING\n" + + ")\n" + + "ROW FORMAT SERDE\n" + + "'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'\n" + + "WITH SERDEPROPERTIES (\n" + + " 'field.delim' = '\\t',\n" + + " 'line.delim' = '\\n',\n" + + " 'serialization.format' = '\\t'\n" + + ")\n" + + "STORED AS \n" + + "INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'\n" + + "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" + + "LOCATION 'oss://{AccessKeyId}:{AccessKeySecret}@{bucket}.{endpoint}/hive/oss_share_feedback'\n" + + "TBLPROPERTIES ('transient_lastDdlTime'='1495603307','hive.table_external'='true'," + + "'hive.row_format_serde'='org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'," + + "'hive.serde_props.field.delim'='\\t','hive.serde_props.line.delim'='\\n'," + + "'hive.serde_props.serialization.format'='\\t'," + + "'hive.stored_input_format'='org.apache.hadoop.mapred.TextInputFormat'," + + "'hive.stored_output_format'='org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'," + + "'hive.location'='oss://{AccessKeyId}:{AccessKeySecret}@{bucket}.{endpoint}/hive/oss_share_feedback'," + + "'hive.table_external'='true');"); + } + + @Test + public void testGenerateHiveExternalTable2() { + CodeGenerator codeGenerator = new DefaultCodeGenerator(); + String createTableDdl = "CREATE EXTERNAL TABLE `oss_share_feedback`(\n" + + " `uid` string, \n" + + " `os` string, \n" + + " `source_id` string, \n" + + " `type` string, \n" + + " `target_key` string, \n" + + " `created_time` string, \n" + + " `updated_time` string)\n" + + "PARTITIONED BY ( \n" + + " `pt_month` string, \n" + + " `pt_day` string)\n" + + "ROW FORMAT delimited \n" + + "fields terminated by ','" + + "STORED AS INPUTFORMAT \n" + + " 'org.apache.hadoop.mapred.TextInputFormat' \n" + + "OUTPUTFORMAT \n" + + " 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" + + "LOCATION\n" + + " 'oss://{AccessKeyId}:{AccessKeySecret}@{bucket}.{endpoint}/hive/oss_share_feedback'\n" + + "TBLPROPERTIES (\n" + + " 'transient_lastDdlTime'='1495603307');"; + DialectNode dialectNode = new DialectNode(createTableDdl); + ReverseContext build = ReverseContext.builder().build(); + Transformer transformer = TransformerFactory.getInstance().get(DialectMeta.getHive()); + Node reverse = transformer.reverse(dialectNode, build); + Table table = transformer.transformTable(reverse, TransformContext.builder().build()); + + + TableConfig config = TableConfig.builder() + .dialectMeta(DialectMeta.DEFAULT_HIVE) + .caseSensitive(false) + .build(); + DdlGeneratorModelRequest request = DdlGeneratorModelRequest.builder() + .after(table) + .config(config) + .build(); + DdlGeneratorResult generate = codeGenerator.generate(request); + List dialectNodes = generate.getDialectNodes(); + DialectNode dialectNode1 = dialectNodes.get(0); + assertEquals(dialectNode1.getNode(), "CREATE EXTERNAL TABLE oss_share_feedback\n" + + "(\n" + + " uid STRING,\n" + + " os STRING,\n" + + " source_id STRING,\n" + + " type STRING,\n" + + " target_key STRING,\n" + + " created_time STRING,\n" + + " updated_time STRING\n" + + ")\n" + + "PARTITIONED BY\n" + + "(\n" + + " pt_month STRING,\n" + + " pt_day STRING\n" + + ")\n" + + "ROW FORMAT DELIMITED\n" + + "FIELDS TERMINATED BY ','\n" + + "STORED AS \n" + + "INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'\n" + + "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" + + "LOCATION 'oss://{AccessKeyId}:{AccessKeySecret}@{bucket}.{endpoint}/hive/oss_share_feedback'\n" + + "TBLPROPERTIES ('transient_lastDdlTime'='1495603307','hive.table_external'='true'," + + "'hive.fields_terminated'=',','hive.stored_input_format'='org.apache.hadoop.mapred.TextInputFormat'," + + "'hive.stored_output_format'='org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'," + + "'hive.location'='oss://{AccessKeyId}:{AccessKeySecret}@{bucket}.{endpoint}/hive/oss_share_feedback'," + + "'hive.table_external'='true');"); + } + @Test public void testGenerator1() { DefaultCodeGenerator defaultCodeGenerator = new DefaultCodeGenerator(); @@ -109,7 +321,7 @@ public void testGenerator1() { .dialectMeta(DialectMeta.DEFAULT_HIVE).build(); DdlTableResult reverse = defaultCodeGenerator.reverse(request); Table table = reverse.getTable(); - assertEquals(table.getSchema(), "default"); + assertEquals(table.getDatabase(), "default"); } @Test diff --git a/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/HiveCodeGeneratorModelTest.java b/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/HiveCodeGeneratorModelTest.java index 5f0a154..9b0de44 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/HiveCodeGeneratorModelTest.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/HiveCodeGeneratorModelTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; /** * code generator @@ -208,7 +209,7 @@ public void testConvertToTable() { DdlTableResult reverse = defaultCodeGenerator.reverse(request); Table table = reverse.getTable(); assertEquals(table.getDatabase(), "d1"); - assertEquals(table.getSchema(), "c1"); + assertNull(table.getSchema()); assertEquals(table.getName(), "a"); assertEquals(table.getColumns().size(), 1); } diff --git a/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/converter/HiveClientConverterTest.java b/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/converter/HiveClientConverterTest.java index 7e7e357..bd6376b 100644 --- a/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/converter/HiveClientConverterTest.java +++ b/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/client/converter/HiveClientConverterTest.java @@ -188,7 +188,7 @@ public void testConvertToTable() { Table table1 = hiveClientConverter.convertToTable(table, HiveTransformContext.builder().build()); assertTrue(table1.isExternal()); List properties1 = table1.getProperties(); - BaseClientProperty baseClientProperty = properties1.get(0); + BaseClientProperty baseClientProperty = properties1.get(1); assertEquals(baseClientProperty.getKey(), HivePropertyKey.STORAGE_FORMAT.getValue()); } @@ -211,7 +211,7 @@ public void testConvertToNode() { .properties(properties) .build(); Node node = hiveClientConverter.covertToNode(table, TableConfig.builder().build()); - assertEquals(node.toString(), "CREATE DIM TABLE IF NOT EXISTS abc \n" + assertEquals(node.toString(), "CREATE TABLE IF NOT EXISTS abc \n" + "(\n" + " n1 BIGINT NOT NULL\n" + ")\n" diff --git a/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/parser/HiveAstBuilderTest.java b/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/parser/HiveAstBuilderTest.java new file mode 100644 index 0000000..505aa18 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-hive/src/test/java/com/aliyun/fastmodel/transform/hive/parser/HiveAstBuilderTest.java @@ -0,0 +1,68 @@ +package com.aliyun.fastmodel.transform.hive.parser; + +import com.aliyun.fastmodel.core.tree.BaseStatement; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.statement.constants.TableDetailType; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.parser.NodeParser; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.api.dialect.DialectNode; +import com.aliyun.fastmodel.transform.hive.HiveTransformer; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; + +import java.util.List; + +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.*; + +/** + * @author 子梁 + * @date 2023/11/10 + */ +public class HiveAstBuilderTest { + HiveTransformer hiveTransformer = new HiveTransformer(); + + NodeParser nodeParser = new NodeParser(); + + @Test + public void testReverse() { + String code = "CREATE TABLE `default.test_ziliang_v1`(\n" + + " `a` string COMMENT 'aa')\n" + + "COMMENT '123'\n" + + "PARTITIONED BY ( \n" + + " `ds` string COMMENT '')\n" + + "ROW FORMAT SERDE \n" + + " 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' \n" + + "STORED AS INPUTFORMAT \n" + + " 'org.apache.hadoop.mapred.TextInputFormat' \n" + + "OUTPUTFORMAT \n" + + " 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'\n" + + "LOCATION\n" + + " 'hdfs://master-1-1.c-aac6b952af7280c8.cn-shanghai.emr.aliyuncs.com:9000/user/hive/warehouse/test_ziliang_v1'\n" + + "TBLPROPERTIES (\n" + + " 'bucketing_version'='2', \n" + + " 'transient_lastDdlTime'='1698806349')"; + BaseStatement reverse = hiveTransformer.reverse(new DialectNode(code), ReverseContext.builder().build()); + MatcherAssert.assertThat(reverse.toString(), Matchers.equalTo("CREATE DIM TABLE `default.test_ziliang_v1` \n" + + "(\n" + + " `a` STRING COMMENT 'aa'\n" + + ")\n" + + "COMMENT '123'\n" + + "PARTITIONED BY\n" + + "(\n" + + " `ds` STRING COMMENT ''\n" + + ")\n" + + "WITH('bucketing_version'='2','transient_lastDdlTime'='1698806349'," + + "'hive.row_format_serde'='org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'," + + "'hive.stored_input_format'='org.apache.hadoop.mapred.TextInputFormat'," + + "'hive.stored_output_format'='org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'," + + "'hive.location'='hdfs://master-1-1.c-aac6b952af7280c8.cn-shanghai.emr.aliyuncs.com:9000/user/hive/warehouse/test_ziliang_v1')")); + } +} \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-hologres/src/main/java/com/aliyun/fastmodel/transform/hologres/parser/tree/datatype/HologresArrayDataType.java b/fastmodel-transform/fastmodel-transform-hologres/src/main/java/com/aliyun/fastmodel/transform/hologres/parser/tree/datatype/HologresArrayDataType.java index 4400ee8..058702a 100644 --- a/fastmodel-transform/fastmodel-transform-hologres/src/main/java/com/aliyun/fastmodel/transform/hologres/parser/tree/datatype/HologresArrayDataType.java +++ b/fastmodel-transform/fastmodel-transform-hologres/src/main/java/com/aliyun/fastmodel/transform/hologres/parser/tree/datatype/HologresArrayDataType.java @@ -13,6 +13,8 @@ import com.aliyun.fastmodel.core.tree.IAstVisitor; import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName; +import com.aliyun.fastmodel.transform.hologres.context.HologresTransformContext; +import com.aliyun.fastmodel.transform.hologres.parser.visitor.HologresExpressionVisitor; import com.aliyun.fastmodel.transform.hologres.parser.visitor.HologresVisitor; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -47,4 +49,9 @@ public R accept(IAstVisitor visitor, C context) { HologresVisitor hologresVisitor = (HologresVisitor)visitor; return hologresVisitor.visitHologresArrayDataType(this, context); } + + @Override + public String toString() { + return new HologresExpressionVisitor(HologresTransformContext.builder().build()).process(this); + } } diff --git a/fastmodel-transform/fastmodel-transform-hologres/src/main/java/com/aliyun/fastmodel/transform/hologres/parser/tree/datatype/HologresGenericDataType.java b/fastmodel-transform/fastmodel-transform-hologres/src/main/java/com/aliyun/fastmodel/transform/hologres/parser/tree/datatype/HologresGenericDataType.java index f3ab77f..d0e1b85 100644 --- a/fastmodel-transform/fastmodel-transform-hologres/src/main/java/com/aliyun/fastmodel/transform/hologres/parser/tree/datatype/HologresGenericDataType.java +++ b/fastmodel-transform/fastmodel-transform-hologres/src/main/java/com/aliyun/fastmodel/transform/hologres/parser/tree/datatype/HologresGenericDataType.java @@ -14,6 +14,8 @@ import com.aliyun.fastmodel.core.tree.datatype.DataTypeParameter; import com.aliyun.fastmodel.core.tree.datatype.GenericDataType; import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName; +import com.aliyun.fastmodel.transform.hologres.context.HologresTransformContext; +import com.aliyun.fastmodel.transform.hologres.parser.visitor.HologresExpressionVisitor; import com.aliyun.fastmodel.transform.hologres.parser.visitor.HologresVisitor; import lombok.Getter; import lombok.ToString; @@ -25,7 +27,6 @@ * @date 2022/6/9 */ @Getter -@ToString public class HologresGenericDataType extends GenericDataType { public HologresGenericDataType(String dataTypeName) { @@ -46,4 +47,9 @@ public R accept(IAstVisitor visitor, C context) { HologresVisitor hologresVisitor = (HologresVisitor)visitor; return hologresVisitor.visitHologresGenericDataType(this, context); } + + @Override + public String toString() { + return new HologresExpressionVisitor(HologresTransformContext.builder().build()).process(this); + } } diff --git a/fastmodel-transform/fastmodel-transform-hologres/src/test/java/com/aliyun/fastmodel/transform/hologres/client/converter/HologresClientConverterTest.java b/fastmodel-transform/fastmodel-transform-hologres/src/test/java/com/aliyun/fastmodel/transform/hologres/client/converter/HologresClientConverterTest.java index a032bd9..f3e3f65 100644 --- a/fastmodel-transform/fastmodel-transform-hologres/src/test/java/com/aliyun/fastmodel/transform/hologres/client/converter/HologresClientConverterTest.java +++ b/fastmodel-transform/fastmodel-transform-hologres/src/test/java/com/aliyun/fastmodel/transform/hologres/client/converter/HologresClientConverterTest.java @@ -33,10 +33,10 @@ import com.aliyun.fastmodel.transform.api.client.dto.table.Table; import com.aliyun.fastmodel.transform.api.client.dto.table.TableConfig; import com.aliyun.fastmodel.transform.hologres.client.property.ClusterKey; -import com.aliyun.fastmodel.transform.hologres.client.property.DictEncodingColumn; import com.aliyun.fastmodel.transform.hologres.client.property.ColumnStatus; -import com.aliyun.fastmodel.transform.hologres.client.property.Status; +import com.aliyun.fastmodel.transform.hologres.client.property.DictEncodingColumn; import com.aliyun.fastmodel.transform.hologres.client.property.DistributionKey; +import com.aliyun.fastmodel.transform.hologres.client.property.Status; import com.aliyun.fastmodel.transform.hologres.client.property.TimeToLiveSeconds; import com.aliyun.fastmodel.transform.hologres.context.HologresTransformContext; import com.aliyun.fastmodel.transform.hologres.parser.tree.datatype.HologresDataTypeName; @@ -47,7 +47,9 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -209,9 +211,14 @@ public void convertToTableComplexDataType() { Table convertToTable = hologresClientConverter.convertToTable(table, HologresTransformContext.builder().build()); assertEquals(convertToTable.getColumns().size(), 1); Column column = convertToTable.getColumns().get(0); - assertEquals(column.toString(), - "Column(id=c1, name=c1, dataType=VARCHAR, length=10, comment=comment, precision=null, scale=null, primaryKey=true, nullable=true, " - + "partitionKey=false, partitionKeyIndex=null)"); + assertEquals("c1", column.getId()); + assertEquals("VARCHAR", column.getDataType()); + assertEquals("comment", column.getComment()); + assertNull(column.getPrecision()); + assertNull(column.getScale()); + assertFalse(column.isPartitionKey()); + assertTrue(column.isNullable()); + assertNull(column.getPartitionKeyIndex()); } @Test @@ -231,9 +238,14 @@ public void convertToTableComplexDecimal() { Table convertToTable = hologresClientConverter.convertToTable(table, HologresTransformContext.builder().build()); assertEquals(convertToTable.getColumns().size(), 1); Column column = convertToTable.getColumns().get(0); - assertEquals( - "Column(id=c1, name=c1, dataType=DECIMAL, length=null, comment=comment, precision=10, scale=11, primaryKey=false, nullable=false, " - + "partitionKey=false, partitionKeyIndex=null)", column.toString()); + assertEquals("c1", column.getId()); + assertEquals("DECIMAL", column.getDataType()); + assertEquals("comment", column.getComment()); + assertEquals(Integer.valueOf(10), column.getPrecision()); + assertEquals(Integer.valueOf(11), column.getScale()); + assertFalse(column.isPartitionKey()); + assertFalse(column.isNullable()); + assertNull(column.getPartitionKeyIndex()); } @Test @@ -253,9 +265,14 @@ public void convertToTablePartition() { Table convertToTable = hologresClientConverter.convertToTable(table, HologresTransformContext.builder().build()); assertEquals(convertToTable.getColumns().size(), 1); Column column = convertToTable.getColumns().get(0); - assertEquals( - "Column(id=c1, name=c1, dataType=DECIMAL, length=null, comment=comment, precision=10, scale=11, primaryKey=false, nullable=false, " - + "partitionKey=true, partitionKeyIndex=0)", column.toString()); + assertEquals("c1", column.getId()); + assertEquals("DECIMAL", column.getDataType()); + assertEquals("comment", column.getComment()); + assertEquals(Integer.valueOf(10), column.getPrecision()); + assertEquals(Integer.valueOf(11), column.getScale()); + assertTrue(column.isPartitionKey()); + assertFalse(column.isNullable()); + assertEquals(Integer.valueOf(0), column.getPartitionKeyIndex()); } @Test diff --git a/fastmodel-transform/fastmodel-transform-spark/src/main/java/com/aliyun/fastmodel/transform/spark/format/SparkVisitor.java b/fastmodel-transform/fastmodel-transform-spark/src/main/java/com/aliyun/fastmodel/transform/spark/format/SparkVisitor.java index 1f7ec3c..76b663a 100644 --- a/fastmodel-transform/fastmodel-transform-spark/src/main/java/com/aliyun/fastmodel/transform/spark/format/SparkVisitor.java +++ b/fastmodel-transform/fastmodel-transform-spark/src/main/java/com/aliyun/fastmodel/transform/spark/format/SparkVisitor.java @@ -69,7 +69,7 @@ private BiConsumer hiveFormat() { appendNumBuckets(node); //append row format - String rowFormat = HiveHelper.appendRowFormat(node); + String rowFormat = HiveHelper.appendRowFormat(node, elementIndent); if (StringUtils.isNotBlank(rowFormat)) { appendLineIfNecessary(); builder.append(rowFormat); diff --git a/fastmodel-transform/fastmodel-transform-spark/src/test/java/com/aliyun/fastmodel/transform/spark/parser/SparkLanguageParserTest.java b/fastmodel-transform/fastmodel-transform-spark/src/test/java/com/aliyun/fastmodel/transform/spark/parser/SparkLanguageParserTest.java index 77e316e..93d9430 100644 --- a/fastmodel-transform/fastmodel-transform-spark/src/test/java/com/aliyun/fastmodel/transform/spark/parser/SparkLanguageParserTest.java +++ b/fastmodel-transform/fastmodel-transform-spark/src/test/java/com/aliyun/fastmodel/transform/spark/parser/SparkLanguageParserTest.java @@ -20,7 +20,7 @@ public void parseNode() { SparkLanguageParser sparkLanguageParser = new SparkLanguageParser(); Node o = sparkLanguageParser.parseNode(parseNode); assertNotNull(o); - assertEquals(o.toString(), "CREATE DIM TABLE a \n" + assertEquals(o.toString(), "CREATE TABLE a \n" + "(\n" + " a BIGINT\n" + ")\n" diff --git a/fastmodel-transform/fastmodel-transform-spark/src/test/java/com/aliyun/fastmodel/transform/spark/parser/SparkTransformerTest.java b/fastmodel-transform/fastmodel-transform-spark/src/test/java/com/aliyun/fastmodel/transform/spark/parser/SparkTransformerTest.java index 1ed4d10..da9e53b 100644 --- a/fastmodel-transform/fastmodel-transform-spark/src/test/java/com/aliyun/fastmodel/transform/spark/parser/SparkTransformerTest.java +++ b/fastmodel-transform/fastmodel-transform-spark/src/test/java/com/aliyun/fastmodel/transform/spark/parser/SparkTransformerTest.java @@ -84,7 +84,7 @@ public void testTransformWithCsv() { public void testReverse() { SparkTransformer sparkTransformer = new SparkTransformer(); BaseStatement reverse = sparkTransformer.reverse(new DialectNode("create table a (b bigint) comment 'abc';")); - assertEquals(reverse.toString(), "CREATE DIM TABLE a \n" + assertEquals(reverse.toString(), "CREATE TABLE a \n" + "(\n" + " b BIGINT\n" + ")\n" diff --git a/fastmodel-transform/fastmodel-transform-sqlite/pom.xml b/fastmodel-transform/fastmodel-transform-sqlite/pom.xml index 9a1ebea..c124e0c 100644 --- a/fastmodel-transform/fastmodel-transform-sqlite/pom.xml +++ b/fastmodel-transform/fastmodel-transform-sqlite/pom.xml @@ -35,6 +35,34 @@ fastmodel-common ${project.parent.version} + + commons-io + commons-io + test + + + com.alibaba + fastjson + test + + + org.slf4j + slf4j-api + test + + + + + org.antlr + antlr4-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + + + \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-sqlite/src/main/antlr4/com/aliyun/fastmodel/transform/sqlite/parser/SQLiteLexer.g4 b/fastmodel-transform/fastmodel-transform-sqlite/src/main/antlr4/com/aliyun/fastmodel/transform/sqlite/parser/SQLiteLexer.g4 new file mode 100644 index 0000000..f247c32 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-sqlite/src/main/antlr4/com/aliyun/fastmodel/transform/sqlite/parser/SQLiteLexer.g4 @@ -0,0 +1,243 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2020 by Martin Mirchev + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and + * associated documentation files (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT + * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * Project : sqlite-parser; an ANTLR4 grammar for SQLite https://github.com/bkiers/sqlite-parser + * Developed by : Bart Kiers, bart@big-o.nl + */ + +// $antlr-format alignTrailingComments on, columnLimit 150, maxEmptyLinesToKeep 1, reflowComments off, useTab off +// $antlr-format allowShortRulesOnASingleLine on, alignSemicolons ownLine + +lexer grammar SQLiteLexer; + +options { caseInsensitive = true; } + +SCOL: ';'; +DOT: '.'; +OPEN_PAR: '('; +CLOSE_PAR: ')'; +COMMA: ','; +ASSIGN: '='; +STAR: '*'; +PLUS: '+'; +MINUS: '-'; +TILDE: '~'; +PIPE2: '||'; +DIV: '/'; +MOD: '%'; +LT2: '<<'; +GT2: '>>'; +AMP: '&'; +PIPE: '|'; +LT: '<'; +LT_EQ: '<='; +GT: '>'; +GT_EQ: '>='; +EQ: '=='; +NOT_EQ1: '!='; +NOT_EQ2: '<>'; + +// http://www.sqlite.org/lang_keywords.html +ABORT_: 'ABORT'; +ACTION_: 'ACTION'; +ADD_: 'ADD'; +AFTER_: 'AFTER'; +ALL_: 'ALL'; +ALTER_: 'ALTER'; +ANALYZE_: 'ANALYZE'; +AND_: 'AND'; +AS_: 'AS'; +ASC_: 'ASC'; +ATTACH_: 'ATTACH'; +AUTOINCREMENT_: 'AUTOINCREMENT'; +BEFORE_: 'BEFORE'; +BEGIN_: 'BEGIN'; +BETWEEN_: 'BETWEEN'; +BY_: 'BY'; +CASCADE_: 'CASCADE'; +CASE_: 'CASE'; +CAST_: 'CAST'; +CHECK_: 'CHECK'; +COLLATE_: 'COLLATE'; +COLUMN_: 'COLUMN'; +COMMIT_: 'COMMIT'; +CONFLICT_: 'CONFLICT'; +CONSTRAINT_: 'CONSTRAINT'; +CREATE_: 'CREATE'; +CROSS_: 'CROSS'; +CURRENT_DATE_: 'CURRENT_DATE'; +CURRENT_TIME_: 'CURRENT_TIME'; +CURRENT_TIMESTAMP_: 'CURRENT_TIMESTAMP'; +DATABASE_: 'DATABASE'; +DEFAULT_: 'DEFAULT'; +DEFERRABLE_: 'DEFERRABLE'; +DEFERRED_: 'DEFERRED'; +DELETE_: 'DELETE'; +DESC_: 'DESC'; +DETACH_: 'DETACH'; +DISTINCT_: 'DISTINCT'; +DROP_: 'DROP'; +EACH_: 'EACH'; +ELSE_: 'ELSE'; +END_: 'END'; +ESCAPE_: 'ESCAPE'; +EXCEPT_: 'EXCEPT'; +EXCLUSIVE_: 'EXCLUSIVE'; +EXISTS_: 'EXISTS'; +EXPLAIN_: 'EXPLAIN'; +FAIL_: 'FAIL'; +FOR_: 'FOR'; +FOREIGN_: 'FOREIGN'; +FROM_: 'FROM'; +FULL_: 'FULL'; +GLOB_: 'GLOB'; +GROUP_: 'GROUP'; +HAVING_: 'HAVING'; +IF_: 'IF'; +IGNORE_: 'IGNORE'; +IMMEDIATE_: 'IMMEDIATE'; +IN_: 'IN'; +INDEX_: 'INDEX'; +INDEXED_: 'INDEXED'; +INITIALLY_: 'INITIALLY'; +INNER_: 'INNER'; +INSERT_: 'INSERT'; +INSTEAD_: 'INSTEAD'; +INTERSECT_: 'INTERSECT'; +INTO_: 'INTO'; +IS_: 'IS'; +ISNULL_: 'ISNULL'; +JOIN_: 'JOIN'; +KEY_: 'KEY'; +LEFT_: 'LEFT'; +LIKE_: 'LIKE'; +LIMIT_: 'LIMIT'; +MATCH_: 'MATCH'; +NATURAL_: 'NATURAL'; +NO_: 'NO'; +NOT_: 'NOT'; +NOTNULL_: 'NOTNULL'; +NULL_: 'NULL'; +OF_: 'OF'; +OFFSET_: 'OFFSET'; +ON_: 'ON'; +OR_: 'OR'; +ORDER_: 'ORDER'; +OUTER_: 'OUTER'; +PLAN_: 'PLAN'; +PRAGMA_: 'PRAGMA'; +PRIMARY_: 'PRIMARY'; +QUERY_: 'QUERY'; +RAISE_: 'RAISE'; +RECURSIVE_: 'RECURSIVE'; +REFERENCES_: 'REFERENCES'; +REGEXP_: 'REGEXP'; +REINDEX_: 'REINDEX'; +RELEASE_: 'RELEASE'; +RENAME_: 'RENAME'; +REPLACE_: 'REPLACE'; +RESTRICT_: 'RESTRICT'; +RETURNING_: 'RETURNING'; +RIGHT_: 'RIGHT'; +ROLLBACK_: 'ROLLBACK'; +ROW_: 'ROW'; +ROWS_: 'ROWS'; +SAVEPOINT_: 'SAVEPOINT'; +SELECT_: 'SELECT'; +SET_: 'SET'; +TABLE_: 'TABLE'; +TEMP_: 'TEMP'; +TEMPORARY_: 'TEMPORARY'; +THEN_: 'THEN'; +TO_: 'TO'; +TRANSACTION_: 'TRANSACTION'; +TRIGGER_: 'TRIGGER'; +UNION_: 'UNION'; +UNIQUE_: 'UNIQUE'; +UPDATE_: 'UPDATE'; +USING_: 'USING'; +VACUUM_: 'VACUUM'; +VALUES_: 'VALUES'; +VIEW_: 'VIEW'; +VIRTUAL_: 'VIRTUAL'; +WHEN_: 'WHEN'; +WHERE_: 'WHERE'; +WITH_: 'WITH'; +WITHOUT_: 'WITHOUT'; +FIRST_VALUE_: 'FIRST_VALUE'; +OVER_: 'OVER'; +PARTITION_: 'PARTITION'; +RANGE_: 'RANGE'; +PRECEDING_: 'PRECEDING'; +UNBOUNDED_: 'UNBOUNDED'; +CURRENT_: 'CURRENT'; +FOLLOWING_: 'FOLLOWING'; +CUME_DIST_: 'CUME_DIST'; +DENSE_RANK_: 'DENSE_RANK'; +LAG_: 'LAG'; +LAST_VALUE_: 'LAST_VALUE'; +LEAD_: 'LEAD'; +NTH_VALUE_: 'NTH_VALUE'; +NTILE_: 'NTILE'; +PERCENT_RANK_: 'PERCENT_RANK'; +RANK_: 'RANK'; +ROW_NUMBER_: 'ROW_NUMBER'; +GENERATED_: 'GENERATED'; +ALWAYS_: 'ALWAYS'; +STORED_: 'STORED'; +TRUE_: 'TRUE'; +FALSE_: 'FALSE'; +WINDOW_: 'WINDOW'; +NULLS_: 'NULLS'; +FIRST_: 'FIRST'; +LAST_: 'LAST'; +FILTER_: 'FILTER'; +GROUPS_: 'GROUPS'; +EXCLUDE_: 'EXCLUDE'; +TIES_: 'TIES'; +OTHERS_: 'OTHERS'; +DO_: 'DO'; +NOTHING_: 'NOTHING'; + +IDENTIFIER: + '"' (~'"' | '""')* '"' + | '`' (~'`' | '``')* '`' + | '[' ~']'* ']' + | [A-Z_] [A-Z_0-9]* +; // TODO check: needs more chars in set + +NUMERIC_LITERAL: ((DIGIT+ ('.' DIGIT*)?) | ('.' DIGIT+)) ('E' [-+]? DIGIT+)? | '0x' HEX_DIGIT+; + +BIND_PARAMETER: '?' DIGIT* | [:@$] IDENTIFIER; + +STRING_LITERAL: '\'' ( ~'\'' | '\'\'')* '\''; + +BLOB_LITERAL: 'X' STRING_LITERAL; + +SINGLE_LINE_COMMENT: '--' ~[\r\n]* (('\r'? '\n') | EOF) -> channel(HIDDEN); + +MULTILINE_COMMENT: '/*' .*? '*/' -> channel(HIDDEN); + +SPACES: [ \u000B\t\r\n] -> channel(HIDDEN); + +UNEXPECTED_CHAR: .; + +fragment HEX_DIGIT: [0-9A-F]; +fragment DIGIT: [0-9]; diff --git a/fastmodel-transform/fastmodel-transform-sqlite/src/main/antlr4/com/aliyun/fastmodel/transform/sqlite/parser/SQLiteParser.g4 b/fastmodel-transform/fastmodel-transform-sqlite/src/main/antlr4/com/aliyun/fastmodel/transform/sqlite/parser/SQLiteParser.g4 new file mode 100644 index 0000000..dd762b5 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-sqlite/src/main/antlr4/com/aliyun/fastmodel/transform/sqlite/parser/SQLiteParser.g4 @@ -0,0 +1,915 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2014 by Bart Kiers + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and + * associated documentation files (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT + * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * Project : sqlite-parser; an ANTLR4 grammar for SQLite https://github.com/bkiers/sqlite-parser + * Developed by: + * Bart Kiers, bart@big-o.nl + * Martin Mirchev, marti_2203@abv.bg + * Mike Lische, mike@lischke-online.de + */ + +// $antlr-format alignTrailingComments on, columnLimit 130, minEmptyLines 1, maxEmptyLinesToKeep 1, reflowComments off +// $antlr-format useTab off, allowShortRulesOnASingleLine off, allowShortBlocksOnASingleLine on, alignSemicolons ownLine + +parser grammar SQLiteParser; + +options { + tokenVocab = SQLiteLexer; +} + +parse: (sql_stmt_list)* EOF +; + +sql_stmt_list: + SCOL* sql_stmt (SCOL+ sql_stmt)* SCOL* +; + +sql_stmt: (EXPLAIN_ (QUERY_ PLAN_)?)? ( + alter_table_stmt + | analyze_stmt + | attach_stmt + | begin_stmt + | commit_stmt + | create_index_stmt + | create_table_stmt + | create_trigger_stmt + | create_view_stmt + | create_virtual_table_stmt + | delete_stmt + | delete_stmt_limited + | detach_stmt + | drop_stmt + | insert_stmt + | pragma_stmt + | reindex_stmt + | release_stmt + | rollback_stmt + | savepoint_stmt + | select_stmt + | update_stmt + | update_stmt_limited + | vacuum_stmt + ) +; + +alter_table_stmt: + ALTER_ TABLE_ (schema_name DOT)? table_name ( + RENAME_ ( + TO_ new_table_name = table_name + | COLUMN_? old_column_name = column_name TO_ new_column_name = column_name + ) + | ADD_ COLUMN_? column_def + | DROP_ COLUMN_? column_name + ) +; + +analyze_stmt: + ANALYZE_ (schema_name | (schema_name DOT)? table_or_index_name)? +; + +attach_stmt: + ATTACH_ DATABASE_? expr AS_ schema_name +; + +begin_stmt: + BEGIN_ (DEFERRED_ | IMMEDIATE_ | EXCLUSIVE_)? ( + TRANSACTION_ transaction_name? + )? +; + +commit_stmt: (COMMIT_ | END_) TRANSACTION_? +; + +rollback_stmt: + ROLLBACK_ TRANSACTION_? (TO_ SAVEPOINT_? savepoint_name)? +; + +savepoint_stmt: + SAVEPOINT_ savepoint_name +; + +release_stmt: + RELEASE_ SAVEPOINT_? savepoint_name +; + +create_index_stmt: + CREATE_ UNIQUE_? INDEX_ (IF_ NOT_ EXISTS_)? (schema_name DOT)? index_name ON_ table_name OPEN_PAR + indexed_column (COMMA indexed_column)* CLOSE_PAR (WHERE_ expr)? +; + +indexed_column: (column_name | expr) (COLLATE_ collation_name)? asc_desc? +; + +create_table_stmt: + CREATE_ (TEMP_ | TEMPORARY_)? TABLE_ (IF_ NOT_ EXISTS_)? ( + schema_name DOT + )? table_name ( + OPEN_PAR column_def (COMMA column_def)*? (COMMA table_constraint)* CLOSE_PAR ( + WITHOUT_ row_ROW_ID = IDENTIFIER + )? + | AS_ select_stmt + ) +; + +column_def: + column_name type_name? column_constraint* +; + +type_name: + name+? ( + OPEN_PAR signed_number CLOSE_PAR + | OPEN_PAR signed_number COMMA signed_number CLOSE_PAR + )? +; + +column_constraint: (CONSTRAINT_ name)? ( + (PRIMARY_ KEY_ asc_desc? conflict_clause? AUTOINCREMENT_?) + | (NOT_? NULL_ | UNIQUE_) conflict_clause? + | CHECK_ OPEN_PAR expr CLOSE_PAR + | DEFAULT_ (signed_number | literal_value | OPEN_PAR expr CLOSE_PAR) + | COLLATE_ collation_name + | foreign_key_clause + | (GENERATED_ ALWAYS_)? AS_ OPEN_PAR expr CLOSE_PAR ( + STORED_ + | VIRTUAL_ + )? + ) +; + +signed_number: (PLUS | MINUS)? NUMERIC_LITERAL +; + +table_constraint: (CONSTRAINT_ name)? ( + (PRIMARY_ KEY_ | UNIQUE_) OPEN_PAR indexed_column ( + COMMA indexed_column + )* CLOSE_PAR conflict_clause? + | CHECK_ OPEN_PAR expr CLOSE_PAR + | FOREIGN_ KEY_ OPEN_PAR column_name (COMMA column_name)* CLOSE_PAR foreign_key_clause + ) +; + +foreign_key_clause: + REFERENCES_ foreign_table ( + OPEN_PAR column_name (COMMA column_name)* CLOSE_PAR + )? ( + ON_ (DELETE_ | UPDATE_) ( + SET_ (NULL_ | DEFAULT_) + | CASCADE_ + | RESTRICT_ + | NO_ ACTION_ + ) + | MATCH_ name + )* (NOT_? DEFERRABLE_ (INITIALLY_ (DEFERRED_ | IMMEDIATE_))?)? +; + +conflict_clause: + ON_ CONFLICT_ ( + ROLLBACK_ + | ABORT_ + | FAIL_ + | IGNORE_ + | REPLACE_ + ) +; + +create_trigger_stmt: + CREATE_ (TEMP_ | TEMPORARY_)? TRIGGER_ (IF_ NOT_ EXISTS_)? ( + schema_name DOT + )? trigger_name (BEFORE_ | AFTER_ | INSTEAD_ OF_)? ( + DELETE_ + | INSERT_ + | UPDATE_ (OF_ column_name ( COMMA column_name)*)? + ) ON_ table_name (FOR_ EACH_ ROW_)? (WHEN_ expr)? BEGIN_ ( + (update_stmt | insert_stmt | delete_stmt | select_stmt) SCOL + )+ END_ +; + +create_view_stmt: + CREATE_ (TEMP_ | TEMPORARY_)? VIEW_ (IF_ NOT_ EXISTS_)? ( + schema_name DOT + )? view_name (OPEN_PAR column_name (COMMA column_name)* CLOSE_PAR)? AS_ select_stmt +; + +create_virtual_table_stmt: + CREATE_ VIRTUAL_ TABLE_ (IF_ NOT_ EXISTS_)? (schema_name DOT)? table_name USING_ module_name ( + OPEN_PAR module_argument (COMMA module_argument)* CLOSE_PAR + )? +; + +with_clause: + WITH_ RECURSIVE_? cte_table_name AS_ OPEN_PAR select_stmt CLOSE_PAR ( + COMMA cte_table_name AS_ OPEN_PAR select_stmt CLOSE_PAR + )* +; + +cte_table_name: + table_name (OPEN_PAR column_name ( COMMA column_name)* CLOSE_PAR)? +; + +recursive_cte: + cte_table_name AS_ OPEN_PAR initial_select UNION_ ALL_? recursive_select CLOSE_PAR +; + +common_table_expression: + table_name (OPEN_PAR column_name ( COMMA column_name)* CLOSE_PAR)? AS_ OPEN_PAR select_stmt CLOSE_PAR +; + +delete_stmt: + with_clause? DELETE_ FROM_ qualified_table_name (WHERE_ expr)? returning_clause? +; + +delete_stmt_limited: + with_clause? DELETE_ FROM_ qualified_table_name (WHERE_ expr)? returning_clause? ( + order_by_stmt? limit_stmt + )? +; + +detach_stmt: + DETACH_ DATABASE_? schema_name +; + +drop_stmt: + DROP_ object = (INDEX_ | TABLE_ | TRIGGER_ | VIEW_) ( + IF_ EXISTS_ + )? (schema_name DOT)? any_name +; + +/* + SQLite understands the following binary operators, in order from highest to lowest precedence: + || + * / % + + - + << >> & | + < <= > >= + = == != <> IS IS NOT IN LIKE GLOB MATCH REGEXP + AND + OR + */ +expr: + literal_value + | BIND_PARAMETER + | ((schema_name DOT)? table_name DOT)? column_name + | unary_operator expr + | expr PIPE2 expr + | expr ( STAR | DIV | MOD) expr + | expr ( PLUS | MINUS) expr + | expr ( LT2 | GT2 | AMP | PIPE) expr + | expr ( LT | LT_EQ | GT | GT_EQ) expr + | expr ( + ASSIGN + | EQ + | NOT_EQ1 + | NOT_EQ2 + | IS_ + | IS_ NOT_ + | IN_ + | LIKE_ + | GLOB_ + | MATCH_ + | REGEXP_ + ) expr + | expr AND_ expr + | expr OR_ expr + | function_name OPEN_PAR ((DISTINCT_? expr ( COMMA expr)*) | STAR)? CLOSE_PAR filter_clause? over_clause? + | OPEN_PAR expr (COMMA expr)* CLOSE_PAR + | CAST_ OPEN_PAR expr AS_ type_name CLOSE_PAR + | expr COLLATE_ collation_name + | expr NOT_? (LIKE_ | GLOB_ | REGEXP_ | MATCH_) expr ( + ESCAPE_ expr + )? + | expr ( ISNULL_ | NOTNULL_ | NOT_ NULL_) + | expr IS_ NOT_? expr + | expr NOT_? BETWEEN_ expr AND_ expr + | expr NOT_? IN_ ( + OPEN_PAR (select_stmt | expr ( COMMA expr)*)? CLOSE_PAR + | ( schema_name DOT)? table_name + | (schema_name DOT)? table_function_name OPEN_PAR (expr (COMMA expr)*)? CLOSE_PAR + ) + | ((NOT_)? EXISTS_)? OPEN_PAR select_stmt CLOSE_PAR + | CASE_ expr? (WHEN_ expr THEN_ expr)+ (ELSE_ expr)? END_ + | raise_function +; + +raise_function: + RAISE_ OPEN_PAR ( + IGNORE_ + | (ROLLBACK_ | ABORT_ | FAIL_) COMMA error_message + ) CLOSE_PAR +; + +literal_value: + NUMERIC_LITERAL + | STRING_LITERAL + | BLOB_LITERAL + | NULL_ + | TRUE_ + | FALSE_ + | CURRENT_TIME_ + | CURRENT_DATE_ + | CURRENT_TIMESTAMP_ +; + +value_row: + OPEN_PAR expr (COMMA expr)* CLOSE_PAR +; + +values_clause: + VALUES_ value_row (COMMA value_row)* +; + +insert_stmt: + with_clause? ( + INSERT_ + | REPLACE_ + | INSERT_ OR_ ( + REPLACE_ + | ROLLBACK_ + | ABORT_ + | FAIL_ + | IGNORE_ + ) + ) INTO_ (schema_name DOT)? table_name (AS_ table_alias)? ( + OPEN_PAR column_name ( COMMA column_name)* CLOSE_PAR + )? ( + ( + ( values_clause | select_stmt ) upsert_clause? + ) + | DEFAULT_ VALUES_ + ) returning_clause? +; + +returning_clause: + RETURNING_ result_column (COMMA result_column)* +; + +upsert_clause: + ON_ CONFLICT_ ( + OPEN_PAR indexed_column (COMMA indexed_column)* CLOSE_PAR (WHERE_ expr)? + )? DO_ ( + NOTHING_ + | UPDATE_ SET_ ( + (column_name | column_name_list) ASSIGN expr ( + COMMA (column_name | column_name_list) ASSIGN expr + )* (WHERE_ expr)? + ) + ) +; + +pragma_stmt: + PRAGMA_ (schema_name DOT)? pragma_name ( + ASSIGN pragma_value + | OPEN_PAR pragma_value CLOSE_PAR + )? +; + +pragma_value: + signed_number + | name + | STRING_LITERAL +; + +reindex_stmt: + REINDEX_ (collation_name | (schema_name DOT)? (table_name | index_name))? +; + +select_stmt: + common_table_stmt? select_core (compound_operator select_core)* order_by_stmt? limit_stmt? +; + +join_clause: + table_or_subquery (join_operator table_or_subquery join_constraint?)* +; + +select_core: + ( + SELECT_ (DISTINCT_ | ALL_)? result_column (COMMA result_column)* ( + FROM_ (table_or_subquery (COMMA table_or_subquery)* | join_clause) + )? (WHERE_ whereExpr=expr)? ( + GROUP_ BY_ groupByExpr+=expr (COMMA groupByExpr+=expr)* ( + HAVING_ havingExpr=expr + )?)? ( + WINDOW_ window_name AS_ window_defn ( + COMMA window_name AS_ window_defn + )* + )? + ) + | values_clause +; + +factored_select_stmt: + select_stmt +; + +simple_select_stmt: + common_table_stmt? select_core order_by_stmt? limit_stmt? +; + +compound_select_stmt: + common_table_stmt? select_core ( + (UNION_ ALL_? | INTERSECT_ | EXCEPT_) select_core + )+ order_by_stmt? limit_stmt? +; + +table_or_subquery: ( + (schema_name DOT)? table_name (AS_? table_alias)? ( + INDEXED_ BY_ index_name + | NOT_ INDEXED_ + )? + ) + | (schema_name DOT)? table_function_name OPEN_PAR expr (COMMA expr)* CLOSE_PAR ( + AS_? table_alias + )? + | OPEN_PAR (table_or_subquery (COMMA table_or_subquery)* | join_clause) CLOSE_PAR + | OPEN_PAR select_stmt CLOSE_PAR (AS_? table_alias)? +; + +result_column: + STAR + | table_name DOT STAR + | expr ( AS_? column_alias)? +; + +join_operator: + COMMA + | NATURAL_? (LEFT_ OUTER_? | INNER_ | CROSS_)? JOIN_ +; + +join_constraint: + ON_ expr + | USING_ OPEN_PAR column_name ( COMMA column_name)* CLOSE_PAR +; + +compound_operator: + UNION_ ALL_? + | INTERSECT_ + | EXCEPT_ +; + +update_stmt: + with_clause? UPDATE_ ( + OR_ (ROLLBACK_ | ABORT_ | REPLACE_ | FAIL_ | IGNORE_) + )? qualified_table_name SET_ (column_name | column_name_list) ASSIGN expr ( + COMMA (column_name | column_name_list) ASSIGN expr + )* ( + FROM_ (table_or_subquery (COMMA table_or_subquery)* | join_clause) + )? (WHERE_ expr)? returning_clause? +; + +column_name_list: + OPEN_PAR column_name (COMMA column_name)* CLOSE_PAR +; + +update_stmt_limited: + with_clause? UPDATE_ ( + OR_ (ROLLBACK_ | ABORT_ | REPLACE_ | FAIL_ | IGNORE_) + )? qualified_table_name SET_ (column_name | column_name_list) ASSIGN expr ( + COMMA (column_name | column_name_list) ASSIGN expr + )* (WHERE_ expr)? returning_clause? (order_by_stmt? limit_stmt)? +; + +qualified_table_name: (schema_name DOT)? table_name (AS_ alias)? ( + INDEXED_ BY_ index_name + | NOT_ INDEXED_ + )? +; + +vacuum_stmt: + VACUUM_ schema_name? (INTO_ filename)? +; + +filter_clause: + FILTER_ OPEN_PAR WHERE_ expr CLOSE_PAR +; + +window_defn: + OPEN_PAR base_window_name? (PARTITION_ BY_ expr (COMMA expr)*)? ( + ORDER_ BY_ ordering_term (COMMA ordering_term)* + ) frame_spec? CLOSE_PAR +; + +over_clause: + OVER_ ( + window_name + | OPEN_PAR base_window_name? (PARTITION_ BY_ expr (COMMA expr)*)? ( + ORDER_ BY_ ordering_term (COMMA ordering_term)* + )? frame_spec? CLOSE_PAR + ) +; + +frame_spec: + frame_clause ( + EXCLUDE_ ( + NO_ OTHERS_ + | CURRENT_ ROW_ + | GROUP_ + | TIES_ + ) + )? +; + +frame_clause: (RANGE_ | ROWS_ | GROUPS_) ( + frame_single + | BETWEEN_ frame_left AND_ frame_right + ) +; + +simple_function_invocation: + simple_func OPEN_PAR (expr (COMMA expr)* | STAR) CLOSE_PAR +; + +aggregate_function_invocation: + aggregate_func OPEN_PAR (DISTINCT_? expr (COMMA expr)* | STAR)? CLOSE_PAR filter_clause? +; + +window_function_invocation: + window_function OPEN_PAR (expr (COMMA expr)* | STAR)? CLOSE_PAR filter_clause? OVER_ ( + window_defn + | window_name + ) +; + +common_table_stmt: //additional structures + WITH_ RECURSIVE_? common_table_expression (COMMA common_table_expression)* +; + +order_by_stmt: + ORDER_ BY_ ordering_term (COMMA ordering_term)* +; + +limit_stmt: + LIMIT_ expr ((OFFSET_ | COMMA) expr)? +; + +ordering_term: + expr (COLLATE_ collation_name)? asc_desc? (NULLS_ (FIRST_ | LAST_))? +; + +asc_desc: + ASC_ + | DESC_ +; + +frame_left: + expr PRECEDING_ + | expr FOLLOWING_ + | CURRENT_ ROW_ + | UNBOUNDED_ PRECEDING_ +; + +frame_right: + expr PRECEDING_ + | expr FOLLOWING_ + | CURRENT_ ROW_ + | UNBOUNDED_ FOLLOWING_ +; + +frame_single: + expr PRECEDING_ + | UNBOUNDED_ PRECEDING_ + | CURRENT_ ROW_ +; + +// unknown + +window_function: + (FIRST_VALUE_ | LAST_VALUE_) OPEN_PAR expr CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr_asc_desc frame_clause + ? CLOSE_PAR + | (CUME_DIST_ | PERCENT_RANK_) OPEN_PAR CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr? CLOSE_PAR + | (DENSE_RANK_ | RANK_ | ROW_NUMBER_) OPEN_PAR CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr_asc_desc + CLOSE_PAR + | (LAG_ | LEAD_) OPEN_PAR expr offset? default_value? CLOSE_PAR OVER_ OPEN_PAR partition_by? + order_by_expr_asc_desc CLOSE_PAR + | NTH_VALUE_ OPEN_PAR expr COMMA signed_number CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr_asc_desc + frame_clause? CLOSE_PAR + | NTILE_ OPEN_PAR expr CLOSE_PAR OVER_ OPEN_PAR partition_by? order_by_expr_asc_desc CLOSE_PAR +; + +offset: + COMMA signed_number +; + +default_value: + COMMA signed_number +; + +partition_by: + PARTITION_ BY_ expr+ +; + +order_by_expr: + ORDER_ BY_ expr+ +; + +order_by_expr_asc_desc: + ORDER_ BY_ expr_asc_desc +; + +expr_asc_desc: + expr asc_desc? (COMMA expr asc_desc?)* +; + +//TODO BOTH OF THESE HAVE TO BE REWORKED TO FOLLOW THE SPEC +initial_select: + select_stmt +; + +recursive_select: + select_stmt +; + +unary_operator: + MINUS + | PLUS + | TILDE + | NOT_ +; + +error_message: + STRING_LITERAL +; + +module_argument: // TODO check what exactly is permitted here + expr + | column_def +; + +column_alias: + IDENTIFIER + | STRING_LITERAL +; + +keyword: + ABORT_ + | ACTION_ + | ADD_ + | AFTER_ + | ALL_ + | ALTER_ + | ANALYZE_ + | AND_ + | AS_ + | ASC_ + | ATTACH_ + | AUTOINCREMENT_ + | BEFORE_ + | BEGIN_ + | BETWEEN_ + | BY_ + | CASCADE_ + | CASE_ + | CAST_ + | CHECK_ + | COLLATE_ + | COLUMN_ + | COMMIT_ + | CONFLICT_ + | CONSTRAINT_ + | CREATE_ + | CROSS_ + | CURRENT_DATE_ + | CURRENT_TIME_ + | CURRENT_TIMESTAMP_ + | DATABASE_ + | DEFAULT_ + | DEFERRABLE_ + | DEFERRED_ + | DELETE_ + | DESC_ + | DETACH_ + | DISTINCT_ + | DROP_ + | EACH_ + | ELSE_ + | END_ + | ESCAPE_ + | EXCEPT_ + | EXCLUSIVE_ + | EXISTS_ + | EXPLAIN_ + | FAIL_ + | FOR_ + | FOREIGN_ + | FROM_ + | FULL_ + | GLOB_ + | GROUP_ + | HAVING_ + | IF_ + | IGNORE_ + | IMMEDIATE_ + | IN_ + | INDEX_ + | INDEXED_ + | INITIALLY_ + | INNER_ + | INSERT_ + | INSTEAD_ + | INTERSECT_ + | INTO_ + | IS_ + | ISNULL_ + | JOIN_ + | KEY_ + | LEFT_ + | LIKE_ + | LIMIT_ + | MATCH_ + | NATURAL_ + | NO_ + | NOT_ + | NOTNULL_ + | NULL_ + | OF_ + | OFFSET_ + | ON_ + | OR_ + | ORDER_ + | OUTER_ + | PLAN_ + | PRAGMA_ + | PRIMARY_ + | QUERY_ + | RAISE_ + | RECURSIVE_ + | REFERENCES_ + | REGEXP_ + | REINDEX_ + | RELEASE_ + | RENAME_ + | REPLACE_ + | RESTRICT_ + | RIGHT_ + | ROLLBACK_ + | ROW_ + | ROWS_ + | SAVEPOINT_ + | SELECT_ + | SET_ + | TABLE_ + | TEMP_ + | TEMPORARY_ + | THEN_ + | TO_ + | TRANSACTION_ + | TRIGGER_ + | UNION_ + | UNIQUE_ + | UPDATE_ + | USING_ + | VACUUM_ + | VALUES_ + | VIEW_ + | VIRTUAL_ + | WHEN_ + | WHERE_ + | WITH_ + | WITHOUT_ + | FIRST_VALUE_ + | OVER_ + | PARTITION_ + | RANGE_ + | PRECEDING_ + | UNBOUNDED_ + | CURRENT_ + | FOLLOWING_ + | CUME_DIST_ + | DENSE_RANK_ + | LAG_ + | LAST_VALUE_ + | LEAD_ + | NTH_VALUE_ + | NTILE_ + | PERCENT_RANK_ + | RANK_ + | ROW_NUMBER_ + | GENERATED_ + | ALWAYS_ + | STORED_ + | TRUE_ + | FALSE_ + | WINDOW_ + | NULLS_ + | FIRST_ + | LAST_ + | FILTER_ + | GROUPS_ + | EXCLUDE_ +; + +// TODO: check all names below + +name: + any_name +; + +function_name: + any_name +; + +schema_name: + any_name +; + +table_name: + any_name +; + +table_or_index_name: + any_name +; + +column_name: + any_name +; + +collation_name: + any_name +; + +foreign_table: + any_name +; + +index_name: + any_name +; + +trigger_name: + any_name +; + +view_name: + any_name +; + +module_name: + any_name +; + +pragma_name: + any_name +; + +savepoint_name: + any_name +; + +table_alias: + any_name +; + +transaction_name: + any_name +; + +window_name: + any_name +; + +alias: + any_name +; + +filename: + any_name +; + +base_window_name: + any_name +; + +simple_func: + any_name +; + +aggregate_func: + any_name +; + +table_function_name: + any_name +; + +any_name: + IDENTIFIER + | keyword + | STRING_LITERAL + | OPEN_PAR any_name CLOSE_PAR +; diff --git a/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/SqliteTransformer.java b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/SqliteTransformer.java index 7a41d8f..2939bee 100644 --- a/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/SqliteTransformer.java +++ b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/SqliteTransformer.java @@ -4,6 +4,7 @@ import com.aliyun.fastmodel.transform.api.Transformer; import com.aliyun.fastmodel.transform.api.builder.BuilderFactory; import com.aliyun.fastmodel.transform.api.builder.StatementBuilder; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; import com.aliyun.fastmodel.transform.api.context.TransformContext; import com.aliyun.fastmodel.transform.api.dialect.Dialect; import com.aliyun.fastmodel.transform.api.dialect.DialectMeta; @@ -12,7 +13,9 @@ import com.aliyun.fastmodel.transform.api.dialect.DialectNode; import com.aliyun.fastmodel.transform.api.dialect.IVersion; import com.aliyun.fastmodel.transform.sqlite.context.SqliteContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SqliteLanguageParser; import com.google.auto.service.AutoService; +import com.google.common.base.Preconditions; /** * sqlite transformer @@ -24,6 +27,8 @@ @Dialect(value = Constants.SQLITE) public class SqliteTransformer implements Transformer { + private final SqliteLanguageParser sqliteLanguageParser = new SqliteLanguageParser(); + @Override public DialectNode transform(BaseStatement source, TransformContext context) { DialectMeta dialectMeta = new DialectMeta(DialectName.SQLITE, IVersion.getDefault()); @@ -35,4 +40,10 @@ public DialectNode transform(BaseStatement source, TransformContext context) { } return builder.build(source, mysqlTransformContext); } + + @Override + public BaseStatement reverse(DialectNode dialectNode, ReverseContext context) { + Preconditions.checkNotNull(dialectNode.getNode()); + return (BaseStatement)sqliteLanguageParser.parseNode(dialectNode.getNode(), context); + } } diff --git a/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/format/SqliteExpressionVisitor.java b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/format/SqliteExpressionVisitor.java new file mode 100644 index 0000000..8f7ea0d --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/format/SqliteExpressionVisitor.java @@ -0,0 +1,27 @@ +package com.aliyun.fastmodel.transform.sqlite.format; + +import com.aliyun.fastmodel.common.utils.StripUtils; +import com.aliyun.fastmodel.core.formatter.ExpressionVisitor; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import org.apache.commons.lang3.StringUtils; + +/** + * sqlite expression visitor + * + * @author panguanjing + * @date 2023/8/29 + */ +public class SqliteExpressionVisitor extends ExpressionVisitor { + + @Override + public String visitIdentifier(Identifier node, Void context) { + String value = StringUtils.isNotBlank(node.getOrigin()) ? + StripUtils.strip(node.getOrigin()) : node.getValue(); + if (!node.isDelimited()) { + return value; + } else { + String strip = StripUtils.strip(value); + return StripUtils.addDoubleStrip(strip); + } + } +} diff --git a/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/format/SqliteVisitor.java b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/format/SqliteVisitor.java index f2050ec..f83ed89 100644 --- a/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/format/SqliteVisitor.java +++ b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/format/SqliteVisitor.java @@ -2,6 +2,7 @@ import com.aliyun.fastmodel.core.formatter.FastModelVisitor; import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; +import com.aliyun.fastmodel.core.tree.expr.BaseExpression; import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; import com.aliyun.fastmodel.transform.api.datatype.DataTypeConverter; @@ -74,4 +75,8 @@ protected BaseDataType convert(BaseDataType dataType) { } } + @Override + protected String formatExpression(BaseExpression baseExpression) { + return new SqliteExpressionVisitor().process(baseExpression); + } } diff --git a/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/parser/SqliteLanguageParser.java b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/parser/SqliteLanguageParser.java new file mode 100644 index 0000000..40da221 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/parser/SqliteLanguageParser.java @@ -0,0 +1,58 @@ +package com.aliyun.fastmodel.transform.sqlite.parser; + +import java.util.function.Function; + +import com.aliyun.fastmodel.common.parser.ThrowingErrorListener; +import com.aliyun.fastmodel.common.parser.lexer.CaseChangingCharStream; +import com.aliyun.fastmodel.common.utils.StripUtils; +import com.aliyun.fastmodel.core.exception.ParseException; +import com.aliyun.fastmodel.core.parser.LanguageParser; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.sqlite.parser.visitor.SqliteAstBuilder; +import com.google.auto.service.AutoService; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CodePointCharStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.atn.PredictionMode; + +/** + * SparkLanguageParser + * + * @author panguanjing + * @date 2023/2/18 + */ +@AutoService(LanguageParser.class) +public class SqliteLanguageParser implements LanguageParser { + + public static final ThrowingErrorListener LISTENER = new ThrowingErrorListener(); + + @Override + public Node parseNode(String text, ReverseContext context) throws ParseException { + return getNode(text, context, SQLiteParser::parse); + } + + private Node getNode(String text, ReverseContext context, Function functionalInterface) { + String code = StripUtils.appendSemicolon(text); + CodePointCharStream charStream = CharStreams.fromString(code); + CaseChangingCharStream caseChangingCharStream = new CaseChangingCharStream(charStream, true); + SQLiteLexer lexer = new SQLiteLexer(caseChangingCharStream); + lexer.removeErrorListeners(); + lexer.addErrorListener(LISTENER); + CommonTokenStream commonTokenStream = new CommonTokenStream(lexer); + SQLiteParser sparkParser = new SQLiteParser(commonTokenStream); + sparkParser.removeErrorListeners(); + sparkParser.addErrorListener(LISTENER); + ParserRuleContext tree; + try { + sparkParser.getInterpreter().setPredictionMode(PredictionMode.SLL); + tree = functionalInterface.apply(sparkParser); + } catch (Throwable e) { + commonTokenStream.seek(0); + sparkParser.getInterpreter().setPredictionMode(PredictionMode.LL); + tree = functionalInterface.apply(sparkParser); + } + return tree.accept(new SqliteAstBuilder(context)); + } +} diff --git a/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/parser/visitor/SqliteAstBuilder.java b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/parser/visitor/SqliteAstBuilder.java new file mode 100644 index 0000000..a118d14 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-sqlite/src/main/java/com/aliyun/fastmodel/transform/sqlite/parser/visitor/SqliteAstBuilder.java @@ -0,0 +1,255 @@ +package com.aliyun.fastmodel.transform.sqlite.parser.visitor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; + +import com.aliyun.fastmodel.common.parser.ParserHelper; +import com.aliyun.fastmodel.common.utils.StripUtils; +import com.aliyun.fastmodel.core.exception.ParseException; +import com.aliyun.fastmodel.core.tree.BaseStatement; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.QualifiedName; +import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; +import com.aliyun.fastmodel.core.tree.datatype.DataTypeParameter; +import com.aliyun.fastmodel.core.tree.datatype.GenericDataType; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.expr.literal.StringLiteral; +import com.aliyun.fastmodel.core.tree.statement.CompositeStatement; +import com.aliyun.fastmodel.core.tree.statement.constants.TableDetailType; +import com.aliyun.fastmodel.core.tree.statement.constants.TableType; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.core.tree.statement.table.CreateAdsTable; +import com.aliyun.fastmodel.core.tree.statement.table.CreateDimTable; +import com.aliyun.fastmodel.core.tree.statement.table.CreateDwsTable; +import com.aliyun.fastmodel.core.tree.statement.table.CreateFactTable; +import com.aliyun.fastmodel.core.tree.statement.table.CreateOdsTable; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable.TableBuilder; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.BaseConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.DimConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.NotNullConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.PrimaryConstraint; +import com.aliyun.fastmodel.core.tree.util.IdentifierUtil; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.Any_nameContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.Column_constraintContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.Column_defContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.Create_table_stmtContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.Foreign_key_clauseContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.Foreign_tableContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.NameContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.ParseContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.Sql_stmt_listContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.Table_constraintContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParser.Type_nameContext; +import com.aliyun.fastmodel.transform.sqlite.parser.SQLiteParserBaseVisitor; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.BooleanUtils; + +/** + * SqliteAstBuilder + * + * @author panguanjing + * @date 2023/8/28 + */ +public class SqliteAstBuilder extends SQLiteParserBaseVisitor { + + private final ReverseContext context; + + public SqliteAstBuilder(ReverseContext context) { + this.context = context == null ? ReverseContext.builder().build() : context; + } + + @Override + public Node visitParse(ParseContext ctx) { + List sql_stmt_listContexts = ctx.sql_stmt_list(); + List visit = ParserHelper.visit(this, sql_stmt_listContexts, BaseStatement.class); + if (sql_stmt_listContexts.size() == 1) { + return visit(sql_stmt_listContexts.get(0)); + } + return new CompositeStatement(visit); + } + + @Override + public Node visitSql_stmt_list(Sql_stmt_listContext ctx) { + List visit = ParserHelper.visit(this, ctx.sql_stmt(), BaseStatement.class); + if (visit.size() == 0) { + throw new ParseException("parse error with text:" + ParserHelper.getOrigin(ctx)); + } + if (visit.size() == 1) { + return visit.get(0); + } + return new CompositeStatement(visit); + } + + @Override + public Node visitCreate_table_stmt(Create_table_stmtContext ctx) { + //由用户指定路由的context的key内容 + + Optional schemaName = ParserHelper.visitIfPresent(this, ctx.schema_name(), Identifier.class); + Optional identifier = ParserHelper.visitIfPresent(this, ctx.table_name(), Identifier.class); + QualifiedName t = null; + if (schemaName.isPresent()) { + t = QualifiedName.of(Lists.newArrayList(schemaName.get(), identifier.get())); + } else { + t = QualifiedName.of(Lists.newArrayList(identifier.get())); + } + List constraints = new ArrayList<>(); + //all columns + List columnDefinitions = ImmutableList.of(); + if (ctx.column_def() != null) { + columnDefinitions = ParserHelper.visit(this, ctx.column_def(), ColumnDefinition.class); + } + //if primary constraint not exist + if (ctx.table_constraint() != null) { + constraints = ParserHelper.visit(this, ctx.table_constraint(), BaseConstraint.class); + } + boolean ifNotExist = ctx.IF_() != null && ctx.NOT_() != null; + TableBuilder tableBuilder = null; + TableDetailType tableDetailType = context.getReverseTableType(); + if (tableDetailType == null) { + tableBuilder = CreateTable.builder(); + } else { + if (tableDetailType.getParent() == TableType.ODS) { + tableBuilder = CreateOdsTable.builder(); + } else if (tableDetailType.getParent() == TableType.DIM) { + tableBuilder = CreateDimTable.builder(); + } else if (tableDetailType.getParent() == TableType.FACT) { + tableBuilder = CreateFactTable.builder(); + } else if (tableDetailType.getParent() == TableType.DWS) { + tableBuilder = CreateDwsTable.builder(); + } else if (tableDetailType.getParent() == TableType.ADS) { + tableBuilder = CreateAdsTable.builder(); + } + } + return tableBuilder + .tableName(t) + .detailType(tableDetailType) + .ifNotExist(ifNotExist) + .columns(columnDefinitions) + .constraints(constraints) + .build(); + } + + @Override + public Node visitColumn_def(Column_defContext ctx) { + Identifier colName = (Identifier)visit(ctx.column_name()); + BaseDataType baseDataType = null; + if (ctx.type_name() != null) { + baseDataType = (BaseDataType)visit(ctx.type_name()); + } + List baseConstraints = null; + if (ctx.column_constraint() != null) { + baseConstraints = ParserHelper.visit(this, ctx.column_constraint(), BaseConstraint.class); + } + Boolean primary = isPrimary(baseConstraints); + return ColumnDefinition.builder() + .colName(colName) + .dataType(baseDataType) + .primary(primary) + .notNull(isNotNull(primary, baseConstraints)) + .build(); + } + + private Boolean isNotNull(Boolean primary, List baseConstraints) { + if (BooleanUtils.isTrue(primary)) { + return true; + } + boolean match = baseConstraints.stream().anyMatch(baseConstraint -> baseConstraint instanceof NotNullConstraint); + if (match) { + return true; + } + return null; + } + + private Boolean isPrimary(List baseConstraints) { + return baseConstraints.stream().anyMatch(baseConstraint -> baseConstraint instanceof PrimaryConstraint); + } + + @Override + public Node visitColumn_constraint(Column_constraintContext ctx) { + Optional optionalIdentifier = ParserHelper.visitIfPresent(this, ctx.name(), Identifier.class); + Identifier identifier = null; + identifier = optionalIdentifier.orElseGet(IdentifierUtil::sysIdentifier); + + if (ctx.PRIMARY_() != null) { + return new PrimaryConstraint(identifier, Lists.newArrayList()); + } + + if (ctx.NOT_() != null && ctx.NULL_() != null) { + return new NotNullConstraint(identifier); + } + return null; + } + + @Override + public Node visitTable_constraint(Table_constraintContext ctx) { + NameContext name = ctx.name(); + Identifier constraintName = null; + if (name != null) { + constraintName = (Identifier)visit(name); + } else { + constraintName = IdentifierUtil.sysIdentifier(); + } + if (ctx.PRIMARY_() != null) { + List list = ParserHelper.visit(this, ctx.indexed_column(), Identifier.class); + return new PrimaryConstraint(constraintName, list); + } + if (ctx.FOREIGN_() != null) { + List list = ParserHelper.visit(this, ctx.column_name(), Identifier.class); + Foreign_key_clauseContext foreignKeyClauseContext = ctx.foreign_key_clause(); + QualifiedName qualifiedName = (QualifiedName)visit(foreignKeyClauseContext.foreign_table()); + List referenceColumns = ParserHelper.visit(this, foreignKeyClauseContext.column_name(), Identifier.class); + //如果为空，那么用左表的列输出 + if (referenceColumns.isEmpty()) { + referenceColumns = new CopyOnWriteArrayList<>(list); + } + return new DimConstraint( + constraintName, list, + qualifiedName, + referenceColumns + ); + } + return super.visitTable_constraint(ctx); + } + + @Override + public Node visitForeign_table(Foreign_tableContext ctx) { + Identifier identifier = (Identifier)visit(ctx.any_name()); + return QualifiedName.of(Lists.newArrayList(identifier)); + } + + @Override + public Node visitAny_name(Any_nameContext ctx) { + if (ctx.IDENTIFIER() != null) { + String text = ctx.IDENTIFIER().getText(); + Identifier identifier = new Identifier(text); + if (identifier.isDelimited()) { + return new Identifier(StripUtils.strip(text), true); + } + return identifier; + } + if (ctx.keyword() != null) { + return new Identifier(ctx.keyword().getText()); + } + if (ctx.STRING_LITERAL() != null) { + StringLiteral stringLiteral = new StringLiteral(StripUtils.strip(ctx.STRING_LITERAL().getText())); + return new Identifier(stringLiteral.getValue()); + } + return new Identifier(ctx.getText()); + } + + @Override + public Node visitType_name(Type_nameContext ctx) { + List list = ParserHelper.visit(this, ctx.name(), Identifier.class); + List dataTypeParameter = null; + if (ctx.signed_number() != null) { + dataTypeParameter = ParserHelper.visit(this, ctx.signed_number(), DataTypeParameter.class); + } + Identifier first = list.get(0); + return new GenericDataType(first.getValue(), dataTypeParameter); + } +} diff --git a/fastmodel-transform/fastmodel-transform-sqlite/src/test/java/com/aliyun/fastmodel/transform/sqlite/SqliteTransformerTest.java b/fastmodel-transform/fastmodel-transform-sqlite/src/test/java/com/aliyun/fastmodel/transform/sqlite/SqliteTransformerTest.java index de72017..a063174 100644 --- a/fastmodel-transform/fastmodel-transform-sqlite/src/test/java/com/aliyun/fastmodel/transform/sqlite/SqliteTransformerTest.java +++ b/fastmodel-transform/fastmodel-transform-sqlite/src/test/java/com/aliyun/fastmodel/transform/sqlite/SqliteTransformerTest.java @@ -59,4 +59,9 @@ public void transform() { + " d_column REAL\n" + ")", transform.getNode()); } + + @Test + public void testTransformWithDoubleQuote() { + + } } \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-sqlite/src/test/java/com/aliyun/fastmodel/transform/sqlite/parser/SqliteLanguageParserTest.java b/fastmodel-transform/fastmodel-transform-sqlite/src/test/java/com/aliyun/fastmodel/transform/sqlite/parser/SqliteLanguageParserTest.java new file mode 100644 index 0000000..14e1df1 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-sqlite/src/test/java/com/aliyun/fastmodel/transform/sqlite/parser/SqliteLanguageParserTest.java @@ -0,0 +1,124 @@ +package com.aliyun.fastmodel.transform.sqlite.parser; + +import java.util.List; +import java.util.stream.Collectors; + +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.statement.CompositeStatement; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.BaseConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.DimConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.PrimaryConstraint; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Desc: + * + * @author panguanjing + * @date 2023/8/28 + */ +@Slf4j +public class SqliteLanguageParserTest { + + SqliteLanguageParser sqliteLanguageParser = new SqliteLanguageParser(); + + @Test + public void parseNode() { + CreateTable createTable = sqliteLanguageParser.parseNode( + "CREATE TABLE overall_team_standings (overall_team_standings_id AUTO_INCREMENT PRIMARY KEY, Rank INT, Team " + + "VARCHAR, Round1 INT, Round2 INT, Round3 INT, Round4 INT, Round5 INT, Total Points INT)"); + assertEquals("overall_team_standings", createTable.getQualifiedName().toString()); + assertEquals(9, createTable.getColumnDefines().size()); + assertEquals("CREATE DIM TABLE overall_team_standings \n" + + "(\n" + + " overall_team_standings_id AUTO_INCREMENT PRIMARY KEY,\n" + + " Rank INT,\n" + + " Team VARCHAR,\n" + + " Round1 INT,\n" + + " Round2 INT,\n" + + " Round3 INT,\n" + + " Round4 INT,\n" + + " Round5 INT,\n" + + " Total Points\n" + + ")", createTable.toString()); + } + + @Data + static class SpiderData { + private String instruction; + private String output; + } + + @Test + public void testReverse() { + CreateTable createTable = sqliteLanguageParser.parseNode("CREATE TABLE \"lists\"\n" + + "(\n" + + "user_id INTEGER\n" + + "references lists_users (user_id),\n" + + "list_id INTEGER not null\n" + + "key_5 key COMMENT 'primary',\n" + + "title TEXT COMMENT 'list title',\n" + + "list_movie_number INTEGER,\n" + + "key_8 TEXT COMMENT 'list update timestamp utc',\n" + + "key_9 TEXT COMMENT 'list creation timestamp utc',\n" + + "llowers INTEGER COMMENT 'list followers',\n" + + "url TEXT COMMENT 'list url',\n" + + "list_comments INTEGER,\n" + + "key_13 TEXT COMMENT 'list description',\n" + + "list_cover TEXT COMMENT 'list cover image url',\n" + + "list_first_image_url TEXT,\n" + + "key_16 TEXT COMMENT 'list second image url',\n" + + "list_third_image_url TEXT\n" + + ")"); + List columnDefines = createTable.getColumnDefines(); + assertEquals(14, columnDefines.size()); + String collect = columnDefines.stream().map(c -> { + return c.getDataType().toString(); + }).collect(Collectors.joining(",")); + assertEquals("INTEGER,INTEGER,TEXT,INTEGER,TEXT,TEXT,INTEGER,TEXT,INTEGER,TEXT,TEXT,TEXT,TEXT,TEXT", collect); + } + + @Test + public void testReverseBatch() { + CompositeStatement createTable = sqliteLanguageParser.parseNode( + "create table a (a text)\n\ncreate table b (a integer)" + ); + assertEquals(2, createTable.getStatements().size()); + } + + @Test + public void testReverseForeignKey() { + CreateTable createTable = sqliteLanguageParser.parseNode( + "CREATE TABLE `countries` (`COUNTRY_ID` varchar(2) NOT NULL,`COUNTRY_NAME` varchar(40) DEFAULT NULL," + + "`REGION_ID` decimal(10,0) DEFAULT NULL,PRIMARY KEY (`COUNTRY_ID`),FOREIGN KEY (`REGION_ID`) REFERENCES regions (`REGION_ID`), " + + "FOREIGN KEY (`COUNTRY_NAME`) REFERENCES regions (`COUNTRY_NAME`))"); + List constraintStatements = createTable.getConstraintStatements(); + assertEquals(3, constraintStatements.size()); + BaseConstraint baseConstraint = constraintStatements.get(0); + assertTrue(baseConstraint instanceof PrimaryConstraint); + baseConstraint = constraintStatements.get(1); + assertTrue(baseConstraint instanceof DimConstraint); + DimConstraint constraint = (DimConstraint)baseConstraint; + assertEquals("REGION_ID", constraint.getColNames().stream().map(Identifier::getValue).collect(Collectors.joining())); + } + + @Test + public void testReverseForeignKeyWithoutColumns() { + CreateTable createTable = sqliteLanguageParser.parseNode( + "CREATE TABLE `countries` (`COUNTRY_ID` varchar(2) NOT NULL,`COUNTRY_NAME` varchar(40) DEFAULT NULL," + + "`REGION_ID` decimal(10,0) DEFAULT NULL,PRIMARY KEY (`COUNTRY_ID`),FOREIGN KEY (`REGION_ID`) REFERENCES regions, " + + "FOREIGN KEY (`COUNTRY_NAME`) REFERENCES regions (`COUNTRY_NAME`))"); + + List constraintStatements = createTable.getConstraintStatements(); + assertEquals(3, constraintStatements.size()); + DimConstraint dimConstraint = (DimConstraint)constraintStatements.get(1); + List referenceColNames = dimConstraint.getReferenceColNames(); + assertEquals(1, referenceColNames.size()); + } +} \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-starrocks/pom.xml b/fastmodel-transform/fastmodel-transform-starrocks/pom.xml new file mode 100644 index 0000000..47e63c1 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/pom.xml @@ -0,0 +1,40 @@ + + + + fastmodel-transform + com.aliyun.fastmodel + ${revision} + + 4.0.0 + + fastmodel-transform-starrocks + + + + + com.aliyun.fastmodel + fastmodel-transform-api + ${project.parent.version} + + + com.google.auto.service + auto-service + + + org.antlr + antlr4-runtime + + + + com.aliyun.fastmodel + fastmodel-common + ${project.parent.version} + + + com.alibaba + fastjson + + + \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/antlr4/com/aliyun/fastmodel/transform/starrocks/parser/StarRocks.g4 b/fastmodel-transform/fastmodel-transform-starrocks/src/main/antlr4/com/aliyun/fastmodel/transform/starrocks/parser/StarRocks.g4 new file mode 100644 index 0000000..e506e4f --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/antlr4/com/aliyun/fastmodel/transform/starrocks/parser/StarRocks.g4 @@ -0,0 +1,2621 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +grammar StarRocks; +import StarRocksLex; + +sqlStatements + : singleStatement+ EOF + ; + +singleStatement + : (statement (SEMICOLON | EOF)) | emptyStatement + ; +emptyStatement + : SEMICOLON + ; + +statement + // Query Statement + : queryStatement + + // Warehouse Statement + | showWarehousesStatement + | showClustersStatement + + // Database Statement + | useDatabaseStatement + | useCatalogStatement + | setCatalogStatement + | showDatabasesStatement + | alterDbQuotaStatement + | createDbStatement + | dropDbStatement + | showCreateDbStatement + | alterDatabaseRenameStatement + | recoverDbStmt + | showDataStmt + + // Table Statement + | createTableStatement + | createTableAsSelectStatement + | createTemporaryTableStatement + | createTableLikeStatement + | showCreateTableStatement + | dropTableStatement + | recoverTableStatement + | truncateTableStatement + | showTableStatement + | descTableStatement + | showTableStatusStatement + | showColumnStatement + | refreshTableStatement + | alterTableStatement + | cancelAlterTableStatement + | showAlterStatement + + // View Statement + | createViewStatement + | alterViewStatement + | dropViewStatement + + // Partition Statement + | showPartitionsStatement + | recoverPartitionStatement + + // Index Statement + | createIndexStatement + | dropIndexStatement + | showIndexStatement + + // Task Statement + | submitTaskStatement + | dropTaskStatement + + // Materialized View Statement + | createMaterializedViewStatement + | showMaterializedViewsStatement + | dropMaterializedViewStatement + | alterMaterializedViewStatement + | refreshMaterializedViewStatement + | cancelRefreshMaterializedViewStatement + + // Catalog Statement + | createExternalCatalogStatement + | dropExternalCatalogStatement + | showCatalogsStatement + | showCreateExternalCatalogStatement + + // DML Statement + | insertStatement + | updateStatement + | deleteStatement + + // Routine Statement + | createRoutineLoadStatement + | alterRoutineLoadStatement + | stopRoutineLoadStatement + | resumeRoutineLoadStatement + | pauseRoutineLoadStatement + | showRoutineLoadStatement + | showRoutineLoadTaskStatement + | showCreateRoutineLoadStatement + + // StreamLoad Statement + | showStreamLoadStatement + + // Admin Statement + | adminSetConfigStatement + | adminSetReplicaStatusStatement + | adminShowConfigStatement + | adminShowReplicaDistributionStatement + | adminShowReplicaStatusStatement + | adminRepairTableStatement + | adminCancelRepairTableStatement + | adminCheckTabletsStatement + | killStatement + | syncStatement + | executeScriptStatement + + // Cluster Management Statement + | alterSystemStatement + | cancelAlterSystemStatement + | showComputeNodesStatement + + // Analyze Statement + | analyzeStatement + | dropStatsStatement + | createAnalyzeStatement + | dropAnalyzeJobStatement + | analyzeHistogramStatement + | dropHistogramStatement + | showAnalyzeStatement + | showStatsMetaStatement + | showHistogramMetaStatement + | killAnalyzeStatement + + // Profile Statement + | analyzeProfileStatement + + // Resource Group Statement + | createResourceGroupStatement + | dropResourceGroupStatement + | alterResourceGroupStatement + | showResourceGroupStatement + + // External Resource Statement + | createResourceStatement + | alterResourceStatement + | dropResourceStatement + | showResourceStatement + + // UDF Statement + | showFunctionsStatement + | dropFunctionStatement + | createFunctionStatement + + // Load Statement + | loadStatement + | showLoadStatement + | showLoadWarningsStatement + | cancelLoadStatement + | alterLoadStatement + + // Show Statement + | showAuthorStatement + | showBackendsStatement + | showBrokerStatement + | showCharsetStatement + | showCollationStatement + | showDeleteStatement + | showDynamicPartitionStatement + | showEventsStatement + | showEnginesStatement + | showFrontendsStatement + | showPluginsStatement + | showRepositoriesStatement + | showOpenTableStatement + | showPrivilegesStatement + | showProcedureStatement + | showProcStatement + | showProcesslistStatement + | showProfilelistStatement + | showRunningQueriesStatement + | showStatusStatement + | showTabletStatement + | showTransactionStatement + | showTriggersStatement + | showUserPropertyStatement + | showVariablesStatement + | showWarningStatement + | helpStatement + + // authz Statement + | createUserStatement + | dropUserStatement + | alterUserStatement + | showUserStatement + | showAuthenticationStatement + | executeAsStatement + | createRoleStatement + | alterRoleStatement + | dropRoleStatement + | showRolesStatement + | grantRoleStatement + | revokeRoleStatement + | setRoleStatement + | setDefaultRoleStatement + | grantPrivilegeStatement + | revokePrivilegeStatement + | showGrantsStatement + | createSecurityIntegrationStatement + | alterSecurityIntegrationStatement + | dropSecurityIntegrationStatement + | showSecurityIntegrationStatement + | showCreateSecurityIntegrationStatement + | createRoleMappingStatement + | alterRoleMappingStatement + | dropRoleMappingStatement + | showRoleMappingStatement + | refreshRoleMappingStatement + + // Security Policy + | createMaskingPolicyStatement + | dropMaskingPolicyStatement + | alterMaskingPolicyStatement + | showMaskingPolicyStatement + | showCreateMaskingPolicyStatement + + | createRowAccessPolicyStatement + | dropRowAccessPolicyStatement + | alterRowAccessPolicyStatement + | showRowAccessPolicyStatement + | showCreateRowAccessPolicyStatement + + // Backup Restore Statement + | backupStatement + | cancelBackupStatement + | showBackupStatement + | restoreStatement + | cancelRestoreStatement + | showRestoreStatement + | showSnapshotStatement + | createRepositoryStatement + | dropRepositoryStatement + + // Sql BlackList And WhiteList Statement + | addSqlBlackListStatement + | delSqlBlackListStatement + | showSqlBlackListStatement + | showWhiteListStatement + + // Export Statement + | exportStatement + | cancelExportStatement + | showExportStatement + + // Plugin Statement + | installPluginStatement + | uninstallPluginStatement + + // File Statement + | createFileStatement + | dropFileStatement + | showSmallFilesStatement + + // Set Statement + | setStatement + | setUserPropertyStatement + + // Storage Volume Statement + | createStorageVolumeStatement + | alterStorageVolumeStatement + | dropStorageVolumeStatement + | showStorageVolumesStatement + | descStorageVolumeStatement + | setDefaultStorageVolumeStatement + + // Pipe Statement + | createPipeStatement + | dropPipeStatement + | alterPipeStatement + | showPipeStatement + | descPipeStatement + + // Compaction Statement + | cancelCompactionStatement + + // FailPoint Statement + | updateFailPointStatusStatement + | showFailPointStatement + + // Unsupported Statement + | unsupportedStatement + ; + +// ---------------------------------------- DataBase Statement --------------------------------------------------------- + +useDatabaseStatement + : USE qualifiedName + ; + +useCatalogStatement + : USE string + ; + +setCatalogStatement + : SET CATALOG identifierOrString + ; + +showDatabasesStatement + : SHOW DATABASES ((FROM | IN) catalog=qualifiedName)? ((LIKE pattern=string) | (WHERE expression))? + | SHOW SCHEMAS ((LIKE pattern=string) | (WHERE expression))? + ; + +alterDbQuotaStatement + : ALTER DATABASE identifier SET DATA QUOTA identifier + | ALTER DATABASE identifier SET REPLICA QUOTA INTEGER_VALUE + ; + +createDbStatement + : CREATE (DATABASE | SCHEMA) (IF NOT EXISTS)? (catalog=identifier '.')? database=identifier charsetDesc? collateDesc? properties? + ; + +dropDbStatement + : DROP (DATABASE | SCHEMA) (IF EXISTS)? (catalog=identifier '.')? database=identifier FORCE? + ; + +showCreateDbStatement + : SHOW CREATE (DATABASE | SCHEMA) identifier + ; + +alterDatabaseRenameStatement + : ALTER DATABASE identifier RENAME identifier + ; + +recoverDbStmt + : RECOVER (DATABASE | SCHEMA) identifier + ; + +showDataStmt + : SHOW DATA + | SHOW DATA FROM qualifiedName + ; + +// ------------------------------------------- Table Statement --------------------------------------------------------- + +createTableStatement + : CREATE EXTERNAL? TABLE (IF NOT EXISTS)? qualifiedName + '(' columnDesc (',' columnDesc)* (',' indexDesc)* ')' + engineDesc? + charsetDesc? + keyDesc? + withRowAccessPolicy* + comment? + partitionDesc? + distributionDesc? + orderByDesc? + rollupDesc? + properties? + extProperties? + ; + +columnDesc + : identifier type charsetName? KEY? aggDesc? (NULL | NOT NULL)? + (defaultDesc | AUTO_INCREMENT | generatedColumnDesc)? + (withMaskingPolicy)? + comment? + ; + +charsetName + : CHAR SET identifier + | CHARSET identifier + | CHARACTER SET identifier + ; + +defaultDesc + : DEFAULT (string | NULL | CURRENT_TIMESTAMP | '(' qualifiedName '(' ')' ')') + ; + +generatedColumnDesc + : AS expression + ; + +indexDesc + : INDEX indexName=identifier identifierList indexType? comment? + ; + +engineDesc + : ENGINE EQ identifier + ; + +charsetDesc + : DEFAULT? (CHAR SET | CHARSET | CHARACTER SET) EQ? identifierOrString + ; + +collateDesc + : DEFAULT? COLLATE EQ? identifierOrString + ; + +keyDesc + : (AGGREGATE | UNIQUE | PRIMARY | DUPLICATE) KEY identifierList + ; + +orderByDesc + : ORDER BY identifierList + ; + +aggDesc + : SUM + | MAX + | MIN + | REPLACE + | HLL_UNION + | BITMAP_UNION + | PERCENTILE_UNION + | REPLACE_IF_NOT_NULL + ; + +rollupDesc + : ROLLUP '(' rollupItem (',' rollupItem)* ')' + ; + +rollupItem + : rollupName=identifier identifierList (dupKeys)? (fromRollup)? properties? + ; + +dupKeys + : DUPLICATE KEY identifierList + ; + +fromRollup + : FROM identifier + ; + +withMaskingPolicy + : WITH MASKING POLICY policyName=qualifiedName (USING identifierList)? + ; + +withRowAccessPolicy + : WITH ROW ACCESS POLICY policyName=qualifiedName (ON identifierList)? + ; + +createTemporaryTableStatement + : CREATE TEMPORARY TABLE qualifiedName + queryStatement + ; + +createTableAsSelectStatement + : CREATE TABLE (IF NOT EXISTS)? qualifiedName + ('(' identifier (',' identifier)* ')')? + keyDesc? + comment? + partitionDesc? + distributionDesc? + properties? + AS queryStatement + ; + +dropTableStatement + : DROP TEMPORARY? TABLE (IF EXISTS)? qualifiedName FORCE? + ; + +alterTableStatement + : ALTER TABLE qualifiedName alterClause (',' alterClause)* + | ALTER TABLE qualifiedName ADD ROLLUP rollupItem (',' rollupItem)* + | ALTER TABLE qualifiedName DROP ROLLUP identifier (',' identifier)* + ; + +createIndexStatement + : CREATE INDEX indexName=identifier + ON qualifiedName identifierList indexType? + comment? + ; + +dropIndexStatement + : DROP INDEX indexName=identifier ON qualifiedName + ; + +indexType + : USING BITMAP + ; + +showTableStatement + : SHOW FULL? TABLES ((FROM | IN) db=qualifiedName)? ((LIKE pattern=string) | (WHERE expression))? + ; + +showCreateTableStatement + : SHOW CREATE (TABLE | VIEW | MATERIALIZED VIEW) table=qualifiedName + ; + +showColumnStatement + : SHOW FULL? (COLUMNS | FIELDS) ((FROM | IN) table=qualifiedName) ((FROM | IN) db=qualifiedName)? + ((LIKE pattern=string) | (WHERE expression))? + ; + +showTableStatusStatement + : SHOW TABLE STATUS ((FROM | IN) db=qualifiedName)? ((LIKE pattern=string) | (WHERE expression))? + ; + +refreshTableStatement + : REFRESH EXTERNAL TABLE qualifiedName (PARTITION '(' string (',' string)* ')')? + ; + +showAlterStatement + : SHOW ALTER TABLE (COLUMN | ROLLUP) ((FROM | IN) db=qualifiedName)? + (WHERE expression)? (ORDER BY sortItem (',' sortItem)*)? (limitElement)? + | SHOW ALTER MATERIALIZED VIEW ((FROM | IN) db=qualifiedName)? + (WHERE expression)? (ORDER BY sortItem (',' sortItem)*)? (limitElement)? + ; + +descTableStatement + : (DESC | DESCRIBE) table=qualifiedName ALL? + ; + +createTableLikeStatement + : CREATE (EXTERNAL)? TABLE (IF NOT EXISTS)? qualifiedName + partitionDesc? + distributionDesc? + properties? + LIKE qualifiedName + ; + +showIndexStatement + : SHOW (INDEX | INDEXES | KEY | KEYS) ((FROM | IN) table=qualifiedName) ((FROM | IN) db=qualifiedName)? + ; + +recoverTableStatement + : RECOVER TABLE qualifiedName + ; + +truncateTableStatement + : TRUNCATE TABLE qualifiedName partitionNames? + ; + +cancelAlterTableStatement + : CANCEL ALTER TABLE (COLUMN | ROLLUP)? FROM qualifiedName ('(' INTEGER_VALUE (',' INTEGER_VALUE)* ')')? + | CANCEL ALTER MATERIALIZED VIEW FROM qualifiedName + ; + +showPartitionsStatement + : SHOW TEMPORARY? PARTITIONS FROM table=qualifiedName + (WHERE expression)? + (ORDER BY sortItem (',' sortItem)*)? limitElement? + ; + +recoverPartitionStatement + : RECOVER PARTITION identifier FROM table=qualifiedName + ; + +// ------------------------------------------- View Statement ---------------------------------------------------------- + +createViewStatement + : CREATE (OR REPLACE)? VIEW (IF NOT EXISTS)? qualifiedName + ('(' columnNameWithComment (',' columnNameWithComment)* ')')? + withRowAccessPolicy* + comment? AS queryStatement + ; + +alterViewStatement + : ALTER VIEW qualifiedName ('(' columnNameWithComment (',' columnNameWithComment)* ')')? AS queryStatement + | ALTER VIEW qualifiedName applyMaskingPolicyClause + | ALTER VIEW qualifiedName applyRowAccessPolicyClause + ; + +dropViewStatement + : DROP VIEW (IF EXISTS)? qualifiedName + ; + +columnNameWithComment + : columnName=identifier withMaskingPolicy? comment? + ; + +// ------------------------------------------- Task Statement ---------------------------------------------------------- + +submitTaskStatement + : SUBMIT setVarHint* TASK qualifiedName? + AS (createTableAsSelectStatement | insertStatement ) + ; + +dropTaskStatement + : DROP TASK qualifiedName + ; + +// ------------------------------------------- Materialized View Statement --------------------------------------------- + +createMaterializedViewStatement + : CREATE MATERIALIZED VIEW (IF NOT EXISTS)? mvName=qualifiedName + ('(' columnNameWithComment (',' columnNameWithComment)* ')')? + withRowAccessPolicy* + comment? + materializedViewDesc* + AS queryStatement + ; + +materializedViewDesc + : (PARTITION BY primaryExpression) + | distributionDesc + | orderByDesc + | refreshSchemeDesc + | properties + ; + +showMaterializedViewsStatement + : SHOW MATERIALIZED VIEWS ((FROM | IN) db=qualifiedName)? ((LIKE pattern=string) | (WHERE expression))? + ; + +dropMaterializedViewStatement + : DROP MATERIALIZED VIEW (IF EXISTS)? mvName=qualifiedName + ; + +alterMaterializedViewStatement + : ALTER MATERIALIZED VIEW mvName=qualifiedName ( + refreshSchemeDesc | + tableRenameClause | + modifyTablePropertiesClause | + swapTableClause ) + | ALTER MATERIALIZED VIEW mvName=qualifiedName statusDesc + | ALTER MATERIALIZED VIEW qualifiedName applyMaskingPolicyClause + | ALTER MATERIALIZED VIEW qualifiedName applyRowAccessPolicyClause + ; + +refreshMaterializedViewStatement + : REFRESH MATERIALIZED VIEW mvName=qualifiedName (PARTITION partitionRangeDesc)? FORCE? (WITH (SYNC | ASYNC) MODE)? + ; + +cancelRefreshMaterializedViewStatement + : CANCEL REFRESH MATERIALIZED VIEW mvName=qualifiedName + ; + +// ------------------------------------------- Admin Statement --------------------------------------------------------- + +adminSetConfigStatement + : ADMIN SET FRONTEND CONFIG '(' property ')' + ; +adminSetReplicaStatusStatement + : ADMIN SET REPLICA STATUS properties + ; +adminShowConfigStatement + : ADMIN SHOW FRONTEND CONFIG (LIKE pattern=string)? + ; + +adminShowReplicaDistributionStatement + : ADMIN SHOW REPLICA DISTRIBUTION FROM qualifiedName partitionNames? + ; + +adminShowReplicaStatusStatement + : ADMIN SHOW REPLICA STATUS FROM qualifiedName partitionNames? (WHERE where=expression)? + ; + +adminRepairTableStatement + : ADMIN REPAIR TABLE qualifiedName partitionNames? + ; + +adminCancelRepairTableStatement + : ADMIN CANCEL REPAIR TABLE qualifiedName partitionNames? + ; + +adminCheckTabletsStatement + : ADMIN CHECK tabletList PROPERTIES '('property')' + ; + +killStatement + : KILL (CONNECTION? | QUERY) INTEGER_VALUE + ; + +syncStatement + : SYNC + ; + +// ------------------------------------------- Cluster Management Statement --------------------------------------------- + +alterSystemStatement + : ALTER SYSTEM alterClause + ; + +cancelAlterSystemStatement + : CANCEL DECOMMISSION BACKEND string (',' string)* + ; + +showComputeNodesStatement + : SHOW COMPUTE NODES + ; + +// ------------------------------------------- Catalog Statement ------------------------------------------------------- + +createExternalCatalogStatement + : CREATE EXTERNAL CATALOG catalogName=identifierOrString comment? properties + ; + +showCreateExternalCatalogStatement + : SHOW CREATE CATALOG catalogName=identifierOrString + ; + +dropExternalCatalogStatement + : DROP CATALOG catalogName=identifierOrString + ; + +showCatalogsStatement + : SHOW CATALOGS + ; + + +// ---------------------------------------- Warehouse Statement --------------------------------------------------------- + +createWarehouseStatement + : CREATE (WAREHOUSE) (IF NOT EXISTS)? warehouseName=identifierOrString + properties? + ; + +showWarehousesStatement + : SHOW WAREHOUSES ((LIKE pattern=string) | (WHERE expression))? + ; + +dropWarehouseStatement + : DROP WAREHOUSE (IF EXISTS)? warehouseName=identifierOrString + ; + +alterWarehouseStatement + : ALTER WAREHOUSE identifier ADD CLUSTER + | ALTER WAREHOUSE identifier REMOVE CLUSTER + | ALTER WAREHOUSE identifier SET propertyList + ; + +showClustersStatement + : SHOW CLUSTERS FROM WAREHOUSE identifier + ; + +suspendWarehouseStatement + : SUSPEND WAREHOUSE (IF EXISTS)? identifier + ; + +resumeWarehouseStatement + : RESUME WAREHOUSE (IF EXISTS)? identifier + ; + +// ---------------------------------------- Storage Volume Statement --------------------------------------------------- + +createStorageVolumeStatement + : CREATE STORAGE VOLUME (IF NOT EXISTS)? storageVolumeName=identifierOrString typeDesc locationsDesc + comment? properties + ; + +typeDesc + : TYPE EQ identifier + ; + +locationsDesc + : LOCATIONS EQ stringList + ; + +showStorageVolumesStatement + : SHOW STORAGE VOLUMES (LIKE pattern=string)? + ; + +dropStorageVolumeStatement + : DROP STORAGE VOLUME (IF EXISTS)? storageVolumeName=identifierOrString + ; + +alterStorageVolumeStatement + : ALTER STORAGE VOLUME identifierOrString alterStorageVolumeClause (',' alterStorageVolumeClause)* + ; + +alterStorageVolumeClause + : modifyStorageVolumeCommentClause + | modifyStorageVolumePropertiesClause + ; + +modifyStorageVolumePropertiesClause + : SET propertyList + ; + +modifyStorageVolumeCommentClause + : COMMENT '=' string + ; + +descStorageVolumeStatement + : (DESC | DESCRIBE) STORAGE VOLUME identifierOrString + ; + +setDefaultStorageVolumeStatement + : SET identifierOrString AS DEFAULT STORAGE VOLUME + ; + +// ------------------------------------------- FailPoint Statement ----------------------------------------------------- + +updateFailPointStatusStatement + : ADMIN DISABLE FAILPOINT string (ON BACKEND string)? + | ADMIN ENABLE FAILPOINT string (WITH INTEGER_VALUE TIMES)? (ON BACKEND string)? + | ADMIN ENABLE FAILPOINT string (WITH DECIMAL_VALUE PROBABILITY)? (ON BACKEND string)? + ; + +showFailPointStatement + : SHOW FAILPOINTS ((LIKE pattern=string))? (ON BACKEND string)? + ; + +// ------------------------------------------- Alter Clause ------------------------------------------------------------ + +alterClause + //Alter system clause + : addFrontendClause + | dropFrontendClause + | modifyFrontendHostClause + | addBackendClause + | dropBackendClause + | decommissionBackendClause + | modifyBackendHostClause + | addComputeNodeClause + | dropComputeNodeClause + | modifyBrokerClause + | alterLoadErrorUrlClause + | createImageClause + | cleanTabletSchedQClause + + //Alter table clause + | createIndexClause + | dropIndexClause + | tableRenameClause + | swapTableClause + | modifyTablePropertiesClause + | addColumnClause + | addColumnsClause + | dropColumnClause + | modifyColumnClause + | columnRenameClause + | reorderColumnsClause + | rollupRenameClause + | compactionClause + | modifyCommentClause + + //Apply Policy clause + | applyMaskingPolicyClause + | applyRowAccessPolicyClause + + //Alter partition clause + | addPartitionClause + | dropPartitionClause + | distributionClause + | truncatePartitionClause + | modifyPartitionClause + | replacePartitionClause + | partitionRenameClause + ; + +// ---------Alter system clause--------- + +addFrontendClause + : ADD (FOLLOWER | OBSERVER) string + ; + +dropFrontendClause + : DROP (FOLLOWER | OBSERVER) string + ; + +modifyFrontendHostClause + : MODIFY FRONTEND HOST string TO string + ; + +addBackendClause + : ADD BACKEND string (',' string)* + ; + +dropBackendClause + : DROP BACKEND string (',' string)* FORCE? + ; + +decommissionBackendClause + : DECOMMISSION BACKEND string (',' string)* + ; + +modifyBackendHostClause + : MODIFY BACKEND HOST string TO string + ; + +addComputeNodeClause + : ADD COMPUTE NODE string (',' string)* + ; + +dropComputeNodeClause + : DROP COMPUTE NODE string (',' string)* + ; + +modifyBrokerClause + : ADD BROKER identifierOrString string (',' string)* + | DROP BROKER identifierOrString string (',' string)* + | DROP ALL BROKER identifierOrString + ; + +alterLoadErrorUrlClause + : SET LOAD ERRORS HUB properties? + ; + +createImageClause + : CREATE IMAGE + ; + +cleanTabletSchedQClause + : CLEAN TABLET SCHEDULER QUEUE + ; + +// ---------Alter table clause--------- + +createIndexClause + : ADD INDEX indexName=identifier identifierList indexType? comment? + ; + +dropIndexClause + : DROP INDEX indexName=identifier + ; + +tableRenameClause + : RENAME identifier + ; + +swapTableClause + : SWAP WITH identifier + ; + +modifyTablePropertiesClause + : SET propertyList + ; + +modifyCommentClause + : COMMENT '=' string + ; + +addColumnClause + : ADD COLUMN columnDesc (FIRST | AFTER identifier)? ((TO | IN) rollupName=identifier)? properties? + ; + +addColumnsClause + : ADD COLUMN '(' columnDesc (',' columnDesc)* ')' ((TO | IN) rollupName=identifier)? properties? + ; + +dropColumnClause + : DROP COLUMN identifier (FROM rollupName=identifier)? properties? + ; + +modifyColumnClause + : MODIFY COLUMN columnDesc (FIRST | AFTER identifier)? (FROM rollupName=identifier)? properties? + ; + +columnRenameClause + : RENAME COLUMN oldColumn=identifier newColumn=identifier + ; + +reorderColumnsClause + : ORDER BY identifierList (FROM rollupName=identifier)? properties? + ; + +rollupRenameClause + : RENAME ROLLUP rollupName=identifier newRollupName=identifier + ; + +compactionClause + : (BASE | CUMULATIVE)? COMPACT (identifier | identifierList)? + ; + +applyMaskingPolicyClause + : MODIFY COLUMN columnName=identifier SET MASKING POLICY policyName=qualifiedName (USING identifierList)? + | MODIFY COLUMN columnName=identifier UNSET MASKING POLICY + ; + +applyRowAccessPolicyClause + : ADD ROW ACCESS POLICY policyName=qualifiedName (ON identifierList)? + | DROP ROW ACCESS POLICY policyName=qualifiedName + | DROP ALL ROW ACCESS POLICIES + ; + +// ---------Alter partition clause--------- + +addPartitionClause + : ADD TEMPORARY? (singleRangePartition | PARTITIONS multiRangePartition) distributionDesc? properties? + | ADD TEMPORARY? (singleItemListPartitionDesc | multiItemListPartitionDesc) distributionDesc? properties? + ; + +dropPartitionClause + : DROP TEMPORARY? PARTITION (IF EXISTS)? identifier FORCE? + ; + +truncatePartitionClause + : TRUNCATE partitionNames + ; + +modifyPartitionClause + : MODIFY PARTITION (identifier | identifierList | '(' ASTERISK_SYMBOL ')') SET propertyList + | MODIFY PARTITION distributionDesc + ; + +replacePartitionClause + : REPLACE parName=partitionNames WITH tempParName=partitionNames properties? + ; + +partitionRenameClause + : RENAME PARTITION parName=identifier newParName=identifier + ; + +// ------------------------------------------- DML Statement ----------------------------------------------------------- + +insertStatement + : explainDesc? INSERT (INTO | OVERWRITE) qualifiedName partitionNames? + (WITH LABEL label=identifier)? columnAliases? + (queryStatement | (VALUES expressionsWithDefault (',' expressionsWithDefault)*)) + ; + +updateStatement + : explainDesc? withClause? UPDATE qualifiedName SET assignmentList fromClause (WHERE where=expression)? + ; + +deleteStatement + : explainDesc? withClause? DELETE FROM qualifiedName partitionNames? (USING using=relations)? (WHERE where=expression)? + ; + +// ------------------------------------------- Routine Statement ----------------------------------------------------------- +createRoutineLoadStatement + : CREATE ROUTINE LOAD (db=qualifiedName '.')? name=identifier ON table=qualifiedName + (loadProperties (',' loadProperties)*)? + jobProperties? + FROM source=identifier + dataSourceProperties? + ; + +alterRoutineLoadStatement + : ALTER ROUTINE LOAD FOR (db=qualifiedName '.')? name=identifier + (loadProperties (',' loadProperties)*)? + jobProperties? + dataSource? + ; + +dataSource + : FROM source=identifier dataSourceProperties + ; + +loadProperties + : colSeparatorProperty + | rowDelimiterProperty + | importColumns + | WHERE expression + | partitionNames + ; + +colSeparatorProperty + : COLUMNS TERMINATED BY string + ; + +rowDelimiterProperty + : ROWS TERMINATED BY string + ; + +importColumns + : COLUMNS columnProperties + ; + +columnProperties + : '(' + (qualifiedName | assignment) (',' (qualifiedName | assignment))* + ')' + ; + +jobProperties + : properties + ; + +dataSourceProperties + : propertyList + ; + +stopRoutineLoadStatement + : STOP ROUTINE LOAD FOR (db=qualifiedName '.')? name=identifier + ; + +resumeRoutineLoadStatement + : RESUME ROUTINE LOAD FOR (db=qualifiedName '.')? name=identifier + ; + +pauseRoutineLoadStatement + : PAUSE ROUTINE LOAD FOR (db=qualifiedName '.')? name=identifier + ; + +showRoutineLoadStatement + : SHOW ALL? ROUTINE LOAD (FOR (db=qualifiedName '.')? name=identifier)? + (FROM db=qualifiedName)? + (WHERE expression)? (ORDER BY sortItem (',' sortItem)*)? (limitElement)? + ; + +showRoutineLoadTaskStatement + : SHOW ROUTINE LOAD TASK + (FROM db=qualifiedName)? + WHERE expression + ; + +showCreateRoutineLoadStatement + : SHOW CREATE ROUTINE LOAD (db=qualifiedName '.')? name=identifier + ; + +showStreamLoadStatement + : SHOW ALL? STREAM LOAD (FOR (db=qualifiedName '.')? name=identifier)? + (FROM db=qualifiedName)? + (WHERE expression)? (ORDER BY sortItem (',' sortItem)*)? (limitElement)? + ; +// ------------------------------------------- Analyze Statement ------------------------------------------------------- + +analyzeStatement + : ANALYZE (FULL | SAMPLE)? TABLE qualifiedName ('(' identifier (',' identifier)* ')')? + (WITH (SYNC | ASYNC) MODE)? + properties? + ; + +dropStatsStatement + : DROP STATS qualifiedName + ; + +analyzeHistogramStatement + : ANALYZE TABLE qualifiedName UPDATE HISTOGRAM ON identifier (',' identifier)* + (WITH (SYNC | ASYNC) MODE)? + (WITH bucket=INTEGER_VALUE BUCKETS)? + properties? + ; + +dropHistogramStatement + : ANALYZE TABLE qualifiedName DROP HISTOGRAM ON identifier (',' identifier)* + ; + +createAnalyzeStatement + : CREATE ANALYZE (FULL | SAMPLE)? ALL properties? + | CREATE ANALYZE (FULL | SAMPLE)? DATABASE db=identifier properties? + | CREATE ANALYZE (FULL | SAMPLE)? TABLE qualifiedName ('(' identifier (',' identifier)* ')')? properties? + ; + +dropAnalyzeJobStatement + : DROP ANALYZE INTEGER_VALUE + ; + +showAnalyzeStatement + : SHOW ANALYZE (JOB | STATUS)? (WHERE expression)? + ; + +showStatsMetaStatement + : SHOW STATS META (WHERE expression)? + ; + +showHistogramMetaStatement + : SHOW HISTOGRAM META (WHERE expression)? + ; + +killAnalyzeStatement + : KILL ANALYZE INTEGER_VALUE + ; + +// ----------------------------------------- Analyze Profile Statement ------------------------------------------------- + +analyzeProfileStatement + : ANALYZE PROFILE FROM string + | ANALYZE PROFILE FROM string ',' INTEGER_VALUE (',' INTEGER_VALUE)* + ; + +// ------------------------------------------- Work Group Statement ---------------------------------------------------- + +createResourceGroupStatement + : CREATE RESOURCE GROUP (IF NOT EXISTS)? (OR REPLACE)? identifier + (TO classifier (',' classifier)*)? WITH '(' property (',' property)* ')' + ; + +dropResourceGroupStatement + : DROP RESOURCE GROUP identifier + ; + +alterResourceGroupStatement + : ALTER RESOURCE GROUP identifier ADD classifier (',' classifier)* + | ALTER RESOURCE GROUP identifier DROP '(' INTEGER_VALUE (',' INTEGER_VALUE)* ')' + | ALTER RESOURCE GROUP identifier DROP ALL + | ALTER RESOURCE GROUP identifier WITH '(' property (',' property)* ')' + ; + +showResourceGroupStatement + : SHOW RESOURCE GROUP identifier + | SHOW RESOURCE GROUPS ALL? + ; + +createResourceStatement + : CREATE EXTERNAL? RESOURCE resourceName=identifierOrString properties? + ; + +alterResourceStatement + : ALTER RESOURCE resourceName=identifierOrString SET properties + ; + +dropResourceStatement + : DROP RESOURCE resourceName=identifierOrString + ; + +showResourceStatement + : SHOW RESOURCES + ; + +classifier + : '(' expressionList ')' + ; + +// ------------------------------------------- UDF Statement ---------------------------------------------------- + +showFunctionsStatement + : SHOW FULL? (BUILTIN|GLOBAL)? FUNCTIONS ((FROM | IN) db=qualifiedName)? ((LIKE pattern=string) | (WHERE expression))? + ; + +dropFunctionStatement + : DROP GLOBAL? FUNCTION qualifiedName '(' typeList ')' + ; + +createFunctionStatement + : CREATE GLOBAL? functionType=(TABLE | AGGREGATE)? FUNCTION qualifiedName '(' typeList ')' RETURNS returnType=type (INTERMEDIATE intermediateType = type)? properties? + ; + +typeList + : type? ( ',' type)* (',' DOTDOTDOT) ? + ; + +// ------------------------------------------- Load Statement ---------------------------------------------------------- + +loadStatement + : LOAD LABEL label=labelName + data=dataDescList? + broker=brokerDesc? + (BY system=identifierOrString)? + (PROPERTIES props=propertyList)? + | LOAD LABEL label=labelName + data=dataDescList? + resource=resourceDesc + (PROPERTIES props=propertyList)? + ; + +labelName + : (db=identifier '.')? label=identifier + ; + +dataDescList + : '(' dataDesc (',' dataDesc)* ')' + ; + +dataDesc + : DATA INFILE srcFiles=stringList + NEGATIVE? + INTO TABLE dstTableName=identifier + partitions=partitionNames? + (COLUMNS TERMINATED BY colSep=string)? + (ROWS TERMINATED BY rowSep=string)? + format=fileFormat? + (formatPropsField=formatProps)? + colList=columnAliases? + (COLUMNS FROM PATH AS colFromPath=identifierList)? + (SET colMappingList=classifier)? + (WHERE where=expression)? + | DATA FROM TABLE srcTableName=identifier + NEGATIVE? + INTO TABLE dstTableName=identifier + partitions=partitionNames? + (SET colMappingList=classifier)? + (WHERE where=expression)? + ; + +formatProps + : '(' + (SKIP_HEADER '=' INTEGER_VALUE)? + (TRIM_SPACE '=' booleanValue)? + (ENCLOSE '=' encloseCharacter=string)? + (ESCAPE '=' escapeCharacter=string)? + ')' + ; + +brokerDesc + : WITH BROKER props=propertyList? + | WITH BROKER name=identifierOrString props=propertyList? + ; + +resourceDesc + : WITH RESOURCE name=identifierOrString props=propertyList? + ; + +showLoadStatement + : SHOW LOAD (ALL)? (FROM identifier)? (WHERE expression)? (ORDER BY sortItem (',' sortItem)*)? limitElement? + ; + +showLoadWarningsStatement + : SHOW LOAD WARNINGS (FROM identifier)? (WHERE expression)? limitElement? + | SHOW LOAD WARNINGS ON string + ; + +cancelLoadStatement + : CANCEL LOAD (FROM identifier)? (WHERE expression)? + ; + +alterLoadStatement + : ALTER LOAD FOR (db=qualifiedName '.')? name=identifier + jobProperties? + ; + +// ------------------------------------------- Compaction Statement ---------------------------------------------------------- + +cancelCompactionStatement + : CANCEL COMPACTION WHERE expression + ; + +// ------------------------------------------- Show Statement ---------------------------------------------------------- + +showAuthorStatement + : SHOW AUTHORS + ; + +showBackendsStatement + : SHOW BACKENDS + ; + +showBrokerStatement + : SHOW BROKER + ; + +showCharsetStatement + : SHOW (CHAR SET | CHARSET | CHARACTER SET) ((LIKE pattern=string) | (WHERE expression))? + ; + +showCollationStatement + : SHOW COLLATION ((LIKE pattern=string) | (WHERE expression))? + ; + +showDeleteStatement + : SHOW DELETE ((FROM | IN) db=qualifiedName)? + ; + +showDynamicPartitionStatement + : SHOW DYNAMIC PARTITION TABLES ((FROM | IN) db=qualifiedName)? + ; + +showEventsStatement + : SHOW EVENTS ((FROM | IN) catalog=qualifiedName)? ((LIKE pattern=string) | (WHERE expression))? + ; + +showEnginesStatement + : SHOW ENGINES + ; + +showFrontendsStatement + : SHOW FRONTENDS + ; + +showPluginsStatement + : SHOW PLUGINS + ; + +showRepositoriesStatement + : SHOW REPOSITORIES + ; + +showOpenTableStatement + : SHOW OPEN TABLES + ; +showPrivilegesStatement + : SHOW PRIVILEGES + ; + +showProcedureStatement + : SHOW (PROCEDURE | FUNCTION) STATUS ((LIKE pattern=string) | (WHERE where=expression))? + ; + +showProcStatement + : SHOW PROC path=string + ; + +showProcesslistStatement + : SHOW FULL? PROCESSLIST + ; + +showProfilelistStatement + : SHOW PROFILELIST (LIMIT limit =INTEGER_VALUE)? + ; + +showRunningQueriesStatement + : SHOW RUNNING QUERIES (LIMIT limit =INTEGER_VALUE)? + ; + +showStatusStatement + : SHOW varType? STATUS ((LIKE pattern=string) | (WHERE expression))? + ; + +showTabletStatement + : SHOW TABLET INTEGER_VALUE + | SHOW TABLET FROM qualifiedName partitionNames? (WHERE expression)? (ORDER BY sortItem (',' sortItem)*)? (limitElement)? + ; + +showTransactionStatement + : SHOW TRANSACTION ((FROM | IN) db=qualifiedName)? (WHERE expression)? + ; + +showTriggersStatement + : SHOW FULL? TRIGGERS ((FROM | IN) catalog=qualifiedName)? ((LIKE pattern=string) | (WHERE expression))? + ; + +showUserPropertyStatement + : SHOW PROPERTY (FOR string)? (LIKE string)? + ; + +showVariablesStatement + : SHOW varType? VARIABLES ((LIKE pattern=string) | (WHERE expression))? + ; + +showWarningStatement + : SHOW (WARNINGS | ERRORS) (limitElement)? + ; + +helpStatement + : HELP identifierOrString + ; + +// ------------------------------------------- Authz Statement ----------------------------------------------------- + +createUserStatement + : CREATE USER (IF NOT EXISTS)? user authOption? (DEFAULT ROLE roleList)? + ; + +dropUserStatement + : DROP USER (IF EXISTS)? user + ; + +alterUserStatement + : ALTER USER (IF EXISTS)? user authOption + | ALTER USER (IF EXISTS)? user DEFAULT ROLE (NONE| ALL | roleList) + ; + +showUserStatement + : SHOW (USER | USERS) + ; + +showAuthenticationStatement + : SHOW ALL AUTHENTICATION #showAllAuthentication + | SHOW AUTHENTICATION (FOR user)? #showAuthenticationForUser + ; + +executeAsStatement + : EXECUTE AS user (WITH NO REVERT)? + ; + +createRoleStatement + : CREATE ROLE (IF NOT EXISTS)? roleList comment? + ; + +alterRoleStatement + : ALTER ROLE (IF EXISTS)? roleList SET COMMENT '=' string + ; + +dropRoleStatement + : DROP ROLE (IF EXISTS)? roleList + ; + +showRolesStatement + : SHOW ROLES + ; + +grantRoleStatement + : GRANT identifierOrStringList TO USER? user #grantRoleToUser + | GRANT identifierOrStringList TO ROLE identifierOrString #grantRoleToRole + ; + +revokeRoleStatement + : REVOKE identifierOrStringList FROM USER? user #revokeRoleFromUser + | REVOKE identifierOrStringList FROM ROLE identifierOrString #revokeRoleFromRole + ; + +setRoleStatement + : SET ROLE DEFAULT + | SET ROLE NONE + | SET ROLE ALL (EXCEPT roleList)? + | SET ROLE roleList + ; + +setDefaultRoleStatement + : SET DEFAULT ROLE (NONE | ALL | roleList) TO user; + +grantRevokeClause + : (USER? user | ROLE identifierOrString) + ; + +grantPrivilegeStatement + : GRANT IMPERSONATE ON USER user (',' user)* TO grantRevokeClause (WITH GRANT OPTION)? #grantOnUser + | GRANT privilegeTypeList ON privObjectNameList TO grantRevokeClause (WITH GRANT OPTION)? #grantOnTableBrief + + | GRANT privilegeTypeList ON GLOBAL? FUNCTION privFunctionObjectNameList + TO grantRevokeClause (WITH GRANT OPTION)? #grantOnFunc + | GRANT privilegeTypeList ON SYSTEM TO grantRevokeClause (WITH GRANT OPTION)? #grantOnSystem + | GRANT privilegeTypeList ON privObjectType privObjectNameList + TO grantRevokeClause (WITH GRANT OPTION)? #grantOnPrimaryObj + | GRANT privilegeTypeList ON ALL privObjectTypePlural + (IN isAll=ALL DATABASES| IN DATABASE identifierOrString)? TO grantRevokeClause + (WITH GRANT OPTION)? #grantOnAll + ; + +revokePrivilegeStatement + : REVOKE IMPERSONATE ON USER user (',' user)* FROM grantRevokeClause #revokeOnUser + | REVOKE privilegeTypeList ON privObjectNameList FROM grantRevokeClause #revokeOnTableBrief + | REVOKE privilegeTypeList ON GLOBAL? FUNCTION privFunctionObjectNameList + FROM grantRevokeClause #revokeOnFunc + | REVOKE privilegeTypeList ON SYSTEM FROM grantRevokeClause #revokeOnSystem + | REVOKE privilegeTypeList ON privObjectType privObjectNameList + FROM grantRevokeClause #revokeOnPrimaryObj + | REVOKE privilegeTypeList ON ALL privObjectTypePlural + (IN isAll=ALL DATABASES| IN DATABASE identifierOrString)? FROM grantRevokeClause #revokeOnAll + ; + +showGrantsStatement + : SHOW GRANTS + | SHOW GRANTS FOR USER? user + | SHOW GRANTS FOR ROLE identifierOrString + ; + +createSecurityIntegrationStatement + : CREATE SECURITY INTEGRATION identifier properties + ; + +alterSecurityIntegrationStatement + : ALTER SECURITY INTEGRATION identifier SET propertyList + ; + +dropSecurityIntegrationStatement + : DROP SECURITY INTEGRATION identifier + ; + +showSecurityIntegrationStatement + : SHOW SECURITY INTEGRATIONS + ; + +showCreateSecurityIntegrationStatement + : SHOW CREATE SECURITY INTEGRATION identifier + ; + +createRoleMappingStatement + : CREATE ROLE MAPPING identifier properties + ; + +alterRoleMappingStatement + : ALTER ROLE MAPPING identifier SET propertyList + ; + +dropRoleMappingStatement + : DROP ROLE MAPPING identifier + ; + +showRoleMappingStatement + : SHOW ROLE MAPPINGS + ; + +refreshRoleMappingStatement + : REFRESH ALL ROLE MAPPINGS + ; + +authOption + : IDENTIFIED BY PASSWORD? string #authWithoutPlugin + | IDENTIFIED WITH identifierOrString ((BY | AS) string)? #authWithPlugin + ; + +privObjectName + : identifierOrStringOrStar ('.' identifierOrStringOrStar)? + ; + +privObjectNameList + : privObjectName (',' privObjectName)* + ; + +privFunctionObjectNameList + : qualifiedName '(' typeList ')' (',' qualifiedName '(' typeList ')')* + ; + +privilegeTypeList + : privilegeType (',' privilegeType)* + ; + +privilegeType + : ALL PRIVILEGES? + | ALTER | APPLY | BLACKLIST + | CREATE ( + DATABASE| TABLE| VIEW| FUNCTION| GLOBAL FUNCTION| MATERIALIZED VIEW| RESOURCE| RESOURCE GROUP| EXTERNAL CATALOG | POLICY | STORAGE VOLUME) + | DELETE | DROP | EXPORT | FILE | IMPERSONATE | INSERT | GRANT | NODE | OPERATE + | PLUGIN | REPOSITORY| REFRESH | SELECT | UPDATE | USAGE + ; + +privObjectType + : CATALOG | DATABASE | MATERIALIZED VIEW | POLICY | RESOURCE | RESOURCE GROUP| STORAGE VOLUME | SYSTEM | TABLE| VIEW + ; + +privObjectTypePlural + : CATALOGS | DATABASES | FUNCTIONS | GLOBAL FUNCTIONS | MATERIALIZED VIEWS | POLICIES | RESOURCES | RESOURCE GROUPS + | STORAGE VOLUMES | TABLES | USERS | VIEWS + ; + +// ---------------------------------------- Security Policy Statement --------------------------------------------------- + +createMaskingPolicyStatement + : CREATE (OR REPLACE)? MASKING POLICY (IF NOT EXISTS)? policyName=qualifiedName + AS '(' policySignature (',' policySignature)* ')' RETURNS type ARROW expression comment? + ; + +dropMaskingPolicyStatement + : DROP MASKING POLICY (IF EXISTS)? policyName=qualifiedName FORCE? + ; + +alterMaskingPolicyStatement + : ALTER MASKING POLICY (IF EXISTS)? policyName=qualifiedName SET BODY ARROW expression + | ALTER MASKING POLICY (IF EXISTS)? policyName=qualifiedName SET COMMENT '=' string + | ALTER MASKING POLICY (IF EXISTS)? policyName=qualifiedName RENAME TO newPolicyName=identifier + ; + +showMaskingPolicyStatement + : SHOW MASKING POLICIES ((FROM | IN) db=qualifiedName)? + ; + +showCreateMaskingPolicyStatement + : SHOW CREATE MASKING POLICY policyName=qualifiedName + ; + +createRowAccessPolicyStatement + : CREATE (OR REPLACE)? ROW ACCESS POLICY (IF NOT EXISTS)? policyName=qualifiedName + AS '(' policySignature (',' policySignature)* ')' RETURNS BOOLEAN ARROW expression comment? + ; + +dropRowAccessPolicyStatement + : DROP ROW ACCESS POLICY (IF EXISTS)? policyName=qualifiedName FORCE? + ; + +alterRowAccessPolicyStatement + : ALTER ROW ACCESS POLICY (IF EXISTS)? policyName=qualifiedName SET BODY ARROW expression + | ALTER ROW ACCESS POLICY (IF EXISTS)? policyName=qualifiedName SET COMMENT '=' string + | ALTER ROW ACCESS POLICY (IF EXISTS)? policyName=qualifiedName RENAME TO newPolicyName=identifier + ; + +showRowAccessPolicyStatement + : SHOW ROW ACCESS POLICIES ((FROM | IN) db=qualifiedName)? + ; + +showCreateRowAccessPolicyStatement + : SHOW CREATE ROW ACCESS POLICY policyName=qualifiedName + ; + +policySignature : identifier type; + +// ---------------------------------------- Backup Restore Statement --------------------------------------------------- + +backupStatement + : BACKUP SNAPSHOT qualifiedName + TO identifier + (ON '(' tableDesc (',' tableDesc) * ')')? + (PROPERTIES propertyList)? + ; + +cancelBackupStatement + : CANCEL BACKUP ((FROM | IN) identifier)? + ; + +showBackupStatement + : SHOW BACKUP ((FROM | IN) identifier)? + ; + +restoreStatement + : RESTORE SNAPSHOT qualifiedName + FROM identifier + (ON '(' restoreTableDesc (',' restoreTableDesc) * ')')? + (PROPERTIES propertyList)? + ; + +cancelRestoreStatement + : CANCEL RESTORE ((FROM | IN) identifier)? + ; + +showRestoreStatement + : SHOW RESTORE ((FROM | IN) identifier)? (WHERE where=expression)? + ; + +showSnapshotStatement + : SHOW SNAPSHOT ON identifier + (WHERE expression)? + ; + +createRepositoryStatement + : CREATE (READ ONLY)? REPOSITORY repoName=identifier + WITH BROKER brokerName=identifierOrString? + ON LOCATION location=string + (PROPERTIES propertyList)? + ; + +dropRepositoryStatement + : DROP REPOSITORY identifier + ; + +// ------------------------------------ Sql BlackList And WhiteList Statement ------------------------------------------ + +addSqlBlackListStatement + : ADD SQLBLACKLIST string + ; + +delSqlBlackListStatement + : DELETE SQLBLACKLIST INTEGER_VALUE (',' INTEGER_VALUE)* + ; + +showSqlBlackListStatement + : SHOW SQLBLACKLIST + ; + +showWhiteListStatement + : SHOW WHITELIST + ; + +// ------------------------------------------- Export Statement -------------------------------------------------------- + +exportStatement + : EXPORT TABLE tableDesc columnAliases? TO string properties? brokerDesc? + ; + +cancelExportStatement + : CANCEL EXPORT ((FROM | IN) catalog=qualifiedName)? ((LIKE pattern=string) | (WHERE expression))? + ; + +showExportStatement + : SHOW EXPORT ((FROM | IN) catalog=qualifiedName)? + ((LIKE pattern=string) | (WHERE expression))? + (ORDER BY sortItem (',' sortItem)*)? (limitElement)? + ; + +// ------------------------------------------- Plugin Statement -------------------------------------------------------- + +installPluginStatement + : INSTALL PLUGIN FROM identifierOrString properties? + ; + +uninstallPluginStatement + : UNINSTALL PLUGIN identifierOrString + ; + +// ------------------------------------------- File Statement ---------------------------------------------------------- + +createFileStatement + : CREATE FILE string ((FROM | IN) catalog=qualifiedName)? properties + ; + +dropFileStatement + : DROP FILE string ((FROM | IN) catalog=qualifiedName)? properties + ; + +showSmallFilesStatement + : SHOW FILE ((FROM | IN) catalog=qualifiedName)? + ; + +// -------------------------------------------- Pipe Statement --------------------------------------------------------- + +createPipeStatement + : CREATE PIPE (IF NOT EXISTS)? qualifiedName + properties? + AS insertStatement + ; + +dropPipeStatement + : DROP PIPE (IF EXISTS)? qualifiedName + ; + +alterPipeClause + : SUSPEND | + RESUME | + RETRY ALL | + RETRY FILE fileName=string | + SET propertyList + ; + +alterPipeStatement + : ALTER PIPE qualifiedName alterPipeClause + ; + +descPipeStatement + : (DESC | DESCRIBE) PIPE qualifiedName + ; + +showPipeStatement + : SHOW PIPES ((LIKE pattern=string) | (WHERE expression) | (FROM qualifiedName))? + (ORDER BY sortItem (',' sortItem)*)? limitElement? + ; + + +// ------------------------------------------- Set Statement ----------------------------------------------------------- + +setStatement + : SET setVar (',' setVar)* + ; + +setVar + : (CHAR SET | CHARSET | CHARACTER SET) (identifierOrString | DEFAULT) #setNames + | NAMES (charset = identifierOrString | DEFAULT) + (COLLATE (collate = identifierOrString | DEFAULT))? #setNames + | PASSWORD '=' (string | PASSWORD '(' string ')') #setPassword + | PASSWORD FOR user '=' (string | PASSWORD '(' string ')') #setPassword + | userVariable '=' expression #setUserVar + | varType? identifier '=' setExprOrDefault #setSystemVar + | systemVariable '=' setExprOrDefault #setSystemVar + | varType? TRANSACTION transaction_characteristics #setTransaction + ; + +transaction_characteristics + : transaction_access_mode + | isolation_level + | transaction_access_mode ',' isolation_level + | isolation_level ',' transaction_access_mode + ; + +transaction_access_mode + : READ ONLY + | READ WRITE + ; + +isolation_level + : ISOLATION LEVEL isolation_types + ; + +isolation_types + : READ UNCOMMITTED + | READ COMMITTED + | REPEATABLE READ + | SERIALIZABLE + ; + +setExprOrDefault + : DEFAULT + | ON + | ALL + | expression + ; + +setUserPropertyStatement + : SET PROPERTY (FOR string)? userPropertyList + ; + +roleList + : identifierOrString (',' identifierOrString)* + ; + +executeScriptStatement + : ADMIN EXECUTE ON (FRONTEND | INTEGER_VALUE) string + ; + +unsupportedStatement + : START TRANSACTION (WITH CONSISTENT SNAPSHOT)? + | BEGIN WORK? + | COMMIT WORK? (AND NO? CHAIN)? (NO? RELEASE)? + | ROLLBACK WORK? (AND NO? CHAIN)? (NO? RELEASE)? + | LOCK TABLES lock_item (',' lock_item)* + | UNLOCK TABLES + ; + +lock_item + : identifier (AS? alias=identifier)? lock_type + ; + +lock_type + : READ LOCAL? + | LOW_PRIORITY? WRITE + ; + +// ------------------------------------------- Query Statement --------------------------------------------------------- + +queryStatement + : (explainDesc | optimizerTrace) ? queryRelation outfile?; + +queryRelation + : withClause? queryNoWith + ; + +withClause + : WITH commonTableExpression (',' commonTableExpression)* + ; + +queryNoWith + : queryPrimary (ORDER BY sortItem (',' sortItem)*)? (limitElement)? + ; + +temporalClause + : AS OF expression + | FOR SYSTEM_TIME AS OF TIMESTAMP string + | FOR SYSTEM_TIME BETWEEN expression AND expression + | FOR SYSTEM_TIME FROM expression TO expression + | FOR SYSTEM_TIME ALL + ; + +queryPrimary + : querySpecification #queryPrimaryDefault + | subquery #queryWithParentheses + | left=queryPrimary operator=INTERSECT setQuantifier? right=queryPrimary #setOperation + | left=queryPrimary operator=(UNION | EXCEPT | MINUS) + setQuantifier? right=queryPrimary #setOperation + ; + +subquery + : '(' queryRelation ')' + ; + +rowConstructor + :'(' expressionList ')' + ; + +sortItem + : expression ordering = (ASC | DESC)? (NULLS nullOrdering=(FIRST | LAST))? + ; + +limitElement + : LIMIT limit =INTEGER_VALUE (OFFSET offset=INTEGER_VALUE)? + | LIMIT offset =INTEGER_VALUE ',' limit=INTEGER_VALUE + ; + +querySpecification + : SELECT setVarHint* setQuantifier? selectItem (',' selectItem)* + fromClause + ((WHERE where=expression)? (GROUP BY groupingElement)? (HAVING having=expression)? + (QUALIFY qualifyFunction=selectItem comparisonOperator limit=INTEGER_VALUE)?) + ; + +fromClause + : (FROM relations)? #from + | FROM DUAL #dual + ; + +groupingElement + : ROLLUP '(' (expressionList)? ')' #rollup + | CUBE '(' (expressionList)? ')' #cube + | GROUPING SETS '(' groupingSet (',' groupingSet)* ')' #multipleGroupingSets + | expressionList #singleGroupingSet + ; + +groupingSet + : '(' expression? (',' expression)* ')' + ; + +commonTableExpression + : name=identifier (columnAliases)? AS '(' queryRelation ')' + ; + +setQuantifier + : DISTINCT + | ALL + ; + +selectItem + : expression (AS? (identifier | string))? #selectSingle + | qualifiedName '.' ASTERISK_SYMBOL #selectAll + | ASTERISK_SYMBOL #selectAll + ; + +relations + : relation (',' LATERAL? relation)* + ; + +relation + : relationPrimary joinRelation* + | '(' relationPrimary joinRelation* ')' + ; + +relationPrimary + : qualifiedName temporalClause? partitionNames? tabletList? replicaList? ( + AS? alias=identifier)? bracketHint? #tableAtom + | '(' VALUES rowConstructor (',' rowConstructor)* ')' + (AS? alias=identifier columnAliases?)? #inlineTable + | subquery (AS? alias=identifier columnAliases?)? #subqueryWithAlias + | qualifiedName '(' expressionList ')' + (AS? alias=identifier columnAliases?)? #tableFunction + | TABLE '(' qualifiedName '(' expressionList ')' ')' + (AS? alias=identifier columnAliases?)? #normalizedTableFunction + | FILES propertyList + (AS? alias=identifier columnAliases?)? #fileTableFunction + | '(' relations ')' #parenthesizedRelation + ; + +joinRelation + : crossOrInnerJoinType bracketHint? + LATERAL? rightRelation=relationPrimary joinCriteria? + | outerAndSemiJoinType bracketHint? + LATERAL? rightRelation=relationPrimary joinCriteria + ; + +crossOrInnerJoinType + : JOIN | INNER JOIN + | CROSS | CROSS JOIN + ; + +outerAndSemiJoinType + : LEFT JOIN | RIGHT JOIN | FULL JOIN + | LEFT OUTER JOIN | RIGHT OUTER JOIN + | FULL OUTER JOIN + | LEFT SEMI JOIN | RIGHT SEMI JOIN + | LEFT ANTI JOIN | RIGHT ANTI JOIN + ; + +bracketHint + : '[' identifier (',' identifier)* ']' + ; + +setVarHint + : '/*+' SET_VAR '(' hintMap (',' hintMap)* ')' '*/' + ; + +hintMap + : k=identifierOrString '=' v=literalExpression + ; + +joinCriteria + : ON expression + | USING '(' identifier (',' identifier)* ')' + ; + +columnAliases + : '(' identifier (',' identifier)* ')' + ; + +// partitionNames should not support string, it should be identifier here only for compatibility with historical bugs +partitionNames + : TEMPORARY? (PARTITION | PARTITIONS) '(' identifierOrString (',' identifierOrString)* ')' + | TEMPORARY? (PARTITION | PARTITIONS) identifierOrString + | keyPartitions + ; + +keyPartitions + : PARTITION '(' keyPartition (',' keyPartition)* ')' #keyPartitionList + ; + +tabletList + : TABLET '(' INTEGER_VALUE (',' INTEGER_VALUE)* ')' + ; + +replicaList + : REPLICA '(' INTEGER_VALUE (',' INTEGER_VALUE)* ')' + ; + +// ------------------------------------------- Expression -------------------------------------------------------------- + +/** + * Operator precedences are shown in the following list, from highest precedence to the lowest. + * + * ! + * - (unary minus), ~ (unary bit inversion) + * ^ + * *, /, DIV, %, MOD + * -, + + * & + * | + * = (comparison), <=>, >=, >, <=, <, <>, !=, IS, LIKE, REGEXP + * BETWEEN, CASE WHEN + * NOT + * AND, && + * XOR + * OR, || + * = (assignment) + */ + +expressionsWithDefault + : '(' expressionOrDefault (',' expressionOrDefault)* ')' + ; + +expressionOrDefault + : expression | DEFAULT + ; + +mapExpressionList + : mapExpression (',' mapExpression)* + ; + +mapExpression + : key=expression ':' value=expression + ; + +expressionSingleton + : expression EOF + ; + +expression + : booleanExpression #expressionDefault + | NOT expression #logicalNot + | left=expression operator=(AND|LOGICAL_AND) right=expression #logicalBinary + | left=expression operator=(OR|LOGICAL_OR) right=expression #logicalBinary + ; + +expressionList + : expression (',' expression)* + ; + +booleanExpression + : predicate #booleanExpressionDefault + | booleanExpression IS NOT? NULL #isNull + | left = booleanExpression comparisonOperator right = predicate #comparison + | booleanExpression comparisonOperator '(' queryRelation ')' #scalarSubquery + ; + +predicate + : valueExpression (predicateOperations[$valueExpression.ctx])? + | tupleInSubquery + ; + +tupleInSubquery + : '(' expression (',' expression)+ ')' NOT? IN '(' queryRelation ')' + ; + +predicateOperations [ParserRuleContext value] + : NOT? IN '(' queryRelation ')' #inSubquery + | NOT? IN '(' expressionList ')' #inList + | NOT? BETWEEN lower = valueExpression AND upper = predicate #between + | NOT? (LIKE | RLIKE | REGEXP) pattern=valueExpression #like + ; + +valueExpression + : primaryExpression #valueExpressionDefault + | left = valueExpression operator = BITXOR right = valueExpression #arithmeticBinary + | left = valueExpression operator = ( + ASTERISK_SYMBOL + | SLASH_SYMBOL + | PERCENT_SYMBOL + | INT_DIV + | MOD) + right = valueExpression #arithmeticBinary + | left = valueExpression operator = (PLUS_SYMBOL | MINUS_SYMBOL) + right = valueExpression #arithmeticBinary + | left = valueExpression operator = BITAND right = valueExpression #arithmeticBinary + | left = valueExpression operator = BITOR right = valueExpression #arithmeticBinary + | left = valueExpression operator = BIT_SHIFT_LEFT right = valueExpression #arithmeticBinary + | left = valueExpression operator = BIT_SHIFT_RIGHT right = valueExpression #arithmeticBinary + | left = valueExpression operator = BIT_SHIFT_RIGHT_LOGICAL right = valueExpression #arithmeticBinary + ; + +primaryExpression + : userVariable #userVariableExpression + | systemVariable #systemVariableExpression + | functionCall #functionCallExpression + | '{' FN functionCall '}' #odbcFunctionCallExpression + | primaryExpression COLLATE (identifier | string) #collate + | literalExpression #literal + | columnReference #columnRef + | base = primaryExpression (DOT_IDENTIFIER | '.' fieldName = identifier ) #dereference + | left = primaryExpression CONCAT right = primaryExpression #concat + | operator = (MINUS_SYMBOL | PLUS_SYMBOL | BITNOT) primaryExpression #arithmeticUnary + | operator = LOGICAL_NOT primaryExpression #arithmeticUnary + | '(' expression ')' #parenthesizedExpression + | EXISTS '(' queryRelation ')' #exists + | subquery #subqueryExpression + | CAST '(' expression AS type ')' #cast + | CONVERT '(' expression ',' type ')' #convert + | CASE caseExpr=expression whenClause+ (ELSE elseExpression=expression)? END #simpleCase + | CASE whenClause+ (ELSE elseExpression=expression)? END #searchedCase + | arrayType? '[' (expressionList)? ']' #arrayConstructor + | mapType '{' (mapExpressionList)? '}' #mapConstructor + | MAP '{' (mapExpressionList)? '}' #mapConstructor + | value=primaryExpression '[' index=valueExpression ']' #collectionSubscript + | primaryExpression '[' start=INTEGER_VALUE? ':' end=INTEGER_VALUE? ']' #arraySlice + | primaryExpression ARROW string #arrowExpression + | (identifier | identifierList) '->' expression #lambdaFunctionExpr + | identifierList '->' '('(expressionList)?')' #lambdaFunctionExpr + ; + +literalExpression + : NULL #nullLiteral + | booleanValue #booleanLiteral + | number #numericLiteral + | (DATE | DATETIME) string #dateLiteral + | string #stringLiteral + | interval #intervalLiteral + | unitBoundary #unitBoundaryLiteral + | binary #binaryLiteral + ; + +functionCall + : EXTRACT '(' identifier FROM valueExpression ')' #extract + | GROUPING '(' (expression (',' expression)*)? ')' #groupingOperation + | GROUPING_ID '(' (expression (',' expression)*)? ')' #groupingOperation + | informationFunctionExpression #informationFunction + | specialDateTimeExpression #specialDateTime + | specialFunctionExpression #specialFunction + | aggregationFunction over? #aggregationFunctionCall + | windowFunction over #windowFunctionCall + | qualifiedName '(' (expression (',' expression)*)? ')' over? #simpleFunctionCall + ; + +aggregationFunction + : AVG '(' setQuantifier? expression ')' + | COUNT '(' ASTERISK_SYMBOL? ')' + | COUNT '(' (setQuantifier bracketHint?)? (expression (',' expression)*)? ')' + | MAX '(' setQuantifier? expression ')' + | MIN '(' setQuantifier? expression ')' + | SUM '(' setQuantifier? expression ')' + | ARRAY_AGG '(' setQuantifier? expression (ORDER BY sortItem (',' sortItem)*)? ')' + | GROUP_CONCAT '(' setQuantifier? expression (',' expression)* (ORDER BY sortItem (',' sortItem)*)? (SEPARATOR expression)? ')' + ; + +userVariable + : AT identifierOrString + ; + +systemVariable + : AT AT (varType '.')? identifier + ; + +columnReference + : identifier + ; + +informationFunctionExpression + : name = CATALOG '(' ')' + | name = DATABASE '(' ')' + | name = SCHEMA '(' ')' + | name = USER '(' ')' + | name = CURRENT_USER ('(' ')')? + | name = CURRENT_ROLE ('(' ')')? + ; + +specialDateTimeExpression + : name = CURRENT_DATE ('(' ')')? + | name = CURRENT_TIME ('(' ')')? + | name = CURRENT_TIMESTAMP ('(' ')')? + | name = LOCALTIME ('(' ')')? + | name = LOCALTIMESTAMP ('(' ')')? + ; + +specialFunctionExpression + : CHAR '(' expression ')' + | DAY '(' expression ')' + | HOUR '(' expression ')' + | IF '(' (expression (',' expression)*)? ')' + | LEFT '(' expression ',' expression ')' + | LIKE '(' expression ',' expression ')' + | MINUTE '(' expression ')' + | MOD '(' expression ',' expression ')' + | MONTH '(' expression ')' + | QUARTER '(' expression ')' + | REGEXP '(' expression ',' expression ')' + | REPLACE '(' (expression (',' expression)*)? ')' + | RIGHT '(' expression ',' expression ')' + | RLIKE '(' expression ',' expression ')' + | SECOND '(' expression ')' + | TIMESTAMPADD '(' unitIdentifier ',' expression ',' expression ')' + | TIMESTAMPDIFF '(' unitIdentifier ',' expression ',' expression ')' + //| WEEK '(' expression ')' TODO: Support week(expr) function + | YEAR '(' expression ')' + | PASSWORD '(' string ')' + | FLOOR '(' expression ')' + | CEIL '(' expression ')' + ; + +windowFunction + : name = ROW_NUMBER '(' ')' + | name = RANK '(' ')' + | name = DENSE_RANK '(' ')' + | name = CUME_DIST '(' ')' + | name = PERCENT_RANK '(' ')' + | name = NTILE '(' expression? ')' + | name = LEAD '(' (expression ignoreNulls? (',' expression)*)? ')' ignoreNulls? + | name = LAG '(' (expression ignoreNulls? (',' expression)*)? ')' ignoreNulls? + | name = FIRST_VALUE '(' (expression ignoreNulls? (',' expression)*)? ')' ignoreNulls? + | name = LAST_VALUE '(' (expression ignoreNulls? (',' expression)*)? ')' ignoreNulls? + ; + +whenClause + : WHEN condition=expression THEN result=expression + ; + +over + : OVER '(' + (bracketHint? PARTITION BY partition+=expression (',' partition+=expression)*)? + (ORDER BY sortItem (',' sortItem)*)? + windowFrame? + ')' + ; + +ignoreNulls + : IGNORE NULLS + ; + +windowFrame + : frameType=RANGE start=frameBound + | frameType=ROWS start=frameBound + | frameType=RANGE BETWEEN start=frameBound AND end=frameBound + | frameType=ROWS BETWEEN start=frameBound AND end=frameBound + ; + +frameBound + : UNBOUNDED boundType=PRECEDING #unboundedFrame + | UNBOUNDED boundType=FOLLOWING #unboundedFrame + | CURRENT ROW #currentRowBound + | expression boundType=(PRECEDING | FOLLOWING) #boundedFrame + ; + +// ------------------------------------------- COMMON AST -------------------------------------------------------------- + +tableDesc + : qualifiedName partitionNames? + ; + +restoreTableDesc + : qualifiedName partitionNames? (AS identifier)? + ; + +explainDesc + : (DESC | DESCRIBE | EXPLAIN) (LOGICAL | ANALYZE | VERBOSE | COSTS | SCHEDULER)? + ; + +optimizerTrace + : TRACE (OPTIMIZER | REWRITE) + ; + +partitionDesc + : PARTITION BY RANGE identifierList '(' (rangePartitionDesc (',' rangePartitionDesc)*)? ')' #partitionRangeIdentifier + | PARTITION BY RANGE primaryExpression '(' (rangePartitionDesc (',' rangePartitionDesc)*)? ')' #partititonRangeExpression + | PARTITION BY LIST? identifierList '(' (listPartitionDesc (',' listPartitionDesc)*)? ')' #partitionListIdentifier + | PARTITION BY LIST? identifierList #partitionListIdentiifer + | PARTITION BY functionCall '(' (rangePartitionDesc (',' rangePartitionDesc)*)? ')' #partitionFunctionCall + | PARTITION BY functionCall #partitionFunctionCall + ; + +listPartitionDesc + : singleItemListPartitionDesc + | multiItemListPartitionDesc + ; + +singleItemListPartitionDesc + : PARTITION (IF NOT EXISTS)? identifier VALUES IN stringList propertyList? + ; + +multiItemListPartitionDesc + : PARTITION (IF NOT EXISTS)? identifier VALUES IN '(' stringList (',' stringList)* ')' propertyList? + ; + +stringList + : '(' string (',' string)* ')' + ; + +rangePartitionDesc + : singleRangePartition + | multiRangePartition + ; + +singleRangePartition + : PARTITION (IF NOT EXISTS)? identifier VALUES partitionKeyDesc propertyList? + ; + +multiRangePartition + : START '(' start=string ')' END '(' end=string ')' EVERY '(' interval ')' + | START '(' start=string ')' END '(' end=string ')' EVERY '(' INTEGER_VALUE ')' + ; + +partitionRangeDesc + : START '(' string ')' END '(' string ')' + ; + +partitionKeyDesc + : LESS THAN (MAXVALUE | partitionValueList) + | '[' partitionValueList ',' partitionValueList ')' + ; + +partitionValueList + : '(' partitionValue (',' partitionValue)* ')' + ; + +keyPartition + : partitionColName=identifier '=' partitionColValue=literalExpression + ; + +partitionValue + : MAXVALUE | string + ; + +distributionClause + : DISTRIBUTED BY HASH identifierList (BUCKETS INTEGER_VALUE)? + | DISTRIBUTED BY HASH identifierList + ; + +distributionDesc + : DISTRIBUTED BY HASH identifierList (BUCKETS INTEGER_VALUE)? + | DISTRIBUTED BY HASH identifierList + | DISTRIBUTED BY RANDOM (BUCKETS INTEGER_VALUE)? + ; + +refreshSchemeDesc + : REFRESH (IMMEDIATE | DEFERRED)? (ASYNC + | ASYNC (START '(' string ')')? EVERY '(' interval ')' + | INCREMENTAL + | MANUAL) + ; + +statusDesc + : ACTIVE + | INACTIVE + ; + +properties + : PROPERTIES '(' property (',' property)* ')' + ; + +extProperties + : BROKER properties + ; + +propertyList + : '(' property (',' property)* ')' + ; + +userPropertyList + : property (',' property)* + ; + +property + : key=string '=' value=string + ; + +varType + : GLOBAL + | LOCAL + | SESSION + | VERBOSE + ; + +comment + : COMMENT string + ; + +outfile + : INTO OUTFILE file=string fileFormat? properties? + ; + +fileFormat + : FORMAT AS (identifier | string) + ; + +string + : SINGLE_QUOTED_TEXT + | DOUBLE_QUOTED_TEXT + ; + +binary + : BINARY_SINGLE_QUOTED_TEXT + | BINARY_DOUBLE_QUOTED_TEXT + ; + +comparisonOperator + : EQ | NEQ | LT | LTE | GT | GTE | EQ_FOR_NULL + ; + +booleanValue + : TRUE | FALSE + ; + +interval + : INTERVAL value=expression from=unitIdentifier + ; + +unitIdentifier + : YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND | QUARTER + ; + +unitBoundary + : FLOOR | CEIL + ; + +type + : baseType + | decimalType + | arrayType + | structType + | mapType + ; + +arrayType + : ARRAY '<' type '>' + ; + +mapType + : MAP '<' type ',' type '>' + ; + +subfieldDesc + : identifier type + ; + +subfieldDescs + : subfieldDesc (',' subfieldDesc)* + ; + +structType + : STRUCT '<' subfieldDescs '>' + ; + +typeParameter + : '(' INTEGER_VALUE ')' + ; + +baseType + : name=BOOLEAN + | name=TINYINT typeParameter? + | name=SMALLINT typeParameter? + | name=SIGNED INT? + | name=SIGNED INTEGER? + | name=UNSIGNED INT? + | name=UNSIGNED INTEGER? + | name=INT typeParameter? + | name=INTEGER typeParameter? + | name=BIGINT typeParameter? + | name=LARGEINT typeParameter? + | name=FLOAT + | name=DOUBLE + | name=DATE + | name=DATETIME + | name=TIME + | name=CHAR typeParameter? + | name=VARCHAR typeParameter? + | name=STRING + | name=TEXT + | name=BITMAP + | name=HLL + | name=PERCENTILE + | name=JSON + | name=VARBINARY typeParameter? + | name=BINARY typeParameter? + ; + +decimalType + : name=(DECIMAL | DECIMALV2 | DECIMAL32 | DECIMAL64 | DECIMAL128 | NUMERIC | NUMBER ) + ('(' precision=INTEGER_VALUE (',' scale=INTEGER_VALUE)? ')')? + ; + +qualifiedName + : identifier (DOT_IDENTIFIER | '.' identifier)* + ; + +identifier + : LETTER_IDENTIFIER #unquotedIdentifier + | nonReserved #unquotedIdentifier + | DIGIT_IDENTIFIER #digitIdentifier + | BACKQUOTED_IDENTIFIER #backQuotedIdentifier + ; + +identifierList + : '(' identifier (',' identifier)* ')' + ; + +identifierOrString + : identifier + | string + ; + +identifierOrStringList + : identifierOrString (',' identifierOrString)* + ; + +identifierOrStringOrStar + : ASTERISK_SYMBOL + | identifier + | string + ; + +user + : identifierOrString # userWithoutHost + | identifierOrString '@' identifierOrString # userWithHost + | identifierOrString '@' '[' identifierOrString ']' # userWithHostAndBlanket + ; + +assignment + : identifier EQ expressionOrDefault + ; + +assignmentList + : assignment (',' assignment)* + ; + +number + : DECIMAL_VALUE #decimalValue + | DOUBLE_VALUE #doubleValue + | INTEGER_VALUE #integerValue + ; + +nonReserved + : ACCESS | ACTIVE | AFTER | AGGREGATE | APPLY | ASYNC | AUTHORS | AVG | ADMIN | ANTI | AUTHENTICATION | AUTO_INCREMENT + | ARRAY_AGG + | BACKEND | BACKENDS | BACKUP | BEGIN | BITMAP_UNION | BLACKLIST | BINARY | BODY | BOOLEAN | BROKER | BUCKETS + | BUILTIN | BASE + | CAST | CANCEL | CATALOG | CATALOGS | CEIL | CHAIN | CHARSET | CLEAN | CLUSTER | CLUSTERS | CURRENT | COLLATION | COLUMNS + | CUME_DIST | CUMULATIVE | COMMENT | COMMIT | COMMITTED | COMPUTE | CONNECTION | CONSISTENT | COSTS | COUNT + | CONFIG | COMPACT + | DATA | DATE | DATETIME | DAY | DECOMMISSION | DISABLE | DISTRIBUTION | DUPLICATE | DYNAMIC | DISTRIBUTED + | ENABLE | END | ENGINE | ENGINES | ERRORS | EVENTS | EXECUTE | EXTERNAL | EXTRACT | EVERY | ENCLOSE | ESCAPE | EXPORT + | FAILPOINT | FAILPOINTS | FIELDS | FILE | FILTER | FIRST | FLOOR | FOLLOWING | FORMAT | FN | FRONTEND | FRONTENDS | FOLLOWER | FREE + | FUNCTIONS + | GLOBAL | GRANTS | GROUP_CONCAT + | HASH | HISTOGRAM | HELP | HLL_UNION | HOST | HOUR | HUB + | IDENTIFIED | IMAGE | IMPERSONATE | INACTIVE | INCREMENTAL | INDEXES | INSTALL | INTEGRATION | INTEGRATIONS | INTERMEDIATE + | INTERVAL | ISOLATION + | JOB + | LABEL | LAST | LESS | LEVEL | LIST | LOCAL | LOCATION | LOGICAL | LOW_PRIORITY | LOCK | LOCATIONS + | MASKING | MANUAL | MAP | MAPPING | MAPPINGS | MATERIALIZED | MAX | META | MIN | MINUTE | MODE | MODIFY | MONTH | MERGE | MINUS + | NAME | NAMES | NEGATIVE | NO | NODE | NODES | NONE | NULLS | NUMBER | NUMERIC + | OBSERVER | OF | OFFSET | ONLY | OPTIMIZER | OPEN | OPERATE | OPTION | OVERWRITE + | PARTITIONS | PASSWORD | PATH | PAUSE | PENDING | PERCENTILE_UNION | PLUGIN | PLUGINS | POLICY | POLICIES + | PERCENT_RANK | PRECEDING | PROC | PROCESSLIST | PROFILE | PROFILELIST | PRIVILEGES | PROBABILITY | PROPERTIES | PROPERTY | PIPE | PIPES + | QUARTER | QUERY | QUEUE | QUOTA | QUALIFY + | REMOVE | REWRITE | RANDOM | RANK | RECOVER | REFRESH | REPAIR | REPEATABLE | REPLACE_IF_NOT_NULL | REPLICA | REPOSITORY + | REPOSITORIES + | RESOURCE | RESOURCES | RESTORE | RESUME | RETURNS | RETRY | REVERT | ROLE | ROLES | ROLLUP | ROLLBACK | ROUTINE | ROW + | SAMPLE | SCHEDULER | SECOND | SECURITY | SEPARATOR | SERIALIZABLE |SEMI | SESSION | SETS | SIGNED | SNAPSHOT | SQLBLACKLIST | START + | STREAM | SUM | STATUS | STOP | SKIP_HEADER | SWAP + | STORAGE| STRING | STRUCT | STATS | SUBMIT | SUSPEND | SYNC | SYSTEM_TIME + | TABLES | TABLET | TASK | TEMPORARY | TIMESTAMP | TIMESTAMPADD | TIMESTAMPDIFF | THAN | TIME | TIMES | TRANSACTION | TRACE + | TRIM_SPACE + | TRIGGERS | TRUNCATE | TYPE | TYPES + | UNBOUNDED | UNCOMMITTED | UNSET | UNINSTALL | USAGE | USER | USERS | UNLOCK + | VALUE | VARBINARY | VARIABLES | VIEW | VIEWS | VERBOSE | VOLUME | VOLUMES + | WARNINGS | WEEK | WHITELIST | WORK | WRITE | WAREHOUSE | WAREHOUSES + | YEAR + | DOTDOTDOT + ; diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/antlr4/imports/StarRocksLex.g4 b/fastmodel-transform/fastmodel-transform-starrocks/src/main/antlr4/imports/StarRocksLex.g4 new file mode 100644 index 0000000..6e5f446 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/antlr4/imports/StarRocksLex.g4 @@ -0,0 +1,553 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +lexer grammar StarRocksLex; + +tokens { + CONCAT +} + +ACCESS: 'ACCESS'; +ACTIVE: 'ACTIVE'; +ADD: 'ADD'; +ADMIN: 'ADMIN'; +AFTER: 'AFTER'; +AGGREGATE: 'AGGREGATE'; +ALL: 'ALL'; +ALTER: 'ALTER'; +ANALYZE: 'ANALYZE'; +AND: 'AND'; +ANTI: 'ANTI'; +APPLY: 'APPLY'; +ARRAY: 'ARRAY'; +ARRAY_AGG: 'ARRAY_AGG'; +AS: 'AS'; +ASC: 'ASC'; +ASYNC: 'ASYNC'; +AUTHORS: 'AUTHORS'; +AUTHENTICATION: 'AUTHENTICATION'; +AUTO_INCREMENT: 'AUTO_INCREMENT'; +AVG: 'AVG'; +BACKEND: 'BACKEND'; +BACKENDS: 'BACKENDS'; +BACKUP: 'BACKUP'; +BASE: 'BASE'; +BEGIN: 'BEGIN'; +BETWEEN: 'BETWEEN'; +BIGINT: 'BIGINT'; +BINARY: 'BINARY'; +BITMAP: 'BITMAP'; +BITMAP_UNION: 'BITMAP_UNION'; +BLACKLIST: 'BLACKLIST'; +BODY: 'BODY'; +BOOLEAN: 'BOOLEAN'; +BOTH: 'BOTH'; +BROKER: 'BROKER'; +BUCKETS: 'BUCKETS'; +BUILTIN: 'BUILTIN'; +BY: 'BY'; +CANCEL: 'CANCEL'; +CASE: 'CASE'; +CAST: 'CAST'; +CATALOG: 'CATALOG'; +CATALOGS: 'CATALOGS'; +CEIL: 'CEIL'; +CHAIN: 'CHAIN'; +CHAR: 'CHAR'; +CHARACTER: 'CHARACTER'; +CHARSET: 'CHARSET'; +CHECK: 'CHECK'; +CLEAN: 'CLEAN'; +CLUSTER : 'CLUSTER'; +CLUSTERS : 'CLUSTERS'; +COLLATE: 'COLLATE'; +COLLATION: 'COLLATION'; +COLUMN: 'COLUMN'; +COLUMNS: 'COLUMNS'; +COMMENT: 'COMMENT'; +COMMIT: 'COMMIT'; +COMMITTED: 'COMMITTED'; +COMPACT: 'COMPACT'; +COMPACTION: 'COMPACTION'; +COMPUTE: 'COMPUTE'; +CONFIG: 'CONFIG'; +CONNECTION: 'CONNECTION'; +CONSISTENT: 'CONSISTENT'; +CONVERT: 'CONVERT'; +COSTS: 'COSTS'; +COUNT: 'COUNT'; +CREATE: 'CREATE'; +CROSS: 'CROSS'; +CUBE: 'CUBE'; +CUME_DIST: 'CUME_DIST'; +CUMULATIVE: 'CUMULATIVE'; +CURRENT: 'CURRENT'; +CURRENT_DATE: 'CURRENT_DATE'; +CURRENT_ROLE: 'CURRENT_ROLE'; +CURRENT_TIME: 'CURRENT_TIME'; +CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; +CURRENT_USER: 'CURRENT_USER'; +DATA: 'DATA'; +DATABASE: 'DATABASE'; +DATABASES: 'DATABASES'; +DATE: 'DATE'; +DATETIME: 'DATETIME'; +DAY: 'DAY'; +DECIMAL: 'DECIMAL'; +DECIMALV2: 'DECIMALV2'; +DECIMAL32: 'DECIMAL32'; +DECIMAL64: 'DECIMAL64'; +DECIMAL128: 'DECIMAL128'; +DECOMMISSION: 'DECOMMISSION'; +DEFAULT: 'DEFAULT'; +DELETE: 'DELETE'; +DENSE_RANK: 'DENSE_RANK'; +DEFERRED: 'DEFERRED'; +NTILE: 'NTILE'; +DESC: 'DESC'; +DESCRIBE: 'DESCRIBE'; +DISABLE: 'DISABLE'; +DISTINCT: 'DISTINCT'; +DISTRIBUTED: 'DISTRIBUTED'; +DISTRIBUTION: 'DISTRIBUTION'; +DOUBLE: 'DOUBLE'; +DROP: 'DROP'; +DUAL: 'DUAL'; +DUPLICATE: 'DUPLICATE'; +DYNAMIC: 'DYNAMIC'; +ELSE: 'ELSE'; +ENABLE: 'ENABLE'; +ENCLOSE: 'ENCLOSE'; +END: 'END'; +ENGINE: 'ENGINE'; +ENGINES: 'ENGINES'; +ERRORS: 'ERRORS'; +ESCAPE: 'ESCAPE'; +EVENTS: 'EVENTS'; +EXCEPT: 'EXCEPT'; +EXECUTE: 'EXECUTE'; +EXISTS: 'EXISTS'; +EXPLAIN: 'EXPLAIN'; +EXPORT: 'EXPORT'; +EXTERNAL: 'EXTERNAL'; +EXTRACT: 'EXTRACT'; +EVERY: 'EVERY'; +FAILPOINT: 'FAILPOINT'; +FAILPOINTS: 'FAILPOINTS'; +FALSE: 'FALSE'; +FIELDS: 'FIELDS'; +FILE: 'FILE'; +FILES: 'FILES'; +FILTER: 'FILTER'; +FIRST: 'FIRST'; +FIRST_VALUE: 'FIRST_VALUE'; +FLOAT: 'FLOAT'; +FLOOR: 'FLOOR'; +FN: 'FN'; +FOLLOWING: 'FOLLOWING'; +FOLLOWER: 'FOLLOWER'; +FOR: 'FOR'; +FORCE: 'FORCE'; +FORMAT: 'FORMAT'; +FREE: 'FREE'; +FROM: 'FROM'; +FRONTEND: 'FRONTEND'; +FRONTENDS: 'FRONTENDS'; +FULL: 'FULL'; +FUNCTION: 'FUNCTION'; +FUNCTIONS: 'FUNCTIONS'; +GLOBAL: 'GLOBAL'; +GRANT: 'GRANT'; +GRANTS: 'GRANTS'; +GROUP: 'GROUP'; +GROUPS: 'GROUPS'; +GROUPING: 'GROUPING'; +GROUPING_ID: 'GROUPING_ID'; +GROUP_CONCAT: 'GROUP_CONCAT'; +HASH: 'HASH'; +HAVING: 'HAVING'; +HELP: 'HELP'; +HISTOGRAM: 'HISTOGRAM'; +HLL: 'HLL'; +HLL_UNION: 'HLL_UNION'; +HOST: 'HOST'; +HOUR: 'HOUR'; +HUB: 'HUB'; +IDENTIFIED: 'IDENTIFIED'; +IF: 'IF'; +IMPERSONATE: 'IMPERSONATE'; +IMMEDIATE: 'IMMEDIATE'; +IGNORE: 'IGNORE'; +IMAGE: 'IMAGE'; +IN: 'IN'; +INACTIVE: 'INACTIVE'; +INCREMENTAL: 'INCREMENTAL'; +INDEX: 'INDEX'; +INDEXES: 'INDEXES'; +INFILE: 'INFILE'; +INNER: 'INNER'; +INSTALL: 'INSTALL'; +INSERT: 'INSERT'; +INT: 'INT'; +INTEGER: 'INTEGER'; +INTEGRATION: 'INTEGRATION'; +INTEGRATIONS: 'INTEGRATIONS'; +INTERMEDIATE: 'INTERMEDIATE'; +INTERSECT: 'INTERSECT'; +INTERVAL: 'INTERVAL'; +INTO: 'INTO'; +OVERWRITE: 'OVERWRITE'; +IS: 'IS'; +ISOLATION: 'ISOLATION'; +JOB: 'JOB'; +JOIN: 'JOIN'; +JSON: 'JSON'; +KEY: 'KEY'; +KEYS: 'KEYS'; +KILL: 'KILL'; +LABEL: 'LABEL'; +LAG: 'LAG'; +LARGEINT: 'LARGEINT'; +LAST: 'LAST'; +LAST_VALUE: 'LAST_VALUE'; +LATERAL: 'LATERAL'; +LEAD: 'LEAD'; +LEFT: 'LEFT'; +LESS: 'LESS'; +LEVEL: 'LEVEL'; +LIKE: 'LIKE'; +LIMIT: 'LIMIT'; +LIST: 'LIST'; +LOAD: 'LOAD'; +LOCAL: 'LOCAL'; +LOCALTIME: 'LOCALTIME'; +LOCALTIMESTAMP: 'LOCALTIMESTAMP'; +LOCATION: 'LOCATION'; +LOCATIONS: 'LOCATIONS'; +LOGICAL: 'LOGICAL'; +MANUAL: 'MANUAL'; +MAP: 'MAP'; +MAPPING: 'MAPPING'; +MAPPINGS: 'MAPPINGS'; +MASKING: 'MASKING'; +MATERIALIZED: 'MATERIALIZED'; +MAX: 'MAX'; +MAXVALUE: 'MAXVALUE'; +MERGE: 'MERGE'; +MIN: 'MIN'; +MINUTE: 'MINUTE'; +MINUS: 'MINUS'; +META: 'META'; +MOD: 'MOD'; +MODE: 'MODE'; +MODIFY: 'MODIFY'; +MONTH: 'MONTH'; +NAME: 'NAME'; +NAMES: 'NAMES'; +NEGATIVE: 'NEGATIVE'; +NO: 'NO'; +NODE: 'NODE'; +NODES: 'NODES'; +NONE: 'NONE'; +NOT: 'NOT'; +NULL: 'NULL'; +NULLS: 'NULLS'; +NUMBER: 'NUMBER'; +NUMERIC: 'NUMERIC'; +OBSERVER: 'OBSERVER'; +OF: 'OF'; +OFFSET: 'OFFSET'; +ON: 'ON'; +ONLY: 'ONLY'; +OPEN: 'OPEN'; +OPERATE: 'OPERATE'; +OPTIMIZER: 'OPTIMIZER'; +OPTION: 'OPTION'; +OR: 'OR'; +ORDER: 'ORDER'; +OUTER: 'OUTER'; +OUTFILE: 'OUTFILE'; +OVER: 'OVER'; +PARTITION: 'PARTITION'; +PARTITIONS: 'PARTITIONS'; +PASSWORD: 'PASSWORD'; +PATH: 'PATH'; +PAUSE: 'PAUSE'; +PENDING: 'PENDING'; +PERCENT_RANK: 'PERCENT_RANK'; +PERCENTILE: 'PERCENTILE'; +PERCENTILE_UNION: 'PERCENTILE_UNION'; +PLUGIN: 'PLUGIN'; +PLUGINS: 'PLUGINS'; +PIPE: 'PIPE'; +PIPES: 'PIPES'; +POLICY: 'POLICY'; +POLICIES: 'POLICIES'; +PRECEDING: 'PRECEDING'; +PRIMARY: 'PRIMARY'; +PRIVILEGES: 'PRIVILEGES'; +PROBABILITY: 'PROBABILITY'; +PROC: 'PROC'; +PROCEDURE: 'PROCEDURE'; +PROCESSLIST: 'PROCESSLIST'; +PROFILE: 'PROFILE'; +PROFILELIST: 'PROFILELIST'; +PROPERTIES: 'PROPERTIES'; +PROPERTY: 'PROPERTY'; +QUALIFY: 'QUALIFY'; +QUARTER: 'QUARTER'; +QUERY: 'QUERY'; +QUERIES: 'QUERIES'; +QUEUE: 'QUEUE'; +QUOTA: 'QUOTA'; +RANDOM: 'RANDOM'; +RANGE: 'RANGE'; +RANK: 'RANK'; +READ: 'READ'; +RECOVER: 'RECOVER'; +REFRESH: 'REFRESH'; +REWRITE: 'REWRITE'; +REGEXP: 'REGEXP'; +RELEASE: 'RELEASE'; +REMOVE: 'REMOVE'; +RENAME: 'RENAME'; +REPAIR: 'REPAIR'; +REPEATABLE: 'REPEATABLE'; +REPLACE: 'REPLACE'; +REPLACE_IF_NOT_NULL: 'REPLACE_IF_NOT_NULL'; +REPLICA: 'REPLICA'; +REPOSITORY: 'REPOSITORY'; +REPOSITORIES: 'REPOSITORIES'; +RESOURCE: 'RESOURCE'; +RESOURCES: 'RESOURCES'; +RESTORE: 'RESTORE'; +RESUME: 'RESUME'; +RETURNS: 'RETURNS'; +RETRY: 'RETRY'; +REVOKE: 'REVOKE'; +REVERT: 'REVERT'; +RIGHT: 'RIGHT'; +RLIKE: 'RLIKE'; +ROLE: 'ROLE'; +ROLES: 'ROLES'; +ROLLBACK: 'ROLLBACK'; +ROLLUP: 'ROLLUP'; +ROUTINE: 'ROUTINE'; +ROW: 'ROW'; +ROWS: 'ROWS'; +ROW_NUMBER: 'ROW_NUMBER'; +RUNNING: 'RUNNING'; +SAMPLE: 'SAMPLE'; +SCHEDULER: 'SCHEDULER'; +SCHEMA: 'SCHEMA'; +SCHEMAS: 'SCHEMAS'; +SECOND: 'SECOND'; +SECURITY: 'SECURITY'; +SELECT: 'SELECT'; +SEMI: 'SEMI'; +SEPARATOR: 'SEPARATOR'; +SERIALIZABLE: 'SERIALIZABLE'; +SESSION: 'SESSION'; +SET: 'SET'; +SETS: 'SETS'; +SET_VAR: 'SET_VAR'; +SIGNED: 'SIGNED'; +SKIP_HEADER: 'SKIP_HEADER'; +SHOW: 'SHOW'; +SMALLINT: 'SMALLINT'; +SNAPSHOT: 'SNAPSHOT'; +SQLBLACKLIST: 'SQLBLACKLIST'; +START: 'START'; +STATS: 'STATS'; +STATUS: 'STATUS'; +STOP: 'STOP'; +STORAGE: 'STORAGE'; +STREAM: 'STREAM'; +STRING: 'STRING'; +TEXT: 'TEXT'; +SUBMIT: 'SUBMIT'; +SUM: 'SUM'; +SUSPEND: 'SUSPEND'; +SYNC: 'SYNC'; +SYSTEM: 'SYSTEM'; +SYSTEM_TIME: 'SYSTEM_TIME'; +SWAP: 'SWAP'; +STRUCT: 'STRUCT'; +TABLE: 'TABLE'; +TABLES: 'TABLES'; +TABLET: 'TABLET'; +TASK: 'TASK'; +TEMPORARY: 'TEMPORARY'; +TERMINATED: 'TERMINATED'; +THAN: 'THAN'; +THEN: 'THEN'; +TIME: 'TIME'; +TIMES: 'TIMES'; +TIMESTAMP: 'TIMESTAMP'; +TIMESTAMPADD: 'TIMESTAMPADD'; +TIMESTAMPDIFF: 'TIMESTAMPDIFF'; +TINYINT: 'TINYINT'; +TRANSACTION: 'TRANSACTION'; +TO: 'TO'; +TRACE: 'TRACE'; +TRIGGERS: 'TRIGGERS'; +TRIM_SPACE: 'TRIM_SPACE'; +TRUE: 'TRUE'; +TRUNCATE: 'TRUNCATE'; +TYPE: 'TYPE'; +TYPES: 'TYPES'; +UNBOUNDED: 'UNBOUNDED'; +UNCOMMITTED: 'UNCOMMITTED'; +UNION: 'UNION'; +UNIQUE: 'UNIQUE'; +UNINSTALL: 'UNINSTALL'; +UNSET: 'UNSET'; +UNSIGNED: 'UNSIGNED'; +UPDATE: 'UPDATE'; +USAGE: 'USAGE'; +USE: 'USE'; +USER: 'USER'; +USERS: 'USERS'; +USING: 'USING'; +VALUE: 'VALUE'; +VALUES: 'VALUES'; +VARBINARY: 'VARBINARY'; +VARCHAR: 'VARCHAR'; +VARIABLES: 'VARIABLES'; +VERBOSE: 'VERBOSE'; +VIEW: 'VIEW'; +VIEWS: 'VIEWS'; +VOLUME: 'VOLUME'; +VOLUMES: 'VOLUMES'; +WAREHOUSE: 'WAREHOUSE'; +WAREHOUSES: 'WAREHOUSES'; +WARNINGS: 'WARNINGS'; +WEEK: 'WEEK'; +WHEN: 'WHEN'; +WHERE: 'WHERE'; +WHITELIST: 'WHITELIST'; +WITH: 'WITH'; +WORK: 'WORK'; +WRITE: 'WRITE'; +YEAR: 'YEAR'; +LOCK: 'LOCK'; +UNLOCK: 'UNLOCK'; +LOW_PRIORITY: 'LOW_PRIORITY'; + +EQ : '='; +NEQ : '<>' | '!='; +LT : '<'; +LTE : '<='; +GT : '>'; +GTE : '>='; +EQ_FOR_NULL: '<=>'; + +PLUS_SYMBOL: '+'; +MINUS_SYMBOL: '-'; +ASTERISK_SYMBOL: '*'; +SLASH_SYMBOL: '/'; +PERCENT_SYMBOL: '%'; + +LOGICAL_OR: '||'; +LOGICAL_AND: '&&'; +LOGICAL_NOT: '!'; + +INT_DIV: 'DIV'; +BITAND: '&'; +BITOR: '|'; +BITXOR: '^'; +BITNOT: '~'; +BIT_SHIFT_LEFT: 'BITSHIFTLEFT'; +BIT_SHIFT_RIGHT: 'BITSHIFTRIGHT'; +BIT_SHIFT_RIGHT_LOGICAL: 'BITSHIFTRIGHTLOGICAL'; + +ARROW: '->'; +AT: '@'; + +INTEGER_VALUE + : DIGIT+ + ; + +DECIMAL_VALUE + : DIGIT+ '.' DIGIT* + | '.' DIGIT+ + ; + +DOUBLE_VALUE + : DIGIT+ ('.' DIGIT*)? EXPONENT + | '.' DIGIT+ EXPONENT + ; + +SINGLE_QUOTED_TEXT + : '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\'' + ; + +DOUBLE_QUOTED_TEXT + : '"' ('\\'. | '""' | ~('"'| '\\'))* '"' + ; + +BINARY_SINGLE_QUOTED_TEXT + : 'X\'' (~('\'' | '\\'))* '\'' + ; + +BINARY_DOUBLE_QUOTED_TEXT + : 'X"' (~('"'| '\\'))* '"' + ; + +LETTER_IDENTIFIER + : (LETTER | '_') (LETTER | DIGIT | '_')* + ; + +DIGIT_IDENTIFIER + : DIGIT (LETTER | DIGIT | '_')+ + ; + +BACKQUOTED_IDENTIFIER + : '`' ( ~'`' | '``' )* '`' + ; + +// Prevent recognize string: .123somelatin AS ((.123), DECIMAL_LITERAL), ((somelatin), IDENTIFIER) +// it must recoginze: .123somelatin AS ((.), DOT), (123somelatin, IDENTIFIER) +DOT_IDENTIFIER + : '.' DIGIT_IDENTIFIER + ; + +fragment EXPONENT + : 'E' [+-]? DIGIT+ + ; + +fragment DIGIT + : [0-9] + ; + +fragment LETTER + : [a-zA-Z_$\u0080-\uffff] + ; + +SIMPLE_COMMENT + : '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN) + ; + +BRACKETED_COMMENT + : '/*' ('+'? [ \r\n\t\u3000]* | ~'+' .*?) '*/' -> channel(HIDDEN) + ; + +SEMICOLON: ';'; + +DOTDOTDOT: '...'; + +WS + : [ \r\n\t\u3000]+ -> channel(HIDDEN) + ; diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/StarRocksTransformer.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/StarRocksTransformer.java new file mode 100644 index 0000000..4ff8cf2 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/StarRocksTransformer.java @@ -0,0 +1,64 @@ +package com.aliyun.fastmodel.transform.starrocks; + +import com.aliyun.fastmodel.core.tree.BaseStatement; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.transform.api.Transformer; +import com.aliyun.fastmodel.transform.api.builder.BuilderFactory; +import com.aliyun.fastmodel.transform.api.builder.StatementBuilder; +import com.aliyun.fastmodel.transform.api.client.dto.table.Table; +import com.aliyun.fastmodel.transform.api.client.dto.table.TableConfig; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.api.context.TransformContext; +import com.aliyun.fastmodel.transform.api.dialect.Dialect; +import com.aliyun.fastmodel.transform.api.dialect.DialectMeta; +import com.aliyun.fastmodel.transform.api.dialect.DialectName; +import com.aliyun.fastmodel.transform.api.dialect.DialectName.Constants; +import com.aliyun.fastmodel.transform.api.dialect.DialectNode; +import com.aliyun.fastmodel.transform.api.dialect.IVersion; +import com.aliyun.fastmodel.transform.starrocks.client.converter.StarRocksClientConverter; +import com.aliyun.fastmodel.transform.starrocks.context.StarRocksContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksLanguageParser; +import com.google.auto.service.AutoService; + +/** + * star rocks transformer + * + * @author panguanjing + * @date 2023/9/5 + */ +@AutoService(Transformer.class) +@Dialect(value = Constants.STARROCKS) +public class StarRocksTransformer implements Transformer { + + private final StarRocksLanguageParser starRocksLanguageParser = new StarRocksLanguageParser(); + + private final StarRocksClientConverter starRocksClientConverter = new StarRocksClientConverter(); + + @Override + public DialectNode transform(BaseStatement source, TransformContext context) { + DialectMeta dialectMeta = new DialectMeta(DialectName.STARROCKS, IVersion.getDefault()); + StarRocksContext mysqlTransformContext = new StarRocksContext(context); + StatementBuilder builder = BuilderFactory.getInstance().getBuilder(source, dialectMeta, mysqlTransformContext); + if (builder == null) { + throw new UnsupportedOperationException( + "UnSupported statement transform with target Dialect, source: " + source.getClass()); + } + return builder.build(source, mysqlTransformContext); + } + + @Override + public BaseStatement reverse(DialectNode dialectNode, ReverseContext context) { + return (BaseStatement)starRocksLanguageParser.parseNode(dialectNode.getNode(), context); + } + + @Override + public Node reverseTable(Table table, ReverseContext context) { + return starRocksClientConverter.covertToNode(table, TableConfig.builder().build()); + } + + @Override + public Table transformTable(Node table, TransformContext context) { + StarRocksContext starRocksContext = new StarRocksContext(context); + return starRocksClientConverter.convertToTable(table, starRocksContext); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/builder/DefaultBuilder.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/builder/DefaultBuilder.java new file mode 100644 index 0000000..f39fde8 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/builder/DefaultBuilder.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.fastmodel.transform.starrocks.builder; + +import com.aliyun.fastmodel.core.tree.BaseStatement; +import com.aliyun.fastmodel.transform.api.builder.BuilderAnnotation; +import com.aliyun.fastmodel.transform.api.builder.StatementBuilder; +import com.aliyun.fastmodel.transform.api.dialect.DialectName.Constants; +import com.aliyun.fastmodel.transform.api.dialect.DialectNode; +import com.aliyun.fastmodel.transform.starrocks.context.StarRocksContext; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksOutVisitor; +import com.google.auto.service.AutoService; + +/** + * 默认的builder实现 + * + * @author panguanjing + */ +@BuilderAnnotation(dialect = Constants.STARROCKS, values = {BaseStatement.class}) +@AutoService(StatementBuilder.class) +public class DefaultBuilder implements StatementBuilder { + + @Override + public DialectNode build(BaseStatement source, StarRocksContext context) { + StarRocksOutVisitor starRocksVisitor = new StarRocksOutVisitor(context); + Boolean process = starRocksVisitor.process(source, 0); + StringBuilder builder = starRocksVisitor.getBuilder(); + return new DialectNode(builder.toString(), process); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksClientConverter.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksClientConverter.java new file mode 100644 index 0000000..2beaa05 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksClientConverter.java @@ -0,0 +1,408 @@ +package com.aliyun.fastmodel.transform.starrocks.client.converter; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.QualifiedName; +import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; +import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName; +import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName.Dimension; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.expr.enums.DateTimeEnum; +import com.aliyun.fastmodel.core.tree.expr.literal.IntervalLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.LongLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.StringLiteral; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; +import com.aliyun.fastmodel.core.tree.statement.table.PartitionedBy; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.BaseConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.PrimaryConstraint; +import com.aliyun.fastmodel.core.tree.util.IdentifierUtil; +import com.aliyun.fastmodel.transform.api.client.PropertyConverter; +import com.aliyun.fastmodel.transform.api.client.converter.BaseClientConverter; +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.api.client.dto.property.StringProperty; +import com.aliyun.fastmodel.transform.api.client.dto.table.Column; +import com.aliyun.fastmodel.transform.api.client.dto.table.Table; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.MultiRangePartitionProperty; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.SingleRangePartitionProperty; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.TablePartitionRaw; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.ArrayClientPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.BaseClientPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.LessThanClientPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.MultiRangeClientPartition; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.PartitionClientValue; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.SingleRangeClientPartition; +import com.aliyun.fastmodel.transform.starrocks.context.StarRocksContext; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksOutVisitor; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksLanguageParser; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype.StarRocksDataTypeName; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ArrayPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.LessThanPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiRangePartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionDesc; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.RangePartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleRangePartition; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; + +/** + * StarRocksClientConverter + * + * @author panguanjing + * @date 2023/9/16 + */ +public class StarRocksClientConverter extends BaseClientConverter { + + private final StarRocksLanguageParser starRocksLanguageParser = new StarRocksLanguageParser(); + + private final StarRocksPropertyConverter starRocksPropertyConverter = new StarRocksPropertyConverter(); + + @Override + public PropertyConverter getPropertyConverter() { + return starRocksPropertyConverter; + } + + @Override + protected BaseDataType getDataType(Column column) { + String dataTypeName = column.getDataType(); + if (StringUtils.isBlank(dataTypeName)) { + throw new IllegalArgumentException("dataType name can't be null:" + column.getName()); + } + IDataTypeName byValue = StarRocksDataTypeName.getByValue(dataTypeName); + Dimension dimension = byValue.getDimension(); + ReverseContext context = ReverseContext.builder().build(); + if (dimension == null || dimension == Dimension.ZERO) { + return starRocksLanguageParser.parseDataType(dataTypeName, context); + } + if (dimension == Dimension.ONE) { + boolean isValidLength = column.getLength() != null && column.getLength() > 0; + if (isValidLength) { + String dt = String.format(ONE_DIMENSION, dataTypeName, column.getLength()); + return starRocksLanguageParser.parseDataType(dt, context); + } + } + if (dimension == Dimension.TWO) { + if (column.getPrecision() != null) { + if (column.getScale() == null) { + return starRocksLanguageParser.parseDataType(String.format(ONE_DIMENSION, dataTypeName, column.getPrecision()), context); + } + return starRocksLanguageParser.parseDataType(String.format(TWO_DIMENSION, dataTypeName, column.getPrecision(), column.getScale()), + context); + } + } + return starRocksLanguageParser.parseDataType(dataTypeName, context); + } + + @Override + protected PartitionedBy toPartitionedBy(Table table, List columns) { + //convert to starRockPartitionBy + List properties = table.getProperties(); + List list = columns.stream() + .filter(Column::isPartitionKey) + .sorted(Comparator.comparing(Column::getPartitionKeyIndex)) + .map(x -> { + List clientProperties = x.getProperties(); + return ColumnDefinition.builder().colName(new Identifier(x.getName())) + .properties(toProperty(table, clientProperties)) + .build(); + }) + .collect(Collectors.toList()); + List rangePartitions = toRangePartition(properties); + return new RangePartitionedBy(list, rangePartitions); + } + + @Override + protected List toBaseClientProperty(CreateTable createTable) { + List propertyList = Lists.newArrayList(); + if (!createTable.isPropertyEmpty()) { + List properties = createTable.getProperties(); + for (Property property : properties) { + BaseClientProperty baseClientProperty = getPropertyConverter().create(property.getName(), property.getValue()); + propertyList.add(baseClientProperty); + } + } + + PartitionedBy partitionedBy = createTable.getPartitionedBy(); + if (createTable.isPartitionEmpty()) { + return propertyList; + } + if (partitionedBy instanceof RangePartitionedBy) { + RangePartitionedBy rangePartitionedBy = (RangePartitionedBy)partitionedBy; + List rangePartitions = rangePartitionedBy.getRangePartitions(); + //结构化的返回 + List list = Lists.newArrayList(); + for (PartitionDesc partitionDesc : rangePartitions) { + BaseClientProperty baseClientProperty = parseRangePartition(partitionDesc); + if (baseClientProperty != null) { + list.add(baseClientProperty); + } + } + //如果解析不了，那么使用visitor的方式进行解析 + if (list.isEmpty()) { + StarRocksOutVisitor starRocksOutVisitor = new StarRocksOutVisitor(StarRocksContext.builder().build()); + starRocksOutVisitor.visitRangePartitionedBy(rangePartitionedBy, 0); + TablePartitionRaw baseClientProperty = new TablePartitionRaw(); + baseClientProperty.setKey(StarRocksProperty.TABLE_PARTITION_RAW.getValue()); + baseClientProperty.setValueString(starRocksOutVisitor.getBuilder().toString()); + propertyList.add(baseClientProperty); + } else { + propertyList.addAll(list); + } + } + //list partition by 只支持raw的方式 + if (partitionedBy instanceof ListPartitionedBy) { + StarRocksOutVisitor starRocksOutVisitor = new StarRocksOutVisitor(StarRocksContext.builder().build()); + ListPartitionedBy listPartitionedBy = (ListPartitionedBy)partitionedBy; + starRocksOutVisitor.visitListPartitionedBy(listPartitionedBy, 0); + StringProperty baseClientProperty = new StringProperty(); + baseClientProperty.setKey(StarRocksProperty.TABLE_PARTITION_RAW.getValue()); + baseClientProperty.setValueString(starRocksOutVisitor.getBuilder().toString()); + propertyList.add(baseClientProperty); + } + + return propertyList; + } + + /** + * 如果是三层模型，那么必须是a.b.c三层定义 + * 如果是二层模型，那么传入的a.b, 那么只会将a认为是database + * + * @param createTable + * @param transformContext + * @return + */ + @Override + protected String toSchema(CreateTable createTable, StarRocksContext transformContext) { + QualifiedName qualifiedName = createTable.getQualifiedName(); + if (!qualifiedName.isJoinPath()) { + return transformContext.getSchema(); + } + boolean isSecondSchema = qualifiedName.getOriginalParts().size() == SECOND_INDEX; + if (isSecondSchema) { + return transformContext.getSchema(); + } + boolean isThirdSchema = qualifiedName.getOriginalParts().size() == THIRD_INDEX; + if (isThirdSchema) { + return qualifiedName.getParts().get(1); + } + return transformContext.getSchema(); + } + + /** + * get database + * + * @param createTable + * @param database + * @return + */ + protected String toDatabase(CreateTable createTable, String database) { + QualifiedName qualifiedName = createTable.getQualifiedName(); + boolean isThirdSchema = qualifiedName.isJoinPath() && qualifiedName.getOriginalParts().size() == THIRD_INDEX; + if (isThirdSchema) { + return qualifiedName.getFirst(); + } + boolean isSecond = qualifiedName.isJoinPath() && qualifiedName.getOriginalParts().size() == SECOND_INDEX; + if (isSecond) { + return qualifiedName.getFirst(); + } + return database; + } + + /** + * 将partition desc转为 baseClientProperty + * + * @param partitionDesc + */ + private BaseClientProperty parseRangePartition(PartitionDesc partitionDesc) { + if (partitionDesc instanceof SingleRangePartition) { + SingleRangePartition singleRangePartition = (SingleRangePartition)partitionDesc; + SingleRangeClientPartition rangePartitionValue = new SingleRangeClientPartition(); + rangePartitionValue.setName(singleRangePartition.getName().getValue()); + rangePartitionValue.setIfNotExists(singleRangePartition.isIfNotExists()); + List propertyList = singleRangePartition.getPropertyList(); + if (propertyList != null && !propertyList.isEmpty()) { + LinkedHashMap linkedHashMap = Maps.newLinkedHashMap(); + for (Property p : propertyList) { + linkedHashMap.put(p.getName(), p.getValue()); + } + rangePartitionValue.setProperties(linkedHashMap); + } + BaseClientPartitionKey partitionKey = toClientPartitionKey(singleRangePartition.getPartitionKey()); + rangePartitionValue.setPartitionKey(partitionKey); + + //property + SingleRangePartitionProperty singleRangePartitionProperty = new SingleRangePartitionProperty(); + singleRangePartitionProperty.setValue(rangePartitionValue); + return singleRangePartitionProperty; + + } else if (partitionDesc instanceof MultiRangePartition) { + MultiRangePartition multiRangePartition = (MultiRangePartition)partitionDesc; + MultiRangeClientPartition multiRangePartitionValue = new MultiRangeClientPartition(); + multiRangePartitionValue.setStart(multiRangePartition.getStart().getValue()); + multiRangePartitionValue.setEnd(multiRangePartition.getEnd().getValue()); + if (multiRangePartition.getLongLiteral() != null) { + LongLiteral longLiteral = multiRangePartition.getLongLiteral(); + multiRangePartitionValue.setInterval(longLiteral.getValue()); + } + IntervalLiteral intervalLiteral = multiRangePartition.getIntervalLiteral(); + if (intervalLiteral != null) { + DateTimeEnum fromDateTime = intervalLiteral.getFromDateTime(); + multiRangePartitionValue.setDateTimeEnum(fromDateTime); + LongLiteral value = (LongLiteral)intervalLiteral.getValue(); + multiRangePartitionValue.setInterval(value.getValue()); + } + //property + MultiRangePartitionProperty multiRangePartitionProperty = new MultiRangePartitionProperty(); + multiRangePartitionProperty.setValue(multiRangePartitionValue); + return multiRangePartitionProperty; + } + return null; + } + + private BaseClientPartitionKey toClientPartitionKey(PartitionKey partitionKey) { + if (partitionKey instanceof ArrayPartitionKey) { + ArrayPartitionKey arrayPartitionKey = (ArrayPartitionKey)partitionKey; + List partitionValues1 = arrayPartitionKey.getPartitionValues(); + ArrayClientPartitionKey arrayClientPartitionKey = new ArrayClientPartitionKey(); + List> partitionValues = partitionValues1.stream().map(x -> { + List partitionValueList = x.getPartitionValueList(); + return partitionValueList.stream().map(p -> { + PartitionClientValue partitionClientValue = new PartitionClientValue(); + partitionClientValue.setMaxValue(p.isMaxValue()); + if (p.getStringLiteral() != null) { + partitionClientValue.setValue(p.getStringLiteral().getValue()); + } + return partitionClientValue; + }).collect(Collectors.toList()); + }).collect(Collectors.toList()); + arrayClientPartitionKey.setPartitionValue(partitionValues); + return arrayClientPartitionKey; + } else if (partitionKey instanceof LessThanPartitionKey) { + LessThanPartitionKey lessThanPartitionKey = (LessThanPartitionKey)partitionKey; + LessThanClientPartitionKey lessThanClientPartitionKey = new LessThanClientPartitionKey(); + lessThanClientPartitionKey.setMaxValue(lessThanPartitionKey.isMaxValue()); + List list = lessThanPartitionKey.getPartitionValues().getPartitionValueList().stream().map( + x -> { + PartitionClientValue partitionClientValue = new PartitionClientValue(); + partitionClientValue.setMaxValue(x.isMaxValue()); + if (x.getStringLiteral() != null) { + partitionClientValue.setValue(x.getStringLiteral().getValue()); + } + return partitionClientValue; + } + ).collect(Collectors.toList()); + lessThanClientPartitionKey.setPartitionValueList(list); + return lessThanClientPartitionKey; + } + return null; + } + + private List toRangePartition(List properties) { + if (properties == null || properties.isEmpty()) { + return Lists.newArrayList(); + } + List first = properties.stream().filter( + p -> StringUtils.equalsIgnoreCase(p.getKey(), StarRocksProperty.TABLE_RANGE_PARTITION.getValue())).collect(Collectors.toList()); + List list = Lists.newArrayList(); + for (BaseClientProperty baseClientProperty : first) { + toPartitionDesc(list, baseClientProperty); + } + return list; + } + + private void toPartitionDesc(List list, BaseClientProperty baseClientProperty) { + if (baseClientProperty instanceof SingleRangePartitionProperty) { + SingleRangePartitionProperty value = (SingleRangePartitionProperty)baseClientProperty; + SingleRangeClientPartition rangePartitionValue = value.getValue(); + BaseClientPartitionKey partitionKeyValue = rangePartitionValue.getPartitionKey(); + PartitionKey partitionKey = null; + if (partitionKeyValue instanceof LessThanClientPartitionKey) { + LessThanClientPartitionKey lessThanClientPartitionKey = (LessThanClientPartitionKey)partitionKeyValue; + List partitionValueList = Optional.ofNullable(lessThanClientPartitionKey.getPartitionValueList()).orElse( + Lists.newArrayList()); + List collect = partitionValueList + .stream().map(x -> { + if (x.getValue() != null) { + PartitionValue partitionValue = new PartitionValue(x.isMaxValue(), new StringLiteral(x.getValue())); + return partitionValue; + } else { + PartitionValue partitionValue = new PartitionValue(x.isMaxValue(), null); + return partitionValue; + } + }) + .collect(Collectors.toList()); + partitionKey = new LessThanPartitionKey( + lessThanClientPartitionKey.isMaxValue(), + new ListPartitionValue(collect) + ); + } else if (partitionKeyValue instanceof ArrayClientPartitionKey) { + ArrayClientPartitionKey arrayClientPartitionKey = (ArrayClientPartitionKey)partitionKeyValue; + List collect = arrayClientPartitionKey.getPartitionValue().stream().map( + x -> { + List partitionValueList = Lists.newArrayList(); + for (PartitionClientValue partitionClientValue : x) { + PartitionValue partitionValue = new PartitionValue(partitionClientValue.isMaxValue(), + new StringLiteral(partitionClientValue.getValue())); + partitionValueList.add(partitionValue); + } + return new ListPartitionValue(partitionValueList); + } + ).collect(Collectors.toList()); + partitionKey = new ArrayPartitionKey(collect); + } + PartitionDesc rangePartition = new SingleRangePartition( + new Identifier(rangePartitionValue.getName()), + rangePartitionValue.isIfNotExists(), + partitionKey, + null + ); + list.add(rangePartition); + } else if (baseClientProperty instanceof MultiRangePartitionProperty) { + MultiRangePartitionProperty multiRangePartitionProperty = (MultiRangePartitionProperty)baseClientProperty; + MultiRangeClientPartition value = multiRangePartitionProperty.getValue(); + LongLiteral longLiteral = new LongLiteral(String.valueOf(value.getInterval())); + PartitionDesc rangePartition = new MultiRangePartition( + new StringLiteral(value.getStart()), + new StringLiteral(value.getEnd()), + value.getDateTimeEnum() != null ? new IntervalLiteral(longLiteral, value.getDateTimeEnum()) : null, + value.getDateTimeEnum() == null ? longLiteral : null + ); + list.add(rangePartition); + } + } + + protected BaseConstraint setPrimaryConstraintColumns(List columns) { + if (CollectionUtils.isEmpty(columns)) { + return null; + } + List primaryKeyColumns = columns.stream().filter( + c -> BooleanUtils.isTrue(c.isPrimaryKey()) + ).collect(Collectors.toList()); + BaseConstraint baseConstraint = null; + if (primaryKeyColumns.size() > 0) { + List list = new ArrayList<>(); + for (Column column : primaryKeyColumns) { + list.add(new Identifier(column.getName())); + column.setPrimaryKey(false); + column.setNullable(false); + } + baseConstraint = new PrimaryConstraint(IdentifierUtil.sysIdentifier(), list); + } + return baseConstraint; + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksPropertyConverter.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksPropertyConverter.java new file mode 100644 index 0000000..b46a171 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksPropertyConverter.java @@ -0,0 +1,53 @@ +package com.aliyun.fastmodel.transform.starrocks.client.converter; + +import java.util.Map; +import java.util.function.Function; + +import com.aliyun.fastmodel.transform.api.client.converter.BasePropertyConverter; +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.DistributeBucketsNum; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.DistributeHash; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.ReplicationNum; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import com.google.common.collect.Maps; + +/** + * StarRocksPropertyConverter + * + * @author panguanjing + * @date 2023/9/16 + */ +public class StarRocksPropertyConverter extends BasePropertyConverter { + + private Map> functionMap = Maps.newHashMap(); + + public StarRocksPropertyConverter() { + init(); + } + + private void init() { + functionMap.put(StarRocksProperty.TABLE_DISTRIBUTED_BUCKETS.getValue(), + (k) -> { + DistributeBucketsNum distributeBucketsNum = new DistributeBucketsNum(); + distributeBucketsNum.setValueString(k); + return distributeBucketsNum; + }); + + functionMap.put(StarRocksProperty.TABLE_DISTRIBUTED_HASH.getValue(), (k) -> { + DistributeHash distributeHash = new DistributeHash(); + distributeHash.setValueString(k); + return distributeHash; + }); + + functionMap.put(StarRocksProperty.TABLE_REPLICATION_NUM.getValue(), (k) -> { + ReplicationNum distributeHash = new ReplicationNum(); + distributeHash.setValueString(k); + return distributeHash; + }); + } + + @Override + protected Map> getFunctionMap() { + return functionMap; + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/column/AggrColumnProperty.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/column/AggrColumnProperty.java new file mode 100644 index 0000000..bbc2d9a --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/column/AggrColumnProperty.java @@ -0,0 +1,30 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.column; + +import java.util.Locale; + +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.AggDesc; + +/** + * aggr column property + * + * @author panguanjing + * @date 2023/9/17 + */ +public class AggrColumnProperty extends BaseClientProperty { + + public AggrColumnProperty() { + this.setKey(StarRocksProperty.COLUMN_AGG_DESC.getValue()); + } + + @Override + public String valueString() { + return value.name(); + } + + @Override + public void setValueString(String value) { + this.value = AggDesc.valueOf(value.toUpperCase(Locale.ROOT)); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/DistributeBucketsNum.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/DistributeBucketsNum.java new file mode 100644 index 0000000..ea09eb5 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/DistributeBucketsNum.java @@ -0,0 +1,30 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table; + +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import org.apache.commons.lang3.StringUtils; + +/** + * 分桶列的个数 + * + * @author panguanjing + * @date 2023/9/20 + */ +public class DistributeBucketsNum extends BaseClientProperty { + public DistributeBucketsNum() { + setKey(StarRocksProperty.TABLE_DISTRIBUTED_BUCKETS.getValue()); + } + + @Override + public String valueString() { + return String.valueOf(value); + } + + @Override + public void setValueString(String value) { + if (!StringUtils.isNumeric(value)) { + return; + } + this.value = Integer.parseInt(value); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/DistributeHash.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/DistributeHash.java new file mode 100644 index 0000000..524a6e9 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/DistributeHash.java @@ -0,0 +1,34 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table; + +import java.util.List; + +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; + +/** + * 分桶列的个数 + * + * @author panguanjing + * @date 2023/9/20 + */ +public class DistributeHash extends BaseClientProperty> { + + public DistributeHash() { + setKey(StarRocksProperty.TABLE_DISTRIBUTED_HASH.getValue()); + } + + @Override + public String valueString() { + return String.join(",", value); + } + + @Override + public void setValueString(String value) { + if (StringUtils.isBlank(value)) { + return; + } + this.value = Lists.newArrayList(StringUtils.split(value, ",")); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/MultiRangePartitionProperty.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/MultiRangePartitionProperty.java new file mode 100644 index 0000000..3ed60fa --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/MultiRangePartitionProperty.java @@ -0,0 +1,34 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table; + +import com.alibaba.fastjson.JSON; + +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.MultiRangeClientPartition; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import org.apache.commons.lang3.StringUtils; + +/** + * multi range partition + * + * @author panguanjing + * @date 2023/9/17 + */ +public class MultiRangePartitionProperty extends BaseClientProperty { + + public MultiRangePartitionProperty() { + this.setKey(StarRocksProperty.TABLE_RANGE_PARTITION.getValue()); + } + + @Override + public String valueString() { + return JSON.toJSONString(value); + } + + @Override + public void setValueString(String value) { + if (StringUtils.isNotBlank(value)) { + return; + } + this.setValue(JSON.parseObject(value, MultiRangeClientPartition.class)); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/ReplicationNum.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/ReplicationNum.java new file mode 100644 index 0000000..8e2f4f0 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/ReplicationNum.java @@ -0,0 +1,30 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table; + +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import org.apache.commons.lang3.StringUtils; + +/** + * ReplicationNum + * + * @author panguanjing + * @date 2023/9/22 + */ +public class ReplicationNum extends BaseClientProperty { + public ReplicationNum() { + setKey(StarRocksProperty.TABLE_REPLICATION_NUM.getValue()); + } + + @Override + public String valueString() { + return String.valueOf(value); + } + + @Override + public void setValueString(String value) { + if (StringUtils.isBlank(value)) { + return; + } + this.value = Integer.parseInt(value); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/SingleRangePartitionProperty.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/SingleRangePartitionProperty.java new file mode 100644 index 0000000..2e86ef6 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/SingleRangePartitionProperty.java @@ -0,0 +1,34 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table; + +import com.alibaba.fastjson.JSON; + +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.SingleRangeClientPartition; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import org.apache.commons.lang3.StringUtils; + +/** + * single range partition + * + * @author panguanjing + * @date 2023/9/17 + */ +public class SingleRangePartitionProperty extends BaseClientProperty { + + public SingleRangePartitionProperty() { + this.setKey(StarRocksProperty.TABLE_RANGE_PARTITION.getValue()); + } + + @Override + public String valueString() { + return JSON.toJSONString(value); + } + + @Override + public void setValueString(String value) { + if (StringUtils.isNotBlank(value)) { + return; + } + this.setValue(JSON.parseObject(value, SingleRangeClientPartition.class)); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/TablePartitionRaw.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/TablePartitionRaw.java new file mode 100644 index 0000000..311ad7d --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/TablePartitionRaw.java @@ -0,0 +1,27 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table; + +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; + +/** + * 分区定义的文本化描述 + * + * @author panguanjing + * @date 2023/9/20 + */ +public class TablePartitionRaw extends BaseClientProperty { + + public TablePartitionRaw() { + this.setKey(StarRocksProperty.TABLE_PARTITION_RAW.getValue()); + } + + @Override + public String valueString() { + return this.value; + } + + @Override + public void setValueString(String value) { + this.value = value; + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/ArrayClientPartitionKey.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/ArrayClientPartitionKey.java new file mode 100644 index 0000000..e8e758b --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/ArrayClientPartitionKey.java @@ -0,0 +1,25 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table.partition; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * ArrayClientPartitionKey + * + * @author panguanjing + * @date 2023/10/17 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ArrayClientPartitionKey extends BaseClientPartitionKey { + + private List> partitionValue; +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/BaseClientPartitionKey.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/BaseClientPartitionKey.java new file mode 100644 index 0000000..c18f729 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/BaseClientPartitionKey.java @@ -0,0 +1,10 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table.partition; + +/** + * partition key + * + * @author panguanjing + * @date 2023/10/17 + */ +public abstract class BaseClientPartitionKey { +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/LessThanClientPartitionKey.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/LessThanClientPartitionKey.java new file mode 100644 index 0000000..bf69170 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/LessThanClientPartitionKey.java @@ -0,0 +1,27 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table.partition; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * Desc: + * + * @author panguanjing + * @date 2023/10/16 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class LessThanClientPartitionKey extends BaseClientPartitionKey{ + + private boolean maxValue; + + private List partitionValueList; +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/MultiRangeClientPartition.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/MultiRangeClientPartition.java new file mode 100644 index 0000000..016f616 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/MultiRangeClientPartition.java @@ -0,0 +1,35 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table.partition; + +import com.aliyun.fastmodel.core.tree.expr.enums.DateTimeEnum; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * multi range partition value + * + * @author panguanjing + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class MultiRangeClientPartition { + /** + * start + */ + private String start; + /** + * end + */ + private String end; + /** + * interval + */ + private Long interval; + /** + * time enum + */ + private DateTimeEnum dateTimeEnum; +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/PartitionClientValue.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/PartitionClientValue.java new file mode 100644 index 0000000..85b3d50 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/PartitionClientValue.java @@ -0,0 +1,27 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table.partition; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Desc: + * + * @author panguanjing + * @date 2023/10/23 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class PartitionClientValue { + /** + * max value + */ + private boolean maxValue; + /** + * value + */ + private String value; +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/SingleRangeClientPartition.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/SingleRangeClientPartition.java new file mode 100644 index 0000000..c8e4315 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/client/property/table/partition/SingleRangeClientPartition.java @@ -0,0 +1,39 @@ +package com.aliyun.fastmodel.transform.starrocks.client.property.table.partition; + +import java.util.LinkedHashMap; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * range partition value + * + * @author panguanjing + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SingleRangeClientPartition { + /** + * column name + */ + private String name; + /** + * if not exists + */ + private boolean ifNotExists; + /** + * partition key + */ + private BaseClientPartitionKey partitionKey; + /** + * properties + */ + private LinkedHashMap properties; + + + +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/context/StarRocksContext.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/context/StarRocksContext.java new file mode 100644 index 0000000..b3c29a3 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/context/StarRocksContext.java @@ -0,0 +1,31 @@ +package com.aliyun.fastmodel.transform.starrocks.context; + +import com.aliyun.fastmodel.transform.api.context.TransformContext; + +/** + * StarRocksContext + * + * @author panguanjing + * @date 2023/9/5 + */ +public class StarRocksContext extends TransformContext { + public StarRocksContext(TransformContext context) { + super(context); + } + + public StarRocksContext(Builder tBuilder) { + super(tBuilder); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder extends TransformContext.Builder { + + @Override + public StarRocksContext build() { + return new StarRocksContext(this); + } + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksExpressionVisitor.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksExpressionVisitor.java new file mode 100644 index 0000000..6a521e6 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksExpressionVisitor.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2022. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.fastmodel.transform.starrocks.format; + +import com.aliyun.fastmodel.common.utils.StripUtils; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.transform.api.format.DefaultExpressionVisitor; +import com.aliyun.fastmodel.transform.starrocks.context.StarRocksContext; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype.StarRocksGenericDataType; +import com.aliyun.fastmodel.transform.starrocks.parser.util.StarRocksReservedWordUtil; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import org.apache.commons.lang3.StringUtils; + +/** + * hologres的表达式格式化 + * + * @author panguanjing + * @date 2022/6/24 + */ +public class StarRocksExpressionVisitor extends DefaultExpressionVisitor implements StarRocksAstVisitor { + + private final StarRocksContext starRocksContext; + + public StarRocksExpressionVisitor(StarRocksContext starRocksContext) { + this.starRocksContext = starRocksContext; + } + + @Override + public String visitStarRocksGenericDataType(StarRocksGenericDataType starRocksGenericDataType, Void context) { + return super.visitGenericDataType(starRocksGenericDataType, context); + } + + @Override + public String visitIdentifier(Identifier node, Void context) { + String value = StringUtils.isNotBlank(node.getOrigin()) ? + StripUtils.strip(node.getOrigin()) : node.getValue(); + if (!node.isDelimited()) { + boolean reservedKeyWord = StarRocksReservedWordUtil.isReservedKeyWord(value); + //如果node是关键字，那么进行转义处理 + if (reservedKeyWord) { + return StripUtils.addPrefix(value); + } + return value; + } else { + String strip = StripUtils.strip(value); + return StripUtils.addPrefix(strip); + } + } + + @Override + public String formatStringLiteral(String s) { + if (s == null) { + return null; + } + String result = s.replace("\\", "\\\\"); + result = result.replace("\"", "\\\""); + return "\"" + result + "\""; + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksOutVisitor.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksOutVisitor.java new file mode 100644 index 0000000..3ce3bc0 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksOutVisitor.java @@ -0,0 +1,514 @@ +package com.aliyun.fastmodel.transform.starrocks.format; + +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.aliyun.fastmodel.core.formatter.ExpressionVisitor; +import com.aliyun.fastmodel.core.formatter.FastModelVisitor; +import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; +import com.aliyun.fastmodel.core.tree.expr.BaseExpression; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.expr.literal.ListStringLiteral; +import com.aliyun.fastmodel.core.tree.statement.table.AddCols; +import com.aliyun.fastmodel.core.tree.statement.table.ChangeCol; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; +import com.aliyun.fastmodel.core.tree.statement.table.DropCol; +import com.aliyun.fastmodel.core.tree.statement.table.RenameCol; +import com.aliyun.fastmodel.core.tree.statement.table.RenameTable; +import com.aliyun.fastmodel.core.tree.statement.table.SetTableProperties; +import com.aliyun.fastmodel.core.tree.statement.table.UnSetTableProperties; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.BaseConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.PrimaryConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.UniqueConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.index.TableIndex; +import com.aliyun.fastmodel.core.tree.util.PropertyUtil; +import com.aliyun.fastmodel.transform.starrocks.context.StarRocksContext; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.AggregateConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.DuplicateConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ArrayPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.LessThanPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiItemListPartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiRangePartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionDesc; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.RangePartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleItemListPartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleRangePartition; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import com.google.common.base.Objects; +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; + +/** + * StarRocksVisitor + * + * @author panguanjing + * @date 2023/9/5 + */ +public class StarRocksOutVisitor extends FastModelVisitor implements StarRocksAstVisitor { + + public static final String MAXVALUE = "MAXVALUE"; + private final StarRocksContext starRocksContext; + + public StarRocksOutVisitor(StarRocksContext context) { + this.starRocksContext = context; + } + + @Override + public Boolean visitCreateTable(CreateTable node, Integer indent) { + boolean columnEmpty = node.isColumnEmpty(); + //maxcompute不支持没有列的表 + boolean executable = !columnEmpty; + builder.append("CREATE TABLE "); + if (node.isNotExists()) { + builder.append("IF NOT EXISTS "); + } + String tableName = getCode(node.getQualifiedName()); + builder.append(tableName); + if (!columnEmpty) { + builder.append("\n(\n"); + String elementIndent = indentString(indent + 1); + String columnList = formatColumnList(node.getColumnDefines(), elementIndent); + builder.append(columnList); + + if (!node.isIndexEmpty()) { + for (TableIndex tableIndex : node.getTableIndexList()) { + builder.append(",\n"); + process(tableIndex, indent + 1); + } + } + builder.append("\n").append(")"); + } + + //format engine + if (node.getProperties() != null) { + Optional first = node.getProperties().stream().filter( + p -> StringUtils.equalsIgnoreCase(p.getName(), StarRocksProperty.TABLE_ENGINE.getValue())).findFirst(); + first.ifPresent(property -> builder.append("\nENGINE=").append(property.getValue())); + } + + //constraint + if (!node.isConstraintEmpty()) { + appendConstraint(node, indent); + } + + if (node.getComment() != null) { + builder.append("\n"); + builder.append(formatComment(node.getComment(), true)); + } + //format partitioned by + if (!node.isPartitionEmpty()) { + builder.append("\n"); + process(node.getPartitionedBy(), indent); + } else { + String propertyValue = PropertyUtil.getPropertyValue(node.getProperties(), StarRocksProperty.TABLE_PARTITION_RAW.getValue()); + if (StringUtils.isNotBlank(propertyValue)) { + builder.append("\n"); + builder.append(propertyValue); + } + } + + //distribute desc + appendDistributeBy(node.getProperties()); + + + if (!node.isPropertyEmpty()) { + String prop = formatProperty(node.getProperties()); + if (StringUtils.isNotBlank(prop)) { + builder.append("\n"); + builder.append("PROPERTIES ("); + builder.append(prop); + builder.append(")"); + } + } + return executable; + } + + private void appendDistributeBy(List properties) { + String propertyValue = PropertyUtil.getPropertyValue(properties, StarRocksProperty.TABLE_DISTRIBUTED_HASH.getValue()); + if (StringUtils.isBlank(propertyValue)) { + return; + } + builder.append("\n"); + builder.append("DISTRIBUTED BY HASH"); + String formatPropertyValue = formatIdentifierListValue(propertyValue); + builder.append("(").append(formatPropertyValue).append(")"); + + String buckets = PropertyUtil.getPropertyValue(properties, StarRocksProperty.TABLE_DISTRIBUTED_BUCKETS.getValue()); + if (StringUtils.isNotBlank(buckets)) { + builder.append(" BUCKETS "); + builder.append(buckets); + } + } + + private String formatIdentifierListValue(String propertyValue) { + return Lists.newArrayList(propertyValue.split(",")).stream() + .map(Identifier::new).map(this::formatExpression).collect(Collectors.joining(",")); + } + + @Override + protected String formatProperty(List properties) { + List collect = properties.stream().filter( + p -> { + StarRocksProperty byValue = StarRocksProperty.getByValue(p.getName()); + return byValue == null || byValue.isSupportPrint(); + } + ).collect(Collectors.toList()); + return super.formatProperty(collect); + } + + @Override + protected String formatColumnDefinition(ColumnDefinition column, Integer max) { + BaseDataType dataType = column.getDataType(); + String col = formatColName(column.getColName(), max); + StringBuilder sb = new StringBuilder().append(col); + if (dataType != null) { + sb.append(" ").append(formatDataType(dataType)); + } + List columnProperties = column.getColumnProperties(); + //charset + String charSet = PropertyUtil.getPropertyValue(columnProperties, StarRocksProperty.COLUMN_CHAR_SET.getValue()); + if (StringUtils.isNotBlank(charSet)) { + sb.append(" ").append("CHARSET ").append(charSet); + } + //key + String key = PropertyUtil.getPropertyValue(columnProperties, StarRocksProperty.COLUMN_KEY.getValue()); + if (StringUtils.isNotBlank(key)) { + sb.append(" ").append("KEY"); + } + //aggDesc + String propertyValue = PropertyUtil.getPropertyValue(columnProperties, StarRocksProperty.COLUMN_AGG_DESC.getValue()); + if (StringUtils.isNotBlank(propertyValue)) { + sb.append(" ").append(propertyValue); + } + boolean isNotNull = column.getNotNull() != null && column.getNotNull(); + if (isNotNull) { + sb.append(" NOT NULL"); + } else if (BooleanUtils.isFalse(column.getNotNull())) { + sb.append(" NULL"); + } + if (column.getDefaultValue() != null) { + sb.append(" DEFAULT ").append(formatExpression(column.getDefaultValue())); + } + sb.append(formatComment(column.getComment(), isEndNewLine(sb.toString()))); + return sb.toString(); + } + + private void appendConstraint(CreateTable node, Integer indent) { + List constraintStatements = node.getConstraintStatements(); + if (CollectionUtils.isEmpty(constraintStatements)) { + return; + } + for (BaseConstraint baseConstraint : node.getConstraintStatements()) { + builder.append("\n"); + process(baseConstraint, indent); + } + } + + @Override + public Boolean visitAggregateConstraint(AggregateConstraint aggregateConstraint, Integer context) { + builder.append("AGGREGATE KEY ("); + List colNames = aggregateConstraint.getColumns(); + builder.append(colNames.stream().map(this::formatExpression).collect(Collectors.joining(","))); + builder.append(")"); + return true; + } + + @Override + public Boolean visitDuplicateConstraint(DuplicateConstraint duplicateConstraint, Integer context) { + builder.append("DUPLICATE KEY ("); + List colNames = duplicateConstraint.getColumns(); + builder.append(colNames.stream().map(this::formatExpression).collect(Collectors.joining(","))); + builder.append(")"); + return true; + } + + @Override + public Boolean visitPrimaryConstraint(PrimaryConstraint primaryConstraint, Integer ident) { + builder.append("PRIMARY KEY ("); + List colNames = primaryConstraint.getColNames(); + builder.append(colNames.stream().map(this::formatExpression).collect(Collectors.joining(","))); + builder.append(")"); + return true; + } + + @Override + protected String formatStringLiteral(String s) { + return new StarRocksExpressionVisitor(starRocksContext).formatStringLiteral(s); + } + + @Override + public Boolean visitUniqueConstraint(UniqueConstraint uniqueConstraint, Integer indent) { + builder.append("UNIQUE KEY ("); + builder.append(uniqueConstraint.getColumnNames().stream().map(this::formatExpression).collect(Collectors.joining(","))); + builder.append(")"); + return true; + } + + @Override + protected String formatExpression(BaseExpression baseExpression) { + return new StarRocksExpressionVisitor(starRocksContext).process(baseExpression); + } + + @Override + public Boolean visitChangeCol(ChangeCol renameCol, Integer context) { + //starRocks不支持修改列名 + Identifier oldColName = renameCol.getOldColName(); + ColumnDefinition columnDefinition = renameCol.getColumnDefinition(); + Identifier targetColName = columnDefinition.getColName(); + if (!Objects.equal(oldColName, targetColName)) { + return false; + } + builder.append("ALTER TABLE ").append(getCode(renameCol.getQualifiedName())); + builder.append(" MODIFY COLUMN"); + builder.append(" ").append(formatColumnDefinition(columnDefinition, 0)); + return true; + } + + @Override + public Boolean visitDropCol(DropCol dropCol, Integer context) { + return super.visitDropCol(dropCol, context); + } + + @Override + public Boolean visitRenameTable(RenameTable renameTable, Integer context) { + builder.append("ALTER TABLE "); + builder.append(getCode(renameTable.getQualifiedName())); + builder.append(" RENAME ").append(getCode(renameTable.getTarget())); + return true; + } + + /** + * starRocks不支持修改列名 + * + * @param renameCol + * @param context + * @return + */ + @Override + public Boolean visitRenameCol(RenameCol renameCol, Integer context) { + return !super.visitRenameCol(renameCol, context); + } + + @Override + public Boolean visitSetTableProperties(SetTableProperties setTableProperties, Integer context) { + builder.append("ALTER TABLE "); + builder.append(formatName(setTableProperties.getQualifiedName())); + builder.append(" SET ("); + builder.append(formatProperty(setTableProperties.getProperties())); + builder.append(")"); + return true; + } + + @Override + public Boolean visitUnSetTableProperties(UnSetTableProperties unSetTableProperties, Integer context) { + builder.append("ALTER TABLE "); + builder.append(formatName(unSetTableProperties.getQualifiedName())); + builder.append(" SET ("); + List propertyKeys = unSetTableProperties.getPropertyKeys(); + String a = propertyKeys.stream().map( + k -> formatStringLiteral(k) + "=" + "\"\"" + ).collect(Collectors.joining(",")); + builder.append(a); + builder.append(")"); + return true; + } + + @Override + public Boolean visitAddCols(AddCols addCols, Integer context) { + builder.append("ALTER TABLE ").append(getCode(addCols.getQualifiedName())); + builder.append(" ADD COLUMN\n").append('(').append("\n"); + String columnList = formatColumnList(addCols.getColumnDefineList(), indentString(context + 1)); + builder.append(columnList); + builder.append("\n"); + builder.append(')'); + return true; + } + + @Override + public Boolean visitRangePartitionedBy(RangePartitionedBy starRocksPartitionedBy, Integer indent) { + builder.append("PARTITION BY RANGE ("); + List partitioned = starRocksPartitionedBy.getColumnDefinitions(); + String collect = partitioned.stream().map( + x -> this.formatExpression(x.getColName()) + ).collect(Collectors.joining(",")); + builder.append(collect).append(")"); + List rangePartitions = starRocksPartitionedBy.getRangePartitions(); + if (CollectionUtils.isNotEmpty(rangePartitions)) { + builder.append("\n"); + } + Iterator iterator = rangePartitions.iterator(); + builder.append("("); + if (iterator.hasNext()) { + builder.append("\n"); + process(iterator.next(), indent + 1); + while (iterator.hasNext()) { + builder.append(",\n"); + process(iterator.next(), indent + 1); + } + } + if (CollectionUtils.isNotEmpty(rangePartitions)) { + builder.append("\n"); + } + builder.append(")"); + return true; + } + + @Override + public Boolean visitListPartitionedBy(ListPartitionedBy listPartitionedBy, Integer indent) { + builder.append("PARTITION BY LIST ("); + List partitioned = listPartitionedBy.getColumnDefinitions(); + String collect = partitioned.stream().map( + x -> formatExpression(x.getColName()) + ).collect(Collectors.joining(",")); + builder.append(collect).append(")"); + builder.append("\n"); + List rangePartitions = listPartitionedBy.getRangePartitions(); + Iterator iterator = rangePartitions.iterator(); + if (iterator.hasNext()) { + builder.append("(\n"); + process(iterator.next(), indent + 1); + while (iterator.hasNext()) { + builder.append(",\n"); + process(iterator.next(), indent + 1); + } + builder.append("\n)"); + } + return true; + } + + @Override + public Boolean visitSingleItemListPartition(SingleItemListPartition singleItemListPartition, Integer context) { + builder.append("PARTITION "); + if (singleItemListPartition.isIfNotExists()) { + builder.append("IF NOT EXISTS "); + } + builder.append(formatExpression(singleItemListPartition.getName())); + builder.append(" VALUES IN ("); + String collect = getListString(singleItemListPartition.getListStringLiteral()); + builder.append(collect); + builder.append(")"); + appendProperty(singleItemListPartition.getPropertyList()); + return true; + } + + private void appendProperty(List propertyList) { + if (propertyList == null || propertyList.isEmpty()) { + return; + } + builder.append(" "); + builder.append("("); + builder.append(formatProperty(propertyList)); + builder.append(")"); + } + + private String getListString(ListStringLiteral listStringLiteral) { + return listStringLiteral.getStringLiteralList().stream().map( + this::formatExpression + ).collect(Collectors.joining(",")); + } + + private String getListString(ListPartitionValue listStringLiteral) { + return listStringLiteral.getPartitionValueList().stream().map( + this::formatPartitionValue + ).collect(Collectors.joining(",")); + } + + private String formatPartitionValue(PartitionValue partitionValue) { + if (partitionValue.isMaxValue()) { + return MAXVALUE; + } + return formatExpression(partitionValue.getStringLiteral()); + } + + @Override + public Boolean visitMultiItemListPartition(MultiItemListPartition multiItemListPartition, Integer context) { + builder.append("PARTITION "); + if (multiItemListPartition.isIfNotExists()) { + builder.append("IF NOT EXISTS "); + } + builder.append(formatExpression(multiItemListPartition.getName())); + builder.append(" VALUES IN ("); + String collect = multiItemListPartition.getListStringLiterals().stream().map( + x -> "(" + getListString(x) + ")" + ).collect(Collectors.joining(",")); + builder.append(collect); + builder.append(")"); + appendProperty(multiItemListPartition.getPropertyList()); + return true; + } + + @Override + public Boolean visitSingleRangePartition(SingleRangePartition singleRangePartition, Integer context) { + String s = indentString(context); + builder.append(s).append("PARTITION "); + if (singleRangePartition.isIfNotExists()) { + builder.append("IF NOT EXISTS "); + } + builder.append(formatExpression(singleRangePartition.getName())); + builder.append(" VALUES "); + process(singleRangePartition.getPartitionKey()); + + if (singleRangePartition.getPropertyList() != null) { + String property = formatProperty(singleRangePartition.getPropertyList()); + builder.append(property); + } + return true; + } + + @Override + public Boolean visitLessThanPartitionKey(LessThanPartitionKey lessThanPartitionKey, Integer context) { + builder.append("LESS THAN "); + if (lessThanPartitionKey.isMaxValue()) { + builder.append(MAXVALUE); + } else { + String collect = lessThanPartitionKey.getPartitionValues().getPartitionValueList().stream() + .map(this::formatPartitionValue) + .collect(Collectors.joining(",")); + builder.append("("); + builder.append(collect); + builder.append(")"); + } + return true; + } + + @Override + public Boolean visitArrayPartitionKey(ArrayPartitionKey arrayPartitionKey, Integer context) { + builder.append("["); + String value = arrayPartitionKey.getPartitionValues().stream().map( + p -> "(" + getListString(p) + ")" + ).collect(Collectors.joining(",")); + builder.append(value); + builder.append(")"); + return true; + } + + @Override + public Boolean visitMultiRangePartition(MultiRangePartition multiRangePartition, Integer context) { + String s = indentString(context); + builder.append(s).append("START"); + builder.append("(").append(formatExpression(multiRangePartition.getStart())); + builder.append(")"); + builder.append(" "); + builder.append("END"); + builder.append("("); + builder.append(formatExpression(multiRangePartition.getEnd())); + builder.append(")"); + builder.append(" EVERY ("); + if (multiRangePartition.getIntervalLiteral() != null) { + builder.append(formatExpression(multiRangePartition.getIntervalLiteral())); + } else if (multiRangePartition.getLongLiteral() != null) { + builder.append(formatExpression(multiRangePartition.getLongLiteral())); + } + builder.append(")"); + return true; + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksProperty.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksProperty.java new file mode 100644 index 0000000..876d15b --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksProperty.java @@ -0,0 +1,105 @@ +package com.aliyun.fastmodel.transform.starrocks.format; + +import com.aliyun.fastmodel.transform.api.format.PropertyKey; +import org.apache.commons.lang3.StringUtils; + +/** + * StarRocksProperty + * $$是代表系统属性，不会进行输出 + * + * @author panguanjing + * @date 2023/9/14 + */ +public enum StarRocksProperty implements PropertyKey { + /** + * engine + */ + TABLE_ENGINE("engine"), + + /** + * distribute hash + */ + TABLE_DISTRIBUTED_HASH("distributed_hash"), + + /** + * distribute buckets + */ + TABLE_DISTRIBUTED_BUCKETS("distributed_buckets"), + + /** + * index type + */ + TABLE_INDEX_TYPE("index_type"), + /** + * index comment + */ + TABLE_INDEX_COMMENT("index_comment"), + + /** + * rollup + */ + TABLE_ROLLUP("rollup"), + + /** + * range partition + */ + TABLE_RANGE_PARTITION("range_partition"), + + /** + * table range partition raw + */ + TABLE_PARTITION_RAW("range_partition_raw"), + + /** + * char set + */ + COLUMN_CHAR_SET("char_set"), + + /** + * column key + */ + COLUMN_KEY("column_key"), + + /** + * column agg desc + */ + COLUMN_AGG_DESC("column_agg_desc"), + + /** + * REPLICATION_NUM + */ + TABLE_REPLICATION_NUM("replication_num", true); + + private final String value; + + private final boolean supportPrint; + + StarRocksProperty(String value) { + this(value, false); + } + + StarRocksProperty(String value, boolean supportPrint) { + this.value = value; + this.supportPrint = supportPrint; + } + + public static StarRocksProperty getByValue(String value) { + StarRocksProperty[] starRocksProperties = StarRocksProperty.values(); + for (StarRocksProperty starRocksProperty : starRocksProperties) { + if (StringUtils.equalsIgnoreCase(starRocksProperty.getValue(), value)) { + return starRocksProperty; + } + } + return null; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isSupportPrint() { + return supportPrint; + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/StarRocksLanguageParser.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/StarRocksLanguageParser.java new file mode 100644 index 0000000..ed0af9e --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/StarRocksLanguageParser.java @@ -0,0 +1,64 @@ +package com.aliyun.fastmodel.transform.starrocks.parser; + +import java.util.function.Function; + +import com.aliyun.fastmodel.common.parser.ThrowingErrorListener; +import com.aliyun.fastmodel.common.parser.lexer.CaseChangingCharStream; +import com.aliyun.fastmodel.common.utils.StripUtils; +import com.aliyun.fastmodel.core.exception.ParseException; +import com.aliyun.fastmodel.core.parser.LanguageParser; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstBuilder; +import com.google.auto.service.AutoService; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CodePointCharStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.atn.PredictionMode; + +/** + * StarRocksLanguageParser + * + * @author panguanjing + * @date 2023/9/5 + */ +@AutoService(LanguageParser.class) +public class StarRocksLanguageParser implements LanguageParser { + + public static final ThrowingErrorListener LISTENER = new ThrowingErrorListener(); + + @Override + public Node parseNode(String text, ReverseContext context) throws ParseException { + return getNode(text, context, StarRocksParser::sqlStatements); + } + + private Node getNode(String text, ReverseContext context, Function functionalInterface) { + String code = StripUtils.appendSemicolon(text); + CodePointCharStream charStream = CharStreams.fromString(code); + CaseChangingCharStream caseChangingCharStream = new CaseChangingCharStream(charStream, true); + StarRocksLexer lexer = new StarRocksLexer(caseChangingCharStream); + lexer.removeErrorListeners(); + lexer.addErrorListener(LISTENER); + CommonTokenStream commonTokenStream = new CommonTokenStream(lexer); + StarRocksParser sparkParser = new StarRocksParser(commonTokenStream); + sparkParser.removeErrorListeners(); + sparkParser.addErrorListener(LISTENER); + ParserRuleContext tree; + try { + sparkParser.getInterpreter().setPredictionMode(PredictionMode.SLL); + tree = functionalInterface.apply(sparkParser); + } catch (Throwable e) { + commonTokenStream.seek(0); + sparkParser.getInterpreter().setPredictionMode(PredictionMode.LL); + tree = functionalInterface.apply(sparkParser); + } + return tree.accept(new StarRocksAstBuilder(context)); + } + + @Override + public BaseDataType parseDataType(String code, ReverseContext context) throws ParseException { + return (BaseDataType)getNode(code, context, StarRocksParser::type); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/AggDesc.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/AggDesc.java new file mode 100644 index 0000000..aee2cab --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/AggDesc.java @@ -0,0 +1,42 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree; + +/** + * column agg desc + * + * @author panguanjing + * @date 2023/9/16 + */ +public enum AggDesc { + /** + * sum + */ + SUM, + /** + * max + */ + MAX, + /** + * min + */ + MIN, + /** + * replace + */ + REPLACE, + /** + * hll union + */ + HLL_UNION, + /** + * bitmap union + */ + BITMAP_UNION, + /** + * percentile union + */ + PERCENTILE_UNION, + /** + * replace if not null + */ + REPLACE_IF_NOT_NULL; +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/AggregateConstraint.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/AggregateConstraint.java new file mode 100644 index 0000000..1c780aa --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/AggregateConstraint.java @@ -0,0 +1,44 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.AstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.CustomConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import lombok.Getter; + +/** + * aggregate constraint + * + * @author panguanjing + * @date 2023/9/11 + */ +@Getter +public class AggregateConstraint extends CustomConstraint { + + public static final String TYPE = "AGGREGATE"; + + private final List columns; + + public AggregateConstraint(Identifier constraintName, List columns, Boolean enable) { + super(constraintName, enable, TYPE); + this.columns = columns; + } + + public AggregateConstraint(Identifier constraintName, List columns) { + this(constraintName, columns, true); + } + + @Override + public R accept(AstVisitor visitor, C context) { + StarRocksAstVisitor starRocksAstVisitor = (StarRocksAstVisitor)visitor; + return starRocksAstVisitor.visitAggregateConstraint(this, context); + } + + @Override + public List getChildren() { + return columns; + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/DuplicateConstraint.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/DuplicateConstraint.java new file mode 100644 index 0000000..8d953b7 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/DuplicateConstraint.java @@ -0,0 +1,44 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.CustomConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import lombok.Getter; + +/** + * aggregate constraint + * + * @author panguanjing + * @date 2023/9/11 + */ +@Getter +public class DuplicateConstraint extends CustomConstraint { + + public static final String TYPE = "DUPLICATE"; + + private final List columns; + + public DuplicateConstraint(Identifier constraintName, List columns, Boolean enable) { + super(constraintName, enable, TYPE); + this.columns = columns; + } + + public DuplicateConstraint(Identifier constraintName, List columns) { + this(constraintName, columns, true); + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksAstVisitor = (StarRocksAstVisitor)visitor; + return starRocksAstVisitor.visitDuplicateConstraint(this, context); + } + + @Override + public List getChildren() { + return columns; + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksDataTypeName.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksDataTypeName.java new file mode 100644 index 0000000..27b1ce3 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksDataTypeName.java @@ -0,0 +1,233 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype; + +import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName; +import com.aliyun.fastmodel.transform.api.datatype.simple.ISimpleDataTypeName; +import com.aliyun.fastmodel.transform.api.datatype.simple.SimpleDataTypeName; +import org.apache.commons.lang3.StringUtils; + +/** + * starRocks DataType Name + * + * @author panguanjing + * @date 2023/9/12 + */ +public enum StarRocksDataTypeName implements ISimpleDataTypeName { + /** + * boolean + */ + BOOLEAN("BOOLEAN", Dimension.ZERO, SimpleDataTypeName.BOOLEAN), + + /** + * tinyint + */ + TINYINT("TINYINT", Dimension.ONE, SimpleDataTypeName.NUMBER), + + /** + * small int + */ + SMALLINT("SMALLINT", Dimension.ONE, SimpleDataTypeName.NUMBER), + + /** + * signed int + */ + SIGNED_INT("SIGNED INT", Dimension.ZERO, SimpleDataTypeName.NUMBER), + + /** + * singed integer + */ + SIGNED_INTEGER("SIGNED INTEGER", Dimension.ZERO, SimpleDataTypeName.NUMBER), + + /** + * int + */ + INT("INT", Dimension.ONE, SimpleDataTypeName.NUMBER), + + /** + * integer + */ + INTEGER("INTEGER", Dimension.ONE, SimpleDataTypeName.NUMBER), + + /** + * bigint + */ + BIGINT("BIGINT", Dimension.ONE, SimpleDataTypeName.NUMBER), + + /** + * large int + */ + LARGEINT("LARGEINT", Dimension.ONE, SimpleDataTypeName.NUMBER), + + /** + * float + */ + FLOAT("FLOAT", Dimension.ZERO, SimpleDataTypeName.NUMBER), + + /** + * double + */ + DOUBLE("DOUBLE", Dimension.ZERO, SimpleDataTypeName.NUMBER), + + /** + * date + */ + DATE("DATE", Dimension.ZERO, SimpleDataTypeName.DATE), + + /** + * datetime + */ + DATETIME("DATETIME", Dimension.ZERO, SimpleDataTypeName.DATE), + + /** + * time + */ + TIME("TIME", Dimension.ZERO, SimpleDataTypeName.DATE), + + /** + * char + */ + CHAR("CHAR", Dimension.ONE, SimpleDataTypeName.STRING), + + /** + * varchar + */ + VARCHAR("VARCHAR", Dimension.ONE, SimpleDataTypeName.STRING), + + /** + * string + */ + STRING("STRING", Dimension.ZERO, SimpleDataTypeName.STRING), + + /** + * text + */ + TEXT("TEXT", Dimension.ZERO, SimpleDataTypeName.STRING), + + /** + * bitmap + */ + BITMAP("BITMAP", Dimension.ZERO, SimpleDataTypeName.NUMBER), + + /** + * hll + */ + HLL("HLL", Dimension.ZERO, SimpleDataTypeName.NUMBER), + + /** + * percentile + */ + PERCENTILE("PERCENTILE", Dimension.ZERO, SimpleDataTypeName.NUMBER), + + /** + * json + */ + JSON("JSON", Dimension.ZERO, SimpleDataTypeName.STRING), + + /** + * varbinary + */ + VARBINARY("VARBINARY", Dimension.ONE, SimpleDataTypeName.STRING), + + /** + * binary + */ + BINARY("BINARY", Dimension.ONE, SimpleDataTypeName.STRING), + + /** + * decimal + */ + DECIMAL("DECIMAL", Dimension.TWO, SimpleDataTypeName.NUMBER), + + /** + * decimal + */ + DECIMALV2("DECIMALV2", Dimension.TWO, SimpleDataTypeName.NUMBER), + + /** + * decimal32 + */ + DECIMAL32("DECIMAL32", Dimension.TWO, SimpleDataTypeName.NUMBER), + + /** + * decimal64 + */ + DECIMAL64("DECIMAL64", Dimension.TWO, SimpleDataTypeName.NUMBER), + + /** + * decimal128 + */ + DECIMAL128("DECIMAL128", Dimension.TWO, SimpleDataTypeName.NUMBER), + + /** + * numeric + */ + NUMERIC("NUMERIC", Dimension.TWO, SimpleDataTypeName.NUMBER), + + /** + * number + */ + NUMBER("NUMBER", Dimension.TWO, SimpleDataTypeName.NUMBER), + + /** + * Array + */ + ARRAY("ARRAY", Dimension.MULTIPLE, SimpleDataTypeName.STRING), + + /** + * MAP + */ + Map("MAP", Dimension.MULTIPLE, SimpleDataTypeName.STRING), + ; + + /** + * multi prefix + */ + public static final String MULTI_PREFIX = "<"; + + private final String value; + + private final Dimension dimension; + + private final SimpleDataTypeName simpleDataTypeName; + + StarRocksDataTypeName(String value, Dimension dimension, + SimpleDataTypeName simpleDataTypeName) { + this.value = value; + this.dimension = dimension; + this.simpleDataTypeName = simpleDataTypeName; + } + + public static IDataTypeName getByValue(String value) { + String v = value; + if (v.indexOf(MULTI_PREFIX) > 0) { + v = v.substring(0, v.indexOf(MULTI_PREFIX)).trim(); + } + StarRocksDataTypeName[] starRocksDataTypeNames = StarRocksDataTypeName.values(); + for (StarRocksDataTypeName s : starRocksDataTypeNames) { + if (StringUtils.equalsIgnoreCase(s.getValue(), v)) { + return s; + } + } + throw new IllegalArgumentException("not support the dataType with value:" + value); + } + + @Override + public SimpleDataTypeName getSimpleDataTypeName() { + return simpleDataTypeName; + } + + @Override + public String getName() { + return name(); + } + + @Override + public String getValue() { + return value; + } + + @Override + public Dimension getDimension() { + return dimension; + } + +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksGenericDataType.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksGenericDataType.java new file mode 100644 index 0000000..44b9480 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksGenericDataType.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2022. Aliyun.com All right reserved. This software is the + * confidential and proprietary information of Aliyun.com ("Confidential + * Information"). You shall not disclose such Confidential Information and shall + * use it only in accordance with the terms of the license agreement you entered + * into with Aliyun.com. + */ + +package com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.NodeLocation; +import com.aliyun.fastmodel.core.tree.datatype.DataTypeParameter; +import com.aliyun.fastmodel.core.tree.datatype.GenericDataType; +import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName; +import com.aliyun.fastmodel.transform.starrocks.context.StarRocksContext; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksExpressionVisitor; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import lombok.Getter; + +/** + * HologresGenericDataType + * + * @author panguanjing + * @date 2022/6/9 + */ +@Getter +public class StarRocksGenericDataType extends GenericDataType { + + public StarRocksGenericDataType(String dataTypeName) { + super(dataTypeName); + } + + public StarRocksGenericDataType(String dataTypeName, List arguments) { + super(dataTypeName, arguments); + } + + public StarRocksGenericDataType(NodeLocation location, String origin, String dataTypeName, + List arguments) { + super(location, origin, dataTypeName, arguments); + } + + @Override + public IDataTypeName getTypeName() { + return StarRocksDataTypeName.getByValue(getName()); + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksAstVisitor = (StarRocksAstVisitor)visitor; + return starRocksAstVisitor.visitStarRocksGenericDataType(this, context); + } + + @Override + public String toString() { + return new StarRocksExpressionVisitor(StarRocksContext.builder().build()).process(this); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ArrayPartitionKey.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ArrayPartitionKey.java new file mode 100644 index 0000000..c34b8ed --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ArrayPartitionKey.java @@ -0,0 +1,33 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import lombok.Getter; + +/** + * list partition value + * + * @author panguanjing + * @date 2023/9/14 + */ +@Getter +public class ArrayPartitionKey extends PartitionKey { + + private final List partitionValues; + + public ArrayPartitionKey(List partitionValues) {this.partitionValues = partitionValues;} + + @Override + public List getChildren() { + return partitionValues; + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitArrayPartitionKey(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/LessThanPartitionKey.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/LessThanPartitionKey.java new file mode 100644 index 0000000..909f6b5 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/LessThanPartitionKey.java @@ -0,0 +1,49 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.expr.literal.StringLiteral; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import lombok.Getter; + +/** + * less than + * + * @author panguanjing + * @date 2023/9/14 + */ +@Getter +public class LessThanPartitionKey extends PartitionKey { + + private final boolean maxValue; + + private final ListPartitionValue partitionValues; + + public LessThanPartitionKey(boolean maxValue, ListPartitionValue partitionValues) { + this.maxValue = maxValue; + this.partitionValues = partitionValues; + } + + public LessThanPartitionKey(ListPartitionValue partitionValues) { + this.partitionValues = partitionValues; + this.maxValue =false; + } + + @Override + public List getChildren() { + if (maxValue) { + return ImmutableList.of(); + } + return Lists.newArrayList(partitionValues); + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitLessThanPartitionKey(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ListPartitionValue.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ListPartitionValue.java new file mode 100644 index 0000000..ec9bbc7 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ListPartitionValue.java @@ -0,0 +1,41 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.AbstractNode; +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.NodeLocation; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import lombok.Getter; + +/** + * list partition value + * + * @author panguanjing + * @date 2023/10/23 + */ +@Getter +public class ListPartitionValue extends AbstractNode { + + private final List partitionValueList; + + public ListPartitionValue(NodeLocation location, + List partitionValueList) { + super(location); + this.partitionValueList = partitionValueList; + } + + public ListPartitionValue(List partitionValueList) {this.partitionValueList = partitionValueList;} + + @Override + public List getChildren() { + return partitionValueList; + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitListPartitionValue(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ListPartitionedBy.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ListPartitionedBy.java new file mode 100644 index 0000000..7768932 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/ListPartitionedBy.java @@ -0,0 +1,36 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.core.tree.statement.table.PartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import lombok.Getter; + +/** + * StarRocksPartitionedBy + * + * @author panguanjing + * @date 2023/9/13 + */ +@Getter +public class ListPartitionedBy extends PartitionedBy { + + /** + * rangePartitions + */ + private final List rangePartitions; + + public ListPartitionedBy(List columnDefinitions, + List rangePartitions) { + super(columnDefinitions); + this.rangePartitions = rangePartitions; + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitListPartitionedBy(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/MultiItemListPartition.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/MultiItemListPartition.java new file mode 100644 index 0000000..202b3b3 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/MultiItemListPartition.java @@ -0,0 +1,59 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.expr.literal.ListStringLiteral; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import com.google.common.collect.ImmutableList; +import lombok.Getter; + +/** + * single item list + * + * @author panguanjing + * @date 2023/10/17 + */ +@Getter +public class MultiItemListPartition extends PartitionDesc { + + private final Identifier name; + private final boolean ifNotExists; + private final List listStringLiterals; + private final List propertyList; + + public MultiItemListPartition(Identifier name, boolean ifNotExists, + List listStringLiterals, List propertyList) { + this.name = name; + this.ifNotExists = ifNotExists; + this.listStringLiterals = listStringLiterals; + this.propertyList = propertyList; + } + + @Override + public List getChildren() { + ImmutableList.Builder builder = ImmutableList.builder(); + if (name != null) { + builder.add(name); + } + if (name != null) { + builder.add(name); + } + if (listStringLiterals != null) { + builder.addAll(listStringLiterals); + } + if (propertyList != null) { + builder.addAll(propertyList); + } + return builder.build(); + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitMultiItemListPartition(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/MultiRangePartition.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/MultiRangePartition.java new file mode 100644 index 0000000..a7cfc31 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/MultiRangePartition.java @@ -0,0 +1,58 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.expr.literal.IntervalLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.LongLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.StringLiteral; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import com.google.common.collect.ImmutableList; +import lombok.Getter; + +/** + * multi range partition + * + * @author panguanjing + * @date 2023/9/13 + */ +@Getter +public class MultiRangePartition extends PartitionDesc { + private final StringLiteral start; + private final StringLiteral end; + private final IntervalLiteral intervalLiteral; + private final LongLiteral longLiteral; + + public MultiRangePartition(StringLiteral start, StringLiteral end, IntervalLiteral intervalLiteral, + LongLiteral longLiteral) { + this.start = start; + this.end = end; + this.intervalLiteral = intervalLiteral; + this.longLiteral = longLiteral; + } + + @Override + public List getChildren() { + ImmutableList.Builder builder = ImmutableList.builder(); + if (start != null) { + builder.add(start); + } + if (end != null) { + builder.add(end); + } + if (intervalLiteral != null) { + builder.add(intervalLiteral); + } + if (longLiteral != null) { + builder.add(longLiteral); + } + return builder.build(); + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitMultiRangePartition(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionDesc.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionDesc.java new file mode 100644 index 0000000..d1d5029 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionDesc.java @@ -0,0 +1,20 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import com.aliyun.fastmodel.core.tree.AbstractNode; +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; + +/** + * RangePartition + * + * @author panguanjing + * @date 2023/9/13 + */ +public abstract class PartitionDesc extends AbstractNode { + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitPartitionDesc(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionKey.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionKey.java new file mode 100644 index 0000000..e46ce38 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionKey.java @@ -0,0 +1,20 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import com.aliyun.fastmodel.core.tree.AbstractNode; +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; + +/** + * partition key + * + * @author panguanjing + * @date 2023/9/13 + */ +public abstract class PartitionKey extends AbstractNode { + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitPartitionKey(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionValue.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionValue.java new file mode 100644 index 0000000..2516ce8 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/PartitionValue.java @@ -0,0 +1,46 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.AbstractNode; +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.expr.literal.StringLiteral; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import com.google.common.collect.ImmutableList; +import lombok.Getter; + +/** + * PartitionValue + * + * @author panguanjing + * @date 2023/10/23 + */ +@Getter +public class PartitionValue extends AbstractNode { + + private final boolean maxValue; + + private final StringLiteral stringLiteral; + + public PartitionValue(StringLiteral stringLiteral) { + this.stringLiteral = stringLiteral; + this.maxValue = false; + } + + public PartitionValue(boolean maxValue, StringLiteral stringLiteral) { + this.maxValue = maxValue; + this.stringLiteral = stringLiteral; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitPartitionValue(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/RangePartitionedBy.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/RangePartitionedBy.java new file mode 100644 index 0000000..ba175f4 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/RangePartitionedBy.java @@ -0,0 +1,36 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.core.tree.statement.table.PartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import lombok.Getter; + +/** + * StarRocksPartitionedBy + * + * @author panguanjing + * @date 2023/9/13 + */ +@Getter +public class RangePartitionedBy extends PartitionedBy { + + /** + * rangePartitions + */ + private final List rangePartitions; + + public RangePartitionedBy(List columnDefinitions, + List rangePartitions) { + super(columnDefinitions); + this.rangePartitions = rangePartitions; + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitRangePartitionedBy(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/SingleItemListPartition.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/SingleItemListPartition.java new file mode 100644 index 0000000..fa631ad --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/SingleItemListPartition.java @@ -0,0 +1,59 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.expr.literal.ListStringLiteral; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import com.google.common.collect.ImmutableList; +import lombok.Getter; + +/** + * single item list + * + * @author panguanjing + * @date 2023/10/17 + */ +@Getter +public class SingleItemListPartition extends PartitionDesc { + + private final Identifier name; + private final boolean ifNotExists; + private final ListStringLiteral listStringLiteral; + private final List propertyList; + + public SingleItemListPartition(Identifier name, boolean ifNotExists, + ListStringLiteral listStringLiteral, List propertyList) { + this.name = name; + this.ifNotExists = ifNotExists; + this.listStringLiteral = listStringLiteral; + this.propertyList = propertyList; + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitSingleItemListPartition(this, context); + } + + @Override + public List getChildren() { + ImmutableList.Builder builder = ImmutableList.builder(); + if (name != null) { + builder.add(name); + } + if (name != null) { + builder.add(name); + } + if (listStringLiteral != null) { + builder.add(listStringLiteral); + } + if (propertyList != null) { + builder.addAll(propertyList); + } + return builder.build(); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/SingleRangePartition.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/SingleRangePartition.java new file mode 100644 index 0000000..7cde507 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/partition/SingleRangePartition.java @@ -0,0 +1,53 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.partition; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.transform.starrocks.parser.visitor.StarRocksAstVisitor; +import com.google.common.collect.ImmutableList; +import lombok.Getter; + +/** + * single range partition + * + * @author panguanjing + * @date 2023/9/13 + */ +@Getter +public class SingleRangePartition extends PartitionDesc { + private final Identifier name; + private final boolean ifNotExists; + private final PartitionKey partitionKey; + private final List propertyList; + + public SingleRangePartition(Identifier name, boolean ifNotExists, PartitionKey partitionKey, List propertyList) { + this.name = name; + this.ifNotExists = ifNotExists; + this.partitionKey = partitionKey; + this.propertyList = propertyList; + } + + @Override + public List getChildren() { + ImmutableList.Builder builder = ImmutableList.builder(); + if (name != null) { + builder.add(name); + } + if (partitionKey != null) { + builder.add(partitionKey); + } + if (propertyList != null) { + builder.addAll(propertyList); + } + return builder.build(); + } + + @Override + public R accept(IAstVisitor visitor, C context) { + StarRocksAstVisitor starRocksVisitor = (StarRocksAstVisitor)visitor; + return starRocksVisitor.visitSingleRangePartition(this, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/util/StarRocksReservedWordUtil.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/util/StarRocksReservedWordUtil.java new file mode 100644 index 0000000..fed9543 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/util/StarRocksReservedWordUtil.java @@ -0,0 +1,38 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.util; + +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksLexer; +import com.google.common.base.Strings; +import org.antlr.v4.runtime.Vocabulary; + +/** + * hologres reserved keyword + * + * @author panguanjing + * @date 2023/11/6 + */ +public class StarRocksReservedWordUtil { + + private static final Pattern IDENTIFIER = Pattern.compile("'([A-Z_]+)'"); + + private static final Set SET = new HashSet<>(); + + static { + Vocabulary vocabulary = StarRocksLexer.VOCABULARY; + for (int i = 0; i <= vocabulary.getMaxTokenType(); i++) { + String name = Strings.nullToEmpty(vocabulary.getLiteralName(i)); + Matcher matcher = IDENTIFIER.matcher(name); + if (matcher.matches()) { + SET.add(matcher.group(1)); + } + } + } + + public static boolean isReservedKeyWord(String word) { + return SET.contains(word.toUpperCase()); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/visitor/StarRocksAstBuilder.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/visitor/StarRocksAstBuilder.java new file mode 100644 index 0000000..aa84cc0 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/visitor/StarRocksAstBuilder.java @@ -0,0 +1,609 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.visitor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.aliyun.fastmodel.common.parser.ParserHelper; +import com.aliyun.fastmodel.common.utils.StripUtils; +import com.aliyun.fastmodel.core.exception.ParseException; +import com.aliyun.fastmodel.core.tree.BaseStatement; +import com.aliyun.fastmodel.core.tree.Comment; +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.QualifiedName; +import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; +import com.aliyun.fastmodel.core.tree.datatype.DataTypeParameter; +import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName; +import com.aliyun.fastmodel.core.tree.datatype.NumericParameter; +import com.aliyun.fastmodel.core.tree.datatype.TypeParameter; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.expr.enums.DateTimeEnum; +import com.aliyun.fastmodel.core.tree.expr.literal.BaseLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.BooleanLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.CurrentTimestamp; +import com.aliyun.fastmodel.core.tree.expr.literal.IntervalLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.ListStringLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.LongLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.NullLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.StringLiteral; +import com.aliyun.fastmodel.core.tree.statement.CompositeStatement; +import com.aliyun.fastmodel.core.tree.statement.misc.EmptyStatement; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; +import com.aliyun.fastmodel.core.tree.statement.table.PartitionedBy; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.BaseConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.PrimaryConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.UniqueConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.index.IndexColumnName; +import com.aliyun.fastmodel.core.tree.statement.table.index.TableIndex; +import com.aliyun.fastmodel.core.tree.util.IdentifierUtil; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksBaseVisitor; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.AggDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.ArrayTypeContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.BackQuotedIdentifierContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.BaseTypeContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.BooleanLiteralContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.ColumnDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.CommentContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.CreateTableStatementContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.DecimalTypeContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.DecimalValueContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.DefaultDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.DigitIdentifierContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.DistributionDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.DoubleValueContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.EngineDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.IdentifierListContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.IndexDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.IndexTypeContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.IntegerValueContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.IntervalContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.KeyDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.LiteralContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.MapTypeContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.MultiItemListPartitionDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.MultiRangePartitionContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.NullLiteralContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.NumericLiteralContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.PartitionDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.PartitionKeyDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.PartitionListIdentifierContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.PartitionRangeIdentifierContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.PartitionValueContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.PartitionValueListContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.PropertiesContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.PropertyContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.QualifiedNameContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.RollupDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.SingleItemListPartitionDescContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.SingleRangePartitionContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.SingleStatementContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.SqlStatementsContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.StringContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.StringLiteralContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.TypeContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.TypeListContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.TypeParameterContext; +import com.aliyun.fastmodel.transform.starrocks.parser.StarRocksParser.UnquotedIdentifierContext; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.AggregateConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.DuplicateConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype.StarRocksDataTypeName; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype.StarRocksGenericDataType; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ArrayPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.LessThanPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiItemListPartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiRangePartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionDesc; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.RangePartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleItemListPartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleRangePartition; +import com.google.common.collect.Lists; +import org.antlr.v4.runtime.Token; + +import static com.aliyun.fastmodel.common.parser.ParserHelper.getLocation; +import static com.aliyun.fastmodel.common.parser.ParserHelper.getOrigin; + +/** + * StarRocksAstBuilder + * + * @author panguanjing + * @date 2023/9/11 + */ +public class StarRocksAstBuilder extends StarRocksBaseVisitor { + + private final ReverseContext reverseContext; + + public StarRocksAstBuilder(ReverseContext context) { + this.reverseContext = context; + } + + @Override + public Node visitSqlStatements(SqlStatementsContext ctx) { + List sqlStatementContexts = ctx.singleStatement(); + List list = new ArrayList<>(); + if (sqlStatementContexts == null) { + throw new ParseException("can't parse the empty context"); + } + if (sqlStatementContexts.size() == 1) { + return visit(sqlStatementContexts.get(0)); + } + for (SingleStatementContext sqlStatementContext : sqlStatementContexts) { + Node node = visit(sqlStatementContext); + list.add((BaseStatement)node); + } + return new CompositeStatement(getLocation(ctx), getOrigin(ctx), list); + } + + @Override + public Node visitSingleStatement(SingleStatementContext ctx) { + if (ctx.statement() != null) { + return visit(ctx.statement()); + } + if (ctx.emptyStatement() != null) { + return new EmptyStatement(); + } + return super.visitSingleStatement(ctx); + } + + @Override + public Node visitCreateTableStatement(CreateTableStatementContext ctx) { + // table name + QualifiedNameContext qualifiedNameContext = ctx.qualifiedName(); + QualifiedName qualifiedName = (QualifiedName)visit(qualifiedNameContext); + // comment + CommentContext comment = ctx.comment(); + Optional tableComment = ParserHelper.visitIfPresent(this, comment, Comment.class); + //columns + List columns = ParserHelper.visit(this, ctx.columnDesc(), ColumnDefinition.class); + //index + List listTableIndex = ParserHelper.visit(this, ctx.indexDesc(), TableIndex.class); + //constraint + Optional list = ParserHelper.visitIfPresent(this, ctx.keyDesc(), BaseConstraint.class); + List constraints = null; + if (list.isPresent()) { + constraints = Lists.newArrayList(list.get()); + } + PropertiesContext properties = ctx.properties(); + List propertyList = Lists.newArrayList(); + if (properties != null) { + propertyList = ParserHelper.visit(this, properties.property(), Property.class); + } + //extend properties + List extendProperties = toExtend(ctx); + List all = Lists.newArrayList(); + if (propertyList != null) { + all.addAll(propertyList); + } + all.addAll(extendProperties); + + //partition by + PartitionDescContext partitionDescContext = ctx.partitionDesc(); + PartitionedBy partitionedBy = null; + if (partitionDescContext != null) { + partitionedBy = (PartitionedBy)visit(partitionDescContext); + } + return CreateTable.builder() + .tableName(qualifiedName) + .tableIndex(listTableIndex) + .columns(columns) + .constraints(constraints) + .comment(tableComment.orElse(null)) + .properties(all) + .partition(partitionedBy) + .build(); + } + + @Override + public Node visitIndexDesc(IndexDescContext ctx) { + //index desc + Identifier indexName = (Identifier)visit(ctx.indexName); + List indexColumns = ParserHelper.visit(this, ctx.identifierList().identifier(), Identifier.class); + List properties = Lists.newArrayList(); + IndexTypeContext indexTypeContext = ctx.indexType(); + if (indexTypeContext != null) { + Property property = new Property(StarRocksProperty.TABLE_INDEX_TYPE.getValue(), "USING BITMAP"); + properties.add(property); + } + if (ctx.comment() != null) { + Comment comment = (Comment)visit(ctx.comment()); + Property property = new Property(StarRocksProperty.TABLE_INDEX_COMMENT.getValue(), comment.getComment()); + properties.add(property); + } + return new TableIndex( + indexName, + indexColumns.stream().map(i -> new IndexColumnName(i, null, null)).collect(Collectors.toList()), + properties + ); + } + + @Override + public Node visitPartitionRangeIdentifier(PartitionRangeIdentifierContext ctx) { + IdentifierListContext identifierListContext = ctx.identifierList(); + List list = ParserHelper.visit(this, identifierListContext.identifier(), Identifier.class); + List columnDefinitionList = + list.stream().map(c -> ColumnDefinition.builder().colName(c).build()).collect(Collectors.toList()); + List rangePartitions = ParserHelper.visit(this, ctx.rangePartitionDesc(), PartitionDesc.class); + return new RangePartitionedBy( + columnDefinitionList, rangePartitions + ); + } + + @Override + public Node visitPartitionListIdentifier(PartitionListIdentifierContext ctx) { + List visit = ParserHelper.visit(this, ctx.identifierList().identifier(), Identifier.class); + List columnDefines = visit.stream().map( + i -> ColumnDefinition.builder() + .colName(i) + .build() + ).collect(Collectors.toList()); + List rangePartitions = null; + if (ctx.listPartitionDesc() != null) { + rangePartitions = ParserHelper.visit(this, ctx.listPartitionDesc(), PartitionDesc.class); + } + return new ListPartitionedBy(columnDefines, rangePartitions); + } + + @Override + public Node visitSingleItemListPartitionDesc(SingleItemListPartitionDescContext ctx) { + Identifier identifier = (Identifier)visit(ctx.identifier()); + ListStringLiteral listStringLiteral = (ListStringLiteral)visit(ctx.stringList()); + List properList = null; + if (ctx.propertyList() != null) { + properList = ParserHelper.visit(this, ctx.propertyList().property(), Property.class); + } + return new SingleItemListPartition(identifier, ctx.IF() != null, listStringLiteral, properList); + } + + @Override + public Node visitMultiItemListPartitionDesc(MultiItemListPartitionDescContext ctx) { + Identifier identifier = (Identifier)visit(ctx.identifier()); + List listStringLiteral = ParserHelper.visit(this, ctx.stringList(), ListStringLiteral.class); + List properList = null; + if (ctx.propertyList() != null) { + properList = ParserHelper.visit(this, ctx.propertyList().property(), Property.class); + } + return new MultiItemListPartition(identifier, ctx.IF() != null, listStringLiteral, properList); + } + + @Override + public Node visitSingleRangePartition(SingleRangePartitionContext ctx) { + Identifier identifier = (Identifier)visit(ctx.identifier()); + PartitionKey partitionKey = (PartitionKey)visit(ctx.partitionKeyDesc()); + List propertyList = null; + if (ctx.propertyList() != null) { + propertyList = ParserHelper.visit(this, ctx.propertyList().property(), Property.class); + } + return new SingleRangePartition(identifier, ctx.IF() != null, partitionKey, propertyList); + } + + @Override + public Node visitMultiRangePartition(MultiRangePartitionContext ctx) { + StringLiteral start = (StringLiteral)visit(ctx.start); + StringLiteral end = (StringLiteral)visit(ctx.end); + IntervalLiteral intervalLiteral = null; + LongLiteral longLiteral = null; + if (ctx.INTEGER_VALUE() != null) { + longLiteral = new LongLiteral(ctx.INTEGER_VALUE().getText()); + } else { + intervalLiteral = (IntervalLiteral)visit(ctx.interval()); + } + return new MultiRangePartition(start, end, intervalLiteral, longLiteral); + } + + @Override + public Node visitPartitionKeyDesc(PartitionKeyDescContext ctx) { + if (ctx.LESS() != null) { + ListPartitionValue visit = (ListPartitionValue)visit(ctx.partitionValueList().get(0)); + return new LessThanPartitionKey( + ctx.MAXVALUE() != null, + visit + ); + } else { + List list = ParserHelper.visit(this, ctx.partitionValueList(), ListPartitionValue.class); + return new ArrayPartitionKey(list); + } + } + + @Override + public Node visitPartitionValueList(PartitionValueListContext ctx) { + List list = ParserHelper.visit(this, ctx.partitionValue(), PartitionValue.class); + return new ListPartitionValue( + getLocation(ctx), + list + ); + } + + @Override + public Node visitPartitionValue(PartitionValueContext ctx) { + StringLiteral stringLiteral = null; + if (ctx.string() != null) { + stringLiteral = (StringLiteral)visit(ctx.string()); + } + PartitionValue partitionValue = new PartitionValue( + ctx.MAXVALUE() != null, + stringLiteral + ); + return partitionValue; + } + + private List toExtend(CreateTableStatementContext ctx) { + List list = Lists.newArrayList(); + EngineDescContext engineDescContext = ctx.engineDesc(); + if (engineDescContext != null) { + Property property = (Property)visit(engineDescContext); + list.add(property); + } + DistributionDescContext distributionDescContext = ctx.distributionDesc(); + if (distributionDescContext != null) { + List property = toList(distributionDescContext); + list.addAll(property); + } + //roll up + RollupDescContext rollupDescContext = ctx.rollupDesc(); + if (rollupDescContext != null) { + Property property = (Property)visit(rollupDescContext); + list.add(property); + } + return list; + } + + @Override + public Node visitArrayType(ArrayTypeContext ctx) { + BaseDataType baseDataType = (BaseDataType)visit(ctx.type()); + return new StarRocksGenericDataType( + StarRocksDataTypeName.ARRAY.getValue(), + Lists.newArrayList(new TypeParameter(baseDataType)) + ); + } + + @Override + public Node visitRollupDesc(RollupDescContext ctx) { + return super.visitRollupDesc(ctx); + } + + public List toList(DistributionDescContext ctx) { + List list = null; + if (ctx.identifierList() != null) { + list = ParserHelper.visit(this, ctx.identifierList().identifier(), Identifier.class); + } + List propertyList = Lists.newArrayList(); + if (list != null) { + Property property = new Property(StarRocksProperty.TABLE_DISTRIBUTED_HASH.getValue(), + list.stream().map(Identifier::getValue).collect(Collectors.joining(","))); + propertyList.add(property); + } + if (ctx.BUCKETS() != null) { + Property property = new Property(StarRocksProperty.TABLE_DISTRIBUTED_BUCKETS.getValue(), ctx.INTEGER_VALUE().getText()); + propertyList.add(property); + } + return propertyList; + } + + @Override + public Node visitEngineDesc(EngineDescContext ctx) { + Identifier identifier = (Identifier)visit(ctx.identifier()); + return new Property(StarRocksProperty.TABLE_ENGINE.getValue(), identifier.getValue()); + } + + @Override + public Node visitKeyDesc(KeyDescContext ctx) { + IdentifierListContext identifierListContext = ctx.identifierList(); + List list = ParserHelper.visit(this, identifierListContext.identifier(), Identifier.class); + if (ctx.AGGREGATE() != null) { + return new AggregateConstraint(IdentifierUtil.sysIdentifier(), list, true); + } + if (ctx.PRIMARY() != null) { + return new PrimaryConstraint(IdentifierUtil.sysIdentifier(), list); + } + if (ctx.UNIQUE() != null) { + return new UniqueConstraint(IdentifierUtil.sysIdentifier(), list); + } + if (ctx.DUPLICATE() != null) { + return new DuplicateConstraint(IdentifierUtil.sysIdentifier(), list, true); + } + return super.visitKeyDesc(ctx); + } + + @Override + public Node visitColumnDesc(ColumnDescContext ctx) { + //column name + Identifier identifier = (Identifier)visit(ctx.identifier()); + TypeContext type = ctx.type(); + //data type + BaseDataType baseDataType = (BaseDataType)visit(type); + //comment + Optional comment = ParserHelper.visitIfPresent(this, ctx.comment(), Comment.class); + //not null + Boolean notNull = null; + if (ctx.NOT() != null && ctx.NULL() != null) { + notNull = true; + } else if (ctx.NULL() != null) { + notNull = false; + } + //default value + DefaultDescContext defaultDescContext = ctx.defaultDesc(); + BaseLiteral baseLiteral = null; + if (defaultDescContext != null) { + baseLiteral = (BaseLiteral)visit(ctx.defaultDesc()); + } + + //agg desc + AggDescContext aggDescContext = ctx.aggDesc(); + List properties = Lists.newArrayList(); + if (aggDescContext != null) { + Property property = new Property(StarRocksProperty.COLUMN_AGG_DESC.getValue(), aggDescContext.getText()); + properties.add(property); + } + if (ctx.charsetName() != null) { + Identifier identifier1 = (Identifier)visit(ctx.charsetName().identifier()); + Property property = new Property(StarRocksProperty.COLUMN_CHAR_SET.getValue(), identifier1.getValue()); + properties.add(property); + } + if (ctx.KEY() != null) { + Property property = new Property(StarRocksProperty.COLUMN_KEY.getValue(), "KEY"); + properties.add(property); + } + return ColumnDefinition.builder() + .dataType(baseDataType) + .comment(comment.orElse(null)) + .colName(identifier) + .properties(properties) + .notNull(notNull) + .defaultValue(baseLiteral) + .build(); + } + + @Override + public Node visitDefaultDesc(DefaultDescContext ctx) { + if (ctx.string() != null) { + return visit(ctx.string()); + } + if (ctx.NULL() != null) { + return new NullLiteral(); + } + if (ctx.CURRENT_TIMESTAMP() != null) { + return new CurrentTimestamp(); + } + return super.visitDefaultDesc(ctx); + } + + @Override + public Node visitTypeList(TypeListContext ctx) { + return super.visitTypeList(ctx); + } + + @Override + public Node visitProperty(PropertyContext ctx) { + StringLiteral key = (StringLiteral)visit(ctx.key); + StringLiteral value = (StringLiteral)visit(ctx.value); + return new Property(key.getValue(), value); + } + + @Override + public Node visitMapType(MapTypeContext ctx) { + return super.visitMapType(ctx); + } + + @Override + public Node visitBaseType(BaseTypeContext ctx) { + Token name = ctx.name; + IDataTypeName byValue = StarRocksDataTypeName.getByValue(name.getText()); + List list = Lists.newArrayList(); + if (ctx.typeParameter() != null) { + DataTypeParameter dataTypeParameter = (DataTypeParameter)visit(ctx.typeParameter()); + list.add(dataTypeParameter); + } + return new StarRocksGenericDataType(getLocation(ctx), getOrigin(ctx), byValue.getValue(), list); + } + + @Override + public Node visitTypeParameter(TypeParameterContext ctx) { + return new NumericParameter(ctx.INTEGER_VALUE().getText()); + } + + @Override + public Node visitDecimalType(DecimalTypeContext ctx) { + Token name = ctx.name; + IDataTypeName byValue = StarRocksDataTypeName.getByValue(name.getText()); + List list = Lists.newArrayList(); + if (ctx.precision != null) { + DataTypeParameter p = new NumericParameter(ctx.precision.getText()); + list.add(p); + } + if (ctx.scale != null) { + DataTypeParameter p = new NumericParameter(ctx.scale.getText()); + list.add(p); + } + return new StarRocksGenericDataType(byValue.getValue(), list); + } + + @Override + public Node visitComment(CommentContext ctx) { + StringLiteral stringLiteral = (StringLiteral)visit(ctx.string()); + return new Comment(stringLiteral.getValue()); + } + + @Override + public Node visitString(StringContext ctx) { + return new StringLiteral( + getLocation(ctx), + getOrigin(ctx), + StripUtils.strip(ctx.getText())); + } + + @Override + public Node visitInterval(IntervalContext ctx) { + BaseLiteral baseLiteral = (BaseLiteral)visit(ctx.value); + DateTimeEnum dateTimeEnum = DateTimeEnum.getByCode(ctx.from.getText()); + return new IntervalLiteral(baseLiteral, dateTimeEnum); + } + + @Override + public Node visitLiteral(LiteralContext ctx) { + return super.visitLiteral(ctx); + } + + @Override + public Node visitNullLiteral(NullLiteralContext ctx) { + return new NullLiteral(); + } + + @Override + public Node visitBooleanLiteral(BooleanLiteralContext ctx) { + return new BooleanLiteral(ctx.booleanValue().getText()); + } + + @Override + public Node visitNumericLiteral(NumericLiteralContext ctx) { + return visit(ctx.number()); + } + + @Override + public Node visitDecimalValue(DecimalValueContext ctx) { + return super.visitDecimalValue(ctx); + } + + @Override + public Node visitDoubleValue(DoubleValueContext ctx) { + return super.visitDoubleValue(ctx); + } + + @Override + public Node visitIntegerValue(IntegerValueContext ctx) { + return super.visitIntegerValue(ctx); + } + + @Override + public Node visitStringLiteral(StringLiteralContext ctx) { + return visit(ctx.string()); + } + + @Override + public Node visitQualifiedName(QualifiedNameContext ctx) { + List list = ParserHelper.visit(this, ctx.identifier(), Identifier.class); + return QualifiedName.of(list); + } + + @Override + public Node visitUnquotedIdentifier(UnquotedIdentifierContext ctx) { + return ParserHelper.getIdentifier(ctx); + } + + @Override + public Node visitDigitIdentifier(DigitIdentifierContext ctx) { + return ParserHelper.getIdentifier(ctx); + } + + @Override + public Node visitBackQuotedIdentifier(BackQuotedIdentifierContext ctx) { + return ParserHelper.getIdentifier(ctx); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/visitor/StarRocksAstVisitor.java b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/visitor/StarRocksAstVisitor.java new file mode 100644 index 0000000..5a8dc00 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/main/java/com/aliyun/fastmodel/transform/starrocks/parser/visitor/StarRocksAstVisitor.java @@ -0,0 +1,192 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.visitor; + +import com.aliyun.fastmodel.core.tree.IAstVisitor; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.AggregateConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.DuplicateConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype.StarRocksGenericDataType; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ArrayPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.LessThanPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiItemListPartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiRangePartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionDesc; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.RangePartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleItemListPartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleRangePartition; + +/** + * star rocks visitor + * + * @author panguanjing + * @date 2023/9/12 + */ +public interface StarRocksAstVisitor extends IAstVisitor { + + /** + * visit starRocks GenericDataType + * + * @param starRocksGenericDataType genericDataType + * @param context context + * @return R + */ + default R visitStarRocksGenericDataType(StarRocksGenericDataType starRocksGenericDataType, C context) { + return visitGenericDataType(starRocksGenericDataType, context); + } + + /** + * visit starRocks partitioned by + * + * @param starRocksPartitionedBy + * @param context + * @return + */ + default R visitRangePartitionedBy(RangePartitionedBy starRocksPartitionedBy, C context) { + return visitNode(starRocksPartitionedBy, context); + } + + /** + * visitAggregateConstraint + * + * @param aggregateConstraint + * @param context + * @return + */ + default R visitAggregateConstraint(AggregateConstraint aggregateConstraint, C context) { + return visitNode(aggregateConstraint, context); + } + + /** + * visit duplicate constraint + * + * @param duplicateConstraint + * @param context + * @return + */ + default R visitDuplicateConstraint(DuplicateConstraint duplicateConstraint, C context) { + return visitNode(duplicateConstraint, context); + } + + /** + * visitListPartitionedBy + * + * @param listPartitionedBy + * @param context + * @return + */ + default R visitListPartitionedBy(ListPartitionedBy listPartitionedBy, C context) { + return visitNode(listPartitionedBy, context); + } + + /** + * single range partition + * + * @param singleRangePartition + * @param context + * @return + */ + default R visitSingleRangePartition(SingleRangePartition singleRangePartition, C context) { + return visitNode(singleRangePartition, context); + } + + /** + * multi range partition + * + * @param multiRangePartition + * @param context + * @return + */ + default R visitMultiRangePartition(MultiRangePartition multiRangePartition, C context) { + return visitNode(multiRangePartition, context); + } + + /** + * visit partition key + * + * @param partitionKey + * @param context + * @return + */ + default R visitPartitionKey(PartitionKey partitionKey, C context) { + return visitNode(partitionKey, context); + } + + /** + * visit array partition key + * + * @param arrayPartitionKey arrayPartitionKey + * @param context context + * @return + */ + default R visitArrayPartitionKey(ArrayPartitionKey arrayPartitionKey, C context) { + return visitPartitionKey(arrayPartitionKey, context); + } + + /** + * visit less than partition key + * + * @param lessThanPartitionKey + * @param context + * @return + */ + default R visitLessThanPartitionKey(LessThanPartitionKey lessThanPartitionKey, C context) { + return visitPartitionKey(lessThanPartitionKey, context); + } + + /** + * visit single item partition + * + * @param singleItemListPartition + * @param context + * @return + */ + default R visitSingleItemListPartition(SingleItemListPartition singleItemListPartition, C context) { + return visitPartitionDesc(singleItemListPartition, context); + } + + /** + * visit partition desc + * + * @param partitionDesc + * @param context + * @return + */ + default R visitPartitionDesc(PartitionDesc partitionDesc, C context) { + return visitNode(partitionDesc, context); + } + + /** + * visit multiItemListPartition + * + * @param multiItemListPartition + * @param context + * @return + */ + default R visitMultiItemListPartition(MultiItemListPartition multiItemListPartition, C context) { + return visitPartitionDesc(multiItemListPartition, context); + } + + /** + * visit list partition value + * + * @param listPartitionValue + * @param context + * @return + */ + default R visitListPartitionValue(ListPartitionValue listPartitionValue, C context) { + return visitNode(listPartitionValue, context); + } + + /** + * visit partition value + * + * @param partitionValue + * @param context + * @return + */ + default R visitPartitionValue(PartitionValue partitionValue, C context) { + return visitNode(partitionValue, context); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/StarRocksTransformerTest.java b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/StarRocksTransformerTest.java new file mode 100644 index 0000000..c3790d6 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/StarRocksTransformerTest.java @@ -0,0 +1,198 @@ +package com.aliyun.fastmodel.transform.starrocks; + +import java.util.List; +import java.util.Optional; + +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.QualifiedName; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.api.client.dto.table.Column; +import com.aliyun.fastmodel.transform.api.client.dto.table.Table; +import com.aliyun.fastmodel.transform.api.context.TransformContext; +import com.aliyun.fastmodel.transform.api.dialect.DialectNode; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.TablePartitionRaw; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Desc: + * + * @author panguanjing + * @date 2023/9/6 + */ +public class StarRocksTransformerTest { + + StarRocksTransformer starRocksTransformer = new StarRocksTransformer(); + + @Test + public void transform() { + CreateTable createTable = CreateTable + .builder() + .tableName(QualifiedName.of("ab")) + .build(); + DialectNode transform = starRocksTransformer.transform(createTable); + assertEquals("CREATE TABLE ab", transform.getNode()); + } + + @Test + public void testTransformTable() { + DialectNode dialectNode = new DialectNode( + "CREATE TABLE example_db.table_range\n" + + "(\n" + + " k1 DATE,\n" + + " k2 ARRAY,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k1, k2, k3)\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + " PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + " PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + " PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\n" + + " \"storage_medium\" = \"SSD\", \n" + + " \"storage_cooldown_time\" = \"2015-06-04 00:00:00\"\n" + + ");" + ); + Node node = starRocksTransformer.reverse(dialectNode); + Table table = starRocksTransformer.transformTable(node, TransformContext.builder().build()); + assertNull(table.getSchema()); + assertEquals("example_db", table.getDatabase()); + assertEquals("table_range", table.getName()); + List columns = table.getColumns(); + assertEquals(5, columns.size()); + Column column = columns.get(1); + assertEquals("ARRAY", column.getDataType()); + List properties = table.getProperties(); + assertEquals(8, properties.size()); + Optional first = properties.stream().filter( + c -> StringUtils.equalsIgnoreCase(c.getKey(), StarRocksProperty.TABLE_ENGINE.getValue())).findFirst(); + assertEquals(StarRocksProperty.TABLE_ENGINE.getValue(), first.get().getKey()); + } + + @Test + public void testTransformTableArray() { + DialectNode dialectNode = new DialectNode( + "CREATE TABLE example_db.table_range\n" + + "(\n" + + " k1 DATE,\n" + + " k2 ARRAY,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k1, k2, k3)\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + " PARTITION p202101 VALUES [(\"20210101\"), (\"20210201\")),\n" + + " PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + " PARTITION p202103 VALUES [(\"20210301\"), (MAXVALUE))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\n" + + " \"storage_medium\" = \"SSD\", \n" + + " \"storage_cooldown_time\" = \"2015-06-04 00:00:00\"\n" + + ");" + ); + Node node = starRocksTransformer.reverse(dialectNode); + Table table = starRocksTransformer.transformTable(node, TransformContext.builder().build()); + List columns = table.getColumns(); + assertEquals(5, columns.size()); + Column column = columns.get(1); + assertEquals("ARRAY", column.getDataType()); + List properties = table.getProperties(); + assertEquals(8, properties.size()); + Optional first = properties.stream().filter(c -> { + return StringUtils.equalsIgnoreCase(c.getKey(), StarRocksProperty.TABLE_ENGINE.getValue()); + }).findFirst(); + assertEquals(StarRocksProperty.TABLE_ENGINE.getValue(), first.get().getKey()); + } + + @Test + public void testTransformWithoutPartition() { + DialectNode dialectNode = new DialectNode( + "CREATE TABLE IF NOT EXISTS ruoyun_db.fml_simple\n" + + "(\n" + + " c1 TINYINT NOT NULL COMMENT \"ti_comment\",\n" + + " c2 SMALLINT NOT NULL COMMENT \"si_comment\",\n" + + " c3 STRING NULL\n" + + ")\n" + + "PRIMARY KEY (c1,c2)\n" + + "COMMENT \"table_comment\"\n" + + "DISTRIBUTED BY HASH(c1,c2) BUCKETS 4\n" + + "PROPERTIES (\"replication_num\"=\"3\")" + ); + Node node = starRocksTransformer.reverse(dialectNode); + Table table = starRocksTransformer.transformTable(node, TransformContext.builder().build()); + List columns = table.getColumns(); + assertEquals(3, columns.size()); + } + + @Test + public void testTransform() { + DialectNode dialectNode = new DialectNode( + "CREATE TABLE IF NOT EXISTS ruoyun_db.fml_lower_upper_bound_partition\n" + + "(\n" + + " c1 TINYINT NOT NULL COMMENT \"c1 comment\",\n" + + " c2 DATE NOT NULL COMMENT \"c2 comment\",\n" + + " c3 INT NOT NULL COMMENT \"\",\n" + + " c4 STRING NULL COMMENT \"c4 comment\"\n" + + ")\n" + + "PRIMARY KEY (c1,c2,c3)\n" + + "COMMENT \"table_comment\"\n" + + "PARTITION BY RANGE (c3,c2)\n" + + "(\n" + + " PARTITION IF NOT EXISTS pt1 VALUES [(\"1021-01-01\",\"2021-01-01\"),(\"1022-01-01\",\"2022-01-01\")),\n" + + " PARTITION IF NOT EXISTS pt2 VALUES [(\"1023-01-01\",\"2023-01-01\"),(\"1024-01-01\",\"2024-01-01\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(c1,c2) BUCKETS 4\n" + + "PROPERTIES (\"replication_num\"=\"3\");" + ); + Node node = starRocksTransformer.reverse(dialectNode); + Table table = starRocksTransformer.transformTable(node, TransformContext.builder().build()); + List columns = table.getColumns(); + Column column = columns.get(1); + assertEquals(Integer.valueOf(1), column.getPartitionKeyIndex()); + column = columns.get(2); + assertEquals(Integer.valueOf(0), column.getPartitionKeyIndex()); + } + + @Test + public void testReversePartitionRaw() { + DialectNode dialectNode = new DialectNode( + "CREATE TABLE IF NOT EXISTS ruoyun_db.fml_raw_partition\n" + + "(\n" + + " c1 TINYINT NOT NULL COMMENT \"c1 comment\",\n" + + " c2 DATE NOT NULL COMMENT \"c2 comment\",\n" + + " c3 INT NOT NULL COMMENT \"\",\n" + + " c4 STRING NULL COMMENT \"c4 comment\"\n" + + ")\n" + + "PRIMARY KEY (c1,c2,c3)\n" + + "COMMENT \"table_comment\"\n" + + "PARTITION BY RANGE (c2)()\n" + + "DISTRIBUTED BY HASH(c1,c2) BUCKETS 4\n" + + "PROPERTIES (\"replication_num\"=\"3\")" + ); + Node node = starRocksTransformer.reverse(dialectNode); + Table table = starRocksTransformer.transformTable(node, TransformContext.builder().build()); + List properties = table.getProperties(); + assertEquals(4, properties.size()); + Optional first = properties.stream().filter(p -> { + return p instanceof TablePartitionRaw; + }).findFirst(); + BaseClientProperty baseClientProperty = first.get(); + TablePartitionRaw tablePartitionRaw = (TablePartitionRaw)baseClientProperty; + assertEquals("PARTITION BY RANGE (c2)()", tablePartitionRaw.getValue()); + } +} \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksClientConverterTest.java b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksClientConverterTest.java new file mode 100644 index 0000000..36619dc --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksClientConverterTest.java @@ -0,0 +1,195 @@ +package com.aliyun.fastmodel.transform.starrocks.client.converter; + +import java.util.ArrayList; +import java.util.List; + +import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.QualifiedName; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.expr.enums.DateTimeEnum; +import com.aliyun.fastmodel.core.tree.expr.literal.IntervalLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.ListStringLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.LongLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.StringLiteral; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; +import com.aliyun.fastmodel.transform.api.client.PropertyConverter; +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.api.client.dto.table.Column; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.SingleRangePartitionProperty; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.ArrayClientPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.BaseClientPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.LessThanClientPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.MultiRangeClientPartition; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.PartitionClientValue; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.SingleRangeClientPartition; +import com.aliyun.fastmodel.transform.starrocks.format.StarRocksProperty; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ArrayPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.LessThanPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiRangePartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionDesc; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.RangePartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleRangePartition; +import com.google.common.collect.Lists; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Desc: + * + * @author panguanjing + * @date 2023/9/16 + */ +public class StarRocksClientConverterTest { + + StarRocksClientConverter starRocksClientConverter = new StarRocksClientConverter(); + + @Test + public void getPropertyConverter() { + PropertyConverter propertyConverter = starRocksClientConverter.getPropertyConverter(); + assertNotNull(propertyConverter); + } + + @Test + public void testToBaseClientProperty() { + List columns = Lists.newArrayList(); + columns.add(ColumnDefinition.builder().colName(new Identifier("k1")).build()); + List rangePartitions = Lists.newArrayList(); + List partitionValues = Lists.newArrayList(); + ArrayList value = Lists.newArrayList(new PartitionValue(new StringLiteral("2020-01-01")), + new PartitionValue(new StringLiteral("2022-01-01"))); + partitionValues.add(new ListPartitionValue(value)); + List properties = Lists.newArrayList(new Property("a", "b")); + SingleRangePartition singleRangePartition = new SingleRangePartition( + new Identifier("abc"), + false, + new ArrayPartitionKey(partitionValues), + properties + ); + rangePartitions.add(singleRangePartition); + CreateTable table = CreateTable.builder() + .tableName(QualifiedName.of("abc")) + .partition(new RangePartitionedBy(columns, rangePartitions)) + .build(); + List baseClientProperties = starRocksClientConverter.toBaseClientProperty(table); + assertEquals(1, baseClientProperties.size()); + BaseClientProperty baseClientProperty = baseClientProperties.get(0); + assertEquals(StarRocksProperty.TABLE_RANGE_PARTITION.getValue(), baseClientProperty.getKey()); + + SingleRangePartitionProperty singleRangePartitionProperty = (SingleRangePartitionProperty)baseClientProperty; + SingleRangeClientPartition value1 = singleRangePartitionProperty.getValue(); + assertEquals("abc", value1.getName()); + BaseClientPartitionKey partitionKey = value1.getPartitionKey(); + ArrayClientPartitionKey arrayClientPartitionKey = (ArrayClientPartitionKey)partitionKey; + List> partitionValue = arrayClientPartitionKey.getPartitionValue(); + assertEquals(1, partitionValue.size()); + List strings = partitionValue.get(0); + assertEquals(2, strings.size()); + } + + @Test + public void testToBaseClientPropertyLessThan() { + List columns = Lists.newArrayList(); + columns.add(ColumnDefinition.builder().colName(new Identifier("k1")).build()); + List rangeParations = Lists.newArrayList(); + ArrayList value = Lists.newArrayList(new PartitionValue(new StringLiteral("2020-01-01")), + new PartitionValue(new StringLiteral("2022-01-01"))); + ListPartitionValue e = new ListPartitionValue(value); + SingleRangePartition singleRangePartition = new SingleRangePartition( + new Identifier("abc"), + false, + new LessThanPartitionKey(new ListPartitionValue(value)), + null + ); + rangeParations.add(singleRangePartition); + CreateTable table = CreateTable.builder() + .tableName(QualifiedName.of("abc")) + .partition(new RangePartitionedBy(columns, rangeParations)) + .build(); + List baseClientProperties = starRocksClientConverter.toBaseClientProperty(table); + assertEquals(1, baseClientProperties.size()); + BaseClientProperty baseClientProperty = baseClientProperties.get(0); + assertEquals(StarRocksProperty.TABLE_RANGE_PARTITION.getValue(), baseClientProperty.getKey()); + SingleRangeClientPartition value1 = (SingleRangeClientPartition)baseClientProperty.getValue(); + assertEquals("abc", value1.getName()); + assertFalse(value1.isIfNotExists()); + BaseClientPartitionKey partitionKey = value1.getPartitionKey(); + LessThanClientPartitionKey lessThanClientPartitionKey = (LessThanClientPartitionKey)partitionKey; + assertEquals(2, lessThanClientPartitionKey.getPartitionValueList().size()); + assertFalse(lessThanClientPartitionKey.isMaxValue()); + } + + @Test + public void testToBaseClientPropertyMultiRange() { + List columns = Lists.newArrayList(); + columns.add(ColumnDefinition.builder().colName(new Identifier("k1")).build()); + List rangePartitions = Lists.newArrayList(); + ArrayList value = Lists.newArrayList(new StringLiteral("2020-01-01"), new StringLiteral("2022-01-01")); + ListStringLiteral e = new ListStringLiteral(value); + MultiRangePartition singleRangePartition = new MultiRangePartition( + new StringLiteral("2021-01-01"), + new StringLiteral("2022-01-01"), + null, + new LongLiteral("1") + ); + rangePartitions.add(singleRangePartition); + CreateTable table = CreateTable.builder() + .tableName(QualifiedName.of("abc")) + .partition(new RangePartitionedBy(columns, rangePartitions)) + .build(); + List baseClientProperties = starRocksClientConverter.toBaseClientProperty(table); + assertEquals(1, baseClientProperties.size()); + BaseClientProperty baseClientProperty = baseClientProperties.get(0); + assertEquals(StarRocksProperty.TABLE_RANGE_PARTITION.getValue(), baseClientProperty.getKey()); + MultiRangeClientPartition value1 = (MultiRangeClientPartition)baseClientProperty.getValue(); + String start = value1.getStart(); + assertEquals("2021-01-01", start); + assertEquals("2022-01-01", value1.getEnd()); + assertEquals(null, value1.getDateTimeEnum()); + assertTrue(value1.getInterval().intValue() == 1); + } + + @Test + public void testToBaseClientPropertyMultiRangeInterval() { + List columns = Lists.newArrayList(); + columns.add(ColumnDefinition.builder().colName(new Identifier("k1")).build()); + List rangePartitions = Lists.newArrayList(); + ArrayList value = Lists.newArrayList(new StringLiteral("2020-01-01"), new StringLiteral("2022-01-01")); + ListStringLiteral e = new ListStringLiteral(value); + MultiRangePartition singleRangePartition = new MultiRangePartition( + new StringLiteral("2021-01-01"), + new StringLiteral("2022-01-01"), + new IntervalLiteral(new LongLiteral("10"), DateTimeEnum.DAY), + null + ); + rangePartitions.add(singleRangePartition); + CreateTable table = CreateTable.builder() + .tableName(QualifiedName.of("abc")) + .partition(new RangePartitionedBy(columns, rangePartitions)) + .build(); + List baseClientProperties = starRocksClientConverter.toBaseClientProperty(table); + assertEquals(1, baseClientProperties.size()); + BaseClientProperty baseClientProperty = baseClientProperties.get(0); + assertEquals(StarRocksProperty.TABLE_RANGE_PARTITION.getValue(), baseClientProperty.getKey()); + MultiRangeClientPartition value1 = (MultiRangeClientPartition)baseClientProperty.getValue(); + String start = value1.getStart(); + assertEquals("2021-01-01", start); + assertEquals("2022-01-01", value1.getEnd()); + assertEquals(DateTimeEnum.DAY, value1.getDateTimeEnum()); + assertTrue(value1.getInterval() == Long.parseLong("10")); + } + + @Test + public void getDataType() { + Column column = Column.builder() + .dataType("int") + .build(); + starRocksClientConverter.getDataType(column); + } +} \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksPropertyConverterTest.java b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksPropertyConverterTest.java new file mode 100644 index 0000000..b068ff2 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/converter/StarRocksPropertyConverterTest.java @@ -0,0 +1,25 @@ +package com.aliyun.fastmodel.transform.starrocks.client.converter; + +import java.util.Map; +import java.util.function.Function; + +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Desc: + * + * @author panguanjing + * @date 2023/9/16 + */ +public class StarRocksPropertyConverterTest { + + StarRocksPropertyConverter starRocksPropertyConverter = new StarRocksPropertyConverter(); + @Test + public void getFunctionMap() { + Map> functionMap = starRocksPropertyConverter.getFunctionMap(); + assertEquals(3, functionMap.size()); + } +} \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/generator/StarRocksGeneratorAlterTest.java b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/generator/StarRocksGeneratorAlterTest.java new file mode 100644 index 0000000..b1f2ea5 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/generator/StarRocksGeneratorAlterTest.java @@ -0,0 +1,256 @@ +package com.aliyun.fastmodel.transform.starrocks.client.generator; + +import java.util.List; +import java.util.stream.Collectors; + +import com.aliyun.fastmodel.transform.api.client.CodeGenerator; +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.api.client.dto.property.StringProperty; +import com.aliyun.fastmodel.transform.api.client.dto.request.DdlGeneratorModelRequest; +import com.aliyun.fastmodel.transform.api.client.dto.result.DdlGeneratorResult; +import com.aliyun.fastmodel.transform.api.client.dto.table.Column; +import com.aliyun.fastmodel.transform.api.client.dto.table.Table; +import com.aliyun.fastmodel.transform.api.client.dto.table.TableConfig; +import com.aliyun.fastmodel.transform.api.client.generator.DefaultCodeGenerator; +import com.aliyun.fastmodel.transform.api.dialect.DialectMeta; +import com.aliyun.fastmodel.transform.api.dialect.DialectName; +import com.aliyun.fastmodel.transform.api.dialect.DialectNode; +import com.aliyun.fastmodel.transform.api.dialect.IVersion; +import com.google.common.collect.Lists; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * ddl generator test + * + * @author panguanjing + * @date 2023/9/17 + */ +public class StarRocksGeneratorAlterTest { + CodeGenerator codeGenerator = new DefaultCodeGenerator(); + + @Test + public void testGeneratorAddColumn() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List oneColumns = Lists.newArrayList(); + Column e1 = Column.builder() + .dataType("int") + .name("c1") + .nullable(true) + .build(); + oneColumns.add(e1); + + List twoColumns = Lists.newArrayList(); + twoColumns.add(e1); + twoColumns.add(Column.builder() + .dataType("int") + .name("c2") + .comment("comment") + .build()); + Table after = Table.builder() + .database("autotest") + .name("abc") + .columns(twoColumns) + .comment("comment") + .build(); + request.setAfter(after); + Table before = Table.builder() + .database("autotest") + .name("abc") + .columns(oneColumns) + .comment("comment2") + .build(); + request.setBefore(before); + request.setConfig(TableConfig.builder() + .dialectMeta(DialectMeta.getByNameAndVersion(DialectName.STARROCKS.getValue(), IVersion.DEFAULT_VERSION)) + .build()); + DdlGeneratorResult generate = codeGenerator.generate(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals("ALTER TABLE autotest.abc ADD COLUMN\n" + + "(\n" + + " c2 INT NOT NULL COMMENT \"comment\"\n" + + ");\n" + + "ALTER TABLE autotest.abc SET COMMENT \"comment\"", + dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining(";\n"))); + } + + @Test + public void testGeneratorDropColumn() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List beforeColumns = Lists.newArrayList(); + + beforeColumns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(false) + .build()); + + Column build = Column.builder() + .dataType("int") + .name("c2") + .partitionKeyIndex(0) + .partitionKey(true) + .comment("comment") + .build(); + beforeColumns.add(build); + + List afterColumns = Lists.newArrayList(); + afterColumns.add(build); + + Table before = Table.builder() + .database("autotest") + .name("abc") + .columns(beforeColumns) + .comment("comment1") + .build(); + + Table after = Table.builder() + .database("autotest") + .name("abc") + .columns(afterColumns) + .build(); + + request.setBefore(before); + request.setAfter(after); + request.setConfig(TableConfig.builder() + .dialectMeta(DialectMeta.getByNameAndVersion(DialectName.STARROCKS.getValue(), IVersion.DEFAULT_VERSION)) + .build()); + DdlGeneratorResult generate = codeGenerator.generate(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals("ALTER TABLE autotest.abc DROP COLUMN c1,\n" + + "ALTER TABLE autotest.abc SET COMMENT \"\"", dialectNodes.stream().filter(DialectNode::isExecutable).map(DialectNode::getNode) + .collect(Collectors.joining(",\n"))); + } + + @Test + public void testGeneratorModifyColumn() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List beforeColumns = Lists.newArrayList(); + + beforeColumns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(false) + .build()); + + Column build = Column.builder() + .dataType("int") + .name("c1") + .comment("comment") + .nullable(true) + .build(); + + List afterColumns = Lists.newArrayList(); + afterColumns.add(build); + + Table before = Table.builder() + .database("autotest") + .name("abc") + .columns(beforeColumns) + .build(); + + Table after = Table.builder() + .database("autotest") + .name("abc") + .columns(afterColumns) + .build(); + + request.setBefore(before); + request.setAfter(after); + request.setConfig(TableConfig.builder() + .dialectMeta(DialectMeta.getByNameAndVersion(DialectName.STARROCKS.getValue(), IVersion.DEFAULT_VERSION)) + .build()); + DdlGeneratorResult generate = codeGenerator.generate(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals("ALTER TABLE autotest.abc MODIFY COLUMN c1 INT NULL COMMENT \"comment\"", dialectNodes.stream().filter(DialectNode::isExecutable) + .map(DialectNode::getNode) + .collect(Collectors.joining(",\n"))); + } + + @Test + public void testGeneratorSetProperties() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List beforeColumns = Lists.newArrayList(); + + beforeColumns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(false) + .build()); + + StringProperty stringProperty = new StringProperty(); + stringProperty.setKey("default.replication_num"); + stringProperty.setValueString("2"); + List properties = Lists.newArrayList( + stringProperty + ); + Table before = Table.builder() + .database("autotest") + .name("abc") + .columns(beforeColumns) + .build(); + + Table after = Table.builder() + .database("autotest") + .name("abc") + .columns(beforeColumns) + .properties(properties) + .build(); + + request.setBefore(before); + request.setAfter(after); + request.setConfig(TableConfig.builder() + .dialectMeta(DialectMeta.getByNameAndVersion(DialectName.STARROCKS.getValue(), IVersion.DEFAULT_VERSION)) + .build()); + DdlGeneratorResult generate = codeGenerator.generate(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals("ALTER TABLE autotest.abc SET (\"default.replication_num\"=\"2\")", dialectNodes.stream().filter(DialectNode::isExecutable) + .map(DialectNode::getNode) + .collect(Collectors.joining(",\n"))); + + } + + @Test + public void testGeneratorUnSetProperties() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List beforeColumns = Lists.newArrayList(); + + beforeColumns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(false) + .build()); + + StringProperty stringProperty = new StringProperty(); + stringProperty.setKey("default.replication_num"); + stringProperty.setValueString("2"); + List properties = Lists.newArrayList( + stringProperty + ); + Table before = Table.builder() + .database("autotest") + .name("abc") + .columns(beforeColumns) + .properties(properties) + .build(); + + Table after = Table.builder() + .database("autotest") + .name("abc") + .columns(beforeColumns) + .build(); + + request.setBefore(before); + request.setAfter(after); + request.setConfig(TableConfig.builder() + .dialectMeta(DialectMeta.getByNameAndVersion(DialectName.STARROCKS.getValue(), IVersion.DEFAULT_VERSION)) + .build()); + DdlGeneratorResult generate = codeGenerator.generate(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals("ALTER TABLE autotest.abc SET (\"default.replication_num\"=\"\")", dialectNodes.stream().filter(DialectNode::isExecutable) + .map(DialectNode::getNode) + .collect(Collectors.joining(",\n"))); + + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/generator/StarRocksGeneratorTest.java b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/generator/StarRocksGeneratorTest.java new file mode 100644 index 0000000..4b42cc7 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/client/generator/StarRocksGeneratorTest.java @@ -0,0 +1,694 @@ +package com.aliyun.fastmodel.transform.starrocks.client.generator; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import com.aliyun.fastmodel.core.tree.expr.enums.DateTimeEnum; +import com.aliyun.fastmodel.transform.api.client.CodeGenerator; +import com.aliyun.fastmodel.transform.api.client.dto.property.BaseClientProperty; +import com.aliyun.fastmodel.transform.api.client.dto.property.StringProperty; +import com.aliyun.fastmodel.transform.api.client.dto.request.DdlGeneratorModelRequest; +import com.aliyun.fastmodel.transform.api.client.dto.result.DdlGeneratorResult; +import com.aliyun.fastmodel.transform.api.client.dto.table.Column; +import com.aliyun.fastmodel.transform.api.client.dto.table.Table; +import com.aliyun.fastmodel.transform.api.client.dto.table.TableConfig; +import com.aliyun.fastmodel.transform.api.client.generator.DefaultCodeGenerator; +import com.aliyun.fastmodel.transform.api.dialect.DialectMeta; +import com.aliyun.fastmodel.transform.api.dialect.DialectName; +import com.aliyun.fastmodel.transform.api.dialect.DialectNode; +import com.aliyun.fastmodel.transform.api.dialect.IVersion; +import com.aliyun.fastmodel.transform.starrocks.client.property.column.AggrColumnProperty; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.DistributeBucketsNum; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.DistributeHash; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.MultiRangePartitionProperty; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.ReplicationNum; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.SingleRangePartitionProperty; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.TablePartitionRaw; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.ArrayClientPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.LessThanClientPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.MultiRangeClientPartition; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.PartitionClientValue; +import com.aliyun.fastmodel.transform.starrocks.client.property.table.partition.SingleRangeClientPartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.AggDesc; +import com.google.common.collect.Lists; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * ddl generator test + * + * @author panguanjing + * @date 2023/9/17 + */ +public class StarRocksGeneratorTest { + CodeGenerator codeGenerator = new DefaultCodeGenerator(); + + /** + * 通过string property传入 + */ + @Test + public void testGeneratorProperties1() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("decimal") + .precision(10) + .scale(4) + .name("c1") + .nullable(true) + .build()); + + List properties = Lists.newArrayList(); + StringProperty stringProperty = new StringProperty(); + stringProperty.setKey("replication_num"); + stringProperty.setValueString("3"); + properties.add(stringProperty); + Table after = Table.builder() + .database("autotest") + .name("abc") + .columns(columns) + .comment("comment") + .properties(properties) + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes().stream() + .filter(DialectNode::getExecutable) + .collect(Collectors.toList()); + assertEquals("CREATE TABLE IF NOT EXISTS autotest.abc\n" + + "(\n" + + " c1 DECIMAL(10,4) NULL\n" + + ")\n" + + "COMMENT \"comment\"\n" + + "PROPERTIES (\"replication_num\"=\"3\")", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + /** + * 通过固定的property传入 + */ + @Test + public void testGeneratorProperties2() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("decimal") + .precision(10) + .scale(4) + .name("c1") + .nullable(true) + .build()); + + List properties = Lists.newArrayList(); + ReplicationNum stringProperty = new ReplicationNum(); + stringProperty.setValueString("3"); + properties.add(stringProperty); + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .properties(properties) + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes().stream() + .filter(DialectNode::getExecutable) + .collect(Collectors.toList()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 DECIMAL(10,4) NULL\n" + + ")\n" + + "COMMENT \"comment\"\n" + + "PROPERTIES (\"replication_num\"=\"3\")", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testGeneratorSupportDecimal() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("decimal") + .precision(10) + .scale(4) + .name("c1") + .nullable(true) + .build()); + + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes().stream() + .filter(DialectNode::getExecutable) + .collect(Collectors.toList()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 DECIMAL(10,4) NULL\n" + + ")\n" + + "COMMENT \"comment\"", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testGeneratorSupportArrayInt() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("array") + .name("c1") + .nullable(true) + .build()); + + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes().stream() + .filter(DialectNode::getExecutable) + .collect(Collectors.toList()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 ARRAY NULL\n" + + ")\n" + + "COMMENT \"comment\"", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testGeneratorSupportPartitionRaw() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(true) + .build()); + List properties = Lists.newArrayList(); + TablePartitionRaw e = new TablePartitionRaw(); + e.setValueString("PARTITION BY RANGE (pay_dt) (\n" + + " PARTITION p1 VALUES LESS THAN (\"20210102\"),\n" + + " PARTITION p2 VALUES LESS THAN (\"20210103\"),\n" + + " PARTITION p3 VALUES LESS THAN MAXVALUE\n" + + ")"); + properties.add(e); + + DistributeHash distributeHash = new DistributeHash(); + distributeHash.setValueString("c1"); + properties.add(distributeHash); + + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .properties(properties) + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes().stream() + .filter(DialectNode::getExecutable) + .collect(Collectors.toList()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 INT NULL\n" + + ")\n" + + "COMMENT \"comment\"\n" + + "PARTITION BY RANGE (pay_dt) (\n" + + " PARTITION p1 VALUES LESS THAN (\"20210102\"),\n" + + " PARTITION p2 VALUES LESS THAN (\"20210103\"),\n" + + " PARTITION p3 VALUES LESS THAN MAXVALUE\n" + + ")\n" + + "DISTRIBUTED BY HASH(c1)", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testGeneratorSupportDistribute() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(true) + .build()); + List properties = Lists.newArrayList(); + DistributeBucketsNum e = new DistributeBucketsNum(); + e.setValueString("1"); + properties.add(e); + + DistributeHash distributeHash = new DistributeHash(); + distributeHash.setValueString("c1");//多个列用,分割 + properties.add(distributeHash); + + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .properties(properties) + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes().stream() + .filter(DialectNode::getExecutable) + .collect(Collectors.toList()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 INT NULL\n" + + ")\n" + + "COMMENT \"comment\"\n" + + "DISTRIBUTED BY HASH(c1) BUCKETS 1", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testGeneratorSingleRangePartitionProperty() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(true) + .build()); + + columns.add(Column.builder() + .dataType("int") + .name("c2") + .partitionKeyIndex(0) + .partitionKey(true) + .comment("comment") + .build()); + List properties = Lists.newArrayList(); + //first + SingleRangePartitionProperty e = new SingleRangePartitionProperty(); + SingleRangeClientPartition value = new SingleRangeClientPartition(); + value.setName("a1"); + value.setIfNotExists(true); + List> partitionValues = Lists.newArrayList(); + PartitionClientValue p1 = PartitionClientValue.builder() + .value("2021-01-01") + .build(); + partitionValues.add(Lists.newArrayList(p1)); + PartitionClientValue p2 = PartitionClientValue.builder() + .value("2022-01-02") + .build(); + PartitionClientValue p3 = PartitionClientValue.builder() + .maxValue(true) + .build(); + ; + partitionValues.add(Lists.newArrayList(p2, p3)); + ArrayClientPartitionKey partitionKey = ArrayClientPartitionKey.builder() + .partitionValue(partitionValues) + .build(); + value.setPartitionKey(partitionKey); + e.setValue(value); + properties.add(e); + + //second + //first + e = new SingleRangePartitionProperty(); + value = new SingleRangeClientPartition(); + value.setName("a2"); + value.setIfNotExists(false); + partitionValues = Lists.newArrayList(); + p1 = PartitionClientValue.builder() + .value("2021-01-01") + .build(); + partitionValues.add(Lists.newArrayList(p1)); + p2 = PartitionClientValue.builder() + .value("2022-01-02") + .build(); + p3 = PartitionClientValue.builder() + .maxValue(true) + .build(); + partitionValues.add(Lists.newArrayList(p2, p3)); + partitionKey = ArrayClientPartitionKey.builder() + .partitionValue(partitionValues) + .build(); + value.setPartitionKey(partitionKey); + e.setValue(value); + properties.add(e); + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .properties(properties) + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 INT NULL,\n" + + " c2 INT NOT NULL COMMENT \"comment\"\n" + + ")\n" + + "COMMENT \"comment\"\n" + + "PARTITION BY RANGE (c2)\n" + + "(\n" + + " PARTITION IF NOT EXISTS a1 VALUES [(\"2021-01-01\"),(\"2022-01-02\",MAXVALUE)),\n" + + " PARTITION a2 VALUES [(\"2021-01-01\"),(\"2022-01-02\",MAXVALUE))\n" + + ")", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testGeneratorSingleRangePartitionPropertyWithLessThan() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(true) + .build()); + + columns.add(Column.builder() + .dataType("int") + .name("c2") + .partitionKeyIndex(0) + .partitionKey(true) + .comment("comment") + .build()); + List properties = Lists.newArrayList(); + SingleRangePartitionProperty e = new SingleRangePartitionProperty(); + SingleRangeClientPartition value = new SingleRangeClientPartition(); + value.setName("a1"); + value.setIfNotExists(true); + LessThanClientPartitionKey partitionKey = LessThanClientPartitionKey.builder() + .maxValue(false) + .partitionValueList(Lists.newArrayList(PartitionClientValue.builder().value("2020-01-01").build(), + PartitionClientValue.builder().value("2021-02-03").build())) + .build(); + value.setPartitionKey(partitionKey); + e.setValue(value); + properties.add(e); + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .properties(properties) + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 INT NULL,\n" + + " c2 INT NOT NULL COMMENT \"comment\"\n" + + ")\n" + + "COMMENT \"comment\"\n" + + "PARTITION BY RANGE (c2)\n" + + "(\n" + + " PARTITION IF NOT EXISTS a1 VALUES LESS THAN (\"2020-01-01\",\"2021-02-03\")\n" + + ")", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + private DdlGeneratorResult getDdlGeneratorResult(DdlGeneratorModelRequest request) { + request.setConfig(TableConfig.builder() + .dialectMeta(DialectMeta.getByNameAndVersion(DialectName.STARROCKS.getValue(), IVersion.DEFAULT_VERSION)) + .build()); + return codeGenerator.generate(request); + } + + @Test + public void testGeneratorMultiRange() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + List columnProp = Lists.newArrayList(); + AggrColumnProperty aggrColumnProperty = new AggrColumnProperty(); + aggrColumnProperty.setValueString(AggDesc.MIN.name()); + columnProp.add(aggrColumnProperty); + + columns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(true) + .properties(columnProp) + .build()); + + columns.add(Column.builder() + .dataType("int") + .name("c2") + .partitionKeyIndex(0) + .partitionKey(true) + .comment("comment") + .build()); + List properties = Lists.newArrayList(); + SingleRangePartitionProperty e = new SingleRangePartitionProperty(); + SingleRangeClientPartition value = new SingleRangeClientPartition(); + value.setName("a1"); + value.setIfNotExists(true); + ArrayList strings = Lists.newArrayList(PartitionClientValue.builder().value("2001-01-01").build(), + PartitionClientValue.builder().value("2002-01-01").build()); + LessThanClientPartitionKey partitionKeyValue = LessThanClientPartitionKey.builder() + .partitionValueList(strings) + .build(); + value.setPartitionKey(partitionKeyValue); + e.setValue(value); + properties.add(e); + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .properties(properties) + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 INT MIN NULL,\n" + + " c2 INT NOT NULL COMMENT \"comment\"\n" + + ")\n" + + "COMMENT \"comment\"\n" + + "PARTITION BY RANGE (c2)\n" + + "(\n" + + " PARTITION IF NOT EXISTS a1 VALUES LESS THAN (\"2001-01-01\",\"2002-01-01\")\n" + + ")", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testGeneratorMultiRangeInterval() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + List columnProp = Lists.newArrayList(); + AggrColumnProperty aggrColumnProperty = new AggrColumnProperty(); + aggrColumnProperty.setValueString(AggDesc.MIN.name()); + columnProp.add(aggrColumnProperty); + + columns.add(Column.builder() + .dataType("int") + .name("c1") + .nullable(true) + .properties(columnProp) + .build()); + + columns.add(Column.builder() + .dataType("int") + .name("c2") + .partitionKeyIndex(0) + .partitionKey(true) + .comment("comment") + .build()); + List properties = Lists.newArrayList(); + MultiRangePartitionProperty e = new MultiRangePartitionProperty(); + MultiRangeClientPartition v = new MultiRangeClientPartition(); + v.setStart("2020-01-01"); + v.setEnd("2023-01-01"); + v.setDateTimeEnum(DateTimeEnum.DAY); + v.setInterval(1L); + e.setValue(v); + properties.add(e); + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .properties(properties) + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 INT MIN NULL,\n" + + " c2 INT NOT NULL COMMENT \"comment\"\n" + + ")\n" + + "COMMENT \"comment\"\n" + + "PARTITION BY RANGE (c2)\n" + + "(\n" + + " START(\"2020-01-01\") END(\"2023-01-01\") EVERY (INTERVAL 1 DAY)\n" + + ")", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testPrimaryKey() { + List columns = Lists.newArrayList(); + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + columns.add(Column.builder() + .dataType("tinyint") + .name("c1") + .nullable(false) + .primaryKey(true) + .comment("ti_comment") + .build()); + columns.add(Column.builder() + .dataType("smallint") + .name("c2") + .nullable(false) + .primaryKey(false) + .comment("si_comment") + .build()); + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals(1, dialectNodes.size()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " c1 TINYINT NOT NULL COMMENT \"ti_comment\",\n" + + " c2 SMALLINT NOT NULL COMMENT \"si_comment\"\n" + + ")\n" + + "PRIMARY KEY (c1)\n" + + "COMMENT \"comment\"", dialectNodes.get(0).getNode()); + } + + @Test + public void testKeyWords() { + List columns = Lists.newArrayList(); + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + columns.add(Column.builder() + .dataType("tinyint") + .name("add") + .nullable(false) + .primaryKey(true) + .comment("ti_comment") + .build()); + columns.add(Column.builder() + .dataType("smallint") + .name("c2") + .nullable(false) + .primaryKey(true) + .comment("si_comment") + .build()); + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes(); + assertEquals(1, dialectNodes.size()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " `add` TINYINT NOT NULL COMMENT \"ti_comment\",\n" + + " c2 SMALLINT NOT NULL COMMENT \"si_comment\"\n" + + ")\n" + + "PRIMARY KEY (`add`,c2)\n" + + "COMMENT \"comment\"", dialectNodes.get(0).getNode()); + } + + @Test + public void testGeneratorDistribute() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("decimal") + .precision(10) + .scale(4) + .name("add") + .nullable(true) + .primaryKey(true) + .build()); + + List properties = Lists.newArrayList(); + DistributeHash distributeHash = new DistributeHash(); + distributeHash.setValueString("add"); + properties.add(distributeHash); + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("comment") + .properties(properties) + .build(); + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes().stream() + .filter(DialectNode::getExecutable) + .collect(Collectors.toList()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " `add` DECIMAL(10,4) NOT NULL\n" + + ")\n" + + "PRIMARY KEY (`add`)\n" + + "COMMENT \"comment\"\n" + + "DISTRIBUTED BY HASH(`add`)", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testGeneratorCommentWithConverter() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("decimal") + .precision(10) + .scale(4) + .name("add") + .nullable(true) + .primaryKey(true) + .build()); + + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("\"\"comment") + .build(); + + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes().stream() + .filter(DialectNode::getExecutable) + .collect(Collectors.toList()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " `add` DECIMAL(10,4) NOT NULL\n" + + ")\n" + + "PRIMARY KEY (`add`)\n" + + "COMMENT \"\\\"\\\"comment\"", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } + + @Test + public void testGeneratorCommentWithSingleQuote() { + DdlGeneratorModelRequest request = new DdlGeneratorModelRequest(); + List columns = Lists.newArrayList(); + columns.add(Column.builder() + .dataType("decimal") + .precision(10) + .scale(4) + .name("add") + .nullable(true) + .primaryKey(true) + .build()); + + Table after = Table.builder() + .name("abc") + .columns(columns) + .comment("'\\comment'") + .build(); + + request.setAfter(after); + DdlGeneratorResult generate = getDdlGeneratorResult(request); + List dialectNodes = generate.getDialectNodes().stream() + .filter(DialectNode::getExecutable) + .collect(Collectors.toList()); + assertEquals("CREATE TABLE IF NOT EXISTS abc\n" + + "(\n" + + " `add` DECIMAL(10,4) NOT NULL\n" + + ")\n" + + "PRIMARY KEY (`add`)\n" + + "COMMENT \"'\\\\comment'\"", dialectNodes.stream().map(DialectNode::getNode).collect(Collectors.joining("\n"))); + } +} diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksOutVisitorTest.java b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksOutVisitorTest.java new file mode 100644 index 0000000..f941c7f --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/format/StarRocksOutVisitorTest.java @@ -0,0 +1,298 @@ +package com.aliyun.fastmodel.transform.starrocks.format; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.Property; +import com.aliyun.fastmodel.core.tree.QualifiedName; +import com.aliyun.fastmodel.core.tree.datatype.DataTypeParameter; +import com.aliyun.fastmodel.core.tree.datatype.TypeParameter; +import com.aliyun.fastmodel.core.tree.expr.Identifier; +import com.aliyun.fastmodel.core.tree.expr.enums.DateTimeEnum; +import com.aliyun.fastmodel.core.tree.expr.literal.IntervalLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.ListStringLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.LongLiteral; +import com.aliyun.fastmodel.core.tree.expr.literal.StringLiteral; +import com.aliyun.fastmodel.core.tree.statement.table.AddCols; +import com.aliyun.fastmodel.core.tree.statement.table.ChangeCol; +import com.aliyun.fastmodel.core.tree.statement.table.ColumnDefinition; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; +import com.aliyun.fastmodel.core.tree.statement.table.DropCol; +import com.aliyun.fastmodel.core.tree.statement.table.PartitionedBy; +import com.aliyun.fastmodel.core.tree.statement.table.RenameTable; +import com.aliyun.fastmodel.core.tree.statement.table.SetTableProperties; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.BaseConstraint; +import com.aliyun.fastmodel.core.tree.statement.table.constraint.PrimaryConstraint; +import com.aliyun.fastmodel.core.tree.util.DataTypeUtil; +import com.aliyun.fastmodel.core.tree.util.IdentifierUtil; +import com.aliyun.fastmodel.transform.starrocks.context.StarRocksContext; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.AggDesc; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.AggregateConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.DuplicateConstraint; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype.StarRocksGenericDataType; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.LessThanPartitionKey; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.ListPartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiItemListPartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.MultiRangePartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionDesc; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionValue; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.RangePartitionedBy; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleItemListPartition; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.SingleRangePartition; +import com.google.common.collect.Lists; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * StarRocksVisitorTest + * + * @author panguanjing + * @date 2023/9/12 + */ +public class StarRocksOutVisitorTest { + + StarRocksContext context = StarRocksContext.builder().build(); + StarRocksOutVisitor starRocksVisitor = new StarRocksOutVisitor(context); + + @Test + public void testVisitCreateTable() { + StarRocksContext context = StarRocksContext.builder().build(); + StarRocksOutVisitor starRocksVisitor = new StarRocksOutVisitor(context); + List columns = Lists.newArrayList(); + List columnProperties = toColumnProperties(); + ColumnDefinition columnDefinition = ColumnDefinition.builder() + .colName(new Identifier("c1")) + .dataType(new StarRocksGenericDataType("int")) + .defaultValue(new StringLiteral("2001-01-01")) + .properties(columnProperties) + .build(); + columns.add(columnDefinition); + columns.add(ColumnDefinition.builder().colName(new Identifier("c2")).dataType(new StarRocksGenericDataType("int")).build()); + columns.add(ColumnDefinition.builder().colName(new Identifier("c3")).dataType(new StarRocksGenericDataType("int")).build()); + List constraints = toConstraint(); + PartitionedBy partition = toPartition(); + List properties = toProperty(); + CreateTable createTable = CreateTable.builder() + .tableName(QualifiedName.of("abc")) + .partition(partition) + .columns(columns) + .constraints(constraints) + .properties(properties) + .build(); + starRocksVisitor.visitCreateTable(createTable, 0); + String s = starRocksVisitor.getBuilder().toString(); + assertEquals("CREATE TABLE abc\n" + + "(\n" + + " c1 INT SUM DEFAULT \"2001-01-01\",\n" + + " c2 INT,\n" + + " c3 INT\n" + + ")\n" + + "ENGINE=mysql\n" + + "PRIMARY KEY (c1)\n" + + "DUPLICATE KEY (c2)\n" + + "AGGREGATE KEY (c2)\n" + + "PARTITION BY RANGE (c1,c2)\n" + + "(\n" + + " PARTITION p1 VALUES LESS THAN (\"2010-01-10\"),\n" + + " START(\"2001-01-01\") END(\"2020-01-01\") EVERY (INTERVAL 10 HOUR)\n" + + ")", s); + } + + @Test + public void testVisitSetTableProperties() { + List propertiesList = Lists.newArrayList(); + propertiesList.add(new Property("replication_num", "3")); + SetTableProperties setTableProperties = new SetTableProperties( + QualifiedName.of("abc.bcd"), + propertiesList + ); + StarRocksContext context = StarRocksContext.builder().build(); + StarRocksOutVisitor starRocksVisitor = new StarRocksOutVisitor(context); + starRocksVisitor.visitSetTableProperties(setTableProperties, 0); + String s = starRocksVisitor.getBuilder().toString(); + assertEquals("ALTER TABLE abc.bcd SET (\"replication_num\"=\"3\")", s); + } + + @Test + public void testRenameTable() { + RenameTable renameTable = new RenameTable(QualifiedName.of("a"), QualifiedName.of("b")); + StarRocksContext context = StarRocksContext.builder().build(); + StarRocksOutVisitor starRocksVisitor = new StarRocksOutVisitor(context); + starRocksVisitor.visitRenameTable(renameTable, 0); + String s = starRocksVisitor.getBuilder().toString(); + assertEquals("ALTER TABLE a RENAME b", s); + } + + @Test + public void testAddCols() { + List list = Lists.newArrayList(); + ColumnDefinition columnDefinition = ColumnDefinition.builder() + .colName(new Identifier("c1")) + .dataType(new StarRocksGenericDataType("int")) + .build(); + list.add(columnDefinition); + + List arguments = Lists.newArrayList(); + arguments.add(new TypeParameter(DataTypeUtil.simpleType("STRING", null))); + ColumnDefinition columnDefinition2 = ColumnDefinition.builder() + .colName(new Identifier("c2")) + .dataType(new StarRocksGenericDataType("array", arguments)) + .build(); + list.add(columnDefinition2); + AddCols addCols = new AddCols(QualifiedName.of("a"), list); + starRocksVisitor.visitAddCols(addCols, 0); + assertEquals("ALTER TABLE a ADD COLUMN\n" + + "(\n" + + " c1 INT,\n" + + " c2 ARRAY\n" + + ")", starRocksVisitor.getBuilder().toString()); + } + + @Test + public void testDistributeBy() { + List columns = Lists.newArrayList(); + List columnProperties = Lists.newArrayList(); + columns.add(ColumnDefinition.builder().colName(new Identifier("c1")).properties(columnProperties).build()); + List properties = Lists.newArrayList(); + properties.add(new Property(StarRocksProperty.TABLE_DISTRIBUTED_HASH.getValue(), "c1")); + properties.add(new Property(StarRocksProperty.TABLE_DISTRIBUTED_BUCKETS.getValue(), "4")); + CreateTable createTable = CreateTable.builder() + .tableName(QualifiedName.of("ab")) + .columns(columns) + .properties(properties) + .build(); + starRocksVisitor.visitCreateTable(createTable, 0); + assertEquals("CREATE TABLE ab\n" + + "(\n" + + " c1\n" + + ")\n" + + "DISTRIBUTED BY HASH(c1) BUCKETS 4", starRocksVisitor.getBuilder().toString()); + } + + @Test + public void testPartitionValueRaw() { + List properties = Lists.newArrayList(); + properties.add(new Property(StarRocksProperty.TABLE_PARTITION_RAW.getValue(), "PARTITION BY RANGE (pay_dt) (\n" + + " PARTITION p1 VALUES LESS THAN (\"20210102\"),\n" + + " PARTITION p2 VALUES LESS THAN (\"20210103\"),\n" + + " PARTITION p3 VALUES LESS THAN MAXVALUE\n" + + ")")); + List columns = Lists.newArrayList(); + ColumnDefinition e = ColumnDefinition.builder() + .colName(new Identifier("c1")) + .dataType(DataTypeUtil.simpleType("BIGINT", null)) + .build(); + columns.add(e); + CreateTable createTable = CreateTable.builder() + .tableName(QualifiedName.of("abc")) + .columns(columns) + .properties(properties) + .build(); + starRocksVisitor.visitCreateTable(createTable, 0); + String s = starRocksVisitor.getBuilder().toString(); + assertEquals("CREATE TABLE abc\n" + + "(\n" + + " c1 BIGINT\n" + + ")\n" + + "PARTITION BY RANGE (pay_dt) (\n" + + " PARTITION p1 VALUES LESS THAN (\"20210102\"),\n" + + " PARTITION p2 VALUES LESS THAN (\"20210103\"),\n" + + " PARTITION p3 VALUES LESS THAN MAXVALUE\n" + + ")", s); + } + + @Test + public void testVisitDropColumn() { + DropCol dropCol = new DropCol(QualifiedName.of("abc.bcd"), new Identifier("c1")); + starRocksVisitor.visitDropCol(dropCol, 0); + String s = starRocksVisitor.getBuilder().toString(); + assertEquals("ALTER TABLE abc.bcd DROP COLUMN c1", s); + } + + @Test + public void testVisitChangeColumn() { + ChangeCol changeCol = new ChangeCol(QualifiedName.of("abc.bcd"), new Identifier("c1"), + ColumnDefinition.builder().defaultValue(new StringLiteral("1")).colName(new Identifier("c1")) + .notNull(false) + .dataType(new StarRocksGenericDataType("string")).build()); + starRocksVisitor.visitChangeCol(changeCol, 0); + String s = starRocksVisitor.getBuilder().toString(); + assertEquals("ALTER TABLE abc.bcd MODIFY COLUMN c1 STRING NULL DEFAULT \"1\"", s); + } + + @Test + public void testVisitListPartition() { + List columns = Lists.newArrayList( + ColumnDefinition.builder().colName(new Identifier("k1")).build() + ); + ListStringLiteral listStringLiteral = new ListStringLiteral(Lists.newArrayList(new StringLiteral("2021-01-01"))); + List property = Lists.newArrayList(); + property.add(new Property("test", "test_value")); + List stringLiterals = Lists.newArrayList(); + ListStringLiteral e = new ListStringLiteral(Lists.newArrayList(new StringLiteral("2021-01-01"))); + stringLiterals.add(e); + e = new ListStringLiteral(Lists.newArrayList(new StringLiteral("2021-01-01"), new StringLiteral("2023-01-01"))); + stringLiterals.add(e); + List rangePartitons = Lists.newArrayList( + new SingleItemListPartition(new Identifier("p1"), true, listStringLiteral, property), + new MultiItemListPartition(new Identifier("p2"), false, stringLiterals, null) + ); + ListPartitionedBy listPartitionedBy = new ListPartitionedBy( + columns, + rangePartitons + ); + Boolean aBoolean = starRocksVisitor.visitListPartitionedBy(listPartitionedBy, 0); + assertTrue(aBoolean); + assertEquals("PARTITION BY LIST (k1)\n" + + "(\n" + + "PARTITION IF NOT EXISTS p1 VALUES IN (\"2021-01-01\") (\"test\"=\"test_value\"),\n" + + "PARTITION p2 VALUES IN ((\"2021-01-01\"),(\"2021-01-01\",\"2023-01-01\"))\n" + + ")", starRocksVisitor.getBuilder().toString()); + } + + private List toColumnProperties() { + List properties = Lists.newArrayList(); + properties.add(new Property(StarRocksProperty.COLUMN_AGG_DESC.getValue(), AggDesc.SUM.name())); + return properties; + } + + private List toProperty() { + List list = Lists.newArrayList(); + list.add(new Property(StarRocksProperty.TABLE_ENGINE.getValue(), "mysql")); + return list; + } + + private PartitionedBy toPartition() { + List columns = Lists.newArrayList(); + columns.add(ColumnDefinition.builder().colName(new Identifier("c1")).build()); + columns.add(ColumnDefinition.builder().colName(new Identifier("c2")).build()); + List rangePartition = Lists.newArrayList(); + LessThanPartitionKey partitionKey = new LessThanPartitionKey(false, + new ListPartitionValue(Lists.newArrayList(new PartitionValue(false, new StringLiteral("2010-01-10"))))); + SingleRangePartition singleRangePartition = new SingleRangePartition( + new Identifier("p1"), false, partitionKey, null); + + MultiRangePartition multiRangePartition = new MultiRangePartition(new StringLiteral("2001-01-01"), new StringLiteral("2020-01-01"), + new IntervalLiteral(new LongLiteral("10"), DateTimeEnum.HOUR), null); + rangePartition.add(singleRangePartition); + rangePartition.add(multiRangePartition); + return new RangePartitionedBy( + columns, rangePartition + ); + } + + private List toConstraint() { + PrimaryConstraint primaryConstraint = new PrimaryConstraint( + IdentifierUtil.sysIdentifier(), + Lists.newArrayList(new Identifier("c1")) + ); + DuplicateConstraint duplicateConstraint = new DuplicateConstraint( + IdentifierUtil.sysIdentifier(), Lists.newArrayList(new Identifier("c2")), true + ); + + AggregateConstraint aggregateConstraint = new AggregateConstraint(IdentifierUtil.sysIdentifier(), Lists.newArrayList(new Identifier("c2"))); + return Lists.newArrayList(primaryConstraint, duplicateConstraint, aggregateConstraint); + } +} \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/StarRocksLanguageParserTest.java b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/StarRocksLanguageParserTest.java new file mode 100644 index 0000000..e1bcb7c --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/StarRocksLanguageParserTest.java @@ -0,0 +1,157 @@ +package com.aliyun.fastmodel.transform.starrocks.parser; + +import java.util.List; + +import com.aliyun.fastmodel.core.tree.Node; +import com.aliyun.fastmodel.core.tree.datatype.BaseDataType; +import com.aliyun.fastmodel.core.tree.statement.table.CreateTable; +import com.aliyun.fastmodel.core.tree.statement.table.PartitionedBy; +import com.aliyun.fastmodel.transform.api.context.ReverseContext; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype.StarRocksDataTypeName; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.PartitionDesc; +import com.aliyun.fastmodel.transform.starrocks.parser.tree.partition.RangePartitionedBy; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Desc: + * + * @author panguanjing + * @date 2023/9/11 + */ +public class StarRocksLanguageParserTest { + StarRocksLanguageParser starRocksLanguageParser = new StarRocksLanguageParser(); + + @Test + public void parseNode() { + Node node = starRocksLanguageParser.parseNode("CREATE TABLE example_db.table_hash\n" + + "(\n" + + " k1 TINYINT,\n" + + " k2 DECIMAL(10, 2) DEFAULT \"10.5\",\n" + + " v1 CHAR(10) REPLACE,\n" + + " v2 INT SUM\n" + + ")\n" + + "ENGINE=olap\n" + + "AGGREGATE KEY(k1, k2)\n" + + "COMMENT \"my first starrocks table\"\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 10\n" + + "PROPERTIES (\"storage_type\"=\"column\");"); + assertNotNull(node); + CreateTable createTable = (CreateTable)node; + assertEquals("example_db.table_hash", createTable.getQualifiedName().toString()); + assertEquals(4, createTable.getColumnDefines().size()); + } + + @Test + public void testParseNodeWithPartition() { + Node node = starRocksLanguageParser.parseNode("CREATE TABLE example_db.table_range\n" + + "(\n" + + " k1 DATE,\n" + + " k2 INT,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k1, k2, k3)\n" + + "PARTITION BY RANGE (k1)\n" + + "(\n" + + " PARTITION p1 VALUES LESS THAN (\"2014-01-01\"),\n" + + " PARTITION p2 VALUES LESS THAN (\"2014-06-01\"),\n" + + " PARTITION p3 VALUES LESS THAN (\"2014-12-01\")\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\n" + + " \"storage_medium\" = \"SSD\", \n" + + " \"storage_cooldown_time\" = \"2015-06-04 00:00:00\"\n" + + ");"); + assertNotNull(node); + CreateTable createTable = (CreateTable)node; + PartitionedBy partitionedBy = createTable.getPartitionedBy(); + RangePartitionedBy starRocksPartitionedBy = (RangePartitionedBy)partitionedBy; + List rangePartitions = starRocksPartitionedBy.getRangePartitions(); + assertEquals(3, rangePartitions.size()); + } + + @Test + public void testParseWithFixPartition() { + Node node = starRocksLanguageParser.parseNode("CREATE TABLE table_range\n" + + "(\n" + + " k1 DATE,\n" + + " k2 INT,\n" + + " k3 SMALLINT,\n" + + " v1 VARCHAR(2048),\n" + + " v2 DATETIME DEFAULT \"2014-02-04 15:36:00\"\n" + + ")\n" + + "ENGINE=olap\n" + + "DUPLICATE KEY(k1, k2, k3)\n" + + "PARTITION BY RANGE (k1, k2, k3)\n" + + "(\n" + + " PARTITION p1 VALUES [(\"2014-01-01\", \"10\", \"200\"), (\"2014-01-01\", \"20\", \"300\")),\n" + + " PARTITION p2 VALUES [(\"2014-06-01\", \"100\", \"200\"), (\"2014-07-01\", \"100\", \"300\"))\n" + + ")\n" + + "DISTRIBUTED BY HASH(k2) BUCKETS 10\n" + + "PROPERTIES(\n" + + " \"storage_medium\" = \"SSD\"\n" + + ");"); + assertNotNull(node); + } + + @Test + public void testParseWithHllColumns() { + Node node = starRocksLanguageParser.parseNode("CREATE TABLE example_db.example_table\n" + + "(\n" + + " k1 TINYINT,\n" + + " k2 DECIMAL(10, 2) DEFAULT \"10.5\",\n" + + " v1 HLL HLL_UNION,\n" + + " v2 HLL HLL_UNION\n" + + ")\n" + + "ENGINE=olap\n" + + "AGGREGATE KEY(k1, k2)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 10\n" + + "PROPERTIES (\"storage_type\"=\"column\");"); + assertNotNull(node); + } + + @Test + public void testWithBitMapUnion() { + Node node = starRocksLanguageParser.parseNode("CREATE TABLE example_db.example_table\n" + + "(\n" + + " k1 TINYINT,\n" + + " k2 DECIMAL(10, 2) DEFAULT \"10.5\",\n" + + " v1 BITMAP BITMAP_UNION,\n" + + " v2 BITMAP BITMAP_UNION\n" + + ")\n" + + "ENGINE=olap\n" + + "AGGREGATE KEY(k1, k2)\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 10\n" + + "PROPERTIES (\"storage_type\"=\"column\");"); + assertNotNull(node); + } + + @Test + public void testParseWithIndex() { + Node createTableNode = starRocksLanguageParser.parseNode("CREATE TABLE example_db.table_hash\n" + + "(\n" + + " k1 TINYINT,\n" + + " k2 DECIMAL(10, 2) DEFAULT \"10.5\",\n" + + " v1 CHAR(10) REPLACE,\n" + + " v2 INT SUM,\n" + + " INDEX k1_idx (k1) USING BITMAP COMMENT 'xxxxxx'\n" + + ")\n" + + "ENGINE=olap\n" + + "AGGREGATE KEY(k1, k2)\n" + + "COMMENT \"my first starrocks table\"\n" + + "DISTRIBUTED BY HASH(k1) BUCKETS 10\n" + + "PROPERTIES (\"storage_type\"=\"column\");"); + assertNotNull(createTableNode); + } + + @Test + public void testParseDataType() { + BaseDataType baseDataType = starRocksLanguageParser.parseDataType("array", ReverseContext.builder().build()); + assertEquals(baseDataType.getTypeName().getValue(), StarRocksDataTypeName.ARRAY.getValue()); + } +} \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksDataTypeNameTest.java b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksDataTypeNameTest.java new file mode 100644 index 0000000..e82fa03 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/tree/datatype/StarRocksDataTypeNameTest.java @@ -0,0 +1,26 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.tree.datatype; + +import com.aliyun.fastmodel.core.tree.datatype.IDataTypeName; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Desc: + * + * @author panguanjing + * @date 2023/9/21 + */ +public class StarRocksDataTypeNameTest { + + @Test + public void testGetByValue() { + IDataTypeName byValue = StarRocksDataTypeName.getByValue("array"); + assertEquals(StarRocksDataTypeName.ARRAY, byValue); + byValue = StarRocksDataTypeName.getByValue("json"); + assertEquals(StarRocksDataTypeName.JSON, byValue); + + IDataTypeName byValue1 = StarRocksDataTypeName.getByValue("map"); + assertEquals(StarRocksDataTypeName.Map, byValue1); + } +} \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/util/StarRocksReservedWordUtilTest.java b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/util/StarRocksReservedWordUtilTest.java new file mode 100644 index 0000000..b849276 --- /dev/null +++ b/fastmodel-transform/fastmodel-transform-starrocks/src/test/java/com/aliyun/fastmodel/transform/starrocks/parser/util/StarRocksReservedWordUtilTest.java @@ -0,0 +1,20 @@ +package com.aliyun.fastmodel.transform.starrocks.parser.util; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * Desc: + * + * @author panguanjing + * @date 2023/11/6 + */ +public class StarRocksReservedWordUtilTest { + + @Test + public void isReservedKeyWord() { + boolean reservedKeyWord = StarRocksReservedWordUtil.isReservedKeyWord("add"); + assertTrue(reservedKeyWord); + } +} \ No newline at end of file diff --git a/fastmodel-transform/fastmodel-transform-zen/src/test/java/com/aliyun/aliyun/transform/zen/compare/ZenNodeCompareTest.java b/fastmodel-transform/fastmodel-transform-zen/src/test/java/com/aliyun/aliyun/transform/zen/compare/ZenNodeCompareTest.java index 1db6438..b17a1ca 100644 --- a/fastmodel-transform/fastmodel-transform-zen/src/test/java/com/aliyun/aliyun/transform/zen/compare/ZenNodeCompareTest.java +++ b/fastmodel-transform/fastmodel-transform-zen/src/test/java/com/aliyun/aliyun/transform/zen/compare/ZenNodeCompareTest.java @@ -46,12 +46,12 @@ public void testCompare() { .build()); BaseStatement beforeStatement = compareResult.getBeforeStatement(); BaseStatement afterStatement = compareResult.getAfterStatement(); - assertEquals(beforeStatement.toString(), "CREATE DIM TABLE dim_shop \n" + assertEquals(beforeStatement.toString(), "CREATE TABLE dim_shop \n" + "(\n" + " user_id STRING COMMENT 'user_id',\n" + " user_name STRING COMMENT 'user_name'\n" + ")"); - assertEquals(afterStatement.toString(), "CREATE DIM TABLE dim_shop \n" + assertEquals(afterStatement.toString(), "CREATE TABLE dim_shop \n" + "(\n" + " user_id STRING COMMENT 'user_id',\n" + " user_name STRING COMMENT 'user_name',\n" diff --git a/fastmodel-transform/pom.xml b/fastmodel-transform/pom.xml index 7073eb1..7c3fa12 100644 --- a/fastmodel-transform/pom.xml +++ b/fastmodel-transform/pom.xml @@ -45,6 +45,7 @@ fastmodel-transform-spark fastmodel-transform-adbmysql fastmodel-transform-sqlite + fastmodel-transform-starrocks diff --git a/pom.xml b/pom.xml index 5bbd30b..f42cb9c 100644 --- a/pom.xml +++ b/pom.xml @@ -55,13 +55,13 @@ - 8 - 8 + 11 + 11 UTF-8 0.8.5 - 0.5.8-jdk8 + 0.5.9 1.4.1 - 4.9.3 + 4.13.1 @@ -144,24 +144,91 @@ + + + jdk11 + + 0.5.9 + + + true + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + 11 + true + true + true + true + + + + org.antlr + antlr4-maven-plugin + 4.13.0 + + + + antlr4 + + + + + true + + + + + + + jdk8 + + 0.5.9-jdk8 + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 8 + 8 + true + true + true + true + + + + org.antlr + antlr4-maven-plugin + 4.9.3 + + + + antlr4 + + + + + true + + + + + + + - - org.antlr - antlr4-maven-plugin - ${dep.antlr.version} - - - - antlr4 - - - - - true - - org.jacoco jacoco-maven-plugin @@ -202,22 +269,34 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.0 + 3.8.1 - ${java.compile.version} - ${java.version} - ${java.version} - ${project.build.sourceEncoding} + 11 + 11 true true true true + + org.antlr + antlr4-maven-plugin + 4.13.0 + + + + antlr4 + + + + + true + + org.jacoco jacoco-maven-plugin - prepare-agent @@ -267,4 +346,5 @@ + \ No newline at end of file