From 96702e0f16dbc396fbf16b0d32ebc1f8e402e7c8 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Thu, 22 Aug 2024 17:41:34 +0800 Subject: [PATCH 01/23] =?UTF-8?q?add(member):=E5=BC=A0=E5=A4=A9=E7=91=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../process/function/FactorialFunction.java | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java new file mode 100644 index 00000000000..abd2df49752 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; + +import java.math.BigDecimal; + +/** + * FactorialFunction + * description: factorial(numeric)--returns the factorial of a non-negative + * integer + */ +public class FactorialFunction implements ValueParser { + + private ValueParser numberParser; + + /** + * Constructor + * + * @param expr + */ + public FactorialFunction(Function expr) { + numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + /** + * parse + * + * @param sourceData + * @param rowIndex + * @return + */ + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object numberObj = numberParser.parse(sourceData, rowIndex, context); + BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); + + // Ensure the number is a non-negative integer + if (numberValue.scale() > 0 || numberValue.compareTo(BigDecimal.ZERO) < 0) { + throw new IllegalArgumentException("Factorial is only defined for non-negative integers."); + } + + return factorial(numberValue.intValue()); + } + + /** + * Helper method to calculate factorial + * + * @param n + * @return + */ + private BigDecimal factorial(int n) { + BigDecimal result = BigDecimal.ONE; + for (int i = 2; i <= n; i++) { + result = result.multiply(BigDecimal.valueOf(i)); + } + return result; + } +} From 35eceba7ead77a147f40216d82990d200e45df49 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Thu, 22 Aug 2024 20:50:33 +0800 Subject: [PATCH 02/23] [SDK] Transform support factorial function #10859 --- .../process/operator/OperatorTools.java | 2 ++ ...TransformArithmeticFunctionsProcessor.java | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java index fe361263a43..70ed13630e4 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java @@ -43,6 +43,7 @@ import org.apache.inlong.sdk.transform.process.function.ToDateFunction; import org.apache.inlong.sdk.transform.process.function.ToTimestampFunction; import org.apache.inlong.sdk.transform.process.function.UnixTimestampFunction; +import org.apache.inlong.sdk.transform.process.function.FactorialFunction; import org.apache.inlong.sdk.transform.process.parser.AdditionParser; import org.apache.inlong.sdk.transform.process.parser.ColumnParser; import org.apache.inlong.sdk.transform.process.parser.DateParser; @@ -55,6 +56,7 @@ import org.apache.inlong.sdk.transform.process.parser.TimestampParser; import org.apache.inlong.sdk.transform.process.parser.ValueParser; + import net.sf.jsqlparser.expression.DateValue; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Function; diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 7ec41e4c328..be2d3cd79f9 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -174,6 +174,41 @@ public void testLog10Function() throws Exception { Assert.assertEquals(output2.get(0), "result=3.0"); } + @Test + public void testFactorialFunction() throws Exception { + String transformSql = "select factorial(numeric1) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + // case1: 测试非负整数的阶乘 + List output1 = processor.transform("5|4|6|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=120"); + + // case2: 测试0的阶乘 + List output2 = processor.transform("0|4|6|8"); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=1"); + + // case3: 测试带有小数的输入，应抛出异常 + try { + List output3 = processor.transform("5.5|4|6|8"); + Assert.fail("Expected an exception for non-integer input"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("Factorial is only defined for non-negative integers.", e.getMessage()); + } + + // case4: 测试负数的输入，应抛出异常 + try { + List output4 = processor.transform("-5|4|6|8"); + Assert.fail("Expected an exception for negative input"); + } catch (IllegalArgumentException e) { + Assert.assertEquals("Factorial is only defined for non-negative integers.", e.getMessage()); + } + } +} + @Test public void testLog2Function() throws Exception { String transformSql = "select log2(numeric1) from source"; From 15cf7ee2e4affa45a69955668d8fde9a049f184b Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Sun, 25 Aug 2024 15:19:56 +0800 Subject: [PATCH 03/23] [INLONG-10873][SDK] Transform support factorial function --- .../TestTransformArithmeticFunctionsProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index be2d3cd79f9..35bf13326af 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -181,17 +181,17 @@ public void testFactorialFunction() throws Exception { TransformProcessor processor = TransformProcessor .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), SinkEncoderFactory.createKvEncoder(kvSink)); - // case1: 测试非负整数的阶乘 + // case1: 5！ List output1 = processor.transform("5|4|6|8"); Assert.assertEquals(1, output1.size()); Assert.assertEquals(output1.get(0), "result=120"); - // case2: 测试0的阶乘 + // case2: 0！ List output2 = processor.transform("0|4|6|8"); Assert.assertEquals(1, output2.size()); Assert.assertEquals(output2.get(0), "result=1"); - // case3: 测试带有小数的输入，应抛出异常 + // case3: 5.5！ try { List output3 = processor.transform("5.5|4|6|8"); Assert.fail("Expected an exception for non-integer input"); @@ -199,7 +199,7 @@ public void testFactorialFunction() throws Exception { Assert.assertEquals("Factorial is only defined for non-negative integers.", e.getMessage()); } - // case4: 测试负数的输入，应抛出异常 + // case4: -5！ try { List output4 = processor.transform("-5|4|6|8"); Assert.fail("Expected an exception for negative input"); From 8a79da627ad8488451ed040efa069c68bcde869a Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Sun, 25 Aug 2024 15:40:35 +0800 Subject: [PATCH 04/23] [INLONG-10873][SDK] Transform support factorial function --- .../process/TestTransformArithmeticFunctionsProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 35bf13326af..2ceacfb61f7 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -207,7 +207,6 @@ public void testFactorialFunction() throws Exception { Assert.assertEquals("Factorial is only defined for non-negative integers.", e.getMessage()); } } -} @Test public void testLog2Function() throws Exception { From 4ba63067e0e6933f12f48143b50170df63d7c87f Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Sun, 25 Aug 2024 16:10:22 +0800 Subject: [PATCH 05/23] [INLONG-10873][SDK] Transform support factorial function --- .../transform/process/function/FactorialFunction.java | 1 + .../sdk/transform/process/operator/OperatorTools.java | 10 ++++++---- .../TestTransformArithmeticFunctionsProcessor.java | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java index abd2df49752..3d7efa6a532 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.inlong.sdk.transform.process.function; import org.apache.inlong.sdk.transform.decode.SourceData; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java index 70ed13630e4..2522e2fb44e 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java @@ -56,7 +56,6 @@ import org.apache.inlong.sdk.transform.process.parser.TimestampParser; import org.apache.inlong.sdk.transform.process.parser.ValueParser; - import net.sf.jsqlparser.expression.DateValue; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Function; @@ -96,7 +95,7 @@ public class OperatorTools { public static final String CHILD_KEY = "$child"; - private static final Map> functionMap = new HashMap<>(); + private static final Map, ValueParser>> functionMap = new HashMap<>(); static { functionMap.put("concat", ConcatFunction::new); @@ -133,6 +132,7 @@ public class OperatorTools { func -> new TimestampExtractFunction(TimestampExtractFunction.TimestampExtractFunctionType.SECOND, func)); functionMap.put("round", RoundFunction::new); + functionMap.put("factorial", FactorialFunction::new); functionMap.put("from_unixtime", FromUnixTimeFunction::new); functionMap.put("unix_timestamp", UnixTimestampFunction::new); functionMap.put("to_timestamp", ToTimestampFunction::new); @@ -191,8 +191,8 @@ public static ValueParser buildParser(Expression expr) { } else { // TODO Function func = (Function) expr; - java.util.function.Function valueParserConstructor = - functionMap.get(func.getName().toLowerCase()); + java.util.function.Function valueParserConstructor = functionMap + .get(func.getName().toLowerCase()); if (valueParserConstructor != null) { return valueParserConstructor.apply(func); } else { @@ -205,6 +205,7 @@ public static ValueParser buildParser(Expression expr) { /** * parseBigDecimal + * * @param value * @return */ @@ -238,6 +239,7 @@ public static Timestamp parseTimestamp(Object value) { /** * compareValue + * * @param left * @param right * @return diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 2ceacfb61f7..a081bc84ed6 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -208,6 +208,7 @@ public void testFactorialFunction() throws Exception { } } + @Test public void testLog2Function() throws Exception { String transformSql = "select log2(numeric1) from source"; From 1579c5cf40120e6f743c1aaf6fdf46bd10939969 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Sun, 25 Aug 2024 16:53:06 +0800 Subject: [PATCH 06/23] [INLONG-10873][SDK] Transform support factorial function --- .../sdk/transform/process/operator/OperatorTools.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java index 2522e2fb44e..a5e754202c8 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java @@ -95,7 +95,7 @@ public class OperatorTools { public static final String CHILD_KEY = "$child"; - private static final Map, ValueParser>> functionMap = new HashMap<>(); + private static final Map> functionMap = new HashMap<>(); static { functionMap.put("concat", ConcatFunction::new); @@ -191,8 +191,8 @@ public static ValueParser buildParser(Expression expr) { } else { // TODO Function func = (Function) expr; - java.util.function.Function valueParserConstructor = functionMap - .get(func.getName().toLowerCase()); + java.util.function.Function valueParserConstructor = + functionMap.get(func.getName().toLowerCase()); if (valueParserConstructor != null) { return valueParserConstructor.apply(func); } else { From ef7d6f9bc93f5627e5baa801ed86ecec2f11d071 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Sun, 25 Aug 2024 16:54:10 +0800 Subject: [PATCH 07/23] [INLONG-10873][SDK] Transform support factorial function --- .../inlong/sdk/transform/process/operator/OperatorTools.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java index a5e754202c8..2e7bf5d6744 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java @@ -192,7 +192,7 @@ public static ValueParser buildParser(Expression expr) { // TODO Function func = (Function) expr; java.util.function.Function valueParserConstructor = - functionMap.get(func.getName().toLowerCase()); + functionMap.get(func.getName().toLowerCase()); if (valueParserConstructor != null) { return valueParserConstructor.apply(func); } else { From 411b2578a9731bcabed61ea4205cb25bee35504f Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Sun, 1 Sep 2024 14:49:09 +0800 Subject: [PATCH 08/23] fix some bugs --- .../transform/process/function/FactorialFunction.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java index 3d7efa6a532..11a5a20962a 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java @@ -41,6 +41,10 @@ public class FactorialFunction implements ValueParser { * @param expr */ public FactorialFunction(Function expr) { + if (expr == null || expr.getParameters() == null || expr.getParameters().getExpressions() == null + || expr.getParameters().getExpressions().get(0) == null) { + throw new IllegalArgumentException("Invalid expression for factorial function."); + } numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); } @@ -54,6 +58,9 @@ public FactorialFunction(Function expr) { @Override public Object parse(SourceData sourceData, int rowIndex, Context context) { Object numberObj = numberParser.parse(sourceData, rowIndex, context); + if (numberObj == null) { + throw new IllegalArgumentException("Number object cannot be null."); + } BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); // Ensure the number is a non-negative integer @@ -71,6 +78,9 @@ public Object parse(SourceData sourceData, int rowIndex, Context context) { * @return */ private BigDecimal factorial(int n) { + if (n < 0) { + throw new IllegalArgumentException("Factorial is not defined for negative numbers."); + } BigDecimal result = BigDecimal.ONE; for (int i = 2; i <= n; i++) { result = result.multiply(BigDecimal.valueOf(i)); From b004ac437d1d0b03ca748406901af2ab42167a8b Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Sun, 1 Sep 2024 23:19:45 +0800 Subject: [PATCH 09/23] [INLONG-10873][SDK]Transform support factorial function --- .../inlong/sdk/transform/process/operator/OperatorTools.java | 4 ++-- .../process/TestTransformArithmeticFunctionsProcessor.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java index 2e7bf5d6744..9210b691b81 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java @@ -25,6 +25,7 @@ import org.apache.inlong.sdk.transform.process.function.DateExtractFunction.DateExtractFunctionType; import org.apache.inlong.sdk.transform.process.function.DateFormatFunction; import org.apache.inlong.sdk.transform.process.function.ExpFunction; +import org.apache.inlong.sdk.transform.process.function.FactorialFunction; import org.apache.inlong.sdk.transform.process.function.FloorFunction; import org.apache.inlong.sdk.transform.process.function.FromUnixTimeFunction; import org.apache.inlong.sdk.transform.process.function.LnFunction; @@ -43,7 +44,6 @@ import org.apache.inlong.sdk.transform.process.function.ToDateFunction; import org.apache.inlong.sdk.transform.process.function.ToTimestampFunction; import org.apache.inlong.sdk.transform.process.function.UnixTimestampFunction; -import org.apache.inlong.sdk.transform.process.function.FactorialFunction; import org.apache.inlong.sdk.transform.process.parser.AdditionParser; import org.apache.inlong.sdk.transform.process.parser.ColumnParser; import org.apache.inlong.sdk.transform.process.parser.DateParser; @@ -191,7 +191,7 @@ public static ValueParser buildParser(Expression expr) { } else { // TODO Function func = (Function) expr; - java.util.function.Function valueParserConstructor = + java.util.function.Function valueParserConstructor = functionMap.get(func.getName().toLowerCase()); if (valueParserConstructor != null) { return valueParserConstructor.apply(func); diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index a081bc84ed6..2ceacfb61f7 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -208,7 +208,6 @@ public void testFactorialFunction() throws Exception { } } - @Test public void testLog2Function() throws Exception { String transformSql = "select log2(numeric1) from source"; From ef0ca019d0cd75892d7f8c7feac27484e9f0a42c Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Mon, 2 Sep 2024 10:45:43 +0800 Subject: [PATCH 10/23] [INLONG-10873][SDK]Transform support factorial function --- ...TransformArithmeticFunctionsProcessor.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 2ceacfb61f7..9d26e8fc776 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -176,22 +176,35 @@ public void testLog10Function() throws Exception { @Test public void testFactorialFunction() throws Exception { - String transformSql = "select factorial(numeric1) from source"; + + List fields = new ArrayList<>(); + FieldInfo numeric1 = new FieldInfo(); + numeric1.setName("numeric1"); + fields.add(numeric1); + + CsvSourceInfo csvSource = new CsvSourceInfo("UTF-8", '|', '\\', fields); + + KvSinkInfo kvSink = new KvSinkInfo("UTF-8", fields); + + String transformSql = "select factorial(numeric1) as result from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), SinkEncoderFactory.createKvEncoder(kvSink)); - // case1: 5！ + + // case1: 5! List output1 = processor.transform("5|4|6|8"); Assert.assertEquals(1, output1.size()); Assert.assertEquals(output1.get(0), "result=120"); - // case2: 0！ + // case2: 0! List output2 = processor.transform("0|4|6|8"); Assert.assertEquals(1, output2.size()); Assert.assertEquals(output2.get(0), "result=1"); - // case3: 5.5！ + // case3: 5.5! try { List output3 = processor.transform("5.5|4|6|8"); Assert.fail("Expected an exception for non-integer input"); @@ -199,7 +212,7 @@ public void testFactorialFunction() throws Exception { Assert.assertEquals("Factorial is only defined for non-negative integers.", e.getMessage()); } - // case4: -5！ + // case4: -5! try { List output4 = processor.transform("-5|4|6|8"); Assert.fail("Expected an exception for negative input"); From e5ce8416d87a7ad8983eff268b6dad5a0abb370a Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Mon, 2 Sep 2024 16:01:06 +0800 Subject: [PATCH 11/23] [INLONG-10873][SDK]Transform support factorial function --- .../process/TestTransformArithmeticFunctionsProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 9d26e8fc776..a3164ceea7a 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -197,7 +197,7 @@ public void testFactorialFunction() throws Exception { // case1: 5! List output1 = processor.transform("5|4|6|8"); Assert.assertEquals(1, output1.size()); - Assert.assertEquals(output1.get(0), "result=120"); + Assert.assertEquals("result=120", output1.get(0)); // case2: 0! List output2 = processor.transform("0|4|6|8"); From a9843f68bec2c8b29a398fd7bc91f98090f88675 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Mon, 2 Sep 2024 17:37:36 +0800 Subject: [PATCH 12/23] [INLONG-10873][SDK]Transform support factorial function --- .../process/TestTransformArithmeticFunctionsProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index a3164ceea7a..2f93319fdf1 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -186,7 +186,7 @@ public void testFactorialFunction() throws Exception { KvSinkInfo kvSink = new KvSinkInfo("UTF-8", fields); - String transformSql = "select factorial(numeric1) as result from source"; + String transformSql = "select factorial(numeric1) as numeric1 from source"; TransformConfig config = new TransformConfig(transformSql); @@ -197,7 +197,7 @@ public void testFactorialFunction() throws Exception { // case1: 5! List output1 = processor.transform("5|4|6|8"); Assert.assertEquals(1, output1.size()); - Assert.assertEquals("result=120", output1.get(0)); + Assert.assertEquals(output1.get(0), "result=120"); // case2: 0! List output2 = processor.transform("0|4|6|8"); From dd039f6926e55660e955a6010e6522b79489eee1 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Mon, 2 Sep 2024 19:00:46 +0800 Subject: [PATCH 13/23] [INLONG-10873][SDK]Transform support factorial function --- .../process/TestTransformArithmeticFunctionsProcessor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 2f93319fdf1..841b3e30be3 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -183,7 +183,6 @@ public void testFactorialFunction() throws Exception { fields.add(numeric1); CsvSourceInfo csvSource = new CsvSourceInfo("UTF-8", '|', '\\', fields); - KvSinkInfo kvSink = new KvSinkInfo("UTF-8", fields); String transformSql = "select factorial(numeric1) as numeric1 from source"; @@ -197,12 +196,12 @@ public void testFactorialFunction() throws Exception { // case1: 5! List output1 = processor.transform("5|4|6|8"); Assert.assertEquals(1, output1.size()); - Assert.assertEquals(output1.get(0), "result=120"); + Assert.assertEquals(output1.get(0), "numeric1=120"); // case2: 0! List output2 = processor.transform("0|4|6|8"); Assert.assertEquals(1, output2.size()); - Assert.assertEquals(output2.get(0), "result=1"); + Assert.assertEquals(output2.get(0), "numeric1=1"); // case3: 5.5! try { From 8c1b6eca0e10c091c9b37a5a7c925cf8fdd1b75a Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Mon, 2 Sep 2024 20:50:07 +0800 Subject: [PATCH 14/23] [INLONG-10873][SDK]Transform support factorial function --- .../sdk/transform/process/function/FactorialFunction.java | 3 +++ .../org/apache/inlong/sdk/transform/process/history.java | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/history.java diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java index 11a5a20962a..46d6317a66c 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java @@ -63,6 +63,9 @@ public Object parse(SourceData sourceData, int rowIndex, Context context) { } BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); + System.out.println("Parsed number: " + numberValue); + System.out.println("Scale: " + numberValue.scale()); + // Ensure the number is a non-negative integer if (numberValue.scale() > 0 || numberValue.compareTo(BigDecimal.ZERO) < 0) { throw new IllegalArgumentException("Factorial is only defined for non-negative integers."); diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/history.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/history.java new file mode 100644 index 00000000000..f89cc8e6c18 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/history.java @@ -0,0 +1,6 @@ +public static void main(String[] args) { + BigDecimal number = new BigDecimal("5.5"); + if (number.scale() > 0) { + throw new IllegalArgumentException("Factorial is only defined for non-negative integers."); + } +} From b6dec70dcffdb7895788a36ea7810370eff37f75 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Mon, 2 Sep 2024 20:50:52 +0800 Subject: [PATCH 15/23] [INLONG-10873][SDK]Transform support factorial function --- .../org/apache/inlong/sdk/transform/process/history.java | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/history.java diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/history.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/history.java deleted file mode 100644 index f89cc8e6c18..00000000000 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/history.java +++ /dev/null @@ -1,6 +0,0 @@ -public static void main(String[] args) { - BigDecimal number = new BigDecimal("5.5"); - if (number.scale() > 0) { - throw new IllegalArgumentException("Factorial is only defined for non-negative integers."); - } -} From 31f4d8afe5673752d63fa0cf2bf10f4384a08d95 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Mon, 2 Sep 2024 23:09:26 +0800 Subject: [PATCH 16/23] [INLONG-10873][SDK]Transform support factorial function --- .../process/function/FactorialFunction.java | 1 + .../TestTransformArithmeticFunctionsProcessor.java | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java index 46d6317a66c..45d87f0ccbb 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java @@ -63,6 +63,7 @@ public Object parse(SourceData sourceData, int rowIndex, Context context) { } BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); + // 调试输出 System.out.println("Parsed number: " + numberValue); System.out.println("Scale: " + numberValue.scale()); diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 841b3e30be3..82ee342a9fc 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -196,24 +196,24 @@ public void testFactorialFunction() throws Exception { // case1: 5! List output1 = processor.transform("5|4|6|8"); Assert.assertEquals(1, output1.size()); - Assert.assertEquals(output1.get(0), "numeric1=120"); + Assert.assertEquals("numeric1=120", output1.get(0)); // case2: 0! List output2 = processor.transform("0|4|6|8"); Assert.assertEquals(1, output2.size()); - Assert.assertEquals(output2.get(0), "numeric1=1"); + Assert.assertEquals("numeric1=1", output2.get(0)); - // case3: 5.5! + // case3: 5.5! - Non-integer input try { - List output3 = processor.transform("5.5|4|6|8"); + processor.transform("5.5|4|6|8"); Assert.fail("Expected an exception for non-integer input"); } catch (IllegalArgumentException e) { Assert.assertEquals("Factorial is only defined for non-negative integers.", e.getMessage()); } - // case4: -5! + // case4: -5! - Negative input try { - List output4 = processor.transform("-5|4|6|8"); + processor.transform("-5|4|6|8"); Assert.fail("Expected an exception for negative input"); } catch (IllegalArgumentException e) { Assert.assertEquals("Factorial is only defined for non-negative integers.", e.getMessage()); From a7dcba66b66718a89f7c7e70b98c9e31e4cebf2d Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Tue, 3 Sep 2024 11:24:09 +0800 Subject: [PATCH 17/23] [INLONG-10873][SDK]Transform support factorial function --- .../process/function/FactorialFunction.java | 51 ++++++++++++++----- ...TransformArithmeticFunctionsProcessor.java | 18 +++---- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java index 45d87f0ccbb..65355026cb4 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java @@ -22,6 +22,7 @@ import org.apache.inlong.sdk.transform.process.operator.OperatorTools; import org.apache.inlong.sdk.transform.process.parser.ValueParser; +import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Function; import java.math.BigDecimal; @@ -31,45 +32,66 @@ * description: factorial(numeric)--returns the factorial of a non-negative * integer */ +@Slf4j public class FactorialFunction implements ValueParser { - private ValueParser numberParser; + private final ValueParser numberParser; /** * Constructor - * + * * @param expr */ public FactorialFunction(Function expr) { if (expr == null || expr.getParameters() == null || expr.getParameters().getExpressions() == null + || expr.getParameters().getExpressions().isEmpty() || expr.getParameters().getExpressions().get(0) == null) { - throw new IllegalArgumentException("Invalid expression for factorial function."); + log.error("Invalid expression for factorial function."); + numberParser = null; + } else { + numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); } - numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); } /** * parse - * + * * @param sourceData * @param rowIndex * @return */ @Override public Object parse(SourceData sourceData, int rowIndex, Context context) { - Object numberObj = numberParser.parse(sourceData, rowIndex, context); + if (numberParser == null) { + log.error("Number parser is not initialized."); + return null; + } + + Object numberObj; + try { + numberObj = numberParser.parse(sourceData, rowIndex, context); + } catch (Exception e) { + log.error("Error parsing number object", e); + return null; + } + if (numberObj == null) { - throw new IllegalArgumentException("Number object cannot be null."); + log.warn("Parsed number object is null."); + return null; } - BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); - // 调试输出 - System.out.println("Parsed number: " + numberValue); - System.out.println("Scale: " + numberValue.scale()); + BigDecimal numberValue; + try { + numberValue = OperatorTools.parseBigDecimal(numberObj); + } catch (Exception e) { + log.error("Error converting parsed object to BigDecimal", e); + return null; + } // Ensure the number is a non-negative integer if (numberValue.scale() > 0 || numberValue.compareTo(BigDecimal.ZERO) < 0) { - throw new IllegalArgumentException("Factorial is only defined for non-negative integers."); + log.warn("Factorial is only defined for non-negative integers. Invalid input: {}", numberValue); + return null; } return factorial(numberValue.intValue()); @@ -77,13 +99,14 @@ public Object parse(SourceData sourceData, int rowIndex, Context context) { /** * Helper method to calculate factorial - * + * * @param n * @return */ private BigDecimal factorial(int n) { if (n < 0) { - throw new IllegalArgumentException("Factorial is not defined for negative numbers."); + log.error("Factorial is not defined for negative numbers."); + return null; } BigDecimal result = BigDecimal.ONE; for (int i = 2; i <= n; i++) { diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 82ee342a9fc..542eef709ed 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -204,20 +204,14 @@ public void testFactorialFunction() throws Exception { Assert.assertEquals("numeric1=1", output2.get(0)); // case3: 5.5! - Non-integer input - try { - processor.transform("5.5|4|6|8"); - Assert.fail("Expected an exception for non-integer input"); - } catch (IllegalArgumentException e) { - Assert.assertEquals("Factorial is only defined for non-negative integers.", e.getMessage()); - } + List output3 = processor.transform("5.5|4|6|8"); + Assert.assertEquals(1, output3.size()); + Assert.assertNull(output3.get(0)); // case4: -5! - Negative input - try { - processor.transform("-5|4|6|8"); - Assert.fail("Expected an exception for negative input"); - } catch (IllegalArgumentException e) { - Assert.assertEquals("Factorial is only defined for non-negative integers.", e.getMessage()); - } + List output4 = processor.transform("-5|4|6|8"); + Assert.assertEquals(1, output4.size()); + Assert.assertNull(output4.get(0)); } @Test From e1ac5e74968dd93600e466bfd18380d90f465532 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Tue, 3 Sep 2024 12:34:29 +0800 Subject: [PATCH 18/23] [INLONG-10873][SDK]Transform support factorial function --- .../TestTransformArithmeticFunctionsProcessor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 542eef709ed..058c12f1078 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -193,22 +193,22 @@ public void testFactorialFunction() throws Exception { .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), SinkEncoderFactory.createKvEncoder(kvSink)); - // case1: 5! + // case1: Valid input - 5! List output1 = processor.transform("5|4|6|8"); Assert.assertEquals(1, output1.size()); Assert.assertEquals("numeric1=120", output1.get(0)); - // case2: 0! + // case2: Valid input - 0! List output2 = processor.transform("0|4|6|8"); Assert.assertEquals(1, output2.size()); Assert.assertEquals("numeric1=1", output2.get(0)); - // case3: 5.5! - Non-integer input + // case3: Non-integer input (5.5) should return null List output3 = processor.transform("5.5|4|6|8"); Assert.assertEquals(1, output3.size()); Assert.assertNull(output3.get(0)); - // case4: -5! - Negative input + // case4: Negative input (-5) should return null List output4 = processor.transform("-5|4|6|8"); Assert.assertEquals(1, output4.size()); Assert.assertNull(output4.get(0)); From 4e00c686ce9bf0d3cd00c2c69ef32707973959e8 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Tue, 3 Sep 2024 13:47:58 +0800 Subject: [PATCH 19/23] [INLONG-11002][SDK]Transform SQL support Fibonacci function --- .../process/function/FactorialFunction.java | 117 ------------------ .../process/function/FibonacciFunction.java | 79 ++++++++++++ .../process/operator/OperatorTools.java | 8 +- ...TransformArithmeticFunctionsProcessor.java | 39 +++--- 4 files changed, 97 insertions(+), 146 deletions(-) delete mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FibonacciFunction.java diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java deleted file mode 100644 index 65355026cb4..00000000000 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FactorialFunction.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.inlong.sdk.transform.process.function; - -import org.apache.inlong.sdk.transform.decode.SourceData; -import org.apache.inlong.sdk.transform.process.Context; -import org.apache.inlong.sdk.transform.process.operator.OperatorTools; -import org.apache.inlong.sdk.transform.process.parser.ValueParser; - -import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.expression.Function; - -import java.math.BigDecimal; - -/** - * FactorialFunction - * description: factorial(numeric)--returns the factorial of a non-negative - * integer - */ -@Slf4j -public class FactorialFunction implements ValueParser { - - private final ValueParser numberParser; - - /** - * Constructor - * - * @param expr - */ - public FactorialFunction(Function expr) { - if (expr == null || expr.getParameters() == null || expr.getParameters().getExpressions() == null - || expr.getParameters().getExpressions().isEmpty() - || expr.getParameters().getExpressions().get(0) == null) { - log.error("Invalid expression for factorial function."); - numberParser = null; - } else { - numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); - } - } - - /** - * parse - * - * @param sourceData - * @param rowIndex - * @return - */ - @Override - public Object parse(SourceData sourceData, int rowIndex, Context context) { - if (numberParser == null) { - log.error("Number parser is not initialized."); - return null; - } - - Object numberObj; - try { - numberObj = numberParser.parse(sourceData, rowIndex, context); - } catch (Exception e) { - log.error("Error parsing number object", e); - return null; - } - - if (numberObj == null) { - log.warn("Parsed number object is null."); - return null; - } - - BigDecimal numberValue; - try { - numberValue = OperatorTools.parseBigDecimal(numberObj); - } catch (Exception e) { - log.error("Error converting parsed object to BigDecimal", e); - return null; - } - - // Ensure the number is a non-negative integer - if (numberValue.scale() > 0 || numberValue.compareTo(BigDecimal.ZERO) < 0) { - log.warn("Factorial is only defined for non-negative integers. Invalid input: {}", numberValue); - return null; - } - - return factorial(numberValue.intValue()); - } - - /** - * Helper method to calculate factorial - * - * @param n - * @return - */ - private BigDecimal factorial(int n) { - if (n < 0) { - log.error("Factorial is not defined for negative numbers."); - return null; - } - BigDecimal result = BigDecimal.ONE; - for (int i = 2; i <= n; i++) { - result = result.multiply(BigDecimal.valueOf(i)); - } - return result; - } -} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FibonacciFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FibonacciFunction.java new file mode 100644 index 00000000000..59f4ce81096 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FibonacciFunction.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; + +import java.math.BigDecimal; + +/** + * FibonacciFunction + * description: fibonacci(numeric)--returns the nth Fibonacci number + */ + +public class FibonacciFunction implements ValueParser { + + private final ValueParser numberParser; + + /** + * Constructor + * + * @param expr + */ + public FibonacciFunction(Function expr) { + numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + /** + * parse + * + * @param sourceData + * @param rowIndex + * @return + */ + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object numberObj = numberParser.parse(sourceData, rowIndex, context); + BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); + int n = numberValue.intValue(); + return fibonacci(n); + } + + /** + * Calculate the nth Fibonacci number. + * + * @param n the position in the Fibonacci sequence + * @return the nth Fibonacci number + */ + private long fibonacci(int n) { + if (n <= 1) + return n; + long prev = 0, curr = 1; + for (int i = 2; i <= n; i++) { + long temp = curr; + curr = curr + prev; + prev = temp; + } + return curr; + } +} \ No newline at end of file diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java index 9210b691b81..97f8d98704f 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java @@ -25,7 +25,7 @@ import org.apache.inlong.sdk.transform.process.function.DateExtractFunction.DateExtractFunctionType; import org.apache.inlong.sdk.transform.process.function.DateFormatFunction; import org.apache.inlong.sdk.transform.process.function.ExpFunction; -import org.apache.inlong.sdk.transform.process.function.FactorialFunction; +import org.apache.inlong.sdk.transform.process.function.FibonacciFunction; import org.apache.inlong.sdk.transform.process.function.FloorFunction; import org.apache.inlong.sdk.transform.process.function.FromUnixTimeFunction; import org.apache.inlong.sdk.transform.process.function.LnFunction; @@ -132,7 +132,7 @@ public class OperatorTools { func -> new TimestampExtractFunction(TimestampExtractFunction.TimestampExtractFunctionType.SECOND, func)); functionMap.put("round", RoundFunction::new); - functionMap.put("factorial", FactorialFunction::new); + functionMap.put("factorial", FibonacciFunction::new); functionMap.put("from_unixtime", FromUnixTimeFunction::new); functionMap.put("unix_timestamp", UnixTimestampFunction::new); functionMap.put("to_timestamp", ToTimestampFunction::new); @@ -191,8 +191,8 @@ public static ValueParser buildParser(Expression expr) { } else { // TODO Function func = (Function) expr; - java.util.function.Function valueParserConstructor = - functionMap.get(func.getName().toLowerCase()); + java.util.function.Function valueParserConstructor = functionMap + .get(func.getName().toLowerCase()); if (valueParserConstructor != null) { return valueParserConstructor.apply(func); } else { diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 058c12f1078..a75d984156b 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -175,43 +175,32 @@ public void testLog10Function() throws Exception { } @Test - public void testFactorialFunction() throws Exception { - - List fields = new ArrayList<>(); - FieldInfo numeric1 = new FieldInfo(); - numeric1.setName("numeric1"); - fields.add(numeric1); - - CsvSourceInfo csvSource = new CsvSourceInfo("UTF-8", '|', '\\', fields); - KvSinkInfo kvSink = new KvSinkInfo("UTF-8", fields); - - String transformSql = "select factorial(numeric1) as numeric1 from source"; - + public void testFibonacciFunction() throws Exception { + String transformSql = "select fibonacci(numeric1) from source"; TransformConfig config = new TransformConfig(transformSql); - TransformProcessor processor = TransformProcessor .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), SinkEncoderFactory.createKvEncoder(kvSink)); - // case1: Valid input - 5! - List output1 = processor.transform("5|4|6|8"); + // case1: fibonacci(0) + List output1 = processor.transform("0|4|6|8", new HashMap<>()); Assert.assertEquals(1, output1.size()); - Assert.assertEquals("numeric1=120", output1.get(0)); + Assert.assertEquals(output1.get(0), "result=0"); - // case2: Valid input - 0! - List output2 = processor.transform("0|4|6|8"); + // case2: fibonacci(1) + List output2 = processor.transform("1|4|6|8", new HashMap<>()); Assert.assertEquals(1, output2.size()); - Assert.assertEquals("numeric1=1", output2.get(0)); + Assert.assertEquals(output2.get(0), "result=1"); - // case3: Non-integer input (5.5) should return null - List output3 = processor.transform("5.5|4|6|8"); + // case3: fibonacci(7) + List output3 = processor.transform("7|4|6|8", new HashMap<>()); Assert.assertEquals(1, output3.size()); - Assert.assertNull(output3.get(0)); + Assert.assertEquals(output3.get(0), "result=13"); - // case4: Negative input (-5) should return null - List output4 = processor.transform("-5|4|6|8"); + // case4: fibonacci(10) + List output4 = processor.transform("10|4|6|8", new HashMap<>()); Assert.assertEquals(1, output4.size()); - Assert.assertNull(output4.get(0)); + Assert.assertEquals(output4.get(0), "result=55"); } @Test From e46810b41593c56e692b00ac516666fa1b650203 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Wed, 4 Sep 2024 10:49:24 +0800 Subject: [PATCH 20/23] [INLONG-11002][SDK]Transform SQL support Fibonacci function --- .../inlong/sort/formats/binlog/InLongBinlog.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/binlog/InLongBinlog.java b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/binlog/InLongBinlog.java index 487fc6483e4..c8efd07dd7e 100644 --- a/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/binlog/InLongBinlog.java +++ b/inlong-sort/sort-formats/format-common/src/main/java/org/apache/inlong/sort/formats/binlog/InLongBinlog.java @@ -1,19 +1,3 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: InLongBinlog.proto From 5103fa642798d21794c01ad228ba56da093b96d9 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Wed, 4 Sep 2024 11:01:13 +0800 Subject: [PATCH 21/23] [INLONG-11002][SDK]Transform SQL support Fibonacci function --- .../sdk/transform/process/operator/OperatorTools.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java index 14049feec76..75b61783602 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java @@ -25,6 +25,7 @@ import org.apache.inlong.sdk.transform.process.function.DateExtractFunction.DateExtractFunctionType; import org.apache.inlong.sdk.transform.process.function.DateFormatFunction; import org.apache.inlong.sdk.transform.process.function.ExpFunction; +import org.apache.inlong.sdk.transform.process.function.FibonacciFunction; import org.apache.inlong.sdk.transform.process.function.FloorFunction; import org.apache.inlong.sdk.transform.process.function.FromUnixTimeFunction; import org.apache.inlong.sdk.transform.process.function.LnFunction; @@ -43,9 +44,7 @@ import org.apache.inlong.sdk.transform.process.function.ToDateFunction; import org.apache.inlong.sdk.transform.process.function.ToTimestampFunction; import org.apache.inlong.sdk.transform.process.function.UnixTimestampFunction; -import org.apache.inlong.sdk.transform.process.parser.AdditionParser; import org.apache.inlong.sdk.transform.process.parser.ColumnParser; -import org.apache.inlong.sdk.transform.process.parser.ParserTools; import org.apache.inlong.sdk.transform.process.parser.ValueParser; import net.sf.jsqlparser.expression.Expression; @@ -65,6 +64,8 @@ import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; +import java.util.HashMap; +import java.util.Map; /** * OperatorTools @@ -116,6 +117,8 @@ public class OperatorTools { functionMap.put("from_unixtime", FromUnixTimeFunction::new); functionMap.put("unix_timestamp", UnixTimestampFunction::new); functionMap.put("to_timestamp", ToTimestampFunction::new); + functionMap.put("fibonacci", FibonacciFunction::new); + } public static ExpressionOperator buildOperator(Expression expr) { @@ -151,8 +154,8 @@ public static ValueParser buildParser(Expression expr) { } else { // TODO Function func = (Function) expr; - java.util.function.Function valueParserConstructor = - functionMap.get(func.getName().toLowerCase()); + java.util.function.Function valueParserConstructor = functionMap + .get(func.getName().toLowerCase()); if (valueParserConstructor != null) { return valueParserConstructor.apply(func); } else { From 467d901fb76fcaf51d582c77ff5f6a91116a27e8 Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Thu, 5 Sep 2024 17:36:48 +0800 Subject: [PATCH 22/23] [INLONG-11002][SDK]Transform SQL support Fibonacci function --- .github/workflows/ci_ut.yml | 2 +- .github/workflows/ci_ut_flink18.yml | 96 ++++ .../GroupDetail/DataStream/PreviewModal.tsx | 6 +- .../pages/ModuleAudit/AuditModule/config.tsx | 7 +- .../service/datatype/KvDataTypeOperator.java | 26 +- .../service/group/InlongGroupServiceImpl.java | 4 +- .../resource/queue/tubemq/TubeMQOperator.java | 3 +- .../sort/DefaultSortConfigOperator.java | 1 + .../sdk/transform/decode/CsvSourceData.java | 10 +- .../transform/decode/CsvSourceDecoder.java | 8 +- .../sdk/transform/decode/JsonSourceData.java | 24 +- .../transform/decode/JsonSourceDecoder.java | 55 +- .../sdk/transform/decode/SourceData.java | 2 +- .../sdk/transform/encode/CsvSinkEncoder.java | 6 +- .../sdk/transform/encode/KvSinkEncoder.java | 6 +- .../sdk/transform/encode/SinkEncoder.java | 2 + .../inlong/sdk/transform/pojo/FieldInfo.java | 2 +- .../transform/process/TransformProcessor.java | 34 +- .../transform/process/ValueParserNode.java | 34 ++ .../process/converter/DoubleConverter.java | 26 + .../process/converter/LongConverter.java | 26 + .../process/function/AsinFunction.java | 51 ++ .../process/function/Atan2Function.java | 58 +++ .../process/function/AtanFunction.java | 48 ++ .../process/function/ContainsFunction.java | 56 ++ .../process/function/CoshFunction.java | 48 ++ .../process/function/CotFunction.java | 55 ++ .../process/function/DateDiffFunction.java | 89 ++++ .../process/function/DateExtractFunction.java | 20 +- .../process/function/FromBase64Function.java | 66 +++ .../function/FromUnixTimeFunction.java | 2 +- .../process/function/HexFunction.java | 2 +- .../process/function/IfFunction.java | 55 ++ .../process/function/IfNullFunction.java | 72 +++ .../process/function/InsertFunction.java | 101 ++++ .../process/function/LpadFunction.java | 81 +++ .../process/function/PiFunction.java | 40 ++ .../process/function/RadiansFunction.java | 49 ++ .../process/function/RpadFunction.java | 81 +++ .../process/function/Sha2Function.java | 75 +++ .../process/function/ShaFunction.java | 54 ++ .../process/function/SubstringFunction.java | 4 +- .../process/function/TanhFunction.java | 48 ++ .../function/TimestampAddFunction.java | 2 +- .../process/operator/AndOperator.java | 1 + .../process/operator/EqualsToOperator.java | 1 + .../operator/GreaterThanEqualsOperator.java | 1 + .../process/operator/GreaterThanOperator.java | 1 + .../operator/MinorThanEqualsOperator.java | 1 + .../process/operator/MinorThanOperator.java | 1 + .../process/operator/NotEqualsToOperator.java | 1 + .../process/operator/NotOperator.java | 1 + .../process/operator/OperatorTools.java | 64 +-- .../process/operator/OrOperator.java | 1 + .../process/operator/ParenthesisOperator.java | 1 + .../process/operator/TransformOperator.java | 31 ++ .../process/parser/BitwiseAndParser.java | 61 +++ .../parser/BitwiseLeftShiftParser.java | 67 +++ .../process/parser/BitwiseOrParser.java | 62 +++ .../parser/BitwiseRightShiftParser.java | 66 +++ .../process/parser/BitwiseXorParser.java | 62 +++ .../process/parser/DoubleParser.java | 1 + .../transform/process/parser/SignParser.java | 26 +- ...TransformArithmeticFunctionsProcessor.java | 489 ++++++++++++++++++ ...TransformExpressionOperatorsProcessor.java | 354 +++++++++++++ ...tTransformFromBase64FunctionProcessor.java | 90 ++++ .../process/TestTransformProcessor.java | 143 +++-- ...TestTransformStringFunctionsProcessor.java | 206 ++++++++ ...stTransformTemporalFunctionsProcessor.java | 58 +++ inlong-sort/sort-core/pom.xml | 54 +- inlong-sort/sort-dist/pom.xml | 75 ++- inlong-sort/sort-end-to-end-tests/pom.xml | 9 + .../sort-end-to-end-tests-v1.18/pom.xml | 209 ++++++++ .../tests/utils/FlinkContainerTestEnv.java | 241 +++++++++ .../utils/FlinkContainerTestEnvJRE11.java | 55 ++ .../utils/FlinkContainerTestEnvJRE8.java | 55 ++ .../sort/tests/utils/PlaceholderResolver.java | 150 ++++++ .../inlong/sort/tests/utils/TestUtils.java | 124 +++++ .../src/test/resources/log4j2-test.properties | 47 ++ inlong-sort/sort-formats/pom.xml | 3 +- 80 files changed, 4123 insertions(+), 226 deletions(-) create mode 100644 .github/workflows/ci_ut_flink18.yml create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/ValueParserNode.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/converter/DoubleConverter.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/converter/LongConverter.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AsinFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Atan2Function.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AtanFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ContainsFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/CoshFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/CotFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateDiffFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromBase64Function.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/IfFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/IfNullFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/InsertFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LpadFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/PiFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RadiansFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RpadFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TanhFunction.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/TransformOperator.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseAndParser.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseLeftShiftParser.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseOrParser.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseRightShiftParser.java create mode 100644 inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseXorParser.java create mode 100644 inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformExpressionOperatorsProcessor.java create mode 100644 inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformFromBase64FunctionProcessor.java create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/pom.xml create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnv.java create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE11.java create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE8.java create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/PlaceholderResolver.java create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/TestUtils.java create mode 100644 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/resources/log4j2-test.properties diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 5c433872bf8..0d67d46320b 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -101,7 +101,7 @@ jobs: CI: false - name: Unit test with Maven - run: mvn --batch-mode --update-snapshots -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13 + run: mvn --batch-mode --update-snapshots -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13,!:sort-end-to-end-tests-v1.18 env: CI: false diff --git a/.github/workflows/ci_ut_flink18.yml b/.github/workflows/ci_ut_flink18.yml new file mode 100644 index 00000000000..5c2f2709f27 --- /dev/null +++ b/.github/workflows/ci_ut_flink18.yml @@ -0,0 +1,96 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: + InLong Unit Test For Flink 1.18 + +on: + push: + paths: + - '.github/workflows/ci_ut_flink18.yml' + - 'inlong-sort/**' + - '!**.md' + + pull_request: + paths: + - '.github/workflows/ci_ut_flink18.yml' + - 'inlong-sort/**' + - '!**.md' + +jobs: + unit-test: + name: Unit Test + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + # Release space size + - name: Remove unnecessary packages + run: | + echo "=== Before pruning ===" + df -h + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache + echo "=== After pruning ===" + df -h + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: adopt + + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/inlong + key: ${{ runner.os }}-inlong-flink18-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-inlong-flink18 + + - name: Build for Flink 1.18 with Maven + run: mvn --update-snapshots -e -V clean install -U -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + env: + CI: false + + - name: Unit test for Flink 1.18 with Maven + run: mvn --update-snapshots -e -V verify -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 + env: + CI: false + + - name: Upload unit test results + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: surefire-reports + path: ./**/target/surefire-reports/ + if-no-files-found: ignore + + - name: Upload integration test results + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: failsafe-reports + path: ./**/target/failsafe-reports/ + if-no-files-found: ignore + + - name: Clean up build packages + run: mvn clean \ No newline at end of file diff --git a/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx b/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx index fa9edff445c..6593c59e6c3 100644 --- a/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx +++ b/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx @@ -69,9 +69,9 @@ const Comp: React.FC = ({ inlongGroupId, inlongStreamId, ...modalProps }) exitsId = true; } const width = - (cur['fieldName'].length > cur['fieldValue'].length - ? cur['fieldName'].length - : cur['fieldValue'].length) * 10; + (cur['fieldName']?.length > cur['fieldValue']?.length + ? cur['fieldName']?.length + : cur['fieldValue']?.length) * 10; acc.push({ title: cur['fieldName'], key: cur['fieldName'], diff --git a/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/config.tsx b/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/config.tsx index 81d05bd15d0..2ef04c5bd57 100644 --- a/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/config.tsx +++ b/inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/config.tsx @@ -350,13 +350,12 @@ export const getTableColumns = (source, dim) => { title: item.auditName, dataIndex: item.auditId, render: text => { - let color = 'black'; if (text?.includes('+')) { - color = 'red'; + return {text}; } else if (text?.includes('-')) { - color = 'green'; + return {text}; } - return {text}; + return {text}; }, })); return [ diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/datatype/KvDataTypeOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/datatype/KvDataTypeOperator.java index 1eb73888cd3..8b0a40af379 100644 --- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/datatype/KvDataTypeOperator.java +++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/datatype/KvDataTypeOperator.java @@ -23,12 +23,14 @@ import org.apache.inlong.manager.common.util.CommonBeanUtils; import org.apache.inlong.manager.pojo.consume.BriefMQMessage.FieldInfo; import org.apache.inlong.manager.pojo.stream.InlongStreamInfo; +import org.apache.inlong.sdk.transform.decode.KvUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import java.util.List; +import java.util.Map; @Slf4j @Service @@ -51,18 +53,20 @@ public List parseFields(String str, InlongStreamInfo streamInfo) thro if (StringUtils.isNotBlank(streamInfo.getKvSeparator())) { kvSeparator = (char) Integer.parseInt(streamInfo.getKvSeparator()); } - String[] bodys = StringUtils.split(str, separator); - if (bodys.length != fields.size()) { - log.warn( - "The number of reported fields does not match the number of stream fields for groupId={}, streamId={}, reported field size ={}, stream field size ={}", - streamInfo.getInlongGroupId(), streamInfo.getInlongStreamId(), bodys.length, fields.size()); - return fields; + Character escapeChar = null; + if (StringUtils.isNotBlank(streamInfo.getDataEscapeChar())) { + escapeChar = streamInfo.getDataEscapeChar().charAt(0); } - for (int i = 0; i < bodys.length; i++) { - String body = bodys[i]; - String[] values = StringUtils.split(body, kvSeparator); - fields.get(i).setFieldName(values[0]); - fields.get(i).setFieldValue(values[1]); + Character lineSeparator = null; + if (StringUtils.isNotBlank(streamInfo.getLineSeparator())) { + lineSeparator = (char) Integer.parseInt(streamInfo.getLineSeparator()); + } + List> rowValues = + KvUtils.splitKv(str, separator, kvSeparator, escapeChar, '\"', lineSeparator); + for (Map row : rowValues) { + for (FieldInfo fieldInfo : fields) { + fieldInfo.setFieldValue(row.get(fieldInfo.getFieldName())); + } } } catch (Exception e) { log.warn("parse fields failed for groupId = {}, streamId = {}", streamInfo.getInlongGroupId(), diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java index 54407681917..3ee94a23ce0 100644 --- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java +++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java @@ -205,7 +205,7 @@ public String save(InlongGroupRequest request, String operator) { Preconditions.expectNotNull(request, "inlong group request cannot be empty"); String groupId = request.getInlongGroupId(); - InlongGroupEntity entity = groupMapper.selectByGroupId(groupId); + InlongGroupEntity entity = groupMapper.selectByGroupIdWithoutTenant(groupId); if (entity != null) { LOGGER.error("groupId={} has already exists", groupId); throw new BusinessException(ErrorCodeEnum.GROUP_DUPLICATE); @@ -278,7 +278,7 @@ public List batchSave(List groupRequestList, St @Override public Boolean exist(String groupId) { Preconditions.expectNotNull(groupId, ErrorCodeEnum.GROUP_ID_IS_EMPTY.getMessage()); - InlongGroupEntity entity = groupMapper.selectByGroupId(groupId); + InlongGroupEntity entity = groupMapper.selectByGroupIdWithoutTenant(groupId); LOGGER.debug("success to check inlong group {}, exist? {}", groupId, entity != null); return entity != null; } diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java index 050de078050..66d1fcf59a5 100644 --- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java +++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java @@ -71,6 +71,7 @@ public class TubeMQOperator { private static final String CREATE_USER = "&createUser="; private static final String CONF_MOD_AUTH_TOKEN = "&confModAuthToken="; private static final String MSG_COUNT = "&msgCount="; + private static final String FILTER_CONDS = "&filterConds="; private static final String QUERY_TOPIC_PATH = "/webapi.htm?method=admin_query_cluster_topic_view"; private static final String QUERY_BROKER_PATH = "/webapi.htm?method=admin_query_broker_run_status"; @@ -288,7 +289,7 @@ public List queryLastMessage(TubeClusterInfo tubeCluster, String } String url = "http://" + brokerUrl + QUERY_MESSAGE_PATH + TOPIC_NAME + topicName + MSG_COUNT - + request.getMessageCount(); + + request.getMessageCount() + FILTER_CONDS + streamInfo.getInlongStreamId(); TubeMessageResponse response = HttpUtils.request(restTemplate, url, HttpMethod.GET, null, new HttpHeaders(), TubeMessageResponse.class); if (response.getErrCode() != SUCCESS_CODE && response.getErrCode() != 200) { diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java index 116a8f0ea21..17ca4b9bad2 100644 --- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java +++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java @@ -175,6 +175,7 @@ private DataFlowConfig getDataFlowConfig(InlongGroupInfo groupInfo, InlongStream .dataflowId(String.valueOf(sink.getId())) .sourceConfig(getSourceConfig(groupInfo, streamInfo, sink)) .auditTag(String.valueOf(sink.getId())) + .transformSql(sink.getTransformSql()) .sinkConfig(getSinkConfig(groupInfo, streamInfo, sink)) .inlongGroupId(groupInfo.getInlongGroupId()) .inlongStreamId(streamInfo.getInlongStreamId()) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceData.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceData.java index d4492b4b853..e0bd9f794ce 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceData.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceData.java @@ -28,14 +28,14 @@ */ public class CsvSourceData implements SourceData { - private List> rows = new ArrayList<>(); + private List> rows = new ArrayList<>(); - private Map currentRow; + private Map currentRow; public CsvSourceData() { } - public void putField(String fieldName, String fieldValue) { + public void putField(String fieldName, Object fieldValue) { this.currentRow.put(fieldName, fieldValue); } @@ -50,11 +50,11 @@ public int getRowCount() { } @Override - public String getField(int rowNum, String fieldName) { + public Object getField(int rowNum, String fieldName) { if (rowNum >= this.rows.size()) { return null; } - Map targetRow = this.rows.get(rowNum); + Map targetRow = this.rows.get(rowNum); return targetRow.get(fieldName); } } diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java index fb95dadc430..7b3dedb6373 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java @@ -76,9 +76,13 @@ public SourceData decode(String srcString, Context context) { int fieldIndex = 0; for (FieldInfo field : fields) { String fieldName = field.getName(); - String fieldValue = null; + Object fieldValue = null; if (fieldIndex < fieldValues.length) { - fieldValue = fieldValues[fieldIndex]; + try { + fieldValue = field.getConverter().convert(fieldValues[fieldIndex]); + } catch (Exception e) { + throw new RuntimeException(e); + } } sourceData.putField(fieldName, fieldValue); fieldIndex++; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceData.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceData.java index 5f92c34965c..539f4b06dc5 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceData.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceData.java @@ -111,20 +111,22 @@ public String getField(int rowNum, String fieldName) { // error data return ""; } + // node is not array if (!node.isArray()) { current = newElement; - } else { - if (!newElement.isJsonArray()) { - // error data - return ""; - } - JsonArray newArray = newElement.getAsJsonArray(); - if (node.getArrayIndex() >= newArray.size()) { - // error data - return ""; - } - current = newArray.get(node.getArrayIndex()); + continue; + } + // node is an array + if (!newElement.isJsonArray()) { + // error data + return ""; + } + JsonArray newArray = newElement.getAsJsonArray(); + if (node.getArrayIndex() >= newArray.size()) { + // error data + return ""; } + current = newArray.get(node.getArrayIndex()); } return current.getAsString(); } catch (Exception e) { diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceDecoder.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceDecoder.java index 2d16d92bc1a..426ae167a28 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceDecoder.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceDecoder.java @@ -85,39 +85,40 @@ public SourceData decode(byte[] srcBytes, Context context) { public SourceData decode(String srcString, Context context) { JsonObject root = gson.fromJson(srcString, JsonObject.class); JsonArray childRoot = null; - if (CollectionUtils.isNotEmpty(childNodes)) { - JsonElement current = root; - for (JsonNode node : childNodes) { - if (!current.isJsonObject()) { + if (CollectionUtils.isEmpty(childNodes)) { + return new JsonSourceData(root, null); + } + JsonElement current = root; + for (JsonNode node : childNodes) { + if (!current.isJsonObject()) { + // error data + return new JsonSourceData(root, null); + } + JsonElement newElement = current.getAsJsonObject().get(node.getName()); + if (newElement == null) { + // error data + return new JsonSourceData(root, null); + } + if (!node.isArray()) { + current = newElement; + } else { + if (!newElement.isJsonArray()) { // error data - return new JsonSourceData(root, childRoot); + return new JsonSourceData(root, null); } - JsonElement newElement = current.getAsJsonObject().get(node.getName()); - if (newElement == null) { + JsonArray newArray = newElement.getAsJsonArray(); + if (node.getArrayIndex() >= newArray.size()) { // error data - return new JsonSourceData(root, childRoot); + return new JsonSourceData(root, null); } - if (!node.isArray()) { - current = newElement; - } else { - if (!newElement.isJsonArray()) { - // error data - return new JsonSourceData(root, childRoot); - } - JsonArray newArray = newElement.getAsJsonArray(); - if (node.getArrayIndex() >= newArray.size()) { - // error data - return new JsonSourceData(root, childRoot); - } - current = newArray.get(node.getArrayIndex()); - } - } - if (!current.isJsonArray()) { - // error data - return new JsonSourceData(root, childRoot); + current = newArray.get(node.getArrayIndex()); } - childRoot = current.getAsJsonArray(); } + if (!current.isJsonArray()) { + // error data + return new JsonSourceData(root, null); + } + childRoot = current.getAsJsonArray(); return new JsonSourceData(root, childRoot); } } diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceData.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceData.java index 2c39948f2d0..cf5f9c0fbe8 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceData.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceData.java @@ -26,5 +26,5 @@ public interface SourceData { int getRowCount(); - String getField(int rowNum, String fieldName); + Object getField(int rowNum, String fieldName); } diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/CsvSinkEncoder.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/CsvSinkEncoder.java index ce47a0072c7..89f6f364a08 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/CsvSinkEncoder.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/CsvSinkEncoder.java @@ -66,7 +66,11 @@ public String encode(SinkData sinkData, Context context) { } else { for (String fieldName : sinkData.keyList()) { String fieldValue = sinkData.getField(fieldName); - EscapeUtils.escapeContent(builder, delimiter, escapeChar, fieldValue); + if (StringUtils.equals(fieldName, ALL_SOURCE_FIELD_SIGN)) { + builder.append(fieldValue); + } else { + EscapeUtils.escapeContent(builder, delimiter, escapeChar, fieldValue); + } builder.append(delimiter); } } diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/KvSinkEncoder.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/KvSinkEncoder.java index 7460ec95c29..2822374c412 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/KvSinkEncoder.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/KvSinkEncoder.java @@ -63,7 +63,11 @@ public String encode(SinkData sinkData, Context context) { if (fields == null || fields.size() == 0) { for (String fieldName : sinkData.keyList()) { String fieldValue = sinkData.getField(fieldName); - builder.append(fieldName).append(kvDelimiter).append(fieldValue).append(entryDelimiter); + if (StringUtils.equals(fieldName, ALL_SOURCE_FIELD_SIGN)) { + builder.append(fieldValue).append(entryDelimiter); + } else { + builder.append(fieldName).append(kvDelimiter).append(fieldValue).append(entryDelimiter); + } } } else { for (FieldInfo field : fields) { diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkEncoder.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkEncoder.java index 7f845a99d61..a63f9702956 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkEncoder.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkEncoder.java @@ -27,6 +27,8 @@ */ public interface SinkEncoder { + public static final String ALL_SOURCE_FIELD_SIGN = "*"; + Output encode(SinkData sinkData, Context context); List getFields(); diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/FieldInfo.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/FieldInfo.java index 1027dad944b..2a7834112a1 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/FieldInfo.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/FieldInfo.java @@ -28,7 +28,7 @@ public class FieldInfo { private String name; - private TypeConverter converter; + private TypeConverter converter = TypeConverter.DefaultTypeConverter(); public FieldInfo() { diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java index 9944268dda6..acb7e62e070 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java @@ -31,23 +31,23 @@ import com.google.common.collect.ImmutableMap; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserManager; +import net.sf.jsqlparser.statement.select.AllColumns; import net.sf.jsqlparser.statement.select.PlainSelect; import net.sf.jsqlparser.statement.select.Select; import net.sf.jsqlparser.statement.select.SelectExpressionItem; import net.sf.jsqlparser.statement.select.SelectItem; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.StringReader; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; /** * TransformProcessor - * + * */ public class TransformProcessor { @@ -61,7 +61,9 @@ public class TransformProcessor { private PlainSelect transformSelect; private ExpressionOperator where; - private Map selectItemMap; + private List selectItems; + + private boolean includeAllSourceFields = false; public static TransformProcessor create( TransformConfig config, @@ -91,7 +93,7 @@ private void initTransformSql() throws JSQLParserException { this.transformSelect = (PlainSelect) select.getSelectBody(); this.where = OperatorTools.buildOperator(this.transformSelect.getWhere()); List items = this.transformSelect.getSelectItems(); - this.selectItemMap = new HashMap<>(items.size()); + this.selectItems = new ArrayList<>(items.size()); List fields = this.encoder.getFields(); for (int i = 0; i < items.size(); i++) { SelectItem item = items.get(i); @@ -108,8 +110,12 @@ private void initTransformSql() throws JSQLParserException { fieldName = exprItem.getAlias().getName(); } } - this.selectItemMap.put(fieldName, - OperatorTools.buildParser(exprItem.getExpression())); + this.selectItems + .add(new ValueParserNode(fieldName, OperatorTools.buildParser(exprItem.getExpression()))); + } else if (item instanceof AllColumns) { + fieldName = item.toString(); + this.encoder.getFields().clear(); + this.selectItems.add(new ValueParserNode(fieldName, null)); } } } @@ -137,10 +143,18 @@ public List transform(I input, Map extParams) { // parse value SinkData sinkData = new DefaultSinkData(); - for (Entry entry : this.selectItemMap.entrySet()) { - String fieldName = entry.getKey(); + for (ValueParserNode node : this.selectItems) { + String fieldName = node.getFieldName(); + ValueParser parser = node.getParser(); + if (parser == null && StringUtils.equals(fieldName, SinkEncoder.ALL_SOURCE_FIELD_SIGN)) { + if (input instanceof String) { + sinkData.addField(fieldName, (String) input); + } else { + sinkData.addField(fieldName, ""); + } + continue; + } try { - ValueParser parser = entry.getValue(); Object fieldValue = parser.parse(sourceData, i, context); sinkData.addField(fieldName, String.valueOf(fieldValue)); } catch (Throwable t) { diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/ValueParserNode.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/ValueParserNode.java new file mode 100644 index 00000000000..e36c0c9c6a9 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/ValueParserNode.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process; + +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * ValueParserNode + */ +@AllArgsConstructor +@Data +public class ValueParserNode { + + private String fieldName; + private ValueParser parser; +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/converter/DoubleConverter.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/converter/DoubleConverter.java new file mode 100644 index 00000000000..52afbda16b7 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/converter/DoubleConverter.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.converter; + +public class DoubleConverter implements TypeConverter { + + @Override + public Object convert(String value) throws Exception { + return Double.parseDouble(value); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/converter/LongConverter.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/converter/LongConverter.java new file mode 100644 index 00000000000..5a18f8ee138 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/converter/LongConverter.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.converter; + +public class LongConverter implements TypeConverter { + + @Override + public Object convert(String value) throws Exception { + return Long.parseLong(value); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AsinFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AsinFunction.java new file mode 100644 index 00000000000..ac45933509f --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AsinFunction.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; + +import java.math.BigDecimal; + +/** + * AsinFunction + * description: asin(numeric)--returns the arc sine of numeric + */ +@TransformFunction(names = {"asin"}) +public class AsinFunction implements ValueParser { + + private ValueParser numberParser; + + public AsinFunction(Function expr) { + numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object numberObj = numberParser.parse(sourceData, rowIndex, context); + if (numberObj == null) { + throw new NullPointerException("Parsed number object is null"); + } + BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); + return Math.asin(numberValue.doubleValue()); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Atan2Function.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Atan2Function.java new file mode 100644 index 00000000000..733a323347c --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Atan2Function.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; + +import java.math.BigDecimal; + +/** + * Atan2Function + * description: atan2(numeric)--returns the arc tangent of a coordinate (numeric1, numeric2). + */ +@TransformFunction(names = {"atan2"}) +public class Atan2Function implements ValueParser { + + private ValueParser xParser; + private ValueParser yParser; + + public Atan2Function(Function expr) { + xParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + yParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(1)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object xObj = xParser.parse(sourceData, rowIndex, context); + Object yObj = yParser.parse(sourceData, rowIndex, context); + + if (xObj == null) { + throw new NullPointerException("Parsed number object on the x-axis is null"); + } + + BigDecimal xValue = OperatorTools.parseBigDecimal(xObj); + BigDecimal yValue = OperatorTools.parseBigDecimal(yObj); + + return Math.atan2(xValue.doubleValue(), yValue.doubleValue()); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AtanFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AtanFunction.java new file mode 100644 index 00000000000..13a7d0bac61 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AtanFunction.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; + +import java.math.BigDecimal; + +/** + * AtanFunction + * description: atan(numeric)--returns the arc tangent of numeric + */ +@TransformFunction(names = {"atan"}) +public class AtanFunction implements ValueParser { + + private ValueParser numberParser; + + public AtanFunction(Function expr) { + numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object numberObj = numberParser.parse(sourceData, rowIndex, context); + BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); + return Math.atan(numberValue.doubleValue()); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ContainsFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ContainsFunction.java new file mode 100644 index 00000000000..e2905eea07e --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ContainsFunction.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; + +import java.util.List; + +/** + * ContainsFunction + * description: contains(left, right) - Returns a boolean. + * The value is True if right is found inside left, otherwise, returns False. + * Both left or right must be of STRING type. + */ +@TransformFunction(names = {"contains"}) +public class ContainsFunction implements ValueParser { + + private ValueParser leftStrParser; + private ValueParser rightStrParser; + + public ContainsFunction(Function expr) { + List expressions = expr.getParameters().getExpressions(); + leftStrParser = OperatorTools.buildParser(expressions.get(0)); + rightStrParser = OperatorTools.buildParser(expressions.get(1)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object leftStrObj = leftStrParser.parse(sourceData, rowIndex, context); + Object rightStrObj = rightStrParser.parse(sourceData, rowIndex, context); + String leftStr = OperatorTools.parseString(leftStrObj); + String rightStr = OperatorTools.parseString(rightStrObj); + return (leftStr == null || rightStr == null) ? null : leftStr.contains(rightStr); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/CoshFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/CoshFunction.java new file mode 100644 index 00000000000..0de7e84f03e --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/CoshFunction.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; + +import java.math.BigDecimal; + +/** + * CoshFunction + * description: cosh(numeric)--returns the hyperbolic cosine of numeric + */ +@TransformFunction(names = {"cosh"}) +public class CoshFunction implements ValueParser { + + private ValueParser numberParser; + + public CoshFunction(Function expr) { + numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object numberObj = numberParser.parse(sourceData, rowIndex, context); + BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); + return Math.cosh(numberValue.doubleValue()); + } +} \ No newline at end of file diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/CotFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/CotFunction.java new file mode 100644 index 00000000000..34bbe3231c2 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/CotFunction.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; + +import java.math.BigDecimal; + +/** + * CotFunction + * description: cot(numeric) -- returns the cotangent of the numeric (in radians) + */ +@TransformFunction(names = {"cot"}) +public class CotFunction implements ValueParser { + + private final ValueParser valueParser; + + public CotFunction(Function expr) { + this.valueParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object valueObj = valueParser.parse(sourceData, rowIndex, context); + + BigDecimal value = OperatorTools.parseBigDecimal(valueObj); + + // Calculate tan(x) and take the inverse to find cot(x) + double tanValue = Math.tan(value.doubleValue()); + if (tanValue == 0) { + throw new ArithmeticException("Cotangent undefined for this input, tan(x) is zero."); + } + return 1.0 / tanValue; + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateDiffFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateDiffFunction.java new file mode 100644 index 00000000000..77b2c93dfc6 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateDiffFunction.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.List; + +/** + * DateDiffFunction + * description: DATEDIFF(d1, d2) + * - return null if one of the two parameters is null or "" + * - return null if one of the two parameters has an incorrect date format + * - return the number of days between the dates d1->d2. + */ +@TransformFunction(names = {"datediff", "date_diff"}) +public class DateDiffFunction implements ValueParser { + + private final ValueParser leftDateParser; + private final ValueParser rightDateParser; + private static final DateTimeFormatter DEFAULT_FORMAT_DATE_TIME = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final DateTimeFormatter DEFAULT_FORMAT_DATE = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + public DateDiffFunction(Function expr) { + List expressions = expr.getParameters().getExpressions(); + leftDateParser = OperatorTools.buildParser(expressions.get(0)); + rightDateParser = OperatorTools.buildParser(expressions.get(1)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object leftDateObj = leftDateParser.parse(sourceData, rowIndex, context); + Object rightDateObj = rightDateParser.parse(sourceData, rowIndex, context); + if (leftDateObj == null || rightDateObj == null) { + return null; + } + String leftDate = OperatorTools.parseString(leftDateObj); + String rightDate = OperatorTools.parseString(rightDateObj); + if (leftDate.isEmpty() || rightDate.isEmpty()) { + return null; + } + try { + LocalDate left = getLocalDate(leftDate); + LocalDate right = getLocalDate(rightDate); + return ChronoUnit.DAYS.between(right, left); + } catch (Exception e) { + return null; + } + } + + public LocalDate getLocalDate(String dateString) { + DateTimeFormatter formatter = null; + LocalDate dateTime = null; + if (dateString.indexOf(' ') != -1) { + formatter = DEFAULT_FORMAT_DATE_TIME; + dateTime = LocalDateTime.parse(dateString, formatter).toLocalDate(); + } else { + formatter = DEFAULT_FORMAT_DATE; + dateTime = LocalDate.parse(dateString, formatter); + } + return dateTime; + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java index ff0eb6b01eb..a1ab7e2dfa6 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java @@ -42,6 +42,7 @@ * - dayofyear(date)--returns the day of a year (an integer between 1 and 366) from SQL date * - dayofmonth(date)--returns the day of a month (an integer between 1 and 31) from SQL date * - dayofweek(date)--returns the day of a week (an integer between 1(Sunday) and 7(Saturday)) from SQL date + * - dayname(date)--returns the name of the day of the week from SQL date */ public abstract class DateExtractFunction implements ValueParser { @@ -50,7 +51,7 @@ public abstract class DateExtractFunction implements ValueParser { private static final TemporalField weekOfYearField = WeekFields.of(Locale.getDefault()).weekOfWeekBasedYear(); public enum DateExtractFunctionType { - YEAR, QUARTER, MONTH, WEEK, DAY_OF_YEAR, DAY_OF_MONTH, DAY_OF_WEEK + YEAR, QUARTER, MONTH, WEEK, DAY_OF_YEAR, DAY_OF_MONTH, DAY_OF_WEEK, DAY_NAME } @TransformFunction(names = {"year"}) @@ -85,7 +86,7 @@ public WeekExtractFunction(Function expr) { } } - @TransformFunction(names = {"day_of_year"}) + @TransformFunction(names = {"day_of_year", "dayofyear"}) public static class DayOfYearExtractFunction extends DateExtractFunction { public DayOfYearExtractFunction(Function expr) { @@ -93,7 +94,7 @@ public DayOfYearExtractFunction(Function expr) { } } - @TransformFunction(names = {"day_of_month"}) + @TransformFunction(names = {"day_of_month", "dayofmonth"}) public static class DayOfMonthExtractFunction extends DateExtractFunction { public DayOfMonthExtractFunction(Function expr) { @@ -101,7 +102,7 @@ public DayOfMonthExtractFunction(Function expr) { } } - @TransformFunction(names = {"day_of_week"}) + @TransformFunction(names = {"day_of_week", "dayofweek"}) public static class DayOfWeekExtractFunction extends DateExtractFunction { public DayOfWeekExtractFunction(Function expr) { @@ -109,6 +110,14 @@ public DayOfWeekExtractFunction(Function expr) { } } + @TransformFunction(names = {"day_name", "dayname"}) + public static class DayNameExtractFunction extends DateExtractFunction { + + public DayNameExtractFunction(Function expr) { + super(DateExtractFunctionType.DAY_NAME, expr); + } + } + public DateExtractFunction(DateExtractFunctionType type, Function expr) { this.type = type; List expressions = expr.getParameters().getExpressions(); @@ -142,6 +151,9 @@ public Object parse(SourceData sourceData, int rowIndex, Context context) { // dayofweek(between 1 and 7) case DAY_OF_WEEK: return localDate.getDayOfWeek().getValue() % 7 + 1; + // dayname(between Sunday and Saturday) + case DAY_NAME: + return localDate.getDayOfWeek().name(); default: return null; } diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromBase64Function.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromBase64Function.java new file mode 100644 index 00000000000..d64891d4711 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromBase64Function.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; + +/** + * FromBase64Function + * description: Returns the base64-decoded result from string; returns NULL if string is NULL + */ +@TransformFunction(names = {"from_base64"}) +public class FromBase64Function implements ValueParser { + + private final ValueParser stringParser; + + public FromBase64Function(Function expr) { + List expressions = expr.getParameters().getExpressions(); + stringParser = OperatorTools.buildParser(expressions.get(0)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object stringObj = stringParser.parse(sourceData, rowIndex, context); + if (stringObj == null) { + return null; + } + String encodedString = OperatorTools.parseString(stringObj); + + if (encodedString == null) { + return null; + } + + try { + byte[] decodedBytes = Base64.getDecoder().decode(encodedString); + return new String(decodedBytes, StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + // handle decoding exceptions and log exception information + throw new RuntimeException("Invalid Base64 input: " + encodedString, e); + } + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromUnixTimeFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromUnixTimeFunction.java index b7b68a0e733..6a897ee3e04 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromUnixTimeFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromUnixTimeFunction.java @@ -41,7 +41,7 @@ * format(default is ‘yyyy-MM-dd HH:mm:ss’). numeric is an internal timestamp value representing seconds * since ‘1970-01-01 00:00:00’ UTC, such as produced by the UNIX_TIMESTAMP() function. */ -@TransformFunction(names = {"from_unix_time"}) +@TransformFunction(names = {"from_unix_time", "form_unixtime"}) public class FromUnixTimeFunction implements ValueParser { private ValueParser numericParser; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java index c641209162a..0d33056c728 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java @@ -36,7 +36,7 @@ * - If the input argument is a string, the HEX function converts each character in the string to its corresponding hexadecimal ASCII encoding and returns the hexadecimal representation of the entire string. */ @TransformFunction(names = {"hex"}) -class HexFunction implements ValueParser { +public class HexFunction implements ValueParser { private static final Pattern BIG_DECIMAL_PATTERN = Pattern.compile("^[-+]?\\d+(\\.\\d+)?([eE][-+]?\\d+)?$"); diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/IfFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/IfFunction.java new file mode 100644 index 00000000000..bda7b9301cc --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/IfFunction.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.ExpressionOperator; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; + +import java.util.List; + +/** + * IfFunction + * description: if(expr,r1,r2) -- expr is an expression, if it holds, return r1; otherwise, return r2 + */ +@TransformFunction(names = {"if"}) +public class IfFunction implements ValueParser { + + private final ExpressionOperator expressionOperator; + private final ValueParser tureValueParser; + private final ValueParser falseValueParser; + + public IfFunction(Function expr) { + List expressions = expr.getParameters().getExpressions(); + expressionOperator = OperatorTools.buildOperator(expressions.get(0)); + tureValueParser = OperatorTools.buildParser(expressions.get(1)); + falseValueParser = OperatorTools.buildParser(expressions.get(2)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + boolean condition = expressionOperator.check(sourceData, rowIndex, context); + return condition ? tureValueParser.parse(sourceData, rowIndex, context) + : falseValueParser.parse(sourceData, rowIndex, context); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/IfNullFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/IfNullFunction.java new file mode 100644 index 00000000000..f19b291d7e0 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/IfNullFunction.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; + +import java.util.List; + +/** + * IfNullFunction + * description: IFNULL(expr1,expr2) + * - return expr1 if expr1 is not NULL + * - return expr2 otherwise + */ +@Slf4j +@TransformFunction(names = {"ifnull", "if_null"}) +public class IfNullFunction implements ValueParser { + + private final ValueParser firstExprParser; + private final ValueParser secondExprParser; + + public IfNullFunction(Function expr) { + List expressions = expr.getParameters().getExpressions(); + firstExprParser = OperatorTools.buildParser(expressions.get(0)); + secondExprParser = OperatorTools.buildParser(expressions.get(1)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + try { + Object firstExprObj = firstExprParser.parse(sourceData, rowIndex, context); + if (firstExprObj == null) { + return parseSecondExpr(sourceData, rowIndex, context); + } + return firstExprObj; + } catch (Exception e) { + log.error("Value parsing failed", e); + return parseSecondExpr(sourceData, rowIndex, context); + } + } + + private Object parseSecondExpr(SourceData sourceData, int rowIndex, Context context) { + try { + return secondExprParser.parse(sourceData, rowIndex, context); + } catch (Exception e) { + log.error("Value parsing failed", e); + return null; + } + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/InsertFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/InsertFunction.java new file mode 100644 index 00000000000..3473bc9d793 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/InsertFunction.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; + +import java.util.List; +/** + * InsertFunction + * + * Description: + * Returns a string where a specified substring is replaced by another string, starting at a given position and for a specified length. + * If the position is out of the string's bounds, the original string is returned. + * If the length exceeds the remaining length of the string from the given position, the replacement continues to the end of the string. + * If any argument is null, the function returns null. + * + * Arguments: + * - str: The original string. + * - pos: The position to start the replacement (1-based index). + * - len: The number of characters to replace. + * - newstr: The string to insert. + * + * Examples: + * - INSERT('12345678', 3, 4, 'word') = '12word78' + * - INSERT('12345678', -1, 4, 'word') = '12345678' + * - INSERT('12345678', 3, 100, 'word') = '12word' + */ +@TransformFunction(names = {"insert"}) +public class InsertFunction implements ValueParser { + + private ValueParser strParser; + private ValueParser posParser; + private ValueParser lenParser; + private ValueParser newStrParser; + + public InsertFunction(Function expr) { + List expressions = expr.getParameters().getExpressions(); + strParser = OperatorTools.buildParser(expressions.get(0)); + posParser = OperatorTools.buildParser(expressions.get(1)); + lenParser = OperatorTools.buildParser(expressions.get(2)); + newStrParser = OperatorTools.buildParser(expressions.get(3)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object strObject = strParser.parse(sourceData, rowIndex, context); + Object posObject = posParser.parse(sourceData, rowIndex, context); + Object lenObject = lenParser.parse(sourceData, rowIndex, context); + Object newStrObject = newStrParser.parse(sourceData, rowIndex, context); + + if (strObject == null || posObject == null || lenObject == null || newStrObject == null) { + return null; + } + + String str = OperatorTools.parseString(strObject); + int pos = OperatorTools.parseBigDecimal(posObject).intValue(); + int len = OperatorTools.parseBigDecimal(lenObject).intValue(); + String newStr = OperatorTools.parseString(newStrObject); + + if (str == null || newStr == null) { + return null; + } + + if (pos < 1 || pos > str.length()) { + return str; + } + + int startIndex = pos - 1; + int endIndex = Math.min(startIndex + len, str.length()); + + StringBuilder result = new StringBuilder(); + result.append(str, 0, startIndex); + result.append(newStr); + if (endIndex < str.length()) { + result.append(str, endIndex, str.length()); + } + + return result.toString(); + } +} \ No newline at end of file diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LpadFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LpadFunction.java new file mode 100644 index 00000000000..cce51ba1e42 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LpadFunction.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; + +import java.util.List; + +/** + * LpadFunction + * description: Lpad(s1,len,s2) Fill string s2 at the beginning of string s1 to make the string length len + * - return null if any of the three parameters is null or len is less than 0 + * - return the substring of s1 with subscripts in the range of [0, len) if len is less than or equal to the length of s1 + * - if s2 is "" + * - return "" if len is longer than the length of s1 + * - if s2 is not "" + * - return the filled string + */ +@TransformFunction(names = {"lpad"}) +public class LpadFunction implements ValueParser { + + private final ValueParser leftStringParser; + private final ValueParser lengthParser; + private final ValueParser rightStringParser; + + public LpadFunction(Function expr) { + List expressions = expr.getParameters().getExpressions(); + leftStringParser = OperatorTools.buildParser(expressions.get(0)); + lengthParser = OperatorTools.buildParser(expressions.get(1)); + rightStringParser = OperatorTools.buildParser(expressions.get(2)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object leftStringObj = leftStringParser.parse(sourceData, rowIndex, context); + Object lengthObj = lengthParser.parse(sourceData, rowIndex, context); + Object rightStringObj = rightStringParser.parse(sourceData, rowIndex, context); + if (leftStringObj == null || lengthObj == null || rightStringObj == null) { + return null; + } + int len = Integer.parseInt(OperatorTools.parseString(lengthObj)); + if (len < 0) { + return null; + } + String leftStr = OperatorTools.parseString(leftStringObj); + if (len <= leftStr.length()) { + return leftStr.substring(0, len); + } + String rightStr = OperatorTools.parseString(rightStringObj); + if (rightStr.isEmpty()) { + return ""; + } + int padLen = len - leftStr.length(); + StringBuilder builder = new StringBuilder(padLen); + while (builder.length() < padLen) { + builder.append(rightStr); + } + return builder.substring(0, padLen).concat(leftStr); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/PiFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/PiFunction.java new file mode 100644 index 00000000000..722955ebf51 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/PiFunction.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; +/** + * PiFunction + * returns the mathematical constant PI + */ +@TransformFunction(names = {"pi"}) +public class PiFunction implements ValueParser { + + public PiFunction(Function expr) { + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + return String.valueOf(Math.PI); + } + +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RadiansFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RadiansFunction.java new file mode 100644 index 00000000000..635d18b9b8f --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RadiansFunction.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; + +/** + * RadiansFunction + * description: + * - RADIANS(x)--returns radians of x, Convert degrees to radians + */ +@TransformFunction(names = {"radians"}) +public class RadiansFunction implements ValueParser { + + private ValueParser degreeParser; + + public RadiansFunction(Function expr) { + degreeParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object degreeObj = degreeParser.parse(sourceData, rowIndex, context); + if (degreeObj == null) { + return null; + } + return Math.toRadians(OperatorTools.parseBigDecimal(degreeObj).doubleValue()); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RpadFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RpadFunction.java new file mode 100644 index 00000000000..c3357c7b552 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RpadFunction.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; + +import java.util.List; + +/** + * RpadFunction + * description: RPAD(s1,len,s2) : Fill string s2 at the end of string s1 to make the length of the string len + * - return null if any of the three parameters is null or len is less than 0 + * - return the substring of s1 with subscripts in the range of [0, len) if len is less than or equal to the length of s1 + * - if s2 is "" + * - return "" if len is longer than the length of s1 + * - if s2 is not "" + * - return the filled string + */ +@TransformFunction(names = {"rpad"}) +public class RpadFunction implements ValueParser { + + private final ValueParser leftStringParser; + private final ValueParser lengthParser; + private final ValueParser rightStringParser; + + public RpadFunction(Function expr) { + List expressions = expr.getParameters().getExpressions(); + leftStringParser = OperatorTools.buildParser(expressions.get(0)); + lengthParser = OperatorTools.buildParser(expressions.get(1)); + rightStringParser = OperatorTools.buildParser(expressions.get(2)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object leftStringObj = leftStringParser.parse(sourceData, rowIndex, context); + Object lengthObj = lengthParser.parse(sourceData, rowIndex, context); + Object rightStringObj = rightStringParser.parse(sourceData, rowIndex, context); + if (leftStringObj == null || lengthObj == null || rightStringObj == null) { + return null; + } + int len = Integer.parseInt(OperatorTools.parseString(lengthObj)); + if (len < 0) { + return null; + } + String leftStr = OperatorTools.parseString(leftStringObj); + if (len <= leftStr.length()) { + return leftStr.substring(0, len); + } + String rightStr = OperatorTools.parseString(rightStringObj); + if (rightStr.isEmpty()) { + return ""; + } + StringBuilder builder = new StringBuilder(len); + builder.append(leftStr); + while (builder.length() < len) { + builder.append(rightStr); + } + return builder.substring(0, len); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java new file mode 100644 index 00000000000..ad120712bfd --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import org.apache.commons.codec.digest.DigestUtils; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.apache.commons.codec.digest.MessageDigestAlgorithms.SHA_224; + +/** + * Sha2Function + * description: SHA2(str, hash_length): Calculates the SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, and SHA-512) + * return NULL If either argument is NULL or the hash length(224 256 384 512) is not one of the permitted values + * return a hash value containing the desired number of bits. + */ +@TransformFunction(names = {"sha2"}) +public class Sha2Function implements ValueParser { + + private final ValueParser msgParser; + private final ValueParser lenParser; + + public Sha2Function(Function expr) { + List expressions = expr.getParameters().getExpressions(); + msgParser = OperatorTools.buildParser(expressions.get(0)); + lenParser = OperatorTools.buildParser(expressions.get(1)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object msgObj = msgParser.parse(sourceData, rowIndex, context); + Object lenObj = lenParser.parse(sourceData, rowIndex, context); + if (msgObj == null || lenObj == null) { + return null; + } + String msg = msgObj.toString(); + int len = Integer.parseInt(lenObj.toString()); + switch (len) { + case 0: + case 256: + return DigestUtils.sha256Hex(msg.getBytes(StandardCharsets.UTF_8)); + case 224: + return new DigestUtils(SHA_224).digestAsHex(msg.getBytes(StandardCharsets.UTF_8)); + case 384: + return DigestUtils.sha384Hex(msg.getBytes(StandardCharsets.UTF_8)); + case 512: + return DigestUtils.sha512Hex(msg.getBytes(StandardCharsets.UTF_8)); + default: + return null; + } + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java new file mode 100644 index 00000000000..7da5e6c3a9d --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; +import org.apache.commons.codec.digest.DigestUtils; + +import java.nio.charset.StandardCharsets; + +/** + * ShaFunction + * description: sha(string): Compute the SHA-1 160 bit checksum of a string. + * return NULL if the parameter is NULL + * return a string of 40 hexadecimal digits. + */ +@TransformFunction(names = {"sha"}) +public class ShaFunction implements ValueParser { + + private final ValueParser msgParser; + + public ShaFunction(Function expr) { + msgParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object msgObj = msgParser.parse(sourceData, rowIndex, context); + if (msgObj == null) { + return null; + } + String msg = msgObj.toString(); + return DigestUtils.sha1Hex(msg.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SubstringFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SubstringFunction.java index 772dc2950cf..727c0484b4b 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SubstringFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SubstringFunction.java @@ -32,7 +32,7 @@ * description: substring(string FROM INT1 [ FOR INT2 ])--returns a substring of STRING starting from position INT1 with * length INT2 (to the end by default) */ -@TransformFunction(names = {"substring"}) +@TransformFunction(names = {"substring", "substr"}) public class SubstringFunction implements ValueParser { private ValueParser stringParser; @@ -41,6 +41,7 @@ public class SubstringFunction implements ValueParser { /** * Constructor + * * @param expr */ public SubstringFunction(Function expr) { @@ -55,6 +56,7 @@ public SubstringFunction(Function expr) { /** * parse + * * @param sourceData * @param rowIndex * @return diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TanhFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TanhFunction.java new file mode 100644 index 00000000000..dd19eb57081 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TanhFunction.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.function; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import org.apache.inlong.sdk.transform.process.parser.ValueParser; + +import net.sf.jsqlparser.expression.Function; + +import java.math.BigDecimal; + +/** + * TanhFunction + * description: tanh(numeric)--returns the hyperbolic tangent of numeric + */ +@TransformFunction(names = {"tanh"}) +public class TanhFunction implements ValueParser { + + private ValueParser numberParser; + + public TanhFunction(Function expr) { + numberParser = OperatorTools.buildParser(expr.getParameters().getExpressions().get(0)); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + Object numberObj = numberParser.parse(sourceData, rowIndex, context); + BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj); + return Math.tanh(numberValue.doubleValue()); + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TimestampAddFunction.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TimestampAddFunction.java index 9bcbb8c8e42..29b4636d81d 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TimestampAddFunction.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TimestampAddFunction.java @@ -36,7 +36,7 @@ * The unit of the time interval is specified by the unit parameter, which should be one of the following values: * FRAC_SECOND, SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, or YEAR. */ -@TransformFunction(names = {"timestamp_add"}) +@TransformFunction(names = {"timestamp_add", "timestampadd"}) public class TimestampAddFunction implements ValueParser { private ValueParser intervalParser; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/AndOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/AndOperator.java index a9dcd426062..f438d4295cd 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/AndOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/AndOperator.java @@ -26,6 +26,7 @@ * AndOperator * */ +@TransformOperator(values = AndExpression.class) public class AndOperator implements ExpressionOperator { private final ExpressionOperator left; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/EqualsToOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/EqualsToOperator.java index 709537e8a01..13010c854bf 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/EqualsToOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/EqualsToOperator.java @@ -27,6 +27,7 @@ * EqualsToOperator * */ +@TransformOperator(values = EqualsTo.class) public class EqualsToOperator implements ExpressionOperator { private final ValueParser left; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanEqualsOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanEqualsOperator.java index 3a53968e10e..e703afdbda7 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanEqualsOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanEqualsOperator.java @@ -27,6 +27,7 @@ * GreaterThanEqualsOperator * */ +@TransformOperator(values = GreaterThanEquals.class) public class GreaterThanEqualsOperator implements ExpressionOperator { private final ValueParser left; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanOperator.java index a1cd8c2ea20..ba73ef4c4df 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanOperator.java @@ -27,6 +27,7 @@ * GreaterThanOperator * */ +@TransformOperator(values = GreaterThan.class) public class GreaterThanOperator implements ExpressionOperator { private final ValueParser left; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanEqualsOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanEqualsOperator.java index 4248cf1d366..e8104e2cc3b 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanEqualsOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanEqualsOperator.java @@ -27,6 +27,7 @@ * MinorThanEqualsOperator * */ +@TransformOperator(values = MinorThanEquals.class) public class MinorThanEqualsOperator implements ExpressionOperator { private final ValueParser left; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanOperator.java index 21ecc0400a3..ecae167c5c4 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanOperator.java @@ -27,6 +27,7 @@ * MinorThanOperator * */ +@TransformOperator(values = MinorThan.class) public class MinorThanOperator implements ExpressionOperator { private final ValueParser left; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotEqualsToOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotEqualsToOperator.java index 98bf102b4f4..d3fec5c8c2c 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotEqualsToOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotEqualsToOperator.java @@ -27,6 +27,7 @@ * NotEqualsToOperator * */ +@TransformOperator(values = NotEqualsTo.class) public class NotEqualsToOperator implements ExpressionOperator { private final ValueParser left; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotOperator.java index d8b9ff07e06..306177ffbab 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/NotOperator.java @@ -26,6 +26,7 @@ * NotOperator * */ +@TransformOperator(values = NotExpression.class) public class NotOperator implements ExpressionOperator { private final ExpressionOperator node; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java index 75b61783602..99dc99b2f63 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java @@ -47,19 +47,13 @@ import org.apache.inlong.sdk.transform.process.parser.ColumnParser; import org.apache.inlong.sdk.transform.process.parser.ValueParser; +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Function; -import net.sf.jsqlparser.expression.NotExpression; -import net.sf.jsqlparser.expression.Parenthesis; -import net.sf.jsqlparser.expression.operators.conditional.AndExpression; -import net.sf.jsqlparser.expression.operators.conditional.OrExpression; -import net.sf.jsqlparser.expression.operators.relational.EqualsTo; -import net.sf.jsqlparser.expression.operators.relational.GreaterThan; -import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; -import net.sf.jsqlparser.expression.operators.relational.MinorThan; -import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; -import net.sf.jsqlparser.expression.operators.relational.NotEqualsTo; import org.apache.commons.lang.ObjectUtils; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; import java.math.BigDecimal; import java.sql.Date; @@ -69,10 +63,15 @@ /** * OperatorTools - * + * */ +@Slf4j public class OperatorTools { + private static final String OPERATOR_PATH = "org.apache.inlong.sdk.transform.process.operator"; + + private final static Map, Class> operatorMap = Maps.newConcurrentMap(); + public static final String ROOT_KEY = "$root"; public static final String CHILD_KEY = "$child"; @@ -122,26 +121,8 @@ public class OperatorTools { } public static ExpressionOperator buildOperator(Expression expr) { - if (expr instanceof AndExpression) { - return new AndOperator((AndExpression) expr); - } else if (expr instanceof OrExpression) { - return new OrOperator((OrExpression) expr); - } else if (expr instanceof Parenthesis) { - return new ParenthesisOperator((Parenthesis) expr); - } else if (expr instanceof NotExpression) { - return new NotOperator((NotExpression) expr); - } else if (expr instanceof EqualsTo) { - return new EqualsToOperator((EqualsTo) expr); - } else if (expr instanceof NotEqualsTo) { - return new NotEqualsToOperator((NotEqualsTo) expr); - } else if (expr instanceof GreaterThan) { - return new GreaterThanOperator((GreaterThan) expr); - } else if (expr instanceof GreaterThanEquals) { - return new GreaterThanEqualsOperator((GreaterThanEquals) expr); - } else if (expr instanceof MinorThan) { - return new MinorThanOperator((MinorThan) expr); - } else if (expr instanceof MinorThanEquals) { - return new MinorThanEqualsOperator((MinorThanEquals) expr); + if (expr != null) { + return getTransformOperator(expr); } return null; } @@ -215,19 +196,18 @@ public static int compareValue(Comparable left, Comparable right) { if (right == null) { return 1; } - if (left instanceof String) { - if (right instanceof String) { - return ObjectUtils.compare(left, right); - } else { - BigDecimal leftValue = parseBigDecimal(left); - return ObjectUtils.compare(leftValue, right); - } + + if (((Object) left).getClass() == ((Object) right).getClass()) { + return ObjectUtils.compare(left, right); } else { - if (right instanceof String) { + try { + BigDecimal leftValue = parseBigDecimal(left); BigDecimal rightValue = parseBigDecimal(right); - return ObjectUtils.compare(left, rightValue); - } else { - return ObjectUtils.compare(left, right); + return ObjectUtils.compare(leftValue, rightValue); + } catch (Exception e) { + String leftValue = parseString(left); + String rightValue = parseString(right); + return ObjectUtils.compare(leftValue, rightValue); } } } diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OrOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OrOperator.java index b5de7f279ed..9efccba3f22 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OrOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OrOperator.java @@ -26,6 +26,7 @@ * OrOperator * */ +@TransformOperator(values = OrExpression.class) public class OrOperator implements ExpressionOperator { private final ExpressionOperator left; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/ParenthesisOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/ParenthesisOperator.java index 0ca1334fcee..de71c84b4d5 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/ParenthesisOperator.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/ParenthesisOperator.java @@ -26,6 +26,7 @@ * ParenthesisOperator * */ +@TransformOperator(values = Parenthesis.class) public class ParenthesisOperator implements ExpressionOperator { private final ExpressionOperator node; diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/TransformOperator.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/TransformOperator.java new file mode 100644 index 00000000000..ee9c676665e --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/TransformOperator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.operator; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.TYPE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target(TYPE) +public @interface TransformOperator { + + Class[] values(); +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseAndParser.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseAndParser.java new file mode 100644 index 00000000000..b8920434ef2 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseAndParser.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.parser; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; + +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.operators.arithmetic.BitwiseAnd; + +import java.math.BigInteger; + +/** + * BitwiseAndParser + */ +@Slf4j +@TransformParser(values = BitwiseAnd.class) +public class BitwiseAndParser implements ValueParser { + + private final ValueParser left; + + private final ValueParser right; + + public BitwiseAndParser(BitwiseAnd expr) { + this.left = OperatorTools.buildParser(expr.getLeftExpression()); + this.right = OperatorTools.buildParser(expr.getRightExpression()); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + try { + Object leftObj = this.left.parse(sourceData, rowIndex, context); + Object rightObj = this.right.parse(sourceData, rowIndex, context); + if (leftObj == null || rightObj == null) { + return null; + } + BigInteger leftValue = OperatorTools.parseBigDecimal(leftObj).toBigInteger(); + BigInteger rightValue = OperatorTools.parseBigDecimal(rightObj).toBigInteger(); + return Long.toUnsignedString(leftValue.and(rightValue).longValue()); + } catch (Exception e) { + log.error("Value parsing failed", e); + return null; + } + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseLeftShiftParser.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseLeftShiftParser.java new file mode 100644 index 00000000000..c68bc89e543 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseLeftShiftParser.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.parser; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; + +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.operators.arithmetic.BitwiseLeftShift; + +import java.math.BigInteger; + +/** + * BitwiseLeftShiftParser + * + */ +@Slf4j +@TransformParser(values = BitwiseLeftShift.class) +public class BitwiseLeftShiftParser implements ValueParser { + + private final ValueParser left; + + private final ValueParser right; + + public BitwiseLeftShiftParser(BitwiseLeftShift expr) { + this.left = OperatorTools.buildParser(expr.getLeftExpression()); + this.right = OperatorTools.buildParser(expr.getRightExpression()); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + try { + Object leftObj = this.left.parse(sourceData, rowIndex, context); + Object rightObj = this.right.parse(sourceData, rowIndex, context); + if (leftObj == null || rightObj == null) { + return null; + } + BigInteger leftValue = OperatorTools.parseBigDecimal(leftObj).toBigInteger(); + String unsignedRight = Long.toUnsignedString(OperatorTools.parseBigDecimal(rightObj).longValue()); + int cmp = new BigInteger(unsignedRight).compareTo(new BigInteger("65")); + if (cmp >= 0) { + return Long.toUnsignedString(leftValue.shiftLeft(65).longValue()); + } else { + return Long.toUnsignedString(leftValue.shiftLeft(Integer.parseInt(unsignedRight)).longValue()); + } + } catch (Exception e) { + log.error("Value parsing failed", e); + return null; + } + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseOrParser.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseOrParser.java new file mode 100644 index 00000000000..2cbf5e9d09c --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseOrParser.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.parser; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; + +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.operators.arithmetic.BitwiseOr; + +import java.math.BigInteger; + +/** + * BitwiseOrParser + * + */ +@Slf4j +@TransformParser(values = BitwiseOr.class) +public class BitwiseOrParser implements ValueParser { + + private final ValueParser left; + + private final ValueParser right; + + public BitwiseOrParser(BitwiseOr expr) { + this.left = OperatorTools.buildParser(expr.getLeftExpression()); + this.right = OperatorTools.buildParser(expr.getRightExpression()); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + try { + Object leftObj = this.left.parse(sourceData, rowIndex, context); + Object rightObj = this.right.parse(sourceData, rowIndex, context); + if (leftObj == null || rightObj == null) { + return null; + } + BigInteger leftValue = OperatorTools.parseBigDecimal(leftObj).toBigInteger(); + BigInteger rightValue = OperatorTools.parseBigDecimal(rightObj).toBigInteger(); + return Long.toUnsignedString(leftValue.or(rightValue).longValue()); + } catch (Exception e) { + log.error("Value parsing failed", e); + return null; + } + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseRightShiftParser.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseRightShiftParser.java new file mode 100644 index 00000000000..7b6085725b8 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseRightShiftParser.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.parser; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; + +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.operators.arithmetic.BitwiseRightShift; + +import java.math.BigInteger; + +/** + * BitwiseRightShiftParser + */ +@Slf4j +@TransformParser(values = BitwiseRightShift.class) +public class BitwiseRightShiftParser implements ValueParser { + + private final ValueParser left; + + private final ValueParser right; + + public BitwiseRightShiftParser(BitwiseRightShift expr) { + this.left = OperatorTools.buildParser(expr.getLeftExpression()); + this.right = OperatorTools.buildParser(expr.getRightExpression()); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + try { + Object leftObj = this.left.parse(sourceData, rowIndex, context); + Object rightObj = this.right.parse(sourceData, rowIndex, context); + if (leftObj == null || rightObj == null) { + return null; + } + BigInteger leftValue = OperatorTools.parseBigDecimal(leftObj).toBigInteger(); + String unsignedRight = Long.toUnsignedString(OperatorTools.parseBigDecimal(rightObj).longValue()); + int cmp = new BigInteger(unsignedRight).compareTo(new BigInteger("65")); + if (cmp >= 0) { + return Long.toUnsignedString(leftValue.shiftRight(65).longValue()); + } else { + return Long.toUnsignedString(leftValue.shiftRight(Integer.parseInt(unsignedRight)).longValue()); + } + } catch (Exception e) { + log.error("Value parsing failed", e); + return null; + } + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseXorParser.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseXorParser.java new file mode 100644 index 00000000000..3277bd91edc --- /dev/null +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseXorParser.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process.parser; + +import org.apache.inlong.sdk.transform.decode.SourceData; +import org.apache.inlong.sdk.transform.process.Context; +import org.apache.inlong.sdk.transform.process.operator.OperatorTools; + +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.expression.operators.arithmetic.BitwiseXor; + +import java.math.BigInteger; + +/** + * BitwiseXorParser + * + */ +@Slf4j +@TransformParser(values = BitwiseXor.class) +public class BitwiseXorParser implements ValueParser { + + private final ValueParser left; + + private final ValueParser right; + + public BitwiseXorParser(BitwiseXor expr) { + this.left = OperatorTools.buildParser(expr.getLeftExpression()); + this.right = OperatorTools.buildParser(expr.getRightExpression()); + } + + @Override + public Object parse(SourceData sourceData, int rowIndex, Context context) { + try { + Object leftObj = this.left.parse(sourceData, rowIndex, context); + Object rightObj = this.right.parse(sourceData, rowIndex, context); + if (leftObj == null || rightObj == null) { + return null; + } + BigInteger leftValue = OperatorTools.parseBigDecimal(leftObj).toBigInteger(); + BigInteger rightValue = OperatorTools.parseBigDecimal(rightObj).toBigInteger(); + return Long.toUnsignedString(leftValue.xor(rightValue).longValue()); + } catch (Exception e) { + log.error("Value parsing failed", e); + return null; + } + } +} diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/DoubleParser.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/DoubleParser.java index ad39558a112..a88b17f6baa 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/DoubleParser.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/DoubleParser.java @@ -24,6 +24,7 @@ /** * LongParser + * */ @TransformParser(values = DoubleValue.class) public class DoubleParser implements ValueParser { diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/SignParser.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/SignParser.java index ff97aadfdb2..7a744f90154 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/SignParser.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/SignParser.java @@ -21,29 +21,43 @@ import org.apache.inlong.sdk.transform.process.Context; import org.apache.inlong.sdk.transform.process.operator.OperatorTools; +import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.expression.SignedExpression; import java.math.BigDecimal; /** * SignParser - * */ +@Slf4j @TransformParser(values = SignedExpression.class) public class SignParser implements ValueParser { - private final Integer sign; + private final char sign; private final ValueParser number; public SignParser(SignedExpression expr) { - sign = expr.getSign() == '-' ? -1 : 1; + sign = expr.getSign(); number = OperatorTools.buildParser(expr.getExpression()); } @Override public Object parse(SourceData sourceData, int rowIndex, Context context) { - Object numberObject = number.parse(sourceData, rowIndex, context); - BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObject); - return numberValue.multiply(new BigDecimal(sign)); + try { + Object numberObject = number.parse(sourceData, rowIndex, context); + if (numberObject == null) { + return null; + } + BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObject); + switch (sign) { + case '-': + return numberValue.multiply(new BigDecimal(-1)); + case '~': + return Long.toUnsignedString(numberValue.toBigInteger().not().longValue()); + } + } catch (Exception e) { + log.error("Value parsing failed", e); + } + return null; } } diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java index 3da557f1f65..cc3c4423289 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java @@ -224,6 +224,147 @@ public void testModuloFunction() throws Exception { } + @Test + public void testIfNullFunction() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: ifnull(5, 3) + transformSql = "select ifnull(numeric1,numeric2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "5|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=5", output.get(0)); + + // case2: ifnull(null,3) + transformSql = "select ifnull(xxd,numeric2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "5|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=3", output.get(0)); + + // case3: ifnull(6 / 3,'YES') + transformSql = "select ifnull(numeric1 / numeric2,'YES') from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "6|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=2", output.get(0)); + + // case4: ifnull(6 / 0,'YES') + transformSql = "select ifnull(numeric1 / numeric2,'YES') from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "6|0|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=YES", output.get(0)); + + // case5: ifnull(6 / 0,3 / 0) + transformSql = "select ifnull(numeric1 / numeric2,numeric3 / numeric2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "6|0|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + } + + @Test + public void testShaFunction() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: sha("") + transformSql = "select sha(numeric1) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=da39a3ee5e6b4b0d3255bfef95601890afd80709", output.get(0)); + + // case2: sha("5") + data = "5|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=ac3478d69a3c81fa62e60f5c3696165a4e5e6ac4", output.get(0)); + + // case3: sha(null) + transformSql = "select sha(xxd) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "3|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + } + + @Test + public void testSha2Function() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: sha2("",3) + transformSql = "select sha2(numeric1,numeric2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "|3|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case2: sha2("5",224) + data = "5|224|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=b51d18b551043c1f145f22dbde6f8531faeaf68c54ed9dd79ce24d17", output.get(0)); + + // case3: sha2("5",0) + data = "5|0|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39d", output.get(0)); + + // case4: sha2(null,224) + transformSql = "select sha2(xxd,224) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "3|224|3|5"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + } + @Test public void testMd5Function() throws Exception { String transformSql = "select md5(numeric1) from source"; @@ -257,6 +398,181 @@ public void testMd5Function() throws Exception { Assert.assertEquals("result=null", output4.get(0)); } + @Test + public void testBitwiseInversionOperator() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: ~-4 + transformSql = "select ~numeric1 from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "-4|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=3", output.get(0)); + + // case2: ~4 + data = "4|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=18446744073709551611", output.get(0)); + + // case3: ~0 + data = "0|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=18446744073709551615", output.get(0)); + + // case4: ~~-4 + transformSql = "select ~(~numeric1) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "-4|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=18446744073709551612", output.get(0)); + + } + @Test + public void testBitwiseAndOperator() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: 18446744073709551615 & -1 + transformSql = "select numeric1 & numeric2 from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "18446744073709551615|-1|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=18446744073709551615", output.get(0)); + + // case2: 18446744073709551615 & 0 + data = "18446744073709551615|0|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=0", output.get(0)); + } + @Test + public void testBitwiseOrOperator() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: 18446744073709551615 | -1 + transformSql = "select numeric1 | numeric2 from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "18446744073709551615|-1|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=18446744073709551615", output.get(0)); + + // case2: 4 | 3 + data = "4|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=7", output.get(0)); + } + @Test + public void testBitwiseRightShiftOperator() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: 4 >> -1 + transformSql = "select numeric1 >> numeric2 from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "4|-1|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=0", output.get(0)); + + // case2: 9223372036854775808 >> 2 + data = "9223372036854775808|2|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=2305843009213693952", output.get(0)); + + // case3: 9223372036854775808 >> 9223372036854775808 + data = "9223372036854775808|9223372036854775808|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=0", output.get(0)); + } + @Test + public void testBitwiseLeftShiftOperator() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: 9223372036854775807 << 1 + transformSql = "select numeric1 << numeric2 from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "9223372036854775807|1|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=18446744073709551614", output.get(0)); + + // case2: 18446744073709551615 << 18446744073709551615 + data = "18446744073709551615|18446744073709551615|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=0", output.get(0)); + + // case3: 9223372036854775807 << -1 + data = "9223372036854775807|-1|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=0", output.get(0)); + } + @Test + public void testBitwiseXorOperator() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + // case1: 4 ^ 3 + transformSql = "select numeric1 ^ numeric2 from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "4|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=7", output.get(0)); + + // case2: 4 ^ -1 + data = "4|-1|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=18446744073709551611", output.get(0)); + } + @Test public void testRoundFunction() throws Exception { String transformSql = "select round(numeric1) from source"; @@ -359,6 +675,23 @@ public void testLnFunction() throws Exception { Assert.assertEquals(output2.get(0), "result=2.302585092994046"); } + @Test + public void testRadiansFunction() throws Exception { + String transformSql = "select radians(numeric1) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + // case1: radians(10) + List output1 = processor.transform("10|4|6|8", new HashMap<>()); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0.17453292519943295"); + // case2: radians(18.97) + List output2 = processor.transform("18.97|4|6|8", new HashMap<>()); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=0.33108895910332425"); + } + @Test public void testLog10Function() throws Exception { String transformSql = "select log10(numeric1) from source"; @@ -648,6 +981,150 @@ public void testRandFunction() throws Exception { Assert.assertTrue(result >= 0.0 && result < 1.0); } + @Test + public void testCotFunction() throws Exception { + String transformSql = "select cot(numeric1) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + + // case1: cot(1) + List output1 = processor.transform("1|4|6|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0.6420926159343306"); + + // case2: cot(0.5) + List output2 = processor.transform("0.5|4|6|8"); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=1.830487721712452"); + + // case3: cot(-1) + List output3 = processor.transform("-1|4|6|8"); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "result=-0.6420926159343306"); + } + + @Test + public void testTanhFunction() throws Exception { + String transformSql = "select tanh(numeric1) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + + // case1: tanh(1) + List output1 = processor.transform("1|4|6|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0.7615941559557649"); + + // case2: tanh(0) + List output2 = processor.transform("0|4|6|8"); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=0.0"); + + // case3: tanh(-1) + List output3 = processor.transform("-1|4|6|8"); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "result=-0.7615941559557649"); + } + + @Test + public void testCoshFunction() throws Exception { + String transformSql = "select cosh(numeric1) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + + // case1: cosh(1) + List output1 = processor.transform("1|4|6|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=1.543080634815244"); + + // case2: cosh(0) + List output2 = processor.transform("0|4|6|8"); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=1.0"); + + // case3: cosh(-1) + List output3 = processor.transform("-1|4|6|8"); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "result=1.543080634815244"); + } + + @Test + public void testAsinFunction() throws Exception { + String transformSql = "select asin(numeric1) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + + // case1: asin(0.5) + List output1 = processor.transform("0.5|4|6|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0.5235987755982989"); + + // case2: asin(0) + List output2 = processor.transform("0|4|6|8"); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=0.0"); + + // case3: asin(-0.5) + List output3 = processor.transform("-0.5|4|6|8"); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "result=-0.5235987755982989"); + } + + @Test + public void testAtanFunction() throws Exception { + String transformSql = "select atan(numeric1) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + + // case1: atan(1) + List output1 = processor.transform("1|4|6|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0.7853981633974483"); + + // case2: atan(0) + List output2 = processor.transform("0|4|6|8"); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=0.0"); + + // case3: atan(-1) + List output3 = processor.transform("-1|4|6|8"); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "result=-0.7853981633974483"); + } + + @Test + public void testAtan2Function() throws Exception { + String transformSql = "select atan2(numeric1, numeric2) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + + // case1: atan2(1, 1) + List output1 = processor.transform("1|1|6|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0.7853981633974483"); + + // case2: atan2(1, 0) + List output2 = processor.transform("1|0|6|8"); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=1.5707963267948966"); + + // case3: atan2(0, -1) + List output3 = processor.transform("0|-1|6|8"); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "result=3.141592653589793"); + } + @Test public void testHexFunction() throws Exception { String transformSql1 = "select hex(numeric1) from source"; @@ -677,4 +1154,16 @@ public void testHexFunction() throws Exception { Assert.assertEquals(output5.get(0), "result=616263"); } + @Test + public void testPiFunction() throws Exception { + String transformSql1 = "select pi() from source"; + TransformConfig config1 = new TransformConfig(transformSql1); + TransformProcessor processor1 = TransformProcessor + .create(config1, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + // case: pi() + List output1 = processor1.transform("1007|4|6|8", new HashMap<>()); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=3.141592653589793"); + } } diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformExpressionOperatorsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformExpressionOperatorsProcessor.java new file mode 100644 index 00000000000..67e4e331d41 --- /dev/null +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformExpressionOperatorsProcessor.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process; + +import org.apache.inlong.sdk.transform.decode.SourceDecoderFactory; +import org.apache.inlong.sdk.transform.encode.SinkEncoderFactory; +import org.apache.inlong.sdk.transform.pojo.CsvSourceInfo; +import org.apache.inlong.sdk.transform.pojo.FieldInfo; +import org.apache.inlong.sdk.transform.pojo.KvSinkInfo; +import org.apache.inlong.sdk.transform.pojo.TransformConfig; +import org.apache.inlong.sdk.transform.process.converter.DoubleConverter; +import org.apache.inlong.sdk.transform.process.converter.LongConverter; +import org.apache.inlong.sdk.transform.process.converter.TypeConverter; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * TestArithmeticFunctionsTransformProcessor + * description: test the arithmetic functions in transform processor + */ +public class TestTransformExpressionOperatorsProcessor { + + private static final List srcFields = new ArrayList<>(); + private static final List dstFields = new ArrayList<>(); + private static final CsvSourceInfo csvSource; + private static final KvSinkInfo kvSink; + + static { + srcFields.add(new FieldInfo("numeric1", new DoubleConverter())); + srcFields.add(new FieldInfo("string2", TypeConverter.DefaultTypeConverter())); + srcFields.add(new FieldInfo("numeric3", new DoubleConverter())); + srcFields.add(new FieldInfo("numeric4", new LongConverter())); + + FieldInfo field = new FieldInfo(); + field.setName("result"); + dstFields.add(field); + csvSource = new CsvSourceInfo("UTF-8", '|', '\\', srcFields); + kvSink = new KvSinkInfo("UTF-8", dstFields); + } + + @Test + public void testEqualsToOperator() throws Exception { + String transformSql = "select if(string2 = 4,1,0) from source"; + TransformConfig config = new TransformConfig(transformSql); + // case1: "3.14159265358979323846|4a|4|8" + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor.transform("3.14159265358979323846|4a|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0"); + // case2: "3.14159265358979323846|4|4|8" + List output2 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output2.get(0), "result=1"); + + transformSql = "select if(numeric3 = 4,1,0) from source"; + config = new TransformConfig(transformSql); + // case3: "3.14159265358979323846|4|4|8" + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output3 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output3.get(0), "result=1"); + // case4: "3.14159265358979323846|4|4.2|8" + List output4 = processor.transform("3.14159265358979323846|4|4.2|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output4.get(0), "result=0"); + } + + @Test + public void testNotEqualsToOperator() throws Exception { + String transformSql = "select if(string2 != 4,1,0) from source"; + TransformConfig config = new TransformConfig(transformSql); + // case1: "3.14159265358979323846|4a|4|8" + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor.transform("3.14159265358979323846|4a|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=1"); + // case2: "3.14159265358979323846|4|4|8" + List output2 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output2.get(0), "result=0"); + + transformSql = "select if(numeric3 != 4,1,0) from source"; + config = new TransformConfig(transformSql); + // case3: "3.14159265358979323846|4|4|8" + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output3 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output3.get(0), "result=0"); + // case4: "3.14159265358979323846|4|4.2|8" + List output4 = processor.transform("3.14159265358979323846|4|4.2|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output4.get(0), "result=1"); + } + + @Test + public void testGreaterThanEqualsOperator() throws Exception { + String transformSql = "select if(string2 >= 4,1,0) from source"; + TransformConfig config = new TransformConfig(transformSql); + // case1: "3.14159265358979323846|3a|4|8" + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor.transform("3.14159265358979323846|3a|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0"); + // case2: "3.14159265358979323846|5|4|8" + List output2 = processor.transform("3.14159265358979323846|5|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output2.get(0), "result=1"); + + transformSql = "select if(numeric3 >= 4,1,0) from source"; + config = new TransformConfig(transformSql); + // case3: "3.14159265358979323846|4|4|8" + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output3 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output3.get(0), "result=1"); + // case4: "3.14159265358979323846|4|3.2|8" + List output4 = processor.transform("3.14159265358979323846|4|3.2|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output4.get(0), "result=0"); + } + + @Test + public void testGreaterThanOperator() throws Exception { + String transformSql = "select if(string2 > 4.1,1,0) from source"; + TransformConfig config = new TransformConfig(transformSql); + // case1: "3.14159265358979323846|3a|4|8" + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor.transform("3.14159265358979323846|3a|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0"); + // case2: "3.14159265358979323846|5|4|8" + List output2 = processor.transform("3.14159265358979323846|5|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output2.get(0), "result=1"); + + transformSql = "select if(numeric3 > 4.1,1,0) from source"; + config = new TransformConfig(transformSql); + // case3: "3.14159265358979323846|4|4|8" + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output3 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output3.get(0), "result=0"); + // case4: "3.14159265358979323846|4|4.2|8" + List output4 = processor.transform("3.14159265358979323846|4|4.2|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output4.get(0), "result=1"); + } + + @Test + public void testMinorThanEqualsOperator() throws Exception { + String transformSql = "select if(string2 <= 4,1,0) from source"; + TransformConfig config = new TransformConfig(transformSql); + // case1: "3.14159265358979323846|3a|4|8" + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor.transform("3.14159265358979323846|3a|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=1"); + // case2: "3.14159265358979323846|5|4|8" + List output2 = processor.transform("3.14159265358979323846|5|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output2.get(0), "result=0"); + + transformSql = "select if(numeric3 <= 4,1,0) from source"; + config = new TransformConfig(transformSql); + // case3: "3.14159265358979323846|4|4|8" + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output3 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output3.get(0), "result=1"); + // case4: "3.14159265358979323846|4|4.2|8" + List output4 = processor.transform("3.14159265358979323846|4|4.2|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output4.get(0), "result=0"); + } + + @Test + public void testMinorThanOperator() throws Exception { + String transformSql = "select if(string2 < 4.1,1,0) from source"; + TransformConfig config = new TransformConfig(transformSql); + // case1: "3.14159265358979323846|3a|4|8" + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor.transform("3.14159265358979323846|3a|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=1"); + // case2: "3.14159265358979323846|5|4|8" + List output2 = processor.transform("3.14159265358979323846|5|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output2.get(0), "result=0"); + + transformSql = "select if(numeric3 < 4,1,0) from source"; + config = new TransformConfig(transformSql); + // case3: "3.14159265358979323846|4|4|8" + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output3 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output3.get(0), "result=0"); + // case4: "3.14159265358979323846|4|3.2|8" + List output4 = processor.transform("3.14159265358979323846|4|3.2|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output4.get(0), "result=1"); + } + + @Test + public void testNotOperator() throws Exception { + String transformSql = "select if(!(string2 < 4),1,0) from source"; + TransformConfig config = new TransformConfig(transformSql); + // case1: "3.14159265358979323846|3a|4|8" + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor.transform("3.14159265358979323846|3a|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0"); + // case2: "3.14159265358979323846|5|4|8" + List output2 = processor.transform("3.14159265358979323846|5|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output2.get(0), "result=1"); + + transformSql = "select if(!(numeric3 < 3.9),1,0) from source"; + config = new TransformConfig(transformSql); + // case3: "3.14159265358979323846|4|4|8" + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output3 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output3.get(0), "result=1"); + // case4: "3.14159265358979323846|4|3.2|8" + List output4 = processor.transform("3.14159265358979323846|4|3.2|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output4.get(0), "result=0"); + } + + @Test + public void testOrOperator() throws Exception { + String transformSql = "select if((string2 < 4) or (numeric4 > 5),1,0) from source"; + TransformConfig config = new TransformConfig(transformSql); + // case1: "3.14159265358979323846|3a|4|8" + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor.transform("3.14159265358979323846|3a|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=1"); + // case2: "3.14159265358979323846|5|4|8" + List output2 = processor.transform("3.14159265358979323846|5|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output2.get(0), "result=1"); + // case3: "3.14159265358979323846|5|4|4" + List output3 = processor.transform("3.14159265358979323846|5|4|4"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output3.get(0), "result=0"); + + transformSql = "select if((numeric3 < 4) or (numeric4 > 5),1,0) from source"; + config = new TransformConfig(transformSql); + // case4: "3.14159265358979323846|4|4|8" + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output4 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output4.get(0), "result=1"); + // case5: "3.14159265358979323846|4|3.2|8" + List output5 = processor.transform("3.14159265358979323846|4|3.2|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output5.get(0), "result=1"); + // case6: "3.14159265358979323846|4|4.2|5" + List output6 = processor.transform("3.14159265358979323846|4|4.2|5"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output6.get(0), "result=0"); + } + + @Test + public void testAndOperator() throws Exception { + String transformSql = "select if((string2 < 4) and (numeric4 > 5),1,0) from source"; + TransformConfig config = new TransformConfig(transformSql); + // case1: "3.14159265358979323846|3a|4|4" + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor.transform("3.14159265358979323846|3a|4|4"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=0"); + // case2: "3.14159265358979323846|5|4|8" + List output2 = processor.transform("3.14159265358979323846|5|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output2.get(0), "result=0"); + // case3: "3.14159265358979323846|3|4|8" + List output3 = processor.transform("3.14159265358979323846|3|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output3.get(0), "result=1"); + + transformSql = "select if((numeric3 < 4) and (numeric4 > 5),1,0) from source"; + config = new TransformConfig(transformSql); + // case4: "3.14159265358979323846|4|4|8" + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output4 = processor.transform("3.14159265358979323846|4|4|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output4.get(0), "result=0"); + // case5: "3.14159265358979323846|4|3.2|4" + List output5 = processor.transform("3.14159265358979323846|4|3.2|4"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output5.get(0), "result=0"); + // case6: "3.14159265358979323846|4|3.2|8" + List output6 = processor.transform("3.14159265358979323846|4|3.2|8"); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output6.get(0), "result=1"); + } +} diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformFromBase64FunctionProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformFromBase64FunctionProcessor.java new file mode 100644 index 00000000000..9684c5fe7cd --- /dev/null +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformFromBase64FunctionProcessor.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sdk.transform.process; + +import org.apache.inlong.sdk.transform.decode.SourceDecoderFactory; +import org.apache.inlong.sdk.transform.encode.SinkEncoderFactory; +import org.apache.inlong.sdk.transform.pojo.CsvSourceInfo; +import org.apache.inlong.sdk.transform.pojo.FieldInfo; +import org.apache.inlong.sdk.transform.pojo.KvSinkInfo; +import org.apache.inlong.sdk.transform.pojo.TransformConfig; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +/** + * TestTransformFromBase64FunctionProcessor + * description: test the from_base64 function in transform processor + */ +public class TestTransformFromBase64FunctionProcessor { + + private static final List srcFields = new ArrayList<>(); + private static final List dstFields = new ArrayList<>(); + private static final CsvSourceInfo csvSource; + private static final KvSinkInfo kvSink; + + static { + for (int i = 1; i < 4; i++) { + FieldInfo field = new FieldInfo(); + field.setName("string" + i); + srcFields.add(field); + } + for (int i = 1; i < 4; i++) { + FieldInfo field = new FieldInfo(); + field.setName("numeric" + i); + srcFields.add(field); + } + FieldInfo field = new FieldInfo(); + field.setName("result"); + dstFields.add(field); + csvSource = new CsvSourceInfo("UTF-8", '|', '\\', srcFields); + kvSink = new KvSinkInfo("UTF-8", dstFields); + } + @Test + public void testFromBase64Function() throws Exception { + String transformSql = "select from_base64(string1) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + + // case1: from_base64('aGVsbG8gd29ybGQ=') -> 'hello world' + List output1 = processor.transform("aGVsbG8gd29ybGQ=|apple|banana|cloud|1", new HashMap<>()); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=hello world"); + + String transformSql2 = "select from_base64(stringX) from source"; + TransformConfig config2 = new TransformConfig(transformSql2); + TransformProcessor processor2 = TransformProcessor + .create(config2, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + // case2: from_base64(null) -> null + List output2 = processor2.transform("|apple|banana|cloud|1", new HashMap<>()); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=null"); + + // case3: from_base64('QXBhY2hlIEluTG9uZw==') -> 'Apache InLong' + List output3 = processor.transform("QXBhY2hlIEluTG9uZw==|apple|banana|cloud|1", new HashMap<>()); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "result=Apache InLong"); + } +} diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java index 20af097de60..8448260252d 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java @@ -44,13 +44,7 @@ public class TestTransformProcessor { @Test public void testCsv2Kv() throws Exception { - List fields = new ArrayList<>(); - FieldInfo ftime = new FieldInfo(); - ftime.setName("ftime"); - fields.add(ftime); - FieldInfo extinfo = new FieldInfo(); - extinfo.setName("extinfo"); - fields.add(extinfo); + List fields = this.getTestFieldList("ftime", "extinfo"); CsvSourceInfo csvSource = new CsvSourceInfo("UTF-8", '|', '\\', fields); KvSinkInfo kvSink = new KvSinkInfo("UTF-8", fields); String transformSql = "select ftime,extinfo from source where extinfo='ok'"; @@ -97,13 +91,7 @@ public void testCsv2KvNoField() throws Exception { @Test public void testKv2Csv() throws Exception { - List fields = new ArrayList<>(); - FieldInfo ftime = new FieldInfo(); - ftime.setName("ftime"); - fields.add(ftime); - FieldInfo extinfo = new FieldInfo(); - extinfo.setName("extinfo"); - fields.add(extinfo); + List fields = this.getTestFieldList("ftime", "extinfo"); KvSourceInfo kvSource = new KvSourceInfo("UTF-8", fields); CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields); String transformSql = "select ftime,extinfo from source where extinfo='ok'"; @@ -148,7 +136,7 @@ public void testKv2CsvNoField() throws Exception { @Test public void testJson2Csv() throws Exception { - List fields1 = this.getTestFieldList(); + List fields1 = this.getTestFieldList("sid", "packageID", "msgTime", "msg"); JsonSourceInfo jsonSource1 = new JsonSourceInfo("UTF-8", "msgs"); CsvSinkInfo csvSink1 = new CsvSinkInfo("UTF-8", '|', '\\', fields1); String transformSql1 = "select $root.sid,$root.packageID,$child.msgTime,$child.msg from source"; @@ -170,7 +158,7 @@ public void testJson2Csv() throws Exception { Assert.assertEquals(output1.get(0), "value1|value2|1713243918000|value4"); Assert.assertEquals(output1.get(1), "value1|value2|1713243918000|v4"); // case2 - List fields2 = this.getTestFieldList2(); + List fields2 = this.getTestFieldList("id", "itemId", "subItemId", "msg"); JsonSourceInfo jsonSource2 = new JsonSourceInfo("UTF-8", "items"); CsvSinkInfo csvSink2 = new CsvSinkInfo("UTF-8", '|', '\\', fields2); String transformSql2 = @@ -205,7 +193,7 @@ public void testJson2Csv() throws Exception { @Test public void testJson2CsvForOne() throws Exception { - List fields = this.getTestFieldList(); + List fields = this.getTestFieldList("sid", "packageID", "msgTime", "msg"); JsonSourceInfo jsonSource = new JsonSourceInfo("UTF-8", ""); CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields); String transformSql = "select $root.sid,$root.packageID,$root.msgs(1).msgTime,$root.msgs(0).msg from source"; @@ -229,7 +217,7 @@ public void testJson2CsvForOne() throws Exception { @Test public void testPb2Csv() throws Exception { - List fields = this.getTestFieldList(); + List fields = this.getTestFieldList("sid", "packageID", "msgTime", "msg"); String transformBase64 = this.getPbTestDescription(); PbSourceInfo pbSource = new PbSourceInfo("UTF-8", transformBase64, "SdkDataRequest", "msgs"); CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields); @@ -246,36 +234,13 @@ public void testPb2Csv() throws Exception { Assert.assertEquals(output.get(1), "sid|1|1713243918002|msgValue42"); } - private List getTestFieldList() { + private List getTestFieldList(String... fieldNames) { List fields = new ArrayList<>(); - FieldInfo sid = new FieldInfo(); - sid.setName("sid"); - fields.add(sid); - FieldInfo packageID = new FieldInfo(); - packageID.setName("packageID"); - fields.add(packageID); - FieldInfo msgTime = new FieldInfo(); - msgTime.setName("msgTime"); - fields.add(msgTime); - FieldInfo msg = new FieldInfo(); - msg.setName("msg"); - fields.add(msg); - return fields; - } - private List getTestFieldList2() { - List fields = new ArrayList<>(); - FieldInfo id = new FieldInfo(); - id.setName("id"); - fields.add(id); - FieldInfo itemId = new FieldInfo(); - itemId.setName("itemId"); - fields.add(itemId); - FieldInfo subItemId = new FieldInfo(); - subItemId.setName("subItemId"); - fields.add(subItemId); - FieldInfo msg = new FieldInfo(); - msg.setName("msg"); - fields.add(msg); + for (String fieldName : fieldNames) { + FieldInfo field = new FieldInfo(); + field.setName(fieldName); + fields.add(field); + } return fields; } @@ -309,7 +274,7 @@ private String getPbTestDescription() { @Test public void testPb2CsvForOne() throws Exception { - List fields = this.getTestFieldList(); + List fields = this.getTestFieldList("sid", "packageID", "msgTime", "msg"); String transformBase64 = this.getPbTestDescription(); PbSourceInfo pbSource = new PbSourceInfo("UTF-8", transformBase64, "SdkDataRequest", null); CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields); @@ -327,7 +292,7 @@ public void testPb2CsvForOne() throws Exception { @Test public void testPb2CsvForAdd() throws Exception { - List fields = this.getTestFieldList(); + List fields = this.getTestFieldList("sid", "packageID", "msgTime", "msg"); String transformBase64 = this.getPbTestDescription(); PbSourceInfo pbSource = new PbSourceInfo("UTF-8", transformBase64, "SdkDataRequest", null); CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields); @@ -351,7 +316,7 @@ public void testPb2CsvForAdd() throws Exception { @Test public void testPb2CsvForConcat() throws Exception { - List fields = this.getTestFieldList(); + List fields = this.getTestFieldList("sid", "packageID", "msgTime", "msg"); String transformBase64 = this.getPbTestDescription(); PbSourceInfo pbSource = new PbSourceInfo("UTF-8", transformBase64, "SdkDataRequest", "msgs"); CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields); @@ -371,7 +336,7 @@ public void testPb2CsvForConcat() throws Exception { @Test public void testPb2CsvForNow() throws Exception { - List fields = this.getTestFieldList(); + List fields = this.getTestFieldList("sid", "packageID", "msgTime", "msg"); String transformBase64 = this.getPbTestDescription(); PbSourceInfo pbSource = new PbSourceInfo("UTF-8", transformBase64, "SdkDataRequest", "msgs"); CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields); @@ -385,4 +350,80 @@ public void testPb2CsvForNow() throws Exception { List output = processor.transform(srcBytes, new HashMap<>()); Assert.assertEquals(2, output.size()); } + @Test + public void testCsv2Star() throws Exception { + List fields = this.getTestFieldList("ftime", "extinfo"); + CsvSourceInfo csvSource = new CsvSourceInfo("UTF-8", '|', '\\', fields); + CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', new ArrayList<>()); + String transformSql = "select *"; + TransformConfig config = new TransformConfig(transformSql); + // case1 + TransformProcessor processor1 = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createCsvEncoder(csvSink)); + + List output1 = processor1.transform("2024-04-28 00:00:00|ok", new HashMap<>()); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "2024-04-28 00:00:00|ok"); + // case2 + config.setTransformSql("select * from source where extinfo!='ok'"); + TransformProcessor processor2 = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createCsvEncoder(csvSink)); + + List output2 = processor2.transform("2024-04-28 00:00:00|ok", new HashMap<>()); + Assert.assertEquals(0, output2.size()); + // case3 + config.setTransformSql("select *,extinfo,ftime from source where extinfo!='ok'"); + TransformProcessor processor3 = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createCsvEncoder(csvSink)); + + List output3 = processor3.transform("2024-04-28 00:00:00|nok", new HashMap<>()); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "2024-04-28 00:00:00|nok|nok|2024-04-28 00:00:00"); + // case4 + CsvSourceInfo csvSourceNoField = new CsvSourceInfo("UTF-8", '|', '\\', new ArrayList<>()); + CsvSinkInfo csvSinkNoField = new CsvSinkInfo("UTF-8", '|', '\\', new ArrayList<>()); + config.setTransformSql("select *,$2,$1 from source where $2='nok'"); + TransformProcessor processor4 = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSourceNoField), + SinkEncoderFactory.createCsvEncoder(csvSinkNoField)); + + List output4 = processor4.transform("2024-04-28 00:00:00|nok", new HashMap<>()); + Assert.assertEquals(1, output4.size()); + Assert.assertEquals(output4.get(0), "2024-04-28 00:00:00|nok|nok|2024-04-28 00:00:00"); + } + + @Test + public void testKv2Star() throws Exception { + List fields = this.getTestFieldList("ftime", "extinfo"); + KvSourceInfo kvSource = new KvSourceInfo("UTF-8", fields); + KvSinkInfo kvSink = new KvSinkInfo("UTF-8", new ArrayList<>()); + String transformSql = "select *"; + TransformConfig config = new TransformConfig(transformSql); + // case1 + TransformProcessor processor1 = TransformProcessor + .create(config, SourceDecoderFactory.createKvDecoder(kvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output1 = processor1.transform("ftime=2024-04-28 00:00:00&extinfo=ok", new HashMap<>()); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "ftime=2024-04-28 00:00:00&extinfo=ok"); + // case2 + config.setTransformSql("select * from source where extinfo!='ok'"); + TransformProcessor processor2 = TransformProcessor + .create(config, SourceDecoderFactory.createKvDecoder(kvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + List output2 = processor2.transform("ftime=2024-04-28 00:00:00&extinfo=ok", new HashMap<>()); + Assert.assertEquals(0, output2.size()); + // case3 + config.setTransformSql("select *,extinfo e1,ftime f1 from source where extinfo!='ok'"); + TransformProcessor processor3 = TransformProcessor + .create(config, SourceDecoderFactory.createKvDecoder(kvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + + List output3 = processor3.transform("ftime=2024-04-28 00:00:00&extinfo=nok", new HashMap<>()); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "ftime=2024-04-28 00:00:00&extinfo=nok&e1=nok&f1=2024-04-28 00:00:00"); + } } diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java index 10cfa740c97..f099d728dc7 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java @@ -428,6 +428,146 @@ public void testStrcmpFunction() throws Exception { Assert.assertEquals("result=null", output.get(0)); } + @Test + public void testRpadFunction() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + transformSql = "select rpad(string1,numeric1,string2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + // case1: rpad('he',7,'xxd') + data = "he|xxd|cloud|7|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=hexxdxx", output.get(0)); + + // case2: rpad('he',1,'xxd') + data = "he|xxd|cloud|1|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=h", output.get(0)); + + // case3: rpad('he',1,'') + data = "he||cloud|1|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=h", output.get(0)); + + // case4: rpad('he',-1,'xxd') + data = "he|xxd|cloud|-1|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case5: rpad(null,5,'xxd') + transformSql = "select rpad(xxd,numeric1,string2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "he|xxd|cloud|5|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case6: rpad('he',null,'xxd') + transformSql = "select rpad(string1,xxd,string2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "he|xxd|cloud|5|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case7: rpad('he',5,null) + transformSql = "select rpad(string1,numeric1,xxd) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "he|xxd|cloud|5|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + } + + @Test + public void testLpadFunction() throws Exception { + String transformSql = null, data = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + transformSql = "select lpad(string1,numeric1,string2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + // case1: lpad('he',7,'xxd') + data = "he|xxd|cloud|7|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=xxdxxhe", output.get(0)); + + // case2: lpad('he',1,'xxd') + data = "he|xxd|cloud|1|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=h", output.get(0)); + + // case3: lpad('he',1,'') + data = "he||cloud|1|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=h", output.get(0)); + + // case4: lpad('he',-1,'xxd') + data = "he|xxd|cloud|-1|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case5: lpad(null,5,'xxd') + transformSql = "select lpad(xxd,numeric1,string2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "he|xxd|cloud|5|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case6: lpad('he',null,'xxd') + transformSql = "select lpad(string1,xxd,string2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "he|xxd|cloud|5|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case7: lpad('he',5,null) + transformSql = "select lpad(string1,numeric1,xxd) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + data = "he|xxd|cloud|5|3|3"; + output = processor.transform(data, new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + } + @Test public void testRightFunction() throws Exception { String transformSql = "select right(string1,numeric1) from source"; @@ -550,4 +690,70 @@ public void testTranslateFunction() throws Exception { Assert.assertEquals(output3.get(0), "result=Apache Inlong"); } + @Test + public void testInsertFunction() throws Exception { + String transformSql1 = "select insert(string1, numeric1, numeric2, string2) from source"; + TransformConfig config1 = new TransformConfig(transformSql1); + TransformProcessor processor1 = TransformProcessor + .create(config1, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + + // case1: insert('12345678', 3, 4, 'word') -> '12word78' + List output1 = processor1.transform("12345678|word|cloud|3|4|0", new HashMap<>()); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals("result=12word78", output1.get(0)); + + // case2: insert('12345678', -1, 4, 'word') -> '12345678' + List output2 = processor1.transform("12345678|word|cloud|-1|4|0", new HashMap<>()); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals("result=12345678", output2.get(0)); + + // case3: insert('12345678', 3, 100, 'word') -> '12word' + List output3 = processor1.transform("12345678|word|cloud|3|100|0", new HashMap<>()); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals("result=12word", output3.get(0)); + + // case4: insert('', 3, 4, 'word') -> '' + List output4 = processor1.transform("|word|cloud|3|4|0", new HashMap<>()); + Assert.assertEquals(1, output4.size()); + Assert.assertEquals("result=", output4.get(0)); + + // case5: insert('12345678', 3, 4, '') -> '1278' + List output5 = processor1.transform("12345678||cloud|3|4|0", new HashMap<>()); + Assert.assertEquals(1, output5.size()); + Assert.assertEquals("result=1278", output5.get(0)); + } + + @Test + public void testContainsFunction() throws Exception { + String transformSql = "select contains(string1, string2) from source"; + TransformConfig config = new TransformConfig(transformSql); + TransformProcessor processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + // case1: contains('Transform SQL', 'SQL') + List output1 = processor.transform("Transform SQL|SQL", new HashMap<>()); + Assert.assertEquals(1, output1.size()); + Assert.assertEquals(output1.get(0), "result=true"); + // case2: contains('', 'SQL') + List output2 = processor.transform("|SQL", new HashMap<>()); + Assert.assertEquals(1, output2.size()); + Assert.assertEquals(output2.get(0), "result=false"); + // case3: contains('Transform SQL', '') + List output3 = processor.transform("Transform SQL|", new HashMap<>()); + Assert.assertEquals(1, output3.size()); + Assert.assertEquals(output3.get(0), "result=true"); + // case4: contains('Transform SQL', 'Transformer') + List output4 = processor.transform("Transform SQL|Transformer", new HashMap<>()); + Assert.assertEquals(1, output4.size()); + Assert.assertEquals(output4.get(0), "result=false"); + // case5: contains('Transform SQL', 'm S') + List output5 = processor.transform("Transform SQL|m S", new HashMap<>()); + Assert.assertEquals(1, output5.size()); + Assert.assertEquals(output5.get(0), "result=true"); + // case6: contains('', '') + List output6 = processor.transform("|", new HashMap<>()); + Assert.assertEquals(1, output6.size()); + Assert.assertEquals(output6.get(0), "result=true"); + } } diff --git a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java index 354b8b7f185..14c1fab04ba 100644 --- a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java +++ b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java @@ -216,6 +216,16 @@ public void testDateExtractFunction() throws Exception { List output7 = processor7.transform("2024-02-29", new HashMap<>()); Assert.assertEquals(1, output7.size()); Assert.assertEquals(output7.get(0), "result=5"); + + String transformSql8 = "select dayname(string1) from source"; + TransformConfig config8 = new TransformConfig(transformSql8); + TransformProcessor processor8 = TransformProcessor + .create(config8, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + // case8: dayname(2024-02-29) (Thursday) + List output8 = processor8.transform("2024-02-29", new HashMap<>()); + Assert.assertEquals(1, output8.size()); + Assert.assertEquals(output8.get(0), "result=THURSDAY"); } @Test @@ -379,6 +389,54 @@ public void testToTimestampFunction() throws Exception { Assert.assertEquals(output4.get(0), "result=1970-01-01 00:00:00.0"); } + @Test + public void testDateDiffFunction() throws Exception { + String transformSql = null; + TransformConfig config = null; + TransformProcessor processor = null; + List output = null; + + transformSql = "select datediff(string1,string2) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + // case1: datediff('1970-01-01','1970-01-02') + output = processor.transform("1970-01-01|1970-01-02", new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=-1", output.get(0)); + + // case2: datediff('1970-01-02','1970-01-01') + output = processor.transform("1970-01-02|1970-01-01", new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=1", output.get(0)); + + // case3: datediff('2018-12-10 12:30:00', '2018-12-09 13:30:00') + output = processor.transform("2018-12-10 12:30:00|2018-12-09 13:30:00", new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=1", output.get(0)); + + // case4: datediff('2018-12-10 12:30:00', '') + output = processor.transform("2018-12-10 12:30:00|", new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case5: datediff('2018-12', '2018-12-12') + output = processor.transform("2018-12|2018-12-12", new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + + // case6: datediff('1970-01-01',null) + transformSql = "select datediff(string1,xxd) from source"; + config = new TransformConfig(transformSql); + processor = TransformProcessor + .create(config, SourceDecoderFactory.createCsvDecoder(csvSource), + SinkEncoderFactory.createKvEncoder(kvSink)); + output = processor.transform("1970-01-01|1970-01-02", new HashMap<>()); + Assert.assertEquals(1, output.size()); + Assert.assertEquals("result=null", output.get(0)); + } + @Test public void testLocalTimeFunction() throws Exception { String transformSql1 = "select localtime() from source"; diff --git a/inlong-sort/sort-core/pom.xml b/inlong-sort/sort-core/pom.xml index 2ef9506520a..e4881e6043c 100644 --- a/inlong-sort/sort-core/pom.xml +++ b/inlong-sort/sort-core/pom.xml @@ -72,18 +72,6 @@ ${mysql.jdbc.version} provided - - org.apache.inlong - sort-format-inlongmsg-base - ${project.version} - test - - - org.apache.inlong - sort-format-csv - ${project.version} - test - org.apache.hadoop hadoop-common @@ -106,6 +94,18 @@ flink-table-common provided + + org.apache.inlong + sort-format-csv + ${project.version} + test + + + org.apache.inlong + sort-format-inlongmsg-base + ${project.version} + test + org.apache.inlong sort-flink-dependencies-${sort.flink.version} @@ -251,6 +251,18 @@ flink-table-common provided + + org.apache.inlong + sort-format-csv + ${project.version} + test + + + org.apache.inlong + sort-format-inlongmsg-base + ${project.version} + test + org.apache.inlong sort-flink-dependencies-${sort.flink.version} @@ -372,6 +384,24 @@ ${project.version} test + + org.apache.inlong + sort-connector-jdbc-v1.18 + ${project.version} + test + + + org.apache.inlong + sort-connector-elasticsearch6-v1.18 + ${project.version} + test + + + org.apache.inlong + sort-connector-elasticsearch7-v1.18 + ${project.version} + test + diff --git a/inlong-sort/sort-dist/pom.xml b/inlong-sort/sort-dist/pom.xml index 0f28f495bf6..4541bb37eab 100644 --- a/inlong-sort/sort-dist/pom.xml +++ b/inlong-sort/sort-dist/pom.xml @@ -55,31 +55,6 @@ sort-format-common ${project.version} - - org.apache.inlong - sort-format-base - ${project.version} - - - org.apache.inlong - sort-format-csv - ${project.version} - - - org.apache.inlong - sort-format-inlongmsg-base - ${project.version} - - - org.apache.inlong - sort-format-inlongmsg-csv - ${project.version} - - - org.apache.inlong - sort-format-inlongmsg-kv - ${project.version} - org.apache.inlong sort-format-rowdata-kv @@ -134,6 +109,31 @@ sort-format-json-v1.13 ${project.version} + + org.apache.inlong + sort-format-csv + ${project.version} + + + org.apache.inlong + sort-format-inlongmsg-base + ${project.version} + + + org.apache.inlong + sort-format-inlongmsg-csv + ${project.version} + + + org.apache.inlong + sort-format-inlongmsg-kv + ${project.version} + + + org.apache.inlong + sort-format-base + ${project.version} + org.apache.flink flink-sql-parquet_${scala.binary.version} @@ -172,6 +172,31 @@ sort-format-json-v1.15 ${project.version} + + org.apache.inlong + sort-format-csv + ${project.version} + + + org.apache.inlong + sort-format-inlongmsg-base + ${project.version} + + + org.apache.inlong + sort-format-inlongmsg-csv + ${project.version} + + + org.apache.inlong + sort-format-inlongmsg-kv + ${project.version} + + + org.apache.inlong + sort-format-base + ${project.version} + org.apache.flink flink-sql-parquet diff --git a/inlong-sort/sort-end-to-end-tests/pom.xml b/inlong-sort/sort-end-to-end-tests/pom.xml index 04b87c02826..6c6319cd4e8 100644 --- a/inlong-sort/sort-end-to-end-tests/pom.xml +++ b/inlong-sort/sort-end-to-end-tests/pom.xml @@ -52,6 +52,15 @@ sort-end-to-end-tests-v1.15 + + v1.18 + + true + + + sort-end-to-end-tests-v1.18 + + diff --git a/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/pom.xml b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/pom.xml new file mode 100644 index 00000000000..22c8e6fc6bd --- /dev/null +++ b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/pom.xml @@ -0,0 +1,209 @@ + + + + 4.0.0 + + org.apache.inlong + sort-end-to-end-tests + 1.14.0-SNAPSHOT + + + sort-end-to-end-tests-v1.18 + Apache InLong - Sort End to End Tests v1.18 + + + ${project.parent.parent.parent.basedir} + 1.18.1 + 6.8.17 + 2.15.3-18.0 + + + + + org.apache.inlong + sort-dist + ${project.version} + test + + + org.testcontainers + testcontainers + + + org.testcontainers + postgresql + ${testcontainers.version} + + + org.postgresql + postgresql + test + + + org.testcontainers + elasticsearch + ${testcontainers.version} + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + + org.elasticsearch.client + elasticsearch-rest-client + ${elasticsearch.version} + + + + org.apache.flink + flink-shaded-jackson + ${flink.shaded.jackson.version} + + + org.apache.flink + flink-test-utils + ${flink.version} + test + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.logging.log4j + log4j-core + + + + + org.apache.inlong + sort-flink-dependencies-v1.18 + ${project.version} + test + + + org.apache.flink + flink-core + ${flink.version} + test + + + org.apache.flink + flink-json + ${flink.version} + test + + + org.apache.flink + flink-avro + ${flink.version} + test + + + org.apache.flink + flink-csv + ${flink.version} + test + + + org.apache.flink + flink-sql-avro + ${flink.version} + test + + + org.apache.flink + flink-runtime + ${flink.version} + test + + + org.apache.flink + flink-table-common + ${flink.version} + test + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + org.apache.inlong + sort-dist + ${project.version} + sort-dist.jar + jar + ${project.build.directory}/dependencies + + + + + + copy-jars + + copy + + validate + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + end-to-end-tests-v1.18 + integration-test + + + **/*.* + + 1 + + ${project.basedir} + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${plugin.surefire.version} + + + + diff --git a/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnv.java b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnv.java new file mode 100644 index 00000000000..de6166442ea --- /dev/null +++ b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnv.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.tests.utils; + +import org.apache.commons.io.IOUtils; +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.client.deployment.StandaloneClusterId; +import org.apache.flink.client.program.rest.RestClusterClient; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestOptions; +import org.apache.flink.runtime.client.JobStatusMessage; +import org.apache.flink.runtime.jobmaster.JobMaster; +import org.apache.flink.runtime.taskexecutor.TaskExecutor; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.util.TestLogger; +import org.junit.AfterClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container.ExecResult; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.Transferable; + +import javax.annotation.Nullable; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; + +import static org.apache.flink.util.Preconditions.checkState; + +/** + * End to end base test environment for test sort-connectors. + * Every link : MySQL -> Xxx (Test connector) -> MySQL + */ +public abstract class FlinkContainerTestEnv extends TestLogger { + + static final Logger JM_LOG = LoggerFactory.getLogger(JobMaster.class); + static final Logger TM_LOG = LoggerFactory.getLogger(TaskExecutor.class); + static final Logger LOG = LoggerFactory.getLogger(FlinkContainerTestEnv.class); + + private static final Path SORT_DIST_JAR = TestUtils.getResource("sort-dist.jar"); + // ------------------------------------------------------------------------------------------ + // Flink Variables + // ------------------------------------------------------------------------------------------ + static final int JOB_MANAGER_REST_PORT = 8081; + static final int DEBUG_PORT = 20000; + static final String FLINK_BIN = "bin"; + static final String INTER_CONTAINER_JM_ALIAS = "jobmanager"; + static final String INTER_CONTAINER_TM_ALIAS = "taskmanager"; + static final String FLINK_PROPERTIES = String.join("\n", Arrays.asList( + "jobmanager.rpc.address: jobmanager", + "taskmanager.numberOfTaskSlots: 10", + "parallelism.default: 4", + "env.java.opts.jobmanager: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=20000", + "env.java.opts.taskmanager: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=20000", + // this is needed for oracle-cdc tests. + // see https://stackoverflow.com/a/47062742/4915129 + "env.java.opts: -Doracle.jdbc.timezoneAsRegion=false")); + + @ClassRule + public static final Network NETWORK = Network.newNetwork(); + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Nullable + private static RestClusterClient restClusterClient; + + static GenericContainer jobManager; + static GenericContainer taskManager; + + @AfterClass + public static void after() { + if (restClusterClient != null) { + restClusterClient.close(); + } + if (jobManager != null) { + jobManager.stop(); + } + if (taskManager != null) { + taskManager.stop(); + } + } + + /** + * Submits a SQL job to the running cluster. + * + *

NOTE: You should not use {@code '\t'}. + */ + public void submitSQLJob(String sqlFile, Path... jars) + throws IOException, InterruptedException { + final List commands = new ArrayList<>(); + String containerSqlFile = copyToContainerTmpPath(jobManager, sqlFile); + commands.add(FLINK_BIN + "/flink run -d"); + commands.add("-c org.apache.inlong.sort.Entrance"); + commands.add(copyToContainerTmpPath(jobManager, constructDistJar(jars))); + commands.add("--sql.script.file"); + commands.add(containerSqlFile); + + ExecResult execResult = + jobManager.execInContainer("bash", "-c", String.join(" ", commands)); + LOG.info(execResult.getStdout()); + if (execResult.getExitCode() != 0) { + LOG.error(execResult.getStderr()); + throw new AssertionError("Failed when submitting the SQL job."); + } + } + + /** + * Get {@link RestClusterClient} connected to this FlinkContainer. + * + *

This method lazily initializes the REST client on-demand. + */ + public RestClusterClient getRestClusterClient() { + checkState( + jobManager.isRunning(), + "Cluster client should only be retrieved for a running cluster"); + try { + final Configuration clientConfiguration = new Configuration(); + clientConfiguration.set(RestOptions.ADDRESS, jobManager.getHost()); + clientConfiguration.set( + RestOptions.PORT, jobManager.getMappedPort(JOB_MANAGER_REST_PORT)); + this.restClusterClient = + new RestClusterClient<>(clientConfiguration, StandaloneClusterId.getInstance()); + } catch (Exception e) { + throw new IllegalStateException( + "Failed to create client for Flink container cluster", e); + } + return restClusterClient; + } + + /** + * Polling to detect task status until the task successfully into {@link JobStatus.RUNNING} + * + * @param timeout + */ + public void waitUntilJobRunning(Duration timeout) { + RestClusterClient clusterClient = getRestClusterClient(); + Deadline deadline = Deadline.fromNow(timeout); + while (deadline.hasTimeLeft()) { + Collection jobStatusMessages; + try { + jobStatusMessages = clusterClient.listJobs().get(10, TimeUnit.SECONDS); + } catch (Exception e) { + LOG.warn("Error when fetching job status.", e); + continue; + } + if (jobStatusMessages != null && !jobStatusMessages.isEmpty()) { + JobStatusMessage message = jobStatusMessages.iterator().next(); + JobStatus jobStatus = message.getJobState(); + if (jobStatus.isTerminalState()) { + throw new ValidationException( + String.format( + "Job has been terminated! JobName: %s, JobID: %s, Status: %s", + message.getJobName(), + message.getJobId(), + message.getJobState())); + } else if (jobStatus == JobStatus.RUNNING) { + return; + } + } + } + } + + /** + * Copy all other dependencies into user jar 'lib/' entry. + * Flink per-job mode only support upload one jar to cluster. + */ + private String constructDistJar(Path... jars) throws IOException { + + File newJar = temporaryFolder.newFile("sort-dist.jar"); + try ( + JarFile jarFile = new JarFile(SORT_DIST_JAR.toFile()); + JarOutputStream jos = new JarOutputStream(new FileOutputStream(newJar))) { + jarFile.stream().forEach(entry -> { + try (InputStream is = jarFile.getInputStream(entry)) { + jos.putNextEntry(entry); + jos.write(IOUtils.toByteArray(is)); + jos.closeEntry(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + for (Path jar : jars) { + try (InputStream is = new FileInputStream(jar.toFile())) { + jos.putNextEntry(new JarEntry("lib/" + jar.getFileName().toString())); + jos.write(IOUtils.toByteArray(is)); + jos.closeEntry(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + } + return newJar.getAbsolutePath(); + } + + // Should not a big file, all file data will load into memory, then copy to container. + private String copyToContainerTmpPath(GenericContainer container, String filePath) throws IOException { + Path path = Paths.get(filePath); + byte[] fileData = Files.readAllBytes(path); + String containerPath = "/tmp/" + path.getFileName(); + container.copyFileToContainer(Transferable.of(fileData), containerPath); + return containerPath; + } +} diff --git a/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE11.java b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE11.java new file mode 100644 index 00000000000..9033740822f --- /dev/null +++ b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE11.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.tests.utils; + +import org.junit.BeforeClass; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; + +import java.util.stream.Stream; + +public abstract class FlinkContainerTestEnvJRE11 extends FlinkContainerTestEnv { + + @BeforeClass + public static void before() { + LOG.info("Starting containers..."); + jobManager = + new GenericContainer<>("flink:1.18.1-scala_2.12") + .withCommand("jobmanager") + .withNetwork(NETWORK) + .withNetworkAliases(INTER_CONTAINER_JM_ALIAS) + .withExposedPorts(JOB_MANAGER_REST_PORT, DEBUG_PORT) + .withEnv("FLINK_PROPERTIES", FLINK_PROPERTIES) + .withExposedPorts(JOB_MANAGER_REST_PORT) + .withLogConsumer(new Slf4jLogConsumer(JM_LOG)); + taskManager = + new GenericContainer<>("flink:1.18.1-scala_2.12") + .withCommand("taskmanager") + .withNetwork(NETWORK) + .withNetworkAliases(INTER_CONTAINER_TM_ALIAS) + .withExposedPorts(DEBUG_PORT) + .withEnv("FLINK_PROPERTIES", FLINK_PROPERTIES) + .dependsOn(jobManager) + .withLogConsumer(new Slf4jLogConsumer(TM_LOG)); + + Startables.deepStart(Stream.of(jobManager)).join(); + Startables.deepStart(Stream.of(taskManager)).join(); + LOG.info("Containers are started."); + } +} diff --git a/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE8.java b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE8.java new file mode 100644 index 00000000000..de982da4ba0 --- /dev/null +++ b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/FlinkContainerTestEnvJRE8.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.tests.utils; + +import org.junit.BeforeClass; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; + +import java.util.stream.Stream; + +public abstract class FlinkContainerTestEnvJRE8 extends FlinkContainerTestEnv { + + @BeforeClass + public static void before() { + LOG.info("Starting containers..."); + jobManager = + new GenericContainer<>("flink:1.18.1-scala_2.12-java8") + .withCommand("jobmanager") + .withNetwork(NETWORK) + .withNetworkAliases(INTER_CONTAINER_JM_ALIAS) + .withExposedPorts(JOB_MANAGER_REST_PORT, DEBUG_PORT) + .withEnv("FLINK_PROPERTIES", FLINK_PROPERTIES) + .withExposedPorts(JOB_MANAGER_REST_PORT) + .withLogConsumer(new Slf4jLogConsumer(JM_LOG)); + taskManager = + new GenericContainer<>("flink:1.18.1-scala_2.12-java8") + .withCommand("taskmanager") + .withNetwork(NETWORK) + .withNetworkAliases(INTER_CONTAINER_TM_ALIAS) + .withExposedPorts(DEBUG_PORT) + .withEnv("FLINK_PROPERTIES", FLINK_PROPERTIES) + .dependsOn(jobManager) + .withLogConsumer(new Slf4jLogConsumer(TM_LOG)); + + Startables.deepStart(Stream.of(jobManager)).join(); + Startables.deepStart(Stream.of(taskManager)).join(); + LOG.info("Containers are started."); + } +} diff --git a/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/PlaceholderResolver.java b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/PlaceholderResolver.java new file mode 100644 index 00000000000..0c283336999 --- /dev/null +++ b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/PlaceholderResolver.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.tests.utils; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * A file placeholder replacement tool. + */ +public class PlaceholderResolver { + + /** + * Default placeholder prefix + */ + public static final String DEFAULT_PLACEHOLDER_PREFIX = "${"; + + /** + * Default placeholder suffix + */ + public static final String DEFAULT_PLACEHOLDER_SUFFIX = "}"; + + /** + * Default singleton resolver + */ + private static PlaceholderResolver defaultResolver = new PlaceholderResolver(); + + /** + * Placeholder prefix + */ + private String placeholderPrefix = DEFAULT_PLACEHOLDER_PREFIX; + + /** + * Placeholder suffix + */ + private String placeholderSuffix = DEFAULT_PLACEHOLDER_SUFFIX; + + private PlaceholderResolver() { + + } + + private PlaceholderResolver(String placeholderPrefix, String placeholderSuffix) { + this.placeholderPrefix = placeholderPrefix; + this.placeholderSuffix = placeholderSuffix; + } + + public static PlaceholderResolver getDefaultResolver() { + return defaultResolver; + } + + public static PlaceholderResolver getResolver(String placeholderPrefix, String placeholderSuffix) { + return new PlaceholderResolver(placeholderPrefix, placeholderSuffix); + } + + /** + * Replace template string with special placeholder according to replace function. + * @param content template string with special placeholder + * @param rule placeholder replacement rule + * @return new replaced string + */ + public String resolveByRule(String content, Function rule) { + int start = content.indexOf(this.placeholderPrefix); + if (start == -1) { + return content; + } + StringBuilder result = new StringBuilder(content); + while (start != -1) { + int end = result.indexOf(this.placeholderSuffix, start); + // get placeholder actual value (e.g. ${id}, get the value represent id) + String placeholder = result.substring(start + this.placeholderPrefix.length(), end); + // replace placeholder value + String replaceContent = placeholder.trim().isEmpty() ? "" : rule.apply(placeholder); + result.replace(start, end + this.placeholderSuffix.length(), replaceContent); + start = result.indexOf(this.placeholderPrefix, start + replaceContent.length()); + } + return result.toString(); + } + + /** + * Replace template string with special placeholder according to replace function. + * @param file template file with special placeholder + * @param rule placeholder replacement rule + * @return new replaced string + */ + public Path resolveByRule(Path file, Function rule) { + try { + List newContents = Files.readAllLines(file, StandardCharsets.UTF_8) + .stream() + .map(content -> resolveByRule(content, rule)) + .collect(Collectors.toList()); + Path newPath = Paths.get(file.getParent().toString(), file.getFileName() + "$"); + Files.write(newPath, String.join(System.lineSeparator(), newContents).getBytes(StandardCharsets.UTF_8)); + return newPath; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Replace template string with special placeholder according to properties file. + * Key is the content of the placeholder

+ * e.g: content = product:${id}:detail:${did}
+ * valueMap = id -> 1; pid -> 2
+ * return: product:1:detail:2
+ * + * @param content template string with special placeholder + * @param valueMap placeholder replacement map + * @return new replaced string + */ + public String resolveByMap(String content, final Map valueMap) { + return resolveByRule(content, placeholderValue -> String.valueOf(valueMap.get(placeholderValue))); + } + + /** + * Replace template string with special placeholder according to properties file. + * Key is the content of the placeholder

+ * e.g: content = product:${id}:detail:${did}
+ * valueMap = id -> 1; pid -> 2
+ * return: product:1:detail:2
+ * + * @param file template string with special placeholder + * @param valueMap placeholder replacement map + * @return new replaced string + */ + public Path resolveByMap(Path file, final Map valueMap) { + return resolveByRule(file, placeholderValue -> String.valueOf(valueMap.get(placeholderValue))); + } +} diff --git a/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/TestUtils.java b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/TestUtils.java new file mode 100644 index 00000000000..8daff533da2 --- /dev/null +++ b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/java/org/apache/inlong/sort/tests/utils/TestUtils.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.sort.tests.utils; + +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; + +/** + * Test util for test container. + */ +public class TestUtils { + + private static final ParameterProperty MODULE_DIRECTORY = + new ParameterProperty<>("moduleDir", Paths::get); + + /** + * Searches for a resource file matching the given regex in the given directory. This method is + * primarily intended to be used for the initialization of static {@link Path} fields for + * resource file(i.e. jar, config file) that reside in the modules {@code target} directory. + * + * @param resourceNameRegex regex pattern to match against + * @return Path pointing to the matching jar + * @throws RuntimeException if none or multiple resource files could be found + */ + public static Path getResource(final String resourceNameRegex) { + // if the property is not set then we are most likely running in the IDE, where the working + // directory is the + // module of the test that is currently running, which is exactly what we want + Path moduleDirectory = MODULE_DIRECTORY.get(Paths.get("").toAbsolutePath()); + + try (Stream dependencyResources = Files.walk(moduleDirectory)) { + final List matchingResources = + dependencyResources + .filter( + jar -> Pattern.compile(resourceNameRegex) + .matcher(jar.toAbsolutePath().toString()) + .find()) + .collect(Collectors.toList()); + switch (matchingResources.size()) { + case 0: + throw new RuntimeException( + new FileNotFoundException( + String.format( + "No resource file could be found that matches the pattern %s. " + + "This could mean that the test module must be rebuilt via maven.", + resourceNameRegex))); + case 1: + return matchingResources.get(0); + default: + throw new RuntimeException( + new IOException( + String.format( + "Multiple resource files were found matching the pattern %s. Matches=%s", + resourceNameRegex, matchingResources))); + } + } catch (final IOException ioe) { + throw new RuntimeException("Could not search for resource resource files.", ioe); + } + } + + /** + * A simple system properties value getter with default value when could not find the system property. + * @param + */ + static class ParameterProperty { + + private final String propertyName; + private final Function converter; + + public ParameterProperty(final String propertyName, final Function converter) { + this.propertyName = propertyName; + this.converter = converter; + } + + /** + * Retrieves the value of this property, or the given default if no value was set. + * + * @return the value of this property, or the given default if no value was set + */ + public V get(final V defaultValue) { + final String value = System.getProperty(propertyName); + return value == null ? defaultValue : converter.apply(value); + } + } + + @Test + public void testReplaceholder() { + String before = "today is ${date}, today weather is ${weather}"; + Map maps = new HashMap<>(); + maps.put("date", "2024.07.15"); + maps.put("weather", "song"); + String after = PlaceholderResolver.getDefaultResolver().resolveByMap(before, maps); + assertEquals(after, "today is 2024.07.15, today weather is song"); + } +} diff --git a/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/resources/log4j2-test.properties b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/resources/log4j2-test.properties new file mode 100644 index 00000000000..3e95477751b --- /dev/null +++ b/inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.18/src/test/resources/log4j2-test.properties @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +rootLogger=INFO, STDOUT + +appender.console.type=Console +appender.console.name=STDOUT +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=%-4r [%t] %-5p %c %x - %m%n + +appender.jm.type = File +appender.jm.name = jobmanager +appender.jm.fileName = target/logs/jobmanager.log +appender.jm.layout.type = PatternLayout +appender.jm.layout.pattern = - %m%n + +appender.tm.type = File +appender.tm.name = taskmanager +appender.tm.fileName = target/logs/taskmanager.log +appender.tm.layout.type = PatternLayout +appender.tm.layout.pattern = - %m%n + +logger.jm=INFO, jobmanager +logger.jm.name=org.apache.flink.runtime.jobmaster.JobMaster +logger.jm.additivity=false + +logger.tm=INFO, taskmanager +logger.tm.name=org.apache.flink.runtime.taskexecutor.TaskExecutor +logger.tm.additivity=false + + + diff --git a/inlong-sort/sort-formats/pom.xml b/inlong-sort/sort-formats/pom.xml index e36378306f4..3c392001f3e 100644 --- a/inlong-sort/sort-formats/pom.xml +++ b/inlong-sort/sort-formats/pom.xml @@ -249,7 +249,8 @@ v1.18 format-common - format-row/format-json-v1.18 + format-row + format-rowdata From 951651decff795b1129b1a1c53cd56f37fd0a9bd Mon Sep 17 00:00:00 2001 From: Zhang Tianrui <15903703590@163.com> Date: Thu, 5 Sep 2024 17:57:53 +0800 Subject: [PATCH 23/23] [INLONG-11002][SDK]Transform SQL support Fibonacci function --- .../inlong/sdk/transform/process/operator/OperatorTools.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java index ae5985cff6b..59b10bdb485 100644 --- a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java +++ b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java @@ -21,8 +21,6 @@ import org.apache.inlong.sdk.transform.process.function.CeilFunction; import org.apache.inlong.sdk.transform.process.function.ConcatFunction; import org.apache.inlong.sdk.transform.process.function.CosFunction; -import org.apache.inlong.sdk.transform.process.function.DateExtractFunction; -import org.apache.inlong.sdk.transform.process.function.DateExtractFunction.DateExtractFunctionType; import org.apache.inlong.sdk.transform.process.function.DateFormatFunction; import org.apache.inlong.sdk.transform.process.function.ExpFunction; import org.apache.inlong.sdk.transform.process.function.FibonacciFunction; @@ -40,7 +38,6 @@ import org.apache.inlong.sdk.transform.process.function.SinhFunction; import org.apache.inlong.sdk.transform.process.function.SqrtFunction; import org.apache.inlong.sdk.transform.process.function.SubstringFunction; -import org.apache.inlong.sdk.transform.process.function.TimestampExtractFunction; import org.apache.inlong.sdk.transform.process.function.ToDateFunction; import org.apache.inlong.sdk.transform.process.function.ToTimestampFunction; import org.apache.inlong.sdk.transform.process.function.UnixTimestampFunction; @@ -52,8 +49,6 @@ import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.Function; import org.apache.commons.lang.ObjectUtils; -import org.reflections.Reflections; -import org.reflections.scanners.Scanners; import java.math.BigDecimal; import java.sql.Date;