From ffcb1666d09ed9edd9e8e138bab21bc5612eb4e5 Mon Sep 17 00:00:00 2001 From: An Phi Date: Mon, 3 Jun 2024 16:59:08 -0400 Subject: [PATCH] Revert "relation: add support for pivot()" (#824) * Revert "relation: support column name with quotes (#822)" This reverts commit 1e1685464d1a97ebd41707203f01dd1fb949c46e. * Revert "relation: add support for pivot (#807)" This reverts commit f2615ca37795f640c0b66d71eaa75113f7ebd7dc. --- .../m3parser/antlr/core/M3CoreParser.g4 | 8 +- .../antlr/AntlrContextToM3CoreInstance.java | 13 +- .../resources/platform/pure/relation.pure | 2 +- .../relation/functions/pivot.pure | 168 ------------- .../compiled/RelationExtensionCompiled.java | 4 +- .../RelationNativeImplementation.java | 91 ------- .../relation/compiled/natives/Pivot.java | 63 ----- .../relation/compiled/natives/PivotArray.java | 47 ---- .../relation/TestFunctionTester.java | 125 +++------- .../RelationExtensionInterpreted.java | 5 - .../relation/interpreted/natives/Pivot.java | 222 ------------------ .../src/test/java/TestFunctionTester.java | 124 ---------- .../external/relation/shared/TestTDS.java | 217 +---------------- .../java/extension/relation/TestTestTDS.java | 46 ++-- .../grammar/relational.pure | 7 - 15 files changed, 72 insertions(+), 1070 deletions(-) delete mode 100644 legend-pure-functions/legend-pure-functions-relation/legend-pure-m2-functions-relation-pure/src/main/resources/platform_functions_relation/relation/functions/pivot.pure delete mode 100644 legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Pivot.java delete mode 100644 legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/PivotArray.java delete mode 100644 legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Pivot.java delete mode 100644 legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/TestFunctionTester.java diff --git a/legend-pure-core/legend-pure-m3-core/src/main/antlr4/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/core/M3CoreParser.g4 b/legend-pure-core/legend-pure-m3-core/src/main/antlr4/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/core/M3CoreParser.g4 index 44a01fc7f0..9d2130fcf7 100644 --- a/legend-pure-core/legend-pure-m3-core/src/main/antlr4/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/core/M3CoreParser.g4 +++ b/legend-pure-core/legend-pure-m3-core/src/main/antlr4/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/core/M3CoreParser.g4 @@ -316,8 +316,10 @@ atomicExpression: columnBuilders: TILDE (oneColSpec | (BRACKET_OPEN oneColSpec(COMMA oneColSpec)* BRACKET_CLOSE)) ; -oneColSpec: columnName (COLON (type | lambdaParam lambdaPipe) extraFunction?)? + +oneColSpec: identifier ((COLON (type | lambdaParam lambdaPipe) extraFunction? ))? ; + extraFunction: (COLON lambdaParam lambdaPipe) ; @@ -468,9 +470,7 @@ type: ( qualifiedName (LESSTHAN (typeArguments? (PIPE multiplicityArguments)?) G unitName ; -columnType: (QUESTION | columnName) COLON type -; -columnName: identifier | STRING +columnType : (QUESTION | identifier) COLON type ; multiplicity: BRACKET_OPEN multiplicityArgument BRACKET_CLOSE diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/AntlrContextToM3CoreInstance.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/AntlrContextToM3CoreInstance.java index 0d99285d06..c950d1ea35 100644 --- a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/AntlrContextToM3CoreInstance.java +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/AntlrContextToM3CoreInstance.java @@ -1105,9 +1105,7 @@ else if (ctx.columnBuilders() != null) MutableList extraFunction = Lists.mutable.empty(); ListIterate.forEach(ctx.columnBuilders().oneColSpec(), oneColSpec -> { - M3Parser.ColumnNameContext colNameCtx = oneColSpec.columnName(); - String colName = colNameCtx.STRING() != null ? this.removeQuotes(colNameCtx.STRING()) : colNameCtx.identifier().getText(); - columnNames.add(this.repository.newStringCoreInstance(colName)); + columnNames.add(this.repository.newStringCoreInstance(oneColSpec.identifier().getText())); String returnType = null; if (oneColSpec.lambdaParam() != null && oneColSpec.lambdaPipe() != null) { @@ -1123,7 +1121,7 @@ else if (oneColSpec.type() != null) GenericType returnGType = type(oneColSpec.type(), typeParametersNames, "", importId, addLines); returnType = returnGType._rawType().getName(); } - columnInstances.add(_Column.getColumnInstance(colName, false, relationTypeGenericType, returnType, src, processorSupport)); + columnInstances.add(_Column.getColumnInstance(oneColSpec.identifier().getText(), false, relationTypeGenericType, returnType, src, processorSupport)); }); relationTypeGenericType._rawTypeCoreInstance(_RelationType.build(columnInstances, this.sourceInformation.getPureSourceInformation(ctx.getStart(), ctx.getStart(), ctx.getStop()), processorSupport)); @@ -2141,12 +2139,7 @@ public GenericType type(TypeContext ctx, MutableList typeParametersNames _RelationType.build( ListIterate.collect( ctx.columnType(), - c -> - { - M3Parser.ColumnNameContext colNameCtx = c.columnName(); - String colName = colNameCtx != null ? colNameCtx.STRING() != null ? this.removeQuotes(colNameCtx.STRING()) : colNameCtx.identifier().getText() : ""; - return _Column.getColumnInstance(c.QUESTION() != null ? "" : colName, c.QUESTION() != null, genericTypeInstance, this.type(c.type(), typeParametersNames, spacePlusTabs(space, 5), importId, addLines), srcInfo, processorSupport); - } + c -> _Column.getColumnInstance(c.QUESTION() != null ? "" : c.identifier().getText(), c.QUESTION() != null, genericTypeInstance, this.type(c.type(), typeParametersNames, spacePlusTabs(space, 5), importId, addLines), srcInfo, processorSupport) ), srcInfo, processorSupport ) ); diff --git a/legend-pure-core/legend-pure-m3-core/src/main/resources/platform/pure/relation.pure b/legend-pure-core/legend-pure-m3-core/src/main/resources/platform/pure/relation.pure index e53b4013b8..dbaa6f0c54 100644 --- a/legend-pure-core/legend-pure-m3-core/src/main/resources/platform/pure/relation.pure +++ b/legend-pure-core/legend-pure-m3-core/src/main/resources/platform/pure/relation.pure @@ -44,7 +44,7 @@ Class meta::pure::metamodel::relation::AggColSpec // Where T is of type R reduce : Function[1]; } -Class meta::pure::metamodel::relation::AggColSpecArray // Where T is of type RelationType +Class meta::pure::metamodel::relation::AggColSpecArray { aggSpecs : AggColSpec[*]; } diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-m2-functions-relation-pure/src/main/resources/platform_functions_relation/relation/functions/pivot.pure b/legend-pure-functions/legend-pure-functions-relation/legend-pure-m2-functions-relation-pure/src/main/resources/platform_functions_relation/relation/functions/pivot.pure deleted file mode 100644 index 7a0086935e..0000000000 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-m2-functions-relation-pure/src/main/resources/platform_functions_relation/relation/functions/pivot.pure +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import meta::pure::test::pct::*; -import meta::pure::metamodel::relation::*; - -native function <> meta::pure::functions::relation::pivot(r:Relation[1], cols:ColSpecArray[1], agg:AggColSpec<{T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1]; - -native function <> meta::pure::functions::relation::pivot(r:Relation[1], cols:ColSpec[1], agg:AggColSpec<{T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1]; - -native function <> meta::pure::functions::relation::pivot(r:Relation[1], cols:ColSpecArray[1], agg:AggColSpecArray<{T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1]; - -native function <> meta::pure::functions::relation::pivot(r:Relation[1], cols:ColSpec[1], agg:AggColSpecArray<{T[1]->K[0..1]},{K[*]->V[1]}, R>[1]):Relation[1]; - - -function <> meta::pure::functions::relation::tests::pivot::testSimplePivotBySingleSingle(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] -{ - let expr = { - |#TDS - city, country, year, treePlanted - NYC, USA, 2011, 5000 - NYC, USA, 2000, 5000 - SAN, USA, 2000, 2000 - SAN, USA, 2011, 100 - LND, UK, 2011, 3000 - SAN, USA, 2011, 2500 - NYC, USA, 2000, 10000 - NYC, USA, 2012, 7600 - NYC, USA, 2012, 7600 - #->pivot(~[year], ~[newCol : x | $x.treePlanted : y | $y->plus()]) - }; - - let res = $f->eval($expr); - - assertEquals( '#TDS\n'+ - ' city,country,2000__|__newCol,2011__|__newCol,2012__|__newCol\n' + - ' LND,UK,null,3000,null\n' + - ' NYC,USA,15000,5000,15200\n' + - ' SAN,USA,2000,2600,null\n' + - '#', $res->toString()); -} - - -function <> meta::pure::functions::relation::tests::pivot::testSimplePivotByMultipleSingle(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] -{ - let expr = { - |#TDS - city, country, year, treePlanted - NYC, USA, 2011, 5000 - NYC, USA, 2000, 5000 - SAN, USA, 2000, 2000 - SAN, USA, 2011, 100 - LND, UK, 2011, 3000 - SAN, USA, 2011, 2500 - NYC, USA, 2000, 10000 - NYC, USA, 2012, 7600 - NYC, USA, 2012, 7600 - #->pivot(~[country, city], ~sum : x | $x.treePlanted : y | $y->plus()) - }; - - let res = $f->eval($expr); - - assertEquals( '#TDS\n'+ - ' year,UK__|__LND__|__sum,USA__|__NYC__|__sum,USA__|__SAN__|__sum\n' + - ' 2000,null,15000,2000\n' + - ' 2011,3000,5000,2600\n' + - ' 2012,null,15200,null\n' + - '#', $res->toString()); -} - - -function <> meta::pure::functions::relation::tests::pivot::testSimplePivotBySingleMultiple(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] -{ - let expr = { - |#TDS - city, country, year, treePlanted - NYC, USA, 2011, 5000 - NYC, USA, 2000, 5000 - SAN, USA, 2000, 2000 - SAN, USA, 2011, 100 - LND, UK, 2011, 3000 - SAN, USA, 2011, 2500 - NYC, USA, 2000, 10000 - NYC, USA, 2012, 7600 - NYC, USA, 2012, 7600 - #->pivot(~[year], ~[sum : x | $x.treePlanted : y | $y->plus(), count : x | 1 : y | $y->plus()]) - }; - - let res = $f->eval($expr); - - assertEquals( '#TDS\n'+ - ' city,country,2000__|__sum,2000__|__count,2011__|__sum,2011__|__count,2012__|__sum,2012__|__count\n' + - ' LND,UK,null,null,3000,1,null,null\n' + - ' NYC,USA,15000,2,5000,1,15200,2\n' + - ' SAN,USA,2000,1,2600,2,null,null\n' + - '#', $res->toString()); -} - - -function <> meta::pure::functions::relation::tests::pivot::testSimplePivotByMultipleMultiple(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] -{ - let expr = { - |#TDS - city, country, year, treePlanted - NYC, USA, 2011, 5000 - NYC, USA, 2000, 5000 - SAN, USA, 2000, 2000 - SAN, USA, 2011, 100 - LND, UK, 2011, 3000 - SAN, USA, 2011, 2500 - NYC, USA, 2000, 10000 - NYC, USA, 2012, 7600 - NYC, USA, 2012, 7600 - #->pivot(~[country, city], ~[sum : x | $x.treePlanted : y | $y->plus(), count : x | 1 : y | $y->plus()]) - }; - - let res = $f->eval($expr); - - assertEquals( '#TDS\n'+ - ' year,UK__|__LND__|__sum,UK__|__LND__|__count,USA__|__NYC__|__sum,USA__|__NYC__|__count,USA__|__SAN__|__sum,USA__|__SAN__|__count\n' + - ' 2000,null,null,15000,2,2000,1\n' + - ' 2011,3000,1,5000,1,2600,2\n' + - ' 2012,null,null,15200,2,null,null\n' + - '#', $res->toString()); -} - -function <> meta::pure::functions::relation::tests::pivot::testSimplePivotChained(f:Function<{Function<{->T[m]}>[1]->T[m]}>[1]):Boolean[1] -{ - let expr = { - |#TDS - city, country, year, treePlanted - NYC, USA, 2011, 5000 - NYC, USA, 2000, 5000 - SAN, USA, 2000, 2000 - SAN, USA, 2011, 100 - LND, UK, 2011, 3000 - SAN, USA, 2011, 2500 - NYC, USA, 2000, 10000 - NYC, USA, 2012, 7600 - NYC, USA, 2012, 7600 - #->pivot(~[year], ~['newCol' : x | $x.treePlanted : y | $y->plus()])->cast(@Relation<(city:String,country:String,'2000__|__newCol':Integer,'2011__|__newCol':Integer,'2012__|__newCol':Integer)>)->filter(x|$x.city=='NYC') - }; - - let res = $f->eval($expr); - - assertEquals( '#TDS\n'+ - ' city,country,2000__|__newCol,2011__|__newCol,2012__|__newCol\n' + - ' NYC,USA,15000,5000,15200\n' + - '#', $res->toString()); -} - -// TODO: test pivot with no aggregation function specified -// TODO when we support specifying pivot with specific values -// - test pivot with values specified -// - test empty value for pivot -// - test incompatible types (check for error - see AbstractTestCast, AbstractTestPathToElement) -// - test mixed values for pivot (check for error - see AbstractTestCast, AbstractTestPathToElement) diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java index 0bebdb2905..99e5850333 100644 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java +++ b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java @@ -60,9 +60,7 @@ public List getExtraNatives() new Slice(), new Distinct(), new Select(), - new SelectArray(), - new Pivot(), - new PivotArray() + new SelectArray() ); } diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java index 566e7b0002..b2b6ac7f98 100644 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java +++ b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java @@ -26,11 +26,8 @@ import org.eclipse.collections.api.set.primitive.MutableIntSet; import org.eclipse.collections.api.tuple.Pair; import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.*; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enum; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.SimpleFunctionExpression; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification; import org.finos.legend.pure.m3.execution.ExecutionSupport; import org.finos.legend.pure.m3.navigation.M3Properties; import org.finos.legend.pure.m3.navigation.ProcessorSupport; @@ -44,8 +41,6 @@ import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortInfo; import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; -import java.util.Objects; - import static org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS.readCsv; public class RelationNativeImplementation @@ -232,7 +227,6 @@ public static class AggColSpecTrans public Function2 map; public Function2 reduce; public String reduceType; - public AggColSpec aggColSpec; public AggColSpecTrans(String newColName, Function2 map, Function2 reduce, String reduceType) { @@ -240,13 +234,6 @@ public AggColSpecTrans(String newColName, Function2 map, Function2 reduce, Strin this.map = map; this.reduce = reduce; this.reduceType = reduceType; - this.aggColSpec = null; - } - - public AggColSpecTrans(String newColName, Function2 map, Function2 reduce, String reduceType, AggColSpec aggColSpec) - { - this(newColName, map, reduce, reduceType); - this.aggColSpec = aggColSpec; } } @@ -313,84 +300,6 @@ private static void performMapReduce(Function2 map, Function2 reduce, ExecutionS } } - public static Relation pivot(Relation rel, ColSpec pivotCols, MutableList aggColSpecTrans, ExecutionSupport es) - { - return pivot(rel, Lists.mutable.with(pivotCols._name()), aggColSpecTrans, es); - } - - public static Relation pivot(Relation rel, ColSpecArray pivotCols, MutableList aggColSpecTransAll, ExecutionSupport es) - { - return pivot(rel, Lists.mutable.withAll(pivotCols._names()), aggColSpecTransAll, es); - } - - private static Relation pivot(Relation rel, ListIterable pivotCols, MutableList aggColSpecTransAll, ExecutionSupport es) - { - ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); - TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel); - - // TODO: right now we make assumption that the map expression is really simple so we can safely extract the column(s) - // used for aggregation, we make sure these column(s) are not part of the groupBy calculation - ListIterable columnsUsedInAggregation = aggColSpecTransAll.collect(col -> - { - try - { - ValueSpecification lambda = ((LambdaFunction) col.aggColSpec._map())._expressionSequence().getFirst(); - if (lambda instanceof SimpleFunctionExpression && ((SimpleFunctionExpression) lambda)._func() instanceof Column) - { - return ((SimpleFunctionExpression) lambda)._func()._name(); - } - return null; - } - catch (Exception e) - { - // do nothing, the shape is not as expected, we will try to inspect no further - return null; - } - }).select(Objects::nonNull); - - // these are the columns not being aggregated on, which will be used for groupBy calculation before transposing - ListIterable groupByColumns = tds.getColumnNames().reject(c -> columnsUsedInAggregation.anySatisfy(a -> a.equals(c)) || pivotCols.anySatisfy(a -> a.equals(c))).withAll(pivotCols); - - // create the big group-by table by processing all aggregations - Pair>> sorted = tds.sort(groupByColumns.collect(c -> new SortInfo(c, SortDirection.ASC))); - TestTDSCompiled temp = (TestTDSCompiled) sorted.getOne()._distinct(sorted.getTwo()); - temp = aggColSpecTransAll.injectInto(temp, (existing, aggColSpecTrans) -> - { - int size = sorted.getTwo().size(); - switch (aggColSpecTrans.reduceType) - { - case "String": - { - String[] finalRes = new String[size]; - performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sorted, (o, j) -> finalRes[j] = (String) o); - existing.addColumn(aggColSpecTrans.newColName, DataType.STRING, finalRes); - break; - } - case "Integer": - { - int[] finalResInt = new int[size]; - performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sorted, (o, j) -> finalResInt[j] = (int) (long) o); - existing.addColumn(aggColSpecTrans.newColName, DataType.INT, finalResInt); - break; - } - case "Float": - { - double[] finalResDouble = new double[size]; - performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sorted, (o, j) -> finalResDouble[j] = (double) o); - existing.addColumn(aggColSpecTrans.newColName, DataType.FLOAT, finalResDouble); - break; - } - } - - return existing; - }); - - // transposing the table to complete pivoting - TestTDSCompiled result = (TestTDSCompiled) temp.applyPivot(groupByColumns.reject(pivotCols::contains), pivotCols, aggColSpecTransAll.collect(col -> col.newColName)); - - return new TDSContainer(result, ps); - } - public static Relation project(RichIterable objects, RichIterable colFuncs, ExecutionSupport es) { ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport(); diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Pivot.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Pivot.java deleted file mode 100644 index 8523aedb1d..0000000000 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/Pivot.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; - -import org.eclipse.collections.api.list.ListIterable; -import org.finos.legend.pure.m4.coreinstance.CoreInstance; -import org.finos.legend.pure.runtime.java.compiled.generation.ProcessorContext; -import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; -import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; - -public class Pivot extends AbstractNative implements Native -{ - public Pivot() - { - super("pivot_Relation_1__ColSpec_1__AggColSpec_1__Relation_1_", "pivot_Relation_1__ColSpecArray_1__AggColSpec_1__Relation_1_"); - } - - @Override - public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) - { - StringBuilder result = new StringBuilder("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.pivot"); - result.append('('); - result.append(transformedParams.get(0)); - result.append(", "); - result.append(transformedParams.get(1)); - result.append(", "); - result.append("Lists.mutable.with(" + transformedParams.get(2) + ")"); - - processAggColSpec(transformedParams, result); - - result.append(", es)"); - return result.toString(); - } - - static void processAggColSpec(ListIterable transformedParams, StringBuilder result) - { - result.append(".collect("); - result.append("new DefendedFunction, org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.AggColSpecTrans>()\n" + - "{\n" + - " @Override\n" + - " public org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.AggColSpecTrans valueOf(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.AggColSpec c)\n" + - " {\n"); - result.append("return new org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.AggColSpecTrans(c._name(),"); - result.append("(Function2)PureCompiledLambda.getPureFunction(c._map(),es),"); - result.append("(Function2)PureCompiledLambda.getPureFunction(c._reduce(),es),"); - result.append("((org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType)c._reduce()._classifierGenericType()._typeArguments().toList().get(0)._rawType())._returnType()._rawType()._name(),"); - result.append("c);"); - result.append(" }\n" + - "})"); - } -} \ No newline at end of file diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/PivotArray.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/PivotArray.java deleted file mode 100644 index bae0c78bf6..0000000000 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/compiled/natives/PivotArray.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives; - -import org.eclipse.collections.api.list.ListIterable; -import org.finos.legend.pure.m4.coreinstance.CoreInstance; -import org.finos.legend.pure.runtime.java.compiled.generation.ProcessorContext; -import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.AbstractNative; -import org.finos.legend.pure.runtime.java.compiled.generation.processors.natives.Native; - -import static org.finos.legend.pure.runtime.java.extension.external.relation.compiled.natives.Pivot.processAggColSpec; - -public class PivotArray extends AbstractNative implements Native -{ - public PivotArray() - { - super("pivot_Relation_1__ColSpec_1__AggColSpecArray_1__Relation_1_", "pivot_Relation_1__ColSpecArray_1__AggColSpecArray_1__Relation_1_"); - } - - @Override - public String build(CoreInstance topLevelElement, CoreInstance functionExpression, ListIterable transformedParams, ProcessorContext processorContext) - { - StringBuilder result = new StringBuilder("org.finos.legend.pure.runtime.java.extension.external.relation.compiled.RelationNativeImplementation.pivot"); - result.append('('); - result.append(transformedParams.get(0)); - result.append(", "); - result.append(transformedParams.get(1)); - result.append(", "); - result.append("Lists.mutable.withAll(" + transformedParams.get(2) + "._aggSpecs())"); - processAggColSpec(transformedParams, result); - result.append(", es)"); - return result.toString(); - } - -} \ No newline at end of file diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestFunctionTester.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestFunctionTester.java index aea75eca7d..2ee583d38f 100644 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestFunctionTester.java +++ b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-compiled-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestFunctionTester.java @@ -19,7 +19,6 @@ import org.finos.legend.pure.runtime.java.compiled.execution.FunctionExecutionCompiledBuilder; import org.junit.After; import org.junit.BeforeClass; -import org.junit.Test; public class TestFunctionTester extends PureExpressionTest { @@ -37,90 +36,32 @@ public void cleanRuntime() protected static FunctionExecution getFunctionExecution() { +// return new FunctionExecutionInterpreted(); return new FunctionExecutionCompiledBuilder().build(); } - @Test - public void testPlaygroundCompiledFunction() + @org.junit.Test + public void testFunction() { -// compileTestSource("fromString.pure", -// "function test():Any[*]\n" + -// "{" + -// "print(#TDS\n" + -// " city, country, year, treePlanted\n" + -// " NYC, USA, 2011, 5000\n" + -// " NYC, USA, 2000, 5000\n" + -// " SAN, USA, 2000, 2000\n" + -// " SAN, USA, 2011, 100\n" + -// " LND, UK, 2011, 3000\n" + -// " SAN, USA, 2011, 2500\n" + -// " NYC, USA, 2000, 10000\n" + -// " NYC, USA, 2012, 7600\n" + -// " NYC, USA, 2012, 7600\n" + -// " #->pivot(~year, ~'newCol' : x | $x.treePlanted : y | $y->plus())->toString(),1);\n" + -// "}"); -// this.execute("test():Any[*]"); -// runtime.delete("fromString.pure"); - -//-------------------------------------------------------------------- - -// compileTestSource("fromString.pure", -// "function test():Any[*]\n" + -// "{" + -// "print(#TDS\n" + -// " value, str\n" + -// " 1, a\n" + -// " 3, ewe\n" + -// " 4, qw\n" + -// " #\n" + -// " ->filter\n" + -// " (\n" + -// " x|$x.value == 1" + -// " )->toString(),1);\n" + -// "}"); -// this.execute("test():Any[*]"); -// runtime.delete("fromString.pure"); - -//-------------------------------------------------------------------- - -// compileTestSource("fromString.pure", -// "function test():Any[*]\n" + -// "{" + -// "print(#TDS\n" + -// " value, str\n" + -// " 1, a\n" + -// " 3, ewe\n" + -// " 4, qw\n" + -// " #\n" + -// " ->filter\n" + -// " (\n" + -// " x|$x.value == 1" + -// " )->toString(),1);\n" + -// "}"); -// this.execute("test():Any[*]"); -// runtime.delete("fromString.pure"); - -//-------------------------------------------------------------------- - -// compileTestSource("fromString.pure", -// "function test():Any[*]\n" + -// "{" + -// "print(#TDS\n" + -// " value, str\n" + -// " 1, a\n" + -// " 3, ewe\n" + -// " 4, qw\n" + -// " #\n" + -// " ->filter\n" + -// " (\n" + -// " x|$x.value == 1" + -// " )->toString(),1);\n" + -// "}"); -// this.execute("test():Any[*]"); -// runtime.delete("fromString.pure"); - -//-------------------------------------------------------------------- + compileTestSource("fromString.pure", + "function test():Any[*]\n" + + "{" + + "print(#TDS\n" + + " value, str\n" + + " 1, a\n" + + " 3, ewe\n" + + " 4, qw\n" + + " #\n" + + " ->filter\n" + + " (\n" + + " x|$x.value == 1" + + " )->toString(),1);\n" + + "}"); + this.execute("test():Any[*]"); + runtime.delete("fromString.pure"); +// //-------------------------------------------------------------------- +// // compileTestSource("fromString.pure", // "function test():Any[*]\n" + // "{ " + @@ -138,9 +79,9 @@ public void testPlaygroundCompiledFunction() // "}\n"); // this.execute("test():Any[*]"); // runtime.delete("fromString.pure"); - -//-------------------------------------------------------------------- - +// +// //-------------------------------------------------------------------- +// // compileTestSource("fromString.pure", // "function test():Any[*]\n" + // "{ " + @@ -158,9 +99,9 @@ public void testPlaygroundCompiledFunction() // "}\n"); // this.execute("test():Any[*]"); // runtime.delete("fromString.pure"); - -//-------------------------------------------------------------------- - +// +// //-------------------------------------------------------------------- +// // compileTestSource("fromString.pure", // "function test():Any[*]\n" + // "{ " + @@ -178,9 +119,9 @@ public void testPlaygroundCompiledFunction() // "}\n"); // this.execute("test():Any[*]"); // runtime.delete("fromString.pure"); - -//-------------------------------------------------------------------- - +// +// //-------------------------------------------------------------------- +// // compileTestSource("fromString.pure", // "function test():Any[*]\n" + // "{ " + @@ -199,9 +140,8 @@ public void testPlaygroundCompiledFunction() // this.execute("test():Any[*]"); // runtime.delete("fromString.pure"); // - -//-------------------------------------------------------------------- - +// +// //-------------------------------------------------------------------- // // compileTestSource("fromString.pure", // "function test():Any[*]\n" + @@ -221,5 +161,6 @@ public void testPlaygroundCompiledFunction() // this.execute("test():Any[*]"); // runtime.delete("fromString.pure"); // +// //-------------------------------------------------------------------- } } diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java index 1d7291a92a..f200da3973 100644 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java +++ b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/RelationExtensionInterpreted.java @@ -32,7 +32,6 @@ import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Join; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Limit; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Map; -import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Pivot; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Project; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Rename; import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.Select; @@ -71,10 +70,6 @@ public RelationExtensionInterpreted() Tuples.pair("groupBy_Relation_1__ColSpec_1__AggColSpec_1__Relation_1_", GroupBy::new), Tuples.pair("groupBy_Relation_1__ColSpec_1__AggColSpecArray_1__Relation_1_", GroupBy::new), Tuples.pair("groupBy_Relation_1__ColSpecArray_1__AggColSpecArray_1__Relation_1_", GroupBy::new), - Tuples.pair("pivot_Relation_1__ColSpecArray_1__AggColSpecArray_1__Relation_1_", Pivot::new), - Tuples.pair("pivot_Relation_1__ColSpecArray_1__AggColSpec_1__Relation_1_", Pivot::new), - Tuples.pair("pivot_Relation_1__ColSpec_1__AggColSpecArray_1__Relation_1_", Pivot::new), - Tuples.pair("pivot_Relation_1__ColSpec_1__AggColSpec_1__Relation_1_", Pivot::new), Tuples.pair("project_C_MANY__FuncColSpecArray_1__Relation_1_", Project::new), Tuples.pair("columns_Relation_1__Column_MANY_", Columns::new), Tuples.pair("select_Relation_1__ColSpec_1__Relation_1_", Select::new), diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Pivot.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Pivot.java deleted file mode 100644 index 3f38546756..0000000000 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/interpreted/natives/Pivot.java +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives; - -import io.deephaven.csv.parsers.DataType; -import org.eclipse.collections.api.block.procedure.Procedure2; -import org.eclipse.collections.api.list.FixedSizeList; -import org.eclipse.collections.api.list.ListIterable; -import org.eclipse.collections.api.list.MutableList; -import org.eclipse.collections.api.map.MutableMap; -import org.eclipse.collections.api.tuple.Pair; -import org.eclipse.collections.impl.factory.Lists; -import org.finos.legend.pure.m3.compiler.Context; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunctionCoreInstanceWrapper; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.*; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Type; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.SimpleFunctionExpression; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification; -import org.finos.legend.pure.m3.exception.PureExecutionException; -import org.finos.legend.pure.m3.navigation.*; -import org.finos.legend.pure.m3.navigation._package._Package; -import org.finos.legend.pure.m3.navigation.relation._Column; -import org.finos.legend.pure.m3.navigation.relation._RelationType; -import org.finos.legend.pure.m4.ModelRepository; -import org.finos.legend.pure.m4.coreinstance.CoreInstance; -import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.Shared; -import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSCoreInstance; -import org.finos.legend.pure.runtime.java.extension.external.relation.interpreted.natives.shared.TDSWithCursorCoreInstance; -import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortDirection; -import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortInfo; -import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS; -import org.finos.legend.pure.runtime.java.interpreted.ExecutionSupport; -import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted; -import org.finos.legend.pure.runtime.java.interpreted.VariableContext; -import org.finos.legend.pure.runtime.java.interpreted.natives.InstantiationContext; -import org.finos.legend.pure.runtime.java.interpreted.profiler.Profiler; - -import java.util.Objects; -import java.util.Stack; - -public class Pivot extends Shared -{ - public Pivot(FunctionExecutionInterpreted functionExecution, ModelRepository repository) - { - super(functionExecution, repository); - } - - @Override - public CoreInstance execute(ListIterable params, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, Context context, ProcessorSupport processorSupport) throws PureExecutionException - { - TestTDS tds = getTDS(params, 0, processorSupport); - RelationType relationType = getRelationType(params, 0); - - CoreInstance pivotColSpec = Instance.getValueForMetaPropertyToOneResolved(params.get(1), M3Properties.values, processorSupport); - ListIterable pivotCols; - if (pivotColSpec instanceof ColSpec) - { - pivotCols = Lists.mutable.with(((ColSpec) pivotColSpec)._name()); - } - else if (pivotColSpec instanceof ColSpecArray) - { - pivotCols = ((ColSpecArray) pivotColSpec)._names().toList(); - } - else - { - throw new RuntimeException("Not Possible"); - } - - CoreInstance aggColSpec = Instance.getValueForMetaPropertyToOneResolved(params.get(2), M3Properties.values, processorSupport); - ListIterable aggColSpecs; - if (aggColSpec instanceof AggColSpec) - { - aggColSpecs = Lists.mutable.with(((AggColSpec) aggColSpec)); - } - else if (aggColSpec instanceof AggColSpecArray) - { - aggColSpecs = ((AggColSpecArray) aggColSpec)._aggSpecs().toList(); - } - else - { - throw new RuntimeException("Not Possible"); - } - - // TODO: right now we make assumption that the map expression is really simple so we can safely extract the column(s) - // used for aggregation, we make sure these column(s) are not part of the groupBy calculation - ListIterable columnsUsedInAggregation = aggColSpecs.collect(col -> - { - try - { - ValueSpecification lambda = ((LambdaFunction) col._map())._expressionSequence().getFirst(); - if (lambda instanceof SimpleFunctionExpression && ((SimpleFunctionExpression) lambda)._func() instanceof Column) - { - return ((SimpleFunctionExpression) lambda)._func()._name(); - } - return null; - } - catch (Exception e) - { - // do nothing, the shape is not as expected, we will try to inspect no further - return null; - } - }).select(Objects::nonNull); - - // these are the columns not being aggregated on, which will be used for groupBy calculation before transposing - ListIterable groupByColumns = tds.getColumnNames().reject(c -> columnsUsedInAggregation.anySatisfy(a -> a.equals(c)) || pivotCols.anySatisfy(a -> a.equals(c))).withAll(pivotCols); - - // create the big group-by table by processing all aggregations - Pair>> sorted = tds.sort(groupByColumns.collect(c -> new SortInfo(c, SortDirection.ASC))); - TestTDS temp = aggColSpecs.injectInto(null, (a, b) -> processOneAggColSpec(a, b, sorted, resolvedTypeParameters, resolvedMultiplicityParameters, variableContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType)); - - // transposing the table to complete pivoting - TestTDS result = temp.applyPivot(groupByColumns.reject(pivotCols::contains), pivotCols, aggColSpecs.collect(AggColSpecAccessor::_name)); - - // populate the generic type for the return at execution time since it cannot be reasoned out at compile time - CoreInstance returnGenericType = getReturnGenericType(resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, processorSupport); - returnGenericType.getValueForMetaPropertyToMany("typeArguments").getFirst().setKeyValues(Lists.mutable.with("rawType"), Lists.mutable.with(_RelationType.build(result.getColumnWithTypes().collect(c -> - { - String type; - switch (c.getTwo()) - { - case FLOAT: - { - type = M3Paths.Float; - break; - } - case DOUBLE: - { - type = M3Paths.Decimal; - break; - } - case INT: - { - type = M3Paths.Integer; - break; - } - case CHAR: - case STRING: - { - type = M3Paths.String; - break; - } - default: - throw new RuntimeException("ERROR " + c.getTwo() + " not supported in pivot!"); - } - return _Column.getColumnInstance(c.getOne(), false, (GenericType) returnGenericType, type, null, processorSupport); - }), functionExpressionToUseInStack.getSourceInformation(), processorSupport))); - - return ValueSpecificationBootstrap.wrapValueSpecification(new TDSCoreInstance(result, returnGenericType, repository, processorSupport), false, processorSupport); - } - - private TestTDS processOneAggColSpec(TestTDS tds, AggColSpec aggColSpec, Pair>> sorted, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, VariableContext variableContext, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, RelationType relationType) - { - String name = aggColSpec.getValueForMetaPropertyToOne("name").getName(); - LambdaFunction mapF = (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(aggColSpec.getValueForMetaPropertyToOne("map")); - LambdaFunction reduceF = (LambdaFunction) LambdaFunctionCoreInstanceWrapper.toLambdaFunction(aggColSpec.getValueForMetaPropertyToOne("reduce")); - - VariableContext mapFVarContext = this.getParentOrEmptyVariableContextForLambda(variableContext, mapF); - VariableContext reduceFVarContext = this.getParentOrEmptyVariableContextForLambda(variableContext, reduceF); - - Type type = ((FunctionType) reduceF._classifierGenericType()._typeArguments().getFirst()._rawType())._returnType()._rawType(); - - FixedSizeList parameters = Lists.fixedSize.with((CoreInstance) null); - - int size = sorted.getTwo().size(); - DataType resType = null; - Object _finalRes = null; - if (type == _Package.getByUserPath("String", processorSupport)) - { - String[] finalRes = new String[size]; - performAggregation(sorted, mapF, reduceF, (j, val) -> finalRes[j] = PrimitiveUtilities.getStringValue(val), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, size, parameters, mapFVarContext, reduceFVarContext); - resType = DataType.STRING; - _finalRes = finalRes; - } - if (type == _Package.getByUserPath("Integer", processorSupport)) - { - int[] finalRes = new int[size]; - performAggregation(sorted, mapF, reduceF, (j, val) -> finalRes[j] = PrimitiveUtilities.getIntegerValue(val).intValue(), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, size, parameters, mapFVarContext, reduceFVarContext); - resType = DataType.INT; - _finalRes = finalRes; - } - if (type == _Package.getByUserPath("Float", processorSupport)) - { - double[] finalRes = new double[size]; - performAggregation(sorted, mapF, reduceF, (j, val) -> finalRes[j] = PrimitiveUtilities.getFloatValue(val).doubleValue(), resolvedTypeParameters, resolvedMultiplicityParameters, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport, processorSupport, relationType, size, parameters, mapFVarContext, reduceFVarContext); - resType = DataType.FLOAT; - _finalRes = finalRes; - } - return tds == null ? sorted.getOne()._distinct(sorted.getTwo()).addColumn(name, resType, _finalRes) : tds.addColumn(name, resType, _finalRes); - } - - private void performAggregation(Pair>> res, LambdaFunction mapF, LambdaFunction reduceF, Procedure2 setter, Stack> resolvedTypeParameters, Stack> resolvedMultiplicityParameters, CoreInstance functionExpressionToUseInStack, Profiler profiler, InstantiationContext instantiationContext, ExecutionSupport executionSupport, ProcessorSupport processorSupport, RelationType relationType, int size, FixedSizeList parameters, VariableContext mapFVarContext, VariableContext reduceFVarContext) - { - for (int j = 0; j < size; j++) - { - Pair r = res.getTwo().get(j); - MutableList subList = Lists.mutable.empty(); - for (int i = r.getOne(); i < r.getTwo(); i++) - { - parameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(new TDSWithCursorCoreInstance(res.getOne(), i, "", null, relationType, -1, repository, false), true, processorSupport)); - subList.add(this.functionExecution.executeFunction(false, mapF, parameters, resolvedTypeParameters, resolvedMultiplicityParameters, mapFVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport).getValueForMetaPropertyToOne("values")); - } - parameters.set(0, ValueSpecificationBootstrap.wrapValueSpecification(subList, true, processorSupport)); - CoreInstance re = this.functionExecution.executeFunction(false, reduceF, parameters, resolvedTypeParameters, resolvedMultiplicityParameters, reduceFVarContext, functionExpressionToUseInStack, profiler, instantiationContext, executionSupport); - setter.value(j, re.getValueForMetaPropertyToOne("values")); - } - } -} diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/TestFunctionTester.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/TestFunctionTester.java deleted file mode 100644 index 4600f8022b..0000000000 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-interpreted-functions-relation/src/test/java/TestFunctionTester.java +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import org.finos.legend.pure.m3.execution.FunctionExecution; -import org.finos.legend.pure.m3.tests.function.base.PureExpressionTest; -import org.finos.legend.pure.runtime.java.interpreted.FunctionExecutionInterpreted; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * This is a tester for interpreted function, so we don't need to test via the Pure IDE, hence a faster dev feedback loop - */ -public class TestFunctionTester extends PureExpressionTest -{ - @BeforeClass - public static void setUp() - { - setUpRuntime(getFunctionExecution()); - } - - @After - public void cleanRuntime() - { - runtime.delete("fromString.pure"); - } - - protected static FunctionExecution getFunctionExecution() - { - return new FunctionExecutionInterpreted(); - } - - // TO FIX: this should work? - // @Test - // public void testColumnsOfRelationAny() - // { - // compileTestSource("fromString.pure", - // "function test():Any[*]\n" + - // "{" + - // "#TDS\n" + - // " city, country, year, treePlanted\n" + - // " NYC, USA, 2011, 5000\n" + - // " #->cast(@meta::pure::metamodel::relation::Relation)->columns();\n" + - // "}"); - // this.execute("test():Any[*]"); - // runtime.delete("fromString.pure"); - // } - - @Test - public void testPlaygroundInterpretedFunction() - { -// compileTestSource("fromString.pure", -// "function test():Any[*]\n" + -// "{" + -// "print(#TDS\n" + -// " city, country, year, treePlanted\n" + -// " NYC, USA, 2011, 5000\n" + -// " NYC, USA, 2000, 5000\n" + -// " SAN, USA, 2000, 2000\n" + -// " SAN, USA, 2011, 100\n" + -// " LND, UK, 2011, 3000\n" + -// " SAN, USA, 2011, 2500\n" + -// " NYC, USA, 2000, 10000\n" + -// " NYC, USA, 2012, 7600\n" + -// " NYC, USA, 2012, 7600\n" + -// "#->pivot(~[country,city], ~[sum : x | $x.treePlanted : y | $y->plus(), count : x | $x : y | $y->size()])->toString(),1);\n" + -// "}"); -// this.execute("test():Any[*]"); -// runtime.delete("fromString.pure"); - -//-------------------------------------------------------------------- - -// compileTestSource("fromString.pure", -// "function test():Any[*]\n" + -// "{" + -// "print(#TDS\n" + -// " city, country, year, treePlanted\n" + -// " NYC, USA, 2011, 5000\n" + -// " NYC, USA, 2000, 5000\n" + -// " SAN, USA, 2000, 2000\n" + -// " SAN, USA, 2011, 100\n" + -// " LND, UK, 2011, 3000\n" + -// " SAN, USA, 2011, 2500\n" + -// " NYC, USA, 2000, 10000\n" + -// " NYC, USA, 2012, 7600\n" + -// " NYC, USA, 2012, 7600\n" + -// "#->pivot(~[country,city], ~[sum : x | $x.treePlanted : y | $y->plus(), count : x | $x : y | $y->size()])->cast(@meta::pure::metamodel::relation::Relation<(year: Number)>)->toString(),1);\n" + -// "}"); -// this.execute("test():Any[*]"); -// runtime.delete("fromString.pure"); - -//-------------------------------------------------------------------- - -// compileTestSource("fromString.pure", -// "function test():Any[*]\n" + -// "{" + -// "print(#TDS\n" + -// " city, country, year, treePlanted\n" + -// " NYC, USA, 2011, 5000\n" + -// " NYC, USA, 2000, 5000\n" + -// " SAN, USA, 2000, 2000\n" + -// " SAN, USA, 2011, 100\n" + -// " LND, UK, 2011, 3000\n" + -// " SAN, USA, 2011, 2500\n" + -// " NYC, USA, 2000, 10000\n" + -// " NYC, USA, 2012, 7600\n" + -// " NYC, USA, 2012, 7600\n" + -// "#->pivot(~[year], ~['newCol' : x | $x.treePlanted : y | $y->plus()])->toString(),1);\n" + -// "}"); -// this.execute("test():Any[*]"); -// runtime.delete("fromString.pure"); - } -} diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java index 51c9e5f7cc..5075fc42e1 100644 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java +++ b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-shared-functions-relation/src/main/java/org/finos/legend/pure/runtime/java/extension/external/relation/shared/TestTDS.java @@ -32,7 +32,6 @@ import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; import org.eclipse.collections.impl.tuple.Tuples; import org.eclipse.collections.impl.utility.ArrayIterate; -import org.eclipse.collections.impl.utility.ListIterate; import java.io.ByteArrayInputStream; import java.lang.reflect.Array; @@ -152,7 +151,7 @@ protected TestTDS(MutableList columnOrdered, MutableMap indices, int start, int break; } default: - throw new RuntimeException("ERROR " + copy.columnType.get(columnName) + " not supported in reorder!"); + throw new RuntimeException("ERROR " + copy.columnType.get(columnName) + " not supported in drop!"); } } } public String toString() { - RichIterable columns = this.columnsOrdered; + RichIterable columns = this.dataByColumnName.keysView(); MutableList rows = Lists.mutable.empty(); for (int i = 0; i < rowCount; i++) { @@ -999,206 +998,4 @@ public MutableList getColumnNames() { return this.columnsOrdered; } - - static class PivotColumnInfo - { - private final MutableList> columnValues; - private final String aggColumnName; - private final DataType columnType; - private final String columnName; - - public PivotColumnInfo(MutableList> columnValues, String aggColumnName, DataType columnType) - { - this.columnValues = columnValues; - this.aggColumnName = aggColumnName; - this.columnType = columnType; - // TODO: we might need to rethink this column naming strategy, it could break in some edge cases - this.columnName = ListIterate.collect(columnValues, Pair::getTwo).with(aggColumnName).select(Objects::nonNull).makeString("__|__"); - } - - public String getColumnName() - { - return this.columnName; - } - - public String getAggColumnName() - { - return this.aggColumnName; - } - - public DataType getColumnType() - { - return this.columnType; - } - - public boolean match(TestTDS tds, int row) - { - return columnValues.allSatisfy(col -> - { - DataType columnType = tds.columnType.get(col.getOne()); - switch (columnType) - { - case INT: - { - return ((Integer) ((int[]) tds.dataByColumnName.get(col.getOne()))[row]).toString().equals(col.getTwo()); - } - case CHAR: - { - return ((Character) ((char[]) tds.dataByColumnName.get(col.getOne()))[row]).toString().equals(col.getTwo()); - } - case STRING: - { - return ((String[]) tds.dataByColumnName.get(col.getOne()))[row].equals(col.getTwo()); - } - case DOUBLE: - { - return ((Double) ((double[]) tds.dataByColumnName.get(col.getOne()))[row]).toString().equals(col.getTwo()); - } - default: - throw new RuntimeException("ERROR " + columnType + " not supported in pivot!"); - } - }); - } - } - - public TestTDS applyPivot(ListIterable nonTransposeColumns, ListIterable pivotColumns, ListIterable aggColumns) - { - // compute the different unique combinations of values - Pair>> sortedByPivotColumns = this.sort(pivotColumns.collect(c -> new SortInfo(c, SortDirection.ASC))); - MutableList newColumns = Lists.mutable.empty(); - for (int i = 0; i < sortedByPivotColumns.getTwo().size(); i++) - { - Pair r = sortedByPivotColumns.getTwo().get(i); - for (String aggColumnName : aggColumns) - { - newColumns.add(new PivotColumnInfo(pivotColumns.collect(c -> - { - Object valuesAsObject = sortedByPivotColumns.getOne().dataByColumnName.get(c); - DataType columnType = this.columnType.get(c); - switch (columnType) - { - case INT: - { - return Tuples.pair(c, ((Integer) ((int[]) valuesAsObject)[r.getOne()]).toString()); - } - case CHAR: - { - return Tuples.pair(c, ((Character) ((char[]) valuesAsObject)[r.getOne()]).toString()); - } - case STRING: - { - return Tuples.pair(c, ((String[]) valuesAsObject)[r.getOne()]); - } - case DOUBLE: - { - return Tuples.pair(c, ((Double) ((double[]) valuesAsObject)[r.getOne()]).toString()); - } - default: - throw new RuntimeException("ERROR " + columnType + " not supported in pivot!"); - } - }).toList(), aggColumnName, this.columnType.get(aggColumnName))); - } - } - - Pair>> sortedByNonTransposeColumns = this.sort(nonTransposeColumns.collect(c -> new SortInfo(c, SortDirection.ASC))); - TestTDS result = this._distinct(sortedByNonTransposeColumns.getTwo()).removeColumns(this.columnsOrdered.reject(nonTransposeColumns::contains).toSet()); - result = newColumns.injectInto(result, (tds, newColInfo) -> - { - int size = (int) tds.rowCount; - String name = newColInfo.getColumnName(); - Object dataAsObject; - boolean[] isNull = new boolean[size]; - Arrays.fill(isNull, Boolean.TRUE); - switch (newColInfo.getColumnType()) - { - case INT: - { - int[] values = new int[size]; - for (int i = 0; i < size; i++) - { - for (int j = sortedByNonTransposeColumns.getTwo().get(i).getOne(); j < sortedByNonTransposeColumns.getTwo().get(i).getTwo(); j++) - { - if (newColInfo.match(sortedByNonTransposeColumns.getOne(), j)) - { - values[i] = ((int[]) sortedByNonTransposeColumns.getOne().dataByColumnName.get(newColInfo.getAggColumnName()))[j]; - isNull[i] = Boolean.FALSE; - } - } - } - dataAsObject = values; - break; - } - case STRING: - { - String[] values = new String[size]; - for (int i = 0; i < size; i++) - { - for (int j = sortedByNonTransposeColumns.getTwo().get(i).getOne(); j < sortedByNonTransposeColumns.getTwo().get(i).getTwo(); j++) - { - if (newColInfo.match(sortedByNonTransposeColumns.getOne(), j)) - { - values[i] = ((String[]) sortedByNonTransposeColumns.getOne().dataByColumnName.get(newColInfo.getAggColumnName()))[j]; - isNull[i] = Boolean.FALSE; - } - } - } - dataAsObject = values; - break; - } - case CHAR: - { - char[] values = new char[size]; - for (int i = 0; i < size; i++) - { - for (int j = sortedByNonTransposeColumns.getTwo().get(i).getOne(); j < sortedByNonTransposeColumns.getTwo().get(i).getTwo(); j++) - { - if (newColInfo.match(sortedByNonTransposeColumns.getOne(), j)) - { - values[i] = ((char[]) sortedByNonTransposeColumns.getOne().dataByColumnName.get(newColInfo.getAggColumnName()))[j]; - isNull[i] = Boolean.FALSE; - } - } - } - dataAsObject = values; - break; - } - case DOUBLE: - { - double[] values = new double[size]; - for (int i = 0; i < size; i++) - { - for (int j = sortedByNonTransposeColumns.getTwo().get(i).getOne(); j < sortedByNonTransposeColumns.getTwo().get(i).getTwo(); j++) - { - if (newColInfo.match(sortedByNonTransposeColumns.getOne(), j)) - { - values[i] = ((double[]) sortedByNonTransposeColumns.getOne().dataByColumnName.get(newColInfo.getAggColumnName()))[j]; - isNull[i] = Boolean.FALSE; - } - } - } - dataAsObject = values; - break; - } - default: - { - throw new RuntimeException("ERROR " + newColInfo.getColumnType() + " not supported in pivot!"); - } - } - tds.dataByColumnName.put(name, dataAsObject); - if (!newColInfo.getColumnType().equals(DataType.STRING)) - { - tds.isNullByColumn.put(name, isNull); - } - tds.columnsOrdered.add(name); - tds.columnType.put(name, newColInfo.getColumnType()); - return tds; - }); - - return result; - } - - public MutableList> getColumnWithTypes() - { - return this.columnsOrdered.collect(col -> Tuples.pair(col, this.columnType.get(col))); - } } diff --git a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-shared-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestTestTDS.java b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-shared-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestTestTDS.java index 22ff581534..9af242e9c7 100644 --- a/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-shared-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestTestTDS.java +++ b/legend-pure-functions/legend-pure-functions-relation/legend-pure-runtime-java-extension-shared-functions-relation/src/test/java/org/finos/legend/pure/runtime/java/extension/relation/TestTestTDS.java @@ -298,16 +298,16 @@ public void testJoin() TestTDS t = tds1.join(tds2); - Assert.assertEquals("id, name, extra\n" + - "1, A, X\n" + - "1, A, Y\n" + - "1, A, Z\n" + - "2, B, X\n" + - "2, B, Y\n" + - "2, B, Z\n" + - "3, C, X\n" + - "3, C, Y\n" + - "3, C, Z", t.toString()); + Assert.assertEquals("id, extra, name\n" + + "1, X, A\n" + + "1, Y, A\n" + + "1, Z, A\n" + + "2, X, B\n" + + "2, Y, B\n" + + "2, Z, B\n" + + "3, X, C\n" + + "3, Y, C\n" + + "3, Z, C", t.toString()); Assert.assertEquals(initialTds1, tds1.toString()); Assert.assertEquals(initialTds2, tds2.toString()); @@ -334,13 +334,13 @@ public void testJoinWithNull() TestTDS t = left.compensateLeft(res).join(third); - Assert.assertEquals("id, id2, extra, name, extraInt, id, boom\n" + - "1, 4, More David, David, 1, 1, A1\n" + - "3, 4, More David, David, 1, 3, A3\n" + - "1, 1, More George 2, George, 2, 1, A1\n" + - "3, 1, More George 2, George, 2, 3, A3\n" + - "1, NULL, NULL, Sachin, NULL, 1, A1\n" + - "3, NULL, NULL, Sachin, NULL, 3, A3", t.toString()); + Assert.assertEquals("id, id2, extra, name, boom, extraInt\n" + + "1, 4, More David, David, A1, 1\n" + + "3, 4, More David, David, A3, 1\n" + + "1, 1, More George 2, George, A1, 2\n" + + "3, 1, More George 2, George, A3, 2\n" + + "1, NULL, NULL, Sachin, A1, NULL\n" + + "3, NULL, NULL, Sachin, A3, NULL", t.toString()); Assert.assertEquals(initialRes, res.toString()); Assert.assertEquals(initialLeft, left.toString()); @@ -389,11 +389,11 @@ public void testSentinel() TestTDS res = new TestTDS(resTDS); - Assert.assertEquals("id, name, id2, col, other\n" + - "1, George, 1, More George 1, 1\n" + - "1, George, 1, More George 2, 2\n" + - "2, Pierre, NULL, NULL, NULL\n" + - "3, Sachin, NULL, NULL, NULL\n" + - "4, David, 4, More David, 1", res.toString()); + Assert.assertEquals("id, col, other, id2, name\n" + + "1, More George 1, 1, 1, George\n" + + "1, More George 2, 2, 1, George\n" + + "2, NULL, NULL, NULL, Pierre\n" + + "3, NULL, NULL, NULL, Sachin\n" + + "4, More David, 1, 4, David", res.toString()); } } \ No newline at end of file diff --git a/legend-pure-store/legend-pure-store-relational/legend-pure-m2-store-relational-pure/src/main/resources/platform_store_relational/grammar/relational.pure b/legend-pure-store/legend-pure-store-relational/legend-pure-m2-store-relational-pure/src/main/resources/platform_store_relational/grammar/relational.pure index 70df236874..5216db5857 100644 --- a/legend-pure-store/legend-pure-store-relational/legend-pure-m2-store-relational-pure/src/main/resources/platform_store_relational/grammar/relational.pure +++ b/legend-pure-store/legend-pure-store-relational/legend-pure-m2-store-relational-pure/src/main/resources/platform_store_relational/grammar/relational.pure @@ -213,7 +213,6 @@ Class meta::relational::metamodel::relation::SelectSQLQuery extends Relation, SQ data : RootJoinTreeNode[0..1]; filteringOperation : RelationalOperationElement[*]; groupBy : RelationalOperationElement[*]; - pivot : Pivot[0..1]; havingOperation : RelationalOperationElement[*]; orderBy : OrderBy[*]; fromRow : meta::relational::metamodel::Literal[0..1]; @@ -498,9 +497,3 @@ Class meta::relational::metamodel::Window extends meta::relational::metamodel::R sortDirection: meta::relational::metamodel::SortDirection[0..1] ; } - -Class meta::relational::metamodel::Pivot -{ - pivotColumns : RelationalOperationElement[*]; - aggColumns : RelationalOperationElement[*]; -}