Skip to content

Commit

Permalink
Revert "relation: add support for pivot()" (finos#824)
Browse files Browse the repository at this point in the history 
* Revert "relation: support column name with quotes (finos#822)"

This reverts commit 1e16854.

* Revert "relation: add support for pivot (finos#807)"

This reverts commit f2615ca.
  • Loading branch information
@akphi
akphi authored Jun 3, 2024
1 parent 87db8cf commit ffcb166
Show file tree
Hide file tree
Showing 15 changed files with 72 additions and 1,070 deletions.
8 changes: 4 additions & 4 deletions ...antlr4/org/finos/legend/pure/m3/serialization/grammar/m3parser/antlr/core/M3CoreParser.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,8 +316,10 @@ atomicExpression:

columnBuilders: TILDE (oneColSpec | (BRACKET_OPEN oneColSpec(COMMA oneColSpec)* BRACKET_CLOSE))
;
oneColSpec: columnName (COLON (type | lambdaParam lambdaPipe) extraFunction?)?

oneColSpec: identifier ((COLON (type | lambdaParam lambdaPipe) extraFunction? ))?
;

extraFunction: (COLON lambdaParam lambdaPipe)
;

Expand Down Expand Up @@ -468,9 +470,7 @@ type: ( qualifiedName (LESSTHAN (typeArguments? (PIPE multiplicityArguments)?) G
unitName
;

columnType: (QUESTION | columnName) COLON type
;
columnName: identifier | STRING
columnType : (QUESTION | identifier) COLON type
;

multiplicity: BRACKET_OPEN multiplicityArgument BRACKET_CLOSE
Expand Down
13 changes: 3 additions & 10 deletions ...nos/legend/pure/m3/serialization/grammar/m3parser/antlr/AntlrContextToM3CoreInstance.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1105,9 +1105,7 @@ else if (ctx.columnBuilders() != null)
MutableList<CoreInstance> extraFunction = Lists.mutable.empty();
ListIterate.forEach(ctx.columnBuilders().oneColSpec(), oneColSpec ->
{
M3Parser.ColumnNameContext colNameCtx = oneColSpec.columnName();
String colName = colNameCtx.STRING() != null ? this.removeQuotes(colNameCtx.STRING()) : colNameCtx.identifier().getText();
columnNames.add(this.repository.newStringCoreInstance(colName));
columnNames.add(this.repository.newStringCoreInstance(oneColSpec.identifier().getText()));
String returnType = null;
if (oneColSpec.lambdaParam() != null && oneColSpec.lambdaPipe() != null)
{
Expand All @@ -1123,7 +1121,7 @@ else if (oneColSpec.type() != null)
GenericType returnGType = type(oneColSpec.type(), typeParametersNames, "", importId, addLines);
returnType = returnGType._rawType().getName();
}
columnInstances.add(_Column.getColumnInstance(colName, false, relationTypeGenericType, returnType, src, processorSupport));
columnInstances.add(_Column.getColumnInstance(oneColSpec.identifier().getText(), false, relationTypeGenericType, returnType, src, processorSupport));
});
relationTypeGenericType._rawTypeCoreInstance(_RelationType.build(columnInstances, this.sourceInformation.getPureSourceInformation(ctx.getStart(), ctx.getStart(), ctx.getStop()), processorSupport));

Expand Down Expand Up @@ -2141,12 +2139,7 @@ public GenericType type(TypeContext ctx, MutableList<String> typeParametersNames
_RelationType.build(
ListIterate.collect(
ctx.columnType(),
c ->
{
M3Parser.ColumnNameContext colNameCtx = c.columnName();
String colName = colNameCtx != null ? colNameCtx.STRING() != null ? this.removeQuotes(colNameCtx.STRING()) : colNameCtx.identifier().getText() : "";
return _Column.getColumnInstance(c.QUESTION() != null ? "" : colName, c.QUESTION() != null, genericTypeInstance, this.type(c.type(), typeParametersNames, spacePlusTabs(space, 5), importId, addLines), srcInfo, processorSupport);
}
c -> _Column.getColumnInstance(c.QUESTION() != null ? "" : c.identifier().getText(), c.QUESTION() != null, genericTypeInstance, this.type(c.type(), typeParametersNames, spacePlusTabs(space, 5), importId, addLines), srcInfo, processorSupport)
), srcInfo, processorSupport
)
);
Expand Down
2 changes: 1 addition & 1 deletion legend-pure-core/legend-pure-m3-core/src/main/resources/platform/pure/relation.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ Class meta::pure::metamodel::relation::AggColSpec<Z,V,T> // Where T is of type R
reduce : Function<V>[1];
}

Class meta::pure::metamodel::relation::AggColSpecArray<A,B,T> // Where T is of type RelationType
Class meta::pure::metamodel::relation::AggColSpecArray<A,B,T>
{
aggSpecs : AggColSpec<A,B,Any>[*];
}
Expand Down
168 changes: 0 additions & 168 deletions ...elation-pure/src/main/resources/platform_functions_relation/relation/functions/pivot.pure
View file

This file was deleted.

4 changes: 1 addition & 3 deletions ...end/pure/runtime/java/extension/external/relation/compiled/RelationExtensionCompiled.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,7 @@ public List<Native> getExtraNatives()
new Slice(),
new Distinct(),
new Select(),
new SelectArray(),
new Pivot(),
new PivotArray()
new SelectArray()
);
}

Expand Down
91 changes: 0 additions & 91 deletions .../pure/runtime/java/extension/external/relation/compiled/RelationNativeImplementation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,8 @@
import org.eclipse.collections.api.set.primitive.MutableIntSet;
import org.eclipse.collections.api.tuple.Pair;
import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.*;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enum;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.SimpleFunctionExpression;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification;
import org.finos.legend.pure.m3.execution.ExecutionSupport;
import org.finos.legend.pure.m3.navigation.M3Properties;
import org.finos.legend.pure.m3.navigation.ProcessorSupport;
Expand All @@ -44,8 +41,6 @@
import org.finos.legend.pure.runtime.java.extension.external.relation.shared.SortInfo;
import org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS;

import java.util.Objects;

import static org.finos.legend.pure.runtime.java.extension.external.relation.shared.TestTDS.readCsv;

public class RelationNativeImplementation
Expand Down Expand Up @@ -232,21 +227,13 @@ public static class AggColSpecTrans
public Function2 map;
public Function2 reduce;
public String reduceType;
public AggColSpec<?, ?, ?> aggColSpec;

public AggColSpecTrans(String newColName, Function2 map, Function2 reduce, String reduceType)
{
this.newColName = newColName;
this.map = map;
this.reduce = reduce;
this.reduceType = reduceType;
this.aggColSpec = null;
}

public AggColSpecTrans(String newColName, Function2 map, Function2 reduce, String reduceType, AggColSpec<?, ?, ?> aggColSpec)
{
this(newColName, map, reduce, reduceType);
this.aggColSpec = aggColSpec;
}
}

Expand Down Expand Up @@ -313,84 +300,6 @@ private static void performMapReduce(Function2 map, Function2 reduce, ExecutionS
}
}

public static <T> Relation<? extends Object> pivot(Relation<? extends T> rel, ColSpec<?> pivotCols, MutableList<AggColSpecTrans> aggColSpecTrans, ExecutionSupport es)
{
return pivot(rel, Lists.mutable.with(pivotCols._name()), aggColSpecTrans, es);
}

public static <T> Relation<? extends Object> pivot(Relation<? extends T> rel, ColSpecArray<?> pivotCols, MutableList<AggColSpecTrans> aggColSpecTransAll, ExecutionSupport es)
{
return pivot(rel, Lists.mutable.withAll(pivotCols._names()), aggColSpecTransAll, es);
}

private static <T> Relation<? extends Object> pivot(Relation<? extends T> rel, ListIterable<String> pivotCols, MutableList<AggColSpecTrans> aggColSpecTransAll, ExecutionSupport es)
{
ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
TestTDSCompiled tds = RelationNativeImplementation.getTDS(rel);

// TODO: right now we make assumption that the map expression is really simple so we can safely extract the column(s)
// used for aggregation, we make sure these column(s) are not part of the groupBy calculation
ListIterable<String> columnsUsedInAggregation = aggColSpecTransAll.collect(col ->
{
try
{
ValueSpecification lambda = ((LambdaFunction<?>) col.aggColSpec._map())._expressionSequence().getFirst();
if (lambda instanceof SimpleFunctionExpression && ((SimpleFunctionExpression) lambda)._func() instanceof Column)
{
return ((SimpleFunctionExpression) lambda)._func()._name();
}
return null;
}
catch (Exception e)
{
// do nothing, the shape is not as expected, we will try to inspect no further
return null;
}
}).select(Objects::nonNull);

// these are the columns not being aggregated on, which will be used for groupBy calculation before transposing
ListIterable<String> groupByColumns = tds.getColumnNames().reject(c -> columnsUsedInAggregation.anySatisfy(a -> a.equals(c)) || pivotCols.anySatisfy(a -> a.equals(c))).withAll(pivotCols);

// create the big group-by table by processing all aggregations
Pair<TestTDS, MutableList<Pair<Integer, Integer>>> sorted = tds.sort(groupByColumns.collect(c -> new SortInfo(c, SortDirection.ASC)));
TestTDSCompiled temp = (TestTDSCompiled) sorted.getOne()._distinct(sorted.getTwo());
temp = aggColSpecTransAll.injectInto(temp, (existing, aggColSpecTrans) ->
{
int size = sorted.getTwo().size();
switch (aggColSpecTrans.reduceType)
{
case "String":
{
String[] finalRes = new String[size];
performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sorted, (o, j) -> finalRes[j] = (String) o);
existing.addColumn(aggColSpecTrans.newColName, DataType.STRING, finalRes);
break;
}
case "Integer":
{
int[] finalResInt = new int[size];
performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sorted, (o, j) -> finalResInt[j] = (int) (long) o);
existing.addColumn(aggColSpecTrans.newColName, DataType.INT, finalResInt);
break;
}
case "Float":
{
double[] finalResDouble = new double[size];
performMapReduce(aggColSpecTrans.map, aggColSpecTrans.reduce, es, size, sorted, (o, j) -> finalResDouble[j] = (double) o);
existing.addColumn(aggColSpecTrans.newColName, DataType.FLOAT, finalResDouble);
break;
}
}

return existing;
});

// transposing the table to complete pivoting
TestTDSCompiled result = (TestTDSCompiled) temp.applyPivot(groupByColumns.reject(pivotCols::contains), pivotCols, aggColSpecTransAll.collect(col -> col.newColName));

return new TDSContainer(result, ps);
}

public static Relation<?> project(RichIterable<?> objects, RichIterable<? extends ColFuncSpecTrans> colFuncs, ExecutionSupport es)
{
ProcessorSupport ps = ((CompiledExecutionSupport) es).getProcessorSupport();
Expand Down
Loading

0 comments on commit ffcb166

Please sign in to comment.