Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Route using map rather than fold to prevent state from parameters affecting the outer state #3310

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Route using map rather than fold to prevent state from parameters affecting the outer state #3310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions ...gend-engine-core-executionPlan-execution/legend-engine-executionPlan-dependencies/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,17 @@
</dependency>
<!-- ECLIPSE COLLECTIONS -->

<!-- JACKSON -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<!-- JACKSON -->

<!-- COMMONS CODEC -->
<dependency>
<groupId>commons-codec</groupId>
Expand Down
5 changes: 5 additions & 0 deletions ...dencies/src/main/java/org/finos/legend/engine/plan/dependencies/domain/date/PureDate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@

package org.finos.legend.engine.plan.dependencies.domain.date;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
import org.eclipse.collections.api.tuple.Pair;
import org.eclipse.collections.impl.block.factory.Comparators;
import org.eclipse.collections.impl.tuple.Tuples;
Expand All @@ -40,6 +43,7 @@
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;

@JsonSerialize(using = ToStringSerializer.class)
public class PureDate implements org.finos.legend.pure.m4.coreinstance.primitive.date.PureDate, Serializable
{
private static final long serialVersionUID = 8608696099577042248L;
Expand Down Expand Up @@ -1632,6 +1636,7 @@ else if ((second instanceof Float) || (second instanceof Double) || (second inst
* @param string string
* @return Pure date
*/
@JsonCreator(mode = JsonCreator.Mode.DELEGATING)
public static PureDate parsePureDate(String string)
{
return parsePureDate(string, 0, string.length());
Expand Down
2 changes: 1 addition & 1 deletion ...execution/src/main/java/org/finos/legend/engine/plan/execution/result/ConstantResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public <T> T accept(ResultVisitor<T> resultVisitor)
return resultVisitor.visit(this);
}

public Object stream() throws IOException
public String stream() throws IOException
{
return ConstantResultHelper.stream(this.getValue());
}
Expand Down
13 changes: 2 additions & 11 deletions ...n/java/org/finos/legend/engine/plan/execution/result/transformer/SetImplTransformers.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,11 @@

public class SetImplTransformers
{
public static final Function<Object, Object> TEMPORARY_DATATYPE_TRANSFORMER = (Function<Object, Object>) o ->
public static final Function<Object, Object> TEMPORARY_DATATYPE_TRANSFORMER = o ->
{
if (o instanceof Timestamp)
{
return DateFunctions.fromSQLTimestamp((Timestamp) o);
}
if (o instanceof java.sql.Date)
{
return StrictDate.fromSQLDate((java.sql.Date) o);
}
if (o instanceof Date)
{
PureDate pureDate = PureDate.fromDate((Date) o);
return pureDate.toString();
return PureDate.fromDate((Date) o);
}
return o;
};
Expand Down
2 changes: 1 addition & 1 deletion ...ine-pure-code-compiled-core/src/main/resources/core/pure/router/metamodel/clustering.pure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ function meta::pure::router::clustering::areClustersCompatible(cluster1:Clustere
function meta::pure::router::clustering::isFunctionSupportedByCluster(cluster:ClusteredValueSpecification[1], f:FunctionExpression[1]):Boolean[1]
{
$cluster->match([
sc:StoreMappingClusteredValueSpecification[1] |
sc:StoreClusteredValueSpecification[1] |
$sc.s.supports->toOne()->eval($f->evaluateAndDeactivate()),
ef:ExternalFormatClusteredValueSpecification[1] |
$f->meta::pure::router::externalFormat::clustering::externalFormatSupportsFunction(),
Expand Down
96 changes: 40 additions & 56 deletions ...e-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 20 additions & 0 deletions ...end-engine-pure-code-compiled-core/src/main/resources/core/pure/router/store/routing.pure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,26 @@ function <<access.private>> meta::pure::router::store::routing::wrapValueSpecifi
function meta::pure::router::store::routing::specializedFunctionExpressionRouterProcessorForStore(extensions:meta::pure::extension::Extension[*]):Pair<Function<{FunctionExpression[1]->Boolean[1]}>, Function<{Function<Any>[1], FunctionExpression[1], RoutingState[1], meta::pure::runtime::ExecutionContext[1], Map<VariableExpression, ValueSpecification>[1], Map<String, List<Any>>[1], meta::pure::extension::Extension[*], DebugContext[1]->RoutingState[1]}>>[*]
{
[
pair(fe:FunctionExpression[1] | $fe.func == letFunction_String_1__T_m__T_m_,
{f:Function<Any>[1], fe:FunctionExpression[1], state:RoutingState[1], executionContext:meta::pure::runtime::ExecutionContext[1], vars:Map<VariableExpression, ValueSpecification>[1], inScopeVars:Map<String, List<Any>>[1], extensions:meta::pure::extension::Extension[*], debug:DebugContext[1] |

let assigmentRouted = routeValueSpecification(
^$state(value = $fe.parametersValues->at(1)),
$executionContext,
$vars,
$inScopeVars,
$extensions,
$debug
);

let wrappedValueSpec = $assigmentRouted.value->evaluateAndDeactivate()->match([
evs: ExtendedRoutedValueSpecification[1] | $evs,
vs: ValueSpecification[1] | $assigmentRouted.routingStrategy.wrapValueSpec($vs, 'from wrapper', $executionContext, $extensions, $debug)
]);
let processedLet = ^$fe(parametersValues = $fe.parametersValues->at(0)->concatenate($wrappedValueSpec));
^$state(value = $processedLet, routingStrategy = meta::pure::router::platform::routing::getPlatformRoutingStrategy());
}
),
pair(fe:FunctionExpression[1] | $fe.func->in([meta::pure::mapping::from_T_m__Runtime_1__T_m_]),
{f:Function<Any>[1], fe:FunctionExpression[1], state:RoutingState[1], executionContext:meta::pure::runtime::ExecutionContext[1], vars:Map<VariableExpression, ValueSpecification>[1], inScopeVars:Map<String, List<Any>>[1], extensions:meta::pure::extension::Extension[*], debug:DebugContext[1] |
let resolvedParameters = $fe.parametersValues->tail()->map(p|$p->evaluateAndDeactivate()->match([v:VariableExpression[1] |let iv = meta::pure::functions::meta::resolve($v, $vars, $inScopeVars)->cast(@InstanceValue).values,
Expand Down
2 changes: 1 addition & 1 deletion ...ine-pure-code-compiled-core/src/main/resources/core/pure/serialization/toPureGrammar.pure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ function meta::pure::metamodel::serialization::grammar::printFunctionExpression(
|$main
);
$val->elementToPath() + '.all()';),
pair(|$name == 'letFunction',|'let '+$functionExpression.parametersValues->at(0)->cast(@InstanceValue).values->toOne()->toString()+ ' = ' +$functionExpression.parametersValues->at(0)->printValueSpecification($space)),
pair(|$name == 'letFunction',|'let '+$functionExpression.parametersValues->at(0)->cast(@InstanceValue).values->toOne()->toString()+ ' = ' +$functionExpression.parametersValues->at(1)->printValueSpecification($space)),
pair(|$name == 'colSpec', |'~'+$functionExpression.parametersValues->at(0)->cast(@InstanceValue).values->at(0)->toString()->printColName()),
pair(|$name == 'funcColSpec' || $name == 'funcColSpec2', | '~'+printColSpec($functionExpression, $configuration, $space)),
pair(|$name == 'aggColSpec' || $name == 'aggColSpec2', | '~'+printColSpec($functionExpression, $configuration, $space)),
Expand Down
4 changes: 2 additions & 2 deletions ...re/src/main/java/org/finos/legend/engine/shared/core/api/result/ConstantResultHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@

public class ConstantResultHelper
{
public static Object stream(Object value) throws IOException
public static String stream(Object value) throws IOException
{
return ObjectMapperFactory.getNewStandardObjectMapper().writeValueAsString(value);
}

public static Object stream(Object value, ObjectMapper objectMapper) throws IOException
public static String stream(Object value, ObjectMapper objectMapper) throws IOException
{
return objectMapper.writeValueAsString(value);
}
Expand Down
6 changes: 2 additions & 4 deletions ...on/src/main/java/org/finos/legend/engine/pure/runtime/execution/shared/LegendExecute.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.finos.legend.engine.plan.execution.result.StreamingResult;
import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat;
import org.finos.legend.engine.shared.core.identity.Identity;
import org.finos.legend.engine.shared.core.identity.factory.IdentityFactory;
import org.finos.legend.engine.shared.core.kerberos.SubjectTools;
import org.finos.legend.pure.m3.exception.PureExecutionException;
import org.finos.legend.pure.m4.coreinstance.CoreInstance;
Expand Down Expand Up @@ -51,8 +50,7 @@ public static String doExecute(String jsonPlan, Map<String, Object> variables, M
.build();

// execute plan
Result result = planExecutor.executeWithArgs(executeArgs);
try (AutoCloseable ignore = result::close)
try (Result result = planExecutor.executeWithArgs(executeArgs))
{
if (result instanceof StreamingResult)
{
Expand All @@ -62,7 +60,7 @@ public static String doExecute(String jsonPlan, Map<String, Object> variables, M
}
else if (result instanceof ConstantResult)
{
return ((ConstantResult) result).stream().toString();
return ((ConstantResult) result).stream();
}
else
{
Expand Down
67 changes: 35 additions & 32 deletions ...tionalStore-PCT-pure/src/main/resources/core_external_test_connection/pct_relational.pure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,38 +452,41 @@ function meta::relational::tests::pct::process::reprocess(a:Any[1], state:Proces
{
$a->match(
[
z:FunctionExpression[1]| if($z.func == new_Class_1__String_1__KeyExpression_MANY__T_1_,
| meta::relational::tests::pct::process::reprocessNew($z->reactivate()->evaluateAndDeactivate(), $z.genericType.rawType->toOne()->cast(@Class<Any>), $state);,
| if (!$z.func->in(
[
meta::pure::functions::relation::funcColSpecArray_FuncColSpec_MANY__P_1__FuncColSpecArray_1_,
meta::pure::functions::relation::funcColSpecArray2_FuncColSpec_MANY__P_1__FuncColSpecArray_1_,
meta::pure::functions::relation::funcColSpec_Function_1__String_1__T_1__FuncColSpec_1_,
meta::pure::functions::relation::funcColSpec2_Function_1__String_1__T_1__FuncColSpec_1_,
meta::pure::functions::relation::aggColSpecArray_AggColSpec_MANY__P_1__AggColSpecArray_1_,
meta::pure::functions::relation::aggColSpecArray2_AggColSpec_MANY__P_1__AggColSpecArray_1_,
meta::pure::functions::relation::aggColSpec_Function_1__Function_1__String_1__T_1__AggColSpec_1_,
meta::pure::functions::relation::aggColSpec2_Function_1__Function_1__String_1__T_1__AggColSpec_1_
]),
| let repro = $z.parametersValues->evaluateAndDeactivate()->map(x|$x->reprocess($state));
^$state
(
current = ^$z(parametersValues = if ($z->cast(@FunctionExpression).func == write_Relation_1__RelationElementAccessor_1__Integer_1_,
| let reproCurrent = $repro.current->cast(@ValueSpecification);
let tds = $reproCurrent->at(1)->cast(@FunctionExpression).parametersValues->at(0);
$reproCurrent->at(0)->concatenate($tds);,
| $repro.current->cast(@ValueSpecification))
)->evaluateAndDeactivate(),
mapping = $repro.mapping->first(),
replaced = $repro.replaced,
csvs = $repro.csvs
);,
| ^$state
(
current = $z
)
)
);,
z:FunctionExpression[1]|
let ifElseIf = [
pair(
|$z.func == new_Class_1__String_1__KeyExpression_MANY__T_1_,
|meta::relational::tests::pct::process::reprocessNew($z->reactivate()->evaluateAndDeactivate(), $z.genericType.rawType->toOne()->cast(@Class<Any>), $state)
),
pair(
|!$z.func->in(
[
meta::pure::functions::relation::funcColSpecArray_FuncColSpec_MANY__P_1__FuncColSpecArray_1_,
meta::pure::functions::relation::funcColSpecArray2_FuncColSpec_MANY__P_1__FuncColSpecArray_1_,
meta::pure::functions::relation::funcColSpec_Function_1__String_1__T_1__FuncColSpec_1_,
meta::pure::functions::relation::funcColSpec2_Function_1__String_1__T_1__FuncColSpec_1_,
meta::pure::functions::relation::aggColSpecArray_AggColSpec_MANY__P_1__AggColSpecArray_1_,
meta::pure::functions::relation::aggColSpecArray2_AggColSpec_MANY__P_1__AggColSpecArray_1_,
meta::pure::functions::relation::aggColSpec_Function_1__Function_1__String_1__T_1__AggColSpec_1_,
meta::pure::functions::relation::aggColSpec2_Function_1__Function_1__String_1__T_1__AggColSpec_1_
]
),
|let repro = $z.parametersValues->evaluateAndDeactivate()->map(x|$x->reprocess($state));
^$state
(
current = ^$z(parametersValues = if ($z->cast(@FunctionExpression).func == write_Relation_1__RelationElementAccessor_1__Integer_1_,
| let reproCurrent = $repro.current->cast(@ValueSpecification);
let tds = $reproCurrent->at(1)->cast(@FunctionExpression).parametersValues->at(0);
$reproCurrent->at(0)->concatenate($tds);,
| $repro.current->cast(@ValueSpecification))
)->evaluateAndDeactivate(),
mapping = $repro.mapping->first(),
replaced = $repro.replaced,
csvs = $repro.csvs
);
)
];
if($ifElseIf, |^$state(current = $z));,
ix:InstanceValue[1]|
let i = $ix->evaluateAndDeactivate();
let type = $i.genericType.rawType->toOne();
Expand Down
Loading