From 84c75762f4e7211becdcfcefab13511f2a12f6f7 Mon Sep 17 00:00:00 2001 From: Svyatoslav Reyentenko Date: Mon, 19 Aug 2013 18:12:49 +0400 Subject: [PATCH 1/3] Macros initial implementation --- .../groovy/transformations/MacroExpand.java | 14 ++ .../service/impl/GroovyTemplateService.scala | 9 +- .../MacroASTTransformation.scala | 154 ++++++++++++++++++ .../PhaseContainerASTTransformation.scala | 8 +- .../src/test/resources/groovy/Macros.genesis | 43 +++++ .../genesis/service/impl/MacrosTest.scala | 40 +++++ 6 files changed, 261 insertions(+), 7 deletions(-) create mode 100644 backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroExpand.java create mode 100644 backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala create mode 100644 backend/src/test/resources/groovy/Macros.genesis create mode 100644 backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala diff --git a/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroExpand.java b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroExpand.java new file mode 100644 index 000000000..3728a0092 --- /dev/null +++ b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroExpand.java @@ -0,0 +1,14 @@ +package com.griddynamics.genesis.template.dsl.groovy.transformations; + +import org.codehaus.groovy.transform.GroovyASTTransformationClass; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.SOURCE) +@Target({ElementType.TYPE, ElementType.METHOD}) +@GroovyASTTransformationClass({"com.griddynamics.genesis.template.dsl.groovy.transformations.MacroASTTransformation"}) +public @interface MacroExpand { +} diff --git a/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala b/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala index 51c04c1ec..ad6db89f7 100755 --- a/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala +++ b/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala @@ -47,7 +47,7 @@ import com.griddynamics.genesis.annotation.RemoteGateway import org.codehaus.groovy.control.CompilerConfiguration import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer import support.VariablesSupport -import com.griddynamics.genesis.template.dsl.groovy.transformations.{PhaseContainer, Context} +import com.griddynamics.genesis.template.dsl.groovy.transformations.{MacroExpand, PhaseContainer, Context} @RemoteGateway("groovy template service") class GroovyTemplateService(val templateRepoService : TemplateRepoService, @@ -147,8 +147,11 @@ class GroovyTemplateService(val templateRepoService : TemplateRepoService, try { val compilerConfiguration = new CompilerConfiguration() - compilerConfiguration.addCompilationCustomizers(new ASTTransformationCustomizer(classOf[Context]), - new ASTTransformationCustomizer(classOf[PhaseContainer])) + compilerConfiguration.addCompilationCustomizers( + new ASTTransformationCustomizer(classOf[MacroExpand]), + new ASTTransformationCustomizer(classOf[Context]), + new ASTTransformationCustomizer(classOf[PhaseContainer]) + ) val groovyShell = new GroovyShell(binding, compilerConfiguration) groovyShell.evaluate(body) projectId.foreach (evaluateIncludes(_, templateDecl.includes, groovyShell)) diff --git a/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala new file mode 100644 index 000000000..ddcf783ff --- /dev/null +++ b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala @@ -0,0 +1,154 @@ +package com.griddynamics.genesis.template.dsl.groovy.transformations + +import org.codehaus.groovy.transform.{ASTTransformation, GroovyASTTransformation} +import org.codehaus.groovy.control.{SourceUnit, CompilePhase} +import com.griddynamics.genesis.util.Logging +import org.codehaus.groovy.ast.{CodeVisitorSupport, ASTNode} +import org.codehaus.groovy.ast.expr._ +import scala.collection.mutable +import org.codehaus.groovy.ast.stmt.{ExpressionStatement, Statement, BlockStatement} +import java.util +import scala.collection.JavaConversions._ + +@GroovyASTTransformation(phase = CompilePhase.CONVERSION) +class MacroASTTransformation extends ASTTransformation with Logging { + def visit(nodes: Array[ASTNode], source: SourceUnit) { + log.trace(s"Inside a macros transformer: ${nodes.length}") + val methods = source.getAST.getStatementBlock.getStatements + val it = methods.iterator().next() + val collector = new MacroCollector + it.visit(new NameMatchArgumentTransformer("template", collector)) + if (collector.macros.size > 0) { + methods.iterator().next().visit(new MacroExpandVisitor(collector.macros.toMap)) + } + } +} + +class MacroExpandVisitor(val macrodefs: Map[String, ClosureExpression]) extends CodeVisitorSupport with Logging { + + val nameExtract: PartialFunction[Expression, String] = { + case call if call.isInstanceOf[MethodCallExpression] => call.asInstanceOf[MethodCallExpression].getMethodAsString + } + + def replaceStatement(statement: BlockStatement, expression: ExpressionStatement): Statement = { + val initialStatements: util.List[Statement] = statement.getStatements + val position = initialStatements.indexOf(expression) + expression.getExpression match { + case mce: MethodCallExpression => + val key: String = nameExtract(expression.getExpression) + log.trace(s"Expanding call of $key at position $position") + macrodefs.get(key).map(e => { + initialStatements.set(position, e.getCode) + }) orElse(throw new IllegalArgumentException(s"Macro $key not found anywhere in template")) + case other => + throw new IllegalArgumentException("Macro call must have a form of function call: " + expression.getExpression) + } + val cleared = initialStatements.filterNot(_.getStatementLabel == "macro") + new BlockStatement(cleared, statement.getVariableScope) + } + + override def visitClosureExpression(call: ClosureExpression) { + call.getCode match { + case bs: BlockStatement => { + bs.getStatements.foreach( statement => + statement match { + case es: ExpressionStatement => { + if (es.getStatementLabel == "macro") { + call.setCode(replaceStatement(bs, es)) + } else { + es.getExpression match { + case call: MethodCallExpression => { + this.visitMethodCallExpression(call) + } + case other => + } + } + } + case other => + } + ) + } + case other => + } + } + + + + override def visitMethodCallExpression(call: MethodCallExpression) { + call.getArguments match { + case ale: ArgumentListExpression => { + ale.visit(this) + } + case _ => + } + } +} + +class MacroCollector extends ExpressionTransformer with Logging { + var macros: mutable.Map[String, ClosureExpression] = new mutable.HashMap[String, ClosureExpression]() + def transform(expression: Expression) : Expression = { + expression match { + case a: ClosureExpression => { + a.getCode match { + case block: BlockStatement => { + val statements: util.List[Statement] = block.getStatements + val seq: Seq[Statement] = statements.toSeq + val macroDefs: Seq[Statement] = seq.filter(evalStatement) + if (macroDefs.size > 0) { + val rest: Seq[Statement] = seq.diff(macroDefs) + val content = new BlockStatement(rest.toArray, block.getVariableScope) + a.setCode(content) + } + macroDefs.foreach(m => { + m match { + case statement: ExpressionStatement => { + statement.getExpression match { + case mce: MethodCallExpression => saveExpr(mce) + case other => + } + } + case other => + } + }) + a + } + case x => a + } + + } + case x => x + } + } + + def saveExpr(expression: MethodCallExpression) { + def parseArgurmentList(mapExpr: NamedArgumentListExpression): (String, ClosureExpression) = { + val top = mapExpr.getMapEntryExpressions.iterator().next() + val key: String = top.getKeyExpression.asInstanceOf[ConstantExpression].getValue.toString + val closure = top.getValueExpression.asInstanceOf[ClosureExpression] + (key, closure) + } + val (name, body) = expression.getArguments match { + case mapExpr: NamedArgumentListExpression => + parseArgurmentList(mapExpr) + case tuple: TupleExpression => + parseArgurmentList(tuple.getExpression(0).asInstanceOf[NamedArgumentListExpression]) + } + macros(name) = body + } + + def evalStatement(statement: Statement): Boolean = { + statement match { + case es: ExpressionStatement => { + es.getExpression match { + case mce: MethodCallExpression => { + mce.getMethodAsString == "defmacro" + } + case other => false + } + } + case other => { + false + } + } + } +} diff --git a/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/PhaseContainerASTTransformation.scala b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/PhaseContainerASTTransformation.scala index 4ce0ec276..66aab813f 100644 --- a/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/PhaseContainerASTTransformation.scala +++ b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/PhaseContainerASTTransformation.scala @@ -18,14 +18,14 @@ class PhaseContainerASTTransformation extends ASTTransformation with Logging{ def visit(nodes: Array[ASTNode], source: SourceUnit) { val methods = source.getAST.getStatementBlock.getStatements val it = methods.iterator().next() - it.visit(new PhaseEraser) + it.visit(new NameMatchArgumentTransformer("steps", new PhaseTransformer)) } } -class PhaseEraser extends CodeVisitorSupport with Logging { +class NameMatchArgumentTransformer(name: String, transformer: ExpressionTransformer) extends CodeVisitorSupport with Logging { override def visitMethodCallExpression(call: MethodCallExpression) { - if (call.getMethodAsString == "steps") { - val expr = call.getArguments.transformExpression(new PhaseTransformer) + if (call.getMethodAsString == name) { + val expr = call.getArguments.transformExpression(transformer) call.setArguments(expr) } else { call.getArguments.visit(this) diff --git a/backend/src/test/resources/groovy/Macros.genesis b/backend/src/test/resources/groovy/Macros.genesis new file mode 100644 index 000000000..cefd07fbc --- /dev/null +++ b/backend/src/test/resources/groovy/Macros.genesis @@ -0,0 +1,43 @@ +package groovy +template { + name("Macros") + version("0.1") + createWorkflow("create") + destroyWorkflow("destroy") + + + defmacro "create_steps": { + teststep { + text = "test from macro" + } + } + + workflow("create") { + steps { + teststep { + text = "test input" + } + teststep { + text = "another input" + } + } + } + + workflow("macros") { + steps { + teststep { + text = "It was here" + } + macro:create_steps() + } + } + + workflow("destroy") { + steps { + teststep { + phase = "undeploy" + text = "destroy" + } + } + } +} \ No newline at end of file diff --git a/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala b/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala new file mode 100644 index 000000000..5e14a2631 --- /dev/null +++ b/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala @@ -0,0 +1,40 @@ +package com.griddynamics.genesis.service.impl + +import org.scalatest.junit.AssertionsForJUnit +import org.scalatest.mock.MockitoSugar +import org.springframework.core.convert.support.DefaultConversionService +import com.griddynamics.genesis.template.ListVarDSFactory +import com.griddynamics.genesis.util.{Logging, IoUtil} +import org.mockito.{Matchers, Mockito} +import org.mockito.Mockito._ +import scala.Some +import org.junit.Test +import com.griddynamics.genesis.template.support.DatabagDataSourceFactory +import com.griddynamics.genesis.cache.NullCacheManager +import com.griddynamics.genesis.template.VersionedTemplate +import com.griddynamics.genesis.api +import com.griddynamics.genesis.plugin.StepBuilder + + +class MacrosTest extends AssertionsForJUnit with MockitoSugar with DSLTestUniverse with Logging { + val templateService = new GroovyTemplateService(templateRepoService, + List(new DoNothingStepBuilderFactory), new DefaultConversionService, + Seq(new ListVarDSFactory, new DependentListVarDSFactory, + new DatabagDataSourceFactory(databagRepository)), databagRepository, configService, NullCacheManager) + + val body = IoUtil.streamAsString(classOf[GroovyTemplateServiceTest].getResourceAsStream("/groovy/Macros.genesis")) + + Mockito.when(templateRepository.listSources()).thenReturn(Map(VersionedTemplate("1") -> body)) + when(configService.get(Matchers.any(), Matchers.any())).thenReturn(Some(new api.Configuration(Some(0), "", 0, None, Map()))) + + @Test + def testPhaseApplied() { + val createWorkflow = templateService.findTemplate(0, "Macros", "0.1", 0).flatMap(_.getWorkflow("macros")).get + val steps = createWorkflow.embody(Map()) + expectResult(2)(steps.regular.size) + val initialPhase: Option[StepBuilder] = steps.regular.find(_.phase == "auto_0") + assert(initialPhase.isDefined) + assert(initialPhase.get.getPrecedingPhases.isEmpty) + } +} + From 7d1a51d11fe4b68a0909823c1144b95bc74f5caf Mon Sep 17 00:00:00 2001 From: Svyatoslav Reyentenko Date: Wed, 21 Aug 2013 16:01:15 +0400 Subject: [PATCH 2/3] [#1028][feature] - Reuse DSL code with macros --- .../exceptions/DSLSyntaxException.java | 8 + .../exceptions/MacroParametersException.java | 8 + .../service/impl/GroovyTemplateService.scala | 4 +- .../MacroASTTransformation.scala | 217 ++++++++++++++++-- .../src/test/resources/groovy/Macros.genesis | 39 +++- .../genesis/service/impl/MacrosTest.scala | 60 ++++- 6 files changed, 309 insertions(+), 27 deletions(-) create mode 100644 backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/DSLSyntaxException.java create mode 100644 backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/MacroParametersException.java diff --git a/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/DSLSyntaxException.java b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/DSLSyntaxException.java new file mode 100644 index 000000000..6f319a23e --- /dev/null +++ b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/DSLSyntaxException.java @@ -0,0 +1,8 @@ +package com.griddynamics.genesis.template.dsl.groovy.transformations.exceptions; + + +public class DSLSyntaxException extends Exception { + public DSLSyntaxException(String message) { + super(message); + } +} diff --git a/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/MacroParametersException.java b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/MacroParametersException.java new file mode 100644 index 000000000..7bd8f1562 --- /dev/null +++ b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/MacroParametersException.java @@ -0,0 +1,8 @@ +package com.griddynamics.genesis.template.dsl.groovy.transformations.exceptions; + + +public class MacroParametersException extends DSLSyntaxException { + public MacroParametersException(String message) { + super(message); + } +} diff --git a/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala b/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala index ad6db89f7..fe9a5c4f8 100755 --- a/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala +++ b/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala @@ -48,6 +48,7 @@ import org.codehaus.groovy.control.CompilerConfiguration import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer import support.VariablesSupport import com.griddynamics.genesis.template.dsl.groovy.transformations.{MacroExpand, PhaseContainer, Context} +import scala.tools.ant.sabbus.CompilationFailure @RemoteGateway("groovy template service") class GroovyTemplateService(val templateRepoService : TemplateRepoService, @@ -156,7 +157,8 @@ class GroovyTemplateService(val templateRepoService : TemplateRepoService, groovyShell.evaluate(body) projectId.foreach (evaluateIncludes(_, templateDecl.includes, groovyShell)) } catch { - case e: GroovyRuntimeException => throw new IllegalStateException("can't process template", e) + case compilation: CompilationFailure => throw new IllegalStateException("Conpilation error: " + compilation.message, compilation) + case e: GroovyRuntimeException => throw new IllegalStateException("can't process template: " + e.getMessage, e) } templateDecl.bodies.headOption.map { body => DslDelegate(body).to(builder).newTemplate() } } diff --git a/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala index ddcf783ff..64170bac7 100644 --- a/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala +++ b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala @@ -3,12 +3,14 @@ package com.griddynamics.genesis.template.dsl.groovy.transformations import org.codehaus.groovy.transform.{ASTTransformation, GroovyASTTransformation} import org.codehaus.groovy.control.{SourceUnit, CompilePhase} import com.griddynamics.genesis.util.Logging -import org.codehaus.groovy.ast.{CodeVisitorSupport, ASTNode} +import org.codehaus.groovy.ast.{Parameter, CodeVisitorSupport, ASTNode} import org.codehaus.groovy.ast.expr._ import scala.collection.mutable import org.codehaus.groovy.ast.stmt.{ExpressionStatement, Statement, BlockStatement} import java.util import scala.collection.JavaConversions._ +import com.griddynamics.genesis.template.dsl.groovy.transformations.exceptions.MacroParametersException +import org.codehaus.groovy.control.messages.SimpleMessage @GroovyASTTransformation(phase = CompilePhase.CONVERSION) class MacroASTTransformation extends ASTTransformation with Logging { @@ -16,21 +18,171 @@ class MacroASTTransformation extends ASTTransformation with Logging { log.trace(s"Inside a macros transformer: ${nodes.length}") val methods = source.getAST.getStatementBlock.getStatements val it = methods.iterator().next() - val collector = new MacroCollector + val collector = new MacroCollector(source) it.visit(new NameMatchArgumentTransformer("template", collector)) if (collector.macros.size > 0) { - methods.iterator().next().visit(new MacroExpandVisitor(collector.macros.toMap)) + val replacementMap = new mutable.HashMap[Statement, Expression]() + log.trace(s"Found ${collector.macros.size} macros") + methods.iterator().next().visit(new MacroExpandVisitor(collector.macros.toMap, replacementMap, source)) } + log.trace("Done") } } -class MacroExpandVisitor(val macrodefs: Map[String, ClosureExpression]) extends CodeVisitorSupport with Logging { +trait SourceReporting { + def source: SourceUnit + def error[B](message: String)(value: B) = { + source.getErrorCollector.addError(new SimpleMessage(message, "", source)) + value + } +} + +class ApplyVariablesVisitor(values: Map[String, Expression], replacements: mutable.Map[Statement, Expression]) extends CodeVisitorSupport with ExpressionTransformer + with Logging { + override def visitExpressionStatement(statement: ExpressionStatement) { + val expression: Expression = statement.getExpression + //expression.visit(this) + val toChange: Expression = expression + val newExpression: Expression = toChange.transformExpression(this) + statement.setExpression(newExpression) + super.visitExpressionStatement(statement) + } + + + override def visitMapEntryExpression(expression: MapEntryExpression) { + val key: Expression = expression.getKeyExpression + val value: Expression = expression.getValueExpression + expression.setKeyExpression(transform(key)) + expression.setValueExpression(transform(value)) + super.visitMapEntryExpression(expression) + } + + override def visitBinaryExpression(expression: BinaryExpression) { + expression.setLeftExpression(transform(expression.getLeftExpression)) + expression.setRightExpression(transform(expression.getRightExpression)) + super.visitBinaryExpression(expression) + } + + def transform(expression: Expression): Expression = { + expression match { + case variable: VariableExpression => { + values.get(variable.getName).getOrElse(variable) + } + case constant: ConstantExpression if constant.getText.startsWith("$") => + values.get(constant.getText).getOrElse(constant) + case other => other + } + } +} +class MacroExpandVisitor(val macrodefs: Map[String, Macro], replacements: mutable.Map[Statement,Expression], val source: SourceUnit) extends CodeVisitorSupport with Logging with SourceReporting { val nameExtract: PartialFunction[Expression, String] = { case call if call.isInstanceOf[MethodCallExpression] => call.asInstanceOf[MethodCallExpression].getMethodAsString } + private def substitute(code: Macro, call: MethodCallExpression): Statement = { + val arguments: Expression = call.getArguments + //possible variants: ArgumentListExpression, Map + val passed: Seq[PassedParameter] = arguments match { + case list: ArgumentListExpression => list.getExpressions.zipWithIndex.map({case (expr, index) => new PositionedParameter(index, expr)}).toSeq + case tuple: TupleExpression => tuple.getExpressions.flatMap(e => e match { + case named: NamedArgumentListExpression => { + named.getMapEntryExpressions.map(expr => expr.getKeyExpression match { + case constant: ConstantExpression => Some(NamedParameter(constant.getText, expr.getValueExpression)) + case other => error("Only constants allowed in macro named arguments")(None) + }) + } + }).flatten + case other => error("Macro parameters can be passed as comma-separated list or as named pairs")(Seq()) + } + //apply + val values: Map[String, Option[Expression]] = code.parameters.zipWithIndex.map({case ((name, value), idx) => { + val found = passed.find({ + case named: NamedParameter => named.name == name + case pos: PositionedParameter => pos.position == idx + }).map(_.value).orElse(value) + (name, found) + }}).toMap + val unsassigned = values.collect({case (name, value) if value.isEmpty || value.get == null => name}) + if (unsassigned.size > 0) { + error(s"Some arguments for macro `${code.name}` were not assigned and they don't have default value: " + unsassigned.mkString(", "))(code.code) + } else { + applyArguments(values.map({case (name, expr) => (name, expr.get)}), code.code) + } + } + + private def applyArguments(values: Map[String, Expression], code: BlockStatement) : BlockStatement = { + val visitor = new ApplyVariablesVisitor(values, replacements) + log.debug(s"Applying variables $values") + val copy = copyBlock(code) + log.trace(s"Initial code is: $code") + copy.visit(visitor) + log.debug(s"Copy with expanded variables: $copy") + copy + } - def replaceStatement(statement: BlockStatement, expression: ExpressionStatement): Statement = { + private def copyBlock(original: BlockStatement): BlockStatement = { + val bs = new BlockStatement() + addBlockToBlock(original, bs) + bs + } + + private def addBlockToBlock(source: BlockStatement, target: BlockStatement) { + source.getStatements.foreach(addStatement(target)) + } + + private val copy: PartialFunction[Expression, Expression] = { + case variable: VariableExpression => new VariableExpression(variable.getName) + case constant: ConstantExpression => new ConstantExpression(constant.getText) + case binary: BinaryExpression => new BinaryExpression(copy(binary.getLeftExpression), binary.getOperation, + copy(binary.getRightExpression)) + case closure: ClosureExpression => copyClosureExpression(closure) + case arguments: ArgumentListExpression => new ArgumentListExpression(arguments.getExpressions.map(copy)) + case mapEntry: MapEntryExpression => new MapEntryExpression(copy(mapEntry.getKeyExpression), copy(mapEntry.getValueExpression)) + } + + private def copyClosureExpression(closure: ClosureExpression): ClosureExpression = { + def copyParameters (closure: ClosureExpression) = { + closure.getParameters.map(p => new Parameter(p.getType, p.getName, copy(p.getInitialExpression))) + } + closure.getCode match { + case bs: BlockStatement => new ClosureExpression(copyParameters(closure), copyBlock(bs)) + } + } + + private def addBinaryExpression(be: BinaryExpression, bs: BlockStatement) = { + bs.addStatement(new ExpressionStatement(copy(be))) + } + + private def addMethodCall(objectExpr: Expression, method: String, arguments: Expression, statement: BlockStatement) { + val newExpr = new MethodCallExpression(objectExpr, method, copy(arguments)) + statement.addStatement(new ExpressionStatement(newExpr)) + } + + private def addMapExpression(map: MapExpression, statement: BlockStatement) { + val copies = map.getMapEntryExpressions.map(entry => { + copy(entry).asInstanceOf[MapEntryExpression] + }) + val newExpr = new MapExpression(copies.toList) + statement.addStatement(new ExpressionStatement(newExpr)) + } + + private def addStatement(blockStatement: BlockStatement)(statement: Statement) { + statement match { + case bs: BlockStatement => addBlockToBlock(bs, blockStatement) + case es: ExpressionStatement => { + val expr: Expression = es.getExpression + expr match { + case constant: ConstantExpression => blockStatement.addStatement(es) + case de: DeclarationExpression => blockStatement.addStatement(es) + case be: BinaryExpression => addBinaryExpression(be, blockStatement) + case method: MethodCallExpression => addMethodCall(method.getObjectExpression, method.getMethodAsString, method.getArguments, blockStatement) + case map: MapExpression => addMapExpression(map, blockStatement) + } + } + } + } + + private def replaceStatement(statement: BlockStatement, expression: ExpressionStatement): Statement = { val initialStatements: util.List[Statement] = statement.getStatements val position = initialStatements.indexOf(expression) expression.getExpression match { @@ -38,10 +190,12 @@ class MacroExpandVisitor(val macrodefs: Map[String, ClosureExpression]) extends val key: String = nameExtract(expression.getExpression) log.trace(s"Expanding call of $key at position $position") macrodefs.get(key).map(e => { - initialStatements.set(position, e.getCode) - }) orElse(throw new IllegalArgumentException(s"Macro $key not found anywhere in template")) + val code: Statement = if (e.parameters.isEmpty) e.code else substitute(e, mce) + initialStatements.set(position, code) + code + }) orElse error(s"Macro $key not found anywhere in template")(None) case other => - throw new IllegalArgumentException("Macro call must have a form of function call: " + expression.getExpression) + error("Macro call must have a form of function call: " + expression.getExpression)_ } val cleared = initialStatements.filterNot(_.getStatementLabel == "macro") new BlockStatement(cleared, statement.getVariableScope) @@ -84,8 +238,23 @@ class MacroExpandVisitor(val macrodefs: Map[String, ClosureExpression]) extends } } -class MacroCollector extends ExpressionTransformer with Logging { - var macros: mutable.Map[String, ClosureExpression] = new mutable.HashMap[String, ClosureExpression]() +case class Macro(name: String, parameters: List[Macro.ParameterDefinition], code: BlockStatement) { + val parametersMap = parameters.toMap + val zippedMap = parameters.zipWithIndex.map({case (definition,index) => (index,definition)}).toMap +} + +sealed trait PassedParameter { + def value: Expression +} +case class NamedParameter(name: String, value: Expression) extends PassedParameter +case class PositionedParameter(position: Int, value: Expression) extends PassedParameter + +object Macro { + type ParameterDefinition = (String, Option[Expression]) +} + +class MacroCollector(val source: SourceUnit) extends ExpressionTransformer with Logging with SourceReporting { + var macros: mutable.Map[String, Macro] = new mutable.HashMap[String, Macro]() def transform(expression: Expression) : Expression = { expression match { case a: ClosureExpression => { @@ -121,19 +290,33 @@ class MacroCollector extends ExpressionTransformer with Logging { } def saveExpr(expression: MethodCallExpression) { - def parseArgurmentList(mapExpr: NamedArgumentListExpression): (String, ClosureExpression) = { + def parseArgumentList(mapExpr: NamedArgumentListExpression): Option[(String, ClosureExpression)] = { val top = mapExpr.getMapEntryExpressions.iterator().next() val key: String = top.getKeyExpression.asInstanceOf[ConstantExpression].getValue.toString - val closure = top.getValueExpression.asInstanceOf[ClosureExpression] - (key, closure) + top.getValueExpression match { + case closure: ClosureExpression => Some(key, closure) + case other => error(s"Macro body must be a closure: $key; actual $other")(None) + } + } + def parseMacro(name: String, closure: ClosureExpression): Option[Macro] = { + val parameters = closure.getParameters.map(parameter => { + new Macro.ParameterDefinition(parameter.getName, Some(parameter.getInitialExpression)) + }) + closure.getCode match { + case bs: BlockStatement => Some(Macro(name, parameters.toList, closure.getCode.asInstanceOf[BlockStatement])) + case other => error(s"Macro body must be a block statement: $name; actual: $other")(None) + } + } - val (name, body) = expression.getArguments match { + val nameAndCode: Option[(String, ClosureExpression)] = expression.getArguments match { case mapExpr: NamedArgumentListExpression => - parseArgurmentList(mapExpr) + parseArgumentList(mapExpr) case tuple: TupleExpression => - parseArgurmentList(tuple.getExpression(0).asInstanceOf[NamedArgumentListExpression]) + tuple.getExpression(0) match { + case arguments: NamedArgumentListExpression => parseArgumentList(arguments) + } } - macros(name) = body + nameAndCode.flatMap ({case (name,body) => parseMacro(name,body) }) map (result => macros(result.name) = result) } def evalStatement(statement: Statement): Boolean = { diff --git a/backend/src/test/resources/groovy/Macros.genesis b/backend/src/test/resources/groovy/Macros.genesis index cefd07fbc..788ea97d6 100644 --- a/backend/src/test/resources/groovy/Macros.genesis +++ b/backend/src/test/resources/groovy/Macros.genesis @@ -5,13 +5,36 @@ template { createWorkflow("create") destroyWorkflow("destroy") + def MY_CONSTANT = "Set from constant" - defmacro "create_steps": { + defmacro "create_steps": { $message = "default" -> + 1 + 2 teststep { - text = "test from macro" + text = $message } } + defmacro "simple": { + teststep { + text = MY_CONSTANT + } + } + + defmacro "map": { $key, $mapValue -> + withMap { + text = "Test" + map = { [ $key: $mapValue ] } + } + } + + defmacro "bad_require": { $message -> + require { + $message { 1 == 2 } + "Oops again" { 2 == 1 } + } + + } + workflow("create") { steps { teststep { @@ -23,12 +46,22 @@ template { } } + workflow("maps") { + steps { + macro:map("operation", "subst") + } + } + workflow("macros") { + macro:bad_require("Oops") steps { teststep { - text = "It was here" + text = "Static" } + macro:create_steps("Passed from macro call") + macro:create_steps($message: "Set with map") macro:create_steps() + macro:simple() } } diff --git a/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala b/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala index 5e14a2631..617ec646f 100644 --- a/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala +++ b/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala @@ -13,12 +13,18 @@ import com.griddynamics.genesis.template.support.DatabagDataSourceFactory import com.griddynamics.genesis.cache.NullCacheManager import com.griddynamics.genesis.template.VersionedTemplate import com.griddynamics.genesis.api -import com.griddynamics.genesis.plugin.StepBuilder +import com.griddynamics.genesis.plugin.{StepBuilderFactory, StepBuilder} +import com.griddynamics.genesis.service.TemplateDefinition +import com.griddynamics.genesis.api.Failure +import com.griddynamics.genesis.workflow.Step +import scala.beans.BeanProperty +import scala.collection.mutable +import scala.collection.JavaConversions._ class MacrosTest extends AssertionsForJUnit with MockitoSugar with DSLTestUniverse with Logging { val templateService = new GroovyTemplateService(templateRepoService, - List(new DoNothingStepBuilderFactory), new DefaultConversionService, + List(new DoNothingStepBuilderFactory, new StepWithMapFactory), new DefaultConversionService, Seq(new ListVarDSFactory, new DependentListVarDSFactory, new DatabagDataSourceFactory(databagRepository)), databagRepository, configService, NullCacheManager) @@ -28,13 +34,55 @@ class MacrosTest extends AssertionsForJUnit with MockitoSugar with DSLTestUniver when(configService.get(Matchers.any(), Matchers.any())).thenReturn(Some(new api.Configuration(Some(0), "", 0, None, Map()))) @Test - def testPhaseApplied() { - val createWorkflow = templateService.findTemplate(0, "Macros", "0.1", 0).flatMap(_.getWorkflow("macros")).get - val steps = createWorkflow.embody(Map()) - expectResult(2)(steps.regular.size) + def testStepsInserted() { + val template: Option[TemplateDefinition] = templateService.findTemplate(0, "Macros", "0.1", 0) + val workflow = template.flatMap(_.getWorkflow("macros")).get + val steps = workflow.embody(Map()) + expectResult(5)(steps.regular.size) val initialPhase: Option[StepBuilder] = steps.regular.find(_.phase == "auto_0") + val secondPhase: Option[StepBuilder] = steps.regular.find(_.phase == "auto_1") assert(initialPhase.isDefined) assert(initialPhase.get.getPrecedingPhases.isEmpty) + assert(secondPhase.isDefined) + assert(secondPhase.get.getPrecedingPhases.contains("auto_0")) + steps.regular.zip(Seq("Static", "Passed from macro call", + "Set with map", "default", "Set from constant")).map({ + case (step, message) => step.newStep.actualStep match { + case nothing: DoNothingStep => expectResult(message)(nothing.name) + } + }) + } + + @Test + def testFullRequireFromMacro() { + val template: Option[TemplateDefinition] = templateService.findTemplate(0, "Macros", "0.1", 0) + assert(template.isDefined) + val result = template.map(_.getValidWorkflow("macros")).get + expectResult(Failure(compoundServiceErrors = Seq("Oops", "Oops again")))(result) + } + + @Test + def testMacroWithMap() { + val template: Option[TemplateDefinition] = templateService.findTemplate(0, "Macros", "0.1", 0) + val result = template.get.getWorkflow("maps").get + val steps = result.embody(Map()) + val values: Map[String, String] = steps.regular(0).newStep.actualStep.asInstanceOf[StepWithMap].values + expectResult(Map("operation" -> "subst"))(values) + } +} + +case class StepWithMap(name: String, values: Map[String,String]) extends Step { + override def stepDescription = "Best step ever!" +} + +class StepWithMapFactory extends StepBuilderFactory { + val stepName = "withMap" + + def newStepBuilder = new StepBuilder { + @BeanProperty var text: String = _ + @BeanProperty var map: java.util.Map[String, String] = _ + + def getDetails = StepWithMap(text, map.toMap) } } From 38ee5125401a81a3727e428737f3b2bff43df5a8 Mon Sep 17 00:00:00 2001 From: Svyatoslav Reyentenko Date: Fri, 23 Aug 2013 14:05:41 +0400 Subject: [PATCH 3/3] [#1028][feature] - Reusing code with macros --- .../transformations/MacroASTTransformation.scala | 1 + backend/src/test/resources/groovy/Macros.genesis | 12 +++++++++++- .../genesis/service/impl/MacrosTest.scala | 12 +++++++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala index 64170bac7..beec21f02 100644 --- a/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala +++ b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala @@ -138,6 +138,7 @@ class MacroExpandVisitor(val macrodefs: Map[String, Macro], replacements: mutabl case closure: ClosureExpression => copyClosureExpression(closure) case arguments: ArgumentListExpression => new ArgumentListExpression(arguments.getExpressions.map(copy)) case mapEntry: MapEntryExpression => new MapEntryExpression(copy(mapEntry.getKeyExpression), copy(mapEntry.getValueExpression)) + case prop: PropertyExpression => new PropertyExpression(copy(prop.getObjectExpression), copy(prop.getProperty)) } private def copyClosureExpression(closure: ClosureExpression): ClosureExpression = { diff --git a/backend/src/test/resources/groovy/Macros.genesis b/backend/src/test/resources/groovy/Macros.genesis index 788ea97d6..40cdddacf 100644 --- a/backend/src/test/resources/groovy/Macros.genesis +++ b/backend/src/test/resources/groovy/Macros.genesis @@ -22,11 +22,18 @@ template { defmacro "map": { $key, $mapValue -> withMap { - text = "Test" + text = $vars.myvar map = { [ $key: $mapValue ] } } } + defmacro "defvar": { $name -> + $name = { + description = "Variable from template" + isOptional = false + } + } + defmacro "bad_require": { $message -> require { $message { 1 == 2 } @@ -47,6 +54,9 @@ template { } workflow("maps") { + variables { + macro:defvar(myvar) + } steps { macro:map("operation", "subst") } diff --git a/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala b/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala index 617ec646f..8ef877679 100644 --- a/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala +++ b/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala @@ -14,7 +14,7 @@ import com.griddynamics.genesis.cache.NullCacheManager import com.griddynamics.genesis.template.VersionedTemplate import com.griddynamics.genesis.api import com.griddynamics.genesis.plugin.{StepBuilderFactory, StepBuilder} -import com.griddynamics.genesis.service.TemplateDefinition +import com.griddynamics.genesis.service.{VariableDescription, TemplateDefinition} import com.griddynamics.genesis.api.Failure import com.griddynamics.genesis.workflow.Step import scala.beans.BeanProperty @@ -65,9 +65,15 @@ class MacrosTest extends AssertionsForJUnit with MockitoSugar with DSLTestUniver def testMacroWithMap() { val template: Option[TemplateDefinition] = templateService.findTemplate(0, "Macros", "0.1", 0) val result = template.get.getWorkflow("maps").get - val steps = result.embody(Map()) - val values: Map[String, String] = steps.regular(0).newStep.actualStep.asInstanceOf[StepWithMap].values + val steps = result.embody(Map("myvar" -> "1024")) + val variables: Seq[VariableDescription] = result.variableDescriptions + assert(variables.size == 1) + assert(variables.exists(_.name == "myvar")) + val step: StepWithMap = steps.regular(0).newStep.actualStep.asInstanceOf[StepWithMap] + val values: Map[String, String] = step.values + val text = step.name expectResult(Map("operation" -> "subst"))(values) + expectResult("1024")(text) } }