Fix macro reification if else and more... (#199)

* fixed Dollar handling
* fixed expression handling after BkClose
* fixed expression handling after BrClose
* fixed POpen type detection
* fixed new handling
* added hasTempStore function
* use hasTempStore
* fixed sharp handling for function type hint
* updated gitignore
* new version 1.0.29
* fixed array access on new
* fixed colon after macro
* fixed arrow functions

.gitignore

@@ -21,4 +21,5 @@ check-style-report.json
 check-style-report.xml
 coverage.json
 lcov.info
-display.hxml
+display.hxml
+codecov.json

CHANGELOG.md

@@ -2,12 +2,24 @@
 
 ## dev branch / next version (1.x.x)
 
+## version 1.0.29 (2020-11-01)
+
 - Added TokenTreeDef ([#191](https://github.com/HaxeCheckstyle/tokentree/issues/191))
 - Added support for abstract classes / interfaces ([#195](https://github.com/HaxeCheckstyle/tokentree/issues/195))
+- Added hasTempStore function ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
 - Fixed negative const / Binop(OpSub) detection ([#189](https://github.com/HaxeCheckstyle/tokentree/issues/189))
 - Fixed final with multiple vars ([#191](https://github.com/HaxeCheckstyle/tokentree/issues/191))
 - Fixed is operator ([#194](https://github.com/HaxeCheckstyle/tokentree/issues/194))
 - Fixed semicolon with multiple binops ([#197](https://github.com/HaxeCheckstyle/tokentree/issues/197) + [#198](https://github.com/HaxeCheckstyle/tokentree/issues/198))
+- Fixed Dollar handling ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
+- Fixed expression handling after BkClose ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
+- Fixed expression handling after BrClose ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
+- Fixed POpen type detection ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
+- Fixed new handling ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
+- Fixed sharp handling for function type hint ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
+- Fixed macro complextype ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
+- Fixed array access on new ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
+- Fixed arrow functions ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199))
 - Refactored enums to use CamelCase ([#191](https://github.com/HaxeCheckstyle/tokentree/issues/191))
 - Refactored to reduce usage of Type.enumEq ([#195](https://github.com/HaxeCheckstyle/tokentree/issues/195) + [#196](https://github.com/HaxeCheckstyle/tokentree/issues/196))
 - Removed `is` operator ([#195](https://github.com/HaxeCheckstyle/tokentree/issues/195))

haxelib.json

@@ -8,7 +8,7 @@
         "AlexHaxe"
     ],
     "releasenote": "fixed @in handling, detects more POpen types - see CHANGELOG for details",
-    "version": "1.0.28",
+    "version": "1.0.29",
     "url": "https://github.com/HaxeCheckstyle/tokentree",
     "dependencies": {}
 }

package.json

@@ -1,6 +1,6 @@
 {
     "name": "tokentree",
-    "version": "1.0.28",
+    "version": "1.0.29",
     "description": "TokenTree library used by haxe-checkstyle, haxe-formatter and haxe-languageserver",
     "repository": {
         "type": "git",

src/tokentree/TokenStream.hx

@@ -96,6 +96,10 @@ class TokenStream {
 		}
 	}
 
+	public function hasTempStore():Bool {
+		return tempStore.length > 0;
+	}
+
 	public function getTempStore():Array<TokenTree> {
 		return tempStore;
 	}

src/tokentree/utils/TokenTreeCheckUtils.hx

@@ -49,8 +49,7 @@ class TokenTreeCheckUtils {
 
 	public static function isOpGtTypedefExtension(token:TokenTree):Bool {
 		return switch (token.tok) {
-			case Binop(OpGt): (token.access().parent().matches(BrOpen).parent()
-					.matches(Binop(OpAssign)).parent().isCIdent().parent().matches(Kwd(KwdTypedef))
+			case Binop(OpGt): (token.access().parent().matches(BrOpen).parent().matches(Binop(OpAssign)).parent().isCIdent().parent().matches(Kwd(KwdTypedef))
 					.token != null);
 			default: false;
 		}
@@ -199,8 +198,7 @@ class TokenTreeCheckUtils {
 					if (!child.tok.match(At)) {
 						continue;
 					}
-					var enumTok = child.access().firstChild().matches(DblDot).firstChild()
-						.matches(Kwd(KwdEnum));
+					var enumTok = child.access().firstChild().matches(DblDot).firstChild().matches(Kwd(KwdEnum));
 					if (!enumTok.exists()) {
 						continue;
 					}
@@ -216,8 +214,7 @@ class TokenTreeCheckUtils {
 	**/
 	public static function isTypeStructure(typedefToken:TokenTree):Bool {
 		var afterAssign = typedefToken.access().firstChild().isCIdent().firstOf(Binop(OpAssign)).firstChild();
-		return afterAssign.matches(BrOpen).exists()
-			|| afterAssign.isCIdent().firstOf(Binop(OpAnd)).exists();
+		return afterAssign.matches(BrOpen).exists() || afterAssign.isCIdent().firstOf(Binop(OpAnd)).exists();
 	}
 
 	public static function isTypeEnum(enumToken:TokenTree):Bool {
@@ -234,8 +231,7 @@ class TokenTreeCheckUtils {
 	}
 
 	public static function isTypeMacroClass(classToken:TokenTree):Bool {
-		return classToken.tok.match(Kwd(KwdClass))
-			&& classToken.access().parent().matches(Kwd(KwdMacro)).exists();
+		return classToken.tok.match(Kwd(KwdClass)) && classToken.access().parent().matches(Kwd(KwdMacro)).exists();
 	}
 
 	public static function isBrOpenAnonTypeOrTypedef(token:TokenTree):Bool {
@@ -534,13 +530,8 @@ class TokenTreeCheckUtils {
 		if (hasAtParent(token)) {
 			return At;
 		}
-		var lastChild:TokenTree = token.getLastChild();
-		if (lastChild != null) {
-			switch (lastChild.tok) {
-				case Arrow:
-					return Parameter;
-				default:
-			}
+		if (token.hasChildren() && checkPOpenForArrowChildren(token)) {
+			return Parameter;
 		}
 		while ((parent != null) && (parent.tok != Root)) {
 			switch (parent.tok) {
@@ -589,7 +580,14 @@ class TokenTreeCheckUtils {
 				switch (parent.parent.tok) {
 					case Kwd(KwdFunction):
 						if (parent.previousSibling == null) {
-							return Parameter;
+							var pOpen:Null<TokenTree> = parent.access().firstOf(POpen).token;
+							if (pOpen == null) {
+								return Parameter;
+							}
+							if (pOpen.index == token.index) {
+								return Parameter;
+							}
+							return Expression;
 						}
 						return Call;
 					case Kwd(KwdAbstract): return Parameter;
@@ -608,6 +606,26 @@ class TokenTreeCheckUtils {
 		return Expression;
 	}
 
+	static function checkPOpenForArrowChildren(token:TokenTree):Bool {
+		var skip:Bool = true;
+		for (child in token.children) {
+			switch (child.tok) {
+				case PClose:
+					skip = false;
+				default:
+			}
+			if (skip) {
+				continue;
+			}
+			switch (child.tok) {
+				case Arrow:
+					return true;
+				default:
+			}
+		}
+		return false;
+	}
+
 	public static function hasAtParent(token:TokenTree):Bool {
 		var parent:TokenTree = token.parent;
 		while (parent.tok != Root) {

src/tokentree/walk/WalkArrayAccess.hx

@@ -43,5 +43,10 @@ class WalkArrayAccess {
 			}
 		}
 		bkOpen.addChild(stream.consumeTokenDef(BkClose));
+		switch (stream.token()) {
+			case BkOpen | Dot | Binop(_) | Const(CIdent("is")):
+				WalkStatement.walkStatementContinue(stream, bkOpen);
+			default:
+		}
 	}
 }

src/tokentree/walk/WalkBlock.hx

@@ -15,6 +15,7 @@ class WalkBlock {
 			parent.addChild(openTok);
 			stream.applyTempStore(openTok);
 			walkBlockContinue(stream, openTok);
+			stream.applyTempStore(openTok);
 		}
 		else {
 			WalkStatement.walkStatement(stream, parent);
@@ -56,7 +57,7 @@ class WalkBlock {
 					return;
 				default:
 			}
-			WalkStatement.walkStatementContinue(stream, parent);
+			walkAfterBlock(stream, parent);
 			if (stream.hasMore()) {
 				switch (stream.token()) {
 					case Semicolon:
@@ -67,4 +68,43 @@ class WalkBlock {
 			}
 		}
 	}
+
+	@:access(tokentree.walk.WalkStatement)
+	static function walkAfterBlock(stream:TokenStream, parent:TokenTree) {
+		if (!stream.hasMore()) return;
+		switch (stream.token()) {
+			case Dot:
+				WalkStatement.walkStatementWithoutSemicolon(stream, parent);
+			case DblDot:
+				WalkStatement.walkDblDot(stream, parent);
+			case Semicolon:
+				return;
+			case Arrow:
+				WalkStatement.walkStatementWithoutSemicolon(stream, parent);
+			case Binop(_):
+				WalkStatement.walkStatementWithoutSemicolon(stream, parent);
+			case Const(CIdent("is")):
+				WalkStatement.walkStatementWithoutSemicolon(stream, parent);
+			case Unop(_):
+				if (parent.isCIdentOrCString()) {
+					WalkStatement.walkStatementWithoutSemicolon(stream, parent);
+				}
+			case Question:
+				WalkQuestion.walkQuestion(stream, parent);
+			case POpen:
+				WalkStatement.walkStatementWithoutSemicolon(stream, parent);
+			case CommentLine(_), Comment(_):
+				var nextTokDef:Null<TokenTreeDef> = stream.peekNonCommentToken();
+				if (nextTokDef == null) {
+					return;
+				}
+				switch (nextTokDef) {
+					case Dot, DblDot, Binop(_), Unop(_), Question:
+						WalkComment.walkComment(stream, parent);
+						WalkStatement.walkStatementContinue(stream, parent);
+					default:
+				}
+			default:
+		}
+	}
 }

src/tokentree/walk/WalkClass.hx

@@ -62,7 +62,12 @@ class WalkClass {
 					WalkFinal.walkFinal(stream, parent);
 				#end
 				case Comment(_), CommentLine(_):
-					parent.addChild(stream.consumeToken());
+					if (stream.hasTempStore()) {
+						stream.consumeToTempStore();
+					}
+					else {
+						parent.addChild(stream.consumeToken());
+					}
 				default:
 					switch (TokenStream.MODE) {
 						case Relaxed: WalkStatement.walkStatement(stream, parent);

src/tokentree/walk/WalkFile.hx

@@ -18,7 +18,12 @@ class WalkFile {
 				case At:
 					stream.addToTempStore(WalkAt.walkAt(stream));
 				case Comment(_), CommentLine(_):
-					WalkComment.walkComment(stream, parent);
+					if (stream.hasTempStore()) {
+						stream.consumeToTempStore();
+					}
+					else {
+						WalkComment.walkComment(stream, parent);
+					}
 				case Kwd(KwdClass), Kwd(KwdInterface), Kwd(KwdEnum), Kwd(KwdTypedef), Kwd(KwdAbstract):
 					WalkType.walkType(stream, parent);
 				case PClose, BrClose, BkClose, Semicolon, Comma:

src/tokentree/walk/WalkFunction.hx

@@ -21,6 +21,17 @@ class WalkFunction {
 		WalkComment.walkComment(stream, name);
 		WalkFunction.walkFunctionParameters(stream, name);
 		WalkComment.walkComment(stream, name);
+
+		switch (stream.token()) {
+			case Sharp(_):
+				WalkSharp.walkSharp(stream, name, WalkStatement.walkStatement);
+				switch (stream.token()) {
+					case DblDot | BrOpen:
+					default: return;
+				}
+			default:
+		}
+
 		if (stream.tokenForMatch().match(DblDot)) {
 			var dblDot:Null<TokenTree> = stream.consumeToken();
 			name.addChild(dblDot);

src/tokentree/walk/WalkInterface.hx

@@ -40,7 +40,12 @@ class WalkInterface {
 					WalkFinal.walkFinal(stream, parent);
 				#end
 				case Comment(_), CommentLine(_):
-					parent.addChild(stream.consumeToken());
+					if (stream.hasTempStore()) {
+						stream.consumeToTempStore();
+					}
+					else {
+						parent.addChild(stream.consumeToken());
+					}
 				default:
 					stream.consumeToTempStore();
 			}

src/tokentree/walk/WalkNew.hx

@@ -15,8 +15,11 @@ class WalkNew {
 			default:
 		}
 		WalkComment.walkComment(stream, name);
-		if (stream.tokenForMatch().match(Dot)) {
-			WalkStatement.walkStatement(stream, name);
+
+		switch (stream.token()) {
+			case Dot | Binop(_) | Const(CIdent("is")) | BkOpen:
+				WalkStatement.walkStatement(stream, name);
+			default:
 		}
 	}
 }

src/tokentree/walk/WalkPOpen.hx

@@ -9,6 +9,11 @@ class WalkPOpen {
 		if (walkTrailingComments) {
 			WalkComment.walkComment(stream, parent);
 		}
+		switch (stream.token()) {
+			case Arrow:
+				WalkStatement.walkStatementContinue(stream, pOpen);
+			default:
+		}
 		return pOpen;
 	}
 

src/tokentree/walk/WalkQuestion.hx

@@ -12,6 +12,7 @@ class WalkQuestion {
 		}
 		WalkStatement.walkStatement(stream, question);
 		WalkComment.walkComment(stream, question);
+		if (!stream.tokenForMatch().match(DblDot)) return;
 		var dblDotTok:TokenTree = stream.consumeTokenDef(DblDot);
 		question.addChild(dblDotTok);
 		WalkStatement.walkStatement(stream, dblDotTok);
@@ -20,29 +21,29 @@ class WalkQuestion {
 	public static function isTernary(parent:TokenTree):Bool {
 		var lastChild:Null<TokenTree> = parent.getLastChild();
 		if (lastChild == null) {
-			return switch (parent.tok) {
-				case Const(_): true;
-				default: false;
+			switch (parent.tok) {
+				case Const(_):
+					return true;
+				default:
+					lastChild = parent;
 			}
 		}
-		else {
-			return switch (lastChild.tok) {
-				case Const(_): true;
-				case BkOpen: true;
-				case BrOpen: true;
-				case Binop(OpAdd), Binop(OpSub): true;
-				case Unop(_): true;
-				case Kwd(KwdCast): true;
-				case Kwd(KwdNew): true;
-				case Kwd(KwdTrue), Kwd(KwdFalse), Kwd(KwdNull): true;
-				case Kwd(KwdThis), Kwd(KwdMacro), Kwd(KwdUntyped): true;
-				case Kwd(KwdFunction): true;
-				case Dollar(_): true;
-				case POpen: true;
-				case PClose: true;
-				case DblDot: true;
-				default: false;
-			}
+		return switch (lastChild.tok) {
+			case Const(_): true;
+			case BkOpen: true;
+			case BrOpen: true;
+			case Binop(OpAdd), Binop(OpSub): true;
+			case Unop(_): true;
+			case Kwd(KwdCast): true;
+			case Kwd(KwdNew): true;
+			case Kwd(KwdTrue), Kwd(KwdFalse), Kwd(KwdNull): true;
+			case Kwd(KwdThis), Kwd(KwdMacro), Kwd(KwdUntyped): true;
+			case Kwd(KwdFunction): true;
+			case Dollar(_): true;
+			case POpen: true;
+			case PClose: true;
+			case DblDot: true;
+			default: false;
 		}
 	}
 }