moved inline markup code to haxeparser (#212)

* moved inline markup code to haxeparser

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## dev branch / next version (1.x.x)
 
+## version 1.2.1 (2022-09-07)
+
+- Moved inline markup code to haxeparser ([#212](https://github.com/HaxeCheckstyle/tokentree/issues/212))
+
 ## version 1.2.0 (2022-08-27)
 
 - Added support for inline markup ([#211](https://github.com/HaxeCheckstyle/tokentree/issues/211))

haxe_libraries/haxeparser.hxml

@@ -1,4 +1,4 @@
-# @install: lix --silent download "gh://github.com/HaxeCheckstyle/haxeparser#7230c32c5bed0a2eef4f05260d79d25627648abe" into haxeparser/3.3.0/github/7230c32c5bed0a2eef4f05260d79d25627648abe
+# @install: lix --silent download "gh://github.com/HaxeCheckstyle/haxeparser#1ea2a8915956ac3af6f1a11c67e0e56261b75929" into haxeparser/3.3.0/github/1ea2a8915956ac3af6f1a11c67e0e56261b75929
 -lib hxparse
--cp ${HAXE_LIBCACHE}/haxeparser/3.3.0/github/7230c32c5bed0a2eef4f05260d79d25627648abe/src
+-cp ${HAXE_LIBCACHE}/haxeparser/3.3.0/github/1ea2a8915956ac3af6f1a11c67e0e56261b75929/src
 -D haxeparser=3.3.0

haxe_libraries/test-adapter.hxml

@@ -1,5 +1,5 @@
-# @install: lix --silent download "haxelib:/test-adapter#2.0.1" into test-adapter/2.0.1/haxelib
+# @install: lix --silent download "haxelib:/test-adapter#2.0.4" into test-adapter/2.0.4/haxelib
 -lib json2object
--cp ${HAXE_LIBCACHE}/test-adapter/2.0.1/haxelib/
--D test-adapter=2.0.1
+-cp ${HAXE_LIBCACHE}/test-adapter/2.0.4/haxelib/
+-D test-adapter=2.0.4
 --macro _testadapter.Macro.init()

haxelib.json

@@ -7,8 +7,8 @@
     "contributors": [
         "AlexHaxe"
     ],
-    "releasenote": "added support for inline markup - see CHANGELOG for details",
-    "version": "1.2.0",
+    "releasenote": "moved inline markup code to haxeparser - see CHANGELOG for details",
+    "version": "1.2.1",
     "url": "https://github.com/HaxeCheckstyle/tokentree",
     "dependencies": {}
 }

package.json

@@ -1,6 +1,6 @@
 {
     "name": "tokentree",
-    "version": "1.2.0",
+    "version": "1.2.1",
     "description": "TokenTree library used by haxe-checkstyle, haxe-formatter and haxe-languageserver",
     "repository": {
         "type": "git",

src/tokentree/ToTokenTreeDef.hx

@@ -27,6 +27,7 @@ abstract ToTokenTreeDef(TokenTreeDef) {
 					case CString(s, kind): Const(CString(s, kind));
 					case CIdent(s): Const(CIdent(s));
 					case CRegexp(r, opt): Const(CRegexp(r, opt));
+					case CMarkup(s): Const(CMarkup(s));
 				}
 			case Sharp(s): Sharp(s);
 			case Dollar(s): Dollar(s);

src/tokentree/TokenStream.hx

@@ -82,100 +82,6 @@ class TokenStream {
 		}
 	}
 
-	public function consumeInlineMarkup():Null<TokenTree> {
-		if (current >= tokens.length - 2) {
-			return null;
-		}
-		var startPos = tokens[current].pos;
-		var text:String = bytes.readString(startPos.min, bytes.length - startPos.min);
-		var startReg = ~/^<([a-zA-Z_][a-zA-Z0-9_:-]*)/;
-		if (!startReg.match(text)) {
-			return null;
-		}
-		var tagName = startReg.matched(1);
-		var startTag = '<$tagName';
-		var endTag = '</$tagName>';
-
-		var depth = 0;
-		var index = 0;
-		while (true) {
-			var indexStartTag = text.indexOf(startTag, index);
-			var indexEndTag = text.indexOf(endTag, index);
-			if ((indexStartTag == -1) && (indexEndTag == -1)) {
-				return null;
-			}
-			if (indexStartTag == -1) {
-				indexStartTag = indexEndTag + 1;
-			}
-			if (indexEndTag == -1) {
-				indexEndTag = indexStartTag + 1;
-			}
-
-			if (indexStartTag < indexEndTag) {
-				index = indexStartTag + startTag.length;
-				switch (text.charAt(index)) {
-					case " " | "/" | ">":
-					default:
-						continue;
-				}
-				depth++;
-				var indexSelfClosing = text.indexOf("/>", index);
-				var indexTagClosing = text.indexOf(">", index);
-				var indexOpenTag = text.indexOf("<", index);
-
-				if ((indexSelfClosing == -1) && (indexTagClosing == -1) && (indexOpenTag == -1)) {
-					return null;
-				}
-				if (indexSelfClosing == -1) {
-					indexSelfClosing = Std.int(Math.max(indexTagClosing, indexOpenTag)) + 1;
-				}
-				if (indexTagClosing == -1) {
-					indexTagClosing = Std.int(Math.max(indexSelfClosing, indexOpenTag)) + 1;
-				}
-				if (indexOpenTag == -1) {
-					indexOpenTag = Std.int(Math.max(indexSelfClosing, indexTagClosing)) + 1;
-				}
-				if (indexSelfClosing < indexTagClosing && indexSelfClosing < indexOpenTag) {
-					index = indexSelfClosing + 2;
-					depth--;
-				}
-				if (indexTagClosing < indexSelfClosing && indexTagClosing < indexOpenTag) {
-					index = indexTagClosing + 1;
-				}
-				if (indexOpenTag < indexSelfClosing && indexOpenTag < indexTagClosing) {
-					index = indexOpenTag;
-				}
-			}
-			if (indexEndTag < indexStartTag) {
-				index = indexEndTag + endTag.length;
-				depth--;
-			}
-			if (depth <= 0) {
-				break;
-			}
-		}
-		text = text.substr(0, index);
-		var textBytes = ByteData.ofString(text);
-		var endPos = startPos.min + textBytes.length;
-		var pos = {
-			file: startPos.file,
-			min: startPos.min,
-			max: endPos
-		}
-		var token = new TokenTree(Const(CMarkup(text)), "", pos, current);
-		forwardToPos(endPos);
-		return token;
-	}
-
-	function forwardToPos(pos:Int) {
-		while (current < tokens.length) {
-			current++;
-			if (tokens[current].pos.min >= pos) {
-				return;
-			}
-		}
-	}
-
 	public function consumeToTempStore() {
 		tempStore.push(consumeToken());
 	}

src/tokentree/TokenTreeDef.hx

@@ -6,7 +6,7 @@ import haxeparser.Data;
 enum TokenTreeDef {
 	Root;
 	Kwd(k:Keyword);
-	Const(c:TokenTreeConstant);
+	Const(c:haxeparser.Data.Constant);
 	Sharp(s:String);
 	Dollar(s:String);
 	Unop(op:haxe.macro.Expr.Unop);

src/tokentree/walk/WalkStatement.hx

@@ -34,11 +34,6 @@ class WalkStatement {
 				WalkBinopSub.walkBinopSub(stream, parent);
 				return;
 			case Binop(OpLt):
-				var markup:Null<TokenTree> = stream.consumeInlineMarkup();
-				if (markup != null) {
-					parent.addChild(markup);
-					return;
-				}
 				if (stream.isTypedParam()) {
 					WalkLtGt.walkLtGt(stream, parent);
 					if (stream.tokenForMatch().match(Arrow)) {

test/tokentree/TestTokenTreeBuilder.hx

@@ -1,11 +1,11 @@
 package tokentree;
 
-import tokentree.walk.WalkStatement;
 import byte.ByteData;
 import haxeparser.Data.Token;
 import haxeparser.HaxeLexer;
 import tokentree.TokenTreeBuilder.TokenTreeEntryPoint;
 import tokentree.walk.WalkFile;
+import tokentree.walk.WalkStatement;
 
 class TestTokenTreeBuilder extends TokenTreeBuilder {
 	public static function parseCode(code:String):TestTokenTreeBuilder {