NLP-ENGINE-423 add fetch attrs for punct and ctrl

VisualText · Jul 22, 2024 · 24732c8 · 24732c8
1 parent 0ec9e4d
commit 24732c8
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 12 deletions.
diff --git a/lite/dicttok.cpp b/lite/dicttok.cpp
@@ -772,26 +772,18 @@ switch (typ)
 		// Put this as a variable on the node!
 		int ansi = (unsigned char) *str;
 		replaceNum(node,_T("CTRL"),ansi);
-		makeTextAttrs(node,last);
+		findTokAttrs(node,last,con,str,lcstr);
 		}
 		break;
 	case PNALPHA:
 	case PNEMOJI:
-		makeTextAttrs(node,last);
-		con = NULL;
-		if (str && *str) {
-			con = cg_->findWordConcept(str);
-		}
-		if (!con && lcstr && *lcstr) {
-			con = cg_->findWordConcept(lcstr);
-		}
-		findAttrs(node, con, str, false);
+		findTokAttrs(node,last,con,str,lcstr);
 		break;
 	case PNNUM:	// Placed here for easy reference.
 		++totnums_;
 		// Fall through to PNPUNCT for now.
 	case PNPUNCT:
-		makeTextAttrs(node,last);
+		findTokAttrs(node,last,con,str,lcstr);
 		break;
 	case PNWHITE:
 		prevwh_ = true;
@@ -808,6 +800,18 @@ else if (root_)	// Sanity check.
 return node;
 }
 
+void DICTTok::findTokAttrs(Node<Pn> *node, 	Node<Pn> *last, CONCEPT *con, _TCHAR *str, _TCHAR *lcstr) {
+	makeTextAttrs(node,last);
+	con = NULL;
+	if (str && *str) {
+		con = cg_->findWordConcept(str);
+	}
+	if (!con && lcstr && *lcstr) {
+		con = cg_->findWordConcept(lcstr);
+	}
+	findAttrs(node, con, str, false);
+}
+
 void DICTTok::makeTextAttrs(Node<Pn> *node, Node<Pn> *last) {
 	if (prevwh_)
 		replaceNum(node,_T("SP"),1);
@@ -821,7 +825,6 @@ void DICTTok::makeTextAttrs(Node<Pn> *node, Node<Pn> *last) {
 	lines_ = tabs_ = 0;
 }
 
-
 inline bool DICTTok::findAttrs(Node<Pn> *node, CONCEPT *con, _TCHAR *str, bool isSuggested) {
 	_TCHAR attrName[NAMESIZ];
 	_TCHAR bufval[NAMESIZ];

diff --git a/lite/dicttok.h b/lite/dicttok.h
@@ -96,6 +96,7 @@ class DICTTok : public Algo
 		Htab *htab
 		);
 
+	void findTokAttrs(Node<Pn> *node, Node<Pn> *last, CONCEPT *con, _TCHAR *str, _TCHAR *lcstr);
 	void makeTextAttrs(Node<Pn> *node, Node<Pn> *last);
 
 	inline bool findAttrs(