Further progress on DataFlex; lex-tester test app for testing DataF…

…lex programmatically (#370) * Rename: kDataTokenBlockComment{Asterisk -> Skip} * Hack to properly reset Flex lexing state in lex-tester * Fix CompareTokens output header written state not resetting correctly * Allow DataFlex_target.cpp to still be swapped in for DataFlex.c * Fix block comment lexing issues * DataFlex re-entrancy support * Add a few more test routines to lex-tester These require manual code modification to use, for the time being at least * Distinguish floats with exponents in lex-tester * Shuffle algorithm for lex-tester * More thorough lex fragment test routine
DarkRTA · Oct 8, 2024 · 20d8e4e · 20d8e4e
1 parent 193fb4f
commit 20d8e4e
Show file tree

Hide file tree

Showing 7 changed files with 723 additions and 186 deletions.
diff --git a/src/system/obj/DataFlex.c b/src/system/obj/DataFlex.c
diff --git a/src/system/obj/DataFlex.h b/src/system/obj/DataFlex.h
@@ -20,7 +20,7 @@ enum DataToken {
     kDataTokenComment,
     kDataTokenBlockCommentStart,
     kDataTokenBlockCommentText,
-    kDataTokenBlockCommentAsterisk,
+    kDataTokenBlockCommentSkip,
     kDataTokenBlockCommentNewline,
     kDataTokenBlockCommentEnd,
 
@@ -89,14 +89,40 @@ enum DataToken {
 };
 #endif
 
-extern void yyrestart(FILE*);
-extern int yylex();
+#ifdef YY_REENTRANT
+
+extern int yylex_init(void **out_globals);
+extern int yylex_destroy(void *yy_globals);
+
+#ifndef YY_LAST_ARG
+#define YY_LAST_ARG , void *yy_globals
+#define YY_ONLY_ARG void *yy_globals
+#endif
+
+#else
 
 extern FILE *yyin, *yyout;
 
 extern char *yytext;
 extern int yyleng;
 
+#ifndef YY_LAST_ARG
+#define YY_LAST_ARG
+#define YY_ONLY_ARG void
+#endif
+
+#endif
+
+extern int yylex(YY_ONLY_ARG);
+extern void yyrestart(FILE *file YY_LAST_ARG);
+
+extern char *yyget_text(YY_ONLY_ARG);
+extern int yyget_leng(YY_ONLY_ARG);
+
+#ifdef DATAFLEX_TESTER
+extern void yy_actually_restart(YY_ONLY_ARG);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/src/system/obj/DataFlex.l b/src/system/obj/DataFlex.l
@@ -11,7 +11,7 @@
 %option noyywrap
 
 %{
-/* Flex is stupid and doesn't include this under C mode for malloc/realloc */
+/* This version of Flex is stupid and doesn't include this under C mode for malloc/realloc */
 #include <stdlib.h>
 
 #include "DataFlex.h"
@@ -52,9 +52,9 @@ COMMENT             ;[^\n]*
 
 %x BLOCK_COMMENT
 /* BUG: /** won't start a block */
-BCOMMENT_START      \/\*[^\n\*]*
+BCOMMENT_START      (\/\*)+[^\n\*]*
 BCOMMENT_TEXT       [^\n\*]*
-BCOMMENT_SKIP       \*+
+BCOMMENT_SKIP       \*+[^\n\/\*]*
 BCOMMENT_END        \*+\/
 
 %%
@@ -64,7 +64,7 @@ BCOMMENT_END        \*+\/
 {QUOTED_SYMBOL}                                     { return kDataTokenQuotedSymbol; }
 {BCOMMENT_START}                                    { BEGIN(BLOCK_COMMENT); TESTER_RETURN(kDataTokenBlockCommentStart); }
 <BLOCK_COMMENT>{BCOMMENT_TEXT}                      { /* ignore text in block comments */ TESTER_RETURN(kDataTokenBlockCommentText); }
-<BLOCK_COMMENT>{BCOMMENT_SKIP}                      { /* ignore *s in block comments */ TESTER_RETURN(kDataTokenBlockCommentAsterisk); }
+<BLOCK_COMMENT>{BCOMMENT_SKIP}                      { /* ignore *s in block comments */ TESTER_RETURN(kDataTokenBlockCommentSkip); }
 <BLOCK_COMMENT>\n                                   { gDataLine++; TESTER_RETURN(kDataTokenBlockCommentNewline); }
 <BLOCK_COMMENT>{BCOMMENT_END}                       { BEGIN(INITIAL); TESTER_RETURN(kDataTokenBlockCommentEnd); }
 {COMMENT}                                           { /* skip comments */ TESTER_RETURN(kDataTokenComment); }
@@ -73,7 +73,7 @@ kDataUnhandled                                      { return kDataTokenUnhandled
 {HEX_NUMBER}                                        { return kDataTokenHex; }
 {INTEGER}                                           { return kDataTokenInt; }
 {FLOAT}                                             { return kDataTokenFloat; }
-{FLOAT_EXPONENT}                                    { return kDataTokenFloat; }
+{FLOAT_EXPONENT}                                    { TESTER_RETURN(kDataTokenFloatExp); return kDataTokenFloat; }
 
 #include_opt                                        { return kDataTokenIncludeOptional; }
 #include                                            { return kDataTokenInclude; }
@@ -97,3 +97,11 @@ kDataUnhandled                                      { return kDataTokenUnhandled
 \[                                                  { return kDataTokenPropertyOpen; }
 \]                                                  { return kDataTokenPropertyClose; }
 %%
+
+#ifdef DATAFLEX_TESTER
+/* This version of Flex is even more stupid and doesn't reset the lexing state when restarting */
+void yy_actually_restart(YY_ONLY_ARG) {
+    BEGIN(INITIAL);
+    yyrestart(NULL YY_CALL_LAST_ARG);
+}
+#endif
diff --git a/src/system/obj/DataFlex_target.cpp b/src/system/obj/DataFlex_target.cpp
@@ -4,6 +4,16 @@
 #define YY_FLEX_MAJOR_VERSION 2
 #define YY_FLEX_MINOR_VERSION 5
 
+#if defined(DATAFLEX_TESTER) && defined(__cplusplus)
+#define DATAFLEX_NAMESPACE yytarget::
+#define DATAFLEX_NAMESPACE_BEGIN namespace yytarget {
+#define DATAFLEX_NAMESPACE_END }
+#else
+#define DATAFLEX_NAMESPACE
+#define DATAFLEX_NAMESPACE_BEGIN
+#define DATAFLEX_NAMESPACE_END
+#endif
+
 #include <stdio.h>
 
 
@@ -45,7 +55,7 @@
 #define YY_USE_PROTOS
 #endif
 
-namespace yytarget {
+DATAFLEX_NAMESPACE_BEGIN
 
 #ifdef YY_USE_CONST
 #define yyconst const
@@ -129,7 +139,7 @@ int yylex_destroy YY_PROTO((void* scanner));
 #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1)
 
 /* Special action meaning "start processing a new file". */
-#define YY_NEW_FILE yytarget::yyrestart( YY_G(yyin) YY_CALL_LAST_ARG )
+#define YY_NEW_FILE DATAFLEX_NAMESPACE yyrestart( YY_G(yyin) YY_CALL_LAST_ARG )
 
 #define YY_END_OF_BUFFER_CHAR 0
 
@@ -581,19 +591,24 @@ static char *yy_last_accepting_cpos;
 char *yytext;
 #endif
 
-}
+DATAFLEX_NAMESPACE_END
 
 #define INITIAL 0
 /*
  * requires the following commit of flex:
  * https://github.com/westes/flex/commit/fe84af1738b78edeca58752c5a549a175236420a
+ *
+ * run with the following command-line args in the root of the repo:
+ * -L "-osrc/system/obj/DataFlex.c" "src/system/obj/DataFlex.l"
  */
 /* %option nounistd - not supported on the version of flex used */
 #define YY_NEVER_INTERACTIVE 1
-/* Flex is stupid and doesn't include this under C mode for malloc/realloc */
+/* This version of Flex is stupid and doesn't include this under C mode for malloc/realloc */
 #include <stdlib.h>
 
-#include "DataFlex_target.h"
+#include "DataFlex.h"
+
+DATAFLEX_NAMESPACE_BEGIN
 
 #define YY_INPUT(buf, result, max_size) \
     (result) = (DataInput((buf), 1) != 0)
@@ -607,12 +622,15 @@ char *yytext;
 #endif
 
 #define ECHO TESTER_RETURN(kDataTokenNotRecognized) /* don't echo unmatched characters */
+
+DATAFLEX_NAMESPACE_END
+
 /* TODO: '-' has a significant usage outside of SIGN */
 #define BLOCK_COMMENT 1
 
 /* BUG: /** won't start a block */
 
-namespace yytarget {
+DATAFLEX_NAMESPACE_BEGIN
 
 #ifndef YY_EXTRA_TYPE
 #define YY_EXTRA_TYPE void *
@@ -788,7 +806,7 @@ static int yy_top_state YY_PROTO(( YY_ONLY_ARG ));
 #define YY_NO_TOP_STATE 1
 #endif
 
-}
+DATAFLEX_NAMESPACE_END
 
 #ifdef YY_MALLOC_DECL
 YY_MALLOC_DECL
@@ -805,7 +823,7 @@ YY_MALLOC_DECL
 #endif
 #endif
 
-namespace yytarget {
+DATAFLEX_NAMESPACE_BEGIN
 
 /* Amount of stuff to slurp up with each read. */
 #ifndef YY_READ_BUF_SIZE
@@ -1033,7 +1051,7 @@ YY_RULE_SETUP
 	YY_BREAK
 case 7:
 YY_RULE_SETUP
-{ /* ignore *s in block comments */ TESTER_RETURN(kDataTokenBlockCommentAsterisk); }
+{ /* ignore *s in block comments */ TESTER_RETURN(kDataTokenBlockCommentSkip); }
 	YY_BREAK
 case 8:
 YY_RULE_SETUP
@@ -1065,7 +1083,7 @@ YY_RULE_SETUP
 	YY_BREAK
 case 15:
 YY_RULE_SETUP
-{ return kDataTokenFloat; }
+{ TESTER_RETURN(kDataTokenFloatExp); return kDataTokenFloat; }
 	YY_BREAK
 case 16:
 YY_RULE_SETUP
@@ -1400,7 +1418,7 @@ YY_DECL_LAST_ARG
 		if ( number_to_move == YY_MORE_ADJ )
 			{
 			ret_val = EOB_ACT_END_OF_FILE;
-			yytarget::yyrestart( YY_G(yyin)  YY_CALL_LAST_ARG);
+			DATAFLEX_NAMESPACE yyrestart( YY_G(yyin)  YY_CALL_LAST_ARG);
 			}
 
 		else
@@ -1586,7 +1604,7 @@ static int input(YY_ONLY_ARG)
 					 */
 
 					/* Reset buffer status. */
-					yytarget::yyrestart( YY_G(yyin) YY_CALL_LAST_ARG);
+					DATAFLEX_NAMESPACE yyrestart( YY_G(yyin) YY_CALL_LAST_ARG);
 
 					/* fall through */
 
@@ -2334,4 +2352,13 @@ int main()
 	}
 #endif
 
+
+#ifdef DATAFLEX_TESTER
+/* This version of Flex is even more stupid and doesn't reset the lexing state when restarting */
+void yy_actually_restart(YY_ONLY_ARG) {
+    BEGIN(INITIAL);
+    yyrestart(NULL YY_CALL_LAST_ARG);
 }
+#endif
+
+DATAFLEX_NAMESPACE_END
diff --git a/src/system/obj/DataFlex_target.h b/src/system/obj/DataFlex_target.h
@@ -7,13 +7,41 @@
 #include "DataFlex.h"
 
 namespace yytarget {
-    extern void yyrestart(FILE*);
-    extern int yylex();
+
+#ifdef YY_REENTRANT
+
+    extern int yylex_init(void **out_globals);
+    extern int yylex_destroy(void *yy_globals);
+
+#ifndef YY_LAST_ARG
+#define YY_LAST_ARG , void *yy_globals
+#define YY_ONLY_ARG void *yy_globals
+#endif
+
+#else
 
     extern FILE *yyin, *yyout;
 
     extern char *yytext;
     extern int yyleng;
+
+#ifndef YY_LAST_ARG
+#define YY_LAST_ARG
+#define YY_ONLY_ARG void
+#endif
+
+#endif
+
+    extern int yylex(YY_ONLY_ARG);
+    extern void yyrestart(FILE *file YY_LAST_ARG);
+
+    extern char *yyget_text(YY_ONLY_ARG);
+    extern int yyget_leng(YY_ONLY_ARG);
+
+#ifdef DATAFLEX_TESTER
+    extern void yy_actually_restart(YY_ONLY_ARG);
+#endif
+
 }
 
 #endif
diff --git a/tools/lex-tester/CMakeLists.txt b/tools/lex-tester/CMakeLists.txt
@@ -23,4 +23,6 @@ target_include_directories(DataFlexTester
 target_compile_definitions(DataFlexTester
     PUBLIC
     "DATAFLEX_TESTER"
+    "YY_REENTRANT"
+    "YY_USE_PROTOS"
 )