Merge pull request #992 from ThrowTheSwitch/bugfix/parsing_issues

Fix parsing issue

bin/projectinator.rb

@@ -109,11 +109,11 @@ def extract_mixins(config:)
 
     # Handle any inline Ruby string expansion
     load_paths.each do |load_path|
-      load_path.replace( @system_wrapper.module_eval( load_path ) ) if (load_path =~ RUBY_STRING_REPLACEMENT_PATTERN)
+      load_path.replace( @system_wrapper.module_eval( load_path ) ) if (load_path =~ PATTERNS::RUBY_STRING_REPLACEMENT)
     end
 
     enabled.each do |mixin|
-      mixin.replace( @system_wrapper.module_eval( mixin ) ) if (mixin =~ RUBY_STRING_REPLACEMENT_PATTERN)
+      mixin.replace( @system_wrapper.module_eval( mixin ) ) if (mixin =~ PATTERNS::RUBY_STRING_REPLACEMENT)
     end
 
     # Remove the :mixins section of the configuration

lib/ceedling/configurator.rb

@@ -132,7 +132,7 @@ def prepare_plugins_load_paths(plugins_load_path, config)
     # Plugins must be loaded before generic path evaluation & magic that happen later.
     # So, perform path magic here as discrete step.
     config[:plugins][:load_paths].each do |path|
-      path.replace( @system_wrapper.module_eval( path ) ) if (path =~ RUBY_STRING_REPLACEMENT_PATTERN)
+      path.replace( @system_wrapper.module_eval( path ) ) if (path =~ PATTERNS::RUBY_STRING_REPLACEMENT)
       FilePathUtils::standardize( path )
     end
 
@@ -460,7 +460,7 @@ def eval_environment_variables(config)
       # Process value array
       items.each do |item|
         # Process each item for Ruby string replacement
-        if item.is_a? String and item =~ RUBY_STRING_REPLACEMENT_PATTERN
+        if item.is_a? String and item =~ PATTERNS::RUBY_STRING_REPLACEMENT
           item.replace( @system_wrapper.module_eval( item ) )
         end
       end
@@ -706,7 +706,7 @@ def eval_path_entries( container )
     end
 
     paths.each do |path|
-      path.replace( @system_wrapper.module_eval( path ) ) if (path =~ RUBY_STRING_REPLACEMENT_PATTERN)
+      path.replace( @system_wrapper.module_eval( path ) ) if (path =~ PATTERNS::RUBY_STRING_REPLACEMENT)
     end
   end
 
@@ -721,7 +721,7 @@ def traverse_hash_eval_string_arrays(config)
       if config.all? { |item| item.is_a?( String ) }
         # Expand in place each string item in the array
         config.each do |item|
-          item.replace( @system_wrapper.module_eval( item ) ) if (item =~ RUBY_STRING_REPLACEMENT_PATTERN)
+          item.replace( @system_wrapper.module_eval( item ) ) if (item =~ PATTERNS::RUBY_STRING_REPLACEMENT)
         end
       end
 

lib/ceedling/constants.rb

@@ -65,6 +65,15 @@ class StdErrRedirect
   TCSH = :tcsh
 end
 
+class PATTERNS
+  GLOB = /[\*\?\{\}\[\]]/
+  RUBY_STRING_REPLACEMENT = /#\{.+\}/
+  TOOL_EXECUTOR_ARGUMENT_REPLACEMENT = /(\$\{(\d+)\})/
+  TEST_STDOUT_STATISTICS  = /\n-+\s*(\d+)\s+Tests\s+(\d+)\s+Failures\s+(\d+)\s+Ignored\s+(OK|FAIL)\s*/i
+  TEST_SOURCE_FILE  = /TEST_SOURCE_FILE\s*\(\s*\"\s*([^"]+)\s*\"\s*\)/
+  TEST_INCLUDE_PATH = /TEST_INCLUDE_PATH\s*\(\s*\"\s*([^"]+)\s*\"\s*\)/
+end
+
 GIT_COMMIT_SHA_FILENAME = 'GIT_COMMIT_SHA'
 
 # Escaped newline literal (literally double-slash-n) for "encoding" multiline strings as single string
@@ -95,10 +104,6 @@ class StdErrRedirect
 UNITY_H_FILE           = 'unity.h'
 UNITY_INTERNALS_H_FILE = 'unity_internals.h'
 
-# Do-nothing macros defined in unity.h for extra build context to be used by build tools like Ceedling
-UNITY_TEST_SOURCE_FILE  = 'TEST_SOURCE_FILE'
-UNITY_TEST_INCLUDE_PATH = 'TEST_INCLUDE_PATH'
-
 RUNNER_BUILD_CMDLINE_ARGS_DEFINE = 'UNITY_USE_COMMAND_LINE_ARGS'
 
 CMOCK_SYM       = :cmock
@@ -134,12 +139,6 @@ class StdErrRedirect
 PREPROCESS_FULL_EXPANSION_DIR  = 'full_expansion'
 PREPROCESS_DIRECTIVES_ONLY_DIR = 'directives_only'
 
-# Match presence of any glob pattern characters
-GLOB_PATTERN = /[\*\?\{\}\[\]]/
-RUBY_STRING_REPLACEMENT_PATTERN = /#\{.+\}/
-TOOL_EXECUTOR_ARGUMENT_REPLACEMENT_PATTERN = /(\$\{(\d+)\})/
-TEST_STDOUT_STATISTICS_PATTERN  = /\n-+\s*(\d+)\s+Tests\s+(\d+)\s+Failures\s+(\d+)\s+Ignored\s+(OK|FAIL)\s*/i
-
 NULL_FILE_PATH = '/dev/null'
 
 TESTS_BASE_PATH   = TEST_ROOT_NAME

lib/ceedling/file_path_utils.rb

@@ -52,7 +52,7 @@ def self.no_decorators(path)
     path = self.no_aggregation_decorators(path)
 
     # Find first occurrence of glob specifier: *, ?, {, }, [, ]
-    find_index = (path =~ GLOB_PATTERN)
+    find_index = (path =~ PATTERNS::GLOB)
 
     # Return empty path if first character is part of a glob
     return '' if find_index == 0

lib/ceedling/generator_helper.rb

@@ -26,7 +26,7 @@ def test_crash?(test_filename, executable, shell_result)
     end
 
     # No test results found in test executable output
-    if (shell_result[:output] =~ TEST_STDOUT_STATISTICS_PATTERN).nil?
+    if (shell_result[:output] =~ PATTERNS::TEST_STDOUT_STATISTICS).nil?
       # No debug logging here because we log this condition in the error log handling below
       crash = true
     end
@@ -57,7 +57,7 @@ def log_test_results_crash(executable, shell_result, backtrace)
       notice += "> Produced no output (including no final test result counts).\n"
 
     # Check for no test results
-    elsif ((shell_result[:output] =~ TEST_STDOUT_STATISTICS_PATTERN).nil?)
+    elsif ((shell_result[:output] =~ PATTERNS::TEST_STDOUT_STATISTICS).nil?)
       # Mirror style of generic tool_executor failure output
       notice += "> Produced some output but contains no final test result counts.\n"
     end

lib/ceedling/generator_test_results.rb

@@ -105,7 +105,7 @@ def process_and_write_results(executable, unity_shell_result, results_file, test
     results[:time] = unity_shell_result[:time] unless unity_shell_result[:time].nil?
 
     # Process test statistics
-    if (unity_shell_result[:output] =~ TEST_STDOUT_STATISTICS_PATTERN)
+    if (unity_shell_result[:output] =~ PATTERNS::TEST_STDOUT_STATISTICS)
       results[:counts][:total] =   $1.to_i
       results[:counts][:failed] =  $2.to_i
       results[:counts][:ignored] = $3.to_i
@@ -115,7 +115,7 @@ def process_and_write_results(executable, unity_shell_result, results_file, test
     end
 
     # Remove test statistics lines
-    output_string = unity_shell_result[:output].sub( TEST_STDOUT_STATISTICS_PATTERN, '' )
+    output_string = unity_shell_result[:output].sub( PATTERNS::TEST_STDOUT_STATISTICS, '' )
 
     # Process test executable results line-by-line
     output_string.lines do |line|

lib/ceedling/include_pathinator.rb

@@ -28,7 +28,7 @@ def validate_test_build_directive_paths
         # TODO: When Ceedling's base project path handling is resolved, enable this path redefinition
         # path = File.join( @base_path, path )
         unless @file_wrapper.exist?(path)
-          error = "'#{path}' specified by #{UNITY_TEST_INCLUDE_PATH}() within #{test_filepath} not found"
+          error = "'#{path}' specified by TEST_INCLUDE_PATH() within #{test_filepath} not found"
           raise CeedlingException.new( error )
         end
       end
@@ -51,7 +51,7 @@ def validate_header_files_collection
 
     if headers.length == 0
       error = "No header files found in project.\n" +
-              "Add search paths to :paths ↳ :include in your project file and/or use #{UNITY_TEST_INCLUDE_PATH}() in your test files.\n" +
+              "Add search paths to :paths ↳ :include in your project file and/or use TEST_INCLUDE_PATH() in your test files.\n" +
               "Verify header files with `ceedling paths:include` and\\or `ceedling files:include`."
       @loginator.log( error, Verbosity::COMPLAIN )
     end

lib/ceedling/objects.yml

@@ -162,11 +162,14 @@ file_finder:
 file_finder_helper:
   compose: loginator
 
+parsing_parcels:
+
 test_context_extractor:
   compose:
     - configurator
     - file_wrapper
     - loginator
+    - parsing_parcels
 
 include_pathinator:
   compose:
@@ -287,6 +290,8 @@ preprocessinator_file_handler:
     - loginator
 
 preprocessinator_extractor:
+  compose:
+    - parsing_parcels
 
 build_batchinator:
   compose:

lib/ceedling/parsing_parcels.rb

@@ -0,0 +1,77 @@
+# =========================================================================
+#   Ceedling - Test-Centered Build System for C
+#   ThrowTheSwitch.org
+#   Copyright (c) 2010-25 Mike Karlesky, Mark VanderVoord, & Greg Williams
+#   SPDX-License-Identifier: MIT
+# =========================================================================
+
+require 'ceedling/encodinator'
+
+# This is a collection of parsing aids to be used in other modules
+class ParsingParcels
+
+  # This parser accepts a collection of lines which it will sweep through and tidy, giving the purified
+  # lines to the block (one line at a time) for further analysis. It analyzes a single line at a time, 
+  # which is far more memory efficient and faster for large files. However, this requires it to also 
+  # handle backslash line continuations as a single line at this point.
+  def code_lines(input)
+    comment_block = false
+    full_line = ''
+    input.each_line do |line|
+        m = line.match /(.*)\\\s*$/
+      if (!m.nil?)
+          full_line += m[1]
+      elsif full_line.empty?
+        _line, comment_block = clean_code_line( line, comment_block )
+        yield( _line )
+      else
+        _line, comment_block = clean_code_line( full_line + line, comment_block )
+        yield( _line )
+        full_line = ''
+      end
+    end    
+  end
+
+  private ######################################################################
+
+  def clean_code_line(line, comment_block)
+    _line = line.clean_encoding
+
+    # Remove line comments
+    _line.gsub!(/\/\/.*$/, '')
+
+    # Handle end of previously begun comment block
+    if comment_block
+      if _line.include?( '*/' )
+        # Turn off comment block handling state
+        comment_block = false
+
+        # Remove everything up to end of comment block
+        _line.gsub!(/^.*\*\//, '')
+      else
+        # Ignore contents of the line if its entirely within a comment block
+        return '', comment_block        
+      end
+
+    end
+
+    # Block comments inside a C string are valid C, but we remove to simplify other parsing.
+    # No code we care about will be inside a C string.
+    # Note that we're not attempting the complex case of multiline string enclosed comment blocks
+    _line.gsub!(/"\s*\/\*.*"/, '')
+
+    # Remove single-line block comments
+    _line.gsub!(/\/\*.*\*\//, '')
+
+    # Handle beginning of any remaining multiline comment block
+    if _line.include?( '/*' )
+      comment_block = true
+
+      # Remove beginning of block comment
+      _line.gsub!(/\/\*.*/, '')
+    end
+
+    return _line, comment_block
+  end
+
+end

lib/ceedling/preprocessinator_extractor.rb

@@ -7,9 +7,12 @@
 
 require 'ceedling/constants'
 require 'ceedling/encodinator'
+require 'ceedling/parsing_parcels'
 
 class PreprocessinatorExtractor 
 
+  constructor :parsing_parcels
+
   ##
   ## Preprocessor Expansion Output Handling
   ## ======================================
@@ -138,11 +141,11 @@ def extract_test_directive_macro_calls(file_contents)
     # Look for TEST_SOURCE_FILE("...") and TEST_INCLUDE_PATH("...") in a string (i.e. a file's contents as a string)
 
     regexes = [
-      /#{UNITY_TEST_SOURCE_FILE}.+?"\)/,
-      /#{UNITY_TEST_INCLUDE_PATH}.+?"\)/
+      /(#{PATTERNS::TEST_SOURCE_FILE})/,
+      /(#{PATTERNS::TEST_INCLUDE_PATH})/
     ]
 
-    return extract_tokens_by_regex_list( file_contents, *regexes )
+    return extract_tokens_by_regex_list( file_contents, *regexes ).map(&:first)
   end
 
 
@@ -199,7 +202,7 @@ def extract_multiline_directives(file_contents, directive)
     #  - Captures all text (non-greedily) after '#<directive>' on a first line through 0 or more line continuations up to a final newline.
     #  - Line continuations comprise a final '\' on a given line followed by whitespace & newline, wrapping to the next
     #    line up to a final '\' on that next line.
-    regex = /(#\s*#{directive}\s+.*?(\\\s*\n.*?)*)\n/
+    regex = /(#\s*#{directive}[^\n]*)\n/
 
     tokens = extract_tokens_by_regex_list( file_contents, regex )
 
@@ -227,7 +230,9 @@ def extract_tokens_by_regex_list(file_contents, *regexes)
 
     # For each regex provided, extract all matches from the source string
     regexes.each do |regex|
-      tokens += file_contents.scan( regex )
+      @parsing_parcels.code_lines( file_contents ) do |line|
+        tokens += line.scan( regex )
+      end
     end
 
     return tokens