Add support in regular expressions for UTF-8 whitespace detection

lib/liquid.rb

@@ -36,8 +36,8 @@ module Liquid
   VariableEnd                 = /\}\}/
   VariableIncompleteEnd       = /\}\}?/
   QuotedString                = /"[^"]*"|'[^']*'/
-  QuotedFragment              = /#{QuotedString}|(?:[^\s,\|'"]|#{QuotedString})+/o
-  TagAttributes               = /(\w[\w-]*)\s*\:\s*(#{QuotedFragment})/o
+  QuotedFragment              = /#{QuotedString}|(?:[^[[:space:]],\|'"]|#{QuotedString})+/o
+  TagAttributes               = /(\w[\w-]*)[[:space:]]*\:[[:space:]]*(#{QuotedFragment})/o
   AnyStartingTag              = /#{TagStart}|#{VariableStart}/o
   PartialTemplateParser       = /#{TagStart}.*?#{TagEnd}|#{VariableStart}.*?#{VariableIncompleteEnd}/om
   TemplateParser              = /(#{PartialTemplateParser}|#{AnyStartingTag})/om

lib/liquid/block_body.rb

@@ -4,11 +4,11 @@
 
 module Liquid
   class BlockBody
-    LiquidTagToken      = /\A\s*(#{TagName})\s*(.*?)\z/o
-    FullToken           = /\A#{TagStart}#{WhitespaceControl}?(\s*)(#{TagName})(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
-    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
+    LiquidTagToken      = /\A[[:space:]]*(#{TagName})[[:space:]]*(.*?)\z/o
+    FullToken           = /\A#{TagStart}#{WhitespaceControl}?([[:space:]]*)(#{TagName})([[:space:]]*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
+    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?[[:space:]]*(\w+)[[:space:]]*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
     ContentOfVariable   = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
-    WhitespaceOrNothing = /\A\s*\z/
+    WhitespaceOrNothing = /\A[[:space:]]*\z/
     TAGSTART            = "{%"
     VARSTART            = "{{"
 

lib/liquid/expression.rb

@@ -18,7 +18,7 @@ class Expression
 
     # Use an atomic group (?>...) to avoid pathological backtracing from
     # malicious input as described in https://github.com/Shopify/liquid/issues/1357
-    RANGES_REGEX         = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
+    RANGES_REGEX         = /\A\([[:space:]]*(?>(\S+)[[:space:]]*\.\.)[[:space:]]*(\S+)[[:space:]]*\)\z/
 
     def self.parse(markup)
       return nil unless markup

lib/liquid/lexer.rb

@@ -21,8 +21,8 @@ class Lexer
     STRING_LITERAL        = Regexp.union(SINGLE_STRING_LITERAL, DOUBLE_STRING_LITERAL)
     NUMBER_LITERAL        = /-?\d+(\.\d+)?/
     DOTDOT                = /\.\./
-    COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=\s)/
-    WHITESPACE_OR_NOTHING = /\s*/
+    COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=[[:space:]])/
+    WHITESPACE_OR_NOTHING = /[[:space:]]*/
 
     def initialize(input)
       @ss = StringScanner.new(input)

lib/liquid/tags/assign.rb

@@ -14,7 +14,7 @@ module Liquid
   # @liquid_syntax_keyword variable_name The name of the variable being created.
   # @liquid_syntax_keyword value The value you want to assign to the variable.
   class Assign < Tag
-    Syntax = /(#{VariableSignature}+)\s*=\s*(.*)\s*/om
+    Syntax = /(#{VariableSignature}+)[[:space:]]*=[[:space:]]*(.*)[[:space:]]*/om
 
     # @api private
     def self.raise_syntax_error(parse_context)

lib/liquid/tags/case.rb

@@ -24,7 +24,7 @@ module Liquid
   # @liquid_syntax_keyword third_expression An expression to be rendered when the variable's value has no match.
   class Case < Block
     Syntax     = /(#{QuotedFragment})/o
-    WhenSyntax = /(#{QuotedFragment})(?:(?:\s+or\s+|\s*\,\s*)(#{QuotedFragment}.*))?/om
+    WhenSyntax = /(#{QuotedFragment})(?:(?:[[:space:]]+or[[:space:]]+|[[:space:]]*\,[[:space:]]*)(#{QuotedFragment}.*))?/om
 
     attr_reader :blocks, :left
 

lib/liquid/tags/cycle.rb

@@ -16,7 +16,7 @@ module Liquid
   #   {% cycle string, string, ... %}
   class Cycle < Tag
     SimpleSyntax = /\A#{QuotedFragment}+/o
-    NamedSyntax  = /\A(#{QuotedFragment})\s*\:\s*(.*)/om
+    NamedSyntax  = /\A(#{QuotedFragment})[[:space:]]*\:[[:space:]]*(.*)/om
 
     attr_reader :variables
 
@@ -61,7 +61,7 @@ def render_to_output_buffer(context, output)
 
     def variables_from_string(markup)
       markup.split(',').collect do |var|
-        var =~ /\s*(#{QuotedFragment})\s*/o
+        var =~ /[[:space:]]*(#{QuotedFragment})[[:space:]]*/o
         Regexp.last_match(1) ? parse_expression(Regexp.last_match(1)) : nil
       end.compact
     end

lib/liquid/tags/for.rb

@@ -25,7 +25,7 @@ module Liquid
   # @liquid_optional_param range [untyped] A custom numeric range to iterate over.
   # @liquid_optional_param reversed [untyped] Iterate in reverse order.
   class For < Block
-    Syntax = /\A(#{VariableSegment}+)\s+in\s+(#{QuotedFragment}+)\s*(reversed)?/o
+    Syntax = /\A(#{VariableSegment}+)[[:space:]]+in[[:space:]]+(#{QuotedFragment}+)[[:space:]]*(reversed)?/o
 
     attr_reader :collection_name, :variable_name, :limit, :from
 

lib/liquid/tags/if.rb

@@ -14,8 +14,8 @@ module Liquid
   # @liquid_syntax_keyword condition The condition to evaluate.
   # @liquid_syntax_keyword expression The expression to render if the condition is met.
   class If < Block
-    Syntax                  = /(#{QuotedFragment})\s*([=!<>a-z_]+)?\s*(#{QuotedFragment})?/o
-    ExpressionsAndOperators = /(?:\b(?:\s?and\s?|\s?or\s?)\b|(?:\s*(?!\b(?:\s?and\s?|\s?or\s?)\b)(?:#{QuotedFragment}|\S+)\s*)+)/o
+    Syntax                  = /(#{QuotedFragment})[[:space:]]*([=!<>a-z_]+)?[[:space:]]*(#{QuotedFragment})?/o
+    ExpressionsAndOperators = /(?:\b(?:[[:space:]]?and[[:space:]]?|[[:space:]]?or[[:space:]]?)\b|(?:[[:space:]]*(?!\b(?:[[:space:]]?and[[:space:]]?|[[:space:]]?or[[:space:]]?)\b)(?:#{QuotedFragment}|\S+)[[:space:]]*)+)/o
     BOOLEAN_OPERATORS       = %w(and or).freeze
 
     attr_reader :blocks

lib/liquid/tags/include.rb

@@ -20,7 +20,7 @@ module Liquid
   class Include < Tag
     prepend Tag::Disableable
 
-    SYNTAX = /(#{QuotedFragment}+)(\s+(?:with|for)\s+(#{QuotedFragment}+))?(\s+(?:as)\s+(#{VariableSegment}+))?/o
+    SYNTAX = /(#{QuotedFragment}+)([[:space:]]+(?:with|for)[[:space:]]+(#{QuotedFragment}+))?([[:space:]]+(?:as)[[:space:]]+(#{VariableSegment}+))?/o
     Syntax = SYNTAX
 
     attr_reader :template_name_expr, :variable_name_expr, :attributes

lib/liquid/tags/inline_comment.rb

@@ -12,7 +12,7 @@ def initialize(tag_name, markup, options)
       #
       # As such, we're forcing users to put a # symbol on every line otherwise this
       # tag will throw an error.
-      if markup.match?(/\n\s*[^#\s]/)
+      if markup.match?(/\n[[:space:]]*[^#[[:space:]]]/)
         raise SyntaxError, options[:locale].t("errors.syntax.inline_comment_invalid")
       end
     end

lib/liquid/tags/raw.rb

@@ -13,7 +13,7 @@ module Liquid
   #   {% endraw %}
   # @liquid_syntax_keyword expression The expression to be output without being rendered.
   class Raw < Block
-    Syntax = /\A\s*\z/
+    Syntax = /\A[[:space:]]*\z/
 
     def initialize(tag_name, markup, parse_context)
       super

lib/liquid/tags/render.rb

@@ -27,7 +27,7 @@ module Liquid
   # @liquid_syntax_keyword filename The name of the snippet to render, without the `.liquid` extension.
   class Render < Tag
     FOR = 'for'
-    SYNTAX = /(#{QuotedString}+)(\s+(with|#{FOR})\s+(#{QuotedFragment}+))?(\s+(?:as)\s+(#{VariableSegment}+))?/o
+    SYNTAX = /(#{QuotedString}+)([[:space:]]+(with|#{FOR})[[:space:]]+(#{QuotedFragment}+))?([[:space:]]+(?:as)[[:space:]]+(#{VariableSegment}+))?/o
 
     disable_tags "include"
 

lib/liquid/tags/table_row.rb

@@ -24,7 +24,7 @@ module Liquid
   # @liquid_optional_param offset [number] The 1-based index to start iterating at.
   # @liquid_optional_param range [untyped] A custom numeric range to iterate over.
   class TableRow < Block
-    Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)/o
+    Syntax = /(\w+)[[:space:]]+in[[:space:]]+(#{QuotedFragment}+)/o
 
     attr_reader :variable_name, :collection_name, :attributes
 

lib/liquid/variable.rb

@@ -12,9 +12,9 @@ module Liquid
   #   {{ user | link }}
   #
   class Variable
-    FilterMarkupRegex        = /#{FilterSeparator}\s*(.*)/om
-    FilterParser             = /(?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+/o
-    FilterArgsRegex          = /(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o
+    FilterMarkupRegex        = /#{FilterSeparator}[[:space:]]*(.*)/om
+    FilterParser             = /(?:[[:space:]]+|#{QuotedFragment}|#{ArgumentSeparator})+/o
+    FilterArgsRegex          = /(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})[[:space:]]*((?:\w+[[:space:]]*\:[[:space:]]*)?#{QuotedFragment})/o
     JustTagAttributes        = /\A#{TagAttributes}\z/o
     MarkupWithQuotedFragment = /(#{QuotedFragment})(.*)/om
 

performance/shopify/paginate.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class Paginate < Liquid::Block
-  Syntax = /(#{Liquid::QuotedFragment})\s*(by\s*(\d+))?/
+  Syntax = /(#{Liquid::QuotedFragment})[[:space:]]*(by[[:space:]]*(\d+))?/
 
   def initialize(tag_name, markup, options)
     super

test/integration/tags/include_tag_test.rb

@@ -29,7 +29,7 @@ def read_template_file(_template_path)
 end
 
 class CustomInclude < Liquid::Tag
-  Syntax = /(#{Liquid::QuotedFragment}+)(\s+(?:with|for)\s+(#{Liquid::QuotedFragment}+))?/o
+  Syntax = /(#{Liquid::QuotedFragment}+)([[:space:]]+(?:with|for)[[:space:]]+(#{Liquid::QuotedFragment}+))?/o
 
   def initialize(tag_name, markup, tokens)
     markup =~ Syntax

test/integration/whitespace_test.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+require 'timeout'
+
+class WhitespaceTest < Minitest::Test
+  include Liquid
+
+
+  def test_if_0xa0_utf8_whitespace_parses_correctly
+    utf8_0xa0 = "\u00A0"
+    assert_template_result('one', "{% if foo ==#{utf8_0xa0}1 %}one{% endif %}", { 'foo' => IntegerDrop.new('1') })
+  end
+
+end