20321 fix consistent interpolations check

lib/i18n/tasks/interpolations.rb

@@ -3,25 +3,30 @@
 module I18n::Tasks
   module Interpolations
     class << self
-      attr_accessor :variable_regex
+      attr_accessor :variable_regex, :tag_pairs, :tag_with_localized_value_regex
     end
-    @variable_regex = /%{[^}]+}/.freeze
+    @variable_regex = /%\{[^}]+\}|\{\{.*?\}\}|\{%.*?%\}/.freeze
+    @tag_pairs = [
+      ['{{', '}}'],
+      ['%{', '}'],
+      ['{%', '%}']
+    ].freeze
+    @tag_with_localized_value_regex = /\{\{\s?(("[^"]+")|('[^']+'))\s?\|.*?\}\}/
 
     def inconsistent_interpolations(locales: nil, base_locale: nil) # rubocop:disable Metrics/AbcSize
       locales ||= self.locales
       base_locale ||= self.base_locale
       result = empty_forest
-      variable_regex = I18n::Tasks::Interpolations.variable_regex
 
       data[base_locale].key_values.each do |key, value|
         next if !value.is_a?(String) || ignore_key?(key, :inconsistent_interpolations)
 
-        base_vars = Set.new(value.scan(variable_regex))
+        base_vars = get_normalized_variables_set(value)
         (locales - [base_locale]).each do |current_locale|
           node = data[current_locale].first.children[key]
           next unless node&.value.is_a?(String)
 
-          if base_vars != Set.new(node.value.scan(variable_regex))
+          if base_vars != get_normalized_variables_set(node.value)
             result.merge!(node.walk_to_root.reduce(nil) { |c, p| [p.derive(children: c)] })
           end
         end
@@ -30,5 +35,27 @@ def inconsistent_interpolations(locales: nil, base_locale: nil) # rubocop:disabl
       result.each { |root| root.data[:type] = :inconsistent_interpolations }
       result
     end
+
+    def get_normalized_variables_set(string)
+      Set.new(string.scan(I18n::Tasks::Interpolations.variable_regex).map { |variable| normalize(variable) })
+    end
+
+    def normalize(variable)
+      normalized = nil
+      if (match = variable.match(I18n::Tasks::Interpolations.tag_with_localized_value_regex))
+        variable = variable.sub(match[1], 'localized input')
+      end
+      I18n::Tasks::Interpolations.tag_pairs.each do |start, end_|
+        next unless variable.start_with?(start)
+
+        normalized = variable.delete_prefix(start).delete_suffix(end_).strip
+        normalized = start + normalized + end_
+        break
+      end
+
+      fail 'No start/end tag pair detected' if normalized.nil?
+
+      normalized
+    end
   end
 end

lib/i18n/tasks/translators/base_translator.rb

@@ -104,7 +104,7 @@ def parse_value(untranslated, each_translated, opts)
         end
       end
 
-      INTERPOLATION_KEY_RE = /%\{[^}]+}/.freeze
+      INTERPOLATION_KEY_RE = /%\{[^}]+\}|\{\{.*?\}\}|\{%.*?%\}/.freeze
       UNTRANSLATABLE_STRING = 'X__'
 
       # @param [String] value

lib/i18n/tasks/translators/deepl_translator.rb

@@ -23,6 +23,11 @@ def initialize(*)
 
     def translate_values(list, from:, to:, **options)
       results = []
+
+      if (glossary = glossary_for(from, to))
+        options.merge!({ glossary_id: glossary.id })
+      end
+
       list.each_slice(BATCH_SIZE) do |parts|
         res = DeepL.translate(
           parts,
@@ -93,6 +98,13 @@ def to_deepl_target_locale(locale)
       end
     end
 
+    # Find the largest glossary given a language pair
+    def glossary_for(source, target)
+      DeepL.glossaries.list.select do |glossary|
+        glossary.source_lang == source && glossary.target_lang == target
+      end.max_by(&:entry_count)
+    end
+
     def configure_api_key!
       api_key = @i18n_tasks.translation_config[:deepl_api_key]
       host = @i18n_tasks.translation_config[:deepl_host]

spec/deepl_translate_spec.rb

@@ -5,9 +5,20 @@
 require 'deepl'
 
 RSpec.describe 'DeepL Translation' do
-  nil_value_test  = ['nil-value-key', nil, nil]
-  text_test       = ['key', "Hello, %{user} O'Neill! How are you?", "¡Hola, %{user} O'Neill! ¿Qué tal estás?"]
-  html_test_plrl  = ['html-key.html.one', '<span>Hello %{count}</span>', '<span>Hola %{count}</span>']
+  nil_value_test = ['nil-value-key', nil, nil]
+
+  text_test = [
+    'key',
+    "Hello, %{user} O'Neill! How are you? {{ Check out this Liquid tag, it should not be translated }} \
+    {% That applies to this Liquid tag as well %}",
+    "¡Hola, %{user} O'Neill! ¿Qué tal estás? {{ Check out this Liquid tag, it should not be translated }} \
+    {% That applies to this Liquid tag as well %}"
+  ]
+
+  html_test_plrl = [
+    'html-key.html.one', '<span>Hello %{count} {{ count }} {% count %}</span>',
+    '<span>Hola %{count} {{ count }} {% count %}</span>'
+  ]
   array_test      = ['array-key', ['Hello.', nil, '', 'Goodbye.'], ['Hola.', nil, '', 'Adiós.']]
   array_hash_test = ['array-hash-key',
                      [{ 'hash_key1' => 'How are you?' }, { 'hash_key2' => nil }, { 'hash_key3' => 'Well.' }],

spec/interpolations_spec.rb

@@ -5,9 +5,6 @@
 RSpec.describe 'Interpolations' do
   let!(:task) { I18n::Tasks::BaseTask.new }
 
-  let(:base_keys) { { 'a' => 'hello %{world}', 'b' => 'foo', 'c' => { 'd' => 'hello %{name}' }, 'e' => 'ok' } }
-  let(:test_keys) { { 'a' => 'hello', 'b' => 'foo %{bar}', 'c' => { 'd' => 'hola %{amigo}' }, 'e' => 'ok' } }
-
   around do |ex|
     TestCodebase.setup(
       'config/i18n-tasks.yml' => { base_locale: 'en', locales: %w[es] }.to_yaml,
@@ -19,13 +16,63 @@
     TestCodebase.teardown
   end
 
-  it '#inconsistent_interpolation' do
-    wrong  = task.inconsistent_interpolations
-    leaves = wrong.leaves.to_a
+  context 'when using ruby string interpolations' do
+    let(:base_keys) { { 'a' => 'hello %{world}', 'b' => 'foo', 'c' => { 'd' => 'hello %{name}' }, 'e' => 'ok' } }
+    let(:test_keys) { { 'a' => 'hello', 'b' => 'foo %{bar}', 'c' => { 'd' => 'hola %{amigo}' }, 'e' => 'ok' } }
+
+    it 'detects inconsistent interpolations' do
+      wrong  = task.inconsistent_interpolations
+      leaves = wrong.leaves.to_a
+
+      expect(leaves.size).to eq 3
+      expect(leaves[0].full_key).to eq 'es.a'
+      expect(leaves[1].full_key).to eq 'es.b'
+      expect(leaves[2].full_key).to eq 'es.c.d'
+    end
+  end
+
+  context 'when using liquid tags' do
+    let(:base_keys) do
+      {
+        a: 'hello {{ world }}',
+        b: 'foo',
+        c: {
+          d: 'hello {{ name }}'
+        },
+        e: 'ok',
+        f: 'inconsistent {{ whitespace}}',
+        g: 'includes a {% comment %}',
+        h: 'wrong {% comment %}',
+        i: 'with localized value: {{ "thanks" | owner_invoice }}',
+        j: 'with wrong function: {{ "thanks" | owner_invoices }}'
+      }
+    end
+    let(:test_keys) do
+      {
+        a: 'hello',
+        b: 'foo {{ bar }}',
+        c: {
+          d: 'hola {{ amigo }}'
+        },
+        e: 'ok',
+        f: '{{whitespace }} inconsistentes',
+        g: 'incluye un {% comment %}',
+        h: '{% commentario %} equivocado',
+        i: 'con valor localizado: {{ "gracias" | owner_invoice }}',
+        j: 'con función incorrecta: {{ "thanks" | owner_invoice }}'
+      }
+    end
+
+    it 'detects inconsistent interpolations' do
+      wrong  = task.inconsistent_interpolations
+      leaves = wrong.leaves.to_a
 
-    expect(leaves.size).to eq 3
-    expect(leaves[0].full_key).to eq 'es.a'
-    expect(leaves[1].full_key).to eq 'es.b'
-    expect(leaves[2].full_key).to eq 'es.c.d'
+      expect(leaves.size).to eq 5
+      expect(leaves[0].full_key).to eq 'es.a'
+      expect(leaves[1].full_key).to eq 'es.b'
+      expect(leaves[2].full_key).to eq 'es.c.d'
+      expect(leaves[3].full_key).to eq 'es.h'
+      expect(leaves[4].full_key).to eq 'es.j'
+    end
   end
 end