Add diacritics setting (#1636)

Setting to enable further char_filter mappings for diacritics.

.gitignore

@@ -8,6 +8,7 @@
 /config/config.php
 /config/databases.yml
 /config/propel.ini
+/config/diacritics_mapping.yml
 
 # Internal use
 /cache/

apps/qubit/modules/settings/actions/diacriticsAction.class.php

@@ -0,0 +1,103 @@
+<?php
+/*
+ * This file is part of the Access to Memory (AtoM) software.
+ *
+ * Access to Memory (AtoM) is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Access to Memory (AtoM) is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Access to Memory (AtoM).  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+class SettingsDiacriticsAction extends SettingsEditAction
+{
+    // Arrays not allowed in class constants
+    public static $NAMES = [
+        'diacritics',
+        'mappings',
+    ];
+
+    public function earlyExecute()
+    {
+        parent::earlyExecute();
+
+        $this->updateMessage = $this->i18n->__('Diacritics settings saved.');
+
+        $this->settingDefaults = [
+            'diacritics' => 0,
+        ];
+    }
+
+    public function processForm()
+    {
+        foreach ($this->form as $field) {
+            $this->processField($field);
+        }
+    }
+
+    protected function addField($name)
+    {
+        switch ($name) {
+            case 'diacritics':
+                $this->form->setDefault($name, $this->settingDefaults[$name]);
+                $this->form->setWidget($name, new sfWidgetFormSelectRadio(['choices' => [0 => $this->i18n->__('Disabled'), 1 => $this->i18n->__('Enabled')]], ['class' => 'radio']));
+                $this->form->setValidator($name, new sfValidatorChoice(['choices' => [1, 0]]));
+
+                break;
+
+            case 'mappings':
+                $this->form->setWidget($name, new sfWidgetFormInputFile([], ['accept' => '.yml,.yaml']));
+                $this->form->setValidator($name, new sfValidatorFile(['mime_types' => ['text/plain']]));
+
+                break;
+        }
+    }
+
+    protected function processField($field)
+    {
+        switch ($name = $field->getName()) {
+            case 'diacritics':
+                parent::processField($field);
+
+                break;
+
+            case 'mappings':
+                $file = $this->form->getValue('mappings');
+
+                $diacriticsMappingPath = sfConfig::get('sf_config_dir').DIRECTORY_SEPARATOR.'diacritics_mapping.yml';
+
+                if (null !== $file) {
+                    try {
+                        sfYaml::load($file->getTempName());
+
+                        if (!move_uploaded_file($file->getTempName(), $diacriticsMappingPath)) {
+                            $this->getUser()->setFlash('error', $this->context->i18n->__('Unable to upload diacritics mapping yaml file.'));
+                            unset($this->updateMessage);
+
+                            return;
+                        }
+                    } catch (Exception $e) {
+                        QubitSetting::findAndSave('diacritics', 0, ['sourceCulture' => true]);
+                        unlink($diacriticsMappingPath);
+                        $this->getUser()->setFlash('error', $this->context->i18n->__('Unable to upload diacritis mapping yaml file.'));
+                        unset($this->updateMessage);
+                    }
+                } else {
+                    // Reset diacritics settings when uploading yaml fails
+                    QubitSetting::findAndSave('diacritics', 0, ['sourceCulture' => true]);
+                    unlink($diacriticsMappingPath);
+                    $this->getUser()->setFlash('error', $this->context->i18n->__('Unable to upload diacritis mapping yaml file.'));
+                    unset($this->updateMessage);
+                }
+
+                break;
+        }
+    }
+}

apps/qubit/modules/settings/actions/editAction.class.php

@@ -1,5 +1,4 @@
 <?php
-
 /*
  * This file is part of the Access to Memory (AtoM) software.
  *
@@ -28,7 +27,7 @@ public function execute($request)
 
         // Handle posted data
         if ($request->isMethod('post')) {
-            $this->form->bind($request->getPostParameters());
+            $this->form->bind($request->getPostParameters(), $request->getFiles());
 
             if ($this->form->isValid()) {
                 $this->processForm();

apps/qubit/modules/settings/actions/menuComponent.class.php

@@ -39,6 +39,10 @@ public function execute($request)
                 'label' => $i18n->__('Default template'),
                 'action' => 'template',
             ],
+            [
+                'label' => $i18n->__('Diacritics'),
+                'action' => 'diacritics',
+            ],
             [
                 'label' => $i18n->__('Digital object derivatives'),
                 'action' => 'digitalObjectDerivatives',

apps/qubit/modules/settings/templates/diacriticsSuccess.php

@@ -0,0 +1,66 @@
+<?php decorate_with('layout_2col.php'); ?>
+
+<?php slot('sidebar'); ?>
+
+<?php echo get_component('settings', 'menu'); ?>
+
+<?php end_slot(); ?>
+
+<?php slot('title'); ?>
+
+<h1>
+  <?php echo __('Diacritics'); ?>
+</h1>
+
+<?php end_slot(); ?>
+
+<?php slot('content'); ?>
+<div class="alert alert-info">
+  <p>
+    <?php echo __('Please rebuild the search index after uploading diacritics mappings.'); ?>
+  </p>
+  <pre>$ php symfony search:populate</pre>
+</div>
+
+<div class="alert alert-info">
+  <p>
+    <?php echo __('Example CSV:'); ?>
+  </p>
+  <pre>type: mapping<br/>mappings:<br/>  - À => A<br/>  - Á => A</pre>
+</div>
+
+<?php echo $form->renderGlobalErrors(); ?>
+
+<?php echo $form->renderFormTag(url_for(['module' => 'settings', 'action' => 'diacritics'])); ?>
+
+<?php echo $form->renderHiddenFields(); ?>
+
+<div id="content">
+
+  <fieldset class="collapsible">
+    <legend>
+      <?php echo __('Diacritics settings'); ?>
+    </legend>
+
+    <?php echo $form->diacritics->label(__('Diacritics'))->renderRow(); ?>
+  </fieldset>
+
+  <fieldset class="collapsible">
+    <legend>
+      <?php echo __('CSV Mapping YAML'); ?>
+    </legend>
+
+    <?php echo $form->mappings->label(__('Mappings YAML'))->renderRow(); ?>
+  </fieldset>
+
+</div>
+
+<section class="actions">
+  <ul>
+    <li><input class="c-btn c-btn-submit" type="submit" value="<?php echo __('Save'); ?>" /></li>
+  </ul>
+</section>
+
+</form>
+
+<?php end_slot(); ?>

plugins/arDominionB5Plugin/modules/settings/templates/diacriticsSuccess.php

@@ -0,0 +1,74 @@
+<?php decorate_with('layout_2col.php'); ?>
+
+<?php slot('sidebar'); ?>
+
+<?php echo get_component('settings', 'menu'); ?>
+
+<?php end_slot(); ?>
+
+<?php slot('title'); ?>
+<h1>
+  <?php echo __('Diacritics settings'); ?>
+</h1>
+<?php end_slot(); ?>
+
+<?php slot('content'); ?>
+
+<div class="alert alert-info">
+  <p>
+    <?php echo __('Please rebuild the search index after uploading diacritics mappings.'); ?>
+  </p>
+  <pre>$ php symfony search:populate</pre>
+</div>
+
+<?php echo $form->renderGlobalErrors(); ?>
+
+<?php echo $form->renderFormTag(url_for(['module' => 'settings', 'action' => 'diacritics'])); ?>
+
+<?php echo $form->renderHiddenFields(); ?>
+
+<div class="accordion mb-3">
+  <div class="accordion-item">
+    <h2 class="accordion-header" id="diacritics-heading">
+      <button class="accordion-button" type="button" data-bs-toggle="collapse" data-bs-target="#diacritics-collapse"
+        aria-expanded="true" aria-controls="diacritics-collapse">
+        <?php echo __('Diacritics Settings'); ?>
+      </button>
+    </h2>
+    <div id="diacritics-collapse" class="accordion-collapse collapse show" aria-labelledby="diacritics-heading">
+      <div class="accordion-body">
+        <?php echo render_field($form->diacritics->label(__('Diacritics'))); ?>
+      </div>
+    </div>
+  </div>
+  <div class="accordion-item">
+    <h2 class="accordion-header" id="mappings-heading">
+      <button class="accordion-button" type="button" data-bs-toggle="collapse" data-bs-target="#mappings-collapse"
+        aria-expanded="false" aria-controls="mappings-collapse">
+        <?php echo __('CSV Mapping YAML'); ?>
+      </button>
+    </h2>
+
+    <div id="mappings-collapse" class="accordion-collapse collapse show" aria-labelledby="sending-heading">
+
+      <div class="alert alert-info m-3 mb-0">
+        <p>
+          <?php echo __('Example CSV:'); ?>
+        </p>
+        <pre>type: mapping<br/>mappings:<br/>  - À => A<br/>  - Á => A</pre>
+      </div>
+
+      <div class="accordion-body">
+        <?php echo render_field($form->mappings->label(__('Mappings YAML'))); ?>
+      </div>
+    </div>
+  </div>
+</div>
+
+<section class="actions">
+  <input class="btn atom-btn-outline-success" type="submit" value="<?php echo __('Save'); ?>" />
+</section>
+
+</form>
+
+<?php end_slot(); ?>

plugins/arElasticSearchPlugin/lib/arElasticSearchPlugin.class.php

@@ -119,10 +119,12 @@ public static function loadMappings()
     {
         // Find mapping.yml
         $finder = sfFinder::type('file')->name('mapping.yml');
-        $files = array_unique(array_merge(
-            $finder->in(sfConfig::get('sf_config_dir')),
-            $finder->in(ProjectConfiguration::getActive()->getPluginSubPaths('/config'))
-        ));
+        $files = array_unique(
+            array_merge(
+                $finder->in(sfConfig::get('sf_config_dir')),
+                $finder->in(ProjectConfiguration::getActive()->getPluginSubPaths('/config'))
+            )
+        );
 
         if (!count($files)) {
             throw new sfException('You must create a mapping.xml file.');
@@ -135,6 +137,24 @@ public static function loadMappings()
         return $esMapping;
     }
 
+    public function loadDiacriticsMappings()
+    {
+        // Find diacritics_mapping.yml
+        $diacriticsFinder = sfFinder::type('file')->name('diacritics_mapping.yml');
+        $diacriticsFiles = array_unique(
+            array_merge(
+                $diacriticsFinder->in(sfConfig::get('sf_config_dir')),
+                $diacriticsFinder->in(ProjectConfiguration::getActive()->getPluginSubPaths('/config'))
+            )
+        );
+
+        if (!count($diacriticsFiles)) {
+            throw new sfException('You must create a diacritics_mapping.yml file.');
+        }
+
+        return sfYaml::load(array_shift($diacriticsFiles));
+    }
+
     /**
      * Optimize index.
      *
@@ -272,10 +292,12 @@ public function populate($options = [])
             }
         }
 
-        $this->log(vsprintf(
-            'Index populated with %s documents in %s seconds.',
-            [$total, $timer->elapsed()]
-        ));
+        $this->log(
+            vsprintf(
+                'Index populated with %s documents in %s seconds.',
+                [$total, $timer->elapsed()]
+            )
+        );
 
         if (!$showErrors) {
             return;
@@ -468,6 +490,10 @@ public static function modelClassFromQubitObjectClass($className)
      */
     protected function initialize()
     {
+        if (sfConfig::get('app_diacritics')) {
+            $this->config['index']['configuration']['analysis']['char_filter']['diacritics_lowercase'] = $this->loadDiacriticsMappings();
+        }
+
         try {
             $this->index->open();
         } catch (Exception $e) {
@@ -479,7 +505,17 @@ protected function initialize()
                     && isset($this->config['index']['configuration']['analysis']['char_filter']['strip_md'])
                 ) {
                     foreach ($this->config['index']['configuration']['analysis']['analyzer'] as $key => $analyzer) {
-                        $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter'] = ['strip_md'];
+                        $filters = ['strip_md'];
+
+                        if ($this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']) {
+                            $filters = array_merge($filters, $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter']);
+                        }
+
+                        if (sfConfig::get('app_diacritics')) {
+                            $filters = array_merge($filters, ['diacritics_lowercase']);
+                        }
+
+                        $this->config['index']['configuration']['analysis']['analyzer'][$key]['char_filter'] = $filters;
                     }
                 }