ISSUE-14: LoD Reconciling + CSV Export + Release Improvements

ami.install

@@ -53,7 +53,9 @@ function ami_update_8902() {
     ->setDescription(new TranslatableMarkup('A Zip file containing accompanying Files for the Source Data'))
     ->setSetting('file_extensions', 'zip')
     ->setSetting('upload_validators', $validators)
-    ->setRequired(TRUE)
+    ->setSetting('uri_scheme', 'private')
+    ->setSetting('file_directory', '/ami/zip')
+    ->setRequired(FALSE)
     ->setDisplayOptions('view', [
       'label' => 'above',
       'type' => 'file',
@@ -100,4 +102,17 @@ function ami_update_8903() {
         "plugin_configuration"
      ],
    ]));
-}
+}
+
+/**
+ * Update 8903 - Make Private default upload location for ami Set ZIP files.
+ */
+function ami_update_8904() {
+  $field_storage_definition = \Drupal::entityDefinitionUpdateManager()->getFieldStorageDefinition('zip_file', 'ami_set_entity');
+  $field_storage_definition->setSetting('uri_scheme', 'private');
+  $field_storage_definition->setSetting('file_directory', '/ami/zip');
+  \Drupal::entityDefinitionUpdateManager()->updateFieldStorageDefinition($field_storage_definition);
+}
+
+
+

ami.links.task.yml

@@ -38,3 +38,15 @@ ami_set_entity.delete_process_form:
   base_route: entity.ami_set_entity.canonical
   title: Delete Processed ADOs
   weight: 12
+
+ami_set_entity.reconcile_form:
+  route_name: entity.ami_set_entity.reconcile_form
+  base_route: entity.ami_set_entity.canonical
+  title: Reconcile LoD
+  weight: 13
+
+entity.ami_set_entity.reconcileedit_form:
+  route_name: entity.ami_set_entity.reconcileedit_form
+  base_route: entity.ami_set_entity.canonical
+  title: Edit Reconciled LoD
+  weight: 14

ami.module

@@ -3,7 +3,14 @@
  * @file
  * Contains ami.module.
  */
+
+use Drupal\Core\Language\Language;
 use Drupal\Core\Routing\RouteMatchInterface;
+use Drupal\Core\Form\FormStateInterface;
+use Drupal\Core\Url;
+use Drupal\format_strawberryfield\Event\FormatStrawberryfieldFormAlterEvent;
+use Drupal\format_strawberryfield\FormatStrawberryfieldEventType;
+
 /**
  * Implements hook_help().
  */
@@ -24,3 +31,102 @@ function ami_help($route_name, RouteMatchInterface $route_match) {
       }
   }
 }
+
+function ami_form_metadatadisplay_entity_edit_form_alter(&$form,FormStateInterface $form_state, $form_id) {
+
+  // Add our AMI options here
+  $form['preview']['entity_type'] = [
+    '#type' => 'radios',
+    '#weight' => -10,
+    '#title' => t('Entity type'),
+    '#description' => t('The Entity Type you want to preview.'),
+    '#options' => [
+      'ado' => t('Archipelago Digital Objects'),
+      'ami' => t('AMI Sets'),
+    ],
+    '#default_value' => $form_state->getValue('entity_type', NULL) ?? 'ado'
+  ];
+
+
+  $form['preview']['ado_context_preview']['#states'] = [
+      'visible' => [
+        ':input[name="entity_type"]' => ['value' => 'ado'],
+      ],
+    ];
+
+  $form['preview']['ado_amiset_preview'] = [
+    '#type' => 'entity_autocomplete',
+    '#weight' => -9,
+    '#title' => t('Ami Set to preview'),
+    '#description' => t('The AMI Set to be used to preview the data.'),
+    '#target_type' => 'ami_set_entity',
+    '#maxlength' => 1024,
+    '#ajax' => [
+      'callback' => '\Drupal\ami\Controller\AmiRowAutocompleteHandler::rowAjaxCallback',
+      'event' => 'autocompleteclose change',
+    ],
+    '#states' => [
+      'visible' => [':input[name="entity_type"]' => ['value' => 'ami']],
+    ],
+  ];
+
+  $form['preview']['ado_amiset_row_context_preview'] = [
+    '#type' => 'textfield',
+    '#weight' => -8,
+    '#title' => t('Row to preview'),
+    '#states' => [
+      'visible' => [
+        'input[name="entity_type"]' => ['value' => 'ami'],
+        'input[name="ado_amiset_preview"' => ['filled' => true],
+      ],
+    ],
+  ];
+  $ami_set = $form_state->getValue('ado_amiset_preview', NULL);
+  if (is_scalar($ami_set)) {
+    $form['preview']['ado_amiset_row_context_preview']['#autocomplete_route_name'] = 'ami.rowsbylabel.autocomplete';
+    $form['preview']['ado_amiset_row_context_preview']['#autocomplete_route_parameters'] = [
+      'ami_set_entity' => $ami_set
+    ];
+  }
+
+  $form['preview']['button_preview'][
+    '#states'] = [
+      'visible' => [
+        'input[name="ado_context_preview"' => ['filled' => true],
+        'input[name="entity_type"]' => ['value' => 'ado'],
+      ],
+    ];
+
+  $form['preview']['button_preview_amiset'] = [
+    '#type' => 'button',
+    '#op' => 'preview',
+    '#weight' => -7,
+    '#value' => t('Show preview for AMI Set'),
+    '#ajax' => [
+      'callback' => '\Drupal\ami\Controller\AmiRowAutocompleteHandler::ajaxPreviewAmiSet',
+    ],
+    '#states' => [
+      'visible' => [
+        'input[name="ado_amiset_preview"' => ['filled' => true],
+        'input[name="entity_type"]' => ['value' => 'ami']
+      ],
+    ],
+  ];
+
+  $form['preview']['render_native'] = [
+    '#type' => 'checkbox',
+    '#weight' => 10,
+    '#defaut_value' => FALSE,
+    '#title' => 'Show Preview using native Output Format (e.g HTML)',
+    '#states' => [
+      'visible' => [
+        ['input[name="ado_context_preview"' => ['filled' => true]],
+        'or',
+        ['input[name="ado_amiset_preview"' => ['filled' => true],
+          'input[name="ado_amiset_row_context_preview"' => ['filled' => true]]
+      ],
+    ],
+  ];
+
+  return $form;
+}

ami.permissions.yml

@@ -1,6 +1,9 @@
 Bulk Patch Strawberryfield:
   title: 'JSON Patch Archipelago Digital Objects in Bulk using Views Batch Operations'
   description: 'Allows JSON Patch actions to be executed against a "Strawberryfield" bearing Entity. Still on Patch time, individual permissions may apply and/or override this permission'
+CSV Export Archipelago Digital Objects:
+  title: 'Export Archipelago Digital Objects in Bulk using Views Batch Operations to CSV'
+  description: 'CSV Export actions to be executed against a "Strawberryfield" bearing Entity. Still on export time, individual permissions may apply and/or override this permission'
 Multi Import Digital Objects:
   title: 'Import or Update assets from remote sources or files using Archipelago Multi Importer'
   description: 'Allows AMI to be used to import or update Digital Objects and assets. Still on import(create) and update time, individual permissions may apply and/or override this permission'

ami.routing.yml

@@ -86,3 +86,32 @@ entity.ami_set_entity.delete_process_form:
     _title: 'Process Ami Set'
   requirements:
     _entity_access: 'ami_set_entity.deleteados'
+
+entity.ami_set_entity.reconcile_form:
+  path: '/amiset/{ami_set_entity}/reconcile'
+  defaults:
+    _entity_form: ami_set_entity.reconcile
+    _title: 'Reconcile LoD'
+  requirements:
+    _entity_access: 'ami_set_entity.process'
+entity.ami_set_entity.reconcileedit_form:
+  path: '/amiset/{ami_set_entity}/editreconcile'
+  defaults:
+    _entity_form: ami_set_entity.editreconcile
+    _title: 'Clean Reconciled LoD'
+  requirements:
+    _entity_access: 'ami_set_entity.process'
+
+ami.rowsbylabel.autocomplete:
+  path: '/admin/amiset/autocomplete/{ami_set_entity}/rowsbylabel'
+  options:
+    parameters:
+      ami_set_entity:
+        type: 'entity:ami_set_entity'
+  defaults:
+    _controller: '\Drupal\ami\Controller\AmiRowAutocompleteHandler::handleAutocomplete'
+    _format: json
+  requirements:
+    _entity_access: 'ami_set_entity.view'
+    _permission: 'access content'
+    _csrf_token: 'TRUE'

ami.services.yml

@@ -5,6 +5,11 @@ services:
     arguments: ['@entity_type.manager']
   ami.utility:
     class: Drupal\ami\AmiUtilityService
-    arguments: [ '@file_system', '@file.usage', '@entity_type.manager', '@stream_wrapper_manager', '@plugin.manager.archiver', '@config.factory', '@current_user', '@language_manager', '@transliteration', '@module_handler', '@logger.factory', '@strawberryfield.utility', '@entity_field.manager', '@entity_type.bundle.info', '@http_client']
+    arguments: [ '@file_system', '@file.usage', '@entity_type.manager', '@stream_wrapper_manager', '@plugin.manager.archiver', '@config.factory', '@current_user', '@language_manager', '@transliteration', '@module_handler', '@logger.factory', '@strawberryfield.utility', '@entity_field.manager', '@entity_type.bundle.info', '@http_client', '@ami.lod', '@keyvalue', '@strawberryfield.file_metadata_extractor']
+    tags:
+      - { name: backend_overridable }
+  ami.lod:
+    class: Drupal\ami\AmiLoDService
+    arguments: [ '@file_system', '@file.usage', '@entity_type.manager', '@stream_wrapper_manager', '@plugin.manager.archiver', '@config.factory', '@current_user', '@language_manager', '@transliteration', '@module_handler', '@logger.factory', '@strawberryfield.utility', '@http_client', '@keyvalue']
     tags:
       - { name: backend_overridable }

composer.json

@@ -24,7 +24,7 @@
         "swaggest/json-diff": "^3.7.5",
         "phpoffice/phpspreadsheet": "^1.15.0",
         "maennchen/zipstream-php": "^1.2 || ^2.1",
-        "drupal/google_api_client": "^3.0",
+        "drupal/google_api_client": "^3.0 || ^4.0",
         "ramsey/uuid": "^4.1"
     },
     "minimum-stability": "dev",

src/AmiBatchQueue.php

@@ -53,7 +53,15 @@ public static function takeOne(string $queue_name, string $set_id, array &$conte
     try {
       // Only process Items of this Set if $context['set_id'] is set.
       if ($item = $queue->claimItem()) {
-        $ado_title = isset($item->data->info['row']['uuid']) ? 'ADO with UUID '.$item->data->info['row']['uuid'] : 'Unidentifed ADO without UUID';
+        // Let's figure out the type of queue running here, ADO or Attached file
+        if (!empty($item->data->info['filename']) && !empty($item->data->info['file_column']) && !empty($item->data->info['processed_row'])) {
+          $ado_title = 'File ' . $item->data->info['filename'];
+          $ado_title .= isset($item->data->info['uuid']) ? ' for ADO with UUID ' . $item->data->info['uuid'] : 'for Unidentifed ADO without UUID ';
+        }
+        else {
+         $ado_title = isset($item->data->info['row']['uuid']) ? 'ADO with UUID ' . $item->data->info['row']['uuid'] : 'Unidentifed ADO without UUID';
+        }
+
         $title = t('For %name processing %adotitle, <b>%count</b> items remaining', [
           '%name' => $context['results']['queue_label'],
           '%adotitle' => $ado_title,
@@ -136,11 +144,17 @@ public static function finish($success, $results, $operations) {
         )
       );
     }
-    // Cleanup and remove the queue. This is a live batch operation.
-    /** @var \Drupal\Core\Queue\QueueFactory $queue_factory */
-    $queue_name = $results['queue_name'];
-    $queue_factory = \Drupal::service('queue');
-    $queue_factory->get($queue_name)->deleteQueue();
+    // If the queue fails for whatever reason the $context may be lost
+    if (isset($results['queue_name'])) {
+      // Cleanup and remove the queue. This is a live batch operation.
+      /** @var \Drupal\Core\Queue\QueueFactory $queue_factory */
+      $queue_name = $results['queue_name'];
+      $queue_factory = \Drupal::service('queue');
+      $queue_factory->get($queue_name)->deleteQueue();
+    }
+    else {
+      \Drupal::messenger()->addError(\Drupal::translation('The Batch Operation failed. Please check your logs, available Filesystem space and try again'));
+    }
   }
 
 }