Merge pull request #727 from UN-OCHA/develop

Develop -> Main - v2.4.0

composer.json

@@ -71,6 +71,7 @@
         "reliefweb/simple-datepicker": "^v1.3",
         "symfony/uid": "^6.2",
         "unocha/common_design": "^9",
+        "unocha/ocha_ai": "^1.0",
         "unocha/ocha_monitoring": "^1.0",
         "webflo/drupal-finder": "^1.2.2"
     },

config/block.block.common_design_subtheme_ochaaichatpopup.yml

@@ -0,0 +1,33 @@
+uuid: b9a70ca4-c0f1-435d-9d2f-4ad6abc77a99
+langcode: en
+status: true
+dependencies:
+  module:
+    - node
+    - ocha_ai_chat
+    - system
+  theme:
+    - common_design_subtheme
+id: common_design_subtheme_ochaaichatpopup
+theme: common_design_subtheme
+region: content
+weight: 0
+provider: null
+plugin: ocha_ai_chat_chat_popup
+settings:
+  id: ocha_ai_chat_chat_popup
+  label: 'OCHA AI Chat popup'
+  label_display: '0'
+  provider: ocha_ai_chat
+visibility:
+  'entity_bundle:node':
+    id: 'entity_bundle:node'
+    negate: false
+    context_mapping:
+      node: '@node.node_route_context:node'
+    bundles:
+      report: report
+  request_path:
+    id: request_path
+    negate: true
+    pages: '/node/*/*'

config/core.extension.yml

@@ -51,6 +51,8 @@ module:
   monitoring: 0
   mysql: 0
   node: 0
+  ocha_ai: 0
+  ocha_ai_chat: 0
   ocha_monitoring: 0
   options: 0
   page_cache: 0

config/ocha_ai.settings.yml

@@ -0,0 +1,81 @@
+plugins:
+  completion:
+    aws_bedrock:
+      model: amazon.titan-text-express-v1
+      endpoint: ''
+      version: ''
+      region: REPLACE_WITH_REGION
+      api_key: REPLACE_WITH_KEY
+      max_tokens: 512
+      prompt_template: |-
+        {{ context }}
+
+        Based solely on the information above, please answer the following question. Please do not make things up and say you don't know if you cannot answer.
+
+        {{ question }}
+      api_secret: REPLACE_WITH_SECRET
+      role_arn: null
+    azure_openai:
+      model: gpt-3.5-turbo
+      endpoint: REPLACE_WITH_ENDPOINT
+      version: '2023-05-15'
+      region: REPLACE_WITH_REGION
+      api_key: REPLACE_WITH_KEY
+      max_tokens: 512
+      prompt_template: |-
+        Based solely on the information below, please answer the user's question. Please do not make things up and say you don't know if you cannot answer.
+
+        {{ context }}
+  embedding:
+    aws_bedrock:
+      model: amazon.titan-embed-text-v1
+      endpoint: ''
+      version: ''
+      region: REPLACE_WITH_REGION
+      api_key: REPLACE_WITH_KEY
+      batch_size: 1
+      dimensions: 1536
+      max_tokens: 8192
+      api_secret: REPLACE_WITH_SECRET
+      role_arn: null
+    azure_openai:
+      model: text-embedding-ada-002
+      endpoint: REPLACE_WITH_ENDPOINT
+      version: '2023-05-15'
+      region: REPLACE_WITH_REGION
+      api_key: REPLACE_WITH_KEY
+      batch_size: 16
+      dimensions: 1536
+      max_tokens: 4096
+  source:
+    reliefweb:
+      api_url: 'https://api.reliefweb.int/v1'
+      converter_url: 'https://reliefweb.int/search/converter/json'
+      site_url: 'https://reliefweb.int'
+      appname: ocha-ai
+      cache_enabled: true
+      cache_lifetime: 600
+  text_extractor:
+    mupdf:
+      mutool: /usr/bin/mutool
+  text_splitter:
+    token:
+      length: 384
+      overlap: 96
+    sentence:
+      length: 4
+      overlap: 2
+    page:
+      length: 1
+      overlap: 0
+  vector_store:
+    elasticsearch:
+      url: REPLACE_WITH_URL
+      base_index_name: ocha_ai
+      shards: 1
+      replicas: 0
+      nested_object_limit: 100000
+      indexing_batch_size: 10
+      topk: 5
+      min_similarity: 0.3
+      cutoff_coefficient: 0.5

config/ocha_ai_chat.settings.yml

@@ -0,0 +1,19 @@
+defaults:
+  form:
+    instructions:
+      value: ''
+      format: markdown_editor
+  plugins:
+    completion:
+      plugin_id: aws_bedrock
+    embedding:
+      plugin_id: aws_bedrock
+    source:
+      plugin_id: reliefweb
+    text_extractor:
+      application/pdf:
+        plugin_id: mupdf
+    text_splitter:
+      plugin_id: token
+    vector_store:
+      plugin_id: elasticsearch

config/user.role.beta_tester.yml

@@ -1,9 +1,12 @@
 uuid: d32877fc-1a02-4626-902d-577e71464493
 langcode: en
 status: true
-dependencies: {  }
+dependencies:
+  module:
+    - ocha_ai_chat
 id: beta_tester
 label: 'Beta tester'
 weight: 7
 is_admin: null
-permissions: {  }
+permissions:
+  - 'access ocha ai chat'

docker/Dockerfile

@@ -3,15 +3,16 @@ FROM public.ecr.aws/unocha/php-k8s:8.2-stable as builder
 
 ARG  BRANCH_ENVIRONMENT
 
-ENV  NODE_ENV=$BRANCH_ENVIRONMENT
-
 # Copy only the files necessary to build the code (see .dockerignore in root).
 COPY . /srv/www
 
 WORKDIR /srv/www
 
 # Clean up previous composer installation and run new one.
-RUN rm -rf ./vendor && composer install --no-interaction --no-dev
+RUN rm -rf ./vendor && \
+  COMPOSER_ALLOW_SUPERUSER=1 \
+  COMPOSER_MEMORY_LIMIT=-1 \
+  composer install --no-interaction --no-dev --prefer-dist
 
 # Copy settings to default site location.
 RUN cp -a docker/settings.php docker/services.yml docker/memcache.services.yml html/sites/default

html/modules/custom/reliefweb_entities/reliefweb_entities.module

@@ -17,6 +17,7 @@ use Drupal\Core\Render\Element;
 use Drupal\media\MediaInterface;
 use Drupal\reliefweb_entities\BundleEntityInterface;
 use Drupal\reliefweb_entities\BundleEntityStorageInterface;
+use Drupal\reliefweb_entities\Entity\Report;
 use Drupal\reliefweb_entities\EntityFormAlterServiceBase;
 use Drupal\reliefweb_utility\Helpers\ClassHelper;
 use Drupal\reliefweb_utility\Helpers\DateHelper;
@@ -842,3 +843,63 @@ function reliefweb_entities_node_delete(EntityInterface $entity) {
     }
   }
 }
+
+/**
+ * Implements hook_form_FORM_ID_alter() for `ocha_ai_chat_chat_form`.
+ */
+function reliefweb_entities_form_ocha_ai_chat_chat_form_alter(array &$form, FormStateInterface $form_state, string $form_id) {
+  $url = \Drupal::request()?->query?->get('url');
+  if (isset($url)) {
+    $router = \Drupal::service('router.no_access_checks');
+    $parameters = $router->match($url);
+    $node = $parameters['node'] ?? NULL;
+
+    if (isset($node) && $node instanceof Report) {
+      // No need to show the source when chatting with a single report.
+      if (isset($form['source'])) {
+        $form['source']['#access'] = FALSE;
+      }
+
+      // Only English documents are supported due to LLM limitations.
+      $is_english_report = FALSE;
+      foreach ($node->field_language as $item) {
+        if ($item->target_id == 267) {
+          $is_english_report = TRUE;
+          break;
+        }
+      }
+      if (!$is_english_report) {
+        $reason = t('Sorry, only English reports are supported.');
+      }
+
+      // Non supported content formats.
+      foreach ($node->field_content_format as $item) {
+        if ($item->target_id == 12) {
+          $reason = t('Sorry, maps are not supported.');
+          break;
+        }
+        elseif ($item->target_id == 12570) {
+          $reason = t('Sorry, infographics are not supported.');
+          break;
+        }
+        elseif ($item->target_id == 38974) {
+          $reason = t('Sorry, interactive reports are not supported.');
+          break;
+        }
+      }
+
+      if (!empty($reason)) {
+        foreach (Element::children($form) as $key) {
+          $form[$key]['#access'] = FALSE;
+        }
+        $form['unsupported'] = [
+          '#type' => 'inline_template',
+          '#template' => '<p class="ocha-ai-chat-chat-form__unsupported">{{ reason }}</p>',
+          '#context' => [
+            'reason' => $reason,
+          ],
+        ];
+      }
+    }
+  }
+}

html/modules/custom/reliefweb_files/src/Plugin/Field/FieldType/ReliefWebFile.php

@@ -877,8 +877,8 @@ protected function extractPreview($source_uri, $destination_uri, $page, $rotatio
 
     $mutool = \Drupal::state()->get('mutool', '/usr/bin/mutool');
     if (is_executable($mutool)) {
-      // @todo add max dimensions.
-      $command = "{$mutool} draw -R {$rotation} -o {$destination} {$source} {$page}";
+      $options = \Drupal::state()->get('mutool_options', '');
+      $command = "{$mutool} draw {$options} -R {$rotation} -o {$destination} {$source} {$page}";
       exec($command, $output, $return_val);
       // @todo log error?
       return empty($return_val) && @file_exists($destination_uri);

html/themes/custom/common_design_subtheme/templates/cd/cd-header/cd-ocha.html.twig

@@ -3,7 +3,7 @@
   {% block related_platforms %}
     <li class="cd-ocha-dropdown__link"><a href="https://response.reliefweb.int">{{ 'Reliefweb Response'|t }}</a></li>
     <li class="cd-ocha-dropdown__link"><a href="https://data.humdata.org/">{{ 'Humanitarian Data Exchange'|t }}</a></li>
-    <li class="cd-ocha-dropdown__link"><a href="https://hum-insight.info/">{{ 'Humanitarian InSight'|t }}</a></li>
+    <li class="cd-ocha-dropdown__link"><a href="https://humanitarianaction.info">{{ 'Humanitarian Action'|t }}</a></li>
     <li class="cd-ocha-dropdown__link"><a href="https://cerf.un.org">{{ 'Central Emergency Response Fund'|t }}</a></li>
   {% endblock %}
 

html/themes/custom/common_design_subtheme/templates/layout/page--ai--chat--popup.html.twig

@@ -0,0 +1,52 @@
+{#
+/**
+ * @file
+ * Theme override to display a single page.
+ *
+ * The doctype, html, head and body tags are not in this template. Instead they
+ * can be found in the html.html.twig template in this directory.
+ *
+ * Available variables:
+ *
+ * General utility variables:
+ * - base_path: The base URL path of the Drupal installation. Will usually be
+ *   "/" unless you have installed Drupal in a sub-directory.
+ * - is_front: A flag indicating if the current page is the front page.
+ * - logged_in: A flag indicating if the user is registered and signed in.
+ * - is_admin: A flag indicating if the user has permission to access
+ *   administration pages.
+ *
+ * Site identity:
+ * - front_page: The URL of the front page. Use this instead of base_path when
+ *   linking to the front page. This includes the language domain or prefix.
+ *
+ * Page content (in order of occurrence in the default page.html.twig):
+ * - node: Fully loaded node, if there is an automatically-loaded node
+ *   associated with the page and the node ID is the second argument in the
+ *   page's path (e.g. node/12345 and node/12345/revisions, but not
+ *   comment/reply/12345).
+ *
+ * Regions:
+ * - page.header: Items for the header region.
+ * - page.primary_menu: Items for the primary menu region.
+ * - page.secondary_menu: Items for the secondary menu region.
+ * - page.highlighted: Items for the highlighted content region.
+ * - page.help: Dynamic help text, mostly for admin pages.
+ * - page.content: The main content of the current page.
+ * - page.sidebar_first: Items for the first sidebar.
+ * - page.sidebar_second: Items for the second sidebar.
+ * - page.footer: Items for the footer region.
+ * - page.breadcrumb: Items for the breadcrumb region.
+ *
+ * @see template_preprocess_page()
+ * @see html.html.twig
+ */
+#}
+<div class="ocha-ai-chat">
+  {# Link to skip to the main content is in html.html.twig #}
+  <main aria-label="{{ 'Chat'|t }}" id="main-content">
+    <div class="ocha-ai-chat__content">
+      {{ page.content }}
+    </div>
+  </main>
+</div>