NGSTACK-843 page indexing implementation from fina + FieldMapper impl…

bundle/DependencyInjection/Configuration.php

@@ -4,6 +4,7 @@
 
 namespace Netgen\Bundle\IbexaSearchExtraBundle\DependencyInjection;
 
+use Ibexa\Contracts\Core\Repository\LanguageService;
 use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
 use Symfony\Component\Config\Definition\Builder\TreeBuilder;
 use Symfony\Component\Config\Definition\ConfigurationInterface;
@@ -25,6 +26,7 @@ public function getConfigTreeBuilder(): TreeBuilder
         $this->addIndexableFieldTypeSection($rootNode);
         $this->addSearchResultExtractorSection($rootNode);
         $this->addAsynchronousIndexingSection($rootNode);
+        $this->addPageIndexingSection($rootNode);
 
         return $treeBuilder;
     }
@@ -73,4 +75,94 @@ private function addAsynchronousIndexingSection(ArrayNodeDefinition $nodeDefinit
                 ->end()
             ->end();
     }
+
+    private function addPageIndexingSection(ArrayNodeDefinition $nodeDefinition): void
+    {
+        $keyValidator = static function ($v) {
+            foreach (array_keys($v) as $key) {
+                if (!is_string($key)) {
+                    return true;
+                }
+            }
+            return false;
+        };
+        $nodeDefinition
+            ->children()
+                ->arrayNode('page_indexing')
+                ->addDefaultsIfNotSet()
+                ->info('Page indexing configuration')
+                ->children()
+                    ->booleanNode('enabled')
+                        ->info('Use layouts page text indexing')
+                        ->defaultFalse()
+                    ->end()
+                    ->arrayNode('sites')
+                        ->useAttributeAsKey('name')
+                        ->normalizeKeys(false)
+                        ->validate()
+                            ->ifTrue($keyValidator)
+                            ->thenInvalid('Site root name must be of string type.')
+                        ->end()
+                        ->arrayPrototype()
+                            ->children()
+                                ->integerNode('tree_root_location_id')
+                                    ->info('Site root id')
+                                    ->beforeNormalization()->always(static fn ($v) => is_string($v) ? (int)$v : $v)->end()
+                                ->end()
+                                ->arrayNode('languages_siteaccess_map')
+                                    ->info('Language key mapped to page siteaccess')
+                                    ->useAttributeAsKey('name')
+                                    ->normalizeKeys(false)
+                                    ->validate()
+                                        ->ifTrue($keyValidator)
+                                        ->thenInvalid('Page name must be of string type.')
+                                    ->end()
+                                    ->scalarPrototype()
+                                        ->validate()
+                                            ->ifTrue(static fn ($v) => !is_string($v))
+                                            ->thenInvalid('Siteaccess name must be of string type.')
+                                        ->end()
+                                    ->end()
+                                ->end()
+                                ->arrayNode('fields')
+                                    ->info('Config for separating page text by importance of the html tags and classes, used to index content to separate solr fields')
+                                    ->validate()
+                                        ->ifTrue($keyValidator)
+                                        ->thenInvalid('Array key (level of field importance) must be of string type.')
+                                    ->end()
+                                    ->arrayPrototype()
+                                        ->useAttributeAsKey('name')
+                                        ->normalizeKeys(false)
+                                        ->scalarPrototype()
+                                            ->validate()
+                                                ->ifTrue(static fn ($v) => !is_string($v))
+                                                ->thenInvalid('HTML selector must be of string type.')
+                                            ->end()
+                                        ->end()
+                                    ->end()
+                                ->end()
+                                ->arrayNode('allowed_content_types')
+                                    ->info('Content types to index')
+                                    ->useAttributeAsKey('name')
+                                    ->normalizeKeys(false)
+                                    ->scalarPrototype()
+                                        ->validate()
+                                            ->ifTrue(static fn ($v) => !is_string($v))
+                                            ->thenInvalid('Content type identifier must be of string type.')
+                                        ->end()
+                                    ->end()
+                                ->end()
+                                ->scalarNode('host')
+                                    ->info('Host to index page from, defined in .env files')
+                                    ->validate()
+                                        ->ifTrue(static fn ($v) => !is_string($v))
+                                        ->thenInvalid('Host must be of string type.')
+                                    ->end()
+                                ->end()
+                            ->end()
+                    ->end()
+                ->end()
+            ->end();
+    }
+
 }

bundle/DependencyInjection/NetgenIbexaSearchExtraExtension.php

@@ -16,6 +16,13 @@
 
 class NetgenIbexaSearchExtraExtension extends Extension implements PrependExtensionInterface
 {
+    private static array $defaultConfiguration = [
+        'tree_root_location_id' => null,
+        'languages_siteaccess_map' => [],
+        'host' => null,
+        'fields' => [],
+        'allowed_content_types' => []
+    ];
     public function getAlias(): string
     {
         return 'netgen_ibexa_search_extra';
@@ -82,12 +89,11 @@ private function loadBundleSolrEngine(ContainerBuilder $container): void
     private function processExtensionConfiguration(array $configs, ContainerBuilder $container): void
     {
         $configuration = $this->getConfiguration($configs, $container);
-
         $configuration = $this->processConfiguration($configuration, $configs);
-
         $this->processIndexableFieldTypeConfiguration($configuration, $container);
         $this->processSearchResultExtractorConfiguration($configuration, $container);
         $this->processAsynchronousIndexingConfiguration($configuration, $container);
+        $this->processPageIndexingConfiguration($configuration, $container);
     }
 
     private function processSearchResultExtractorConfiguration(array $configuration, ContainerBuilder $container): void
@@ -117,4 +123,65 @@ private function processAsynchronousIndexingConfiguration(array $configuration,
             $configuration['use_asynchronous_indexing'],
         );
     }
+
+    private function processPageIndexingConfiguration(array $configuration, ContainerBuilder $container): void
+    {
+        $container->setParameter(
+            'netgen_ibexa_search_extra.page_indexing.sites',
+            $configuration['page_indexing']['sites'] ?? [],
+        );
+
+        $container->setParameter(
+            'netgen_ibexa_search_extra.page_indexing.enabled',
+            $configuration['page_indexing']['enabled'] ?? false,
+        );
+
+        if (!$configuration['page_indexing']['enabled']) {
+            return;
+        }
+
+        if ($configuration['page_indexing']['sites'] === []) {
+            $container->setParameter(
+                'netgen_ibexa_search_extra.page_indexing.sites',
+                [
+                    'default' => self::$defaultConfiguration
+                ]
+            );
+            return;
+        }
+        foreach ($container->getParameter('netgen_ibexa_search_extra.page_indexing.sites') as $siteName => $config) {
+            $this->setPageIndexingSitesParameters($configuration, $container, $siteName);
+        }
+    }
+
+    private function setPageIndexingSitesParameters(array $configuration, ContainerBuilder $container, string $siteName): void
+    {
+        /** @var array $pageIndexingSitesConfig */
+        $pageIndexingSitesConfig = $container->getParameter('netgen_ibexa_search_extra.page_indexing.sites');
+
+        if (!array_key_exists('tree_root_location_id', $container->getParameter('netgen_ibexa_search_extra.page_indexing.sites')[$siteName])) {
+            $pageIndexingSitesConfig[$siteName]['tree_root_location_id'] = null;
+        }
+
+        if (!array_key_exists('languages_siteaccess_map', $container->getParameter('netgen_ibexa_search_extra.page_indexing.sites')[$siteName])) {
+            $pageIndexingSitesConfig[$siteName]['languages_siteaccess_map'] = [];
+        }
+
+        if (!array_key_exists('host', $container->getParameter('netgen_ibexa_search_extra.page_indexing.sites')[$siteName])) {
+            $pageIndexingSitesConfig[$siteName]['host'] = null;
+        }
+
+        if (!array_key_exists('fields', $container->getParameter('netgen_ibexa_search_extra.page_indexing.sites')[$siteName])) {
+            $pageIndexingSitesConfig[$siteName]['fields'] = [];
+        }
+
+        if (!array_key_exists('allowed_content_types', $container->getParameter('netgen_ibexa_search_extra.page_indexing.sites')[$siteName])) {
+            $pageIndexingSitesConfig[$siteName]['allowed_content_types'] = [];
+        }
+
+        $container->setParameter(
+            'netgen_ibexa_search_extra.page_indexing.sites',
+            $pageIndexingSitesConfig,
+        );
+    }
 }

bundle/NetgenIbexaSearchExtraBundle.php

@@ -24,5 +24,7 @@ public function build(ContainerBuilder $container): void
         $container->addCompilerPass(new Compiler\FieldType\RichTextIndexablePass());
         $container->addCompilerPass(new Compiler\SearchResultExtractorPass());
         $container->addCompilerPass(new Compiler\RawFacetBuilderDomainVisitorPass());
+        $container->addCompilerPass(new Compiler\PageIndexingPass());
+        $container->addCompilerPass(new Compiler\ElasticsearchExtensibleDocumentFactoryPass());
     }
 }

composer.json

@@ -15,7 +15,9 @@
         "ext-dom": "*",
         "ibexa/core": "^4.6",
         "symfony/messenger": "^5.4",
-        "symfony/proxy-manager-bridge": "^5.4"
+        "symfony/proxy-manager-bridge": "^5.4",
+        "ext-libxml": "*",
+        "ext-curl": "*"
     },
     "require-dev": {
         "ibexa/fieldtype-richtext": "^4.5",
@@ -30,7 +32,8 @@
     },
     "suggest": {
         "netgen/ibexa-site-api": "Boost your site-building productivity with Ibexa CMS",
-        "ibexa/solr": "Supports advanced capabilities with Ibexa search API"
+        "ibexa/solr": "Supports advanced capabilities with Ibexa search API",
+        "ibexa/elasticsearch": "Needed for layouts indexer"
     },
     "autoload": {
         "psr-4": {
@@ -51,5 +54,10 @@
     },
     "scripts": {
         "test": "@php vendor/bin/phpunit --colors=always"
+    },
+    "config": {
+        "allow-plugins": {
+            "php-http/discovery": false
+        }
     }
 }