Merge branch 'master' into config-labels

kitodo · May 23, 2024 · 9ab293b · 9ab293b
2 parents a2c3cd1 + 97fcb4f
commit 9ab293b
Show file tree

Hide file tree

Showing 67 changed files with 9,681 additions and 969 deletions.
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -22,18 +22,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           config-file: ./.github/codeql.yml
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
         with:
           category: "/language:${{ matrix.language }}"
diff --git a/.github/workflows/phpstan.yml b/.github/workflows/phpstan.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         uses: php-actions/composer@v6

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -14,7 +14,7 @@ jobs:
         typo3: [ 10.4, 11.5 ]
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: Build/Test/runTests.sh -s composerInstall -t ${{ matrix.typo3 }}
@@ -24,3 +24,8 @@ jobs:
 
       - name: Run functional tests
         run: Build/Test/runTests.sh -s functional
+
+      - name: Upload coverage reports
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/Build/Documentation/docker-compose.t3docs.yml b/Build/Documentation/docker-compose.t3docs.yml
@@ -1,7 +1,7 @@
 version: '3'
 services:
   t3docs:
-    image: t3docs/render-documentation:latest
+    image: ghcr.io/t3docs/render-documentation:latest
     volumes:
       - ../../:/PROJECT:ro
       - ../../Documentation-GENERATED-temp:/RESULT
diff --git a/Build/Test/docker-compose.yml b/Build/Test/docker-compose.yml
@@ -3,21 +3,21 @@
 version: "2.3"
 services:
   mysql:
-    image: mysql:${MYSQL_VERSION}
+    image: docker.io/mysql:${MYSQL_VERSION}
     environment:
       MYSQL_ROOT_PASSWORD: funcp
     tmpfs:
       - /var/lib/mysql/:rw,noexec,nosuid
 
   mariadb:
-    image: mariadb:${MARIADB_VERSION}
+    image: docker.io/mariadb:${MARIADB_VERSION}
     environment:
       MYSQL_ROOT_PASSWORD: funcp
     tmpfs:
       - /var/lib/mysql/:rw,noexec,nosuid
 
   web:
-    image: typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
+    image: docker.io/typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
     user: "${HOST_UID}:${HOST_GID}"
     stop_grace_period: 1s
     volumes:
@@ -45,13 +45,13 @@ services:
       "
 
   solr:
-    image: solr:8
+    image: docker.io/solr:8
     volumes:
       - ${DLF_ROOT}/Configuration/ApacheSolr/configsets:/var/solr/data/configsets
       - ./solr/contrib/ocrsearch:/opt/solr/contrib/ocrsearch
 
   composer_install:
-    image: typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
+    image: docker.io/typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
     user: "${HOST_UID}:${HOST_GID}"
     volumes:
       - ${DLF_ROOT}:${DLF_ROOT}
@@ -71,7 +71,7 @@ services:
       "
 
   functional:
-    image: typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
+    image: docker.io/typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
     links:
       - ${DBMS}
       - web
@@ -85,7 +85,8 @@ services:
       typo3DatabaseUsername: root
       typo3DatabasePassword: funcp
       typo3DatabaseHost: ${DBMS}
-      dlfTestingSolrHost: solr
+    env_file:
+      - ./test.env
     working_dir: ${DLF_ROOT}
     extra_hosts:
       - "host.docker.internal:host-gateway"
@@ -126,7 +127,7 @@ services:
       "
 
   unit:
-    image: typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
+    image: docker.io/typo3/core-testing-${DOCKER_PHP_IMAGE}:latest
     user: "${HOST_UID}:${HOST_GID}"
     volumes:
       - ${DLF_ROOT}:${DLF_ROOT}

diff --git a/Build/Test/test.env b/Build/Test/test.env
@@ -0,0 +1 @@
+dlfTestingSolrHost=solr
diff --git a/Classes/Command/BaseCommand.php b/Classes/Command/BaseCommand.php
@@ -89,12 +89,18 @@ class BaseCommand extends Command
      */
     protected ConfigurationManager $configurationManager;
 
+    /**
+     * @var PersistenceManager
+     */
+    protected PersistenceManager $persistenceManager;
+
     public function __construct(
         CollectionRepository $collectionRepository,
         DocumentRepository $documentRepository,
         LibraryRepository $libraryRepository,
         StructureRepository $structureRepository,
-        ConfigurationManager $configurationManager
+        ConfigurationManager $configurationManager,
+        PersistenceManager $persistenceManager
     ) {
         parent::__construct();
 
@@ -103,6 +109,7 @@ public function __construct(
         $this->libraryRepository = $libraryRepository;
         $this->structureRepository = $structureRepository;
         $this->configurationManager = $configurationManager;
+        $this->persistenceManager = $persistenceManager;
     }
 
     /**
@@ -202,23 +209,23 @@ protected function saveToDatabase(Document $document): bool
         if ($doc === null) {
             return false;
         }
-        $persistenceManager = GeneralUtility::makeInstance(PersistenceManager::class);
+
         $doc->cPid = $this->storagePid;
 
         $metadata = $doc->getToplevelMetadata($this->storagePid);
 
         // set title data
         $document->setTitle($metadata['title'][0] ? : '');
-        $document->setTitleSorting($metadata['title_sorting'][0]);
+        $document->setTitleSorting($metadata['title_sorting'][0] ? : '');
         $document->setPlace(implode('; ', $metadata['place']));
         $document->setYear(implode('; ', $metadata['year']));
 
         // Remove appended "valueURI" from authors' names for storing in database.
         foreach ($metadata['author'] as $i => $author) {
-            $splitName = explode(chr(31), $author);
+            $splitName = explode(pack('C', 31), $author);
             $metadata['author'][$i] = $splitName[0];
         }
-        $document->setAuthor(implode('; ', $metadata['author']));
+        $document->setAuthor($this->getAuthors($metadata['author']));
         $document->setThumbnail($doc->thumbnail ? : '');
         $document->setMetsLabel($metadata['mets_label'][0] ? : '');
         $document->setMetsOrderlabel($metadata['mets_orderlabel'][0] ? : '');
@@ -227,7 +234,7 @@ protected function saveToDatabase(Document $document): bool
         $document->setStructure($structure);
 
         if (is_array($metadata['collection'])) {
-            $this->addCollections($document, $metadata['collection'], $persistenceManager);
+            $this->addCollections($document, $metadata['collection']);
         }
 
         // set identifiers
@@ -268,7 +275,7 @@ protected function saveToDatabase(Document $document): bool
             $this->documentRepository->update($document);
         }
 
-        $persistenceManager->persistAll();
+        $this->persistenceManager->persistAll();
 
         return true;
     }
@@ -324,11 +331,10 @@ protected function getParentDocumentUidForSaving(Document $document): int
      * 
      * @param Document &$document
      * @param array $collections
-     * @param PersistenceManager $persistenceManager
      *
      * @return void
      */
-    private function addCollections(Document &$document, array $collections, PersistenceManager $persistenceManager): void
+    private function addCollections(Document &$document, array $collections): void
     {
         foreach ($collections as $collection) {
             $documentCollection = $this->collectionRepository->findOneByIndexName($collection);
@@ -343,13 +349,47 @@ private function addCollections(Document &$document, array $collections, Persist
                 // add to CollectionRepository
                 $this->collectionRepository->add($documentCollection);
                 // persist collection to prevent duplicates
-                $persistenceManager->persistAll();
+                $this->persistenceManager->persistAll();
             }
             // add to document
             $document->addCollection($documentCollection);
         }
     }
 
+    /**
+     * Get authors considering that database field can't accept
+     * more than 255 characters.
+     *
+     * @access private
+     * 
+     * @param array $metadataAuthor
+     *
+     * @return string
+     */
+    private function getAuthors(array $metadataAuthor): string
+    {
+        $authors = '';
+        $delimiter = '; ';
+        $ellipsis = 'et al.';
+
+        $count = count($metadataAuthor);
+
+        for ($i = 0; $i < $count; $i++) {
+            // Build the next part to add
+            $nextPart = ($i === 0 ? '' : $delimiter) . $metadataAuthor[$i];
+            // Check if adding this part and ellipsis in future would exceed the character limit
+            if (strlen($authors . $nextPart . $delimiter . $ellipsis) > 255) {
+                // Add ellipsis and stop adding more authors
+                $authors .= $delimiter . $ellipsis;
+                break;
+            }
+            // Add the part to the main string
+            $authors .= $nextPart;
+        }
+
+        return $authors;
+    }
+
     /**
      * If owner is not set set but found by metadata, take it or take default library, if nothing found in database then create new owner.
      *

diff --git a/Classes/Command/ReindexCommand.php b/Classes/Command/ReindexCommand.php
@@ -79,6 +79,18 @@ public function configure(): void
                 'a',
                 InputOption::VALUE_NONE,
                 'Reindex all documents on the given page.'
+            )
+            ->addOption(
+                'index-limit',
+                'l',
+                InputOption::VALUE_OPTIONAL,
+                'Reindex the given amount of documents on the given page.'
+            )
+            ->addOption(
+                'index-begin',
+                'b',
+                InputOption::VALUE_OPTIONAL,
+                'Reindex documents on the given page starting from the given value.'
             );
     }
 
@@ -145,6 +157,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         if (!empty($input->getOption('all'))) {
             // Get all documents.
             $documents = $this->documentRepository->findAll();
+        } elseif (
+            !empty($input->getOption('index-limit'))
+            && $input->getOption('index-begin') >= 0
+        ) {
+            // Get all documents for given limit and start.
+            $documents = $this->documentRepository->findAll()
+                ->getQuery()
+                ->setLimit((int) $input->getOption('index-limit'))
+                ->setOffset((int) $input->getOption('index-begin'))
+                ->execute();
+
+            $io->writeln($input->getOption('index-limit') . ' documents starting from ' . $input->getOption('index-begin') . ' will be indexed.');
         } elseif (
             !empty($input->getOption('coll'))
             && !is_array($input->getOption('coll'))
@@ -181,10 +205,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 // add to index
                 Indexer::add($document, $this->documentRepository);
             }
-            // Clear document registry to prevent memory exhaustion.
-            AbstractDocument::clearRegistry();
+            // Clear document cache to prevent memory exhaustion.
+            AbstractDocument::clearDocumentCache();
         }
 
+        // Clear state of persistence manager to prevent memory exhaustion.
+        $this->persistenceManager->clearState();
+
         $io->success('All done!');
 
         return BaseCommand::SUCCESS;

diff --git a/Classes/Common/AbstractDocument.php b/Classes/Common/AbstractDocument.php
@@ -211,13 +211,6 @@ abstract class AbstractDocument
      */
     protected string $recordId = '';
 
-    /**
-     * @access protected
-     * @static
-     * @var array (AbstractDocument) This holds the singleton object of the document
-     */
-    protected static array $registry = [];
-
     /**
      * @access protected
      * @var int This holds the UID of the root document or zero if not multi-volumed
@@ -598,18 +591,18 @@ public static function &getInstance(string $location, array $settings = [], bool
     }
 
     /**
-     * This clears the static registry to prevent memory exhaustion
+     * Clear document cache.
      *
      * @access public
      *
      * @static
      *
      * @return void
      */
-    public static function clearRegistry(): void
+    public static function clearDocumentCache(): void
     {
-        // Reset registry array.
-        self::$registry = [];
+        $cache = GeneralUtility::makeInstance(CacheManager::class)->getCache('tx_dlf_doc');
+        $cache->flush();
     }
 
     /**
@@ -1030,7 +1023,8 @@ protected function magicGetHasFulltext(): bool
     }
 
     /**
-     * This builds an array of the document's metadata
+     * This magic method is called each time an invisible property is referenced from the object
+     * It builds an array of the document's metadata
      *
      * @access protected
      *