Merge branch 'master' into ext-tables

kitodo · May 29, 2024 · 1374b17 · 1374b17
2 parents 88e7024 + 8d37dc1
commit 1374b17
Show file tree

Hide file tree

Showing 37 changed files with 900 additions and 576 deletions.
diff --git a/.github/ISSUE_TEMPLATE/task-for-the-development-fund.md b/.github/ISSUE_TEMPLATE/task-for-the-development-fund.md
@@ -2,7 +2,7 @@
 name: Task for the development fund
 about: A working package which may be sponsored by the Kitodo e.V. development fund.
 title: "[FUND] "
-labels: ⭐ development fund 2024
+labels: ⭐ development fund 2025
 assignees: ''
 
 ---

diff --git a/.github/codeql.yml b/.github/codeql.yml
@@ -1,16 +1,16 @@
-   name: "CodeQL Configuration"
+name: CodeQL Configuration
 
-   queries:
-     - uses: security-and-quality
+queries:
+  - uses: security-and-quality
 
-   query-filters:
-     - exclude:
-         problem.severity:
-           - note
+query-filters:
+  - exclude:
+      problem.severity:
+        - note
 
-   paths-ignore:
-     - Resources/Public/JavaScript/jPlayer
-     - Resources/Public/JavaScript/jQuery
-     - Resources/Public/JavaScript/jQueryUI
-     - Resources/Public/JavaScript/OpenLayers
-     - Resources/Public/JavaScript/Toastify
+paths-ignore:
+  - Resources/Public/JavaScript/jPlayer
+  - Resources/Public/JavaScript/jQuery
+  - Resources/Public/JavaScript/jQueryUI
+  - Resources/Public/JavaScript/OpenLayers
+  - Resources/Public/JavaScript/Toastify
diff --git a/.github/phpstan.neon b/.github/phpstan.neon
@@ -1,31 +1,31 @@
 parameters:
-    ignoreErrors:
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::countByPid\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsListed\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsSortable\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByFeUserId\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByIndexName\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByLocation\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByPid\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByRecordId\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneBySessionId\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByType\(\)\.#'
-            - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByUid\(\)\.#'
-            - '#Call to an undefined method Psr\\Http\\Message\\RequestFactoryInterface::request\(\)\.#'
-            - '#Call to an undefined method Solarium\\Core\\Query\\DocumentInterface::setField\(\)\.#'
-            - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\IiifResourceInterface::getHeight\(\)\.#'
-            - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\IiifResourceInterface::getWidth\(\)\.#'
-            - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\IiifResourceInterface::getPossibleTextAnnotationContainers\(\)\.#'
-            - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\IiifResourceInterface::getTextAnnotations\(\)\.#'
-            - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\ManifestInterface::getOriginalJsonArray\(\)\.#'
-            - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\RangeInterface::getMemberRangesAndRanges\(\)\.#'
-            - '#Constant LOG_SEVERITY_ERROR not found\.#'
-            - '#Constant LOG_SEVERITY_NOTICE not found\.#'
-            - '#Constant LOG_SEVERITY_WARNING not found\.#'
-            - '#Constant TYPO3_MODE not found\.#'
-    level: 5
-    paths:
-        - ../Classes/
-    excludePaths:
-        - ../Classes/Controller/OaiPmhController.php
-    treatPhpDocTypesAsCertain: false
+  ignoreErrors:
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::countByPid\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsListed\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findByIsSortable\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByFeUserId\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByIndexName\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByLocation\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByPid\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByRecordId\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneBySessionId\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByType\(\)\.#'
+    - '#Call to an undefined method Kitodo\\Dlf\\Domain\\Repository\\[a-zA-Z]+Repository::findOneByUid\(\)\.#'
+    - '#Call to an undefined method Psr\\Http\\Message\\RequestFactoryInterface::request\(\)\.#'
+    - '#Call to an undefined method Solarium\\Core\\Query\\DocumentInterface::setField\(\)\.#'
+    - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\IiifResourceInterface::getHeight\(\)\.#'
+    - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\IiifResourceInterface::getWidth\(\)\.#'
+    - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\IiifResourceInterface::getPossibleTextAnnotationContainers\(\)\.#'
+    - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\IiifResourceInterface::getTextAnnotations\(\)\.#'
+    - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\ManifestInterface::getOriginalJsonArray\(\)\.#'
+    - '#Call to an undefined method Ubl\\Iiif\\Presentation\\Common\\Model\\Resources\\RangeInterface::getMemberRangesAndRanges\(\)\.#'
+    - '#Constant LOG_SEVERITY_ERROR not found\.#'
+    - '#Constant LOG_SEVERITY_NOTICE not found\.#'
+    - '#Constant LOG_SEVERITY_WARNING not found\.#'
+    - '#Constant TYPO3_MODE not found\.#'
+  level: 5
+  paths:
+    - ../Classes/
+  excludePaths:
+    - ../Classes/Controller/OaiPmhController.php
+  treatPhpDocTypesAsCertain: false
diff --git a/.github/pull.yml b/.github/pull.yml
@@ -18,3 +18,6 @@ rules:
   - base: 4.x
     upstream: kitodo:4.x
     mergeMethod: hardreset
+  - base: 5.x
+    upstream: kitodo:5.x
+    mergeMethod: hardreset
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -1,14 +1,14 @@
-name: "CodeQL"
+name: CodeQL
 
 on:
   push:
-    branches: [ "master", "1.x", "2.x", "3.2.x", "3.3.x", "4.x" ]
+    branches: [ "master", "1.x", "2.x", "3.2.x", "3.3.x", "4.x", "5.x" ]
   pull_request:
     branches: [ "master" ]
 
 jobs:
   analyze:
-    name: Analyze
+    name: Static Code Analysis
     runs-on: ubuntu-latest
     permissions:
       actions: read

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -1,10 +1,10 @@
-name: Build Documentation
+name: Documentation
 
 on: [ push, pull_request ]
 
 jobs:
     tests:
-        name: Documentation
+        name: Build Test
         runs-on: ubuntu-latest
         steps:
             - name: Checkout

diff --git a/.github/workflows/phpstan.yml b/.github/workflows/phpstan.yml
@@ -8,6 +8,7 @@ on:
 
 jobs:
   phpstan:
+    name: Static Code Analysis
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -8,6 +8,7 @@ on:
 
 jobs:
   test:
+    name: TYPO3
     runs-on: ubuntu-latest
     strategy:
       matrix:

diff --git a/Classes/Command/BaseCommand.php b/Classes/Command/BaseCommand.php
@@ -22,6 +22,7 @@
 use Kitodo\Dlf\Domain\Model\Collection;
 use Kitodo\Dlf\Domain\Model\Document;
 use Kitodo\Dlf\Domain\Model\Library;
+use Kitodo\Dlf\Validation\DocumentValidator;
 use Symfony\Component\Console\Command\Command;
 use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -123,25 +124,16 @@ public function __construct(
      *
      * @param int $storagePid The storage pid
      *
-     * @return bool
+     * @return void
      */
-    protected function initializeRepositories(int $storagePid): bool
+    protected function initializeRepositories(int $storagePid): void
     {
-        if (MathUtility::canBeInterpretedAsInteger($storagePid)) {
-            $frameworkConfiguration = $this->configurationManager->getConfiguration(ConfigurationManagerInterface::CONFIGURATION_TYPE_FRAMEWORK);
-
-            $frameworkConfiguration['persistence']['storagePid'] = MathUtility::forceIntegerInRange((int) $storagePid, 0);
-            $this->configurationManager->setConfiguration($frameworkConfiguration);
-
-            // Get extension configuration.
-            $this->extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('dlf');
-        } else {
-            return false;
-        }
-        $this->storagePid = MathUtility::forceIntegerInRange((int) $storagePid, 0);
+        $frameworkConfiguration = $this->configurationManager->getConfiguration(ConfigurationManagerInterface::CONFIGURATION_TYPE_FRAMEWORK);
+        $frameworkConfiguration['persistence']['storagePid'] = MathUtility::forceIntegerInRange($storagePid, 0);
+        $this->configurationManager->setConfiguration($frameworkConfiguration);
+        $this->extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('dlf');
+        $this->storagePid = MathUtility::forceIntegerInRange($storagePid, 0);
         $this->persistenceManager = GeneralUtility::makeInstance(PersistenceManager::class);
-
-        return true;
     }
 
     /**
@@ -216,71 +208,70 @@ protected function saveToDatabase(Document $document): bool
         $doc->cPid = $this->storagePid;
 
         $metadata = $doc->getToplevelMetadata($this->storagePid);
+        $validator = new DocumentValidator($metadata, explode(',', $this->extConf['general']['requiredMetadataFields']));
+
+        if ($validator->hasAllMandatoryMetadataFields()) {
+            // set title data
+            $document->setTitle($metadata['title'][0] ? : '');
+            $document->setTitleSorting($metadata['title_sorting'][0] ? : '');
+            $document->setPlace(implode('; ', $metadata['place']));
+            $document->setYear(implode('; ', $metadata['year']));
+            $document->setAuthor($this->getAuthors($metadata['author']));
+            $document->setThumbnail($doc->thumbnail ? : '');
+            $document->setMetsLabel($metadata['mets_label'][0] ? : '');
+            $document->setMetsOrderlabel($metadata['mets_orderlabel'][0] ? : '');
+
+            $structure = $this->structureRepository->findOneByIndexName($metadata['type'][0]);
+            $document->setStructure($structure);
+
+            if (is_array($metadata['collection'])) {
+                $this->addCollections($document, $metadata['collection']);
+            }
 
-        // set title data
-        $document->setTitle($metadata['title'][0] ? : '');
-        $document->setTitleSorting($metadata['title_sorting'][0] ? : '');
-        $document->setPlace(implode('; ', $metadata['place']));
-        $document->setYear(implode('; ', $metadata['year']));
-
-        // Remove appended "valueURI" from authors' names for storing in database.
-        foreach ($metadata['author'] as $i => $author) {
-            $splitName = explode(pack('C', 31), $author);
-            $metadata['author'][$i] = $splitName[0];
-        }
-        $document->setAuthor($this->getAuthors($metadata['author']));
-        $document->setThumbnail($doc->thumbnail ? : '');
-        $document->setMetsLabel($metadata['mets_label'][0] ? : '');
-        $document->setMetsOrderlabel($metadata['mets_orderlabel'][0] ? : '');
-
-        $structure = $this->structureRepository->findOneByIndexName($metadata['type'][0]);
-        $document->setStructure($structure);
+            // set identifiers
+            $document->setProdId($metadata['prod_id'][0] ? : '');
+            $document->setOpacId($metadata['opac_id'][0] ? : '');
+            $document->setUnionId($metadata['union_id'][0] ? : '');
+
+            $document->setRecordId($metadata['record_id'][0]);
+            $document->setUrn($metadata['urn'][0] ? : '');
+            $document->setPurl($metadata['purl'][0] ? : '');
+            $document->setDocumentFormat($metadata['document_format'][0] ? : '');
+
+            // set access
+            $document->setLicense($metadata['license'][0] ? : '');
+            $document->setTerms($metadata['terms'][0] ? : '');
+            $document->setRestrictions($metadata['restrictions'][0] ? : '');
+            $document->setOutOfPrint($metadata['out_of_print'][0] ? : '');
+            $document->setRightsInfo($metadata['rights_info'][0] ? : '');
+            $document->setStatus(0);
+
+            $this->setOwner($metadata['owner'][0]);
+            $document->setOwner($this->owner);
+
+            // set volume data
+            $document->setVolume($metadata['volume'][0] ? : '');
+            $document->setVolumeSorting($metadata['volume_sorting'][0] ? : $metadata['mets_order'][0] ? : '');
+
+            // Get UID of parent document.
+            if ($document->getDocumentFormat() === 'METS') {
+                $document->setPartof($this->getParentDocumentUidForSaving($document));
+            }
 
-        if (is_array($metadata['collection'])) {
-            $this->addCollections($document, $metadata['collection']);
-        }
+            if ($document->getUid() === null) {
+                // new document
+                $this->documentRepository->add($document);
+            } else {
+                // update of existing document
+                $this->documentRepository->update($document);
+            }
 
-        // set identifiers
-        $document->setProdId($metadata['prod_id'][0] ? : '');
-        $document->setOpacId($metadata['opac_id'][0] ? : '');
-        $document->setUnionId($metadata['union_id'][0] ? : '');
-
-        $document->setRecordId($metadata['record_id'][0] ? : ''); // (?) $doc->recordId
-        $document->setUrn($metadata['urn'][0] ? : '');
-        $document->setPurl($metadata['purl'][0] ? : '');
-        $document->setDocumentFormat($metadata['document_format'][0] ? : '');
-
-        // set access
-        $document->setLicense($metadata['license'][0] ? : '');
-        $document->setTerms($metadata['terms'][0] ? : '');
-        $document->setRestrictions($metadata['restrictions'][0] ? : '');
-        $document->setOutOfPrint($metadata['out_of_print'][0] ? : '');
-        $document->setRightsInfo($metadata['rights_info'][0] ? : '');
-        $document->setStatus(0);
-
-        $this->setOwner($metadata['owner'][0]);
-        $document->setOwner($this->owner);
-
-        // set volume data
-        $document->setVolume($metadata['volume'][0] ? : '');
-        $document->setVolumeSorting($metadata['volume_sorting'][0] ? : $metadata['mets_order'][0] ? : '');
-
-        // Get UID of parent document.
-        if ($document->getDocumentFormat() === 'METS') {
-            $document->setPartof($this->getParentDocumentUidForSaving($document));
-        }
+            $this->persistenceManager->persistAll();
 
-        if ($document->getUid() === null) {
-            // new document
-            $this->documentRepository->add($document);
-        } else {
-            // update of existing document
-            $this->documentRepository->update($document);
+            return true;
         }
 
-        $this->persistenceManager->persistAll();
-
-        return true;
+        return false;
     }
 
     /**
@@ -346,7 +337,7 @@ private function addCollections(Document &$document, array $collections): void
                 $documentCollection = GeneralUtility::makeInstance(Collection::class);
                 $documentCollection->setIndexName($collection);
                 $documentCollection->setLabel($collection);
-                $documentCollection->setOaiName((!empty($this->extConf['publishNewCollections']) ? Helper::getCleanString($collection) : ''));
+                $documentCollection->setOaiName((!empty($this->extConf['general']['publishNewCollections']) ? Helper::getCleanString($collection) : ''));
                 $documentCollection->setIndexSearch('');
                 $documentCollection->setDescription('');
                 // add to CollectionRepository
@@ -371,6 +362,12 @@ private function addCollections(Document &$document, array $collections): void
      */
     private function getAuthors(array $metadataAuthor): string
     {
+        // Remove appended "valueURI" from authors' names for storing in database.
+        foreach ($metadataAuthor as $i => $author) {
+            $splitName = explode(pack('C', 31), $author);
+            $metadataAuthor[$i] = $splitName[0];
+        }
+
         $authors = '';
         $delimiter = '; ';
         $ellipsis = 'et al.';

diff --git a/Classes/Command/DeleteCommand.php b/Classes/Command/DeleteCommand.php
@@ -81,7 +81,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $io = new SymfonyStyle($input, $output);
         $io->title($this->getDescription());
 
-        $this->initializeRepositories($input->getOption('pid'));
+        $this->initializeRepositories((int) $input->getOption('pid'));
 
         if ($this->storagePid == 0) {
             $io->error('ERROR: No valid PID (' . $this->storagePid . ') given.');

diff --git a/Classes/Command/HarvestCommand.php b/Classes/Command/HarvestCommand.php
@@ -108,7 +108,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $io = new SymfonyStyle($input, $output);
         $io->title($this->getDescription());
 
-        $this->initializeRepositories($input->getOption('pid'));
+        $this->initializeRepositories((int) $input->getOption('pid'));
 
         if ($this->storagePid == 0) {
             $io->error('ERROR: No valid PID (' . $this->storagePid . ') given.');