Add command to prune deleted records

bin/cli

@@ -29,6 +29,7 @@ use Exception;
 use OCC\OaiPmh2\Console\AddRecordCommand;
 use OCC\OaiPmh2\Console\BulkUpdateCommand;
 use OCC\OaiPmh2\Console\DeleteRecordCommand;
+use OCC\OaiPmh2\Console\PruneRecordsCommand;
 use OCC\OaiPmh2\Console\PruneResumptionTokensCommand;
 use OCC\OaiPmh2\Console\UpdateFormatsCommand;
 
@@ -38,6 +39,7 @@ $commands = [
     new AddRecordCommand(),
     new BulkUpdateCommand(),
     new DeleteRecordCommand(),
+    new PruneRecordsCommand(),
     new PruneResumptionTokensCommand(),
     new UpdateFormatsCommand()
 ];

config/config.dist.yml

@@ -72,6 +72,29 @@ metadataPrefix: {
     }
 }
 
+#
+# Deleted records policy
+#
+# This states if and how the repository keeps track of deleted records. You can
+# delete records by importing empty records with the same identifiers and
+# metadata prefixes. Depending on the deleted records policy those records will
+# be either marked as deleted or completely removed from the database.
+# "no" means the repository does not provide any information about deletions.
+# "persistent" means the repository consistently provides information about
+# deletions.
+# "transient" - The repository may provide information about deletions. This is
+# handled exactly the same as "persistent", but you are allowed to manually
+# prune deleted records from the database (see below).
+#
+# ["no"|"persistent"|"transient"]
+#
+# Run "bin/cli oai:records:prune" after changing the deleted records policy to
+# "no" to remove all deleted records from the database.
+# If your policy is "transient" and you want to clean up deleted records from
+# the database anyway, run the command with the "--force" flag.
+#
+deletedRecords: 'transient'
+
 #
 # Maximum number of records to return per request
 #

src/Configuration.php

@@ -40,11 +40,12 @@
  * @property-read string $adminEmail
  * @property-read string $database
  * @property-read array $metadataPrefix
+ * @property-read string $deletedRecords
  * @property-read int $maxRecords
  * @property-read int $tokenValid
  *
  * @template TKey of string
- * @template TValue
+ * @template TValue of array|int|string
  */
 class Configuration
 {
@@ -102,6 +103,11 @@ protected function getValidationConstraints(): Assert\Collection
                     ])
                 ])
             ],
+            'deletedRecords' => [
+                new Assert\Type('string'),
+                new Assert\Choice(['no', 'persistent', 'transient']),
+                new Assert\NotBlank()
+            ],
             'maxRecords' => [
                 new Assert\Type('int'),
                 new Assert\Range([

src/Console/PruneRecordsCommand.php

@@ -0,0 +1,92 @@
+<?php
+
+/**
+ * OAI-PMH 2.0 Data Provider
+ * Copyright (C) 2023 Sebastian Meyer <[email protected]>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+namespace OCC\OaiPmh2\Console;
+
+use OCC\OaiPmh2\Configuration;
+use OCC\OaiPmh2\Database;
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Prune deleted records from database.
+ *
+ * @author Sebastian Meyer <[email protected]>
+ * @package opencultureconsulting/oai-pmh2
+ */
+#[AsCommand(
+    name: 'oai:records:prune',
+    description: 'Prune deleted records from database'
+)]
+class PruneRecordsCommand extends Command
+{
+    protected function configure(): void
+    {
+        $this->addOption(
+            'force',
+            null,
+            InputOption::VALUE_NONE,
+            'Deletes records even under "transient" policy.'
+        );
+        parent::configure();
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $policy = Configuration::getInstance()->deletedRecords;
+        $forced = (bool) $input->getOption('force');
+        if (
+            $policy === 'no'
+            or ($policy === 'transient' && $forced)
+        ) {
+            $deleted = Database::getInstance()->pruneDeletedRecords();
+            $output->writeln([
+                '',
+                sprintf(
+                    ' [OK] %d records are deleted and were successfully removed! ',
+                    $deleted
+                ),
+                ''
+            ]);
+            return Command::SUCCESS;
+        } else {
+            if ($policy === 'persistent') {
+                $output->writeln([
+                    '',
+                    ' [ERROR] Under "persistent" policy removal of deleted records is not allowed. ',
+                    ''
+                ]);
+                return Command::FAILURE;
+            } else {
+                $output->writeln([
+                    '',
+                    ' [INFO] Use the "--force" option to remove deleted records under "transient" policy. ',
+                    ''
+                ]);
+                return Command::INVALID;
+            }
+        }
+    }
+}

src/Console/PruneResumptionTokensCommand.php

@@ -46,7 +46,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $output->writeln([
             '',
             sprintf(
-                ' [OK] %d resumption tokens are expired and were successfully deleted. ',
+                ' [OK] %d resumption tokens are expired and were successfully deleted! ',
                 $expired
             ),
             ''

src/Console/UpdateFormatsCommand.php

@@ -66,7 +66,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 ++$added;
                 $output->writeln([
                     sprintf(
-                        ' [OK] Metadata format "%s" added or updated successfully. ',
+                        ' [OK] Metadata format "%s" added or updated successfully! ',
                         $prefix
                     )
                 ]);
@@ -87,7 +87,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                     ++$deleted;
                     $output->writeln([
                         sprintf(
-                            ' [OK] Metadata format "%s" and all associated records deleted successfully. ',
+                            ' [OK] Metadata format "%s" and all associated records deleted successfully! ',
                             $prefix
                         )
                     ]);

src/Database.php

@@ -23,6 +23,7 @@
 namespace OCC\OaiPmh2;
 
 use DateTime;
+use Doctrine\Common\Collections\Criteria;
 use Doctrine\DBAL\DriverManager;
 use Doctrine\DBAL\Schema\AbstractAsset;
 use Doctrine\DBAL\Tools\DsnParser;
@@ -149,11 +150,11 @@ public function getMetadataFormats(?string $identifier = null): Result
         if (isset($identifier)) {
             $dql->innerJoin(
                     'format.records',
-                    'records',
+                    'record',
                     'WITH',
                     $dql->expr()->andX(
-                        $dql->expr()->eq('records.identifier', ':identifier'),
-                        $dql->expr()->neq('records.data', '')
+                        $dql->expr()->eq('record.identifier', ':identifier'),
+                        $dql->expr()->isNotNull('record.content')
                     )
                 )
                 ->setParameter('identifier', $identifier);
@@ -328,20 +329,56 @@ public function idDoesExist(string $identifier): bool
         return (bool) $query->getOneOrNullResult(AbstractQuery::HYDRATE_SINGLE_SCALAR);
     }
 
+    /**
+     * Prune deleted records.
+     *
+     * @return int The number of removed records
+     */
+    public function pruneDeletedRecords(): int
+    {
+        $repository = $this->entityManager->getRepository(Record::class);
+        $criteria = Criteria::create()->where(Criteria::expr()->isNull('content'));
+        $records = $repository->matching($criteria);
+        foreach ($records as $record) {
+            $this->entityManager->remove($record);
+        }
+        $this->entityManager->flush();
+        $this->pruneOrphanSets();
+        return count($records);
+    }
+
+    /**
+     * Prune orphan sets.
+     *
+     * @return void
+     */
+    public function pruneOrphanSets(): void
+    {
+        $repository = $this->entityManager->getRepository(Set::class);
+        $sets = $repository->findAll();
+        foreach ($sets as $set) {
+            if ($set->isEmpty()) {
+                $this->entityManager->remove($set);
+            }
+        }
+        $this->entityManager->flush();
+    }
+
     /**
      * Prune expired resumption tokens.
      *
      * @return int The number of deleted tokens
      */
     public function pruneResumptionTokens(): int
     {
-        $dql = $this->entityManager->createQueryBuilder();
-        $dql->delete(Token::class, 'token')
-            ->where($dql->expr()->lt('token.validUntil', ':now'))
-            ->setParameter('now', new DateTime());
-        $query = $dql->getQuery();
-        /** @var int */
-        return $query->execute();
+        $repository = $this->entityManager->getRepository(Token::class);
+        $criteria = Criteria::create()->where(Criteria::expr()->lt('validUntil', new DateTime()));
+        $tokens = $repository->matching($criteria);
+        foreach ($tokens as $token) {
+            $this->entityManager->remove($token);
+        }
+        $this->entityManager->flush();
+        return count($tokens);
     }
 
     /**
@@ -357,6 +394,7 @@ public function removeMetadataFormat(string $prefix): bool
         if (isset($format)) {
             $this->entityManager->remove($format);
             $this->entityManager->flush();
+            $this->pruneOrphanSets();
             return true;
         } else {
             return false;

src/Database/Record.php

@@ -64,8 +64,8 @@ class Record
     /**
      * The record's content.
      */
-    #[ORM\Column(type: 'text')]
-    private string $content = '';
+    #[ORM\Column(type: 'text', nullable: true)]
+    private ?string $content = null;
 
     /**
      * Collection of associated sets.
@@ -97,9 +97,9 @@ public function addSet(Set $set): void
     /**
      * Get the record's content.
      *
-     * @return string The record's content
+     * @return ?string The record's content or NULL if deleted
      */
-    public function getContent(): string
+    public function getContent(): ?string
     {
         return $this->content;
     }
@@ -174,21 +174,23 @@ public function removeSet(Set $set): void
     /**
      * Set record's content.
      *
-     * @param string $data The record's content
+     * @param ?string $data The record's content or NULL to mark as deleted
      * @param bool $validate Should the input be validated?
      *
      * @return void
      *
      * @throws ValidationFailedException
      */
-    public function setContent(string $data, bool $validate = true): void
+    public function setContent(?string $data = null, bool $validate = true): void
     {
-        $data = trim($data);
-        if ($validate && $data !== '') {
-            try {
-                $data = $this->validate($data);
-            } catch (ValidationFailedException $exception) {
-                throw $exception;
+        if (isset($data)) {
+            $data = trim($data);
+            if ($validate && $data !== '') {
+                try {
+                    $data = $this->validate($data);
+                } catch (ValidationFailedException $exception) {
+                    throw $exception;
+                }
             }
         }
         $this->content = $data;
@@ -246,16 +248,18 @@ protected function validate(string $xml): string
      *
      * @param string $identifier The record identifier
      * @param Format $format The format
-     * @param string $data The record's content
+     * @param ?string $data The record's content
      *
      * @throws ValidationFailedException
      */
-    public function __construct(string $identifier, Format $format, string $data = '')
+    public function __construct(string $identifier, Format $format, ?string $data = null)
     {
         try {
             $this->identifier = $identifier;
             $this->setFormat($format);
-            $this->setContent($data);
+            if (isset($data)) {
+                $this->setContent($data);
+            }
             $this->setLastChanged();
             $this->sets = new ArrayCollection();
         } catch (ValidationFailedException $exception) {
@@ -270,6 +274,6 @@ public function __construct(string $identifier, Format $format, string $data = '
      */
     public function __toString(): string
     {
-        return $this->content;
+        return $this->content ?? '';
     }
 }