Merge branch 'master' into patch-1

.env

@@ -305,9 +305,15 @@ REQUEST_TERMINATE_TIMEOUT=300s
 
 # Maximum amount of memory a script may consume (128MB)
 # http://php.net/memory-limit
+# @run
 FPM_MEMORY_LIMIT=2048M
 PHP_CLI_MEMORY_LIMIT=2048M
 
+# Temporary directory for HTTP uploaded files (will use system default if not
+# specified).
+# http://php.net/upload-tmp-dir
+# @run
+PHP_UPLOAD_TMP_DIR=/var/alchemy/Phraseanet/tmp/php_upload_tmp
 
 # Php Opcache status. See [opcache Php documentation|
 # https://www.php.net/manual/en/intro.opcache.php].

Dockerfile

@@ -1,5 +1,5 @@
 
-FROM alchemyfr/phraseanet-base:1.0.0 AS builder
+FROM alchemyfr/phraseanet-base:1.1.0 AS builder
 
 COPY --from=composer:2.1.6 /usr/bin/composer /usr/bin/composer
 
@@ -39,8 +39,8 @@ USER app
 
 # Warm up composer cache for faster builds
 COPY docker/caching/composer.* ./
-RUN composer install --prefer-dist --no-dev --no-progress --classmap-authoritative --no-interaction --no-scripts \
-    && rm -rf vendor composer.*
+RUN composer install --prefer-dist --no-dev --no-progress --classmap-authoritative --no-interaction --no-scripts
+#    && rm -rf vendor composer.*
 # End warm up
 
 COPY --chown=app  . .
@@ -72,7 +72,7 @@ CMD []
 # Phraseanet install and setup application image
 #########################################################################
 
-FROM alchemyfr/phraseanet-base:1.0.0 AS phraseanet-setup
+FROM alchemyfr/phraseanet-base:1.1.0 AS phraseanet-setup
 
 COPY --from=builder --chown=app /var/alchemy/Phraseanet /var/alchemy/Phraseanet
 ADD ./docker/phraseanet/root /
@@ -85,7 +85,7 @@ CMD []
 # Phraseanet web application image
 #########################################################################
 
-FROM alchemyfr/phraseanet-base:1.0.0 AS phraseanet-fpm
+FROM alchemyfr/phraseanet-base:1.1.0 AS phraseanet-fpm
 
 COPY --from=builder --chown=app /var/alchemy/Phraseanet /var/alchemy/Phraseanet
 ADD ./docker/phraseanet/root /
@@ -97,7 +97,7 @@ CMD ["php-fpm", "-F"]
 # Phraseanet worker application image
 #########################################################################
 
-FROM alchemyfr/phraseanet-base:1.0.0 AS phraseanet-worker
+FROM alchemyfr/phraseanet-base:1.1.0 AS phraseanet-worker
 
 COPY --from=builder --chown=app /var/alchemy/Phraseanet /var/alchemy/Phraseanet
 ADD ./docker/phraseanet/root /
@@ -139,7 +139,7 @@ HEALTHCHECK CMD wget --spider http://127.0.0.1/login || nginx -s reload || exit
 # phraseanet adapted simplesaml service provider 
 #########################################################################
 
-FROM alchemyfr/phraseanet-base:1.0.0 AS phraseanet-saml-sp
+FROM alchemyfr/phraseanet-base:1.1.0 AS phraseanet-saml-sp
 RUN apt-get update \
     && apt-get install -y \
         apt-transport-https \
@@ -162,5 +162,3 @@ ADD ./docker/phraseanet/saml-sp/root /
 ENTRYPOINT ["/bootstrap/entrypoint.sh"]
 CMD ["/bootstrap/bin/start-servers.sh"]
 HEALTHCHECK CMD wget --spider http://127.0.0.1/ || nginx -s reload || exit
-
-

docker-compose.yml

@@ -105,6 +105,7 @@ services:
     - SESSION_CACHE_LIMITER
     - PHP_LOG_LEVEL
     - PHP_CLI_MEMORY_LIMIT
+    - PHP_UPLOAD_TMP_DIR
     - PHRASEANET_ADMIN_ACCOUNT_ID
     - PHRASEANET_ADMIN_ACCOUNT_EMAIL
     - PHRASEANET_ADMIN_ACCOUNT_PASSWORD
@@ -230,6 +231,7 @@ services:
     - OPCACHE_ENABLED
     - SESSION_CACHE_LIMITER
     - PHP_LOG_LEVEL
+    - PHP_UPLOAD_TMP_DIR
     - PHRASEANET_SCHEME
     - PHRASEANET_HOSTNAME
     - PHRASEANET_APP_PORT

docker/phraseanet/fpm/entrypoint.sh

@@ -5,7 +5,6 @@ set -e
 envsubst < "docker/phraseanet/php.ini.sample" > /usr/local/etc/php/php.ini
 envsubst < "docker/phraseanet/php-fpm.conf.sample" > /usr/local/etc/php-fpm.conf
 envsubst < "docker/phraseanet/root/usr/local/etc/php-fpm.d/zz-docker.conf" > /usr/local/etc/php-fpm.d/zz-docker.conf
-# cat docker/phraseanet/root/usr/local/etc/php-fpm.d/zz-docker.conf  | sed "s/\$REQUEST_TERMINATE_TIMEOUT/$REQUEST_TERMINATE_TIMEOUT/g" > /usr/local/etc/php-fpm.d/zz-docker.conf
 
 if [ ${XDEBUG_ENABLED} == "1" ]; then
     echo "XDEBUG is enabled. YOU MAY KEEP THIS FEATURE DISABLED IN PRODUCTION."
@@ -35,12 +34,6 @@ fi
 chown -R app:app cache
 echo `date +"%Y-%m-%d %H:%M:%S"` " - chown APP:APP on cache/ repository"
 
-#    config \
-#    tmp \
-#    logs \
-#    www
-
-
 if [ -d "plugins/" ];then
 chown -R app:app plugins
 echo `date +"%Y-%m-%d %H:%M:%S"` " - chown APP:APP on plugins/ repository"

docker/phraseanet/php.ini.sample

@@ -817,7 +817,7 @@ file_uploads = On
 ; Temporary directory for HTTP uploaded files (will use system default if not
 ; specified).
 ; http://php.net/upload-tmp-dir
-;upload_tmp_dir =
+upload_tmp_dir = $PHP_UPLOAD_TMP_DIR
 
 ; Maximum allowed size for uploaded files.
 ; http://php.net/upload-max-filesize

docker/phraseanet/setup/entrypoint.sh

@@ -4,6 +4,20 @@ set -e
 envsubst < "docker/phraseanet/php.ini.worker.sample" > /usr/local/etc/php/php.ini
 cat docker/phraseanet/root/usr/local/etc/php-fpm.d/zz-docker.conf  | sed "s/\$REQUEST_TERMINATE_TIMEOUT/$REQUEST_TERMINATE_TIMEOUT/g" > /usr/local/etc/php-fpm.d/zz-docker.conf
 
+if [ -d "$PHP_UPLOAD_TMP_DIR" ]; then
+  echo `date +"%Y-%m-%d %H:%M:%S"` " - The directory: $PHP_UPLOAD_TMP_DIR already exists."
+else
+  echo `date +"%Y-%m-%d %H:%M:%S"` " - The directory: $PHP_UPLOAD_TMP_DIR does not exist. Creating the directory..."
+  mkdir -p "$PHP_UPLOAD_TMP_DIR"
+
+  if [ $? -eq 0 ]; then
+    echo `date +"%Y-%m-%d %H:%M:%S"` " - The directory: $PHP_UPLOAD_TMP_DIR was successfully created."
+  else
+    echo `date +"%Y-%m-%d %H:%M:%S"` " - Failed to create directory: $PHP_UPLOAD_TMP_DIR."
+    exit 1
+  fi
+fi
+
 if [[ -z "$PHRASEANET_APP_PORT" || $PHRASEANET_APP_PORT = "80" || $PHRASEANET_APP_PORT = "443" ]];then
 export PHRASEANET_BASE_URL="$PHRASEANET_SCHEME://$PHRASEANET_HOSTNAME"
 echo  `date +"%Y-%m-%d %H:%M:%S"` " - Phraseanet BASE URL IS : " $PHRASEANET_BASE_URL 
@@ -293,9 +307,14 @@ chown -R app:app backup
 echo `date +"%Y-%m-%d %H:%M:%S"` " - chown APP:APP on www/repository excluding www/thumbnails"
 cd www
 chown -R app:app  $(ls -I thumbnails)
-    
+
 echo `date +"%Y-%m-%d %H:%M:%S"` " - End of chown!"   
 
+if [ -d "$PHP_UPLOAD_TMP_DIR" ]; then
+  echo `date +"%Y-%m-%d %H:%M:%S"` " - Cleaning files older than 2 days in $PHP_UPLOAD_TMP_DIR "
+  find "$PHP_UPLOAD_TMP_DIR" -type f -mtime +2 -exec rm -f {} \;
+fi
+
 echo `date +"%Y-%m-%d %H:%M:%S"` " - End of Phraseanet setup entrypoint.sh"
 
 

lib/Alchemy/Phrasea/Controller/Root/AccountController.php

@@ -318,6 +318,8 @@ public function displayAccount()
 
         $initiatedValidations = $this->getBasketRepository()->findby(['vote_initiator' => $user, ]);
 
+        $this->setSessionFormToken('userAccount');
+
         return $this->render('account/account.html.twig', [
             'user'                  => $user,
             'evt_mngr'              => $manager,
@@ -417,6 +419,10 @@ public function confirmDeleteAccount(Request $request)
      */
     public function updateAccount(Request $request)
     {
+        if (!$this->isCrsfValid($request, 'userAccount')) {
+            return new Response('invalid crsf token form', 403);
+        }
+
         $registrations = $request->request->get('registrations', []);
 
         if (false === is_array($registrations)) {

lib/Alchemy/Phrasea/Controller/Root/DeveloperController.php

@@ -171,6 +171,10 @@ public function authorizeGrantPassword(Request $request, ApiApplication $applica
      */
     public function newApp(Request $request)
     {
+        if (!$this->isCrsfValid($request, 'newApplication')) {
+            return new Response('invalid crsf token form', 403);
+        }
+
         if ($request->request->get('type') === ApiApplication::DESKTOP_TYPE) {
             $form = new \API_OAuth2_Form_DevAppDesktop($request);
         } else {
@@ -223,6 +227,8 @@ public function listApps()
      */
     public function displayFormApp(Request $request)
     {
+        $this->setSessionFormToken('newApplication');
+
         return $this->render('developers/application_form.html.twig', [
             "violations" => null,
             'form'       => null,

lib/Alchemy/Phrasea/Core/Configuration/DisplaySettingService.php

@@ -98,6 +98,10 @@ public function getUserSetting(User $user, $name, $default = null)
             return array_key_exists($name, $this->usersSettings) ? $this->usersSettings[$name] : $default;
         }
 
+        if ($name == 'start_page_query') {
+            return htmlentities($user->getSettings()->get($name)->getValue());
+        }
+
         return $user->getSettings()->get($name)->getValue();
     }
 

lib/Alchemy/Phrasea/Model/Repositories/WorkerRunningJobRepository.php

@@ -328,14 +328,16 @@ private function releaseMutex(int $recordMutexId)
      * mark a job a "finished"
      * nb : after a long job, connection may be lost so we reconnect.
      *      But sometimes (?) a first commit fails (due to reconnect ?), while the second one is ok.
-     *      So here we try 2 times, just in case...
+     *      So here we try 4 times, just in case...
      *
      * @param int $workerRunningJobId
+     * @param MessagePublisher $messagePublisher
+     * @param $jobType
      * @param null $info
      */
-    public function markFinished(int $workerRunningJobId, $info = null)
+    public function markFinished(int $workerRunningJobId, MessagePublisher $messagePublisher, $jobType, $info = null)
     {
-        for($tryout=1; $tryout<=2; $tryout++) {
+        for($wait = 2, $tryout=1; $tryout<=4; $tryout++) {
             try {
                 $this->reconnect();
                 $cnx = $this->getEntityManager()->getConnection()->getWrappedConnection();
@@ -356,8 +358,10 @@ public function markFinished(int $workerRunningJobId, $info = null)
                 throw new Exception(sprintf("updating WorkerRunningJob should return 1 row affected, got %s", $a));
             }
             catch (Exception $e) {
-                if($tryout < 2) {
-                    sleep(1);   // retry in 1 sec
+                if($tryout < 4) {
+                    $messagePublisher->pushLog(sprintf("failed updating WorkerRunningJob to finished with id=%d for %s, attempt %d", $workerRunningJobId, $jobType, $tryout));
+                    sleep($wait);   // retry after more sec
+                    $wait *= 2;
                 }
             }
         }

lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php

@@ -5,6 +5,7 @@
 use Alchemy\Phrasea\Application as PhraseaApplication;
 use Alchemy\Phrasea\Application\Helper\DataboxLoggerAware;
 use Alchemy\Phrasea\Controller\Controller;
+use Alchemy\Phrasea\Filesystem\FilesystemService;
 use Alchemy\Phrasea\Model\Entities\WorkerRunningJob;
 use Alchemy\Phrasea\Model\Repositories\WorkerRunningJobRepository;
 use Alchemy\Phrasea\Plugin\Exception\JsonValidationException;
@@ -224,12 +225,34 @@ public function changeStatusAction(Request $request, $workerId)
 
         /** @var WorkerRunningJob $workerRunningJob */
         $workerRunningJob = $repoWorker->find($workerId);
-
-        $workerRunningJob->setStatus($request->request->get('status'));
+        $subdefOK = false;
         $finishedDate = new \DateTime('now');
 
-        if($request->request->get('finished') == '1') {
-            $workerRunningJob->setFinished($finishedDate)->setFlock(null);
+        if ($workerRunningJob->getWork() == 'subdefCreation') {
+            try {
+                $databox = $this->findDataboxById($workerRunningJob->getDataboxId());
+                $record = $databox->get_record($workerRunningJob->getRecordId());
+                if ($record->has_subdef($workerRunningJob->getWorkOn()) ) {
+                    $filePathToCheck = $record->get_subdef($workerRunningJob->getWorkOn())->getRealPath();
+                    if ($this->getFileSystem()->exists($filePathToCheck)) {
+                        // the subdefinition exist
+                        // so mark as finished
+                        $subdefOK = true;
+                        $workerRunningJob->setStatus(WorkerRunningJob::FINISHED);
+                        $workerRunningJob->setFinished($finishedDate)->setFlock(null);
+                    }
+                }
+            } catch (\Exception $e) {
+            }
+        }
+
+        if (!$subdefOK || $workerRunningJob->getWork() != 'subdefCreation') {
+            $workerRunningJob->setStatus($request->request->get('status'));
+
+
+            if($request->request->get('finished') == '1') {
+                $workerRunningJob->setFinished($finishedDate)->setFlock(null);
+            }
         }
 
         $em = $repoWorker->getEntityManager();
@@ -259,14 +282,48 @@ public function doChangeStatusToCanceledAction(PhraseaApplication $app, Request
     {
         /** @var WorkerRunningJobRepository $repoWorker */
         $repoWorker = $this->app['repo.worker-running-job'];
+        $finishedDate = new \DateTime('now');
+        $em = $repoWorker->getEntityManager();
+
         $workerRunningJobs = $repoWorker->getRunningSinceCreated($request->request->get('hour'), ['subdefCreation', 'writeMetadatas']);
+        $workerRunningJobsForOnlySubdefcreation = $repoWorker->getRunningSinceCreated($request->request->get('hour'), ['subdefCreation']);
 
+        // treat the subdefinition case
+        /** @var WorkerRunningJob $ws */
+        foreach ($workerRunningJobsForOnlySubdefcreation as $ws) {
+            $subdefOK = false;
+            try {
+                $databox = $this->findDataboxById($ws->getDataboxId());
+                $record = $databox->get_record($ws->getRecordId());
+                if ($record->has_subdef($ws->getWorkOn()) ) {
+                    $filePathToCheck = $record->get_subdef($ws->getWorkOn())->getRealPath();
+                    if ($this->getFileSystem()->exists($filePathToCheck)) {
+                        // the subdefinition exist
+                        // so mark as finished
+                        $subdefOK = true;
+                        $ws->setStatus(WorkerRunningJob::FINISHED);
+                        $ws->setFinished($finishedDate)->setFlock(null);
+                    }
+                }
+
+            } catch (\Exception $e) {
+            }
+
+            if (!$subdefOK) {
+                $ws->setStatus(WorkerRunningJob::INTERRUPT);
+                $ws->setFinished($finishedDate)->setFlock(null);
+            }
+            $em->persist($ws);
+        }
+        $em->flush();
+
+        // treat all the rest case
         $repoWorker->updateStatusRunningToCanceledSinceCreated($request->request->get('hour'));
 
-        $finishedDate = new \DateTime('now');
+        // "log docs" the subdefCreation and writeMetadatas action
         /** @var WorkerRunningJob $workerRunningJob */
         foreach ($workerRunningJobs as $workerRunningJob) {
-            $this->updateLogDocs($workerRunningJob, 'canceled', $finishedDate);
+            $this->updateLogDocs($workerRunningJob, $workerRunningJob->getStatus(), $finishedDate);
         }
 
         return $this->app->json(['success' => true]);
@@ -791,4 +848,11 @@ private function getUrlGenerator()
         return $this->app['url_generator'];
     }
 
+    /**
+     * @return FilesystemService
+     */
+    private function getFileSystem()
+    {
+        return $this->app['phraseanet.filesystem'];
+    }
 }

lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php

@@ -159,7 +159,7 @@ public function register(Application $app)
         }));
 
         $app['alchemy_worker.type_based_worker_resolver']->addFactory(MessagePublisher::SUBTITLE_TYPE, new CallableWorkerFactory(function () use ($app) {
-            return (new SubtitleWorker($app['repo.worker-running-job'], $app['conf'], new LazyLocator($app, 'phraseanet.appbox'), $app['alchemy_worker.logger'], $app['dispatcher']))
+            return (new SubtitleWorker($app['repo.worker-running-job'], $app['conf'], new LazyLocator($app, 'phraseanet.appbox'), $app['alchemy_worker.logger'], $app['dispatcher'], $app['alchemy_worker.message.publisher']))
                 ->setFileSystemLocator(new LazyLocator($app, 'filesystem'))
                 ->setTemporaryFileSystemLocator(new LazyLocator($app, 'temporary-filesystem'));
         }));

lib/Alchemy/Phrasea/WorkerManager/Worker/CreateRecordWorker.php

@@ -137,24 +137,6 @@ public function process(array $payload)
             return;
         }
 
-        if ($workerRunningJob != null) {
-            $em->beginTransaction();
-            try {
-                $workerRunningJob
-                    ->setStatus(WorkerRunningJob::FINISHED)
-                    ->setFinished(new \DateTime('now'))
-                ;
-
-                $em->persist($workerRunningJob);
-
-                $em->flush();
-                $em->commit();
-            } catch (\Exception $e) {
-                $em->rollback();
-            }
-
-        }
-
         $lazaretSession = new LazaretSession();
 
         $userRepository = $this->getUserRepository();
@@ -257,6 +239,11 @@ public function process(array $payload)
                 ]
             ]
         );
+
+        if ($workerRunningJob != null) {
+            $this->repoWorkerJob->markFinished($workerRunningJob->getId(), $this->messagePublisher, MessagePublisher::CREATE_RECORD_TYPE);
+        }
+
     }
 
     /**