From 62e44715b6b11ce6a4aceffe9d84cb3e3fcfdf96 Mon Sep 17 00:00:00 2001 From: Nicolas Maillat Date: Thu, 10 Oct 2024 18:14:46 +0200 Subject: [PATCH 1/5] PHRAS-4100 Php upload tmp directory (#4553) * adding PHP_UPLOAD_TMP_DIR * ready for QA * clean in fpm entrypoint * leave php upload_tmp_dir in /tmp for workers * keep only PHP_UPLOAD_TMP_DIR env in fpm and setup containers --- .env | 6 ++++++ docker-compose.yml | 2 ++ docker/phraseanet/fpm/entrypoint.sh | 7 ------- docker/phraseanet/php.ini.sample | 2 +- docker/phraseanet/setup/entrypoint.sh | 21 ++++++++++++++++++++- 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 04b50a5b90..1983bf8287 100644 --- a/.env +++ b/.env @@ -305,9 +305,15 @@ REQUEST_TERMINATE_TIMEOUT=300s # Maximum amount of memory a script may consume (128MB) # http://php.net/memory-limit +# @run FPM_MEMORY_LIMIT=2048M PHP_CLI_MEMORY_LIMIT=2048M +# Temporary directory for HTTP uploaded files (will use system default if not +# specified). +# http://php.net/upload-tmp-dir +# @run +PHP_UPLOAD_TMP_DIR=/var/alchemy/Phraseanet/tmp/php_upload_tmp # Php Opcache status. See [opcache Php documentation| # https://www.php.net/manual/en/intro.opcache.php]. diff --git a/docker-compose.yml b/docker-compose.yml index e5914218fe..b378c472f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,6 +105,7 @@ services: - SESSION_CACHE_LIMITER - PHP_LOG_LEVEL - PHP_CLI_MEMORY_LIMIT + - PHP_UPLOAD_TMP_DIR - PHRASEANET_ADMIN_ACCOUNT_ID - PHRASEANET_ADMIN_ACCOUNT_EMAIL - PHRASEANET_ADMIN_ACCOUNT_PASSWORD @@ -230,6 +231,7 @@ services: - OPCACHE_ENABLED - SESSION_CACHE_LIMITER - PHP_LOG_LEVEL + - PHP_UPLOAD_TMP_DIR - PHRASEANET_SCHEME - PHRASEANET_HOSTNAME - PHRASEANET_APP_PORT diff --git a/docker/phraseanet/fpm/entrypoint.sh b/docker/phraseanet/fpm/entrypoint.sh index b14b8f917f..98caf8a08c 100755 --- a/docker/phraseanet/fpm/entrypoint.sh +++ b/docker/phraseanet/fpm/entrypoint.sh @@ -5,7 +5,6 @@ set -e envsubst < "docker/phraseanet/php.ini.sample" > /usr/local/etc/php/php.ini envsubst < "docker/phraseanet/php-fpm.conf.sample" > /usr/local/etc/php-fpm.conf envsubst < "docker/phraseanet/root/usr/local/etc/php-fpm.d/zz-docker.conf" > /usr/local/etc/php-fpm.d/zz-docker.conf -# cat docker/phraseanet/root/usr/local/etc/php-fpm.d/zz-docker.conf | sed "s/\$REQUEST_TERMINATE_TIMEOUT/$REQUEST_TERMINATE_TIMEOUT/g" > /usr/local/etc/php-fpm.d/zz-docker.conf if [ ${XDEBUG_ENABLED} == "1" ]; then echo "XDEBUG is enabled. YOU MAY KEEP THIS FEATURE DISABLED IN PRODUCTION." @@ -35,12 +34,6 @@ fi chown -R app:app cache echo `date +"%Y-%m-%d %H:%M:%S"` " - chown APP:APP on cache/ repository" -# config \ -# tmp \ -# logs \ -# www - - if [ -d "plugins/" ];then chown -R app:app plugins echo `date +"%Y-%m-%d %H:%M:%S"` " - chown APP:APP on plugins/ repository" diff --git a/docker/phraseanet/php.ini.sample b/docker/phraseanet/php.ini.sample index a4c714dc60..c738feb74f 100644 --- a/docker/phraseanet/php.ini.sample +++ b/docker/phraseanet/php.ini.sample @@ -817,7 +817,7 @@ file_uploads = On ; Temporary directory for HTTP uploaded files (will use system default if not ; specified). ; http://php.net/upload-tmp-dir -;upload_tmp_dir = +upload_tmp_dir = $PHP_UPLOAD_TMP_DIR ; Maximum allowed size for uploaded files. ; http://php.net/upload-max-filesize diff --git a/docker/phraseanet/setup/entrypoint.sh b/docker/phraseanet/setup/entrypoint.sh index 8abaad59c7..02d1ca4aa2 100755 --- a/docker/phraseanet/setup/entrypoint.sh +++ b/docker/phraseanet/setup/entrypoint.sh @@ -4,6 +4,20 @@ set -e envsubst < "docker/phraseanet/php.ini.worker.sample" > /usr/local/etc/php/php.ini cat docker/phraseanet/root/usr/local/etc/php-fpm.d/zz-docker.conf | sed "s/\$REQUEST_TERMINATE_TIMEOUT/$REQUEST_TERMINATE_TIMEOUT/g" > /usr/local/etc/php-fpm.d/zz-docker.conf +if [ -d "$PHP_UPLOAD_TMP_DIR" ]; then + echo `date +"%Y-%m-%d %H:%M:%S"` " - The directory: $PHP_UPLOAD_TMP_DIR already exists." +else + echo `date +"%Y-%m-%d %H:%M:%S"` " - The directory: $PHP_UPLOAD_TMP_DIR does not exist. Creating the directory..." + mkdir -p "$PHP_UPLOAD_TMP_DIR" + + if [ $? -eq 0 ]; then + echo `date +"%Y-%m-%d %H:%M:%S"` " - The directory: $PHP_UPLOAD_TMP_DIR was successfully created." + else + echo `date +"%Y-%m-%d %H:%M:%S"` " - Failed to create directory: $PHP_UPLOAD_TMP_DIR." + exit 1 + fi +fi + if [[ -z "$PHRASEANET_APP_PORT" || $PHRASEANET_APP_PORT = "80" || $PHRASEANET_APP_PORT = "443" ]];then export PHRASEANET_BASE_URL="$PHRASEANET_SCHEME://$PHRASEANET_HOSTNAME" echo `date +"%Y-%m-%d %H:%M:%S"` " - Phraseanet BASE URL IS : " $PHRASEANET_BASE_URL @@ -293,9 +307,14 @@ chown -R app:app backup echo `date +"%Y-%m-%d %H:%M:%S"` " - chown APP:APP on www/repository excluding www/thumbnails" cd www chown -R app:app $(ls -I thumbnails) - + echo `date +"%Y-%m-%d %H:%M:%S"` " - End of chown!" +if [ -d "$PHP_UPLOAD_TMP_DIR" ]; then + echo `date +"%Y-%m-%d %H:%M:%S"` " - Cleaning files older than 2 days in $PHP_UPLOAD_TMP_DIR " + find "$PHP_UPLOAD_TMP_DIR" -type f -mtime +2 -exec rm -f {} \; +fi + echo `date +"%Y-%m-%d %H:%M:%S"` " - End of Phraseanet setup entrypoint.sh" From 8f84487688d7576b9c5125cf86e9e67b6f4a3e31 Mon Sep 17 00:00:00 2001 From: moctardiouf Date: Thu, 17 Oct 2024 13:45:57 +0200 Subject: [PATCH 2/5] PHRAS-4079 Bump base image 1.1.0 (#4554) * use base-image-1.0.1-alpha4 * PHRAS-4079 bump base image to 1.1.0 --- Dockerfile | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index bf7a1ef828..bca1bb3fcf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -FROM alchemyfr/phraseanet-base:1.0.0 AS builder +FROM alchemyfr/phraseanet-base:1.1.0 AS builder COPY --from=composer:2.1.6 /usr/bin/composer /usr/bin/composer @@ -39,8 +39,8 @@ USER app # Warm up composer cache for faster builds COPY docker/caching/composer.* ./ -RUN composer install --prefer-dist --no-dev --no-progress --classmap-authoritative --no-interaction --no-scripts \ - && rm -rf vendor composer.* +RUN composer install --prefer-dist --no-dev --no-progress --classmap-authoritative --no-interaction --no-scripts +# && rm -rf vendor composer.* # End warm up COPY --chown=app . . @@ -72,7 +72,7 @@ CMD [] # Phraseanet install and setup application image ######################################################################### -FROM alchemyfr/phraseanet-base:1.0.0 AS phraseanet-setup +FROM alchemyfr/phraseanet-base:1.1.0 AS phraseanet-setup COPY --from=builder --chown=app /var/alchemy/Phraseanet /var/alchemy/Phraseanet ADD ./docker/phraseanet/root / @@ -85,7 +85,7 @@ CMD [] # Phraseanet web application image ######################################################################### -FROM alchemyfr/phraseanet-base:1.0.0 AS phraseanet-fpm +FROM alchemyfr/phraseanet-base:1.1.0 AS phraseanet-fpm COPY --from=builder --chown=app /var/alchemy/Phraseanet /var/alchemy/Phraseanet ADD ./docker/phraseanet/root / @@ -97,7 +97,7 @@ CMD ["php-fpm", "-F"] # Phraseanet worker application image ######################################################################### -FROM alchemyfr/phraseanet-base:1.0.0 AS phraseanet-worker +FROM alchemyfr/phraseanet-base:1.1.0 AS phraseanet-worker COPY --from=builder --chown=app /var/alchemy/Phraseanet /var/alchemy/Phraseanet ADD ./docker/phraseanet/root / @@ -139,7 +139,7 @@ HEALTHCHECK CMD wget --spider http://127.0.0.1/login || nginx -s reload || exit # phraseanet adapted simplesaml service provider ######################################################################### -FROM alchemyfr/phraseanet-base:1.0.0 AS phraseanet-saml-sp +FROM alchemyfr/phraseanet-base:1.1.0 AS phraseanet-saml-sp RUN apt-get update \ && apt-get install -y \ apt-transport-https \ @@ -162,5 +162,3 @@ ADD ./docker/phraseanet/saml-sp/root / ENTRYPOINT ["/bootstrap/entrypoint.sh"] CMD ["/bootstrap/bin/start-servers.sh"] HEALTHCHECK CMD wget --spider http://127.0.0.1/ || nginx -s reload || exit - - From 59342a6975dd95299fbe74396b5ee591bb2219d1 Mon Sep 17 00:00:00 2001 From: Aina Sitraka <35221835+aynsix@users.noreply.github.com> Date: Wed, 23 Oct 2024 18:46:31 +0300 Subject: [PATCH 3/5] PHRAS-3857 : Check CSRF token on account (#4556) * fix csrf account * fix * csrf new application * fix --- .../Phrasea/Controller/Root/AccountController.php | 6 ++++++ .../Phrasea/Controller/Root/DeveloperController.php | 6 ++++++ templates/web/account/account.html.twig | 1 + templates/web/developers/application_form.html.twig | 1 + .../Tests/Phrasea/Controller/Root/AccountTest.php | 4 +++- .../Tests/Phrasea/Controller/Root/DevelopersTest.php | 9 +++++++-- 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/lib/Alchemy/Phrasea/Controller/Root/AccountController.php b/lib/Alchemy/Phrasea/Controller/Root/AccountController.php index 6a6bc67b1a..8992212868 100644 --- a/lib/Alchemy/Phrasea/Controller/Root/AccountController.php +++ b/lib/Alchemy/Phrasea/Controller/Root/AccountController.php @@ -318,6 +318,8 @@ public function displayAccount() $initiatedValidations = $this->getBasketRepository()->findby(['vote_initiator' => $user, ]); + $this->setSessionFormToken('userAccount'); + return $this->render('account/account.html.twig', [ 'user' => $user, 'evt_mngr' => $manager, @@ -417,6 +419,10 @@ public function confirmDeleteAccount(Request $request) */ public function updateAccount(Request $request) { + if (!$this->isCrsfValid($request, 'userAccount')) { + return new Response('invalid crsf token form', 403); + } + $registrations = $request->request->get('registrations', []); if (false === is_array($registrations)) { diff --git a/lib/Alchemy/Phrasea/Controller/Root/DeveloperController.php b/lib/Alchemy/Phrasea/Controller/Root/DeveloperController.php index 5634819113..98e3b137cb 100644 --- a/lib/Alchemy/Phrasea/Controller/Root/DeveloperController.php +++ b/lib/Alchemy/Phrasea/Controller/Root/DeveloperController.php @@ -171,6 +171,10 @@ public function authorizeGrantPassword(Request $request, ApiApplication $applica */ public function newApp(Request $request) { + if (!$this->isCrsfValid($request, 'newApplication')) { + return new Response('invalid crsf token form', 403); + } + if ($request->request->get('type') === ApiApplication::DESKTOP_TYPE) { $form = new \API_OAuth2_Form_DevAppDesktop($request); } else { @@ -223,6 +227,8 @@ public function listApps() */ public function displayFormApp(Request $request) { + $this->setSessionFormToken('newApplication'); + return $this->render('developers/application_form.html.twig', [ "violations" => null, 'form' => null, diff --git a/templates/web/account/account.html.twig b/templates/web/account/account.html.twig index c843a3561f..458fcc0476 100644 --- a/templates/web/account/account.html.twig +++ b/templates/web/account/account.html.twig @@ -293,6 +293,7 @@ + diff --git a/templates/web/developers/application_form.html.twig b/templates/web/developers/application_form.html.twig index c1ec6cb8bf..9b543afea0 100644 --- a/templates/web/developers/application_form.html.twig +++ b/templates/web/developers/application_form.html.twig @@ -123,5 +123,6 @@ + {% endblock %} diff --git a/tests/Alchemy/Tests/Phrasea/Controller/Root/AccountTest.php b/tests/Alchemy/Tests/Phrasea/Controller/Root/AccountTest.php index 4f42595d3c..0aed6b756f 100644 --- a/tests/Alchemy/Tests/Phrasea/Controller/Root/AccountTest.php +++ b/tests/Alchemy/Tests/Phrasea/Controller/Root/AccountTest.php @@ -384,6 +384,7 @@ public function testUpdateAccount() $app = $this->getApplication(); $client = $this->getClient(); $bases = $notifs = []; + $randomValue = $this->setSessionFormToken('userAccount'); foreach ($app->getDataboxes() as $databox) { foreach ($databox->get_collections() as $collection) { @@ -424,7 +425,8 @@ public function testUpdateAccount() 'form_retryFTP' => '', 'notifications' => $notifs, 'form_defaultdataFTP' => ['document', 'preview', 'caption'], - 'mail_notifications' => '1' + 'mail_notifications' => '1', + 'userAccount_token' => $randomValue ]); $response = $client->getResponse(); diff --git a/tests/Alchemy/Tests/Phrasea/Controller/Root/DevelopersTest.php b/tests/Alchemy/Tests/Phrasea/Controller/Root/DevelopersTest.php index f3fbc55fb5..4f81818540 100644 --- a/tests/Alchemy/Tests/Phrasea/Controller/Root/DevelopersTest.php +++ b/tests/Alchemy/Tests/Phrasea/Controller/Root/DevelopersTest.php @@ -39,6 +39,8 @@ public function testDisplayformApp() */ public function testPostNewAppInvalidArguments() { + $randomValue = $this->setSessionFormToken('newApplication'); + $crawler = self::$DI['client']->request('POST', '/developers/application/', [ 'type' => ApiApplication::WEB_TYPE, 'name' => '', @@ -46,7 +48,8 @@ public function testPostNewAppInvalidArguments() 'website' => 'my.website.com', 'callback' => 'my.callback.com', 'scheme-website' => 'http://', - 'scheme-callback' => 'http://' + 'scheme-callback' => 'http://', + 'newApplication_token' => $randomValue ]); $this->assertTrue(self::$DI['client']->getResponse()->isOk()); @@ -63,6 +66,7 @@ public function testPostNewApp() { $apps = self::$DI['app']['repo.api-applications']->findByCreator(self::$DI['user']); $nbApp = count($apps); + $randomValue = $this->setSessionFormToken('newApplication'); self::$DI['client']->request('POST', '/developers/application/', [ 'type' => ApiApplication::WEB_TYPE, @@ -71,7 +75,8 @@ public function testPostNewApp() 'website' => 'my.website.com', 'callback' => 'my.callback.com', 'scheme-website' => 'http://', - 'scheme-callback' => 'http://' + 'scheme-callback' => 'http://', + 'newApplication_token' => $randomValue ]); $apps = self::$DI['app']['repo.api-applications']->findByCreator(self::$DI['user']); From 47371bb2aa9cd29d97638902653b04e2f8f436a8 Mon Sep 17 00:00:00 2001 From: Aina Sitraka <35221835+aynsix@users.noreply.github.com> Date: Wed, 23 Oct 2024 20:57:10 +0300 Subject: [PATCH 4/5] fix (#4555) --- .../Phrasea/Core/Configuration/DisplaySettingService.php | 4 ++++ templates/web/prod/index.html.twig | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/Alchemy/Phrasea/Core/Configuration/DisplaySettingService.php b/lib/Alchemy/Phrasea/Core/Configuration/DisplaySettingService.php index 11987a9d5c..99c4ff90cd 100644 --- a/lib/Alchemy/Phrasea/Core/Configuration/DisplaySettingService.php +++ b/lib/Alchemy/Phrasea/Core/Configuration/DisplaySettingService.php @@ -98,6 +98,10 @@ public function getUserSetting(User $user, $name, $default = null) return array_key_exists($name, $this->usersSettings) ? $this->usersSettings[$name] : $default; } + if ($name == 'start_page_query') { + return htmlentities($user->getSettings()->get($name)->getValue()); + } + return $user->getSettings()->get($name)->getValue(); } diff --git a/templates/web/prod/index.html.twig b/templates/web/prod/index.html.twig index 519c46f298..5a39d59250 100644 --- a/templates/web/prod/index.html.twig +++ b/templates/web/prod/index.html.twig @@ -986,7 +986,7 @@ {{ 'Aide' | trans }} - + From 3fa25670878adc1180723d62bda0c556ec0eed9f Mon Sep 17 00:00:00 2001 From: Aina Sitraka <35221835+aynsix@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:16:30 +0300 Subject: [PATCH 5/5] PHRAS-4088: improving Job ack in workerRunningJob (#4535) * improve workerrunningjob finished * subdefcreation mark as finished when file exist --- .../WorkerRunningJobRepository.php | 14 ++-- .../AdminConfigurationController.php | 76 +++++++++++++++++-- .../Provider/AlchemyWorkerServiceProvider.php | 2 +- .../Worker/CreateRecordWorker.php | 23 ++---- .../Worker/DeleteRecordWorker.php | 9 +-- .../Worker/DownloadAsyncWorker.php | 15 ++-- .../WorkerManager/Worker/EditRecordWorker.php | 12 +-- .../WorkerManager/Worker/ExportMailWorker.php | 19 +++-- .../Worker/ExposeUploadWorker.php | 2 +- .../WorkerManager/Worker/FtpWorker.php | 14 +--- .../Worker/PopulateIndexWorker.php | 10 +-- .../Worker/ShareBasketWorker.php | 26 +++++-- .../Worker/SubdefCreationWorker.php | 2 +- .../WorkerManager/Worker/SubtitleWorker.php | 10 ++- .../WorkerManager/Worker/WebhookWorker.php | 9 +-- .../Worker/WriteMetadatasWorker.php | 8 +- 16 files changed, 137 insertions(+), 114 deletions(-) diff --git a/lib/Alchemy/Phrasea/Model/Repositories/WorkerRunningJobRepository.php b/lib/Alchemy/Phrasea/Model/Repositories/WorkerRunningJobRepository.php index cd32204602..3e0b281c59 100644 --- a/lib/Alchemy/Phrasea/Model/Repositories/WorkerRunningJobRepository.php +++ b/lib/Alchemy/Phrasea/Model/Repositories/WorkerRunningJobRepository.php @@ -328,14 +328,16 @@ private function releaseMutex(int $recordMutexId) * mark a job a "finished" * nb : after a long job, connection may be lost so we reconnect. * But sometimes (?) a first commit fails (due to reconnect ?), while the second one is ok. - * So here we try 2 times, just in case... + * So here we try 4 times, just in case... * * @param int $workerRunningJobId + * @param MessagePublisher $messagePublisher + * @param $jobType * @param null $info */ - public function markFinished(int $workerRunningJobId, $info = null) + public function markFinished(int $workerRunningJobId, MessagePublisher $messagePublisher, $jobType, $info = null) { - for($tryout=1; $tryout<=2; $tryout++) { + for($wait = 2, $tryout=1; $tryout<=4; $tryout++) { try { $this->reconnect(); $cnx = $this->getEntityManager()->getConnection()->getWrappedConnection(); @@ -356,8 +358,10 @@ public function markFinished(int $workerRunningJobId, $info = null) throw new Exception(sprintf("updating WorkerRunningJob should return 1 row affected, got %s", $a)); } catch (Exception $e) { - if($tryout < 2) { - sleep(1); // retry in 1 sec + if($tryout < 4) { + $messagePublisher->pushLog(sprintf("failed updating WorkerRunningJob to finished with id=%d for %s, attempt %d", $workerRunningJobId, $jobType, $tryout)); + sleep($wait); // retry after more sec + $wait *= 2; } } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php b/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php index 5f4e2684c1..38f3a6a8b0 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Controller/AdminConfigurationController.php @@ -5,6 +5,7 @@ use Alchemy\Phrasea\Application as PhraseaApplication; use Alchemy\Phrasea\Application\Helper\DataboxLoggerAware; use Alchemy\Phrasea\Controller\Controller; +use Alchemy\Phrasea\Filesystem\FilesystemService; use Alchemy\Phrasea\Model\Entities\WorkerRunningJob; use Alchemy\Phrasea\Model\Repositories\WorkerRunningJobRepository; use Alchemy\Phrasea\Plugin\Exception\JsonValidationException; @@ -224,12 +225,34 @@ public function changeStatusAction(Request $request, $workerId) /** @var WorkerRunningJob $workerRunningJob */ $workerRunningJob = $repoWorker->find($workerId); - - $workerRunningJob->setStatus($request->request->get('status')); + $subdefOK = false; $finishedDate = new \DateTime('now'); - if($request->request->get('finished') == '1') { - $workerRunningJob->setFinished($finishedDate)->setFlock(null); + if ($workerRunningJob->getWork() == 'subdefCreation') { + try { + $databox = $this->findDataboxById($workerRunningJob->getDataboxId()); + $record = $databox->get_record($workerRunningJob->getRecordId()); + if ($record->has_subdef($workerRunningJob->getWorkOn()) ) { + $filePathToCheck = $record->get_subdef($workerRunningJob->getWorkOn())->getRealPath(); + if ($this->getFileSystem()->exists($filePathToCheck)) { + // the subdefinition exist + // so mark as finished + $subdefOK = true; + $workerRunningJob->setStatus(WorkerRunningJob::FINISHED); + $workerRunningJob->setFinished($finishedDate)->setFlock(null); + } + } + } catch (\Exception $e) { + } + } + + if (!$subdefOK || $workerRunningJob->getWork() != 'subdefCreation') { + $workerRunningJob->setStatus($request->request->get('status')); + + + if($request->request->get('finished') == '1') { + $workerRunningJob->setFinished($finishedDate)->setFlock(null); + } } $em = $repoWorker->getEntityManager(); @@ -259,14 +282,48 @@ public function doChangeStatusToCanceledAction(PhraseaApplication $app, Request { /** @var WorkerRunningJobRepository $repoWorker */ $repoWorker = $this->app['repo.worker-running-job']; + $finishedDate = new \DateTime('now'); + $em = $repoWorker->getEntityManager(); + $workerRunningJobs = $repoWorker->getRunningSinceCreated($request->request->get('hour'), ['subdefCreation', 'writeMetadatas']); + $workerRunningJobsForOnlySubdefcreation = $repoWorker->getRunningSinceCreated($request->request->get('hour'), ['subdefCreation']); + // treat the subdefinition case + /** @var WorkerRunningJob $ws */ + foreach ($workerRunningJobsForOnlySubdefcreation as $ws) { + $subdefOK = false; + try { + $databox = $this->findDataboxById($ws->getDataboxId()); + $record = $databox->get_record($ws->getRecordId()); + if ($record->has_subdef($ws->getWorkOn()) ) { + $filePathToCheck = $record->get_subdef($ws->getWorkOn())->getRealPath(); + if ($this->getFileSystem()->exists($filePathToCheck)) { + // the subdefinition exist + // so mark as finished + $subdefOK = true; + $ws->setStatus(WorkerRunningJob::FINISHED); + $ws->setFinished($finishedDate)->setFlock(null); + } + } + + } catch (\Exception $e) { + } + + if (!$subdefOK) { + $ws->setStatus(WorkerRunningJob::INTERRUPT); + $ws->setFinished($finishedDate)->setFlock(null); + } + $em->persist($ws); + } + $em->flush(); + + // treat all the rest case $repoWorker->updateStatusRunningToCanceledSinceCreated($request->request->get('hour')); - $finishedDate = new \DateTime('now'); + // "log docs" the subdefCreation and writeMetadatas action /** @var WorkerRunningJob $workerRunningJob */ foreach ($workerRunningJobs as $workerRunningJob) { - $this->updateLogDocs($workerRunningJob, 'canceled', $finishedDate); + $this->updateLogDocs($workerRunningJob, $workerRunningJob->getStatus(), $finishedDate); } return $this->app->json(['success' => true]); @@ -791,4 +848,11 @@ private function getUrlGenerator() return $this->app['url_generator']; } + /** + * @return FilesystemService + */ + private function getFileSystem() + { + return $this->app['phraseanet.filesystem']; + } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php b/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php index 71aba39dfe..c947fd6023 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php @@ -159,7 +159,7 @@ public function register(Application $app) })); $app['alchemy_worker.type_based_worker_resolver']->addFactory(MessagePublisher::SUBTITLE_TYPE, new CallableWorkerFactory(function () use ($app) { - return (new SubtitleWorker($app['repo.worker-running-job'], $app['conf'], new LazyLocator($app, 'phraseanet.appbox'), $app['alchemy_worker.logger'], $app['dispatcher'])) + return (new SubtitleWorker($app['repo.worker-running-job'], $app['conf'], new LazyLocator($app, 'phraseanet.appbox'), $app['alchemy_worker.logger'], $app['dispatcher'], $app['alchemy_worker.message.publisher'])) ->setFileSystemLocator(new LazyLocator($app, 'filesystem')) ->setTemporaryFileSystemLocator(new LazyLocator($app, 'temporary-filesystem')); })); diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/CreateRecordWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/CreateRecordWorker.php index 03847d2fa2..18168332e2 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/CreateRecordWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/CreateRecordWorker.php @@ -137,24 +137,6 @@ public function process(array $payload) return; } - if ($workerRunningJob != null) { - $em->beginTransaction(); - try { - $workerRunningJob - ->setStatus(WorkerRunningJob::FINISHED) - ->setFinished(new \DateTime('now')) - ; - - $em->persist($workerRunningJob); - - $em->flush(); - $em->commit(); - } catch (\Exception $e) { - $em->rollback(); - } - - } - $lazaretSession = new LazaretSession(); $userRepository = $this->getUserRepository(); @@ -257,6 +239,11 @@ public function process(array $payload) ] ] ); + + if ($workerRunningJob != null) { + $this->repoWorkerJob->markFinished($workerRunningJob->getId(), $this->messagePublisher, MessagePublisher::CREATE_RECORD_TYPE); + } + } /** diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/DeleteRecordWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/DeleteRecordWorker.php index ad1fc0a492..59a2184baa 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/DeleteRecordWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/DeleteRecordWorker.php @@ -69,14 +69,7 @@ public function process(array $payload) // tell that the delete is finished if ($workerRunningJob != null) { - $workerRunningJob - ->setStatus(WorkerRunningJob::FINISHED) - ->setFinished(new \DateTime('now')) - ; - - $em->persist($workerRunningJob); - - $em->flush(); + $this->repoWorker->markFinished($workerRunningJob->getId(), $this->messagePublisher, MessagePublisher::DELETE_RECORD_TYPE); } } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/DownloadAsyncWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/DownloadAsyncWorker.php index e567530ac9..35270ab176 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/DownloadAsyncWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/DownloadAsyncWorker.php @@ -385,15 +385,7 @@ public function process(array $payload) ); if ($workerRunningJob != null) { - $this->repoWorkerJob->reconnect(); - $workerRunningJob - ->setStatus(WorkerRunningJob::FINISHED) - ->setFinished(new \DateTime('now')) - ; - - $em->persist($workerRunningJob); - - $em->flush(); + $this->repoWorkerJob->markFinished($workerRunningJob->getId(), $this->getMessagePublisher(),MessagePublisher::DOWNLOAD_ASYNC_TYPE); } sleep(1); @@ -436,6 +428,11 @@ private function getWorkerRunningJobRepository() return $this->app['repo.worker-running-job']; } + private function getMessagePublisher() + { + return $this->app['alchemy_worker.message.publisher']; + } + private function cellRefFromColumnAndRow(int $col, int $row = null) { $r = Coordinate::stringFromColumnIndex($col); diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/EditRecordWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/EditRecordWorker.php index 0997204cf5..0886615a61 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/EditRecordWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/EditRecordWorker.php @@ -200,17 +200,7 @@ public function process(array $payload) ); // tell that we have finished to work on edit - $this->repoWorker->reconnect(); - $em->getConnection()->beginTransaction(); - try { - $workerRunningJob->setStatus(WorkerRunningJob::FINISHED); - $workerRunningJob->setFinished(new \DateTime('now')); - $em->persist($workerRunningJob); - $em->flush(); - $em->commit(); - } catch (\Exception $e) { - $em->rollback(); - } + $this->repoWorker->markFinished($workerRunningJob->getId(), $this->messagePublisher, MessagePublisher::EDIT_RECORD_TYPE); $this->messagePublisher->pushLog(sprintf("record edited databoxname=%s databoxid=%d recordid=%d", $databox->get_viewname(), $payload['databoxId'], $payload['record_id'])); } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/ExportMailWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/ExportMailWorker.php index 0e33106341..a836277085 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/ExportMailWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/ExportMailWorker.php @@ -216,16 +216,7 @@ public function process(array $payload) } if ($workerRunningJob != null) { - $this->repoWorkerJob->reconnect(); - $workerRunningJob - ->setWorkOn(implode(',', $deliverEmails)) - ->setStatus(WorkerRunningJob::FINISHED) - ->setFinished(new \DateTime('now')) - ; - - $em->persist($workerRunningJob); - - $em->flush(); + $this->repoWorkerJob->markFinished($workerRunningJob->getId(), $this->getMessagePublisher(), MessagePublisher::EXPORT_MAIL_TYPE); } sleep(30); @@ -250,4 +241,12 @@ private function getWorkerRunningJobRepository() { return $this->app['repo.worker-running-job']; } + + /** + * @return MessagePublisher + */ + private function getMessagePublisher() + { + return $this->app['alchemy_worker.message.publisher']; + } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/ExposeUploadWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/ExposeUploadWorker.php index 1050e8df37..2bab163278 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/ExposeUploadWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/ExposeUploadWorker.php @@ -373,7 +373,7 @@ public function process(array $payload) } // tell that the upload is finished - $this->finishedJob($workerRunningJob, $em); + $this->repoWorker->markFinished($workerRunningJob->getId(), $this->messagePublisher, MessagePublisher::EXPOSE_UPLOAD_TYPE); } private function getClientAnnotationProfile(Client $exposeClient, $publicationId) diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/FtpWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/FtpWorker.php index 6db13c3bd9..af710f16ad 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/FtpWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/FtpWorker.php @@ -355,18 +355,7 @@ private function doExport(FtpExport $export, array $payload) if (!$processError && $workerRunningJob) { // tell that we have finished to work on this file - $this->repoWorker->reconnect(); - $em->beginTransaction(); - try { - $workerRunningJob->setStatus(WorkerRunningJob::FINISHED); - $workerRunningJob->setFinished(new \DateTime('now')); - $em->persist($workerRunningJob); - $em->flush(); - $em->commit(); - } - catch (Exception $e) { - $em->rollback(); - } + $this->repoWorker->markFinished($workerRunningJob->getId(), $this->getMessagePublisher(), MessagePublisher::FTP_TYPE); } else { // if there is an error $count = isset($payload['count']) ? $payload['count'] + 1 : 2 ; @@ -537,5 +526,4 @@ private function getMessagePublisher() { return $this->app['alchemy_worker.message.publisher']; } - } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/PopulateIndexWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/PopulateIndexWorker.php index 465599fae9..98e45a44fa 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/PopulateIndexWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/PopulateIndexWorker.php @@ -145,15 +145,7 @@ public function process(array $payload) // tell that the populate is finished if ($workerRunningJob != null) { - $this->repoWorker->reconnect(); - $workerRunningJob - ->setStatus(WorkerRunningJob::FINISHED) - ->setFinished(new \DateTime('now')) - ; - - $em->persist($workerRunningJob); - - $em->flush(); + $this->repoWorker->markFinished($workerRunningJob->getId(), $this->messagePublisher, MessagePublisher::POPULATE_INDEX_TYPE); } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/ShareBasketWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/ShareBasketWorker.php index 3fc10e136a..e8d23471f8 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/ShareBasketWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/ShareBasketWorker.php @@ -15,6 +15,7 @@ use Alchemy\Phrasea\Model\Manipulator\TokenManipulator; use Alchemy\Phrasea\Model\Repositories\BasketRepository; use Alchemy\Phrasea\Model\Repositories\UserRepository; +use Alchemy\Phrasea\Model\Repositories\WorkerRunningJobRepository; use Alchemy\Phrasea\Record\RecordReference; use Alchemy\Phrasea\WorkerManager\Queue\MessagePublisher; use DateTime; @@ -405,14 +406,7 @@ public function process(array $payload) $this->getLogger()->info("Basket with Id " . $basket->getId() . " successfully shared !"); if ($workerRunningJob != null) { - $workerRunningJob - ->setStatus(WorkerRunningJob::FINISHED) - ->setFinished(new \DateTime('now')) - ; - - $manager->persist($workerRunningJob); - - $manager->flush(); + $this->getRepoWorkerRunningJob()->markFinished($workerRunningJob->getId(), $this->getMessagePublisher(), MessagePublisher::SHARE_BASKET_TYPE); } // file_put_contents("./tmp/phraseanet-log.txt", sprintf("\n%s; ==== END (N = %d ; dT = %d ==> %0.2f / sec) ====\n\n", time(), $n_participants, time()-$_t0, $n_participants/(max(time()-$_t0, 0.001))), FILE_APPEND); @@ -500,4 +494,20 @@ private function getLogger() { return $this->app['alchemy_worker.logger']; } + + /** + * @return WorkerRunningJobRepository + */ + private function getRepoWorkerRunningJob() + { + return $this->app['repo.worker-running-job']; + } + + /** + * @return MessagePublisher + */ + private function getMessagePublisher() + { + return $this->app['alchemy_worker.message.publisher']; + } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php index 9e647191f6..6869c36f99 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/SubdefCreationWorker.php @@ -199,7 +199,7 @@ public function process(array $payload) $payload['subdefName'], $databox->get_viewname(), $databoxId, $recordId)); // tell that we have finished to work on this file (=unlock) - $this->repoWorker->markFinished($workerRunningJobId); + $this->repoWorker->markFinished($workerRunningJobId, $this->messagePublisher, MessagePublisher::SUBDEF_CREATION_TYPE); $this->getDataboxLogger($databox)->initOrUpdateLogDocsFromWorker($record, $databox, $workerRunningJob, $subdefName, \Session_Logger::EVENT_SUBDEFCREATION, new \DateTime('now'), WorkerRunningJob::FINISHED); } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/SubtitleWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/SubtitleWorker.php index 8119f509a1..74be8f339c 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/SubtitleWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/SubtitleWorker.php @@ -38,13 +38,19 @@ class SubtitleWorker implements WorkerInterface private $workerRunningJob; private $transcriptionsId; - public function __construct(WorkerRunningJobRepository $repoWorker, PropertyAccess $conf, callable $appboxLocator, LoggerInterface $logger, EventDispatcherInterface $dispatcher) + /** + * @var MessagePublisher + */ + private $messagePublisher; + + public function __construct(WorkerRunningJobRepository $repoWorker, PropertyAccess $conf, callable $appboxLocator, LoggerInterface $logger, EventDispatcherInterface $dispatcher, $messagePublisher) { $this->repoWorker = $repoWorker; $this->conf = $conf; $this->appboxLocator = $appboxLocator; $this->logger = $logger; $this->dispatcher = $dispatcher; + $this->messagePublisher = $messagePublisher; } public function process(array $payload) @@ -265,7 +271,7 @@ public function process(array $payload) // $this->deleteTranscription($transcriptionId); // } - $this->jobFinished(); + $this->repoWorker->markFinished($this->workerRunningJob->getId(), $this->messagePublisher, MessagePublisher::SUBTITLE_TYPE); return 0; } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/WebhookWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/WebhookWorker.php index 593afbee5b..ed78038f7d 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/WebhookWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/WebhookWorker.php @@ -137,14 +137,7 @@ public function process(array $payload) } if ($workerRunningJob != null) { - $workerRunningJob - ->setStatus(WorkerRunningJob::FINISHED) - ->setFinished(new \DateTime('now')) - ; - - $em->persist($workerRunningJob); - - $em->flush(); + $this->repoWorkerJob->markFinished($workerRunningJob->getId(), $this->messagePublisher, MessagePublisher::WEBHOOK_TYPE); } } } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php index a32ceadce7..a2ddca88b0 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/WriteMetadatasWorker.php @@ -96,7 +96,7 @@ public function process(array $payload) try { $record = $databox->get_record($recordId); } catch (\Exception $e) { - $this->repoWorker->markFinished($workerRunningJobId, "error " . $e->getMessage()); + $this->repoWorker->markFinished($workerRunningJobId, $this->messagePublisher, MessagePublisher::WRITE_METADATAS_TYPE, "error " . $e->getMessage()); return; } @@ -112,7 +112,7 @@ public function process(array $payload) $this->logger->error("Can't write meta on svg file!"); // tell that we have finished to work on this file ("unlock") - $this->repoWorker->markFinished($workerRunningJobId, "Can't write meta on svg file!"); + $this->repoWorker->markFinished($workerRunningJobId, $this->messagePublisher, MessagePublisher::WRITE_METADATAS_TYPE, "Can't write meta on svg file!"); $this->getDataboxLogger($databox)->initOrUpdateLogDocsFromWorker($record, $databox, $workerRunningJob, $subdefName, \Session_Logger::EVENT_WRITEMETADATAS, new \DateTime('now'), WorkerRunningJob::ERROR); @@ -323,7 +323,7 @@ public function process(array $payload) $this->updateJeton($record); // tell that we have finished to work on this file (=unlock) - $this->repoWorker->markFinished($workerRunningJobId, $stopInfo); + $this->repoWorker->markFinished($workerRunningJobId, $this->messagePublisher, MessagePublisher::WRITE_METADATAS_TYPE, $stopInfo); $this->getDataboxLogger($databox)->initOrUpdateLogDocsFromWorker($record, $databox, $workerRunningJob, $subdefName, \Session_Logger::EVENT_WRITEMETADATAS, new \DateTime('now'), WorkerRunningJob::ERROR); } return ; @@ -333,7 +333,7 @@ public function process(array $payload) $this->updateJeton($record); // tell that we have finished to work on this file (=unlock) - $this->repoWorker->markFinished($workerRunningJobId); + $this->repoWorker->markFinished($workerRunningJobId, $this->messagePublisher, MessagePublisher::WRITE_METADATAS_TYPE); $this->getDataboxLogger($databox)->initOrUpdateLogDocsFromWorker($record, $databox, $workerRunningJob, $subdefName, \Session_Logger::EVENT_WRITEMETADATAS, new \DateTime('now'), WorkerRunningJob::FINISHED); }