From 4f2fadc5d53163a7527b9ea8f578b3a6f21afd8c Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Wed, 1 May 2019 18:22:58 +0300 Subject: [PATCH 001/103] PLAT-9784:adding whitelist of allowed from_emails on partner info --- alpha/lib/model/Partner.php | 13 +++++++++++++ .../lib/model/EmailNotificationTemplate.php | 16 +++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/alpha/lib/model/Partner.php b/alpha/lib/model/Partner.php index 8c24d6cbded..6e64a1dae3d 100644 --- a/alpha/lib/model/Partner.php +++ b/alpha/lib/model/Partner.php @@ -82,6 +82,8 @@ class Partner extends BasePartner const RTC_SERVER_NODE_ENV = 'rtc_server_node_env'; + const CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST = 'allowedFromEmailWhiteList'; + private $cdnWhiteListCache = array(); public function save(PropelPDO $con = null) @@ -199,6 +201,17 @@ public function setPartnerStats( $v) private static $s_config_params = array ( ); + public function getAllowedFromEmailWhiteList() + { + return $this->getFromCustomData( self::CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST, null, array()); + } + public function addAllowedFromEmailWhiteList( $email ) + { + $allowedFromEmailsArr = $this->getFromCustomData( self::CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST, null, array()); + $allowedFromEmailsArr[] = $email; + $this->putInCustomData( self::CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST, $allowedFromEmailsArr); + } + public function getUseDefaultKshow() { return $this->getFromCustomData( "useDefaultKshow" , null , true ); } public function setUseDefaultKshow( $v ) { return $this->putInCustomData( "useDefaultKshow", $v ); } diff --git a/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php b/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php index 28af55aa208..bf1ff4d85ed 100644 --- a/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php +++ b/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php @@ -45,7 +45,21 @@ protected function getJobData(kScope $scope = null) { $jobData = new kEmailNotificationDispatchJobData(); $jobData->setTemplateId($this->getId()); - $jobData->setFromEmail($this->getFromEmail()); + + $email = $this->getFromEmail(); + $partner = PartnerPeer::retrieveByPK($this->getPartnerId()); + $allowedFromEmailWhiteList = $partner->getAllowedFromEmailWhiteList(); + if (in_array($email, $allowedFromEmailWhiteList)) + { + $jobData->setFromEmail($email); + } + else + { + $allowedFromEmailWhiteListStr = implode(',', $allowedFromEmailWhiteList); + KalturaLog::info("from_email requested: $email is not allowed in the partner whitelist: ". $allowedFromEmailWhiteListStr); + $jobData->setFromEmail(kConf::get("partner_notification_email")); + } + $jobData->setFromName($this->getFromName()); $jobData->setPriority($this->getPriority()); $jobData->setConfirmReadingTo($this->getConfirmReadingTo()); From 11ce41e8be23960cde471137cc78d331064e36ec Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Thu, 2 May 2019 02:55:11 +0300 Subject: [PATCH 002/103] PLAT-9784:Admin console -add partner configuration for white list from emails, and adding whitelist of allowed from_emails on partner info --- admin_console/forms/PartnerConfiguration.php | 7 ++++- alpha/lib/model/Partner.php | 9 +++--- api_v3/lib/types/partner/KalturaPartner.php | 7 ++++- .../api/KalturaSystemPartnerConfiguration.php | 6 ++++ .../lib/model/EmailNotificationTemplate.php | 28 +++++++++++-------- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/admin_console/forms/PartnerConfiguration.php b/admin_console/forms/PartnerConfiguration.php index 424c0760a32..2613882dff3 100644 --- a/admin_console/forms/PartnerConfiguration.php +++ b/admin_console/forms/PartnerConfiguration.php @@ -274,6 +274,11 @@ public function init() 'label' => 'Notification Configuration:', 'filters' => array('StringTrim'), )); + + $this->addElement('text', 'allowed_from_email_white_list', array( + 'label' => 'Allowed From Email WhiteList Notifications:', + 'filters' => array('StringTrim'), + )); $this->addElement('checkbox', 'allow_multi_notification', array( 'label' => 'Allow multi-notifications', @@ -908,7 +913,7 @@ public function addAllDisplayGroups($permissionNames) $this->addDisplayGroup(array_merge(array('delivery_profile_ids', 'live_delivery_profile_ids', 'deliveryFormat', 'delivery_profile_type', 'editDeliveryProfiles', 'enforce_delivery', 'checkbox_host', 'host', 'checkbox_cdn_host', 'cdn_host', 'checkbox_thumbnail_host', 'thumbnail_host', 'checkbox_cache_flavor_version', 'cache_flavor_version', 'support_animated_thumbnails'), $permissionNames[self::GROUP_PUBLISHER_DELIVERY_SETTINGS], array ('crossLine')), 'publisherSpecificDeliverySettings', array('legend' => 'Publisher Specific Delivery Settings')); $this->addDisplayGroup(array_merge(array('storage_serve_priority', 'storage_delete_from_kaltura','import_remote_source_for_convert'), $permissionNames[self::GROUP_REMOTE_STORAGE] ,array('crossLine')), 'remoteStorageAccountPolicy', array('legend' => 'Remote Storage Policy')); - $this->addDisplayGroup(array_merge(array('notifications_config', 'allow_multi_notification'), $permissionNames[self::GROUP_NOTIFICATION_CONFIG] ,array('crossLine')), 'advancedNotificationSettings', array('legend' => 'Advanced Notification Settings')); + $this->addDisplayGroup(array_merge(array('notifications_config', 'allowed_from_email_white_list', 'allow_multi_notification'), $permissionNames[self::GROUP_NOTIFICATION_CONFIG] ,array('crossLine')), 'advancedNotificationSettings', array('legend' => 'Advanced Notification Settings')); $this->addDisplayGroup(array_merge(array('def_thumb_offset','def_thumb_density') , $permissionNames[self::GROUP_CONTENT_INGESTION_OPTIONS], array('enable_bulk_upload_notifications_emails', 'bulk_upload_notifications_email', 'crossLine')), 'publisherSpecificIngestionSettings', array('legend' => 'Content Ingestion Options')); $this->addDisplayGroup(array('logout_url', 'crossLine'), 'signSignOn', array('legend' => 'Sign Sign On')); $this->addDisplayGroup(array_merge(array('api_access_control_id', 'restrict_entry_by_metadata'), $permissionNames[self::GROUP_ACCESS_CONTROL], array('crossLine')), 'apiAccessControlIdGroup', array('legend' => 'Access Control')); diff --git a/alpha/lib/model/Partner.php b/alpha/lib/model/Partner.php index 6e64a1dae3d..845da3beff2 100644 --- a/alpha/lib/model/Partner.php +++ b/alpha/lib/model/Partner.php @@ -203,13 +203,12 @@ public function setPartnerStats( $v) public function getAllowedFromEmailWhiteList() { - return $this->getFromCustomData( self::CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST, null, array()); + return $this->getFromCustomData( self::CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST); } - public function addAllowedFromEmailWhiteList( $email ) + + public function setAllowedFromEmailWhiteList( $emails ) { - $allowedFromEmailsArr = $this->getFromCustomData( self::CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST, null, array()); - $allowedFromEmailsArr[] = $email; - $this->putInCustomData( self::CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST, $allowedFromEmailsArr); + $this->putInCustomData( self::CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST, $emails); } public function getUseDefaultKshow() { return $this->getFromCustomData( "useDefaultKshow" , null , true ); } diff --git a/api_v3/lib/types/partner/KalturaPartner.php b/api_v3/lib/types/partner/KalturaPartner.php index d778e4148ff..52453243f50 100644 --- a/api_v3/lib/types/partner/KalturaPartner.php +++ b/api_v3/lib/types/partner/KalturaPartner.php @@ -130,6 +130,11 @@ class KalturaPartner extends KalturaObject implements IFilterable * @var string */ public $notificationsConfig; + + /** + * @var string + */ + public $allowedFromEmailWhiteList; /** * @var int @@ -341,7 +346,7 @@ class KalturaPartner extends KalturaObject implements IFilterable ( 'id' , 'name', 'website' => 'url1' , 'notificationUrl' => 'url2' , 'appearInSearch' , 'createdAt' , 'adminName' , 'adminEmail' , 'description' , 'commercialUse' , 'landingPage' , 'userLandingPage' , 'contentCategories' , 'type' , 'phone' , 'describeYourself' , - 'adultContent' , 'defConversionProfileType' , 'notify' , 'status' , 'allowQuickEdit' , 'mergeEntryLists' , 'notificationsConfig' , + 'adultContent' , 'defConversionProfileType' , 'notify' , 'status' , 'allowQuickEdit' , 'mergeEntryLists' , 'notificationsConfig' , 'allowedFromEmailWhiteList', 'maxUploadSize' , 'partnerPackage' , 'secret' , 'adminSecret' , 'allowMultiNotification', 'adminLoginUsersQuota', 'adminUserId', 'firstName' , 'lastName' , 'country' , 'state' , 'publishersQuota', 'partnerGroupType', 'defaultEntitlementEnforcement', 'defaultDeliveryType', 'defaultEmbedCodeType', 'deliveryTypes', 'embedCodeTypes', 'templatePartnerId', 'ignoreSeoLinks', diff --git a/plugins/admin_console/system_partner/lib/api/KalturaSystemPartnerConfiguration.php b/plugins/admin_console/system_partner/lib/api/KalturaSystemPartnerConfiguration.php index fc235534e1b..fe00474c0a5 100644 --- a/plugins/admin_console/system_partner/lib/api/KalturaSystemPartnerConfiguration.php +++ b/plugins/admin_console/system_partner/lib/api/KalturaSystemPartnerConfiguration.php @@ -132,6 +132,11 @@ class KalturaSystemPartnerConfiguration extends KalturaObject * @var string */ public $notificationsConfig; + + /** + * @var string + */ + public $allowedFromEmailWhiteList; /** * @var bool @@ -406,6 +411,7 @@ class KalturaSystemPartnerConfiguration extends KalturaObject "alwaysAllowedPermissionNames", "importRemoteSourceForConvert", "notificationsConfig", + "allowedFromEmailWhiteList", "allowMultiNotification", //"maxLoginAttempts", "loginBlockPeriod", diff --git a/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php b/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php index bf1ff4d85ed..c27483e46c8 100644 --- a/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php +++ b/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php @@ -38,28 +38,32 @@ public function copy($deepCopy = false) return $returnObj; } - /* (non-PHPdoc) - * @see BatchEventNotificationTemplate::getJobData() - */ - protected function getJobData(kScope $scope = null) + protected function getEmail() { - $jobData = new kEmailNotificationDispatchJobData(); - $jobData->setTemplateId($this->getId()); - $email = $this->getFromEmail(); + $partnerNotificationEmail = "{from_email}"; $partner = PartnerPeer::retrieveByPK($this->getPartnerId()); $allowedFromEmailWhiteList = $partner->getAllowedFromEmailWhiteList(); - if (in_array($email, $allowedFromEmailWhiteList)) + if ($email != $partnerNotificationEmail && in_array($email, array_map('trim',explode(',',$allowedFromEmailWhiteList)))) { - $jobData->setFromEmail($email); + KalturaLog::info("from_email requested: $email is allowed in the partner from_email whitelist: ".$allowedFromEmailWhiteList ); + return $email; } else { - $allowedFromEmailWhiteListStr = implode(',', $allowedFromEmailWhiteList); - KalturaLog::info("from_email requested: $email is not allowed in the partner whitelist: ". $allowedFromEmailWhiteListStr); - $jobData->setFromEmail(kConf::get("partner_notification_email")); + KalturaLog::info("from_email requested is: $email , partner from_email whitelist: ".$allowedFromEmailWhiteList ); + return $partnerNotificationEmail; } + } + /* (non-PHPdoc) + * @see BatchEventNotificationTemplate::getJobData() + */ + protected function getJobData(kScope $scope = null) + { + $jobData = new kEmailNotificationDispatchJobData(); + $jobData->setTemplateId($this->getId()); + $jobData->setFromEmail($this->getEmail()); $jobData->setFromName($this->getFromName()); $jobData->setPriority($this->getPriority()); $jobData->setConfirmReadingTo($this->getConfirmReadingTo()); From 07eec492491cc9b1f2a7bf3e22142db76bcb6d66 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 21 May 2019 16:19:41 +0300 Subject: [PATCH 003/103] PLAT-9879:add storage support for tumbnails --- infra/storage/kFile.class.php | 1108 ++++++++--------- .../model/thumbStorage/kThumbStorageBase.php | 79 ++ .../thumbStorage/kThumbStorageFactory.php | 27 + .../thumbStorage/kThumbStorageInterface.php | 11 + .../model/thumbStorage/kThumbStorageLocal.php | 29 + .../model/thumbStorage/kThumbStorageNone.php | 23 + .../model/thumbStorage/kThumbStorageS3.php | 72 ++ .../model/thumbStorage/kThumbStorageType.php | 12 + .../thumbnail/services/ThumbnailService.php | 35 +- 9 files changed, 820 insertions(+), 576 deletions(-) create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageInterface.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php diff --git a/infra/storage/kFile.class.php b/infra/storage/kFile.class.php index 77dba2f21c0..f98da105990 100644 --- a/infra/storage/kFile.class.php +++ b/infra/storage/kFile.class.php @@ -1,554 +1,554 @@ - 10) - { - // exceeded the recursion depth - return NULL; - } - - $depth ++; - - // create an array to hold directory list - $results = array(); - // create a handler for the directory - $handler = @opendir($directory); - if(! $handler) - return NULL; - - // echo ( "directory: " .$directory . "
" ); - - - $current_path = pathinfo($directory, PATHINFO_DIRNAME) . "/" . pathinfo($directory, PATHINFO_BASENAME) . "/"; - // keep going until all files in directory have been read - while(($file = readdir($handler)) != NULL) - { - // if $file isn't this directory or its parent, - // add it to the results array - if($file != '.' && $file != '..') - { - $match = 1; - if($file_pattern != NULL) - { - $match = preg_match($file_pattern, $file); - } - - if($match > 0) - { - $results[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file; - if($max_results > 1 && count($results) > $max_results) - return $results; - } - - if($should_recurse && is_dir($current_path . $file)) - { - // echo "Recursing... [$should_recurse] [$current_path $file]
"; - - - $child_dir_results = self::recursiveDirList($current_path . $file, $return_directory_as_prefix, $should_recurse, $file_pattern, $depth); - if($child_dir_results) - { - $results = kArray::append($results, $child_dir_results); - } - } - } - } - // tidy up: close the handler - closedir($handler); - - // done! - return $results; - } - - /** - * the result is a list of tuples - files_name , files_size - */ - // TODO - implement recursion - static public function dirListExtended($directory, $return_directory_as_prefix = true, $should_recurse = false, $file_pattern = NULL, $depth = 0, $fetch_content = false) - { - if($depth > 10) - { - // exceeded the recursion depth - return NULL; - } - - // create an array to hold directory list - $results = array(); - - // create a handler for the directory - $handler = @opendir($directory); - if(! $handler) - return NULL; - - // echo ( "directory: " .$directory . "
" ); - - - $current_path = pathinfo($directory, PATHINFO_DIRNAME) . "/" . pathinfo($directory, PATHINFO_BASENAME) . "/"; - // keep going until all files in directory have been read - while(($file = readdir($handler)) != NULL) - { - - // if $file isn't this directory or its parent, - // add it to the results array - if($file != '.' && $file != '..') - { - if(! $file_pattern) - $match = 1; - else - $match = preg_match($file_pattern, $file); - - if($match > 0) - { - $file_full_path = $directory . "/" . $file; - $result = array(); - // first - name (with or without the full path) - $result[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file; - // second - size - $result[] = self::fileSize($file_full_path); - // third - time - $result[] = filemtime($file_full_path); - // forth - content (only if requested - if($fetch_content) - $result[] = file_get_contents($file_full_path); - $results[] = $result; - } - - if($should_recurse && is_dir($current_path . $file)) - { - // echo "Recursing... [$should_recurse] [$current_path $file]
"; - - - $child_dir_results = self::dirListExtended($current_path . $file, $return_directory_as_prefix, $should_recurse, $file_pattern, ++ $depth); - if($child_dir_results) - { - $results = kArray::append($results, $child_dir_results); - } - } - } - } - - // tidy up: close the handler - closedir($handler); - - // done! - return $results; - } - - /** - * copies src to destination. - * Doesn't support non-flat directories! - * One can't use rename because rename isn't supported between partitions. - */ - private static function copyRecursively($src, $dest, $deleteSrc = false) { - if (is_dir($src)) { - - // Generate target directory - if (file_exists ($dest)) { - if (! is_dir($dest)) { - KalturaLog::err("Can't override a file with a directory [$dest]"); - return false; - } - } else { - if (! mkdir($dest)) { - KalturaLog::err("Failed to create directory [$dest]"); - return false; - } - } - - // Copy files - $dir = dir($src); - while ( false !== $entry = $dir->read () ) { - if ($entry == '.' || $entry == '..') { - continue; - } - - $newSrc = $src . DIRECTORY_SEPARATOR . $entry; - if(is_dir($newSrc)) { - KalturaLog::err("Copying of non-flat directroeis is illegal"); - return false; - } - - $res = self::copySingleFile ($newSrc, $dest . DIRECTORY_SEPARATOR . $entry , $deleteSrc); - if (! $res) - return false; - } - - // Delete source - if ($deleteSrc && (! rmdir($src))) { - KalturaLog::err("Failed to delete source directory : [$src]"); - return false; - } - } else { - self::copySingleFile($src, $dest, $deleteSrc); - } - return true; - } - - private static function copySingleFile($src, $dest, $deleteSrc) { - if($deleteSrc) { - // In case of move, first try to move the file before copy & unlink. - $startTime = microtime(true); - if(rename($src, $dest)) - { - KalturaLog::log("rename took : ".(microtime(true) - $startTime)." [$src] to [$dest] size: ".filesize($dest)); - return true; - } - - KalturaLog::err("Failed to rename file : [$src] to [$dest]"); - } - - if (!copy($src,$dest)) { - KalturaLog::err("Failed to copy file : [$src] to [$dest]"); - return false; - } - if ($deleteSrc && (!unlink($src))) { - KalturaLog::err("Failed to delete source file : [$src]"); - return false; - } - return true; - } - - public static function moveFile($from, $to, $override_if_exists = false, $copy = false) - { - $from = str_replace("\\", "/", $from); - $to = str_replace("\\", "/", $to); - - // Validation - if(!file_exists($from)) - { - KalturaLog::err("Source doesn't exist [$from]"); - return false; - } - - if(strpos($to,'\"') !== false) - { - KalturaLog::err("Illegal destination file [$to]"); - return false; - } - - // Preperation - if($override_if_exists && is_file($to)) - { - self::deleteFile($to); - } - - if(! is_dir(dirname($to))) - { - self::fullMkdir($to); - } - - // Copy - return self::copyRecursively($from,$to, !$copy); - } - - public static function linkFile($from, $to, $overrideIfExists = false, $copyIfLinkFailed = true) - { - $from = str_replace("\\", "/", $from); - $to = str_replace("\\", "/", $to); - - if($overrideIfExists && (is_file($to) || is_link($to))) - { - self::deleteFile($to); - } - - if(! is_dir(dirname($to))) - { - self::fullMkdir($to); - } - - if(!file_exists($from)) - { - KalturaLog::err("Source file doesn't exist [$from]"); - return false; - } - - if(strpos($to,'\"') !== false) - { - KalturaLog::err("Illegal destination file [$to]"); - return false; - } - - if(symlink($from, $to)) - return true; - - $out_arr = array(); - $rv = 0; - exec("ln -s \"$from\" \"$to\"", $out_arr, $rv); -// echo "RV($rv)\n"; - if($rv==0) - return true; - - if(!$copyIfLinkFailed) - return false; - - return self::moveFile($from, $to, $overrideIfExists, true); - } - - // - // downloadHeader - 1 only body, 2 - only header, 3 - both body and header - // - static public function downloadUrlToString($sourceUrl, $downloadHeader = 1, $extraHeaders = null) - { - // create a new curl resource - // TODO - remove this hack !!! - // I added it only because for some reason or other I couldn't get hold of the http_get - /* - if ( function_exists ('http_get')) - { - return http_get($sourceUrl, array('redirect' => 5)); - - } - else - */ - { - $ch = curl_init(); - - // set URL and other appropriate options - curl_setopt($ch, CURLOPT_URL, $sourceUrl); - curl_setopt($ch, CURLOPT_USERAGENT, "curl/7.11.1"); - curl_setopt($ch, CURLOPT_HEADER, ($downloadHeader & 2) ? 1 : 0); - curl_setopt($ch, CURLOPT_NOBODY, ($downloadHeader & 1) ? 0 : 1); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE); - if($extraHeaders) - curl_setopt($ch, CURLOPT_HTTPHEADER, $extraHeaders); - - // grab URL and pass it to the browser - $content = curl_exec($ch); - - // close curl resource, and free up system resources - curl_close($ch); - - } - return $content; - } - - public static function getFileData($file_full_path) - { - return new kFileData($file_full_path); - } - - public static function getFileDataWithContent($file_full_path) - { - $add_content = (strpos($file_full_path, ".txt") !== false || strpos($file_full_path, ".log") !== false); - - return new kFileData($file_full_path, $add_content); - - } - - - public static function read_header($ch, $string) - { - $length = strlen($string); - - // we shouldnt return a chunked encoded header as we read the whole response and echo it after curl extracts it - if (stripos($string, "Transfer-Encoding: chunked") === FALSE) - { - header($string); - } - - return $length; - } - - public static function read_body($ch, $string) - { - $length = strlen($string); - echo $string; - return $length; - } - - public static function getRequestHeaders() - { - if(function_exists('apache_request_headers')) - return apache_request_headers(); - - foreach($_SERVER as $key => $value) - { - if(substr($key, 0, 5) == "HTTP_") - { - $key = str_replace(" ", "-", ucwords(strtolower(str_replace("_", " ", substr($key, 5))))); - $out[$key] = $value; - } - } - return $out; - } - - public static function cacheRedirect($url) - { - if (function_exists('apc_store')) - { - $protocol = (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on') ? "https" : "http"; - apc_store("redirect-".$protocol.$_SERVER["REQUEST_URI"], $url, 60); - } - } - - public static function closeDbConnections() - { - // close all opened db connetion while we end an action with a long executing operation such as dumping a file. - // this will limit the number of concurrent db connections as dumping a file make take a long time - - try - { - Propel::close(); - } - catch(Exception $e) - { - KalturaLog::err("closeDbConnections: error closing db $e"); - } - } - - /** - * Check if the file is executable - * @param string $path - * - * @return string - */ - public static function getMediaInfoFormat($path) - { - $mediaInfoParser = new KMediaInfoMediaParser($path); - $mediaInfo = $mediaInfoParser->getMediaInfo(); - return $mediaInfo->containerFormat; - } - - /** - * Try to find the file type by running the file cmd and match the output to a pattern - * It will return empty string if no pattern was matched - */ - public static function findFileTypeByFileCmd($filePath) - { - $fileType = ''; - $realPath = realpath($filePath); - if($realPath) - { - $fileBrief = shell_exec('file -b ' . $realPath); - if(kString::beginsWith($fileBrief,self::MO_PATTERN)) - $fileType = 'application/mo'; - } - return $fileType; - } - - public static function getFileDescription($realPath) - { - return shell_exec('file -b '.$realPath); - } - - -} - -/** - * @package infra - * @subpackage Storage - */ -class kFileData -{ - public $exists; - public $full_path = NULL; - public $name = NULL; - public $size = NULL; - public $timestamp = NULL; - public $ext = NULL; - public $content = NULL; - public $raw_timestamp = NULL; - - public function __construct($full_file_path, $add_content = false) - { - //debugUtils::st(); - $this->full_path = realpath($full_file_path); - $this->exists = file_exists($full_file_path); - $this->name = pathinfo($full_file_path, PATHINFO_BASENAME); - $this->ext = pathinfo($full_file_path, PATHINFO_EXTENSION); - - if($this->exists) - { - $this->size = self::fileSize($full_file_path); - $this->raw_timestamp = filectime($full_file_path); - $this->timestamp = date("Y-m-d H:i:s.", $this->raw_timestamp); - - if($add_content) - { - $this->content = file_get_contents($full_file_path); - } - } - } -} + 10) + { + // exceeded the recursion depth + return NULL; + } + + $depth ++; + + // create an array to hold directory list + $results = array(); + // create a handler for the directory + $handler = @opendir($directory); + if(! $handler) + return NULL; + + // echo ( "directory: " .$directory . "
" ); + + + $current_path = pathinfo($directory, PATHINFO_DIRNAME) . "/" . pathinfo($directory, PATHINFO_BASENAME) . "/"; + // keep going until all files in directory have been read + while(($file = readdir($handler)) != NULL) + { + // if $file isn't this directory or its parent, + // add it to the results array + if($file != '.' && $file != '..') + { + $match = 1; + if($file_pattern != NULL) + { + $match = preg_match($file_pattern, $file); + } + + if($match > 0) + { + $results[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file; + if($max_results > 1 && count($results) > $max_results) + return $results; + } + + if($should_recurse && is_dir($current_path . $file)) + { + // echo "Recursing... [$should_recurse] [$current_path $file]
"; + + + $child_dir_results = self::recursiveDirList($current_path . $file, $return_directory_as_prefix, $should_recurse, $file_pattern, $depth); + if($child_dir_results) + { + $results = kArray::append($results, $child_dir_results); + } + } + } + } + // tidy up: close the handler + closedir($handler); + + // done! + return $results; + } + + /** + * the result is a list of tuples - files_name , files_size + */ + // TODO - implement recursion + static public function dirListExtended($directory, $return_directory_as_prefix = true, $should_recurse = false, $file_pattern = NULL, $depth = 0, $fetch_content = false) + { + if($depth > 10) + { + // exceeded the recursion depth + return NULL; + } + + // create an array to hold directory list + $results = array(); + + // create a handler for the directory + $handler = @opendir($directory); + if(! $handler) + return NULL; + + // echo ( "directory: " .$directory . "
" ); + + + $current_path = pathinfo($directory, PATHINFO_DIRNAME) . "/" . pathinfo($directory, PATHINFO_BASENAME) . "/"; + // keep going until all files in directory have been read + while(($file = readdir($handler)) != NULL) + { + + // if $file isn't this directory or its parent, + // add it to the results array + if($file != '.' && $file != '..') + { + if(! $file_pattern) + $match = 1; + else + $match = preg_match($file_pattern, $file); + + if($match > 0) + { + $file_full_path = $directory . "/" . $file; + $result = array(); + // first - name (with or without the full path) + $result[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file; + // second - size + $result[] = self::fileSize($file_full_path); + // third - time + $result[] = filemtime($file_full_path); + // forth - content (only if requested + if($fetch_content) + $result[] = file_get_contents($file_full_path); + $results[] = $result; + } + + if($should_recurse && is_dir($current_path . $file)) + { + // echo "Recursing... [$should_recurse] [$current_path $file]
"; + + + $child_dir_results = self::dirListExtended($current_path . $file, $return_directory_as_prefix, $should_recurse, $file_pattern, ++ $depth); + if($child_dir_results) + { + $results = kArray::append($results, $child_dir_results); + } + } + } + } + + // tidy up: close the handler + closedir($handler); + + // done! + return $results; + } + + /** + * copies src to destination. + * Doesn't support non-flat directories! + * One can't use rename because rename isn't supported between partitions. + */ + private static function copyRecursively($src, $dest, $deleteSrc = false) { + if (is_dir($src)) { + + // Generate target directory + if (file_exists ($dest)) { + if (! is_dir($dest)) { + KalturaLog::err("Can't override a file with a directory [$dest]"); + return false; + } + } else { + if (! mkdir($dest)) { + KalturaLog::err("Failed to create directory [$dest]"); + return false; + } + } + + // Copy files + $dir = dir($src); + while ( false !== $entry = $dir->read () ) { + if ($entry == '.' || $entry == '..') { + continue; + } + + $newSrc = $src . DIRECTORY_SEPARATOR . $entry; + if(is_dir($newSrc)) { + KalturaLog::err("Copying of non-flat directroeis is illegal"); + return false; + } + + $res = self::copySingleFile ($newSrc, $dest . DIRECTORY_SEPARATOR . $entry , $deleteSrc); + if (! $res) + return false; + } + + // Delete source + if ($deleteSrc && (! rmdir($src))) { + KalturaLog::err("Failed to delete source directory : [$src]"); + return false; + } + } else { + self::copySingleFile($src, $dest, $deleteSrc); + } + return true; + } + + private static function copySingleFile($src, $dest, $deleteSrc) { + if($deleteSrc) { + // In case of move, first try to move the file before copy & unlink. + $startTime = microtime(true); + if(rename($src, $dest)) + { + KalturaLog::log("rename took : ".(microtime(true) - $startTime)." [$src] to [$dest] size: ".filesize($dest)); + return true; + } + + KalturaLog::err("Failed to rename file : [$src] to [$dest]"); + } + + if (!copy($src,$dest)) { + KalturaLog::err("Failed to copy file : [$src] to [$dest]"); + return false; + } + if ($deleteSrc && (!unlink($src))) { + KalturaLog::err("Failed to delete source file : [$src]"); + return false; + } + return true; + } + + public static function moveFile($from, $to, $override_if_exists = false, $copy = false) + { + $from = str_replace("\\", "/", $from); + $to = str_replace("\\", "/", $to); + + // Validation + if(!file_exists($from)) + { + KalturaLog::err("Source doesn't exist [$from]"); + return false; + } + + if(strpos($to,'\"') !== false) + { + KalturaLog::err("Illegal destination file [$to]"); + return false; + } + + // Preperation + if($override_if_exists && is_file($to)) + { + self::deleteFile($to); + } + + if(! is_dir(dirname($to))) + { + self::fullMkdir($to); + } + + // Copy + return self::copyRecursively($from,$to, !$copy); + } + + public static function linkFile($from, $to, $overrideIfExists = false, $copyIfLinkFailed = true) + { + $from = str_replace("\\", "/", $from); + $to = str_replace("\\", "/", $to); + + if($overrideIfExists && (is_file($to) || is_link($to))) + { + self::deleteFile($to); + } + + if(! is_dir(dirname($to))) + { + self::fullMkdir($to); + } + + if(!file_exists($from)) + { + KalturaLog::err("Source file doesn't exist [$from]"); + return false; + } + + if(strpos($to,'\"') !== false) + { + KalturaLog::err("Illegal destination file [$to]"); + return false; + } + + if(symlink($from, $to)) + return true; + + $out_arr = array(); + $rv = 0; + exec("ln -s \"$from\" \"$to\"", $out_arr, $rv); +// echo "RV($rv)\n"; + if($rv==0) + return true; + + if(!$copyIfLinkFailed) + return false; + + return self::moveFile($from, $to, $overrideIfExists, true); + } + + // + // downloadHeader - 1 only body, 2 - only header, 3 - both body and header + // + static public function downloadUrlToString($sourceUrl, $downloadHeader = 1, $extraHeaders = null) + { + // create a new curl resource + // TODO - remove this hack !!! + // I added it only because for some reason or other I couldn't get hold of the http_get + /* + if ( function_exists ('http_get')) + { + return http_get($sourceUrl, array('redirect' => 5)); + + } + else + */ + { + $ch = curl_init(); + + // set URL and other appropriate options + curl_setopt($ch, CURLOPT_URL, $sourceUrl); + curl_setopt($ch, CURLOPT_USERAGENT, "curl/7.11.1"); + curl_setopt($ch, CURLOPT_HEADER, ($downloadHeader & 2) ? 1 : 0); + curl_setopt($ch, CURLOPT_NOBODY, ($downloadHeader & 1) ? 0 : 1); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE); + if($extraHeaders) + curl_setopt($ch, CURLOPT_HTTPHEADER, $extraHeaders); + + // grab URL and pass it to the browser + $content = curl_exec($ch); + + // close curl resource, and free up system resources + curl_close($ch); + + } + return $content; + } + + public static function getFileData($file_full_path) + { + return new kFileData($file_full_path); + } + + public static function getFileDataWithContent($file_full_path) + { + $add_content = (strpos($file_full_path, ".txt") !== false || strpos($file_full_path, ".log") !== false); + + return new kFileData($file_full_path, $add_content); + + } + + + public static function read_header($ch, $string) + { + $length = strlen($string); + + // we shouldnt return a chunked encoded header as we read the whole response and echo it after curl extracts it + if (stripos($string, "Transfer-Encoding: chunked") === FALSE) + { + header($string); + } + + return $length; + } + + public static function read_body($ch, $string) + { + $length = strlen($string); + echo $string; + return $length; + } + + public static function getRequestHeaders() + { + if(function_exists('apache_request_headers')) + return apache_request_headers(); + + foreach($_SERVER as $key => $value) + { + if(substr($key, 0, 5) == "HTTP_") + { + $key = str_replace(" ", "-", ucwords(strtolower(str_replace("_", " ", substr($key, 5))))); + $out[$key] = $value; + } + } + return $out; + } + + public static function cacheRedirect($url) + { + if (function_exists('apc_store')) + { + $protocol = (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on') ? "https" : "http"; + apc_store("redirect-".$protocol.$_SERVER["REQUEST_URI"], $url, 60); + } + } + + public static function closeDbConnections() + { + // close all opened db connetion while we end an action with a long executing operation such as dumping a file. + // this will limit the number of concurrent db connections as dumping a file make take a long time + + try + { + Propel::close(); + } + catch(Exception $e) + { + KalturaLog::err("closeDbConnections: error closing db $e"); + } + } + + /** + * Check if the file is executable + * @param string $path + * + * @return string + */ + public static function getMediaInfoFormat($path) + { + $mediaInfoParser = new KMediaInfoMediaParser($path); + $mediaInfo = $mediaInfoParser->getMediaInfo(); + return $mediaInfo->containerFormat; + } + + /** + * Try to find the file type by running the file cmd and match the output to a pattern + * It will return empty string if no pattern was matched + */ + public static function findFileTypeByFileCmd($filePath) + { + $fileType = ''; + $realPath = realpath($filePath); + if($realPath) + { + $fileBrief = shell_exec('file -b ' . $realPath); + if(kString::beginsWith($fileBrief,self::MO_PATTERN)) + $fileType = 'application/mo'; + } + return $fileType; + } + + public static function getFileDescription($realPath) + { + return shell_exec('file -b '.$realPath); + } + + +} + +/** + * @package infra + * @subpackage Storage + */ +class kFileData +{ + public $exists; + public $full_path = NULL; + public $name = NULL; + public $size = NULL; + public $timestamp = NULL; + public $ext = NULL; + public $content = NULL; + public $raw_timestamp = NULL; + + public function __construct($full_file_path, $add_content = false) + { + //debugUtils::st(); + $this->full_path = realpath($full_file_path); + $this->exists = file_exists($full_file_path); + $this->name = pathinfo($full_file_path, PATHINFO_BASENAME); + $this->ext = pathinfo($full_file_path, PATHINFO_EXTENSION); + + if($this->exists) + { + $this->size = kFile::fileSize($full_file_path); + $this->raw_timestamp = filectime($full_file_path); + $this->timestamp = date("Y-m-d H:i:s.", $this->raw_timestamp); + + if($add_content) + { + $this->content = file_get_contents($full_file_path); + } + } + } +} diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php new file mode 100644 index 00000000000..856d1e9f6b5 --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php @@ -0,0 +1,79 @@ +getPrefix() . '/' . $this->getPath($md5) . '/' .$md5. '.jpg'; + return $path; + } + + protected static function init() + { + if (self::$init) + { + return; + } + self::$init = true; + self::$configParams = kConf::get('thumb_storage','local',array()); + if(isset(self::$configParams['type'])) + { + self::$type = self::$configParams['type']; + } + } + public static function getInstance() + { + self::init(); + $storage = kThumbStorageFactory::getInstance(self::$type); + return $storage; + } + + protected abstract function getRenderer(); + + public function render() + { + $renderer = $this->getRenderer(); + $renderer->output(); + KExternalErrors::dieGracefully(); + } +} \ No newline at end of file diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php new file mode 100644 index 00000000000..0ed13ac4ad4 --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php @@ -0,0 +1,27 @@ +getFullPath($fileName); + kFile::fullMkdir($path); + $ret = kFile::safeFilePutContents($path,$content); + $this->fileName = $path; + return $ret; + } + protected function getRenderer() + { + $renderer = kFileUtils::getDumpFileRenderer($this->fileName, self::MIME_TYPE ,self::MAX_AGE); + return $renderer; + } + public function loadFile($url) + { + $path = $this->getFullPath($url); + $fileData = kFile::getFileData($path); + $this->fileName = $path; + return $fileData->exists; + } +} \ No newline at end of file diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php new file mode 100644 index 00000000000..c750d07482b --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php @@ -0,0 +1,23 @@ +url = $url; + $this->content = $content; + } + protected function getRenderer() + { + $renderer = new kRendererString($this->content ,self::MIME_TYPE ); + return $renderer; + } + public function loadFile($path) + { + return false; + } +} \ No newline at end of file diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php new file mode 100644 index 00000000000..2282979d6d8 --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php @@ -0,0 +1,72 @@ +setS3Options(); + $this->s3Mgr = kFileTransferMgr::getInstance(StorageProfileProtocol::S3 ,$options); + } + + protected function setS3Options() + { + $s3Options = array(); + if (isset(self::$configParams[self::CONF_REGION])) + { + $s3Options['s3Region'] = self::$configParams[self::CONF_REGION]; + } + return $s3Options; + } + + protected function login() + { + $this->s3Mgr->login(self::$configParams[self::CONF_URL], + self::$configParams[self::CONF_USER_NAME], + self::$configParams[self::CONF_PASSWORD]); + } + + + public function saveFile($fileName , $content) + { + $this->login(); + $path = $this->getFullPath($fileName); + $this->s3Mgr->mkDir(dirname($path)); + kFile::fullMkdir(self::LOCAL_TMP.$path); + kFile::safeFilePutContents(self::LOCAL_TMP.$path,$content); + try + { + $this->s3Mgr->putFile($path, self::LOCAL_TMP . $path); + } + catch (Exception $e) + { + KalturaLog::debug($e->getMessage()); + } + kFile::deleteFile(self::LOCAL_TMP.$path); + $this->content = $content; + } + protected function getRenderer() + { + $renderer = new kRendererString($this->content ,self::MIME_TYPE ); + return $renderer; + } + public function loadFile($url) + { + $this->login(); + $path = $this->getFullPath($url); + $this->url = self::$configParams[self::CONF_URL].$path; + try + { + $this->content = $this->s3Mgr->getFile($path); + } + catch (Exception $e) + { + return false; + } + return !empty($this->content); + } +} \ No newline at end of file diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php new file mode 100644 index 00000000000..6578f371a8b --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php @@ -0,0 +1,12 @@ +parseUrl(); - $transformation->validate(); - $imagick = $transformation->execute(); - $tempFilePath = self::saveTransformationResult($imagick); - $renderer = kFileUtils::getDumpFileRenderer($tempFilePath, null); - $renderer->output(); - return; - } - - protected function saveTransformationResult($imagick) - { - $dc = kDataCenterMgr::getCurrentDc(); - $id = $dc["id"].'_'.kString::generateStringId(); - $fileName = "{$id}.jpg"; - $tempFilePath = sys_get_temp_dir().DIRECTORY_SEPARATOR . $fileName; - $imagick->setImageFormat('jpg'); - file_put_contents($tempFilePath, $imagick); - return $tempFilePath; + $transformationUrl = $this->getTransformationStringFromUri(); + $transformation = $this->parseUrl($transformationUrl); + $storage = kThumbStorageBase::getInstance(); + if( !$storage->loadFile($transformationUrl) ) + { + $transformation->validate(); + $imagick = $transformation->execute(); + $storage->saveFile($transformationUrl,$imagick); + } + $storage->render(); } - protected function parseUrl() + protected function parseUrl($url) { - $transformParametersString = $this->getTransformationStringFromUri(); $transformation = new imageTransformation(); - $steps = explode(self::IMAGE_TRANSFORMATION_STEPS_DELIMITER, $transformParametersString); + $steps = explode(self::IMAGE_TRANSFORMATION_STEPS_DELIMITER, $url); $stepsCount = count($steps); for ($i = 1; $i < $stepsCount; $i++) { From bc44b0f29e743c2960bdd38574c1f6f9298949c2 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 21 May 2019 16:19:54 +0300 Subject: [PATCH 004/103] PLAT-9879:add storage support for tumbnails --- infra/storage/kFile.class.php | 1108 ++++++++++++++++----------------- 1 file changed, 554 insertions(+), 554 deletions(-) diff --git a/infra/storage/kFile.class.php b/infra/storage/kFile.class.php index f98da105990..a9909f54fe7 100644 --- a/infra/storage/kFile.class.php +++ b/infra/storage/kFile.class.php @@ -1,554 +1,554 @@ - 10) - { - // exceeded the recursion depth - return NULL; - } - - $depth ++; - - // create an array to hold directory list - $results = array(); - // create a handler for the directory - $handler = @opendir($directory); - if(! $handler) - return NULL; - - // echo ( "directory: " .$directory . "
" ); - - - $current_path = pathinfo($directory, PATHINFO_DIRNAME) . "/" . pathinfo($directory, PATHINFO_BASENAME) . "/"; - // keep going until all files in directory have been read - while(($file = readdir($handler)) != NULL) - { - // if $file isn't this directory or its parent, - // add it to the results array - if($file != '.' && $file != '..') - { - $match = 1; - if($file_pattern != NULL) - { - $match = preg_match($file_pattern, $file); - } - - if($match > 0) - { - $results[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file; - if($max_results > 1 && count($results) > $max_results) - return $results; - } - - if($should_recurse && is_dir($current_path . $file)) - { - // echo "Recursing... [$should_recurse] [$current_path $file]
"; - - - $child_dir_results = self::recursiveDirList($current_path . $file, $return_directory_as_prefix, $should_recurse, $file_pattern, $depth); - if($child_dir_results) - { - $results = kArray::append($results, $child_dir_results); - } - } - } - } - // tidy up: close the handler - closedir($handler); - - // done! - return $results; - } - - /** - * the result is a list of tuples - files_name , files_size - */ - // TODO - implement recursion - static public function dirListExtended($directory, $return_directory_as_prefix = true, $should_recurse = false, $file_pattern = NULL, $depth = 0, $fetch_content = false) - { - if($depth > 10) - { - // exceeded the recursion depth - return NULL; - } - - // create an array to hold directory list - $results = array(); - - // create a handler for the directory - $handler = @opendir($directory); - if(! $handler) - return NULL; - - // echo ( "directory: " .$directory . "
" ); - - - $current_path = pathinfo($directory, PATHINFO_DIRNAME) . "/" . pathinfo($directory, PATHINFO_BASENAME) . "/"; - // keep going until all files in directory have been read - while(($file = readdir($handler)) != NULL) - { - - // if $file isn't this directory or its parent, - // add it to the results array - if($file != '.' && $file != '..') - { - if(! $file_pattern) - $match = 1; - else - $match = preg_match($file_pattern, $file); - - if($match > 0) - { - $file_full_path = $directory . "/" . $file; - $result = array(); - // first - name (with or without the full path) - $result[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file; - // second - size - $result[] = self::fileSize($file_full_path); - // third - time - $result[] = filemtime($file_full_path); - // forth - content (only if requested - if($fetch_content) - $result[] = file_get_contents($file_full_path); - $results[] = $result; - } - - if($should_recurse && is_dir($current_path . $file)) - { - // echo "Recursing... [$should_recurse] [$current_path $file]
"; - - - $child_dir_results = self::dirListExtended($current_path . $file, $return_directory_as_prefix, $should_recurse, $file_pattern, ++ $depth); - if($child_dir_results) - { - $results = kArray::append($results, $child_dir_results); - } - } - } - } - - // tidy up: close the handler - closedir($handler); - - // done! - return $results; - } - - /** - * copies src to destination. - * Doesn't support non-flat directories! - * One can't use rename because rename isn't supported between partitions. - */ - private static function copyRecursively($src, $dest, $deleteSrc = false) { - if (is_dir($src)) { - - // Generate target directory - if (file_exists ($dest)) { - if (! is_dir($dest)) { - KalturaLog::err("Can't override a file with a directory [$dest]"); - return false; - } - } else { - if (! mkdir($dest)) { - KalturaLog::err("Failed to create directory [$dest]"); - return false; - } - } - - // Copy files - $dir = dir($src); - while ( false !== $entry = $dir->read () ) { - if ($entry == '.' || $entry == '..') { - continue; - } - - $newSrc = $src . DIRECTORY_SEPARATOR . $entry; - if(is_dir($newSrc)) { - KalturaLog::err("Copying of non-flat directroeis is illegal"); - return false; - } - - $res = self::copySingleFile ($newSrc, $dest . DIRECTORY_SEPARATOR . $entry , $deleteSrc); - if (! $res) - return false; - } - - // Delete source - if ($deleteSrc && (! rmdir($src))) { - KalturaLog::err("Failed to delete source directory : [$src]"); - return false; - } - } else { - self::copySingleFile($src, $dest, $deleteSrc); - } - return true; - } - - private static function copySingleFile($src, $dest, $deleteSrc) { - if($deleteSrc) { - // In case of move, first try to move the file before copy & unlink. - $startTime = microtime(true); - if(rename($src, $dest)) - { - KalturaLog::log("rename took : ".(microtime(true) - $startTime)." [$src] to [$dest] size: ".filesize($dest)); - return true; - } - - KalturaLog::err("Failed to rename file : [$src] to [$dest]"); - } - - if (!copy($src,$dest)) { - KalturaLog::err("Failed to copy file : [$src] to [$dest]"); - return false; - } - if ($deleteSrc && (!unlink($src))) { - KalturaLog::err("Failed to delete source file : [$src]"); - return false; - } - return true; - } - - public static function moveFile($from, $to, $override_if_exists = false, $copy = false) - { - $from = str_replace("\\", "/", $from); - $to = str_replace("\\", "/", $to); - - // Validation - if(!file_exists($from)) - { - KalturaLog::err("Source doesn't exist [$from]"); - return false; - } - - if(strpos($to,'\"') !== false) - { - KalturaLog::err("Illegal destination file [$to]"); - return false; - } - - // Preperation - if($override_if_exists && is_file($to)) - { - self::deleteFile($to); - } - - if(! is_dir(dirname($to))) - { - self::fullMkdir($to); - } - - // Copy - return self::copyRecursively($from,$to, !$copy); - } - - public static function linkFile($from, $to, $overrideIfExists = false, $copyIfLinkFailed = true) - { - $from = str_replace("\\", "/", $from); - $to = str_replace("\\", "/", $to); - - if($overrideIfExists && (is_file($to) || is_link($to))) - { - self::deleteFile($to); - } - - if(! is_dir(dirname($to))) - { - self::fullMkdir($to); - } - - if(!file_exists($from)) - { - KalturaLog::err("Source file doesn't exist [$from]"); - return false; - } - - if(strpos($to,'\"') !== false) - { - KalturaLog::err("Illegal destination file [$to]"); - return false; - } - - if(symlink($from, $to)) - return true; - - $out_arr = array(); - $rv = 0; - exec("ln -s \"$from\" \"$to\"", $out_arr, $rv); -// echo "RV($rv)\n"; - if($rv==0) - return true; - - if(!$copyIfLinkFailed) - return false; - - return self::moveFile($from, $to, $overrideIfExists, true); - } - - // - // downloadHeader - 1 only body, 2 - only header, 3 - both body and header - // - static public function downloadUrlToString($sourceUrl, $downloadHeader = 1, $extraHeaders = null) - { - // create a new curl resource - // TODO - remove this hack !!! - // I added it only because for some reason or other I couldn't get hold of the http_get - /* - if ( function_exists ('http_get')) - { - return http_get($sourceUrl, array('redirect' => 5)); - - } - else - */ - { - $ch = curl_init(); - - // set URL and other appropriate options - curl_setopt($ch, CURLOPT_URL, $sourceUrl); - curl_setopt($ch, CURLOPT_USERAGENT, "curl/7.11.1"); - curl_setopt($ch, CURLOPT_HEADER, ($downloadHeader & 2) ? 1 : 0); - curl_setopt($ch, CURLOPT_NOBODY, ($downloadHeader & 1) ? 0 : 1); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE); - if($extraHeaders) - curl_setopt($ch, CURLOPT_HTTPHEADER, $extraHeaders); - - // grab URL and pass it to the browser - $content = curl_exec($ch); - - // close curl resource, and free up system resources - curl_close($ch); - - } - return $content; - } - - public static function getFileData($file_full_path) - { - return new kFileData($file_full_path); - } - - public static function getFileDataWithContent($file_full_path) - { - $add_content = (strpos($file_full_path, ".txt") !== false || strpos($file_full_path, ".log") !== false); - - return new kFileData($file_full_path, $add_content); - - } - - - public static function read_header($ch, $string) - { - $length = strlen($string); - - // we shouldnt return a chunked encoded header as we read the whole response and echo it after curl extracts it - if (stripos($string, "Transfer-Encoding: chunked") === FALSE) - { - header($string); - } - - return $length; - } - - public static function read_body($ch, $string) - { - $length = strlen($string); - echo $string; - return $length; - } - - public static function getRequestHeaders() - { - if(function_exists('apache_request_headers')) - return apache_request_headers(); - - foreach($_SERVER as $key => $value) - { - if(substr($key, 0, 5) == "HTTP_") - { - $key = str_replace(" ", "-", ucwords(strtolower(str_replace("_", " ", substr($key, 5))))); - $out[$key] = $value; - } - } - return $out; - } - - public static function cacheRedirect($url) - { - if (function_exists('apc_store')) - { - $protocol = (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on') ? "https" : "http"; - apc_store("redirect-".$protocol.$_SERVER["REQUEST_URI"], $url, 60); - } - } - - public static function closeDbConnections() - { - // close all opened db connetion while we end an action with a long executing operation such as dumping a file. - // this will limit the number of concurrent db connections as dumping a file make take a long time - - try - { - Propel::close(); - } - catch(Exception $e) - { - KalturaLog::err("closeDbConnections: error closing db $e"); - } - } - - /** - * Check if the file is executable - * @param string $path - * - * @return string - */ - public static function getMediaInfoFormat($path) - { - $mediaInfoParser = new KMediaInfoMediaParser($path); - $mediaInfo = $mediaInfoParser->getMediaInfo(); - return $mediaInfo->containerFormat; - } - - /** - * Try to find the file type by running the file cmd and match the output to a pattern - * It will return empty string if no pattern was matched - */ - public static function findFileTypeByFileCmd($filePath) - { - $fileType = ''; - $realPath = realpath($filePath); - if($realPath) - { - $fileBrief = shell_exec('file -b ' . $realPath); - if(kString::beginsWith($fileBrief,self::MO_PATTERN)) - $fileType = 'application/mo'; - } - return $fileType; - } - - public static function getFileDescription($realPath) - { - return shell_exec('file -b '.$realPath); - } - - -} - -/** - * @package infra - * @subpackage Storage - */ -class kFileData -{ - public $exists; - public $full_path = NULL; - public $name = NULL; - public $size = NULL; - public $timestamp = NULL; - public $ext = NULL; - public $content = NULL; - public $raw_timestamp = NULL; - - public function __construct($full_file_path, $add_content = false) - { - //debugUtils::st(); - $this->full_path = realpath($full_file_path); - $this->exists = file_exists($full_file_path); - $this->name = pathinfo($full_file_path, PATHINFO_BASENAME); - $this->ext = pathinfo($full_file_path, PATHINFO_EXTENSION); - - if($this->exists) - { - $this->size = kFile::fileSize($full_file_path); - $this->raw_timestamp = filectime($full_file_path); - $this->timestamp = date("Y-m-d H:i:s.", $this->raw_timestamp); - - if($add_content) - { - $this->content = file_get_contents($full_file_path); - } - } - } -} + 10) + { + // exceeded the recursion depth + return NULL; + } + + $depth ++; + + // create an array to hold directory list + $results = array(); + // create a handler for the directory + $handler = @opendir($directory); + if(! $handler) + return NULL; + + // echo ( "directory: " .$directory . "
" ); + + + $current_path = pathinfo($directory, PATHINFO_DIRNAME) . "/" . pathinfo($directory, PATHINFO_BASENAME) . "/"; + // keep going until all files in directory have been read + while(($file = readdir($handler)) != NULL) + { + // if $file isn't this directory or its parent, + // add it to the results array + if($file != '.' && $file != '..') + { + $match = 1; + if($file_pattern != NULL) + { + $match = preg_match($file_pattern, $file); + } + + if($match > 0) + { + $results[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file; + if($max_results > 1 && count($results) > $max_results) + return $results; + } + + if($should_recurse && is_dir($current_path . $file)) + { + // echo "Recursing... [$should_recurse] [$current_path $file]
"; + + + $child_dir_results = self::recursiveDirList($current_path . $file, $return_directory_as_prefix, $should_recurse, $file_pattern, $depth); + if($child_dir_results) + { + $results = kArray::append($results, $child_dir_results); + } + } + } + } + // tidy up: close the handler + closedir($handler); + + // done! + return $results; + } + + /** + * the result is a list of tuples - files_name , files_size + */ + // TODO - implement recursion + static public function dirListExtended($directory, $return_directory_as_prefix = true, $should_recurse = false, $file_pattern = NULL, $depth = 0, $fetch_content = false) + { + if($depth > 10) + { + // exceeded the recursion depth + return NULL; + } + + // create an array to hold directory list + $results = array(); + + // create a handler for the directory + $handler = @opendir($directory); + if(! $handler) + return NULL; + + // echo ( "directory: " .$directory . "
" ); + + + $current_path = pathinfo($directory, PATHINFO_DIRNAME) . "/" . pathinfo($directory, PATHINFO_BASENAME) . "/"; + // keep going until all files in directory have been read + while(($file = readdir($handler)) != NULL) + { + + // if $file isn't this directory or its parent, + // add it to the results array + if($file != '.' && $file != '..') + { + if(! $file_pattern) + $match = 1; + else + $match = preg_match($file_pattern, $file); + + if($match > 0) + { + $file_full_path = $directory . "/" . $file; + $result = array(); + // first - name (with or without the full path) + $result[] = ($return_directory_as_prefix ? $directory . "/" : "") . $file; + // second - size + $result[] = self::fileSize($file_full_path); + // third - time + $result[] = filemtime($file_full_path); + // forth - content (only if requested + if($fetch_content) + $result[] = file_get_contents($file_full_path); + $results[] = $result; + } + + if($should_recurse && is_dir($current_path . $file)) + { + // echo "Recursing... [$should_recurse] [$current_path $file]
"; + + + $child_dir_results = self::dirListExtended($current_path . $file, $return_directory_as_prefix, $should_recurse, $file_pattern, ++ $depth); + if($child_dir_results) + { + $results = kArray::append($results, $child_dir_results); + } + } + } + } + + // tidy up: close the handler + closedir($handler); + + // done! + return $results; + } + + /** + * copies src to destination. + * Doesn't support non-flat directories! + * One can't use rename because rename isn't supported between partitions. + */ + private static function copyRecursively($src, $dest, $deleteSrc = false) { + if (is_dir($src)) { + + // Generate target directory + if (file_exists ($dest)) { + if (! is_dir($dest)) { + KalturaLog::err("Can't override a file with a directory [$dest]"); + return false; + } + } else { + if (! mkdir($dest)) { + KalturaLog::err("Failed to create directory [$dest]"); + return false; + } + } + + // Copy files + $dir = dir($src); + while ( false !== $entry = $dir->read () ) { + if ($entry == '.' || $entry == '..') { + continue; + } + + $newSrc = $src . DIRECTORY_SEPARATOR . $entry; + if(is_dir($newSrc)) { + KalturaLog::err("Copying of non-flat directroeis is illegal"); + return false; + } + + $res = self::copySingleFile ($newSrc, $dest . DIRECTORY_SEPARATOR . $entry , $deleteSrc); + if (! $res) + return false; + } + + // Delete source + if ($deleteSrc && (! rmdir($src))) { + KalturaLog::err("Failed to delete source directory : [$src]"); + return false; + } + } else { + self::copySingleFile($src, $dest, $deleteSrc); + } + return true; + } + + private static function copySingleFile($src, $dest, $deleteSrc) { + if($deleteSrc) { + // In case of move, first try to move the file before copy & unlink. + $startTime = microtime(true); + if(rename($src, $dest)) + { + KalturaLog::log("rename took : ".(microtime(true) - $startTime)." [$src] to [$dest] size: ".filesize($dest)); + return true; + } + + KalturaLog::err("Failed to rename file : [$src] to [$dest]"); + } + + if (!copy($src,$dest)) { + KalturaLog::err("Failed to copy file : [$src] to [$dest]"); + return false; + } + if ($deleteSrc && (!unlink($src))) { + KalturaLog::err("Failed to delete source file : [$src]"); + return false; + } + return true; + } + + public static function moveFile($from, $to, $override_if_exists = false, $copy = false) + { + $from = str_replace("\\", "/", $from); + $to = str_replace("\\", "/", $to); + + // Validation + if(!file_exists($from)) + { + KalturaLog::err("Source doesn't exist [$from]"); + return false; + } + + if(strpos($to,'\"') !== false) + { + KalturaLog::err("Illegal destination file [$to]"); + return false; + } + + // Preperation + if($override_if_exists && is_file($to)) + { + self::deleteFile($to); + } + + if(! is_dir(dirname($to))) + { + self::fullMkdir($to); + } + + // Copy + return self::copyRecursively($from,$to, !$copy); + } + + public static function linkFile($from, $to, $overrideIfExists = false, $copyIfLinkFailed = true) + { + $from = str_replace("\\", "/", $from); + $to = str_replace("\\", "/", $to); + + if($overrideIfExists && (is_file($to) || is_link($to))) + { + self::deleteFile($to); + } + + if(! is_dir(dirname($to))) + { + self::fullMkdir($to); + } + + if(!file_exists($from)) + { + KalturaLog::err("Source file doesn't exist [$from]"); + return false; + } + + if(strpos($to,'\"') !== false) + { + KalturaLog::err("Illegal destination file [$to]"); + return false; + } + + if(symlink($from, $to)) + return true; + + $out_arr = array(); + $rv = 0; + exec("ln -s \"$from\" \"$to\"", $out_arr, $rv); +// echo "RV($rv)\n"; + if($rv==0) + return true; + + if(!$copyIfLinkFailed) + return false; + + return self::moveFile($from, $to, $overrideIfExists, true); + } + + // + // downloadHeader - 1 only body, 2 - only header, 3 - both body and header + // + static public function downloadUrlToString($sourceUrl, $downloadHeader = 1, $extraHeaders = null) + { + // create a new curl resource + // TODO - remove this hack !!! + // I added it only because for some reason or other I couldn't get hold of the http_get + /* + if ( function_exists ('http_get')) + { + return http_get($sourceUrl, array('redirect' => 5)); + + } + else + */ + { + $ch = curl_init(); + + // set URL and other appropriate options + curl_setopt($ch, CURLOPT_URL, $sourceUrl); + curl_setopt($ch, CURLOPT_USERAGENT, "curl/7.11.1"); + curl_setopt($ch, CURLOPT_HEADER, ($downloadHeader & 2) ? 1 : 0); + curl_setopt($ch, CURLOPT_NOBODY, ($downloadHeader & 1) ? 0 : 1); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE); + if($extraHeaders) + curl_setopt($ch, CURLOPT_HTTPHEADER, $extraHeaders); + + // grab URL and pass it to the browser + $content = curl_exec($ch); + + // close curl resource, and free up system resources + curl_close($ch); + + } + return $content; + } + + public static function getFileData($file_full_path) + { + return new kFileData($file_full_path); + } + + public static function getFileDataWithContent($file_full_path) + { + $add_content = (strpos($file_full_path, ".txt") !== false || strpos($file_full_path, ".log") !== false); + + return new kFileData($file_full_path, $add_content); + + } + + + public static function read_header($ch, $string) + { + $length = strlen($string); + + // we shouldnt return a chunked encoded header as we read the whole response and echo it after curl extracts it + if (stripos($string, "Transfer-Encoding: chunked") === FALSE) + { + header($string); + } + + return $length; + } + + public static function read_body($ch, $string) + { + $length = strlen($string); + echo $string; + return $length; + } + + public static function getRequestHeaders() + { + if(function_exists('apache_request_headers')) + return apache_request_headers(); + + foreach($_SERVER as $key => $value) + { + if(substr($key, 0, 5) == "HTTP_") + { + $key = str_replace(" ", "-", ucwords(strtolower(str_replace("_", " ", substr($key, 5))))); + $out[$key] = $value; + } + } + return $out; + } + + public static function cacheRedirect($url) + { + if (function_exists('apc_store')) + { + $protocol = (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on') ? "https" : "http"; + apc_store("redirect-".$protocol.$_SERVER["REQUEST_URI"], $url, 60); + } + } + + public static function closeDbConnections() + { + // close all opened db connetion while we end an action with a long executing operation such as dumping a file. + // this will limit the number of concurrent db connections as dumping a file make take a long time + + try + { + Propel::close(); + } + catch(Exception $e) + { + KalturaLog::err("closeDbConnections: error closing db $e"); + } + } + + /** + * Check if the file is executable + * @param string $path + * + * @return string + */ + public static function getMediaInfoFormat($path) + { + $mediaInfoParser = new KMediaInfoMediaParser($path); + $mediaInfo = $mediaInfoParser->getMediaInfo(); + return $mediaInfo->containerFormat; + } + + /** + * Try to find the file type by running the file cmd and match the output to a pattern + * It will return empty string if no pattern was matched + */ + public static function findFileTypeByFileCmd($filePath) + { + $fileType = ''; + $realPath = realpath($filePath); + if($realPath) + { + $fileBrief = shell_exec('file -b ' . $realPath); + if(kString::beginsWith($fileBrief,self::MO_PATTERN)) + $fileType = 'application/mo'; + } + return $fileType; + } + + public static function getFileDescription($realPath) + { + return shell_exec('file -b '.$realPath); + } + + +} + +/** + * @package infra + * @subpackage Storage + */ +class kFileData +{ + public $exists; + public $full_path = NULL; + public $name = NULL; + public $size = NULL; + public $timestamp = NULL; + public $ext = NULL; + public $content = NULL; + public $raw_timestamp = NULL; + + public function __construct($full_file_path, $add_content = false) + { + //debugUtils::st(); + $this->full_path = realpath($full_file_path); + $this->exists = file_exists($full_file_path); + $this->name = pathinfo($full_file_path, PATHINFO_BASENAME); + $this->ext = pathinfo($full_file_path, PATHINFO_EXTENSION); + + if($this->exists) + { + $this->size = kFile::fileSize($full_file_path); + $this->raw_timestamp = filectime($full_file_path); + $this->timestamp = date("Y-m-d H:i:s.", $this->raw_timestamp); + + if($add_content) + { + $this->content = file_get_contents($full_file_path); + } + } + } +} From bd92c19e54ee2894e013aa13f96e2ec6c7c3ae83 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 21 May 2019 16:25:26 +0300 Subject: [PATCH 005/103] PLAT-9879:add storage support for tumbnails --- .../thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php | 6 +++--- .../thumbnail/lib/model/thumbStorage/kThumbStorageNone.php | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php index 9f42d94bff9..57ee526ef9a 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php @@ -6,17 +6,17 @@ class kThumbStorageLocal extends kThumbStorageBase implements kThumbStorageInterface { - public function saveFile($fileName , $content) + public function saveFile($fileName, $content) { $path = $this->getFullPath($fileName); kFile::fullMkdir($path); - $ret = kFile::safeFilePutContents($path,$content); + $ret = kFile::safeFilePutContents($path, $content); $this->fileName = $path; return $ret; } protected function getRenderer() { - $renderer = kFileUtils::getDumpFileRenderer($this->fileName, self::MIME_TYPE ,self::MAX_AGE); + $renderer = kFileUtils::getDumpFileRenderer($this->fileName,self::MIME_TYPE,self::MAX_AGE); return $renderer; } public function loadFile($url) diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php index c750d07482b..2092f20580c 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php @@ -13,7 +13,7 @@ public function saveFile($url,$content) } protected function getRenderer() { - $renderer = new kRendererString($this->content ,self::MIME_TYPE ); + $renderer = new kRendererString($this->content,self::MIME_TYPE); return $renderer; } public function loadFile($path) From 59dcd6459f02fa9b7c8f5f606bdf167aeb3caae0 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 21 May 2019 16:28:58 +0300 Subject: [PATCH 006/103] PLAT-9879:add storage support for tumbnails --- .../lib/model/thumbStorage/kThumbStorageBase.php | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php index 856d1e9f6b5..4b53a1a3efe 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php @@ -17,6 +17,8 @@ abstract class kThumbStorageBase const MIME_TYPE = 'image/jpeg'; const MAX_AGE = 86400; const LOCAL_TMP = '/tmp/'; + const DEFAULT_PATH = 'thumb'; + const CONF_SECTION_NAME = 'storage_path'; const CONF_PATH = 'path'; @@ -24,10 +26,11 @@ abstract class kThumbStorageBase const CONF_PASSWORD = 'password'; const CONF_REGION = 'region'; const CONF_URL = 'url'; + const CONF_TYPE = 'type'; protected function getPrefix() { - $prefix = 'thumb'; + $prefix = self::DEFAULT_PATH; if(isset(self::$configParams[self::CONF_PATH])) { $prefix = self::$configParams[self::CONF_PATH]; @@ -37,7 +40,7 @@ protected function getPrefix() protected function getPath($md5) { - $path = substr($md5,0,3). '/' .substr($md5,3,3); + $path = substr($md5, 0, 3). '/' .substr($md5, 3, 3); return $path; } @@ -55,10 +58,10 @@ protected static function init() return; } self::$init = true; - self::$configParams = kConf::get('thumb_storage','local',array()); - if(isset(self::$configParams['type'])) + self::$configParams = kConf::get(self::CONF_SECTION_NAME, 'local', array()); + if(isset(self::$configParams[self::CONF_TYPE])) { - self::$type = self::$configParams['type']; + self::$type = self::$configParams[self::CONF_TYPE]; } } public static function getInstance() From 79c65b3a319960434965f7e3c80a289789dc5a16 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 21 May 2019 16:37:51 +0300 Subject: [PATCH 007/103] PLAT-9879:add storage support for tumbnails --- .../lib/model/thumbStorage/kThumbStorageBase.php | 1 + .../lib/model/thumbStorage/kThumbStorageLocal.php | 5 +++-- .../lib/model/thumbStorage/kThumbStorageNone.php | 2 ++ .../thumbnail/lib/model/thumbStorage/kThumbStorageS3.php | 8 ++++---- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php index 4b53a1a3efe..a11ecf3a6e5 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php @@ -64,6 +64,7 @@ protected static function init() self::$type = self::$configParams[self::CONF_TYPE]; } } + public static function getInstance() { self::init(); diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php index 57ee526ef9a..674f7cf48df 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php @@ -14,11 +14,12 @@ public function saveFile($fileName, $content) $this->fileName = $path; return $ret; } + protected function getRenderer() { - $renderer = kFileUtils::getDumpFileRenderer($this->fileName,self::MIME_TYPE,self::MAX_AGE); - return $renderer; + return kFileUtils::getDumpFileRenderer($this->fileName,self::MIME_TYPE,self::MAX_AGE); } + public function loadFile($url) { $path = $this->getFullPath($url); diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php index 2092f20580c..0f1e7f36c6d 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php @@ -11,11 +11,13 @@ public function saveFile($url,$content) $this->url = $url; $this->content = $content; } + protected function getRenderer() { $renderer = new kRendererString($this->content,self::MIME_TYPE); return $renderer; } + public function loadFile($path) { return false; diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php index 2282979d6d8..3e537d9d4db 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php @@ -7,6 +7,7 @@ class kThumbStorageS3 extends kThumbStorageBase implements kThumbStorageInterface { protected $s3Mgr; + function __construct() { $options = $this->setS3Options(); @@ -30,12 +31,10 @@ protected function login() self::$configParams[self::CONF_PASSWORD]); } - - public function saveFile($fileName , $content) + public function saveFile($fileName, $content) { $this->login(); $path = $this->getFullPath($fileName); - $this->s3Mgr->mkDir(dirname($path)); kFile::fullMkdir(self::LOCAL_TMP.$path); kFile::safeFilePutContents(self::LOCAL_TMP.$path,$content); try @@ -54,11 +53,12 @@ protected function getRenderer() $renderer = new kRendererString($this->content ,self::MIME_TYPE ); return $renderer; } + public function loadFile($url) { $this->login(); $path = $this->getFullPath($url); - $this->url = self::$configParams[self::CONF_URL].$path; + $this->url = self::$configParams[self::CONF_URL] . $path; try { $this->content = $this->s3Mgr->getFile($path); From 8b67086b760e5d0a7b27abe53b2079a1fef4ad79 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Sat, 25 May 2019 14:34:24 +0300 Subject: [PATCH 008/103] PLAT-9889 --- .../services/ESearchService.php | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/plugins/search/providers/elastic_search/services/ESearchService.php b/plugins/search/providers/elastic_search/services/ESearchService.php index 8fe3aa17a88..441ec5c15fe 100644 --- a/plugins/search/providers/elastic_search/services/ESearchService.php +++ b/plugins/search/providers/elastic_search/services/ESearchService.php @@ -16,10 +16,16 @@ class ESearchService extends KalturaBaseService function searchEntryAction(KalturaESearchEntryParams $searchParams, KalturaPager $pager = null) { $entrySearch = new kEntrySearch(); - list($coreResults, $objectCount) = $this->initAndSearch($entrySearch, $searchParams, $pager); + list($coreResults, $objectCount, $aggregations) = $this->initAndSearch($entrySearch, $searchParams, $pager); $response = new KalturaESearchEntryResponse(); $response->objects = KalturaESearchEntryResultArray::fromDbArray($coreResults, $this->getResponseProfile()); $response->totalCount = $objectCount; + if($aggregations) + { + $aggregationResponse = new KalturaESearchAggregationResponse(); + $aggregationResponse->resultToApi($aggregations); + $response->aggregations = $aggregationResponse->aggs; + } return $response; } @@ -70,7 +76,7 @@ function searchUserAction(KalturaESearchUserParams $searchParams, KalturaPager $ public function entryExportToCsvAction (KalturaMediaEsearchExportToCsvJobData $data) { if(!$data->userName || !$data->userMail) - throw new KalturaAPIException(APIErrors::USER_EMAIL_NOT_FOUND, $kuser); + throw new KalturaAPIException(APIErrors::USER_EMAIL_NOT_FOUND); $kJobdData = $data->toObject(new kMediaEsearchExportToCsvJobData()); @@ -87,12 +93,12 @@ public function entryExportToCsvAction (KalturaMediaEsearchExportToCsvJobData $d */ protected function initAndSearch($coreSearchObject, $searchParams, $pager) { - list($coreSearchOperator, $objectStatusesArr, $objectId, $kPager, $coreOrder) = + list($coreSearchOperator, $objectStatusesArr, $objectId, $kPager, $coreOrder ,$aggregations) = self::initSearchActionParams($searchParams, $pager); - $elasticResults = $coreSearchObject->doSearch($coreSearchOperator, $kPager, $objectStatusesArr, $objectId, $coreOrder); + $elasticResults = $coreSearchObject->doSearch($coreSearchOperator, $kPager, $objectStatusesArr, $objectId, $coreOrder, $aggregations); - list($coreResults, $objectCount) = kESearchCoreAdapter::transformElasticToCoreObject($elasticResults, $coreSearchObject); - return array($coreResults, $objectCount); + list($coreResults, $objectCount, $aggregationsResult) = kESearchCoreAdapter::transformElasticToCoreObject($elasticResults, $coreSearchObject); + return array($coreResults, $objectCount, $aggregationsResult); } protected static function initSearchActionParams($searchParams, KalturaPager $pager = null) @@ -115,7 +121,7 @@ protected static function initSearchActionParams($searchParams, KalturaPager $pa $kPager = $pager->toObject(); } - return array($coreParams->getSearchOperator(), $objectStatusesArr, $coreParams->getObjectId(), $kPager, $coreParams->getOrderBy()); + return array($coreParams->getSearchOperator(), $objectStatusesArr, $coreParams->getObjectId(), $kPager, $coreParams->getOrderBy(), $coreParams->getAggregations()); } } \ No newline at end of file From ce58b8fec4c22fb8130c6bd70873a6084fdce60b Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Sat, 25 May 2019 14:36:23 +0300 Subject: [PATCH 009/103] PLAT-9889 --- .../lib/api/types/KalturaESearchParams.php | 122 +++++++++--------- .../KalturaESearchAggregation.php | 29 +++++ .../items/KalturaESearchAggregationItem.php | 38 ++++++ .../items/KalturaESearchAggregationsArray.php | 54 ++++++++ .../KalturaESearchCategoryAggregationItem.php | 32 +++++ ...KalturaESearchCuepointsAggregationItem.php | 33 +++++ .../KalturaESearchEntryAggregationItem.php | 32 +++++ .../KalturaESearchMetadataAggregationItem.php | 32 +++++ .../KalturaESearchAggregationBucket.php | 28 ++++ .../KalturaESearchAggregationBucketsArray.php | 12 ++ .../KalturaESearchAggregationResponse.php | 50 +++++++ ...KalturaESearchAggregationResponseArray.php | 26 ++++ .../KalturaESearchAggregationResponseItem.php | 22 ++++ .../aggregations/ESearchAggregations.php | 29 +++++ .../items/ESearchAggregationItem.php | 59 +++++++++ .../items/ESearchCategoryAggregationItem.php | 27 ++++ .../items/ESearchCuepointsAggregationItem.php | 31 +++++ .../items/ESearchEntryAggregationItem.php | 27 ++++ .../items/ESearchMetadataAggregationItem.php | 25 ++++ 19 files changed, 650 insertions(+), 58 deletions(-) create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/KalturaESearchAggregation.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationsArray.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucket.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseArray.php create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseItem.php create mode 100644 plugins/search/providers/elastic_search/lib/model/aggregations/ESearchAggregations.php create mode 100644 plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php create mode 100644 plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCategoryAggregationItem.php create mode 100644 plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php create mode 100644 plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php create mode 100644 plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchMetadataAggregationItem.php diff --git a/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php b/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php index 451a08164ba..6e31068c2fe 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php +++ b/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php @@ -1,58 +1,64 @@ -operator) - { - $searchOperator->operator = KalturaSearchOperatorType::SEARCH_AND; - } - } - -} +operator) + { + $searchOperator->operator = KalturaSearchOperatorType::SEARCH_AND; + } + } + +} diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/KalturaESearchAggregation.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/KalturaESearchAggregation.php new file mode 100644 index 00000000000..8ddd48e346e --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/KalturaESearchAggregation.php @@ -0,0 +1,29 @@ +getFieldEnumMap(); + if(isset($fieldEnumMap[$this->fieldName])) + { + $coreFieldName = $fieldEnumMap[$this->fieldName]; + $object_to_fill->setFieldName($coreFieldName); + $props_to_skip[] = 'fieldName'; + } + + return parent::toObject($object_to_fill, $props_to_skip); + } +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationsArray.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationsArray.php new file mode 100644 index 00000000000..7bbb83af295 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationsArray.php @@ -0,0 +1,54 @@ +fromObject($obj, $responseProfile); + $newArr[] = $nObj; + } + + return $newArr; + } + +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php new file mode 100644 index 00000000000..c0a0090f748 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php @@ -0,0 +1,32 @@ + ESearchCategoryAggregationFieldName::CATEGORY_NAME); + } + + public function coreToApiResponse($aggregation) + { + + } + +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php new file mode 100644 index 00000000000..24c4001cdf5 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php @@ -0,0 +1,33 @@ + ESearchCuePointsAggregationFieldName::TAGS, + KalturaESearchCuePointAggregateByFieldName::TYPE => ESearchCuePointsAggregationFieldName::TYPE); + } + + public function coreToApiResponse($aggregation) + { + + } + +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php new file mode 100644 index 00000000000..87f631da349 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php @@ -0,0 +1,32 @@ + ESearchEntryAggregationFieldName::ENTRY_TYPE, + KalturaESearchEntryAggregateByFieldName::MEDIA_TYPE => ESearchEntryAggregationFieldName::MEDIA_TYPE, + KalturaESearchEntryAggregateByFieldName::TAGS => ESearchEntryAggregationFieldName::TAGS, + KalturaESearchEntryAggregateByFieldName::ACCESS_CONTROL_PROFILE => ESearchEntryAggregationFieldName::ACCESS_CONTROL_PROFILE); + } +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php new file mode 100644 index 00000000000..5f1c2262055 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -0,0 +1,32 @@ +'key', + 'count'=>'doc_count' + ); + + protected function getMapBetweenObjects() + { + return array_merge(parent::getMapBetweenObjects(), self::$map_between_objects); + } +} diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php new file mode 100644 index 00000000000..3268607d86b --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php @@ -0,0 +1,12 @@ +$response) + { + list ($responseObject, $fieldName) = $this->getApiObjects($key); + $agg = new KalturaESearchAggregationResponseItem(); + $agg->fieldName = $fieldName; + $agg->name = $responseObject; + $buckets=null; + if(isset($response['buckets'])) + { + $buckets = $response['buckets']; + } + elseif (isset($response['NestedBucket'])) + { + $buckets = $response['NestedBucket']['buckets']; + } + if ($buckets) + { + $agg->buckets = new KalturaESearchAggregationBucketsArray(); + foreach ($buckets as $bucket) + { + $reponseBucket = new KalturaESearchAggregationBucket(); + $reponseBucket->fromArray($bucket); + $agg->buckets[] = $reponseBucket; + } + } + $this->aggs[] = $agg; + } + } + +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseArray.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseArray.php new file mode 100644 index 00000000000..01269b7559d --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseArray.php @@ -0,0 +1,26 @@ +fromObject($obj, $responseProfile); + $outputArray[] = $nObj; + } + return $outputArray; + } + + +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseItem.php new file mode 100644 index 00000000000..6987c0f12c2 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseItem.php @@ -0,0 +1,22 @@ +aggregations; + } + + /** + * @param array $aggregations + */ + public function setAggregations($aggregations) + { + $this->aggregations = $aggregations; + } + +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php new file mode 100644 index 00000000000..6f786094079 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php @@ -0,0 +1,59 @@ +size) + { + $this->size = self::DEFAULT_SIZE; + } + return $this->size; + } + + /** + * @param int $size + */ + public function setSize($size) + { + $this->size = $size; + } + + public function getFieldName() + { + return $this->fieldName; + } + + public function setFieldName($fieldName) + { + $this->fieldName = $fieldName; + } + + public abstract function getAggregationCommand(); + + public abstract function getAggregationKey(); + + public function getCommandKey() + { + return $this->getAggregationKey().':'.$this->getFieldName(); + } + + +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCategoryAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCategoryAggregationItem.php new file mode 100644 index 00000000000..1aca52d99c6 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCategoryAggregationItem.php @@ -0,0 +1,27 @@ + + array('field' => $this->fieldName, + 'size' =>$this->getSize())); + } +} diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php new file mode 100644 index 00000000000..32d6c225077 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php @@ -0,0 +1,31 @@ + + array('path' => "cue_points"), + 'aggs'=>array('NestedBucket' => + array('terms' => + array ('field' => $this->getFieldName(), + 'size' =>$this->getSize())))); + } + + public function getAggregationKey() + { + return self::KEY; + } +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php new file mode 100644 index 00000000000..ceaf4d7e521 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php @@ -0,0 +1,27 @@ + + array('field' => $this->fieldName, + 'size' =>$this->getSize())); + } + public function getAggregationKey() + { + return self::KEY; + } + +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchMetadataAggregationItem.php new file mode 100644 index 00000000000..e73db305284 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchMetadataAggregationItem.php @@ -0,0 +1,25 @@ + + array('path' => "metadata"), + 'aggs'=>array('NestedBucket' => + array('terms' => + array ('field' => 'metadata.value_text.raw' , + 'size' =>$this->getSize())))); + } + + public function getAggregationKey() + { + return self::KEY; + } +} \ No newline at end of file From a88dc7ee055d6d37a22e6082b344fe81bdfc8602 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Sat, 25 May 2019 14:37:31 +0300 Subject: [PATCH 010/103] PLAT-9889 --- .../lib/api/types/KalturaESearchParams.php | 128 +++++++++--------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php b/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php index 6e31068c2fe..05c45542edf 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php +++ b/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php @@ -1,64 +1,64 @@ -operator) - { - $searchOperator->operator = KalturaSearchOperatorType::SEARCH_AND; - } - } - -} +operator) + { + $searchOperator->operator = KalturaSearchOperatorType::SEARCH_AND; + } + } + +} From 8801a98ca1e8c0d3c886434f1b420f47e3747876 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Sat, 25 May 2019 14:39:48 +0300 Subject: [PATCH 011/103] PLAT-9889 --- .../lib/model/ESearchParams.php | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/plugins/search/providers/elastic_search/lib/model/ESearchParams.php b/plugins/search/providers/elastic_search/lib/model/ESearchParams.php index 9ba9abc754a..92080c821ef 100644 --- a/plugins/search/providers/elastic_search/lib/model/ESearchParams.php +++ b/plugins/search/providers/elastic_search/lib/model/ESearchParams.php @@ -30,6 +30,11 @@ class ESearchParams extends BaseObject */ protected $useHighlight; + /** + * @var ESearchAggregations + */ + protected $aggregations; + /** * @return ESearchOperator */ @@ -110,4 +115,20 @@ public function setUseHighlight($useHighlight) $this->useHighlight = $useHighlight; } + /*** + * @param ESearchAggregations $aggregations + */ + public function setAggregations($aggregations) + { + $this->aggregations = $aggregations; + } + + /*** + * @return ESearchAggregations + */ + public function getAggregations() + { + return $this->aggregations; + } + } From 0865a7c3b37e7e580fffff08eed756434c26365a Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Sat, 25 May 2019 14:41:24 +0300 Subject: [PATCH 012/103] PLAT-9889 --- .../lib/model/search/kBaseESearch.php | 3 ++- .../lib/model/search/kBaseSearch.php | 23 ++++++++++++++++++- .../lib/model/search/kEntrySearch.php | 8 +++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/model/search/kBaseESearch.php b/plugins/search/providers/elastic_search/lib/model/search/kBaseESearch.php index 46596c9d3ee..2783a682b16 100644 --- a/plugins/search/providers/elastic_search/lib/model/search/kBaseESearch.php +++ b/plugins/search/providers/elastic_search/lib/model/search/kBaseESearch.php @@ -31,13 +31,14 @@ protected function execSearch(ESearchOperator $eSearchOperator) return $result; } - protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null) + protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null, ESearchAggregations $aggregations=null) { $partnerId = kBaseElasticEntitlement::$partnerId; $this->initQueryAttributes($partnerId, $objectId); $this->initBaseFilter($partnerId, $statuses, $objectId); $this->initPager($pager); $this->initOrderBy($order); + $this->initAggregations($aggregations); } protected function addGlobalHighlights() diff --git a/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php b/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php index 3ae84b7986f..3c53934588d 100644 --- a/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php +++ b/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php @@ -23,7 +23,7 @@ public function __construct() $this->forceInnerHitsSizeOverride = false; } - public abstract function doSearch(ESearchOperator $eSearchOperator, kPager $pager = null, $statuses = array(), $objectId = null, ESearchOrderBy $order = null); + public abstract function doSearch(ESearchOperator $eSearchOperator, kPager $pager = null, $statuses = array(), $objectId = null, ESearchOrderBy $order = null, ESearchAggregations $aggregations = null); /** * @return ESearchQueryAttributes @@ -64,6 +64,27 @@ protected function initOrderBy(ESearchOrderBy $order = null) } } + protected function initAggregations(ESearchAggregations $aggregations = null) + { + if(!$aggregations) + { + return; + } + + $aggs = array(); + + foreach ($aggregations->getAggregations() as $aggregation) + { + /* var $aggregation ESearchAggregationItem */ + $aggregationKey = $aggregation->getAggregationKey().':'.$aggregation->getFieldName(); + $aggs[$aggregationKey] = $aggregation->getAggregationCommand(); + } + if($aggs) + { + $this->query['body']['aggs'] = $aggs; + } + } + protected function getSortConditions(ESearchOrderBy $order) { $orderItems = $order->getOrderItems(); diff --git a/plugins/search/providers/elastic_search/lib/model/search/kEntrySearch.php b/plugins/search/providers/elastic_search/lib/model/search/kEntrySearch.php index 657b9020b67..265a257db10 100644 --- a/plugins/search/providers/elastic_search/lib/model/search/kEntrySearch.php +++ b/plugins/search/providers/elastic_search/lib/model/search/kEntrySearch.php @@ -26,24 +26,24 @@ protected function handleDisplayInSearch() $this->mainBoolQuery->addToFilter($displayInSearchQuery); } - public function doSearch(ESearchOperator $eSearchOperator, kPager $pager = null, $entriesStatus = array(), $objectId = null, ESearchOrderBy $order = null) + public function doSearch(ESearchOperator $eSearchOperator, kPager $pager = null, $entriesStatus = array(), $objectId = null, ESearchOrderBy $order = null, ESearchAggregations $aggregations = null) { kEntryElasticEntitlement::init(); if (!count($entriesStatus)) $entriesStatus = array(entryStatus::READY); - $this->initQuery($entriesStatus, $objectId, $pager, $order); + $this->initQuery($entriesStatus, $objectId, $pager, $order, $aggregations); $this->initEntitlement($eSearchOperator, $objectId); $result = $this->execSearch($eSearchOperator); return $result; } - protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null) + protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null, ESearchAggregations $aggregations = null) { $this->query = array( 'index' => ElasticIndexMap::ELASTIC_ENTRY_INDEX, 'type' => ElasticIndexMap::ELASTIC_ENTRY_TYPE ); - parent::initQuery($statuses, $objectId, $pager, $order); + parent::initQuery($statuses, $objectId, $pager, $order, $aggregations); } protected function initEntitlement(ESearchOperator $eSearchOperator, $objectId) From 28e0ca66413e4592fad7f6b60f618b1f4db33207 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Sat, 25 May 2019 14:43:15 +0300 Subject: [PATCH 013/103] PLAT-9889 --- .../KalturaESearchCategoryAggregateByFieldName.php | 10 ++++++++++ .../KalturaESearchCuePointAggregateByFieldName.php | 11 +++++++++++ .../KalturaESearchEntryAggregateByFieldName.php | 13 +++++++++++++ .../KalturaESearchMetadataAggregateByFieldName.php | 10 ++++++++++ 4 files changed, 44 insertions(+) create mode 100644 plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchCategoryAggregateByFieldName.php create mode 100644 plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchCuePointAggregateByFieldName.php create mode 100644 plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryAggregateByFieldName.php create mode 100644 plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchMetadataAggregateByFieldName.php diff --git a/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchCategoryAggregateByFieldName.php b/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchCategoryAggregateByFieldName.php new file mode 100644 index 00000000000..a9e8237ea74 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchCategoryAggregateByFieldName.php @@ -0,0 +1,10 @@ + Date: Sat, 25 May 2019 14:45:13 +0300 Subject: [PATCH 014/103] PLAT-9889 --- .../lib/model/enum/ESearchAggregateByFieldName.php | 14 ++++++++++++++ .../enum/ESearchCategoryAggregationFieldName.php | 10 ++++++++++ .../enum/ESearchCuePointsAggregationFieldName.php | 10 ++++++++++ .../enum/ESearchEntryAggregationFieldName.php | 13 +++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 plugins/search/providers/elastic_search/lib/model/enum/ESearchAggregateByFieldName.php create mode 100644 plugins/search/providers/elastic_search/lib/model/enum/ESearchCategoryAggregationFieldName.php create mode 100644 plugins/search/providers/elastic_search/lib/model/enum/ESearchCuePointsAggregationFieldName.php create mode 100644 plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryAggregationFieldName.php diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchAggregateByFieldName.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchAggregateByFieldName.php new file mode 100644 index 00000000000..4afd5fc6d27 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchAggregateByFieldName.php @@ -0,0 +1,14 @@ + Date: Mon, 3 Jun 2019 14:58:52 +0300 Subject: [PATCH 015/103] merge --- .../model/thumbStorage/kThumbStorageBase.php | 83 +++++++++++++++++++ .../thumbStorage/kThumbStorageFactory.php | 27 ++++++ .../thumbStorage/kThumbStorageInterface.php | 11 +++ .../model/thumbStorage/kThumbStorageLocal.php | 30 +++++++ .../model/thumbStorage/kThumbStorageNone.php | 25 ++++++ .../model/thumbStorage/kThumbStorageS3.php | 72 ++++++++++++++++ .../model/thumbStorage/kThumbStorageType.php | 12 +++ .../thumbnail/lib/model/thumbnailSource.php | 10 +++ .../thumbnail/services/ThumbnailService.php | 62 ++++++++++++++ 9 files changed, 332 insertions(+) create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageInterface.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageLocal.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php create mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php create mode 100644 plugins/thumbnail/lib/model/thumbnailSource.php create mode 100644 plugins/thumbnail/services/ThumbnailService.php diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php new file mode 100644 index 00000000000..a11ecf3a6e5 --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php @@ -0,0 +1,83 @@ +getPrefix() . '/' . $this->getPath($md5) . '/' .$md5. '.jpg'; + return $path; + } + + protected static function init() + { + if (self::$init) + { + return; + } + self::$init = true; + self::$configParams = kConf::get(self::CONF_SECTION_NAME, 'local', array()); + if(isset(self::$configParams[self::CONF_TYPE])) + { + self::$type = self::$configParams[self::CONF_TYPE]; + } + } + + public static function getInstance() + { + self::init(); + $storage = kThumbStorageFactory::getInstance(self::$type); + return $storage; + } + + protected abstract function getRenderer(); + + public function render() + { + $renderer = $this->getRenderer(); + $renderer->output(); + KExternalErrors::dieGracefully(); + } +} \ No newline at end of file diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php new file mode 100644 index 00000000000..0ed13ac4ad4 --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php @@ -0,0 +1,27 @@ +getFullPath($fileName); + kFile::fullMkdir($path); + $ret = kFile::safeFilePutContents($path, $content); + $this->fileName = $path; + return $ret; + } + + protected function getRenderer() + { + return kFileUtils::getDumpFileRenderer($this->fileName,self::MIME_TYPE,self::MAX_AGE); + } + + public function loadFile($url) + { + $path = $this->getFullPath($url); + $fileData = kFile::getFileData($path); + $this->fileName = $path; + return $fileData->exists; + } +} \ No newline at end of file diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php new file mode 100644 index 00000000000..0f1e7f36c6d --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php @@ -0,0 +1,25 @@ +url = $url; + $this->content = $content; + } + + protected function getRenderer() + { + $renderer = new kRendererString($this->content,self::MIME_TYPE); + return $renderer; + } + + public function loadFile($path) + { + return false; + } +} \ No newline at end of file diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php new file mode 100644 index 00000000000..3e537d9d4db --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php @@ -0,0 +1,72 @@ +setS3Options(); + $this->s3Mgr = kFileTransferMgr::getInstance(StorageProfileProtocol::S3 ,$options); + } + + protected function setS3Options() + { + $s3Options = array(); + if (isset(self::$configParams[self::CONF_REGION])) + { + $s3Options['s3Region'] = self::$configParams[self::CONF_REGION]; + } + return $s3Options; + } + + protected function login() + { + $this->s3Mgr->login(self::$configParams[self::CONF_URL], + self::$configParams[self::CONF_USER_NAME], + self::$configParams[self::CONF_PASSWORD]); + } + + public function saveFile($fileName, $content) + { + $this->login(); + $path = $this->getFullPath($fileName); + kFile::fullMkdir(self::LOCAL_TMP.$path); + kFile::safeFilePutContents(self::LOCAL_TMP.$path,$content); + try + { + $this->s3Mgr->putFile($path, self::LOCAL_TMP . $path); + } + catch (Exception $e) + { + KalturaLog::debug($e->getMessage()); + } + kFile::deleteFile(self::LOCAL_TMP.$path); + $this->content = $content; + } + protected function getRenderer() + { + $renderer = new kRendererString($this->content ,self::MIME_TYPE ); + return $renderer; + } + + public function loadFile($url) + { + $this->login(); + $path = $this->getFullPath($url); + $this->url = self::$configParams[self::CONF_URL] . $path; + try + { + $this->content = $this->s3Mgr->getFile($path); + } + catch (Exception $e) + { + return false; + } + return !empty($this->content); + } +} \ No newline at end of file diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php new file mode 100644 index 00000000000..6578f371a8b --- /dev/null +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php @@ -0,0 +1,12 @@ +getTransformationStringFromUri(); + $transformation = $this->parseUrl($transformationUrl); + $storage = kThumbStorageBase::getInstance(); + if( !$storage->loadFile($transformationUrl) ) + { + $transformation->validate(); + $imagick = $transformation->execute(); + $storage->saveFile($transformationUrl,$imagick); + } + $storage->render(); + } + + protected function parseUrl($url) + { + $transformation = new imageTransformation(); + $steps = explode(self::IMAGE_TRANSFORMATION_STEPS_DELIMITER, $url); + $stepsCount = count($steps); + for ($i = 1; $i < $stepsCount; $i++) + { + if(!empty($steps[$i])) + { + $transformation->addImageTransformationStep(thumbnailEngine::parseImageTransformationStep($steps[$i])); + } + } + + return $transformation; + } + + protected function getTransformationStringFromUri() + { + $uri = $_SERVER['REQUEST_URI']; + $transformParametersStart = strpos($uri,self::PARTNER_TOKEN); + if($transformParametersStart === false) + { + throw new KalturaAPIException(KalturaThumbnailErrors::MISSING_PARTNER_PARAMETER_IN_URL); + } + + return substr($uri, $transformParametersStart + 3); + } +} \ No newline at end of file From 243141270b04a3cd4d6c1332914f0f8c848366ab Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Mon, 3 Jun 2019 15:08:59 +0300 Subject: [PATCH 016/103] merge --- .../model/thumbStorage/kThumbStorageBase.php | 32 ++++---- .../thumbStorage/kThumbStorageFactory.php | 2 +- .../thumbStorage/kThumbStorageInterface.php | 7 +- .../model/thumbStorage/kThumbStorageLocal.php | 41 ++++++++-- .../model/thumbStorage/kThumbStorageNone.php | 15 ++-- .../model/thumbStorage/kThumbStorageS3.php | 76 ++++++++++++++----- .../thumbnail/services/ThumbnailService.php | 49 ++---------- 7 files changed, 132 insertions(+), 90 deletions(-) diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php index a11ecf3a6e5..a6099fcef32 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php @@ -1,7 +1,7 @@ getPrefix() . '/' . $this->getPath($md5) . '/' .$md5. '.jpg'; + $path = $this->getPrefix() . DIRECTORY_SEPARATOR . $this->getPath($md5) . DIRECTORY_SEPARATOR .$md5. '.jpg'; return $path; } @@ -57,6 +54,7 @@ protected static function init() { return; } + self::$init = true; self::$configParams = kConf::get(self::CONF_SECTION_NAME, 'local', array()); if(isset(self::$configParams[self::CONF_TYPE])) @@ -67,16 +65,24 @@ protected static function init() public static function getInstance() { - self::init(); - $storage = kThumbStorageFactory::getInstance(self::$type); + if(kApiCache::isCacheEnabled()) + { + self::init(); + $storage = kThumbStorageFactory::getInstance(self::$type); + } + else + { + $storage = kThumbStorageFactory::getInstance(kThumbStorageType::NONE); + } + return $storage; } - protected abstract function getRenderer(); + protected abstract function getRenderer($lastModified = null); - public function render() + public function render($lastModified = null) { - $renderer = $this->getRenderer(); + $renderer = $this->getRenderer($lastModified); $renderer->output(); KExternalErrors::dieGracefully(); } diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php index 0ed13ac4ad4..a2045f0d5a7 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageFactory.php @@ -1,7 +1,7 @@ getFullPath($fileName); + KalturaLog::debug("Saving file to:" . $url); + $path = $this->getFullPath($url); kFile::fullMkdir($path); $ret = kFile::safeFilePutContents($path, $content); + if(!$ret) + { + KalturaLog::err("Failed to save thumbnail file"); + throw new kThumbnailException(kThumbnailException::CACHE_ERROR, kThumbnailException::CACHE_ERROR); + } + $this->fileName = $path; return $ret; } - protected function getRenderer() + protected function getRenderer($lastModified = null) { - return kFileUtils::getDumpFileRenderer($this->fileName,self::MIME_TYPE,self::MAX_AGE); + return kFileUtils::getDumpFileRenderer($this->fileName, self::MIME_TYPE, self::MAX_AGE, 0, $lastModified); } - public function loadFile($url) + public function loadFile($url, $lastModified = null) { + KalturaLog::debug("loading file from path:" . $url); $path = $this->getFullPath($url); $fileData = kFile::getFileData($path); + if(!$fileData->exists) + { + KalturaLog::debug("file wasn't found" . $url); + return false; + } + + if($lastModified && $fileData->last_modified < $lastModified) + { + KalturaLog::debug("file was created before entry changed" . $fileData->last_modified); + return false; + } + $this->fileName = $path; - return $fileData->exists; + return true; + } + + public function deleteFile($url) + { + KalturaLog::debug("deleting file to:" . $url); + $path = $this->getFullPath($url); + kFile::deleteFile($path); } } \ No newline at end of file diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php index 0f1e7f36c6d..7f209228c0b 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageNone.php @@ -1,24 +1,29 @@ url = $url; $this->content = $content; } - protected function getRenderer() + protected function getRenderer($lastModified = null) { - $renderer = new kRendererString($this->content,self::MIME_TYPE); + $renderer = new kRendererString($this->content,self::MIME_TYPE, self::MAX_AGE, $lastModified); return $renderer; } - public function loadFile($path) + public function loadFile($url, $lastModified = null) + { + return false; + } + + public function deleteFile($url) { return false; } diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php index 3e537d9d4db..2210a9a89fd 100644 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php +++ b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageS3.php @@ -1,17 +1,27 @@ setS3Options(); - $this->s3Mgr = kFileTransferMgr::getInstance(StorageProfileProtocol::S3 ,$options); + $options = $this->setS3Options(); + if(!isset(self::$configParams[self::CONF_URL]) || + !isset(self::$configParams[self::CONF_USER_NAME]) || + !isset(self::$configParams[self::CONF_PASSWORD])) + { + throw new kThumbnailException(kThumbnailException::MISSING_S3_CONFIGURATION, kThumbnailException::MISSING_S3_CONFIGURATION); + } + + $this->s3Mgr = kFileTransferMgr::getInstance(StorageProfileProtocol::S3, $options); + $this->login(); + $this->s3Mgr->registerStreamWrapper(); } protected function setS3Options() @@ -21,6 +31,7 @@ protected function setS3Options() { $s3Options['s3Region'] = self::$configParams[self::CONF_REGION]; } + return $s3Options; } @@ -33,40 +44,65 @@ protected function login() public function saveFile($fileName, $content) { - $this->login(); $path = $this->getFullPath($fileName); - kFile::fullMkdir(self::LOCAL_TMP.$path); - kFile::safeFilePutContents(self::LOCAL_TMP.$path,$content); - try + $this->url = self::getUrl($path); + if(kFile::filePutContents($this->url, $content)) { - $this->s3Mgr->putFile($path, self::LOCAL_TMP . $path); + $this->content = $content; } - catch (Exception $e) + else { - KalturaLog::debug($e->getMessage()); + KalturaLog::err("Failed to save thumbnail file"); + throw new kThumbnailException(kThumbnailException::CACHE_ERROR, kThumbnailException::CACHE_ERROR); } - kFile::deleteFile(self::LOCAL_TMP.$path); - $this->content = $content; } - protected function getRenderer() + + protected function getRenderer($lastModified = null) { - $renderer = new kRendererString($this->content ,self::MIME_TYPE ); + $renderer = new kRendererString($this->content ,self::MIME_TYPE, $lastModified); return $renderer; } - public function loadFile($url) + public function loadFile($url, $lastModified = null) { - $this->login(); + KalturaLog::debug("loading file from S3 " . $url); $path = $this->getFullPath($url); - $this->url = self::$configParams[self::CONF_URL] . $path; + $this->url = self::getUrl($path); try { - $this->content = $this->s3Mgr->getFile($path); + if(file_exists($this->url)) + { + if($lastModified) + { + $s3lastModified = filemtime($this->url); + if($lastModified > $s3lastModified) + { + KalturaLog::debug("file was created before entry changed" . $s3lastModified); + return false; + } + } + + $this->content = $this->s3Mgr->getFile($path); + return true; + } } catch (Exception $e) { - return false; + KalturaLog::debug("failed to load file " . $e->getMessage()); + throw new kThumbnailException(kThumbnailException::CACHE_ERROR, kThumbnailException::CACHE_ERROR); } - return !empty($this->content); + + return false; + } + + public function deleteFile($url) + { + KalturaLog::debug("deleting file from s3:" . $url); + return $this->s3Mgr->delFile($url); + } + + protected static function getUrl($path) + { + return 's3://' . $path; } } \ No newline at end of file diff --git a/plugins/thumbnail/services/ThumbnailService.php b/plugins/thumbnail/services/ThumbnailService.php index 99c4ef0307a..7f68befe240 100644 --- a/plugins/thumbnail/services/ThumbnailService.php +++ b/plugins/thumbnail/services/ThumbnailService.php @@ -5,58 +5,25 @@ * @subpackage api.services */ -//require_once ("/opt/kaltura/app/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageBase.php"); class ThumbnailService extends KalturaBaseUserService { - const PARTNER_INDEX = 0; - const IMAGE_TRANSFORMATION_STEPS_DELIMITER = "/"; - const PARTNER_TOKEN = "/p/"; - /** * Retrieves a thumbnail according to the required transformation * @action transform - * @return bool - * + * @param string $transformString */ - public function transformAction() + public function transformAction($transformString) { - $transformationUrl = $this->getTransformationStringFromUri(); - $transformation = $this->parseUrl($transformationUrl); + $transformation = thumbnailStringParser::parseTransformString($transformString); + $transformation->validate(); + $lastModified = $transformation->getLastModified(); $storage = kThumbStorageBase::getInstance(); - if( !$storage->loadFile($transformationUrl) ) + if(!$storage->loadFile($transformString, $lastModified)) { - $transformation->validate(); $imagick = $transformation->execute(); - $storage->saveFile($transformationUrl,$imagick); - } - $storage->render(); - } - - protected function parseUrl($url) - { - $transformation = new imageTransformation(); - $steps = explode(self::IMAGE_TRANSFORMATION_STEPS_DELIMITER, $url); - $stepsCount = count($steps); - for ($i = 1; $i < $stepsCount; $i++) - { - if(!empty($steps[$i])) - { - $transformation->addImageTransformationStep(thumbnailEngine::parseImageTransformationStep($steps[$i])); - } - } - - return $transformation; - } - - protected function getTransformationStringFromUri() - { - $uri = $_SERVER['REQUEST_URI']; - $transformParametersStart = strpos($uri,self::PARTNER_TOKEN); - if($transformParametersStart === false) - { - throw new KalturaAPIException(KalturaThumbnailErrors::MISSING_PARTNER_PARAMETER_IN_URL); + $storage->saveFile($transformString, $imagick, $lastModified); } - return substr($uri, $transformParametersStart + 3); + $storage->render($lastModified); } } \ No newline at end of file From 57ec449e1463bd7240a8820b70f4f824b61fb237 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Mon, 3 Jun 2019 15:11:16 +0300 Subject: [PATCH 017/103] merge --- .../lib/model/thumbStorage/kThumbStorageType.php | 12 ------------ plugins/thumbnail/lib/model/thumbnailSource.php | 10 ---------- 2 files changed, 22 deletions(-) delete mode 100644 plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php delete mode 100644 plugins/thumbnail/lib/model/thumbnailSource.php diff --git a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php b/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php deleted file mode 100644 index 6578f371a8b..00000000000 --- a/plugins/thumbnail/lib/model/thumbStorage/kThumbStorageType.php +++ /dev/null @@ -1,12 +0,0 @@ - Date: Mon, 3 Jun 2019 19:56:23 +0300 Subject: [PATCH 018/103] merge --- .../response/KalturaESearchAggregationResponse.php | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php index 353a01a7960..06ac9e30c21 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php @@ -6,18 +6,14 @@ class KalturaESearchAggregationResponse extends KalturaObject { - /*** - * @var KalturaESearchAggregationResponseArray - */ - public $aggs; - protected function getApiObjects($aggregationName) { return explode(':', $aggregationName); } - public function resultToApi($aggregationResults, $aggregationRequest) + public function resultToApi($aggregationResults) { + $aggs = new KalturaESearchAggregationResponseArray(); foreach ($aggregationResults as $key=>$response) { list ($responseObject, $fieldName) = $this->getApiObjects($key); @@ -43,8 +39,9 @@ public function resultToApi($aggregationResults, $aggregationRequest) $agg->buckets[] = $reponseBucket; } } - $this->aggs[] = $agg; + $aggs[] = $agg; } + return $aggs; } } \ No newline at end of file From 14e6a3b0a713debe1d9491c58d62964c49d0309e Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Mon, 3 Jun 2019 19:58:00 +0300 Subject: [PATCH 019/103] merge --- .../aggregations/items/ESearchCuepointsAggregationItem.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php index 32d6c225077..7bdba5fe135 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php @@ -1,9 +1,7 @@ Date: Mon, 3 Jun 2019 20:01:31 +0300 Subject: [PATCH 020/103] merge --- .../items/ESearchCuepointsAggregationItem.php | 2 +- .../aggregations/items/ESearchEntryAggregationItem.php | 2 +- .../elastic_search/lib/model/kESearchCoreAdapter.php | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php index 7bdba5fe135..0afcaa74ab4 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php @@ -6,7 +6,7 @@ class ESearchCuepointsAggregationItem extends ESearchAggregationItem { - /*** + /** * var ESearchCuePointsAggregationFieldName */ protected $fieldName; diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php index ceaf4d7e521..f76b4d510e5 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php @@ -6,7 +6,7 @@ class ESearchEntryAggregationItem extends ESearchAggregationItem { - /*** + /** * var ESearchEntryAggregationFieldName */ protected $fieldName; diff --git a/plugins/search/providers/elastic_search/lib/model/kESearchCoreAdapter.php b/plugins/search/providers/elastic_search/lib/model/kESearchCoreAdapter.php index 982556cc95f..9dd56b52ae5 100644 --- a/plugins/search/providers/elastic_search/lib/model/kESearchCoreAdapter.php +++ b/plugins/search/providers/elastic_search/lib/model/kESearchCoreAdapter.php @@ -30,11 +30,11 @@ class kESearchCoreAdapter public static function transformElasticToCoreObject($elasticResults, $coreSearchObject) { - list($objectData, $objectOrder, $objectCount, $objectHighlight) = self::getElasticResultAsArray($elasticResults, + list($objectData, $objectOrder, $objectCount, $objectHighlight, $aggregations) = self::getElasticResultAsArray($elasticResults, $coreSearchObject->getQueryAttributes()->getQueryHighlightsAttributes()); $objects = $coreSearchObject->fetchCoreObjectsByIds(array_keys($objectData)); $coreResults = self::getCoreESearchResults($objects, $objectData, $objectOrder, $objectHighlight); - return array($coreResults, $objectCount); + return array($coreResults, $objectCount, $aggregations); } public static function getElasticResultAsArray($elasticResults, $queryHighlightsAttribute) @@ -57,8 +57,10 @@ public static function getElasticResultAsArray($elasticResults, $queryHighlights if(isset($elasticResults[self::HITS_KEY][self::TOTAL_KEY])) $objectCount = $elasticResults[self::HITS_KEY][self::TOTAL_KEY]; - - return array($objectData, $objectOrder, $objectCount, $objectHighlight); + + $aggregations = isset($elasticResults['aggregations']) ? $elasticResults['aggregations'] : null; + + return array($objectData, $objectOrder, $objectCount, $objectHighlight, $aggregations ); } private static function getCoreESearchResults($coreObjects, $objectsData, $objectsOrder, $objectHighlight) From d12449d46075a26c4231f2c55fee36b60115c591 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Mon, 3 Jun 2019 20:02:18 +0300 Subject: [PATCH 021/103] merge --- .../lib/api/types/results/KalturaESearchEntryResponse.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/search/providers/elastic_search/lib/api/types/results/KalturaESearchEntryResponse.php b/plugins/search/providers/elastic_search/lib/api/types/results/KalturaESearchEntryResponse.php index a31dd405793..d9a7e76677e 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/results/KalturaESearchEntryResponse.php +++ b/plugins/search/providers/elastic_search/lib/api/types/results/KalturaESearchEntryResponse.php @@ -10,4 +10,10 @@ class KalturaESearchEntryResponse extends KalturaESearchResponse * @readonly */ public $objects; + + /** + * @var KalturaESearchAggregationResponseArray + * @readonly + */ + public $aggregations; } From 92a3489cf6becced747c99fb6e386eba482611ef Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Mon, 3 Jun 2019 20:07:17 +0300 Subject: [PATCH 022/103] merge --- .../providers/elastic_search/services/ESearchService.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/search/providers/elastic_search/services/ESearchService.php b/plugins/search/providers/elastic_search/services/ESearchService.php index 441ec5c15fe..d9ceb848006 100644 --- a/plugins/search/providers/elastic_search/services/ESearchService.php +++ b/plugins/search/providers/elastic_search/services/ESearchService.php @@ -23,8 +23,7 @@ function searchEntryAction(KalturaESearchEntryParams $searchParams, KalturaPager if($aggregations) { $aggregationResponse = new KalturaESearchAggregationResponse(); - $aggregationResponse->resultToApi($aggregations); - $response->aggregations = $aggregationResponse->aggs; + $response->aggregations = $aggregationResponse->resultToApi($aggregations); } return $response; } From b76e10ef6d5084e9be968f9181a56406f055ae7e Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 4 Jun 2019 09:31:28 +0300 Subject: [PATCH 023/103] integration --- .../lib/model/enum/ESearchCuePointsAggregationFieldName.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchCuePointsAggregationFieldName.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchCuePointsAggregationFieldName.php index 086c2b27423..544756ce93c 100644 --- a/plugins/search/providers/elastic_search/lib/model/enum/ESearchCuePointsAggregationFieldName.php +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchCuePointsAggregationFieldName.php @@ -5,6 +5,6 @@ */ interface ESearchCuePointsAggregationFieldName extends BaseEnum { - const TAGS = 'cue_point_tags.raw'; - const TYPE = 'cue_point_type'; + const TAGS = 'cue_points.cue_point_tags.raw'; + const TYPE = 'cue_points.cue_point_type'; } \ No newline at end of file From b0009fd0d593548292f1a1799cb183fcafc46ccc Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 4 Jun 2019 09:43:20 +0300 Subject: [PATCH 024/103] integration --- .../KalturaESearchCategoryAggregationItem.php | 23 +- ...KalturaESearchCuepointsAggregationItem.php | 18 +- .../KalturaESearchEntryAggregationItem.php | 20 + .../KalturaESearchMetadataAggregationItem.php | 18 +- .../KalturaESearchAggregationResponse.php | 19 +- .../elastic_search/lib/elasticSearchUtils.php | 439 +++++++++--------- .../enum/ESearchEntryAggregationFieldName.php | 2 +- 7 files changed, 316 insertions(+), 223 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php index c0a0090f748..8805785a6f9 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php @@ -15,7 +15,9 @@ class KalturaESearchCategoryAggregationItem extends KalturaESearchAggregationIte public function toObject($object_to_fill = null, $props_to_skip = array()) { if (!$object_to_fill) + { $object_to_fill = new ESearchCategoryAggregationItem(); + } return parent::toObject($object_to_fill, $props_to_skip); } @@ -24,9 +26,28 @@ public function getFieldEnumMap() return array (KalturaESearchCategoryAggregateByFieldName::CATEGORY_NAME => ESearchCategoryAggregationFieldName::CATEGORY_NAME); } - public function coreToApiResponse($aggregation) + protected function fixCategoryName($coreName) { } + public function coreToApiResponse($coreRespone) + { + $bucketsArray = new KalturaESearchAggregationBucketsArray(); + $buckets = $coreRespone['buckets']; + if ($buckets) + { + foreach ($buckets as $bucket) + { + $reponseBucket = new KalturaESearchAggregationBucket(); + $reponseBucket->fromArray($bucket); + list(,$categoryName) = elasticSearchUtils::reverseFormatCategoryNameStatus($reponseBucket->value); + $reponseBucket->value = $categoryName; + $bucketsArray[] = $reponseBucket; + } + + } + return $bucketsArray; + } + } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php index 24c4001cdf5..03a1324ad23 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php @@ -14,7 +14,9 @@ class KalturaESearchCuepointsAggregationItem extends KalturaESearchAggregationIt public function toObject($object_to_fill = null, $props_to_skip = array()) { if (!$object_to_fill) + { $object_to_fill = new ESearchCuepointsAggregationItem(); + } return parent::toObject($object_to_fill, $props_to_skip); } @@ -25,9 +27,19 @@ public function getFieldEnumMap() KalturaESearchCuePointAggregateByFieldName::TYPE => ESearchCuePointsAggregationFieldName::TYPE); } - public function coreToApiResponse($aggregation) + public function coreToApiResponse($coreRespone) { - + $bucketsArray = new KalturaESearchAggregationBucketsArray(); + $buckets = $coreRespone['NestedBucket']['buckets']; + if ($buckets) + { + foreach ($buckets as $bucket) + { + $reponseBucket = new KalturaESearchAggregationBucket(); + $reponseBucket->fromArray($bucket); + $bucketsArray[] = $reponseBucket; + } + } + return $bucketsArray; } - } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php index 87f631da349..d23f9594305 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php @@ -17,7 +17,9 @@ class KalturaESearchEntryAggregationItem extends KalturaESearchAggregationItem public function toObject($object_to_fill = null, $props_to_skip = array()) { if (!$object_to_fill) + { $object_to_fill = new ESearchEntryAggregationItem(); + } return parent::toObject($object_to_fill, $props_to_skip); } @@ -29,4 +31,22 @@ public function getFieldEnumMap() KalturaESearchEntryAggregateByFieldName::TAGS => ESearchEntryAggregationFieldName::TAGS, KalturaESearchEntryAggregateByFieldName::ACCESS_CONTROL_PROFILE => ESearchEntryAggregationFieldName::ACCESS_CONTROL_PROFILE); } + + public function coreToApiResponse($coreRespone) + { + $bucketsArray = new KalturaESearchAggregationBucketsArray(); + $buckets = $coreRespone['buckets']; + if ($buckets) + { + foreach ($buckets as $bucket) + { + $reponseBucket = new KalturaESearchAggregationBucket(); + $reponseBucket->fromArray($bucket); + $bucketsArray[] = $reponseBucket; + } + } + return $bucketsArray; + } + + } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php index 5f1c2262055..4fbe2791faf 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -15,7 +15,9 @@ class KalturaESearchMetadataAggregationItem extends KalturaESearchAggregationIte public function toObject($object_to_fill = null, $props_to_skip = array()) { if (!$object_to_fill) + { $object_to_fill = new ESearchMetadataAggregationItem(); + } return parent::toObject($object_to_fill, $props_to_skip); } @@ -24,9 +26,21 @@ public function getFieldEnumMap() return array (); } - public function coreToApiResponse($response, $request) + public function coreToApiResponse($coreRespone) { - return $response['MetadataAggs']['buckets']; + $bucketsArray = new KalturaESearchAggregationBucketsArray(); + $buckets = $coreRespone['NestedBucket']['buckets']; + if ($buckets) + { + foreach ($buckets as $bucket) + { + $reponseBucket = new KalturaESearchAggregationBucket(); + $reponseBucket->fromArray($bucket); + $bucketsArray[] = $reponseBucket; + } + } + return $bucketsArray; } + } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php index 06ac9e30c21..cd034af009c 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php @@ -11,6 +11,16 @@ protected function getApiObjects($aggregationName) return explode(':', $aggregationName); } + protected function mapAggregationCoreObjects($coreObject) + { + $map = array(ESearchCategoryAggregationItem::KEY => 'KalturaESearchCategoryAggregationItem', + ESearchCuepointsAggregationItem::KEY => 'KalturaESearchCuepointsAggregationItem', + ESearchMetadataAggregationItem::KEY => 'KalturaESearchMetadataAggregationItem', + ESearchEntryAggregationItem::KEY => 'KalturaESearchEntryAggregationItem'); + $ret = isset($map[$coreObject]) ? $map[$coreObject] : null; + return $ret; + } + public function resultToApi($aggregationResults) { $aggs = new KalturaESearchAggregationResponseArray(); @@ -20,6 +30,13 @@ public function resultToApi($aggregationResults) $agg = new KalturaESearchAggregationResponseItem(); $agg->fieldName = $fieldName; $agg->name = $responseObject; + + $itemObjectName = $this->mapAggregationCoreObjects($responseObject); + $objectItemHandler = new $itemObjectName(); + $agg->buckets = $objectItemHandler->coreToApiResponse($response); + $aggs[] = $agg; + } +/* $buckets=null; if(isset($response['buckets'])) { @@ -40,7 +57,7 @@ public function resultToApi($aggregationResults) } } $aggs[] = $agg; - } + }*/ return $aggs; } diff --git a/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php b/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php index 834e8310e72..77b318b67e0 100644 --- a/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php +++ b/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php @@ -1,215 +1,224 @@ -', '
', - '
', '
', '
', '
', '

', '

', '

'); - - protected static $elastic_negation_booleans = array('0', 'false', 'off', 'no'); - - /** - * return the analyzed language field name - * @param $language - * @param $fieldName - * @param $delimiter - * @return null|string - */ - public static function getAnalyzedFieldName($language, $fieldName, $delimiter) - { - $fieldMap = array( - 'english' => 'english', - 'arabic' => 'arabic', - 'brazilian' => 'brazilian', - 'chinese' => 'cjk', - 'korean' => 'cjk', - 'japanese' => 'cjk', - 'danish' => 'danish', - 'dutch' => 'dutch', - 'finnish' => 'finnish', - 'french' => 'french', - 'german' => 'german', - 'greek' => 'greek', - 'hindi' => 'hindi', - 'indonesian' => 'indonesian', - 'italian' => 'italian', - 'norwegian' => 'norwegian', - 'portuguese' => 'portuguese', - 'russian' => 'russian', - 'spanish' => 'spanish', - 'swedish' => 'swedish', - 'turkish' => 'turkish', - 'thai' => 'thai', - ); - - $language = strtolower($language); - if(isset($fieldMap[$language])) - return $fieldName.$delimiter.$fieldMap[$language]; - - return null; - } - - public static function getSynonymFieldName($language, $fieldName, $delimiter) - { - $fieldMap = array( - 'english' => kESearchQueryManager::SYNONYM_FIELD_SUFFIX, - ); - - $language = strtolower($language); - if(isset($fieldMap[$language])) - return $fieldName.$delimiter.$fieldMap[$language]; - } - - public static function formatGroupIdCreationMode($groupId, $creationMode) - { - return sprintf("g%scm%s", $groupId, $creationMode); - } - - public static function formatPartnerStatus($partnerId, $status) - { - return sprintf("p%ss%s", $partnerId, $status); - } - - public static function formatCategoryIdStatus($categoryId, $status) - { - return sprintf("c%ss%s", $categoryId, $status); - } - - public static function formatCategoryFullIdStatus($categoryId, $status) - { - return sprintf("s%sfid>%s", $status, $categoryId); - } - - public static function formatParentCategoryIdStatus($categoryId, $status) - { - return sprintf("p%ss%s", $categoryId, $status); - } - - public static function formatCategoryNameStatus($categoryName, $status) - { - return sprintf("s%sc>%s", $status, $categoryName); - } - - public static function formatParentCategoryNameStatus($categoryName, $status) - { - return sprintf("s%sp>%s", $status, $categoryName); - } - - public static function formatCategoryEntryStatus($status) - { - return sprintf("ces%s", $status); - } - - public static function formatCategoryUserPermissionLevel($userId, $permissionLevel) - { - return sprintf("uid%spl%s", $userId, $permissionLevel); - } - - public static function formatCategoryUserPermissionName($userId, $permissionName) - { - return sprintf("uid%spn%s", $userId, $permissionName); - } - - public static function getCategoryUserAllPermissionLevels($userId) - { - $formatPermissions = array(); - $permissionLevelReflection = new ReflectionClass('CategoryKuserPermissionLevel'); - $permissionLevels = $permissionLevelReflection->getConstants(); - foreach ($permissionLevels as $permissionLevel) - $formatPermissions[] = elasticSearchUtils::formatCategoryUserPermissionLevel($userId, $permissionLevel); - - return $formatPermissions; - } - - public static function getCategoryUserAllPermissionNames($userId) - { - $formatPermissions = array(); - $permissionNames = array( - PermissionName::CATEGORY_SUBSCRIBE, - PermissionName::CATEGORY_CONTRIBUTE, - PermissionName::CATEGORY_MODERATE, - PermissionName::CATEGORY_EDIT, - PermissionName::CATEGORY_VIEW, - ); - - foreach ($permissionNames as $permissionName) - $formatPermissions[] = elasticSearchUtils::formatCategoryUserPermissionName($userId, $permissionName); - - return $formatPermissions; - } - - public static function formatSearchTerm($searchTerm) - { - //remove extra spaces - $term = preg_replace('/\s+/', ' ', $searchTerm); - //lowercase and trim - $term = strtolower($term); - $term = trim($term); - return $term; - } - - public static function isMaster($elasticClient, $elasticHostName) - { - $masterInfo = $elasticClient->getMasterInfo(); - if(isset($masterInfo[0]['node']) && $masterInfo[0]['node'] == $elasticHostName) - return true; - - return false; - } - - public static function cleanEmptyValues(&$body) - { - foreach ($body as $key => $value) - { - if(is_null($value) || $value === '') - unset($body[$key]); - if(is_array($value) && ( count($value) == 0 || ( (count($value) == 1 && (isset($value[0])) && $value[0] === '' ) ) )) - unset($body[$key]); - } - } - - public static function getNumOfFragmentsByConfigKey($highlightConfigKey) - { - $highlightConfig = kConf::get('highlights', 'elastic'); - //return null to use elastic default num of fragments - $numOfFragments = isset($highlightConfig[$highlightConfigKey]) ? $highlightConfig[$highlightConfigKey] : null; - return $numOfFragments; - } - - public static function getBooleanValue($value) - { - return !in_array(self::formatSearchTerm($value), self::$elastic_negation_booleans); - } - - /** - * Go over the array and decode html and strip tags from all of its leafs - * @param array $cmd - */ - public static function prepareForInsertToElastic(&$cmd) - { - array_walk_recursive($cmd, array('elasticSearchUtils','prepareElasticLeafInput')); - } - - public static function prepareElasticLeafInput(&$value, $key) - { - if(is_string($value)) - { - self::filterHtmlFromLeaf($value); - $value = trim($value); - $value = @iconv('utf-8', 'utf-8//IGNORE', $value); - } - } - - public static function filterHtmlFromLeaf(&$value) - { - $value = html_entity_decode($value); - $value = str_replace(self::$html_chars_to_replace, " ", $value); - $value = strip_tags($value); - } - -} +', '
', + '
', '

', '
', '
', '

', '

', '

'); + + protected static $elastic_negation_booleans = array('0', 'false', 'off', 'no'); + + /** + * return the analyzed language field name + * @param $language + * @param $fieldName + * @param $delimiter + * @return null|string + */ + public static function getAnalyzedFieldName($language, $fieldName, $delimiter) + { + $fieldMap = array( + 'english' => 'english', + 'arabic' => 'arabic', + 'brazilian' => 'brazilian', + 'chinese' => 'cjk', + 'korean' => 'cjk', + 'japanese' => 'cjk', + 'danish' => 'danish', + 'dutch' => 'dutch', + 'finnish' => 'finnish', + 'french' => 'french', + 'german' => 'german', + 'greek' => 'greek', + 'hindi' => 'hindi', + 'indonesian' => 'indonesian', + 'italian' => 'italian', + 'norwegian' => 'norwegian', + 'portuguese' => 'portuguese', + 'russian' => 'russian', + 'spanish' => 'spanish', + 'swedish' => 'swedish', + 'turkish' => 'turkish', + 'thai' => 'thai', + ); + + $language = strtolower($language); + if(isset($fieldMap[$language])) + return $fieldName.$delimiter.$fieldMap[$language]; + + return null; + } + + public static function getSynonymFieldName($language, $fieldName, $delimiter) + { + $fieldMap = array( + 'english' => kESearchQueryManager::SYNONYM_FIELD_SUFFIX, + ); + + $language = strtolower($language); + if(isset($fieldMap[$language])) + return $fieldName.$delimiter.$fieldMap[$language]; + } + + public static function formatGroupIdCreationMode($groupId, $creationMode) + { + return sprintf("g%scm%s", $groupId, $creationMode); + } + + public static function formatPartnerStatus($partnerId, $status) + { + return sprintf("p%ss%s", $partnerId, $status); + } + + public static function formatCategoryIdStatus($categoryId, $status) + { + return sprintf("c%ss%s", $categoryId, $status); + } + + public static function formatCategoryFullIdStatus($categoryId, $status) + { + return sprintf("s%sfid>%s", $status, $categoryId); + } + + public static function formatParentCategoryIdStatus($categoryId, $status) + { + return sprintf("p%ss%s", $categoryId, $status); + } + + public static function formatCategoryNameStatus($categoryName, $status) + { + return sprintf("s%sc>%s", $status, $categoryName); + } + + public static function reverseFormatCategoryNameStatus($categoryNameStatus) + { + list($status , $categoryName) = explode ('>',$categoryNameStatus); + $status = substr($status,1,strlen ($status)-1 ); + return array($status, $categoryName); + } + + + + public static function formatParentCategoryNameStatus($categoryName, $status) + { + return sprintf("s%sp>%s", $status, $categoryName); + } + + public static function formatCategoryEntryStatus($status) + { + return sprintf("ces%s", $status); + } + + public static function formatCategoryUserPermissionLevel($userId, $permissionLevel) + { + return sprintf("uid%spl%s", $userId, $permissionLevel); + } + + public static function formatCategoryUserPermissionName($userId, $permissionName) + { + return sprintf("uid%spn%s", $userId, $permissionName); + } + + public static function getCategoryUserAllPermissionLevels($userId) + { + $formatPermissions = array(); + $permissionLevelReflection = new ReflectionClass('CategoryKuserPermissionLevel'); + $permissionLevels = $permissionLevelReflection->getConstants(); + foreach ($permissionLevels as $permissionLevel) + $formatPermissions[] = elasticSearchUtils::formatCategoryUserPermissionLevel($userId, $permissionLevel); + + return $formatPermissions; + } + + public static function getCategoryUserAllPermissionNames($userId) + { + $formatPermissions = array(); + $permissionNames = array( + PermissionName::CATEGORY_SUBSCRIBE, + PermissionName::CATEGORY_CONTRIBUTE, + PermissionName::CATEGORY_MODERATE, + PermissionName::CATEGORY_EDIT, + PermissionName::CATEGORY_VIEW, + ); + + foreach ($permissionNames as $permissionName) + $formatPermissions[] = elasticSearchUtils::formatCategoryUserPermissionName($userId, $permissionName); + + return $formatPermissions; + } + + public static function formatSearchTerm($searchTerm) + { + //remove extra spaces + $term = preg_replace('/\s+/', ' ', $searchTerm); + //lowercase and trim + $term = strtolower($term); + $term = trim($term); + return $term; + } + + public static function isMaster($elasticClient, $elasticHostName) + { + $masterInfo = $elasticClient->getMasterInfo(); + if(isset($masterInfo[0]['node']) && $masterInfo[0]['node'] == $elasticHostName) + return true; + + return false; + } + + public static function cleanEmptyValues(&$body) + { + foreach ($body as $key => $value) + { + if(is_null($value) || $value === '') + unset($body[$key]); + if(is_array($value) && ( count($value) == 0 || ( (count($value) == 1 && (isset($value[0])) && $value[0] === '' ) ) )) + unset($body[$key]); + } + } + + public static function getNumOfFragmentsByConfigKey($highlightConfigKey) + { + $highlightConfig = kConf::get('highlights', 'elastic'); + //return null to use elastic default num of fragments + $numOfFragments = isset($highlightConfig[$highlightConfigKey]) ? $highlightConfig[$highlightConfigKey] : null; + return $numOfFragments; + } + + public static function getBooleanValue($value) + { + return !in_array(self::formatSearchTerm($value), self::$elastic_negation_booleans); + } + + /** + * Go over the array and decode html and strip tags from all of its leafs + * @param array $cmd + */ + public static function prepareForInsertToElastic(&$cmd) + { + array_walk_recursive($cmd, array('elasticSearchUtils','prepareElasticLeafInput')); + } + + public static function prepareElasticLeafInput(&$value, $key) + { + if(is_string($value)) + { + self::filterHtmlFromLeaf($value); + $value = trim($value); + $value = @iconv('utf-8', 'utf-8//IGNORE', $value); + } + } + + public static function filterHtmlFromLeaf(&$value) + { + $value = html_entity_decode($value); + $value = str_replace(self::$html_chars_to_replace, " ", $value); + $value = strip_tags($value); + } + +} diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryAggregationFieldName.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryAggregationFieldName.php index 7735f04bcba..0b3c27268c6 100644 --- a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryAggregationFieldName.php +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryAggregationFieldName.php @@ -8,6 +8,6 @@ interface ESearchEntryAggregationFieldName extends BaseEnum { const ENTRY_TYPE = 'entry_type'; const MEDIA_TYPE = 'media_type'; - const TAGS = 'tags'; + const TAGS = 'tags.raw'; const ACCESS_CONTROL_PROFILE = 'access_control_profile_id'; } \ No newline at end of file From 0c2df088bcb57ce208bbaad39ed04250919ed4f1 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 4 Jun 2019 19:48:55 +0300 Subject: [PATCH 025/103] testing --- .../items/KalturaESearchAggregationItem.php | 2 ++ .../KalturaESearchCategoryAggregationItem.php | 14 +++++------ ...KalturaESearchCuepointsAggregationItem.php | 10 ++++---- .../KalturaESearchEntryAggregationItem.php | 10 ++++---- .../KalturaESearchMetadataAggregationItem.php | 24 +++++++++++++++---- .../KalturaESearchAggregationBucket.php | 6 ++--- .../KalturaESearchAggregationBucketsArray.php | 2 +- .../KalturaESearchAggregationResponse.php | 22 ----------------- .../KalturaESearchAggregationResponseItem.php | 4 ++-- .../aggregations/ESearchAggregations.php | 10 ++++++++ .../items/ESearchAggregationItem.php | 14 ++++------- .../items/ESearchCategoryAggregationItem.php | 7 ------ .../items/ESearchCuepointsAggregationItem.php | 11 ++++----- .../items/ESearchEntryAggregationItem.php | 6 ----- .../items/ESearchMetadataAggregationItem.php | 19 ++++++++++----- 15 files changed, 78 insertions(+), 83 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php index 75fe01bf3b9..fd5d0f3dc8c 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php @@ -35,4 +35,6 @@ public function toObject($object_to_fill = null, $props_to_skip = array()) return parent::toObject($object_to_fill, $props_to_skip); } + + } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php index 8805785a6f9..1c977be506c 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php @@ -31,19 +31,19 @@ protected function fixCategoryName($coreName) } - public function coreToApiResponse($coreRespone) + public function coreToApiResponse($coreResponse) { $bucketsArray = new KalturaESearchAggregationBucketsArray(); - $buckets = $coreRespone['buckets']; + $buckets = $coreResponse[ESearchAggregations::BUCKETS]; if ($buckets) { foreach ($buckets as $bucket) { - $reponseBucket = new KalturaESearchAggregationBucket(); - $reponseBucket->fromArray($bucket); - list(,$categoryName) = elasticSearchUtils::reverseFormatCategoryNameStatus($reponseBucket->value); - $reponseBucket->value = $categoryName; - $bucketsArray[] = $reponseBucket; + $responseBucket = new KalturaESearchAggregationBucket(); + $responseBucket->fromArray($bucket); + list(,$categoryName) = elasticSearchUtils::reverseFormatCategoryNameStatus($responseBucket->value); + $responseBucket->value = $categoryName; + $bucketsArray[] = $responseBucket; } } diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php index 03a1324ad23..2ccd508b589 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php @@ -27,17 +27,17 @@ public function getFieldEnumMap() KalturaESearchCuePointAggregateByFieldName::TYPE => ESearchCuePointsAggregationFieldName::TYPE); } - public function coreToApiResponse($coreRespone) + public function coreToApiResponse($coreResponse) { $bucketsArray = new KalturaESearchAggregationBucketsArray(); - $buckets = $coreRespone['NestedBucket']['buckets']; + $buckets = $coreResponse[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; if ($buckets) { foreach ($buckets as $bucket) { - $reponseBucket = new KalturaESearchAggregationBucket(); - $reponseBucket->fromArray($bucket); - $bucketsArray[] = $reponseBucket; + $responseBucket = new KalturaESearchAggregationBucket(); + $responseBucket->fromArray($bucket); + $bucketsArray[] = $responseBucket; } } return $bucketsArray; diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php index d23f9594305..28faef5f8a3 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php @@ -32,17 +32,17 @@ public function getFieldEnumMap() KalturaESearchEntryAggregateByFieldName::ACCESS_CONTROL_PROFILE => ESearchEntryAggregationFieldName::ACCESS_CONTROL_PROFILE); } - public function coreToApiResponse($coreRespone) + public function coreToApiResponse($coreResponse) { $bucketsArray = new KalturaESearchAggregationBucketsArray(); - $buckets = $coreRespone['buckets']; + $buckets = $coreResponse[ESearchAggregations::BUCKETS]; if ($buckets) { foreach ($buckets as $bucket) { - $reponseBucket = new KalturaESearchAggregationBucket(); - $reponseBucket->fromArray($bucket); - $bucketsArray[] = $reponseBucket; + $responseBucket = new KalturaESearchAggregationBucket(); + $responseBucket->fromArray($bucket); + $bucketsArray[] = $responseBucket; } } return $bucketsArray; diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php index 4fbe2791faf..671bbd7c63d 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -26,17 +26,33 @@ public function getFieldEnumMap() return array (); } + protected function getMetadataFieldNameFromXpath($xpath) + { + $token = explode("'", $xpath); + return $token[3]; + } + public function coreToApiResponse($coreRespone) { $bucketsArray = new KalturaESearchAggregationBucketsArray(); - $buckets = $coreRespone['NestedBucket']['buckets']; + $buckets = $coreRespone[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; if ($buckets) { foreach ($buckets as $bucket) { - $reponseBucket = new KalturaESearchAggregationBucket(); - $reponseBucket->fromArray($bucket); - $bucketsArray[] = $reponseBucket; + //get the field name from the xpath + $metadataFieldName = $this->getMetadataFieldNameFromXpath($bucket[ESearchAggregations::KEY]); + + // loop over the subaggs + $subBuckets = $bucket[ESearchMetadataAggregationItem::SUB_AGG][ESearchAggregations::BUCKETS]; + foreach($subBuckets as $subBucket) + { + $responseBucket = new KalturaESearchAggregationBucket(); + $responseBucket->value = $metadataFieldName.':'. $subBucket[ESearchAggregations::KEY]; + $responseBucket->count = $subBucket[ESearchAggregations::DOC_COUNT]; + $bucketsArray[] = $responseBucket; + } + } } return $bucketsArray; diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucket.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucket.php index 85ce5da7e02..3e00cd1dc14 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucket.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucket.php @@ -6,7 +6,7 @@ class KalturaESearchAggregationBucket extends KalturaObject { /** - * @var KalturaString + * @var string */ public $value; @@ -17,8 +17,8 @@ class KalturaESearchAggregationBucket extends KalturaObject private static $map_between_objects = array( - 'value'=>'key', - 'count'=>'doc_count' + 'value'=>ESearchAggregations::KEY, + 'count'=>ESearchAggregations::DOC_COUNT ); protected function getMapBetweenObjects() diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php index 3268607d86b..612c81ef0b0 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php @@ -7,6 +7,6 @@ class KalturaESearchAggregationBucketsArray extends KalturaTypedArray { public function __construct() { - parent::__construct("KalturaESearchAggregationBucket"); + parent::__construct("KalturaESearchAggregationBucket"); } } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php index cd034af009c..2c96564e867 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php @@ -36,28 +36,6 @@ public function resultToApi($aggregationResults) $agg->buckets = $objectItemHandler->coreToApiResponse($response); $aggs[] = $agg; } -/* - $buckets=null; - if(isset($response['buckets'])) - { - $buckets = $response['buckets']; - } - elseif (isset($response['NestedBucket'])) - { - $buckets = $response['NestedBucket']['buckets']; - } - if ($buckets) - { - $agg->buckets = new KalturaESearchAggregationBucketsArray(); - foreach ($buckets as $bucket) - { - $reponseBucket = new KalturaESearchAggregationBucket(); - $reponseBucket->fromArray($bucket); - $agg->buckets[] = $reponseBucket; - } - } - $aggs[] = $agg; - }*/ return $aggs; } diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseItem.php index 6987c0f12c2..4325a1282f7 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponseItem.php @@ -6,12 +6,12 @@ class KalturaESearchAggregationResponseItem extends KalturaObject { /** - * @var KalturaString + * @var string */ public $name; /** - * @var KalturaString + * @var string */ public $fieldName; diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/ESearchAggregations.php b/plugins/search/providers/elastic_search/lib/model/aggregations/ESearchAggregations.php index f238ec847e6..39f9d89c9c8 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/ESearchAggregations.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/ESearchAggregations.php @@ -5,6 +5,16 @@ */ class ESearchAggregations extends BaseObject { + const BUCKET = 'bucket'; + const DOC_COUNT = 'doc_count'; + const TERMS = 'terms'; + const SIZE = 'size'; + const AGGS = 'aggs'; + const FIELD = 'field'; + const NESTED = 'nested'; + const PATH = 'path'; + const KEY = 'key'; + const BUCKETS = 'buckets'; /** * @var array */ diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php index 6f786094079..926650e71cd 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php @@ -5,9 +5,8 @@ */ abstract class ESearchAggregationItem extends BaseObject { - const SIZE = 'size'; - const DEFAULT_SIZE = 5; + const NESTED_BUCKET = 'NestedBucket'; /** * @var int @@ -46,14 +45,11 @@ public function setFieldName($fieldName) $this->fieldName = $fieldName; } - public abstract function getAggregationCommand(); - - public abstract function getAggregationKey(); - - public function getCommandKey() + public function getAggregationCommand() { - return $this->getAggregationKey().':'.$this->getFieldName(); + return array(ESearchAggregations::TERMS => + array(ESearchAggregations::FIELD => $this->fieldName, ESearchAggregations::SIZE => $this->getSize())); } - + public abstract function getAggregationKey(); } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCategoryAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCategoryAggregationItem.php index 1aca52d99c6..e58b77fe298 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCategoryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCategoryAggregationItem.php @@ -17,11 +17,4 @@ public function getAggregationKey() { return self::KEY; } - - public function getAggregationCommand() - { - return array('terms' => - array('field' => $this->fieldName, - 'size' =>$this->getSize())); - } } diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php index 0afcaa74ab4..1d9be678c2d 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchCuepointsAggregationItem.php @@ -14,12 +14,11 @@ class ESearchCuepointsAggregationItem extends ESearchAggregationItem const KEY = 'cue_points'; public function getAggregationCommand() { - return array ('nested' => - array('path' => "cue_points"), - 'aggs'=>array('NestedBucket' => - array('terms' => - array ('field' => $this->getFieldName(), - 'size' =>$this->getSize())))); + return array (ESearchAggregations::NESTED => + array(ESearchAggregations::PATH => "cue_points"), + ESearchAggregations::AGGS=>array(self::NESTED_BUCKET => + parent::getAggregationCommand() + )); } public function getAggregationKey() diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php index f76b4d510e5..c28c904e8fb 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchEntryAggregationItem.php @@ -13,12 +13,6 @@ class ESearchEntryAggregationItem extends ESearchAggregationItem const KEY = 'entries'; - public function getAggregationCommand() - { - return array('terms' => - array('field' => $this->fieldName, - 'size' =>$this->getSize())); - } public function getAggregationKey() { return self::KEY; diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchMetadataAggregationItem.php index e73db305284..279fbb78b12 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchMetadataAggregationItem.php @@ -7,15 +7,22 @@ class ESearchMetadataAggregationItem extends ESearchAggregationItem { const KEY = 'metadata'; + const SUB_AGG = 'subagg'; public function getAggregationCommand() { - return array ('nested' => - array('path' => "metadata"), - 'aggs'=>array('NestedBucket' => - array('terms' => - array ('field' => 'metadata.value_text.raw' , - 'size' =>$this->getSize())))); + + return array (ESearchAggregations::NESTED => + array(ESearchAggregations::PATH => "metadata"), + ESearchAggregations::AGGS=> + array(self::NESTED_BUCKET => + array( + ESearchAggregations::TERMS => + array (ESearchAggregations::FIELD => 'metadata.xpath', ESearchAggregations::SIZE =>$this->getSize()), + ESearchAggregations::AGGS => + array (self::SUB_AGG => + array(ESearchAggregations::TERMS => + array (ESearchAggregations::FIELD => 'metadata.value_text.raw', ESearchAggregations::SIZE =>$this->getSize())))))); } public function getAggregationKey() From 12e7129d6263f31032136f34532a88a888028d04 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 4 Jun 2019 19:50:14 +0300 Subject: [PATCH 026/103] testing --- .../items/KalturaESearchMetadataAggregationItem.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php index 671bbd7c63d..8029d0cd4cb 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -32,10 +32,10 @@ protected function getMetadataFieldNameFromXpath($xpath) return $token[3]; } - public function coreToApiResponse($coreRespone) + public function coreToApiResponse($coreResponse) { $bucketsArray = new KalturaESearchAggregationBucketsArray(); - $buckets = $coreRespone[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; + $buckets = $coreResponse[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; if ($buckets) { foreach ($buckets as $bucket) From 99504198f40e671a3fe9d881435ac81f41a04838 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Tue, 4 Jun 2019 19:52:34 +0300 Subject: [PATCH 027/103] testing --- .../types/aggregations/items/KalturaESearchAggregationItem.php | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php index fd5d0f3dc8c..66f68fc075f 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php @@ -36,5 +36,6 @@ public function toObject($object_to_fill = null, $props_to_skip = array()) return parent::toObject($object_to_fill, $props_to_skip); } + abstract public function coreToApiResponse($coreResponse); } \ No newline at end of file From 7a9136014d4d917c786906b52f6ff32a26658a1e Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Wed, 5 Jun 2019 06:08:42 +0300 Subject: [PATCH 028/103] Changing values coreToApi --- .../aggregations/items/KalturaESearchAggregationItem.php | 2 +- .../items/KalturaESearchCategoryAggregationItem.php | 2 +- .../items/KalturaESearchCuepointsAggregationItem.php | 6 +++++- .../items/KalturaESearchEntryAggregationItem.php | 2 +- .../items/KalturaESearchMetadataAggregationItem.php | 2 +- .../response/KalturaESearchAggregationResponse.php | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php index 66f68fc075f..0d967654f2e 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php @@ -36,6 +36,6 @@ public function toObject($object_to_fill = null, $props_to_skip = array()) return parent::toObject($object_to_fill, $props_to_skip); } - abstract public function coreToApiResponse($coreResponse); + abstract public function coreToApiResponse($coreResponse,$fieldName = null); } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php index 1c977be506c..fcc8e93d4d9 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php @@ -31,7 +31,7 @@ protected function fixCategoryName($coreName) } - public function coreToApiResponse($coreResponse) + public function coreToApiResponse($coreResponse, $fieldName = null) { $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregations::BUCKETS]; diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php index 2ccd508b589..a481c5e5e2d 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php @@ -27,7 +27,7 @@ public function getFieldEnumMap() KalturaESearchCuePointAggregateByFieldName::TYPE => ESearchCuePointsAggregationFieldName::TYPE); } - public function coreToApiResponse($coreResponse) + public function coreToApiResponse($coreResponse, $fieldName = null) { $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; @@ -37,6 +37,10 @@ public function coreToApiResponse($coreResponse) { $responseBucket = new KalturaESearchAggregationBucket(); $responseBucket->fromArray($bucket); + if($fieldName === ESearchCuePointsAggregationFieldName::TYPE) + { + $responseBucket->value = kPluginableEnumsManager::coreToApi('CuePointType',$responseBucket->value); + } $bucketsArray[] = $responseBucket; } } diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php index 28faef5f8a3..aed6a012c1e 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php @@ -32,7 +32,7 @@ public function getFieldEnumMap() KalturaESearchEntryAggregateByFieldName::ACCESS_CONTROL_PROFILE => ESearchEntryAggregationFieldName::ACCESS_CONTROL_PROFILE); } - public function coreToApiResponse($coreResponse) + public function coreToApiResponse($coreResponse, $fieldName = null) { $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregations::BUCKETS]; diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php index 8029d0cd4cb..1af33fb5a64 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -32,7 +32,7 @@ protected function getMetadataFieldNameFromXpath($xpath) return $token[3]; } - public function coreToApiResponse($coreResponse) + public function coreToApiResponse($coreResponse, $fieldName=null) { $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php index 2c96564e867..144eb4be6b2 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php @@ -33,7 +33,7 @@ public function resultToApi($aggregationResults) $itemObjectName = $this->mapAggregationCoreObjects($responseObject); $objectItemHandler = new $itemObjectName(); - $agg->buckets = $objectItemHandler->coreToApiResponse($response); + $agg->buckets = $objectItemHandler->coreToApiResponse($response,$fieldName); $aggs[] = $agg; } return $aggs; From 5bd1ab59096be61d877630f2c8c55f2cf639858b Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Wed, 5 Jun 2019 07:35:35 +0300 Subject: [PATCH 029/103] testing --- .../items/KalturaESearchAggregationsArray.php | 43 +------------------ 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationsArray.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationsArray.php index 7bbb83af295..09ca6d2fac4 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationsArray.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationsArray.php @@ -6,49 +6,8 @@ class KalturaESearchAggregationsArray extends KalturaTypedArray { - public function __construct() { return parent::__construct("KalturaESearchAggregationItem"); } - - public static function fromDbArray($arr, KalturaDetachedResponseProfile $responseProfile = null) - { - KalturaLog::debug(print_r($arr, true)); - $newArr = new KalturaESearchAggregationsArray(); - if ($arr == null) - return $newArr; - - foreach ($arr as $obj) - { - switch (get_class($obj)) - { - case 'ESearchCategoryAggregationItem': - $nObj = new KalturaESearchCategoryAggregationItem(); - break; - - case 'ESearchMetadataAggregationItem': - $nObj = new KalturaESearchMetadataAggregationItem(); - break; - - case 'ESearchEntryAggregationItem': - $nObj = new KalturaESearchEntryAggregationItem(); - break; - - case 'ESearchCuepointsAggregationItem': - $nObj = new KalturaESearchCuepointsAggregationItem (); - break; - - default: - $nObj = KalturaPluginManager::loadObject('KalturaESearchAggregationItem', get_class($obj)); - break; - } - - $nObj->fromObject($obj, $responseProfile); - $newArr[] = $nObj; - } - - return $newArr; - } - -} \ No newline at end of file +} From a69d456554ad6a97f66094757baeae80007d0606 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Wed, 5 Jun 2019 17:54:34 +0300 Subject: [PATCH 030/103] testing --- .../KalturaESearchCategoryAggregationItem.php | 6 +++++- .../KalturaESearchCuepointsAggregationItem.php | 7 ++++++- .../items/KalturaESearchEntryAggregationItem.php | 7 ++++++- .../KalturaESearchMetadataAggregationItem.php | 14 +++++++++----- .../response/KalturaESearchAggregationResponse.php | 12 ++++++------ 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php index fcc8e93d4d9..8a1cc2af124 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php @@ -33,6 +33,9 @@ protected function fixCategoryName($coreName) public function coreToApiResponse($coreResponse, $fieldName = null) { + $agg = new KalturaESearchAggregationResponseItem(); + $agg->fieldName = $fieldName; + $agg->name = 'category'; $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregations::BUCKETS]; if ($buckets) @@ -47,7 +50,8 @@ public function coreToApiResponse($coreResponse, $fieldName = null) } } - return $bucketsArray; + $agg->buckets = $bucketsArray; + return array($agg); } } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php index a481c5e5e2d..3a5edaf8072 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php @@ -29,6 +29,10 @@ public function getFieldEnumMap() public function coreToApiResponse($coreResponse, $fieldName = null) { + $agg = new KalturaESearchAggregationResponseItem(); + $agg->fieldName = $fieldName; + $agg->name = 'cuepoints'; + $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; if ($buckets) @@ -44,6 +48,7 @@ public function coreToApiResponse($coreResponse, $fieldName = null) $bucketsArray[] = $responseBucket; } } - return $bucketsArray; + $agg->buckets = $bucketsArray; + return ($agg); } } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php index aed6a012c1e..44e9175f55a 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php @@ -34,10 +34,14 @@ public function getFieldEnumMap() public function coreToApiResponse($coreResponse, $fieldName = null) { + $agg = new KalturaESearchAggregationResponseItem(); + $agg->fieldName = $fieldName; + $agg->name = 'entry'; $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregations::BUCKETS]; if ($buckets) { + foreach ($buckets as $bucket) { $responseBucket = new KalturaESearchAggregationBucket(); @@ -45,7 +49,8 @@ public function coreToApiResponse($coreResponse, $fieldName = null) $bucketsArray[] = $responseBucket; } } - return $bucketsArray; + $agg->buckets = $bucketsArray; + return array($agg); } diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php index 1af33fb5a64..ee0454f7f78 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -34,28 +34,32 @@ protected function getMetadataFieldNameFromXpath($xpath) public function coreToApiResponse($coreResponse, $fieldName=null) { + $ret = array(); $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; if ($buckets) { foreach ($buckets as $bucket) { + $agg = new KalturaESearchAggregationResponseItem(); + $agg->name = 'metadata'; + //get the field name from the xpath $metadataFieldName = $this->getMetadataFieldNameFromXpath($bucket[ESearchAggregations::KEY]); - + $agg->fieldName = $metadataFieldName; // loop over the subaggs $subBuckets = $bucket[ESearchMetadataAggregationItem::SUB_AGG][ESearchAggregations::BUCKETS]; foreach($subBuckets as $subBucket) { $responseBucket = new KalturaESearchAggregationBucket(); - $responseBucket->value = $metadataFieldName.':'. $subBucket[ESearchAggregations::KEY]; - $responseBucket->count = $subBucket[ESearchAggregations::DOC_COUNT]; + $responseBucket->fromArray($subBucket); $bucketsArray[] = $responseBucket; } - + $agg->buckets = $bucketsArray; + $ret[] = $agg; } } - return $bucketsArray; + return $ret; } diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php index 144eb4be6b2..2e541a3af80 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php @@ -27,14 +27,14 @@ public function resultToApi($aggregationResults) foreach ($aggregationResults as $key=>$response) { list ($responseObject, $fieldName) = $this->getApiObjects($key); - $agg = new KalturaESearchAggregationResponseItem(); - $agg->fieldName = $fieldName; - $agg->name = $responseObject; - $itemObjectName = $this->mapAggregationCoreObjects($responseObject); $objectItemHandler = new $itemObjectName(); - $agg->buckets = $objectItemHandler->coreToApiResponse($response,$fieldName); - $aggs[] = $agg; + $aggsResponses = $objectItemHandler->coreToApiResponse($response,$fieldName); + foreach ($aggsResponses as $aggsResponse) + { + $aggs[] = $aggsResponse; + } + } return $aggs; } From 9cb8bcd4e034e40333b34547e74bb82c0b7d0e27 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Thu, 6 Jun 2019 01:15:00 +0300 Subject: [PATCH 031/103] testing --- .../items/KalturaESearchCategoryAggregationItem.php | 2 +- .../items/KalturaESearchCuepointsAggregationItem.php | 2 +- .../aggregations/items/KalturaESearchEntryAggregationItem.php | 2 +- .../items/KalturaESearchMetadataAggregationItem.php | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php index 8a1cc2af124..15f2f27225d 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php @@ -35,7 +35,7 @@ public function coreToApiResponse($coreResponse, $fieldName = null) { $agg = new KalturaESearchAggregationResponseItem(); $agg->fieldName = $fieldName; - $agg->name = 'category'; + $agg->name = ESearchCategoryAggregationItem::KEY; $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregations::BUCKETS]; if ($buckets) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php index 3a5edaf8072..64743329f2d 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php @@ -31,7 +31,7 @@ public function coreToApiResponse($coreResponse, $fieldName = null) { $agg = new KalturaESearchAggregationResponseItem(); $agg->fieldName = $fieldName; - $agg->name = 'cuepoints'; + $agg->name = ESearchCuepointsAggregationItem::KEY; $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php index 44e9175f55a..9f5ff85850c 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php @@ -36,7 +36,7 @@ public function coreToApiResponse($coreResponse, $fieldName = null) { $agg = new KalturaESearchAggregationResponseItem(); $agg->fieldName = $fieldName; - $agg->name = 'entry'; + $agg->name = ESearchEntryAggregationItem::KEY; $bucketsArray = new KalturaESearchAggregationBucketsArray(); $buckets = $coreResponse[ESearchAggregations::BUCKETS]; if ($buckets) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php index ee0454f7f78..9fcdf654361 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -42,7 +42,7 @@ public function coreToApiResponse($coreResponse, $fieldName=null) foreach ($buckets as $bucket) { $agg = new KalturaESearchAggregationResponseItem(); - $agg->name = 'metadata'; + $agg->name = ESearchMetadataAggregationItem::KEY; //get the field name from the xpath $metadataFieldName = $this->getMetadataFieldNameFromXpath($bucket[ESearchAggregations::KEY]); From d7ab99fac3b62325e524fc64f1d3306530f50404 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Thu, 6 Jun 2019 05:03:39 +0300 Subject: [PATCH 032/103] integration tests --- .../lib/api/types/KalturaESearchEntryParams.php | 8 +++++++- .../lib/api/types/KalturaESearchParams.php | 7 +------ .../errors/KalturaESearchAggregationErrors.php | 10 ++++++++++ .../items/KalturaESearchAggregationItem.php | 8 ++++++++ .../items/KalturaESearchMetadataAggregationItem.php | 10 +++++++--- .../elastic_search/lib/model/search/kBaseSearch.php | 2 +- 6 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 plugins/search/providers/elastic_search/lib/api/types/aggregations/errors/KalturaESearchAggregationErrors.php diff --git a/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchEntryParams.php b/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchEntryParams.php index ffdf7ee1c9e..9185180914a 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchEntryParams.php +++ b/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchEntryParams.php @@ -10,9 +10,15 @@ class KalturaESearchEntryParams extends KalturaESearchParams */ public $searchOperator; + /** + * @var KalturaESearchAggregation + */ + public $aggregations; + + private static $mapBetweenObjects = array ( - "searchOperator", + "searchOperator","aggregations" ); protected function initStatuses() diff --git a/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php b/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php index 05c45542edf..e13ef78aea6 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php +++ b/plugins/search/providers/elastic_search/lib/api/types/KalturaESearchParams.php @@ -20,17 +20,12 @@ abstract class KalturaESearchParams extends KalturaObject */ public $orderBy; - /** - * @var KalturaESearchAggregation - */ - public $aggregations; private static $mapBetweenObjects = array ( "objectStatuses", "objectId", - "orderBy", - "aggregations" + "orderBy" ); public function getMapBetweenObjects() diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/errors/KalturaESearchAggregationErrors.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/errors/KalturaESearchAggregationErrors.php new file mode 100644 index 00000000000..6fba065aaf3 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/errors/KalturaESearchAggregationErrors.php @@ -0,0 +1,10 @@ +fieldName)) + { + throw new KalturaAPIException(KalturaESearchAggregationErrors::FIELD_NAME_MUST_BE_SUPPLIED); + } + parent::validateForUsage($sourceObject, $propertiesToSkip); + } } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php index 9fcdf654361..bdd17b20684 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -35,7 +35,7 @@ protected function getMetadataFieldNameFromXpath($xpath) public function coreToApiResponse($coreResponse, $fieldName=null) { $ret = array(); - $bucketsArray = new KalturaESearchAggregationBucketsArray(); + $buckets = $coreResponse[ESearchAggregationItem::NESTED_BUCKET][ESearchAggregations::BUCKETS]; if ($buckets) { @@ -43,11 +43,11 @@ public function coreToApiResponse($coreResponse, $fieldName=null) { $agg = new KalturaESearchAggregationResponseItem(); $agg->name = ESearchMetadataAggregationItem::KEY; - //get the field name from the xpath $metadataFieldName = $this->getMetadataFieldNameFromXpath($bucket[ESearchAggregations::KEY]); $agg->fieldName = $metadataFieldName; - // loop over the subaggs + // loop over the sub aggregations + $bucketsArray = new KalturaESearchAggregationBucketsArray(); $subBuckets = $bucket[ESearchMetadataAggregationItem::SUB_AGG][ESearchAggregations::BUCKETS]; foreach($subBuckets as $subBucket) { @@ -62,5 +62,9 @@ public function coreToApiResponse($coreResponse, $fieldName=null) return $ret; } + public function validateForUsage($sourceObject, $propertiesToSkip = array()) + { + KalturaObject::validateForUsage($sourceObject, $propertiesToSkip); + } } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php b/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php index 3c53934588d..ca6e34abef5 100644 --- a/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php +++ b/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php @@ -23,7 +23,7 @@ public function __construct() $this->forceInnerHitsSizeOverride = false; } - public abstract function doSearch(ESearchOperator $eSearchOperator, kPager $pager = null, $statuses = array(), $objectId = null, ESearchOrderBy $order = null, ESearchAggregations $aggregations = null); + public abstract function doSearch(ESearchOperator $eSearchOperator, kPager $pager = null, $statuses = array(), $objectId = null, ESearchOrderBy $order = null); /** * @return ESearchQueryAttributes From e160a3d30f4548dc8e3112f886cef2ac0017a405 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Thu, 6 Jun 2019 05:06:25 +0300 Subject: [PATCH 033/103] integration tests --- .../aggregations/errors/KalturaESearchAggregationErrors.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/errors/KalturaESearchAggregationErrors.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/errors/KalturaESearchAggregationErrors.php index 6fba065aaf3..a44bb3dce6b 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/errors/KalturaESearchAggregationErrors.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/errors/KalturaESearchAggregationErrors.php @@ -6,5 +6,5 @@ class KalturaESearchAggregationErrors extends KalturaESearchErrors { - const FIELD_NAME_MUST_BE_SUPPLIED = 'FIELD_NAME_MUST_BE_SUPPLIED;;field name must be supplied'; + const AGGREGATION_FIELD_NAME_MUST_BE_SUPPLIED = 'AGGREGATION_FIELD_NAME_MUST_BE_SUPPLIED;;Aggregation field name must be supplied'; } \ No newline at end of file From b928ebfab585373519a2261cc7ea452603531231 Mon Sep 17 00:00:00 2001 From: MosheMaorKaltura Date: Thu, 6 Jun 2019 05:06:57 +0300 Subject: [PATCH 034/103] integration tests --- .../types/aggregations/items/KalturaESearchAggregationItem.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php index bccfdc0688f..cd437a841ea 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php @@ -42,7 +42,7 @@ public function validateForUsage($sourceObject, $propertiesToSkip = array()) { if(is_null($this->fieldName)) { - throw new KalturaAPIException(KalturaESearchAggregationErrors::FIELD_NAME_MUST_BE_SUPPLIED); + throw new KalturaAPIException(KalturaESearchAggregationErrors::AGGREGATION_FIELD_NAME_MUST_BE_SUPPLIED); } parent::validateForUsage($sourceObject, $propertiesToSkip); } From 44f4ea84fa40026b0b4cfd69bfaaa24a3c33bbb4 Mon Sep 17 00:00:00 2001 From: hilak Date: Fri, 7 Jun 2019 17:22:39 -0400 Subject: [PATCH 035/103] Entry creation date update script. --- alpha/scripts/utils/setEntryCreationDate.php | 40 ++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 alpha/scripts/utils/setEntryCreationDate.php diff --git a/alpha/scripts/utils/setEntryCreationDate.php b/alpha/scripts/utils/setEntryCreationDate.php new file mode 100644 index 00000000000..0e7232d252e --- /dev/null +++ b/alpha/scripts/utils/setEntryCreationDate.php @@ -0,0 +1,40 @@ + \n"; + exit; +} +$mapping = $argv[1]; +$isDateString = ($argv[2] == "true"); +$dryRun = ($argv[3] == "dryrun"); + +require_once(__DIR__ . '/../bootstrap.php'); + +KalturaStatement::setDryRun($dryRun); + +$entryMappings = file($mapping, FILE_IGNORE_NEW_LINES); + +$counter = 0; +foreach ($entryMappings as $entryMapping) +{ + list ($entryId,$createdAt) = explode(",", $entryMapping); + $createdAt = $isDateString ? strtotime($createdAt) : $createdAt; + + $entry = entryPeer::retrieveByPK($entryId); + if(!$entry) + { + echo "Entry id [$entryId] not found\n"; + continue; + } + + $entry->setCreatedAt($createdAt); + $entry->setAvailableFrom($createdAt); + $entry->save(); + + kEventsManager::flushEvents(); + kMemoryManager::clearMemory(); +} + +KalturaLog::debug('Done'); From 834e8fe074f3ff691cd3a88e6328d1517b39a562 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Mon, 1 Jul 2019 14:27:46 +0300 Subject: [PATCH 036/103] SUP-16933:use eSearch instead of sphinx for playlist execure from filter --- .../kaltura/lib/myPlaylistUtils.class.php | 88 ++++++- api_v3/lib/KalturaErrors.php | 2 + api_v3/services/PlaylistService.php | 14 +- .../elastic/mapping/entry_mapping.json | 3 +- .../api/enum/KalturaESearchEntryFieldName.php | 1 + .../items/entry/KalturaESearchEntryItem.php | 1 + .../lib/model/enum/ESearchEntryFieldName.php | 1 + .../model/enum/ESearchEntryFilterFields.php | 54 ++++ .../lib/model/enum/ESearchFilterItemType.php | 9 + .../filters/ESearchCaptionQueryFromFilter.php | 2 +- .../filters/ESearchEntryQueryFromFilter.php | 238 ++++++++++++++++++ .../model/filters/ESearchQueryFromFilter.php | 188 +++++++++++--- .../lib/model/items/ESearchEntryItem.php | 1 + 13 files changed, 563 insertions(+), 39 deletions(-) create mode 100644 plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php create mode 100644 plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index 5851119524a..4e4ced5b11d 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -26,6 +26,10 @@ public static function setIsAdminKs($v) self::$isAdminKs = $v; } + protected static $moderationStatusesNotIn = array( + entry::ENTRY_MODERATION_STATUS_PENDING_MODERATION, + entry::ENTRY_MODERATION_STATUS_REJECTED); + /** * Playlist is an entry of type ENTRY_TYPE_PLAYLIST = 5. * Within this type there are 3 media_types to tell the difference between dynamic,static and external playslits: @@ -483,6 +487,78 @@ public static function getDynamicPlaylistFilters($xml) } return $entry_filters; } + + public static function executeDynamicPlaylistViaEsearch ($partnerId , $xml , $pager = null) + { + list ($totalResults, $listOfFilters) = self::getPlaylistFilterListStruct($xml); + if (!$listOfFilters) + { + return null; + } + $entry_filters = self::fillEntryFilterFromXml($listOfFilters, $partnerId); + $entryKPager = new kPager(); + if ($pager) + { + $pager->toObject($entryKPager); + } + $entryQueryToFilterESearch = new ESearchEntryQueryFromFilter(); + $entryIds= array(); + foreach ($entry_filters as $entry_filter) + { + list ($currEntryIds, $count) = $entryQueryToFilterESearch->retrieveElasticQueryEntryIds($entry_filter, $entryKPager); + if (count($currEntryIds) > $entry_filter->get('_limit')) + { + $currEntryIds = array_slice($currEntryIds,0,$entry_filter->get('_limit')); + } + $entryIds = array_merge ($entryIds, $currEntryIds); + } + return self::getEntriesFromEntryIds(array_unique($entryIds)); + } + + protected static function getEntriesFromEntryIds($entryIds) + { + $entriesResult = array(); + foreach ($entryIds as $entryId) + { + $dbEntry = entryPeer::retrieveByPK($entryId); + if ($dbEntry) + { + $entriesResult[] = $dbEntry; + } + } + return $entriesResult; + } + + protected static function fillEntryFilterFromXml($list_of_filters, $partnerId) + { + $entry_filters = array(); + for ($i = 0; $i < count($list_of_filters) ; $i++) + { + $entry_filter_xml = $list_of_filters[$i]; + self::replaceContextTokens($entry_filter_xml); + $entry_filter = new entryFilter(); + $entry_filter->fillObjectFromXml($entry_filter_xml, "_", null); + self::updateEntryFilterFields($entry_filter, $partnerId); + $entry_filters[] = $entry_filter; + } + return $entry_filters; + } + + protected static function updateEntryFilterFields($entryFilter, $partnerId) + { + self::updateEntryFilter($entryFilter, $partnerId); + $typeArray = array (entryType::MEDIA_CLIP, entryType::MIX, entryType::LIVE_STREAM ); + $typeArray = array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::MEDIA_CLIP)); + $typeArray = array_unique(array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::LIVE_STREAM))); + $entryFilter->set ( "_in_type" , implode(',',$typeArray) ); + $entryFilter->set( "_eq_status" , entryStatus::READY ); //needed? + $entryFilter->set("_notin_moderation_status", self::$moderationStatusesNotIn); + if (!self::$isAdminKs) + { + self::addSchedulingToCriteria(null, $entryFilter, true); + } + } + public static function executeDynamicPlaylist ( $partner_id , $xml , $filter = null ,$detailed = true, $pager = null ) { @@ -946,7 +1022,7 @@ private static function addSchedulingCriterion(Criteria $c, $field, $min, $max, $c->addAnd($criterion); } - private static function addSchedulingToCriteria(Criteria $c, entryFilter $filter = null) + private static function addSchedulingToCriteria($c, entryFilter $filter = null, $isEsearch = false) { $min = 0; $max = kApiCache::getTime(); @@ -972,7 +1048,10 @@ private static function addSchedulingToCriteria(Criteria $c, entryFilter $filter $filter->unsetByName('_gte_start_date'); } } - self::addSchedulingCriterion($c, entryPeer::START_DATE, $min, $max, $allowNull); + if(!$isEsearch) + { + self::addSchedulingCriterion($c, entryPeer::START_DATE, $min, $max, $allowNull); + } $min = kApiCache::getTime(); @@ -999,7 +1078,10 @@ private static function addSchedulingToCriteria(Criteria $c, entryFilter $filter $filter->unsetByName('_gte_end_date'); } } - self::addSchedulingCriterion($c, entryPeer::END_DATE, $min, $max, $allowNull); + if(!$isEsearch) + { + self::addSchedulingCriterion($c, entryPeer::END_DATE, $min, $max, $allowNull); + } } private static function addModerationToCriteria(Criteria $c) diff --git a/api_v3/lib/KalturaErrors.php b/api_v3/lib/KalturaErrors.php index 03edd0cc55c..e71b3ff0502 100644 --- a/api_v3/lib/KalturaErrors.php +++ b/api_v3/lib/KalturaErrors.php @@ -753,5 +753,7 @@ class KalturaErrors extends APIErrors const MAP_ALREADY_EXIST = "MAP_ALREADY_EXIST;NAME,HOST;Map already exist for this map name {@NAME@} and host {@HOST@}"; const MAP_CANNOT_BE_CREATED_ON_FILE_SYSTEM = "MAP_CANNOT_BE_CREATED_ON_FILE_SYSTEM;;Map cannnot be created on file system"; const HOST_NAME_CONTAINS_ASTRIX = "HOST_NAME_CONTAINS_ASTRIX;HOST_NAME;Host name contains *, use # instead {@HOST_NAME@}"; + + const SEARCH_ITEM_TYPE_NOT_FOUND = 'SEARCH_ITEM_TYPE_NOT_FOUND;SEARCH_ITEM_TYPE,ELASTIC_FIELD_NAME; Search item type [@SEARCH_ITEM_TYPE@] not found for field: [@ELASTIC_FIELD_NAME@]'; } diff --git a/api_v3/services/PlaylistService.php b/api_v3/services/PlaylistService.php index b15269e5da1..cb6e8a9c04b 100644 --- a/api_v3/services/PlaylistService.php +++ b/api_v3/services/PlaylistService.php @@ -351,7 +351,19 @@ function executeFromContentAction($playlistType, $playlistContent, $detailed = f $entryList = array(); if ($playlistType == KalturaPlaylistType::DYNAMIC) - $entryList = myPlaylistUtils::executeDynamicPlaylist($this->getPartnerId(), $playlistContent, null, true, $pager); + { + if (strpos($playlistContent, 'advancedSearch') === false || + (strpos($playlistContent, 'advancedSearch') && strpos($playlistContent, 'KalturaMetadataSearchItem')) ) + { + $entryList = myPlaylistUtils::executeDynamicPlaylistViaEsearch($this->getPartnerId(), $playlistContent, $pager); + } + else + { + $entryList = myPlaylistUtils::executeDynamicPlaylist($this->getPartnerId(), $playlistContent, null, true, $pager); + } + + } + else if ($playlistType == KalturaPlaylistType::STATIC_LIST) $entryList = myPlaylistUtils::executeStaticPlaylistFromEntryIdsString($playlistContent, null, true, $pager); diff --git a/configurations/elastic/mapping/entry_mapping.json b/configurations/elastic/mapping/entry_mapping.json index 88a3ce8ce25..d1bf9a8deb9 100644 --- a/configurations/elastic/mapping/entry_mapping.json +++ b/configurations/elastic/mapping/entry_mapping.json @@ -626,8 +626,7 @@ "normalizer" : "kaltura_keyword_normalizer" }, "partner_sort_value": { - "type" : "keyword", - "normalizer" : "kaltura_keyword_normalizer" + "type" : "integer" }, "redirect_entry_id": { "type" : "keyword", diff --git a/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php b/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php index 3c590e32625..be898c4a6a9 100644 --- a/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php +++ b/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php @@ -39,4 +39,5 @@ class KalturaESearchEntryFieldName extends KalturaStringEnum const IS_LIVE = 'is_live'; const USER_NAMES = "user_names"; const ROOT_ID = 'root_id'; + const PARTNER_SORT_VALUE = 'partner_sort_value'; } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php b/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php index 89124cfc9bc..6c27a232b31 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php @@ -58,6 +58,7 @@ class KalturaESearchEntryItem extends KalturaESearchAbstractEntryItem KalturaESearchEntryFieldName::IS_LIVE => ESearchEntryFieldName::IS_LIVE, KalturaESearchEntryFieldName::USER_NAMES => ESearchEntryFieldName::USER_NAMES, KalturaESearchEntryFieldName::ROOT_ID => ESearchEntryFieldName::ROOT_ID, + KalturaESearchEntryFieldName::PARTNER_SORT_VALUE => ESearchEntryFieldName::PARTNER_SORT_VALUE ); protected function getMapBetweenObjects() diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php index de84c28d317..4378a5f1fe0 100644 --- a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php @@ -41,4 +41,5 @@ interface ESearchEntryFieldName extends BaseEnum const DISPLAY_IN_SEARCH = 'display_in_search'; const ROOT_ID = 'root_id'; const PRIVACY_BY_CONTEXTS = 'privacy_by_contexts'; + const PARTNER_SORT_VALUE = 'partner_sort_value'; } diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php new file mode 100644 index 00000000000..4f4a93663b6 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php @@ -0,0 +1,54 @@ +updateEntryPager($entryPager, $filterOnEntryIds); - $query = $this->createElasticQueryFromFilter($filter); + list($query, $kEsearchOrderBy ) = $this->createElasticQueryFromFilter($filter); $elasticResults = $entrySearch->doSearch($query, $entryPager, self::$validStatuses); diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php new file mode 100644 index 00000000000..e14d5efc6d4 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php @@ -0,0 +1,238 @@ + ESearchEntryFieldName::ID, + ESearchEntryFilterFields::USER_ID => ESearchEntryFieldName::USER_ID, + ESearchEntryFilterFields::GROUP_ID => ESearchEntryFieldName::USER_ID, + ESearchEntryFilterFields::CREATOR_ID => ESearchEntryFieldName::CREATOR_ID, + ESearchEntryFilterFields::NAME => ESearchEntryFieldName::NAME, + ESearchEntryFilterFields::TAGS => ESearchEntryFieldName::TAGS, + ESearchEntryFilterFields::TAGS_NAME => ESearchEntryFieldName::TAGS, + ESearchEntryFilterFields::ADMIN_TAGS => ESearchEntryFieldName::ADMIN_TAGS, + ESearchEntryFilterFields::TAGS_ADMIN_TAGS => ESearchEntryFieldName::ADMIN_TAGS, + ESearchEntryFilterFields::TAGS_ADMIN_TAGS_NAME => ESearchEntryFieldName::ADMIN_TAGS, + ESearchEntryFilterFields::CONVERSION_PROFILE_ID => ESearchEntryFieldName::CONVERSION_PROFILE_ID, + ESearchEntryFilterFields::REDIRECT_ENTRY_ID => ESearchEntryFieldName::REDIRECT_ENTRY_ID, + ESearchEntryFilterFields::ENTITLED_USER_EDIT => ESearchEntryFieldName::ENTITLED_USER_EDIT, + ESearchEntryFilterFields::ENTITLED_USER_PUBLISH => ESearchEntryFieldName::ENTITLED_USER_PUBLISH, + ESearchEntryFilterFields::ENTITLED_USER_VIEW => ESearchEntryFieldName::ENTITLED_USER_VIEW, + ESearchEntryFilterFields::DISPLAY_IN_SEARCH => ESearchEntryFieldName::DISPLAY_IN_SEARCH, + ESearchEntryFilterFields::PARENT_ENTRY_ID => ESearchEntryFieldName::PARENT_ENTRY_ID, + ESearchEntryFilterFields::MEDIA_TYPE => ESearchEntryFieldName::MEDIA_TYPE, + ESearchEntryFilterFields::SOURCE_TYPE => ESearchEntryFieldName::SOURCE_TYPE, + ESearchEntryFilterFields::LENGTH_IN_MSECS => ESearchEntryFieldName::LENGTH_IN_MSECS, + ESearchEntryFilterFields::TYPE => ESearchEntryFieldName::ENTRY_TYPE, + ESearchEntryFilterFields::MODERATION_STATUS => ESearchEntryFieldName::MODERATION_STATUS, + ESearchEntryFilterFields::CREATED_AT => ESearchEntryFieldName::CREATED_AT, + ESearchEntryFilterFields::UPDATED_AT => ESearchEntryFieldName::UPDATED_AT, + ESearchEntryFilterFields::ACCESS_CONTROL_ID => ESearchEntryFieldName::ACCESS_CONTROL_ID, + ESearchEntryFilterFields::USER_NAMES => ESearchEntryFieldName::USER_NAMES, + ESearchEntryFilterFields::START_DATE => ESearchEntryFieldName::START_DATE, + ESearchEntryFilterFields::END_DATE => ESearchEntryFieldName::END_DATE, + ESearchEntryFilterFields::REFERENCE_ID => ESearchEntryFieldName::REFERENCE_ID, + ESearchEntryFilterFields::ROOT_ENTRY_ID => ESearchEntryFieldName::ROOT_ID, + ESearchEntryFilterFields::DURATION => ESearchEntryFieldName::LENGTH_IN_MSECS, + ESearchEntryFilterFields::CATEGORIES => ESearchCategoryEntryNameItem::CATEGORY_NAMES_MAPPING_FIELD, + ESearchEntryFilterFields::CATEGORIES_IDS => ESearchCategoryEntryIdItem::CATEGORY_IDS_MAPPING_FIELD, + ESearchEntryFilterFields::CATEGORIES_ANCESTOR_ID => ESearchCategoryEntryFieldName::ANCESTOR_ID, + ESearchEntryFilterFields::CATEGORIES_FULL_NAME => ESearchCategoryEntryFieldName::FULL_IDS, + ESearchEntryFilterFields::PARTNER_SORT_VALUE => ESearchEntryFieldName::PARTNER_SORT_VALUE, + ESearchEntryFilterFields::SEARCH_TEXT => ESearchUnifiedItem::UNIFIED, + ESearchEntryFilterFields::FREE_TEXT => ESearchUnifiedItem::UNIFIED + ); + + if(array_key_exists($field, $fieldsMap)) + { + return $fieldsMap[$field]; + } + else + { + return null; + } + } + + protected function getSphinxToElasticOrderBy($field) + { + $fieldsMap = array( + ESearchEntryFilterFields::VIEWS => ESearchEntryOrderByFieldName::VIEWS, + ESearchEntryFilterFields::PLAYS => ESearchEntryOrderByFieldName::PLAYS + ); + + if(array_key_exists($field, $fieldsMap)) + { + return $fieldsMap[$field]; + } + else + { + return null; + } + } + + public function createElasticQueryFromFilter(baseObjectFilter $filter) + { + $this->init(); + $kEsearchOrderBy = null; + foreach($filter->fields as $field => $fieldValue) + { + if ($field === entryFilter::ORDER && !is_null($fieldValue) && $fieldValue!= '') + { + $kEsearchOrderBy = $this->getKESearchOrderBy($fieldValue); + continue; + } + + list($operator, $fieldName, $fieldValue, $shouldContinue) = $this->splitIntoParameters($filter, $field, $fieldValue); + if ($shouldContinue) + { + continue; + } + $searchItemType = $this->getSphinxToElasticSearchItemType($operator); + $elasticFieldName = $this->getSphinxToElasticFieldName($fieldName); + if($elasticFieldName && $searchItemType) + { + $this->AddFieldPartToQuery($searchItemType, $elasticFieldName, $fieldValue); + } + + } + $this->addNestedQueryPart(); + $operator = new ESearchOperator(); + $operator->setOperator(ESearchOperatorType::AND_OP); + $operator->setSearchItems($this->searchItems); + return array($operator, $kEsearchOrderBy); + } + + protected function splitIntoParameters($filter, $field, $fieldValue ) + { + $fieldParts = explode(entryFilter::FILTER_PREFIX, $field, 3); + list( , $operator, $fieldName) = $fieldParts; + + if ($field === ESearchEntryFilterFields::FREE_TEXT) + { + $operator = baseObjectFilter::IN; + $fieldName = $field; + } + + if(!in_array($fieldName, static::getSupportedFields()) || is_null($fieldValue) || $fieldValue === '') + { + return array(null , null, null, true); + } + if(in_array($fieldName, self::$puserFields)) + { + $kuser = kuserPeer::getKuserByPartnerAndUid($filter->getPartnerSearchScope(), $fieldValue); + if (!$kuser) + { + throw new KalturaAPIException(KalturaErrors::INVALID_USER_ID, $fieldValue); + } + $fieldValue = $kuser->getId(); + } + + if ($fieldName === ESearchEntryFilterFields::STATUS && ($operator === baseObjectFilter::EQ ||$operator === baseObjectFilter::IN )) + { + self::$validStatuses = explode(',',$fieldValue); + return array(null , null, null, true); + } + + if ($fieldName === ESearchEntryFilterFields::DURATION) + { + $fieldValue = $fieldValue * 1000; + } + return array($operator, $fieldName, $fieldValue, false); + } + + protected function getKESearchOrderBy($fieldValue) + { + $fieldNameSortField = substr($fieldValue,1); + if (!$this->getSphinxToElasticOrderBy($fieldNameSortField)) + { + return null; + } + $eSearchOrderByItem = new ESearchEntryOrderByItem(); + if ($fieldValue[0] === self::DESC) + { + $eSearchOrderByItem->setSortOrder(ESearchSortOrder::ORDER_BY_DESC); + } + else if ($fieldValue[0] === self::ASC) + { + $eSearchOrderByItem->setSortOrder(ESearchSortOrder::ORDER_BY_ASC); + } + $eSearchOrderByItem->setSortField($fieldNameSortField); + + $orderItems = array($eSearchOrderByItem); + $kEsearchOrderBy = new ESearchOrderBy(); + $kEsearchOrderBy->setOrderItems($orderItems); + return $kEsearchOrderBy; + } +} \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php index e859d8afc6e..13482a3bc18 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php @@ -5,6 +5,15 @@ class ESearchQueryFromFilter protected $searchItems; protected $nestedSearchItem; protected static $validStatuses = array(entryStatus::READY,entryStatus::NO_CONTENT); + protected static $categoryFilterFields = array( + ESearchBaseCategoryEntryItem::CATEGORY_NAMES_MAPPING_FIELD => array(ESearchCategoryEntryFieldName::NAME, 'ESearchCategoryEntryNameItem'), + ESearchBaseCategoryEntryItem::CATEGORY_IDS_MAPPING_FIELD => array(ESearchCategoryEntryFieldName::ID, 'ESearchCategoryEntryIdItem'), + ESearchCategoryEntryFieldName::ANCESTOR_ID => array(ESearchCategoryEntryFieldName::ANCESTOR_ID, 'ESearchCategoryEntryAncestorIdItem'), + ESearchCategoryEntryFieldName::ANCESTOR_NAME => array(ESearchCategoryEntryFieldName::ANCESTOR_NAME, 'ESearchCategoryEntryAncestorNameItem') + ); + + CONST FIELD_NAME = 0; + CONST FIELD_CLASS = 1; public function __construct() { @@ -46,16 +55,16 @@ public function createElasticQueryFromFilter(baseObjectFilter $filter) $operator = new ESearchOperator(); $operator->setOperator(ESearchOperatorType::AND_OP); $operator->setSearchItems($this->searchItems); - return $operator; + return array($operator, null); } public function retrieveElasticQueryEntryIds(baseObjectFilter $filter, kPager $pager) { - $query = $this->createElasticQueryFromFilter($filter); + list($query, $kEsearchOrderBy ) = $this->createElasticQueryFromFilter($filter); $entrySearch = new kEntrySearch(); $entrySearch->setFilterOnlyContext(); - $elasticResults = $entrySearch->doSearch($query, $pager, self::$validStatuses); + $elasticResults = $entrySearch->doSearch($query, $pager, self::$validStatuses,null, $kEsearchOrderBy); list($coreResults, $objectOrder, $objectCount, $objectHighlight) = kESearchCoreAdapter::getElasticResultAsArray($elasticResults, $entrySearch->getQueryAttributes()->getQueryHighlightsAttributes()); @@ -70,41 +79,89 @@ protected function AddFieldPartToQuery($searchItemType, $elasticFieldName, $fiel { case ESearchFilterItemType::EXACT_MATCH: $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH); - $this->addToSearchItemsByField($elasticFieldName, $searchItem); break; case ESearchFilterItemType::EXACT_MATCH_MULTI_OR : - $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); + if ($elasticFieldName === ESearchCategoryEntryFieldName::FULL_IDS) + { + $searchItem = $this->addMultiQueryCategoryFullName($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); + } + else + { + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); + } break; case ESearchFilterItemType::EXACT_MATCH_MULTI_AND : - $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::AND_OP); + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::AND_OP); break; case ESearchFilterItemType::PARTIAL : $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL); - $this->addToSearchItemsByField($elasticFieldName, $searchItem); break; case ESearchFilterItemType::PARTIAL_MULTI_OR : - $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::OR_OP); + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::OR_OP); break; case ESearchFilterItemType::PARTIAL_MULTI_AND : - $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::AND_OP); + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::AND_OP); + break; + + case ESearchFilterItemType::EXACT_MATCH_NOT: + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::NOT_OP); + break; + + case ESearchFilterItemType::NOT_CONTAINS: + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::NOT_OP); + break; + + case ESearchFilterItemType::IS_EMPTY: + $searchItem = $this->addSearchItem($elasticFieldName, null, ESearchItemType::EXISTS); + if ($fieldValue) + { + $searchItem = $this->createOperator(ESearchOperatorType::NOT_OP, array($searchItem), $elasticFieldName); + } break; case ESearchFilterItemType::RANGE_GTE : - $this->addRangeGteQuery($elasticFieldName, $fieldValue); + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setGreaterThanOrEqual'); break; case ESearchFilterItemType::RANGE_LTE : - $this->addRangeLteQuery($elasticFieldName, $fieldValue); + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setLessThanOrEqual'); + break; + + case ESearchFilterItemType::RANGE_LT: + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setLessThan'); + break; + + case ESearchFilterItemType::RANGE_GT: + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setGreaterThan'); + break; + + case ESearchFilterItemType::RANGE_LTE_OR_NULL: + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setLessThanOrEqual'); + $searchItem = $this->allowNullValues($searchItem,$elasticFieldName); + break; + + case ESearchFilterItemType::RANGE_GTE_OR_NULL: + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setGreaterThanOrEqual'); + $searchItem = $this->allowNullValues($searchItem,$elasticFieldName); + break; + + case ESearchFilterItemType::MATCH_AND: + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::AND_OP); + break; + + case ESearchFilterItemType::MATCH_OR: + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); break; default: - KalturaLog::debug("Skip field [$elasticFieldName] as it has no search item type [$searchItemType]"); + throw new KalturaAPIException(KalturaErrors::SEARCH_ITEM_TYPE_NOT_FOUND,$searchItemType, $elasticFieldName); } + $this->addToSearchItemsByField($elasticFieldName, $searchItem); } protected function addMultiQuery($elasticFieldName, $fieldValue, $searchType, $operatorType) @@ -121,37 +178,84 @@ protected function addMultiQuery($elasticFieldName, $fieldValue, $searchType, $o $operator = $this->getEsearchOperatorByField($elasticFieldName); $operator->setOperator($operatorType); $operator->setSearchItems($innerSearchItems); - $this->addToSearchItemsByField($elasticFieldName, $operator); + return $operator; } } - protected function addRangeGteQuery($elasticFieldName, $fieldValue) + protected function addSearchItem($elasticFieldName, $value, $itemType, $range = false) { - $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::RANGE, true); - $range = new ESearchRange(); - $range->setGreaterThanOrEqual($fieldValue); - $searchItem->setRange($range); - $this->addToSearchItemsByField($elasticFieldName, $searchItem); + $searchItem = $this->createSearchItemByFieldType($elasticFieldName); + if (property_exists ($searchItem, 'fieldName') && !$searchItem->getFieldName()) + { + $searchItem->setFieldName($elasticFieldName); + } + if(!$range) + { + $searchItem->setSearchTerm($value); + } + $searchItem->setItemType($itemType); + return $searchItem; } - protected function addRangeLteQuery($elasticFieldName, $fieldValue) + protected function getSearchItemForCategories($values, $searchType) { - $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::RANGE, true); - $range = new ESearchRange(); - $range->setLessThanOrEqual($fieldValue); - $searchItem->setRange($range); - $this->addToSearchItemsByField($elasticFieldName, $searchItem); + $innerSearchItems = array(); + if(count($values)) + { + if (count($values) > 1) + { + //all values excepts from the last value should be parent category p1>p2>c3 + for ($i = 0; $i < count($values) - 1; $i++) + { + $innerSearchItems[] = $this->addSearchItem(KalturaESearchCategoryEntryFieldName::ANCESTOR_NAME, $values[$i], $searchType); + } + } + $innerSearchItems[] = $this->addSearchItem(ESearchBaseCategoryEntryItem::CATEGORY_NAMES_MAPPING_FIELD, $values[count($values) - 1], $searchType); + } + return $innerSearchItems; } - protected function addSearchItem($elasticFieldName, $value, $itemType, $range = false) + protected function addMultiQueryCategoryFullName($elasticFieldName, $fieldValue, $searchType, $operatorType) { - $searchItem = $this->createSearchItemByFieldType($elasticFieldName); - $searchItem->setFieldName($elasticFieldName); - if(!$range) + $categoriesValues = array(); + $andOperator = array(); + $values = $this->createValuesArray($fieldValue); + foreach ($values as $value) { - $searchItem->setSearchTerm($value); + $categoriesValues[] = explode(categoryPeer::CATEGORY_SEPARATOR, $value); } - $searchItem->setItemType($itemType); + foreach ($categoriesValues as $categoriesValue) + { + $searchItemsAndArray = $this->getSearchItemForCategories($categoriesValue, $searchType); + $andOperator[] = $this->createOperator(ESearchOperatorType::AND_OP, $searchItemsAndArray, $elasticFieldName); + } + $orOprator = $this->createOperator($operatorType, $andOperator, $elasticFieldName); + return $orOprator; + } + + protected function allowNullValues($searchItem,$elasticFieldName) + { + $notSearchItem = $this->addSearchItem($elasticFieldName, null, ESearchItemType::EXISTS); + $notOprator = $this->createOperator(ESearchOperatorType::NOT_OP, array($notSearchItem), $elasticFieldName); + $orOprator = $this->createOperator(ESearchOperatorType::OR_OP, array($notOprator,$searchItem), $elasticFieldName); + return $orOprator; + } + + + protected function createOperator($opretorType,$searchItemsArray,$elasticFieldName) + { + $operatorGeneral = $this->getEsearchOperatorByField($elasticFieldName); + $operatorGeneral->setOperator($opretorType); + $operatorGeneral->setSearchItems($searchItemsArray); + return $operatorGeneral; + } + + protected function getSearchItemWithRange($elasticFieldName, $fieldValue, $rangeProperty) + { + $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::RANGE, true); + $range = new ESearchRange(); + $range->$rangeProperty($fieldValue); + $searchItem->setRange($range); return $searchItem; } @@ -207,6 +311,16 @@ protected function addNestedQueryPart() protected function createSearchItemByFieldType($elasticFieldName) { + if (in_array($elasticFieldName ,array_keys(self::$categoryFilterFields))) + { + $eSearchCategoryEntry = new self::$categoryFilterFields[$elasticFieldName][self::FIELD_CLASS](); + $eSearchCategoryEntry->setFieldName(self::$categoryFilterFields[$elasticFieldName][self::FIELD_NAME]); + return $eSearchCategoryEntry; + } + if ($elasticFieldName === ESearchUnifiedItem::UNIFIED) + { + return new ESearchUnifiedItem(); + } return new ESearchEntryItem(); } @@ -220,7 +334,17 @@ protected function getSphinxToElasticSearchItemType($operator) baseObjectFilter::LTE => ESearchFilterItemType::RANGE_LTE, baseObjectFilter::LIKE => ESearchFilterItemType::EXACT_MATCH, baseObjectFilter::MULTI_LIKE_OR => ESearchFilterItemType::EXACT_MATCH_MULTI_OR, - baseObjectFilter::MULTI_LIKE_AND => ESearchFilterItemType::EXACT_MATCH_MULTI_AND); + baseObjectFilter::MULTI_LIKE_AND => ESearchFilterItemType::EXACT_MATCH_MULTI_AND, + baseObjectFilter::LTE_OR_NULL => ESearchFilterItemType::RANGE_LTE_OR_NULL, + baseObjectFilter::LT => ESearchFilterItemType::RANGE_LT, + baseObjectFilter::GT => ESearchFilterItemType::RANGE_GT, + baseObjectFilter::GTE_OR_NULL => ESearchFilterItemType::RANGE_GTE_OR_NULL, + baseObjectFilter::MATCH_AND => ESearchFilterItemType::MATCH_AND, + baseObjectFilter::MATCH_OR => ESearchFilterItemType::MATCH_OR, + baseObjectFilter::NOT_CONTAINS => ESearchFilterItemType::NOT_CONTAINS, + baseObjectFilter::IS_EMPTY => ESearchFilterItemType::IS_EMPTY + ); + if(array_key_exists($operator, $operatorsMap)) { diff --git a/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php b/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php index 671327a8704..d1db7521cd8 100644 --- a/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php +++ b/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php @@ -51,6 +51,7 @@ class ESearchEntryItem extends ESearchItem 'is_live' => array('ESearchItemType::EXACT_MATCH'=> ESearchItemType::EXACT_MATCH), 'user_names' => array('ESearchItemType::EXACT_MATCH'=> ESearchItemType::EXACT_MATCH, 'ESearchItemType::PARTIAL'=> ESearchItemType::PARTIAL, 'ESearchItemType::STARTS_WITH'=> ESearchItemType::STARTS_WITH, 'ESearchItemType::EXISTS'=> ESearchItemType::EXISTS, ESearchUnifiedItem::UNIFIED), 'root_id' => array('ESearchItemType::EXACT_MATCH'=> ESearchItemType::EXACT_MATCH), + 'partner_sort_value' => array('ESearchItemType::RANGE' => ESearchItemType::RANGE) ); protected static $field_boost_values = array( From 2d6aca3232c581a46463d6b7ee37bbeb9acc0c50 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Mon, 1 Jul 2019 15:17:42 +0300 Subject: [PATCH 037/103] SUP-16933: small fix --- alpha/apps/kaltura/lib/myPlaylistUtils.class.php | 2 +- .../lib/model/filters/ESearchEntryQueryFromFilter.php | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index 4e4ced5b11d..2558ef85a99 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -551,7 +551,7 @@ protected static function updateEntryFilterFields($entryFilter, $partnerId) $typeArray = array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::MEDIA_CLIP)); $typeArray = array_unique(array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::LIVE_STREAM))); $entryFilter->set ( "_in_type" , implode(',',$typeArray) ); - $entryFilter->set( "_eq_status" , entryStatus::READY ); //needed? + $entryFilter->set( "_eq_status" , entryStatus::READY ); $entryFilter->set("_notin_moderation_status", self::$moderationStatusesNotIn); if (!self::$isAdminKs) { diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php index e14d5efc6d4..76136fc6add 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php @@ -45,8 +45,6 @@ class ESearchEntryQueryFromFilter extends ESearchQueryFromFilter ESearchEntryFilterFields::END_DATE, ESearchEntryFilterFields::REFERENCE_ID, ESearchEntryFilterFields::ROOT_ENTRY_ID, - ESearchEntryFilterFields::VIEWS, - ESearchEntryFilterFields::PLAYS, ESearchEntryFilterFields::DURATION, ESearchEntryFilterFields::CATEGORIES, ESearchEntryFilterFields::CATEGORIES_IDS, From 84b0fc68936d3a71dd3543ee1d18532c82974f8e Mon Sep 17 00:00:00 2001 From: TalDubovKaltura Date: Tue, 2 Jul 2019 16:33:49 -0400 Subject: [PATCH 038/103] PSVAMB-7785 - Adding new feature flip for allowing live streaming entries view in feed --- alpha/lib/enums/PermissionName.php | 2 ++ configurations/admin.template.ini | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/alpha/lib/enums/PermissionName.php b/alpha/lib/enums/PermissionName.php index 26f30a5ab60..0ad1814e355 100644 --- a/alpha/lib/enums/PermissionName.php +++ b/alpha/lib/enums/PermissionName.php @@ -275,4 +275,6 @@ interface PermissionName extends BaseEnum const FEATURE_ANALYTICS_PERSISTENT_SESSION_ID = 'FEATURE_ANALYTICS_PERSISTENT_SESSION_ID'; const FEATURE_LIVE_ANALYTICS_DASHBOARD = 'FEATURE_LIVE_ANALYTICS_DASHBOARD'; + + const FEATURE_LIVE_ENTRIES_IN_FEED = 'FEATURE_LIVE_ENTRIES_IN_FEED'; } diff --git a/configurations/admin.template.ini b/configurations/admin.template.ini index 8851dfa8709..418d373c52d 100644 --- a/configurations/admin.template.ini +++ b/configurations/admin.template.ini @@ -1037,6 +1037,14 @@ moduls.limitAllowedActions.label = "Limit allowed actions" moduls.limitAllowedActions.permissionName = FEATURE_LIMIT_ALLOWED_ACTIONS moduls.limitAllowedActions.group = GROUP_ENABLE_DISABLE_FEATURES +moduls.feedWithLiveEntries.enabled = true +moduls.feedWithLiveEntries.permissionType = 2 +moduls.feedWithLiveEntries.label = "Show live entries in feed" +moduls.feedWithLiveEntries.permissionName = FEATURE_LIVE_ENTRIES_IN_FEED +moduls.feedWithLiveEntries.basePermissionType = +moduls.feedWithLiveEntries.basePermissionName = +moduls.feedWithLiveEntries.group = GROUP_ENABLE_DISABLE_FEATURES + ; ACCESS PERMISSIONS access.error = * From 74d8c3e70b952617fb3a04ba9b9b3b683a228c1e Mon Sep 17 00:00:00 2001 From: TalDubovKaltura Date: Tue, 2 Jul 2019 16:34:43 -0400 Subject: [PATCH 039/103] PSVAMB-7785 - Adding logic of viewing live streaming entries inside the feed page --- .../KalturaSyndicationFeedRenderer.php | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php b/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php index f7131c2500b..5ae86e19610 100644 --- a/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php +++ b/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php @@ -140,12 +140,12 @@ public function __construct($feedId, $feedProcessingKey = null, $ks = null, $sta // add partner to default criteria myPartnerUtils::addPartnerToCriteria('category', $this->syndicationFeed->partnerId, true); myPartnerUtils::addPartnerToCriteria('asset', $this->syndicationFeed->partnerId, true); - + myPartnerUtils::resetPartnerFilter('entry'); entryPeer::onlyReadyCriteriaFilter(); - if($this->shouldAddNextLink()) + if($this->shouldAddNextLink() ) { $this->addLinkForNextIteration = true; @@ -156,17 +156,17 @@ public function __construct($feedId, $feedProcessingKey = null, $ks = null, $sta } $this->baseCriteria = clone entryPeer::getDefaultCriteriaFilter(); - + $startDateCriterion = $this->baseCriteria->getNewCriterion(entryPeer::START_DATE, time(), Criteria::LESS_EQUAL); $startDateCriterion->addOr($this->baseCriteria->getNewCriterion(entryPeer::START_DATE, null)); $this->baseCriteria->addAnd($startDateCriterion); - + $endDateCriterion = $this->baseCriteria->getNewCriterion(entryPeer::END_DATE, time(), Criteria::GREATER_EQUAL); $endDateCriterion->addOr($this->baseCriteria->getNewCriterion(entryPeer::END_DATE, null)); $this->baseCriteria->addAnd($endDateCriterion); - + $this->baseCriteria->addAnd(entryPeer::PARTNER_ID, $this->syndicationFeed->partnerId); - + if($this->addLinkForNextIteration) { $this->addExternalAttachedFilter(); @@ -183,7 +183,13 @@ public function __construct($feedId, $feedProcessingKey = null, $ks = null, $sta } else { - $this->baseCriteria->addAnd(entryPeer::TYPE, array(entryType::MEDIA_CLIP, entryType::MIX), Criteria::IN); + $mediaEntrytypes = array(entryType::MEDIA_CLIP, entryType::MIX); + $partner = PartnerPeer::retrieveByPK($this->getSyndicationFeedDb()->getPartnerId()); + $liveEntriesView = $partner->getEnabledService(KalturaPermissionName::FEATURE_LIVE_ENTRIES_IN_FEED); + if($liveEntriesView) { + $mediaEntrytypes[] = entryType::LIVE_STREAM; + } + $this->baseCriteria->addAnd(entryPeer::TYPE, $mediaEntrytypes, Criteria::IN); } $this->baseCriteria->addAnd(entryPeer::MODERATION_STATUS, array( entry::ENTRY_MODERATION_STATUS_REJECTED, From 6028767362feb736e56d575acf517d42cafa538c Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Wed, 3 Jul 2019 15:50:01 +0300 Subject: [PATCH 040/103] SUP-16933:use eSearch instead of sphinx for dynamic playlist execute from filter/content actions --- .../kaltura/lib/myPlaylistUtils.class.php | 142 +++++++++++------- api_v3/services/PlaylistService.php | 88 +++++++++-- .../filters/ESearchEntryQueryFromFilter.php | 106 ++++++++----- .../model/filters/ESearchQueryFromFilter.php | 16 +- 4 files changed, 240 insertions(+), 112 deletions(-) diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index 2558ef85a99..e1c21805ddb 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -14,22 +14,35 @@ class myPlaylistUtils const CAPTION_FILES_LABEL = "label"; const CAPTION_FILES_PATH = "path"; const CAPTION_FILES_ID = "captionId"; + const KALTURA_CLASS = 'kalturaClass'; private static $user_cache = null; private static $isAdminKs = false; private static $playlistContext; - + + private static $moderationStatusesNotIn = array( + entry::ENTRY_MODERATION_STATUS_PENDING_MODERATION, + entry::ENTRY_MODERATION_STATUS_REJECTED + ); + + private static $dates = array( + '_lteornull_start_date', + '_gteornull_start_date', + '_lte_start_date', + '_gte_start_date', + '_lteornull_end_date', + '_gteornull_end_date', + '_lte_end_date', + '_gte_end_date' + ); + public static function setIsAdminKs($v) { self::$isAdminKs = $v; } - protected static $moderationStatusesNotIn = array( - entry::ENTRY_MODERATION_STATUS_PENDING_MODERATION, - entry::ENTRY_MODERATION_STATUS_REJECTED); - /** * Playlist is an entry of type ENTRY_TYPE_PLAYLIST = 5. * Within this type there are 3 media_types to tell the difference between dynamic,static and external playslits: @@ -488,14 +501,41 @@ public static function getDynamicPlaylistFilters($xml) return $entry_filters; } - public static function executeDynamicPlaylistViaEsearch ($partnerId , $xml , $pager = null) + public static function getEntryFiltersFromXml($listOfFilters, $partnerId) { - list ($totalResults, $listOfFilters) = self::getPlaylistFilterListStruct($xml); if (!$listOfFilters) { return null; } - $entry_filters = self::fillEntryFilterFromXml($listOfFilters, $partnerId); + return self::fillEntryFilterFromXml($listOfFilters, $partnerId); + } + + //splitting entry filters into filters that will run in Esearch and in Sphinx: + // Elastic - advancesSearch is not set OR advancesSearch is set and includes KalturaMetadataSearchItem + // Sphinx - advancesSearch is set and NOT including KalturaMetadataSearchItem + public static function splitEntryFilters($xml) + { + $entryFiltersViaEsearch = array(); + $entryFiltersViaSphinx = array(); + list ($totalResults, $entryFilters) = self::getPlaylistFilterListStruct($xml); + + foreach ($entryFilters as $entryFilter) + { + if (!isset($entryFilter->advancedSearch) || + ( isset($entryFilter->advancedSearch) && (string)$entryFilter->advancedSearch[self::KALTURA_CLASS] === PlaylistService::KALTURA_METADATA_SEARCH_ITEM)) + { + $entryFiltersViaEsearch[] = $entryFilter; + } + else + { + $entryFiltersViaSphinx[] = $entryFilter; + } + } + return array($entryFiltersViaEsearch, $entryFiltersViaSphinx, $totalResults); + } + + public static function executeDynamicPlaylistViaEsearch ($entryFilters ,$pager = null) + { $entryKPager = new kPager(); if ($pager) { @@ -503,45 +543,35 @@ public static function executeDynamicPlaylistViaEsearch ($partnerId , $xml , $pa } $entryQueryToFilterESearch = new ESearchEntryQueryFromFilter(); $entryIds= array(); - foreach ($entry_filters as $entry_filter) + foreach ($entryFilters as $entryFilter) { - list ($currEntryIds, $count) = $entryQueryToFilterESearch->retrieveElasticQueryEntryIds($entry_filter, $entryKPager); - if (count($currEntryIds) > $entry_filter->get('_limit')) + list ($currEntryIds, $count) = $entryQueryToFilterESearch->retrieveElasticQueryEntryIds($entryFilter, $entryKPager); + if (count($currEntryIds) > $entryFilter->getLimit()) { - $currEntryIds = array_slice($currEntryIds,0,$entry_filter->get('_limit')); + $currEntryIds = array_slice($currEntryIds,0,$entryFilter->getLimit()); } $entryIds = array_merge ($entryIds, $currEntryIds); } - return self::getEntriesFromEntryIds(array_unique($entryIds)); - } - protected static function getEntriesFromEntryIds($entryIds) - { - $entriesResult = array(); - foreach ($entryIds as $entryId) - { - $dbEntry = entryPeer::retrieveByPK($entryId); - if ($dbEntry) - { - $entriesResult[] = $dbEntry; - } - } - return $entriesResult; + $entryIds = array_unique($entryIds); + $entryIdsOrder = array_flip($entryIds); + $entries = entryPeer::retrieveByPKs($entryIds); + usort($entries, build_sorter($entryIdsOrder)); + return $entries; } - protected static function fillEntryFilterFromXml($list_of_filters, $partnerId) + protected static function fillEntryFilterFromXml($listOfFilters, $partnerId) { - $entry_filters = array(); - for ($i = 0; $i < count($list_of_filters) ; $i++) + $entryFilters = array(); + foreach ($listOfFilters as $filter) { - $entry_filter_xml = $list_of_filters[$i]; - self::replaceContextTokens($entry_filter_xml); - $entry_filter = new entryFilter(); - $entry_filter->fillObjectFromXml($entry_filter_xml, "_", null); - self::updateEntryFilterFields($entry_filter, $partnerId); - $entry_filters[] = $entry_filter; + self::replaceContextTokens($filter); + $entryFilter = new entryFilter(); + $entryFilter->fillObjectFromXml($filter, '_'); + self::updateEntryFilterFields($entryFilter, $partnerId); + $entryFilters[] = $entryFilter; } - return $entry_filters; + return $entryFilters; } protected static function updateEntryFilterFields($entryFilter, $partnerId) @@ -550,20 +580,32 @@ protected static function updateEntryFilterFields($entryFilter, $partnerId) $typeArray = array (entryType::MEDIA_CLIP, entryType::MIX, entryType::LIVE_STREAM ); $typeArray = array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::MEDIA_CLIP)); $typeArray = array_unique(array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::LIVE_STREAM))); - $entryFilter->set ( "_in_type" , implode(',',$typeArray) ); - $entryFilter->set( "_eq_status" , entryStatus::READY ); - $entryFilter->set("_notin_moderation_status", self::$moderationStatusesNotIn); - if (!self::$isAdminKs) + $entryFilter->setTypeIn($typeArray); + $entryFilter->setStatusEquel(entryStatus::READY); + $entryFilter->setModerationStatusNotIn(self::$moderationStatusesNotIn); + if (self::$isAdminKs) { - self::addSchedulingToCriteria(null, $entryFilter, true); + self::unsetDates($entryFilter); } } - - public static function executeDynamicPlaylist ( $partner_id , $xml , $filter = null ,$detailed = true, $pager = null ) + protected static function unsetDates($filter) + { + foreach (self::$dates as $date) + if ($filter->is_set($date)) + { + $filter->unsetByName($date); + } + } + + public static function executeDynamicPlaylist ( $partner_id, $xml, $filter = null, $detailed = true, $pager = null ) { list ( $total_results , $list_of_filters ) = self::getPlaylistFilterListStruct ( $xml ); - + return self::executeDynamicPlaylistFromFilters($total_results, $list_of_filters, $partner_id, $filter , $detailed, $pager); + } + + public static function executeDynamicPlaylistFromFilters($total_results, $list_of_filters, $partner_id, $filter = null, $detailed = true, $pager = null) + { $entry_filters = array(); if ( ! $list_of_filters ) return null; @@ -1022,7 +1064,7 @@ private static function addSchedulingCriterion(Criteria $c, $field, $min, $max, $c->addAnd($criterion); } - private static function addSchedulingToCriteria($c, entryFilter $filter = null, $isEsearch = false) + private static function addSchedulingToCriteria(Criteria $c, entryFilter $filter = null) { $min = 0; $max = kApiCache::getTime(); @@ -1048,10 +1090,7 @@ private static function addSchedulingToCriteria($c, entryFilter $filter = null, $filter->unsetByName('_gte_start_date'); } } - if(!$isEsearch) - { - self::addSchedulingCriterion($c, entryPeer::START_DATE, $min, $max, $allowNull); - } + self::addSchedulingCriterion($c, entryPeer::START_DATE, $min, $max, $allowNull); $min = kApiCache::getTime(); @@ -1078,10 +1117,7 @@ private static function addSchedulingToCriteria($c, entryFilter $filter = null, $filter->unsetByName('_gte_end_date'); } } - if(!$isEsearch) - { - self::addSchedulingCriterion($c, entryPeer::END_DATE, $min, $max, $allowNull); - } + self::addSchedulingCriterion($c, entryPeer::END_DATE, $min, $max, $allowNull); } private static function addModerationToCriteria(Criteria $c) diff --git a/api_v3/services/PlaylistService.php b/api_v3/services/PlaylistService.php index cb6e8a9c04b..46e15bc94bc 100644 --- a/api_v3/services/PlaylistService.php +++ b/api_v3/services/PlaylistService.php @@ -10,6 +10,9 @@ */ class PlaylistService extends KalturaEntryService { + const ADVANCED_SEARCH = 'advancedSearch'; + const KALTURA_METADATA_SEARCH_ITEM = 'KalturaMetadataSearchItem'; + /* (non-PHPdoc) * @see KalturaBaseService::globalPartnerAllowed() */ @@ -345,31 +348,84 @@ function executeAction( $id , $detailed = false, KalturaContext $playlistContext function executeFromContentAction($playlistType, $playlistContent, $detailed = false, $pager = null) { myDbHelper::$use_alternative_con = myDbHelper::DB_HELPER_CONN_PROPEL3; - if ($this->getKs() && is_object($this->getKs()) && $this->getKs()->isAdmin()) + { myPlaylistUtils::setIsAdminKs(true); + } + list($entryFiltersViaEsearch, $entryFiltersViaSphinx, $totalResults) = myPlaylistUtils::splitEntryFilters($playlistContent); + $pagerSeparateQueries = self::decideWhereHandlingPager($pager,$entryFiltersViaEsearch, $entryFiltersViaSphinx); + $entryList = self::handlePlaylistByType($playlistType, $entryFiltersViaEsearch, $entryFiltersViaSphinx, $this->getPartnerId(), $pagerSeparateQueries, $pager, $totalResults, $playlistContent); + myEntryUtils::updatePuserIdsForEntries($entryList); + KalturaLog::debug("entry ids count: " . count($entryList)); + return KalturaBaseEntryArray::fromDbArray($entryList, $this->getResponseProfile()); + } - $entryList = array(); + protected static function handlePlaylistByType($playlistType, $entryFiltersViaEsearch, $entryFiltersViaSphinx, $partnerId, $pagerSeperateQueries, $pager, $totalResults, $playlistContent) + { if ($playlistType == KalturaPlaylistType::DYNAMIC) { - if (strpos($playlistContent, 'advancedSearch') === false || - (strpos($playlistContent, 'advancedSearch') && strpos($playlistContent, 'KalturaMetadataSearchItem')) ) - { - $entryList = myPlaylistUtils::executeDynamicPlaylistViaEsearch($this->getPartnerId(), $playlistContent, $pager); - } - else + $entryList = self::handlingDynamicPlaylist($entryFiltersViaEsearch,$entryFiltersViaSphinx, $partnerId, $pagerSeperateQueries, $pager, $totalResults); + } + else if ($playlistType == KalturaPlaylistType::STATIC_LIST) + { + $entryList = myPlaylistUtils::executeStaticPlaylistFromEntryIdsString($playlistContent, null, true, $pager); + } + return $entryList; + } + + protected static function handlingDynamicPlaylist($entryFiltersViaEsearch, $entryFiltersViaSphinx, $partnerId, $pagerSeparateQueries, $pager, $totalResults) + { + $entryListEsearch = array(); + $entryListSphinx = array(); + if ($entryFiltersViaEsearch) + { + $entryFiltersViaEsearch = myPlaylistUtils::getEntryFiltersFromXml($entryFiltersViaEsearch, $partnerId); + $entryListEsearch = myPlaylistUtils::executeDynamicPlaylistViaEsearch($entryFiltersViaEsearch, $pagerSeparateQueries); + } + if ($entryFiltersViaSphinx) + { + $entryListSphinx = myPlaylistUtils::executeDynamicPlaylistFromFilters($totalResults, $entryFiltersViaSphinx, $partnerId, null, true, $pagerSeparateQueries); + } + $entryListMerged = array_merge($entryListEsearch,$entryListSphinx); + $entryListUnique = self::getEntryListUnique($entryListMerged); + $entryList = self::getEntryListByPager($entryListUnique, $pager, $pagerSeparateQueries); + return $entryList; + } + + //When queries are going to run both in Esearch and Sphinx, pager handling will be done after merging the results -> ($pagerSeperateQueries = null) + //In case only one of them will run (for example only in sphinx), pager handling will be done only there (in this example: in sphinx query) -> ($pagerSeperateQueries = $pager) + protected static function decideWhereHandlingPager($pager,$entryFiltersViaEsearch, $entryFiltersViaSphinx) + { + $pagerSeparateQueries = $pager; + if ($entryFiltersViaEsearch && $entryFiltersViaSphinx) + { + $pagerSeparateQueries = null; + } + return $pagerSeparateQueries; + } + + protected static function getEntryListUnique($entryListMerged) + { + $entryList = array(); + foreach ($entryListMerged as $currentEntry) + { + if (!in_array($currentEntry, $entryList)) { - $entryList = myPlaylistUtils::executeDynamicPlaylist($this->getPartnerId(), $playlistContent, null, true, $pager); + $entryList[] = $currentEntry; } - } + return $entryList; + } - else if ($playlistType == KalturaPlaylistType::STATIC_LIST) - $entryList = myPlaylistUtils::executeStaticPlaylistFromEntryIdsString($playlistContent, null, true, $pager); - - myEntryUtils::updatePuserIdsForEntries($entryList); - - return KalturaBaseEntryArray::fromDbArray($entryList, $this->getResponseProfile()); + protected static function getEntryListByPager($entryList, $pager, $pagerSeparateQueries) + { + if ( $pager && is_null($pagerSeparateQueries)) + { + $startOffset = $pager->calcOffset(); + $pageSize = $pager->calcPageSize(); + $entryList = array_slice($entryList, $startOffset, $pageSize); + } + return $entryList; } /** diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php index 76136fc6add..8c63436e356 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php @@ -110,7 +110,9 @@ protected function getSphinxToElasticFieldName($field) ESearchEntryFilterFields::CATEGORIES_FULL_NAME => ESearchCategoryEntryFieldName::FULL_IDS, ESearchEntryFilterFields::PARTNER_SORT_VALUE => ESearchEntryFieldName::PARTNER_SORT_VALUE, ESearchEntryFilterFields::SEARCH_TEXT => ESearchUnifiedItem::UNIFIED, - ESearchEntryFilterFields::FREE_TEXT => ESearchUnifiedItem::UNIFIED + ESearchEntryFilterFields::FREE_TEXT => ESearchUnifiedItem::UNIFIED, + ESearchEntryFilterFields::PLAYS => ESearchEntryOrderByFieldName::PLAYS, + ESearchEntryFilterFields::VIEWS => ESearchEntryOrderByFieldName::VIEWS ); if(array_key_exists($field, $fieldsMap)) @@ -123,16 +125,34 @@ protected function getSphinxToElasticFieldName($field) } } - protected function getSphinxToElasticOrderBy($field) - { + protected function getMediaEntryElasticOrderBy($field) + { //TODO: VALIDATE ALL OF THEM $fieldsMap = array( - ESearchEntryFilterFields::VIEWS => ESearchEntryOrderByFieldName::VIEWS, - ESearchEntryFilterFields::PLAYS => ESearchEntryOrderByFieldName::PLAYS + KalturaMediaEntryOrderBy::MEDIA_TYPE_ASC, + KalturaMediaEntryOrderBy::MEDIA_TYPE_DESC, + KalturaMediaEntryOrderBy::PLAYS_ASC, + KalturaMediaEntryOrderBy::PLAYS_DESC, + KalturaMediaEntryOrderBy::VIEWS_ASC, + KalturaMediaEntryOrderBy::VIEWS_DESC, + KalturaMediaEntryOrderBy::DURATION_ASC, + KalturaMediaEntryOrderBy::DURATION_DESC, + KalturaMediaEntryOrderBy::NAME_ASC, + KalturaMediaEntryOrderBy::NAME_DESC, + KalturaMediaEntryOrderBy::CREATED_AT_ASC, + KalturaMediaEntryOrderBy::CREATED_AT_DESC, + KalturaMediaEntryOrderBy::UPDATED_AT_ASC, + KalturaMediaEntryOrderBy::UPDATED_AT_DESC, + KalturaMediaEntryOrderBy::START_DATE_ASC, + KalturaMediaEntryOrderBy::START_DATE_DESC, + KalturaMediaEntryOrderBy::END_DATE_ASC, + KalturaMediaEntryOrderBy::END_DATE_DESC, + KalturaMediaEntryOrderBy::PARTNER_SORT_VALUE_ASC, + KalturaMediaEntryOrderBy::PARTNER_SORT_VALUE_DESC, ); - if(array_key_exists($field, $fieldsMap)) + if(in_array($field, $fieldsMap)) { - return $fieldsMap[$field]; + return $field; } else { @@ -151,20 +171,18 @@ public function createElasticQueryFromFilter(baseObjectFilter $filter) $kEsearchOrderBy = $this->getKESearchOrderBy($fieldValue); continue; } - list($operator, $fieldName, $fieldValue, $shouldContinue) = $this->splitIntoParameters($filter, $field, $fieldValue); if ($shouldContinue) { continue; } - $searchItemType = $this->getSphinxToElasticSearchItemType($operator); - $elasticFieldName = $this->getSphinxToElasticFieldName($fieldName); - if($elasticFieldName && $searchItemType) - { - $this->AddFieldPartToQuery($searchItemType, $elasticFieldName, $fieldValue); - } - + $this->addingFieldPartIntoQuery($operator, $fieldName, $fieldValue); } + return $this->createFinalOperator($kEsearchOrderBy); + } + + protected function createFinalOperator($kEsearchOrderBy) + { $this->addNestedQueryPart(); $operator = new ESearchOperator(); $operator->setOperator(ESearchOperatorType::AND_OP); @@ -172,21 +190,47 @@ public function createElasticQueryFromFilter(baseObjectFilter $filter) return array($operator, $kEsearchOrderBy); } + protected function addingFieldPartIntoQuery($operator, $fieldName, $fieldValue) + { + $searchItemType = $this->getSphinxToElasticSearchItemType($operator); + $elasticFieldName = $this->getSphinxToElasticFieldName($fieldName); + if($elasticFieldName && $searchItemType) + { + $this->AddFieldPartToQuery($searchItemType, $elasticFieldName, $fieldValue); + } + } + protected function splitIntoParameters($filter, $field, $fieldValue ) { $fieldParts = explode(entryFilter::FILTER_PREFIX, $field, 3); list( , $operator, $fieldName) = $fieldParts; + list($operator, $fieldName) = self::handlingFreeTextField($field, $operator, $fieldName); + if(!in_array($fieldName, static::getSupportedFields()) || is_null($fieldValue) || $fieldValue === '') + { + return array(null , null, null, true); + } + if ($fieldName === ESearchEntryFilterFields::STATUS && ($operator === baseObjectFilter::EQ ||$operator === baseObjectFilter::IN )) + { + self::$validStatuses = explode(',',$fieldValue); + return array(null , null, null, true); + } + $fieldValue = self::changeFieldValueByPuserIdAndDuration($fieldName, $filter, $fieldValue); + return array($operator, $fieldName, $fieldValue, false); + } + + protected static function handlingFreeTextField($field, $operator, $fieldName) + { if ($field === ESearchEntryFilterFields::FREE_TEXT) { $operator = baseObjectFilter::IN; $fieldName = $field; } + return array($operator, $fieldName); + } - if(!in_array($fieldName, static::getSupportedFields()) || is_null($fieldValue) || $fieldValue === '') - { - return array(null , null, null, true); - } + protected static function changeFieldValueByPuserIdAndDuration($fieldName, $filter, $fieldValue) + { if(in_array($fieldName, self::$puserFields)) { $kuser = kuserPeer::getKuserByPartnerAndUid($filter->getPartnerSearchScope(), $fieldValue); @@ -196,41 +240,35 @@ protected function splitIntoParameters($filter, $field, $fieldValue ) } $fieldValue = $kuser->getId(); } - - if ($fieldName === ESearchEntryFilterFields::STATUS && ($operator === baseObjectFilter::EQ ||$operator === baseObjectFilter::IN )) - { - self::$validStatuses = explode(',',$fieldValue); - return array(null , null, null, true); - } - - if ($fieldName === ESearchEntryFilterFields::DURATION) + else if ($fieldName === ESearchEntryFilterFields::DURATION) { $fieldValue = $fieldValue * 1000; } - return array($operator, $fieldName, $fieldValue, false); + return $fieldValue; } protected function getKESearchOrderBy($fieldValue) { - $fieldNameSortField = substr($fieldValue,1); - if (!$this->getSphinxToElasticOrderBy($fieldNameSortField)) + if (!$this->getMediaEntryElasticOrderBy($fieldValue)) { return null; } $eSearchOrderByItem = new ESearchEntryOrderByItem(); - if ($fieldValue[0] === self::DESC) + $sortField = $this->getSphinxToElasticFieldName(substr($fieldValue,1)); + $eSearchOrderByItem->setSortField($sortField); + $fieldNameSortOrder = $fieldValue[0]; + if ($fieldNameSortOrder === self::DESC) { $eSearchOrderByItem->setSortOrder(ESearchSortOrder::ORDER_BY_DESC); } - else if ($fieldValue[0] === self::ASC) + else if ($fieldNameSortOrder === self::ASC) { $eSearchOrderByItem->setSortOrder(ESearchSortOrder::ORDER_BY_ASC); } - $eSearchOrderByItem->setSortField($fieldNameSortField); - $orderItems = array($eSearchOrderByItem); $kEsearchOrderBy = new ESearchOrderBy(); $kEsearchOrderBy->setOrderItems($orderItems); return $kEsearchOrderBy; + } } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php index 13482a3bc18..3b451d96f48 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php @@ -12,8 +12,9 @@ class ESearchQueryFromFilter ESearchCategoryEntryFieldName::ANCESTOR_NAME => array(ESearchCategoryEntryFieldName::ANCESTOR_NAME, 'ESearchCategoryEntryAncestorNameItem') ); - CONST FIELD_NAME = 0; - CONST FIELD_CLASS = 1; + const FIELD_NAME_LOCATION = 0; + const FIELD_CLASS_LOCATION = 1; + const FIELD_NAME = 'fieldName'; public function __construct() { @@ -185,7 +186,7 @@ protected function addMultiQuery($elasticFieldName, $fieldValue, $searchType, $o protected function addSearchItem($elasticFieldName, $value, $itemType, $range = false) { $searchItem = $this->createSearchItemByFieldType($elasticFieldName); - if (property_exists ($searchItem, 'fieldName') && !$searchItem->getFieldName()) + if (property_exists ($searchItem, self::FIELD_NAME) && !$searchItem->getFieldName()) { $searchItem->setFieldName($elasticFieldName); } @@ -203,8 +204,7 @@ protected function getSearchItemForCategories($values, $searchType) if(count($values)) { if (count($values) > 1) - { - //all values excepts from the last value should be parent category p1>p2>c3 + { //all values excepts from the last value should be parent category p1>p2>c3 for ($i = 0; $i < count($values) - 1; $i++) { $innerSearchItems[] = $this->addSearchItem(KalturaESearchCategoryEntryFieldName::ANCESTOR_NAME, $values[$i], $searchType); @@ -241,7 +241,6 @@ protected function allowNullValues($searchItem,$elasticFieldName) return $orOprator; } - protected function createOperator($opretorType,$searchItemsArray,$elasticFieldName) { $operatorGeneral = $this->getEsearchOperatorByField($elasticFieldName); @@ -313,8 +312,8 @@ protected function createSearchItemByFieldType($elasticFieldName) { if (in_array($elasticFieldName ,array_keys(self::$categoryFilterFields))) { - $eSearchCategoryEntry = new self::$categoryFilterFields[$elasticFieldName][self::FIELD_CLASS](); - $eSearchCategoryEntry->setFieldName(self::$categoryFilterFields[$elasticFieldName][self::FIELD_NAME]); + $eSearchCategoryEntry = new self::$categoryFilterFields[$elasticFieldName][self::FIELD_CLASS_LOCATION](); + $eSearchCategoryEntry->setFieldName(self::$categoryFilterFields[$elasticFieldName][self::FIELD_NAME_LOCATION]); return $eSearchCategoryEntry; } if ($elasticFieldName === ESearchUnifiedItem::UNIFIED) @@ -345,7 +344,6 @@ protected function getSphinxToElasticSearchItemType($operator) baseObjectFilter::IS_EMPTY => ESearchFilterItemType::IS_EMPTY ); - if(array_key_exists($operator, $operatorsMap)) { return $operatorsMap[$operator]; From 5870cdd02b050c1b41d40c9fbea73762bd41c69f Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Wed, 3 Jul 2019 22:48:18 +0300 Subject: [PATCH 041/103] SUP-16933:use eSearch instead of sphinx for dynamic playlist execute from filter/content actions --- .../kaltura/lib/myPlaylistUtils.class.php | 33 ++++++-- api_v3/services/PlaylistService.php | 2 +- .../model/enum/ESearchEntryFilterFields.php | 1 + .../filters/ESearchEntryQueryFromFilter.php | 80 +++++++++++-------- 4 files changed, 75 insertions(+), 41 deletions(-) diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index e1c21805ddb..f3d292ddf8e 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -518,7 +518,6 @@ public static function splitEntryFilters($xml) $entryFiltersViaEsearch = array(); $entryFiltersViaSphinx = array(); list ($totalResults, $entryFilters) = self::getPlaylistFilterListStruct($xml); - foreach ($entryFilters as $entryFilter) { if (!isset($entryFilter->advancedSearch) || @@ -534,7 +533,7 @@ public static function splitEntryFilters($xml) return array($entryFiltersViaEsearch, $entryFiltersViaSphinx, $totalResults); } - public static function executeDynamicPlaylistViaEsearch ($entryFilters ,$pager = null) + public static function executeDynamicPlaylistViaEsearch ($entryFilters ,$totalResults, $pager = null) { $entryKPager = new kPager(); if ($pager) @@ -546,20 +545,42 @@ public static function executeDynamicPlaylistViaEsearch ($entryFilters ,$pager = foreach ($entryFilters as $entryFilter) { list ($currEntryIds, $count) = $entryQueryToFilterESearch->retrieveElasticQueryEntryIds($entryFilter, $entryKPager); - if (count($currEntryIds) > $entryFilter->getLimit()) + $entryIds = self::mergeEntriesByLimit($entryIds, $currEntryIds, $entryFilter->getLimit()); + $totalResults = max (0, $totalResults - count($entryIds)); + if ( $totalResults == 0 ) { - $currEntryIds = array_slice($currEntryIds,0,$entryFilter->getLimit()); + break; } - $entryIds = array_merge ($entryIds, $currEntryIds); } + return array(self::getEntriesSorted($entryIds), $totalResults); + } + protected static function getEntriesSorted($entryIds) + { $entryIds = array_unique($entryIds); $entryIdsOrder = array_flip($entryIds); $entries = entryPeer::retrieveByPKs($entryIds); - usort($entries, build_sorter($entryIdsOrder)); + usort($entries, self::buildSorter($entryIdsOrder)); return $entries; } + protected static function mergeEntriesByLimit($entryIds, $currEntryIds, $limit) + { + if ($limit && count($currEntryIds) > $limit) + { + $currEntryIds = array_slice($currEntryIds,0, $limit); + } + return array_merge ($entryIds, $currEntryIds); + } + + protected static function buildSorter($objectsOrder) + { + return function ($a, $b) use ($objectsOrder) + { + return ($objectsOrder[$a->getId()] > $objectsOrder[$b->getId()]) ? 1 : -1; + }; + } + protected static function fillEntryFilterFromXml($listOfFilters, $partnerId) { $entryFilters = array(); diff --git a/api_v3/services/PlaylistService.php b/api_v3/services/PlaylistService.php index 46e15bc94bc..f2027bdb36a 100644 --- a/api_v3/services/PlaylistService.php +++ b/api_v3/services/PlaylistService.php @@ -380,7 +380,7 @@ protected static function handlingDynamicPlaylist($entryFiltersViaEsearch, $entr if ($entryFiltersViaEsearch) { $entryFiltersViaEsearch = myPlaylistUtils::getEntryFiltersFromXml($entryFiltersViaEsearch, $partnerId); - $entryListEsearch = myPlaylistUtils::executeDynamicPlaylistViaEsearch($entryFiltersViaEsearch, $pagerSeparateQueries); + list($entryListEsearch, $totalResults) = myPlaylistUtils::executeDynamicPlaylistViaEsearch($entryFiltersViaEsearch, $totalResults, $pagerSeparateQueries); } if ($entryFiltersViaSphinx) { diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php index 4f4a93663b6..223856a3022 100644 --- a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php @@ -50,5 +50,6 @@ class ESearchEntryFilterFields extends KalturaEnum const PARTNER_SORT_VALUE = 'partner_sort_value'; const SEARCH_TEXT = 'search_text'; const FREE_TEXT = '_free_text'; + const TOTAL_RANK = 'total_rank'; } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php index 8c63436e356..8a21d05e601 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php @@ -52,7 +52,8 @@ class ESearchEntryQueryFromFilter extends ESearchQueryFromFilter ESearchEntryFilterFields::CATEGORIES_FULL_NAME, ESearchEntryFilterFields::PARTNER_SORT_VALUE, ESearchEntryFilterFields::SEARCH_TEXT, - ESearchEntryFilterFields::FREE_TEXT + ESearchEntryFilterFields::FREE_TEXT, + ESearchEntryFilterFields::TOTAL_RANK ); @@ -111,8 +112,7 @@ protected function getSphinxToElasticFieldName($field) ESearchEntryFilterFields::PARTNER_SORT_VALUE => ESearchEntryFieldName::PARTNER_SORT_VALUE, ESearchEntryFilterFields::SEARCH_TEXT => ESearchUnifiedItem::UNIFIED, ESearchEntryFilterFields::FREE_TEXT => ESearchUnifiedItem::UNIFIED, - ESearchEntryFilterFields::PLAYS => ESearchEntryOrderByFieldName::PLAYS, - ESearchEntryFilterFields::VIEWS => ESearchEntryOrderByFieldName::VIEWS + ESearchEntryFilterFields::TOTAL_RANK => ESearchEntryOrderByFieldName::VOTES ); if(array_key_exists($field, $fieldsMap)) @@ -125,35 +125,43 @@ protected function getSphinxToElasticFieldName($field) } } - protected function getMediaEntryElasticOrderBy($field) - { //TODO: VALIDATE ALL OF THEM + protected function getSphinxToElasticOrderBy($field) + { $fieldsMap = array( - KalturaMediaEntryOrderBy::MEDIA_TYPE_ASC, - KalturaMediaEntryOrderBy::MEDIA_TYPE_DESC, - KalturaMediaEntryOrderBy::PLAYS_ASC, - KalturaMediaEntryOrderBy::PLAYS_DESC, - KalturaMediaEntryOrderBy::VIEWS_ASC, - KalturaMediaEntryOrderBy::VIEWS_DESC, - KalturaMediaEntryOrderBy::DURATION_ASC, - KalturaMediaEntryOrderBy::DURATION_DESC, - KalturaMediaEntryOrderBy::NAME_ASC, - KalturaMediaEntryOrderBy::NAME_DESC, - KalturaMediaEntryOrderBy::CREATED_AT_ASC, - KalturaMediaEntryOrderBy::CREATED_AT_DESC, - KalturaMediaEntryOrderBy::UPDATED_AT_ASC, - KalturaMediaEntryOrderBy::UPDATED_AT_DESC, - KalturaMediaEntryOrderBy::START_DATE_ASC, - KalturaMediaEntryOrderBy::START_DATE_DESC, - KalturaMediaEntryOrderBy::END_DATE_ASC, - KalturaMediaEntryOrderBy::END_DATE_DESC, - KalturaMediaEntryOrderBy::PARTNER_SORT_VALUE_ASC, - KalturaMediaEntryOrderBy::PARTNER_SORT_VALUE_DESC, + self::ASC.ESearchEntryFilterFields::MEDIA_TYPE, + self::DESC.ESearchEntryFilterFields::MEDIA_TYPE, + KalturaMediaEntryOrderBy::PLAYS_ASC => self::ASC.ESearchEntryOrderByFieldName::PLAYS, + KalturaMediaEntryOrderBy::PLAYS_DESC => self::DESC.ESearchEntryOrderByFieldName::PLAYS, + KalturaMediaEntryOrderBy::VIEWS_ASC => self::ASC.ESearchEntryOrderByFieldName::VIEWS, + KalturaMediaEntryOrderBy::VIEWS_DESC => self::DESC.ESearchEntryOrderByFieldName::VIEWS, + KalturaMediaEntryOrderBy::DURATION_ASC => self::ASC.ESearchEntryFieldName::LENGTH_IN_MSECS, + KalturaMediaEntryOrderBy::DURATION_DESC => self::DESC.ESearchEntryFieldName::LENGTH_IN_MSECS, + KalturaMediaEntryOrderBy::NAME_ASC => self::ASC. ESearchEntryOrderByFieldName::NAME, + KalturaMediaEntryOrderBy::NAME_DESC => self::DESC. ESearchEntryOrderByFieldName::NAME, + KalturaMediaEntryOrderBy::CREATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, + KalturaMediaEntryOrderBy::CREATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT, + KalturaMediaEntryOrderBy::UPDATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::UPDATED_AT, + KalturaMediaEntryOrderBy::UPDATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::UPDATED_AT, + KalturaMediaEntryOrderBy::RANK_ASC => self::ASC.ESearchEntryOrderByFieldName::VOTES, + KalturaMediaEntryOrderBy::RANK_DESC => self::DESC.ESearchEntryOrderByFieldName::VOTES, + self::ASC.ESearchEntryFilterFields::TOTAL_RANK => self::ASC.ESearchEntryOrderByFieldName::VOTES, + self::DESC.ESearchEntryFilterFields::TOTAL_RANK => self::DESC.ESearchEntryOrderByFieldName::VOTES, + KalturaMediaEntryOrderBy::START_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::START_DATE, + KalturaMediaEntryOrderBy::START_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::START_DATE, + KalturaMediaEntryOrderBy::END_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::END_DATE, + KalturaMediaEntryOrderBy::END_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::END_DATE, + KalturaMediaEntryOrderBy::RECENT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, + KalturaMediaEntryOrderBy::RECENT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT ); if(in_array($field, $fieldsMap)) { return $field; } + else if (array_key_exists($field, $fieldsMap)) + { + return $fieldsMap[$field]; + } else { return null; @@ -204,7 +212,6 @@ protected function splitIntoParameters($filter, $field, $fieldValue ) { $fieldParts = explode(entryFilter::FILTER_PREFIX, $field, 3); list( , $operator, $fieldName) = $fieldParts; - list($operator, $fieldName) = self::handlingFreeTextField($field, $operator, $fieldName); if(!in_array($fieldName, static::getSupportedFields()) || is_null($fieldValue) || $fieldValue === '') { @@ -249,13 +256,22 @@ protected static function changeFieldValueByPuserIdAndDuration($fieldName, $filt protected function getKESearchOrderBy($fieldValue) { - if (!$this->getMediaEntryElasticOrderBy($fieldValue)) + $fieldValue = $this->getSphinxToElasticOrderBy($fieldValue); + if (!$fieldValue) { return null; } + $eSearchOrderByItem = self::getESearchOrderByItem($fieldValue); + $orderItems = array($eSearchOrderByItem); + $kEsearchOrderBy = new ESearchOrderBy(); + $kEsearchOrderBy->setOrderItems($orderItems); + return $kEsearchOrderBy; + } + + protected static function getESearchOrderByItem($fieldValue) + { $eSearchOrderByItem = new ESearchEntryOrderByItem(); - $sortField = $this->getSphinxToElasticFieldName(substr($fieldValue,1)); - $eSearchOrderByItem->setSortField($sortField); + $eSearchOrderByItem->setSortField(substr($fieldValue,1)); $fieldNameSortOrder = $fieldValue[0]; if ($fieldNameSortOrder === self::DESC) { @@ -265,10 +281,6 @@ protected function getKESearchOrderBy($fieldValue) { $eSearchOrderByItem->setSortOrder(ESearchSortOrder::ORDER_BY_ASC); } - $orderItems = array($eSearchOrderByItem); - $kEsearchOrderBy = new ESearchOrderBy(); - $kEsearchOrderBy->setOrderItems($orderItems); - return $kEsearchOrderBy; - + return $eSearchOrderByItem; } } \ No newline at end of file From 986e354b1bb2e6e2e8af33414a774e12bfb5095c Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Wed, 3 Jul 2019 23:33:51 +0300 Subject: [PATCH 042/103] SUP-16933:fixing indentation --- .../kaltura/lib/myPlaylistUtils.class.php | 2 +- .../filters/ESearchEntryQueryFromFilter.php | 44 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index f3d292ddf8e..76d74e78ac3 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -547,7 +547,7 @@ public static function executeDynamicPlaylistViaEsearch ($entryFilters ,$totalRe list ($currEntryIds, $count) = $entryQueryToFilterESearch->retrieveElasticQueryEntryIds($entryFilter, $entryKPager); $entryIds = self::mergeEntriesByLimit($entryIds, $currEntryIds, $entryFilter->getLimit()); $totalResults = max (0, $totalResults - count($entryIds)); - if ( $totalResults == 0 ) + if ( $totalResults === 0 ) { break; } diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php index 8a21d05e601..89587ec0677 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php @@ -130,28 +130,28 @@ protected function getSphinxToElasticOrderBy($field) $fieldsMap = array( self::ASC.ESearchEntryFilterFields::MEDIA_TYPE, self::DESC.ESearchEntryFilterFields::MEDIA_TYPE, - KalturaMediaEntryOrderBy::PLAYS_ASC => self::ASC.ESearchEntryOrderByFieldName::PLAYS, - KalturaMediaEntryOrderBy::PLAYS_DESC => self::DESC.ESearchEntryOrderByFieldName::PLAYS, - KalturaMediaEntryOrderBy::VIEWS_ASC => self::ASC.ESearchEntryOrderByFieldName::VIEWS, - KalturaMediaEntryOrderBy::VIEWS_DESC => self::DESC.ESearchEntryOrderByFieldName::VIEWS, - KalturaMediaEntryOrderBy::DURATION_ASC => self::ASC.ESearchEntryFieldName::LENGTH_IN_MSECS, - KalturaMediaEntryOrderBy::DURATION_DESC => self::DESC.ESearchEntryFieldName::LENGTH_IN_MSECS, - KalturaMediaEntryOrderBy::NAME_ASC => self::ASC. ESearchEntryOrderByFieldName::NAME, - KalturaMediaEntryOrderBy::NAME_DESC => self::DESC. ESearchEntryOrderByFieldName::NAME, - KalturaMediaEntryOrderBy::CREATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, - KalturaMediaEntryOrderBy::CREATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT, - KalturaMediaEntryOrderBy::UPDATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::UPDATED_AT, - KalturaMediaEntryOrderBy::UPDATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::UPDATED_AT, - KalturaMediaEntryOrderBy::RANK_ASC => self::ASC.ESearchEntryOrderByFieldName::VOTES, - KalturaMediaEntryOrderBy::RANK_DESC => self::DESC.ESearchEntryOrderByFieldName::VOTES, - self::ASC.ESearchEntryFilterFields::TOTAL_RANK => self::ASC.ESearchEntryOrderByFieldName::VOTES, - self::DESC.ESearchEntryFilterFields::TOTAL_RANK => self::DESC.ESearchEntryOrderByFieldName::VOTES, - KalturaMediaEntryOrderBy::START_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::START_DATE, - KalturaMediaEntryOrderBy::START_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::START_DATE, - KalturaMediaEntryOrderBy::END_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::END_DATE, - KalturaMediaEntryOrderBy::END_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::END_DATE, - KalturaMediaEntryOrderBy::RECENT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, - KalturaMediaEntryOrderBy::RECENT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT + KalturaMediaEntryOrderBy::PLAYS_ASC => self::ASC.ESearchEntryOrderByFieldName::PLAYS, + KalturaMediaEntryOrderBy::PLAYS_DESC => self::DESC.ESearchEntryOrderByFieldName::PLAYS, + KalturaMediaEntryOrderBy::VIEWS_ASC => self::ASC.ESearchEntryOrderByFieldName::VIEWS, + KalturaMediaEntryOrderBy::VIEWS_DESC => self::DESC.ESearchEntryOrderByFieldName::VIEWS, + KalturaMediaEntryOrderBy::DURATION_ASC => self::ASC.ESearchEntryFieldName::LENGTH_IN_MSECS, + KalturaMediaEntryOrderBy::DURATION_DESC => self::DESC.ESearchEntryFieldName::LENGTH_IN_MSECS, + KalturaMediaEntryOrderBy::NAME_ASC => self::ASC. ESearchEntryOrderByFieldName::NAME, + KalturaMediaEntryOrderBy::NAME_DESC => self::DESC. ESearchEntryOrderByFieldName::NAME, + KalturaMediaEntryOrderBy::CREATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, + KalturaMediaEntryOrderBy::CREATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT, + KalturaMediaEntryOrderBy::UPDATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::UPDATED_AT, + KalturaMediaEntryOrderBy::UPDATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::UPDATED_AT, + KalturaMediaEntryOrderBy::RANK_ASC => self::ASC.ESearchEntryOrderByFieldName::VOTES, + KalturaMediaEntryOrderBy::RANK_DESC => self::DESC.ESearchEntryOrderByFieldName::VOTES, + self::ASC.ESearchEntryFilterFields::TOTAL_RANK => self::ASC.ESearchEntryOrderByFieldName::VOTES, + self::DESC.ESearchEntryFilterFields::TOTAL_RANK => self::DESC.ESearchEntryOrderByFieldName::VOTES, + KalturaMediaEntryOrderBy::START_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::START_DATE, + KalturaMediaEntryOrderBy::START_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::START_DATE, + KalturaMediaEntryOrderBy::END_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::END_DATE, + KalturaMediaEntryOrderBy::END_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::END_DATE, + KalturaMediaEntryOrderBy::RECENT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, + KalturaMediaEntryOrderBy::RECENT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT ); if(in_array($field, $fieldsMap)) From 10bb3a7d3dba4421ca3a6142836066dda6430e52 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Wed, 3 Jul 2019 23:55:06 +0300 Subject: [PATCH 043/103] SUP-16933:limiting the entry filter according to total results --- alpha/apps/kaltura/lib/myPlaylistUtils.class.php | 10 +++++++--- .../lib/model/filters/ESearchEntryQueryFromFilter.php | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index 76d74e78ac3..29b7047fa2d 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -545,7 +545,7 @@ public static function executeDynamicPlaylistViaEsearch ($entryFilters ,$totalRe foreach ($entryFilters as $entryFilter) { list ($currEntryIds, $count) = $entryQueryToFilterESearch->retrieveElasticQueryEntryIds($entryFilter, $entryKPager); - $entryIds = self::mergeEntriesByLimit($entryIds, $currEntryIds, $entryFilter->getLimit()); + $entryIds = self::mergeEntriesByLimit($entryIds, $currEntryIds, $entryFilter->getLimit(),$totalResults); $totalResults = max (0, $totalResults - count($entryIds)); if ( $totalResults === 0 ) { @@ -564,9 +564,13 @@ protected static function getEntriesSorted($entryIds) return $entries; } - protected static function mergeEntriesByLimit($entryIds, $currEntryIds, $limit) + protected static function mergeEntriesByLimit($entryIds, $currEntryIds, $limit, $totalResults) { - if ($limit && count($currEntryIds) > $limit) + if ($totalResults < $limit) + { + $limit = $totalResults; + } + if (count($currEntryIds) > $limit) { $currEntryIds = array_slice($currEntryIds,0, $limit); } diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php index 89587ec0677..4a3b9b27b8e 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php @@ -130,9 +130,9 @@ protected function getSphinxToElasticOrderBy($field) $fieldsMap = array( self::ASC.ESearchEntryFilterFields::MEDIA_TYPE, self::DESC.ESearchEntryFilterFields::MEDIA_TYPE, - KalturaMediaEntryOrderBy::PLAYS_ASC => self::ASC.ESearchEntryOrderByFieldName::PLAYS, + KalturaMediaEntryOrderBy::PLAYS_ASC => self::ASC.ESearchEntryOrderByFieldName::PLAYS, KalturaMediaEntryOrderBy::PLAYS_DESC => self::DESC.ESearchEntryOrderByFieldName::PLAYS, - KalturaMediaEntryOrderBy::VIEWS_ASC => self::ASC.ESearchEntryOrderByFieldName::VIEWS, + KalturaMediaEntryOrderBy::VIEWS_ASC => self::ASC.ESearchEntryOrderByFieldName::VIEWS, KalturaMediaEntryOrderBy::VIEWS_DESC => self::DESC.ESearchEntryOrderByFieldName::VIEWS, KalturaMediaEntryOrderBy::DURATION_ASC => self::ASC.ESearchEntryFieldName::LENGTH_IN_MSECS, KalturaMediaEntryOrderBy::DURATION_DESC => self::DESC.ESearchEntryFieldName::LENGTH_IN_MSECS, @@ -283,4 +283,4 @@ protected static function getESearchOrderByItem($fieldValue) } return $eSearchOrderByItem; } -} \ No newline at end of file +} From 1250edc93084148838a3d74b87eca81f6e8a7302 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Thu, 4 Jul 2019 01:05:30 +0300 Subject: [PATCH 044/103] SUP-16933:small fix --- alpha/apps/kaltura/lib/myPlaylistUtils.class.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index 29b7047fa2d..eae9e711314 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -617,9 +617,11 @@ protected static function updateEntryFilterFields($entryFilter, $partnerId) protected static function unsetDates($filter) { foreach (self::$dates as $date) - if ($filter->is_set($date)) { - $filter->unsetByName($date); + if ($filter->is_set($date)) + { + $filter->unsetByName($date); + } } } From d9a17de5391e57a0ae5f79acb4d875ef3e16a158 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 4 Jul 2019 13:44:31 +0300 Subject: [PATCH 045/103] PLAT-9902 SCC to SRT parsing when uploading captions --- configurations/batch/batch.ini.template | 10 +- infra/general/kSCCParser.php | 158 ++++++++++++++++++ .../content/caption/base/CaptionPlugin.php | 10 +- .../KAsyncParseSccCaptionAsset.class.php | 128 ++++++++++++++ .../KAsyncParseSccCaptionAssetExe.php | 13 ++ .../KalturaParseSccCaptionAssetJobData.php | 37 ++++ .../sccCaptionsContentManager.php | 142 ++++++++++++++++ .../base/lib/kCaptionsContentManager.php | 39 ++++- .../enums/ParseSccCaptionAssetBatchType.php | 25 +++ .../model/kParseSccCaptionAssetJobData.php | 71 ++++++++ .../base/services/CaptionAssetService.php | 12 ++ 11 files changed, 641 insertions(+), 4 deletions(-) create mode 100644 infra/general/kSCCParser.php create mode 100644 plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAsset.class.php create mode 100644 plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php create mode 100644 plugins/content/caption/base/lib/api/KalturaParseSccCaptionAssetJobData.php create mode 100644 plugins/content/caption/base/lib/captionManagers/sccCaptionsContentManager.php create mode 100644 plugins/content/caption/base/lib/model/enums/ParseSccCaptionAssetBatchType.php create mode 100644 plugins/content/caption/base/lib/model/kParseSccCaptionAssetJobData.php diff --git a/configurations/batch/batch.ini.template b/configurations/batch/batch.ini.template index c3d43b6b671..18999456301 100644 --- a/configurations/batch/batch.ini.template +++ b/configurations/batch/batch.ini.template @@ -830,4 +830,12 @@ type = KAsyncReportExport maximumExecutionTime = 12000 scriptPath = batches/ReportExport/KAsyncReportExportExe.php params.localTempPath = @TMP_DIR@/reports -params.sharedTempPath = @WEB_DIR@/apptemp-shared/reports \ No newline at end of file +params.sharedTempPath = @WEB_DIR@/apptemp-shared/reports + +[KAsyncParseSccCaptionAsset : JobHandlerWorker] +id = 770 +name = KAsyncParseSccCaptionAsset +friendlyName = parse scc caption asset +type = KAsyncParseSccCaptionAsset +maximumExecutionTime = 300 +scriptPath = ../plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php \ No newline at end of file diff --git a/infra/general/kSCCParser.php b/infra/general/kSCCParser.php new file mode 100644 index 00000000000..51c931fceab --- /dev/null +++ b/infra/general/kSCCParser.php @@ -0,0 +1,158 @@ +\d{2}):(?P\d{2}):(?P\d{2})(:|;)(?P\d{2})(?P.+)\r?(\n|$)?\s*\r?(\n|$)/sU'; + + const commandCodes = array('94ae','9420','942c','942f','947a','97a1','97a2','9723','9120', '91a1', '91a2', '9123', '91a4', '9125', '9126', '91a7', '91a8', '9129', '912a', '91ab', '912c', '91ad', '91ae', '912f', '94a8'); + + private static $rowCodes = array('91','92','15','16','97','13','94'); + + private static $columnsCodes = array('d0','51','c2','43','c4','45','46','c7','c8','49','4a','cb','4c','cd','70','f1','62','e3','64','e5','e6','67','68','e9','ea','6b','ec','6d','52','d3','54','d5','d6','57','58','d9','da','5b','dc','5d','5e','df','f2','73','f4','75','76','f7','f8','79','7a','fb','7c','fd','fe','7f'); + + private static $singleRowCode = array('10'); + + private static $singleColumnCode = array('d0','51','c2','43','c4','45','46','c7','c8','49','4a','cb','4c','cd','52','d3','54','d5','d6','57','58','d9','da','5b','dc','5d','5e','df'); + + private static $specialChars = array('91b0'=>'®','9131'=>'°','9132'=>'½','91b3'=>'¿','9134'=>'™','91b5'=>'¢','91b6'=>'£','9137' => '♪','9138'=>'à','91b9'=>' ','91ba'=>'è','913b'=>'â', + '91bc'=>'ê','913d'=>'î','913e'=>'ô','91bf'=>'û'); + + private static $extendedChars = array('9220'=>'Á','92a1'=>'É','92a2'=>'Ó','9223'=>'Ú','92a4'=>'Ü','9225'=>'ü','9226'=>'‘','92a7'=>'¡','92a8'=>'*','9229'=>'’','922a'=>'—','92ab'=>'©', + '922c'=>'℠','92ad'=>'•','92ae'=>'“','922f'=>'”','92b0'=>'À','9231'=>'Â','9232'=>'Ç','92b3'=>'È','9234'=>'Ê','92b5'=>'Ë','92b6'=>'ë','9237'=>'Î', + '9238'=>'Ï','92b9'=>'ï','92ba'=>'Ô','923b'=>'Ù','92bc'=>'ù','923d'=>'Û','923e'=>'«','92bf'=>'»','1320'=>'Ã','13a1'=>'ã','13a2'=>'Í','1323'=>'Ì', + '13a4'=>'ì','1325'=>'Ò','1326'=>'ò','13a7'=>'Õ','13a8'=>'õ','1329'=>'{','132a'=>'}','13ab'=>'\\','132c'=>'^','13ad'=>'_','13ae'=>'¦','132f'=>'~', + '13b0'=>'Ä','1331'=>'ä','1332'=>'Ö','13b3'=>'ö','1334'=>'ß','13b5'=>'¥','13b6'=>'¤','1337'=>'|','1338'=>'Å','13b9'=>'å','13ba'=>'Ø','133b'=>'ø', + '9bbc'=>'┌','9b3d'=>'┐','9b3e'=>'└','9bbf'=>'┘'); + + private static $twoByteCharset = array('20' =>' ','a1' =>'!','a2' =>'"','23' =>'#','a4' =>'$','25' =>'%','26' =>'&','a7' =>"'",'a8' =>'(','29' =>')','2a' =>'á','ab' =>'+', + '2c' =>',','ad' =>'-','ae' =>'.','2f' =>'/','b0' =>'0','31' =>'1','32' =>'2','b3' =>'3','34' =>'4','b5' =>'5','b6' =>'6','37' =>'7', + '38' =>'8','b9' =>'9','ba' =>':','3b' =>';','bc' =>'<','3d' =>'=','3e' =>'>','bf' =>'?','40' =>'@','c1' =>'A','c2' =>'B','43' =>'C', + 'c4' =>'D','45' =>'E','46' =>'F','c7' =>'G','c8' =>'H','49' =>'I','4a' =>'J','cb' =>'K','4c' =>'L','cd' =>'M','ce' =>'N','4f' =>'O', + 'd0' =>'P','51' =>'Q','52' =>'R','d3' =>'S','54' =>'T','d5' =>'U','d6' =>'V','57' =>'W','58' =>'X','d9' =>'Y','da' =>'Z','5b' =>'[', + 'dc' =>'é','5d' =>']','5e' =>'í','df' =>'ó','e0' =>'ú','61' =>'a','62' =>'b','e3' =>'c','64' =>'d','e5' =>'e','e6' =>'f','67' =>'g', + '68' =>'h','e9' =>'i','ea' =>'j','6b' =>'k','ec' =>'l','6d' =>'m','6e' =>'n','ef' =>'o','70' =>'p','f1' =>'q','f2' =>'r','73' =>'s', + 'f4' =>'t','75' =>'u','76' =>'v','f7' =>'w','f8' =>'x','79' =>'y','7a' =>'z','fb' =>'ç','7c' =>'÷','fd' =>'Ñ','fe' =>'ñ'); + + /** + * @param string $content + * @return array + */ + public static function parseToSrt($content) + { + if (kString::beginsWith($content, "\xff\xfe")) + { + $content = iconv('utf-16', 'utf-8', substr($content, 2)); + } + + if (!preg_match_all(self::timingRegex, $content, $matches) || !count($matches) || !count($matches[0])) + { + KalturaLog::err("Content regex not found"); + print("Content regex not found"); + return array(); + } + $text = ''; + $rowCount = 1; + foreach ($matches[0] as $index => $match) + { + $content = $matches['content'][$index]; + $startTime = self::maketime($matches, $index); + $endTime = $startTime; + if (isset( $matches['content'][$index + 1])) + { + $endTime = self::maketime($matches, $index + 1); + } + $translatedContent = self::tranlasteContent($content); + if ($translatedContent) + { + $text .= "$rowCount\n$startTime --> $endTime\n$translatedContent\n\n"; + $rowCount++; + } + } + return $text; + } + + /** + * @param $content + * @return null|string + */ + private static function tranlasteContent($content) + { + //remove unncessary scc commands that are irrelevant to the text. + $content = str_replace(self::commandCodes, "", $content); + + $parts = preg_split('/\s+/', trim($content)); + $text = null; + for ($index = 0; $index < count($parts); $index++) + { + $code = $parts[$index]; + switch (true) + { + case isset(self::$specialChars[$code]): + { + $text .= self::$specialChars[$code]; + // in case we have double padding for commands we need to ignore next command + if (isset($parts[$index + 1]) && $code == $parts[$index + 1]) + { + $index++; + } + break; + } + case isset(self::$extendedChars[$code]): + { + $text .= self::$extendedChars[$code]; + break; + } + default: + { + $codes = str_split($code, 2); + //check if codes represent location-cursor move + if ((in_array($codes[0], self::$rowCodes) && in_array($codes[1], self::$columnsCodes)) + || (in_array($codes[0], self::$singleRowCode) && in_array($codes[1], self::$singleColumnCode)) + ) + { + $text .= ' '; + // in case we have double padding for commands we need to ignore next command + if (isset($parts[$index + 1]) && $code == $parts[$index + 1]) + { + $index++; + } + + } + else + { + //check if codes represent 2Byte chars and add them to text. + foreach ($codes as $charsetCode) + { + if (isset(self::$twoByteCharset[$charsetCode])) + { + $text .= self::$twoByteCharset[$charsetCode]; + } + } + } + } + } + } + return trim($text); + } + + /** + * @param $matches + * @param $index + * @return array + */ + protected static function maketime($matches, $index) + { + $Hours = $matches['startHours'][$index]; + $Minutes = $matches['startMinutes'][$index]; + $Seconds = $matches['startSeconds'][$index]; + $Frame = round($matches['startFrame'][$index] * 41.7); + return "$Hours:$Minutes:$Seconds,$Frame"; + + } +} diff --git a/plugins/content/caption/base/CaptionPlugin.php b/plugins/content/caption/base/CaptionPlugin.php index 7e586d213fa..99bf9efabb8 100644 --- a/plugins/content/caption/base/CaptionPlugin.php +++ b/plugins/content/caption/base/CaptionPlugin.php @@ -224,7 +224,7 @@ public static function getEventConsumers() public static function getEnums($baseEnumName = null) { if(is_null($baseEnumName)) - return array('CaptionAssetType', 'CaptionObjectFeatureType', 'ParseMultiLanguageCaptionAssetBatchType'); + return array('CaptionAssetType', 'CaptionObjectFeatureType', 'ParseMultiLanguageCaptionAssetBatchType','ParseSccCaptionAssetBatchType'); if($baseEnumName == 'assetType') return array('CaptionAssetType'); @@ -233,7 +233,7 @@ public static function getEnums($baseEnumName = null) return array('CaptionObjectFeatureType'); if ($baseEnumName == 'BatchJobType') - return array('ParseMultiLanguageCaptionAssetBatchType'); + return array('ParseMultiLanguageCaptionAssetBatchType', 'ParseSccCaptionAssetBatchType'); return array(); } @@ -255,6 +255,12 @@ public static function loadObject($baseClass, $enumValue, array $constructorArgs if($baseClass == 'KalturaJobData' && $enumValue == self::getApiValue(ParseMultiLanguageCaptionAssetBatchType::PARSE_MULTI_LANGUAGE_CAPTION_ASSET)) return new KalturaParseMultiLanguageCaptionAssetJobData(); + if($baseClass == 'KalturaJobData' && $enumValue == self::getApiValue(ParseSccCaptionAssetBatchType::PARSE_SCC_CAPTION_ASSET)) + return new KalturaParseSccCaptionAssetJobData(); + + if($baseClass == 'kJobData' && $enumValue == self::getBatchJobTypeCoreValue(ParseSccCaptionAssetBatchType::PARSE_SCC_CAPTION_ASSET)) + return new KalturaParseSccCaptionAssetJobData(); + return null; } diff --git a/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAsset.class.php b/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAsset.class.php new file mode 100644 index 00000000000..a97621b8ef3 --- /dev/null +++ b/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAsset.class.php @@ -0,0 +1,128 @@ +parseSccCaption($job, $job->data); + } + + protected function parseSccCaption(KalturaBatchJob $job, KalturaParseSccCaptionAssetJobData $data) + { + $this->updateJob($job, "Start parsing scc caption asset [$data->sccCaptionAssetId]", KalturaBatchJobStatus::QUEUED); + + $this->captionClientPlugin = KalturaCaptionClientPlugin::get(self::$kClient); + + $captionAssetId = $data->sccCaptionAssetId; + $fileLoc = $data->fileLocation; + + $content = kEncryptFileUtils::getEncryptedFileContent($fileLoc, $data->fileEncryptionKey, kConf::get("encryption_iv")); + if (!$content) + { + $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_GET_FILE', "Error: " . 'UNABLE_TO_GET_FILE', KalturaBatchJobStatus::FAILED, $data); + return $job; + } + + self::impersonate($job->partnerId); + $parsedContent = kSCCParser::parseToSrt($content); + + $sccCaptionAsset = $this->captionClientPlugin->captionAsset->get($captionAssetId); + if (!$sccCaptionAsset) + { + $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_GET_ORIGINAL_SCC_ASSET', "Error: " . 'UNABLE_TO_GET_ORIGINAL_SCC_ASSET', KalturaBatchJobStatus::FAILED, $data); + return $job; + } + + $res = $this->deleteCaptionAsset($captionAssetId); + if (!$res) + { + $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_DELETE_ORIGINAL_SCC_ASSET', "Error: " . 'UNABLE_TO_DELETE_ORIGINAL_SCC_ASSET', KalturaBatchJobStatus::FAILED, $data); + return $job; + } + + $captionsCreated = $this->cloneCaptionAssetToSrtAndSetContent($sccCaptionAsset->entryId, $sccCaptionAsset, $parsedContent); + self::unimpersonate(); + if ($captionsCreated) + { + $this->closeJob($job, null, null, "Finished parsing scc captions to srt", KalturaBatchJobStatus::FINISHED); + return $job; + } + else + throw new kApplicativeException(KalturaBatchJobAppErrors::MISSING_ASSETS, "NABLE_TO_CREATE_ASSET_WITH_SRT_CAPTION"); + } + + + /** + * @param $id + * @return bool + */ + private function deleteCaptionAsset($id) + { + try + { + $this->captionClientPlugin->captionAsset->delete($id); + return true; + } + catch(Exception $e) + { + KalturaLog::info("problem with caption content-setting id - $id - " . $e->getMessage()); + return false; + } + } + + /** + * @param $entryId + * @param $captionAsset + * @param $content + * @return bool + */ + private function cloneCaptionAssetToSrtAndSetContent($entryId, $captionAsset, $content) + { + $captionAsset->id = null; + $captionAsset->entryId = null; + $captionAsset->languageCode = null; + $captionAsset->partnerId = null; + $captionAsset->createdAt = null; + $captionAsset->updatedAt = null; + $captionAsset->version = null; + $captionAsset->size = null; + $captionAsset->description = null; + $captionAsset->status = null; + + $captionAsset->fileExt = 'srt'; + $captionAsset->format = KalturaCaptionType::SRT; + + try + { + $newCaptionAsset = $this->captionClientPlugin->captionAsset->add($entryId, $captionAsset); + } + catch (Exception $e) + { + KalturaLog::info("problem adding srt caption asset - " . $e->getMessage()); + return false; + } + $contentResource = new KalturaStringResource(); + $contentResource->content = $content; + return $this->captionClientPlugin->captionAsset->setContent($newCaptionAsset->id, $contentResource); + } + +} \ No newline at end of file diff --git a/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php b/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php new file mode 100644 index 00000000000..86445f469d7 --- /dev/null +++ b/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php @@ -0,0 +1,13 @@ +run(); +$instance->done(); diff --git a/plugins/content/caption/base/lib/api/KalturaParseSccCaptionAssetJobData.php b/plugins/content/caption/base/lib/api/KalturaParseSccCaptionAssetJobData.php new file mode 100644 index 00000000000..3f754c93b8b --- /dev/null +++ b/plugins/content/caption/base/lib/api/KalturaParseSccCaptionAssetJobData.php @@ -0,0 +1,37 @@ + ((?:[0-9]{2}:)?[0-9]{2}:[0-9]{2}\,[0-9]{3})( .*)?$#'; + + /* (non-PHPdoc) + * @see kCaptionsContentManager::parse() + */ + public function parse($content) + { + if (kString::beginsWith($content, "\xff\xfe")) + { + $content = iconv('utf-16', 'utf-8', substr($content, 2)); + } + + $matches = null; + $regex = '/(?P\d+)\s*\r?\n\s*(?P\d{1,2}):(?P\d{1,2}):(?P\d{1,2})[,\.](?P\d{1,3})\s*-->\s*(?P\d{1,2}):(?P\d{1,2}):(?P\d{1,2})[,\.](?P\d{1,3})\s*\r?\n((?P.+)\r?(\n|$))?\s*\r?(\n|$)/sU'; + if(!preg_match_all($regex, $content, $matches) || !count($matches) || !count($matches[0])) + { + KalturaLog::err("Content regex not found"); + return array(); + } + + $itemsData = array(); + foreach($matches[0] as $index => $match) + { + $startHours = intval($matches['startHours'][$index]); + $startMinutes = intval($matches['startMinutes'][$index]); + $startSeconds = intval($matches['startSeconds'][$index]); + $startMilliseconds = intval($matches['startMilliseconds'][$index]); + $endHours = intval($matches['endHours'][$index]); + $endMinutes = intval($matches['endMinutes'][$index]); + $endSeconds = intval($matches['endSeconds'][$index]); + $endMilliseconds = intval($matches['endMilliseconds'][$index]); + $content = $matches['content'][$index]; + + $itemsData[] = array( + 'startTime' => $this->makeTime($startHours, $startMinutes, $startSeconds, $startMilliseconds), + 'endTime' => $this->makeTime($endHours, $endMinutes, $endSeconds, $endMilliseconds), + 'content' => array(array('text' => $content)), + ); + } + return $itemsData; + } + + private function makeTime($hours, $minutes, $seconds, $milliseconds) + { + $ret = $hours * 60; + $ret += $minutes; + $ret *= 60; + $ret += $seconds; + $ret *= 1000; + $ret += $milliseconds; + return $ret; + } + + /* (non-PHPdoc) + * @see kCaptionsContentManager::getContent() + */ + public function getContent($content) + { + if (kString::beginsWith($content, "\xff\xfe")) + { + $content = iconv('utf-16', 'utf-8', substr($content, 2)); + } + + $replace = array( + '/^[\d]+\s*[\r\n]+/' => '', + '/[\r\n]+\s*[\d]+\s*[\r\n]+/' => '', + '/[\d]{2}:[\d]{2}:[\d]{2},[\d]{3}/' => '', + '/\s+-->\s+/' => ' ', + '/\s+/' => ' ', + ); + return preg_replace(array_keys($replace), $replace, $content); + } + + /** + * @return srtCaptionsContentManager + */ + public static function get() + { + return new srtCaptionsContentManager(); + } + + public function buildFile($content, $clipStartTime, $clipEndTime, $globalOffset = 0) + { + $newFileContent = $this->createCaptionsFile($content, $clipStartTime, $clipEndTime, self::SRT_TIMECODE_PATTERN, $globalOffset); + return $newFileContent; + } + + + protected function createAdjustedTimeLine($matches, $clipStartTime, $clipEndTime, $globalOffset) + { + $startCaption = self::parseCaptionTime($matches[1]); + $endCaption = self::parseCaptionTime($matches[2]); + if (!TimeOffsetUtils::onTimeRange($startCaption, $endCaption, $clipStartTime, $clipEndTime)) + return null; + $adjustedStartTime = TimeOffsetUtils::getAdjustedStartTime($startCaption, $clipStartTime, $globalOffset); + $adjustedEndTime = TimeOffsetUtils::getAdjustedEndTime($endCaption, $clipStartTime, $clipEndTime, $globalOffset); + $timeLine = $this->formatSrtTimeStamp($adjustedStartTime) . ' --> ' . $this->formatSrtTimeStamp($adjustedEndTime). kCaptionsContentManager::UNIX_LINE_ENDING; + return $timeLine; + } + + + /** + * @param int $timeStamp + * @return string + */ + private function formatSrtTimeStamp($timeInMili) + { + $seconds = $timeInMili / 1000; + $remainder = round($seconds - ($seconds >> 0), 3) * 1000; + $formatted_remainder = sprintf("%03d", $remainder); + return gmdate('H:i:s,', $seconds).$formatted_remainder; + } + + /** + * @param $time + * @return string + */ + public function parseCaptionTime($time) + { + $time = str_replace(',','.',$time); + $captionTime = parent::parseCaptionTime($time); + return $captionTime; + } + + /** + * @param string $content + * @param string $toAppend + * @return string + */ + public function merge($content, $toAppend) + { + return $content . $toAppend; + } +} \ No newline at end of file diff --git a/plugins/content/caption/base/lib/kCaptionsContentManager.php b/plugins/content/caption/base/lib/kCaptionsContentManager.php index 3668601abe7..a263a88655e 100644 --- a/plugins/content/caption/base/lib/kCaptionsContentManager.php +++ b/plugins/content/caption/base/lib/kCaptionsContentManager.php @@ -143,7 +143,13 @@ public static function getApiContentManager($type) return KalturaPluginManager::loadObject('kCaptionsContentManager', $type); } } - + + /** + * @param $captionAsset + * @param $fileLocation + * @param null $key + * @return BatchJob + */ public static function addParseMultiLanguageCaptionAssetJob($captionAsset, $fileLocation, $key = null) { $batchJob = new BatchJob(); @@ -166,6 +172,37 @@ public static function addParseMultiLanguageCaptionAssetJob($captionAsset, $file return kJobsManager::addJob($batchJob, $jobData, $jobType); } + /** + * @param $captionAsset + * @param $fileLocation + * @param null $syncKey + * @return BatchJob + */ + public static function addParseSccCaptionAssetJob($captionAsset) + { + $batchJob = new BatchJob(); + + $syncKey = $captionAsset->getSyncKey(CaptionAsset::FILE_SYNC_ASSET_SUB_TYPE_ASSET); + $fileSync = kFileSyncUtils::getLocalFileSyncForKey($syncKey); + $fileLocation = $fileSync->getFullPath(); + + $id = $captionAsset->getId(); + $entryId = $captionAsset->getEntryId(); + + $jobData = new kParseSccCaptionAssetJobData(); + $jobData->setSccCaptionAssetId($id); + $jobData->setFileLocation($fileLocation); + $jobData->setFileEncryptionKey($fileSync->getEncryptionKey()); + + $jobType = CaptionPlugin::getBatchJobTypeCoreValue(ParseSccCaptionAssetBatchType::PARSE_SCC_CAPTION_ASSET); + $batchJob->setObjectType(BatchJobObjectType::ASSET); + $batchJob->setEntryId($entryId); + $batchJob->setPartnerId($captionAsset->getPartnerId()); + $batchJob->setObjectId($id); + + return kJobsManager::addJob($batchJob, $jobData, $jobType); + } + diff --git a/plugins/content/caption/base/lib/model/enums/ParseSccCaptionAssetBatchType.php b/plugins/content/caption/base/lib/model/enums/ParseSccCaptionAssetBatchType.php new file mode 100644 index 00000000000..3db5470d63b --- /dev/null +++ b/plugins/content/caption/base/lib/model/enums/ParseSccCaptionAssetBatchType.php @@ -0,0 +1,25 @@ + self::PARSE_SCC_CAPTION_ASSET + ); + } + + /** + * @return array + */ + public static function getAdditionalDescriptions() + { + return array(); + } + } + diff --git a/plugins/content/caption/base/lib/model/kParseSccCaptionAssetJobData.php b/plugins/content/caption/base/lib/model/kParseSccCaptionAssetJobData.php new file mode 100644 index 00000000000..c1fc8e3d8e2 --- /dev/null +++ b/plugins/content/caption/base/lib/model/kParseSccCaptionAssetJobData.php @@ -0,0 +1,71 @@ +sccCaptionAssetId; + } + + /** + * @param string $captionAssetId + */ + public function setSccCaptionAssetId($captionAssetId) + { + $this->sccCaptionAssetId = $captionAssetId; + } + + /** + * @return string $fileLocation + */ + public function getFileLocation() + { + return $this->fileLocation; + } + + /** + * @param string $fileLocation + */ + public function setFileLocation($fileLocation) + { + $this->fileLocation = $fileLocation; + } + + /** + * @return string $fileEncryptionKey + */ + public function getFileEncryptionKey() + { + return $this->fileEncryptionKey; + } + + /** + * @param string $fileEncryptionKey + */ + public function setFileEncryptionKey($fileEncryptionKey) + { + $this->fileEncryptionKey = $fileEncryptionKey; + } + +} \ No newline at end of file diff --git a/plugins/content/caption/base/services/CaptionAssetService.php b/plugins/content/caption/base/services/CaptionAssetService.php index e55ca3d185e..3f22d6e1c8f 100644 --- a/plugins/content/caption/base/services/CaptionAssetService.php +++ b/plugins/content/caption/base/services/CaptionAssetService.php @@ -10,6 +10,7 @@ class CaptionAssetService extends KalturaAssetService { const MAX_SERVE_WEBVTT_FILE_SIZE = 1048576; + const SSC_CAPTION_FILE_EXT = 'scc'; protected function kalturaNetworkAllowed($actionName) { @@ -118,6 +119,17 @@ function setContentAction($id, KalturaContentResource $contentResource) $contentResource->validateAsset($dbCaptionAsset); $kContentResource = $contentResource->toObject(); $this->attachContentResource($dbCaptionAsset, $kContentResource); + + if (strtolower($dbCaptionAsset->getFileExt()) === self::SSC_CAPTION_FILE_EXT) + { + //start scc batch Job. + kCaptionsContentManager::addParseSccCaptionAssetJob($dbCaptionAsset); + + $captionAsset = new KalturaCaptionAsset(); + $captionAsset->fromObject($dbCaptionAsset, $this->getResponseProfile()); + return $captionAsset; + } + $contentResource->entryHandled($dbCaptionAsset->getentry()); $newStatuses = array( From 55e46f2cc2532f28aca094e15f89d189ef3d32c9 Mon Sep 17 00:00:00 2001 From: Jess Portnoy Date: Fri, 5 Jul 2019 16:00:45 +0100 Subject: [PATCH 046/103] Same problem as with https://github.com/kaltura/server/pull/8515/commits/3718727ce18bfb519fa9cb0b103d3da5d1dc1a3a. Sigh. --- deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql diff --git a/deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql b/deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql new file mode 100644 index 00000000000..6ae812509d3 --- /dev/null +++ b/deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql @@ -0,0 +1 @@ +ALTER TABLE kaltura_sphinx_log.sphinx_log ADD type INT AFTER created_at; From 1d6d17ca2c61ef443ea45754707c915ecb47d388 Mon Sep 17 00:00:00 2001 From: Jess Portnoy Date: Fri, 5 Jul 2019 16:02:38 +0100 Subject: [PATCH 047/103] And, again. --- .../sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 deployment/updates/sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql diff --git a/deployment/updates/sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql b/deployment/updates/sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql new file mode 100644 index 00000000000..85ad506328f --- /dev/null +++ b/deployment/updates/sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql @@ -0,0 +1 @@ +ALTER TABLE kaltura_sphinx_log.sphinx_log ADD index_name VARCHAR(128) AFTER type; From cec3b1c586d2afadaf78ee2025df3b393e588796 Mon Sep 17 00:00:00 2001 From: hilak Date: Sat, 6 Jul 2019 05:14:27 -0400 Subject: [PATCH 048/103] Update setEntryCreationDate.php --- alpha/scripts/utils/setEntryCreationDate.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpha/scripts/utils/setEntryCreationDate.php b/alpha/scripts/utils/setEntryCreationDate.php index 0e7232d252e..4a7a73ae644 100644 --- a/alpha/scripts/utils/setEntryCreationDate.php +++ b/alpha/scripts/utils/setEntryCreationDate.php @@ -7,8 +7,8 @@ exit; } $mapping = $argv[1]; -$isDateString = ($argv[2] == "true"); -$dryRun = ($argv[3] == "dryrun"); +$isDateString = ($argv[2] === "true"); +$dryRun = ($argv[3] === "dryrun"); require_once(__DIR__ . '/../bootstrap.php'); From cd76277083300927b805c9de6afa2724eae5d1f4 Mon Sep 17 00:00:00 2001 From: hilak Date: Sat, 6 Jul 2019 05:19:05 -0400 Subject: [PATCH 049/103] Update entry.php --- alpha/lib/model/entry.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alpha/lib/model/entry.php b/alpha/lib/model/entry.php index 0cd95626c45..9f4c3ef1343 100644 --- a/alpha/lib/model/entry.php +++ b/alpha/lib/model/entry.php @@ -1843,6 +1843,9 @@ public function getSourceEntryId() { return $this->getFromCustomData( "sourceE public function setReachedMaxRecordingDuration ( $v ) { $this->putInCustomData ( "reachedMaxRecordingDuration" , (bool) $v ); } public function getReachedMaxRecordingDuration() { return (bool) $this->getFromCustomData( "reachedMaxRecordingDuration" ,null, false ); } + public function setOriginalCreationDate ( $v ) { $this->putInCustomData ( "originalCreationDate" , $v); } + public function getOriginalCreationDate() { return $this->getFromCustomData( "originalCreationDate", null, null); } + public function getParentEntry() { From be9a14832dde3e0260758ed066667212b60e2514 Mon Sep 17 00:00:00 2001 From: hilak Date: Sat, 6 Jul 2019 05:20:05 -0400 Subject: [PATCH 050/103] Update setEntryCreationDate.php --- alpha/scripts/utils/setEntryCreationDate.php | 1 + 1 file changed, 1 insertion(+) diff --git a/alpha/scripts/utils/setEntryCreationDate.php b/alpha/scripts/utils/setEntryCreationDate.php index 4a7a73ae644..a5102a924ff 100644 --- a/alpha/scripts/utils/setEntryCreationDate.php +++ b/alpha/scripts/utils/setEntryCreationDate.php @@ -29,6 +29,7 @@ continue; } + $entry->setOriginalCreationDate($entry->getCreatedAt()); $entry->setCreatedAt($createdAt); $entry->setAvailableFrom($createdAt); $entry->save(); From 6c3461a2fb939b0cd16f5e898036fe328c779191 Mon Sep 17 00:00:00 2001 From: TalDubovKaltura Date: Sun, 7 Jul 2019 11:01:59 +0300 Subject: [PATCH 051/103] PSVAMB-7785 - CR Hila&Moshe: conventions and naming --- alpha/lib/enums/PermissionName.php | 2 +- api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php | 5 +++-- configurations/admin.template.ini | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/alpha/lib/enums/PermissionName.php b/alpha/lib/enums/PermissionName.php index 0ad1814e355..86db4697b8d 100644 --- a/alpha/lib/enums/PermissionName.php +++ b/alpha/lib/enums/PermissionName.php @@ -276,5 +276,5 @@ interface PermissionName extends BaseEnum const FEATURE_ANALYTICS_PERSISTENT_SESSION_ID = 'FEATURE_ANALYTICS_PERSISTENT_SESSION_ID'; const FEATURE_LIVE_ANALYTICS_DASHBOARD = 'FEATURE_LIVE_ANALYTICS_DASHBOARD'; - const FEATURE_LIVE_ENTRIES_IN_FEED = 'FEATURE_LIVE_ENTRIES_IN_FEED'; + const FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED = 'FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED'; } diff --git a/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php b/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php index 5ae86e19610..26e6b9af741 100644 --- a/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php +++ b/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php @@ -185,8 +185,9 @@ public function __construct($feedId, $feedProcessingKey = null, $ks = null, $sta { $mediaEntrytypes = array(entryType::MEDIA_CLIP, entryType::MIX); $partner = PartnerPeer::retrieveByPK($this->getSyndicationFeedDb()->getPartnerId()); - $liveEntriesView = $partner->getEnabledService(KalturaPermissionName::FEATURE_LIVE_ENTRIES_IN_FEED); - if($liveEntriesView) { + $liveEntriesView = $partner->getEnabledService(KalturaPermissionName::FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED); + if($liveEntriesView) + { $mediaEntrytypes[] = entryType::LIVE_STREAM; } $this->baseCriteria->addAnd(entryPeer::TYPE, $mediaEntrytypes, Criteria::IN); diff --git a/configurations/admin.template.ini b/configurations/admin.template.ini index 418d373c52d..47bb996a151 100644 --- a/configurations/admin.template.ini +++ b/configurations/admin.template.ini @@ -1039,8 +1039,8 @@ moduls.limitAllowedActions.group = GROUP_ENABLE_DISABLE_FEATURES moduls.feedWithLiveEntries.enabled = true moduls.feedWithLiveEntries.permissionType = 2 -moduls.feedWithLiveEntries.label = "Show live entries in feed" -moduls.feedWithLiveEntries.permissionName = FEATURE_LIVE_ENTRIES_IN_FEED +moduls.feedWithLiveEntries.label = "Include live entries in feed" +moduls.feedWithLiveEntries.permissionName = FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED moduls.feedWithLiveEntries.basePermissionType = moduls.feedWithLiveEntries.basePermissionName = moduls.feedWithLiveEntries.group = GROUP_ENABLE_DISABLE_FEATURES From a3055ba0db3b21303b74770d7eee62d40f89d2a4 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Sun, 7 Jul 2019 16:09:23 +0300 Subject: [PATCH 052/103] PLAT-9784:adding sender to the header of the mail --- alpha/lib/model/Partner.php | 9 ++++--- ...KDispatchEmailNotificationEngine.class.php | 13 +++++++-- .../lib/model/EmailNotificationTemplate.php | 27 ++++++++++--------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/alpha/lib/model/Partner.php b/alpha/lib/model/Partner.php index 3abe0b6a4f2..698b8cab123 100644 --- a/alpha/lib/model/Partner.php +++ b/alpha/lib/model/Partner.php @@ -82,10 +82,10 @@ class Partner extends BasePartner const RTC_SERVER_NODE_ENV = 'rtc_server_node_env'; - const CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST = 'allowedFromEmailWhiteList'; - const ANALYTICS_HOST = "analytics_host"; + const CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST = 'allowedFromEmailWhiteList'; + private $cdnWhiteListCache = array(); public function save(PropelPDO $con = null) @@ -210,6 +210,7 @@ public function getAllowedFromEmailWhiteList() public function setAllowedFromEmailWhiteList( $emails ) { + $emails = implode(',',array_map('trim',explode(',',$emails))); $this->putInCustomData( self::CUSTOM_DATA_ALLOWED_FROM_EMAIL_WHITELIST, $emails); } @@ -238,7 +239,7 @@ public function setReturnDuplicateKshow( $v ) public function getAllowQuickEdit() { - return (int)$this->getFromCustomData( "allowQuickEdit" , null , true ); + return $this->getFromCustomData( "allowQuickEdit" , null , true ); } public function setAllowQuickEdit( $v ) @@ -422,7 +423,7 @@ public function setNotificationsConfig( $v ) public function getAllowMultiNotification() { - return (int)$this->getFromCustomData( "allowMultiNotification" , null ); + return $this->getFromCustomData( "allowMultiNotification" , null ); } public function setAllowMultiNotification( $v ) diff --git a/plugins/event_notification/providers/email/lib/batch/KDispatchEmailNotificationEngine.class.php b/plugins/event_notification/providers/email/lib/batch/KDispatchEmailNotificationEngine.class.php index ad4ca83ae40..97fbddaa10b 100644 --- a/plugins/event_notification/providers/email/lib/batch/KDispatchEmailNotificationEngine.class.php +++ b/plugins/event_notification/providers/email/lib/batch/KDispatchEmailNotificationEngine.class.php @@ -12,6 +12,8 @@ class KDispatchEmailNotificationEngine extends KDispatchEventNotificationEngine const BCC_RECIPIENT_TYPE = 'BCC'; const REPLYTO_RECIPIENT_TYPE = 'ReplyTo'; + + const FROM_EMAIL = '{from_email}'; /** * Old kaltura default @@ -216,8 +218,15 @@ protected function sendEmail(KalturaEmailNotificationTemplate $emailNotification $email = str_replace(array_keys($contentParameters), $contentParameters, $email); $name = str_replace(array_keys($contentParameters), $contentParameters, $name); } - - $this::$mailer->Sender = $email; + + if (isset($contentParameters[self::FROM_EMAIL])) + { + $this::$mailer->Sender = $contentParameters[self::FROM_EMAIL]; + } + else + { + $this::$mailer->Sender = $email; + } $this::$mailer->From = $email; $this::$mailer->FromName = $name; } diff --git a/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php b/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php index c27483e46c8..202e8450259 100644 --- a/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php +++ b/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php @@ -19,6 +19,7 @@ class EmailNotificationTemplate extends BatchEventNotificationTemplate implement const CUSTOM_DATA_MESSAGE_ID = 'messageID'; const CUSTOM_DATA_CUSTOM_HEADERS = 'customHeaders'; const CUSTOM_DATA_BODY_FILE_VERSION = 'bodyFileVersion'; + const FROM_EMAIL = '{from_email}'; const FILE_SYNC_BODY = 1; @@ -38,22 +39,22 @@ public function copy($deepCopy = false) return $returnObj; } - protected function getEmail() + protected function getAllowedEmail() { - $email = $this->getFromEmail(); - $partnerNotificationEmail = "{from_email}"; + $allowedFromEmailWhiteList = array(); + $fromEmailNotification = $this->getFromEmail(); $partner = PartnerPeer::retrieveByPK($this->getPartnerId()); - $allowedFromEmailWhiteList = $partner->getAllowedFromEmailWhiteList(); - if ($email != $partnerNotificationEmail && in_array($email, array_map('trim',explode(',',$allowedFromEmailWhiteList)))) + if ($partner) { - KalturaLog::info("from_email requested: $email is allowed in the partner from_email whitelist: ".$allowedFromEmailWhiteList ); - return $email; - } - else - { - KalturaLog::info("from_email requested is: $email , partner from_email whitelist: ".$allowedFromEmailWhiteList ); - return $partnerNotificationEmail; + $allowedFromEmailWhiteList = $partner->getAllowedFromEmailWhiteList(); + if ($fromEmailNotification !== self::FROM_EMAIL && in_array($fromEmailNotification, explode(',',$allowedFromEmailWhiteList))) + { + KalturaLog::info("from_email requested: $fromEmailNotification is allowed on the partner from_email whitelist: ".$allowedFromEmailWhiteList ); + return $fromEmailNotification; + } } + KalturaLog::info("from_email requested: $fromEmailNotification . partner from_email whitelist: ".$allowedFromEmailWhiteList ); + return self::FROM_EMAIL; } /* (non-PHPdoc) @@ -63,7 +64,7 @@ protected function getJobData(kScope $scope = null) { $jobData = new kEmailNotificationDispatchJobData(); $jobData->setTemplateId($this->getId()); - $jobData->setFromEmail($this->getEmail()); + $jobData->setFromEmail($this->getAllowedEmail()); $jobData->setFromName($this->getFromName()); $jobData->setPriority($this->getPriority()); $jobData->setConfirmReadingTo($this->getConfirmReadingTo()); From aa78f159c6546bbb969c15c568fdd61bcd52e931 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Sun, 7 Jul 2019 16:17:37 +0300 Subject: [PATCH 053/103] PLAT-9784:adding sender to the header of the mail --- alpha/lib/model/Partner.php | 4 ++-- .../providers/email/lib/model/EmailNotificationTemplate.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/alpha/lib/model/Partner.php b/alpha/lib/model/Partner.php index 698b8cab123..2c1bf682ffe 100644 --- a/alpha/lib/model/Partner.php +++ b/alpha/lib/model/Partner.php @@ -239,7 +239,7 @@ public function setReturnDuplicateKshow( $v ) public function getAllowQuickEdit() { - return $this->getFromCustomData( "allowQuickEdit" , null , true ); + return (int)$this->getFromCustomData( "allowQuickEdit" , null , true ); } public function setAllowQuickEdit( $v ) @@ -423,7 +423,7 @@ public function setNotificationsConfig( $v ) public function getAllowMultiNotification() { - return $this->getFromCustomData( "allowMultiNotification" , null ); + return (int)$this->getFromCustomData( "allowMultiNotification" , null ); } public function setAllowMultiNotification( $v ) diff --git a/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php b/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php index 202e8450259..aba3b0ff9ef 100644 --- a/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php +++ b/plugins/event_notification/providers/email/lib/model/EmailNotificationTemplate.php @@ -47,9 +47,9 @@ protected function getAllowedEmail() if ($partner) { $allowedFromEmailWhiteList = $partner->getAllowedFromEmailWhiteList(); - if ($fromEmailNotification !== self::FROM_EMAIL && in_array($fromEmailNotification, explode(',',$allowedFromEmailWhiteList))) + if ($fromEmailNotification !== self::FROM_EMAIL + && in_array($fromEmailNotification, explode(',',$allowedFromEmailWhiteList))) { - KalturaLog::info("from_email requested: $fromEmailNotification is allowed on the partner from_email whitelist: ".$allowedFromEmailWhiteList ); return $fromEmailNotification; } } From 499700dacc6ff3aebb4abf6e682eddcff5dbe839 Mon Sep 17 00:00:00 2001 From: Jess Portnoy Date: Sun, 7 Jul 2019 15:22:15 +0100 Subject: [PATCH 054/103] Place both alters in one SQL file as they're related. --- .../sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql | 1 + deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql diff --git a/deployment/updates/sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql b/deployment/updates/sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql index 85ad506328f..9db6f19b0ae 100644 --- a/deployment/updates/sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql +++ b/deployment/updates/sql/2019_07_05_add_sphinx_index_column_sphinx_log.sql @@ -1 +1,2 @@ +ALTER TABLE kaltura_sphinx_log.sphinx_log ADD type INT AFTER created_at; ALTER TABLE kaltura_sphinx_log.sphinx_log ADD index_name VARCHAR(128) AFTER type; diff --git a/deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql b/deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql deleted file mode 100644 index 6ae812509d3..00000000000 --- a/deployment/updates/sql/2019_07_05_add_type_column_sphinx_log.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE kaltura_sphinx_log.sphinx_log ADD type INT AFTER created_at; From 9c5f26a4524acebbb3508b074a26da55a9794342 Mon Sep 17 00:00:00 2001 From: MosheMaorkaltura Date: Mon, 8 Jul 2019 07:34:27 +0300 Subject: [PATCH 055/103] Updated version --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 2e0c689d683..55cd493cc42 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -Orion-15.2.0 +Orion-15.3.0 From b5f483fee27cf4e72e96145e3c97203bd1994bb3 Mon Sep 17 00:00:00 2001 From: Jess Portnoy Date: Mon, 8 Jul 2019 14:03:17 +0100 Subject: [PATCH 056/103] partner->getSecrets() should accept the option $otp and pass it along to userLoginByEmail() At the moment, attempting to call this function when OTP was enabled on the partner will fail. --- api_v3/services/PartnerService.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api_v3/services/PartnerService.php b/api_v3/services/PartnerService.php index 621b7e5ac2e..9b44378b343 100644 --- a/api_v3/services/PartnerService.php +++ b/api_v3/services/PartnerService.php @@ -210,18 +210,19 @@ public function getAction ($id = null) * @param int $partnerId * @param string $adminEmail * @param string $cmsPassword + * @param string $otp * @return KalturaPartner * @ksIgnored * * @throws APIErrors::ADMIN_KUSER_NOT_FOUND */ - public function getSecretsAction( $partnerId , $adminEmail , $cmsPassword ) + public function getSecretsAction( $partnerId , $adminEmail , $cmsPassword, $otp = null ) { KalturaResponseCacher::disableCache(); $adminKuser = null; try { - $adminKuser = UserLoginDataPeer::userLoginByEmail($adminEmail, $cmsPassword, $partnerId); + $adminKuser = UserLoginDataPeer::userLoginByEmail($adminEmail, $cmsPassword, $partnerId, $otp); } catch (kUserException $e) { throw new KalturaAPIException ( APIErrors::ADMIN_KUSER_NOT_FOUND, "The data you entered is invalid" ); From 4e6d6e3c6b42101a9029707640eab9d8ec2c34ca Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Mon, 8 Jul 2019 16:23:11 +0300 Subject: [PATCH 057/103] SUP-16933:code review remarks --- alpha/apps/kaltura/lib/myPlaylistUtils.class.php | 2 +- api_v3/services/PlaylistService.php | 4 ++-- .../model/filters/ESearchEntryQueryFromFilter.php | 15 ++++++++------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index eae9e711314..c17d801c70d 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -581,7 +581,7 @@ protected static function buildSorter($objectsOrder) { return function ($a, $b) use ($objectsOrder) { - return ($objectsOrder[$a->getId()] > $objectsOrder[$b->getId()]) ? 1 : -1; + return ($objectsOrder[$a->getId()] - $objectsOrder[$b->getId()]); }; } diff --git a/api_v3/services/PlaylistService.php b/api_v3/services/PlaylistService.php index f2027bdb36a..7825dc60f87 100644 --- a/api_v3/services/PlaylistService.php +++ b/api_v3/services/PlaylistService.php @@ -409,9 +409,9 @@ protected static function getEntryListUnique($entryListMerged) $entryList = array(); foreach ($entryListMerged as $currentEntry) { - if (!in_array($currentEntry, $entryList)) + if (!array_key_exists ($currentEntry->getId(), $entryList)) { - $entryList[] = $currentEntry; + $entryList[$currentEntry->getId()] = $currentEntry; } } return $entryList; diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php index 4a3b9b27b8e..c77c6a17ff6 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php @@ -179,11 +179,12 @@ public function createElasticQueryFromFilter(baseObjectFilter $filter) $kEsearchOrderBy = $this->getKESearchOrderBy($fieldValue); continue; } - list($operator, $fieldName, $fieldValue, $shouldContinue) = $this->splitIntoParameters($filter, $field, $fieldValue); - if ($shouldContinue) + $fieldParts = $this->splitIntoParameters($filter, $field, $fieldValue); + if (!$fieldParts) { continue; } + list($operator, $fieldName, $fieldValue) = $fieldParts; $this->addingFieldPartIntoQuery($operator, $fieldName, $fieldValue); } return $this->createFinalOperator($kEsearchOrderBy); @@ -215,15 +216,15 @@ protected function splitIntoParameters($filter, $field, $fieldValue ) list($operator, $fieldName) = self::handlingFreeTextField($field, $operator, $fieldName); if(!in_array($fieldName, static::getSupportedFields()) || is_null($fieldValue) || $fieldValue === '') { - return array(null , null, null, true); + return null; } if ($fieldName === ESearchEntryFilterFields::STATUS && ($operator === baseObjectFilter::EQ ||$operator === baseObjectFilter::IN )) { self::$validStatuses = explode(',',$fieldValue); - return array(null , null, null, true); + return null; } - $fieldValue = self::changeFieldValueByPuserIdAndDuration($fieldName, $filter, $fieldValue); - return array($operator, $fieldName, $fieldValue, false); + $fieldValue = self::translateFieldValue($fieldName, $filter, $fieldValue); + return array($operator, $fieldName, $fieldValue); } protected static function handlingFreeTextField($field, $operator, $fieldName) @@ -236,7 +237,7 @@ protected static function handlingFreeTextField($field, $operator, $fieldName) return array($operator, $fieldName); } - protected static function changeFieldValueByPuserIdAndDuration($fieldName, $filter, $fieldValue) + protected static function translateFieldValue($fieldName, $filter, $fieldValue) { if(in_array($fieldName, self::$puserFields)) { From 499a6f789b6d26a944e86accfdcdc9230de15781 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Mon, 8 Jul 2019 16:47:17 +0300 Subject: [PATCH 058/103] SUP-16933:code review remarks --- api_v3/services/PlaylistService.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/api_v3/services/PlaylistService.php b/api_v3/services/PlaylistService.php index 7825dc60f87..7f04c25f565 100644 --- a/api_v3/services/PlaylistService.php +++ b/api_v3/services/PlaylistService.php @@ -409,10 +409,7 @@ protected static function getEntryListUnique($entryListMerged) $entryList = array(); foreach ($entryListMerged as $currentEntry) { - if (!array_key_exists ($currentEntry->getId(), $entryList)) - { - $entryList[$currentEntry->getId()] = $currentEntry; - } + $entryList[$currentEntry->getId()] = $currentEntry; } return $entryList; } From 0855449a6bb73338844f8014c2d698b6dfaac94a Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Mon, 8 Jul 2019 16:50:30 +0300 Subject: [PATCH 059/103] Revert "SUP-16933:use eSearch instead of sphinx for dynamic playlist execute from filter/content actions" --- .../kaltura/lib/myPlaylistUtils.class.php | 153 +--------- api_v3/lib/KalturaErrors.php | 2 - api_v3/services/PlaylistService.php | 79 +---- .../elastic/mapping/entry_mapping.json | 3 +- .../api/enum/KalturaESearchEntryFieldName.php | 1 - .../items/entry/KalturaESearchEntryItem.php | 1 - .../lib/model/enum/ESearchEntryFieldName.php | 1 - .../model/enum/ESearchEntryFilterFields.php | 55 ---- .../lib/model/enum/ESearchFilterItemType.php | 9 - .../filters/ESearchCaptionQueryFromFilter.php | 2 +- .../filters/ESearchEntryQueryFromFilter.php | 287 ------------------ .../model/filters/ESearchQueryFromFilter.php | 186 ++---------- .../lib/model/items/ESearchEntryItem.php | 1 - 13 files changed, 46 insertions(+), 734 deletions(-) delete mode 100644 plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php delete mode 100644 plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index c17d801c70d..5851119524a 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -14,30 +14,13 @@ class myPlaylistUtils const CAPTION_FILES_LABEL = "label"; const CAPTION_FILES_PATH = "path"; const CAPTION_FILES_ID = "captionId"; - const KALTURA_CLASS = 'kalturaClass'; private static $user_cache = null; private static $isAdminKs = false; private static $playlistContext; - - private static $moderationStatusesNotIn = array( - entry::ENTRY_MODERATION_STATUS_PENDING_MODERATION, - entry::ENTRY_MODERATION_STATUS_REJECTED - ); - - private static $dates = array( - '_lteornull_start_date', - '_gteornull_start_date', - '_lte_start_date', - '_gte_start_date', - '_lteornull_end_date', - '_gteornull_end_date', - '_lte_end_date', - '_gte_end_date' - ); - + public static function setIsAdminKs($v) { self::$isAdminKs = $v; @@ -500,139 +483,11 @@ public static function getDynamicPlaylistFilters($xml) } return $entry_filters; } - - public static function getEntryFiltersFromXml($listOfFilters, $partnerId) - { - if (!$listOfFilters) - { - return null; - } - return self::fillEntryFilterFromXml($listOfFilters, $partnerId); - } - - //splitting entry filters into filters that will run in Esearch and in Sphinx: - // Elastic - advancesSearch is not set OR advancesSearch is set and includes KalturaMetadataSearchItem - // Sphinx - advancesSearch is set and NOT including KalturaMetadataSearchItem - public static function splitEntryFilters($xml) - { - $entryFiltersViaEsearch = array(); - $entryFiltersViaSphinx = array(); - list ($totalResults, $entryFilters) = self::getPlaylistFilterListStruct($xml); - foreach ($entryFilters as $entryFilter) - { - if (!isset($entryFilter->advancedSearch) || - ( isset($entryFilter->advancedSearch) && (string)$entryFilter->advancedSearch[self::KALTURA_CLASS] === PlaylistService::KALTURA_METADATA_SEARCH_ITEM)) - { - $entryFiltersViaEsearch[] = $entryFilter; - } - else - { - $entryFiltersViaSphinx[] = $entryFilter; - } - } - return array($entryFiltersViaEsearch, $entryFiltersViaSphinx, $totalResults); - } - - public static function executeDynamicPlaylistViaEsearch ($entryFilters ,$totalResults, $pager = null) - { - $entryKPager = new kPager(); - if ($pager) - { - $pager->toObject($entryKPager); - } - $entryQueryToFilterESearch = new ESearchEntryQueryFromFilter(); - $entryIds= array(); - foreach ($entryFilters as $entryFilter) - { - list ($currEntryIds, $count) = $entryQueryToFilterESearch->retrieveElasticQueryEntryIds($entryFilter, $entryKPager); - $entryIds = self::mergeEntriesByLimit($entryIds, $currEntryIds, $entryFilter->getLimit(),$totalResults); - $totalResults = max (0, $totalResults - count($entryIds)); - if ( $totalResults === 0 ) - { - break; - } - } - return array(self::getEntriesSorted($entryIds), $totalResults); - } - - protected static function getEntriesSorted($entryIds) - { - $entryIds = array_unique($entryIds); - $entryIdsOrder = array_flip($entryIds); - $entries = entryPeer::retrieveByPKs($entryIds); - usort($entries, self::buildSorter($entryIdsOrder)); - return $entries; - } - - protected static function mergeEntriesByLimit($entryIds, $currEntryIds, $limit, $totalResults) - { - if ($totalResults < $limit) - { - $limit = $totalResults; - } - if (count($currEntryIds) > $limit) - { - $currEntryIds = array_slice($currEntryIds,0, $limit); - } - return array_merge ($entryIds, $currEntryIds); - } - - protected static function buildSorter($objectsOrder) - { - return function ($a, $b) use ($objectsOrder) - { - return ($objectsOrder[$a->getId()] - $objectsOrder[$b->getId()]); - }; - } - - protected static function fillEntryFilterFromXml($listOfFilters, $partnerId) - { - $entryFilters = array(); - foreach ($listOfFilters as $filter) - { - self::replaceContextTokens($filter); - $entryFilter = new entryFilter(); - $entryFilter->fillObjectFromXml($filter, '_'); - self::updateEntryFilterFields($entryFilter, $partnerId); - $entryFilters[] = $entryFilter; - } - return $entryFilters; - } - - protected static function updateEntryFilterFields($entryFilter, $partnerId) - { - self::updateEntryFilter($entryFilter, $partnerId); - $typeArray = array (entryType::MEDIA_CLIP, entryType::MIX, entryType::LIVE_STREAM ); - $typeArray = array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::MEDIA_CLIP)); - $typeArray = array_unique(array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::LIVE_STREAM))); - $entryFilter->setTypeIn($typeArray); - $entryFilter->setStatusEquel(entryStatus::READY); - $entryFilter->setModerationStatusNotIn(self::$moderationStatusesNotIn); - if (self::$isAdminKs) - { - self::unsetDates($entryFilter); - } - } - - protected static function unsetDates($filter) - { - foreach (self::$dates as $date) - { - if ($filter->is_set($date)) - { - $filter->unsetByName($date); - } - } - } - - public static function executeDynamicPlaylist ( $partner_id, $xml, $filter = null, $detailed = true, $pager = null ) + + public static function executeDynamicPlaylist ( $partner_id , $xml , $filter = null ,$detailed = true, $pager = null ) { list ( $total_results , $list_of_filters ) = self::getPlaylistFilterListStruct ( $xml ); - return self::executeDynamicPlaylistFromFilters($total_results, $list_of_filters, $partner_id, $filter , $detailed, $pager); - } - - public static function executeDynamicPlaylistFromFilters($total_results, $list_of_filters, $partner_id, $filter = null, $detailed = true, $pager = null) - { + $entry_filters = array(); if ( ! $list_of_filters ) return null; diff --git a/api_v3/lib/KalturaErrors.php b/api_v3/lib/KalturaErrors.php index e71b3ff0502..03edd0cc55c 100644 --- a/api_v3/lib/KalturaErrors.php +++ b/api_v3/lib/KalturaErrors.php @@ -753,7 +753,5 @@ class KalturaErrors extends APIErrors const MAP_ALREADY_EXIST = "MAP_ALREADY_EXIST;NAME,HOST;Map already exist for this map name {@NAME@} and host {@HOST@}"; const MAP_CANNOT_BE_CREATED_ON_FILE_SYSTEM = "MAP_CANNOT_BE_CREATED_ON_FILE_SYSTEM;;Map cannnot be created on file system"; const HOST_NAME_CONTAINS_ASTRIX = "HOST_NAME_CONTAINS_ASTRIX;HOST_NAME;Host name contains *, use # instead {@HOST_NAME@}"; - - const SEARCH_ITEM_TYPE_NOT_FOUND = 'SEARCH_ITEM_TYPE_NOT_FOUND;SEARCH_ITEM_TYPE,ELASTIC_FIELD_NAME; Search item type [@SEARCH_ITEM_TYPE@] not found for field: [@ELASTIC_FIELD_NAME@]'; } diff --git a/api_v3/services/PlaylistService.php b/api_v3/services/PlaylistService.php index 7f04c25f565..b15269e5da1 100644 --- a/api_v3/services/PlaylistService.php +++ b/api_v3/services/PlaylistService.php @@ -10,9 +10,6 @@ */ class PlaylistService extends KalturaEntryService { - const ADVANCED_SEARCH = 'advancedSearch'; - const KALTURA_METADATA_SEARCH_ITEM = 'KalturaMetadataSearchItem'; - /* (non-PHPdoc) * @see KalturaBaseService::globalPartnerAllowed() */ @@ -348,81 +345,19 @@ function executeAction( $id , $detailed = false, KalturaContext $playlistContext function executeFromContentAction($playlistType, $playlistContent, $detailed = false, $pager = null) { myDbHelper::$use_alternative_con = myDbHelper::DB_HELPER_CONN_PROPEL3; + if ($this->getKs() && is_object($this->getKs()) && $this->getKs()->isAdmin()) - { myPlaylistUtils::setIsAdminKs(true); - } - list($entryFiltersViaEsearch, $entryFiltersViaSphinx, $totalResults) = myPlaylistUtils::splitEntryFilters($playlistContent); - $pagerSeparateQueries = self::decideWhereHandlingPager($pager,$entryFiltersViaEsearch, $entryFiltersViaSphinx); - $entryList = self::handlePlaylistByType($playlistType, $entryFiltersViaEsearch, $entryFiltersViaSphinx, $this->getPartnerId(), $pagerSeparateQueries, $pager, $totalResults, $playlistContent); - myEntryUtils::updatePuserIdsForEntries($entryList); - KalturaLog::debug("entry ids count: " . count($entryList)); - return KalturaBaseEntryArray::fromDbArray($entryList, $this->getResponseProfile()); - } - protected static function handlePlaylistByType($playlistType, $entryFiltersViaEsearch, $entryFiltersViaSphinx, $partnerId, $pagerSeperateQueries, $pager, $totalResults, $playlistContent) - { + $entryList = array(); if ($playlistType == KalturaPlaylistType::DYNAMIC) - { - $entryList = self::handlingDynamicPlaylist($entryFiltersViaEsearch,$entryFiltersViaSphinx, $partnerId, $pagerSeperateQueries, $pager, $totalResults); - } + $entryList = myPlaylistUtils::executeDynamicPlaylist($this->getPartnerId(), $playlistContent, null, true, $pager); else if ($playlistType == KalturaPlaylistType::STATIC_LIST) - { $entryList = myPlaylistUtils::executeStaticPlaylistFromEntryIdsString($playlistContent, null, true, $pager); - } - return $entryList; - } - - protected static function handlingDynamicPlaylist($entryFiltersViaEsearch, $entryFiltersViaSphinx, $partnerId, $pagerSeparateQueries, $pager, $totalResults) - { - $entryListEsearch = array(); - $entryListSphinx = array(); - if ($entryFiltersViaEsearch) - { - $entryFiltersViaEsearch = myPlaylistUtils::getEntryFiltersFromXml($entryFiltersViaEsearch, $partnerId); - list($entryListEsearch, $totalResults) = myPlaylistUtils::executeDynamicPlaylistViaEsearch($entryFiltersViaEsearch, $totalResults, $pagerSeparateQueries); - } - if ($entryFiltersViaSphinx) - { - $entryListSphinx = myPlaylistUtils::executeDynamicPlaylistFromFilters($totalResults, $entryFiltersViaSphinx, $partnerId, null, true, $pagerSeparateQueries); - } - $entryListMerged = array_merge($entryListEsearch,$entryListSphinx); - $entryListUnique = self::getEntryListUnique($entryListMerged); - $entryList = self::getEntryListByPager($entryListUnique, $pager, $pagerSeparateQueries); - return $entryList; - } - - //When queries are going to run both in Esearch and Sphinx, pager handling will be done after merging the results -> ($pagerSeperateQueries = null) - //In case only one of them will run (for example only in sphinx), pager handling will be done only there (in this example: in sphinx query) -> ($pagerSeperateQueries = $pager) - protected static function decideWhereHandlingPager($pager,$entryFiltersViaEsearch, $entryFiltersViaSphinx) - { - $pagerSeparateQueries = $pager; - if ($entryFiltersViaEsearch && $entryFiltersViaSphinx) - { - $pagerSeparateQueries = null; - } - return $pagerSeparateQueries; - } - - protected static function getEntryListUnique($entryListMerged) - { - $entryList = array(); - foreach ($entryListMerged as $currentEntry) - { - $entryList[$currentEntry->getId()] = $currentEntry; - } - return $entryList; - } - - protected static function getEntryListByPager($entryList, $pager, $pagerSeparateQueries) - { - if ( $pager && is_null($pagerSeparateQueries)) - { - $startOffset = $pager->calcOffset(); - $pageSize = $pager->calcPageSize(); - $entryList = array_slice($entryList, $startOffset, $pageSize); - } - return $entryList; + + myEntryUtils::updatePuserIdsForEntries($entryList); + + return KalturaBaseEntryArray::fromDbArray($entryList, $this->getResponseProfile()); } /** diff --git a/configurations/elastic/mapping/entry_mapping.json b/configurations/elastic/mapping/entry_mapping.json index d1bf9a8deb9..88a3ce8ce25 100644 --- a/configurations/elastic/mapping/entry_mapping.json +++ b/configurations/elastic/mapping/entry_mapping.json @@ -626,7 +626,8 @@ "normalizer" : "kaltura_keyword_normalizer" }, "partner_sort_value": { - "type" : "integer" + "type" : "keyword", + "normalizer" : "kaltura_keyword_normalizer" }, "redirect_entry_id": { "type" : "keyword", diff --git a/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php b/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php index be898c4a6a9..3c590e32625 100644 --- a/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php +++ b/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php @@ -39,5 +39,4 @@ class KalturaESearchEntryFieldName extends KalturaStringEnum const IS_LIVE = 'is_live'; const USER_NAMES = "user_names"; const ROOT_ID = 'root_id'; - const PARTNER_SORT_VALUE = 'partner_sort_value'; } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php b/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php index 6c27a232b31..89124cfc9bc 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php @@ -58,7 +58,6 @@ class KalturaESearchEntryItem extends KalturaESearchAbstractEntryItem KalturaESearchEntryFieldName::IS_LIVE => ESearchEntryFieldName::IS_LIVE, KalturaESearchEntryFieldName::USER_NAMES => ESearchEntryFieldName::USER_NAMES, KalturaESearchEntryFieldName::ROOT_ID => ESearchEntryFieldName::ROOT_ID, - KalturaESearchEntryFieldName::PARTNER_SORT_VALUE => ESearchEntryFieldName::PARTNER_SORT_VALUE ); protected function getMapBetweenObjects() diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php index 4378a5f1fe0..de84c28d317 100644 --- a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php @@ -41,5 +41,4 @@ interface ESearchEntryFieldName extends BaseEnum const DISPLAY_IN_SEARCH = 'display_in_search'; const ROOT_ID = 'root_id'; const PRIVACY_BY_CONTEXTS = 'privacy_by_contexts'; - const PARTNER_SORT_VALUE = 'partner_sort_value'; } diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php deleted file mode 100644 index 223856a3022..00000000000 --- a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php +++ /dev/null @@ -1,55 +0,0 @@ -updateEntryPager($entryPager, $filterOnEntryIds); - list($query, $kEsearchOrderBy ) = $this->createElasticQueryFromFilter($filter); + $query = $this->createElasticQueryFromFilter($filter); $elasticResults = $entrySearch->doSearch($query, $entryPager, self::$validStatuses); diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php deleted file mode 100644 index c77c6a17ff6..00000000000 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php +++ /dev/null @@ -1,287 +0,0 @@ - ESearchEntryFieldName::ID, - ESearchEntryFilterFields::USER_ID => ESearchEntryFieldName::USER_ID, - ESearchEntryFilterFields::GROUP_ID => ESearchEntryFieldName::USER_ID, - ESearchEntryFilterFields::CREATOR_ID => ESearchEntryFieldName::CREATOR_ID, - ESearchEntryFilterFields::NAME => ESearchEntryFieldName::NAME, - ESearchEntryFilterFields::TAGS => ESearchEntryFieldName::TAGS, - ESearchEntryFilterFields::TAGS_NAME => ESearchEntryFieldName::TAGS, - ESearchEntryFilterFields::ADMIN_TAGS => ESearchEntryFieldName::ADMIN_TAGS, - ESearchEntryFilterFields::TAGS_ADMIN_TAGS => ESearchEntryFieldName::ADMIN_TAGS, - ESearchEntryFilterFields::TAGS_ADMIN_TAGS_NAME => ESearchEntryFieldName::ADMIN_TAGS, - ESearchEntryFilterFields::CONVERSION_PROFILE_ID => ESearchEntryFieldName::CONVERSION_PROFILE_ID, - ESearchEntryFilterFields::REDIRECT_ENTRY_ID => ESearchEntryFieldName::REDIRECT_ENTRY_ID, - ESearchEntryFilterFields::ENTITLED_USER_EDIT => ESearchEntryFieldName::ENTITLED_USER_EDIT, - ESearchEntryFilterFields::ENTITLED_USER_PUBLISH => ESearchEntryFieldName::ENTITLED_USER_PUBLISH, - ESearchEntryFilterFields::ENTITLED_USER_VIEW => ESearchEntryFieldName::ENTITLED_USER_VIEW, - ESearchEntryFilterFields::DISPLAY_IN_SEARCH => ESearchEntryFieldName::DISPLAY_IN_SEARCH, - ESearchEntryFilterFields::PARENT_ENTRY_ID => ESearchEntryFieldName::PARENT_ENTRY_ID, - ESearchEntryFilterFields::MEDIA_TYPE => ESearchEntryFieldName::MEDIA_TYPE, - ESearchEntryFilterFields::SOURCE_TYPE => ESearchEntryFieldName::SOURCE_TYPE, - ESearchEntryFilterFields::LENGTH_IN_MSECS => ESearchEntryFieldName::LENGTH_IN_MSECS, - ESearchEntryFilterFields::TYPE => ESearchEntryFieldName::ENTRY_TYPE, - ESearchEntryFilterFields::MODERATION_STATUS => ESearchEntryFieldName::MODERATION_STATUS, - ESearchEntryFilterFields::CREATED_AT => ESearchEntryFieldName::CREATED_AT, - ESearchEntryFilterFields::UPDATED_AT => ESearchEntryFieldName::UPDATED_AT, - ESearchEntryFilterFields::ACCESS_CONTROL_ID => ESearchEntryFieldName::ACCESS_CONTROL_ID, - ESearchEntryFilterFields::USER_NAMES => ESearchEntryFieldName::USER_NAMES, - ESearchEntryFilterFields::START_DATE => ESearchEntryFieldName::START_DATE, - ESearchEntryFilterFields::END_DATE => ESearchEntryFieldName::END_DATE, - ESearchEntryFilterFields::REFERENCE_ID => ESearchEntryFieldName::REFERENCE_ID, - ESearchEntryFilterFields::ROOT_ENTRY_ID => ESearchEntryFieldName::ROOT_ID, - ESearchEntryFilterFields::DURATION => ESearchEntryFieldName::LENGTH_IN_MSECS, - ESearchEntryFilterFields::CATEGORIES => ESearchCategoryEntryNameItem::CATEGORY_NAMES_MAPPING_FIELD, - ESearchEntryFilterFields::CATEGORIES_IDS => ESearchCategoryEntryIdItem::CATEGORY_IDS_MAPPING_FIELD, - ESearchEntryFilterFields::CATEGORIES_ANCESTOR_ID => ESearchCategoryEntryFieldName::ANCESTOR_ID, - ESearchEntryFilterFields::CATEGORIES_FULL_NAME => ESearchCategoryEntryFieldName::FULL_IDS, - ESearchEntryFilterFields::PARTNER_SORT_VALUE => ESearchEntryFieldName::PARTNER_SORT_VALUE, - ESearchEntryFilterFields::SEARCH_TEXT => ESearchUnifiedItem::UNIFIED, - ESearchEntryFilterFields::FREE_TEXT => ESearchUnifiedItem::UNIFIED, - ESearchEntryFilterFields::TOTAL_RANK => ESearchEntryOrderByFieldName::VOTES - ); - - if(array_key_exists($field, $fieldsMap)) - { - return $fieldsMap[$field]; - } - else - { - return null; - } - } - - protected function getSphinxToElasticOrderBy($field) - { - $fieldsMap = array( - self::ASC.ESearchEntryFilterFields::MEDIA_TYPE, - self::DESC.ESearchEntryFilterFields::MEDIA_TYPE, - KalturaMediaEntryOrderBy::PLAYS_ASC => self::ASC.ESearchEntryOrderByFieldName::PLAYS, - KalturaMediaEntryOrderBy::PLAYS_DESC => self::DESC.ESearchEntryOrderByFieldName::PLAYS, - KalturaMediaEntryOrderBy::VIEWS_ASC => self::ASC.ESearchEntryOrderByFieldName::VIEWS, - KalturaMediaEntryOrderBy::VIEWS_DESC => self::DESC.ESearchEntryOrderByFieldName::VIEWS, - KalturaMediaEntryOrderBy::DURATION_ASC => self::ASC.ESearchEntryFieldName::LENGTH_IN_MSECS, - KalturaMediaEntryOrderBy::DURATION_DESC => self::DESC.ESearchEntryFieldName::LENGTH_IN_MSECS, - KalturaMediaEntryOrderBy::NAME_ASC => self::ASC. ESearchEntryOrderByFieldName::NAME, - KalturaMediaEntryOrderBy::NAME_DESC => self::DESC. ESearchEntryOrderByFieldName::NAME, - KalturaMediaEntryOrderBy::CREATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, - KalturaMediaEntryOrderBy::CREATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT, - KalturaMediaEntryOrderBy::UPDATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::UPDATED_AT, - KalturaMediaEntryOrderBy::UPDATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::UPDATED_AT, - KalturaMediaEntryOrderBy::RANK_ASC => self::ASC.ESearchEntryOrderByFieldName::VOTES, - KalturaMediaEntryOrderBy::RANK_DESC => self::DESC.ESearchEntryOrderByFieldName::VOTES, - self::ASC.ESearchEntryFilterFields::TOTAL_RANK => self::ASC.ESearchEntryOrderByFieldName::VOTES, - self::DESC.ESearchEntryFilterFields::TOTAL_RANK => self::DESC.ESearchEntryOrderByFieldName::VOTES, - KalturaMediaEntryOrderBy::START_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::START_DATE, - KalturaMediaEntryOrderBy::START_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::START_DATE, - KalturaMediaEntryOrderBy::END_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::END_DATE, - KalturaMediaEntryOrderBy::END_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::END_DATE, - KalturaMediaEntryOrderBy::RECENT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, - KalturaMediaEntryOrderBy::RECENT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT - ); - - if(in_array($field, $fieldsMap)) - { - return $field; - } - else if (array_key_exists($field, $fieldsMap)) - { - return $fieldsMap[$field]; - } - else - { - return null; - } - } - - public function createElasticQueryFromFilter(baseObjectFilter $filter) - { - $this->init(); - $kEsearchOrderBy = null; - foreach($filter->fields as $field => $fieldValue) - { - if ($field === entryFilter::ORDER && !is_null($fieldValue) && $fieldValue!= '') - { - $kEsearchOrderBy = $this->getKESearchOrderBy($fieldValue); - continue; - } - $fieldParts = $this->splitIntoParameters($filter, $field, $fieldValue); - if (!$fieldParts) - { - continue; - } - list($operator, $fieldName, $fieldValue) = $fieldParts; - $this->addingFieldPartIntoQuery($operator, $fieldName, $fieldValue); - } - return $this->createFinalOperator($kEsearchOrderBy); - } - - protected function createFinalOperator($kEsearchOrderBy) - { - $this->addNestedQueryPart(); - $operator = new ESearchOperator(); - $operator->setOperator(ESearchOperatorType::AND_OP); - $operator->setSearchItems($this->searchItems); - return array($operator, $kEsearchOrderBy); - } - - protected function addingFieldPartIntoQuery($operator, $fieldName, $fieldValue) - { - $searchItemType = $this->getSphinxToElasticSearchItemType($operator); - $elasticFieldName = $this->getSphinxToElasticFieldName($fieldName); - if($elasticFieldName && $searchItemType) - { - $this->AddFieldPartToQuery($searchItemType, $elasticFieldName, $fieldValue); - } - } - - protected function splitIntoParameters($filter, $field, $fieldValue ) - { - $fieldParts = explode(entryFilter::FILTER_PREFIX, $field, 3); - list( , $operator, $fieldName) = $fieldParts; - list($operator, $fieldName) = self::handlingFreeTextField($field, $operator, $fieldName); - if(!in_array($fieldName, static::getSupportedFields()) || is_null($fieldValue) || $fieldValue === '') - { - return null; - } - if ($fieldName === ESearchEntryFilterFields::STATUS && ($operator === baseObjectFilter::EQ ||$operator === baseObjectFilter::IN )) - { - self::$validStatuses = explode(',',$fieldValue); - return null; - } - $fieldValue = self::translateFieldValue($fieldName, $filter, $fieldValue); - return array($operator, $fieldName, $fieldValue); - } - - protected static function handlingFreeTextField($field, $operator, $fieldName) - { - if ($field === ESearchEntryFilterFields::FREE_TEXT) - { - $operator = baseObjectFilter::IN; - $fieldName = $field; - } - return array($operator, $fieldName); - } - - protected static function translateFieldValue($fieldName, $filter, $fieldValue) - { - if(in_array($fieldName, self::$puserFields)) - { - $kuser = kuserPeer::getKuserByPartnerAndUid($filter->getPartnerSearchScope(), $fieldValue); - if (!$kuser) - { - throw new KalturaAPIException(KalturaErrors::INVALID_USER_ID, $fieldValue); - } - $fieldValue = $kuser->getId(); - } - else if ($fieldName === ESearchEntryFilterFields::DURATION) - { - $fieldValue = $fieldValue * 1000; - } - return $fieldValue; - } - - protected function getKESearchOrderBy($fieldValue) - { - $fieldValue = $this->getSphinxToElasticOrderBy($fieldValue); - if (!$fieldValue) - { - return null; - } - $eSearchOrderByItem = self::getESearchOrderByItem($fieldValue); - $orderItems = array($eSearchOrderByItem); - $kEsearchOrderBy = new ESearchOrderBy(); - $kEsearchOrderBy->setOrderItems($orderItems); - return $kEsearchOrderBy; - } - - protected static function getESearchOrderByItem($fieldValue) - { - $eSearchOrderByItem = new ESearchEntryOrderByItem(); - $eSearchOrderByItem->setSortField(substr($fieldValue,1)); - $fieldNameSortOrder = $fieldValue[0]; - if ($fieldNameSortOrder === self::DESC) - { - $eSearchOrderByItem->setSortOrder(ESearchSortOrder::ORDER_BY_DESC); - } - else if ($fieldNameSortOrder === self::ASC) - { - $eSearchOrderByItem->setSortOrder(ESearchSortOrder::ORDER_BY_ASC); - } - return $eSearchOrderByItem; - } -} diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php index 3b451d96f48..e859d8afc6e 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php @@ -5,16 +5,6 @@ class ESearchQueryFromFilter protected $searchItems; protected $nestedSearchItem; protected static $validStatuses = array(entryStatus::READY,entryStatus::NO_CONTENT); - protected static $categoryFilterFields = array( - ESearchBaseCategoryEntryItem::CATEGORY_NAMES_MAPPING_FIELD => array(ESearchCategoryEntryFieldName::NAME, 'ESearchCategoryEntryNameItem'), - ESearchBaseCategoryEntryItem::CATEGORY_IDS_MAPPING_FIELD => array(ESearchCategoryEntryFieldName::ID, 'ESearchCategoryEntryIdItem'), - ESearchCategoryEntryFieldName::ANCESTOR_ID => array(ESearchCategoryEntryFieldName::ANCESTOR_ID, 'ESearchCategoryEntryAncestorIdItem'), - ESearchCategoryEntryFieldName::ANCESTOR_NAME => array(ESearchCategoryEntryFieldName::ANCESTOR_NAME, 'ESearchCategoryEntryAncestorNameItem') - ); - - const FIELD_NAME_LOCATION = 0; - const FIELD_CLASS_LOCATION = 1; - const FIELD_NAME = 'fieldName'; public function __construct() { @@ -56,16 +46,16 @@ public function createElasticQueryFromFilter(baseObjectFilter $filter) $operator = new ESearchOperator(); $operator->setOperator(ESearchOperatorType::AND_OP); $operator->setSearchItems($this->searchItems); - return array($operator, null); + return $operator; } public function retrieveElasticQueryEntryIds(baseObjectFilter $filter, kPager $pager) { - list($query, $kEsearchOrderBy ) = $this->createElasticQueryFromFilter($filter); + $query = $this->createElasticQueryFromFilter($filter); $entrySearch = new kEntrySearch(); $entrySearch->setFilterOnlyContext(); - $elasticResults = $entrySearch->doSearch($query, $pager, self::$validStatuses,null, $kEsearchOrderBy); + $elasticResults = $entrySearch->doSearch($query, $pager, self::$validStatuses); list($coreResults, $objectOrder, $objectCount, $objectHighlight) = kESearchCoreAdapter::getElasticResultAsArray($elasticResults, $entrySearch->getQueryAttributes()->getQueryHighlightsAttributes()); @@ -80,89 +70,41 @@ protected function AddFieldPartToQuery($searchItemType, $elasticFieldName, $fiel { case ESearchFilterItemType::EXACT_MATCH: $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH); + $this->addToSearchItemsByField($elasticFieldName, $searchItem); break; case ESearchFilterItemType::EXACT_MATCH_MULTI_OR : - if ($elasticFieldName === ESearchCategoryEntryFieldName::FULL_IDS) - { - $searchItem = $this->addMultiQueryCategoryFullName($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); - } - else - { - $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); - } + $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); break; case ESearchFilterItemType::EXACT_MATCH_MULTI_AND : - $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::AND_OP); + $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::AND_OP); break; case ESearchFilterItemType::PARTIAL : $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL); + $this->addToSearchItemsByField($elasticFieldName, $searchItem); break; case ESearchFilterItemType::PARTIAL_MULTI_OR : - $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::OR_OP); + $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::OR_OP); break; case ESearchFilterItemType::PARTIAL_MULTI_AND : - $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::AND_OP); - break; - - case ESearchFilterItemType::EXACT_MATCH_NOT: - $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::NOT_OP); - break; - - case ESearchFilterItemType::NOT_CONTAINS: - $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::NOT_OP); - break; - - case ESearchFilterItemType::IS_EMPTY: - $searchItem = $this->addSearchItem($elasticFieldName, null, ESearchItemType::EXISTS); - if ($fieldValue) - { - $searchItem = $this->createOperator(ESearchOperatorType::NOT_OP, array($searchItem), $elasticFieldName); - } + $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::AND_OP); break; case ESearchFilterItemType::RANGE_GTE : - $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setGreaterThanOrEqual'); + $this->addRangeGteQuery($elasticFieldName, $fieldValue); break; case ESearchFilterItemType::RANGE_LTE : - $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setLessThanOrEqual'); - break; - - case ESearchFilterItemType::RANGE_LT: - $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setLessThan'); - break; - - case ESearchFilterItemType::RANGE_GT: - $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setGreaterThan'); - break; - - case ESearchFilterItemType::RANGE_LTE_OR_NULL: - $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setLessThanOrEqual'); - $searchItem = $this->allowNullValues($searchItem,$elasticFieldName); - break; - - case ESearchFilterItemType::RANGE_GTE_OR_NULL: - $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setGreaterThanOrEqual'); - $searchItem = $this->allowNullValues($searchItem,$elasticFieldName); - break; - - case ESearchFilterItemType::MATCH_AND: - $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::AND_OP); - break; - - case ESearchFilterItemType::MATCH_OR: - $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); + $this->addRangeLteQuery($elasticFieldName, $fieldValue); break; default: - throw new KalturaAPIException(KalturaErrors::SEARCH_ITEM_TYPE_NOT_FOUND,$searchItemType, $elasticFieldName); + KalturaLog::debug("Skip field [$elasticFieldName] as it has no search item type [$searchItemType]"); } - $this->addToSearchItemsByField($elasticFieldName, $searchItem); } protected function addMultiQuery($elasticFieldName, $fieldValue, $searchType, $operatorType) @@ -179,82 +121,37 @@ protected function addMultiQuery($elasticFieldName, $fieldValue, $searchType, $o $operator = $this->getEsearchOperatorByField($elasticFieldName); $operator->setOperator($operatorType); $operator->setSearchItems($innerSearchItems); - return $operator; + $this->addToSearchItemsByField($elasticFieldName, $operator); } } - protected function addSearchItem($elasticFieldName, $value, $itemType, $range = false) + protected function addRangeGteQuery($elasticFieldName, $fieldValue) { - $searchItem = $this->createSearchItemByFieldType($elasticFieldName); - if (property_exists ($searchItem, self::FIELD_NAME) && !$searchItem->getFieldName()) - { - $searchItem->setFieldName($elasticFieldName); - } - if(!$range) - { - $searchItem->setSearchTerm($value); - } - $searchItem->setItemType($itemType); - return $searchItem; + $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::RANGE, true); + $range = new ESearchRange(); + $range->setGreaterThanOrEqual($fieldValue); + $searchItem->setRange($range); + $this->addToSearchItemsByField($elasticFieldName, $searchItem); } - protected function getSearchItemForCategories($values, $searchType) + protected function addRangeLteQuery($elasticFieldName, $fieldValue) { - $innerSearchItems = array(); - if(count($values)) - { - if (count($values) > 1) - { //all values excepts from the last value should be parent category p1>p2>c3 - for ($i = 0; $i < count($values) - 1; $i++) - { - $innerSearchItems[] = $this->addSearchItem(KalturaESearchCategoryEntryFieldName::ANCESTOR_NAME, $values[$i], $searchType); - } - } - $innerSearchItems[] = $this->addSearchItem(ESearchBaseCategoryEntryItem::CATEGORY_NAMES_MAPPING_FIELD, $values[count($values) - 1], $searchType); - } - return $innerSearchItems; + $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::RANGE, true); + $range = new ESearchRange(); + $range->setLessThanOrEqual($fieldValue); + $searchItem->setRange($range); + $this->addToSearchItemsByField($elasticFieldName, $searchItem); } - protected function addMultiQueryCategoryFullName($elasticFieldName, $fieldValue, $searchType, $operatorType) + protected function addSearchItem($elasticFieldName, $value, $itemType, $range = false) { - $categoriesValues = array(); - $andOperator = array(); - $values = $this->createValuesArray($fieldValue); - foreach ($values as $value) - { - $categoriesValues[] = explode(categoryPeer::CATEGORY_SEPARATOR, $value); - } - foreach ($categoriesValues as $categoriesValue) + $searchItem = $this->createSearchItemByFieldType($elasticFieldName); + $searchItem->setFieldName($elasticFieldName); + if(!$range) { - $searchItemsAndArray = $this->getSearchItemForCategories($categoriesValue, $searchType); - $andOperator[] = $this->createOperator(ESearchOperatorType::AND_OP, $searchItemsAndArray, $elasticFieldName); + $searchItem->setSearchTerm($value); } - $orOprator = $this->createOperator($operatorType, $andOperator, $elasticFieldName); - return $orOprator; - } - - protected function allowNullValues($searchItem,$elasticFieldName) - { - $notSearchItem = $this->addSearchItem($elasticFieldName, null, ESearchItemType::EXISTS); - $notOprator = $this->createOperator(ESearchOperatorType::NOT_OP, array($notSearchItem), $elasticFieldName); - $orOprator = $this->createOperator(ESearchOperatorType::OR_OP, array($notOprator,$searchItem), $elasticFieldName); - return $orOprator; - } - - protected function createOperator($opretorType,$searchItemsArray,$elasticFieldName) - { - $operatorGeneral = $this->getEsearchOperatorByField($elasticFieldName); - $operatorGeneral->setOperator($opretorType); - $operatorGeneral->setSearchItems($searchItemsArray); - return $operatorGeneral; - } - - protected function getSearchItemWithRange($elasticFieldName, $fieldValue, $rangeProperty) - { - $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::RANGE, true); - $range = new ESearchRange(); - $range->$rangeProperty($fieldValue); - $searchItem->setRange($range); + $searchItem->setItemType($itemType); return $searchItem; } @@ -310,16 +207,6 @@ protected function addNestedQueryPart() protected function createSearchItemByFieldType($elasticFieldName) { - if (in_array($elasticFieldName ,array_keys(self::$categoryFilterFields))) - { - $eSearchCategoryEntry = new self::$categoryFilterFields[$elasticFieldName][self::FIELD_CLASS_LOCATION](); - $eSearchCategoryEntry->setFieldName(self::$categoryFilterFields[$elasticFieldName][self::FIELD_NAME_LOCATION]); - return $eSearchCategoryEntry; - } - if ($elasticFieldName === ESearchUnifiedItem::UNIFIED) - { - return new ESearchUnifiedItem(); - } return new ESearchEntryItem(); } @@ -333,16 +220,7 @@ protected function getSphinxToElasticSearchItemType($operator) baseObjectFilter::LTE => ESearchFilterItemType::RANGE_LTE, baseObjectFilter::LIKE => ESearchFilterItemType::EXACT_MATCH, baseObjectFilter::MULTI_LIKE_OR => ESearchFilterItemType::EXACT_MATCH_MULTI_OR, - baseObjectFilter::MULTI_LIKE_AND => ESearchFilterItemType::EXACT_MATCH_MULTI_AND, - baseObjectFilter::LTE_OR_NULL => ESearchFilterItemType::RANGE_LTE_OR_NULL, - baseObjectFilter::LT => ESearchFilterItemType::RANGE_LT, - baseObjectFilter::GT => ESearchFilterItemType::RANGE_GT, - baseObjectFilter::GTE_OR_NULL => ESearchFilterItemType::RANGE_GTE_OR_NULL, - baseObjectFilter::MATCH_AND => ESearchFilterItemType::MATCH_AND, - baseObjectFilter::MATCH_OR => ESearchFilterItemType::MATCH_OR, - baseObjectFilter::NOT_CONTAINS => ESearchFilterItemType::NOT_CONTAINS, - baseObjectFilter::IS_EMPTY => ESearchFilterItemType::IS_EMPTY - ); + baseObjectFilter::MULTI_LIKE_AND => ESearchFilterItemType::EXACT_MATCH_MULTI_AND); if(array_key_exists($operator, $operatorsMap)) { diff --git a/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php b/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php index d1db7521cd8..671327a8704 100644 --- a/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php +++ b/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php @@ -51,7 +51,6 @@ class ESearchEntryItem extends ESearchItem 'is_live' => array('ESearchItemType::EXACT_MATCH'=> ESearchItemType::EXACT_MATCH), 'user_names' => array('ESearchItemType::EXACT_MATCH'=> ESearchItemType::EXACT_MATCH, 'ESearchItemType::PARTIAL'=> ESearchItemType::PARTIAL, 'ESearchItemType::STARTS_WITH'=> ESearchItemType::STARTS_WITH, 'ESearchItemType::EXISTS'=> ESearchItemType::EXISTS, ESearchUnifiedItem::UNIFIED), 'root_id' => array('ESearchItemType::EXACT_MATCH'=> ESearchItemType::EXACT_MATCH), - 'partner_sort_value' => array('ESearchItemType::RANGE' => ESearchItemType::RANGE) ); protected static $field_boost_values = array( From 59e52da4528d6499a84f4c9c74662720ce0a68aa Mon Sep 17 00:00:00 2001 From: Moshe Maor Date: Mon, 8 Jul 2019 16:57:52 +0300 Subject: [PATCH 060/103] Revert "Revert "SUP-16933:use eSearch instead of sphinx for dynamic playlist execute from filter/content actions"" --- .../kaltura/lib/myPlaylistUtils.class.php | 153 +++++++++- api_v3/lib/KalturaErrors.php | 2 + api_v3/services/PlaylistService.php | 79 ++++- .../elastic/mapping/entry_mapping.json | 3 +- .../api/enum/KalturaESearchEntryFieldName.php | 1 + .../items/entry/KalturaESearchEntryItem.php | 1 + .../lib/model/enum/ESearchEntryFieldName.php | 1 + .../model/enum/ESearchEntryFilterFields.php | 55 ++++ .../lib/model/enum/ESearchFilterItemType.php | 9 + .../filters/ESearchCaptionQueryFromFilter.php | 2 +- .../filters/ESearchEntryQueryFromFilter.php | 287 ++++++++++++++++++ .../model/filters/ESearchQueryFromFilter.php | 186 ++++++++++-- .../lib/model/items/ESearchEntryItem.php | 1 + 13 files changed, 734 insertions(+), 46 deletions(-) create mode 100644 plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php create mode 100644 plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php diff --git a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php index 5851119524a..c17d801c70d 100644 --- a/alpha/apps/kaltura/lib/myPlaylistUtils.class.php +++ b/alpha/apps/kaltura/lib/myPlaylistUtils.class.php @@ -14,13 +14,30 @@ class myPlaylistUtils const CAPTION_FILES_LABEL = "label"; const CAPTION_FILES_PATH = "path"; const CAPTION_FILES_ID = "captionId"; + const KALTURA_CLASS = 'kalturaClass'; private static $user_cache = null; private static $isAdminKs = false; private static $playlistContext; - + + private static $moderationStatusesNotIn = array( + entry::ENTRY_MODERATION_STATUS_PENDING_MODERATION, + entry::ENTRY_MODERATION_STATUS_REJECTED + ); + + private static $dates = array( + '_lteornull_start_date', + '_gteornull_start_date', + '_lte_start_date', + '_gte_start_date', + '_lteornull_end_date', + '_gteornull_end_date', + '_lte_end_date', + '_gte_end_date' + ); + public static function setIsAdminKs($v) { self::$isAdminKs = $v; @@ -483,11 +500,139 @@ public static function getDynamicPlaylistFilters($xml) } return $entry_filters; } - - public static function executeDynamicPlaylist ( $partner_id , $xml , $filter = null ,$detailed = true, $pager = null ) + + public static function getEntryFiltersFromXml($listOfFilters, $partnerId) + { + if (!$listOfFilters) + { + return null; + } + return self::fillEntryFilterFromXml($listOfFilters, $partnerId); + } + + //splitting entry filters into filters that will run in Esearch and in Sphinx: + // Elastic - advancesSearch is not set OR advancesSearch is set and includes KalturaMetadataSearchItem + // Sphinx - advancesSearch is set and NOT including KalturaMetadataSearchItem + public static function splitEntryFilters($xml) + { + $entryFiltersViaEsearch = array(); + $entryFiltersViaSphinx = array(); + list ($totalResults, $entryFilters) = self::getPlaylistFilterListStruct($xml); + foreach ($entryFilters as $entryFilter) + { + if (!isset($entryFilter->advancedSearch) || + ( isset($entryFilter->advancedSearch) && (string)$entryFilter->advancedSearch[self::KALTURA_CLASS] === PlaylistService::KALTURA_METADATA_SEARCH_ITEM)) + { + $entryFiltersViaEsearch[] = $entryFilter; + } + else + { + $entryFiltersViaSphinx[] = $entryFilter; + } + } + return array($entryFiltersViaEsearch, $entryFiltersViaSphinx, $totalResults); + } + + public static function executeDynamicPlaylistViaEsearch ($entryFilters ,$totalResults, $pager = null) + { + $entryKPager = new kPager(); + if ($pager) + { + $pager->toObject($entryKPager); + } + $entryQueryToFilterESearch = new ESearchEntryQueryFromFilter(); + $entryIds= array(); + foreach ($entryFilters as $entryFilter) + { + list ($currEntryIds, $count) = $entryQueryToFilterESearch->retrieveElasticQueryEntryIds($entryFilter, $entryKPager); + $entryIds = self::mergeEntriesByLimit($entryIds, $currEntryIds, $entryFilter->getLimit(),$totalResults); + $totalResults = max (0, $totalResults - count($entryIds)); + if ( $totalResults === 0 ) + { + break; + } + } + return array(self::getEntriesSorted($entryIds), $totalResults); + } + + protected static function getEntriesSorted($entryIds) + { + $entryIds = array_unique($entryIds); + $entryIdsOrder = array_flip($entryIds); + $entries = entryPeer::retrieveByPKs($entryIds); + usort($entries, self::buildSorter($entryIdsOrder)); + return $entries; + } + + protected static function mergeEntriesByLimit($entryIds, $currEntryIds, $limit, $totalResults) + { + if ($totalResults < $limit) + { + $limit = $totalResults; + } + if (count($currEntryIds) > $limit) + { + $currEntryIds = array_slice($currEntryIds,0, $limit); + } + return array_merge ($entryIds, $currEntryIds); + } + + protected static function buildSorter($objectsOrder) + { + return function ($a, $b) use ($objectsOrder) + { + return ($objectsOrder[$a->getId()] - $objectsOrder[$b->getId()]); + }; + } + + protected static function fillEntryFilterFromXml($listOfFilters, $partnerId) + { + $entryFilters = array(); + foreach ($listOfFilters as $filter) + { + self::replaceContextTokens($filter); + $entryFilter = new entryFilter(); + $entryFilter->fillObjectFromXml($filter, '_'); + self::updateEntryFilterFields($entryFilter, $partnerId); + $entryFilters[] = $entryFilter; + } + return $entryFilters; + } + + protected static function updateEntryFilterFields($entryFilter, $partnerId) + { + self::updateEntryFilter($entryFilter, $partnerId); + $typeArray = array (entryType::MEDIA_CLIP, entryType::MIX, entryType::LIVE_STREAM ); + $typeArray = array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::MEDIA_CLIP)); + $typeArray = array_unique(array_merge($typeArray, KalturaPluginManager::getExtendedTypes(entryPeer::OM_CLASS, entryType::LIVE_STREAM))); + $entryFilter->setTypeIn($typeArray); + $entryFilter->setStatusEquel(entryStatus::READY); + $entryFilter->setModerationStatusNotIn(self::$moderationStatusesNotIn); + if (self::$isAdminKs) + { + self::unsetDates($entryFilter); + } + } + + protected static function unsetDates($filter) + { + foreach (self::$dates as $date) + { + if ($filter->is_set($date)) + { + $filter->unsetByName($date); + } + } + } + + public static function executeDynamicPlaylist ( $partner_id, $xml, $filter = null, $detailed = true, $pager = null ) { list ( $total_results , $list_of_filters ) = self::getPlaylistFilterListStruct ( $xml ); - + return self::executeDynamicPlaylistFromFilters($total_results, $list_of_filters, $partner_id, $filter , $detailed, $pager); + } + + public static function executeDynamicPlaylistFromFilters($total_results, $list_of_filters, $partner_id, $filter = null, $detailed = true, $pager = null) + { $entry_filters = array(); if ( ! $list_of_filters ) return null; diff --git a/api_v3/lib/KalturaErrors.php b/api_v3/lib/KalturaErrors.php index 03edd0cc55c..e71b3ff0502 100644 --- a/api_v3/lib/KalturaErrors.php +++ b/api_v3/lib/KalturaErrors.php @@ -753,5 +753,7 @@ class KalturaErrors extends APIErrors const MAP_ALREADY_EXIST = "MAP_ALREADY_EXIST;NAME,HOST;Map already exist for this map name {@NAME@} and host {@HOST@}"; const MAP_CANNOT_BE_CREATED_ON_FILE_SYSTEM = "MAP_CANNOT_BE_CREATED_ON_FILE_SYSTEM;;Map cannnot be created on file system"; const HOST_NAME_CONTAINS_ASTRIX = "HOST_NAME_CONTAINS_ASTRIX;HOST_NAME;Host name contains *, use # instead {@HOST_NAME@}"; + + const SEARCH_ITEM_TYPE_NOT_FOUND = 'SEARCH_ITEM_TYPE_NOT_FOUND;SEARCH_ITEM_TYPE,ELASTIC_FIELD_NAME; Search item type [@SEARCH_ITEM_TYPE@] not found for field: [@ELASTIC_FIELD_NAME@]'; } diff --git a/api_v3/services/PlaylistService.php b/api_v3/services/PlaylistService.php index b15269e5da1..7f04c25f565 100644 --- a/api_v3/services/PlaylistService.php +++ b/api_v3/services/PlaylistService.php @@ -10,6 +10,9 @@ */ class PlaylistService extends KalturaEntryService { + const ADVANCED_SEARCH = 'advancedSearch'; + const KALTURA_METADATA_SEARCH_ITEM = 'KalturaMetadataSearchItem'; + /* (non-PHPdoc) * @see KalturaBaseService::globalPartnerAllowed() */ @@ -345,19 +348,81 @@ function executeAction( $id , $detailed = false, KalturaContext $playlistContext function executeFromContentAction($playlistType, $playlistContent, $detailed = false, $pager = null) { myDbHelper::$use_alternative_con = myDbHelper::DB_HELPER_CONN_PROPEL3; - if ($this->getKs() && is_object($this->getKs()) && $this->getKs()->isAdmin()) + { myPlaylistUtils::setIsAdminKs(true); + } + list($entryFiltersViaEsearch, $entryFiltersViaSphinx, $totalResults) = myPlaylistUtils::splitEntryFilters($playlistContent); + $pagerSeparateQueries = self::decideWhereHandlingPager($pager,$entryFiltersViaEsearch, $entryFiltersViaSphinx); + $entryList = self::handlePlaylistByType($playlistType, $entryFiltersViaEsearch, $entryFiltersViaSphinx, $this->getPartnerId(), $pagerSeparateQueries, $pager, $totalResults, $playlistContent); + myEntryUtils::updatePuserIdsForEntries($entryList); + KalturaLog::debug("entry ids count: " . count($entryList)); + return KalturaBaseEntryArray::fromDbArray($entryList, $this->getResponseProfile()); + } - $entryList = array(); + protected static function handlePlaylistByType($playlistType, $entryFiltersViaEsearch, $entryFiltersViaSphinx, $partnerId, $pagerSeperateQueries, $pager, $totalResults, $playlistContent) + { if ($playlistType == KalturaPlaylistType::DYNAMIC) - $entryList = myPlaylistUtils::executeDynamicPlaylist($this->getPartnerId(), $playlistContent, null, true, $pager); + { + $entryList = self::handlingDynamicPlaylist($entryFiltersViaEsearch,$entryFiltersViaSphinx, $partnerId, $pagerSeperateQueries, $pager, $totalResults); + } else if ($playlistType == KalturaPlaylistType::STATIC_LIST) + { $entryList = myPlaylistUtils::executeStaticPlaylistFromEntryIdsString($playlistContent, null, true, $pager); - - myEntryUtils::updatePuserIdsForEntries($entryList); - - return KalturaBaseEntryArray::fromDbArray($entryList, $this->getResponseProfile()); + } + return $entryList; + } + + protected static function handlingDynamicPlaylist($entryFiltersViaEsearch, $entryFiltersViaSphinx, $partnerId, $pagerSeparateQueries, $pager, $totalResults) + { + $entryListEsearch = array(); + $entryListSphinx = array(); + if ($entryFiltersViaEsearch) + { + $entryFiltersViaEsearch = myPlaylistUtils::getEntryFiltersFromXml($entryFiltersViaEsearch, $partnerId); + list($entryListEsearch, $totalResults) = myPlaylistUtils::executeDynamicPlaylistViaEsearch($entryFiltersViaEsearch, $totalResults, $pagerSeparateQueries); + } + if ($entryFiltersViaSphinx) + { + $entryListSphinx = myPlaylistUtils::executeDynamicPlaylistFromFilters($totalResults, $entryFiltersViaSphinx, $partnerId, null, true, $pagerSeparateQueries); + } + $entryListMerged = array_merge($entryListEsearch,$entryListSphinx); + $entryListUnique = self::getEntryListUnique($entryListMerged); + $entryList = self::getEntryListByPager($entryListUnique, $pager, $pagerSeparateQueries); + return $entryList; + } + + //When queries are going to run both in Esearch and Sphinx, pager handling will be done after merging the results -> ($pagerSeperateQueries = null) + //In case only one of them will run (for example only in sphinx), pager handling will be done only there (in this example: in sphinx query) -> ($pagerSeperateQueries = $pager) + protected static function decideWhereHandlingPager($pager,$entryFiltersViaEsearch, $entryFiltersViaSphinx) + { + $pagerSeparateQueries = $pager; + if ($entryFiltersViaEsearch && $entryFiltersViaSphinx) + { + $pagerSeparateQueries = null; + } + return $pagerSeparateQueries; + } + + protected static function getEntryListUnique($entryListMerged) + { + $entryList = array(); + foreach ($entryListMerged as $currentEntry) + { + $entryList[$currentEntry->getId()] = $currentEntry; + } + return $entryList; + } + + protected static function getEntryListByPager($entryList, $pager, $pagerSeparateQueries) + { + if ( $pager && is_null($pagerSeparateQueries)) + { + $startOffset = $pager->calcOffset(); + $pageSize = $pager->calcPageSize(); + $entryList = array_slice($entryList, $startOffset, $pageSize); + } + return $entryList; } /** diff --git a/configurations/elastic/mapping/entry_mapping.json b/configurations/elastic/mapping/entry_mapping.json index 88a3ce8ce25..d1bf9a8deb9 100644 --- a/configurations/elastic/mapping/entry_mapping.json +++ b/configurations/elastic/mapping/entry_mapping.json @@ -626,8 +626,7 @@ "normalizer" : "kaltura_keyword_normalizer" }, "partner_sort_value": { - "type" : "keyword", - "normalizer" : "kaltura_keyword_normalizer" + "type" : "integer" }, "redirect_entry_id": { "type" : "keyword", diff --git a/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php b/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php index 3c590e32625..be898c4a6a9 100644 --- a/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php +++ b/plugins/search/providers/elastic_search/lib/api/enum/KalturaESearchEntryFieldName.php @@ -39,4 +39,5 @@ class KalturaESearchEntryFieldName extends KalturaStringEnum const IS_LIVE = 'is_live'; const USER_NAMES = "user_names"; const ROOT_ID = 'root_id'; + const PARTNER_SORT_VALUE = 'partner_sort_value'; } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php b/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php index 89124cfc9bc..6c27a232b31 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/items/entry/KalturaESearchEntryItem.php @@ -58,6 +58,7 @@ class KalturaESearchEntryItem extends KalturaESearchAbstractEntryItem KalturaESearchEntryFieldName::IS_LIVE => ESearchEntryFieldName::IS_LIVE, KalturaESearchEntryFieldName::USER_NAMES => ESearchEntryFieldName::USER_NAMES, KalturaESearchEntryFieldName::ROOT_ID => ESearchEntryFieldName::ROOT_ID, + KalturaESearchEntryFieldName::PARTNER_SORT_VALUE => ESearchEntryFieldName::PARTNER_SORT_VALUE ); protected function getMapBetweenObjects() diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php index de84c28d317..4378a5f1fe0 100644 --- a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFieldName.php @@ -41,4 +41,5 @@ interface ESearchEntryFieldName extends BaseEnum const DISPLAY_IN_SEARCH = 'display_in_search'; const ROOT_ID = 'root_id'; const PRIVACY_BY_CONTEXTS = 'privacy_by_contexts'; + const PARTNER_SORT_VALUE = 'partner_sort_value'; } diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php new file mode 100644 index 00000000000..223856a3022 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryFilterFields.php @@ -0,0 +1,55 @@ +updateEntryPager($entryPager, $filterOnEntryIds); - $query = $this->createElasticQueryFromFilter($filter); + list($query, $kEsearchOrderBy ) = $this->createElasticQueryFromFilter($filter); $elasticResults = $entrySearch->doSearch($query, $entryPager, self::$validStatuses); diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php new file mode 100644 index 00000000000..c77c6a17ff6 --- /dev/null +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php @@ -0,0 +1,287 @@ + ESearchEntryFieldName::ID, + ESearchEntryFilterFields::USER_ID => ESearchEntryFieldName::USER_ID, + ESearchEntryFilterFields::GROUP_ID => ESearchEntryFieldName::USER_ID, + ESearchEntryFilterFields::CREATOR_ID => ESearchEntryFieldName::CREATOR_ID, + ESearchEntryFilterFields::NAME => ESearchEntryFieldName::NAME, + ESearchEntryFilterFields::TAGS => ESearchEntryFieldName::TAGS, + ESearchEntryFilterFields::TAGS_NAME => ESearchEntryFieldName::TAGS, + ESearchEntryFilterFields::ADMIN_TAGS => ESearchEntryFieldName::ADMIN_TAGS, + ESearchEntryFilterFields::TAGS_ADMIN_TAGS => ESearchEntryFieldName::ADMIN_TAGS, + ESearchEntryFilterFields::TAGS_ADMIN_TAGS_NAME => ESearchEntryFieldName::ADMIN_TAGS, + ESearchEntryFilterFields::CONVERSION_PROFILE_ID => ESearchEntryFieldName::CONVERSION_PROFILE_ID, + ESearchEntryFilterFields::REDIRECT_ENTRY_ID => ESearchEntryFieldName::REDIRECT_ENTRY_ID, + ESearchEntryFilterFields::ENTITLED_USER_EDIT => ESearchEntryFieldName::ENTITLED_USER_EDIT, + ESearchEntryFilterFields::ENTITLED_USER_PUBLISH => ESearchEntryFieldName::ENTITLED_USER_PUBLISH, + ESearchEntryFilterFields::ENTITLED_USER_VIEW => ESearchEntryFieldName::ENTITLED_USER_VIEW, + ESearchEntryFilterFields::DISPLAY_IN_SEARCH => ESearchEntryFieldName::DISPLAY_IN_SEARCH, + ESearchEntryFilterFields::PARENT_ENTRY_ID => ESearchEntryFieldName::PARENT_ENTRY_ID, + ESearchEntryFilterFields::MEDIA_TYPE => ESearchEntryFieldName::MEDIA_TYPE, + ESearchEntryFilterFields::SOURCE_TYPE => ESearchEntryFieldName::SOURCE_TYPE, + ESearchEntryFilterFields::LENGTH_IN_MSECS => ESearchEntryFieldName::LENGTH_IN_MSECS, + ESearchEntryFilterFields::TYPE => ESearchEntryFieldName::ENTRY_TYPE, + ESearchEntryFilterFields::MODERATION_STATUS => ESearchEntryFieldName::MODERATION_STATUS, + ESearchEntryFilterFields::CREATED_AT => ESearchEntryFieldName::CREATED_AT, + ESearchEntryFilterFields::UPDATED_AT => ESearchEntryFieldName::UPDATED_AT, + ESearchEntryFilterFields::ACCESS_CONTROL_ID => ESearchEntryFieldName::ACCESS_CONTROL_ID, + ESearchEntryFilterFields::USER_NAMES => ESearchEntryFieldName::USER_NAMES, + ESearchEntryFilterFields::START_DATE => ESearchEntryFieldName::START_DATE, + ESearchEntryFilterFields::END_DATE => ESearchEntryFieldName::END_DATE, + ESearchEntryFilterFields::REFERENCE_ID => ESearchEntryFieldName::REFERENCE_ID, + ESearchEntryFilterFields::ROOT_ENTRY_ID => ESearchEntryFieldName::ROOT_ID, + ESearchEntryFilterFields::DURATION => ESearchEntryFieldName::LENGTH_IN_MSECS, + ESearchEntryFilterFields::CATEGORIES => ESearchCategoryEntryNameItem::CATEGORY_NAMES_MAPPING_FIELD, + ESearchEntryFilterFields::CATEGORIES_IDS => ESearchCategoryEntryIdItem::CATEGORY_IDS_MAPPING_FIELD, + ESearchEntryFilterFields::CATEGORIES_ANCESTOR_ID => ESearchCategoryEntryFieldName::ANCESTOR_ID, + ESearchEntryFilterFields::CATEGORIES_FULL_NAME => ESearchCategoryEntryFieldName::FULL_IDS, + ESearchEntryFilterFields::PARTNER_SORT_VALUE => ESearchEntryFieldName::PARTNER_SORT_VALUE, + ESearchEntryFilterFields::SEARCH_TEXT => ESearchUnifiedItem::UNIFIED, + ESearchEntryFilterFields::FREE_TEXT => ESearchUnifiedItem::UNIFIED, + ESearchEntryFilterFields::TOTAL_RANK => ESearchEntryOrderByFieldName::VOTES + ); + + if(array_key_exists($field, $fieldsMap)) + { + return $fieldsMap[$field]; + } + else + { + return null; + } + } + + protected function getSphinxToElasticOrderBy($field) + { + $fieldsMap = array( + self::ASC.ESearchEntryFilterFields::MEDIA_TYPE, + self::DESC.ESearchEntryFilterFields::MEDIA_TYPE, + KalturaMediaEntryOrderBy::PLAYS_ASC => self::ASC.ESearchEntryOrderByFieldName::PLAYS, + KalturaMediaEntryOrderBy::PLAYS_DESC => self::DESC.ESearchEntryOrderByFieldName::PLAYS, + KalturaMediaEntryOrderBy::VIEWS_ASC => self::ASC.ESearchEntryOrderByFieldName::VIEWS, + KalturaMediaEntryOrderBy::VIEWS_DESC => self::DESC.ESearchEntryOrderByFieldName::VIEWS, + KalturaMediaEntryOrderBy::DURATION_ASC => self::ASC.ESearchEntryFieldName::LENGTH_IN_MSECS, + KalturaMediaEntryOrderBy::DURATION_DESC => self::DESC.ESearchEntryFieldName::LENGTH_IN_MSECS, + KalturaMediaEntryOrderBy::NAME_ASC => self::ASC. ESearchEntryOrderByFieldName::NAME, + KalturaMediaEntryOrderBy::NAME_DESC => self::DESC. ESearchEntryOrderByFieldName::NAME, + KalturaMediaEntryOrderBy::CREATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, + KalturaMediaEntryOrderBy::CREATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT, + KalturaMediaEntryOrderBy::UPDATED_AT_ASC => self::ASC.ESearchEntryOrderByFieldName::UPDATED_AT, + KalturaMediaEntryOrderBy::UPDATED_AT_DESC => self::DESC.ESearchEntryOrderByFieldName::UPDATED_AT, + KalturaMediaEntryOrderBy::RANK_ASC => self::ASC.ESearchEntryOrderByFieldName::VOTES, + KalturaMediaEntryOrderBy::RANK_DESC => self::DESC.ESearchEntryOrderByFieldName::VOTES, + self::ASC.ESearchEntryFilterFields::TOTAL_RANK => self::ASC.ESearchEntryOrderByFieldName::VOTES, + self::DESC.ESearchEntryFilterFields::TOTAL_RANK => self::DESC.ESearchEntryOrderByFieldName::VOTES, + KalturaMediaEntryOrderBy::START_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::START_DATE, + KalturaMediaEntryOrderBy::START_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::START_DATE, + KalturaMediaEntryOrderBy::END_DATE_ASC => self::ASC.ESearchEntryOrderByFieldName::END_DATE, + KalturaMediaEntryOrderBy::END_DATE_DESC => self::DESC.ESearchEntryOrderByFieldName::END_DATE, + KalturaMediaEntryOrderBy::RECENT_ASC => self::ASC.ESearchEntryOrderByFieldName::CREATED_AT, + KalturaMediaEntryOrderBy::RECENT_DESC => self::DESC.ESearchEntryOrderByFieldName::CREATED_AT + ); + + if(in_array($field, $fieldsMap)) + { + return $field; + } + else if (array_key_exists($field, $fieldsMap)) + { + return $fieldsMap[$field]; + } + else + { + return null; + } + } + + public function createElasticQueryFromFilter(baseObjectFilter $filter) + { + $this->init(); + $kEsearchOrderBy = null; + foreach($filter->fields as $field => $fieldValue) + { + if ($field === entryFilter::ORDER && !is_null($fieldValue) && $fieldValue!= '') + { + $kEsearchOrderBy = $this->getKESearchOrderBy($fieldValue); + continue; + } + $fieldParts = $this->splitIntoParameters($filter, $field, $fieldValue); + if (!$fieldParts) + { + continue; + } + list($operator, $fieldName, $fieldValue) = $fieldParts; + $this->addingFieldPartIntoQuery($operator, $fieldName, $fieldValue); + } + return $this->createFinalOperator($kEsearchOrderBy); + } + + protected function createFinalOperator($kEsearchOrderBy) + { + $this->addNestedQueryPart(); + $operator = new ESearchOperator(); + $operator->setOperator(ESearchOperatorType::AND_OP); + $operator->setSearchItems($this->searchItems); + return array($operator, $kEsearchOrderBy); + } + + protected function addingFieldPartIntoQuery($operator, $fieldName, $fieldValue) + { + $searchItemType = $this->getSphinxToElasticSearchItemType($operator); + $elasticFieldName = $this->getSphinxToElasticFieldName($fieldName); + if($elasticFieldName && $searchItemType) + { + $this->AddFieldPartToQuery($searchItemType, $elasticFieldName, $fieldValue); + } + } + + protected function splitIntoParameters($filter, $field, $fieldValue ) + { + $fieldParts = explode(entryFilter::FILTER_PREFIX, $field, 3); + list( , $operator, $fieldName) = $fieldParts; + list($operator, $fieldName) = self::handlingFreeTextField($field, $operator, $fieldName); + if(!in_array($fieldName, static::getSupportedFields()) || is_null($fieldValue) || $fieldValue === '') + { + return null; + } + if ($fieldName === ESearchEntryFilterFields::STATUS && ($operator === baseObjectFilter::EQ ||$operator === baseObjectFilter::IN )) + { + self::$validStatuses = explode(',',$fieldValue); + return null; + } + $fieldValue = self::translateFieldValue($fieldName, $filter, $fieldValue); + return array($operator, $fieldName, $fieldValue); + } + + protected static function handlingFreeTextField($field, $operator, $fieldName) + { + if ($field === ESearchEntryFilterFields::FREE_TEXT) + { + $operator = baseObjectFilter::IN; + $fieldName = $field; + } + return array($operator, $fieldName); + } + + protected static function translateFieldValue($fieldName, $filter, $fieldValue) + { + if(in_array($fieldName, self::$puserFields)) + { + $kuser = kuserPeer::getKuserByPartnerAndUid($filter->getPartnerSearchScope(), $fieldValue); + if (!$kuser) + { + throw new KalturaAPIException(KalturaErrors::INVALID_USER_ID, $fieldValue); + } + $fieldValue = $kuser->getId(); + } + else if ($fieldName === ESearchEntryFilterFields::DURATION) + { + $fieldValue = $fieldValue * 1000; + } + return $fieldValue; + } + + protected function getKESearchOrderBy($fieldValue) + { + $fieldValue = $this->getSphinxToElasticOrderBy($fieldValue); + if (!$fieldValue) + { + return null; + } + $eSearchOrderByItem = self::getESearchOrderByItem($fieldValue); + $orderItems = array($eSearchOrderByItem); + $kEsearchOrderBy = new ESearchOrderBy(); + $kEsearchOrderBy->setOrderItems($orderItems); + return $kEsearchOrderBy; + } + + protected static function getESearchOrderByItem($fieldValue) + { + $eSearchOrderByItem = new ESearchEntryOrderByItem(); + $eSearchOrderByItem->setSortField(substr($fieldValue,1)); + $fieldNameSortOrder = $fieldValue[0]; + if ($fieldNameSortOrder === self::DESC) + { + $eSearchOrderByItem->setSortOrder(ESearchSortOrder::ORDER_BY_DESC); + } + else if ($fieldNameSortOrder === self::ASC) + { + $eSearchOrderByItem->setSortOrder(ESearchSortOrder::ORDER_BY_ASC); + } + return $eSearchOrderByItem; + } +} diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php index e859d8afc6e..3b451d96f48 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchQueryFromFilter.php @@ -5,6 +5,16 @@ class ESearchQueryFromFilter protected $searchItems; protected $nestedSearchItem; protected static $validStatuses = array(entryStatus::READY,entryStatus::NO_CONTENT); + protected static $categoryFilterFields = array( + ESearchBaseCategoryEntryItem::CATEGORY_NAMES_MAPPING_FIELD => array(ESearchCategoryEntryFieldName::NAME, 'ESearchCategoryEntryNameItem'), + ESearchBaseCategoryEntryItem::CATEGORY_IDS_MAPPING_FIELD => array(ESearchCategoryEntryFieldName::ID, 'ESearchCategoryEntryIdItem'), + ESearchCategoryEntryFieldName::ANCESTOR_ID => array(ESearchCategoryEntryFieldName::ANCESTOR_ID, 'ESearchCategoryEntryAncestorIdItem'), + ESearchCategoryEntryFieldName::ANCESTOR_NAME => array(ESearchCategoryEntryFieldName::ANCESTOR_NAME, 'ESearchCategoryEntryAncestorNameItem') + ); + + const FIELD_NAME_LOCATION = 0; + const FIELD_CLASS_LOCATION = 1; + const FIELD_NAME = 'fieldName'; public function __construct() { @@ -46,16 +56,16 @@ public function createElasticQueryFromFilter(baseObjectFilter $filter) $operator = new ESearchOperator(); $operator->setOperator(ESearchOperatorType::AND_OP); $operator->setSearchItems($this->searchItems); - return $operator; + return array($operator, null); } public function retrieveElasticQueryEntryIds(baseObjectFilter $filter, kPager $pager) { - $query = $this->createElasticQueryFromFilter($filter); + list($query, $kEsearchOrderBy ) = $this->createElasticQueryFromFilter($filter); $entrySearch = new kEntrySearch(); $entrySearch->setFilterOnlyContext(); - $elasticResults = $entrySearch->doSearch($query, $pager, self::$validStatuses); + $elasticResults = $entrySearch->doSearch($query, $pager, self::$validStatuses,null, $kEsearchOrderBy); list($coreResults, $objectOrder, $objectCount, $objectHighlight) = kESearchCoreAdapter::getElasticResultAsArray($elasticResults, $entrySearch->getQueryAttributes()->getQueryHighlightsAttributes()); @@ -70,41 +80,89 @@ protected function AddFieldPartToQuery($searchItemType, $elasticFieldName, $fiel { case ESearchFilterItemType::EXACT_MATCH: $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH); - $this->addToSearchItemsByField($elasticFieldName, $searchItem); break; case ESearchFilterItemType::EXACT_MATCH_MULTI_OR : - $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); + if ($elasticFieldName === ESearchCategoryEntryFieldName::FULL_IDS) + { + $searchItem = $this->addMultiQueryCategoryFullName($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); + } + else + { + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); + } break; case ESearchFilterItemType::EXACT_MATCH_MULTI_AND : - $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::AND_OP); + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::AND_OP); break; case ESearchFilterItemType::PARTIAL : $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL); - $this->addToSearchItemsByField($elasticFieldName, $searchItem); break; case ESearchFilterItemType::PARTIAL_MULTI_OR : - $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::OR_OP); + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::OR_OP); break; case ESearchFilterItemType::PARTIAL_MULTI_AND : - $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::AND_OP); + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::PARTIAL, ESearchOperatorType::AND_OP); + break; + + case ESearchFilterItemType::EXACT_MATCH_NOT: + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::NOT_OP); + break; + + case ESearchFilterItemType::NOT_CONTAINS: + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::NOT_OP); + break; + + case ESearchFilterItemType::IS_EMPTY: + $searchItem = $this->addSearchItem($elasticFieldName, null, ESearchItemType::EXISTS); + if ($fieldValue) + { + $searchItem = $this->createOperator(ESearchOperatorType::NOT_OP, array($searchItem), $elasticFieldName); + } break; case ESearchFilterItemType::RANGE_GTE : - $this->addRangeGteQuery($elasticFieldName, $fieldValue); + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setGreaterThanOrEqual'); break; case ESearchFilterItemType::RANGE_LTE : - $this->addRangeLteQuery($elasticFieldName, $fieldValue); + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setLessThanOrEqual'); + break; + + case ESearchFilterItemType::RANGE_LT: + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setLessThan'); + break; + + case ESearchFilterItemType::RANGE_GT: + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setGreaterThan'); + break; + + case ESearchFilterItemType::RANGE_LTE_OR_NULL: + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setLessThanOrEqual'); + $searchItem = $this->allowNullValues($searchItem,$elasticFieldName); + break; + + case ESearchFilterItemType::RANGE_GTE_OR_NULL: + $searchItem = $this->getSearchItemWithRange($elasticFieldName, $fieldValue,'setGreaterThanOrEqual'); + $searchItem = $this->allowNullValues($searchItem,$elasticFieldName); + break; + + case ESearchFilterItemType::MATCH_AND: + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::AND_OP); + break; + + case ESearchFilterItemType::MATCH_OR: + $searchItem = $this->addMultiQuery($elasticFieldName, $fieldValue, ESearchItemType::EXACT_MATCH, ESearchOperatorType::OR_OP); break; default: - KalturaLog::debug("Skip field [$elasticFieldName] as it has no search item type [$searchItemType]"); + throw new KalturaAPIException(KalturaErrors::SEARCH_ITEM_TYPE_NOT_FOUND,$searchItemType, $elasticFieldName); } + $this->addToSearchItemsByField($elasticFieldName, $searchItem); } protected function addMultiQuery($elasticFieldName, $fieldValue, $searchType, $operatorType) @@ -121,37 +179,82 @@ protected function addMultiQuery($elasticFieldName, $fieldValue, $searchType, $o $operator = $this->getEsearchOperatorByField($elasticFieldName); $operator->setOperator($operatorType); $operator->setSearchItems($innerSearchItems); - $this->addToSearchItemsByField($elasticFieldName, $operator); + return $operator; } } - protected function addRangeGteQuery($elasticFieldName, $fieldValue) + protected function addSearchItem($elasticFieldName, $value, $itemType, $range = false) { - $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::RANGE, true); - $range = new ESearchRange(); - $range->setGreaterThanOrEqual($fieldValue); - $searchItem->setRange($range); - $this->addToSearchItemsByField($elasticFieldName, $searchItem); + $searchItem = $this->createSearchItemByFieldType($elasticFieldName); + if (property_exists ($searchItem, self::FIELD_NAME) && !$searchItem->getFieldName()) + { + $searchItem->setFieldName($elasticFieldName); + } + if(!$range) + { + $searchItem->setSearchTerm($value); + } + $searchItem->setItemType($itemType); + return $searchItem; } - protected function addRangeLteQuery($elasticFieldName, $fieldValue) + protected function getSearchItemForCategories($values, $searchType) { - $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::RANGE, true); - $range = new ESearchRange(); - $range->setLessThanOrEqual($fieldValue); - $searchItem->setRange($range); - $this->addToSearchItemsByField($elasticFieldName, $searchItem); + $innerSearchItems = array(); + if(count($values)) + { + if (count($values) > 1) + { //all values excepts from the last value should be parent category p1>p2>c3 + for ($i = 0; $i < count($values) - 1; $i++) + { + $innerSearchItems[] = $this->addSearchItem(KalturaESearchCategoryEntryFieldName::ANCESTOR_NAME, $values[$i], $searchType); + } + } + $innerSearchItems[] = $this->addSearchItem(ESearchBaseCategoryEntryItem::CATEGORY_NAMES_MAPPING_FIELD, $values[count($values) - 1], $searchType); + } + return $innerSearchItems; } - protected function addSearchItem($elasticFieldName, $value, $itemType, $range = false) + protected function addMultiQueryCategoryFullName($elasticFieldName, $fieldValue, $searchType, $operatorType) { - $searchItem = $this->createSearchItemByFieldType($elasticFieldName); - $searchItem->setFieldName($elasticFieldName); - if(!$range) + $categoriesValues = array(); + $andOperator = array(); + $values = $this->createValuesArray($fieldValue); + foreach ($values as $value) { - $searchItem->setSearchTerm($value); + $categoriesValues[] = explode(categoryPeer::CATEGORY_SEPARATOR, $value); } - $searchItem->setItemType($itemType); + foreach ($categoriesValues as $categoriesValue) + { + $searchItemsAndArray = $this->getSearchItemForCategories($categoriesValue, $searchType); + $andOperator[] = $this->createOperator(ESearchOperatorType::AND_OP, $searchItemsAndArray, $elasticFieldName); + } + $orOprator = $this->createOperator($operatorType, $andOperator, $elasticFieldName); + return $orOprator; + } + + protected function allowNullValues($searchItem,$elasticFieldName) + { + $notSearchItem = $this->addSearchItem($elasticFieldName, null, ESearchItemType::EXISTS); + $notOprator = $this->createOperator(ESearchOperatorType::NOT_OP, array($notSearchItem), $elasticFieldName); + $orOprator = $this->createOperator(ESearchOperatorType::OR_OP, array($notOprator,$searchItem), $elasticFieldName); + return $orOprator; + } + + protected function createOperator($opretorType,$searchItemsArray,$elasticFieldName) + { + $operatorGeneral = $this->getEsearchOperatorByField($elasticFieldName); + $operatorGeneral->setOperator($opretorType); + $operatorGeneral->setSearchItems($searchItemsArray); + return $operatorGeneral; + } + + protected function getSearchItemWithRange($elasticFieldName, $fieldValue, $rangeProperty) + { + $searchItem = $this->addSearchItem($elasticFieldName, $fieldValue, ESearchItemType::RANGE, true); + $range = new ESearchRange(); + $range->$rangeProperty($fieldValue); + $searchItem->setRange($range); return $searchItem; } @@ -207,6 +310,16 @@ protected function addNestedQueryPart() protected function createSearchItemByFieldType($elasticFieldName) { + if (in_array($elasticFieldName ,array_keys(self::$categoryFilterFields))) + { + $eSearchCategoryEntry = new self::$categoryFilterFields[$elasticFieldName][self::FIELD_CLASS_LOCATION](); + $eSearchCategoryEntry->setFieldName(self::$categoryFilterFields[$elasticFieldName][self::FIELD_NAME_LOCATION]); + return $eSearchCategoryEntry; + } + if ($elasticFieldName === ESearchUnifiedItem::UNIFIED) + { + return new ESearchUnifiedItem(); + } return new ESearchEntryItem(); } @@ -220,7 +333,16 @@ protected function getSphinxToElasticSearchItemType($operator) baseObjectFilter::LTE => ESearchFilterItemType::RANGE_LTE, baseObjectFilter::LIKE => ESearchFilterItemType::EXACT_MATCH, baseObjectFilter::MULTI_LIKE_OR => ESearchFilterItemType::EXACT_MATCH_MULTI_OR, - baseObjectFilter::MULTI_LIKE_AND => ESearchFilterItemType::EXACT_MATCH_MULTI_AND); + baseObjectFilter::MULTI_LIKE_AND => ESearchFilterItemType::EXACT_MATCH_MULTI_AND, + baseObjectFilter::LTE_OR_NULL => ESearchFilterItemType::RANGE_LTE_OR_NULL, + baseObjectFilter::LT => ESearchFilterItemType::RANGE_LT, + baseObjectFilter::GT => ESearchFilterItemType::RANGE_GT, + baseObjectFilter::GTE_OR_NULL => ESearchFilterItemType::RANGE_GTE_OR_NULL, + baseObjectFilter::MATCH_AND => ESearchFilterItemType::MATCH_AND, + baseObjectFilter::MATCH_OR => ESearchFilterItemType::MATCH_OR, + baseObjectFilter::NOT_CONTAINS => ESearchFilterItemType::NOT_CONTAINS, + baseObjectFilter::IS_EMPTY => ESearchFilterItemType::IS_EMPTY + ); if(array_key_exists($operator, $operatorsMap)) { diff --git a/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php b/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php index 671327a8704..d1db7521cd8 100644 --- a/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php +++ b/plugins/search/providers/elastic_search/lib/model/items/ESearchEntryItem.php @@ -51,6 +51,7 @@ class ESearchEntryItem extends ESearchItem 'is_live' => array('ESearchItemType::EXACT_MATCH'=> ESearchItemType::EXACT_MATCH), 'user_names' => array('ESearchItemType::EXACT_MATCH'=> ESearchItemType::EXACT_MATCH, 'ESearchItemType::PARTIAL'=> ESearchItemType::PARTIAL, 'ESearchItemType::STARTS_WITH'=> ESearchItemType::STARTS_WITH, 'ESearchItemType::EXISTS'=> ESearchItemType::EXISTS, ESearchUnifiedItem::UNIFIED), 'root_id' => array('ESearchItemType::EXACT_MATCH'=> ESearchItemType::EXACT_MATCH), + 'partner_sort_value' => array('ESearchItemType::RANGE' => ESearchItemType::RANGE) ); protected static $field_boost_values = array( From 7a3f3d344bbffcad96d03b716e5b90e780e2a191 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 8 Jul 2019 19:58:05 +0300 Subject: [PATCH 061/103] PLAT-9902 - support convert captions from scc/srt/dfxp/webvtt to scc/srt/dfxp/webvtt --- configurations/batch/batch.ini.template | 3 +- infra/general/kSCCParser.php | 158 ------------------ .../content/caption/base/CaptionPlugin.php | 12 +- .../KAsyncConvertCaptionAsset.class.php} | 66 +++++--- .../KAsyncConvertCaptionAssetExe.php} | 4 +- .../CopyCaptions/KAsyncCopyCaptions.class.php | 2 +- ... => KalturaConvertCaptionAssetJobData.php} | 18 +- .../sccCaptionsContentManager.php | 142 ---------------- .../base/lib/kCaptionsContentManager.php | 17 +- .../base/lib/model/enums/CaptionType.php | 1 + ...e.php => ConvertCaptionAssetBatchType.php} | 6 +- ...ta.php => kConvertCaptionAssetJobData.php} | 54 +++++- .../caption/base/scripts/convertcaption.py | 57 +++++++ .../base/services/CaptionAssetService.php | 6 +- 14 files changed, 195 insertions(+), 351 deletions(-) delete mode 100644 infra/general/kSCCParser.php rename plugins/content/caption/base/batch/{ParseCaptionAsset/KAsyncParseSccCaptionAsset.class.php => ConvertCaptions/KAsyncConvertCaptionAsset.class.php} (50%) rename plugins/content/caption/base/batch/{ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php => ConvertCaptions/KAsyncConvertCaptionAssetExe.php} (65%) rename plugins/content/caption/base/lib/api/{KalturaParseSccCaptionAssetJobData.php => KalturaConvertCaptionAssetJobData.php} (68%) delete mode 100644 plugins/content/caption/base/lib/captionManagers/sccCaptionsContentManager.php rename plugins/content/caption/base/lib/model/enums/{ParseSccCaptionAssetBatchType.php => ConvertCaptionAssetBatchType.php} (61%) rename plugins/content/caption/base/lib/model/{kParseSccCaptionAssetJobData.php => kConvertCaptionAssetJobData.php} (53%) create mode 100644 plugins/content/caption/base/scripts/convertcaption.py diff --git a/configurations/batch/batch.ini.template b/configurations/batch/batch.ini.template index 18999456301..05d224e9a23 100644 --- a/configurations/batch/batch.ini.template +++ b/configurations/batch/batch.ini.template @@ -838,4 +838,5 @@ name = KAsyncParseSccCaptionAsset friendlyName = parse scc caption asset type = KAsyncParseSccCaptionAsset maximumExecutionTime = 300 -scriptPath = ../plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php \ No newline at end of file +scriptPath = ../plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseCaptionAssetExe.php +params.pythonCmd = @PYTHON_3_PATH@ \ No newline at end of file diff --git a/infra/general/kSCCParser.php b/infra/general/kSCCParser.php deleted file mode 100644 index 51c931fceab..00000000000 --- a/infra/general/kSCCParser.php +++ /dev/null @@ -1,158 +0,0 @@ -\d{2}):(?P\d{2}):(?P\d{2})(:|;)(?P\d{2})(?P.+)\r?(\n|$)?\s*\r?(\n|$)/sU'; - - const commandCodes = array('94ae','9420','942c','942f','947a','97a1','97a2','9723','9120', '91a1', '91a2', '9123', '91a4', '9125', '9126', '91a7', '91a8', '9129', '912a', '91ab', '912c', '91ad', '91ae', '912f', '94a8'); - - private static $rowCodes = array('91','92','15','16','97','13','94'); - - private static $columnsCodes = array('d0','51','c2','43','c4','45','46','c7','c8','49','4a','cb','4c','cd','70','f1','62','e3','64','e5','e6','67','68','e9','ea','6b','ec','6d','52','d3','54','d5','d6','57','58','d9','da','5b','dc','5d','5e','df','f2','73','f4','75','76','f7','f8','79','7a','fb','7c','fd','fe','7f'); - - private static $singleRowCode = array('10'); - - private static $singleColumnCode = array('d0','51','c2','43','c4','45','46','c7','c8','49','4a','cb','4c','cd','52','d3','54','d5','d6','57','58','d9','da','5b','dc','5d','5e','df'); - - private static $specialChars = array('91b0'=>'®','9131'=>'°','9132'=>'½','91b3'=>'¿','9134'=>'™','91b5'=>'¢','91b6'=>'£','9137' => '♪','9138'=>'à','91b9'=>' ','91ba'=>'è','913b'=>'â', - '91bc'=>'ê','913d'=>'î','913e'=>'ô','91bf'=>'û'); - - private static $extendedChars = array('9220'=>'Á','92a1'=>'É','92a2'=>'Ó','9223'=>'Ú','92a4'=>'Ü','9225'=>'ü','9226'=>'‘','92a7'=>'¡','92a8'=>'*','9229'=>'’','922a'=>'—','92ab'=>'©', - '922c'=>'℠','92ad'=>'•','92ae'=>'“','922f'=>'”','92b0'=>'À','9231'=>'Â','9232'=>'Ç','92b3'=>'È','9234'=>'Ê','92b5'=>'Ë','92b6'=>'ë','9237'=>'Î', - '9238'=>'Ï','92b9'=>'ï','92ba'=>'Ô','923b'=>'Ù','92bc'=>'ù','923d'=>'Û','923e'=>'«','92bf'=>'»','1320'=>'Ã','13a1'=>'ã','13a2'=>'Í','1323'=>'Ì', - '13a4'=>'ì','1325'=>'Ò','1326'=>'ò','13a7'=>'Õ','13a8'=>'õ','1329'=>'{','132a'=>'}','13ab'=>'\\','132c'=>'^','13ad'=>'_','13ae'=>'¦','132f'=>'~', - '13b0'=>'Ä','1331'=>'ä','1332'=>'Ö','13b3'=>'ö','1334'=>'ß','13b5'=>'¥','13b6'=>'¤','1337'=>'|','1338'=>'Å','13b9'=>'å','13ba'=>'Ø','133b'=>'ø', - '9bbc'=>'┌','9b3d'=>'┐','9b3e'=>'└','9bbf'=>'┘'); - - private static $twoByteCharset = array('20' =>' ','a1' =>'!','a2' =>'"','23' =>'#','a4' =>'$','25' =>'%','26' =>'&','a7' =>"'",'a8' =>'(','29' =>')','2a' =>'á','ab' =>'+', - '2c' =>',','ad' =>'-','ae' =>'.','2f' =>'/','b0' =>'0','31' =>'1','32' =>'2','b3' =>'3','34' =>'4','b5' =>'5','b6' =>'6','37' =>'7', - '38' =>'8','b9' =>'9','ba' =>':','3b' =>';','bc' =>'<','3d' =>'=','3e' =>'>','bf' =>'?','40' =>'@','c1' =>'A','c2' =>'B','43' =>'C', - 'c4' =>'D','45' =>'E','46' =>'F','c7' =>'G','c8' =>'H','49' =>'I','4a' =>'J','cb' =>'K','4c' =>'L','cd' =>'M','ce' =>'N','4f' =>'O', - 'd0' =>'P','51' =>'Q','52' =>'R','d3' =>'S','54' =>'T','d5' =>'U','d6' =>'V','57' =>'W','58' =>'X','d9' =>'Y','da' =>'Z','5b' =>'[', - 'dc' =>'é','5d' =>']','5e' =>'í','df' =>'ó','e0' =>'ú','61' =>'a','62' =>'b','e3' =>'c','64' =>'d','e5' =>'e','e6' =>'f','67' =>'g', - '68' =>'h','e9' =>'i','ea' =>'j','6b' =>'k','ec' =>'l','6d' =>'m','6e' =>'n','ef' =>'o','70' =>'p','f1' =>'q','f2' =>'r','73' =>'s', - 'f4' =>'t','75' =>'u','76' =>'v','f7' =>'w','f8' =>'x','79' =>'y','7a' =>'z','fb' =>'ç','7c' =>'÷','fd' =>'Ñ','fe' =>'ñ'); - - /** - * @param string $content - * @return array - */ - public static function parseToSrt($content) - { - if (kString::beginsWith($content, "\xff\xfe")) - { - $content = iconv('utf-16', 'utf-8', substr($content, 2)); - } - - if (!preg_match_all(self::timingRegex, $content, $matches) || !count($matches) || !count($matches[0])) - { - KalturaLog::err("Content regex not found"); - print("Content regex not found"); - return array(); - } - $text = ''; - $rowCount = 1; - foreach ($matches[0] as $index => $match) - { - $content = $matches['content'][$index]; - $startTime = self::maketime($matches, $index); - $endTime = $startTime; - if (isset( $matches['content'][$index + 1])) - { - $endTime = self::maketime($matches, $index + 1); - } - $translatedContent = self::tranlasteContent($content); - if ($translatedContent) - { - $text .= "$rowCount\n$startTime --> $endTime\n$translatedContent\n\n"; - $rowCount++; - } - } - return $text; - } - - /** - * @param $content - * @return null|string - */ - private static function tranlasteContent($content) - { - //remove unncessary scc commands that are irrelevant to the text. - $content = str_replace(self::commandCodes, "", $content); - - $parts = preg_split('/\s+/', trim($content)); - $text = null; - for ($index = 0; $index < count($parts); $index++) - { - $code = $parts[$index]; - switch (true) - { - case isset(self::$specialChars[$code]): - { - $text .= self::$specialChars[$code]; - // in case we have double padding for commands we need to ignore next command - if (isset($parts[$index + 1]) && $code == $parts[$index + 1]) - { - $index++; - } - break; - } - case isset(self::$extendedChars[$code]): - { - $text .= self::$extendedChars[$code]; - break; - } - default: - { - $codes = str_split($code, 2); - //check if codes represent location-cursor move - if ((in_array($codes[0], self::$rowCodes) && in_array($codes[1], self::$columnsCodes)) - || (in_array($codes[0], self::$singleRowCode) && in_array($codes[1], self::$singleColumnCode)) - ) - { - $text .= ' '; - // in case we have double padding for commands we need to ignore next command - if (isset($parts[$index + 1]) && $code == $parts[$index + 1]) - { - $index++; - } - - } - else - { - //check if codes represent 2Byte chars and add them to text. - foreach ($codes as $charsetCode) - { - if (isset(self::$twoByteCharset[$charsetCode])) - { - $text .= self::$twoByteCharset[$charsetCode]; - } - } - } - } - } - } - return trim($text); - } - - /** - * @param $matches - * @param $index - * @return array - */ - protected static function maketime($matches, $index) - { - $Hours = $matches['startHours'][$index]; - $Minutes = $matches['startMinutes'][$index]; - $Seconds = $matches['startSeconds'][$index]; - $Frame = round($matches['startFrame'][$index] * 41.7); - return "$Hours:$Minutes:$Seconds,$Frame"; - - } -} diff --git a/plugins/content/caption/base/CaptionPlugin.php b/plugins/content/caption/base/CaptionPlugin.php index 99bf9efabb8..d1d075ce2d5 100644 --- a/plugins/content/caption/base/CaptionPlugin.php +++ b/plugins/content/caption/base/CaptionPlugin.php @@ -224,7 +224,7 @@ public static function getEventConsumers() public static function getEnums($baseEnumName = null) { if(is_null($baseEnumName)) - return array('CaptionAssetType', 'CaptionObjectFeatureType', 'ParseMultiLanguageCaptionAssetBatchType','ParseSccCaptionAssetBatchType'); + return array('CaptionAssetType', 'CaptionObjectFeatureType', 'ParseMultiLanguageCaptionAssetBatchType','ConvertCaptionAssetBatchType'); if($baseEnumName == 'assetType') return array('CaptionAssetType'); @@ -233,7 +233,7 @@ public static function getEnums($baseEnumName = null) return array('CaptionObjectFeatureType'); if ($baseEnumName == 'BatchJobType') - return array('ParseMultiLanguageCaptionAssetBatchType', 'ParseSccCaptionAssetBatchType'); + return array('ParseMultiLanguageCaptionAssetBatchType', 'ConvertCaptionAssetBatchType'); return array(); } @@ -255,11 +255,11 @@ public static function loadObject($baseClass, $enumValue, array $constructorArgs if($baseClass == 'KalturaJobData' && $enumValue == self::getApiValue(ParseMultiLanguageCaptionAssetBatchType::PARSE_MULTI_LANGUAGE_CAPTION_ASSET)) return new KalturaParseMultiLanguageCaptionAssetJobData(); - if($baseClass == 'KalturaJobData' && $enumValue == self::getApiValue(ParseSccCaptionAssetBatchType::PARSE_SCC_CAPTION_ASSET)) - return new KalturaParseSccCaptionAssetJobData(); + if($baseClass == 'KalturaJobData' && $enumValue == self::getApiValue(ConvertCaptionAssetBatchType::CONVERT_CAPTION_ASSET)) + return new KalturaConvertCaptionAssetJobData(); - if($baseClass == 'kJobData' && $enumValue == self::getBatchJobTypeCoreValue(ParseSccCaptionAssetBatchType::PARSE_SCC_CAPTION_ASSET)) - return new KalturaParseSccCaptionAssetJobData(); + if($baseClass == 'kJobData' && $enumValue == self::getBatchJobTypeCoreValue(ConvertCaptionAssetBatchType::CONVERT_CAPTION_ASSET)) + return new KalturaConvertCaptionAssetJobData(); return null; } diff --git a/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAsset.class.php b/plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAsset.class.php similarity index 50% rename from plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAsset.class.php rename to plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAsset.class.php index a97621b8ef3..102596ee102 100644 --- a/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAsset.class.php +++ b/plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAsset.class.php @@ -3,19 +3,21 @@ * @package plugins.caption * @subpackage Scheduler */ -class KAsyncParseSccCaptionAsset extends KJobHandlerWorker +class KAsyncConvertCaptionAsset extends KJobHandlerWorker { /* * @var KalturaCaptionClientPlugin */ private $captionClientPlugin = null; + private $formatToName = array(CaptionType::SRT => 'srt' , CaptionType::DFXP => 'dfxp', CaptionType::WEBVTT => 'webvtt', CaptionType::SCC =>'scc'); + /* (non-PHPdoc) * @see KBatchBase::getType() */ public static function getType() { - return KalturaBatchJobType::PARSE_SCC_CAPTION_ASSET; + return KalturaBatchJobType::CONVERT_CAPTION_ASSET; } /* (non-PHPdoc) @@ -23,17 +25,25 @@ public static function getType() */ protected function exec(KalturaBatchJob $job) { - return $this->parseSccCaption($job, $job->data); + return $this->convertCaption($job, $job->data); } - protected function parseSccCaption(KalturaBatchJob $job, KalturaParseSccCaptionAssetJobData $data) + protected function convertCaption(KalturaBatchJob $job, KalturaConvertCaptionAssetJobData $data) { - $this->updateJob($job, "Start parsing scc caption asset [$data->sccCaptionAssetId]", KalturaBatchJobStatus::QUEUED); + $this->updateJob($job, "Start parsing caption asset [$data->captionAssetId]", KalturaBatchJobStatus::QUEUED); $this->captionClientPlugin = KalturaCaptionClientPlugin::get(self::$kClient); - $captionAssetId = $data->sccCaptionAssetId; + $captionAssetId = $data->captionAssetId; $fileLoc = $data->fileLocation; + if (!array_key_exists($data->fromType,$this->formatToName) || !(array_key_exists($data->toType ,$this->formatToName))) + { + $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNSUPPORTED_FORMAT_TYPES', "Error: " . 'UNSUPPORTED_FORMAT_TYPES', KalturaBatchJobStatus::FAILED, $data); + return $job; + } + + $fromTypeName = $this->formatToName[$data->fromType]; + $toTypeName = $this->formatToName[$data->toType]; $content = kEncryptFileUtils::getEncryptedFileContent($fileLoc, $data->fileEncryptionKey, kConf::get("encryption_iv")); if (!$content) @@ -42,32 +52,48 @@ protected function parseSccCaption(KalturaBatchJob $job, KalturaParseSccCaptionA return $job; } - self::impersonate($job->partnerId); - $parsedContent = kSCCParser::parseToSrt($content); + $output = null; + $return_var = 0; + $tempFile = tempnam(sys_get_temp_dir(), 'captionTranslation.'); + + kFileBase::filePutContents($tempFile, $content); + $script = realpath(dirname(__FILE__) . '/../../') . '/scripts/convertcaption.py'; + $cmd = self::$taskConfig->params->pythonCmd ." $script -i $tempFile -f $fromTypeName -t $toTypeName"; + KalturaLog::debug("Running caption conversion command: $cmd"); + exec($cmd, $output, $return_var); - $sccCaptionAsset = $this->captionClientPlugin->captionAsset->get($captionAssetId); - if (!$sccCaptionAsset) + if ($return_var) { - $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_GET_ORIGINAL_SCC_ASSET', "Error: " . 'UNABLE_TO_GET_ORIGINAL_SCC_ASSET', KalturaBatchJobStatus::FAILED, $data); + $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_PARSE_ASSET', "Error: " . 'UNABLE_TO_PARSE_ASSET', KalturaBatchJobStatus::FAILED, $data); + return $job; + } + + $parsedContent = implode("\n",$output); + + self::impersonate($job->partnerId); + $captionAsset = $this->captionClientPlugin->captionAsset->get($captionAssetId); + if (!$captionAsset) + { + $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_GET_ORIGINAL_ASSET', "Error: " . 'UNABLE_TO_GET_ORIGINAL_ASSET', KalturaBatchJobStatus::FAILED, $data); return $job; } $res = $this->deleteCaptionAsset($captionAssetId); if (!$res) { - $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_DELETE_ORIGINAL_SCC_ASSET', "Error: " . 'UNABLE_TO_DELETE_ORIGINAL_SCC_ASSET', KalturaBatchJobStatus::FAILED, $data); + $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_DELETE_ORIGINAL_ASSET', "Error: " . 'UNABLE_TO_DELETE_ORIGINAL_ASSET', KalturaBatchJobStatus::FAILED, $data); return $job; } - $captionsCreated = $this->cloneCaptionAssetToSrtAndSetContent($sccCaptionAsset->entryId, $sccCaptionAsset, $parsedContent); + $captionsCreated = $this->cloneCaptionAssetToSrtAndSetContent($captionAsset->entryId, $captionAsset, $parsedContent,$data->toType, $toTypeName); self::unimpersonate(); if ($captionsCreated) { - $this->closeJob($job, null, null, "Finished parsing scc captions to srt", KalturaBatchJobStatus::FINISHED); + $this->closeJob($job, null, null, "Finished parsing $fromTypeName captions to $toTypeName", KalturaBatchJobStatus::FINISHED); return $job; } else - throw new kApplicativeException(KalturaBatchJobAppErrors::MISSING_ASSETS, "NABLE_TO_CREATE_ASSET_WITH_SRT_CAPTION"); + throw new kApplicativeException(KalturaBatchJobAppErrors::MISSING_ASSETS, "UNABLE_TO_CREATE_ASSET_WITH_CAPTION"); } @@ -89,13 +115,15 @@ private function deleteCaptionAsset($id) } } - /** + /*** * @param $entryId * @param $captionAsset * @param $content + * @param $format + * @param $fileExt * @return bool */ - private function cloneCaptionAssetToSrtAndSetContent($entryId, $captionAsset, $content) + private function cloneCaptionAssetToSrtAndSetContent($entryId, $captionAsset, $content, $format, $fileExt) { $captionAsset->id = null; $captionAsset->entryId = null; @@ -108,8 +136,8 @@ private function cloneCaptionAssetToSrtAndSetContent($entryId, $captionAsset, $c $captionAsset->description = null; $captionAsset->status = null; - $captionAsset->fileExt = 'srt'; - $captionAsset->format = KalturaCaptionType::SRT; + $captionAsset->fileExt = $fileExt; + $captionAsset->format = $format; try { diff --git a/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php b/plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAssetExe.php similarity index 65% rename from plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php rename to plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAssetExe.php index 86445f469d7..2e66c31ca97 100644 --- a/plugins/content/caption/base/batch/ParseCaptionAsset/KAsyncParseSccCaptionAssetExe.php +++ b/plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAssetExe.php @@ -2,12 +2,12 @@ require_once(__DIR__ . "/../../../../../../batch/bootstrap.php"); /** - * Executes the KAsyncParseSccCaptionAssetExe + * Executes the KAsyncConvertCaptionAssetExe * * @package plugins.caption * @subpackage Scheduler */ -$instance = new KAsyncParseSccCaptionAsset(); +$instance = new KAsyncConvertCaptionAsset(); $instance->run(); $instance->done(); diff --git a/plugins/content/caption/base/batch/CopyCaptions/KAsyncCopyCaptions.class.php b/plugins/content/caption/base/batch/CopyCaptions/KAsyncCopyCaptions.class.php index 1e58d00019e..63f4b185e54 100644 --- a/plugins/content/caption/base/batch/CopyCaptions/KAsyncCopyCaptions.class.php +++ b/plugins/content/caption/base/batch/CopyCaptions/KAsyncCopyCaptions.class.php @@ -100,7 +100,7 @@ private function retrieveCaptionAssetsOnlyFromSupportedTypes($originalCaptionAss private function getUnsupportedFormats() { - $unsupportedFormats = array (CaptionType::CAP); + $unsupportedFormats = array (CaptionType::CAP, CaptionType::SCC); return $unsupportedFormats; } diff --git a/plugins/content/caption/base/lib/api/KalturaParseSccCaptionAssetJobData.php b/plugins/content/caption/base/lib/api/KalturaConvertCaptionAssetJobData.php similarity index 68% rename from plugins/content/caption/base/lib/api/KalturaParseSccCaptionAssetJobData.php rename to plugins/content/caption/base/lib/api/KalturaConvertCaptionAssetJobData.php index 3f754c93b8b..ce4214871bf 100644 --- a/plugins/content/caption/base/lib/api/KalturaParseSccCaptionAssetJobData.php +++ b/plugins/content/caption/base/lib/api/KalturaConvertCaptionAssetJobData.php @@ -3,12 +3,12 @@ * @package plugins.caption * @subpackage api.objects */ -class KalturaParseSccCaptionAssetJobData extends KalturaJobData +class KalturaConvertCaptionAssetJobData extends KalturaJobData { /** * @var string */ - public $sccCaptionAssetId; + public $captionAssetId; /** * @var string @@ -20,11 +20,23 @@ class KalturaParseSccCaptionAssetJobData extends KalturaJobData */ public $fileEncryptionKey; + /** + * @var string + */ + public $fromType; + + /** + * @var string + */ + public $toType; + private static $map_between_objects = array ( - "sccCaptionAssetId", + "captionAssetId", "fileLocation", "fileEncryptionKey", + "fromType", + "toType" ); /* (non-PHPdoc) diff --git a/plugins/content/caption/base/lib/captionManagers/sccCaptionsContentManager.php b/plugins/content/caption/base/lib/captionManagers/sccCaptionsContentManager.php deleted file mode 100644 index 24e961cd345..00000000000 --- a/plugins/content/caption/base/lib/captionManagers/sccCaptionsContentManager.php +++ /dev/null @@ -1,142 +0,0 @@ - ((?:[0-9]{2}:)?[0-9]{2}:[0-9]{2}\,[0-9]{3})( .*)?$#'; - - /* (non-PHPdoc) - * @see kCaptionsContentManager::parse() - */ - public function parse($content) - { - if (kString::beginsWith($content, "\xff\xfe")) - { - $content = iconv('utf-16', 'utf-8', substr($content, 2)); - } - - $matches = null; - $regex = '/(?P\d+)\s*\r?\n\s*(?P\d{1,2}):(?P\d{1,2}):(?P\d{1,2})[,\.](?P\d{1,3})\s*-->\s*(?P\d{1,2}):(?P\d{1,2}):(?P\d{1,2})[,\.](?P\d{1,3})\s*\r?\n((?P.+)\r?(\n|$))?\s*\r?(\n|$)/sU'; - if(!preg_match_all($regex, $content, $matches) || !count($matches) || !count($matches[0])) - { - KalturaLog::err("Content regex not found"); - return array(); - } - - $itemsData = array(); - foreach($matches[0] as $index => $match) - { - $startHours = intval($matches['startHours'][$index]); - $startMinutes = intval($matches['startMinutes'][$index]); - $startSeconds = intval($matches['startSeconds'][$index]); - $startMilliseconds = intval($matches['startMilliseconds'][$index]); - $endHours = intval($matches['endHours'][$index]); - $endMinutes = intval($matches['endMinutes'][$index]); - $endSeconds = intval($matches['endSeconds'][$index]); - $endMilliseconds = intval($matches['endMilliseconds'][$index]); - $content = $matches['content'][$index]; - - $itemsData[] = array( - 'startTime' => $this->makeTime($startHours, $startMinutes, $startSeconds, $startMilliseconds), - 'endTime' => $this->makeTime($endHours, $endMinutes, $endSeconds, $endMilliseconds), - 'content' => array(array('text' => $content)), - ); - } - return $itemsData; - } - - private function makeTime($hours, $minutes, $seconds, $milliseconds) - { - $ret = $hours * 60; - $ret += $minutes; - $ret *= 60; - $ret += $seconds; - $ret *= 1000; - $ret += $milliseconds; - return $ret; - } - - /* (non-PHPdoc) - * @see kCaptionsContentManager::getContent() - */ - public function getContent($content) - { - if (kString::beginsWith($content, "\xff\xfe")) - { - $content = iconv('utf-16', 'utf-8', substr($content, 2)); - } - - $replace = array( - '/^[\d]+\s*[\r\n]+/' => '', - '/[\r\n]+\s*[\d]+\s*[\r\n]+/' => '', - '/[\d]{2}:[\d]{2}:[\d]{2},[\d]{3}/' => '', - '/\s+-->\s+/' => ' ', - '/\s+/' => ' ', - ); - return preg_replace(array_keys($replace), $replace, $content); - } - - /** - * @return srtCaptionsContentManager - */ - public static function get() - { - return new srtCaptionsContentManager(); - } - - public function buildFile($content, $clipStartTime, $clipEndTime, $globalOffset = 0) - { - $newFileContent = $this->createCaptionsFile($content, $clipStartTime, $clipEndTime, self::SRT_TIMECODE_PATTERN, $globalOffset); - return $newFileContent; - } - - - protected function createAdjustedTimeLine($matches, $clipStartTime, $clipEndTime, $globalOffset) - { - $startCaption = self::parseCaptionTime($matches[1]); - $endCaption = self::parseCaptionTime($matches[2]); - if (!TimeOffsetUtils::onTimeRange($startCaption, $endCaption, $clipStartTime, $clipEndTime)) - return null; - $adjustedStartTime = TimeOffsetUtils::getAdjustedStartTime($startCaption, $clipStartTime, $globalOffset); - $adjustedEndTime = TimeOffsetUtils::getAdjustedEndTime($endCaption, $clipStartTime, $clipEndTime, $globalOffset); - $timeLine = $this->formatSrtTimeStamp($adjustedStartTime) . ' --> ' . $this->formatSrtTimeStamp($adjustedEndTime). kCaptionsContentManager::UNIX_LINE_ENDING; - return $timeLine; - } - - - /** - * @param int $timeStamp - * @return string - */ - private function formatSrtTimeStamp($timeInMili) - { - $seconds = $timeInMili / 1000; - $remainder = round($seconds - ($seconds >> 0), 3) * 1000; - $formatted_remainder = sprintf("%03d", $remainder); - return gmdate('H:i:s,', $seconds).$formatted_remainder; - } - - /** - * @param $time - * @return string - */ - public function parseCaptionTime($time) - { - $time = str_replace(',','.',$time); - $captionTime = parent::parseCaptionTime($time); - return $captionTime; - } - - /** - * @param string $content - * @param string $toAppend - * @return string - */ - public function merge($content, $toAppend) - { - return $content . $toAppend; - } -} \ No newline at end of file diff --git a/plugins/content/caption/base/lib/kCaptionsContentManager.php b/plugins/content/caption/base/lib/kCaptionsContentManager.php index a263a88655e..a65081c4a11 100644 --- a/plugins/content/caption/base/lib/kCaptionsContentManager.php +++ b/plugins/content/caption/base/lib/kCaptionsContentManager.php @@ -172,13 +172,14 @@ public static function addParseMultiLanguageCaptionAssetJob($captionAsset, $file return kJobsManager::addJob($batchJob, $jobData, $jobType); } - /** + /*** * @param $captionAsset - * @param $fileLocation - * @param null $syncKey + * @param $fromType + * @param $toType * @return BatchJob + * @throws Exception */ - public static function addParseSccCaptionAssetJob($captionAsset) + public static function addConvertCaptionAssetJob($captionAsset, $fromType, $toType) { $batchJob = new BatchJob(); @@ -189,12 +190,14 @@ public static function addParseSccCaptionAssetJob($captionAsset) $id = $captionAsset->getId(); $entryId = $captionAsset->getEntryId(); - $jobData = new kParseSccCaptionAssetJobData(); - $jobData->setSccCaptionAssetId($id); + $jobData = new kConvertCaptionAssetJobData(); + $jobData->setCaptionAssetId($id); $jobData->setFileLocation($fileLocation); $jobData->setFileEncryptionKey($fileSync->getEncryptionKey()); + $jobData->setFromType($fromType); + $jobData->setToType($toType); - $jobType = CaptionPlugin::getBatchJobTypeCoreValue(ParseSccCaptionAssetBatchType::PARSE_SCC_CAPTION_ASSET); + $jobType = CaptionPlugin::getBatchJobTypeCoreValue(ConvertCaptionAssetBatchType::CONVERT_CAPTION_ASSET); $batchJob->setObjectType(BatchJobObjectType::ASSET); $batchJob->setEntryId($entryId); $batchJob->setPartnerId($captionAsset->getPartnerId()); diff --git a/plugins/content/caption/base/lib/model/enums/CaptionType.php b/plugins/content/caption/base/lib/model/enums/CaptionType.php index dfa6c05b0c7..e2d229dab1e 100644 --- a/plugins/content/caption/base/lib/model/enums/CaptionType.php +++ b/plugins/content/caption/base/lib/model/enums/CaptionType.php @@ -9,4 +9,5 @@ interface CaptionType extends BaseEnum const DFXP = 2; const WEBVTT = 3; const CAP = 4; + const SCC = 5; } \ No newline at end of file diff --git a/plugins/content/caption/base/lib/model/enums/ParseSccCaptionAssetBatchType.php b/plugins/content/caption/base/lib/model/enums/ConvertCaptionAssetBatchType.php similarity index 61% rename from plugins/content/caption/base/lib/model/enums/ParseSccCaptionAssetBatchType.php rename to plugins/content/caption/base/lib/model/enums/ConvertCaptionAssetBatchType.php index 3db5470d63b..757b2e2153d 100644 --- a/plugins/content/caption/base/lib/model/enums/ParseSccCaptionAssetBatchType.php +++ b/plugins/content/caption/base/lib/model/enums/ConvertCaptionAssetBatchType.php @@ -3,14 +3,14 @@ * @package plugins.caption * @subpackage api.enum */ - class ParseSccCaptionAssetBatchType implements IKalturaPluginEnum, BatchJobType + class ConvertCaptionAssetBatchType implements IKalturaPluginEnum, BatchJobType { - const PARSE_SCC_CAPTION_ASSET = 'parsescccaptionasset'; + const CONVERT_CAPTION_ASSET = 'convertcaptionasset'; public static function getAdditionalValues() { return array( - 'PARSE_SCC_CAPTION_ASSET' => self::PARSE_SCC_CAPTION_ASSET + 'CONVERT_CAPTION_ASSET' => self::CONVERT_CAPTION_ASSET ); } diff --git a/plugins/content/caption/base/lib/model/kParseSccCaptionAssetJobData.php b/plugins/content/caption/base/lib/model/kConvertCaptionAssetJobData.php similarity index 53% rename from plugins/content/caption/base/lib/model/kParseSccCaptionAssetJobData.php rename to plugins/content/caption/base/lib/model/kConvertCaptionAssetJobData.php index c1fc8e3d8e2..da5551cc7db 100644 --- a/plugins/content/caption/base/lib/model/kParseSccCaptionAssetJobData.php +++ b/plugins/content/caption/base/lib/model/kConvertCaptionAssetJobData.php @@ -3,12 +3,12 @@ * @package plugins.caption * @subpackage model.data */ -class kParseSccCaptionAssetJobData extends kJobData +class kConvertCaptionAssetJobData extends kJobData { /** * @var string */ - private $sccCaptionAssetId; + private $captionAssetId; /** * @var string @@ -20,20 +20,30 @@ class kParseSccCaptionAssetJobData extends kJobData */ private $fileEncryptionKey; + /** + * @var string + */ + private $fromType; + + /** + * @var string + */ + private $toType; + /** * @return string $multiLanaguageCaptionAssetId */ - public function getSccCaptionAssetId() + public function getCaptionAssetId() { - return $this->sccCaptionAssetId; + return $this->captionAssetId; } /** * @param string $captionAssetId */ - public function setSccCaptionAssetId($captionAssetId) + public function setCaptionAssetId($captionAssetId) { - $this->sccCaptionAssetId = $captionAssetId; + $this->captionAssetId = $captionAssetId; } /** @@ -68,4 +78,36 @@ public function setFileEncryptionKey($fileEncryptionKey) $this->fileEncryptionKey = $fileEncryptionKey; } + /** + * @return string $fromType + */ + public function getFromType() + { + return $this->fromType; + } + + /** + * @param string $fromType + */ + public function setFromType($fromType) + { + $this->fromType= $fromType; + } + + /** + * @return string $toType + */ + public function getToType() + { + return $this->toType; + } + + /** + * @param string $toType + */ + public function setToType($toType) + { + $this->toType= $toType; + } + } \ No newline at end of file diff --git a/plugins/content/caption/base/scripts/convertcaption.py b/plugins/content/caption/base/scripts/convertcaption.py new file mode 100644 index 00000000000..266c4e402ef --- /dev/null +++ b/plugins/content/caption/base/scripts/convertcaption.py @@ -0,0 +1,57 @@ +import sys, getopt, pycaption +import io + +def main(argv): + inputfile = '' + inputType = '' + outputType = '' + + try: + opts, args = getopt.getopt(argv,"h:i:f:t:") + except getopt.GetoptError: + print( 'test.py -i -f -t ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print( 'test.py -i -f -t ') + sys.exit() + elif opt in ("-i", "--ifile"): + inputfile=arg + elif opt in ("-f", "--sfile"): + inputType=arg + elif opt in ("-t", "--tfile"): + outputType=arg + + if inputType == outputType: + print( 'Error: input type and output type are same format') + sys.exit(1) + + with io.open(inputfile) as f: + str1 = f.read() + + if inputType.lower() in ['scc']: + c = pycaption.SCCReader().read(str1) + elif inputType.lower() in ['srt']: + c = pycaption.SRTReader().read(str1) + elif inputType.lower() in ['dfxp']: + c = pycaption.DFXPReader().read(str1) + elif inputType.lower() in ['webvtt']: + c = pycaption.WebVTTReader().read(str1) + else: + print('Error: invalid input type. allowed') + sys.exit(1) + + if outputType.lower() in ['scc']: + print (pycaption.SCCWriter().write(c)) + elif outputType.lower() in ['srt']: + print (pycaption.SRTWriter().write(c)) + elif outputType.lower() in ['dfxp']: + print (pycaption.DFXPWriter().write(c)) + elif outputType.lower() in ['webvtt']: + print (pycaption.WebVTTWriter().write(c)) + else: + print('Error: invalid output type. allowed') + sys.exit(1) + +if __name__ == "__main__": + main(sys.argv[1:]) \ No newline at end of file diff --git a/plugins/content/caption/base/services/CaptionAssetService.php b/plugins/content/caption/base/services/CaptionAssetService.php index 3f22d6e1c8f..7ee3c292d23 100644 --- a/plugins/content/caption/base/services/CaptionAssetService.php +++ b/plugins/content/caption/base/services/CaptionAssetService.php @@ -120,10 +120,10 @@ function setContentAction($id, KalturaContentResource $contentResource) $kContentResource = $contentResource->toObject(); $this->attachContentResource($dbCaptionAsset, $kContentResource); - if (strtolower($dbCaptionAsset->getFileExt()) === self::SSC_CAPTION_FILE_EXT) + if ($dbCaptionAsset->getContainerFormat() == CaptionType::SCC) { - //start scc batch Job. - kCaptionsContentManager::addParseSccCaptionAssetJob($dbCaptionAsset); + //start convert caption batch Job. + kCaptionsContentManager::addConvertCaptionAssetJob($dbCaptionAsset, CaptionType::SCC, CaptionType::SRT); $captionAsset = new KalturaCaptionAsset(); $captionAsset->fromObject($dbCaptionAsset, $this->getResponseProfile()); From 3ce7a71e743807aa918f9924256644eed650187c Mon Sep 17 00:00:00 2001 From: TalDubovKaltura Date: Tue, 9 Jul 2019 16:48:14 +0300 Subject: [PATCH 062/103] PSVAMB-7785 - Add the instructions for updating the admin configuration --- release-notes.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/release-notes.md b/release-notes.md index 27e6d98406f..7768aa1a756 100644 --- a/release-notes.md +++ b/release-notes.md @@ -1,3 +1,24 @@ +# Orion 15.3.0 # + +## Add permission in the admin console to include live entries in the mrss feed ## + +- Issue Type: Feature +- Issue ID: PSVAMB-7785 + +### configuration ### + Add the following to admin.ini: + + moduls.feedWithLiveEntries.enabled = true + moduls.feedWithLiveEntries.permissionType = 2 + moduls.feedWithLiveEntries.label = "Include live entries in feed" + moduls.feedWithLiveEntries.permissionName = FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED + moduls.feedWithLiveEntries.basePermissionType = + moduls.feedWithLiveEntries.basePermissionName = + moduls.feedWithLiveEntries.group = GROUP_ENABLE_DISABLE_FEATURES + +### Deployment scripts ### + None + # Orion 15.1.0 # ## New thumbnail API ## - Issue Type: Epic From b5b056a7fb6db10c714ca4996fe5cea682bbfcda Mon Sep 17 00:00:00 2001 From: root Date: Tue, 9 Jul 2019 18:30:06 +0300 Subject: [PATCH 063/103] PLAT-9902 - convert caption script --- .../caption/base/scripts/convertcaption.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/plugins/content/caption/base/scripts/convertcaption.py b/plugins/content/caption/base/scripts/convertcaption.py index 266c4e402ef..e4a8e07b701 100644 --- a/plugins/content/caption/base/scripts/convertcaption.py +++ b/plugins/content/caption/base/scripts/convertcaption.py @@ -1,5 +1,4 @@ -import sys, getopt, pycaption -import io +import sys, getopt, pycaption, io def main(argv): inputfile = '' @@ -28,26 +27,28 @@ def main(argv): with io.open(inputfile) as f: str1 = f.read() + inputValue = inputType.lower() - if inputType.lower() in ['scc']: + if inputValue == 'scc': c = pycaption.SCCReader().read(str1) - elif inputType.lower() in ['srt']: + elif inputValue == 'srt': c = pycaption.SRTReader().read(str1) - elif inputType.lower() in ['dfxp']: + elif inputValue == 'dfxp': c = pycaption.DFXPReader().read(str1) - elif inputType.lower() in ['webvtt']: + elif inputValue == 'webvtt': c = pycaption.WebVTTReader().read(str1) else: print('Error: invalid input type. allowed') sys.exit(1) - if outputType.lower() in ['scc']: + outputValue = outputType.lower() + if outputValue == 'scc': print (pycaption.SCCWriter().write(c)) - elif outputType.lower() in ['srt']: + elif outputValue == 'srt': print (pycaption.SRTWriter().write(c)) - elif outputType.lower() in ['dfxp']: + elif outputValue == 'dfxp': print (pycaption.DFXPWriter().write(c)) - elif outputType.lower() in ['webvtt']: + elif outputValue == 'webvtt': print (pycaption.WebVTTWriter().write(c)) else: print('Error: invalid output type. allowed') From d424ff3e3f04ab389649a387499202448451952f Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Wed, 10 Jul 2019 13:47:02 +0300 Subject: [PATCH 064/103] PLAT-9942:Add 2FA support to AdminKuser->updatePassword and user->updateLoginData actions --- alpha/lib/model/UserLoginDataPeer.php | 34 ++++++++++++++++++++++++--- api_v3/lib/KalturaBaseUserService.php | 15 ++++++++++-- api_v3/services/AdminUserService.php | 7 ++++-- api_v3/services/UserService.php | 7 ++++-- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/alpha/lib/model/UserLoginDataPeer.php b/alpha/lib/model/UserLoginDataPeer.php index 6beafa8b4f4..2c8ad8865ca 100644 --- a/alpha/lib/model/UserLoginDataPeer.php +++ b/alpha/lib/model/UserLoginDataPeer.php @@ -82,7 +82,7 @@ private static function emailResetPassword($partner_id, $cms_email, $user_name, ); } - public static function updateLoginData($oldLoginEmail, $oldPassword, $newLoginEmail = null, $newPassword = null, $newFirstName = null, $newLastName = null) + public static function updateLoginData($oldLoginEmail, $oldPassword, $newLoginEmail = null, $newPassword = null, $newFirstName = null, $newLastName = null, $otp = null) { // if email is null, no need to do any DB queries if (!$oldLoginEmail) { @@ -120,7 +120,7 @@ public static function updateLoginData($oldLoginEmail, $oldPassword, $newLoginEm } self::checkPasswordValidation ( $newPassword, $loginData ); - + self::validate2FA($loginData, $otp); // update password if requested if ($newPassword && $newPassword != $oldPassword) { $password = $loginData->resetPassword($newPassword, $oldPassword); @@ -153,7 +153,35 @@ public static function updateLoginData($oldLoginEmail, $oldPassword, $newLoginEm return $loginData; } - + + protected static function validate2FA($loginData, $otp) + { + if (self::isUserBelongsToPartnerWith2FA($loginData) && kuserPeer::getAdminUser($loginData->getConfigPartnerId(), $loginData)) + { + if(!$otp) + { + throw new kUserException ('otp is missing', kUserException::MISSING_OTP); + } + $result = authenticationUtils::verify2FACode($loginData, $otp); + if (!$result) + { + throw new kUserException ('otp is invalid', kUserException::INVALID_OTP); + } + } + } + + protected static function isUserBelongsToPartnerWith2FA($loginData) + { + kuserPeer::setUseCriteriaFilter(false); + $dbUser = kuserPeer::getKuserByPartnerAndUid($loginData->getConfigPartnerId(), $loginData->getLoginEmail(), true); + kuserPeer::setUseCriteriaFilter(true); + if (!$dbUser) + { + throw new KalturaAPIException(KalturaErrors::INVALID_USER_ID); + } + return $loginData->isTwoFactorAuthenticationRequired($dbUser); + } + public static function checkPasswordValidation($newPassword, $loginData) { // check that new password structure is valid if ($newPassword && diff --git a/api_v3/lib/KalturaBaseUserService.php b/api_v3/lib/KalturaBaseUserService.php index bb8a5b7ff01..4f9b5e40505 100644 --- a/api_v3/lib/KalturaBaseUserService.php +++ b/api_v3/lib/KalturaBaseUserService.php @@ -40,6 +40,7 @@ public function initService($serviceId, $serviceName, $actionName) * @param string $newPassword * @param string $newFirstName Optional, provide only when you want to update the first name * @param string $newLastName Optional, provide only when you want to update the last name + * @param string $otp the user's one-time password * * @throws KalturaErrors::INVALID_FIELD_VALUE * @throws KalturaErrors::LOGIN_DATA_NOT_FOUND @@ -47,8 +48,10 @@ public function initService($serviceId, $serviceName, $actionName) * @throws KalturaErrors::PASSWORD_STRUCTURE_INVALID * @throws KalturaErrors::PASSWORD_ALREADY_USED * @throws KalturaErrors::LOGIN_ID_ALREADY_USED + * @throws KalturaErrors::INVALID_OTP + * @throws KalturaErrors::MISSING_OTP */ - protected function updateLoginDataImpl( $email , $password , $newEmail = "" , $newPassword = "", $newFirstName = null, $newLastName = null) + protected function updateLoginDataImpl( $email , $password , $newEmail = "" , $newPassword = "", $newFirstName = null, $newLastName = null, $otp = null) { KalturaResponseCacher::disableCache(); @@ -61,7 +64,7 @@ protected function updateLoginDataImpl( $email , $password , $newEmail = "" , $n } try { - UserLoginDataPeer::updateLoginData ( $email , $password, $newEmail, $newPassword, $newFirstName, $newLastName); + UserLoginDataPeer::updateLoginData ( $email , $password, $newEmail, $newPassword, $newFirstName, $newLastName, $otp); } catch (kUserException $e) { $code = $e->getCode(); @@ -90,6 +93,14 @@ protected function updateLoginDataImpl( $email , $password , $newEmail = "" , $n else if ($code == kUserException::LOGIN_ID_ALREADY_USED) { throw new KalturaAPIException(KalturaErrors::LOGIN_ID_ALREADY_USED); } + else if ($code === kUserException::INVALID_OTP) + { + throw new KalturaAPIException(KalturaErrors::INVALID_OTP); + } + else if ($code === kUserException::MISSING_OTP) + { + throw new KalturaAPIException(KalturaErrors::MISSING_OTP); + } throw $e; } } diff --git a/api_v3/services/AdminUserService.php b/api_v3/services/AdminUserService.php index dc43d791af6..78621d47632 100644 --- a/api_v3/services/AdminUserService.php +++ b/api_v3/services/AdminUserService.php @@ -60,6 +60,7 @@ private function throwTranslatedException(KalturaAPIException $e) * @param string $password * @param string $newEmail Optional, provide only when you want to update the email * @param string $newPassword + * @param string $otp the user's one-time password * @return KalturaAdminUser * @ksIgnored * @@ -70,14 +71,16 @@ private function throwTranslatedException(KalturaAPIException $e) * @throws KalturaErrors::PASSWORD_ALREADY_USED * @throws KalturaErrors::INVALID_FIELD_VALUE * @throws KalturaErrors::LOGIN_ID_ALREADY_USED + * @throws KalturaErrors::INVALID_OTP + * @throws KalturaErrors::MISSING_OTP * * @deprecated */ - public function updatePasswordAction( $email , $password , $newEmail = "" , $newPassword = "" ) + public function updatePasswordAction( $email , $password , $newEmail = "" , $newPassword = "", $otp = null) { try { - parent::updateLoginDataImpl($email, $password, $newEmail, $newPassword); + parent::updateLoginDataImpl($email, $password, $newEmail, $newPassword, null, null, $otp); // copy required parameters to a KalturaAdminUser object for backward compatibility $adminUser = new KalturaAdminUser(); diff --git a/api_v3/services/UserService.php b/api_v3/services/UserService.php index f916a8eaf1e..2f28d5f1d56 100644 --- a/api_v3/services/UserService.php +++ b/api_v3/services/UserService.php @@ -330,6 +330,7 @@ public function loginByLoginIdAction($loginId, $password, $partnerId = null, $ex * @param string $newPassword Optional, The user's new password * @param string $newFirstName Optional, The user's new first name * @param string $newLastName Optional, The user's new last name + * @param string $otp the user's one-time password * @ksIgnored * * @throws KalturaErrors::INVALID_FIELD_VALUE @@ -338,10 +339,12 @@ public function loginByLoginIdAction($loginId, $password, $partnerId = null, $ex * @throws KalturaErrors::PASSWORD_STRUCTURE_INVALID * @throws KalturaErrors::PASSWORD_ALREADY_USED * @throws KalturaErrors::LOGIN_ID_ALREADY_USED + * @throws KalturaErrors::INVALID_OTP + * @throws KalturaErrors::MISSING_OTP */ - public function updateLoginDataAction( $oldLoginId , $password , $newLoginId = "" , $newPassword = "", $newFirstName = null, $newLastName = null) + public function updateLoginDataAction( $oldLoginId , $password , $newLoginId = "" , $newPassword = "", $newFirstName = null, $newLastName = null, $otp = null) { - return parent::updateLoginDataImpl($oldLoginId , $password , $newLoginId, $newPassword, $newFirstName, $newLastName); + return parent::updateLoginDataImpl($oldLoginId , $password , $newLoginId, $newPassword, $newFirstName, $newLastName, $otp); } /** From 68a7d1a20a865bccd3d267e45a1baeb41f26f938 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 10 Jul 2019 14:59:33 +0300 Subject: [PATCH 065/103] PLAT-9889: code review remarks --- .../items/KalturaESearchAggregationItem.php | 17 +++++++++++------ .../KalturaESearchCategoryAggregationItem.php | 5 ----- .../KalturaESearchCuepointsAggregationItem.php | 4 ++-- .../KalturaESearchEntryAggregationItem.php | 1 - .../KalturaESearchMetadataAggregationItem.php | 4 +++- .../KalturaESearchAggregationBucketsArray.php | 2 +- .../KalturaESearchAggregationResponse.php | 8 ++++++-- .../items/ESearchAggregationItem.php | 3 +++ .../lib/model/search/kBaseSearch.php | 2 +- .../elastic_search/services/ESearchService.php | 4 ++-- 10 files changed, 29 insertions(+), 21 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php index cd437a841ea..0090785f667 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchAggregationItem.php @@ -11,6 +11,8 @@ abstract class KalturaESearchAggregationItem extends KalturaObject */ public $size; + public $fieldName; + private static $map_between_objects = array( 'size', 'fieldName' @@ -25,18 +27,21 @@ public function getMapBetweenObjects() public function toObject($object_to_fill = null, $props_to_skip = array()) { - $fieldEnumMap = $this->getFieldEnumMap(); - if(isset($fieldEnumMap[$this->fieldName])) + if($object_to_fill) { - $coreFieldName = $fieldEnumMap[$this->fieldName]; - $object_to_fill->setFieldName($coreFieldName); - $props_to_skip[] = 'fieldName'; + $fieldEnumMap = $this->getFieldEnumMap(); + if (isset($fieldEnumMap[$this->fieldName])) + { + $coreFieldName = $fieldEnumMap[$this->fieldName]; + $object_to_fill->setFieldName($coreFieldName); + $props_to_skip[] = 'fieldName'; + } } return parent::toObject($object_to_fill, $props_to_skip); } - abstract public function coreToApiResponse($coreResponse,$fieldName = null); + abstract public function coreToApiResponse($coreResponse, $fieldName = null); public function validateForUsage($sourceObject, $propertiesToSkip = array()) { diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php index 15f2f27225d..75f530da9ee 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCategoryAggregationItem.php @@ -26,11 +26,6 @@ public function getFieldEnumMap() return array (KalturaESearchCategoryAggregateByFieldName::CATEGORY_NAME => ESearchCategoryAggregationFieldName::CATEGORY_NAME); } - protected function fixCategoryName($coreName) - { - - } - public function coreToApiResponse($coreResponse, $fieldName = null) { $agg = new KalturaESearchAggregationResponseItem(); diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php index 64743329f2d..ea193516a02 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchCuepointsAggregationItem.php @@ -43,12 +43,12 @@ public function coreToApiResponse($coreResponse, $fieldName = null) $responseBucket->fromArray($bucket); if($fieldName === ESearchCuePointsAggregationFieldName::TYPE) { - $responseBucket->value = kPluginableEnumsManager::coreToApi('CuePointType',$responseBucket->value); + $responseBucket->value = kPluginableEnumsManager::coreToApi('CuePointType' ,$responseBucket->value); } $bucketsArray[] = $responseBucket; } } $agg->buckets = $bucketsArray; - return ($agg); + return array($agg); } } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php index 9f5ff85850c..fc9c51abd03 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchEntryAggregationItem.php @@ -41,7 +41,6 @@ public function coreToApiResponse($coreResponse, $fieldName = null) $buckets = $coreResponse[ESearchAggregations::BUCKETS]; if ($buckets) { - foreach ($buckets as $bucket) { $responseBucket = new KalturaESearchAggregationBucket(); diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php index bdd17b20684..e89d0e375a7 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -12,6 +12,8 @@ class KalturaESearchMetadataAggregationItem extends KalturaESearchAggregationIte */ public $fieldName; + const FIELD_NAME = 3; + public function toObject($object_to_fill = null, $props_to_skip = array()) { if (!$object_to_fill) @@ -29,7 +31,7 @@ public function getFieldEnumMap() protected function getMetadataFieldNameFromXpath($xpath) { $token = explode("'", $xpath); - return $token[3]; + return $token[self::FIELD_NAME]; } public function coreToApiResponse($coreResponse, $fieldName=null) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php index 612c81ef0b0..b7ebec6c068 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationBucketsArray.php @@ -7,6 +7,6 @@ class KalturaESearchAggregationBucketsArray extends KalturaTypedArray { public function __construct() { - parent::__construct("KalturaESearchAggregationBucket"); + parent::__construct("KalturaESearchAggregationBucket"); } } \ No newline at end of file diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php index 2e541a3af80..a8365a0efd7 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/response/KalturaESearchAggregationResponse.php @@ -16,7 +16,7 @@ protected function mapAggregationCoreObjects($coreObject) $map = array(ESearchCategoryAggregationItem::KEY => 'KalturaESearchCategoryAggregationItem', ESearchCuepointsAggregationItem::KEY => 'KalturaESearchCuepointsAggregationItem', ESearchMetadataAggregationItem::KEY => 'KalturaESearchMetadataAggregationItem', - ESearchEntryAggregationItem::KEY => 'KalturaESearchEntryAggregationItem'); + ESearchEntryAggregationItem::KEY => 'KalturaESearchEntryAggregationItem'); $ret = isset($map[$coreObject]) ? $map[$coreObject] : null; return $ret; } @@ -28,8 +28,12 @@ public function resultToApi($aggregationResults) { list ($responseObject, $fieldName) = $this->getApiObjects($key); $itemObjectName = $this->mapAggregationCoreObjects($responseObject); + if(!$itemObjectName) + { + continue; + } $objectItemHandler = new $itemObjectName(); - $aggsResponses = $objectItemHandler->coreToApiResponse($response,$fieldName); + $aggsResponses = $objectItemHandler->coreToApiResponse($response, $fieldName); foreach ($aggsResponses as $aggsResponse) { $aggs[] = $aggsResponse; diff --git a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php index 926650e71cd..25f236f93fc 100644 --- a/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/model/aggregations/items/ESearchAggregationItem.php @@ -13,6 +13,9 @@ abstract class ESearchAggregationItem extends BaseObject */ protected $size; + /** + * @var string + */ protected $fieldName; /** diff --git a/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php b/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php index ca6e34abef5..3752939b480 100644 --- a/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php +++ b/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php @@ -76,7 +76,7 @@ protected function initAggregations(ESearchAggregations $aggregations = null) foreach ($aggregations->getAggregations() as $aggregation) { /* var $aggregation ESearchAggregationItem */ - $aggregationKey = $aggregation->getAggregationKey().':'.$aggregation->getFieldName(); + $aggregationKey = $aggregation->getAggregationKey() . ':' . $aggregation->getFieldName(); $aggs[$aggregationKey] = $aggregation->getAggregationCommand(); } if($aggs) diff --git a/plugins/search/providers/elastic_search/services/ESearchService.php b/plugins/search/providers/elastic_search/services/ESearchService.php index d9ceb848006..2c4d78b5ea2 100644 --- a/plugins/search/providers/elastic_search/services/ESearchService.php +++ b/plugins/search/providers/elastic_search/services/ESearchService.php @@ -75,7 +75,7 @@ function searchUserAction(KalturaESearchUserParams $searchParams, KalturaPager $ public function entryExportToCsvAction (KalturaMediaEsearchExportToCsvJobData $data) { if(!$data->userName || !$data->userMail) - throw new KalturaAPIException(APIErrors::USER_EMAIL_NOT_FOUND); + throw new KalturaAPIException(APIErrors::USER_EMAIL_NOT_FOUND, ''); $kJobdData = $data->toObject(new kMediaEsearchExportToCsvJobData()); @@ -92,7 +92,7 @@ public function entryExportToCsvAction (KalturaMediaEsearchExportToCsvJobData $d */ protected function initAndSearch($coreSearchObject, $searchParams, $pager) { - list($coreSearchOperator, $objectStatusesArr, $objectId, $kPager, $coreOrder ,$aggregations) = + list($coreSearchOperator, $objectStatusesArr, $objectId, $kPager, $coreOrder, $aggregations) = self::initSearchActionParams($searchParams, $pager); $elasticResults = $coreSearchObject->doSearch($coreSearchOperator, $kPager, $objectStatusesArr, $objectId, $coreOrder, $aggregations); From 3482c16da3ec5d50511cffff05157d94eecd1fd3 Mon Sep 17 00:00:00 2001 From: moshemaorkaltura Date: Wed, 10 Jul 2019 15:07:38 +0300 Subject: [PATCH 066/103] PLAT-9889: code review remarks --- .../elastic_search/lib/elasticSearchUtils.php | 447 +++++++++--------- 1 file changed, 223 insertions(+), 224 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php b/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php index 77b318b67e0..14a9a158a8b 100644 --- a/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php +++ b/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php @@ -1,224 +1,223 @@ -', '
', - '
', '

', '
', '
', '

', '

', '

'); - - protected static $elastic_negation_booleans = array('0', 'false', 'off', 'no'); - - /** - * return the analyzed language field name - * @param $language - * @param $fieldName - * @param $delimiter - * @return null|string - */ - public static function getAnalyzedFieldName($language, $fieldName, $delimiter) - { - $fieldMap = array( - 'english' => 'english', - 'arabic' => 'arabic', - 'brazilian' => 'brazilian', - 'chinese' => 'cjk', - 'korean' => 'cjk', - 'japanese' => 'cjk', - 'danish' => 'danish', - 'dutch' => 'dutch', - 'finnish' => 'finnish', - 'french' => 'french', - 'german' => 'german', - 'greek' => 'greek', - 'hindi' => 'hindi', - 'indonesian' => 'indonesian', - 'italian' => 'italian', - 'norwegian' => 'norwegian', - 'portuguese' => 'portuguese', - 'russian' => 'russian', - 'spanish' => 'spanish', - 'swedish' => 'swedish', - 'turkish' => 'turkish', - 'thai' => 'thai', - ); - - $language = strtolower($language); - if(isset($fieldMap[$language])) - return $fieldName.$delimiter.$fieldMap[$language]; - - return null; - } - - public static function getSynonymFieldName($language, $fieldName, $delimiter) - { - $fieldMap = array( - 'english' => kESearchQueryManager::SYNONYM_FIELD_SUFFIX, - ); - - $language = strtolower($language); - if(isset($fieldMap[$language])) - return $fieldName.$delimiter.$fieldMap[$language]; - } - - public static function formatGroupIdCreationMode($groupId, $creationMode) - { - return sprintf("g%scm%s", $groupId, $creationMode); - } - - public static function formatPartnerStatus($partnerId, $status) - { - return sprintf("p%ss%s", $partnerId, $status); - } - - public static function formatCategoryIdStatus($categoryId, $status) - { - return sprintf("c%ss%s", $categoryId, $status); - } - - public static function formatCategoryFullIdStatus($categoryId, $status) - { - return sprintf("s%sfid>%s", $status, $categoryId); - } - - public static function formatParentCategoryIdStatus($categoryId, $status) - { - return sprintf("p%ss%s", $categoryId, $status); - } - - public static function formatCategoryNameStatus($categoryName, $status) - { - return sprintf("s%sc>%s", $status, $categoryName); - } - - public static function reverseFormatCategoryNameStatus($categoryNameStatus) - { - list($status , $categoryName) = explode ('>',$categoryNameStatus); - $status = substr($status,1,strlen ($status)-1 ); - return array($status, $categoryName); - } - - - - public static function formatParentCategoryNameStatus($categoryName, $status) - { - return sprintf("s%sp>%s", $status, $categoryName); - } - - public static function formatCategoryEntryStatus($status) - { - return sprintf("ces%s", $status); - } - - public static function formatCategoryUserPermissionLevel($userId, $permissionLevel) - { - return sprintf("uid%spl%s", $userId, $permissionLevel); - } - - public static function formatCategoryUserPermissionName($userId, $permissionName) - { - return sprintf("uid%spn%s", $userId, $permissionName); - } - - public static function getCategoryUserAllPermissionLevels($userId) - { - $formatPermissions = array(); - $permissionLevelReflection = new ReflectionClass('CategoryKuserPermissionLevel'); - $permissionLevels = $permissionLevelReflection->getConstants(); - foreach ($permissionLevels as $permissionLevel) - $formatPermissions[] = elasticSearchUtils::formatCategoryUserPermissionLevel($userId, $permissionLevel); - - return $formatPermissions; - } - - public static function getCategoryUserAllPermissionNames($userId) - { - $formatPermissions = array(); - $permissionNames = array( - PermissionName::CATEGORY_SUBSCRIBE, - PermissionName::CATEGORY_CONTRIBUTE, - PermissionName::CATEGORY_MODERATE, - PermissionName::CATEGORY_EDIT, - PermissionName::CATEGORY_VIEW, - ); - - foreach ($permissionNames as $permissionName) - $formatPermissions[] = elasticSearchUtils::formatCategoryUserPermissionName($userId, $permissionName); - - return $formatPermissions; - } - - public static function formatSearchTerm($searchTerm) - { - //remove extra spaces - $term = preg_replace('/\s+/', ' ', $searchTerm); - //lowercase and trim - $term = strtolower($term); - $term = trim($term); - return $term; - } - - public static function isMaster($elasticClient, $elasticHostName) - { - $masterInfo = $elasticClient->getMasterInfo(); - if(isset($masterInfo[0]['node']) && $masterInfo[0]['node'] == $elasticHostName) - return true; - - return false; - } - - public static function cleanEmptyValues(&$body) - { - foreach ($body as $key => $value) - { - if(is_null($value) || $value === '') - unset($body[$key]); - if(is_array($value) && ( count($value) == 0 || ( (count($value) == 1 && (isset($value[0])) && $value[0] === '' ) ) )) - unset($body[$key]); - } - } - - public static function getNumOfFragmentsByConfigKey($highlightConfigKey) - { - $highlightConfig = kConf::get('highlights', 'elastic'); - //return null to use elastic default num of fragments - $numOfFragments = isset($highlightConfig[$highlightConfigKey]) ? $highlightConfig[$highlightConfigKey] : null; - return $numOfFragments; - } - - public static function getBooleanValue($value) - { - return !in_array(self::formatSearchTerm($value), self::$elastic_negation_booleans); - } - - /** - * Go over the array and decode html and strip tags from all of its leafs - * @param array $cmd - */ - public static function prepareForInsertToElastic(&$cmd) - { - array_walk_recursive($cmd, array('elasticSearchUtils','prepareElasticLeafInput')); - } - - public static function prepareElasticLeafInput(&$value, $key) - { - if(is_string($value)) - { - self::filterHtmlFromLeaf($value); - $value = trim($value); - $value = @iconv('utf-8', 'utf-8//IGNORE', $value); - } - } - - public static function filterHtmlFromLeaf(&$value) - { - $value = html_entity_decode($value); - $value = str_replace(self::$html_chars_to_replace, " ", $value); - $value = strip_tags($value); - } - -} +', '
', + '
', '

', '
', '
', '

', '

', '

'); + + protected static $elastic_negation_booleans = array('0', 'false', 'off', 'no'); + + /** + * return the analyzed language field name + * @param $language + * @param $fieldName + * @param $delimiter + * @return null|string + */ + public static function getAnalyzedFieldName($language, $fieldName, $delimiter) + { + $fieldMap = array( + 'english' => 'english', + 'arabic' => 'arabic', + 'brazilian' => 'brazilian', + 'chinese' => 'cjk', + 'korean' => 'cjk', + 'japanese' => 'cjk', + 'danish' => 'danish', + 'dutch' => 'dutch', + 'finnish' => 'finnish', + 'french' => 'french', + 'german' => 'german', + 'greek' => 'greek', + 'hindi' => 'hindi', + 'indonesian' => 'indonesian', + 'italian' => 'italian', + 'norwegian' => 'norwegian', + 'portuguese' => 'portuguese', + 'russian' => 'russian', + 'spanish' => 'spanish', + 'swedish' => 'swedish', + 'turkish' => 'turkish', + 'thai' => 'thai', + ); + + $language = strtolower($language); + if(isset($fieldMap[$language])) + return $fieldName.$delimiter.$fieldMap[$language]; + + return null; + } + + public static function getSynonymFieldName($language, $fieldName, $delimiter) + { + $fieldMap = array( + 'english' => kESearchQueryManager::SYNONYM_FIELD_SUFFIX, + ); + + $language = strtolower($language); + if(isset($fieldMap[$language])) + return $fieldName.$delimiter.$fieldMap[$language]; + } + + public static function formatGroupIdCreationMode($groupId, $creationMode) + { + return sprintf("g%scm%s", $groupId, $creationMode); + } + + public static function formatPartnerStatus($partnerId, $status) + { + return sprintf("p%ss%s", $partnerId, $status); + } + + public static function formatCategoryIdStatus($categoryId, $status) + { + return sprintf("c%ss%s", $categoryId, $status); + } + + public static function formatCategoryFullIdStatus($categoryId, $status) + { + return sprintf("s%sfid>%s", $status, $categoryId); + } + + public static function formatParentCategoryIdStatus($categoryId, $status) + { + return sprintf("p%ss%s", $categoryId, $status); + } + + public static function formatCategoryNameStatus($categoryName, $status) + { + return sprintf("s%sc>%s", $status, $categoryName); + } + + public static function reverseFormatCategoryNameStatus($categoryNameStatus) + { + list($status , $categoryName) = explode ('>',$categoryNameStatus); + $status = substr($status,1,strlen ($status)-1 ); + return array($status, $categoryName); + } + + + public static function formatParentCategoryNameStatus($categoryName, $status) + { + return sprintf("s%sp>%s", $status, $categoryName); + } + + public static function formatCategoryEntryStatus($status) + { + return sprintf("ces%s", $status); + } + + public static function formatCategoryUserPermissionLevel($userId, $permissionLevel) + { + return sprintf("uid%spl%s", $userId, $permissionLevel); + } + + public static function formatCategoryUserPermissionName($userId, $permissionName) + { + return sprintf("uid%spn%s", $userId, $permissionName); + } + + public static function getCategoryUserAllPermissionLevels($userId) + { + $formatPermissions = array(); + $permissionLevelReflection = new ReflectionClass('CategoryKuserPermissionLevel'); + $permissionLevels = $permissionLevelReflection->getConstants(); + foreach ($permissionLevels as $permissionLevel) + $formatPermissions[] = elasticSearchUtils::formatCategoryUserPermissionLevel($userId, $permissionLevel); + + return $formatPermissions; + } + + public static function getCategoryUserAllPermissionNames($userId) + { + $formatPermissions = array(); + $permissionNames = array( + PermissionName::CATEGORY_SUBSCRIBE, + PermissionName::CATEGORY_CONTRIBUTE, + PermissionName::CATEGORY_MODERATE, + PermissionName::CATEGORY_EDIT, + PermissionName::CATEGORY_VIEW, + ); + + foreach ($permissionNames as $permissionName) + $formatPermissions[] = elasticSearchUtils::formatCategoryUserPermissionName($userId, $permissionName); + + return $formatPermissions; + } + + public static function formatSearchTerm($searchTerm) + { + //remove extra spaces + $term = preg_replace('/\s+/', ' ', $searchTerm); + //lowercase and trim + $term = strtolower($term); + $term = trim($term); + return $term; + } + + public static function isMaster($elasticClient, $elasticHostName) + { + $masterInfo = $elasticClient->getMasterInfo(); + if(isset($masterInfo[0]['node']) && $masterInfo[0]['node'] == $elasticHostName) + return true; + + return false; + } + + public static function cleanEmptyValues(&$body) + { + foreach ($body as $key => $value) + { + if(is_null($value) || $value === '') + unset($body[$key]); + if(is_array($value) && ( count($value) == 0 || ( (count($value) == 1 && (isset($value[0])) && $value[0] === '' ) ) )) + unset($body[$key]); + } + } + + public static function getNumOfFragmentsByConfigKey($highlightConfigKey) + { + $highlightConfig = kConf::get('highlights', 'elastic'); + //return null to use elastic default num of fragments + $numOfFragments = isset($highlightConfig[$highlightConfigKey]) ? $highlightConfig[$highlightConfigKey] : null; + return $numOfFragments; + } + + public static function getBooleanValue($value) + { + return !in_array(self::formatSearchTerm($value), self::$elastic_negation_booleans); + } + + /** + * Go over the array and decode html and strip tags from all of its leafs + * @param array $cmd + */ + public static function prepareForInsertToElastic(&$cmd) + { + array_walk_recursive($cmd, array('elasticSearchUtils','prepareElasticLeafInput')); + } + + public static function prepareElasticLeafInput(&$value, $key) + { + if(is_string($value)) + { + self::filterHtmlFromLeaf($value); + $value = trim($value); + $value = @iconv('utf-8', 'utf-8//IGNORE', $value); + } + } + + public static function filterHtmlFromLeaf(&$value) + { + $value = html_entity_decode($value); + $value = str_replace(self::$html_chars_to_replace, " ", $value); + $value = strip_tags($value); + } + +} From 83d86e1fbf535b4a26f7a003f4e655f44a848dca Mon Sep 17 00:00:00 2001 From: moshemaorkaltura Date: Wed, 10 Jul 2019 15:09:05 +0300 Subject: [PATCH 067/103] PLAT-9889: code review remarks --- .../search/providers/elastic_search/lib/elasticSearchUtils.php | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php b/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php index 14a9a158a8b..12e5b5d8fd6 100644 --- a/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php +++ b/plugins/search/providers/elastic_search/lib/elasticSearchUtils.php @@ -102,7 +102,6 @@ public static function reverseFormatCategoryNameStatus($categoryNameStatus) return array($status, $categoryName); } - public static function formatParentCategoryNameStatus($categoryName, $status) { return sprintf("s%sp>%s", $status, $categoryName); From bce360f30d65ef7d8f64b093f68cbdd6afd280bb Mon Sep 17 00:00:00 2001 From: TalDubovKaltura Date: Wed, 10 Jul 2019 15:28:33 +0300 Subject: [PATCH 068/103] PSVAMB-7785 - Naming Change --- alpha/lib/enums/PermissionName.php | 2 +- api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php | 2 +- configurations/admin.template.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alpha/lib/enums/PermissionName.php b/alpha/lib/enums/PermissionName.php index 86db4697b8d..0ad1814e355 100644 --- a/alpha/lib/enums/PermissionName.php +++ b/alpha/lib/enums/PermissionName.php @@ -276,5 +276,5 @@ interface PermissionName extends BaseEnum const FEATURE_ANALYTICS_PERSISTENT_SESSION_ID = 'FEATURE_ANALYTICS_PERSISTENT_SESSION_ID'; const FEATURE_LIVE_ANALYTICS_DASHBOARD = 'FEATURE_LIVE_ANALYTICS_DASHBOARD'; - const FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED = 'FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED'; + const FEATURE_LIVE_ENTRIES_IN_FEED = 'FEATURE_LIVE_ENTRIES_IN_FEED'; } diff --git a/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php b/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php index 26e6b9af741..64e6dedf3b9 100644 --- a/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php +++ b/api_v3/lib/syndication/KalturaSyndicationFeedRenderer.php @@ -185,7 +185,7 @@ public function __construct($feedId, $feedProcessingKey = null, $ks = null, $sta { $mediaEntrytypes = array(entryType::MEDIA_CLIP, entryType::MIX); $partner = PartnerPeer::retrieveByPK($this->getSyndicationFeedDb()->getPartnerId()); - $liveEntriesView = $partner->getEnabledService(KalturaPermissionName::FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED); + $liveEntriesView = $partner->getEnabledService(KalturaPermissionName::FEATURE_LIVE_ENTRIES_IN_FEED); if($liveEntriesView) { $mediaEntrytypes[] = entryType::LIVE_STREAM; diff --git a/configurations/admin.template.ini b/configurations/admin.template.ini index 47bb996a151..fd86b0f40c1 100644 --- a/configurations/admin.template.ini +++ b/configurations/admin.template.ini @@ -1040,7 +1040,7 @@ moduls.limitAllowedActions.group = GROUP_ENABLE_DISABLE_FEATURES moduls.feedWithLiveEntries.enabled = true moduls.feedWithLiveEntries.permissionType = 2 moduls.feedWithLiveEntries.label = "Include live entries in feed" -moduls.feedWithLiveEntries.permissionName = FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED +moduls.feedWithLiveEntries.permissionName = FEATURE_LIVE_ENTRIES_IN_FEED moduls.feedWithLiveEntries.basePermissionType = moduls.feedWithLiveEntries.basePermissionName = moduls.feedWithLiveEntries.group = GROUP_ENABLE_DISABLE_FEATURES From 4c543c8a855a807b3a63a608a40b18a6773d037d Mon Sep 17 00:00:00 2001 From: TalDubovKaltura Date: Wed, 10 Jul 2019 15:30:47 +0300 Subject: [PATCH 069/103] PSVAMB-7785 - Naming Change - release notes update --- release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes.md b/release-notes.md index 7768aa1a756..6f60fc66426 100644 --- a/release-notes.md +++ b/release-notes.md @@ -11,7 +11,7 @@ moduls.feedWithLiveEntries.enabled = true moduls.feedWithLiveEntries.permissionType = 2 moduls.feedWithLiveEntries.label = "Include live entries in feed" - moduls.feedWithLiveEntries.permissionName = FEATURE_INCLUDE_LIVE_ENTRIES_IN_FEED + moduls.feedWithLiveEntries.permissionName = FEATURE_LIVE_ENTRIES_IN_FEED moduls.feedWithLiveEntries.basePermissionType = moduls.feedWithLiveEntries.basePermissionName = moduls.feedWithLiveEntries.group = GROUP_ENABLE_DISABLE_FEATURES From fe4f32448abc49cfcaa1e9ef5c9adc641a5554b5 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Wed, 10 Jul 2019 15:59:17 +0300 Subject: [PATCH 070/103] PLAT-9942:code review remarks --- alpha/lib/model/UserLoginDataPeer.php | 22 +++++++--------------- api_v3/services/AdminUserService.php | 2 +- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/alpha/lib/model/UserLoginDataPeer.php b/alpha/lib/model/UserLoginDataPeer.php index 2c8ad8865ca..8a4187c9e60 100644 --- a/alpha/lib/model/UserLoginDataPeer.php +++ b/alpha/lib/model/UserLoginDataPeer.php @@ -17,6 +17,8 @@ class UserLoginDataPeer extends BaseUserLoginDataPeer implements IRelatedObjectP { const KALTURAS_CMS_PASSWORD_RESET = 51; const LAST_LOGIN_TIME_UPDATE_INTERVAL = 600; // 10 Minutes + const OTP_MISSING = 'otp is missing'; + const OTP_INVALID = 'otp is invalid'; public static function generateNewPassword() { @@ -156,32 +158,22 @@ public static function updateLoginData($oldLoginEmail, $oldPassword, $newLoginEm protected static function validate2FA($loginData, $otp) { - if (self::isUserBelongsToPartnerWith2FA($loginData) && kuserPeer::getAdminUser($loginData->getConfigPartnerId(), $loginData)) + + $dbUser = kuserPeer::getAdminUser($loginData->getConfigPartnerId(), $loginData); + if ($dbUser && $loginData->isTwoFactorAuthenticationRequired($dbUser)) { if(!$otp) { - throw new kUserException ('otp is missing', kUserException::MISSING_OTP); + throw new kUserException (self::OTP_MISSING, kUserException::MISSING_OTP); } $result = authenticationUtils::verify2FACode($loginData, $otp); if (!$result) { - throw new kUserException ('otp is invalid', kUserException::INVALID_OTP); + throw new kUserException (self::OTP_INVALID, kUserException::INVALID_OTP); } } } - protected static function isUserBelongsToPartnerWith2FA($loginData) - { - kuserPeer::setUseCriteriaFilter(false); - $dbUser = kuserPeer::getKuserByPartnerAndUid($loginData->getConfigPartnerId(), $loginData->getLoginEmail(), true); - kuserPeer::setUseCriteriaFilter(true); - if (!$dbUser) - { - throw new KalturaAPIException(KalturaErrors::INVALID_USER_ID); - } - return $loginData->isTwoFactorAuthenticationRequired($dbUser); - } - public static function checkPasswordValidation($newPassword, $loginData) { // check that new password structure is valid if ($newPassword && diff --git a/api_v3/services/AdminUserService.php b/api_v3/services/AdminUserService.php index 78621d47632..1f422968594 100644 --- a/api_v3/services/AdminUserService.php +++ b/api_v3/services/AdminUserService.php @@ -76,7 +76,7 @@ private function throwTranslatedException(KalturaAPIException $e) * * @deprecated */ - public function updatePasswordAction( $email , $password , $newEmail = "" , $newPassword = "", $otp = null) + public function updatePasswordAction( $email , $password , $newEmail = "" , $newPassword = "", $otp = null) { try { From 51fb2718dad65670809bb9c95dc690d33046dd8d Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Wed, 10 Jul 2019 20:12:01 +0300 Subject: [PATCH 071/103] PLAT-9959:Adding Timing-Allow-Origin to the response header --- alpha/apps/kaltura/lib/kManifestRenderers.php | 1 + 1 file changed, 1 insertion(+) diff --git a/alpha/apps/kaltura/lib/kManifestRenderers.php b/alpha/apps/kaltura/lib/kManifestRenderers.php index 1af177a26fe..468e7f05fbf 100644 --- a/alpha/apps/kaltura/lib/kManifestRenderers.php +++ b/alpha/apps/kaltura/lib/kManifestRenderers.php @@ -303,6 +303,7 @@ final public function output() $headers = $this->getHeaders(); $headers[] = "Access-Control-Allow-Origin:*"; $headers[] = "Access-Control-Expose-Headers: Server,range,Content-Length,Content-Range"; + $headers[] = 'Timing-Allow-Origin:*'; foreach ($headers as $header) { if ($this->deliveryCode) From 8c3b105ff53b62c13eb473ba64fbc00ae607ea9d Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Wed, 10 Jul 2019 21:22:39 +0300 Subject: [PATCH 072/103] PLAT-9959:align nearby lines --- alpha/apps/kaltura/lib/kManifestRenderers.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpha/apps/kaltura/lib/kManifestRenderers.php b/alpha/apps/kaltura/lib/kManifestRenderers.php index 468e7f05fbf..aa3054da904 100644 --- a/alpha/apps/kaltura/lib/kManifestRenderers.php +++ b/alpha/apps/kaltura/lib/kManifestRenderers.php @@ -301,8 +301,8 @@ final public function output() $this->applyDomainPrefix(); $headers = $this->getHeaders(); - $headers[] = "Access-Control-Allow-Origin:*"; - $headers[] = "Access-Control-Expose-Headers: Server,range,Content-Length,Content-Range"; + $headers[] = 'Access-Control-Allow-Origin:*'; + $headers[] = 'Access-Control-Expose-Headers: Server,range,Content-Length,Content-Range'; $headers[] = 'Timing-Allow-Origin:*'; foreach ($headers as $header) { From 6b7a3c1875b0d0ebe39f23f742ba9eee770fe62d Mon Sep 17 00:00:00 2001 From: moshemaorkaltura Date: Thu, 11 Jul 2019 11:17:00 +0300 Subject: [PATCH 073/103] PLAT-10001:fix failure in search user and category --- .../providers/elastic_search/lib/model/search/kBaseSearch.php | 2 +- .../elastic_search/lib/model/search/kCategorySearch.php | 2 +- .../providers/elastic_search/lib/model/search/kUserSearch.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php b/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php index 3752939b480..63a3b19630a 100644 --- a/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php +++ b/plugins/search/providers/elastic_search/lib/model/search/kBaseSearch.php @@ -39,7 +39,7 @@ protected function handleDisplayInSearch() protected abstract function execSearch(ESearchOperator $eSearchOperator); - protected abstract function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null); + protected abstract function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null, ESearchAggregations $aggregations=null); protected function initPager(kPager $pager = null) { diff --git a/plugins/search/providers/elastic_search/lib/model/search/kCategorySearch.php b/plugins/search/providers/elastic_search/lib/model/search/kCategorySearch.php index 30686984998..8588654bb50 100644 --- a/plugins/search/providers/elastic_search/lib/model/search/kCategorySearch.php +++ b/plugins/search/providers/elastic_search/lib/model/search/kCategorySearch.php @@ -24,7 +24,7 @@ public function doSearch(ESearchOperator $eSearchOperator, kPager $pager = null, return $result; } - protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null) + protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null, ESearchAggregations $aggregations=null) { $this->query = array( 'index' => ElasticIndexMap::ELASTIC_CATEGORY_INDEX, diff --git a/plugins/search/providers/elastic_search/lib/model/search/kUserSearch.php b/plugins/search/providers/elastic_search/lib/model/search/kUserSearch.php index 4399de38683..0cc229c9b65 100644 --- a/plugins/search/providers/elastic_search/lib/model/search/kUserSearch.php +++ b/plugins/search/providers/elastic_search/lib/model/search/kUserSearch.php @@ -22,7 +22,7 @@ public function doSearch(ESearchOperator $eSearchOperator, kPager $pager = null, return $result; } - protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null) + protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null, ESearchAggregations $aggregations=null) { $this->query = array( 'index' => ElasticIndexMap::ELASTIC_KUSER_INDEX, From 481d46bcf6d75ef2d8d49b8ea4ee2c7470600828 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 11 Jul 2019 16:18:46 +0300 Subject: [PATCH 074/103] PLAT-9902 --- .../KAsyncConvertCaptionAsset.class.php | 101 +++++++++++------- .../base/services/CaptionAssetService.php | 1 - 2 files changed, 62 insertions(+), 40 deletions(-) diff --git a/plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAsset.class.php b/plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAsset.class.php index 102596ee102..d467990b312 100644 --- a/plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAsset.class.php +++ b/plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAsset.class.php @@ -10,7 +10,7 @@ class KAsyncConvertCaptionAsset extends KJobHandlerWorker */ private $captionClientPlugin = null; - private $formatToName = array(CaptionType::SRT => 'srt' , CaptionType::DFXP => 'dfxp', CaptionType::WEBVTT => 'webvtt', CaptionType::SCC =>'scc'); + private $formatToName = array(CaptionType::SRT => 'srt' , CaptionType::DFXP => 'dfxp', CaptionType::WEBVTT => 'webvtt', CaptionType::SCC =>'scc'); /* (non-PHPdoc) * @see KBatchBase::getType() @@ -28,74 +28,97 @@ protected function exec(KalturaBatchJob $job) return $this->convertCaption($job, $job->data); } + /*** + * @param KalturaBatchJob $job + * @param KalturaConvertCaptionAssetJobData $data + * @return KalturaBatchJob|mixed + * @throws Exception + * @throws kApplicativeException + */ protected function convertCaption(KalturaBatchJob $job, KalturaConvertCaptionAssetJobData $data) { $this->updateJob($job, "Start parsing caption asset [$data->captionAssetId]", KalturaBatchJobStatus::QUEUED); - $this->captionClientPlugin = KalturaCaptionClientPlugin::get(self::$kClient); - $captionAssetId = $data->captionAssetId; - $fileLoc = $data->fileLocation; - if (!array_key_exists($data->fromType,$this->formatToName) || !(array_key_exists($data->toType ,$this->formatToName))) + if (!array_key_exists($data->fromType, $this->formatToName) || !(array_key_exists($data->toType, $this->formatToName))) { $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNSUPPORTED_FORMAT_TYPES', "Error: " . 'UNSUPPORTED_FORMAT_TYPES', KalturaBatchJobStatus::FAILED, $data); return $job; } - $fromTypeName = $this->formatToName[$data->fromType]; - $toTypeName = $this->formatToName[$data->toType]; - - $content = kEncryptFileUtils::getEncryptedFileContent($fileLoc, $data->fileEncryptionKey, kConf::get("encryption_iv")); + $content = kEncryptFileUtils::getEncryptedFileContent($data->fileLocation, $data->fileEncryptionKey, kConf::get("encryption_iv")); if (!$content) { $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_GET_FILE', "Error: " . 'UNABLE_TO_GET_FILE', KalturaBatchJobStatus::FAILED, $data); return $job; } - $output = null; - $return_var = 0; - $tempFile = tempnam(sys_get_temp_dir(), 'captionTranslation.'); - - kFileBase::filePutContents($tempFile, $content); - $script = realpath(dirname(__FILE__) . '/../../') . '/scripts/convertcaption.py'; - $cmd = self::$taskConfig->params->pythonCmd ." $script -i $tempFile -f $fromTypeName -t $toTypeName"; - KalturaLog::debug("Running caption conversion command: $cmd"); - exec($cmd, $output, $return_var); - - if ($return_var) + $convertedContent = $this->convertContent($content, $this->formatToName[$data->fromType], $this->formatToName[$data->toType]); + if ($convertedContent === false) { $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_PARSE_ASSET', "Error: " . 'UNABLE_TO_PARSE_ASSET', KalturaBatchJobStatus::FAILED, $data); return $job; } - $parsedContent = implode("\n",$output); + return $this->handleConvertedContent($job, $data, $convertedContent); + } + /*** + * @param $job + * @param $data + * @param $content + * @return mixed + * @throws kApplicativeException + */ + private function handleConvertedContent($job, $data, $content) + { self::impersonate($job->partnerId); - $captionAsset = $this->captionClientPlugin->captionAsset->get($captionAssetId); + $captionAsset = $this->captionClientPlugin->captionAsset->get($data->captionAssetId); if (!$captionAsset) { $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_GET_ORIGINAL_ASSET', "Error: " . 'UNABLE_TO_GET_ORIGINAL_ASSET', KalturaBatchJobStatus::FAILED, $data); return $job; } - $res = $this->deleteCaptionAsset($captionAssetId); + $res = $this->deleteCaptionAsset($data->captionAssetId); if (!$res) { $this->closeJob($job, KalturaBatchJobErrorTypes::RUNTIME, 'UNABLE_TO_DELETE_ORIGINAL_ASSET', "Error: " . 'UNABLE_TO_DELETE_ORIGINAL_ASSET', KalturaBatchJobStatus::FAILED, $data); return $job; } - $captionsCreated = $this->cloneCaptionAssetToSrtAndSetContent($captionAsset->entryId, $captionAsset, $parsedContent,$data->toType, $toTypeName); + $captionsCreated = $this->cloneCaptionAssetToSrtAndSetContent($captionAsset->entryId, $captionAsset, $content,$data->toType, $this->formatToName[$data->toType]); self::unimpersonate(); if ($captionsCreated) { - $this->closeJob($job, null, null, "Finished parsing $fromTypeName captions to $toTypeName", KalturaBatchJobStatus::FINISHED); + $this->closeJob($job, null, null, "Finished parsing " . $this->formatToName[$data->fromType]. " captions to " . $this->formatToName[$data->toType], KalturaBatchJobStatus::FINISHED); return $job; } else throw new kApplicativeException(KalturaBatchJobAppErrors::MISSING_ASSETS, "UNABLE_TO_CREATE_ASSET_WITH_CAPTION"); } + /*** + * @param $content + * @param $fromType + * @param $toType + * @return bool|string + */ + private function convertContent($content, $fromType, $toType) + { + $output = null; + $return_var = 0; + $tempFile = tempnam(sys_get_temp_dir(), 'captionTranslation.'); + + kFileBase::filePutContents($tempFile, $content); + $script = realpath(dirname(__FILE__) . '/../../') . '/scripts/convertcaption.py'; + $cmd = self::$taskConfig->params->pythonCmd ." $script -i $tempFile -f $fromType -t $toType"; + KalturaLog::debug("Running caption conversion command: $cmd"); + exec($cmd, $output, $return_var); + + return $return_var ? false : implode("\n",$output); + + } /** * @param $id @@ -125,23 +148,23 @@ private function deleteCaptionAsset($id) */ private function cloneCaptionAssetToSrtAndSetContent($entryId, $captionAsset, $content, $format, $fileExt) { - $captionAsset->id = null; - $captionAsset->entryId = null; - $captionAsset->languageCode = null; - $captionAsset->partnerId = null; - $captionAsset->createdAt = null; - $captionAsset->updatedAt = null; - $captionAsset->version = null; - $captionAsset->size = null; - $captionAsset->description = null; - $captionAsset->status = null; - - $captionAsset->fileExt = $fileExt; - $captionAsset->format = $format; + $newCaptionAsset = new KalturaCaptionAsset(); + $newCaptionAsset->fileExt = $fileExt; + $newCaptionAsset->format = $format; + $newCaptionAsset->tags = $captionAsset->tags; + $newCaptionAsset->partnerData = $captionAsset->partnerData; + $newCaptionAsset->partnerDescription = $captionAsset->partnerDescription; + $newCaptionAsset->actualSourceAssetParamsIds = $captionAsset->actualSourceAssetParamsIds; + $newCaptionAsset->language = $captionAsset->language; + $newCaptionAsset->isDefault = $captionAsset->isDefault; + $newCaptionAsset->label = $captionAsset->label; + $newCaptionAsset->parentId = $captionAsset->parentId; + $newCaptionAsset->accuracy = $captionAsset->accuracy; + $newCaptionAsset->displayOnPlayer = $captionAsset->displayOnPlayer; try { - $newCaptionAsset = $this->captionClientPlugin->captionAsset->add($entryId, $captionAsset); + $newCaptionAsset = $this->captionClientPlugin->captionAsset->add($entryId, $newCaptionAsset); } catch (Exception $e) { diff --git a/plugins/content/caption/base/services/CaptionAssetService.php b/plugins/content/caption/base/services/CaptionAssetService.php index 7ee3c292d23..ff5e1b5d0e7 100644 --- a/plugins/content/caption/base/services/CaptionAssetService.php +++ b/plugins/content/caption/base/services/CaptionAssetService.php @@ -10,7 +10,6 @@ class CaptionAssetService extends KalturaAssetService { const MAX_SERVE_WEBVTT_FILE_SIZE = 1048576; - const SSC_CAPTION_FILE_EXT = 'scc'; protected function kalturaNetworkAllowed($actionName) { From 14e8bf2810e50903c13499d0a9b7de7bb34ed159 Mon Sep 17 00:00:00 2001 From: Erez Gotlieb Date: Thu, 11 Jul 2019 16:35:44 +0300 Subject: [PATCH 075/103] Update release-notes.md --- release-notes.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/release-notes.md b/release-notes.md index 6f60fc66426..17ca6f9af59 100644 --- a/release-notes.md +++ b/release-notes.md @@ -1,5 +1,28 @@ # Orion 15.3.0 # +## Add new caption type SCC to convert from SCC to SRT automatically on upload of scc captions ## + +- Issue Type: Feature +- Issue ID: PLAT-9902 + +### configuration ### +make sure python3 and pycaption for python 3 are installed + +Add the following to batch.ini: + +enabledWorkers.KAsyncConvertCaptionAsset = xxx (number of workers) +[KAsyncConvertCaptionAsset : JobHandlerWorker] +id = 770 +name = KAsyncConvertCaptionAsset +friendlyName = conver caption asset +type = KAsyncConvertCaptionAsset +maximumExecutionTime = 300 +scriptPath = ../plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAssetExe.php +params.pythonCmd = @LOCATION_OF_PYTHON3@ + +### Deployment scripts ### + None + ## Add permission in the admin console to include live entries in the mrss feed ## - Issue Type: Feature From 7fe8afe28352d039680b13f745d9a642b2c57819 Mon Sep 17 00:00:00 2001 From: Erez Gotlieb Date: Thu, 11 Jul 2019 16:36:46 +0300 Subject: [PATCH 076/103] Update release-notes.md --- release-notes.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/release-notes.md b/release-notes.md index 17ca6f9af59..0e59a9ed390 100644 --- a/release-notes.md +++ b/release-notes.md @@ -10,16 +10,18 @@ make sure python3 and pycaption for python 3 are installed Add the following to batch.ini: -enabledWorkers.KAsyncConvertCaptionAsset = xxx (number of workers) +enabledWorkers.KAsyncConvertCaptionAsset = xxx (number of workers) + [KAsyncConvertCaptionAsset : JobHandlerWorker] id = 770 name = KAsyncConvertCaptionAsset -friendlyName = conver caption asset +friendlyName = convert caption asset type = KAsyncConvertCaptionAsset maximumExecutionTime = 300 scriptPath = ../plugins/content/caption/base/batch/ConvertCaptions/KAsyncConvertCaptionAssetExe.php params.pythonCmd = @LOCATION_OF_PYTHON3@ + ### Deployment scripts ### None From a95b0f814a99750d04903fc0074e73c8414d4b7f Mon Sep 17 00:00:00 2001 From: Erez Gotlieb Date: Thu, 11 Jul 2019 16:38:46 +0300 Subject: [PATCH 077/103] Update release-notes.md --- release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes.md b/release-notes.md index 0e59a9ed390..0de168c97d1 100644 --- a/release-notes.md +++ b/release-notes.md @@ -6,7 +6,7 @@ - Issue ID: PLAT-9902 ### configuration ### -make sure python3 and pycaption for python 3 are installed +make sure python3 and pycaption for python 3 are installed Add the following to batch.ini: From f2f11e5d547e9ca8b93618e2a71b0757e49f657b Mon Sep 17 00:00:00 2001 From: Erez Gotlieb Date: Thu, 11 Jul 2019 16:39:59 +0300 Subject: [PATCH 078/103] Update release-notes.md --- release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes.md b/release-notes.md index 0de168c97d1..af5d79180dd 100644 --- a/release-notes.md +++ b/release-notes.md @@ -5,7 +5,7 @@ - Issue Type: Feature - Issue ID: PLAT-9902 -### configuration ### +### configuration ### make sure python3 and pycaption for python 3 are installed Add the following to batch.ini: From c5154810fbc4800b03cf2c21084cc641f8946a8e Mon Sep 17 00:00:00 2001 From: ZurKaltura Date: Thu, 11 Jul 2019 16:50:32 +0300 Subject: [PATCH 079/103] Orion-15.3.0-PLAT-9974 adding distribute on type to distribution profile --- admin_console/configs/lang/en.php | 3 + .../forms/ProviderProfileConfiguration.php | 26 ++++- .../lib/api/KalturaDistributionProfile.php | 11 ++- .../api/enums/KalturaDistributeTrigger.php | 9 ++ .../lib/kContentDistributionFlowManager.php | 95 ++++++++++++++----- .../lib/model/DistributionProfile.php | 10 +- .../lib/model/enums/kDistributeTrigger.php | 10 ++ var_console/configs/lang/en.php | 3 + 8 files changed, 137 insertions(+), 30 deletions(-) create mode 100644 plugins/content_distribution/lib/api/enums/KalturaDistributeTrigger.php create mode 100644 plugins/content_distribution/lib/model/enums/kDistributeTrigger.php diff --git a/admin_console/configs/lang/en.php b/admin_console/configs/lang/en.php index f50c5751da4..2207dae9543 100644 --- a/admin_console/configs/lang/en.php +++ b/admin_console/configs/lang/en.php @@ -306,6 +306,9 @@ 'Kaltura_Client_ContentDistribution_Enum_DistributionProfileStatus::ENABLED' => 'Enabled', 'Kaltura_Client_ContentDistribution_Enum_DistributionProfileStatus::DISABLED' => 'Disabled', + 'Kaltura_Client_ContentDistribution_Enum_DistributeTrigger::ENTRY_READY' => 'On entry ready', + 'Kaltura_Client_ContentDistribution_Enum_DistributeTrigger::MODERATION_APPROVED' => 'On moderation approved', + 'init client failed' => 'Could not initiate the Kaltura client', 'entry not found' => 'Entry not found', 'partner not found' => 'Publisher not found', diff --git a/plugins/content_distribution/admin/forms/ProviderProfileConfiguration.php b/plugins/content_distribution/admin/forms/ProviderProfileConfiguration.php index 77e290116e6..de42b8c2b5b 100644 --- a/plugins/content_distribution/admin/forms/ProviderProfileConfiguration.php +++ b/plugins/content_distribution/admin/forms/ProviderProfileConfiguration.php @@ -68,16 +68,18 @@ public function init() 'readonly' => true, 'filters' => array('StringTrim'), )); - + $this->addElement('hidden', 'provider_type', array( 'value' => $this->providerType, )); - + + $this->addDistributeTriggerElement(); + $this->addElement('hidden', 'crossLine01', array( 'lable' => 'line', 'decorators' => array('ViewHelper', array('Label', array('placement' => 'append')), array('HtmlTag', array('tag' => 'hr', 'class' => 'crossLine'))) )); - + // TODO - redefine the UI // // $this->addElement('text', 'sunrise_default_offset', array( @@ -101,7 +103,23 @@ public function init() $this->addProfileAction('delete'); $this->addProfileAction('report'); } - + + protected function addDistributeTriggerElement() + { + $distributeTrigger = new Kaltura_Form_Element_EnumSelect('distribute_Trigger', array('enum' => 'Kaltura_Client_ContentDistribution_Enum_DistributeTrigger')); + $distributeTrigger->setLabel('Auto distribute trigger:'); + if($this->distributionProfile->distributeTrigger) + { + $distributeTrigger->setValue($this->distributionProfile->distributeTrigger); + } + else + { + $distributeTrigger->setValue(Kaltura_Client_ContentDistribution_Enum_DistributeTrigger::ENTRY_READY); + } + + $this->addElement($distributeTrigger); + } + /** * @param string $action * @return Zend_Form_DisplayGroup diff --git a/plugins/content_distribution/lib/api/KalturaDistributionProfile.php b/plugins/content_distribution/lib/api/KalturaDistributionProfile.php index 7feadc9fb0e..e52d8572d42 100644 --- a/plugins/content_distribution/lib/api/KalturaDistributionProfile.php +++ b/plugins/content_distribution/lib/api/KalturaDistributionProfile.php @@ -153,7 +153,15 @@ abstract class KalturaDistributionProfile extends KalturaObject implements IFilt * @var int */ public $recommendedDcForExecute; - + + + /** + * The event that trigger the automatic distribute + * @var KalturaDistributeTrigger + */ + public $distributeTrigger; + + /* * mapping between the field on this object (on the left) and the setter/getter on the object (on the right) */ @@ -179,6 +187,7 @@ abstract class KalturaDistributionProfile extends KalturaObject implements IFilt 'recommendedStorageProfileForDownload', 'recommendedDcForDownload', 'recommendedDcForExecute', + 'distributeTrigger', ); public function getMapBetweenObjects() diff --git a/plugins/content_distribution/lib/api/enums/KalturaDistributeTrigger.php b/plugins/content_distribution/lib/api/enums/KalturaDistributeTrigger.php new file mode 100644 index 00000000000..2ae4a44a14c --- /dev/null +++ b/plugins/content_distribution/lib/api/enums/KalturaDistributeTrigger.php @@ -0,0 +1,9 @@ +wasObjectSaved()) + { + if($object->getStatus() == entryStatus::READY && + in_array(entryPeer::MODERATION_STATUS, $modifiedColumns) && + !in_array($object->getModerationStatus(), self::$validModerationStatuses)) + { + return false; + } + return true; + } if($object instanceof asset && $object->getStatus() == asset::FLAVOR_ASSET_STATUS_READY && in_array(assetPeer::STATUS, $modifiedColumns) || in_array(assetPeer::VERSION, $modifiedColumns)) return true; @@ -29,10 +42,15 @@ public function objectChanged(BaseObject $object, array $modifiedColumns) { if($object instanceof entry && $object->getStatus() != entryStatus::DELETED) { - if(in_array(entryPeer::STATUS, $modifiedColumns) && $object->getStatus() == entryStatus::READY) + if((in_array(entryPeer::STATUS, $modifiedColumns) && $object->getStatus() == entryStatus::READY) + || in_array(entryPeer::MODERATION_STATUS, $modifiedColumns)) + { return self::onEntryReady($object); + } else + { return self::onEntryChanged($object, $modifiedColumns); + } } if($object instanceof asset && $object->getStatus() == asset::FLAVOR_ASSET_STATUS_READY) @@ -1155,6 +1173,7 @@ public static function onDistributionDeleteJobFailed(BatchJob $dbBatchJob, kDist /** * @param Metadata $metadata + * @return bool */ public static function onMetadataDeleted(Metadata $metadata) { @@ -1245,9 +1264,10 @@ public static function onMetadataDeleted(Metadata $metadata) return true; } - + /** * @param Metadata $metadata + * @return bool */ public static function onMetadataChanged(Metadata $metadata, $previousVersion) { @@ -1438,9 +1458,10 @@ public static function onMetadataChanged(Metadata $metadata, $previousVersion) return true; } - + /** * @param EntryDistribution $entryDistribution + * @return bool */ public static function onEntryDistributionUpdateRequired(EntryDistribution $entryDistribution) { @@ -1502,10 +1523,11 @@ public static function onEntryDistributionUpdateRequired(EntryDistribution $entr self::submitUpdateEntryDistribution($entryDistribution, $distributionProfile); return true; } - + /** * @param EntryDistribution $entryDistribution * @param array $modifiedColumns + * @return bool */ public static function onEntryDistributionChanged(EntryDistribution $entryDistribution, array $modifiedColumns) { @@ -1523,10 +1545,11 @@ public static function onEntryDistributionChanged(EntryDistribution $entryDistri return true; } - + /** * @param entry $entry * @param array $modifiedColumns + * @return bool */ public static function onEntryChanged(entry $entry, array $modifiedColumns) { @@ -1715,9 +1738,10 @@ public static function onGenericDistributionProviderDeleted(GenericDistributionP $genericDistributionProfiles->save(); } } - + /** * @param entry $entry + * @return bool */ public static function onEntryDeleted(entry $entry) { @@ -1770,42 +1794,64 @@ public static function onEntryDeleted(entry $entry) /** * @param entry $entry + * @return bool */ public static function onEntryReady(entry $entry) { - if(!ContentDistributionPlugin::isAllowedPartner($entry->getPartnerId())) + if (!ContentDistributionPlugin::isAllowedPartner($entry->getPartnerId())) + { return true; + } //no temp entries should be handled if ($entry->getDisplayInSearch() == mySearchUtils::DISPLAY_IN_SEARCH_SYSTEM && $entry->getReplacedEntryId()) + { return true; + } $distributionProfiles = DistributionProfilePeer::retrieveByPartnerId($entry->getPartnerId()); - $entryType = $entry->getType(); foreach($distributionProfiles as $distributionProfile) { - if(!$distributionProfile->shouldAddDistributeByType($entryType)) - continue; - - $entryDistribution = EntryDistributionPeer::retrieveByEntryAndProfileId($entry->getId(), $distributionProfile->getId()); - if($entryDistribution) + if($distributionProfile->getDistributeTrigger() == kDistributeTrigger::MODERATION_APPROVED) { - KalturaLog::info("Found entry distribution object with id [" . $entryDistribution->getId() . "] for distrinution profle [" . $distributionProfile->getId() . "]"); - self::onEntryDistributionUpdateRequired($entryDistribution); - continue; + if(!in_array($entry->getModerationStatus(), self::$validModerationStatuses)) + { + continue; + } } - if($distributionProfile->getSubmitEnabled() == DistributionProfileActionStatus::AUTOMATIC) - { - self::addEntryDistribution($entry, $distributionProfile, true); - } + self::distributeNewEntry($entry, $distributionProfile); } return true; } + protected static function distributeNewEntry(entry $entry, DistributionProfile $distributionProfile) + { + if(!$distributionProfile->shouldAddDistributeByType($entry->getType())) + { + return false; + } + + $entryDistribution = EntryDistributionPeer::retrieveByEntryAndProfileId($entry->getId(), $distributionProfile->getId()); + if($entryDistribution) + { + KalturaLog::info("Found entry distribution object with id [" . $entryDistribution->getId() . "] for distribution profile [" . $distributionProfile->getId() . "]"); + self::onEntryDistributionUpdateRequired($entryDistribution); + return false; + } + + if($distributionProfile->getSubmitEnabled() == DistributionProfileActionStatus::AUTOMATIC) + { + self::addEntryDistribution($entry, $distributionProfile, true); + } + + return true; + } + /** * @param asset $asset + * @return bool */ public static function onAssetVersionChanged(asset $asset) { @@ -1853,9 +1899,10 @@ public static function onAssetVersionChanged(asset $asset) return true; } - + /** * @param asset $asset + * @return bool */ public static function onAssetReadyOrDeleted(asset $asset) { @@ -1939,11 +1986,12 @@ public static function onAssetReadyOrDeleted(asset $asset) return true; } - + /** * @param array $distributionProfiles * @param string $entryId * @param int $entryType + * @return bool */ public static function checkShouldDistributeByProfiles(array $distributionProfiles, $entryId, $entryType) { @@ -1954,10 +2002,11 @@ public static function checkShouldDistributeByProfiles(array $distributionProfil } return false; } - + /** * @param array $entryDistributions * @param array $distributionProfiles + * @return array */ public static function getDistProfilesByEntryDistributions(array $entryDistributions, array $distributionProfiles) { diff --git a/plugins/content_distribution/lib/model/DistributionProfile.php b/plugins/content_distribution/lib/model/DistributionProfile.php index e972566d121..13e3c662be1 100644 --- a/plugins/content_distribution/lib/model/DistributionProfile.php +++ b/plugins/content_distribution/lib/model/DistributionProfile.php @@ -25,6 +25,7 @@ abstract class DistributionProfile extends BaseDistributionProfile implements IS const CUSTOM_DATA_FIELD_RECOMMENDED_DC_EXECUTE = "recommendedDcForExecute"; const CUSTOM_DATA_FIELD_REQUIRED_ASSET_DISTRIBUTION_RULES = "requiredAssetDistributionRules"; const CUSTOM_DATA_FIELD_OPTIONAL_ASSET_DISTRIBUTION_RULES = "optionalAssetDistributionRules"; + const CUSTOM_DATA_FIELD_DISTRIBUTE_TRIGGER = "distributeTrigger"; /** * @return IDistributionProvider @@ -467,7 +468,11 @@ public function getRecommendedDcForDownload() {return $this->getFromCustomDat public function getRecommendedDcForExecute() {return $this->getFromCustomData(self::CUSTOM_DATA_FIELD_RECOMMENDED_DC_EXECUTE);} public function getRequiredAssetDistributionRules() {return $this->getFromCustomData(self::CUSTOM_DATA_FIELD_REQUIRED_ASSET_DISTRIBUTION_RULES, null, array());} public function getOptionalAssetDistributionRules() {return $this->getFromCustomData(self::CUSTOM_DATA_FIELD_OPTIONAL_ASSET_DISTRIBUTION_RULES, null, array());} - + public function getDistributeTrigger() + { + return $this->getFromCustomData(self::CUSTOM_DATA_FIELD_DISTRIBUTE_TRIGGER, null, kDistributeTrigger::ENTRY_READY); + } + public function setSunriseDefaultOffset($v) {return $this->putInCustomData(self::CUSTOM_DATA_FIELD_SUNRISE_DEFAULT_OFFSET, $v);} public function setSunsetDefaultOffset($v) {return $this->putInCustomData(self::CUSTOM_DATA_FIELD_SUNSET_DEFAULT_OFFSET, $v);} public function setRecommendedStorageProfileForDownload($v) {return $this->putInCustomData(self::CUSTOM_DATA_FIELD_RECOMMENDED_STORAGE_PROFILE_DOWNLOAD, $v);} @@ -475,7 +480,8 @@ public function setRecommendedDcForDownload($v) {return $this->putInCustomDat public function setRecommendedDcForExecute($v) {return $this->putInCustomData(self::CUSTOM_DATA_FIELD_RECOMMENDED_DC_EXECUTE, $v);} public function setRequiredAssetDistributionRules($v) {return $this->putInCustomData(self::CUSTOM_DATA_FIELD_REQUIRED_ASSET_DISTRIBUTION_RULES, $v);} public function setOptionalAssetDistributionRules($v) {return $this->putInCustomData(self::CUSTOM_DATA_FIELD_OPTIONAL_ASSET_DISTRIBUTION_RULES, $v);} - + public function setDistributeTrigger($v) {return $this->putInCustomData(self::CUSTOM_DATA_FIELD_DISTRIBUTE_TRIGGER, $v);} + public function getCacheInvalidationKeys() { return array("distributionProfile:id=".strtolower($this->getId())); diff --git a/plugins/content_distribution/lib/model/enums/kDistributeTrigger.php b/plugins/content_distribution/lib/model/enums/kDistributeTrigger.php new file mode 100644 index 00000000000..fc8caa9e72d --- /dev/null +++ b/plugins/content_distribution/lib/model/enums/kDistributeTrigger.php @@ -0,0 +1,10 @@ + 'Enabled', 'Kaltura_Client_ContentDistribution_Enum_DistributionProfileStatus::DISABLED' => 'Disabled', + 'Kaltura_Client_ContentDistribution_Enum_DistributeTrigger::ENTRY_READY' => 'On entry ready', + 'Kaltura_Client_ContentDistribution_Enum_DistributeTrigger::MODERATION_APPROVED' => 'On moderation approved', + 'init client failed' => 'Could not initiate the Kaltura client', 'entry not found' => 'Entry not found', 'partner not found' => 'Publisher not found', From 7698bf627cb448e27b08eeac90f5fde23cb15120 Mon Sep 17 00:00:00 2001 From: ZurKaltura Date: Thu, 11 Jul 2019 18:03:20 +0300 Subject: [PATCH 080/103] Orion-15.2.0-PLAT-9936 zoom chat file support Also, include a change to ignore the owner if you get it from participant --- .../services/AttachmentAssetService.php | 13 ++- .../lib/model/enums/kVendorErrorMessages.php | 3 + plugins/vendor/lib/model/zoom/kZoomEngine.php | 109 ++++++++++++++---- 3 files changed, 103 insertions(+), 22 deletions(-) diff --git a/plugins/content/attachment/services/AttachmentAssetService.php b/plugins/content/attachment/services/AttachmentAssetService.php index e11a423ee16..879b19d944d 100644 --- a/plugins/content/attachment/services/AttachmentAssetService.php +++ b/plugins/content/attachment/services/AttachmentAssetService.php @@ -9,6 +9,8 @@ */ class AttachmentAssetService extends KalturaAssetService { + const MAX_FILE_NAME_LENGTH = 255; + public function initService($serviceId, $serviceName, $actionName) { parent::initService($serviceId, $serviceName, $actionName); @@ -213,14 +215,21 @@ protected function attachFile(AttachmentAsset $attachmentAsset, $fullPath, $copy $attachmentAsset->setStatus(AttachmentAsset::ASSET_STATUS_READY); $attachmentAsset->save(); } - + /** * @param AttachmentAsset $attachmentAsset * @param string $url + * @throws KalturaAPIException */ protected function attachUrl(AttachmentAsset $attachmentAsset, $url) { - $fullPath = myContentStorage::getFSUploadsPath() . '/' . basename($url); + $fileName = basename($url); + if(strlen($fileName) > self::MAX_FILE_NAME_LENGTH) + { + $fileName = md5($url); + } + + $fullPath = myContentStorage::getFSUploadsPath() . '/' . $fileName; if (KCurlWrapper::getDataFromFile($url, $fullPath)) return $this->attachFile($attachmentAsset, $fullPath); diff --git a/plugins/vendor/lib/model/enums/kVendorErrorMessages.php b/plugins/vendor/lib/model/enums/kVendorErrorMessages.php index 98675444b46..24ca6a0d694 100644 --- a/plugins/vendor/lib/model/enums/kVendorErrorMessages.php +++ b/plugins/vendor/lib/model/enums/kVendorErrorMessages.php @@ -15,4 +15,7 @@ interface kVendorErrorMessages extends BaseEnum const TOKEN_PARSING_FAILED = 'Parse Tokens failed, response received from zoom is: '; const FAILED_VERIFICATION = 'ZOOM - verification token is different from existing token'; const MISSING_ENTRY_FOR_ZOOM_MEETING = 'Could not find entry for meeting id: '; + const MISSING_ENTRY_FOR_CHAT = 'Missing entry for the chat file'; + const ERROR_HANDLING_CHAT = 'Error while trying to handle chat file'; + const ERROR_HANDLING_TRANSCRIPT = 'Error while trying to handle transcript file'; } \ No newline at end of file diff --git a/plugins/vendor/lib/model/zoom/kZoomEngine.php b/plugins/vendor/lib/model/zoom/kZoomEngine.php index 1b7553a1fa5..38cb16d1819 100644 --- a/plugins/vendor/lib/model/zoom/kZoomEngine.php +++ b/plugins/vendor/lib/model/zoom/kZoomEngine.php @@ -14,6 +14,7 @@ class kZoomEngine protected static $FILE_VIDEO_TYPES = array('MP4'); protected static $FILE_CAPTION_TYPES = array('TRANSCRIPT'); + protected static $FILE_CHAT_TYPES = array('CHAT'); protected $zoomConfiguration; protected $zoomClient; @@ -77,7 +78,7 @@ protected function handleRecordingTranscriptComplete($event) $zoomIntegration = ZoomHelper::getZoomIntegration(); $dbUser = $this->getEntryOwner($transcript->hostEmail, $zoomIntegration); $this->initUserPermissions($dbUser); - $entry = $this->getZoomEntryByReferenceId($transcript->id); + $entry = $this->getZoomEntryByMeetingId($transcript->id); $this->initUserPermissions($dbUser, true); $captionAssetService = new CaptionAssetService(); $captionAssetService->initService('caption_captionasset', 'captionAsset', 'setContent'); @@ -90,14 +91,21 @@ protected function handleRecordingTranscriptComplete($event) continue; } - $captionAsset = $this->createAssetForTranscription($entry); - $captionAssetResource = new KalturaUrlResource(); - $captionAssetResource->url = $recordingFile->download_url . self::URL_ACCESS_TOKEN . $event->downloadToken; - $captionAssetService->setContentAction($captionAsset->getId(), $captionAssetResource); + try + { + $captionAsset = $this->createAssetForTranscription($entry); + $captionAssetResource = new KalturaUrlResource(); + $captionAssetResource->url = $recordingFile->download_url . self::URL_ACCESS_TOKEN . $event->downloadToken; + $captionAssetService->setContentAction($captionAsset->getId(), $captionAssetResource); + } + catch (Exception $e) + { + ZoomHelper::exitWithError(kVendorErrorMessages::ERROR_HANDLING_TRANSCRIPT); + } } } - protected function getZoomEntryByReferenceId($meetingId) + protected function getZoomEntryByMeetingId($meetingId) { $entryFilter = new entryFilter(); $pager = new KalturaFilterPager(); @@ -112,7 +120,7 @@ protected function getZoomEntryByReferenceId($meetingId) entryPeer::setFilterResults(true); } - !$entry = entryPeer::doSelectOne($c); + $entry = entryPeer::doSelectOne($c); if(!$entry) { ZoomHelper::exitWithError(kVendorErrorMessages::MISSING_ENTRY_FOR_ZOOM_MEETING . $meetingId); @@ -132,25 +140,66 @@ protected function handleRecordingVideoComplete($event) $meeting = $event->object; $dbUser = $this->getEntryOwner($meeting->hostEmail, $zoomIntegration); $this->initUserPermissions($dbUser); - $participantsUsersNames = $this->extractMeetingParticipants($meeting->id, $zoomIntegration); + $participantsUsersNames = $this->extractMeetingParticipants($meeting->id, $zoomIntegration, $dbUser->getPuserId()); $validatedUsers = $this->getValidatedUsers($participantsUsersNames, $zoomIntegration->getPartnerId(), $zoomIntegration->getCreateUserIfNotExist()); + $entry = null; foreach ($meeting->recordingFiles as $recordingFile) { /* @var kZoomRecordingFile $recordingFile */ + if (in_array ($recordingFile->fileType, self::$FILE_VIDEO_TYPES)) + { + $entry = $this->handleVideoRecord($meeting, $dbUser, $zoomIntegration, $validatedUsers, $recordingFile, $event); + } + } - if (!in_array ($recordingFile->fileType, self::$FILE_VIDEO_TYPES)) + foreach ($meeting->recordingFiles as $recordingFile) + { + /* @var kZoomRecordingFile $recordingFile */ + if (in_array ($recordingFile->fileType, self::$FILE_CHAT_TYPES)) { - continue; + $this->handleChatRecord($entry, $meeting, $recordingFile->download_url, $event->downloadToken, $dbUser); } + } + } - $entry = $this->createEntryFromMeeting($meeting, $dbUser); - $this->setEntryCategory($zoomIntegration, $entry); - $this->handleParticipants($entry, $validatedUsers, $zoomIntegration); - $entry->save(); - $url = $recordingFile->download_url . self::URL_ACCESS_TOKEN . $event->downloadToken; - kJobsManager::addImportJob(null, $entry->getId(), $entry->getPartnerId(), $url); + /** + * @param entry $entry + * @param kZoomMeeting $meeting + * @param string $chatDownloadUrl + * @param string $downloadToken + * @param kuser $dbUser + */ + protected function handleChatRecord($entry, $meeting, $chatDownloadUrl, $downloadToken, $dbUser) + { + if(!$entry) + { + ZoomHelper::exitWithError(kVendorErrorMessages::MISSING_ENTRY_FOR_CHAT); + } + try + { + $attachmentAsset = $this->createAttachmentAssetForChatFile($meeting->id, $entry); + $attachmentAssetResource = new KalturaUrlResource(); + $attachmentAssetResource->url = $chatDownloadUrl . self::URL_ACCESS_TOKEN . $downloadToken; + $this->initUserPermissions($dbUser, true); + $attachmentAssetService = new AttachmentAssetService(); + $attachmentAssetService->initService('attachment_attachmentasset', 'attachmentAsset', 'setContent'); + $attachmentAssetService->setContentAction($attachmentAsset->getId(), $attachmentAssetResource); } + catch (Exception $e) + { + ZoomHelper::exitWithError(kVendorErrorMessages::ERROR_HANDLING_CHAT); + } + } + protected function handleVideoRecord($meeting, $dbUser, $zoomIntegration, $validatedUsers, $recordingFile, $event) + { + $entry = $this->createEntryFromMeeting($meeting, $dbUser); + $this->setEntryCategory($zoomIntegration, $entry); + $this->handleParticipants($entry, $validatedUsers, $zoomIntegration); + $entry->save(); + $url = $recordingFile->download_url . self::URL_ACCESS_TOKEN . $event->downloadToken; + kJobsManager::addImportJob(null, $entry->getId(), $entry->getPartnerId(), $url); + return $entry; } /** @@ -263,7 +312,7 @@ protected function createEntryFromMeeting($meeting, $owner) * @param entry $entry * @return CaptionAsset */ - public function createAssetForTranscription($entry) + protected function createAssetForTranscription($entry) { $caption = new CaptionAsset(); $caption->setEntryId($entry->getId()); @@ -274,12 +323,28 @@ public function createAssetForTranscription($entry) return $caption; } + /** + * @param string $meetingId + * @return AttachmentAsset + */ + protected function createAttachmentAssetForChatFile($meetingId, $entry) + { + $attachment = new AttachmentAsset(); + $attachment->setFilename("Meeting {$meetingId} chat file"); + $attachment->setPartnerId($entry->getPartnerId()); + $attachment->setEntryId($entry->getId()); + $attachment->setcontainerFormat(AttachmentType::TEXT); + $attachment->save(); + return $attachment; + } + /** * @param $meetingId * @param ZoomVendorIntegration $zoomIntegration + * @param $meetingOwnerName * @return array participants users names */ - protected function extractMeetingParticipants($meetingId, $zoomIntegration) + protected function extractMeetingParticipants($meetingId, $zoomIntegration, $meetingOwnerName) { if ($zoomIntegration->getHandleParticipantsMode() == kHandleParticipantsMode::IGNORE) { @@ -293,10 +358,15 @@ protected function extractMeetingParticipants($meetingId, $zoomIntegration) $participantsEmails = $participants->getParticipantsEmails(); if($participantsEmails) { + KalturaLog::info('Found the following participants: ' . implode(", ", $participantsEmails)); $result = array(); foreach ($participantsEmails as $participantEmail) { - $result[] = $this->matchZoomUserName($participantEmail, $zoomIntegration); + $userName = $this->matchZoomUserName($participantEmail, $zoomIntegration); + if($meetingOwnerName != $userName) + { + $result[] = $userName; + } } } else @@ -332,7 +402,6 @@ public function getEntryOwner($hostEmail, $zoomIntegration) return $dbUser; } - /** * @param string $userName * @param ZoomVendorIntegration $zoomIntegration From d7dec0f30bce94ad3ceac01e8085b3caca065f66 Mon Sep 17 00:00:00 2001 From: Erez Gotlieb Date: Sun, 14 Jul 2019 12:46:23 +0300 Subject: [PATCH 081/103] Update release-notes.md --- release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes.md b/release-notes.md index af5d79180dd..b527850f127 100644 --- a/release-notes.md +++ b/release-notes.md @@ -23,7 +23,7 @@ params.pythonCmd = @LOCATION_OF_PYTHON3@ ### Deployment scripts ### - None + Install Plugins ## Add permission in the admin console to include live entries in the mrss feed ## From 7e3ba325c4788be14bce1beba83aff345ff0121a Mon Sep 17 00:00:00 2001 From: ZurKaltura Date: Sun, 14 Jul 2019 15:28:00 +0300 Subject: [PATCH 082/103] Orion-15.3.0-PLAT-10002 handling duplicate zoom record complete event --- .../apps/kaltura/lib/kResourceReservation.php | 18 ++- infra/general/BinaryResourceReservation.php | 123 ++++++++++++++++++ infra/general/ResourceReservation.php | 83 +----------- plugins/vendor/lib/model/zoom/kZoomEngine.php | 8 ++ 4 files changed, 152 insertions(+), 80 deletions(-) create mode 100644 infra/general/BinaryResourceReservation.php diff --git a/alpha/apps/kaltura/lib/kResourceReservation.php b/alpha/apps/kaltura/lib/kResourceReservation.php index 697cc918c99..ed6581e066c 100644 --- a/alpha/apps/kaltura/lib/kResourceReservation.php +++ b/alpha/apps/kaltura/lib/kResourceReservation.php @@ -3,24 +3,33 @@ class kResourceReservation { /** - * @var ResourceReservation $resourceReservator + * @var BinaryResourceReservation $resourceReservator */ private $resourceReservator; - function __construct($ttl = null) + function __construct($ttl = null, $binaryReservator = false) { $cache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_RESOURCE_RESERVATION); $ks = kCurrentContext::$ks; if (!$ttl) + { $ttl = kConf::get('ResourceReservationDuration'); - $this->resourceReservator = new ResourceReservation($cache, $ttl, $ks); + } + + if($binaryReservator) + { + $this->resourceReservator = new BinaryResourceReservation($cache, $ttl, $ks); + } + else + { + $this->resourceReservator = new ResourceReservation($cache, $ttl, $ks); + } } /** * will reserve the resource for some time * @param string $resourceId - * * @return bool - true if reserve and false if could not */ public function reserve($resourceId) @@ -30,6 +39,7 @@ public function reserve($resourceId) KalturaLog::info("Resource reservation was done successfully for resource id [$resourceId]"); return true; } + KalturaLog::info("Could not reserve resource id [$resourceId]"); return false; } diff --git a/infra/general/BinaryResourceReservation.php b/infra/general/BinaryResourceReservation.php new file mode 100644 index 00000000000..a053ccadfa7 --- /dev/null +++ b/infra/general/BinaryResourceReservation.php @@ -0,0 +1,123 @@ +cache = $cache; + $this->userToken = $userToken; + $this->ttl = intval($ttl); + if (!$this->ttl) + { + $this->ttl = ResourceReservation::DEFAULT_TIME_IN_CACHE_FOR_RESERVATION; + } + } + + + /** + * will return cache-key for resource + * @param string $resourceId + * @return string + */ + protected function getCacheKeyForResource($resourceId) + { + return "resource_reservation_cache_key_$resourceId"; + } + + /** + * will reserve the resource for some time + * @param string $resourceId + * @return bool - true if reserve and false if could not + */ + public function reserve($resourceId) + { + if ($this->cache) + { + $key = $this->getCacheKeyForResource($resourceId); + if ($this->cache->add($key, $this->userToken, $this->ttl)) + { + return true; + } + } + return false; + } + + /** + * will reserve the resource for some time ignoring if the resource was already reserved + * @param string $resourceId + * + * @return bool - true if reserve and false if could not + */ + public function forceReserve($resourceId) + { + if ($this->cache) + { + return $this->cache->set($this->getCacheKeyForResource($resourceId), $this->userToken, $this->ttl); + } + + return false; + } + + /** + * will delete the reservation of the resource from cache + * @param string $resourceId + * + * @return bool - true if reservation was deleted + */ + public function deleteReservation($resourceId) + { + if ($this->cache) + { + $val = $this->cache->get($this->getCacheKeyForResource($resourceId)); + if (!$val) + { + return true; + } + if ($val == $this->userToken) //only the one that reserve the resource can delete reservation + { + return $this->cache->delete($this->getCacheKeyForResource($resourceId)); + } + } + + return false; + } + + /** + * will return BatchJob objects. + * @param string $resourceId + * + * @return bool - true mean the resource is available + */ + public function checkAvailable($resourceId) + { + if ($this->cache) + { + $val = $this->cache->get($this->getCacheKeyForResource($resourceId)); + if (!$val) + { + return true; + } + } + + return false; + } +} diff --git a/infra/general/ResourceReservation.php b/infra/general/ResourceReservation.php index 4b56798e677..afe4d2ab59d 100644 --- a/infra/general/ResourceReservation.php +++ b/infra/general/ResourceReservation.php @@ -1,102 +1,32 @@ cache = $cache; - $this->userToken = $userToken; - $this->ttl = intval($ttl); - if (!$this->ttl) - $this->ttl = ResourceReservation::DEFAULT_TIME_IN_CACHE_FOR_RESERVATION; - } - - - /** - * will return cache-key for resource - * @param string $resourceId - * @return string - */ - private function getCacheKeyForResource($resourceId) - { - return "resource_reservation_cache_key_$resourceId"; - } - /** * will reserve the resource for some time * @param string $resourceId - * * @return bool - true if reserve and false if could not */ public function reserve($resourceId) { - if ($this->cache) + $result = parent::reserve($resourceId); + if(!$result) { $key = $this->getCacheKeyForResource($resourceId); - if ($this->cache->add($key, $this->userToken, $this->ttl)) - return true; $val = $this->cache->get($key); - if ($val == $this->userToken) //only the one that reserve the resource can ovreride the reserve (time extention) + if ($val == $this->userToken) //only the one that reserve the resource can override the reserve (time extending) + { return $this->cache->set($key, $this->userToken, $this->ttl); + } } - return false; - } - /** - * will reserve the resource for some time ignoring if the resource was already reserved - * @param string $resourceId - * - * @return bool - true if reserve and false if could not - */ - public function forceReserve($resourceId) - { - if ($this->cache) - return $this->cache->set($this->getCacheKeyForResource($resourceId), $this->userToken, $this->ttl); - return false; - } - - /** - * will delete the reservation of the resource from cache - * @param string $resourceId - * - * @return bool - true if reservation was deleted - */ - public function deleteReservation($resourceId) - { - if ($this->cache) - { - $val = $this->cache->get($this->getCacheKeyForResource($resourceId)); - if (!$val) - return true; - if ($val == $this->userToken) //only the one that reserve the resource can delete reservation - return $this->cache->delete($this->getCacheKeyForResource($resourceId)); - } return false; } /** * will return BatchJob objects. * @param string $resourceId - * * @return bool - true mean the resource is available */ public function checkAvailable($resourceId) @@ -107,6 +37,7 @@ public function checkAvailable($resourceId) if (!$val || $val == $this->userToken) //if not in cache or the caller was the one that reserve it return true; } + return false; } } diff --git a/plugins/vendor/lib/model/zoom/kZoomEngine.php b/plugins/vendor/lib/model/zoom/kZoomEngine.php index 38cb16d1819..5de1f9372d8 100644 --- a/plugins/vendor/lib/model/zoom/kZoomEngine.php +++ b/plugins/vendor/lib/model/zoom/kZoomEngine.php @@ -11,6 +11,7 @@ class kZoomEngine const URL_ACCESS_TOKEN = '?access_token='; const REFERENCE_FILTER = '_eq_reference_id'; const ZOOM_PREFIX = 'Zoom_'; + const ZOOM_LOCK_TTL = 120; protected static $FILE_VIDEO_TYPES = array('MP4'); protected static $FILE_CAPTION_TYPES = array('TRANSCRIPT'); @@ -138,6 +139,13 @@ protected function handleRecordingVideoComplete($event) $zoomIntegration = ZoomHelper::getZoomIntegration(); /* @var kZoomMeeting $meeting */ $meeting = $event->object; + + $resourceReservation = new kResourceReservation(self::ZOOM_LOCK_TTL, true); + if(!$resourceReservation->reserve($meeting->id, false)) + { + return; + } + $dbUser = $this->getEntryOwner($meeting->hostEmail, $zoomIntegration); $this->initUserPermissions($dbUser); $participantsUsersNames = $this->extractMeetingParticipants($meeting->id, $zoomIntegration, $dbUser->getPuserId()); From dc9051f29e14cc5d7b3da734ffab2c893ababc84 Mon Sep 17 00:00:00 2001 From: ZurKaltura Date: Sun, 14 Jul 2019 16:45:03 +0300 Subject: [PATCH 083/103] Orion-15.3.0-PLAT-10014 zoom chat and transcript file types (#8638) --- plugins/vendor/lib/model/zoom/kZoomEngine.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/vendor/lib/model/zoom/kZoomEngine.php b/plugins/vendor/lib/model/zoom/kZoomEngine.php index 5de1f9372d8..96005b488a2 100644 --- a/plugins/vendor/lib/model/zoom/kZoomEngine.php +++ b/plugins/vendor/lib/model/zoom/kZoomEngine.php @@ -12,6 +12,8 @@ class kZoomEngine const REFERENCE_FILTER = '_eq_reference_id'; const ZOOM_PREFIX = 'Zoom_'; const ZOOM_LOCK_TTL = 120; + const ZOOM_TRANSCRIPT_FILE_TYPE = 'vtt'; + const ZOOM_CHAT_FILE_TYPE = 'txt'; protected static $FILE_VIDEO_TYPES = array('MP4'); protected static $FILE_CAPTION_TYPES = array('TRANSCRIPT'); @@ -327,6 +329,7 @@ protected function createAssetForTranscription($entry) $caption->setPartnerId($entry->getPartnerId()); $caption->setContainerFormat(CaptionType::WEBVTT); $caption->setStatus(CaptionAsset::ASSET_STATUS_QUEUED); + $caption->setFileExt(self::ZOOM_TRANSCRIPT_FILE_TYPE); $caption->save(); return $caption; } @@ -342,6 +345,7 @@ protected function createAttachmentAssetForChatFile($meetingId, $entry) $attachment->setPartnerId($entry->getPartnerId()); $attachment->setEntryId($entry->getId()); $attachment->setcontainerFormat(AttachmentType::TEXT); + $attachment->setFileExt(self::ZOOM_CHAT_FILE_TYPE); $attachment->save(); return $attachment; } From 5309de7c5fd63d3fa3a01da5048ea113b6f1c21b Mon Sep 17 00:00:00 2001 From: ZurKaltura Date: Sun, 14 Jul 2019 16:51:41 +0300 Subject: [PATCH 084/103] Orion-15.3.0-PLAT-10011 zoom handling duplicate transcription complete events (#8639) --- plugins/vendor/lib/model/zoom/data/kZoomRecordingFile.php | 3 +++ plugins/vendor/lib/model/zoom/kZoomEngine.php | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/vendor/lib/model/zoom/data/kZoomRecordingFile.php b/plugins/vendor/lib/model/zoom/data/kZoomRecordingFile.php index 5a89579e973..622b79d83c7 100644 --- a/plugins/vendor/lib/model/zoom/data/kZoomRecordingFile.php +++ b/plugins/vendor/lib/model/zoom/data/kZoomRecordingFile.php @@ -7,13 +7,16 @@ class kZoomRecordingFile implements iZoomObject { const FILE_TYPE = 'file_type'; const DOWNLOAD_URL = 'download_url'; + const ID = 'id'; public $fileType; public $download_url; + public $id; public function parseData($data) { $this->fileType = $data[self::FILE_TYPE]; $this->download_url = $data[self::DOWNLOAD_URL]; + $this->id = $data[self::ID]; } } diff --git a/plugins/vendor/lib/model/zoom/kZoomEngine.php b/plugins/vendor/lib/model/zoom/kZoomEngine.php index 96005b488a2..afa55928ed5 100644 --- a/plugins/vendor/lib/model/zoom/kZoomEngine.php +++ b/plugins/vendor/lib/model/zoom/kZoomEngine.php @@ -85,11 +85,12 @@ protected function handleRecordingTranscriptComplete($event) $this->initUserPermissions($dbUser, true); $captionAssetService = new CaptionAssetService(); $captionAssetService->initService('caption_captionasset', 'captionAsset', 'setContent'); + $resourceReservation = new kResourceReservation(self::ZOOM_LOCK_TTL, true); foreach ($transcript->recordingFiles as $recordingFile) { /* @var kZoomRecordingFile $recordingFile */ - if (!in_array ($recordingFile->fileType, self::$FILE_CAPTION_TYPES)) + if (!in_array ($recordingFile->fileType, self::$FILE_CAPTION_TYPES) || !$resourceReservation->reserve($recordingFile->id)) { continue; } From d4b3a303809f604ff550f23b30ca6e0d01551f9c Mon Sep 17 00:00:00 2001 From: ZurKaltura Date: Sun, 14 Jul 2019 17:49:40 +0300 Subject: [PATCH 085/103] Orion 15.3.0 plat 10014 zoom file names (#8641) --- plugins/vendor/lib/model/zoom/kZoomEngine.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/vendor/lib/model/zoom/kZoomEngine.php b/plugins/vendor/lib/model/zoom/kZoomEngine.php index afa55928ed5..5a28df6f7d8 100644 --- a/plugins/vendor/lib/model/zoom/kZoomEngine.php +++ b/plugins/vendor/lib/model/zoom/kZoomEngine.php @@ -342,7 +342,7 @@ protected function createAssetForTranscription($entry) protected function createAttachmentAssetForChatFile($meetingId, $entry) { $attachment = new AttachmentAsset(); - $attachment->setFilename("Meeting {$meetingId} chat file"); + $attachment->setFilename("Meeting {$meetingId} chat file." . self::ZOOM_CHAT_FILE_TYPE); $attachment->setPartnerId($entry->getPartnerId()); $attachment->setEntryId($entry->getId()); $attachment->setcontainerFormat(AttachmentType::TEXT); From 19779ff75187a465a61e2ed84b78877fe78b8635 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Mon, 15 Jul 2019 09:52:26 +0300 Subject: [PATCH 086/103] SUP-16933:fix warning --- .../lib/model/filters/ESearchEntryQueryFromFilter.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php index c77c6a17ff6..1ed4887537b 100644 --- a/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php +++ b/plugins/search/providers/elastic_search/lib/model/filters/ESearchEntryQueryFromFilter.php @@ -212,6 +212,10 @@ protected function addingFieldPartIntoQuery($operator, $fieldName, $fieldValue) protected function splitIntoParameters($filter, $field, $fieldValue ) { $fieldParts = explode(entryFilter::FILTER_PREFIX, $field, 3); + if (count($fieldParts) < 3) + { + return null; + } list( , $operator, $fieldName) = $fieldParts; list($operator, $fieldName) = self::handlingFreeTextField($field, $operator, $fieldName); if(!in_array($fieldName, static::getSupportedFields()) || is_null($fieldValue) || $fieldValue === '') From fc97d51dfe35afca9eda94c3af253764ffe2e1f3 Mon Sep 17 00:00:00 2001 From: moshemaorkaltura Date: Mon, 15 Jul 2019 10:38:39 +0300 Subject: [PATCH 087/103] SUP-18254: Fix handling mutiple delete actions on user --- .../bulk_upload/csv/batch/BulkUploadUserEngineCsv.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/bulk_upload/csv/batch/BulkUploadUserEngineCsv.php b/plugins/bulk_upload/csv/batch/BulkUploadUserEngineCsv.php index 50fcd1ba92d..3696eea4fcb 100644 --- a/plugins/bulk_upload/csv/batch/BulkUploadUserEngineCsv.php +++ b/plugins/bulk_upload/csv/batch/BulkUploadUserEngineCsv.php @@ -358,9 +358,13 @@ private function getGroupActionList(&$usersToAddList,&$userGroupToDeleteMap) foreach ($this->groupActionsList as $group) { if (strpos($group->group, "-") !== 0) - $usersToAddList[]=$group; + { + $usersToAddList[] = $group; + } else - $userGroupToDeleteMap[$group->userId] = substr($group->group, 1); + { + $userGroupToDeleteMap[] = array($group->userId, substr($group->group, 1)); + } } } @@ -379,8 +383,9 @@ private function getUsers($usersList) private function deleteUsers($usersMap) { $ret = array(); - foreach ($usersMap as $userId=>$group) + foreach ($usersMap as $user) { + list($userId,$group) = $user; $this->handleMultiRequest($ret); KBatchBase::$kClient->groupUser->delete($userId, $group); } From a06251f6feb841c859eb5fabfbd220163fc88d0c Mon Sep 17 00:00:00 2001 From: ZurKaltura Date: Mon, 15 Jul 2019 12:22:51 +0300 Subject: [PATCH 088/103] Orion-15.3.0-PLAT-10019-resource reserve bug (#8644) --- infra/general/BinaryResourceReservation.php | 1 + infra/general/ResourceReservation.php | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/infra/general/BinaryResourceReservation.php b/infra/general/BinaryResourceReservation.php index a053ccadfa7..98627a62f2c 100644 --- a/infra/general/BinaryResourceReservation.php +++ b/infra/general/BinaryResourceReservation.php @@ -58,6 +58,7 @@ public function reserve($resourceId) return true; } } + return false; } diff --git a/infra/general/ResourceReservation.php b/infra/general/ResourceReservation.php index afe4d2ab59d..59370735f2a 100644 --- a/infra/general/ResourceReservation.php +++ b/infra/general/ResourceReservation.php @@ -10,15 +10,16 @@ class ResourceReservation extends BinaryResourceReservation */ public function reserve($resourceId) { - $result = parent::reserve($resourceId); - if(!$result) + if(parent::reserve($resourceId)) { - $key = $this->getCacheKeyForResource($resourceId); - $val = $this->cache->get($key); - if ($val == $this->userToken) //only the one that reserve the resource can override the reserve (time extending) - { - return $this->cache->set($key, $this->userToken, $this->ttl); - } + return true; + } + + $key = $this->getCacheKeyForResource($resourceId); + $val = $this->cache->get($key); + if ($val == $this->userToken) //only the one that reserve the resource can override the reserve (time extending) + { + return $this->cache->set($key, $this->userToken, $this->ttl); } return false; From 80bf4849c1bd02e3dfa3afb5a720d8f669eb2020 Mon Sep 17 00:00:00 2001 From: yossipapi Date: Mon, 15 Jul 2019 13:11:00 +0300 Subject: [PATCH 089/103] When filtering categoryEntry by created_at add updated at to the query to utilize an existing index --- api_v3/lib/types/filters/KalturaCategoryEntryFilter.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api_v3/lib/types/filters/KalturaCategoryEntryFilter.php b/api_v3/lib/types/filters/KalturaCategoryEntryFilter.php index 59f138069ba..3e8b7e50b87 100644 --- a/api_v3/lib/types/filters/KalturaCategoryEntryFilter.php +++ b/api_v3/lib/types/filters/KalturaCategoryEntryFilter.php @@ -99,6 +99,10 @@ public function getListResponse(KalturaFilterPager $pager, KalturaDetachedRespon if(!kEntitlementUtils::getEntitlementEnforcement() || $this->entryIdEqual == null) $pager->attachToCriteria($c); + //When filtering createdAtGreaterThanOrEqual add updated at filtering with the same value to utilize an existing index + if(isset($this->createdAtGreaterThanOrEqual)) + $c->addAnd(categoryEntryPeer::UPDATED_AT, $this->createdAtGreaterThanOrEqual, Criteria::GREATER_EQUAL); + $dbCategoriesEntry = categoryEntryPeer::doSelect($c); if(kEntitlementUtils::getEntitlementEnforcement() && count($dbCategoriesEntry) && $this->entryIdEqual != null) From e24cc9eac1bae15fd5532ce47be62d1bca40c2fb Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Mon, 15 Jul 2019 14:45:19 +0300 Subject: [PATCH 090/103] PLAT-9998:Adding new type of userEntry called watch later playlist --- .../WatchLaterPlaylistPlugin.php | 83 +++++++++++++++++++ .../KalturaWatchLaterPlaylistErrors.php | 12 +++ .../KalturaWatchLaterPlaylistUserEntry.php | 21 +++++ .../lib/model/WatchLaterPlaylistUserEntry.php | 16 ++++ .../enums/WatchLaterPlaylistUserEntryType.php | 29 +++++++ release-notes.md | 12 +++ 6 files changed, 173 insertions(+) create mode 100644 plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php create mode 100644 plugins/content/watch_later_playlist/errors/KalturaWatchLaterPlaylistErrors.php create mode 100644 plugins/content/watch_later_playlist/lib/api/KalturaWatchLaterPlaylistUserEntry.php create mode 100644 plugins/content/watch_later_playlist/lib/model/WatchLaterPlaylistUserEntry.php create mode 100644 plugins/content/watch_later_playlist/lib/model/enums/WatchLaterPlaylistUserEntryType.php diff --git a/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php b/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php new file mode 100644 index 00000000000..d6661b95de4 --- /dev/null +++ b/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php @@ -0,0 +1,83 @@ +getPluginEnabled(self::PLUGIN_NAME); + } + + /* (non-PHPdoc) + * @see IKalturaEnumerator::getEnums() + */ + public static function getEnums($baseEnumName = null) + { + if (is_null($baseEnumName)) + { + return array('WatchLaterPlaylistUserEntryType'); + } + if ($baseEnumName == 'UserEntryType') + { + return array('WatchLaterPlaylistUserEntryType'); + } + return array(); + } + + public static function getWatchLaterPlaylistUserEntryTypeCoreValue ($valueName) + { + $value = self::getPluginName() . IKalturaEnumerator::PLUGIN_VALUE_DELIMITER . $valueName; + return kPluginableEnumsManager::apiToCore('UserEntryType', $value); + } + + /** + * @return string external API value of dynamic enum. + */ + public static function getApiValue($valueName) + { + return self::getPluginName() . IKalturaEnumerator::PLUGIN_VALUE_DELIMITER . $valueName; + } + + public static function loadObject($baseClass, $enumValue, array $constructorArgs = null) + { + if ( ($baseClass == "KalturaUserEntry") && ($enumValue == self::getWatchLaterPlaylistUserEntryTypeCoreValue(WatchLaterPlaylistUserEntryType::WATCH_LATER_PLAYLIST))) + { + return new KalturaWatchLaterPlaylistUserEntry(); + } + if ( ($baseClass == "UserEntry") && ($enumValue == self::getWatchLaterPlaylistUserEntryTypeCoreValue(WatchLaterPlaylistUserEntryType::WATCH_LATER_PLAYLIST))) + { + return new WatchLaterPlaylistUserEntry(); + } + return null; + } + + public static function getObjectClass($baseClass, $enumValue) + { + if ($baseClass == 'UserEntry' && $enumValue == self::getWatchLaterPlaylistUserEntryTypeCoreValue(WatchLaterPlaylistUserEntryType::WATCH_LATER_PLAYLIST)) + { + return WatchLaterPlaylistUserEntry::WATCH_LATER_PLAYLIST_OM_CLASS; + } + return null; + } + } \ No newline at end of file diff --git a/plugins/content/watch_later_playlist/errors/KalturaWatchLaterPlaylistErrors.php b/plugins/content/watch_later_playlist/errors/KalturaWatchLaterPlaylistErrors.php new file mode 100644 index 00000000000..05c66d4a35b --- /dev/null +++ b/plugins/content/watch_later_playlist/errors/KalturaWatchLaterPlaylistErrors.php @@ -0,0 +1,12 @@ +setType(WatchLaterPlaylistPlugin::getWatchLaterPlaylistUserEntryTypeCoreValue(WatchLaterPlaylistUserEntryType::WATCH_LATER_PLAYLIST)); + parent::__construct(); + } + +} \ No newline at end of file diff --git a/plugins/content/watch_later_playlist/lib/model/enums/WatchLaterPlaylistUserEntryType.php b/plugins/content/watch_later_playlist/lib/model/enums/WatchLaterPlaylistUserEntryType.php new file mode 100644 index 00000000000..2328d52e49f --- /dev/null +++ b/plugins/content/watch_later_playlist/lib/model/enums/WatchLaterPlaylistUserEntryType.php @@ -0,0 +1,29 @@ + self::WATCH_LATER_PLAYLIST, + ); + } + + /* (non-PHPdoc) + * @see IKalturaPluginEnum::getAdditionalDescriptions() + */ + public static function getAdditionalDescriptions() + { + return array( + self::WATCH_LATER_PLAYLIST => 'Watch Later Playlist User Entry Type', + ); + } +} \ No newline at end of file diff --git a/release-notes.md b/release-notes.md index b527850f127..76d6b061e54 100644 --- a/release-notes.md +++ b/release-notes.md @@ -1,5 +1,17 @@ # Orion 15.3.0 # +## Add new type of userEntry called watch later playlist ## + +- Issue Type: Task +- Issue ID: PLAT-9998 + +### configuration ### +Add WatchLaterPlaylist to your plugins.ini + +### Deployment scripts ### +php /opt/kaltura/app/deployment/base/scripts/installPlugins.php + + ## Add new caption type SCC to convert from SCC to SRT automatically on upload of scc captions ## - Issue Type: Feature From b5701ae8f10ce8ab0ceb9f469170a51d430c6a37 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Mon, 15 Jul 2019 14:52:18 +0300 Subject: [PATCH 091/103] PLAT-9998:Adding new type of userEntry called watch later playlist --- .../WatchLaterPlaylistPlugin.php | 2 +- .../errors/KalturaWatchLaterPlaylistErrors.php | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 plugins/content/watch_later_playlist/errors/KalturaWatchLaterPlaylistErrors.php diff --git a/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php b/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php index d6661b95de4..da230a727f8 100644 --- a/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php +++ b/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php @@ -1,6 +1,6 @@ Date: Mon, 15 Jul 2019 14:59:28 +0300 Subject: [PATCH 092/103] PLAT-9998:Adding new type of userEntry called watch later playlist --- .../lib/api/KalturaWatchLaterPlaylistUserEntry.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/content/watch_later_playlist/lib/api/KalturaWatchLaterPlaylistUserEntry.php b/plugins/content/watch_later_playlist/lib/api/KalturaWatchLaterPlaylistUserEntry.php index 867f3d24a1b..036134ec024 100644 --- a/plugins/content/watch_later_playlist/lib/api/KalturaWatchLaterPlaylistUserEntry.php +++ b/plugins/content/watch_later_playlist/lib/api/KalturaWatchLaterPlaylistUserEntry.php @@ -5,7 +5,6 @@ */ class KalturaWatchLaterPlaylistUserEntry extends KalturaUserEntry { - /* (non-PHPdoc) * @see KalturaObject::toObject() */ @@ -18,4 +17,4 @@ public function toObject($dbObject = null, $propertiesToSkip = array()) return parent::toObject($dbObject, $propertiesToSkip); } -} +} \ No newline at end of file From 09d688e9a74649eed27ecca731398a428afca4d5 Mon Sep 17 00:00:00 2001 From: "amir.chervinsky" Date: Mon, 15 Jul 2019 15:27:49 +0300 Subject: [PATCH 093/103] bump KMCng version to v5.12.0 and Analytics version to v1.3.2 --- configurations/base.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configurations/base.ini b/configurations/base.ini index 3dc995821f0..95073f1b057 100644 --- a/configurations/base.ini +++ b/configurations/base.ini @@ -105,7 +105,7 @@ studio_v3_version = v3.3.0 liveanalytics_version = v2.7.3 usagedashboard_version = v2.0.0 live_dashboard_version = v1.5.6 -kmc_analytics_version = v1.3.1 +kmc_analytics_version = v1.3.2 live_rtc_concurrent_streams = 2 rtc_server_node_env = prod @@ -549,4 +549,4 @@ fileDescriptions[] = "XML document text";//mpd on ubuntu 14,12 fileDescriptions[] = "XML document text";//mpd on Fedora 11 [kmcng] -kmcng_version = v5.11.0 +kmcng_version = v5.12.0 From c010ffc4311ccb1bfa1e282311ada9e9c43377e6 Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Tue, 16 Jul 2019 02:05:40 +0300 Subject: [PATCH 094/103] PLAT-9998:code review remarks --- .../content/watch_later/WatchLaterPlugin.php | 77 +++++++++++++++++ .../lib/api/KalturaWatchLaterUserEntry.php} | 6 +- .../KalturaWatchLaterUserEntryFilter.php | 14 ++++ .../lib/model/WatchLaterUserEntry.php | 15 ++++ .../model/enums/WatchLaterUserEntryType.php} | 10 +-- .../WatchLaterPlaylistPlugin.php | 83 ------------------- .../lib/model/WatchLaterPlaylistUserEntry.php | 16 ---- release-notes.md | 2 +- 8 files changed, 115 insertions(+), 108 deletions(-) create mode 100644 plugins/content/watch_later/WatchLaterPlugin.php rename plugins/content/{watch_later_playlist/lib/api/KalturaWatchLaterPlaylistUserEntry.php => watch_later/lib/api/KalturaWatchLaterUserEntry.php} (60%) create mode 100644 plugins/content/watch_later/lib/api/filters/KalturaWatchLaterUserEntryFilter.php create mode 100644 plugins/content/watch_later/lib/model/WatchLaterUserEntry.php rename plugins/content/{watch_later_playlist/lib/model/enums/WatchLaterPlaylistUserEntryType.php => watch_later/lib/model/enums/WatchLaterUserEntryType.php} (51%) delete mode 100644 plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php delete mode 100644 plugins/content/watch_later_playlist/lib/model/WatchLaterPlaylistUserEntry.php diff --git a/plugins/content/watch_later/WatchLaterPlugin.php b/plugins/content/watch_later/WatchLaterPlugin.php new file mode 100644 index 00000000000..46abc8e7659 --- /dev/null +++ b/plugins/content/watch_later/WatchLaterPlugin.php @@ -0,0 +1,77 @@ +typeEqual = WatchLaterPlugin::getApiValue(WatchLaterUserEntryType::WATCH_LATER); + $response = parent::getListResponse($pager, $responseProfile); + return $response; + } +} diff --git a/plugins/content/watch_later/lib/model/WatchLaterUserEntry.php b/plugins/content/watch_later/lib/model/WatchLaterUserEntry.php new file mode 100644 index 00000000000..996eb068466 --- /dev/null +++ b/plugins/content/watch_later/lib/model/WatchLaterUserEntry.php @@ -0,0 +1,15 @@ +setType(WatchLaterPlugin::getWatchLaterUserEntryTypeCoreValue(WatchLaterUserEntryType::WATCH_LATER)); + parent::__construct(); + } +} \ No newline at end of file diff --git a/plugins/content/watch_later_playlist/lib/model/enums/WatchLaterPlaylistUserEntryType.php b/plugins/content/watch_later/lib/model/enums/WatchLaterUserEntryType.php similarity index 51% rename from plugins/content/watch_later_playlist/lib/model/enums/WatchLaterPlaylistUserEntryType.php rename to plugins/content/watch_later/lib/model/enums/WatchLaterUserEntryType.php index 2328d52e49f..e6ca14432b9 100644 --- a/plugins/content/watch_later_playlist/lib/model/enums/WatchLaterPlaylistUserEntryType.php +++ b/plugins/content/watch_later/lib/model/enums/WatchLaterUserEntryType.php @@ -1,11 +1,11 @@ self::WATCH_LATER_PLAYLIST, + 'WATCH_LATER' => self::WATCH_LATER, ); } @@ -23,7 +23,7 @@ public static function getAdditionalValues() public static function getAdditionalDescriptions() { return array( - self::WATCH_LATER_PLAYLIST => 'Watch Later Playlist User Entry Type', + self::WATCH_LATER => 'Watch Later User Entry Type', ); } } \ No newline at end of file diff --git a/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php b/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php deleted file mode 100644 index da230a727f8..00000000000 --- a/plugins/content/watch_later_playlist/WatchLaterPlaylistPlugin.php +++ /dev/null @@ -1,83 +0,0 @@ -getPluginEnabled(self::PLUGIN_NAME); - } - - /* (non-PHPdoc) - * @see IKalturaEnumerator::getEnums() - */ - public static function getEnums($baseEnumName = null) - { - if (is_null($baseEnumName)) - { - return array('WatchLaterPlaylistUserEntryType'); - } - if ($baseEnumName == 'UserEntryType') - { - return array('WatchLaterPlaylistUserEntryType'); - } - return array(); - } - - public static function getWatchLaterPlaylistUserEntryTypeCoreValue ($valueName) - { - $value = self::getPluginName() . IKalturaEnumerator::PLUGIN_VALUE_DELIMITER . $valueName; - return kPluginableEnumsManager::apiToCore('UserEntryType', $value); - } - - /** - * @return string external API value of dynamic enum. - */ - public static function getApiValue($valueName) - { - return self::getPluginName() . IKalturaEnumerator::PLUGIN_VALUE_DELIMITER . $valueName; - } - - public static function loadObject($baseClass, $enumValue, array $constructorArgs = null) - { - if ( ($baseClass == "KalturaUserEntry") && ($enumValue == self::getWatchLaterPlaylistUserEntryTypeCoreValue(WatchLaterPlaylistUserEntryType::WATCH_LATER_PLAYLIST))) - { - return new KalturaWatchLaterPlaylistUserEntry(); - } - if ( ($baseClass == "UserEntry") && ($enumValue == self::getWatchLaterPlaylistUserEntryTypeCoreValue(WatchLaterPlaylistUserEntryType::WATCH_LATER_PLAYLIST))) - { - return new WatchLaterPlaylistUserEntry(); - } - return null; - } - - public static function getObjectClass($baseClass, $enumValue) - { - if ($baseClass == 'UserEntry' && $enumValue == self::getWatchLaterPlaylistUserEntryTypeCoreValue(WatchLaterPlaylistUserEntryType::WATCH_LATER_PLAYLIST)) - { - return WatchLaterPlaylistUserEntry::WATCH_LATER_PLAYLIST_OM_CLASS; - } - return null; - } - } \ No newline at end of file diff --git a/plugins/content/watch_later_playlist/lib/model/WatchLaterPlaylistUserEntry.php b/plugins/content/watch_later_playlist/lib/model/WatchLaterPlaylistUserEntry.php deleted file mode 100644 index dbe66887e7b..00000000000 --- a/plugins/content/watch_later_playlist/lib/model/WatchLaterPlaylistUserEntry.php +++ /dev/null @@ -1,16 +0,0 @@ -setType(WatchLaterPlaylistPlugin::getWatchLaterPlaylistUserEntryTypeCoreValue(WatchLaterPlaylistUserEntryType::WATCH_LATER_PLAYLIST)); - parent::__construct(); - } - -} \ No newline at end of file diff --git a/release-notes.md b/release-notes.md index 76d6b061e54..16b726c3c9a 100644 --- a/release-notes.md +++ b/release-notes.md @@ -6,7 +6,7 @@ - Issue ID: PLAT-9998 ### configuration ### -Add WatchLaterPlaylist to your plugins.ini +Add WatchLater to your plugins.ini ### Deployment scripts ### php /opt/kaltura/app/deployment/base/scripts/installPlugins.php From 3e557b0a988f8fbfb1803710003870af798bb01c Mon Sep 17 00:00:00 2001 From: ravitshalem Date: Tue, 16 Jul 2019 02:20:21 +0300 Subject: [PATCH 095/103] PLAT-9998:editing release notes --- release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes.md b/release-notes.md index 16b726c3c9a..94029029307 100644 --- a/release-notes.md +++ b/release-notes.md @@ -1,6 +1,6 @@ # Orion 15.3.0 # -## Add new type of userEntry called watch later playlist ## +## Add new type of userEntry called watch later ## - Issue Type: Task - Issue ID: PLAT-9998 From dcf6dbb5605fa0c91c976ab4a8d7880083bf353b Mon Sep 17 00:00:00 2001 From: gotlieb Date: Tue, 16 Jul 2019 10:20:22 +0300 Subject: [PATCH 096/103] SUP-17451 - handle user list in case of more than 500 in results of query due to duplicates emails --- plugins/metadata/lib/api/filters/KalturaMetadataFilter.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/metadata/lib/api/filters/KalturaMetadataFilter.php b/plugins/metadata/lib/api/filters/KalturaMetadataFilter.php index d1be36c1ed6..7ae83643093 100644 --- a/plugins/metadata/lib/api/filters/KalturaMetadataFilter.php +++ b/plugins/metadata/lib/api/filters/KalturaMetadataFilter.php @@ -98,7 +98,8 @@ private function validateObjectIdFiltered() $kusers = !empty($objectIds) ? kuserPeer::getKuserByPartnerAndUids(kCurrentContext::getCurrentPartnerId(), $objectIds) : array(); $objectIds = array(); foreach($kusers as $kuser) - $objectIds[] = $kuser->getId(); + $objectIds[$kuser->getPuserId()] = $kuser->getId(); + $objectIds = array_values($objectIds); } elseif($this->metadataObjectTypeEqual == MetadataObjectType::CATEGORY) { From 6227d6197197d6ecebd39106cbdf54ec21f6bffd Mon Sep 17 00:00:00 2001 From: gotlieb Date: Tue, 16 Jul 2019 10:45:09 +0300 Subject: [PATCH 097/103] SUP-17451 - handle user list in case of more than 500 in results of query due to duplicates emails --- plugins/metadata/lib/api/filters/KalturaMetadataFilter.php | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/metadata/lib/api/filters/KalturaMetadataFilter.php b/plugins/metadata/lib/api/filters/KalturaMetadataFilter.php index 7ae83643093..1ee0d948e1e 100644 --- a/plugins/metadata/lib/api/filters/KalturaMetadataFilter.php +++ b/plugins/metadata/lib/api/filters/KalturaMetadataFilter.php @@ -99,7 +99,6 @@ private function validateObjectIdFiltered() $objectIds = array(); foreach($kusers as $kuser) $objectIds[$kuser->getPuserId()] = $kuser->getId(); - $objectIds = array_values($objectIds); } elseif($this->metadataObjectTypeEqual == MetadataObjectType::CATEGORY) { From bf4e0c33c20973e6e9567d92649dff59e9b5f17a Mon Sep 17 00:00:00 2001 From: moshemaorkaltura Date: Tue, 16 Jul 2019 17:37:55 +0300 Subject: [PATCH 098/103] PLAT-10023:issue in shceduled resource search due to esearch aggs --- plugins/beacon/lib/model/search/kScheduledResourceSearch.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/beacon/lib/model/search/kScheduledResourceSearch.php b/plugins/beacon/lib/model/search/kScheduledResourceSearch.php index 9f573c98fde..3b6e9c87433 100644 --- a/plugins/beacon/lib/model/search/kScheduledResourceSearch.php +++ b/plugins/beacon/lib/model/search/kScheduledResourceSearch.php @@ -29,7 +29,7 @@ public function doSearch(ESearchOperator $eSearchOperator, kPager $pager = null, return $result; } - protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null) + protected function initQuery(array $statuses, $objectId, kPager $pager = null, ESearchOrderBy $order = null, ESearchAggregations $aggregations=null) { $this->query = array(elasticClient::ELASTIC_INDEX_KEY => BeaconIndexName::SCHEDULED_RESOURCE_INDEX); $partnerId = kBaseElasticEntitlement::$partnerId; From dd9d28364ca71714db6641760d7435524f72a450 Mon Sep 17 00:00:00 2001 From: gotlieb Date: Tue, 16 Jul 2019 17:59:01 +0300 Subject: [PATCH 099/103] fix admin console otp exploit --- admin_console/controllers/UserController.php | 30 ++++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/admin_console/controllers/UserController.php b/admin_console/controllers/UserController.php index 70ea873952c..f55ac5f0231 100644 --- a/admin_console/controllers/UserController.php +++ b/admin_console/controllers/UserController.php @@ -162,22 +162,28 @@ public function loginAction() $adapter->setTimezoneOffset($request->getPost('timezone_offset')); $auth = Infra_AuthHelper::getAuthInstance(); - $result = $auth->authenticate($adapter); - - if ($result->isValid()) + try { - if ($request->getPost('remember_me')) - Zend_Session::rememberMe(60*60*24*7); // 1 week - - $nextUri = $this->_getParam('next_uri'); - if ($nextUri) - $this->_helper->redirector->gotoUrl($nextUri); + $result = $auth->authenticate($adapter); + if ($result->isValid()) + { + if ($request->getPost('remember_me')) + Zend_Session::rememberMe(60 * 60 * 24 * 7); // 1 week + + $nextUri = $this->_getParam('next_uri'); + if ($nextUri) + $this->_helper->redirector->gotoUrl($nextUri); + else + $this->_helper->redirector('list', 'partner'); + } else - $this->_helper->redirector('list', 'partner'); + { + $loginForm->setDescription('login error'); + } } - else + catch (Exception $ex) { - $loginForm->setDescription('login error'); + $loginForm->setDescription('login error ' . $ex->getMessage()); } } From fd633878f41825fd6a908c13233eb3664a48811c Mon Sep 17 00:00:00 2001 From: moshemaorkaltura Date: Wed, 17 Jul 2019 14:40:16 +0300 Subject: [PATCH 100/103] PLAT-10043: esearch aggs - do not return empty buckets --- .../items/KalturaESearchMetadataAggregationItem.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php index e89d0e375a7..e435cca36c7 100644 --- a/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php +++ b/plugins/search/providers/elastic_search/lib/api/types/aggregations/items/KalturaESearchMetadataAggregationItem.php @@ -51,6 +51,10 @@ public function coreToApiResponse($coreResponse, $fieldName=null) // loop over the sub aggregations $bucketsArray = new KalturaESearchAggregationBucketsArray(); $subBuckets = $bucket[ESearchMetadataAggregationItem::SUB_AGG][ESearchAggregations::BUCKETS]; + if(!$subBuckets) + { + continue; + } foreach($subBuckets as $subBucket) { $responseBucket = new KalturaESearchAggregationBucket(); From 3bd2fcc4d9e1ae56951cd2ab3c2d281706c8d32e Mon Sep 17 00:00:00 2001 From: ZurKaltura Date: Thu, 18 Jul 2019 11:16:46 +0300 Subject: [PATCH 101/103] Orion-15.3.0-PLAT-10020-zoom transcription label and language --- plugins/vendor/lib/model/zoom/kZoomEngine.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/vendor/lib/model/zoom/kZoomEngine.php b/plugins/vendor/lib/model/zoom/kZoomEngine.php index 5a28df6f7d8..8a387757953 100644 --- a/plugins/vendor/lib/model/zoom/kZoomEngine.php +++ b/plugins/vendor/lib/model/zoom/kZoomEngine.php @@ -14,6 +14,7 @@ class kZoomEngine const ZOOM_LOCK_TTL = 120; const ZOOM_TRANSCRIPT_FILE_TYPE = 'vtt'; const ZOOM_CHAT_FILE_TYPE = 'txt'; + const ZOOM_LABEL = 'Zoom'; protected static $FILE_VIDEO_TYPES = array('MP4'); protected static $FILE_CAPTION_TYPES = array('TRANSCRIPT'); @@ -328,6 +329,8 @@ protected function createAssetForTranscription($entry) $caption = new CaptionAsset(); $caption->setEntryId($entry->getId()); $caption->setPartnerId($entry->getPartnerId()); + $caption->setLanguage(KalturaLanguage::EN); + $caption->setLabel(self::ZOOM_LABEL); $caption->setContainerFormat(CaptionType::WEBVTT); $caption->setStatus(CaptionAsset::ASSET_STATUS_QUEUED); $caption->setFileExt(self::ZOOM_TRANSCRIPT_FILE_TYPE); From 68a9de36952ff1023cc048c1cb3e2ec9a1b1cfcb Mon Sep 17 00:00:00 2001 From: Oren Me Date: Thu, 18 Jul 2019 12:03:43 +0300 Subject: [PATCH 102/103] update player to 2.77.1 --- configurations/base.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/base.ini b/configurations/base.ini index 95073f1b057..030f68f4780 100644 --- a/configurations/base.ini +++ b/configurations/base.ini @@ -97,7 +97,7 @@ editors_flex_wrapper_version = v1.01 kdp_wrapper_version = v11.0 kdp3_wrapper_stats_url = kdp3_wrapper_version = v34.0 -html5_version = v2.76 +html5_version = v2.77.1 clipapp_version = v1.3 kmc_secured_login = false studio_version = v2.2.1 From e037e7b9c05182321d8ee418cf97e773c186df5a Mon Sep 17 00:00:00 2001 From: moshemaorkaltura Date: Thu, 18 Jul 2019 14:54:41 +0300 Subject: [PATCH 103/103] PLAT-10043: set access control profile correct type for esearch ags --- .../lib/model/enum/ESearchEntryAggregationFieldName.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryAggregationFieldName.php b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryAggregationFieldName.php index 0b3c27268c6..5961774c055 100644 --- a/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryAggregationFieldName.php +++ b/plugins/search/providers/elastic_search/lib/model/enum/ESearchEntryAggregationFieldName.php @@ -9,5 +9,5 @@ interface ESearchEntryAggregationFieldName extends BaseEnum const ENTRY_TYPE = 'entry_type'; const MEDIA_TYPE = 'media_type'; const TAGS = 'tags.raw'; - const ACCESS_CONTROL_PROFILE = 'access_control_profile_id'; + const ACCESS_CONTROL_PROFILE = 'access_control_id'; } \ No newline at end of file