From 6a8cde0faf4d37da5f44d3a496ecb22fdf11f383 Mon Sep 17 00:00:00 2001 From: Chloe Liban Date: Mon, 8 Jul 2019 18:00:15 +0200 Subject: [PATCH 1/3] Addition of gzip feature --- composer.json | 1 + src/Config/AbstractConfig.php | 25 +++++++++++++++++++++++++ src/Config/SearchConfig.php | 1 + src/Http/Psr7/Request.php | 4 ++++ src/RetryStrategy/ApiWrapper.php | 18 ++++++++++++++++++ tests/Integration/IndexingTest.php | 1 - 6 files changed, 49 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 9ef06fb55..6f2e6d86e 100644 --- a/composer.json +++ b/composer.json @@ -15,6 +15,7 @@ "ext-curl": "*", "ext-json": "*", "ext-mbstring": "*", + "ext-zlib": "^7.3", "psr/http-message": "^1.0", "psr/log": "^1.0", "psr/simple-cache": "^1.0" diff --git a/src/Config/AbstractConfig.php b/src/Config/AbstractConfig.php index 60ee1597a..320ccd6ec 100644 --- a/src/Config/AbstractConfig.php +++ b/src/Config/AbstractConfig.php @@ -29,6 +29,7 @@ public function getDefaultConfig() 'writeTimeout' => $this->defaultWriteTimeout, 'connectTimeout' => $this->defaultConnectTimeout, 'defaultHeaders' => array(), + 'gzipEnabled' => false, ); } @@ -115,4 +116,28 @@ public function setDefaultHeaders(array $defaultHeaders) return $this; } + + /** + * @return boolean + */ + public function getGzipEnabled() + { + return $this->config['gzipEnabled']; + } + + /** + * @param boolean $gzipEnabled + * + * @return $this + */ + public function setGzipEnabled($gzipEnabled) + { + if (!is_bool($gzipEnabled)) { + throw new \InvalidArgumentException('Default configuration for GzipEnabled must be a boolean'); + } + + $this->config['gzipEnabled'] = $gzipEnabled; + + return $this; + } } diff --git a/src/Config/SearchConfig.php b/src/Config/SearchConfig.php index 516cec589..7db9375a1 100644 --- a/src/Config/SearchConfig.php +++ b/src/Config/SearchConfig.php @@ -29,6 +29,7 @@ public function getDefaultConfig() 'defaultHeaders' => array(), 'defaultForwardToReplicas' => null, 'batchSize' => 1000, + 'gzipEnabled' => true, ); } diff --git a/src/Http/Psr7/Request.php b/src/Http/Psr7/Request.php index 757176acb..d032c4d3f 100644 --- a/src/Http/Psr7/Request.php +++ b/src/Http/Psr7/Request.php @@ -62,6 +62,10 @@ public function __construct( $this->updateHostFromUri(); } + if ($this->hasHeader('Content-Encoding')) { + $body = gzencode($body, 9); + } + if ('' !== $body && null !== $body) { $this->stream = stream_for($body); } diff --git a/src/RetryStrategy/ApiWrapper.php b/src/RetryStrategy/ApiWrapper.php index 269129485..3aeb01a6d 100644 --- a/src/RetryStrategy/ApiWrapper.php +++ b/src/RetryStrategy/ApiWrapper.php @@ -111,6 +111,10 @@ public function send($method, $path, $requestOptions = array(), $hosts = null) private function request($method, $path, RequestOptions $requestOptions, $hosts, $timeout, $data = array()) { + if ($this->canEnableGzipCompress($method)) { + $requestOptions->addHeader('Content-Encoding', 'gzip'); + } + $uri = $this->createUri($path) ->withQuery($requestOptions->getBuiltQueryParameters()) ->withScheme('https'); @@ -215,6 +219,14 @@ private function createUri($uri) throw new \InvalidArgumentException('URI must be a string or UriInterface'); } + /** + * @param $method + * @param $uri + * @param array $headers + * @param null $body + * @param string $protocolVersion + * @return Request + */ private function createRequest( $method, $uri, @@ -239,6 +251,12 @@ private function createRequest( return new Request($method, $uri, $headers, $body, $protocolVersion); } + private function canEnableGzipCompress($method) + { + return (strtoupper($method) === 'POST' || strtoupper($method) === 'PUT') + && $this->config->getGzipEnabled(); + } + /** * @param string $level * @param string $message diff --git a/tests/Integration/IndexingTest.php b/tests/Integration/IndexingTest.php index d623f6071..2f0dd0075 100644 --- a/tests/Integration/IndexingTest.php +++ b/tests/Integration/IndexingTest.php @@ -56,7 +56,6 @@ public function testIndexing() $multiResponse->wait(); /* Check 6 first records with getObject */ - $objectID1 = $responses[0][0]['objectIDs'][0]; $objectID2 = $responses[1][0]['objectIDs'][0]; $objectID3 = $responses[2][0]['objectIDs'][0]; From 9304e3dee064e0215c6fcae5c7440efb68efc9f0 Mon Sep 17 00:00:00 2001 From: Chloe Liban Date: Wed, 10 Jul 2019 11:31:58 +0200 Subject: [PATCH 2/3] adds: compression types and tests --- composer.json | 2 +- src/Config/AbstractConfig.php | 17 ++++++++++++----- src/Config/SearchConfig.php | 2 +- src/Http/Psr7/Request.php | 4 ---- src/RetryStrategy/ApiWrapper.php | 24 +++++++++++++----------- tests/Unit/CopyResourcesTest.php | 15 ++++++++++++--- tests/Unit/RequestTestCase.php | 25 +++++++++++++++++++++++++ tests/Unit/SearchTest.php | 27 ++++++++++++++++++++++----- 8 files changed, 86 insertions(+), 30 deletions(-) diff --git a/composer.json b/composer.json index 6f2e6d86e..8e8dbdee1 100644 --- a/composer.json +++ b/composer.json @@ -15,7 +15,7 @@ "ext-curl": "*", "ext-json": "*", "ext-mbstring": "*", - "ext-zlib": "^7.3", + "ext-zlib": "*", "psr/http-message": "^1.0", "psr/log": "^1.0", "psr/simple-cache": "^1.0" diff --git a/src/Config/AbstractConfig.php b/src/Config/AbstractConfig.php index 320ccd6ec..7f68ffd56 100644 --- a/src/Config/AbstractConfig.php +++ b/src/Config/AbstractConfig.php @@ -4,6 +4,9 @@ abstract class AbstractConfig { + const COMPRESSION_TYPE_NONE = 'none'; + const COMPRESSION_TYPE_GZIP = 'gzip'; + protected $config; protected $defaultReadTimeout = 5; @@ -29,7 +32,7 @@ public function getDefaultConfig() 'writeTimeout' => $this->defaultWriteTimeout, 'connectTimeout' => $this->defaultConnectTimeout, 'defaultHeaders' => array(), - 'gzipEnabled' => false, + 'gzipEnabled' => self::COMPRESSION_TYPE_NONE, ); } @@ -118,7 +121,7 @@ public function setDefaultHeaders(array $defaultHeaders) } /** - * @return boolean + * @return bool */ public function getGzipEnabled() { @@ -126,14 +129,18 @@ public function getGzipEnabled() } /** - * @param boolean $gzipEnabled + * @param string $gzipEnabled * * @return $this */ public function setGzipEnabled($gzipEnabled) { - if (!is_bool($gzipEnabled)) { - throw new \InvalidArgumentException('Default configuration for GzipEnabled must be a boolean'); + if (!in_array( + $gzipEnabled, + array(self::COMPRESSION_TYPE_GZIP, self::COMPRESSION_TYPE_NONE), + true + )) { + throw new \InvalidArgumentException('gzipEnabled must be equal to '.self::COMPRESSION_TYPE_GZIP.' or '.self::COMPRESSION_TYPE_NONE); } $this->config['gzipEnabled'] = $gzipEnabled; diff --git a/src/Config/SearchConfig.php b/src/Config/SearchConfig.php index 7db9375a1..d67cfb3a4 100644 --- a/src/Config/SearchConfig.php +++ b/src/Config/SearchConfig.php @@ -29,7 +29,7 @@ public function getDefaultConfig() 'defaultHeaders' => array(), 'defaultForwardToReplicas' => null, 'batchSize' => 1000, - 'gzipEnabled' => true, + 'gzipEnabled' => self::COMPRESSION_TYPE_GZIP, ); } diff --git a/src/Http/Psr7/Request.php b/src/Http/Psr7/Request.php index d032c4d3f..757176acb 100644 --- a/src/Http/Psr7/Request.php +++ b/src/Http/Psr7/Request.php @@ -62,10 +62,6 @@ public function __construct( $this->updateHostFromUri(); } - if ($this->hasHeader('Content-Encoding')) { - $body = gzencode($body, 9); - } - if ('' !== $body && null !== $body) { $this->stream = stream_for($body); } diff --git a/src/RetryStrategy/ApiWrapper.php b/src/RetryStrategy/ApiWrapper.php index 3aeb01a6d..569e7f4cb 100644 --- a/src/RetryStrategy/ApiWrapper.php +++ b/src/RetryStrategy/ApiWrapper.php @@ -111,8 +111,11 @@ public function send($method, $path, $requestOptions = array(), $hosts = null) private function request($method, $path, RequestOptions $requestOptions, $hosts, $timeout, $data = array()) { - if ($this->canEnableGzipCompress($method)) { + $canCompress = $this->canEnableGzipCompress($method); + + if ($canCompress) { $requestOptions->addHeader('Content-Encoding', 'gzip'); + $requestOptions->addHeader('Content-Length', null); } $uri = $this->createUri($path) @@ -139,6 +142,7 @@ private function request($method, $path, RequestOptions $requestOptions, $hosts, $request = $this->createRequest( $method, $uri, + $canCompress, $requestOptions->getHeaders(), $body ); @@ -219,17 +223,10 @@ private function createUri($uri) throw new \InvalidArgumentException('URI must be a string or UriInterface'); } - /** - * @param $method - * @param $uri - * @param array $headers - * @param null $body - * @param string $protocolVersion - * @return Request - */ private function createRequest( $method, $uri, + $canCompress, array $headers = array(), $body = null, $protocolVersion = '1.1' @@ -248,13 +245,18 @@ private function createRequest( } } + if ($canCompress) { + $body = gzencode($body, 9); + $headers['Content-Length'] = strlen($body); + } + return new Request($method, $uri, $headers, $body, $protocolVersion); } private function canEnableGzipCompress($method) { - return (strtoupper($method) === 'POST' || strtoupper($method) === 'PUT') - && $this->config->getGzipEnabled(); + return (AbstractConfig::COMPRESSION_TYPE_GZIP === $this->config->getGzipEnabled()) + && ('POST' === strtoupper($method) || 'PUT' === strtoupper($method)); } /** diff --git a/tests/Unit/CopyResourcesTest.php b/tests/Unit/CopyResourcesTest.php index f6018a444..27f75eb42 100644 --- a/tests/Unit/CopyResourcesTest.php +++ b/tests/Unit/CopyResourcesTest.php @@ -22,7 +22,10 @@ public function testCopySettings() static::$client->copySettings('src', 'dest'); } catch (RequestException $e) { $this->assertEndpointEquals($e->getRequest(), '/1/indexes/src/operation'); - $this->assertBodySubset(array( + $this->assertHeaderIsSet('Content-Encoding', $e->getRequest()); + $this->assertHeaderIsSet('Content-Length', $e->getRequest()); + $this->assertBodyEncoded($e->getRequest()); + $this->assertEncodedBodySubset(array( 'operation' => 'copy', 'destination' => 'dest', 'scope' => array('settings'), @@ -38,7 +41,10 @@ public function testCopySynonyms() static::$client->copySynonyms('src', 'dest'); } catch (RequestException $e) { $this->assertEndpointEquals($e->getRequest(), '/1/indexes/src/operation'); - $this->assertBodySubset(array( + $this->assertHeaderIsSet('Content-Encoding', $e->getRequest()); + $this->assertHeaderIsSet('Content-Length', $e->getRequest()); + $this->assertBodyEncoded($e->getRequest()); + $this->assertEncodedBodySubset(array( 'operation' => 'copy', 'destination' => 'dest', 'scope' => array('synonyms'), @@ -54,7 +60,10 @@ public function testCopyRules() static::$client->copyRules('src', 'dest'); } catch (RequestException $e) { $this->assertEndpointEquals($e->getRequest(), '/1/indexes/src/operation'); - $this->assertBodySubset(array( + $this->assertHeaderIsSet('Content-Encoding', $e->getRequest()); + $this->assertHeaderIsSet('Content-Length', $e->getRequest()); + $this->assertBodyEncoded($e->getRequest()); + $this->assertEncodedBodySubset(array( 'operation' => 'copy', 'destination' => 'dest', 'scope' => array('rules'), diff --git a/tests/Unit/RequestTestCase.php b/tests/Unit/RequestTestCase.php index 0c48ed4fb..cac259543 100644 --- a/tests/Unit/RequestTestCase.php +++ b/tests/Unit/RequestTestCase.php @@ -32,6 +32,14 @@ protected function assertBodySubset($subset, RequestInterface $request) $this->assertArraySubset($subset, $body, true); } + protected function assertEncodedBodySubset( + $subset, + RequestInterface $request + ) { + $body = json_decode(gzdecode($request->getBody()), true); + $this->assertArraySubset($subset, $body, true); + } + protected function assertQueryParametersSubset(array $subset, RequestInterface $request) { $params = $this->requestQueryParametersToArray($request); @@ -44,6 +52,23 @@ protected function assertQueryParametersNotHasKey($key, RequestInterface $reques $this->assertArrayNotHasKey($key, $params); } + protected function assertBodyEncoded(RequestInterface $request) + { + return gzdecode($request->getBody()); + } + + protected function assertHeaderIsSet($headerName, RequestInterface $request) + { + $this->assertArrayHasKey($headerName, $request->getHeaders()); + } + + protected function assertHeaderIsNotSet( + $headerName, + RequestInterface $request + ) { + $this->assertArrayNotHasKey($headerName, $request->getHeaders()); + } + private function requestQueryParametersToArray(RequestInterface $request) { $array = array(); diff --git a/tests/Unit/SearchTest.php b/tests/Unit/SearchTest.php index 743e73895..b10dae260 100644 --- a/tests/Unit/SearchTest.php +++ b/tests/Unit/SearchTest.php @@ -15,7 +15,10 @@ public function testQueryAsNullValue() try { $client->searchUserIds(null); } catch (RequestException $e) { - $this->assertBodySubset(array('query' => ''), $e->getRequest()); + $this->assertHeaderIsSet('Content-Encoding', $e->getRequest()); + $this->assertHeaderIsSet('Content-Length', $e->getRequest()); + $this->assertEncodedBodySubset(array('query' => ''), + $e->getRequest()); } $index = $client->initIndex('foo'); @@ -23,25 +26,37 @@ public function testQueryAsNullValue() try { $index->search(null); } catch (RequestException $e) { - $this->assertBodySubset(array('query' => ''), $e->getRequest()); + $this->assertHeaderIsSet('Content-Encoding', $e->getRequest()); + $this->assertHeaderIsSet('Content-Length', $e->getRequest()); + $this->assertEncodedBodySubset(array('query' => ''), + $e->getRequest()); } try { $index->searchSynonyms(null); } catch (RequestException $e) { - $this->assertBodySubset(array('query' => ''), $e->getRequest()); + $this->assertHeaderIsSet('Content-Encoding', $e->getRequest()); + $this->assertHeaderIsSet('Content-Length', $e->getRequest()); + $this->assertEncodedBodySubset(array('query' => ''), + $e->getRequest()); } try { $index->searchRules(null); } catch (RequestException $e) { - $this->assertBodySubset(array('query' => ''), $e->getRequest()); + $this->assertHeaderIsSet('Content-Encoding', $e->getRequest()); + $this->assertHeaderIsSet('Content-Length', $e->getRequest()); + $this->assertEncodedBodySubset(array('query' => ''), + $e->getRequest()); } try { $index->searchRules(null); } catch (RequestException $e) { - $this->assertBodySubset(array('query' => ''), $e->getRequest()); + $this->assertHeaderIsSet('Content-Encoding', $e->getRequest()); + $this->assertHeaderIsSet('Content-Length', $e->getRequest()); + $this->assertEncodedBodySubset(array('query' => ''), + $e->getRequest()); } $client = PlacesClient::create('id', 'key'); @@ -49,6 +64,8 @@ public function testQueryAsNullValue() try { $client->search(null); } catch (RequestException $e) { + $this->assertHeaderIsNotSet('Content-Encoding', $e->getRequest()); + $this->assertHeaderIsNotSet('Content-Length', $e->getRequest()); $this->assertBodySubset(array('query' => ''), $e->getRequest()); } } From a38eae2ae468e54cb9e64cc0575bb8fceea67830 Mon Sep 17 00:00:00 2001 From: Nuno Maduro Date: Wed, 17 Jul 2019 08:33:09 -0700 Subject: [PATCH 3/3] Adds work in progress concerning gzip/null compressors --- src/Compressors/Compressor.php | 21 +++++++ src/Compressors/CompressorFactory.php | 29 ++++++++++ src/Compressors/GzipCompressor.php | 21 +++++++ src/Compressors/NullCompressor.php | 16 ++++++ src/Config/AbstractConfig.php | 18 +++--- src/Config/SearchConfig.php | 2 +- src/RetryStrategy/ApiWrapper.php | 82 ++++++++++++--------------- 7 files changed, 132 insertions(+), 57 deletions(-) create mode 100644 src/Compressors/Compressor.php create mode 100644 src/Compressors/CompressorFactory.php create mode 100644 src/Compressors/GzipCompressor.php create mode 100644 src/Compressors/NullCompressor.php diff --git a/src/Compressors/Compressor.php b/src/Compressors/Compressor.php new file mode 100644 index 000000000..785d9cdc5 --- /dev/null +++ b/src/Compressors/Compressor.php @@ -0,0 +1,21 @@ +addHeader('Content-Encoding', 'gzip'); + $requestOptions->addHeader('Content-Length', strlen($compressedBody)); + + return $compressedBody; + } +} diff --git a/src/Compressors/NullCompressor.php b/src/Compressors/NullCompressor.php new file mode 100644 index 000000000..7f9293e83 --- /dev/null +++ b/src/Compressors/NullCompressor.php @@ -0,0 +1,16 @@ + $this->defaultWriteTimeout, 'connectTimeout' => $this->defaultConnectTimeout, 'defaultHeaders' => array(), - 'gzipEnabled' => self::COMPRESSION_TYPE_NONE, + 'compressionType' => self::COMPRESSION_TYPE_NONE, ); } @@ -121,29 +121,29 @@ public function setDefaultHeaders(array $defaultHeaders) } /** - * @return bool + * @return string */ - public function getGzipEnabled() + public function getCompressionType() { - return $this->config['gzipEnabled']; + return $this->config['compressionType']; } /** - * @param string $gzipEnabled + * @param string $compressionType * * @return $this */ - public function setGzipEnabled($gzipEnabled) + public function setCompressionType($compressionType) { if (!in_array( - $gzipEnabled, + $compressionType, array(self::COMPRESSION_TYPE_GZIP, self::COMPRESSION_TYPE_NONE), true )) { - throw new \InvalidArgumentException('gzipEnabled must be equal to '.self::COMPRESSION_TYPE_GZIP.' or '.self::COMPRESSION_TYPE_NONE); + throw new \InvalidArgumentException('Compression type not supported'); } - $this->config['gzipEnabled'] = $gzipEnabled; + $this->config['compressionType'] = $compressionType; return $this; } diff --git a/src/Config/SearchConfig.php b/src/Config/SearchConfig.php index d67cfb3a4..e4cbb2583 100644 --- a/src/Config/SearchConfig.php +++ b/src/Config/SearchConfig.php @@ -29,7 +29,7 @@ public function getDefaultConfig() 'defaultHeaders' => array(), 'defaultForwardToReplicas' => null, 'batchSize' => 1000, - 'gzipEnabled' => self::COMPRESSION_TYPE_GZIP, + 'compressionType' => self::COMPRESSION_TYPE_GZIP, ); } diff --git a/src/RetryStrategy/ApiWrapper.php b/src/RetryStrategy/ApiWrapper.php index 569e7f4cb..ce1eb49f1 100644 --- a/src/RetryStrategy/ApiWrapper.php +++ b/src/RetryStrategy/ApiWrapper.php @@ -3,6 +3,7 @@ namespace Algolia\AlgoliaSearch\RetryStrategy; use Algolia\AlgoliaSearch\Algolia; +use Algolia\AlgoliaSearch\Compressors\CompressorFactory; use Algolia\AlgoliaSearch\Config\AbstractConfig; use Algolia\AlgoliaSearch\Exceptions\AlgoliaException; use Algolia\AlgoliaSearch\Exceptions\BadRequestException; @@ -42,6 +43,11 @@ final class ApiWrapper implements ApiWrapperInterface */ private $requestOptionsFactory; + /** + * @var \Algolia\AlgoliaSearch\Compressors\Compressor + */ + private $compressor; + public function __construct( HttpClientInterface $http, AbstractConfig $config, @@ -52,6 +58,8 @@ public function __construct( $this->config = $config; $this->clusterHosts = $clusterHosts; $this->requestOptionsFactory = $RqstOptsFactory ?: new RequestOptionsFactory($config); + + $this->compressor = CompressorFactory::create($this->config->getCompressionType()); } public function read($method, $path, $requestOptions = array(), $defaultRequestOptions = array()) @@ -111,18 +119,17 @@ public function send($method, $path, $requestOptions = array(), $hosts = null) private function request($method, $path, RequestOptions $requestOptions, $hosts, $timeout, $data = array()) { - $canCompress = $this->canEnableGzipCompress($method); - - if ($canCompress) { - $requestOptions->addHeader('Content-Encoding', 'gzip'); - $requestOptions->addHeader('Content-Length', null); - } - $uri = $this->createUri($path) ->withQuery($requestOptions->getBuiltQueryParameters()) ->withScheme('https'); - $body = array_merge($data, $requestOptions->getBody()); + $body = $this->sanitizeBody(array_merge($data, $requestOptions->getBody())); + + if ('POST' === strtoupper($method) || 'PUT' === strtoupper($method)) { + $compressedBody = $this->compressor->compress($requestOptions, $body); + } else { + $compressedBody = $body; + } $logParams = array( 'body' => $body, @@ -132,6 +139,7 @@ private function request($method, $path, RequestOptions $requestOptions, $hosts, ); $retry = 1; + foreach ($hosts as $host) { $uri = $uri->withHost($host); $request = null; @@ -139,13 +147,7 @@ private function request($method, $path, RequestOptions $requestOptions, $hosts, $logParams['host'] = (string) $uri; try { - $request = $this->createRequest( - $method, - $uri, - $canCompress, - $requestOptions->getHeaders(), - $body - ); + $request = new Request($method, $uri, $requestOptions->getHeaders(), $compressedBody, '1.1'); $this->log(LogLevel::DEBUG, 'Sending request.', $logParams); @@ -223,17 +225,24 @@ private function createUri($uri) throw new \InvalidArgumentException('URI must be a string or UriInterface'); } - private function createRequest( - $method, - $uri, - $canCompress, - array $headers = array(), - $body = null, - $protocolVersion = '1.1' - ) { + /** + * @param string $level + * @param string $message + * @param array $context + */ + private function log($level, $message, array $context = array()) + { + Algolia::getLogger()->log($level, 'Algolia API client: '.$message, $context); + } + + /** + * @param string $body + * + * @return string + */ + private function sanitizeBody($body) + { if (is_array($body)) { - // Send an empty body instead of "[]" in case there are - // no content/params to send if (empty($body)) { $body = ''; } else { @@ -245,27 +254,6 @@ private function createRequest( } } - if ($canCompress) { - $body = gzencode($body, 9); - $headers['Content-Length'] = strlen($body); - } - - return new Request($method, $uri, $headers, $body, $protocolVersion); - } - - private function canEnableGzipCompress($method) - { - return (AbstractConfig::COMPRESSION_TYPE_GZIP === $this->config->getGzipEnabled()) - && ('POST' === strtoupper($method) || 'PUT' === strtoupper($method)); - } - - /** - * @param string $level - * @param string $message - * @param array $context - */ - private function log($level, $message, array $context = array()) - { - Algolia::getLogger()->log($level, 'Algolia API client: '.$message, $context); + return $body; } }