Merge pull request #302 from matomo-org/develop

Update master for 1.1.2 release
matomo-org · Jun 8, 2020 · c1e58b8 · c1e58b8
2 parents e432271 + 714b351
commit c1e58b8
Show file tree

Hide file tree

Showing 33 changed files with 595 additions and 64 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,10 @@
 == Changelog ==
 
+= 1.1.2 =
+* Update core to Matomo 3.13.6
+* Improve installation
+* Fix city report cannot be loaded
+
 = 1.1.1 =
 * Fix some settings were not accessible in WP Multisite mode when plugin is network enabled
 * Ensure utf8mb4 upgrade works when when large indexes are disabled

diff --git a/app/core/Concurrency/Lock.php b/app/core/Concurrency/Lock.php
@@ -8,11 +8,14 @@
  */
 namespace Piwik\Concurrency;
 
+use Piwik\ArchiveProcessor\ArchivingStatus;
 use Piwik\Common;
+use Piwik\Date;
 
 class Lock
 {
     const MAX_KEY_LEN = 70;
+    const DEFAULT_TTL = 60;
 
     /**
      * @var LockBackend
@@ -24,18 +27,28 @@ class Lock
     private $lockKey   = null;
     private $lockValue = null;
     private $defaultTtl = null;
+    private $lastExpireTime = null;
 
     public function __construct(LockBackend $backend, $lockKeyStart, $defaultTtl = null)
     {
         $this->backend = $backend;
         $this->lockKeyStart = $lockKeyStart;
         $this->lockKey = $this->lockKeyStart;
-        $this->defaultTtl = $defaultTtl;
+        $this->defaultTtl = $defaultTtl ?: self::DEFAULT_TTL;
     }
 
     public function reexpireLock()
     {
-        $this->expireLock($this->defaultTtl);
+        $timeBetweenReexpires = $this->defaultTtl - ($this->defaultTtl / 4);
+
+        $now = Date::getNowTimestamp();
+        if (!empty($this->lastExpireTime) &&
+            $now <= $this->lastExpireTime + $timeBetweenReexpires
+        ) {
+            return false;
+        }
+
+        return $this->expireLock($this->defaultTtl);
     }
 
     public function getNumberOfAcquiredLocks()
@@ -81,6 +94,8 @@ public function acquireLock($id, $ttlInSeconds = 60)
 
         if ($locked) {
             $this->lockValue = $lockValue;
+            $this->ttlUsed = $ttlInSeconds;
+            $this->lastExpireTime = Date::getNowTimestamp();
         }
 
         return $locked;
@@ -125,6 +140,8 @@ public function expireLock($ttlInSeconds)
                     return false;
                 }
 
+                $this->lastExpireTime = Date::getNowTimestamp();
+
                 return true;
             } else {
                 Common::printDebug('Lock is not acquired, cannot update expiration.');

diff --git a/app/core/Config/IniFileChain.php b/app/core/Config/IniFileChain.php
@@ -97,9 +97,36 @@ public function getFrom($file, $name)
      */
     public function set($name, $value)
     {
+	    $name = $this->replaceSectionInvalidChars($name);
+	    if ($value !== null) {
+		    $value = $this->replaceInvalidChars($value);
+	    }
+
         $this->mergedSettings[$name] = $value;
     }
 
+	private function replaceInvalidChars($value)
+	{
+		if (is_array($value)) {
+			$result = [];
+			foreach ($value as $key => $arrayValue) {
+				$key = $this->replaceInvalidChars($key);
+				if (is_array($arrayValue)) {
+					$arrayValue = $this->replaceInvalidChars($arrayValue);
+				}
+
+				$result[$key] = $arrayValue;
+			}
+			return $result;
+		} else {
+			return preg_replace('/[^a-zA-Z0-9_\[\]-]/', '', $value);
+		}
+	}
+
+	private function replaceSectionInvalidChars($value)
+	{
+		return preg_replace('/[^a-zA-Z0-9_-]/', '', $value);
+	}
     /**
      * Returns all settings. Changes made to the array result will be reflected in the
      * IniFileChain instance.

diff --git a/app/core/DataAccess/ArchiveSelector.php b/app/core/DataAccess/ArchiveSelector.php
@@ -89,11 +89,14 @@ public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params
         ) { // the archive cannot be considered valid for this request (has wrong done flag value)
             return [false, $visits, $visitsConverted, true];
         }
+	    if (!empty($minDatetimeArchiveProcessedUTC) && !is_object($minDatetimeArchiveProcessedUTC)) {
+		    $minDatetimeArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC);
+	    }
 
         // the archive is too old
         if ($minDatetimeArchiveProcessedUTC
             && isset($result['idarchive'])
-            && Date::factory($result['ts_archived'])->isEarlier(Date::factory($minDatetimeArchiveProcessedUTC))
+            && Date::factory($result['ts_archived'])->isEarlier($minDatetimeArchiveProcessedUTC)
         ) {
             return [false, $visits, $visitsConverted, true];
         }

diff --git a/app/core/DataAccess/ArchivingDbAdapter.php b/app/core/DataAccess/ArchivingDbAdapter.php
@@ -31,11 +31,6 @@ class ArchivingDbAdapter
      */
     private $logger;
 
-    /**
-     * @var int
-     */
-    private $lastReexpireTime = null;
-
     public function __construct($wrapped, Lock $archivingLock = null, LoggerInterface $logger = null)
     {
         $this->wrapped = $wrapped;
@@ -107,11 +102,7 @@ private function logSql($sql)
     private function reexpireLock()
     {
         if ($this->archivingLock) {
-            $timeBetweenReexpires = ArchivingStatus::DEFAULT_ARCHIVING_TTL / 4;
-            if ($this->lastReexpireTime + $timeBetweenReexpires < time()) {
-                $this->archivingLock->reexpireLock();
-                $this->lastReexpireTime = time();
-            }
+            $this->archivingLock->reexpireLock();
         }
     }
 }
diff --git a/app/core/DataAccess/Model.php b/app/core/DataAccess/Model.php
@@ -59,7 +59,7 @@ public function getInvalidatedArchiveIdsSafeToDelete($archiveTable)
                   FROM `$archiveTable`
                  WHERE name LIKE 'done%'
                    AND `value` NOT IN (" . ArchiveWriter::DONE_ERROR . ")
-              GROUP BY idsite, date1, date2, period, name";
+              GROUP BY idsite, date1, date2, period, name HAVING count(*) > 1";
 
         $archiveIds = array();
 

diff --git a/app/core/Tracker/FingerprintSalt.php b/app/core/Tracker/FingerprintSalt.php
@@ -0,0 +1,88 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Tracker;
+
+use Piwik\Common;
+use Piwik\Date;
+use Piwik\Exception\InvalidRequestParameterException;
+use Piwik\Exception\UnexpectedWebsiteFoundException;
+use Piwik\Option;
+use Piwik\Piwik;
+use Piwik\SettingsServer;
+use Piwik\Site;
+use Piwik\Db as PiwikDb;
+
+class FingerprintSalt
+{
+    const OPTION_PREFIX = 'fingerprint_salt_';
+    const DELETE_FINGERPRINT_OLDER_THAN_SECONDS = 432000; // 5 days in seconds
+
+    public function generateSalt()
+    {
+        return Common::getRandomString(32);
+    }
+
+    public function deleteOldSalts()
+    {
+        // we want to make sure to delete salts that were created more than three days ago as they are likely not in
+        // use anymore. We should delete them to ensure the fingerprint is truly random for each day because if we used
+        // eg the regular salt then it would technically still be possible to try and regenerate the fingerprint based
+        // on certain information.
+        // Typically, only the salts for today and yesterday are used. However, if someone was to import historical data
+        // for the same day and this takes more than five days, then it could technically happen that we delete a
+        // fingerprint that is still in use now and as such after deletion a few visitors would have a new configId
+        // within one visit and such a new visit would be created. That should be very much edge case though.
+        $deleteSaltsCreatedBefore = Date::getNowTimestamp() - self::DELETE_FINGERPRINT_OLDER_THAN_SECONDS;
+        $options = Option::getLike(self::OPTION_PREFIX . '%');
+        $deleted = array();
+        foreach ($options as $name => $value) {
+            $value = $this->decode($value);
+            if (empty($value['time']) || $value['time'] < $deleteSaltsCreatedBefore) {
+                Option::delete($name);
+                $deleted[] = $name;
+            }
+        }
+
+        return $deleted;
+    }
+
+    public function getDateString(Date $date, $timezone)
+    {
+        $dateString = Date::factory($date->getTimestampUTC(), $timezone)->toString();
+        return $dateString;
+    }
+
+    private function encode($value)
+    {
+        return json_encode($value);
+    }
+
+    private function decode($value)
+    {
+        return @json_decode($value, true);
+    }
+
+    public function getSalt($dateString, $idSite)
+    {
+        $fingerprintSaltKey = self::OPTION_PREFIX . (int) $idSite . '_' . $dateString;
+        $salt = Option::get($fingerprintSaltKey);
+        if (!empty($salt)) {
+            $salt = $this->decode($salt);
+        }
+        if (empty($salt['value'])) {
+            $salt = array(
+                'value' => $this->generateSalt(),
+                'time' => Date::getNowTimestamp()
+            );
+            Option::set($fingerprintSaltKey, $this->encode($salt));
+        }
+        return $salt['value'];
+    }
+}
diff --git a/app/core/Tracker/PageUrl.php b/app/core/Tracker/PageUrl.php
@@ -36,14 +36,13 @@ class PageUrl
      * @param $idSite
      * @return bool|string Returned URL is HTML entities decoded
      */
-    public static function excludeQueryParametersFromUrl($originalUrl, $idSite)
+    public static function excludeQueryParametersFromUrl($originalUrl, $idSite, $additionalParametersToExclude = [])
     {
         $originalUrl = self::cleanupUrl($originalUrl);
 
         $parsedUrl = @parse_url($originalUrl);
         $parsedUrl = self::cleanupHostAndHashTag($parsedUrl, $idSite);
-        $parametersToExclude = self::getQueryParametersToExclude($idSite);
-
+	    $parametersToExclude = array_merge(self::getQueryParametersToExclude($idSite), $additionalParametersToExclude);
         if (empty($parsedUrl['query'])) {
             if (empty($parsedUrl['fragment'])) {
                 return UrlHelper::getParseUrlReverse($parsedUrl);

diff --git a/app/core/Tracker/Request.php b/app/core/Tracker/Request.php
@@ -195,6 +195,16 @@ public static function authenticateSuperUserOrAdminOrWrite($tokenAuth, $idSite)
             return false;
         }
 
+	    // Now checking the list of admin token_auth cached in the Tracker config file
+	    if (!empty($idSite) && $idSite > 0) {
+		    $website = Cache::getCacheWebsiteAttributes($idSite);
+		    $hashedToken = UsersManager::hashTrackingToken((string) $tokenAuth, $idSite);
+
+		    if (array_key_exists('tracking_token_auth', $website)
+		        && in_array($hashedToken, $website['tracking_token_auth'], true)) {
+			    return true;
+		    }
+	    }
         Piwik::postEvent('Request.initAuthenticationObject');
 
         /** @var \Piwik\Auth $auth */
@@ -209,16 +219,6 @@ public static function authenticateSuperUserOrAdminOrWrite($tokenAuth, $idSite)
             return true;
         }
 
-        // Now checking the list of admin token_auth cached in the Tracker config file
-        if (!empty($idSite) && $idSite > 0) {
-            $website = Cache::getCacheWebsiteAttributes($idSite);
-            $hashedToken = UsersManager::hashTrackingToken((string) $tokenAuth, $idSite);
-
-            if (array_key_exists('tracking_token_auth', $website)
-                && in_array($hashedToken, $website['tracking_token_auth'], true)) {
-                return true;
-            }
-        }
 
         Common::printDebug("WARNING! token_auth = $tokenAuth is not valid, Super User / Admin / Write was NOT authenticated");
 

diff --git a/app/core/Tracker/Settings.php b/app/core/Tracker/Settings.php
@@ -10,6 +10,10 @@
 
 use Piwik\Config;
 use Piwik\Container\StaticContainer;
+use Piwik\Date;
+use Piwik\Option;
+use Piwik\SettingsServer;
+use Piwik\Site;
 use Piwik\Tracker;
 use Piwik\DeviceDetector\DeviceDetectorFactory;
 use Piwik\SettingsPiwik;
@@ -57,6 +61,31 @@ public function getConfigId(Request $request, $ipAddress)
         }
 
         $browserLang = substr($request->getBrowserLanguage(), 0, 20); // limit the length of this string to match db
+        $trackerConfig = Config::getInstance()->Tracker;
+
+        $fingerprintSalt = '';
+
+        // fingerprint salt won't work when across multiple sites since all sites could have different timezones
+        // also cant add fingerprint salt for a specific day when we dont create new visit after midnight
+        if (!$this->isSameFingerprintsAcrossWebsites && !empty($trackerConfig['create_new_visit_after_midnight'])) {
+            $cache = Cache::getCacheWebsiteAttributes($request->getIdSite());
+            $date = Date::factory((int) $request->getCurrentTimestamp());
+            $fingerprintSaltKey = new FingerprintSalt();
+            $dateString = $fingerprintSaltKey->getDateString($date, $cache['timezone']);
+
+            if (!empty($cache[FingerprintSalt::OPTION_PREFIX . $dateString])) {
+                $fingerprintSalt = $cache[FingerprintSalt::OPTION_PREFIX . $dateString];
+            } else {
+                // we query the DB directly for requests older than 2-3 days...
+                $fingerprintSalt = $fingerprintSaltKey->getSalt($dateString, $request->getIdSite());
+            }
+
+            $fingerprintSalt .= $dateString;
+
+            if (defined('PIWIK_TEST_MODE') && PIWIK_TEST_MODE) {
+                $fingerprintSalt = ''; // use fixed value so they don't change randomly in tests
+            }
+        }
 
         return $this->getConfigHash(
             $request,
@@ -74,7 +103,8 @@ public function getConfigId(Request $request, $ipAddress)
             $plugin_Silverlight,
             $plugin_Cookie,
             $ipAddress,
-            $browserLang);
+            $browserLang,
+            $fingerprintSalt);
     }
 
     /**
@@ -96,12 +126,13 @@ public function getConfigId(Request $request, $ipAddress)
      * @param $plugin_Cookie
      * @param $ip
      * @param $browserLang
+     * @param $fingerprintHash
      * @return string
      */
     protected function getConfigHash(Request $request, $os, $browserName, $browserVersion, $plugin_Flash, $plugin_Java,
                                      $plugin_Director, $plugin_Quicktime, $plugin_RealPlayer, $plugin_PDF,
                                      $plugin_WindowsMedia, $plugin_Gears, $plugin_Silverlight, $plugin_Cookie, $ip,
-                                     $browserLang)
+                                     $browserLang, $fingerprintHash)
     {
         // prevent the config hash from being the same, across different Piwik instances
         // (limits ability of different Piwik instances to cross-match users)
@@ -114,7 +145,8 @@ protected function getConfigHash(Request $request, $os, $browserName, $browserVe
             . $plugin_WindowsMedia . $plugin_Gears . $plugin_Silverlight . $plugin_Cookie
             . $ip
             . $browserLang
-            . $salt;
+            . $salt
+            . $fingerprintHash;
 
         if (!$this->isSameFingerprintsAcrossWebsites) {
             $configString .= $request->getIdSite();

diff --git a/app/core/Tracker/Visit.php b/app/core/Tracker/Visit.php
@@ -243,7 +243,8 @@ protected function handleExistingVisit($visitIsConverted)
         foreach ($this->requestProcessors as $processor) {
             $processor->onExistingVisit($valuesToUpdate, $this->visitProperties, $this->request);
         }
-
+	    $visitorRecognizer = StaticContainer::get(VisitorRecognizer::class);
+	    $valuesToUpdate = $visitorRecognizer->removeUnchangedValues($this->visitProperties, $valuesToUpdate);
         $this->updateExistingVisit($valuesToUpdate);
 
         $this->visitProperties->setProperty('visit_last_action_time', $this->request->getCurrentTimestamp());