added statshouse python test

server/php-engine.cpp

@@ -2084,7 +2084,13 @@ int main_args_handler(int i, const char *long_option) {
         kprintf("--%s option: can't find ':'\n", long_option);
         return -1;
       }
-      StatsHouseClient::init(std::string(optarg, colon - optarg), atoi(colon + 1));
+      auto host = std::string(optarg, colon - optarg);
+      auto port = atoi(colon + 1);
+      if (host.empty()) {
+        host = "127.0.0.1";
+      }
+
+      StatsHouseClient::init(host, port);
       return 0;
     }
     case 2027: {

tests/python/lib/engine.py

@@ -10,6 +10,7 @@
 
 from .colors import cyan
 from .stats_receiver import StatsReceiver
+from .statshouse import Statshouse
 from .port_generator import get_port
 from .tl_client import send_rpc_request
 
@@ -29,6 +30,7 @@ def __init__(self, engine_bin, working_dir, options=None):
         self._engine_name = os.path.basename(engine_bin).replace('-', '_')
         self._log_file = os.path.join(working_dir, self._engine_name + ".log")
         self._stats_receiver = StatsReceiver(self._engine_name, working_dir)
+        self._statshouse = Statshouse(self._engine_name, working_dir)
         self._rpc_port = get_port()
         self._options = {
             "--log": self._log_file,
@@ -107,12 +109,15 @@ def create_binlog(self):
             raise RuntimeError("Can't create binlog")
         self._binlog_path = binlog_path
 
-    def start(self, start_msgs=None):
+    def start(self, start_msgs=None, start_statshouse=None):
         """
-        Запустить дижек
+        Запустить движок
         :param start_msgs: Сообщение в логе, которое нужно проверить после запуска движка
+        :param start_statshouse: start statshouse "server"
         """
         self._stats_receiver.start()
+        if start_statshouse:
+            self._statshouse.start()
 
         cmd = [self._engine_bin]
         for option, value in self._options.items():
@@ -155,7 +160,7 @@ def start(self, start_msgs=None):
 
     def stop(self):
         """
-        Остановить движек и проверить, что все в порядке
+        Остановить движок и проверить, что все в порядке
         """
         if self._engine_process is None or not self._engine_process.is_running():
             return
@@ -171,6 +176,7 @@ def stop(self):
         self._log_file_read_fd.close()
         self._log_file_write_fd.close()
         self._stats_receiver.stop()
+        self._statshouse.stop()
         if not engine_stopped_properly:
             raise RuntimeError("Can't stop engine properly")
         self._check_status_code(status, signal.SIGTERM)
@@ -237,6 +243,12 @@ def get_stats(self, prefix="", timeout=60):
         stats = self._stats_receiver.stats
         return {k[len(prefix):]: v for k, v in stats.items() if k.startswith(prefix)}
 
+    def check_statshouse_metrics(self, timeout=60):
+        """
+        :param timeout: Время ожидания очередной статы
+        """
+        self._statshouse.wait_metrics(timeout)
+
     def assert_stats(self, expected_added_stats, initial_stats=None,
                      prefix="", message="Can't wait expected stats", timeout=60):
         """

tests/python/lib/kphp_server.py

@@ -38,11 +38,11 @@ def __init__(self, engine_bin, working_dir, options=None, auto_start=False):
         if auto_start:
             self.start()
 
-    def start(self, start_msgs=None):
+    def start(self, start_msgs=None, start_statshouse=None):
         with open(self.get_server_config_path(), mode="w+t", encoding="utf-8") as fd:
             fd.write(
                 "cluster_name: kphp_server_{}".format("".join(chr(ord('A') + int(c)) for c in str(self._http_port))))
-        super(KphpServer, self).start(start_msgs)
+        super(KphpServer, self).start(start_msgs, start_statshouse)
         self._json_logs = []
         self._json_log_file_read_fd = open(self._log_file + ".json", 'r')
 

tests/python/lib/statshouse.py

@@ -0,0 +1,65 @@
+import os
+import subprocess
+import signal
+import time
+from sys import platform
+
+import psutil
+
+from .port_generator import get_port
+
+
+class Statshouse:
+    def __init__(self, engine_name, working_dir):
+        self._working_dir = working_dir
+        self._port = get_port()
+        self._statshouse_proc = None
+        self._statshouse_file = os.path.join(working_dir, engine_name + ".statshouse")
+        self._statshouse_file_write_fd = None
+        self._statshouse_file_read_fd = None
+        self._stats = {}
+
+    @property
+    def port(self):
+        return self._port
+
+    @property
+    def stats(self):
+        return self._stats
+
+    def start(self):
+        print("\nStarting statshouse server on port {}".format(self._port))
+        self._statshouse_file_write_fd = open(self._statshouse_file, 'wb')
+        self._statshouse_file_read_fd = open(self._statshouse_file, 'r')
+        self._statshouse_proc = psutil.Popen(
+            ["nc", "-lu", "" if platform == "darwin" else "-p", str(self._port)],
+            stdout=self._statshouse_file_write_fd,
+            stderr=subprocess.STDOUT,
+            cwd=self._working_dir
+        )
+        if not self._statshouse_proc.is_running():
+            self._statshouse_file_write_fd.close()
+            possible_error = self._statshouse_file_read_fd.read() or "empty out"
+            self._statshouse_file_read_fd.close()
+            RuntimeError("Can't start statshouse server: " + possible_error)
+
+    def stop(self):
+        if self._statshouse_proc is None or not self._statshouse_proc.is_running():
+            return
+
+        print("\nStopping statshouse server on port {}".format(self._port))
+        self._statshouse_proc.send_signal(signal.SIGKILL)
+        os.waitpid(self._statshouse_proc.pid, 0)
+        self._statshouse_file_read_fd.close()
+        self._statshouse_file_write_fd.close()
+
+    def wait_metrics(self, timeout=10):
+        start = time.time()
+        while not self.try_update_stats():
+            if time.time() - start > timeout:
+                raise RuntimeError("Waiting next statshouse metrics timeout")
+            time.sleep(0.1)
+
+    def try_update_stats(self):
+        self._statshouse_file_read_fd.seek(0, os.SEEK_END)
+        return bool(self._statshouse_file_read_fd.tell())

tests/python/tests/stats/test_statshouse_metrics.py

@@ -0,0 +1,18 @@
+from multiprocessing.dummy import Pool as ThreadPool
+from python.lib.testcase import KphpServerAutoTestCase
+
+
+class TestStatshouseMetrics(KphpServerAutoTestCase):
+    WORKERS_NUM = 2
+
+    @classmethod
+    def extra_class_setup(cls):
+        cls.kphp_server.update_options({
+            "--workers-num": cls.WORKERS_NUM,
+            "--statshouse-client": ":" + str(cls.kphp_server._statshouse.port),
+        })
+
+    def test_sending_metrics(self):
+        self.kphp_server.stop()
+        self.kphp_server.start(None, True)
+        self.kphp_server.check_statshouse_metrics()