Add E2E test environment for Teleport (#17815)

* Add caddy e2e test environment

* add helper script to randomize the mocks for e2e testing

* add metrics mocks randomizer helper script + lint

* skip integration tests on caddy env, skip e2e tests on non-caddy env

* add conditions to docker_run to fix flakiness

* use port 3000 in caddy env as well

* fix e2e env setup

* fmt

* fix port number in docker setup file

* fix yielded config in docker setup

* lint

* remove unused const

* change e2e setup

teleport/hatch.toml

@@ -2,3 +2,18 @@
 
 [[envs.default.matrix]]
 python = ["3.12"]
+
+[[envs.default.matrix]]
+python = ["3.12"]
+setup = ["caddy"]
+
+[envs.default.overrides]
+name."^py3.12$".e2e-env = { value = true }
+name."^py3.12-caddy$".e2e-env = { value = true }
+matrix.setup.e2e-env = { value = true, if = ["caddy"] }
+matrix.setup.env-vars = [
+  { key = "USE_TELEPORT_CADDY", value = "true", if = ["caddy"] },
+]
+
+[envs.default]
+e2e-env = false

teleport/tests/common.py

@@ -2,6 +2,9 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
+import os
+
+USE_TELEPORT_CADDY = os.environ.get("USE_TELEPORT_CADDY", False)
 
 INSTANCE = {"teleport_url": "http://127.0.0.1", "diag_port": "3000"}
 

teleport/tests/conftest.py

@@ -5,16 +5,36 @@
 
 import pytest
 
-from datadog_checks.dev import docker_run, get_here
+from datadog_checks.dev import docker_run, get_docker_hostname, get_here
+from datadog_checks.dev.conditions import CheckDockerLogs, CheckEndpoints
 
-from .common import INSTANCE
+from .common import INSTANCE, USE_TELEPORT_CADDY
 
+HOST = get_docker_hostname()
 
-@pytest.fixture(scope='session')
+URL = "http://{}".format(HOST)
+
+
+@pytest.fixture(scope="session")
 def dd_environment():
-    compose_file = os.path.join(get_here(), 'docker', 'docker-compose.yaml')
-    with docker_run(compose_file, sleep=5):
-        yield INSTANCE
+    if USE_TELEPORT_CADDY:
+        compose_file = os.path.join(get_here(), "docker", "caddy", "docker-compose.yaml")
+        conditions = [
+            CheckEndpoints(URL + ":3000/healthz", attempts=120),
+        ]
+        with docker_run(compose_file, conditions=conditions, sleep=5):
+            yield INSTANCE
+    else:
+        compose_file = os.path.join(get_here(), "docker", "teleport", "docker-compose.yaml")
+        with docker_run(
+            compose_file,
+            sleep=5,
+            conditions=[
+                CheckDockerLogs(identifier="teleport-service", patterns=["Starting Teleport"]),
+                CheckEndpoints(URL + ":3000/healthz", attempts=120),
+            ],
+        ):
+            yield {"teleport_url": URL, "diag_port": "3000"}
 
 
 @pytest.fixture

teleport/tests/docker/caddy/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: "3"
+
+services:
+  teleport-caddy:
+    image: caddy:2.6.2-alpine
+    build: .
+    container_name: teleport-caddy
+    volumes:
+      - ./fixtures:/usr/share/caddy
+      - ./etc/caddy/teleport-service:/etc/caddy/
+    ports:
+      - "3000:80"

teleport/tests/docker/caddy/etc/caddy/teleport-service/Caddyfile

@@ -0,0 +1,35 @@
+{
+    debug
+    admin :2019
+}
+:80 {
+    root * /usr/share/caddy/
+    @metrics {
+        method GET
+        path /metrics
+    }
+    route @metrics {
+        rewrite * /{http.request.uri.path}/get.txt
+        file_server
+    }
+
+    @healthz {
+        method GET
+        path /healthz
+    }
+    route @healthz {
+        rewrite * /{http.request.uri.path}/get.json
+        file_server
+    }
+
+      @readyz {
+        method GET
+        path /readyz
+    }
+    route @readyz {
+        rewrite * /{http.request.uri.path}/get.json
+        file_server
+    }
+
+    file_server browse
+}

teleport/tests/docker/caddy/fixtures/healthz/get.json

@@ -0,0 +1 @@
+{ "status": "ok" }

teleport/tests/docker/caddy/fixtures/readyz/get.json

@@ -0,0 +1 @@
+{ "status": "ok" }

teleport/tests/docker/caddy/metrics_mock_randomizer.py

@@ -0,0 +1,83 @@
+import random
+import sys
+import time
+
+
+def main():
+    if len(sys.argv) < 2:
+        sys.exit(0)
+
+    metrics_file = sys.argv[1]
+
+    while True:
+        read_and_get_new_metrics(metrics_file)
+        sleep_time = random.randint(1, 30)
+        time.sleep(sleep_time)
+
+
+def read_and_get_new_metrics(metrics_file):
+    new_content = []
+
+    with open(metrics_file, "r") as metrics:
+        for line in metrics.readlines():
+            line = line.rstrip('\n')
+            if line == "":
+                new_content.append(line)
+                continue
+            if line.startswith("#"):
+                new_content.append(line)
+                continue
+
+            fields = line.split(" ")
+
+            metric_name = fields[0]
+            value_str = fields[-1]
+
+            is_percentage = "percentage" in metric_name
+
+            value = None
+            is_float = False
+
+            if "." in value_str:
+                is_float = True
+                value = float(value_str)
+            else:
+                value = int(value_str)
+
+            new_value = modify_value(value, is_percentage, is_float)
+            new_line = fields[:-1]
+            if metric_name == "process_state":
+                new_value = random.choice([0, 1, 2, 3])
+
+            if "bytes" in metric_name:
+                new_value *= random.choice([10, 1000, 10000])
+                new_value = new_value % 686047984
+            new_line.append(str(new_value))
+            new_content.append(" ".join(new_line))
+
+    with open(metrics_file, "w") as metrics:
+        new_file_content = "\n".join(new_content)
+        metrics.write(new_file_content)
+
+
+def modify_value(value, is_percentage, is_float):
+    if is_float and value == 0.0:
+        value = random.uniform(0.0, 0.2)
+    if not is_float and value == 0:
+        value = random.randint(0, 20)
+    change_percent = random.uniform(0.05, 0.1)
+    change_direction = random.choice([1, -1])
+    change = value * (change_percent * change_direction)
+    new_value = value + change
+
+    if is_percentage:
+        new_value = min(new_value, 1.0)
+        new_value = max(new_value, 0.0)
+
+    if not is_float:
+        return int(new_value)
+    return new_value
+
+
+if __name__ == "__main__":
+    main()

teleport/tests/docker/docker-compose.yaml → teleport/tests/docker/teleport/docker-compose.yaml

@@ -1,5 +1,5 @@
 services:
-  teleport:
+  teleport-service:
     image: public.ecr.aws/gravitational/teleport:14.3
     ports:
       - 3000:3000
@@ -9,3 +9,4 @@ services:
     volumes:
       - ./etc/teleport:/etc/teleport
     command: --diag-addr=0.0.0.0:3000
+    container_name: teleport-service

teleport/tests/test_e2e.py

@@ -4,17 +4,21 @@
 
 import pytest
 
-from .common import COMMON_METRICS, INSTANCE
+from .common import COMMON_METRICS, INSTANCE, USE_TELEPORT_CADDY
+
+pytestmark = [
+    pytest.mark.e2e,
+    pytest.mark.skipif(not USE_TELEPORT_CADDY, reason="Only run e2e tests on caddy environment"),
+]
 
-pytestmark = pytest.mark.e2e
 
 CONFIG = {
-    'init_config': {},
-    'instances': [INSTANCE],
+    "init_config": {},
+    "instances": [INSTANCE],
 }
 
 
 def test_teleport_e2e(dd_agent_check):
-    aggregator = dd_agent_check(CONFIG)
+    aggregator = dd_agent_check()
     aggregator.assert_metric("teleport.health.up", value=1, count=1, tags=["teleport_status:ok"])
     aggregator.assert_metric(f"teleport.{COMMON_METRICS[0]}")

teleport/tests/test_integration.py

@@ -6,9 +6,13 @@
 
 from datadog_checks.teleport import TeleportCheck
 
-from .common import COMMON_METRICS
+from .common import COMMON_METRICS, USE_TELEPORT_CADDY
 
-pytestmark = [pytest.mark.integration, pytest.mark.usefixtures("dd_environment")]
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.usefixtures("dd_environment"),
+    pytest.mark.skipif(not USE_TELEPORT_CADDY, reason="Only run integration tests on non-caddy environment"),
+]
 
 
 def test_connect_ok(aggregator, instance, dd_run_check):