DCGM snap tests

.github/workflows/check.yaml

@@ -27,16 +27,29 @@ jobs:
         with:
           fetch-depth: 0  # Complete git history is required to generate the version from git tags.
 
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
       - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y yamllint
+          python -m pip install --upgrade pip
+          # pin tox to the current major version to avoid
+          # workflows breaking all at once when a new major version is released.
+          python -m pip install 'tox<5'
 
       - name: Lint yaml files
-        run: |
-          yamllint .yamllint snap/snapcraft.yaml
+        run: yamllint .yamllint snap/snapcraft.yaml
+
+      - name: Lint tests
+        run: tox -e lint
 
   build:
+    needs:
+      - lint
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
@@ -45,3 +58,36 @@ jobs:
 
       - name: Verify snap builds successfully
         uses: snapcore/action-build@v1
+
+      - name: Upload the built snap
+        uses: actions/upload-artifact@v4
+        with:
+          name: SNAP_FILE
+          path: dcgm*.snap
+
+  func:
+    needs:
+      - build
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Complete git history is required to generate the version from git tags.
+
+      - name: Download the built snap
+        uses: actions/download-artifact@v4
+        with:
+          name: SNAP_FILE
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install 'tox<5'
+
+      - name: Run unit tests
+        run: tox -e func

.gitignore

@@ -1,8 +1,19 @@
 # This is a template `.gitignore` file for snaps
 
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
 # Tests files and dir
+.pytest_cache/
+.coverage
+.tox
+.venv
 reports/
 **/report/
+htmlcov/
+.mypy_cache
 
 # Log files
 *.log
@@ -17,6 +28,11 @@ reports/
 # version data
 repo-info
 
+# Python builds
+deb_dist/
+dist/
+*.egg-info/
+
 # Snaps
 *.snap
 

README.md

@@ -9,4 +9,4 @@ You can build the snap locally by using the command:
 
 ```shell
 snapcraft --use-lxd
-```
+```

pyproject.toml

@@ -0,0 +1,62 @@
+[tool.flake8]
+ignore = ["C901", "D100", "D101", "D102", "D103", "W503", "W504"]
+exclude = ['.eggs', '.git', '.tox', '.venv', '.build', 'build', 'report']
+max-line-length = 99
+max-complexity = 10
+
+[tool.black]
+line-length = 99
+exclude = '''
+/(
+    | .eggs
+    | .git
+    | .tox
+    | .venv
+    | .build
+    | build
+    | report
+)/
+'''
+
+[tool.isort]
+profile = "black"
+skip_glob = [
+  ".eggs",
+  ".git",
+  ".tox",
+  ".venv",
+  ".build",
+  "build",
+  "report"
+]
+
+[tool.pylint]
+max-line-length = 99
+ignore = ['.eggs', '.git', '.tox', '.venv', '.build', 'report', 'tests']
+
+[tool.mypy]
+warn_unused_ignores = true
+warn_unused_configs = true
+warn_unreachable = true
+disallow_untyped_defs = true
+exclude = ['.eggs', '.git', '.tox', '.venv', '.build', 'report', 'tests']
+
+## Ignore unsupported imports
+[[tool.mypy.overrides]]
+ignore_missing_imports = true
+module = ["setuptools"]
+
+[tool.coverage.run]
+relative_files = true
+source = ["."]
+omit = ["docs/**", "lib/**", "snap/**", "build/**", "setup.py"]
+
+[tool.coverage.report]
+fail_under = 100
+show_missing = true
+
+[tool.coverage.html]
+directory = "tests/unit/report/html"
+
+[tool.coverage.xml]
+output = "tests/unit/report/coverage.xml"

snap/hooks/configure

@@ -3,4 +3,12 @@
 # Register config options if unset,
 # so users can see available options by running
 # `sudo snap get dcgm`.
-[ -z "$(snapctl get nv-hostengine-port)" ] && snapctl set nv-hostengine-port=
+if [ -z "$(snapctl get nv-hostengine-port)" ]; then
+    # Setting to empty string for the nv-hostengine binary to use the default port. (5555)
+    snapctl set nv-hostengine-port="5555"
+fi
+
+if [ -z "$(snapctl get dcgm-exporter-address)" ]; then
+    # Setting to empty string for the dcgm-exporter binary to use the default address. (:9400)
+    snapctl set dcgm-exporter-address=":9400"
+fi

snap/local/run_dcgm_exporter.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+
+# Build the argument list for the dcgm-exporter command
+args=()
+
+# Add the dcgm-exporter-address option if it is set. Default: “:9400”
+dcgm_exporter_address="$(snapctl get dcgm-exporter-address)"
+
+if [ -n "$dcgm_exporter_address" ]; then
+    args+=("-a" "$dcgm_exporter_address")
+fi
+
+exec "$SNAP/bin/dcgm-exporter" "${args[@]}"

snap/snapcraft.yaml

@@ -21,10 +21,15 @@ title: NVIDIA DCGM
 
 apps:
   dcgm-exporter:
-    command: bin/dcgm-exporter
+    command: run_dcgm_exporter.sh
     plugs:
       - network-bind
       - opengl
+    daemon: simple
+    # As this is a dcgm snap, not the dcgm-exporter snap,
+    # user might not be interested in running dcgm-exporter, so disable it by default
+    install-mode: disable
+    restart-condition: on-failure
   dcgmi:
     command: usr/bin/dcgmi
     plugs:
@@ -59,6 +64,7 @@ parts:
     override-build: |
       craftctl default
       chmod +x run_nv_hostengine.sh
+      chmod +x run_dcgm_exporter.sh
   dcgm-exporter:
     after:
       - wrapper

tests/functional/conftest.py

@@ -0,0 +1,22 @@
+import subprocess
+
+import pytest
+
+
+@pytest.fixture(scope="session", autouse=True)
+def install_dcgm_snap():
+    """Install the snap and enable dcgm-exporter service for testing."""
+    snap_build_name = "dcgm_*.snap"
+
+    subprocess.run(
+        f"sudo snap install --dangerous {snap_build_name}",
+        check=True,
+        capture_output=True,
+        shell=True,
+    )
+
+    subprocess.run("sudo snap start dcgm.dcgm-exporter".split(), check=True)
+
+    yield
+
+    subprocess.run("sudo snap remove --purge dcgm".split(), check=True)

tests/functional/requirements.txt

@@ -0,0 +1 @@
+tenacity

tests/functional/test_snap_dcgm.py

@@ -0,0 +1,78 @@
+import json
+import subprocess
+import urllib.request
+
+import pytest
+from tenacity import Retrying, retry, stop_after_delay, wait_fixed
+
+
+@retry(wait=wait_fixed(5), stop=stop_after_delay(30))
+def test_dcgm_exporter():
+    """Test of the dcgm-exporter service and its endpoint."""
+    dcgm_exporter_service = "snap.dcgm.dcgm-exporter"
+    endpoint = "http://localhost:9400/metrics"
+
+    assert 0 == subprocess.call(
+        f"sudo systemctl is-active --quiet {dcgm_exporter_service}".split()
+    ), f"{dcgm_exporter_service} is not running"
+
+    # Check the exporter endpoint, will raise an exception if the endpoint is not reachable
+    response = urllib.request.urlopen(endpoint)
+
+    # The output of the exporter endpoint is not tested
+    # as in a virtual environment it will not have any GPU metrics
+    assert 200 == response.getcode(), "DCGM exporter endpoint returned an error"
+
+
+def test_dcgm_nv_hostengine():
+    """Check the dcgm-nv-hostengine service."""
+    nv_hostengine_service = "snap.dcgm.nv-hostengine"
+    nv_hostengine_port = 5555
+
+    assert 0 == subprocess.call(
+        f"sudo systemctl is-active --quiet {nv_hostengine_service}".split()
+    ), f"{nv_hostengine_service} is not running"
+
+    assert 0 == subprocess.call(
+        f"nc -z localhost {nv_hostengine_port}".split()
+    ), f"{nv_hostengine_service} is not listening on port {nv_hostengine_port}"
+
+
+def test_dcgmi():
+    """Test of the dcgmi command."""
+    result = subprocess.run(
+        "dcgm.dcgmi discovery -l".split(), check=True, capture_output=True, text=True
+    )
+
+    # Test if the command is working and outputs a table with the GPU ID
+    # The table will be empty in a virtual environment, but the command should still work
+    assert "GPU ID" in result.stdout.strip(), "DCGMI didn't produce the expected table"
+
+
+bind_test_data = [
+    ("dcgm.dcgm-exporter", "dcgm-exporter-address", ":9466"),
+    ("dcgm.nv-hostengine", "nv-hostengine-port", "5566"),
+]
+
+
+@pytest.mark.parametrize("service, config, new_value", bind_test_data)
+def test_dcgm_bind_config(service: str, config: str, new_value: str):
+    """Test snap bind configuration."""
+    result = subprocess.run(
+        "sudo snap get dcgm -d".split(), check=True, capture_output=True, text=True
+    )
+    dcgm_snap_config = json.loads(result.stdout.strip())
+    assert config in dcgm_snap_config, f"{config} is not in the snap configuration"
+
+    assert 0 == subprocess.call(
+        f"sudo snap set dcgm {config}={new_value}".split()
+    ), f"Failed to set {config} to {new_value}"
+
+    # restart the service to apply the new configuration
+    subprocess.run(f"sudo snap restart {service}".split(), check=True)
+
+    for attempt in Retrying(wait=wait_fixed(2), stop=stop_after_delay(10)):
+        with attempt:
+            assert 0 == subprocess.call(
+                f"nc -z localhost {new_value.lstrip(':')}".split()
+            ), f"{service} is not listening on {new_value}"

tox.ini

@@ -0,0 +1,42 @@
+[tox]
+skipsdist=True
+envlist = lint, unit, func
+skip_missing_interpreters = True
+
+[testenv]
+basepython = python3
+setenv = PYTHONPATH={toxinidir}
+
+[testenv:lint]
+commands =
+    pflake8
+    pylint --recursive=y .
+    black --check --diff --color .
+    isort --check --diff --color .
+deps =
+    black
+    flake8
+    pyproject-flake8
+    flake8-docstrings
+    pep8-naming
+    flake8-colors
+    colorama
+    isort
+    pylint
+    {[testenv:func]deps}
+
+[testenv:reformat]
+envdir = {toxworkdir}/lint
+deps = {[testenv:lint]deps}
+commands =
+    black .
+    isort .
+
+[testenv:func]
+deps =
+    pytest
+    -r {toxinidir}/tests/functional/requirements.txt
+passenv =
+    TEST_*
+commands =
+    pytest {toxinidir}/tests/functional {posargs:-v}