diff --git a/.codecov.yml b/.codecov.yml index b3da4ec81667d..3b23a3da8bf86 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -186,6 +186,10 @@ coverage: target: 75 flags: - druid + DuckDB: + target: 75 + flags: + - duckdb EKS_Fargate: target: 75 flags: @@ -974,6 +978,11 @@ flags: paths: - druid/datadog_checks/druid - druid/tests + duckdb: + carryforward: true + paths: + - duckdb/datadog_checks/duckdb + - duckdb/tests ecs_fargate: carryforward: true paths: diff --git a/.github/workflows/config/labeler.yml b/.github/workflows/config/labeler.yml index b548315a04731..6084caed5feff 100644 --- a/.github/workflows/config/labeler.yml +++ b/.github/workflows/config/labeler.yml @@ -177,6 +177,8 @@ integration/dotnetclr: - dotnetclr/**/* integration/druid: - druid/**/* +integration/duckdb: +- duckdb/**/* integration/ecs_fargate: - ecs_fargate/**/* integration/eks_anywhere: diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index bf58388e435be..4d0f92ea74c8b 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -1154,6 +1154,26 @@ jobs: minimum-base-package: ${{ inputs.minimum-base-package }} pytest-args: ${{ inputs.pytest-args }} secrets: inherit + j9eb6aa6: + uses: ./.github/workflows/test-target.yml + with: + job-name: DuckDB + target: duckdb + platform: linux + runner: '["ubuntu-22.04"]' + repo: "${{ inputs.repo }}" + python-version: "${{ inputs.python-version }}" + standard: ${{ inputs.standard }} + latest: ${{ inputs.latest }} + agent-image: "${{ inputs.agent-image }}" + agent-image-py2: "${{ inputs.agent-image-py2 }}" + agent-image-windows: "${{ inputs.agent-image-windows }}" + agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}" + test-py2: ${{ inputs.test-py2 }} + test-py3: ${{ inputs.test-py3 }} + minimum-base-package: ${{ inputs.minimum-base-package }} + pytest-args: ${{ inputs.pytest-args }} + secrets: inherit j562bfe5: uses: ./.github/workflows/test-target.yml with: diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8af1cd58fd1d1..747ef923c3f87 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -22,6 +22,7 @@ cryptography,PyPI,BSD-3-Clause,Copyright (c) Individual contributors. cryptography,PyPI,PSF,Copyright (c) Individual contributors. ddtrace,PyPI,BSD-3-Clause,"Copyright 2016 Datadog, Inc." dnspython,PyPI,ISC,Copyright (C) Dnspython Contributors +duckdb,PyPI,MIT,Copyright (c) Hannes Muehleisen flup,Vendor,BSD-3-Clause,Copyright (c) 2005 Allan Saddi. All Rights Reserved. flup-py3,Vendor,BSD-3-Clause,"Copyright (c) 2005, 2006 Allan Saddi All rights reserved." foundationdb,PyPI,Apache-2.0,Copyright 2017 FoundationDB diff --git a/README.md b/README.md index fd081156a41a1..734d8ba46033f 100644 --- a/README.md +++ b/README.md @@ -54,4 +54,4 @@ For more information on integrations, please reference our [documentation][11] a [28]: https://img.shields.io/badge/typing-Mypy-blue.svg [29]: https://github.com/python/mypy [30]: https://img.shields.io/badge/license-BSD--3--Clause-9400d3.svg -[31]: https://spdx.org/licenses/BSD-3-Clause.html +[31]: https://spdx.org/licenses/BSD-3-Clause.html \ No newline at end of file diff --git a/agent_requirements.in b/agent_requirements.in index b96253e8f081f..572dee1782565 100644 --- a/agent_requirements.in +++ b/agent_requirements.in @@ -13,6 +13,7 @@ confluent-kafka==2.6.1 cryptography==43.0.1 ddtrace==2.10.6 dnspython==2.6.1 +duckdb==1.1.1 foundationdb==6.3.24 hazelcast-python-client==5.4.0 importlib-metadata==2.1.3; python_version < '3.8' diff --git a/duckdb/CHANGELOG.md b/duckdb/CHANGELOG.md new file mode 100644 index 0000000000000..1a08d7960b6df --- /dev/null +++ b/duckdb/CHANGELOG.md @@ -0,0 +1,4 @@ +# CHANGELOG - DuckDB + + + diff --git a/duckdb/README.md b/duckdb/README.md new file mode 100644 index 0000000000000..e91e3cd5f2e05 --- /dev/null +++ b/duckdb/README.md @@ -0,0 +1,60 @@ +# Agent Check: DuckDB + +## Overview + +This check monitors [DuckDB][1] through the Datadog Agent. + +Include a high level overview of what this integration does: +- What does your product do (in 1-2 sentences)? +- What value will customers get from this integration, and why is it valuable to them? +- What specific data will your integration monitor, and what's the value of that data? + +## Setup + +Follow the instructions below to install and configure this check for an Agent running on a host. For containerized environments, see the [Autodiscovery Integration Templates][3] for guidance on applying these instructions. + +### Installation + +The DuckDB check is included in the [Datadog Agent][2] package. +No additional installation is needed on your server. + +### Configuration + +1. Edit the `duckdb.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory to start collecting your duckdb performance data. See the [sample duckdb.d/conf.yaml][4] for all available configuration options. + +2. [Restart the Agent][5]. + +### Validation + +[Run the Agent's status subcommand][6] and look for `duckdb` under the Checks section. + +## Data Collected + +### Metrics + +See [metadata.csv][7] for a list of metrics provided by this integration. + +### Events + +The DuckDB integration does not include any events. + +### Service Checks + +The DuckDB integration does not include any service checks. + +See [service_checks.json][8] for a list of service checks provided by this integration. + +## Troubleshooting + +Need help? Contact [Datadog support][9]. + + +[1]: **LINK_TO_INTEGRATION_SITE** +[2]: https://app.datadoghq.com/account/settings/agent/latest +[3]: https://docs.datadoghq.com/agent/kubernetes/integrations/ +[4]: https://github.com/DataDog/integrations-core/blob/master/duckdb/datadog_checks/duckdb/data/conf.yaml.example +[5]: https://docs.datadoghq.com/agent/guide/agent-commands/#start-stop-and-restart-the-agent +[6]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information +[7]: https://github.com/DataDog/integrations-core/blob/master/duckdb/metadata.csv +[8]: https://github.com/DataDog/integrations-core/blob/master/duckdb/assets/service_checks.json +[9]: https://docs.datadoghq.com/help/ diff --git a/duckdb/assets/configuration/spec.yaml b/duckdb/assets/configuration/spec.yaml new file mode 100644 index 0000000000000..b7c19b5c4b2e5 --- /dev/null +++ b/duckdb/assets/configuration/spec.yaml @@ -0,0 +1,34 @@ +name: DuckDB +files: +- name: duckdb.yaml + options: + - template: init_config + options: + - template: init_config/default + - template: instances + options: + - name: db_name + required: true + description: | + The database to connect to (file path). + NOTE: DuckDB databases are stored as files. + value: + example: "/path-to-file/my_database.db" + type: string + - name: connection_attempt + description: | + The number of retries to connect to the database in case of failure + value: + type: integer + default: 3 + display_default: 3 + - template: instances/default + overrides: + min_collection_interval.description: | + This changes the collection interval of this check to avoid + the risk of locking the database file. + If your database file is heavily used in write mode, you changes + further increase this value to minimise the monitoring acccess. + min_collection_interval.value.default: 60 + min_collection_interval.value.example: 60 + min_collection_interval.enabled: true diff --git a/duckdb/assets/dashboards/duckdb_overview.json b/duckdb/assets/dashboards/duckdb_overview.json new file mode 100644 index 0000000000000..96e155112c011 --- /dev/null +++ b/duckdb/assets/dashboards/duckdb_overview.json @@ -0,0 +1,29 @@ +{ + "title": "DuckDB Overview", + "description": "[[suggested_dashboards]]", + "widgets": [ + { + "id": 6432334130190000, + "definition": { + "type": "image", + "url": "https://static.datadoghq.com/static/images/logos/duckdb_small.svg", + "sizing": "contain", + "margin": "md", + "has_background": false, + "has_border": false, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 2, + "height": 2 + } + } + ], + "template_variables": [], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} \ No newline at end of file diff --git a/duckdb/assets/service_checks.json b/duckdb/assets/service_checks.json new file mode 100644 index 0000000000000..6c87723be2c74 --- /dev/null +++ b/duckdb/assets/service_checks.json @@ -0,0 +1,2 @@ + +[] diff --git a/duckdb/changelog.d/1.added b/duckdb/changelog.d/1.added new file mode 100644 index 0000000000000..aa949b47b7b41 --- /dev/null +++ b/duckdb/changelog.d/1.added @@ -0,0 +1 @@ +Initial Release \ No newline at end of file diff --git a/duckdb/datadog_checks/__init__.py b/duckdb/datadog_checks/__init__.py new file mode 100644 index 0000000000000..1517d901c0aae --- /dev/null +++ b/duckdb/datadog_checks/__init__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/duckdb/datadog_checks/duckdb/__about__.py b/duckdb/datadog_checks/duckdb/__about__.py new file mode 100644 index 0000000000000..e9541ce83e9e5 --- /dev/null +++ b/duckdb/datadog_checks/duckdb/__about__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +__version__ = '0.0.1' diff --git a/duckdb/datadog_checks/duckdb/__init__.py b/duckdb/datadog_checks/duckdb/__init__.py new file mode 100644 index 0000000000000..f23ef58f4ed30 --- /dev/null +++ b/duckdb/datadog_checks/duckdb/__init__.py @@ -0,0 +1,7 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .__about__ import __version__ +from .check import DuckdbCheck + +__all__ = ['__version__', 'DuckdbCheck'] diff --git a/duckdb/datadog_checks/duckdb/check.py b/duckdb/datadog_checks/duckdb/check.py new file mode 100644 index 0000000000000..2fb746bb40c53 --- /dev/null +++ b/duckdb/datadog_checks/duckdb/check.py @@ -0,0 +1,166 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import json +import os +import re +import time +from contextlib import closing, contextmanager +from copy import deepcopy + +import duckdb + +from datadog_checks.base import AgentCheck +from datadog_checks.base.utils.db import QueryManager + +from .queries import DEFAULT_QUERIES + + +class DuckdbCheck(AgentCheck): + + # This will be the prefix of every metric and service check the integration sends + __NAMESPACE__ = 'duckdb' + + def __init__(self, name, init_config, instances): + super(DuckdbCheck, self).__init__(name, init_config, instances) + + self.db_name = self.instance.get('db_name') + self.connection_attempt = int(self.instance.get('connection_attempt', 3)) + + self.tags = self.instance.get('tags', []) + self._connection = None + self._connect_params = None + self._tags = [] + self._query_errors = 0 + + manager_queries = deepcopy(DEFAULT_QUERIES) + + self._query_manager = QueryManager( + self, + self._execute_query_raw, + queries=manager_queries, + tags=self.tags, + error_handler=self._executor_error_handler, + ) + self.check_initializations.append(self.initialize_config) + self.check_initializations.append(self._query_manager.compile_queries) + + def check(self, _): + retry_delay = 5 + max_retries = self.connection_attempt + for attempt in range(1, max_retries + 1): + try: + with self.connect() as conn: + if conn: + self._connection = conn + self._query_manager.execute() + break + except Exception as e: + self.log.warning('Unable to connect to the database: "%s" , retrying...', e) + if attempt < max_retries: + time.sleep(retry_delay) + else: + self.log.error('Max connection retries reached') + + def _execute_query_raw(self, query): + with closing(self._connection.cursor()) as cursor: + query = query.format(self.db_name) + curs = cursor.execute(query) + if len(curs.fetchall()) < 1: + self._query_errors += 1 + self.log.warning('Failed to fetch records from query: `%s`.', query) + return None + for row in cursor.execute(query).fetchall(): + # Try to find the field name or version from the query, anything else would fail + pattern_version = r"\bversion\b" + query_version = re.search(pattern_version, query) + if query_version: + query_name = 'version' + else: + pattern = r"(?i)\bname\s*=\s*'([^']+)'" + query_name = re.search(pattern, query).group(1) + try: + yield self._queries_processor(row, query_name) + except Exception as e: + self.log.debug( + 'Unable to process row returned from query "%s", skipping row %s. %s', query_name, row, e + ) + yield row + + def _queries_processor(self, row, query_name): + unprocessed_row = row + # Return database version + if query_name == 'version': + self.submit_version(row) + return unprocessed_row + + self.log.debug('Row processor returned: %s. \nFrom query: "%s"', unprocessed_row, query_name) + return unprocessed_row + + @contextmanager + def connect(self): + conn = None + # Only attempt connection if the Database file exists + if os.path.exists(self.db_name): + try: + # Try to establish the connection in read only mode + conn = duckdb.connect(self.db_name, read_only=True) + self.log.info('Connected to DuckDB database.') + yield conn + except Exception as e: + if 'Conflicting lock' in str(e): + self.log.error('Lock conflict detected') + else: + self.log.error('Unable to connect to DuckDB database. %s.', e) + finally: + if conn: + conn.close() + else: + self.log.error('Database file not found') + + def initialize_config(self): + self._connect_params = json.dumps( + { + 'db_name': self.db_name, + } + ) + global_tags = [ + 'db_name:{}'.format(self.instance.get('db_name')), + ] + if self.tags is not None: + global_tags.extend(self.tags) + self._tags = global_tags + self._query_manager.tags = self._tags + + @AgentCheck.metadata_entrypoint + def submit_version(self, row): + """ + Example version: v1.1.1 + """ + try: + duckdb_version_row = row[0] + duckdb_version = duckdb_version_row[1:] + version_split = duckdb_version.split('.') + + if len(version_split) >= 3: + major = version_split[0] + minor = version_split[1] + patch = version_split[2] + + version_raw = f'{major}.{minor}.{patch}' + + version_parts = { + 'major': major, + 'minor': minor, + 'patch': patch, + } + self.set_metadata('version', version_raw, scheme='parts', final_scheme='semver', part_map=version_parts) + else: + self.log.debug("Malformed DuckDB version format: %s", duckdb_version_row) + except Exception as e: + self.log.warning("Could not retrieve version metadata: %s", e) + + def _executor_error_handler(self, error): + self.log.debug('Error from query "%s"', error) + self._query_errors += 1 + return error diff --git a/duckdb/datadog_checks/duckdb/config_models/__init__.py b/duckdb/datadog_checks/duckdb/config_models/__init__.py new file mode 100644 index 0000000000000..106fff2032f68 --- /dev/null +++ b/duckdb/datadog_checks/duckdb/config_models/__init__.py @@ -0,0 +1,24 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from .instance import InstanceConfig +from .shared import SharedConfig + + +class ConfigMixin: + _config_model_instance: InstanceConfig + _config_model_shared: SharedConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared diff --git a/duckdb/datadog_checks/duckdb/config_models/defaults.py b/duckdb/datadog_checks/duckdb/config_models/defaults.py new file mode 100644 index 0000000000000..c99e7d0251838 --- /dev/null +++ b/duckdb/datadog_checks/duckdb/config_models/defaults.py @@ -0,0 +1,24 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + + +def instance_connection_attempt(): + return 3 + + +def instance_disable_generic_tags(): + return False + + +def instance_empty_default_hostname(): + return False + + +def instance_min_collection_interval(): + return 60 diff --git a/duckdb/datadog_checks/duckdb/config_models/instance.py b/duckdb/datadog_checks/duckdb/config_models/instance.py new file mode 100644 index 0000000000000..7e183b58aad41 --- /dev/null +++ b/duckdb/datadog_checks/duckdb/config_models/instance.py @@ -0,0 +1,63 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class MetricPatterns(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + exclude: Optional[tuple[str, ...]] = None + include: Optional[tuple[str, ...]] = None + + +class InstanceConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + connection_attempt: Optional[int] = None + db_name: str + disable_generic_tags: Optional[bool] = None + empty_default_hostname: Optional[bool] = None + metric_patterns: Optional[MetricPatterns] = None + min_collection_interval: Optional[float] = None + service: Optional[str] = None + tags: Optional[tuple[str, ...]] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'instance_{info.field_name}', identity)(value, field=field) + else: + value = getattr(defaults, f'instance_{info.field_name}', lambda: value)() + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_instance', identity)(model)) diff --git a/duckdb/datadog_checks/duckdb/config_models/shared.py b/duckdb/datadog_checks/duckdb/config_models/shared.py new file mode 100644 index 0000000000000..e39d447dfc4b9 --- /dev/null +++ b/duckdb/datadog_checks/duckdb/config_models/shared.py @@ -0,0 +1,45 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import validators + + +class SharedConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + service: Optional[str] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'shared_{info.field_name}', identity)(value, field=field) + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_shared', identity)(model)) diff --git a/duckdb/datadog_checks/duckdb/config_models/validators.py b/duckdb/datadog_checks/duckdb/config_models/validators.py new file mode 100644 index 0000000000000..70150e85e6124 --- /dev/null +++ b/duckdb/datadog_checks/duckdb/config_models/validators.py @@ -0,0 +1,13 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# Here you can include additional config validators or transformers +# +# def initialize_instance(values, **kwargs): +# if 'my_option' not in values and 'my_legacy_option' in values: +# values['my_option'] = values['my_legacy_option'] +# if values.get('my_number') > 10: +# raise ValueError('my_number max value is 10, got %s' % str(values.get('my_number'))) +# +# return values diff --git a/duckdb/datadog_checks/duckdb/data/conf.yaml.example b/duckdb/datadog_checks/duckdb/data/conf.yaml.example new file mode 100644 index 0000000000000..86f32b93c5534 --- /dev/null +++ b/duckdb/datadog_checks/duckdb/data/conf.yaml.example @@ -0,0 +1,67 @@ +## All options defined here are available to all instances. +# +init_config: + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Additionally, this sets the default `service` for every log source. + # + # service: + +## Every instance is scheduled independently of the others. +# +instances: + + ## @param db_name - string - required + ## The database to connect to (file path). + ## NOTE: DuckDB databases are stored as files. + # + - db_name: /path-to-file/my_database.db + + ## @param connection_attempt - integer - optional - default: 3 + ## The number of retries to connect to the database in case of failure + # + # connection_attempt: + + ## @param tags - list of strings - optional + ## A list of tags to attach to every metric and service check emitted by this instance. + ## + ## Learn more about tagging at https://docs.datadoghq.com/tagging + # + # tags: + # - : + # - : + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Overrides any `service` defined in the `init_config` section. + # + # service: + + ## @param min_collection_interval - number - optional - default: 60 + ## This changes the collection interval of this check to avoid + ## the risk of locking the database file. + ## If your database file is heavily used in write mode, you changes + ## further increase this value to minimise the monitoring acccess. + # + min_collection_interval: 60 + + ## @param empty_default_hostname - boolean - optional - default: false + ## This forces the check to send metrics with no hostname. + ## + ## This is useful for cluster-level checks. + # + # empty_default_hostname: false + + ## @param metric_patterns - mapping - optional + ## A mapping of metrics to include or exclude, with each entry being a regular expression. + ## + ## Metrics defined in `exclude` will take precedence in case of overlap. + # + # metric_patterns: + # include: + # - + # exclude: + # - diff --git a/duckdb/datadog_checks/duckdb/queries.py b/duckdb/datadog_checks/duckdb/queries.py new file mode 100644 index 0000000000000..e09030edbb3be --- /dev/null +++ b/duckdb/datadog_checks/duckdb/queries.py @@ -0,0 +1,68 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +DUCKDB_VERSION = { + 'name': 'version', + 'query': "SELECT version();", + 'columns': [{'name': 'version', 'type': 'source'}], +} + +DUCKDDB_WAL = { + 'name': 'wal_autocheckpoint', + 'query': " SELECT CAST(SUBSTR(value, 1, LENGTH(value) - 3) AS BIGINT) * " + "CASE " + " WHEN RIGHT(value, 3) = 'KiB' THEN 1024 " + " WHEN RIGHT(value, 3) = 'MiB' THEN 1024 * 1024 " + " WHEN RIGHT(value, 3) = 'GiB' THEN 1024 * 1024 * 1024 " + " WHEN RIGHT(value, 3) = 'TiB' THEN 1024 * 1024 * 1024 * 1024 " + " ELSE 1 " + " END AS value_in_bytes FROM duckdb_settings() WHERE name = 'wal_autocheckpoint';", + 'columns': [{'name': 'wal_autocheckpoint', 'type': 'gauge'}], +} + + +DUCKDDB_THREADS = { + 'name': 'worker_threads', + 'query': "select value from duckdb_settings() where name = 'worker_threads';", + 'columns': [{'name': 'worker_threads', 'type': 'gauge'}], +} + + +DUCKDB_MEMORY_LIMIT = { + 'name': 'memory_limit', + 'query': " SELECT CAST(SUBSTR(value, 1, LENGTH(value) - 3) AS BIGINT) * " + "CASE " + " WHEN RIGHT(value, 3) = 'KiB' THEN 1024 " + " WHEN RIGHT(value, 3) = 'MiB' THEN 1024 * 1024 " + " WHEN RIGHT(value, 3) = 'GiB' THEN 1024 * 1024 * 1024 " + " WHEN RIGHT(value, 3) = 'TiB' THEN 1024 * 1024 * 1024 * 1024 " + " ELSE 1 " + " END AS value_in_bytes FROM duckdb_settings() WHERE name = 'memory_limit';", + 'columns': [{'name': 'memory_limit', 'type': 'gauge'}], +} + + +DUCKDB_PART_WRITE_FLUSH_THRESHOLD = { + 'name': 'partitioned_write_flush_threshold', + 'query': " SELECT CAST(value AS INTEGER) AS value_as_integer " + " FROM duckdb_settings() WHERE name = 'partitioned_write_flush_threshold';", + 'columns': [{'name': 'partitioned_write_flush_threshold', 'type': 'gauge'}], +} + +DUCKDB_PART_WRITE_MAX_OPEN_FILES = { + 'name': 'partitioned_write_max_open_files', + 'query': " SELECT CAST(value AS INTEGER) AS value_as_integer " + " FROM duckdb_settings() WHERE name = 'partitioned_write_max_open_files';", + 'columns': [{'name': 'partitioned_write_max_open_files', 'type': 'gauge'}], +} + +DEFAULT_QUERIES = [ + DUCKDB_VERSION, + DUCKDDB_THREADS, + DUCKDDB_WAL, + DUCKDB_MEMORY_LIMIT, + DUCKDB_PART_WRITE_FLUSH_THRESHOLD, + DUCKDB_PART_WRITE_MAX_OPEN_FILES, +] diff --git a/duckdb/hatch.toml b/duckdb/hatch.toml new file mode 100644 index 0000000000000..c79a82d22f2c5 --- /dev/null +++ b/duckdb/hatch.toml @@ -0,0 +1,8 @@ +[env.collectors.datadog-checks] + +[[envs.default.matrix]] +python = ["3.12"] +version = ["1.1.1"] + +[envs.default.overrides] +matrix.version.env-vars = "DUCKDB_VERSION" \ No newline at end of file diff --git a/duckdb/images/IMAGES_README.md b/duckdb/images/IMAGES_README.md new file mode 100644 index 0000000000000..443f3c45e3385 --- /dev/null +++ b/duckdb/images/IMAGES_README.md @@ -0,0 +1,41 @@ +# Marketplace Media Carousel Guidelines + +## Using the media gallery + +Please upload images to use the media gallery. Integrations require a minimum of 3 images. Images should highlight your product, your integration, and a full image of the Datadog integration dashboard. The gallery +can hold a maximum of 8 pieces of media total, and one of these pieces of media +can be a video (guidelines and submission steps below). Images should be +added to your /images directory and referenced in the manifest.json file. + + +## Image and video requirements + +### Images + +``` +File type : .jpg or .png +File size : ~500 KB per image, with a max of 1 MB per image +File dimensions : The image must be between 1440px and 2880px width, with a 16:9 aspect ratio (for example: 1440x810) +File name : Use only letters, numbers, underscores, and hyphens +Color mode : RGB +Color profile : sRGB +Description : 300 characters maximum +``` + +### Video + +To display a video in your media gallery, please send our team the zipped file +or a link to download the video at `marketplace@datadog.com`. In addition, +please upload a thumbnail image for your video as a part of the pull request. +Once approved, we will upload the file to Vimeo and provide you with the +vimeo_id to add to your manifest.json file. Please note that the gallery can +only hold one video. + +``` +File type : MP4 H.264 +File size : Max 1 video; 1 GB maximum size +File dimensions : The aspect ratio must be exactly 16:9, and the resolution must be 1920x1080 or higher +File name : partnerName-appName.mp4 +Run time : Recommendation of 60 seconds or less +Description : 300 characters maximum +``` diff --git a/duckdb/manifest.json b/duckdb/manifest.json new file mode 100644 index 0000000000000..268dffb4db54a --- /dev/null +++ b/duckdb/manifest.json @@ -0,0 +1,53 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "a905fbe6-135f-4189-b027-4bdc58e51e29", + "app_id": "duckdb", + "display_on_public_website": false, + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "Integration for DuckDB", + "title": "DuckDB", + "media": [], + "classifier_tags": [ + "Supported OS::Linux", + "Supported OS::Windows", + "Supported OS::macOS", + "Category::Metrics", + "Offering::Integration", + "Submitted Data Type::Metrics" + ] + }, + "assets": { + "integration": { + "auto_install": true, + "source_type_id": 28902456, + "source_type_name": "DuckDB", + "configuration": { + "spec": "assets/configuration/spec.yaml" + }, + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "duckdb.", + "check": "duckdb.memory_limit", + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + }, + "dashboards": { + "Duckdb Overview": "assets/dashboards/duckdb_overview.json" + } + }, + "author": { + "support_email": "help@datadoghq.com", + "name": "Datadog", + "homepage": "https://www.datadoghq.com", + "sales_email": "info@datadoghq.com" + } +} diff --git a/duckdb/metadata.csv b/duckdb/metadata.csv new file mode 100644 index 0000000000000..0704022a503ab --- /dev/null +++ b/duckdb/metadata.csv @@ -0,0 +1,6 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags +duckdb.memory_limit,gauge,,byte,,The maximum memory of the system.,0,duckdb,,, +duckdb.partitioned_write_flush_threshold,gauge,,,,The threshold in number of rows after which we flush a thread state when writing using PARTITION_BY.,0,duckdb,,, +duckdb.partitioned_write_max_open_files,gauge,,,,The maximum amount of files the system can keep open before flushing to disk when writing using PARTITION_BY.,0,duckdb,,, +duckdb.wal_autocheckpoint,gauge,,byte,,The WAL size threshold at which to automatically trigger a checkpoint.,0,duckdb,,, +duckdb.worker_threads,gauge,,,,The number of total threads used by the system.,0,duckdb,,, diff --git a/duckdb/pyproject.toml b/duckdb/pyproject.toml new file mode 100644 index 0000000000000..801b00ea69063 --- /dev/null +++ b/duckdb/pyproject.toml @@ -0,0 +1,62 @@ +[build-system] +requires = [ + "hatchling>=0.13.0", +] +build-backend = "hatchling.build" + +[project] +name = "datadog-duckdb" +description = "The DuckDB check" +readme = "README.md" +license = "BSD-3-Clause" +requires-python = ">=3.12" +keywords = [ + "datadog", + "datadog agent", + "datadog check", + "duckdb", +] +authors = [ + { name = "Datadog", email = "packages@datadoghq.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Private :: Do Not Upload", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", +] +dependencies = [ + "datadog-checks-base>=37.0.0", +] +dynamic = [ + "version", +] + +[project.optional-dependencies] +deps = [ + "duckdb==1.1.1", +] + +[project.urls] +Source = "https://github.com/DataDog/integrations-core" + +[tool.hatch.version] +path = "datadog_checks/duckdb/__about__.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/datadog_checks", + "/tests", + "/manifest.json", +] + +[tool.hatch.build.targets.wheel] +include = [ + "/datadog_checks/duckdb", +] +dev-mode-dirs = [ + ".", +] diff --git a/duckdb/tests/__init__.py b/duckdb/tests/__init__.py new file mode 100644 index 0000000000000..9103122bf028d --- /dev/null +++ b/duckdb/tests/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/duckdb/tests/common.py b/duckdb/tests/common.py new file mode 100644 index 0000000000000..7fa9e7a75b9a5 --- /dev/null +++ b/duckdb/tests/common.py @@ -0,0 +1,23 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import os + +from datadog_checks.dev import get_here + +HERE = get_here() +DB_NAME = 'data/sample.db' +WRONG_DB_NAME = 'test.db' + +DB = os.path.join(HERE, DB_NAME) + +DEFAULT_INSTANCE = {'db_name': DB} +WRONG_INSTANCE = {'db_name': WRONG_DB_NAME} + +METRICS_MAP = [ + 'duckdb.worker_threads', + 'duckdb.wal_autocheckpoint', + 'duckdb.memory_limit', + 'duckdb.partitioned_write_flush_threshold', + 'duckdb.partitioned_write_max_open_files', +] diff --git a/duckdb/tests/conftest.py b/duckdb/tests/conftest.py new file mode 100644 index 0000000000000..9fbf7220d4ef2 --- /dev/null +++ b/duckdb/tests/conftest.py @@ -0,0 +1,18 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from copy import deepcopy + +import pytest + +from . import common + + +@pytest.fixture(scope='session') +def dd_environment(): + yield common.DEFAULT_INSTANCE + + +@pytest.fixture +def instance(): + return deepcopy(common.DEFAULT_INSTANCE) diff --git a/duckdb/tests/data/sample.db b/duckdb/tests/data/sample.db new file mode 100644 index 0000000000000..021514090f2e9 Binary files /dev/null and b/duckdb/tests/data/sample.db differ diff --git a/duckdb/tests/test_e2e.py b/duckdb/tests/test_e2e.py new file mode 100644 index 0000000000000..c9eab0ad6ab45 --- /dev/null +++ b/duckdb/tests/test_e2e.py @@ -0,0 +1,10 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + + +@pytest.mark.e2e +def test_e2e(dd_agent_check): + aggregator = dd_agent_check() + aggregator.assert_all_metrics_covered() diff --git a/duckdb/tests/test_integration.py b/duckdb/tests/test_integration.py new file mode 100644 index 0000000000000..c5d96a1c15118 --- /dev/null +++ b/duckdb/tests/test_integration.py @@ -0,0 +1,45 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import logging + +from datadog_checks.duckdb import DuckdbCheck + +from . import common + + +def test_check(dd_run_check, aggregator, instance): + instance = common.DEFAULT_INSTANCE + check = DuckdbCheck('duckdb', {}, [instance]) + dd_run_check(check) + + for metric in common.METRICS_MAP: + aggregator.assert_metric(metric) + + +def test_failed_connection(dd_run_check, instance, caplog): + caplog.set_level(logging.ERROR) + instance = common.WRONG_INSTANCE + check = DuckdbCheck('duckdb', {}, [instance]) + dd_run_check(check) + + expected_error = "Database file not found" + assert expected_error in caplog.text + + +def test_version(dd_run_check, instance, datadog_agent): + instance = common.DEFAULT_INSTANCE + check = DuckdbCheck('duckdb', {}, [instance]) + check.check_id = 'test:123' + raw_version = '1.1.1' + major, minor, patch = raw_version.split('.') + version_metadata = { + 'version.scheme': 'semver', + 'version.major': major, + 'version.minor': minor, + 'version.patch': patch, + 'version.raw': raw_version, + } + dd_run_check(check) + + datadog_agent.assert_metadata('test:123', version_metadata) diff --git a/duckdb/tests/test_unit.py b/duckdb/tests/test_unit.py new file mode 100644 index 0000000000000..623d1d0e5e51f --- /dev/null +++ b/duckdb/tests/test_unit.py @@ -0,0 +1,18 @@ +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import pytest + +from datadog_checks.base import AgentCheck # noqa: F401 +from datadog_checks.base.stubs.aggregator import AggregatorStub # noqa: F401 +from datadog_checks.duckdb import DuckdbCheck + + +def test_empty_instance(dd_run_check): + with pytest.raises( + Exception, + match='InstanceConfig`:\ndb_name\n Field required', + ): + check = DuckdbCheck('duckdb', {}, [{}]) + dd_run_check(check)