Skip to content

Commit

Permalink
feat: Add tools to heap event (#430)
Browse files Browse the repository at this point in the history 
* Add add-on data to heap event

Signed-off-by: lrcouto <[email protected]>

* Move addons logic to _get_project_property

Signed-off-by: Ankita Katiyar <[email protected]>

* Add condition for pyproject.toml

Signed-off-by: Ankita Katiyar <[email protected]>

* Fix tests

Signed-off-by: Ankita Katiyar <[email protected]>

* Fix tests

Signed-off-by: Ankita Katiyar <[email protected]>

* add tools to mock

Signed-off-by: lrcouto <[email protected]>

* lint

Signed-off-by: lrcouto <[email protected]>

* Update tools test

Signed-off-by: Ankita Katiyar <[email protected]>

* Add after_context_created tools test

Signed-off-by: lrcouto <[email protected]>

* Update rename to tools

Signed-off-by: Ankita Katiyar <[email protected]>

* Update kedro-telemetry/tests/test_plugin.py

Co-authored-by: Sajid Alam <[email protected]>
Signed-off-by: Ankita Katiyar <[email protected]>

---------

Signed-off-by: lrcouto <[email protected]>
Signed-off-by: Ankita Katiyar <[email protected]>
Signed-off-by: Ankita Katiyar <[email protected]>
Co-authored-by: Ankita Katiyar <[email protected]>
Co-authored-by: Ankita Katiyar <[email protected]>
Co-authored-by: Sajid Alam <[email protected]>
  • Loading branch information
@lrcouto @ankatiyar @SajidAlamQB
4 people authored Nov 30, 2023
1 parent 48e2e76 commit a360bfd
Show file tree
Hide file tree
Showing 3 changed files with 173 additions and 9 deletions.
1 change: 1 addition & 0 deletions kedro-telemetry/RELEASE.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Upcoming release
* Fixed double execution of `after_catalog_created` hook by moving the logic of determining and sending of project statistics from `after_context_created` to the `after_catalog_created` hook.
* Updated the plugin to also share the tools selected during project creation with Heap.

# Release 0.3.0
* Added support for Python 3.11
Expand Down
23 changes: 17 additions & 6 deletions kedro-telemetry/kedro_telemetry/plugin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

import click
import requests
import toml
import yaml
from kedro import __version__ as KEDRO_VERSION
from kedro.framework.cli.cli import KedroCLI
Expand Down Expand Up @@ -78,9 +79,10 @@ def before_command_run(
return

logger.debug("You have opted into product usage analytics.")

hashed_username = _get_hashed_username()
project_properties = _get_project_properties(hashed_username)
project_properties = _get_project_properties(
hashed_username, project_metadata.project_path
)
cli_properties = _format_user_cli_data(
project_properties, masked_command_args
)
Expand Down Expand Up @@ -114,6 +116,7 @@ class KedroTelemetryProjectHooks:
def after_context_created(self, context):
"""Hook implementation to send project statistics data to Heap"""
self.consent = _check_for_telemetry_consent(context.project_path)
self.project_path = context.project_path

@hook_impl
def after_catalog_created(self, catalog):
Expand All @@ -129,7 +132,7 @@ def after_catalog_created(self, catalog):
default_pipeline = pipelines.get("__default__") # __default__
hashed_username = _get_hashed_username()

project_properties = _get_project_properties(hashed_username)
project_properties = _get_project_properties(hashed_username, self.project_path)

project_statistics_properties = _format_project_statistics_data(
project_properties, catalog, default_pipeline, pipelines
Expand All @@ -141,17 +144,25 @@ def after_catalog_created(self, catalog):
)


def _get_project_properties(hashed_username: str) -> Dict:
def _get_project_properties(hashed_username: str, project_path: str) -> Dict:
hashed_package_name = _hash(PACKAGE_NAME) if PACKAGE_NAME else "undefined"

return {
properties = {
"username": hashed_username,
"package_name": hashed_package_name,
"project_version": KEDRO_VERSION,
"telemetry_version": TELEMETRY_VERSION,
"python_version": sys.version,
"os": sys.platform,
}
pyproject_path = Path(project_path) / "pyproject.toml"
if pyproject_path.exists():
with open(pyproject_path) as file:
pyproject_data = toml.load(file)

if "tools" in pyproject_data["tool"]["kedro"]:
properties["tools"] = pyproject_data["tool"]["kedro"]["tools"]

return properties


def _format_user_cli_data(
Expand Down
158 changes: 155 additions & 3 deletions kedro-telemetry/tests/test_plugin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,38 @@
REPO_NAME = "dummy_project"
PACKAGE_NAME = "dummy_package"

MOCK_PYPROJECT_TOOLS = """
[build-system]
requires = [ "setuptools",]
build-backend = "setuptools.build_meta"
[project]
name = "spaceflights"
readme = "README.md"
dynamic = [ "dependencies", "version",]
[project.scripts]
new-proj = "spaceflights.__main__:main"
[tool.kedro]
package_name = "spaceflights"
project_name = "spaceflights"
kedro_init_version = "0.18.14"
tools = "['Linting', 'Testing', 'Custom Logging', 'Documentation', 'Data Structure', 'PySpark']"
[project.entry-points."kedro.hooks"]
[tool.setuptools.dynamic.dependencies]
file = "requirements.txt"
[tool.setuptools.dynamic.version]
attr = "spaceflights.__version__"
[tool.setuptools.packages.find]
where = [ "src",]
namespaces = false
"""


@fixture
def fake_metadata(tmp_path):
Expand Down Expand Up @@ -50,6 +82,15 @@ def identity(arg):
return arg


@fixture
def fake_context():
class MockKedroContext:
# A dummy stand-in for KedroContext sufficient for this test
project_path = Path("")

return MockKedroContext()


@fixture
def fake_default_pipeline():
mock_default_pipeline = modular_pipeline(
Expand Down Expand Up @@ -116,6 +157,53 @@ def test_before_command_run(self, mocker, fake_metadata):
]
assert mocked_heap_call.call_args_list == expected_calls

def test_before_command_run_with_tools(self, mocker, fake_metadata):
mocker.patch(
"kedro_telemetry.plugin._check_for_telemetry_consent", return_value=True
)
mocked_anon_id = mocker.patch("kedro_telemetry.plugin._hash")
mocked_anon_id.return_value = "digested"
mocker.patch("kedro_telemetry.plugin.PACKAGE_NAME", "spaceflights")
mocker.patch(
"kedro_telemetry.plugin._get_hashed_username",
return_value="hashed_username",
)

mocked_heap_call = mocker.patch("kedro_telemetry.plugin._send_heap_event")
mocker.patch("builtins.open", mocker.mock_open(read_data=MOCK_PYPROJECT_TOOLS))
mocker.patch("pathlib.Path.exists", return_value=True)
telemetry_hook = KedroTelemetryCLIHooks()
command_args = ["--version"]
telemetry_hook.before_command_run(fake_metadata, command_args)
expected_properties = {
"username": "hashed_username",
"package_name": "digested",
"project_version": kedro_version,
"telemetry_version": TELEMETRY_VERSION,
"python_version": sys.version,
"os": sys.platform,
"command": "kedro --version",
"tools": "['Linting', 'Testing', 'Custom Logging', 'Documentation', 'Data Structure', 'PySpark']",
}
generic_properties = {
**expected_properties,
"main_command": "--version",
}

expected_calls = [
mocker.call(
event_name="Command run: --version",
identity="hashed_username",
properties=expected_properties,
),
mocker.call(
event_name="CLI command",
identity="hashed_username",
properties=generic_properties,
),
]
assert mocked_heap_call.call_args_list == expected_calls

def test_before_command_run_empty_args(self, mocker, fake_metadata):
mocker.patch(
"kedro_telemetry.plugin._check_for_telemetry_consent", return_value=True
Expand Down Expand Up @@ -325,14 +413,15 @@ def test_confirm_consent_yaml_dump_error(self, mocker, fake_metadata, caplog):


class TestKedroTelemetryProjectHooks:
def test_after_context_created_without_kedro_run(
def test_after_context_created_without_kedro_run( # noqa: PLR0913
self,
mocker,
fake_catalog,
fake_default_pipeline,
fake_sub_pipeline,
fake_context,
):
fake_context = mocker.Mock()

mocker.patch.dict(
pipelines, {"__default__": fake_default_pipeline, "sub": fake_sub_pipeline}
)
Expand All @@ -346,6 +435,8 @@ def test_after_context_created_without_kedro_run(
return_value="hashed_username",
)
mocked_heap_call = mocker.patch("kedro_telemetry.plugin._send_heap_event")
mocker.patch("kedro_telemetry.plugin.open")
mocker.patch("kedro_telemetry.plugin.toml.load")

# Without CLI invoked - i.e. `session.run` in Jupyter/IPython
telemetry_hook = KedroTelemetryProjectHooks()
Expand Down Expand Up @@ -383,8 +474,65 @@ def test_after_context_created_with_kedro_run( # noqa: PLR0913
fake_metadata,
fake_default_pipeline,
fake_sub_pipeline,
fake_context,
):
mocker.patch.dict(
pipelines, {"__default__": fake_default_pipeline, "sub": fake_sub_pipeline}
)
mocker.patch(
"kedro_telemetry.plugin._check_for_telemetry_consent", return_value=True
)
mocker.patch("kedro_telemetry.plugin._hash", return_value="digested")
mocker.patch("kedro_telemetry.plugin.PACKAGE_NAME", "spaceflights")
mocker.patch(
"kedro_telemetry.plugin._get_hashed_username",
return_value="hashed_username",
)
mocked_heap_call = mocker.patch("kedro_telemetry.plugin._send_heap_event")
mocker.patch("kedro_telemetry.plugin.toml.load")
# CLI run first
telemetry_cli_hook = KedroTelemetryCLIHooks()
command_args = ["--version"]
telemetry_cli_hook.before_command_run(fake_metadata, command_args)

# Follow by project run
telemetry_hook = KedroTelemetryProjectHooks()
telemetry_hook.after_context_created(fake_context)
telemetry_hook.after_catalog_created(fake_catalog)

project_properties = {
"username": "hashed_username",
"package_name": "digested",
"project_version": kedro_version,
"telemetry_version": TELEMETRY_VERSION,
"python_version": sys.version,
"os": sys.platform,
}
project_statistics = {
"number_of_datasets": 3,
"number_of_nodes": 2,
"number_of_pipelines": 2,
}
expected_properties = {**project_properties, **project_statistics}

expected_call = mocker.call(
event_name="Kedro Project Statistics",
identity="hashed_username",
properties=expected_properties,
)

# CLI hook makes the first 2 calls, the 3rd one is the Project hook
assert mocked_heap_call.call_args_list[2] == expected_call

def test_after_context_created_with_kedro_run_and_tools( # noqa: PLR0913
self,
mocker,
fake_catalog,
fake_metadata,
fake_default_pipeline,
fake_sub_pipeline,
fake_context,
):
fake_context = mocker.Mock()
mocker.patch.dict(
pipelines, {"__default__": fake_default_pipeline, "sub": fake_sub_pipeline}
)
Expand All @@ -398,6 +546,9 @@ def test_after_context_created_with_kedro_run( # noqa: PLR0913
return_value="hashed_username",
)
mocked_heap_call = mocker.patch("kedro_telemetry.plugin._send_heap_event")
mocker.patch("builtins.open", mocker.mock_open(read_data=MOCK_PYPROJECT_TOOLS))
mocker.patch("pathlib.Path.exists", return_value=True)

# CLI run first
telemetry_cli_hook = KedroTelemetryCLIHooks()
command_args = ["--version"]
Expand All @@ -415,6 +566,7 @@ def test_after_context_created_with_kedro_run( # noqa: PLR0913
"telemetry_version": TELEMETRY_VERSION,
"python_version": sys.version,
"os": sys.platform,
"tools": "['Linting', 'Testing', 'Custom Logging', 'Documentation', 'Data Structure', 'PySpark']",
}
project_statistics = {
"number_of_datasets": 3,
Expand Down

0 comments on commit a360bfd

Please sign in to comment.